A high-performance inference engine for AI models
language model generalization (#217)
uuuvnfc0db29a5a19
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rsindex 9faeb13ab..62f878678 100644--- a/crates/uzu/src/backends/common/backend.rs+++ b/crates/uzu/src/backends/common/backend.rs@@ -11,4 +11,6 @@ pub trait Backend: Debug + Clone {type Event: Event<Backend = Self>;type Kernels: Kernels<Backend = Self>;type Error: Error + Debug;++ const MAX_INLINE_BYTES: usize;}diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex 7bd73f81e..ad3959698 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -1,6 +1,6 @@use super::Backend;-pub trait CommandBuffer {+pub trait CommandBuffer: Clone {type Backend: Backend;fn with_compute_encoder<T>(@@ -13,6 +13,23 @@ pub trait CommandBuffer {callback: impl FnOnce(&<Self::Backend as Backend>::CopyEncoder) -> T,) -> T;+ fn encode_wait_for_event(+ &self,+ event: &<Self::Backend as Backend>::Event,+ value: u64,+ );++ fn encode_signal_event(+ &self,+ event: &<Self::Backend as Backend>::Event,+ value: u64,+ );++ fn add_completed_handler(+ &self,+ handler: impl Fn() + 'static,+ );+fn submit(&self);fn wait_until_completed(&self);diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rsindex 52eda7038..ca5a27cff 100644--- a/crates/uzu/src/backends/common/context.rs+++ b/crates/uzu/src/backends/common/context.rs@@ -1,12 +1,31 @@-use std::rc::Rc;+use std::{path::Path, rc::Rc};use super::{Allocator, Backend};+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DeviceClass {+ Ultra,+ Max,+ Pro,+ Base,+ IPhone,+}++impl DeviceClass {+ pub fn is_high_end(&self) -> bool {+ matches!(self, DeviceClass::Ultra | DeviceClass::Max)+ }+}+pub trait Context: Sized {type Backend: Backend<Context = Self>;fn new() -> Result<Rc<Self>, <Self::Backend as Backend>::Error>;+ fn device_class(&self) -> DeviceClass;++ fn debug_active(&self) -> bool;+fn allocator(&self) -> &Allocator<Self::Backend>;fn create_buffer(@@ -19,4 +38,13 @@ pub trait Context: Sized {) -> Result<<Self::Backend as Backend>::CommandBuffer, <Self::Backend as Backend>::Error>;fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;++ fn enable_capture();++ fn start_capture(+ &self,+ trace_path: &Path,+ ) -> Result<(), <Self::Backend as Backend>::Error>;++ fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;}diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rsindex bd1fbd8e2..6155bcddf 100644--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs@@ -1,11 +1,11 @@//! GEMM kernel parameter structs.//!//! Field names use uppercase for matrix dimensions (M, N, K) to match-//! standard BLAS/GEMM convention and generated Metal headers.+//! standard BLAS/GEMM convention and generated shader headers.#![allow(non_snake_case)]-/// Main GEMM parameters passed to Metal kernels as constant buffer.+/// Main GEMM parameters passed to backend kernels as constant buffer.#[repr(C)]#[derive(Debug, Default, Copy, Clone)]pub struct GEMMParams {diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rsindex 4f56924ed..c8ba798f4 100644--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs@@ -38,10 +38,6 @@ pub enum KVCacheUpdateError<B: Backend> {}impl<B: Backend> KVCacheUpdate<B> {- // Metal's set_bytes supports up to 4KB per bound value.- // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objc- const MAX_INLINE_BYTES: usize = 4096;-pub fn new(context: &B::Context,data_type: DataType,@@ -84,7 +80,7 @@ impl<B: Backend> KVCacheUpdate<B> {if swaps.len() > self.max_sequence_length {return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);}- let max_inline_swaps = (Self::MAX_INLINE_BYTES / size_of::<Swap>()).max(1);+ let max_inline_swaps = (B::MAX_INLINE_BYTES / size_of::<Swap>()).max(1);for layer_data in in_place_data {if layer_data.key_shape != layer_data.value_shape {diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rsindex 7936a116d..c82d14345 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs@@ -85,7 +85,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {);let d_model_u32 = args.d_model as u32;let d_ff_u32 = args.d_ff as u32;- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in kernelconst COL_TILE_MODEL: u32 = 64;let n_tiles_ff = if d_ff_u32 == 0 {0diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rsindex c5cab7be9..9b2caf29a 100644--- a/crates/uzu/src/backends/common/mod.rs+++ b/crates/uzu/src/backends/common/mod.rs@@ -15,7 +15,7 @@ pub use backend::Backend;pub use buffer::Buffer;pub use buffer_lifetime::BufferLifetime;pub use command_buffer::CommandBuffer;-pub use context::Context;+pub use context::{Context, DeviceClass};pub use copy_encoder::CopyEncoder;pub use event::Event;pub use kernel::Kernels;diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rsindex 1a5186643..fc5d832ee 100644--- a/crates/uzu/src/backends/metal/backend.rs+++ b/crates/uzu/src/backends/metal/backend.rs@@ -16,4 +16,8 @@ impl Backend for Metal {type Event = Retained<ProtocolObject<dyn MTLEvent>>;type Kernels = MetalKernels;type Error = MetalError;++ // Metal's set_bytes supports up to 4KB per bound value.+ // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objc+ const MAX_INLINE_BYTES: usize = 4096;}diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex 958b9a5c3..bf564978c 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -1,6 +1,6 @@use std::ops::Deref;-use metal::{MTLCommandBuffer, MTLCommandBufferExt, MTLCommandEncoder};+use metal::{MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler, MTLCommandEncoder, MTLEvent};use objc2::{rc::Retained, runtime::ProtocolObject};use super::Metal;@@ -35,6 +35,29 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {ret}+ fn encode_wait_for_event(+ &self,+ event: &Retained<ProtocolObject<dyn MTLEvent>>,+ value: u64,+ ) {+ self.encode_wait_for_event_value(event, value);+ }++ fn encode_signal_event(+ &self,+ event: &Retained<ProtocolObject<dyn MTLEvent>>,+ value: u64,+ ) {+ self.encode_signal_event_value(event, value);+ }++ fn add_completed_handler(+ &self,+ handler: impl Fn() + 'static,+ ) {+ self.deref().add_completed_handler(&MTLCommandBufferHandler::new(move |_| handler()));+ }+fn submit(&self) {self.commit();}diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex ca3e45905..930233183 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -1,13 +1,14 @@use std::{cell::RefCell, collections::HashMap, env, rc::Rc};use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,- MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions,+ MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandBuffer, MTLCommandQueue,+ MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues,+ MTLLibrary, MTLResourceOptions,};use objc2::{rc::Retained, runtime::ProtocolObject};use super::{Metal, error::MetalError, kernel, metal_extensions::LibraryPipelineExtensions};-use crate::backends::common::{Allocator, Context};+use crate::backends::common::{Allocator, Context, DeviceClass as CommonDeviceClass};/// Apple GPU architecture generation./// Based on Apple GPU family naming convention (e.g., "applegpu_g13p").@@ -200,6 +201,27 @@ impl Context for MetalContext {}))}+ fn device_class(&self) -> CommonDeviceClass {+ let name = self.device.name().to_lowercase();++ if name.contains("ultra") {+ CommonDeviceClass::Ultra+ } else if name.contains("max") {+ CommonDeviceClass::Max+ } else if name.contains("pro") {+ CommonDeviceClass::Pro+ } else if name.contains("iphone") || name.contains("a1") {+ CommonDeviceClass::IPhone+ } else {+ CommonDeviceClass::Base+ }+ }++ fn debug_active(&self) -> bool {+ let upper = std::env::var("METAL_DEVICE_WRAPPER_TYPE").unwrap_or_default().to_ascii_uppercase();+ matches!(upper.as_str(), "1" | "YES" | "TRUE")+ }+fn allocator(&self) -> &Allocator<Metal> {&self.allocator}@@ -220,4 +242,35 @@ impl Context for MetalContext {fn create_event(&self) -> Result<Retained<ProtocolObject<dyn MTLEvent>>, MetalError> {self.device.new_event().ok_or(MetalError::Generic("cannot create event".into()))}++ fn enable_capture() {+ unsafe {+ std::env::set_var("METAL_CAPTURE_ENABLED", "1");+ }+ }++ fn start_capture(+ &self,+ trace_path: &std::path::Path,+ ) -> Result<(), <Self::Backend as crate::backends::common::Backend>::Error> {+ let capture_manager = MTLCaptureManager::shared_capture_manager();+ let capture_descriptor = MTLCaptureDescriptor::new();+ capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);+ capture_descriptor.set_output_path(Some(trace_path));++ self.command_queue.set_label(Some("uzu_command_queue"));+ capture_descriptor.set_capture_object(Some(self.command_queue.as_ref()));++ capture_manager+ .start_capture_with_descriptor_error(&capture_descriptor)+ .map_err(|e| format!("Failed to start GPU capture: {}", e))?;++ Ok(())+ }++ fn stop_capture(&self) -> Result<(), <Self::Backend as crate::backends::common::Backend>::Error> {+ MTLCaptureManager::shared_capture_manager().stop_capture();++ Ok(())+ }}diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.hindex db0130600..ac8bfeba5 100644--- a/crates/uzu/src/backends/metal/generated/matmul.h+++ b/crates/uzu/src/backends/metal/generated/matmul.h@@ -14,7 +14,7 @@ namespace matmul {#include <stdint.h>#endif-/**Main GEMM parameters passed to Metal kernels as constant buffer. */+/**Main GEMM parameters passed to backend kernels as constant buffer. */typedef struct {int32_t M;int32_t N;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rsdeleted file mode 100644index 3ce63cb23..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub use crate::backends::common::gpu_types::{GEMMParams, GEMMSpiltKParams};diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex d0464c58c..b13b6a219 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -1,4 +1,3 @@-pub mod common;mod dispatch_descriptor;mod gemm;diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex 8b6b5b189..64ed3aecb 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -4,9 +4,9 @@ mod context;mod copy_encoder;mod error;mod event;-pub mod kernel;+mod kernel;mod metal_extensions;mod native_buffer;pub use backend::Metal;-pub use context::MetalContext;+pub use kernel::matmul::choose_dispatch_descriptor;diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex fc576252c..941c994b3 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -2,7 +2,6 @@use std::{cell::RefCell, rc::Rc};use std::{collections::HashMap, path::Path, time::Instant};-use metal::MTLCommandBuffer;use objc2::rc::autoreleasepool;#[cfg(feature = "tracing")]@@ -10,17 +9,23 @@ use super::ActivationTrace;use super::{ClassificationOutput, ClassificationStats, ClassifierContext};use crate::{DataType,- backends::metal::Metal,+ backends::common::{Backend, CommandBuffer, kernel::matmul::MatmulKernels},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::{ArrayId, ForwardPassState},session::types::Error,};-pub struct Classifier {- pub context: ClassifierContext<Metal>,+pub struct Classifier<B: Backend + 'static>+where+ B::Kernels: MatmulKernels,+{+ pub context: ClassifierContext<B>,}-impl Classifier {+impl<B: Backend + 'static> Classifier<B>+where+ B::Kernels: MatmulKernels,+{pub fn new(model_path: &Path) -> Result<Self, Error> {let context = ClassifierContext::new(model_path)?;Ok(Self {@@ -79,7 +84,7 @@ impl Classifier {&mut self,token_ids: &[u64],token_positions: &[usize],- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<Metal>>>), Error> {+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {self.forward_pass(token_ids, token_positions)}@@ -88,7 +93,7 @@ impl Classifier {&mut self,token_ids: &[u64],token_positions: &[usize],- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<Metal>>>), Error> {+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {autoreleasepool(|_| {let num_labels = self.context.model_config.model_config.num_labels;let mut state = ForwardPassState::new_classifier(@@ -136,7 +141,7 @@ impl Classifier {self.context.prediction_head.encode(&mut state, &encoding_params, &self.context.command_buffer);- self.context.command_buffer.commit();+ self.context.command_buffer.submit();self.context.command_buffer.wait_until_completed();let logits = self.copy_logits_from_state(&state)?;@@ -178,7 +183,7 @@ impl Classifier {self.context.pooling.encode(&mut state, &encoding_params, &self.context.command_buffer);self.context.prediction_head.encode(&mut state, &encoding_params, &self.context.command_buffer);- self.context.command_buffer.commit();+ self.context.command_buffer.submit();self.context.command_buffer.wait_until_completed();let logits = self.copy_logits_from_state(&state)?;@@ -189,7 +194,7 @@ impl Classifier {fn copy_logits_from_state(&self,- state: &ForwardPassState<Metal>,+ state: &ForwardPassState<B>,) -> Result<Box<[f32]>, Error> {let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);let logits_array = logits_arrays[0].borrow();diff --git a/crates/uzu/src/language_model/gpu_capture.rs b/crates/uzu/src/language_model/gpu_capture.rsindex a16b94014..d7e13f5ef 100644--- a/crates/uzu/src/language_model/gpu_capture.rs+++ b/crates/uzu/src/language_model/gpu_capture.rs@@ -1,29 +1,30 @@use std::{+ marker::PhantomData,path::PathBuf,time::{SystemTime, UNIX_EPOCH},};-use metal::{MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueueExt};--use crate::{backends::metal::MetalContext, utils::env_utils::MetalEnvVar};+use crate::{+ backends::common::{Backend, Context},+ utils::env_utils::EnvVar,+};-pub struct GpuCaptureManager {+pub struct GpuCaptureManager<B: Backend> {capture_prefill_enabled: bool,capture_decode_enabled: bool,first_prefill_captured: bool,first_decode_captured: bool,+ phantom: PhantomData<B>,}-impl GpuCaptureManager {+impl<B: Backend> GpuCaptureManager<B> {pub fn new() -> Self {- let capture_prefill_enabled = MetalEnvVar::CaptureFirstPrefill.is_enabled();- let capture_decode_enabled = MetalEnvVar::CaptureFirstDecode.is_enabled();+ let capture_prefill_enabled = EnvVar::CaptureFirstPrefill.is_enabled();+ let capture_decode_enabled = EnvVar::CaptureFirstDecode.is_enabled();- // Enable Metal capture layer BEFORE device creation if any capture is requested+ // Enable backend capture BEFORE device creation if any capture is requestedif capture_prefill_enabled || capture_decode_enabled {- unsafe {- std::env::set_var("METAL_CAPTURE_ENABLED", "1");- }+ B::Context::enable_capture();}Self {@@ -31,6 +32,7 @@ impl GpuCaptureManager {capture_decode_enabled,first_prefill_captured: false,first_decode_captured: false,+ phantom: PhantomData,}}@@ -50,7 +52,7 @@ impl GpuCaptureManager {pub fn start_capture(&self,- mtl_context: &MetalContext,+ context: &B::Context,capture_type: &str,) -> Result<PathBuf, String> {let timestamp =@@ -60,20 +62,7 @@ impl GpuCaptureManager {.unwrap_or_else(|_| PathBuf::from(".")).join(format!("uzu_first_{}-{}.gputrace", capture_type, timestamp));- let capture_manager = MTLCaptureManager::shared_capture_manager();- let capture_descriptor = MTLCaptureDescriptor::new();- capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);- capture_descriptor.set_output_path(Some(&trace_path));-- mtl_context.command_queue.set_label(Some("uzu_command_queue"));- use objc2::rc::Retained;- unsafe {- capture_descriptor.set_capture_object(Some(&*(Retained::as_ptr(&mtl_context.command_queue).cast())));- }-- capture_manager- .start_capture_with_descriptor_error(&capture_descriptor)- .map_err(|e| format!("Failed to start GPU capture: {}", e))?;+ context.start_capture(&trace_path).map_err(|err| err.to_string())?;println!("GPU capture started for first {}: {:?}", capture_type, trace_path);@@ -82,9 +71,10 @@ impl GpuCaptureManager {pub fn stop_capture(&mut self,+ context: &B::Context,capture_type: &str,- ) {- MTLCaptureManager::shared_capture_manager().stop_capture();+ ) -> Result<(), String> {+ context.stop_capture().map_err(|err| err.to_string())?;println!("GPU capture stopped for {}", capture_type);match capture_type {@@ -92,6 +82,8 @@ impl GpuCaptureManager {"decode" => self.first_decode_captured = true,_ => {},}++ Ok(())}pub fn reset(&mut self) {diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 8c0274fbf..cce1f33e7 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -1,7 +1,6 @@use std::{collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant};use itertools::{Either, Itertools, izip};-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler, MTLCommandEncoder};use super::{LanguageModelGeneratorContext,@@ -12,12 +11,9 @@ use super::{tasks::{LanguageModelGeneratorEncodedTask, LanguageModelGeneratorRunTask},};use crate::{- backends::{- common::{- Context,- kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},- },- metal::Metal,+ backends::common::{+ Backend, CommandBuffer, Context, NativeBuffer,+ kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, matmul::MatmulKernels},},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::{@@ -30,20 +26,25 @@ use crate::{types::Error,},trie::{TrieCreationConfig, TrieNode},- utils::env_utils::MetalEnvVar,};-pub struct LanguageModelGenerator {+pub struct LanguageModelGenerator<B: Backend + 'static>+where+ B::Kernels: MatmulKernels,+{pub decoding_config: DecodingConfig,pub tokens: Vec<u64>,- pub context: LanguageModelGeneratorContext<Metal>,- encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask>,+ pub context: LanguageModelGeneratorContext<B>,+ encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask<B>>,registered_prefix_len: usize,- gpu_capture: GpuCaptureManager,+ gpu_capture: GpuCaptureManager<B>,}-impl LanguageModelGenerator {+impl<B: Backend + 'static> LanguageModelGenerator<B>+where+ B::Kernels: MatmulKernels,+{pub fn new(model_path: &Path,decoding_config: DecodingConfig,@@ -119,7 +120,7 @@ impl LanguageModelGenerator {let token_seeds = repeat_n(0, tokens_length - 1).chain(flat_trie.token_seeds()).chunks(prefill_step_size);- let mut last_state: Option<ForwardPassState<Metal>> = None;+ let mut last_state: Option<ForwardPassState<B>> = None;let mut run_times: Vec<f64> = Vec::new();// Process each prefill step and update the KV cache.@@ -201,7 +202,7 @@ impl LanguageModelGenerator {);if should_capture {- self.gpu_capture.stop_capture("prefill");+ self.gpu_capture.stop_capture(&self.context.context, "prefill").map_err(|_| Error::CaptureFailed)?;}// Register the accepted prompt tokens from this step.@@ -393,7 +394,7 @@ impl LanguageModelGenerator {let is_continuation = current_counter > 0;let batch_size = self.context.async_buffers.batch_size;let slot = pass_idx % batch_size;- let async_event = self.context.async_buffers.event.clone();+let results_buffer = self.context.async_buffers.results.clone();let async_positions_buffer = self.context.async_buffers.positions.clone();let async_seeds_buffer = self.context.async_buffers.seeds.clone();@@ -401,7 +402,7 @@ impl LanguageModelGenerator {let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;let token_position = unsafe {- let ptr = async_positions_buffer.contents().as_ptr() as *const u32;+ let ptr = async_positions_buffer.cpu_ptr().as_ptr() as *const u32;*ptr.add(pass_idx) as usize};@@ -455,11 +456,11 @@ impl LanguageModelGenerator {}self.context.reset_command_buffer();- let root_command_buffer = self.context.command_buffer.clone();+ let root_command_buffer = &self.context.command_buffer;// Wait on previous pass if this is a continuationif is_continuation {- root_command_buffer.encode_wait_for_event_value(&async_event, current_counter);+ root_command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);}self.context.executables.encode(@@ -481,67 +482,65 @@ impl LanguageModelGenerator {let sampling_output_buffer = sampling_output.borrow().buffer().clone();let token_ids_buffer = self.context.scratch_buffers.token_ids.borrow().buffer().clone();- let encoder =- root_command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.context.token_copy_sampled.encode(&sampling_output_buffer, &token_ids_buffer, &encoder);- let results_offset = slot * std::mem::size_of::<u32>();- self.context.token_copy_results.encode(&sampling_output_buffer, (&results_buffer, results_offset), &encoder);- encoder.end_encoding();+ root_command_buffer.with_compute_encoder(|encoder| {+ self.context.token_copy_sampled.encode(&sampling_output_buffer, &token_ids_buffer, encoder);+ let results_offset = slot * std::mem::size_of::<u32>();+ self.context.token_copy_results.encode(&sampling_output_buffer, (&results_buffer, results_offset), encoder);+ });// Scatter + register for all transformer layers- {- let cb = root_command_buffer.clone();- self.context.cache_layers.borrow_mut().update_after_acceptance(- &[0],- None,- &cb,- &self.context.kv_cache_update,- );- self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);- }-- let encoder =- root_command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");-- if let Some(mask_update) = &self.context.mask_update {- let updates: Vec<AttentionBiasUpdate> =- self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);- for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {- if let Some(update) = updates.iter().find(|u| &u.key == window_size) {- if update.unmask_col >= 0 || update.mask_col >= 0 {- mask_update.encode(mask_buffer.borrow().buffer(), update.unmask_col, update.mask_col, &encoder);+ self.context.cache_layers.borrow_mut().update_after_acceptance(+ &[0],+ None,+ &root_command_buffer,+ &self.context.kv_cache_update,+ );+ self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);++ root_command_buffer.with_compute_encoder(|encoder| {+ if let Some(mask_update) = &self.context.mask_update {+ let updates: Vec<AttentionBiasUpdate> =+ self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);+ for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {+ if let Some(update) = updates.iter().find(|u| &u.key == window_size) {+ if update.unmask_col >= 0 || update.mask_col >= 0 {+ mask_update.encode(+ mask_buffer.borrow().buffer(),+ update.unmask_col,+ update.mask_col,+ &encoder,+ );+ }}}}- }-- encoder.end_encoding();+ });// Signal event for next passlet next_counter = current_counter + 1;- root_command_buffer.encode_signal_event_value(&async_event, next_counter);+ root_command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);self.context.async_buffers.counter.set(next_counter);// Add completion handlerlet results_buffer_clone = results_buffer.clone();let callback = Arc::new(std::sync::Mutex::new(Some(on_complete)));- let handler = MTLCommandBufferHandler::new(move |_| {+ let handler = move || {let token = {- let ptr = results_buffer_clone.contents().as_ptr() as *const u32;+ let ptr = results_buffer_clone.cpu_ptr().as_ptr() as *const u32;unsafe { *ptr.add(slot) as u64 }};if let Some(cb) = callback.lock().unwrap().take() {cb(token);}- });+ };- root_command_buffer.add_completed_handler(&handler);- root_command_buffer.commit();+ root_command_buffer.add_completed_handler(handler);+ root_command_buffer.submit();if should_capture {root_command_buffer.wait_until_completed();- self.gpu_capture.stop_capture("decode");+ self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;}Ok(())@@ -603,7 +602,7 @@ impl LanguageModelGenerator {allow_pre_encode: bool,sampling_method: SamplingMethod,should_fill_attention_bias: bool,- ) -> (ForwardPassState<Metal>, f64) {+ ) -> (ForwardPassState<B>, f64) {objc2::rc::autoreleasepool(|_pool| {let run_start = Instant::now();@@ -650,7 +649,7 @@ impl LanguageModelGenerator {}}- self.context.command_buffer.commit();+ root_command_buffer.submit();if allow_pre_encode {self.context.reset_command_buffer();@@ -670,7 +669,10 @@ impl LanguageModelGenerator {root_command_buffer.wait_until_completed();let run_time = run_start.elapsed().as_secs_f64();if should_capture {- self.gpu_capture.stop_capture("decode");+ self.gpu_capture+ .stop_capture(&self.context.context, "decode")+ .map_err(|_| Error::CaptureFailed)+ .expect("failed to stop capture");}(state, run_time)@@ -679,7 +681,7 @@ impl LanguageModelGenerator {fn sample(&mut self,- state: &mut ForwardPassState<Metal>,+ state: &mut ForwardPassState<B>,) -> Result<Vec<u64>, Error> {let sampling_output = state.sampling_output()@@ -704,7 +706,7 @@ impl LanguageModelGenerator {wait_until_completed: bool,) {let command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");- let root_command_buffer = command_buffer.to_owned();+ let root_command_buffer = &command_buffer;{let mut cache_layers = self.context.cache_layers.borrow_mut();@@ -716,7 +718,7 @@ impl LanguageModelGenerator {);}- command_buffer.commit();+ command_buffer.submit();if wait_until_completed {command_buffer.wait_until_completed();@@ -724,9 +726,9 @@ impl LanguageModelGenerator {}fn allow_pre_encode(&self) -> bool {- let metal_debug_active = MetalEnvVar::DeviceWrapperType.is_enabled();+ let debug_active = self.context.context.debug_active();- let result = self.decoding_config.allow_pre_encode && !metal_debug_active;+ let result = self.decoding_config.allow_pre_encode && !debug_active;result}diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rsindex abfbfd3dc..68d1ac12a 100644--- a/crates/uzu/src/language_model/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -227,7 +227,7 @@ whereNone};- let async_batch_size = decoding_config.async_batch_size.resolve(model_path);+ let async_batch_size = decoding_config.async_batch_size.resolve::<B>(model_path, context.as_ref());let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);let seed = PRng::new(decoding_config.sampling_seed.resolve());diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsindex 9604c571d..681bb6db0 100644--- a/crates/uzu/src/language_model/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -1,30 +1,27 @@use std::mem::size_of;-use metal::{BufferExt, MTLBuffer};-use objc2::{rc::Retained, runtime::ProtocolObject};-use super::LanguageModelGeneratorContext;use crate::{- backends::{common::Context, metal::Metal},+ backends::common::{Backend, Context, NativeBuffer},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::ForwardPassState,};-pub struct LanguageModelGeneratorEncodedTask {+pub struct LanguageModelGeneratorEncodedTask<B: Backend> {pub key: String,- predicate_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,+ predicate_buffer: B::NativeBuffer,}-impl LanguageModelGeneratorEncodedTask {- pub fn predicate_buffer(&self) -> &Retained<ProtocolObject<dyn MTLBuffer>> {+impl<B: Backend> LanguageModelGeneratorEncodedTask<B> {+ pub fn predicate_buffer(&self) -> &B::NativeBuffer {&self.predicate_buffer}pub fn disable_execution(&self) {+ let ptr = self.predicate_buffer.cpu_ptr().as_ptr() as *mut u32;+unsafe {- let ptr = self.predicate_buffer.contents().as_ptr() as *mut u32;*ptr = 1;- self.predicate_buffer.did_modify_range(0..size_of::<u32>());}}}@@ -81,12 +78,12 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {)}- pub fn create_state(+ pub fn create_state<B: Backend>(&self,- context: &mut LanguageModelGeneratorContext<Metal>,+ context: &mut LanguageModelGeneratorContext<B>,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,should_fill_attention_bias: bool,- ) -> ForwardPassState<Metal> {+ ) -> ForwardPassState<B> {ForwardPassState::new_llm(context.context.clone(),&context.decoder_config,@@ -110,13 +107,13 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {)}- pub fn build_encoded_task(+ pub fn build_encoded_task<B: Backend>(&self,- context: &LanguageModelGeneratorContext<Metal>,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,+ context: &LanguageModelGeneratorContext<B>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,key: String,- ) -> LanguageModelGeneratorEncodedTask {+ ) -> LanguageModelGeneratorEncodedTask<B> {context.executables.encode(state, parameters, &context.command_buffer);LanguageModelGeneratorEncodedTask {diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex 45c4ec1c1..6034560e8 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -19,6 +19,7 @@ use tokenizers::Tokenizer;use xgrammar::TokenizerInfo;use crate::{+ backends::metal::Metal,config::{MixerConfig, ModelMetadata},forward_pass::cache_layers::CacheLayer,language_model::{@@ -54,8 +55,8 @@ pub struct ChatSession {tokenizer_info: TokenizerInfo,input_processor: Box<dyn InputProcessor>,output_parser: OutputParser,- llm: Option<LanguageModelGenerator>,- static_context: Option<Context>,+ llm: Option<LanguageModelGenerator<Metal>>,+ static_context: Option<Context<Metal>>,}impl ChatSession {@@ -204,9 +205,9 @@ impl ChatSession {fn extend(&mut self,input: Input,- context: Option<&Context>,+ context: Option<&Context<Metal>>,config: RunConfig,- ) -> Result<(Output, Context), Error> {+ ) -> Result<(Output, Context<Metal>), Error> {self.reconfigure_language_model_generator(context)?;let output = self.run_internal(input, config, None::<fn(Output) -> bool>)?;let new_context = self.build_context_from_language_model_generator()?;@@ -321,7 +322,10 @@ impl ChatSession {language_model_generator.decoding_config.generate_suffix_length() == 1 && compiled_grammar.is_none();let generate_output = if can_use_async {- let batch_size = language_model_generator.decoding_config.async_batch_size.resolve(&self.model_path);+ let batch_size = language_model_generator+ .decoding_config+ .async_batch_size+ .resolve::<Metal>(&self.model_path, language_model_generator.context.context.as_ref());Self::run_async_batch(&self.tokenizer,&self.output_parser,@@ -351,7 +355,7 @@ impl ChatSession {tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,- language_model_generator: &mut LanguageModelGenerator,+ language_model_generator: &mut LanguageModelGenerator<Metal>,compiled_grammar: Option<&mut CompiledGrammar>,sampling_method: super::parameter::SamplingMethod,progress: &Option<F>,@@ -409,7 +413,7 @@ impl ChatSession {tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,- llm: &mut LanguageModelGenerator,+ llm: &mut LanguageModelGenerator<Metal>,sampling_method: super::parameter::SamplingMethod,progress: &Option<F>,batch_size: usize,@@ -516,7 +520,7 @@ impl ChatSession {fn reconfigure_language_model_generator(&mut self,- context: Option<&Context>,+ context: Option<&Context<Metal>>,) -> Result<(), Error> {let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;language_model_generator.reset_state();@@ -565,7 +569,7 @@ impl ChatSession {Ok(())}- fn build_context_from_language_model_generator(&self) -> Result<Context, Error> {+ fn build_context_from_language_model_generator(&self) -> Result<Context<Metal>, Error> {let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;let cache_layers =language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.context);@@ -581,7 +585,7 @@ impl ChatSession {impl ChatSession {fn check_finish_reason(run_context: &RunContext,- language_model_generator: &LanguageModelGenerator,+ language_model_generator: &LanguageModelGenerator<Metal>,new_tokens: &[u64],) -> Option<FinishReason> {let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;@@ -617,7 +621,7 @@ impl ChatSession {tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,- language_model_generator: &LanguageModelGenerator,+ language_model_generator: &LanguageModelGenerator<Metal>,generate_results: &[GenerateResult],generate_durations: &[f64],finish_reason: Option<FinishReason>,diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rsindex 9ff58d6b9..c1bd9e835 100644--- a/crates/uzu/src/session/classification_session.rs+++ b/crates/uzu/src/session/classification_session.rs@@ -3,6 +3,7 @@ use std::path::PathBuf;use tokenizers::Tokenizer;use crate::{+ backends::metal::Metal,classifier::{ClassificationOutput, Classifier},config::ModelMetadata,session::{@@ -17,7 +18,7 @@ pub struct ClassificationSession {#[allow(dead_code)]model_metadata: ModelMetadata,tokenizer: Tokenizer,- classifier: Classifier,+ classifier: Classifier<Metal>,input_processor: Box<dyn InputProcessor>,}diff --git a/crates/uzu/src/session/helpers/context.rs b/crates/uzu/src/session/helpers/context.rsindex fa7c63627..84ee21882 100644--- a/crates/uzu/src/session/helpers/context.rs+++ b/crates/uzu/src/session/helpers/context.rs@@ -1,15 +1,15 @@-use crate::{backends::metal::Metal, forward_pass::cache_layers::CacheLayers, session::config::DecodingConfig};+use crate::{backends::common::Backend, forward_pass::cache_layers::CacheLayers, session::config::DecodingConfig};-pub struct Context {+pub struct Context<B: Backend> {pub tokens: Vec<u64>,- pub cache_layers: CacheLayers<Metal>,+ pub cache_layers: CacheLayers<B>,pub decoding_config: DecodingConfig,}-impl Context {+impl<B: Backend> Context<B> {pub fn new(tokens: Vec<u64>,- cache_layers: CacheLayers<Metal>,+ cache_layers: CacheLayers<B>,decoding_config: DecodingConfig,) -> Self {Self {diff --git a/crates/uzu/src/session/parameter/async_batch_size.rs b/crates/uzu/src/session/parameter/async_batch_size.rsindex 4c0093c70..31139ff16 100644--- a/crates/uzu/src/session/parameter/async_batch_size.rs+++ b/crates/uzu/src/session/parameter/async_batch_size.rs@@ -1,6 +1,9 @@use std::path::Path;-use crate::utils::{DeviceClass, ModelSize};+use crate::{+ backends::common::{Backend, Context, DeviceClass},+ utils::ModelSize,+};#[derive(Debug, Clone, Copy)]pub enum AsyncBatchSize {@@ -15,15 +18,16 @@ impl Default for AsyncBatchSize {}impl AsyncBatchSize {- pub fn resolve(+ pub fn resolve<B: Backend>(&self,model_path: &Path,+ context: &B::Context,) -> usize {match self {AsyncBatchSize::Default => {- let device = DeviceClass::detect();+ let device_class = context.device_class();let model_size = ModelSize::from_path(model_path);- default_batch_size(device, model_size)+ default_batch_size(device_class, model_size)},AsyncBatchSize::Custom(value) => *value,}diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rsindex 4f172256f..33c1bd827 100644--- a/crates/uzu/src/session/types/error.rs+++ b/crates/uzu/src/session/types/error.rs@@ -46,6 +46,8 @@ pub enum Error {GrammarReject,#[error("Token {0} out of grammar vocabulary range (0..{1})")]TokenOutOfGrammarRange(u64, usize),+ #[error("Capture failed")]+ CaptureFailed,#[error("Classifier error: {0}")]Classifier(#[from] ClassifierError),}diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex bd125e868..18a6d4897 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -12,16 +12,15 @@ use std::{};use half::{bf16, f16};-use metal::{MTLCommandBuffer, MTLCommandQueue};use ndarray::{IxDyn, s};use num_traits::NumCast;use crate::{ArrayElement, DataType,array::Array,- backends::{- common::kernel::kv_cache_update::KVCacheUpdate,- metal::{Metal, MetalContext},+ backends::common::{+ Backend, CommandBuffer, Context,+ kernel::{kv_cache_update::KVCacheUpdate, matmul::MatmulKernels},},classifier::Classifier,config::ModelMetadata,@@ -156,9 +155,12 @@ pub enum ArrayTransform {// Model Context (internal)// ============================================================================-enum ModelContext {- LanguageModelGenerator(LanguageModelGeneratorContext<Metal>),- Classifier(Classifier),+enum ModelContext<B: Backend + 'static>+where+ B::Kernels: MatmulKernels,+{+ LanguageModelGenerator(LanguageModelGeneratorContext<B>),+ Classifier(Classifier<B>),}// ============================================================================@@ -169,12 +171,18 @@ enum ModelContext {////// Automatically detects whether the model is an LLM or classifier and/// runs the appropriate validation.-pub struct TraceValidator {+pub struct TraceValidator<B: Backend + 'static>+where+ B::Kernels: MatmulKernels,+{model_path: PathBuf,- context: ModelContext,+ context: ModelContext<B>,}-impl TraceValidator {+impl<B: Backend + 'static> TraceValidator<B>+where+ B::Kernels: MatmulKernels,+{/// Create a new trace validator for the given model path.////// Automatically detects the model type from config.json.@@ -202,7 +210,7 @@ impl TraceValidator {AsyncBatchSize::default(),false,);- let mut llm_context = LanguageModelGeneratorContext::<Metal>::new(model_path, &decoding_config)?;+ let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);ModelContext::LanguageModelGenerator(llm_context)@@ -240,7 +248,7 @@ impl TraceValidator {// ========================================================================fn run_llm_validation(- ctx: &LanguageModelGeneratorContext<Metal>,+ ctx: &LanguageModelGeneratorContext<B>,traces_path: &Path,) -> Result<TracerValidationResults, Error> {let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;@@ -274,10 +282,10 @@ impl TraceValidator {None,);- let command_buffer = ctx.context.command_queue.command_buffer().expect("Failed to create command buffer");+ let command_buffer = ctx.context.create_command_buffer().expect("Failed to create command buffer");ctx.executables.encode(&mut state, &EncodingParameters::new(false, false, false), &command_buffer);- command_buffer.commit();+ command_buffer.submit();command_buffer.wait_until_completed();let traces = state.traces().clone();@@ -385,7 +393,7 @@ impl TraceValidator {// ========================================================================fn run_classifier_validation(- classifier: &mut Classifier,+ classifier: &mut Classifier<B>,traces_path: &Path,) -> Result<TracerValidationResults, Error> {let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;@@ -425,7 +433,7 @@ impl TraceValidator {})}- fn handle_missing_tokens(traces_view: &ParameterTree<MetalContext>) -> TracerValidationResults {+ fn handle_missing_tokens(traces_view: &ParameterTree<B::Context>) -> TracerValidationResults {if let Ok(expected_logits) = traces_view.leaf("logits") {let reference_shape = expected_logits.shape().to_vec();let metrics = TracerValidationMetrics {@@ -470,13 +478,13 @@ impl TraceValidator {// ========================================================================fn validate_layer_traces(- traces: &Rc<RefCell<ActivationTrace<Metal>>>,- traces_view: &ParameterTree<MetalContext>,+ traces: &Rc<RefCell<ActivationTrace<B>>>,+ traces_view: &ParameterTree<B::Context>,data_type: DataType,) -> Vec<TracerValidationResult> {let mut results = Vec::new();- let validate = |path: &str, array: &Ref<Array<Metal>>| -> Option<TracerValidationResult> {+ let validate = |path: &str, array: &Ref<Array<B>>| -> Option<TracerValidationResult> {if traces_view.leaf(path).is_ok() {Some(TracerValidationResult {name: path.to_string(),@@ -539,8 +547,8 @@ impl TraceValidator {}fn validate_classifier_traces(- traces: &Rc<RefCell<ActivationTrace<Metal>>>,- traces_view: &ParameterTree<MetalContext>,+ traces: &Rc<RefCell<ActivationTrace<B>>>,+ traces_view: &ParameterTree<B::Context>,data_type: DataType,) -> Vec<TracerValidationResult> {let mut results = Vec::new();@@ -584,8 +592,8 @@ impl TraceValidator {fn validate_array(data_type: DataType,- expected_array: &Array<Metal>,- produced_array: &Ref<Array<Metal>>,+ expected_array: &Array<B>,+ produced_array: &Ref<Array<B>>,transform: Option<ArrayTransform>,) -> TracerValidationMetrics {match data_type {@@ -598,17 +606,17 @@ impl TraceValidator {fn validate_array_with_name(data_type: DataType,- traces_view: &ParameterTree<MetalContext>,+ traces_view: &ParameterTree<B::Context>,expected_array_path: &str,- produced_array: &Ref<Array<Metal>>,+ produced_array: &Ref<Array<B>>,) -> TracerValidationMetrics {let expected_array = traces_view.leaf(expected_array_path).unwrap();Self::validate_array(data_type, &expected_array, produced_array, None)}fn validate_array_of_type<Precision: ArrayElement>(- expected_array: &Array<Metal>,- produced_array: &Ref<Array<Metal>>,+ expected_array: &Array<B>,+ produced_array: &Ref<Array<B>>,transform: Option<ArrayTransform>,) -> TracerValidationMetrics {let expected_view = expected_array.as_view::<Precision>();@@ -778,7 +786,7 @@ impl TraceValidator {// ========================================================================fn load_array_as_vec<SourcePrecision: ArrayElement, TargetPrecision: NumCast>(- traces_view: &ParameterTree<MetalContext>,+ traces_view: &ParameterTree<B::Context>,name: &str,) -> Vec<TargetPrecision> {let array = traces_view.leaf(name).unwrap();@@ -803,7 +811,7 @@ impl TraceValidator {fn ensure_llm_context_capacity(decoding_config: &DecodingConfig,desired_suffix_length: usize,- context: &mut LanguageModelGeneratorContext<Metal>,+ context: &mut LanguageModelGeneratorContext<B>,) {let resolved_prefix_length = decoding_config.context_length.resolve(&context.model_config);let current_suffix_length = std::cmp::max(@@ -847,7 +855,7 @@ impl TraceValidator {.expect("Failed to create sampling kernel");}- fn get_tokens_from_logits(logits: &Array<Metal>) -> Vec<u64> {+ fn get_tokens_from_logits(logits: &Array<B>) -> Vec<u64> {let data_type = logits.data_type();match data_type {DataType::F16 => Self::get_tokens_from_logits_of_type::<f16>(logits),@@ -857,7 +865,7 @@ impl TraceValidator {}}- fn get_tokens_from_logits_of_type<Precision: ArrayElement>(logits: &Array<Metal>) -> Vec<u64> {+ fn get_tokens_from_logits_of_type<Precision: ArrayElement>(logits: &Array<B>) -> Vec<u64> {let sampler = ArgmaxSampler {};sampler.sample(logits.as_view::<Precision>())}diff --git a/crates/uzu/src/utils/device.rs b/crates/uzu/src/utils/device.rsdeleted file mode 100644index b4c317563..000000000--- a/crates/uzu/src/utils/device.rs+++ /dev/null@@ -1,61 +0,0 @@-use metal::{MTLDevice, MTLDeviceExt};--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum DeviceClass {- Ultra,- Max,- Pro,- Base,- IPhone,- Unknown,-}--impl DeviceClass {- pub fn detect() -> Self {- let device = match <dyn MTLDevice>::system_default() {- Some(dev) => dev,- None => return DeviceClass::Unknown,- };-- let name = device.name().to_lowercase();-- if name.contains("ultra") {- DeviceClass::Ultra- } else if name.contains("max") {- DeviceClass::Max- } else if name.contains("pro") {- DeviceClass::Pro- } else if name.contains("iphone") || name.contains("a1") {- DeviceClass::IPhone- } else {- DeviceClass::Base- }- }-- pub fn is_high_end(&self) -> bool {- matches!(self, DeviceClass::Ultra | DeviceClass::Max)- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_device_detection() {- let device = DeviceClass::detect();- println!("Detected device: {:?}", device);- println!("Is high-end: {}", device.is_high_end());- assert_ne!(device, DeviceClass::Unknown);- }-- #[test]- fn test_is_high_end() {- assert!(DeviceClass::Ultra.is_high_end());- assert!(DeviceClass::Max.is_high_end());- assert!(!DeviceClass::Pro.is_high_end());- assert!(!DeviceClass::Base.is_high_end());- assert!(!DeviceClass::IPhone.is_high_end());- assert!(!DeviceClass::Unknown.is_high_end());- }-}diff --git a/crates/uzu/src/utils/env_utils.rs b/crates/uzu/src/utils/env_utils.rsindex 27593aef8..7bedd5c22 100644--- a/crates/uzu/src/utils/env_utils.rs+++ b/crates/uzu/src/utils/env_utils.rs@@ -1,16 +1,14 @@#[derive(Copy, Clone, Debug)]-pub enum MetalEnvVar {- DeviceWrapperType,+pub enum EnvVar {CaptureFirstDecode,CaptureFirstPrefill,}-impl MetalEnvVar {+impl EnvVar {pub fn key(&self) -> &'static str {match self {- MetalEnvVar::DeviceWrapperType => "METAL_DEVICE_WRAPPER_TYPE",- MetalEnvVar::CaptureFirstDecode => "UZU_CAPTURE_FIRST_DECODE",- MetalEnvVar::CaptureFirstPrefill => "UZU_CAPTURE_FIRST_PREFILL",+ EnvVar::CaptureFirstDecode => "UZU_CAPTURE_FIRST_DECODE",+ EnvVar::CaptureFirstPrefill => "UZU_CAPTURE_FIRST_PREFILL",}}diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rsindex da72c7209..c18a5712d 100644--- a/crates/uzu/src/utils/mod.rs+++ b/crates/uzu/src/utils/mod.rs@@ -1,11 +1,9 @@pub mod attention;-pub mod device;pub mod env_utils;pub mod model_size;pub mod storage;pub mod version;-pub use device::DeviceClass;pub use env_utils::*;pub use model_size::ModelSize;pub use storage::*;diff --git a/crates/uzu/tests/allocator_test/caching_allocator.rs b/crates/uzu/tests/allocator_test/caching_allocator.rsindex 16e68bb9d..31773c871 100644--- a/crates/uzu/tests/allocator_test/caching_allocator.rs+++ b/crates/uzu/tests/allocator_test/caching_allocator.rs@@ -1,19 +1,19 @@use std::rc::Rc;use uzu::backends::{- common::{Buffer, BufferLifetime, Context},- metal::{Metal, MetalContext},+ common::{Backend, Buffer, BufferLifetime, Context},+ metal::Metal,};use super::allocator_trait::AllocatorTrait;pub struct CachingAllocator {- pub context: Rc<MetalContext>,+ pub context: Rc<<Metal as Backend>::Context>,}impl CachingAllocator {pub fn new() -> Self {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create MetalContext");Self {context,}diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex e817e2bac..085e0546c 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -12,16 +12,13 @@ use uzu::{DataType,backends::{common::{- CommandBuffer, Context,+ Backend, CommandBuffer, Context, Kernels,kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,attention::{AttentionGemmArguments, AttentionGemmBlock},},},- metal::{- Metal, MetalContext,- kernel::dsl::{AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},- },+ metal::Metal,},};@@ -148,7 +145,7 @@ fn create_test_data(/// Convert ndarray to Metal buffer layout expected by our kernelfn create_query_buffer(queries: &Array4<f32>,- context: &MetalContext,+ context: &<Metal as Backend>::Context,) -> Retained<ProtocolObject<dyn MTLBuffer>> {let (_batch_size, num_heads, seq_len, head_dim) = queries.dim();@@ -173,7 +170,7 @@ fn create_query_buffer(fn create_key_cache_buffer(keys: &Array4<f32>,max_seq_len: usize,- context: &MetalContext,+ context: &<Metal as Backend>::Context,) -> Retained<ProtocolObject<dyn MTLBuffer>> {let (_batch_size, num_kv_heads, seq_len, head_dim) = keys.dim();@@ -198,7 +195,7 @@ fn create_key_cache_buffer(fn create_value_cache_buffer(values: &Array4<f32>,max_seq_len: usize,- context: &MetalContext,+ context: &<Metal as Backend>::Context,) -> Retained<ProtocolObject<dyn MTLBuffer>> {let (_batch_size, num_kv_heads, seq_len, head_dim) = values.dim();@@ -223,7 +220,7 @@ fn create_value_cache_buffer(fn create_mask_buffer(mask: &Array3<f32>,num_heads: usize,- context: &MetalContext,+ context: &<Metal as Backend>::Context,) -> Retained<ProtocolObject<dyn MTLBuffer>> {let (_batch_size, seq_len, _) = mask.dim();@@ -246,7 +243,7 @@ fn create_mask_buffer(fn create_mask_2d_buffer(mask: &Array3<f32>,- context: &MetalContext,+ context: &<Metal as Backend>::Context,) -> Retained<ProtocolObject<dyn MTLBuffer>> {let (_batch_size, seq_len, _) = mask.dim();@@ -265,7 +262,7 @@ fn create_mask_2d_buffer(fn create_sinks_buffer(sinks: &[f32],- context: &MetalContext,+ context: &<Metal as Backend>::Context,) -> Retained<ProtocolObject<dyn MTLBuffer>> {context.device@@ -295,8 +292,8 @@ fn convert_kernel_output(}fn run_single_pass_attention(- kernel: &AttentionSinglePassMetalKernel,- context: &MetalContext,+ kernel: &<<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel,+ context: &<Metal as Backend>::Context,queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,@@ -366,8 +363,8 @@ fn run_single_pass_attention(}fn run_single_pass_attention_with_is_causal(- kernel: &AttentionSinglePassMetalKernel,- context: &MetalContext,+ kernel: &<<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel,+ context: &<Metal as Backend>::Context,queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,@@ -439,7 +436,7 @@ fn run_single_pass_attention_with_is_causal(fn run_gemm_attention(kernel: &AttentionGemmBlock<Metal>,- context: &MetalContext,+ context: &<Metal as Backend>::Context,queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,@@ -549,7 +546,7 @@ fn compare_results(#[test]fn test_single_pass_attention_basic() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 4;@@ -569,9 +566,16 @@ fn test_single_pass_attention_basic() {println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());let is_causal = false; // Non-causal attention for this test- let kernel =- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)- .expect("Failed to create attention single pass metal");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ false,+ false,+ false,+ is_causal,+ )+ .expect("Failed to create attention single pass metal");let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale){@@ -589,7 +593,7 @@ fn test_single_pass_attention_basic() {#[test]fn test_gemm_attention_basic() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 4;@@ -616,7 +620,7 @@ fn test_gemm_attention_basic() {#[test]fn test_gemm_attention_f32_head_dim_128() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 4;@@ -643,7 +647,7 @@ fn test_gemm_attention_f32_head_dim_128() {#[test]fn test_matrix_attention_matches_vector_and_cpu_seq256() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 8;@@ -659,9 +663,16 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);// vector attention path (single-pass)- let kernel =- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, true, true, false, is_causal)- .expect("Failed to create attention single pass metal");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ true,+ true,+ false,+ is_causal,+ )+ .expect("Failed to create attention single pass metal");let vector_output = run_single_pass_attention_with_is_causal(&kernel,&context,@@ -700,7 +711,7 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {#[test]fn test_single_pass_attention_with_mask() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 4;@@ -711,9 +722,16 @@ fn test_single_pass_attention_with_mask() {let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let kernel =- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, true, true, false, false)- .expect("Failed to create AttentionSinglePassMetalKernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ true,+ true,+ false,+ false,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");let kernel_output =match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale) {Ok(output) => output,@@ -740,7 +758,7 @@ fn test_single_pass_attention_with_mask() {#[test]fn test_single_pass_attention_with_sinks() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 4;@@ -754,9 +772,16 @@ fn test_single_pass_attention_with_sinks() {let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();println!("Using sinks: {:?}", sinks);- let kernel =- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, true, false)- .expect("Failed to create AttentionSinglePassMetalKernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ false,+ false,+ true,+ false,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);@@ -776,7 +801,7 @@ fn test_single_pass_attention_with_sinks() {#[test]fn test_single_pass_attention_with_sinks_long_sequence() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 4;@@ -790,9 +815,16 @@ fn test_single_pass_attention_with_sinks_long_sequence() {let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();println!("Long sequence sinks: {:?}", sinks);- let kernel =- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, true, false)- .expect("Failed to create AttentionSinglePassMetalKernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ false,+ false,+ true,+ false,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);@@ -817,7 +849,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {#[test]fn test_single_pass_attention_gqa() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 8;@@ -829,9 +861,16 @@ fn test_single_pass_attention_gqa() {let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);let is_causal = false; // Non-causal attention for this test- let kernel =- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)- .expect("Failed to create AttentionSinglePassMetalKernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ false,+ false,+ false,+ is_causal,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale){@@ -850,9 +889,9 @@ fn test_single_pass_attention_gqa() {}fn run_two_pass_attention(- kernel_pass1: &AttentionTwoPass1MetalKernel,- kernel_pass2: &AttentionTwoPass2MetalKernel,- context: &MetalContext,+ kernel_pass1: &<<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel,+ kernel_pass2: &<<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel,+ context: &<Metal as Backend>::Context,queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,@@ -954,7 +993,7 @@ fn run_two_pass_attention(#[test]fn test_two_pass_attention() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 8;@@ -968,11 +1007,22 @@ fn test_two_pass_attention() {let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);- let kernel_pass1 =- AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)- .expect("Failed to create AttentionTwoPass1MetalKernel");- let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)- .expect("Failed to create AttentionTwoPass2MetalKernel");+ let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ false,+ false,+ false,+ is_causal,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");+ let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");let kernel_output = match run_two_pass_attention(&kernel_pass1,&kernel_pass2,@@ -998,7 +1048,7 @@ fn test_two_pass_attention() {#[test]fn test_two_pass_attention_gqa() {- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");let batch_size = 1;let num_heads = 8;@@ -1012,11 +1062,22 @@ fn test_two_pass_attention_gqa() {let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);- let kernel_pass1 =- AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, is_causal, false, false, false)- .expect("Failed to create AttentionTwoPass1MetalKernel");- let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)- .expect("Failed to create AttentionTwoPass2MetalKernel");+ let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ is_causal,+ false,+ false,+ false,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");+ let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");let kernel_output = match run_two_pass_attention(&kernel_pass1,&kernel_pass2,@@ -1044,7 +1105,7 @@ fn test_two_pass_attention_gqa() {fn perf_two_pass_attention() {use std::time::Instant;- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");// ---- Problem sizes requiring two-pass ----let batch_size = 1;@@ -1063,11 +1124,22 @@ fn perf_two_pass_attention() {);let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);- let kernel_pass1 =- AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, is_causal, false, false, false)- .expect("Failed to create AttentionTwoPass1MetalKernel");- let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)- .expect("Failed to create AttentionTwoPass2MetalKernel");+ let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ is_causal,+ false,+ false,+ false,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");+ let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(+ &context,+ DataType::F32,+ head_dim as u32,+ )+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");// ---- Create buffers ----// For realistic inference, we only process queries for the suffix (new tokens)diff --git a/crates/uzu/tests/audio_codec_test.rs b/crates/uzu/tests/audio_codec_test.rsindex dd5e2475d..3075e4dbb 100644--- a/crates/uzu/tests/audio_codec_test.rs+++ b/crates/uzu/tests/audio_codec_test.rs@@ -6,32 +6,26 @@ use uzu::{array::ArrayContextExt,backends::{common::{- Context,+ Backend, Context, Kernels,kernel::{AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dKernel, AudioClampKernel,AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel,AudioLeakyReluKernel, AudioScaleKernel, AudioTanhKernel,},},- metal::{- MetalContext,- kernel::dsl::{- AudioAddMetalKernel, AudioCausalConv1dMetalKernel, AudioCausalConvTranspose1dMetalKernel,- AudioClampMetalKernel, AudioConv1dMetalKernel, AudioFsqDecodeMetalKernel, AudioFsqEncodeMetalKernel,- AudioHalfSnakeMetalKernel, AudioLeakyReluMetalKernel, AudioScaleMetalKernel, AudioTanhMetalKernel,- },- },+ metal::Metal,},};-fn create_test_context() -> std::rc::Rc<MetalContext> {- <MetalContext as Context>::new().expect("MetalContext")+fn create_test_context() -> std::rc::Rc<<Metal as Backend>::Context> {+ <Metal as Backend>::Context::new().expect("MetalContext")}#[test]fn audio_conv1d_replicate_matches_reference_f32() {let context = create_test_context();- let kernel = AudioConv1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioConv1dKernel::new(&context, DataType::F32)+ .expect("audio runtime");let batch_size = 2usize;let cin = 3usize;@@ -145,7 +139,8 @@ fn audio_conv1d_replicate_matches_reference_f32() {#[test]fn audio_causal_conv1d_matches_reference_f32() {let context = create_test_context();- let kernel = AudioCausalConv1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dKernel::new(&context, DataType::F32)+ .expect("audio runtime");let batch_size = 2usize;let cin = 3usize;@@ -246,7 +241,9 @@ fn audio_causal_conv1d_matches_reference_f32() {#[test]fn audio_causal_conv_transpose1d_matches_reference_f32() {let context = create_test_context();- let kernel = AudioCausalConvTranspose1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConvTranspose1dKernel::new(&context, DataType::F32)+ .expect("audio runtime");let batch_size = 2usize;let cin = 4usize;@@ -360,7 +357,8 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {#[test]fn audio_leaky_relu_matches_reference_f32() {let context = create_test_context();- let kernel = AudioLeakyReluMetalKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioLeakyReluKernel::new(&context, DataType::F32)+ .expect("audio runtime");let n = 1024usize;let slope = 0.01f32;@@ -395,7 +393,8 @@ fn audio_leaky_relu_matches_reference_f32() {#[test]fn audio_tanh_matches_reference_f32() {let context = create_test_context();- let kernel = AudioTanhMetalKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::AudioTanhKernel::new(&context, DataType::F32).expect("audio runtime");let n = 1024usize;let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();@@ -425,8 +424,10 @@ fn audio_tanh_matches_reference_f32() {#[test]fn audio_add_and_scale_match_reference_f32() {let context = create_test_context();- let add_kernel = AudioAddMetalKernel::new(&context, DataType::F32).expect("audio add kernel");- let scale_kernel = AudioScaleMetalKernel::new(&context, DataType::F32).expect("audio scale kernel");+ let add_kernel = <<Metal as Backend>::Kernels as Kernels>::AudioAddKernel::new(&context, DataType::F32)+ .expect("audio add kernel");+ let scale_kernel = <<Metal as Backend>::Kernels as Kernels>::AudioScaleKernel::new(&context, DataType::F32)+ .expect("audio scale kernel");let n = 2048usize;let scale_value = 1.0f32 / 3.0f32;@@ -479,7 +480,8 @@ fn audio_add_and_scale_match_reference_f32() {#[test]fn audio_clamp_matches_reference_f32() {let context = create_test_context();- let kernel = AudioClampMetalKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioClampKernel::new(&context, DataType::F32)+ .expect("audio runtime");let n = 2048usize;let min_value = -1.0f32;@@ -511,7 +513,8 @@ fn audio_clamp_matches_reference_f32() {#[test]fn audio_half_snake_matches_reference_f32() {let context = create_test_context();- let kernel = AudioHalfSnakeMetalKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, DataType::F32)+ .expect("audio runtime");let batch_size = 1usize;let channels = 6usize;@@ -583,7 +586,8 @@ fn audio_half_snake_matches_reference_f32() {#[test]fn audio_fsq_decode_matches_reference() {let context = create_test_context();- let kernel = AudioFsqDecodeMetalKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioFsqDecodeKernel::new(&context, DataType::F32)+ .expect("audio runtime");let batch_size = 1usize;let num_groups = 2usize;@@ -635,7 +639,8 @@ fn audio_fsq_decode_matches_reference() {#[test]fn audio_fsq_encode_matches_reference() {let context = create_test_context();- let kernel = AudioFsqEncodeMetalKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioFsqEncodeKernel::new(&context, DataType::F32)+ .expect("audio runtime");let batch_size = 1usize;let num_groups = 2usize;diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex 4ec6e04ba..488953452 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -5,8 +5,8 @@ use uzu::{DataType,array::ArrayContextExt,backends::{- common::{Context, kernel::kv_cache_update::KVCacheUpdate},- metal::{Metal, MetalContext},+ common::{Backend, Context, kernel::kv_cache_update::KVCacheUpdate},+ metal::Metal,},forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},};@@ -31,7 +31,7 @@ struct Scenario {}fn make_test_layer(- context: &MetalContext,+ context: &<Metal as Backend>::Context,state: KVCacheLayerState,prefix_capacity: usize,suffix_capacity: usize,@@ -148,7 +148,7 @@ fn collect_mask(}fn run_scenario(- context: &MetalContext,+ context: &<Metal as Backend>::Context,scenario: &Scenario,) {let mut layer =@@ -249,7 +249,7 @@ fn run_scenario(#[test]fn kv_cache_state_and_mask_scenarios() {- let Some(context) = MetalContext::new().ok() else {+ let Some(context) = <Metal as Backend>::Context::new().ok() else {return;};@@ -357,7 +357,7 @@ fn kv_cache_state_and_mask_scenarios() {#[test]fn kv_cache_slice_apply_contiguous_window() {- let Some(context) = MetalContext::new().ok() else {+ let Some(context) = <Metal as Backend>::Context::new().ok() else {return;};@@ -403,7 +403,7 @@ fn kv_cache_slice_apply_contiguous_window() {#[test]fn kv_cache_slice_apply_wrap_window() {- let Some(context) = MetalContext::new().ok() else {+ let Some(context) = <Metal as Backend>::Context::new().ok() else {return;};@@ -445,7 +445,7 @@ fn kv_cache_slice_apply_wrap_window() {#[test]fn kv_cache_slice_apply_full_restores_metadata() {- let Some(context) = MetalContext::new().ok() else {+ let Some(context) = <Metal as Backend>::Context::new().ok() else {return;};diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rsindex c2afe6b51..56c054f3a 100644--- a/crates/uzu/tests/kv_cache_update_test.rs+++ b/crates/uzu/tests/kv_cache_update_test.rs@@ -4,11 +4,11 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLResouuse ndarray::{Array, Array3, s};use uzu::backends::{common::{- Context,+ Backend, Context,gpu_types::Swap,kernel::kv_cache_update::{KVCacheUpdate, KVLayerData, create_swaps_direct},},- metal::{Metal, MetalContext},+ metal::Metal,};fn apply_swaps_3d<T: Clone>(@@ -31,7 +31,7 @@ fn apply_swaps_3d<T: Clone>(#[test]fn test_kv_cache_update_kernel() {- let metal_context = match MetalContext::new() {+ let metal_context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Failed to create MetalContext: {:?}. Skipping test.", e);@@ -42,7 +42,7 @@ fn test_kv_cache_update_kernel() {test_random_pattern(&metal_context);}-fn test_random_pattern(context: &MetalContext) {+fn test_random_pattern(context: &<Metal as Backend>::Context) {println!("Testing with random pattern...");let max_sequence_length = 256usize;diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 8d0d232f6..4bf11fe20 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -8,15 +8,15 @@ use uzu::{DataType,backends::{common::{- Context,+ Backend, Context,kernel::matmul::{MatmulArguments, MatmulKernel},},- metal::{Metal, MetalContext, kernel::matmul::choose_dispatch_descriptor},+ metal::{Metal, choose_dispatch_descriptor},},};fn run_metal_matmul(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,a_data: &[bf16],b_data: &[bf16],m: usize,@@ -274,7 +274,7 @@ fn compare_results(#[test]#[ignore]fn matmul_correctness_comprehensive() {- let Some(ctx) = MetalContext::new().ok() else {+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {eprintln!("No Metal device available, skipping test");return;};diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rsindex dbec43f84..85cb7bb02 100644--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs@@ -7,11 +7,11 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},- metal::{- MetalContext,- kernel::dsl::{MoeCountsOffsetsFusedMetalKernel, MoeRouterTopKMetalKernel},+ common::{+ Backend, Kernels,+ kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},},+ metal::Metal,},};@@ -50,7 +50,7 @@ fn cpu_offsets_from_counts(counts: &[u32]) -> (Vec<u32>, u32) {}fn gen_topk_ids_from_logits(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,t: usize,e: usize,k: usize,@@ -77,7 +77,8 @@ fn gen_topk_ids_from_logits(// Use fused router+topk kernellet cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let router_topk_encoder = cb.new_compute_command_encoder().expect("router_topk encoder");- let router_topk = MoeRouterTopKMetalKernel::new(ctx, DataType::BF16).expect("router_topk kernel");+ let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16)+ .expect("router_topk kernel");router_topk.encode(&input_buf,&weight_buf,@@ -123,7 +124,8 @@ fn test_counts_offsets_fused_parity_random() {let num_tiles = ((e + 511) / 512).max(1);let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);- let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");@@ -174,7 +176,8 @@ fn test_counts_offsets_fused_edge_cases() {let num_tiles = ((e + 511) / 512).max(1);let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);- let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);@@ -197,7 +200,8 @@ fn test_counts_offsets_fused_edge_cases() {let num_tiles = ((e + 511) / 512).max(1);let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);- let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rsindex 4eaac6417..b6a53be53 100644--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs@@ -4,18 +4,17 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::{- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,- MoeScatterBucketsMapKernel,- moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels},- },- metal::{- Metal, MetalContext,- kernel::dsl::{- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,- MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,+ common::{+ Backend, Kernels,+ kernel::{+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,+ MoeScatterBucketsMapKernel,+ moe::{+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels,+ },},},+ metal::Metal,},};@@ -169,7 +168,7 @@ fn moe_cpu_reference(// Main entry point - automatically tests both modes for T>1fn run_moe_parity_test(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,t: usize,e: usize,k: usize,@@ -233,7 +232,7 @@ fn run_moe_parity_test(}fn run_moe_parity_test_internal(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,t: usize,e: usize,k: usize,@@ -350,7 +349,8 @@ fn run_moe_parity_test_internal(let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");// Router + TopK (fused kernel)- let router_topk = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk");+ let router_topk =+ <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router+topk");let router_topk_encoder = cb.new_compute_command_encoder().expect("router+topk_encoder");router_topk.encode(&x_buf,@@ -367,13 +367,15 @@ fn run_moe_parity_test_internal();router_topk_encoder.end_encoding();- let fused_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let fused_kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let encoder = cb.new_compute_command_encoder().expect("encoder");fused_kernel.encode(&topk_ids_buf, &offsets_buf, &sumk_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);encoder.end_encoding();let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");- let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("scatter bases kernel");+ let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)+ .expect("scatter bases kernel");scatter_bases_kernel.encode(&partials_buf,&block_bases_buf,@@ -387,8 +389,9 @@ fn run_moe_parity_test_internal(scatter_bases_encoder.end_encoding();let scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");- let scatter_map_kernel = MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16)- .expect("Failed to create MoeScatterBucketsMapMetalKernel");+ let scatter_map_kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16)+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");scatter_map_kernel.encode(&topk_ids_buf,&topk_probs_buf,@@ -459,7 +462,8 @@ fn run_moe_parity_test_internal(};experts.encode(&cb, &args);- let finalize = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");+ let finalize =+ <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");let encoder = cb.new_compute_command_encoder().expect("encoder");finalize.encode(&tok2row_buf,diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rsindex 091a5b67a..07f20f91b 100644--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs@@ -6,11 +6,14 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::moe::{- MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,- MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,+ common::{+ Backend,+ kernel::moe::{+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,+ MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,+ },},- metal::{Metal, MetalContext},+ metal::Metal,},};@@ -35,7 +38,7 @@ fn build_offsets(}fn run_decode_case(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,name: &str,t: usize,d_model: usize,@@ -174,7 +177,7 @@ fn test_two_pass_decode_speed() {}fn run_two_pass_prefill_case(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,name: &str,t: usize,d_model: usize,@@ -310,7 +313,7 @@ fn test_two_pass_prefill_speed() {}fn run_fused_single_token_case(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,name: &str,d_model: usize,d_ff: usize,@@ -450,7 +453,7 @@ fn test_single_token_indirect_vs_fused() {}fn run_indirect_decode_timed(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,t: usize,d_model: usize,d_ff: usize,@@ -557,7 +560,7 @@ fn run_indirect_decode_timed(}fn run_fused_decode_timed(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,d_model: usize,d_ff: usize,e: usize,diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rsindex c74e621e6..77254d876 100644--- a/crates/uzu/tests/moe/moe_finalize_test.rs+++ b/crates/uzu/tests/moe/moe_finalize_test.rs@@ -5,7 +5,10 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,- backends::{common::kernel::MoeFinalizeKernel, metal::kernel::dsl::MoeFinalizeMetalKernel},+ backends::{+ common::{Backend, Kernels, kernel::MoeFinalizeKernel},+ metal::Metal,+ },};use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};@@ -85,7 +88,8 @@ fn test_finalize_correctness() {let y_out_buf = alloc_buffer::<bf16>(&ctx, t * d_model);// Execute finalize kernel- let finalize = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize kernel");+ let finalize = <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16)+ .expect("finalize kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");finalize.encode(diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rsindex 147494163..a81c6af36 100644--- a/crates/uzu/tests/moe/moe_perf_test.rs+++ b/crates/uzu/tests/moe/moe_perf_test.rs@@ -6,18 +6,15 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::{- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,- MoeScatterBucketsMapKernel,- moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},- },- metal::{- Metal,- kernel::dsl::{- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,- MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,+ common::{+ Backend, Kernels,+ kernel::{+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,+ MoeScatterBucketsMapKernel,+ moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},},},+ metal::Metal,},};@@ -119,7 +116,8 @@ fn test_moe_e2e_decode_perf() {let topk_ids_buf = alloc_buffer::<i32>(&ctx, t * k);let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);- let router_topk = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk fused kernel");+ let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)+ .expect("router+topk fused kernel");// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {@@ -179,7 +177,8 @@ fn test_moe_e2e_prefill_perf() {let topk_ids_buf = alloc_buffer::<i32>(&ctx, t * k);let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);- let router_topk = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk fused kernel");+ let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)+ .expect("router+topk fused kernel");// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {@@ -295,15 +294,20 @@ fn test_moe_pipeline_breakdown_decode() {let bucketed_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);// Create kernel structs (use production-validated encoding logic)- let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("counts+offsets fused");- let scatter_bases_kernel =- MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("MoeBlockBasesFromPartialsMetalKernel");+ let counts_offsets_kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("counts+offsets fused");+ let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)+ .expect("<<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel");let scatter_map_kernel =- MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16).expect("MoeScatterBucketsMapMetalKernel");+ <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16)+ .expect("<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");let gather_kernel = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("experts two-pass decode");- let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");- let router_topk_fused_kernel = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk fused");+ let finalize_kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");+ let router_topk_fused_kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)+ .expect("router+topk fused");// Testing: Router + TopK + Counts+Offsets (FUSED)let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rsindex d62a85f1f..d5145c00c 100644--- a/crates/uzu/tests/moe/moe_router_topk_test.rs+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs@@ -6,8 +6,8 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::MoeRouterTopKKernel,- metal::{MetalContext, kernel::dsl::MoeRouterTopKMetalKernel},+ common::{Backend, Kernels, kernel::MoeRouterTopKKernel},+ metal::Metal,},};@@ -128,8 +128,8 @@ pub fn cpu_topk_select_f32(}fn run_router_topk_once(- ctx: &MetalContext,- kernel: &MoeRouterTopKMetalKernel,+ ctx: &<Metal as Backend>::Context,+ kernel: &<<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel,t: usize,d_model: usize,e: usize,@@ -223,7 +223,8 @@ fn run_router_topk_once(#[test]fn test_router_topk_fused_matches_reference() {let ctx = create_ctx();- let kernel = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("kernel");+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("kernel");let configs =[(1usize, 64usize, 32usize, 4usize), (2, 128, 64, 8), (4, 256, 128, 16), (8, 256, 256, 32), (1, 512, 512, 64)];diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rsindex 2a2449b60..f47893c00 100644--- a/crates/uzu/tests/moe/moe_scatter_test.rs+++ b/crates/uzu/tests/moe/moe_scatter_test.rs@@ -6,13 +6,14 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::{- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel, MoeScatterBucketsKernel,- },- metal::kernel::dsl::{- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeRouterTopKMetalKernel,- MoeScatterBucketsMetalKernel,+ common::{+ Backend, Kernels,+ kernel::{+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel,+ MoeScatterBucketsKernel,+ },},+ metal::Metal,},};@@ -81,7 +82,8 @@ fn test_scatter_buckets_parity() {let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);// Use fused router+topk kernel- let router_topk = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router_topk");+ let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)+ .expect("router_topk");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("Failed to create encoder");router_topk.encode(@@ -112,7 +114,8 @@ fn test_scatter_buckets_parity() {let num_tiles = ((e + 511) / 512).max(1);let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);- let fused_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");+ let fused_kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let encoder = cb.new_compute_command_encoder().expect("encoder");fused_kernel.encode(@@ -135,10 +138,11 @@ fn test_scatter_buckets_parity() {let entries = num_blocks * num_tiles * 512usize;let block_bases_buf = alloc_buffer::<u32>(&ctx, entries);- let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(&ctx)- .expect("Failed to create MoeBlockBasesFromPartialsMetalKernel");- let scatter_kernel = MoeScatterBucketsMetalKernel::new(&ctx, DataType::BF16)- .expect("Failed to create MoeScatterBucketsMetalKernel");+ let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel");+ let scatter_kernel =+ <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel::new(&ctx, DataType::BF16)+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);diff --git a/crates/uzu/tests/moe/moe_test_utils.rs b/crates/uzu/tests/moe/moe_test_utils.rsindex aaf08b1be..eb6217eb5 100644--- a/crates/uzu/tests/moe/moe_test_utils.rs+++ b/crates/uzu/tests/moe/moe_test_utils.rs@@ -11,16 +11,19 @@ use bytemuck;use half::bf16;use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use objc2::{rc::Retained, runtime::ProtocolObject};-use uzu::backends::{common::Context, metal::MetalContext};+use uzu::backends::{+ common::{Backend, Context},+ metal::Metal,+};/// Create Metal context for testing-pub fn create_ctx() -> Rc<MetalContext> {- MetalContext::new().expect("Failed to create MetalContext")+pub fn create_ctx() -> Rc<<Metal as Backend>::Context> {+ <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context")}/// Helper to allocate buffer with datapub fn alloc_buffer_with_data<T: bytemuck::NoUninit>(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,data: &[T],) -> Retained<ProtocolObject<dyn MTLBuffer>> {if data.is_empty() {@@ -35,7 +38,7 @@ pub fn alloc_buffer_with_data<T: bytemuck::NoUninit>(/// Helper to allocate empty bufferpub fn alloc_buffer<T>(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,count: usize,) -> Retained<ProtocolObject<dyn MTLBuffer>> {ctx.create_buffer(count * size_of::<T>()).expect("Failed to create buffer")diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rsindex b589da660..c32a7990c 100644--- a/crates/uzu/tests/parameters_loader_test.rs+++ b/crates/uzu/tests/parameters_loader_test.rs@@ -4,14 +4,17 @@ mod common;use half::bf16;use is_close::is_close;use uzu::{- backends::{common::Context, metal::MetalContext},+ backends::{+ common::{Backend, Context},+ metal::Metal,+ },parameters::ParameterLoader,};#[test]fn test_parameter_loader_basic() {let weights_path = crate::common::get_test_weights_path();- let context = MetalContext::new().expect("Failed to create MetalContext");+ let context = <Metal as Backend>::Context::new().expect("Failed to create MetalContext");let file = std::fs::File::open(&weights_path).expect("Weights file not found; run download script");let loader = ParameterLoader::new(&file, context.as_ref()).expect("create loader");diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rsindex 039f710b5..17d4e7d40 100644--- a/crates/uzu/tests/quant_matmul_test.rs+++ b/crates/uzu/tests/quant_matmul_test.rs@@ -8,13 +8,13 @@ use uzu::{DataType,backends::{common::{- Context,+ Backend, Context,kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,QuantizedMatmulType,},},- metal::{Metal, MetalContext},+ metal::Metal,},config::QuantizationMode,};@@ -389,7 +389,7 @@ fn generate_test_quant_params(}fn buffer_from_f32_slice(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,dtype: DataType,values: &[f32],) -> Retained<ProtocolObject<dyn MTLBuffer>> {@@ -417,7 +417,7 @@ fn buffer_from_f32_slice(}fn execute_quantized_matmul(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,batch: usize,input_dim: usize,output_dim: usize,@@ -636,7 +636,7 @@ const QVM_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];fn run_kernel_test(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,batch: usize,output_dim: usize,input_dim: usize,@@ -665,7 +665,7 @@ fn run_kernel_test(#[test]fn test_quant_gmv() {- let ctx = match MetalContext::new() {+ let ctx = match <Metal as Backend>::Context::new() {Ok(c) => c,Err(_) => {println!("Metal not available — skipping QMV test");@@ -709,7 +709,7 @@ fn test_quant_gmv() {#[test]fn test_quant_qvm() {- let ctx = match MetalContext::new() {+ let ctx = match <Metal as Backend>::Context::new() {Ok(c) => c,Err(_) => {println!("Metal not available — skipping QVM test");@@ -753,7 +753,7 @@ fn test_quant_qvm() {#[test]fn test_quant_gmm() {- let ctx = match MetalContext::new() {+ let ctx = match <Metal as Backend>::Context::new() {Ok(c) => c,Err(_) => {println!("Metal not available — skipping QMM test");@@ -797,7 +797,7 @@ fn test_quant_gmm() {#[test]fn test_quant_gmm_transposed() {- let ctx = match MetalContext::new() {+ let ctx = match <Metal as Backend>::Context::new() {Ok(c) => c,Err(_) => {println!("Metal not available — skipping QMM transposed test");@@ -842,7 +842,7 @@ fn test_quant_gmm_transposed() {#[test]#[ignore]fn test_quant_matmul_perf() {- let ctx = match MetalContext::new() {+ let ctx = match <Metal as Backend>::Context::new() {Ok(c) => c,Err(_) => {println!("Metal not available — skipping Perf test");diff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rsindex 2d514f320..4f9b430f9 100644--- a/crates/uzu/tests/rms_norm_kernel_test.rs+++ b/crates/uzu/tests/rms_norm_kernel_test.rs@@ -8,13 +8,10 @@ use uzu::{DataType,backends::{common::{- CommandBuffer, Context,+ Backend, CommandBuffer, Context, Kernels,kernel::{QKNormKernel, RMSNormKernel},},- metal::{- MetalContext,- kernel::dsl::{QKNormMetalKernel, RMSNormMetalKernel},- },+ metal::Metal,},};@@ -247,7 +244,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(OutputT: TestFloat,{// Create Metal context- let mtl_context = match MetalContext::new() {+ let mtl_context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping RMS norm test: {}", e);@@ -303,8 +300,14 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(.expect("Failed to create buffer");// Create RMS norm kernel- let kernel = RMSNormMetalKernel::new(&mtl_context, input_type, scale_type, output_type, accumulation_type)- .expect("Failed to create RMS norm kernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::RMSNormKernel::new(+ &mtl_context,+ input_type,+ scale_type,+ output_type,+ accumulation_type,+ )+ .expect("Failed to create RMS norm kernel");// Create command buffer and encodelet command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();@@ -380,7 +383,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(ScaleT: TestFloat,OutputT: TestFloat,{- let mtl_context = match MetalContext::new() {+ let mtl_context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping RMS norm edge case test: {}", e);@@ -414,8 +417,14 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(.new_buffer(4 * OutputT::size_of(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let kernel = RMSNormMetalKernel::new(&mtl_context, input_type, scale_type, output_type, accumulation_type)- .expect("Failed to create RMS norm kernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::RMSNormKernel::new(+ &mtl_context,+ input_type,+ scale_type,+ output_type,+ accumulation_type,+ )+ .expect("Failed to create RMS norm kernel");let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();@@ -572,7 +581,7 @@ fn perf_rms_norm_with_size(use rand::{RngExt, SeedableRng, rngs::StdRng};// ---- Metal context ----- let mtl_context = match MetalContext::new() {+ let mtl_context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping RMS norm perf test: {}", e);@@ -583,8 +592,14 @@ fn perf_rms_norm_with_size(const EPSILON: f32 = 1e-6;// ---- Create kernel ----- let kernel = RMSNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)- .expect("Failed to create RMS norm kernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::RMSNormKernel::new(+ &mtl_context,+ DataType::F32,+ DataType::F32,+ DataType::F32,+ DataType::F32,+ )+ .expect("Failed to create RMS norm kernel");// ---- Generate random data ----let mut rng = StdRng::seed_from_u64(42);@@ -698,7 +713,7 @@ fn perf_rms_norm_16k() {#[test]fn qk_norm_test() {// Test to verify that the QK norm kernel now accesses the correct data ranges- let mtl_context = match MetalContext::new() {+ let mtl_context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping QK norm buffer addressing test: {}", e);@@ -766,11 +781,23 @@ fn qk_norm_test() {.expect("Failed to create buffer");// Create QK norm kernels- let q_kernel = QKNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)- .expect("Failed to create Q norm kernel");-- let k_kernel = QKNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)- .expect("Failed to create K norm kernel");+ let q_kernel = <<Metal as Backend>::Kernels as Kernels>::QKNormKernel::new(+ &mtl_context,+ DataType::F32,+ DataType::F32,+ DataType::F32,+ DataType::F32,+ )+ .expect("Failed to create Q norm kernel");++ let k_kernel = <<Metal as Backend>::Kernels as Kernels>::QKNormKernel::new(+ &mtl_context,+ DataType::F32,+ DataType::F32,+ DataType::F32,+ DataType::F32,+ )+ .expect("Failed to create K norm kernel");// Test Q head normalization{diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rsindex f89ae0bd1..231fd027f 100644--- a/crates/uzu/tests/sampling_test.rs+++ b/crates/uzu/tests/sampling_test.rs@@ -7,18 +7,13 @@ use uzu::{DataType,backends::{common::{- CommandBuffer, Context,+ Backend, CommandBuffer, Context, Kernels,kernel::{GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel, TopPKernel,sampling::{ArgmaxStrategy, SamplingKernel},},},- metal::{- Metal, MetalContext,- kernel::dsl::{- GumbelMetalKernel, MinPMetalKernel, TemperatureMetalKernel, TopKMetalKernel, TopPMetalKernel,- },- },+ metal::Metal,},language_model::gumbel::{gumbel_float, revidx},session::parameter::SamplingMethod,@@ -80,7 +75,7 @@ fn cpu_reference_min_p(}fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping argmax test: {}", e);@@ -171,7 +166,7 @@ fn test_topp_sampling_from_prob_exact_match(use rand::{SeedableRng, rngs::StdRng};- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {eprintln!("Skipping topP exact match test: {}", e);@@ -310,7 +305,7 @@ fn test_topp_sampling_statistical_large() {use rand::{RngExt, SeedableRng, rngs::StdRng};// ===== 1. Create Metal context =====- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping large statistical test: {}", e);@@ -427,7 +422,7 @@ fn perf_topp_128k_vocab() {use rand::{RngExt, SeedableRng, rngs::StdRng};// ---- Metal context ----- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping perf test: {}", e);@@ -532,7 +527,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {use rand::{RngExt, SeedableRng, rngs::StdRng};// ---- Metal context ----- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping argmax perf test: {}", e);@@ -647,7 +642,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {#[test]fn test_categorical_sampling() {- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping categorical test: {}", e);@@ -769,7 +764,7 @@ fn test_categorical_sampling() {fn test_categorical_sampling_statistical() {use rand::{RngExt, SeedableRng, rngs::StdRng};- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping categorical statistical test: {}", e);@@ -900,7 +895,7 @@ fn perf_categorical_128k_vocab() {use rand::{RngExt, SeedableRng, rngs::StdRng};- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping categorical perf test: {}", e);@@ -997,7 +992,7 @@ fn perf_categorical_128k_vocab() {#[test]fn test_temperature_gpu_cpu_match() {- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping temperature gpu cpu match test: {}", e);@@ -1011,7 +1006,8 @@ fn test_temperature_gpu_cpu_match() {const RTOL: f32 = 1e-6;const ATOL: f32 = 1e-6;- let kernel = TemperatureMetalKernel::new(&context, DataType::F32).expect("Failed to create temperature kernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::TemperatureKernel::new(&context, DataType::F32)+ .expect("Failed to create temperature kernel");let logits: Vec<f32> = (0..BATCH * VOCAB).map(|i| ((i * 37 % 1000) as f32 - 500.0) * 0.01).collect();@@ -1060,7 +1056,7 @@ fn test_temperature_gpu_cpu_match() {fn test_topk_gpu_cpu_match() {use rand::{SeedableRng, rngs::StdRng};- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping topk gpu cpu match test: {}", e);@@ -1072,7 +1068,8 @@ fn test_topk_gpu_cpu_match() {const VOCAB: usize = 1024;const TOPK: u32 = 16;- let kernel = TopKMetalKernel::new(&context, DataType::F32).expect("Failed to create topk kernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::TopKKernel::new(&context, DataType::F32)+ .expect("Failed to create topk kernel");let mut rng = StdRng::seed_from_u64(42);let mut logits = vec![0.0f32; BATCH * VOCAB];@@ -1124,7 +1121,7 @@ fn test_topk_gpu_cpu_match() {fn test_topp_gpu_cpu_match() {use rand::{SeedableRng, rngs::StdRng};- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping topp gpu cpu match test: {}", e);@@ -1136,7 +1133,8 @@ fn test_topp_gpu_cpu_match() {const VOCAB: usize = 1024;const TOPP: f32 = 0.9;- let kernel = TopPMetalKernel::new(&context, DataType::F32).expect("Failed to create topp kernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::TopPKernel::new(&context, DataType::F32)+ .expect("Failed to create topp kernel");let mut rng = StdRng::seed_from_u64(42);let mut logits = vec![0.0f32; BATCH * VOCAB];@@ -1193,7 +1191,7 @@ fn test_topp_gpu_cpu_match() {fn test_minp_gpu_cpu_match() {use rand::{RngExt, SeedableRng, rngs::StdRng};- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping minp gpu cpu match test: {}", e);@@ -1205,7 +1203,8 @@ fn test_minp_gpu_cpu_match() {const VOCAB: usize = 1024;const MINP: f32 = 0.1;- let kernel = MinPMetalKernel::new(&context, DataType::F32).expect("Failed to create minp kernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::MinPKernel::new(&context, DataType::F32)+ .expect("Failed to create minp kernel");let mut rng = StdRng::seed_from_u64(42);let mut logits = vec![0.0f32; BATCH * VOCAB];@@ -1257,7 +1256,7 @@ fn test_minp_sampling_exact_match(use rand::{SeedableRng, rngs::StdRng};- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {eprintln!("Skipping minp exact match test: {}", e);@@ -1376,7 +1375,7 @@ fn test_minp_sampling_match_large() {#[test]fn test_gumbel_gpu_cpu_match() {- let context = match MetalContext::new() {+ let context = match <Metal as Backend>::Context::new() {Ok(ctx) => ctx,Err(e) => {println!("Skipping gumbel gpu cpu match test: {}", e);@@ -1389,7 +1388,8 @@ fn test_gumbel_gpu_cpu_match() {const RTOL: f32 = 0.01;const ATOL: f32 = 1e-6;- let kernel = GumbelMetalKernel::new(&context, DataType::F32).expect("Failed to create gumbel kernel");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::GumbelKernel::new(&context, DataType::F32)+ .expect("Failed to create gumbel kernel");let logits = vec![0.0f32; BATCH * VOCAB];let seeds: Vec<u64> = (0_u64..BATCH as u64).collect();diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rsindex 436d552e7..e33fbd8da 100644--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs@@ -12,13 +12,13 @@ use uzu::{DataType,backends::{common::{- Context,+ Backend, Context, Kernels,kernel::{Conv1dScanKernel,ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},},- metal::{Metal, MetalContext, kernel::dsl::Conv1dScanMetalKernel},+ metal::Metal,},};@@ -198,7 +198,7 @@ impl SSDPrefillFixture {}fn run_prefill_kernel_mode(- ctx: &MetalContext,+ ctx: &<Metal as Backend>::Context,kernel: &SSDPrefillKernels<Metal>,fixture: &SSDPrefillFixture,mode: SSDPrefillMode,@@ -261,8 +261,8 @@ fn run_prefill_kernel_mode(}fn run_conv_scan_once(- ctx: &MetalContext,- kernel: &Conv1dScanMetalKernel,+ ctx: &<Metal as Backend>::Context,+ kernel: &<<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel,suffix_len: usize,channels: usize,kernel_size: i32,@@ -359,7 +359,7 @@ fn run_conv_scan_once(}fn assert_deterministic_for_mode(mode: SSDPrefillMode) {- let Some(ctx) = MetalContext::new().ok() else {+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {eprintln!("Skipping SSD prefill determinism test: no Metal device");return;};@@ -374,7 +374,7 @@ fn assert_deterministic_for_mode(mode: SSDPrefillMode) {}fn assert_matches_cpu_reference(mode: SSDPrefillMode) {- let Some(ctx) = MetalContext::new().ok() else {+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {eprintln!("Skipping SSD prefill reference test: no Metal device");return;};@@ -460,11 +460,12 @@ fn ssd_prefill_single_pass_matches_cpu_reference() {#[test]fn conv1d_scan_is_deterministic() {- let Some(ctx) = MetalContext::new().ok() else {+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {eprintln!("Skipping conv1d scan determinism test: no Metal device");return;};- let kernel = Conv1dScanMetalKernel::new(&ctx, DataType::F32, 0u32, true).unwrap();+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, 0u32, true).unwrap();let suffix_len = 192usize;let channels = 8usize;diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rsindex 222bbeb7a..e952f8eb5 100644--- a/crates/uzu/tests/ssm_update_test.rs+++ b/crates/uzu/tests/ssm_update_test.rs@@ -6,8 +6,8 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLuse uzu::{DataType,backends::{- common::{Context, kernel::SSDUpdateKernel},- metal::{MetalContext, kernel::dsl::SSDUpdateMetalKernel},+ common::{Backend, Context, Kernels, kernel::SSDUpdateKernel},+ metal::Metal,},};@@ -174,7 +174,7 @@ fn ssd_update_ref_bf16(#[test]fn ssd_update_with_z_bf16() {- let Some(ctx) = MetalContext::new().ok() else {+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {eprintln!("Skipping: no Metal device");return;};@@ -238,7 +238,7 @@ fn ssd_update_with_z_bf16() {.new_buffer(bsz * h * dh * n * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let kernel = SSDUpdateMetalKernel::new(&ctx, DataType::BF16).unwrap();+ let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16).unwrap();let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");diff --git a/crates/uzu/tests/tracer_test.rs b/crates/uzu/tests/tracer_test.rsindex a8928a7f2..d78365e79 100644--- a/crates/uzu/tests/tracer_test.rs+++ b/crates/uzu/tests/tracer_test.rs@@ -3,7 +3,7 @@mod common;use std::path::PathBuf;-use uzu::tracer::TraceValidator;+use uzu::{backends::metal::Metal, tracer::TraceValidator};fn build_model_path() -> PathBuf {common::get_test_model_path()@@ -19,7 +19,7 @@ fn test_tracer() {return;}- let mut tracer = match TraceValidator::new(&model_path) {+ let mut tracer = match TraceValidator::<Metal>::new(&model_path) {Ok(t) => t,Err(e) => {println!("Failed to create TraceValidator: {:?}", e);