Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

language model generalization (#217)

uuuvnfc0db29a5a19
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rs
index 9faeb13ab..62f878678 100644
--- a/crates/uzu/src/backends/common/backend.rs
+++ b/crates/uzu/src/backends/common/backend.rs
@@ -11,4 +11,6 @@ pub trait Backend: Debug + Clone {
type Event: Event<Backend = Self>;
type Kernels: Kernels<Backend = Self>;
type Error: Error + Debug;
+
+ const MAX_INLINE_BYTES: usize;
}
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index 7bd73f81e..ad3959698 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -1,6 +1,6 @@
use super::Backend;
-pub trait CommandBuffer {
+pub trait CommandBuffer: Clone {
type Backend: Backend;
fn with_compute_encoder<T>(
@@ -13,6 +13,23 @@ pub trait CommandBuffer {
callback: impl FnOnce(&<Self::Backend as Backend>::CopyEncoder) -> T,
) -> T;
+ fn encode_wait_for_event(
+ &self,
+ event: &<Self::Backend as Backend>::Event,
+ value: u64,
+ );
+
+ fn encode_signal_event(
+ &self,
+ event: &<Self::Backend as Backend>::Event,
+ value: u64,
+ );
+
+ fn add_completed_handler(
+ &self,
+ handler: impl Fn() + 'static,
+ );
+
fn submit(&self);
fn wait_until_completed(&self);
diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rs
index 52eda7038..ca5a27cff 100644
--- a/crates/uzu/src/backends/common/context.rs
+++ b/crates/uzu/src/backends/common/context.rs
@@ -1,12 +1,31 @@
-use std::rc::Rc;
+use std::{path::Path, rc::Rc};
use super::{Allocator, Backend};
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DeviceClass {
+ Ultra,
+ Max,
+ Pro,
+ Base,
+ IPhone,
+}
+
+impl DeviceClass {
+ pub fn is_high_end(&self) -> bool {
+ matches!(self, DeviceClass::Ultra | DeviceClass::Max)
+ }
+}
+
pub trait Context: Sized {
type Backend: Backend<Context = Self>;
fn new() -> Result<Rc<Self>, <Self::Backend as Backend>::Error>;
+ fn device_class(&self) -> DeviceClass;
+
+ fn debug_active(&self) -> bool;
+
fn allocator(&self) -> &Allocator<Self::Backend>;
fn create_buffer(
@@ -19,4 +38,13 @@ pub trait Context: Sized {
) -> Result<<Self::Backend as Backend>::CommandBuffer, <Self::Backend as Backend>::Error>;
fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;
+
+ fn enable_capture();
+
+ fn start_capture(
+ &self,
+ trace_path: &Path,
+ ) -> Result<(), <Self::Backend as Backend>::Error>;
+
+ fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;
}
diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rs
index bd1fbd8e2..6155bcddf 100644
--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs
+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs
@@ -1,11 +1,11 @@
//! GEMM kernel parameter structs.
//!
//! Field names use uppercase for matrix dimensions (M, N, K) to match
-//! standard BLAS/GEMM convention and generated Metal headers.
+//! standard BLAS/GEMM convention and generated shader headers.
#![allow(non_snake_case)]
-/// Main GEMM parameters passed to Metal kernels as constant buffer.
+/// Main GEMM parameters passed to backend kernels as constant buffer.
#[repr(C)]
#[derive(Debug, Default, Copy, Clone)]
pub struct GEMMParams {
diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
index 4f56924ed..c8ba798f4 100644
--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -38,10 +38,6 @@ pub enum KVCacheUpdateError<B: Backend> {
}
impl<B: Backend> KVCacheUpdate<B> {
- // Metal's set_bytes supports up to 4KB per bound value.
- // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objc
- const MAX_INLINE_BYTES: usize = 4096;
-
pub fn new(
context: &B::Context,
data_type: DataType,
@@ -84,7 +80,7 @@ impl<B: Backend> KVCacheUpdate<B> {
if swaps.len() > self.max_sequence_length {
return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);
}
- let max_inline_swaps = (Self::MAX_INLINE_BYTES / size_of::<Swap>()).max(1);
+ let max_inline_swaps = (B::MAX_INLINE_BYTES / size_of::<Swap>()).max(1);
for layer_data in in_place_data {
if layer_data.key_shape != layer_data.value_shape {
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
index 7936a116d..c82d14345 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
@@ -85,7 +85,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
);
let d_model_u32 = args.d_model as u32;
let d_ff_u32 = args.d_ff as u32;
- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel
+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in kernel
const COL_TILE_MODEL: u32 = 64;
let n_tiles_ff = if d_ff_u32 == 0 {
0
diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rs
index c5cab7be9..9b2caf29a 100644
--- a/crates/uzu/src/backends/common/mod.rs
+++ b/crates/uzu/src/backends/common/mod.rs
@@ -15,7 +15,7 @@ pub use backend::Backend;
pub use buffer::Buffer;
pub use buffer_lifetime::BufferLifetime;
pub use command_buffer::CommandBuffer;
-pub use context::Context;
+pub use context::{Context, DeviceClass};
pub use copy_encoder::CopyEncoder;
pub use event::Event;
pub use kernel::Kernels;
diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rs
index 1a5186643..fc5d832ee 100644
--- a/crates/uzu/src/backends/metal/backend.rs
+++ b/crates/uzu/src/backends/metal/backend.rs
@@ -16,4 +16,8 @@ impl Backend for Metal {
type Event = Retained<ProtocolObject<dyn MTLEvent>>;
type Kernels = MetalKernels;
type Error = MetalError;
+
+ // Metal's set_bytes supports up to 4KB per bound value.
+ // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objc
+ const MAX_INLINE_BYTES: usize = 4096;
}
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index 958b9a5c3..bf564978c 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -1,6 +1,6 @@
use std::ops::Deref;
-use metal::{MTLCommandBuffer, MTLCommandBufferExt, MTLCommandEncoder};
+use metal::{MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler, MTLCommandEncoder, MTLEvent};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::Metal;
@@ -35,6 +35,29 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
ret
}
+ fn encode_wait_for_event(
+ &self,
+ event: &Retained<ProtocolObject<dyn MTLEvent>>,
+ value: u64,
+ ) {
+ self.encode_wait_for_event_value(event, value);
+ }
+
+ fn encode_signal_event(
+ &self,
+ event: &Retained<ProtocolObject<dyn MTLEvent>>,
+ value: u64,
+ ) {
+ self.encode_signal_event_value(event, value);
+ }
+
+ fn add_completed_handler(
+ &self,
+ handler: impl Fn() + 'static,
+ ) {
+ self.deref().add_completed_handler(&MTLCommandBufferHandler::new(move |_| handler()));
+ }
+
fn submit(&self) {
self.commit();
}
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index ca3e45905..930233183 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -1,13 +1,14 @@
use std::{cell::RefCell, collections::HashMap, env, rc::Rc};
use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,
- MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions,
+ MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandBuffer, MTLCommandQueue,
+ MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues,
+ MTLLibrary, MTLResourceOptions,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{Metal, error::MetalError, kernel, metal_extensions::LibraryPipelineExtensions};
-use crate::backends::common::{Allocator, Context};
+use crate::backends::common::{Allocator, Context, DeviceClass as CommonDeviceClass};
/// Apple GPU architecture generation.
/// Based on Apple GPU family naming convention (e.g., "applegpu_g13p").
@@ -200,6 +201,27 @@ impl Context for MetalContext {
}))
}
+ fn device_class(&self) -> CommonDeviceClass {
+ let name = self.device.name().to_lowercase();
+
+ if name.contains("ultra") {
+ CommonDeviceClass::Ultra
+ } else if name.contains("max") {
+ CommonDeviceClass::Max
+ } else if name.contains("pro") {
+ CommonDeviceClass::Pro
+ } else if name.contains("iphone") || name.contains("a1") {
+ CommonDeviceClass::IPhone
+ } else {
+ CommonDeviceClass::Base
+ }
+ }
+
+ fn debug_active(&self) -> bool {
+ let upper = std::env::var("METAL_DEVICE_WRAPPER_TYPE").unwrap_or_default().to_ascii_uppercase();
+ matches!(upper.as_str(), "1" | "YES" | "TRUE")
+ }
+
fn allocator(&self) -> &Allocator<Metal> {
&self.allocator
}
@@ -220,4 +242,35 @@ impl Context for MetalContext {
fn create_event(&self) -> Result<Retained<ProtocolObject<dyn MTLEvent>>, MetalError> {
self.device.new_event().ok_or(MetalError::Generic("cannot create event".into()))
}
+
+ fn enable_capture() {
+ unsafe {
+ std::env::set_var("METAL_CAPTURE_ENABLED", "1");
+ }
+ }
+
+ fn start_capture(
+ &self,
+ trace_path: &std::path::Path,
+ ) -> Result<(), <Self::Backend as crate::backends::common::Backend>::Error> {
+ let capture_manager = MTLCaptureManager::shared_capture_manager();
+ let capture_descriptor = MTLCaptureDescriptor::new();
+ capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);
+ capture_descriptor.set_output_path(Some(trace_path));
+
+ self.command_queue.set_label(Some("uzu_command_queue"));
+ capture_descriptor.set_capture_object(Some(self.command_queue.as_ref()));
+
+ capture_manager
+ .start_capture_with_descriptor_error(&capture_descriptor)
+ .map_err(|e| format!("Failed to start GPU capture: {}", e))?;
+
+ Ok(())
+ }
+
+ fn stop_capture(&self) -> Result<(), <Self::Backend as crate::backends::common::Backend>::Error> {
+ MTLCaptureManager::shared_capture_manager().stop_capture();
+
+ Ok(())
+ }
}
diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.h
index db0130600..ac8bfeba5 100644
--- a/crates/uzu/src/backends/metal/generated/matmul.h
+++ b/crates/uzu/src/backends/metal/generated/matmul.h
@@ -14,7 +14,7 @@ namespace matmul {
#include <stdint.h>
#endif
-/**Main GEMM parameters passed to Metal kernels as constant buffer. */
+/**Main GEMM parameters passed to backend kernels as constant buffer. */
typedef struct {
int32_t M;
int32_t N;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
deleted file mode 100644
index 3ce63cb23..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub use crate::backends::common::gpu_types::{GEMMParams, GEMMSpiltKParams};
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index d0464c58c..b13b6a219 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -1,4 +1,3 @@
-pub mod common;
mod dispatch_descriptor;
mod gemm;
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 8b6b5b189..64ed3aecb 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -4,9 +4,9 @@ mod context;
mod copy_encoder;
mod error;
mod event;
-pub mod kernel;
+mod kernel;
mod metal_extensions;
mod native_buffer;
pub use backend::Metal;
-pub use context::MetalContext;
+pub use kernel::matmul::choose_dispatch_descriptor;
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index fc576252c..941c994b3 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -2,7 +2,6 @@
use std::{cell::RefCell, rc::Rc};
use std::{collections::HashMap, path::Path, time::Instant};
-use metal::MTLCommandBuffer;
use objc2::rc::autoreleasepool;
#[cfg(feature = "tracing")]
@@ -10,17 +9,23 @@ use super::ActivationTrace;
use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
use crate::{
DataType,
- backends::metal::Metal,
+ backends::common::{Backend, CommandBuffer, kernel::matmul::MatmulKernels},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::{ArrayId, ForwardPassState},
session::types::Error,
};
-pub struct Classifier {
- pub context: ClassifierContext<Metal>,
+pub struct Classifier<B: Backend + 'static>
+where
+ B::Kernels: MatmulKernels,
+{
+ pub context: ClassifierContext<B>,
}
-impl Classifier {
+impl<B: Backend + 'static> Classifier<B>
+where
+ B::Kernels: MatmulKernels,
+{
pub fn new(model_path: &Path) -> Result<Self, Error> {
let context = ClassifierContext::new(model_path)?;
Ok(Self {
@@ -79,7 +84,7 @@ impl Classifier {
&mut self,
token_ids: &[u64],
token_positions: &[usize],
- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<Metal>>>), Error> {
+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {
self.forward_pass(token_ids, token_positions)
}
@@ -88,7 +93,7 @@ impl Classifier {
&mut self,
token_ids: &[u64],
token_positions: &[usize],
- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<Metal>>>), Error> {
+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {
autoreleasepool(|_| {
let num_labels = self.context.model_config.model_config.num_labels;
let mut state = ForwardPassState::new_classifier(
@@ -136,7 +141,7 @@ impl Classifier {
self.context.prediction_head.encode(&mut state, &encoding_params, &self.context.command_buffer);
- self.context.command_buffer.commit();
+ self.context.command_buffer.submit();
self.context.command_buffer.wait_until_completed();
let logits = self.copy_logits_from_state(&state)?;
@@ -178,7 +183,7 @@ impl Classifier {
self.context.pooling.encode(&mut state, &encoding_params, &self.context.command_buffer);
self.context.prediction_head.encode(&mut state, &encoding_params, &self.context.command_buffer);
- self.context.command_buffer.commit();
+ self.context.command_buffer.submit();
self.context.command_buffer.wait_until_completed();
let logits = self.copy_logits_from_state(&state)?;
@@ -189,7 +194,7 @@ impl Classifier {
fn copy_logits_from_state(
&self,
- state: &ForwardPassState<Metal>,
+ state: &ForwardPassState<B>,
) -> Result<Box<[f32]>, Error> {
let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
let logits_array = logits_arrays[0].borrow();
diff --git a/crates/uzu/src/language_model/gpu_capture.rs b/crates/uzu/src/language_model/gpu_capture.rs
index a16b94014..d7e13f5ef 100644
--- a/crates/uzu/src/language_model/gpu_capture.rs
+++ b/crates/uzu/src/language_model/gpu_capture.rs
@@ -1,29 +1,30 @@
use std::{
+ marker::PhantomData,
path::PathBuf,
time::{SystemTime, UNIX_EPOCH},
};
-use metal::{MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueueExt};
-
-use crate::{backends::metal::MetalContext, utils::env_utils::MetalEnvVar};
+use crate::{
+ backends::common::{Backend, Context},
+ utils::env_utils::EnvVar,
+};
-pub struct GpuCaptureManager {
+pub struct GpuCaptureManager<B: Backend> {
capture_prefill_enabled: bool,
capture_decode_enabled: bool,
first_prefill_captured: bool,
first_decode_captured: bool,
+ phantom: PhantomData<B>,
}
-impl GpuCaptureManager {
+impl<B: Backend> GpuCaptureManager<B> {
pub fn new() -> Self {
- let capture_prefill_enabled = MetalEnvVar::CaptureFirstPrefill.is_enabled();
- let capture_decode_enabled = MetalEnvVar::CaptureFirstDecode.is_enabled();
+ let capture_prefill_enabled = EnvVar::CaptureFirstPrefill.is_enabled();
+ let capture_decode_enabled = EnvVar::CaptureFirstDecode.is_enabled();
- // Enable Metal capture layer BEFORE device creation if any capture is requested
+ // Enable backend capture BEFORE device creation if any capture is requested
if capture_prefill_enabled || capture_decode_enabled {
- unsafe {
- std::env::set_var("METAL_CAPTURE_ENABLED", "1");
- }
+ B::Context::enable_capture();
}
Self {
@@ -31,6 +32,7 @@ impl GpuCaptureManager {
capture_decode_enabled,
first_prefill_captured: false,
first_decode_captured: false,
+ phantom: PhantomData,
}
}
@@ -50,7 +52,7 @@ impl GpuCaptureManager {
pub fn start_capture(
&self,
- mtl_context: &MetalContext,
+ context: &B::Context,
capture_type: &str,
) -> Result<PathBuf, String> {
let timestamp =
@@ -60,20 +62,7 @@ impl GpuCaptureManager {
.unwrap_or_else(|_| PathBuf::from("."))
.join(format!("uzu_first_{}-{}.gputrace", capture_type, timestamp));
- let capture_manager = MTLCaptureManager::shared_capture_manager();
- let capture_descriptor = MTLCaptureDescriptor::new();
- capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);
- capture_descriptor.set_output_path(Some(&trace_path));
-
- mtl_context.command_queue.set_label(Some("uzu_command_queue"));
- use objc2::rc::Retained;
- unsafe {
- capture_descriptor.set_capture_object(Some(&*(Retained::as_ptr(&mtl_context.command_queue).cast())));
- }
-
- capture_manager
- .start_capture_with_descriptor_error(&capture_descriptor)
- .map_err(|e| format!("Failed to start GPU capture: {}", e))?;
+ context.start_capture(&trace_path).map_err(|err| err.to_string())?;
println!("GPU capture started for first {}: {:?}", capture_type, trace_path);
@@ -82,9 +71,10 @@ impl GpuCaptureManager {
pub fn stop_capture(
&mut self,
+ context: &B::Context,
capture_type: &str,
- ) {
- MTLCaptureManager::shared_capture_manager().stop_capture();
+ ) -> Result<(), String> {
+ context.stop_capture().map_err(|err| err.to_string())?;
println!("GPU capture stopped for {}", capture_type);
match capture_type {
@@ -92,6 +82,8 @@ impl GpuCaptureManager {
"decode" => self.first_decode_captured = true,
_ => {},
}
+
+ Ok(())
}
pub fn reset(&mut self) {
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 8c0274fbf..cce1f33e7 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -1,7 +1,6 @@
use std::{collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant};
use itertools::{Either, Itertools, izip};
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler, MTLCommandEncoder};
use super::{
LanguageModelGeneratorContext,
@@ -12,12 +11,9 @@ use super::{
tasks::{LanguageModelGeneratorEncodedTask, LanguageModelGeneratorRunTask},
};
use crate::{
- backends::{
- common::{
- Context,
- kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},
- },
- metal::Metal,
+ backends::common::{
+ Backend, CommandBuffer, Context, NativeBuffer,
+ kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, matmul::MatmulKernels},
},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::{
@@ -30,20 +26,25 @@ use crate::{
types::Error,
},
trie::{TrieCreationConfig, TrieNode},
- utils::env_utils::MetalEnvVar,
};
-pub struct LanguageModelGenerator {
+pub struct LanguageModelGenerator<B: Backend + 'static>
+where
+ B::Kernels: MatmulKernels,
+{
pub decoding_config: DecodingConfig,
pub tokens: Vec<u64>,
- pub context: LanguageModelGeneratorContext<Metal>,
- encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask>,
+ pub context: LanguageModelGeneratorContext<B>,
+ encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask<B>>,
registered_prefix_len: usize,
- gpu_capture: GpuCaptureManager,
+ gpu_capture: GpuCaptureManager<B>,
}
-impl LanguageModelGenerator {
+impl<B: Backend + 'static> LanguageModelGenerator<B>
+where
+ B::Kernels: MatmulKernels,
+{
pub fn new(
model_path: &Path,
decoding_config: DecodingConfig,
@@ -119,7 +120,7 @@ impl LanguageModelGenerator {
let token_seeds = repeat_n(0, tokens_length - 1).chain(flat_trie.token_seeds()).chunks(prefill_step_size);
- let mut last_state: Option<ForwardPassState<Metal>> = None;
+ let mut last_state: Option<ForwardPassState<B>> = None;
let mut run_times: Vec<f64> = Vec::new();
// Process each prefill step and update the KV cache.
@@ -201,7 +202,7 @@ impl LanguageModelGenerator {
);
if should_capture {
- self.gpu_capture.stop_capture("prefill");
+ self.gpu_capture.stop_capture(&self.context.context, "prefill").map_err(|_| Error::CaptureFailed)?;
}
// Register the accepted prompt tokens from this step.
@@ -393,7 +394,7 @@ impl LanguageModelGenerator {
let is_continuation = current_counter > 0;
let batch_size = self.context.async_buffers.batch_size;
let slot = pass_idx % batch_size;
- let async_event = self.context.async_buffers.event.clone();
+
let results_buffer = self.context.async_buffers.results.clone();
let async_positions_buffer = self.context.async_buffers.positions.clone();
let async_seeds_buffer = self.context.async_buffers.seeds.clone();
@@ -401,7 +402,7 @@ impl LanguageModelGenerator {
let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;
let token_position = unsafe {
- let ptr = async_positions_buffer.contents().as_ptr() as *const u32;
+ let ptr = async_positions_buffer.cpu_ptr().as_ptr() as *const u32;
*ptr.add(pass_idx) as usize
};
@@ -455,11 +456,11 @@ impl LanguageModelGenerator {
}
self.context.reset_command_buffer();
- let root_command_buffer = self.context.command_buffer.clone();
+ let root_command_buffer = &self.context.command_buffer;
// Wait on previous pass if this is a continuation
if is_continuation {
- root_command_buffer.encode_wait_for_event_value(&async_event, current_counter);
+ root_command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);
}
self.context.executables.encode(
@@ -481,67 +482,65 @@ impl LanguageModelGenerator {
let sampling_output_buffer = sampling_output.borrow().buffer().clone();
let token_ids_buffer = self.context.scratch_buffers.token_ids.borrow().buffer().clone();
- let encoder =
- root_command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.context.token_copy_sampled.encode(&sampling_output_buffer, &token_ids_buffer, &encoder);
- let results_offset = slot * std::mem::size_of::<u32>();
- self.context.token_copy_results.encode(&sampling_output_buffer, (&results_buffer, results_offset), &encoder);
- encoder.end_encoding();
+ root_command_buffer.with_compute_encoder(|encoder| {
+ self.context.token_copy_sampled.encode(&sampling_output_buffer, &token_ids_buffer, encoder);
+ let results_offset = slot * std::mem::size_of::<u32>();
+ self.context.token_copy_results.encode(&sampling_output_buffer, (&results_buffer, results_offset), encoder);
+ });
// Scatter + register for all transformer layers
- {
- let cb = root_command_buffer.clone();
- self.context.cache_layers.borrow_mut().update_after_acceptance(
- &[0],
- None,
- &cb,
- &self.context.kv_cache_update,
- );
- self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
- }
-
- let encoder =
- root_command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
-
- if let Some(mask_update) = &self.context.mask_update {
- let updates: Vec<AttentionBiasUpdate> =
- self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);
- for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {
- if let Some(update) = updates.iter().find(|u| &u.key == window_size) {
- if update.unmask_col >= 0 || update.mask_col >= 0 {
- mask_update.encode(mask_buffer.borrow().buffer(), update.unmask_col, update.mask_col, &encoder);
+ self.context.cache_layers.borrow_mut().update_after_acceptance(
+ &[0],
+ None,
+ &root_command_buffer,
+ &self.context.kv_cache_update,
+ );
+ self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
+
+ root_command_buffer.with_compute_encoder(|encoder| {
+ if let Some(mask_update) = &self.context.mask_update {
+ let updates: Vec<AttentionBiasUpdate> =
+ self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);
+ for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {
+ if let Some(update) = updates.iter().find(|u| &u.key == window_size) {
+ if update.unmask_col >= 0 || update.mask_col >= 0 {
+ mask_update.encode(
+ mask_buffer.borrow().buffer(),
+ update.unmask_col,
+ update.mask_col,
+ &encoder,
+ );
+ }
}
}
}
- }
-
- encoder.end_encoding();
+ });
// Signal event for next pass
let next_counter = current_counter + 1;
- root_command_buffer.encode_signal_event_value(&async_event, next_counter);
+ root_command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);
self.context.async_buffers.counter.set(next_counter);
// Add completion handler
let results_buffer_clone = results_buffer.clone();
let callback = Arc::new(std::sync::Mutex::new(Some(on_complete)));
- let handler = MTLCommandBufferHandler::new(move |_| {
+ let handler = move || {
let token = {
- let ptr = results_buffer_clone.contents().as_ptr() as *const u32;
+ let ptr = results_buffer_clone.cpu_ptr().as_ptr() as *const u32;
unsafe { *ptr.add(slot) as u64 }
};
if let Some(cb) = callback.lock().unwrap().take() {
cb(token);
}
- });
+ };
- root_command_buffer.add_completed_handler(&handler);
- root_command_buffer.commit();
+ root_command_buffer.add_completed_handler(handler);
+ root_command_buffer.submit();
if should_capture {
root_command_buffer.wait_until_completed();
- self.gpu_capture.stop_capture("decode");
+ self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
}
Ok(())
@@ -603,7 +602,7 @@ impl LanguageModelGenerator {
allow_pre_encode: bool,
sampling_method: SamplingMethod,
should_fill_attention_bias: bool,
- ) -> (ForwardPassState<Metal>, f64) {
+ ) -> (ForwardPassState<B>, f64) {
objc2::rc::autoreleasepool(|_pool| {
let run_start = Instant::now();
@@ -650,7 +649,7 @@ impl LanguageModelGenerator {
}
}
- self.context.command_buffer.commit();
+ root_command_buffer.submit();
if allow_pre_encode {
self.context.reset_command_buffer();
@@ -670,7 +669,10 @@ impl LanguageModelGenerator {
root_command_buffer.wait_until_completed();
let run_time = run_start.elapsed().as_secs_f64();
if should_capture {
- self.gpu_capture.stop_capture("decode");
+ self.gpu_capture
+ .stop_capture(&self.context.context, "decode")
+ .map_err(|_| Error::CaptureFailed)
+ .expect("failed to stop capture");
}
(state, run_time)
@@ -679,7 +681,7 @@ impl LanguageModelGenerator {
fn sample(
&mut self,
- state: &mut ForwardPassState<Metal>,
+ state: &mut ForwardPassState<B>,
) -> Result<Vec<u64>, Error> {
let sampling_output = state
.sampling_output()
@@ -704,7 +706,7 @@ impl LanguageModelGenerator {
wait_until_completed: bool,
) {
let command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");
- let root_command_buffer = command_buffer.to_owned();
+ let root_command_buffer = &command_buffer;
{
let mut cache_layers = self.context.cache_layers.borrow_mut();
@@ -716,7 +718,7 @@ impl LanguageModelGenerator {
);
}
- command_buffer.commit();
+ command_buffer.submit();
if wait_until_completed {
command_buffer.wait_until_completed();
@@ -724,9 +726,9 @@ impl LanguageModelGenerator {
}
fn allow_pre_encode(&self) -> bool {
- let metal_debug_active = MetalEnvVar::DeviceWrapperType.is_enabled();
+ let debug_active = self.context.context.debug_active();
- let result = self.decoding_config.allow_pre_encode && !metal_debug_active;
+ let result = self.decoding_config.allow_pre_encode && !debug_active;
result
}
diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
index abfbfd3dc..68d1ac12a 100644
--- a/crates/uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -227,7 +227,7 @@ where
None
};
- let async_batch_size = decoding_config.async_batch_size.resolve(model_path);
+ let async_batch_size = decoding_config.async_batch_size.resolve::<B>(model_path, context.as_ref());
let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);
let seed = PRng::new(decoding_config.sampling_seed.resolve());
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index 9604c571d..681bb6db0 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -1,30 +1,27 @@
use std::mem::size_of;
-use metal::{BufferExt, MTLBuffer};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
use super::LanguageModelGeneratorContext;
use crate::{
- backends::{common::Context, metal::Metal},
+ backends::common::{Backend, Context, NativeBuffer},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::ForwardPassState,
};
-pub struct LanguageModelGeneratorEncodedTask {
+pub struct LanguageModelGeneratorEncodedTask<B: Backend> {
pub key: String,
- predicate_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
+ predicate_buffer: B::NativeBuffer,
}
-impl LanguageModelGeneratorEncodedTask {
- pub fn predicate_buffer(&self) -> &Retained<ProtocolObject<dyn MTLBuffer>> {
+impl<B: Backend> LanguageModelGeneratorEncodedTask<B> {
+ pub fn predicate_buffer(&self) -> &B::NativeBuffer {
&self.predicate_buffer
}
pub fn disable_execution(&self) {
+ let ptr = self.predicate_buffer.cpu_ptr().as_ptr() as *mut u32;
+
unsafe {
- let ptr = self.predicate_buffer.contents().as_ptr() as *mut u32;
*ptr = 1;
- self.predicate_buffer.did_modify_range(0..size_of::<u32>());
}
}
}
@@ -81,12 +78,12 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
)
}
- pub fn create_state(
+ pub fn create_state<B: Backend>(
&self,
- context: &mut LanguageModelGeneratorContext<Metal>,
+ context: &mut LanguageModelGeneratorContext<B>,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
should_fill_attention_bias: bool,
- ) -> ForwardPassState<Metal> {
+ ) -> ForwardPassState<B> {
ForwardPassState::new_llm(
context.context.clone(),
&context.decoder_config,
@@ -110,13 +107,13 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
)
}
- pub fn build_encoded_task(
+ pub fn build_encoded_task<B: Backend>(
&self,
- context: &LanguageModelGeneratorContext<Metal>,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
+ context: &LanguageModelGeneratorContext<B>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
key: String,
- ) -> LanguageModelGeneratorEncodedTask {
+ ) -> LanguageModelGeneratorEncodedTask<B> {
context.executables.encode(state, parameters, &context.command_buffer);
LanguageModelGeneratorEncodedTask {
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index 45c4ec1c1..6034560e8 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -19,6 +19,7 @@ use tokenizers::Tokenizer;
use xgrammar::TokenizerInfo;
use crate::{
+ backends::metal::Metal,
config::{MixerConfig, ModelMetadata},
forward_pass::cache_layers::CacheLayer,
language_model::{
@@ -54,8 +55,8 @@ pub struct ChatSession {
tokenizer_info: TokenizerInfo,
input_processor: Box<dyn InputProcessor>,
output_parser: OutputParser,
- llm: Option<LanguageModelGenerator>,
- static_context: Option<Context>,
+ llm: Option<LanguageModelGenerator<Metal>>,
+ static_context: Option<Context<Metal>>,
}
impl ChatSession {
@@ -204,9 +205,9 @@ impl ChatSession {
fn extend(
&mut self,
input: Input,
- context: Option<&Context>,
+ context: Option<&Context<Metal>>,
config: RunConfig,
- ) -> Result<(Output, Context), Error> {
+ ) -> Result<(Output, Context<Metal>), Error> {
self.reconfigure_language_model_generator(context)?;
let output = self.run_internal(input, config, None::<fn(Output) -> bool>)?;
let new_context = self.build_context_from_language_model_generator()?;
@@ -321,7 +322,10 @@ impl ChatSession {
language_model_generator.decoding_config.generate_suffix_length() == 1 && compiled_grammar.is_none();
let generate_output = if can_use_async {
- let batch_size = language_model_generator.decoding_config.async_batch_size.resolve(&self.model_path);
+ let batch_size = language_model_generator
+ .decoding_config
+ .async_batch_size
+ .resolve::<Metal>(&self.model_path, language_model_generator.context.context.as_ref());
Self::run_async_batch(
&self.tokenizer,
&self.output_parser,
@@ -351,7 +355,7 @@ impl ChatSession {
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
- language_model_generator: &mut LanguageModelGenerator,
+ language_model_generator: &mut LanguageModelGenerator<Metal>,
compiled_grammar: Option<&mut CompiledGrammar>,
sampling_method: super::parameter::SamplingMethod,
progress: &Option<F>,
@@ -409,7 +413,7 @@ impl ChatSession {
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
- llm: &mut LanguageModelGenerator,
+ llm: &mut LanguageModelGenerator<Metal>,
sampling_method: super::parameter::SamplingMethod,
progress: &Option<F>,
batch_size: usize,
@@ -516,7 +520,7 @@ impl ChatSession {
fn reconfigure_language_model_generator(
&mut self,
- context: Option<&Context>,
+ context: Option<&Context<Metal>>,
) -> Result<(), Error> {
let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
language_model_generator.reset_state();
@@ -565,7 +569,7 @@ impl ChatSession {
Ok(())
}
- fn build_context_from_language_model_generator(&self) -> Result<Context, Error> {
+ fn build_context_from_language_model_generator(&self) -> Result<Context<Metal>, Error> {
let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
let cache_layers =
language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.context);
@@ -581,7 +585,7 @@ impl ChatSession {
impl ChatSession {
fn check_finish_reason(
run_context: &RunContext,
- language_model_generator: &LanguageModelGenerator,
+ language_model_generator: &LanguageModelGenerator<Metal>,
new_tokens: &[u64],
) -> Option<FinishReason> {
let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;
@@ -617,7 +621,7 @@ impl ChatSession {
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
- language_model_generator: &LanguageModelGenerator,
+ language_model_generator: &LanguageModelGenerator<Metal>,
generate_results: &[GenerateResult],
generate_durations: &[f64],
finish_reason: Option<FinishReason>,
diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rs
index 9ff58d6b9..c1bd9e835 100644
--- a/crates/uzu/src/session/classification_session.rs
+++ b/crates/uzu/src/session/classification_session.rs
@@ -3,6 +3,7 @@ use std::path::PathBuf;
use tokenizers::Tokenizer;
use crate::{
+ backends::metal::Metal,
classifier::{ClassificationOutput, Classifier},
config::ModelMetadata,
session::{
@@ -17,7 +18,7 @@ pub struct ClassificationSession {
#[allow(dead_code)]
model_metadata: ModelMetadata,
tokenizer: Tokenizer,
- classifier: Classifier,
+ classifier: Classifier<Metal>,
input_processor: Box<dyn InputProcessor>,
}
diff --git a/crates/uzu/src/session/helpers/context.rs b/crates/uzu/src/session/helpers/context.rs
index fa7c63627..84ee21882 100644
--- a/crates/uzu/src/session/helpers/context.rs
+++ b/crates/uzu/src/session/helpers/context.rs
@@ -1,15 +1,15 @@
-use crate::{backends::metal::Metal, forward_pass::cache_layers::CacheLayers, session::config::DecodingConfig};
+use crate::{backends::common::Backend, forward_pass::cache_layers::CacheLayers, session::config::DecodingConfig};
-pub struct Context {
+pub struct Context<B: Backend> {
pub tokens: Vec<u64>,
- pub cache_layers: CacheLayers<Metal>,
+ pub cache_layers: CacheLayers<B>,
pub decoding_config: DecodingConfig,
}
-impl Context {
+impl<B: Backend> Context<B> {
pub fn new(
tokens: Vec<u64>,
- cache_layers: CacheLayers<Metal>,
+ cache_layers: CacheLayers<B>,
decoding_config: DecodingConfig,
) -> Self {
Self {
diff --git a/crates/uzu/src/session/parameter/async_batch_size.rs b/crates/uzu/src/session/parameter/async_batch_size.rs
index 4c0093c70..31139ff16 100644
--- a/crates/uzu/src/session/parameter/async_batch_size.rs
+++ b/crates/uzu/src/session/parameter/async_batch_size.rs
@@ -1,6 +1,9 @@
use std::path::Path;
-use crate::utils::{DeviceClass, ModelSize};
+use crate::{
+ backends::common::{Backend, Context, DeviceClass},
+ utils::ModelSize,
+};
#[derive(Debug, Clone, Copy)]
pub enum AsyncBatchSize {
@@ -15,15 +18,16 @@ impl Default for AsyncBatchSize {
}
impl AsyncBatchSize {
- pub fn resolve(
+ pub fn resolve<B: Backend>(
&self,
model_path: &Path,
+ context: &B::Context,
) -> usize {
match self {
AsyncBatchSize::Default => {
- let device = DeviceClass::detect();
+ let device_class = context.device_class();
let model_size = ModelSize::from_path(model_path);
- default_batch_size(device, model_size)
+ default_batch_size(device_class, model_size)
},
AsyncBatchSize::Custom(value) => *value,
}
diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rs
index 4f172256f..33c1bd827 100644
--- a/crates/uzu/src/session/types/error.rs
+++ b/crates/uzu/src/session/types/error.rs
@@ -46,6 +46,8 @@ pub enum Error {
GrammarReject,
#[error("Token {0} out of grammar vocabulary range (0..{1})")]
TokenOutOfGrammarRange(u64, usize),
+ #[error("Capture failed")]
+ CaptureFailed,
#[error("Classifier error: {0}")]
Classifier(#[from] ClassifierError),
}
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index bd125e868..18a6d4897 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -12,16 +12,15 @@ use std::{
};
use half::{bf16, f16};
-use metal::{MTLCommandBuffer, MTLCommandQueue};
use ndarray::{IxDyn, s};
use num_traits::NumCast;
use crate::{
ArrayElement, DataType,
array::Array,
- backends::{
- common::kernel::kv_cache_update::KVCacheUpdate,
- metal::{Metal, MetalContext},
+ backends::common::{
+ Backend, CommandBuffer, Context,
+ kernel::{kv_cache_update::KVCacheUpdate, matmul::MatmulKernels},
},
classifier::Classifier,
config::ModelMetadata,
@@ -156,9 +155,12 @@ pub enum ArrayTransform {
// Model Context (internal)
// ============================================================================
-enum ModelContext {
- LanguageModelGenerator(LanguageModelGeneratorContext<Metal>),
- Classifier(Classifier),
+enum ModelContext<B: Backend + 'static>
+where
+ B::Kernels: MatmulKernels,
+{
+ LanguageModelGenerator(LanguageModelGeneratorContext<B>),
+ Classifier(Classifier<B>),
}
// ============================================================================
@@ -169,12 +171,18 @@ enum ModelContext {
///
/// Automatically detects whether the model is an LLM or classifier and
/// runs the appropriate validation.
-pub struct TraceValidator {
+pub struct TraceValidator<B: Backend + 'static>
+where
+ B::Kernels: MatmulKernels,
+{
model_path: PathBuf,
- context: ModelContext,
+ context: ModelContext<B>,
}
-impl TraceValidator {
+impl<B: Backend + 'static> TraceValidator<B>
+where
+ B::Kernels: MatmulKernels,
+{
/// Create a new trace validator for the given model path.
///
/// Automatically detects the model type from config.json.
@@ -202,7 +210,7 @@ impl TraceValidator {
AsyncBatchSize::default(),
false,
);
- let mut llm_context = LanguageModelGeneratorContext::<Metal>::new(model_path, &decoding_config)?;
+ let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());
Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);
ModelContext::LanguageModelGenerator(llm_context)
@@ -240,7 +248,7 @@ impl TraceValidator {
// ========================================================================
fn run_llm_validation(
- ctx: &LanguageModelGeneratorContext<Metal>,
+ ctx: &LanguageModelGeneratorContext<B>,
traces_path: &Path,
) -> Result<TracerValidationResults, Error> {
let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
@@ -274,10 +282,10 @@ impl TraceValidator {
None,
);
- let command_buffer = ctx.context.command_queue.command_buffer().expect("Failed to create command buffer");
+ let command_buffer = ctx.context.create_command_buffer().expect("Failed to create command buffer");
ctx.executables.encode(&mut state, &EncodingParameters::new(false, false, false), &command_buffer);
- command_buffer.commit();
+ command_buffer.submit();
command_buffer.wait_until_completed();
let traces = state.traces().clone();
@@ -385,7 +393,7 @@ impl TraceValidator {
// ========================================================================
fn run_classifier_validation(
- classifier: &mut Classifier,
+ classifier: &mut Classifier<B>,
traces_path: &Path,
) -> Result<TracerValidationResults, Error> {
let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
@@ -425,7 +433,7 @@ impl TraceValidator {
})
}
- fn handle_missing_tokens(traces_view: &ParameterTree<MetalContext>) -> TracerValidationResults {
+ fn handle_missing_tokens(traces_view: &ParameterTree<B::Context>) -> TracerValidationResults {
if let Ok(expected_logits) = traces_view.leaf("logits") {
let reference_shape = expected_logits.shape().to_vec();
let metrics = TracerValidationMetrics {
@@ -470,13 +478,13 @@ impl TraceValidator {
// ========================================================================
fn validate_layer_traces(
- traces: &Rc<RefCell<ActivationTrace<Metal>>>,
- traces_view: &ParameterTree<MetalContext>,
+ traces: &Rc<RefCell<ActivationTrace<B>>>,
+ traces_view: &ParameterTree<B::Context>,
data_type: DataType,
) -> Vec<TracerValidationResult> {
let mut results = Vec::new();
- let validate = |path: &str, array: &Ref<Array<Metal>>| -> Option<TracerValidationResult> {
+ let validate = |path: &str, array: &Ref<Array<B>>| -> Option<TracerValidationResult> {
if traces_view.leaf(path).is_ok() {
Some(TracerValidationResult {
name: path.to_string(),
@@ -539,8 +547,8 @@ impl TraceValidator {
}
fn validate_classifier_traces(
- traces: &Rc<RefCell<ActivationTrace<Metal>>>,
- traces_view: &ParameterTree<MetalContext>,
+ traces: &Rc<RefCell<ActivationTrace<B>>>,
+ traces_view: &ParameterTree<B::Context>,
data_type: DataType,
) -> Vec<TracerValidationResult> {
let mut results = Vec::new();
@@ -584,8 +592,8 @@ impl TraceValidator {
fn validate_array(
data_type: DataType,
- expected_array: &Array<Metal>,
- produced_array: &Ref<Array<Metal>>,
+ expected_array: &Array<B>,
+ produced_array: &Ref<Array<B>>,
transform: Option<ArrayTransform>,
) -> TracerValidationMetrics {
match data_type {
@@ -598,17 +606,17 @@ impl TraceValidator {
fn validate_array_with_name(
data_type: DataType,
- traces_view: &ParameterTree<MetalContext>,
+ traces_view: &ParameterTree<B::Context>,
expected_array_path: &str,
- produced_array: &Ref<Array<Metal>>,
+ produced_array: &Ref<Array<B>>,
) -> TracerValidationMetrics {
let expected_array = traces_view.leaf(expected_array_path).unwrap();
Self::validate_array(data_type, &expected_array, produced_array, None)
}
fn validate_array_of_type<Precision: ArrayElement>(
- expected_array: &Array<Metal>,
- produced_array: &Ref<Array<Metal>>,
+ expected_array: &Array<B>,
+ produced_array: &Ref<Array<B>>,
transform: Option<ArrayTransform>,
) -> TracerValidationMetrics {
let expected_view = expected_array.as_view::<Precision>();
@@ -778,7 +786,7 @@ impl TraceValidator {
// ========================================================================
fn load_array_as_vec<SourcePrecision: ArrayElement, TargetPrecision: NumCast>(
- traces_view: &ParameterTree<MetalContext>,
+ traces_view: &ParameterTree<B::Context>,
name: &str,
) -> Vec<TargetPrecision> {
let array = traces_view.leaf(name).unwrap();
@@ -803,7 +811,7 @@ impl TraceValidator {
fn ensure_llm_context_capacity(
decoding_config: &DecodingConfig,
desired_suffix_length: usize,
- context: &mut LanguageModelGeneratorContext<Metal>,
+ context: &mut LanguageModelGeneratorContext<B>,
) {
let resolved_prefix_length = decoding_config.context_length.resolve(&context.model_config);
let current_suffix_length = std::cmp::max(
@@ -847,7 +855,7 @@ impl TraceValidator {
.expect("Failed to create sampling kernel");
}
- fn get_tokens_from_logits(logits: &Array<Metal>) -> Vec<u64> {
+ fn get_tokens_from_logits(logits: &Array<B>) -> Vec<u64> {
let data_type = logits.data_type();
match data_type {
DataType::F16 => Self::get_tokens_from_logits_of_type::<f16>(logits),
@@ -857,7 +865,7 @@ impl TraceValidator {
}
}
- fn get_tokens_from_logits_of_type<Precision: ArrayElement>(logits: &Array<Metal>) -> Vec<u64> {
+ fn get_tokens_from_logits_of_type<Precision: ArrayElement>(logits: &Array<B>) -> Vec<u64> {
let sampler = ArgmaxSampler {};
sampler.sample(logits.as_view::<Precision>())
}
diff --git a/crates/uzu/src/utils/device.rs b/crates/uzu/src/utils/device.rs
deleted file mode 100644
index b4c317563..000000000
--- a/crates/uzu/src/utils/device.rs
+++ /dev/null
@@ -1,61 +0,0 @@
-use metal::{MTLDevice, MTLDeviceExt};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DeviceClass {
- Ultra,
- Max,
- Pro,
- Base,
- IPhone,
- Unknown,
-}
-
-impl DeviceClass {
- pub fn detect() -> Self {
- let device = match <dyn MTLDevice>::system_default() {
- Some(dev) => dev,
- None => return DeviceClass::Unknown,
- };
-
- let name = device.name().to_lowercase();
-
- if name.contains("ultra") {
- DeviceClass::Ultra
- } else if name.contains("max") {
- DeviceClass::Max
- } else if name.contains("pro") {
- DeviceClass::Pro
- } else if name.contains("iphone") || name.contains("a1") {
- DeviceClass::IPhone
- } else {
- DeviceClass::Base
- }
- }
-
- pub fn is_high_end(&self) -> bool {
- matches!(self, DeviceClass::Ultra | DeviceClass::Max)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_device_detection() {
- let device = DeviceClass::detect();
- println!("Detected device: {:?}", device);
- println!("Is high-end: {}", device.is_high_end());
- assert_ne!(device, DeviceClass::Unknown);
- }
-
- #[test]
- fn test_is_high_end() {
- assert!(DeviceClass::Ultra.is_high_end());
- assert!(DeviceClass::Max.is_high_end());
- assert!(!DeviceClass::Pro.is_high_end());
- assert!(!DeviceClass::Base.is_high_end());
- assert!(!DeviceClass::IPhone.is_high_end());
- assert!(!DeviceClass::Unknown.is_high_end());
- }
-}
diff --git a/crates/uzu/src/utils/env_utils.rs b/crates/uzu/src/utils/env_utils.rs
index 27593aef8..7bedd5c22 100644
--- a/crates/uzu/src/utils/env_utils.rs
+++ b/crates/uzu/src/utils/env_utils.rs
@@ -1,16 +1,14 @@
#[derive(Copy, Clone, Debug)]
-pub enum MetalEnvVar {
- DeviceWrapperType,
+pub enum EnvVar {
CaptureFirstDecode,
CaptureFirstPrefill,
}
-impl MetalEnvVar {
+impl EnvVar {
pub fn key(&self) -> &'static str {
match self {
- MetalEnvVar::DeviceWrapperType => "METAL_DEVICE_WRAPPER_TYPE",
- MetalEnvVar::CaptureFirstDecode => "UZU_CAPTURE_FIRST_DECODE",
- MetalEnvVar::CaptureFirstPrefill => "UZU_CAPTURE_FIRST_PREFILL",
+ EnvVar::CaptureFirstDecode => "UZU_CAPTURE_FIRST_DECODE",
+ EnvVar::CaptureFirstPrefill => "UZU_CAPTURE_FIRST_PREFILL",
}
}
diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rs
index da72c7209..c18a5712d 100644
--- a/crates/uzu/src/utils/mod.rs
+++ b/crates/uzu/src/utils/mod.rs
@@ -1,11 +1,9 @@
pub mod attention;
-pub mod device;
pub mod env_utils;
pub mod model_size;
pub mod storage;
pub mod version;
-pub use device::DeviceClass;
pub use env_utils::*;
pub use model_size::ModelSize;
pub use storage::*;
diff --git a/crates/uzu/tests/allocator_test/caching_allocator.rs b/crates/uzu/tests/allocator_test/caching_allocator.rs
index 16e68bb9d..31773c871 100644
--- a/crates/uzu/tests/allocator_test/caching_allocator.rs
+++ b/crates/uzu/tests/allocator_test/caching_allocator.rs
@@ -1,19 +1,19 @@
use std::rc::Rc;
use uzu::backends::{
- common::{Buffer, BufferLifetime, Context},
- metal::{Metal, MetalContext},
+ common::{Backend, Buffer, BufferLifetime, Context},
+ metal::Metal,
};
use super::allocator_trait::AllocatorTrait;
pub struct CachingAllocator {
- pub context: Rc<MetalContext>,
+ pub context: Rc<<Metal as Backend>::Context>,
}
impl CachingAllocator {
pub fn new() -> Self {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create MetalContext");
Self {
context,
}
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index e817e2bac..085e0546c 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -12,16 +12,13 @@ use uzu::{
DataType,
backends::{
common::{
- CommandBuffer, Context,
+ Backend, CommandBuffer, Context, Kernels,
kernel::{
AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,
attention::{AttentionGemmArguments, AttentionGemmBlock},
},
},
- metal::{
- Metal, MetalContext,
- kernel::dsl::{AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},
- },
+ metal::Metal,
},
};
@@ -148,7 +145,7 @@ fn create_test_data(
/// Convert ndarray to Metal buffer layout expected by our kernel
fn create_query_buffer(
queries: &Array4<f32>,
- context: &MetalContext,
+ context: &<Metal as Backend>::Context,
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
let (_batch_size, num_heads, seq_len, head_dim) = queries.dim();
@@ -173,7 +170,7 @@ fn create_query_buffer(
fn create_key_cache_buffer(
keys: &Array4<f32>,
max_seq_len: usize,
- context: &MetalContext,
+ context: &<Metal as Backend>::Context,
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
let (_batch_size, num_kv_heads, seq_len, head_dim) = keys.dim();
@@ -198,7 +195,7 @@ fn create_key_cache_buffer(
fn create_value_cache_buffer(
values: &Array4<f32>,
max_seq_len: usize,
- context: &MetalContext,
+ context: &<Metal as Backend>::Context,
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
let (_batch_size, num_kv_heads, seq_len, head_dim) = values.dim();
@@ -223,7 +220,7 @@ fn create_value_cache_buffer(
fn create_mask_buffer(
mask: &Array3<f32>,
num_heads: usize,
- context: &MetalContext,
+ context: &<Metal as Backend>::Context,
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
let (_batch_size, seq_len, _) = mask.dim();
@@ -246,7 +243,7 @@ fn create_mask_buffer(
fn create_mask_2d_buffer(
mask: &Array3<f32>,
- context: &MetalContext,
+ context: &<Metal as Backend>::Context,
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
let (_batch_size, seq_len, _) = mask.dim();
@@ -265,7 +262,7 @@ fn create_mask_2d_buffer(
fn create_sinks_buffer(
sinks: &[f32],
- context: &MetalContext,
+ context: &<Metal as Backend>::Context,
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
context
.device
@@ -295,8 +292,8 @@ fn convert_kernel_output(
}
fn run_single_pass_attention(
- kernel: &AttentionSinglePassMetalKernel,
- context: &MetalContext,
+ kernel: &<<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel,
+ context: &<Metal as Backend>::Context,
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
@@ -366,8 +363,8 @@ fn run_single_pass_attention(
}
fn run_single_pass_attention_with_is_causal(
- kernel: &AttentionSinglePassMetalKernel,
- context: &MetalContext,
+ kernel: &<<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel,
+ context: &<Metal as Backend>::Context,
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
@@ -439,7 +436,7 @@ fn run_single_pass_attention_with_is_causal(
fn run_gemm_attention(
kernel: &AttentionGemmBlock<Metal>,
- context: &MetalContext,
+ context: &<Metal as Backend>::Context,
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
@@ -549,7 +546,7 @@ fn compare_results(
#[test]
fn test_single_pass_attention_basic() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 4;
@@ -569,9 +566,16 @@ fn test_single_pass_attention_basic() {
println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());
let is_causal = false; // Non-causal attention for this test
- let kernel =
- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)
- .expect("Failed to create attention single pass metal");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ false,
+ false,
+ false,
+ is_causal,
+ )
+ .expect("Failed to create attention single pass metal");
let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)
{
@@ -589,7 +593,7 @@ fn test_single_pass_attention_basic() {
#[test]
fn test_gemm_attention_basic() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 4;
@@ -616,7 +620,7 @@ fn test_gemm_attention_basic() {
#[test]
fn test_gemm_attention_f32_head_dim_128() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 4;
@@ -643,7 +647,7 @@ fn test_gemm_attention_f32_head_dim_128() {
#[test]
fn test_matrix_attention_matches_vector_and_cpu_seq256() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 8;
@@ -659,9 +663,16 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
// vector attention path (single-pass)
- let kernel =
- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, true, true, false, is_causal)
- .expect("Failed to create attention single pass metal");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ true,
+ true,
+ false,
+ is_causal,
+ )
+ .expect("Failed to create attention single pass metal");
let vector_output = run_single_pass_attention_with_is_causal(
&kernel,
&context,
@@ -700,7 +711,7 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
#[test]
fn test_single_pass_attention_with_mask() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 4;
@@ -711,9 +722,16 @@ fn test_single_pass_attention_with_mask() {
let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let kernel =
- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, true, true, false, false)
- .expect("Failed to create AttentionSinglePassMetalKernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ true,
+ true,
+ false,
+ false,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
let kernel_output =
match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale) {
Ok(output) => output,
@@ -740,7 +758,7 @@ fn test_single_pass_attention_with_mask() {
#[test]
fn test_single_pass_attention_with_sinks() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 4;
@@ -754,9 +772,16 @@ fn test_single_pass_attention_with_sinks() {
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();
println!("Using sinks: {:?}", sinks);
- let kernel =
- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, true, false)
- .expect("Failed to create AttentionSinglePassMetalKernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ false,
+ false,
+ true,
+ false,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);
@@ -776,7 +801,7 @@ fn test_single_pass_attention_with_sinks() {
#[test]
fn test_single_pass_attention_with_sinks_long_sequence() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 4;
@@ -790,9 +815,16 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();
println!("Long sequence sinks: {:?}", sinks);
- let kernel =
- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, true, false)
- .expect("Failed to create AttentionSinglePassMetalKernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ false,
+ false,
+ true,
+ false,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);
@@ -817,7 +849,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
#[test]
fn test_single_pass_attention_gqa() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 8;
@@ -829,9 +861,16 @@ fn test_single_pass_attention_gqa() {
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
let is_causal = false; // Non-causal attention for this test
- let kernel =
- AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)
- .expect("Failed to create AttentionSinglePassMetalKernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ false,
+ false,
+ false,
+ is_causal,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)
{
@@ -850,9 +889,9 @@ fn test_single_pass_attention_gqa() {
}
fn run_two_pass_attention(
- kernel_pass1: &AttentionTwoPass1MetalKernel,
- kernel_pass2: &AttentionTwoPass2MetalKernel,
- context: &MetalContext,
+ kernel_pass1: &<<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel,
+ kernel_pass2: &<<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel,
+ context: &<Metal as Backend>::Context,
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
@@ -954,7 +993,7 @@ fn run_two_pass_attention(
#[test]
fn test_two_pass_attention() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 8;
@@ -968,11 +1007,22 @@ fn test_two_pass_attention() {
let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
- let kernel_pass1 =
- AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)
- .expect("Failed to create AttentionTwoPass1MetalKernel");
- let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)
- .expect("Failed to create AttentionTwoPass2MetalKernel");
+ let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ false,
+ false,
+ false,
+ is_causal,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");
+ let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");
let kernel_output = match run_two_pass_attention(
&kernel_pass1,
&kernel_pass2,
@@ -998,7 +1048,7 @@ fn test_two_pass_attention() {
#[test]
fn test_two_pass_attention_gqa() {
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
let batch_size = 1;
let num_heads = 8;
@@ -1012,11 +1062,22 @@ fn test_two_pass_attention_gqa() {
let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
- let kernel_pass1 =
- AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, is_causal, false, false, false)
- .expect("Failed to create AttentionTwoPass1MetalKernel");
- let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)
- .expect("Failed to create AttentionTwoPass2MetalKernel");
+ let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ is_causal,
+ false,
+ false,
+ false,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");
+ let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");
let kernel_output = match run_two_pass_attention(
&kernel_pass1,
&kernel_pass2,
@@ -1044,7 +1105,7 @@ fn test_two_pass_attention_gqa() {
fn perf_two_pass_attention() {
use std::time::Instant;
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
// ---- Problem sizes requiring two-pass ----
let batch_size = 1;
@@ -1063,11 +1124,22 @@ fn perf_two_pass_attention() {
);
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
- let kernel_pass1 =
- AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, is_causal, false, false, false)
- .expect("Failed to create AttentionTwoPass1MetalKernel");
- let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)
- .expect("Failed to create AttentionTwoPass2MetalKernel");
+ let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ is_causal,
+ false,
+ false,
+ false,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");
+ let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(
+ &context,
+ DataType::F32,
+ head_dim as u32,
+ )
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");
// ---- Create buffers ----
// For realistic inference, we only process queries for the suffix (new tokens)
diff --git a/crates/uzu/tests/audio_codec_test.rs b/crates/uzu/tests/audio_codec_test.rs
index dd5e2475d..3075e4dbb 100644
--- a/crates/uzu/tests/audio_codec_test.rs
+++ b/crates/uzu/tests/audio_codec_test.rs
@@ -6,32 +6,26 @@ use uzu::{
array::ArrayContextExt,
backends::{
common::{
- Context,
+ Backend, Context, Kernels,
kernel::{
AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dKernel, AudioClampKernel,
AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel,
AudioLeakyReluKernel, AudioScaleKernel, AudioTanhKernel,
},
},
- metal::{
- MetalContext,
- kernel::dsl::{
- AudioAddMetalKernel, AudioCausalConv1dMetalKernel, AudioCausalConvTranspose1dMetalKernel,
- AudioClampMetalKernel, AudioConv1dMetalKernel, AudioFsqDecodeMetalKernel, AudioFsqEncodeMetalKernel,
- AudioHalfSnakeMetalKernel, AudioLeakyReluMetalKernel, AudioScaleMetalKernel, AudioTanhMetalKernel,
- },
- },
+ metal::Metal,
},
};
-fn create_test_context() -> std::rc::Rc<MetalContext> {
- <MetalContext as Context>::new().expect("MetalContext")
+fn create_test_context() -> std::rc::Rc<<Metal as Backend>::Context> {
+ <Metal as Backend>::Context::new().expect("MetalContext")
}
#[test]
fn audio_conv1d_replicate_matches_reference_f32() {
let context = create_test_context();
- let kernel = AudioConv1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioConv1dKernel::new(&context, DataType::F32)
+ .expect("audio runtime");
let batch_size = 2usize;
let cin = 3usize;
@@ -145,7 +139,8 @@ fn audio_conv1d_replicate_matches_reference_f32() {
#[test]
fn audio_causal_conv1d_matches_reference_f32() {
let context = create_test_context();
- let kernel = AudioCausalConv1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dKernel::new(&context, DataType::F32)
+ .expect("audio runtime");
let batch_size = 2usize;
let cin = 3usize;
@@ -246,7 +241,9 @@ fn audio_causal_conv1d_matches_reference_f32() {
#[test]
fn audio_causal_conv_transpose1d_matches_reference_f32() {
let context = create_test_context();
- let kernel = AudioCausalConvTranspose1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConvTranspose1dKernel::new(&context, DataType::F32)
+ .expect("audio runtime");
let batch_size = 2usize;
let cin = 4usize;
@@ -360,7 +357,8 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
#[test]
fn audio_leaky_relu_matches_reference_f32() {
let context = create_test_context();
- let kernel = AudioLeakyReluMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioLeakyReluKernel::new(&context, DataType::F32)
+ .expect("audio runtime");
let n = 1024usize;
let slope = 0.01f32;
@@ -395,7 +393,8 @@ fn audio_leaky_relu_matches_reference_f32() {
#[test]
fn audio_tanh_matches_reference_f32() {
let context = create_test_context();
- let kernel = AudioTanhMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::AudioTanhKernel::new(&context, DataType::F32).expect("audio runtime");
let n = 1024usize;
let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();
@@ -425,8 +424,10 @@ fn audio_tanh_matches_reference_f32() {
#[test]
fn audio_add_and_scale_match_reference_f32() {
let context = create_test_context();
- let add_kernel = AudioAddMetalKernel::new(&context, DataType::F32).expect("audio add kernel");
- let scale_kernel = AudioScaleMetalKernel::new(&context, DataType::F32).expect("audio scale kernel");
+ let add_kernel = <<Metal as Backend>::Kernels as Kernels>::AudioAddKernel::new(&context, DataType::F32)
+ .expect("audio add kernel");
+ let scale_kernel = <<Metal as Backend>::Kernels as Kernels>::AudioScaleKernel::new(&context, DataType::F32)
+ .expect("audio scale kernel");
let n = 2048usize;
let scale_value = 1.0f32 / 3.0f32;
@@ -479,7 +480,8 @@ fn audio_add_and_scale_match_reference_f32() {
#[test]
fn audio_clamp_matches_reference_f32() {
let context = create_test_context();
- let kernel = AudioClampMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioClampKernel::new(&context, DataType::F32)
+ .expect("audio runtime");
let n = 2048usize;
let min_value = -1.0f32;
@@ -511,7 +513,8 @@ fn audio_clamp_matches_reference_f32() {
#[test]
fn audio_half_snake_matches_reference_f32() {
let context = create_test_context();
- let kernel = AudioHalfSnakeMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, DataType::F32)
+ .expect("audio runtime");
let batch_size = 1usize;
let channels = 6usize;
@@ -583,7 +586,8 @@ fn audio_half_snake_matches_reference_f32() {
#[test]
fn audio_fsq_decode_matches_reference() {
let context = create_test_context();
- let kernel = AudioFsqDecodeMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioFsqDecodeKernel::new(&context, DataType::F32)
+ .expect("audio runtime");
let batch_size = 1usize;
let num_groups = 2usize;
@@ -635,7 +639,8 @@ fn audio_fsq_decode_matches_reference() {
#[test]
fn audio_fsq_encode_matches_reference() {
let context = create_test_context();
- let kernel = AudioFsqEncodeMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioFsqEncodeKernel::new(&context, DataType::F32)
+ .expect("audio runtime");
let batch_size = 1usize;
let num_groups = 2usize;
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index 4ec6e04ba..488953452 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -5,8 +5,8 @@ use uzu::{
DataType,
array::ArrayContextExt,
backends::{
- common::{Context, kernel::kv_cache_update::KVCacheUpdate},
- metal::{Metal, MetalContext},
+ common::{Backend, Context, kernel::kv_cache_update::KVCacheUpdate},
+ metal::Metal,
},
forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},
};
@@ -31,7 +31,7 @@ struct Scenario {
}
fn make_test_layer(
- context: &MetalContext,
+ context: &<Metal as Backend>::Context,
state: KVCacheLayerState,
prefix_capacity: usize,
suffix_capacity: usize,
@@ -148,7 +148,7 @@ fn collect_mask(
}
fn run_scenario(
- context: &MetalContext,
+ context: &<Metal as Backend>::Context,
scenario: &Scenario,
) {
let mut layer =
@@ -249,7 +249,7 @@ fn run_scenario(
#[test]
fn kv_cache_state_and_mask_scenarios() {
- let Some(context) = MetalContext::new().ok() else {
+ let Some(context) = <Metal as Backend>::Context::new().ok() else {
return;
};
@@ -357,7 +357,7 @@ fn kv_cache_state_and_mask_scenarios() {
#[test]
fn kv_cache_slice_apply_contiguous_window() {
- let Some(context) = MetalContext::new().ok() else {
+ let Some(context) = <Metal as Backend>::Context::new().ok() else {
return;
};
@@ -403,7 +403,7 @@ fn kv_cache_slice_apply_contiguous_window() {
#[test]
fn kv_cache_slice_apply_wrap_window() {
- let Some(context) = MetalContext::new().ok() else {
+ let Some(context) = <Metal as Backend>::Context::new().ok() else {
return;
};
@@ -445,7 +445,7 @@ fn kv_cache_slice_apply_wrap_window() {
#[test]
fn kv_cache_slice_apply_full_restores_metadata() {
- let Some(context) = MetalContext::new().ok() else {
+ let Some(context) = <Metal as Backend>::Context::new().ok() else {
return;
};
diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rs
index c2afe6b51..56c054f3a 100644
--- a/crates/uzu/tests/kv_cache_update_test.rs
+++ b/crates/uzu/tests/kv_cache_update_test.rs
@@ -4,11 +4,11 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLResou
use ndarray::{Array, Array3, s};
use uzu::backends::{
common::{
- Context,
+ Backend, Context,
gpu_types::Swap,
kernel::kv_cache_update::{KVCacheUpdate, KVLayerData, create_swaps_direct},
},
- metal::{Metal, MetalContext},
+ metal::Metal,
};
fn apply_swaps_3d<T: Clone>(
@@ -31,7 +31,7 @@ fn apply_swaps_3d<T: Clone>(
#[test]
fn test_kv_cache_update_kernel() {
- let metal_context = match MetalContext::new() {
+ let metal_context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Failed to create MetalContext: {:?}. Skipping test.", e);
@@ -42,7 +42,7 @@ fn test_kv_cache_update_kernel() {
test_random_pattern(&metal_context);
}
-fn test_random_pattern(context: &MetalContext) {
+fn test_random_pattern(context: &<Metal as Backend>::Context) {
println!("Testing with random pattern...");
let max_sequence_length = 256usize;
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 8d0d232f6..4bf11fe20 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -8,15 +8,15 @@ use uzu::{
DataType,
backends::{
common::{
- Context,
+ Backend, Context,
kernel::matmul::{MatmulArguments, MatmulKernel},
},
- metal::{Metal, MetalContext, kernel::matmul::choose_dispatch_descriptor},
+ metal::{Metal, choose_dispatch_descriptor},
},
};
fn run_metal_matmul(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
a_data: &[bf16],
b_data: &[bf16],
m: usize,
@@ -274,7 +274,7 @@ fn compare_results(
#[test]
#[ignore]
fn matmul_correctness_comprehensive() {
- let Some(ctx) = MetalContext::new().ok() else {
+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
eprintln!("No Metal device available, skipping test");
return;
};
diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
index dbec43f84..85cb7bb02 100644
--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs
+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
@@ -7,11 +7,11 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},
- metal::{
- MetalContext,
- kernel::dsl::{MoeCountsOffsetsFusedMetalKernel, MoeRouterTopKMetalKernel},
+ common::{
+ Backend, Kernels,
+ kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},
},
+ metal::Metal,
},
};
@@ -50,7 +50,7 @@ fn cpu_offsets_from_counts(counts: &[u32]) -> (Vec<u32>, u32) {
}
fn gen_topk_ids_from_logits(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
t: usize,
e: usize,
k: usize,
@@ -77,7 +77,8 @@ fn gen_topk_ids_from_logits(
// Use fused router+topk kernel
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let router_topk_encoder = cb.new_compute_command_encoder().expect("router_topk encoder");
- let router_topk = MoeRouterTopKMetalKernel::new(ctx, DataType::BF16).expect("router_topk kernel");
+ let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16)
+ .expect("router_topk kernel");
router_topk.encode(
&input_buf,
&weight_buf,
@@ -123,7 +124,8 @@ fn test_counts_offsets_fused_parity_random() {
let num_tiles = ((e + 511) / 512).max(1);
let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);
- let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
@@ -174,7 +176,8 @@ fn test_counts_offsets_fused_edge_cases() {
let num_tiles = ((e + 511) / 512).max(1);
let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);
- let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
@@ -197,7 +200,8 @@ fn test_counts_offsets_fused_edge_cases() {
let num_tiles = ((e + 511) / 512).max(1);
let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);
- let kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index 4eaac6417..b6a53be53 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -4,18 +4,17 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::{
- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
- MoeScatterBucketsMapKernel,
- moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels},
- },
- metal::{
- Metal, MetalContext,
- kernel::dsl::{
- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
- MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,
+ common::{
+ Backend, Kernels,
+ kernel::{
+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
+ MoeScatterBucketsMapKernel,
+ moe::{
+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels,
+ },
},
},
+ metal::Metal,
},
};
@@ -169,7 +168,7 @@ fn moe_cpu_reference(
// Main entry point - automatically tests both modes for T>1
fn run_moe_parity_test(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
t: usize,
e: usize,
k: usize,
@@ -233,7 +232,7 @@ fn run_moe_parity_test(
}
fn run_moe_parity_test_internal(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
t: usize,
e: usize,
k: usize,
@@ -350,7 +349,8 @@ fn run_moe_parity_test_internal(
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
// Router + TopK (fused kernel)
- let router_topk = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk");
+ let router_topk =
+ <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router+topk");
let router_topk_encoder = cb.new_compute_command_encoder().expect("router+topk_encoder");
router_topk.encode(
&x_buf,
@@ -367,13 +367,15 @@ fn run_moe_parity_test_internal(
);
router_topk_encoder.end_encoding();
- let fused_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let fused_kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let encoder = cb.new_compute_command_encoder().expect("encoder");
fused_kernel.encode(&topk_ids_buf, &offsets_buf, &sumk_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
encoder.end_encoding();
let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");
- let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("scatter bases kernel");
+ let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)
+ .expect("scatter bases kernel");
scatter_bases_kernel.encode(
&partials_buf,
&block_bases_buf,
@@ -387,8 +389,9 @@ fn run_moe_parity_test_internal(
scatter_bases_encoder.end_encoding();
let scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");
- let scatter_map_kernel = MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16)
- .expect("Failed to create MoeScatterBucketsMapMetalKernel");
+ let scatter_map_kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16)
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");
scatter_map_kernel.encode(
&topk_ids_buf,
&topk_probs_buf,
@@ -459,7 +462,8 @@ fn run_moe_parity_test_internal(
};
experts.encode(&cb, &args);
- let finalize = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");
+ let finalize =
+ <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");
let encoder = cb.new_compute_command_encoder().expect("encoder");
finalize.encode(
&tok2row_buf,
diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rs
index 091a5b67a..07f20f91b 100644
--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs
@@ -6,11 +6,14 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::moe::{
- MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
- MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
+ common::{
+ Backend,
+ kernel::moe::{
+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
+ MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
+ },
},
- metal::{Metal, MetalContext},
+ metal::Metal,
},
};
@@ -35,7 +38,7 @@ fn build_offsets(
}
fn run_decode_case(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
name: &str,
t: usize,
d_model: usize,
@@ -174,7 +177,7 @@ fn test_two_pass_decode_speed() {
}
fn run_two_pass_prefill_case(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
name: &str,
t: usize,
d_model: usize,
@@ -310,7 +313,7 @@ fn test_two_pass_prefill_speed() {
}
fn run_fused_single_token_case(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
name: &str,
d_model: usize,
d_ff: usize,
@@ -450,7 +453,7 @@ fn test_single_token_indirect_vs_fused() {
}
fn run_indirect_decode_timed(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
t: usize,
d_model: usize,
d_ff: usize,
@@ -557,7 +560,7 @@ fn run_indirect_decode_timed(
}
fn run_fused_decode_timed(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
d_model: usize,
d_ff: usize,
e: usize,
diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rs
index c74e621e6..77254d876 100644
--- a/crates/uzu/tests/moe/moe_finalize_test.rs
+++ b/crates/uzu/tests/moe/moe_finalize_test.rs
@@ -5,7 +5,10 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
- backends::{common::kernel::MoeFinalizeKernel, metal::kernel::dsl::MoeFinalizeMetalKernel},
+ backends::{
+ common::{Backend, Kernels, kernel::MoeFinalizeKernel},
+ metal::Metal,
+ },
};
use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};
@@ -85,7 +88,8 @@ fn test_finalize_correctness() {
let y_out_buf = alloc_buffer::<bf16>(&ctx, t * d_model);
// Execute finalize kernel
- let finalize = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize kernel");
+ let finalize = <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16)
+ .expect("finalize kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
finalize.encode(
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index 147494163..a81c6af36 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -6,18 +6,15 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::{
- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
- MoeScatterBucketsMapKernel,
- moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},
- },
- metal::{
- Metal,
- kernel::dsl::{
- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
- MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,
+ common::{
+ Backend, Kernels,
+ kernel::{
+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
+ MoeScatterBucketsMapKernel,
+ moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},
},
},
+ metal::Metal,
},
};
@@ -119,7 +116,8 @@ fn test_moe_e2e_decode_perf() {
let topk_ids_buf = alloc_buffer::<i32>(&ctx, t * k);
let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);
- let router_topk = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk fused kernel");
+ let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)
+ .expect("router+topk fused kernel");
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
@@ -179,7 +177,8 @@ fn test_moe_e2e_prefill_perf() {
let topk_ids_buf = alloc_buffer::<i32>(&ctx, t * k);
let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);
- let router_topk = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk fused kernel");
+ let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)
+ .expect("router+topk fused kernel");
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
@@ -295,15 +294,20 @@ fn test_moe_pipeline_breakdown_decode() {
let bucketed_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);
// Create kernel structs (use production-validated encoding logic)
- let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("counts+offsets fused");
- let scatter_bases_kernel =
- MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("MoeBlockBasesFromPartialsMetalKernel");
+ let counts_offsets_kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("counts+offsets fused");
+ let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)
+ .expect("<<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel");
let scatter_map_kernel =
- MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16).expect("MoeScatterBucketsMapMetalKernel");
+ <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16)
+ .expect("<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");
let gather_kernel = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("experts two-pass decode");
- let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");
- let router_topk_fused_kernel = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk fused");
+ let finalize_kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");
+ let router_topk_fused_kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)
+ .expect("router+topk fused");
// Testing: Router + TopK + Counts+Offsets (FUSED)
let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {
diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rs
index d62a85f1f..d5145c00c 100644
--- a/crates/uzu/tests/moe/moe_router_topk_test.rs
+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs
@@ -6,8 +6,8 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::MoeRouterTopKKernel,
- metal::{MetalContext, kernel::dsl::MoeRouterTopKMetalKernel},
+ common::{Backend, Kernels, kernel::MoeRouterTopKKernel},
+ metal::Metal,
},
};
@@ -128,8 +128,8 @@ pub fn cpu_topk_select_f32(
}
fn run_router_topk_once(
- ctx: &MetalContext,
- kernel: &MoeRouterTopKMetalKernel,
+ ctx: &<Metal as Backend>::Context,
+ kernel: &<<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel,
t: usize,
d_model: usize,
e: usize,
@@ -223,7 +223,8 @@ fn run_router_topk_once(
#[test]
fn test_router_topk_fused_matches_reference() {
let ctx = create_ctx();
- let kernel = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("kernel");
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("kernel");
let configs =
[(1usize, 64usize, 32usize, 4usize), (2, 128, 64, 8), (4, 256, 128, 16), (8, 256, 256, 32), (1, 512, 512, 64)];
diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rs
index 2a2449b60..f47893c00 100644
--- a/crates/uzu/tests/moe/moe_scatter_test.rs
+++ b/crates/uzu/tests/moe/moe_scatter_test.rs
@@ -6,13 +6,14 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::{
- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel, MoeScatterBucketsKernel,
- },
- metal::kernel::dsl::{
- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeRouterTopKMetalKernel,
- MoeScatterBucketsMetalKernel,
+ common::{
+ Backend, Kernels,
+ kernel::{
+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel,
+ MoeScatterBucketsKernel,
+ },
},
+ metal::Metal,
},
};
@@ -81,7 +82,8 @@ fn test_scatter_buckets_parity() {
let topk_probs_buf = alloc_buffer::<bf16>(&ctx, t * k);
// Use fused router+topk kernel
- let router_topk = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router_topk");
+ let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)
+ .expect("router_topk");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("Failed to create encoder");
router_topk.encode(
@@ -112,7 +114,8 @@ fn test_scatter_buckets_parity() {
let num_tiles = ((e + 511) / 512).max(1);
let partials_buf = alloc_buffer::<u32>(&ctx, num_tiles * 512);
- let fused_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("fused kernel");
+ let fused_kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
fused_kernel.encode(
@@ -135,10 +138,11 @@ fn test_scatter_buckets_parity() {
let entries = num_blocks * num_tiles * 512usize;
let block_bases_buf = alloc_buffer::<u32>(&ctx, entries);
- let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(&ctx)
- .expect("Failed to create MoeBlockBasesFromPartialsMetalKernel");
- let scatter_kernel = MoeScatterBucketsMetalKernel::new(&ctx, DataType::BF16)
- .expect("Failed to create MoeScatterBucketsMetalKernel");
+ let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel");
+ let scatter_kernel =
+ <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel::new(&ctx, DataType::BF16)
+ .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);
diff --git a/crates/uzu/tests/moe/moe_test_utils.rs b/crates/uzu/tests/moe/moe_test_utils.rs
index aaf08b1be..eb6217eb5 100644
--- a/crates/uzu/tests/moe/moe_test_utils.rs
+++ b/crates/uzu/tests/moe/moe_test_utils.rs
@@ -11,16 +11,19 @@ use bytemuck;
use half::bf16;
use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use objc2::{rc::Retained, runtime::ProtocolObject};
-use uzu::backends::{common::Context, metal::MetalContext};
+use uzu::backends::{
+ common::{Backend, Context},
+ metal::Metal,
+};
/// Create Metal context for testing
-pub fn create_ctx() -> Rc<MetalContext> {
- MetalContext::new().expect("Failed to create MetalContext")
+pub fn create_ctx() -> Rc<<Metal as Backend>::Context> {
+ <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context")
}
/// Helper to allocate buffer with data
pub fn alloc_buffer_with_data<T: bytemuck::NoUninit>(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
data: &[T],
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
if data.is_empty() {
@@ -35,7 +38,7 @@ pub fn alloc_buffer_with_data<T: bytemuck::NoUninit>(
/// Helper to allocate empty buffer
pub fn alloc_buffer<T>(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
count: usize,
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
ctx.create_buffer(count * size_of::<T>()).expect("Failed to create buffer")
diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rs
index b589da660..c32a7990c 100644
--- a/crates/uzu/tests/parameters_loader_test.rs
+++ b/crates/uzu/tests/parameters_loader_test.rs
@@ -4,14 +4,17 @@ mod common;
use half::bf16;
use is_close::is_close;
use uzu::{
- backends::{common::Context, metal::MetalContext},
+ backends::{
+ common::{Backend, Context},
+ metal::Metal,
+ },
parameters::ParameterLoader,
};
#[test]
fn test_parameter_loader_basic() {
let weights_path = crate::common::get_test_weights_path();
- let context = MetalContext::new().expect("Failed to create MetalContext");
+ let context = <Metal as Backend>::Context::new().expect("Failed to create MetalContext");
let file = std::fs::File::open(&weights_path).expect("Weights file not found; run download script");
let loader = ParameterLoader::new(&file, context.as_ref()).expect("create loader");
diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rs
index 039f710b5..17d4e7d40 100644
--- a/crates/uzu/tests/quant_matmul_test.rs
+++ b/crates/uzu/tests/quant_matmul_test.rs
@@ -8,13 +8,13 @@ use uzu::{
DataType,
backends::{
common::{
- Context,
+ Backend, Context,
kernel::quant_matmul::{
QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,
QuantizedMatmulType,
},
},
- metal::{Metal, MetalContext},
+ metal::Metal,
},
config::QuantizationMode,
};
@@ -389,7 +389,7 @@ fn generate_test_quant_params(
}
fn buffer_from_f32_slice(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
dtype: DataType,
values: &[f32],
) -> Retained<ProtocolObject<dyn MTLBuffer>> {
@@ -417,7 +417,7 @@ fn buffer_from_f32_slice(
}
fn execute_quantized_matmul(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
batch: usize,
input_dim: usize,
output_dim: usize,
@@ -636,7 +636,7 @@ const QVM_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];
const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];
fn run_kernel_test(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
batch: usize,
output_dim: usize,
input_dim: usize,
@@ -665,7 +665,7 @@ fn run_kernel_test(
#[test]
fn test_quant_gmv() {
- let ctx = match MetalContext::new() {
+ let ctx = match <Metal as Backend>::Context::new() {
Ok(c) => c,
Err(_) => {
println!("Metal not available — skipping QMV test");
@@ -709,7 +709,7 @@ fn test_quant_gmv() {
#[test]
fn test_quant_qvm() {
- let ctx = match MetalContext::new() {
+ let ctx = match <Metal as Backend>::Context::new() {
Ok(c) => c,
Err(_) => {
println!("Metal not available — skipping QVM test");
@@ -753,7 +753,7 @@ fn test_quant_qvm() {
#[test]
fn test_quant_gmm() {
- let ctx = match MetalContext::new() {
+ let ctx = match <Metal as Backend>::Context::new() {
Ok(c) => c,
Err(_) => {
println!("Metal not available — skipping QMM test");
@@ -797,7 +797,7 @@ fn test_quant_gmm() {
#[test]
fn test_quant_gmm_transposed() {
- let ctx = match MetalContext::new() {
+ let ctx = match <Metal as Backend>::Context::new() {
Ok(c) => c,
Err(_) => {
println!("Metal not available — skipping QMM transposed test");
@@ -842,7 +842,7 @@ fn test_quant_gmm_transposed() {
#[test]
#[ignore]
fn test_quant_matmul_perf() {
- let ctx = match MetalContext::new() {
+ let ctx = match <Metal as Backend>::Context::new() {
Ok(c) => c,
Err(_) => {
println!("Metal not available — skipping Perf test");
diff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rs
index 2d514f320..4f9b430f9 100644
--- a/crates/uzu/tests/rms_norm_kernel_test.rs
+++ b/crates/uzu/tests/rms_norm_kernel_test.rs
@@ -8,13 +8,10 @@ use uzu::{
DataType,
backends::{
common::{
- CommandBuffer, Context,
+ Backend, CommandBuffer, Context, Kernels,
kernel::{QKNormKernel, RMSNormKernel},
},
- metal::{
- MetalContext,
- kernel::dsl::{QKNormMetalKernel, RMSNormMetalKernel},
- },
+ metal::Metal,
},
};
@@ -247,7 +244,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
OutputT: TestFloat,
{
// Create Metal context
- let mtl_context = match MetalContext::new() {
+ let mtl_context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping RMS norm test: {}", e);
@@ -303,8 +300,14 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
.expect("Failed to create buffer");
// Create RMS norm kernel
- let kernel = RMSNormMetalKernel::new(&mtl_context, input_type, scale_type, output_type, accumulation_type)
- .expect("Failed to create RMS norm kernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::RMSNormKernel::new(
+ &mtl_context,
+ input_type,
+ scale_type,
+ output_type,
+ accumulation_type,
+ )
+ .expect("Failed to create RMS norm kernel");
// Create command buffer and encode
let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
@@ -380,7 +383,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
ScaleT: TestFloat,
OutputT: TestFloat,
{
- let mtl_context = match MetalContext::new() {
+ let mtl_context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping RMS norm edge case test: {}", e);
@@ -414,8 +417,14 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
.new_buffer(4 * OutputT::size_of(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let kernel = RMSNormMetalKernel::new(&mtl_context, input_type, scale_type, output_type, accumulation_type)
- .expect("Failed to create RMS norm kernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::RMSNormKernel::new(
+ &mtl_context,
+ input_type,
+ scale_type,
+ output_type,
+ accumulation_type,
+ )
+ .expect("Failed to create RMS norm kernel");
let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
@@ -572,7 +581,7 @@ fn perf_rms_norm_with_size(
use rand::{RngExt, SeedableRng, rngs::StdRng};
// ---- Metal context ----
- let mtl_context = match MetalContext::new() {
+ let mtl_context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping RMS norm perf test: {}", e);
@@ -583,8 +592,14 @@ fn perf_rms_norm_with_size(
const EPSILON: f32 = 1e-6;
// ---- Create kernel ----
- let kernel = RMSNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)
- .expect("Failed to create RMS norm kernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::RMSNormKernel::new(
+ &mtl_context,
+ DataType::F32,
+ DataType::F32,
+ DataType::F32,
+ DataType::F32,
+ )
+ .expect("Failed to create RMS norm kernel");
// ---- Generate random data ----
let mut rng = StdRng::seed_from_u64(42);
@@ -698,7 +713,7 @@ fn perf_rms_norm_16k() {
#[test]
fn qk_norm_test() {
// Test to verify that the QK norm kernel now accesses the correct data ranges
- let mtl_context = match MetalContext::new() {
+ let mtl_context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping QK norm buffer addressing test: {}", e);
@@ -766,11 +781,23 @@ fn qk_norm_test() {
.expect("Failed to create buffer");
// Create QK norm kernels
- let q_kernel = QKNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)
- .expect("Failed to create Q norm kernel");
-
- let k_kernel = QKNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)
- .expect("Failed to create K norm kernel");
+ let q_kernel = <<Metal as Backend>::Kernels as Kernels>::QKNormKernel::new(
+ &mtl_context,
+ DataType::F32,
+ DataType::F32,
+ DataType::F32,
+ DataType::F32,
+ )
+ .expect("Failed to create Q norm kernel");
+
+ let k_kernel = <<Metal as Backend>::Kernels as Kernels>::QKNormKernel::new(
+ &mtl_context,
+ DataType::F32,
+ DataType::F32,
+ DataType::F32,
+ DataType::F32,
+ )
+ .expect("Failed to create K norm kernel");
// Test Q head normalization
{
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index f89ae0bd1..231fd027f 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -7,18 +7,13 @@ use uzu::{
DataType,
backends::{
common::{
- CommandBuffer, Context,
+ Backend, CommandBuffer, Context, Kernels,
kernel::{
GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel, TopPKernel,
sampling::{ArgmaxStrategy, SamplingKernel},
},
},
- metal::{
- Metal, MetalContext,
- kernel::dsl::{
- GumbelMetalKernel, MinPMetalKernel, TemperatureMetalKernel, TopKMetalKernel, TopPMetalKernel,
- },
- },
+ metal::Metal,
},
language_model::gumbel::{gumbel_float, revidx},
session::parameter::SamplingMethod,
@@ -80,7 +75,7 @@ fn cpu_reference_min_p(
}
fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping argmax test: {}", e);
@@ -171,7 +166,7 @@ fn test_topp_sampling_from_prob_exact_match(
use rand::{SeedableRng, rngs::StdRng};
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
eprintln!("Skipping topP exact match test: {}", e);
@@ -310,7 +305,7 @@ fn test_topp_sampling_statistical_large() {
use rand::{RngExt, SeedableRng, rngs::StdRng};
// ===== 1. Create Metal context =====
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping large statistical test: {}", e);
@@ -427,7 +422,7 @@ fn perf_topp_128k_vocab() {
use rand::{RngExt, SeedableRng, rngs::StdRng};
// ---- Metal context ----
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping perf test: {}", e);
@@ -532,7 +527,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
use rand::{RngExt, SeedableRng, rngs::StdRng};
// ---- Metal context ----
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping argmax perf test: {}", e);
@@ -647,7 +642,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
#[test]
fn test_categorical_sampling() {
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping categorical test: {}", e);
@@ -769,7 +764,7 @@ fn test_categorical_sampling() {
fn test_categorical_sampling_statistical() {
use rand::{RngExt, SeedableRng, rngs::StdRng};
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping categorical statistical test: {}", e);
@@ -900,7 +895,7 @@ fn perf_categorical_128k_vocab() {
use rand::{RngExt, SeedableRng, rngs::StdRng};
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping categorical perf test: {}", e);
@@ -997,7 +992,7 @@ fn perf_categorical_128k_vocab() {
#[test]
fn test_temperature_gpu_cpu_match() {
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping temperature gpu cpu match test: {}", e);
@@ -1011,7 +1006,8 @@ fn test_temperature_gpu_cpu_match() {
const RTOL: f32 = 1e-6;
const ATOL: f32 = 1e-6;
- let kernel = TemperatureMetalKernel::new(&context, DataType::F32).expect("Failed to create temperature kernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::TemperatureKernel::new(&context, DataType::F32)
+ .expect("Failed to create temperature kernel");
let logits: Vec<f32> = (0..BATCH * VOCAB).map(|i| ((i * 37 % 1000) as f32 - 500.0) * 0.01).collect();
@@ -1060,7 +1056,7 @@ fn test_temperature_gpu_cpu_match() {
fn test_topk_gpu_cpu_match() {
use rand::{SeedableRng, rngs::StdRng};
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping topk gpu cpu match test: {}", e);
@@ -1072,7 +1068,8 @@ fn test_topk_gpu_cpu_match() {
const VOCAB: usize = 1024;
const TOPK: u32 = 16;
- let kernel = TopKMetalKernel::new(&context, DataType::F32).expect("Failed to create topk kernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::TopKKernel::new(&context, DataType::F32)
+ .expect("Failed to create topk kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -1124,7 +1121,7 @@ fn test_topk_gpu_cpu_match() {
fn test_topp_gpu_cpu_match() {
use rand::{SeedableRng, rngs::StdRng};
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping topp gpu cpu match test: {}", e);
@@ -1136,7 +1133,8 @@ fn test_topp_gpu_cpu_match() {
const VOCAB: usize = 1024;
const TOPP: f32 = 0.9;
- let kernel = TopPMetalKernel::new(&context, DataType::F32).expect("Failed to create topp kernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::TopPKernel::new(&context, DataType::F32)
+ .expect("Failed to create topp kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -1193,7 +1191,7 @@ fn test_topp_gpu_cpu_match() {
fn test_minp_gpu_cpu_match() {
use rand::{RngExt, SeedableRng, rngs::StdRng};
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping minp gpu cpu match test: {}", e);
@@ -1205,7 +1203,8 @@ fn test_minp_gpu_cpu_match() {
const VOCAB: usize = 1024;
const MINP: f32 = 0.1;
- let kernel = MinPMetalKernel::new(&context, DataType::F32).expect("Failed to create minp kernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::MinPKernel::new(&context, DataType::F32)
+ .expect("Failed to create minp kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -1257,7 +1256,7 @@ fn test_minp_sampling_exact_match(
use rand::{SeedableRng, rngs::StdRng};
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
eprintln!("Skipping minp exact match test: {}", e);
@@ -1376,7 +1375,7 @@ fn test_minp_sampling_match_large() {
#[test]
fn test_gumbel_gpu_cpu_match() {
- let context = match MetalContext::new() {
+ let context = match <Metal as Backend>::Context::new() {
Ok(ctx) => ctx,
Err(e) => {
println!("Skipping gumbel gpu cpu match test: {}", e);
@@ -1389,7 +1388,8 @@ fn test_gumbel_gpu_cpu_match() {
const RTOL: f32 = 0.01;
const ATOL: f32 = 1e-6;
- let kernel = GumbelMetalKernel::new(&context, DataType::F32).expect("Failed to create gumbel kernel");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::GumbelKernel::new(&context, DataType::F32)
+ .expect("Failed to create gumbel kernel");
let logits = vec![0.0f32; BATCH * VOCAB];
let seeds: Vec<u64> = (0_u64..BATCH as u64).collect();
diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rs
index 436d552e7..e33fbd8da 100644
--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs
+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs
@@ -12,13 +12,13 @@ use uzu::{
DataType,
backends::{
common::{
- Context,
+ Backend, Context, Kernels,
kernel::{
Conv1dScanKernel,
ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
},
- metal::{Metal, MetalContext, kernel::dsl::Conv1dScanMetalKernel},
+ metal::Metal,
},
};
@@ -198,7 +198,7 @@ impl SSDPrefillFixture {
}
fn run_prefill_kernel_mode(
- ctx: &MetalContext,
+ ctx: &<Metal as Backend>::Context,
kernel: &SSDPrefillKernels<Metal>,
fixture: &SSDPrefillFixture,
mode: SSDPrefillMode,
@@ -261,8 +261,8 @@ fn run_prefill_kernel_mode(
}
fn run_conv_scan_once(
- ctx: &MetalContext,
- kernel: &Conv1dScanMetalKernel,
+ ctx: &<Metal as Backend>::Context,
+ kernel: &<<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel,
suffix_len: usize,
channels: usize,
kernel_size: i32,
@@ -359,7 +359,7 @@ fn run_conv_scan_once(
}
fn assert_deterministic_for_mode(mode: SSDPrefillMode) {
- let Some(ctx) = MetalContext::new().ok() else {
+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
eprintln!("Skipping SSD prefill determinism test: no Metal device");
return;
};
@@ -374,7 +374,7 @@ fn assert_deterministic_for_mode(mode: SSDPrefillMode) {
}
fn assert_matches_cpu_reference(mode: SSDPrefillMode) {
- let Some(ctx) = MetalContext::new().ok() else {
+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
eprintln!("Skipping SSD prefill reference test: no Metal device");
return;
};
@@ -460,11 +460,12 @@ fn ssd_prefill_single_pass_matches_cpu_reference() {
#[test]
fn conv1d_scan_is_deterministic() {
- let Some(ctx) = MetalContext::new().ok() else {
+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
eprintln!("Skipping conv1d scan determinism test: no Metal device");
return;
};
- let kernel = Conv1dScanMetalKernel::new(&ctx, DataType::F32, 0u32, true).unwrap();
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, 0u32, true).unwrap();
let suffix_len = 192usize;
let channels = 8usize;
diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rs
index 222bbeb7a..e952f8eb5 100644
--- a/crates/uzu/tests/ssm_update_test.rs
+++ b/crates/uzu/tests/ssm_update_test.rs
@@ -6,8 +6,8 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTL
use uzu::{
DataType,
backends::{
- common::{Context, kernel::SSDUpdateKernel},
- metal::{MetalContext, kernel::dsl::SSDUpdateMetalKernel},
+ common::{Backend, Context, Kernels, kernel::SSDUpdateKernel},
+ metal::Metal,
},
};
@@ -174,7 +174,7 @@ fn ssd_update_ref_bf16(
#[test]
fn ssd_update_with_z_bf16() {
- let Some(ctx) = MetalContext::new().ok() else {
+ let Some(ctx) = <Metal as Backend>::Context::new().ok() else {
eprintln!("Skipping: no Metal device");
return;
};
@@ -238,7 +238,7 @@ fn ssd_update_with_z_bf16() {
.new_buffer(bsz * h * dh * n * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let kernel = SSDUpdateMetalKernel::new(&ctx, DataType::BF16).unwrap();
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16).unwrap();
let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
diff --git a/crates/uzu/tests/tracer_test.rs b/crates/uzu/tests/tracer_test.rs
index a8928a7f2..d78365e79 100644
--- a/crates/uzu/tests/tracer_test.rs
+++ b/crates/uzu/tests/tracer_test.rs
@@ -3,7 +3,7 @@
mod common;
use std::path::PathBuf;
-use uzu::tracer::TraceValidator;
+use uzu::{backends::metal::Metal, tracer::TraceValidator};
fn build_model_path() -> PathBuf {
common::get_test_model_path()
@@ -19,7 +19,7 @@ fn test_tracer() {
return;
}
- let mut tracer = match TraceValidator::new(&model_path) {
+ let mut tracer = match TraceValidator::<Metal>::new(&model_path) {
Ok(t) => t,
Err(e) => {
println!("Failed to create TraceValidator: {:?}", e);