A high-performance inference engine for AI models
Use Metal 4 compiler for compute pipelines (#896)
OlshaMBd4e7d6478152
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rsindex 3ae9cafa6..1d54e2191 100644--- a/crates/uzu-engine/src/backends/metal/context.rs+++ b/crates/uzu-engine/src/backends/metal/context.rs@@ -8,8 +8,10 @@ use std::{};use metal::{- MTL4CommandQueue, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCaptureTarget,- MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLFunctionConstantValues, MTLGPUFamily, MTLLibrary,+ MTL4CommandQueue, MTL4Compiler, MTL4CompilerDescriptor, MTLCaptureDescriptor,+ MTLCaptureDestination, MTLCaptureManager, MTLCaptureTarget,+ MTLComputePipelineState, MTLDevice, MTLDeviceExt,+ MTLFunctionConstantValues, MTLGPUFamily, MTLLibrary,MTLResidencySet, MTLResidencySetDescriptor, MTLSparsePageSize,};use objc2::{rc::Retained, runtime::ProtocolObject};@@ -27,7 +29,7 @@ use crate::backends::{decompression,error::MetalError,heaps::MetalHeaps,- metal_extensions::{DeviceExt, LibraryPipelineExtensions},+ metal_extensions::{CompilerPipelineExtensions, DeviceExt},},};@@ -41,6 +43,7 @@ pub struct MetalContext {pub device_name: String,pub(super) residency_set: Arc<Mutex<Retained<ProtocolObject<dyn MTLResidencySet>>>>,pub command_queue: Retained<ProtocolObject<dyn MTL4CommandQueue>>,+ compiler: Retained<ProtocolObject<dyn MTL4Compiler>>,pub(super) block_allocator: Arc<BlockAllocator<MetalDenseBuffer>>,pub(super) heaps: Arc<MetalHeaps>,pub(super) peak_memory_usage: Arc<AtomicUsize>,@@ -91,8 +94,8 @@ impl MetalContext {return Ok(pipeline.clone());}- let pipeline =- self.library(library_data, library_compressed)?.compute_pipeline_state(function_name, constants)?;+ let library = self.library(library_data, library_compressed)?;+ let pipeline = self.compiler.compute_pipeline_state(&library, function_name, constants)?;self.pipeline_cache.lock().insert(cache_key.to_string(), pipeline.clone());Ok(pipeline)@@ -120,6 +123,10 @@ impl Context for MetalContext {let command_queue = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueue)?;command_queue.add_residency_set(&residency_set);+ let compiler = device+ .new_compiler_with_descriptor(&MTL4CompilerDescriptor::new())+ .map_err(|error| MetalError::CannotCreateCompiler(error.to_string()))?;+let block_allocator = BlockAllocator::new(16 * 1024);let heaps = MetalHeaps::new(device.clone(), peak_memory_usage.clone(), MTLSparsePageSize::KB256, 256);@@ -132,6 +139,7 @@ impl Context for MetalContext {device_name,residency_set: Arc::new(Mutex::new(residency_set)),command_queue,+ compiler,block_allocator,heaps,peak_memory_usage,diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rsindex 1225ea329..755bb10c8 100644--- a/crates/uzu-engine/src/backends/metal/error.rs+++ b/crates/uzu-engine/src/backends/metal/error.rs@@ -17,6 +17,8 @@ pub enum MetalError {CannotStartGpuCapture(String),#[error("Cannot create library: {0}")]CannotCreateLibrary(String),+ #[error("Cannot create compiler: {0}")]+ CannotCreateCompiler(String),#[error("Cannot create command queue")]CannotCreateCommandQueue,#[error("Cannot create buffer")]@@ -31,8 +33,6 @@ pub enum MetalError {CommandBufferWait(RecvError),#[error("Command buffer execution failed: {0}")]CommandBufferExecution(String),- #[error("Cannot create function: {0}")]- CannotCreateFunction(String),#[error("Cannot create pipeline state for {function_name}: {error}")]CannotCreatePipelineState {function_name: String,diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rsnew file mode 100644index 000000000..653c31f41--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rs@@ -0,0 +1,50 @@+use metal::{+ MTL4Compiler, MTL4CompilerExt, MTL4ComputePipelineDescriptor, MTL4LibraryFunctionDescriptor,+ MTL4SpecializedFunctionDescriptor, MTLComputePipelineState, MTLFunctionConstantValues, MTLLibrary,+};+use objc2::{rc::Retained, runtime::ProtocolObject};++use crate::backends::metal::error::MetalError;++/// Extensions for a Metal 4 compiler to create compute pipeline states.+pub trait CompilerPipelineExtensions {+ /// Creates a compute pipeline state for a named function in the library.+ /// Optionally specializes the function with constant values.+ fn compute_pipeline_state(+ &self,+ library: &ProtocolObject<dyn MTLLibrary>,+ function_name: &str,+ constants: Option<&MTLFunctionConstantValues>,+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError>;+}++impl CompilerPipelineExtensions for ProtocolObject<dyn MTL4Compiler> {+ fn compute_pipeline_state(+ &self,+ library: &ProtocolObject<dyn MTLLibrary>,+ function_name: &str,+ constants: Option<&MTLFunctionConstantValues>,+ ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {+ let library_function = MTL4LibraryFunctionDescriptor::new();+ library_function.set_library(Some(library));+ library_function.set_name(Some(function_name));++ let pipeline_descriptor = MTL4ComputePipelineDescriptor::new();+ match constants {+ Some(constants) => {+ let specialized_function = MTL4SpecializedFunctionDescriptor::new();+ specialized_function.set_function_descriptor(Some(&library_function));+ specialized_function.set_constant_values(Some(constants));+ pipeline_descriptor.set_compute_function_descriptor(Some(&specialized_function));+ },+ None => pipeline_descriptor.set_compute_function_descriptor(Some(&library_function)),+ }++ self.new_compute_pipeline_state_with_descriptor_compiler_task_options_error(&pipeline_descriptor, None).map_err(+ |error| MetalError::CannotCreatePipelineState {+ function_name: function_name.to_owned(),+ error: error.to_string(),+ },+ )+ }+}diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/library_extensions_pipeline.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/library_extensions_pipeline.rsdeleted file mode 100644index 887383597..000000000--- a/crates/uzu-engine/src/backends/metal/metal_extensions/library_extensions_pipeline.rs+++ /dev/null@@ -1,41 +0,0 @@-use metal::{MTLComputePipelineState, MTLDeviceExt, MTLFunctionConstantValues, MTLLibrary, MTLLibraryExt};-use objc2::{rc::Retained, runtime::ProtocolObject};--use crate::backends::metal::error::MetalError;--/// Extensions for Library to create compute pipeline states-pub trait LibraryPipelineExtensions {- /// Creates a compute pipeline state for a named function in the library.- /// Optionally accepts function constants.- fn compute_pipeline_state(- &self,- function_name: &str,- constants: Option<&MTLFunctionConstantValues>,- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError>;-}--impl LibraryPipelineExtensions for ProtocolObject<dyn MTLLibrary> {- fn compute_pipeline_state(- &self,- function_name: &str,- constants: Option<&MTLFunctionConstantValues>,- ) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {- let function = match constants {- Some(const_values) => self- .new_function_with_name_constant_values(function_name, const_values)- .map_err(|error| MetalError::CannotCreateFunction(format!("{function_name}: {error}")))?,- None => self- .new_function_with_name(function_name)- .ok_or_else(|| MetalError::CannotCreateFunction(function_name.to_owned()))?,- };-- let device = self.device();-- device.new_compute_pipeline_state_with_function(&function).map_err(|nserror| {- MetalError::CannotCreatePipelineState {- function_name: function_name.to_owned(),- error: nserror.to_string(),- }- })- }-}diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rsindex 88e550683..84d04caf7 100644--- a/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs+++ b/crates/uzu-engine/src/backends/metal/metal_extensions/mod.rs@@ -1,13 +1,13 @@+mod compiler_extensions_pipeline;mod data_type;mod device_extensions;mod function_constant_values_extensions_set_value;mod gpu_family_extensions;-mod library_extensions_pipeline;mod sparse_page_size_extensions;+pub use compiler_extensions_pipeline::CompilerPipelineExtensions;pub use data_type::MetalDataTypeExt;pub use device_extensions::DeviceExt;pub use function_constant_values_extensions_set_value::FunctionConstantValuesSetValue;pub use gpu_family_extensions::GpuFamilyExt;-pub use library_extensions_pipeline::LibraryPipelineExtensions;pub use sparse_page_size_extensions::SparsePageSizeExt;