A high-performance inference engine for AI models
M6 driver bug workaround (#901)
uuuvn1da16f3b617d
diff --git a/crates/uzu-engine/src/backends/metal/buffer/sparse.rs b/crates/uzu-engine/src/backends/metal/buffer/sparse.rsindex f4858acc9..9e41a6cad 100644--- a/crates/uzu-engine/src/backends/metal/buffer/sparse.rs+++ b/crates/uzu-engine/src/backends/metal/buffer/sparse.rs@@ -6,16 +6,10 @@ use metal::{MTLSparseTextureMappingMode,};use objc2::{rc::Retained, runtime::ProtocolObject};-use parking_lot::Mutex;use crate::backends::{common::{Buffer, SparseBuffer},- metal::{- Metal, MetalContext,- error::MetalError,- heaps::{MetalHeapPage, MetalHeaps},- metal_extensions::SparsePageSizeExt,- },+ metal::{Metal, MetalContext, error::MetalError, heaps::MetalHeapPage, metal_extensions::SparsePageSizeExt},};pub struct MetalSparseBuffer {@@ -23,14 +17,12 @@ pub struct MetalSparseBuffer {pages: Vec<MetalHeapPage>,length: usize,capacity: usize,- heaps: Arc<MetalHeaps>,- command_queue: Retained<ProtocolObject<dyn MTL4CommandQueue>>,- residency_set: Arc<Mutex<Retained<ProtocolObject<dyn MTLResidencySet>>>>,+ context: Arc<MetalContext>,}impl MetalSparseBuffer {pub(in crate::backends::metal) fn new(- context: &MetalContext,+ context: Arc<MetalContext>,capacity: usize,) -> Result<Self, MetalError> {let page_size = context.heaps.page_size();@@ -55,9 +47,7 @@ impl MetalSparseBuffer {pages: Vec::new(),length: 0,capacity,- heaps: context.heaps.clone(),- command_queue: context.command_queue.clone(),- residency_set: context.residency_set.clone(),+ context,})}@@ -83,15 +73,16 @@ impl SparseBuffer for MetalSparseBuffer {return Ok(());}- let old_page_count = self.length.div_ceil(self.heaps.page_size().in_bytes());- let new_page_count = until.div_ceil(self.heaps.page_size().in_bytes());+ let old_page_count = self.length.div_ceil(self.context.heaps.page_size().in_bytes());+ let new_page_count = until.div_ceil(self.context.heaps.page_size().in_bytes());if new_page_count > old_page_count {- let batches = self.heaps.allocate(new_page_count - old_page_count)?;+ let mut sparse_state = self.context.sparse_state.lock();+ let batches = self.context.heaps.allocate(new_page_count - old_page_count)?;let mut current_page_count = old_page_count;for batch in &batches {let num_pages_in_batch = batch.range().iter().len();- self.command_queue.update_buffer_mappings(+ self.context.sparse_queue.update_buffer_mappings(&self.buffer,Some(batch.heap()),&[MTL4UpdateSparseBufferMappingOperation::new(@@ -102,6 +93,7 @@ impl SparseBuffer for MetalSparseBuffer {);current_page_count += num_pages_in_batch;}+ sparse_state.did_ops = true;self.pages.extend(batches);}@@ -114,16 +106,18 @@ impl SparseBuffer for MetalSparseBuffer {impl Drop for MetalSparseBuffer {fn drop(&mut self) {if self.length > 0 {- self.command_queue.update_buffer_mappings(+ let mut sparse_state = self.context.sparse_state.lock();+ self.context.sparse_queue.update_buffer_mappings(&self.buffer,None,&[MTL4UpdateSparseBufferMappingOperation::new(MTLSparseTextureMappingMode::Unmap,- 0..self.length.div_ceil(self.heaps.page_size().in_bytes()),+ 0..self.length.div_ceil(self.context.heaps.page_size().in_bytes()),0,)],);- let event = self.command_queue.device().new_shared_event().unwrap();+ sparse_state.did_ops = true;+ let event = self.context.sparse_queue.device().new_shared_event().unwrap();let retain = (self.buffer.clone(), std::mem::take(&mut self.pages), event.clone());event.notify_listener_at_value(&MTLSharedEventListener::shared_listener(),@@ -132,10 +126,10 @@ impl Drop for MetalSparseBuffer {let _ = &retain;}),);- self.command_queue.signal_event_value(event.as_ref(), 1);+ self.context.sparse_queue.signal_event_value(event.as_ref(), 1);}- let residency_set_locked = self.residency_set.lock();+ let residency_set_locked = self.context.residency_set.lock();residency_set_locked.remove_allocation(self.buffer.as_ref());residency_set_locked.commit();}diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rsindex 194b581db..985a6d0e5 100644--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs@@ -6,9 +6,9 @@ use std::{use metal::{MTL4ArgumentTable, MTL4ArgumentTableDescriptor, MTL4CommandAllocator, MTL4CommandBuffer, MTL4CommandBufferExt,- MTL4CommandEncoder, MTL4CommandEncoderExt, MTL4CommandQueueExt, MTL4CommitFeedback, MTL4CommitFeedbackExt,- MTL4CommitFeedbackHandler, MTL4CommitOptions, MTL4ComputeCommandEncoder, MTL4ComputeCommandEncoderExt,- MTL4VisibilityOptions, MTLDeviceExt, MTLStages,+ MTL4CommandEncoder, MTL4CommandEncoderExt, MTL4CommandQueue, MTL4CommandQueueExt, MTL4CommitFeedback,+ MTL4CommitFeedbackExt, MTL4CommitFeedbackHandler, MTL4CommitOptions, MTL4ComputeCommandEncoder,+ MTL4ComputeCommandEncoderExt, MTL4VisibilityOptions, MTLDeviceExt, MTLStages,};use objc2::{rc::Retained, runtime::ProtocolObject};use rangemap::RangeSet;@@ -251,6 +251,16 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {type CommandBuffer = MetalCommandBuffer;fn submit(self) -> MetalCommandBufferPending {+ let mut sparse_state_locked = self.context.sparse_state.lock();+ if sparse_state_locked.did_ops {+ sparse_state_locked.did_ops = false;+ sparse_state_locked.event_value += 1;+ let value = sparse_state_locked.event_value;+ self.context.sparse_queue.signal_event_value(&self.context.sparse_event, value);+ self.context.command_queue.wait_for_event_value(&self.context.sparse_event, value);+ }+ drop(sparse_state_locked);+let (sender, receiver) = mpsc::channel();let command_allocator = self.command_allocator.clone();diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rsindex a50fc6aba..90ee23b35 100644--- a/crates/uzu-engine/src/backends/metal/context.rs+++ b/crates/uzu-engine/src/backends/metal/context.rs@@ -9,8 +9,8 @@ use std::{use metal::{MTL4CommandQueue, MTL4Compiler, MTL4CompilerDescriptor, MTLCaptureDescriptor, MTLCaptureDestination,- MTLCaptureManager, MTLCaptureTarget, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLFunctionConstantValues,- MTLGPUFamily, MTLLibrary, MTLResidencySet, MTLResidencySetDescriptor, MTLSparsePageSize,+ MTLCaptureManager, MTLCaptureTarget, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,+ MTLFunctionConstantValues, MTLGPUFamily, MTLLibrary, MTLResidencySet, MTLResidencySetDescriptor, MTLSparsePageSize,};use objc2::{rc::Retained, runtime::ProtocolObject};use parking_lot::Mutex;@@ -33,6 +33,11 @@ use crate::backends::{pub(super) const LARGE_MIN_GPU_CORES: u32 = 30;+pub(super) struct SparseState {+ pub did_ops: bool,+ pub event_value: u64,+}+pub struct MetalContext {pub device: Retained<ProtocolObject<dyn MTLDevice>>,pub gpu_core_count: u32,@@ -41,6 +46,9 @@ pub struct MetalContext {pub device_name: String,pub(super) residency_set: Arc<Mutex<Retained<ProtocolObject<dyn MTLResidencySet>>>>,pub command_queue: Retained<ProtocolObject<dyn MTL4CommandQueue>>,+ pub(super) sparse_queue: Retained<ProtocolObject<dyn MTL4CommandQueue>>,+ pub(super) sparse_event: Retained<ProtocolObject<dyn MTLEvent>>,+ pub(super) sparse_state: Mutex<SparseState>,compiler: Retained<ProtocolObject<dyn MTL4Compiler>>,pub(super) block_allocator: Arc<BlockAllocator<MetalDenseBuffer>>,pub(super) heaps: Arc<MetalHeaps>,@@ -121,6 +129,16 @@ impl Context for MetalContext {let command_queue = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueue)?;command_queue.add_residency_set(&residency_set);+ let sparse_queue = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueue)?;+ sparse_queue.add_residency_set(&residency_set);++ let sparse_event = device.new_event().ok_or(MetalError::CannotCreateEvent)?;++ let sparse_state = SparseState {+ did_ops: false,+ event_value: 0,+ };+let compiler = device.new_compiler_with_descriptor(&MTL4CompilerDescriptor::new()).map_err(|error| MetalError::CannotCreateCompiler(error.to_string()))?;@@ -137,6 +155,9 @@ impl Context for MetalContext {device_name,residency_set: Arc::new(Mutex::new(residency_set)),command_queue,+ sparse_queue,+ sparse_event,+ sparse_state: Mutex::new(sparse_state),compiler,block_allocator,heaps,@@ -171,7 +192,7 @@ impl Context for MetalContext {&self,capacity: usize,) -> Result<<Self::Backend as Backend>::SparseBuffer, <Self::Backend as Backend>::Error> {- MetalSparseBuffer::new(self, capacity)+ MetalSparseBuffer::new(self.weak_self.upgrade().unwrap(), capacity)}fn create_allocation_pool(&self) -> Arc<<Metal as Backend>::AllocationPool> {diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rsindex 755bb10c8..e51351964 100644--- a/crates/uzu-engine/src/backends/metal/error.rs+++ b/crates/uzu-engine/src/backends/metal/error.rs@@ -21,6 +21,8 @@ pub enum MetalError {CannotCreateCompiler(String),#[error("Cannot create command queue")]CannotCreateCommandQueue,+ #[error("Cannot create event")]+ CannotCreateEvent,#[error("Cannot create buffer")]CannotCreateBuffer,#[error("Cannot create heap")]