Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

M6 driver bug workaround (#901)

uuuvn1da16f3b617d
diff --git a/crates/uzu-engine/src/backends/metal/buffer/sparse.rs b/crates/uzu-engine/src/backends/metal/buffer/sparse.rs
index f4858acc9..9e41a6cad 100644
--- a/crates/uzu-engine/src/backends/metal/buffer/sparse.rs
+++ b/crates/uzu-engine/src/backends/metal/buffer/sparse.rs
@@ -6,16 +6,10 @@ use metal::{
MTLSparseTextureMappingMode,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
-use parking_lot::Mutex;
use crate::backends::{
common::{Buffer, SparseBuffer},
- metal::{
- Metal, MetalContext,
- error::MetalError,
- heaps::{MetalHeapPage, MetalHeaps},
- metal_extensions::SparsePageSizeExt,
- },
+ metal::{Metal, MetalContext, error::MetalError, heaps::MetalHeapPage, metal_extensions::SparsePageSizeExt},
};
pub struct MetalSparseBuffer {
@@ -23,14 +17,12 @@ pub struct MetalSparseBuffer {
pages: Vec<MetalHeapPage>,
length: usize,
capacity: usize,
- heaps: Arc<MetalHeaps>,
- command_queue: Retained<ProtocolObject<dyn MTL4CommandQueue>>,
- residency_set: Arc<Mutex<Retained<ProtocolObject<dyn MTLResidencySet>>>>,
+ context: Arc<MetalContext>,
}
impl MetalSparseBuffer {
pub(in crate::backends::metal) fn new(
- context: &MetalContext,
+ context: Arc<MetalContext>,
capacity: usize,
) -> Result<Self, MetalError> {
let page_size = context.heaps.page_size();
@@ -55,9 +47,7 @@ impl MetalSparseBuffer {
pages: Vec::new(),
length: 0,
capacity,
- heaps: context.heaps.clone(),
- command_queue: context.command_queue.clone(),
- residency_set: context.residency_set.clone(),
+ context,
})
}
@@ -83,15 +73,16 @@ impl SparseBuffer for MetalSparseBuffer {
return Ok(());
}
- let old_page_count = self.length.div_ceil(self.heaps.page_size().in_bytes());
- let new_page_count = until.div_ceil(self.heaps.page_size().in_bytes());
+ let old_page_count = self.length.div_ceil(self.context.heaps.page_size().in_bytes());
+ let new_page_count = until.div_ceil(self.context.heaps.page_size().in_bytes());
if new_page_count > old_page_count {
- let batches = self.heaps.allocate(new_page_count - old_page_count)?;
+ let mut sparse_state = self.context.sparse_state.lock();
+ let batches = self.context.heaps.allocate(new_page_count - old_page_count)?;
let mut current_page_count = old_page_count;
for batch in &batches {
let num_pages_in_batch = batch.range().iter().len();
- self.command_queue.update_buffer_mappings(
+ self.context.sparse_queue.update_buffer_mappings(
&self.buffer,
Some(batch.heap()),
&[MTL4UpdateSparseBufferMappingOperation::new(
@@ -102,6 +93,7 @@ impl SparseBuffer for MetalSparseBuffer {
);
current_page_count += num_pages_in_batch;
}
+ sparse_state.did_ops = true;
self.pages.extend(batches);
}
@@ -114,16 +106,18 @@ impl SparseBuffer for MetalSparseBuffer {
impl Drop for MetalSparseBuffer {
fn drop(&mut self) {
if self.length > 0 {
- self.command_queue.update_buffer_mappings(
+ let mut sparse_state = self.context.sparse_state.lock();
+ self.context.sparse_queue.update_buffer_mappings(
&self.buffer,
None,
&[MTL4UpdateSparseBufferMappingOperation::new(
MTLSparseTextureMappingMode::Unmap,
- 0..self.length.div_ceil(self.heaps.page_size().in_bytes()),
+ 0..self.length.div_ceil(self.context.heaps.page_size().in_bytes()),
0,
)],
);
- let event = self.command_queue.device().new_shared_event().unwrap();
+ sparse_state.did_ops = true;
+ let event = self.context.sparse_queue.device().new_shared_event().unwrap();
let retain = (self.buffer.clone(), std::mem::take(&mut self.pages), event.clone());
event.notify_listener_at_value(
&MTLSharedEventListener::shared_listener(),
@@ -132,10 +126,10 @@ impl Drop for MetalSparseBuffer {
let _ = &retain;
}),
);
- self.command_queue.signal_event_value(event.as_ref(), 1);
+ self.context.sparse_queue.signal_event_value(event.as_ref(), 1);
}
- let residency_set_locked = self.residency_set.lock();
+ let residency_set_locked = self.context.residency_set.lock();
residency_set_locked.remove_allocation(self.buffer.as_ref());
residency_set_locked.commit();
}
diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rs
index 194b581db..985a6d0e5 100644
--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs
@@ -6,9 +6,9 @@ use std::{
use metal::{
MTL4ArgumentTable, MTL4ArgumentTableDescriptor, MTL4CommandAllocator, MTL4CommandBuffer, MTL4CommandBufferExt,
- MTL4CommandEncoder, MTL4CommandEncoderExt, MTL4CommandQueueExt, MTL4CommitFeedback, MTL4CommitFeedbackExt,
- MTL4CommitFeedbackHandler, MTL4CommitOptions, MTL4ComputeCommandEncoder, MTL4ComputeCommandEncoderExt,
- MTL4VisibilityOptions, MTLDeviceExt, MTLStages,
+ MTL4CommandEncoder, MTL4CommandEncoderExt, MTL4CommandQueue, MTL4CommandQueueExt, MTL4CommitFeedback,
+ MTL4CommitFeedbackExt, MTL4CommitFeedbackHandler, MTL4CommitOptions, MTL4ComputeCommandEncoder,
+ MTL4ComputeCommandEncoderExt, MTL4VisibilityOptions, MTLDeviceExt, MTLStages,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use rangemap::RangeSet;
@@ -251,6 +251,16 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {
type CommandBuffer = MetalCommandBuffer;
fn submit(self) -> MetalCommandBufferPending {
+ let mut sparse_state_locked = self.context.sparse_state.lock();
+ if sparse_state_locked.did_ops {
+ sparse_state_locked.did_ops = false;
+ sparse_state_locked.event_value += 1;
+ let value = sparse_state_locked.event_value;
+ self.context.sparse_queue.signal_event_value(&self.context.sparse_event, value);
+ self.context.command_queue.wait_for_event_value(&self.context.sparse_event, value);
+ }
+ drop(sparse_state_locked);
+
let (sender, receiver) = mpsc::channel();
let command_allocator = self.command_allocator.clone();
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rs
index a50fc6aba..90ee23b35 100644
--- a/crates/uzu-engine/src/backends/metal/context.rs
+++ b/crates/uzu-engine/src/backends/metal/context.rs
@@ -9,8 +9,8 @@ use std::{
use metal::{
MTL4CommandQueue, MTL4Compiler, MTL4CompilerDescriptor, MTLCaptureDescriptor, MTLCaptureDestination,
- MTLCaptureManager, MTLCaptureTarget, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLFunctionConstantValues,
- MTLGPUFamily, MTLLibrary, MTLResidencySet, MTLResidencySetDescriptor, MTLSparsePageSize,
+ MTLCaptureManager, MTLCaptureTarget, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,
+ MTLFunctionConstantValues, MTLGPUFamily, MTLLibrary, MTLResidencySet, MTLResidencySetDescriptor, MTLSparsePageSize,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use parking_lot::Mutex;
@@ -33,6 +33,11 @@ use crate::backends::{
pub(super) const LARGE_MIN_GPU_CORES: u32 = 30;
+pub(super) struct SparseState {
+ pub did_ops: bool,
+ pub event_value: u64,
+}
+
pub struct MetalContext {
pub device: Retained<ProtocolObject<dyn MTLDevice>>,
pub gpu_core_count: u32,
@@ -41,6 +46,9 @@ pub struct MetalContext {
pub device_name: String,
pub(super) residency_set: Arc<Mutex<Retained<ProtocolObject<dyn MTLResidencySet>>>>,
pub command_queue: Retained<ProtocolObject<dyn MTL4CommandQueue>>,
+ pub(super) sparse_queue: Retained<ProtocolObject<dyn MTL4CommandQueue>>,
+ pub(super) sparse_event: Retained<ProtocolObject<dyn MTLEvent>>,
+ pub(super) sparse_state: Mutex<SparseState>,
compiler: Retained<ProtocolObject<dyn MTL4Compiler>>,
pub(super) block_allocator: Arc<BlockAllocator<MetalDenseBuffer>>,
pub(super) heaps: Arc<MetalHeaps>,
@@ -121,6 +129,16 @@ impl Context for MetalContext {
let command_queue = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueue)?;
command_queue.add_residency_set(&residency_set);
+ let sparse_queue = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueue)?;
+ sparse_queue.add_residency_set(&residency_set);
+
+ let sparse_event = device.new_event().ok_or(MetalError::CannotCreateEvent)?;
+
+ let sparse_state = SparseState {
+ did_ops: false,
+ event_value: 0,
+ };
+
let compiler = device
.new_compiler_with_descriptor(&MTL4CompilerDescriptor::new())
.map_err(|error| MetalError::CannotCreateCompiler(error.to_string()))?;
@@ -137,6 +155,9 @@ impl Context for MetalContext {
device_name,
residency_set: Arc::new(Mutex::new(residency_set)),
command_queue,
+ sparse_queue,
+ sparse_event,
+ sparse_state: Mutex::new(sparse_state),
compiler,
block_allocator,
heaps,
@@ -171,7 +192,7 @@ impl Context for MetalContext {
&self,
capacity: usize,
) -> Result<<Self::Backend as Backend>::SparseBuffer, <Self::Backend as Backend>::Error> {
- MetalSparseBuffer::new(self, capacity)
+ MetalSparseBuffer::new(self.weak_self.upgrade().unwrap(), capacity)
}
fn create_allocation_pool(&self) -> Arc<<Metal as Backend>::AllocationPool> {
diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rs
index 755bb10c8..e51351964 100644
--- a/crates/uzu-engine/src/backends/metal/error.rs
+++ b/crates/uzu-engine/src/backends/metal/error.rs
@@ -21,6 +21,8 @@ pub enum MetalError {
CannotCreateCompiler(String),
#[error("Cannot create command queue")]
CannotCreateCommandQueue,
+ #[error("Cannot create event")]
+ CannotCreateEvent,
#[error("Cannot create buffer")]
CannotCreateBuffer,
#[error("Cannot create heap")]