Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

4.6 KB
Raw Blame
use std::sync::Arc;

use metal::{
    MTL4CommandQueue, MTL4CommandQueueExt, MTL4UpdateSparseBufferMappingOperation, MTLBuffer, MTLDeviceExt,
    MTLEventExt, MTLResidencySet, MTLResourceOptions, MTLSharedEventExt, MTLSharedEventListener,
    MTLSharedEventNotificationBlock, MTLSparseTextureMappingMode,
};
use objc2::{rc::Retained, runtime::ProtocolObject};

use crate::backends::{
    common::{Buffer, GpuBuffer, SparseBuffer},
    metal::{
        Metal, context::MetalContext, error::MetalError, heaps::MetalHeapPage, metal_extensions::SparsePageSizeExt,
    },
};

pub struct MetalSparseBuffer {
    buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
    pages: Vec<MetalHeapPage>,
    length: usize,
    capacity: usize,
    context: Arc<MetalContext>,
}

impl MetalSparseBuffer {
    pub(in crate::backends::metal) fn new(
        context: Arc<MetalContext>,
        capacity: usize,
    ) -> Result<Self, MetalError> {
        let page_size = context.heaps.page_size();

        let buffer = context
            .device
            .new_buffer_with_length_options_placement_sparse_page_size(
                capacity.next_multiple_of(page_size.in_bytes()),
                MTLResourceOptions::STORAGE_MODE_PRIVATE,
                page_size,
            )
            .ok_or(MetalError::CannotCreateBuffer)?;

        let residency_set_locked = context.residency_set.lock();
        residency_set_locked.add_allocation(buffer.as_ref());
        residency_set_locked.commit();
        residency_set_locked.request_residency();
        drop(residency_set_locked);

        Ok(Self {
            buffer,
            pages: Vec::new(),
            length: 0,
            capacity,
            context,
        })
    }

    pub(super) fn mtl_buffer(&self) -> &Retained<ProtocolObject<dyn MTLBuffer>> {
        &self.buffer
    }
}

impl Buffer for MetalSparseBuffer {
    type Backend = Metal;

    fn size(&self) -> usize {
        self.capacity
    }
}

impl GpuBuffer for MetalSparseBuffer {}

impl SparseBuffer for MetalSparseBuffer {
    fn map(
        &mut self,
        until: usize,
    ) -> Result<(), MetalError> {
        if until <= self.length {
            return Ok(());
        }

        let old_page_count = self.length.div_ceil(self.context.heaps.page_size().in_bytes());
        let new_page_count = until.div_ceil(self.context.heaps.page_size().in_bytes());

        if new_page_count > old_page_count {
            let mut sparse_state = self.context.sparse_state.lock();
            let batches = self.context.heaps.allocate(new_page_count - old_page_count)?;
            let mut current_page_count = old_page_count;
            for batch in &batches {
                let num_pages_in_batch = batch.range().iter().len();
                self.context.sparse_queue.update_buffer_mappings(
                    &self.buffer,
                    Some(batch.heap()),
                    &[MTL4UpdateSparseBufferMappingOperation::new(
                        MTLSparseTextureMappingMode::Map,
                        current_page_count..current_page_count + num_pages_in_batch,
                        batch.range().start,
                    )],
                );
                current_page_count += num_pages_in_batch;
            }
            sparse_state.did_ops = true;
            self.pages.extend(batches);
        }

        self.length = until;

        Ok(())
    }
}

impl Drop for MetalSparseBuffer {
    fn drop(&mut self) {
        if self.length > 0 {
            let mut sparse_state = self.context.sparse_state.lock();
            self.context.sparse_queue.update_buffer_mappings(
                &self.buffer,
                None,
                &[MTL4UpdateSparseBufferMappingOperation::new(
                    MTLSparseTextureMappingMode::Unmap,
                    0..self.length.div_ceil(self.context.heaps.page_size().in_bytes()),
                    0,
                )],
            );
            sparse_state.did_ops = true;
            let event = self.context.sparse_queue.device().new_shared_event().unwrap();
            event.set_label(Some("sparse unmap event"));
            let retain = (self.buffer.clone(), std::mem::take(&mut self.pages), event.clone());
            event.notify_listener_at_value(
                &MTLSharedEventListener::shared_listener(),
                1,
                &MTLSharedEventNotificationBlock::new(move |_, _| {
                    let _ = &retain;
                }),
            );
            self.context.sparse_queue.signal_event_value(event.as_ref(), 1);
        }

        let residency_set_locked = self.context.residency_set.lock();
        residency_set_locked.remove_allocation(self.buffer.as_ref());
        residency_set_locked.commit();
    }
}