Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

13.8 KB
Raw Blame
use std::{
    ops::{Deref, DerefMut},
    range::Range,
    sync::Arc,
    time::Duration,
};

use futures::channel::mpsc;
use metal::{
    MTL4ArgumentTable, MTL4ArgumentTableDescriptor, MTL4CommandAllocator, MTL4CommandBuffer, MTL4CommandBufferExt,
    MTL4CommandEncoder, MTL4CommandEncoderExt, MTL4CommandQueue, MTL4CommandQueueExt, MTL4CommitFeedback,
    MTL4CommitFeedbackExt, MTL4CommitFeedbackHandler, MTL4CommitOptions, MTL4ComputeCommandEncoder,
    MTL4VisibilityOptions, MTLDeviceExt, MTLStages,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use rangemap::RangeSet;

use crate::backends::{
    common::{
        Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,
        CommandBufferExecutable, CommandBufferPending, Context, TimestampSpan, allocator::bump::BumpAllocator,
    },
    metal::{
        Metal,
        buffer::MetalBufferExt,
        context::MetalContext,
        error::MetalError,
        kernel::{BlitCopyMetalKernel, BlitFillMetalKernel},
        timestamp_recorder::MetalTimestampRecorder,
    },
};

pub struct MetalCommandBuffer;

impl CommandBuffer for MetalCommandBuffer {
    type Backend = Metal;

    type Encoding = MetalCommandBufferEncoding;
    type Executable = MetalCommandBufferExecutable;
    type Pending = MetalCommandBufferPending;
    type Completed = MetalCommandBufferCompleted;
}

pub(super) struct Access {
    pub(super) range: Range<u64>,
    pub(super) write: bool,
}

pub struct MetalCommandBufferEncoding {
    command_allocator: Retained<ProtocolObject<dyn MTL4CommandAllocator>>,
    command_buffer: Retained<ProtocolObject<dyn MTL4CommandBuffer>>,
    pub(super) compute_encoder: Retained<ProtocolObject<dyn MTL4ComputeCommandEncoder>>,
    pub(super) argument_table: Retained<ProtocolObject<dyn MTL4ArgumentTable>>,
    reads: RangeSet<u64>,
    writes: RangeSet<u64>,
    constant_allocator: Option<BumpAllocator<<Metal as Backend>::GlobalGpuBuffer>>,
    allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
    pub(super) context: Arc<MetalContext>,
    timestamps: Option<MetalTimestampRecorder>,
}

unsafe impl Send for MetalCommandBufferEncoding {}

impl MetalCommandBufferEncoding {
    pub fn new(
        context: Arc<MetalContext>,
        name: Option<&str>,
        allocation_pool: Option<Arc<<Metal as Backend>::AllocationPool>>,
    ) -> Result<Self, MetalError> {
        let command_allocator = if let Some(command_allocator) = context.command_allocator_cache.lock().pop() {
            command_allocator.reset();
            command_allocator
        } else {
            context.device.new_command_allocator().ok_or(MetalError::CannotCreateCommandBuffer)?
        };

        let command_buffer = context.device.new_mtl4_command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?;
        command_buffer.begin_command_buffer_with_allocator(&command_allocator);
        command_buffer.set_label(name);

        let compute_encoder = command_buffer.compute_command_encoder().unwrap();
        ProtocolObject::<dyn MTL4CommandEncoder>::set_label(compute_encoder.as_ref(), name);

        compute_encoder.barrier_after_queue_stages_before_stages_visibility_options(
            MTLStages::Dispatch | MTLStages::ResourceState,
            MTLStages::Dispatch,
            MTL4VisibilityOptions::Device,
        );

        let argument_table_descriptor = MTL4ArgumentTableDescriptor::new();
        argument_table_descriptor.set_label(name.map(|name| format!("{name} argument table")).as_deref());
        argument_table_descriptor.set_max_buffer_bind_count(31);
        let argument_table = context
            .device
            .new_argument_table_with_descriptor(&argument_table_descriptor)
            .map_err(|error| MetalError::CannotCreateArgumentTable(error.to_string()))?;
        compute_encoder.set_argument_table(Some(&argument_table));

        let constant_allocator = BumpAllocator::new(256 * 1024, 1);

        let allocation_pool = allocation_pool.unwrap_or_else(|| context.create_allocation_pool());

        Ok(Self {
            command_allocator,
            command_buffer,
            compute_encoder,
            argument_table,
            reads: RangeSet::new(),
            writes: RangeSet::new(),
            constant_allocator: Some(constant_allocator),
            allocation_pool,
            context,
            timestamps: None,
        })
    }

    pub(super) fn access(
        &mut self,
        accesses: &[Access],
    ) {
        // TODO: more fine grained barriers
        if accesses.iter().any(|access| {
            self.writes.overlaps(&access.range.into()) || (access.write && self.reads.overlaps(&access.range.into()))
        }) {
            self.compute_encoder.barrier_after_encoder_stages_before_encoder_stages_visibility_options(
                MTLStages::Dispatch,
                MTLStages::Dispatch,
                MTL4VisibilityOptions::Device,
            );
            self.reads.clear();
            self.writes.clear();
        }

        for access in accesses {
            if access.write {
                self.writes.insert(access.range.into());
            } else {
                self.reads.insert(access.range.into());
            }
        }
    }
}

impl CommandBufferEncoding for MetalCommandBufferEncoding {
    type CommandBuffer = MetalCommandBuffer;

    fn context(&self) -> &MetalContext {
        &self.context
    }

    fn allocate_constant(
        &mut self,
        size: usize,
    ) -> Result<<Metal as Backend>::ConstantBuffer, MetalError> {
        let alignment = usize::min(size, 64).next_power_of_two();
        self.constant_allocator.as_mut().unwrap().allocate(size, alignment, |size| self.context.create_buffer_gpu(size))
    }

    fn allocate_scratch(
        &mut self,
        size: usize,
    ) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
        self.allocation_pool.allocate(size, |size| self.context.create_buffer_gpu(size))
    }

    fn encode_copy(
        &mut self,
        src: impl BufferRef<Backend = Metal>,
        dst: impl BufferMut<Backend = Metal>,
    ) {
        let size = src.size();
        assert_eq!(size, dst.size());
        if size == 0 {
            return;
        }
        let (src_buffer, src_range) = src.parts();
        let (dst_buffer, dst_range) = dst.as_ref().parts();
        let alignment = ((src_buffer.gpu_address() + src_range.start as u64)
            | (dst_buffer.gpu_address() + dst_range.start as u64))
            & 15;
        let (variant, alignment) = match alignment {
            0 => (0, 0),
            4 | 8 | 12 => (1, 4),
            _ => (2, 1),
        };
        let context = self.context.clone();
        let kernel = context.blit_copy[variant]
            .get_or_init(|| BlitCopyMetalKernel::new(&context, alignment).expect("Failed to create blit copy kernel"));
        kernel.encode(src, dst, size as u64, self);
    }

    fn encode_fill(
        &mut self,
        dst: impl BufferMut<Backend = Metal>,
        value: u8,
    ) {
        let (buffer, range) = dst.as_ref().parts();
        assert!(range.end > range.start);
        assert!(range.start.is_multiple_of(4) && range.end.is_multiple_of(4));
        let aligned = (buffer.gpu_address() + range.start as u64).is_multiple_of(16);
        let context = self.context.clone();
        let kernel = context.blit_fill[usize::from(aligned)]
            .get_or_init(|| BlitFillMetalKernel::new(&context, aligned).expect("Failed to create blit fill kernel"));
        kernel.encode(dst, (range.end - range.start) as u64, u32::from(value) * 0x0101_0101, self);
    }

    fn debug_group(
        &mut self,
        name: &str,
    ) -> MetalDebugGroupSpan<'_> {
        ProtocolObject::<dyn MTL4CommandEncoder>::push_debug_group(self.compute_encoder.as_ref(), name);
        if let Some(timestamps) = &mut self.timestamps {
            timestamps.start(name, &self.context.device, &self.compute_encoder);
        }
        MetalDebugGroupSpan {
            command_buffer: self,
        }
    }

    fn debug_point(
        &mut self,
        name: &str,
    ) {
        ProtocolObject::<dyn MTL4CommandEncoder>::insert_debug_signpost(self.compute_encoder.as_ref(), name);
    }

    fn with_timestamps(mut self) -> Self {
        self.timestamps = Some(MetalTimestampRecorder::default());
        self
    }

    fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {
        let constant_allocator = self.constant_allocator.take().unwrap();
        assert!(constant_allocator.is_done(), "attempted to end encoding while constants are still alive");

        MetalCommandBufferExecutable {
            command_allocator: self.command_allocator.clone(),
            command_buffer: self.command_buffer.clone(),
            constant_allocator,
            allocation_pool: self.allocation_pool.clone(),
            context: self.context.clone(),
            timestamps: self.timestamps.take(),
        }
    }
}

impl Drop for MetalCommandBufferEncoding {
    fn drop(&mut self) {
        self.compute_encoder.barrier_after_stages_before_queue_stages_visibility_options(
            MTLStages::Dispatch,
            MTLStages::Dispatch | MTLStages::ResourceState,
            MTL4VisibilityOptions::Device,
        );
        self.compute_encoder.end_encoding();
        self.command_buffer.end_command_buffer();
    }
}

pub struct MetalDebugGroupSpan<'a> {
    command_buffer: &'a mut MetalCommandBufferEncoding,
}

impl Deref for MetalDebugGroupSpan<'_> {
    type Target = MetalCommandBufferEncoding;

    fn deref(&self) -> &Self::Target {
        self.command_buffer
    }
}

impl DerefMut for MetalDebugGroupSpan<'_> {
    fn deref_mut(&mut self) -> &mut Self::Target {
        self.command_buffer
    }
}

impl Drop for MetalDebugGroupSpan<'_> {
    fn drop(&mut self) {
        let command_buffer = &mut *self.command_buffer;
        if let Some(timestamps) = &mut command_buffer.timestamps {
            timestamps.end(&command_buffer.context.device, &command_buffer.compute_encoder);
        }
        command_buffer.compute_encoder.pop_debug_group();
    }
}

pub struct MetalCommandBufferExecutable {
    command_allocator: Retained<ProtocolObject<dyn MTL4CommandAllocator>>,
    command_buffer: Retained<ProtocolObject<dyn MTL4CommandBuffer>>,
    constant_allocator: BumpAllocator<<Metal as Backend>::GlobalGpuBuffer>,
    allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
    context: Arc<MetalContext>,
    timestamps: Option<MetalTimestampRecorder>,
}

impl CommandBufferExecutable for MetalCommandBufferExecutable {
    type CommandBuffer = MetalCommandBuffer;

    fn submit(self) -> MetalCommandBufferPending {
        let mut sparse_state_locked = self.context.sparse_state.lock();
        if sparse_state_locked.did_ops {
            sparse_state_locked.did_ops = false;
            sparse_state_locked.event_value += 1;
            let value = sparse_state_locked.event_value;
            self.context.sparse_queue.signal_event_value(&self.context.sparse_event, value);
            self.context.command_queue.wait_for_event_value(&self.context.sparse_event, value);
        }
        drop(sparse_state_locked);

        let (sender, receiver) = mpsc::unbounded();

        let command_allocator = self.command_allocator.clone();
        let command_buffer = self.command_buffer.clone();
        let context_clone = self.context.clone();
        let timestamps = self.timestamps;

        let constant_allocator = self.constant_allocator;
        let allocation_pool = self.allocation_pool.clone();
        let feedback_handler = move |feedback: &ProtocolObject<dyn MTL4CommitFeedback>| {
            let message = if let Some(error) = feedback.error() {
                Err(MetalError::CommandBufferExecution(error.to_string()))
            } else {
                timestamps
                    .as_ref()
                    .map_or_else(|| Ok(Box::default()), |timestamps| timestamps.resolve(&context_clone.device))
                    .map(|spans| (Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()), spans))
            };
            let _ = sender.unbounded_send(message);
            context_clone.command_allocator_cache.lock().push(command_allocator.clone());
            let _keep_alive = (&command_buffer, &constant_allocator, &allocation_pool);
        };

        let options = MTL4CommitOptions::new();
        options.add_feedback_handler(&MTL4CommitFeedbackHandler::new(feedback_handler));
        self.context.command_queue.commit_with_options(&[&self.command_buffer], &options);

        MetalCommandBufferPending {
            receiver,
        }
    }
}

pub struct MetalCommandBufferPending {
    receiver: mpsc::UnboundedReceiver<Result<(Duration, Box<[TimestampSpan]>), MetalError>>,
}

impl CommandBufferPending for MetalCommandBufferPending {
    type CommandBuffer = MetalCommandBuffer;

    async fn wait_until_completed(mut self) -> Result<MetalCommandBufferCompleted, MetalError> {
        let start = std::time::Instant::now();
        let (gpu_execution_time, timestamps) = loop {
            if start.elapsed() >= Duration::from_millis(50) {
                break self.receiver.recv().await.map_err(|_| MetalError::CommandBufferWait)?;
            }
            match self.receiver.try_recv() {
                Ok(message) => break message,
                Err(mpsc::TryRecvError::Empty) => std::thread::sleep(Duration::from_micros(20)),
                Err(mpsc::TryRecvError::Closed) => return Err(MetalError::CommandBufferWait),
            }
        }?;
        Ok(MetalCommandBufferCompleted {
            gpu_execution_time,
            timestamps,
        })
    }
}

pub struct MetalCommandBufferCompleted {
    gpu_execution_time: Duration,
    timestamps: Box<[TimestampSpan]>,
}

impl CommandBufferCompleted for MetalCommandBufferCompleted {
    type CommandBuffer = MetalCommandBuffer;

    fn gpu_execution_time(&self) -> Option<Duration> {
        Some(self.gpu_execution_time)
    }

    fn timestamps(&self) -> &[TimestampSpan] {
        &self.timestamps
    }
}