A high-performance inference engine for AI models
cpu command buffer refactor (#295)
uuuvn4a3cdec73b08
diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rsindex 4fd6b0555..d825f48de 100644--- a/crates/uzu/build/cpu/compiler.rs+++ b/crates/uzu/build/cpu/compiler.rs@@ -340,8 +340,8 @@ impl CpuCompiler {(Some(quote! { #buffer_lifetime }),match access {- KernelBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, std::cell::UnsafeCell<Box<[u8]>>> },- KernelBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, std::cell::UnsafeCell<Box<[u8]>>> },+ KernelBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, std::cell::UnsafeCell<std::pin::Pin<Box<[u8]>>>> },+ KernelBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, std::cell::UnsafeCell<std::pin::Pin<Box<[u8]>>>> },},)},@@ -371,9 +371,14 @@ impl CpuCompiler {let argument_ident = &argument.name;match &argument.ty {FunctionArgumentType::Buffer(access) => {- let buffer_ptr = match access {- KernelBufferAccess::Read => quote! { (&*__dsl_buffer.get()).as_ptr() },- KernelBufferAccess::ReadWrite => quote! { (&mut *__dsl_buffer.get()).as_mut_ptr() },+ let (buffer_ptr, buffer_ptr_wrapper) = match access {+ KernelBufferAccess::Read => {+ (quote! { (&*__dsl_buffer.get()).as_ptr() }, quote! { crate::utils::pointers::SendPtr })+ },+ KernelBufferAccess::ReadWrite => (+ quote! { (&mut *__dsl_buffer.get()).as_mut_ptr() },+ quote! { crate::utils::pointers::SendPtrMut },+ ),};if argument.conditional.is_some() {@@ -381,7 +386,7 @@ impl CpuCompiler {let #argument_ident = #argument_ident.map(|__dsl_buffer_impl| unsafe {let (__dsl_buffer, __dsl_offset) = __dsl_buffer_impl.into_parts();- #buffer_ptr.byte_add(__dsl_offset)+ #buffer_ptr_wrapper(#buffer_ptr.byte_add(__dsl_offset))});})} else {@@ -389,7 +394,7 @@ impl CpuCompiler {let #argument_ident = unsafe {let (__dsl_buffer, __dsl_offset) = #argument_ident.into_parts();- #buffer_ptr.byte_add(__dsl_offset)+ #buffer_ptr_wrapper(#buffer_ptr.byte_add(__dsl_offset))};})}@@ -418,9 +423,9 @@ impl CpuCompiler {match &argument.ty {FunctionArgumentType::Buffer(_) => {if argument.conditional.is_some() {- quote! { #argument_ident.map(|p| p as _) }+ quote! { #argument_ident.map(|p| p.as_ptr() as _) }} else {- quote! { #argument_ident as _ }+ quote! { #argument_ident.as_ptr() as _ }}},FunctionArgumentType::Constant(_) => quote! { #argument_ident.as_ref() },diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex 7abf67225..20a7b2331 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -110,7 +110,8 @@ pub trait CommandBufferEncoding {&<Self::CommandBuffer as CommandBuffer>::Completed,<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,>,- ) + 'static,+ ) + Send+ + 'static,);fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;diff --git a/crates/uzu/src/backends/common/encoder.rs b/crates/uzu/src/backends/common/encoder.rsindex 7717599d9..3a0e74009 100644--- a/crates/uzu/src/backends/common/encoder.rs+++ b/crates/uzu/src/backends/common/encoder.rs@@ -118,7 +118,7 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {pub fn add_completion_handler(&mut self,- handler: impl FnOnce(Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>) + 'static,+ handler: impl FnOnce(Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>) + Send + 'static,) {self.command_buffer.add_completion_handler(handler);}diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rsindex 9d1e5e665..bd1bfe19d 100644--- a/crates/uzu/src/backends/cpu/backend.rs+++ b/crates/uzu/src/backends/cpu/backend.rs@@ -1,4 +1,4 @@-use std::cell::{RefCell, UnsafeCell};+use std::{cell::UnsafeCell, pin::Pin, sync::atomic::AtomicU64};use super::{command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels};use crate::backends::common::Backend;@@ -9,8 +9,8 @@ pub struct Cpu;impl Backend for Cpu {type Context = CpuContext;type CommandBuffer = CpuCommandBuffer;- type Buffer = UnsafeCell<Box<[u8]>>;- type Event = RefCell<u64>;+ type Buffer = UnsafeCell<Pin<Box<[u8]>>>;+ type Event = Pin<Box<AtomicU64>>;type Kernels = CpuKernels;type Error = CpuError;diff --git a/crates/uzu/src/backends/cpu/buffer.rs b/crates/uzu/src/backends/cpu/buffer.rsindex b147cf202..72476cae5 100644--- a/crates/uzu/src/backends/cpu/buffer.rs+++ b/crates/uzu/src/backends/cpu/buffer.rs@@ -1,9 +1,9 @@-use std::{cell::UnsafeCell, os::raw::c_void, ptr::NonNull};+use std::{cell::UnsafeCell, os::raw::c_void, pin::Pin, ptr::NonNull};use super::Cpu;use crate::backends::common::Buffer;-impl Buffer for UnsafeCell<Box<[u8]>> {+impl Buffer for UnsafeCell<Pin<Box<[u8]>>> {type Backend = Cpu;fn set_label(diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rsindex 086877080..4fed33635 100644--- a/crates/uzu/src/backends/cpu/command_buffer.rs+++ b/crates/uzu/src/backends/cpu/command_buffer.rs@@ -1,66 +1,84 @@use std::{- cell::{OnceCell, RefCell, UnsafeCell},+ cell::UnsafeCell,+ pin::Pin,+ sync::{+ atomic::{AtomicU64, Ordering},+ mpsc,+ },+ thread,time::{Duration, Instant},};-use super::Cpu;-use crate::backends::{- common::{- AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,- CommandBufferInitial, CommandBufferPending,+use crate::{+ backends::{+ common::{+ AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,+ CommandBufferInitial, CommandBufferPending,+ },+ cpu::{Cpu, error::CpuError},},- cpu::error::CpuError,+ utils::pointers::{SendPtr, SendPtrMut},};-pub struct CpuCommandBuffer {- commands: Vec<Box<dyn Fn()>>,- completion_handlers: Vec<Box<dyn FnOnce(Result<&CpuCommandBuffer, CpuError>)>>,- gpu_execution_time: OnceCell<Duration>,-}+pub struct CpuCommandBuffer;impl CommandBuffer for CpuCommandBuffer {type Backend = Cpu;- type Initial = CpuCommandBuffer;- type Encoding = CpuCommandBuffer;- type Executable = CpuCommandBuffer;- type Pending = CpuCommandBuffer;- type Completed = CpuCommandBuffer;+ type Initial = CpuCommandBufferInitial;+ type Encoding = CpuCommandBufferEncoding;+ type Executable = CpuCommandBufferExecutable;+ type Pending = CpuCommandBufferPending;+ type Completed = CpuCommandBufferCompleted;+}++pub struct CpuCommandBufferInitial {+ command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>,+}++impl CpuCommandBufferInitial {+ pub fn new(command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>) -> CpuCommandBufferInitial {+ CpuCommandBufferInitial {+ command_queue,+ }+ }}-impl CpuCommandBuffer {- pub fn new() -> CpuCommandBuffer {- CpuCommandBuffer {+impl CommandBufferInitial for CpuCommandBufferInitial {+ type CommandBuffer = CpuCommandBuffer;++ fn start_encoding(self) -> CpuCommandBufferEncoding {+ CpuCommandBufferEncoding {+ command_queue: self.command_queue,commands: Vec::new(),completion_handlers: Vec::new(),- gpu_execution_time: OnceCell::new(),}}+}++pub struct CpuCommandBufferEncoding {+ command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>,+ commands: Vec<Box<dyn FnOnce() + Send>>,+ completion_handlers: Vec<Box<dyn FnOnce(Result<&CpuCommandBufferCompleted, CpuError>) + Send + 'static>>,+}+impl CpuCommandBufferEncoding {pub fn push_command(&mut self,- command: impl Fn() + 'static,+ command: impl FnOnce() + Send + 'static,) {self.commands.push(Box::new(command))}}-impl CommandBufferInitial for CpuCommandBuffer {- type CommandBuffer = CpuCommandBuffer;-- fn start_encoding(self) -> CpuCommandBuffer {- self- }-}--impl CommandBufferEncoding for CpuCommandBuffer {+impl CommandBufferEncoding for CpuCommandBufferEncoding {type CommandBuffer = CpuCommandBuffer;fn encode_copy(&mut self,- src: &UnsafeCell<Box<[u8]>>,+ src: &UnsafeCell<Pin<Box<[u8]>>>,src_range: std::ops::Range<usize>,- dst: &mut UnsafeCell<Box<[u8]>>,+ dst: &mut UnsafeCell<Pin<Box<[u8]>>>,dst_range: std::ops::Range<usize>,) {let size = src_range.end - src_range.start;@@ -68,24 +86,24 @@ impl CommandBufferEncoding for CpuCommandBuffer {assert!(unsafe { &*src.get() }.len() >= src_range.end);assert!(dst.get_mut().len() >= dst_range.end);- let src_ptr = unsafe { (&*src.get()).as_ptr().add(src_range.start) };- let dst_ptr = unsafe { dst.get_mut().as_mut_ptr().add(dst_range.start) };- self.push_command(Box::new(move || unsafe {- std::ptr::copy(src_ptr, dst_ptr, size);- }))+ let src_ptr = SendPtr(unsafe { (&*src.get()).as_ptr().add(src_range.start) });+ let dst_ptr = SendPtrMut(unsafe { dst.get_mut().as_mut_ptr().add(dst_range.start) });+ self.push_command(move || unsafe {+ std::ptr::copy(src_ptr.as_ptr(), dst_ptr.as_ptr(), size);+ });}fn encode_fill(&mut self,- dst: &mut UnsafeCell<Box<[u8]>>,+ dst: &mut UnsafeCell<Pin<Box<[u8]>>>,range: std::ops::Range<usize>,value: u8,) {let size = range.end - range.start;- let dst = dst.get_mut()[range].as_ptr() as *mut u8;- self.push_command(Box::new(move || unsafe {- dst.write_bytes(value, size);- }))+ let dst = SendPtrMut(dst.get_mut()[range].as_ptr() as *mut u8);+ self.push_command(move || unsafe {+ dst.as_ptr().write_bytes(value, size);+ });}fn encode_barrier(@@ -97,69 +115,104 @@ impl CommandBufferEncoding for CpuCommandBuffer {fn encode_wait_for_event(&mut self,- event: &RefCell<u64>,+ event: &Pin<Box<AtomicU64>>,value: u64,) {- let event = event.as_ptr();- self.push_command(Box::new(move || unsafe { assert!(*event >= value, "deadlock") }))+ let event = SendPtr(event.as_ref().get_ref() as *const AtomicU64);+ self.push_command(move || unsafe {+ while (&*event.as_ptr()).load(Ordering::Acquire) < value {+ thread::yield_now();+ }+ })}fn encode_signal_event(&mut self,- event: &RefCell<u64>,+ event: &Pin<Box<AtomicU64>>,value: u64,) {- let event = event.as_ptr();- self.push_command(Box::new(move || unsafe { *event = value }))+ let event = SendPtr(event.as_ref().get_ref() as *const AtomicU64);+ self.push_command(Box::new(move || unsafe {+ (&*event.as_ptr()).store(value, Ordering::Release);+ }))}fn add_completion_handler(&mut self,- handler: impl FnOnce(Result<&CpuCommandBuffer, CpuError>) + 'static,+ handler: impl FnOnce(Result<&CpuCommandBufferCompleted, CpuError>) + Send + 'static,) {self.completion_handlers.push(Box::new(handler))}- fn end_encoding(self) -> CpuCommandBuffer {- self+ fn end_encoding(self) -> CpuCommandBufferExecutable {+ CpuCommandBufferExecutable {+ command_queue: self.command_queue,+ commands: self.commands,+ completion_handlers: self.completion_handlers,+ }}}-impl CommandBufferExecutable for CpuCommandBuffer {+pub struct CpuCommandBufferExecutable {+ command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>,+ commands: Vec<Box<dyn FnOnce() + Send>>,+ completion_handlers: Vec<Box<dyn FnOnce(Result<&CpuCommandBufferCompleted, CpuError>) + Send + 'static>>,+}++impl CommandBufferExecutable for CpuCommandBufferExecutable {type CommandBuffer = CpuCommandBuffer;- fn submit(mut self) -> CpuCommandBuffer {- let start = Instant::now();- for command in &self.commands {- command()- }+ fn submit(self) -> CpuCommandBufferPending {+ let (return_sender, return_receiver) = mpsc::channel();- self.gpu_execution_time.set(Instant::now() - start).expect("gpu execution time already set");+ self.command_queue+ .send(Box::new(move || {+ let start = Instant::now();- for completion_handler in self.completion_handlers.drain(..).collect::<Vec<_>>() {- completion_handler(Ok(&self));- }+ for command in self.commands {+ command()+ }- CpuCommandBuffer {- commands: Vec::new(),- completion_handlers: Vec::new(),- gpu_execution_time: self.gpu_execution_time,+ let gpu_execution_time = start.elapsed();++ let completed = CpuCommandBufferCompleted {+ gpu_execution_time,+ };++ for handler in self.completion_handlers {+ handler(Ok(&completed))+ }++ let _ = return_sender.send(completed);+ }))+ .unwrap();++ CpuCommandBufferPending {+ return_receiver,}}}-impl CommandBufferPending for CpuCommandBuffer {+pub struct CpuCommandBufferPending {+ return_receiver: mpsc::Receiver<CpuCommandBufferCompleted>,+}++impl CommandBufferPending for CpuCommandBufferPending {type CommandBuffer = CpuCommandBuffer;- fn wait_until_completed(self) -> Result<CpuCommandBuffer, CpuError> {- Ok(self)+ fn wait_until_completed(self) -> Result<CpuCommandBufferCompleted, CpuError> {+ self.return_receiver.recv().map_err(CpuError::CommandBufferExecutionFailed)}}-impl CommandBufferCompleted for CpuCommandBuffer {+pub struct CpuCommandBufferCompleted {+ gpu_execution_time: Duration,+}++impl CommandBufferCompleted for CpuCommandBufferCompleted {type CommandBuffer = CpuCommandBuffer;fn gpu_execution_time(&self) -> Option<Duration> {- self.gpu_execution_time.get().copied()+ Some(self.gpu_execution_time)}}diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rsindex e0d5e314d..8eff9ec0d 100644--- a/crates/uzu/src/backends/cpu/context.rs+++ b/crates/uzu/src/backends/cpu/context.rs@@ -1,22 +1,37 @@use std::{- cell::{RefCell, UnsafeCell},+ cell::UnsafeCell,path::Path,+ pin::Pin,rc::Rc,+ sync::{atomic::AtomicU64, mpsc},+ thread,};-use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};-use crate::backends::common::{Allocation, AllocationPool, AllocationType, Allocator, Context};+use crate::backends::{+ common::{Allocation, AllocationPool, AllocationType, Allocator, Context},+ cpu::{Cpu, command_buffer::CpuCommandBufferInitial, error::CpuError},+};pub struct CpuContext {allocator: Rc<Allocator<Cpu>>,+ command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>,}impl Context for CpuContext {type Backend = Cpu;fn new() -> Result<Rc<Self>, CpuError> {+ let (command_queue_sender, command_queue_receiever) = mpsc::channel::<Box<dyn FnOnce() + Send>>();++ thread::spawn(|| {+ for command_buffer in command_queue_receiever {+ command_buffer();+ }+ });+Ok(Rc::new_cyclic(|weak_self| CpuContext {allocator: Allocator::new(weak_self.clone()),+ command_queue: command_queue_sender,}))}@@ -38,8 +53,8 @@ impl Context for CpuContext {fn create_buffer(&self,size: usize,- ) -> Result<UnsafeCell<Box<[u8]>>, CpuError> {- Ok(UnsafeCell::new(vec![0; size].into_boxed_slice()))+ ) -> Result<UnsafeCell<Pin<Box<[u8]>>>, CpuError> {+ Ok(UnsafeCell::new(Pin::new(vec![0; size].into_boxed_slice())))}fn create_allocation(@@ -57,12 +72,12 @@ impl Context for CpuContext {self.allocator.create_pool(reusable)}- fn create_command_buffer(&self) -> Result<CpuCommandBuffer, CpuError> {- Ok(CpuCommandBuffer::new())+ fn create_command_buffer(&self) -> Result<CpuCommandBufferInitial, CpuError> {+ Ok(CpuCommandBufferInitial::new(self.command_queue.clone()))}- fn create_event(&self) -> Result<RefCell<u64>, CpuError> {- Ok(RefCell::new(0))+ fn create_event(&self) -> Result<Pin<Box<AtomicU64>>, CpuError> {+ Ok(Box::pin(AtomicU64::new(0)))}fn enable_capture() {}diff --git a/crates/uzu/src/backends/cpu/error.rs b/crates/uzu/src/backends/cpu/error.rsindex e221f160a..57e1ba306 100644--- a/crates/uzu/src/backends/cpu/error.rs+++ b/crates/uzu/src/backends/cpu/error.rs@@ -1,7 +1,11 @@+use std::sync::mpsc::RecvError;+use thiserror::Error;#[derive(Debug, Error)]pub enum CpuError {- #[error("not supported")]+ #[error("Not supported")]NotSupported,+ #[error("Command buffer execution failed")]+ CommandBufferExecutionFailed(RecvError),}diff --git a/crates/uzu/src/backends/cpu/event.rs b/crates/uzu/src/backends/cpu/event.rsindex aa3309c3e..346f7af0f 100644--- a/crates/uzu/src/backends/cpu/event.rs+++ b/crates/uzu/src/backends/cpu/event.rs@@ -1,8 +1,7 @@-use std::cell::RefCell;+use std::{pin::Pin, sync::atomic::AtomicU64};-use super::Cpu;-use crate::backends::common::Event;+use crate::backends::{common::Event, cpu::Cpu};-impl Event for RefCell<u64> {+impl Event for Pin<Box<AtomicU64>> {type Backend = Cpu;}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 99213a5b7..33aa51c67 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -9,6 +9,7 @@ use crate::{},cpu::{Cpu, context::CpuContext},},+ utils::pointers::{SendPtr, SendPtrMut},};pub struct MatmulCpuKernel {@@ -46,24 +47,24 @@ impl MatmulKernel for MatmulCpuKernel {let a_offset = arguments.a_offset as usize;let data_type = self.data_type;- let a_ptr = unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset);- let b_ptr = unsafe { &*arguments.b.get() }.as_ptr();+ let a_ptr = SendPtr(unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset));+ let b_ptr = SendPtr(unsafe { &*arguments.b.get() }.as_ptr());let ab_scale = arguments.ab_scale;- let d_ptr = arguments.d.get_mut().as_mut_ptr();+ let d_ptr = SendPtrMut(arguments.d.get_mut().as_mut_ptr());let (is_accumulate, bias_ptr) = match arguments.c {MatmulArgumentC::Accumulate => (true, None),- MatmulArgumentC::Bias(bias) => (false, Some(unsafe { &*bias.get() }.as_ptr())),+ MatmulArgumentC::Bias(bias) => (false, Some(SendPtr(unsafe { &*bias.get() }.as_ptr()))),MatmulArgumentC::None => (false, None),};command_buffer.push_command(move || match data_type {DataType::F32 => {super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f32>(- a_ptr as *const f32,- b_ptr as *const f32,+ a_ptr.as_ptr() as *const f32,+ b_ptr.as_ptr() as *const f32,ab_scale,- d_ptr as *mut f32,+ d_ptr.as_ptr() as *mut f32,m,n,k,@@ -73,15 +74,15 @@ impl MatmulKernel for MatmulCpuKernel {is_accumulate,);if let Some(bias) = bias_ptr {- apply_bias::<f32>(d_ptr as *mut f32, bias as *const f32, m, n);+ apply_bias::<f32>(d_ptr.as_ptr() as *mut f32, bias.as_ptr() as *const f32, m, n);}},DataType::F16 => {super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f16>(- a_ptr as *const f16,- b_ptr as *const f16,+ a_ptr.as_ptr() as *const f16,+ b_ptr.as_ptr() as *const f16,ab_scale,- d_ptr as *mut f16,+ d_ptr.as_ptr() as *mut f16,m,n,k,@@ -91,15 +92,15 @@ impl MatmulKernel for MatmulCpuKernel {is_accumulate,);if let Some(bias) = bias_ptr {- apply_bias::<f16>(d_ptr as *mut f16, bias as *const f16, m, n);+ apply_bias::<f16>(d_ptr.as_ptr() as *mut f16, bias.as_ptr() as *const f16, m, n);}},DataType::BF16 => {super::gemm::shaders::steel_gemm::matmul_gemm_impl::<bf16>(- a_ptr as *const bf16,- b_ptr as *const bf16,+ a_ptr.as_ptr() as *const bf16,+ b_ptr.as_ptr() as *const bf16,ab_scale,- d_ptr as *mut bf16,+ d_ptr.as_ptr() as *mut bf16,m,n,k,@@ -109,7 +110,7 @@ impl MatmulKernel for MatmulCpuKernel {is_accumulate,);if let Some(bias) = bias_ptr {- apply_bias::<bf16>(d_ptr as *mut bf16, bias as *const bf16, m, n);+ apply_bias::<bf16>(d_ptr.as_ptr() as *mut bf16, bias.as_ptr() as *const bf16, m, n);}},_ => unreachable!(),diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex aa3de6135..b2f294885 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -171,7 +171,7 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {fn add_completion_handler(&mut self,- handler: impl FnOnce(Result<&MetalCommandBufferCompleted, MetalError>) + 'static,+ handler: impl FnOnce(Result<&MetalCommandBufferCompleted, MetalError>) + Send + 'static,) {let cell = Cell::new(Some(handler));self.command_buffer.add_completed_handler(&MTLCommandBufferHandler::new(move |command_buffer| {diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 8b10b09ae..c0f01d1e0 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -34,6 +34,7 @@ use crate::{types::Error,},trie::{TrieCreationConfig, TrieNode},+ utils::pointers::SendPtr,};#[derive(Debug, Clone)]@@ -573,11 +574,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.async_buffers.counter.set(next_counter);// Add completion handler- let results_buffer_ptr = results_buffer.borrow().cpu_ptr().as_ptr() as *const u32;+ let results_buffer_ptr = SendPtr(results_buffer.borrow().cpu_ptr().as_ptr() as *const u32);let handler = move |result: Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>| {result.expect("async decoding forward pass completed with error");- let token = { unsafe { *results_buffer_ptr.add(slot) as u64 } };+ let token = { unsafe { *results_buffer_ptr.as_ptr().add(slot) as u64 } };on_complete(token);};diff --git a/crates/uzu/src/utils/pointers.rs b/crates/uzu/src/utils/pointers.rsindex 9f623c59d..7b9b0556d 100644--- a/crates/uzu/src/utils/pointers.rs+++ b/crates/uzu/src/utils/pointers.rs@@ -1,4 +1,9 @@pub struct SendPtr<T>(pub *const T);+impl<T> SendPtr<T> {+ pub fn as_ptr(self) -> *const T {+ self.0+ }+}unsafe impl<T> Send for SendPtr<T> {}unsafe impl<T> Sync for SendPtr<T> {}impl<T> Clone for SendPtr<T> {@@ -9,6 +14,11 @@ impl<T> Clone for SendPtr<T> {impl<T> Copy for SendPtr<T> {}pub struct SendPtrMut<T>(pub *mut T);+impl<T> SendPtrMut<T> {+ pub fn as_ptr(self) -> *mut T {+ self.0+ }+}unsafe impl<T> Send for SendPtrMut<T> {}unsafe impl<T> Sync for SendPtrMut<T> {}impl<T> Clone for SendPtrMut<T> {