Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

cpu command buffer refactor (#295)

uuuvn4a3cdec73b08
diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rs
index 4fd6b0555..d825f48de 100644
--- a/crates/uzu/build/cpu/compiler.rs
+++ b/crates/uzu/build/cpu/compiler.rs
@@ -340,8 +340,8 @@ impl CpuCompiler {
(
Some(quote! { #buffer_lifetime }),
match access {
- KernelBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, std::cell::UnsafeCell<Box<[u8]>>> },
- KernelBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, std::cell::UnsafeCell<Box<[u8]>>> },
+ KernelBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, std::cell::UnsafeCell<std::pin::Pin<Box<[u8]>>>> },
+ KernelBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, std::cell::UnsafeCell<std::pin::Pin<Box<[u8]>>>> },
},
)
},
@@ -371,9 +371,14 @@ impl CpuCompiler {
let argument_ident = &argument.name;
match &argument.ty {
FunctionArgumentType::Buffer(access) => {
- let buffer_ptr = match access {
- KernelBufferAccess::Read => quote! { (&*__dsl_buffer.get()).as_ptr() },
- KernelBufferAccess::ReadWrite => quote! { (&mut *__dsl_buffer.get()).as_mut_ptr() },
+ let (buffer_ptr, buffer_ptr_wrapper) = match access {
+ KernelBufferAccess::Read => {
+ (quote! { (&*__dsl_buffer.get()).as_ptr() }, quote! { crate::utils::pointers::SendPtr })
+ },
+ KernelBufferAccess::ReadWrite => (
+ quote! { (&mut *__dsl_buffer.get()).as_mut_ptr() },
+ quote! { crate::utils::pointers::SendPtrMut },
+ ),
};
if argument.conditional.is_some() {
@@ -381,7 +386,7 @@ impl CpuCompiler {
let #argument_ident = #argument_ident.map(|__dsl_buffer_impl| unsafe {
let (__dsl_buffer, __dsl_offset) = __dsl_buffer_impl.into_parts();
- #buffer_ptr.byte_add(__dsl_offset)
+ #buffer_ptr_wrapper(#buffer_ptr.byte_add(__dsl_offset))
});
})
} else {
@@ -389,7 +394,7 @@ impl CpuCompiler {
let #argument_ident = unsafe {
let (__dsl_buffer, __dsl_offset) = #argument_ident.into_parts();
- #buffer_ptr.byte_add(__dsl_offset)
+ #buffer_ptr_wrapper(#buffer_ptr.byte_add(__dsl_offset))
};
})
}
@@ -418,9 +423,9 @@ impl CpuCompiler {
match &argument.ty {
FunctionArgumentType::Buffer(_) => {
if argument.conditional.is_some() {
- quote! { #argument_ident.map(|p| p as _) }
+ quote! { #argument_ident.map(|p| p.as_ptr() as _) }
} else {
- quote! { #argument_ident as _ }
+ quote! { #argument_ident.as_ptr() as _ }
}
},
FunctionArgumentType::Constant(_) => quote! { #argument_ident.as_ref() },
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index 7abf67225..20a7b2331 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -110,7 +110,8 @@ pub trait CommandBufferEncoding {
&<Self::CommandBuffer as CommandBuffer>::Completed,
<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
>,
- ) + 'static,
+ ) + Send
+ + 'static,
);
fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;
diff --git a/crates/uzu/src/backends/common/encoder.rs b/crates/uzu/src/backends/common/encoder.rs
index 7717599d9..3a0e74009 100644
--- a/crates/uzu/src/backends/common/encoder.rs
+++ b/crates/uzu/src/backends/common/encoder.rs
@@ -118,7 +118,7 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
pub fn add_completion_handler(
&mut self,
- handler: impl FnOnce(Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>) + 'static,
+ handler: impl FnOnce(Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>) + Send + 'static,
) {
self.command_buffer.add_completion_handler(handler);
}
diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rs
index 9d1e5e665..bd1bfe19d 100644
--- a/crates/uzu/src/backends/cpu/backend.rs
+++ b/crates/uzu/src/backends/cpu/backend.rs
@@ -1,4 +1,4 @@
-use std::cell::{RefCell, UnsafeCell};
+use std::{cell::UnsafeCell, pin::Pin, sync::atomic::AtomicU64};
use super::{command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels};
use crate::backends::common::Backend;
@@ -9,8 +9,8 @@ pub struct Cpu;
impl Backend for Cpu {
type Context = CpuContext;
type CommandBuffer = CpuCommandBuffer;
- type Buffer = UnsafeCell<Box<[u8]>>;
- type Event = RefCell<u64>;
+ type Buffer = UnsafeCell<Pin<Box<[u8]>>>;
+ type Event = Pin<Box<AtomicU64>>;
type Kernels = CpuKernels;
type Error = CpuError;
diff --git a/crates/uzu/src/backends/cpu/buffer.rs b/crates/uzu/src/backends/cpu/buffer.rs
index b147cf202..72476cae5 100644
--- a/crates/uzu/src/backends/cpu/buffer.rs
+++ b/crates/uzu/src/backends/cpu/buffer.rs
@@ -1,9 +1,9 @@
-use std::{cell::UnsafeCell, os::raw::c_void, ptr::NonNull};
+use std::{cell::UnsafeCell, os::raw::c_void, pin::Pin, ptr::NonNull};
use super::Cpu;
use crate::backends::common::Buffer;
-impl Buffer for UnsafeCell<Box<[u8]>> {
+impl Buffer for UnsafeCell<Pin<Box<[u8]>>> {
type Backend = Cpu;
fn set_label(
diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rs
index 086877080..4fed33635 100644
--- a/crates/uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu/src/backends/cpu/command_buffer.rs
@@ -1,66 +1,84 @@
use std::{
- cell::{OnceCell, RefCell, UnsafeCell},
+ cell::UnsafeCell,
+ pin::Pin,
+ sync::{
+ atomic::{AtomicU64, Ordering},
+ mpsc,
+ },
+ thread,
time::{Duration, Instant},
};
-use super::Cpu;
-use crate::backends::{
- common::{
- AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
- CommandBufferInitial, CommandBufferPending,
+use crate::{
+ backends::{
+ common::{
+ AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
+ CommandBufferInitial, CommandBufferPending,
+ },
+ cpu::{Cpu, error::CpuError},
},
- cpu::error::CpuError,
+ utils::pointers::{SendPtr, SendPtrMut},
};
-pub struct CpuCommandBuffer {
- commands: Vec<Box<dyn Fn()>>,
- completion_handlers: Vec<Box<dyn FnOnce(Result<&CpuCommandBuffer, CpuError>)>>,
- gpu_execution_time: OnceCell<Duration>,
-}
+pub struct CpuCommandBuffer;
impl CommandBuffer for CpuCommandBuffer {
type Backend = Cpu;
- type Initial = CpuCommandBuffer;
- type Encoding = CpuCommandBuffer;
- type Executable = CpuCommandBuffer;
- type Pending = CpuCommandBuffer;
- type Completed = CpuCommandBuffer;
+ type Initial = CpuCommandBufferInitial;
+ type Encoding = CpuCommandBufferEncoding;
+ type Executable = CpuCommandBufferExecutable;
+ type Pending = CpuCommandBufferPending;
+ type Completed = CpuCommandBufferCompleted;
+}
+
+pub struct CpuCommandBufferInitial {
+ command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>,
+}
+
+impl CpuCommandBufferInitial {
+ pub fn new(command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>) -> CpuCommandBufferInitial {
+ CpuCommandBufferInitial {
+ command_queue,
+ }
+ }
}
-impl CpuCommandBuffer {
- pub fn new() -> CpuCommandBuffer {
- CpuCommandBuffer {
+impl CommandBufferInitial for CpuCommandBufferInitial {
+ type CommandBuffer = CpuCommandBuffer;
+
+ fn start_encoding(self) -> CpuCommandBufferEncoding {
+ CpuCommandBufferEncoding {
+ command_queue: self.command_queue,
commands: Vec::new(),
completion_handlers: Vec::new(),
- gpu_execution_time: OnceCell::new(),
}
}
+}
+
+pub struct CpuCommandBufferEncoding {
+ command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>,
+ commands: Vec<Box<dyn FnOnce() + Send>>,
+ completion_handlers: Vec<Box<dyn FnOnce(Result<&CpuCommandBufferCompleted, CpuError>) + Send + 'static>>,
+}
+impl CpuCommandBufferEncoding {
pub fn push_command(
&mut self,
- command: impl Fn() + 'static,
+ command: impl FnOnce() + Send + 'static,
) {
self.commands.push(Box::new(command))
}
}
-impl CommandBufferInitial for CpuCommandBuffer {
- type CommandBuffer = CpuCommandBuffer;
-
- fn start_encoding(self) -> CpuCommandBuffer {
- self
- }
-}
-
-impl CommandBufferEncoding for CpuCommandBuffer {
+impl CommandBufferEncoding for CpuCommandBufferEncoding {
type CommandBuffer = CpuCommandBuffer;
fn encode_copy(
&mut self,
- src: &UnsafeCell<Box<[u8]>>,
+ src: &UnsafeCell<Pin<Box<[u8]>>>,
src_range: std::ops::Range<usize>,
- dst: &mut UnsafeCell<Box<[u8]>>,
+ dst: &mut UnsafeCell<Pin<Box<[u8]>>>,
dst_range: std::ops::Range<usize>,
) {
let size = src_range.end - src_range.start;
@@ -68,24 +86,24 @@ impl CommandBufferEncoding for CpuCommandBuffer {
assert!(unsafe { &*src.get() }.len() >= src_range.end);
assert!(dst.get_mut().len() >= dst_range.end);
- let src_ptr = unsafe { (&*src.get()).as_ptr().add(src_range.start) };
- let dst_ptr = unsafe { dst.get_mut().as_mut_ptr().add(dst_range.start) };
- self.push_command(Box::new(move || unsafe {
- std::ptr::copy(src_ptr, dst_ptr, size);
- }))
+ let src_ptr = SendPtr(unsafe { (&*src.get()).as_ptr().add(src_range.start) });
+ let dst_ptr = SendPtrMut(unsafe { dst.get_mut().as_mut_ptr().add(dst_range.start) });
+ self.push_command(move || unsafe {
+ std::ptr::copy(src_ptr.as_ptr(), dst_ptr.as_ptr(), size);
+ });
}
fn encode_fill(
&mut self,
- dst: &mut UnsafeCell<Box<[u8]>>,
+ dst: &mut UnsafeCell<Pin<Box<[u8]>>>,
range: std::ops::Range<usize>,
value: u8,
) {
let size = range.end - range.start;
- let dst = dst.get_mut()[range].as_ptr() as *mut u8;
- self.push_command(Box::new(move || unsafe {
- dst.write_bytes(value, size);
- }))
+ let dst = SendPtrMut(dst.get_mut()[range].as_ptr() as *mut u8);
+ self.push_command(move || unsafe {
+ dst.as_ptr().write_bytes(value, size);
+ });
}
fn encode_barrier(
@@ -97,69 +115,104 @@ impl CommandBufferEncoding for CpuCommandBuffer {
fn encode_wait_for_event(
&mut self,
- event: &RefCell<u64>,
+ event: &Pin<Box<AtomicU64>>,
value: u64,
) {
- let event = event.as_ptr();
- self.push_command(Box::new(move || unsafe { assert!(*event >= value, "deadlock") }))
+ let event = SendPtr(event.as_ref().get_ref() as *const AtomicU64);
+ self.push_command(move || unsafe {
+ while (&*event.as_ptr()).load(Ordering::Acquire) < value {
+ thread::yield_now();
+ }
+ })
}
fn encode_signal_event(
&mut self,
- event: &RefCell<u64>,
+ event: &Pin<Box<AtomicU64>>,
value: u64,
) {
- let event = event.as_ptr();
- self.push_command(Box::new(move || unsafe { *event = value }))
+ let event = SendPtr(event.as_ref().get_ref() as *const AtomicU64);
+ self.push_command(Box::new(move || unsafe {
+ (&*event.as_ptr()).store(value, Ordering::Release);
+ }))
}
fn add_completion_handler(
&mut self,
- handler: impl FnOnce(Result<&CpuCommandBuffer, CpuError>) + 'static,
+ handler: impl FnOnce(Result<&CpuCommandBufferCompleted, CpuError>) + Send + 'static,
) {
self.completion_handlers.push(Box::new(handler))
}
- fn end_encoding(self) -> CpuCommandBuffer {
- self
+ fn end_encoding(self) -> CpuCommandBufferExecutable {
+ CpuCommandBufferExecutable {
+ command_queue: self.command_queue,
+ commands: self.commands,
+ completion_handlers: self.completion_handlers,
+ }
}
}
-impl CommandBufferExecutable for CpuCommandBuffer {
+pub struct CpuCommandBufferExecutable {
+ command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>,
+ commands: Vec<Box<dyn FnOnce() + Send>>,
+ completion_handlers: Vec<Box<dyn FnOnce(Result<&CpuCommandBufferCompleted, CpuError>) + Send + 'static>>,
+}
+
+impl CommandBufferExecutable for CpuCommandBufferExecutable {
type CommandBuffer = CpuCommandBuffer;
- fn submit(mut self) -> CpuCommandBuffer {
- let start = Instant::now();
- for command in &self.commands {
- command()
- }
+ fn submit(self) -> CpuCommandBufferPending {
+ let (return_sender, return_receiver) = mpsc::channel();
- self.gpu_execution_time.set(Instant::now() - start).expect("gpu execution time already set");
+ self.command_queue
+ .send(Box::new(move || {
+ let start = Instant::now();
- for completion_handler in self.completion_handlers.drain(..).collect::<Vec<_>>() {
- completion_handler(Ok(&self));
- }
+ for command in self.commands {
+ command()
+ }
- CpuCommandBuffer {
- commands: Vec::new(),
- completion_handlers: Vec::new(),
- gpu_execution_time: self.gpu_execution_time,
+ let gpu_execution_time = start.elapsed();
+
+ let completed = CpuCommandBufferCompleted {
+ gpu_execution_time,
+ };
+
+ for handler in self.completion_handlers {
+ handler(Ok(&completed))
+ }
+
+ let _ = return_sender.send(completed);
+ }))
+ .unwrap();
+
+ CpuCommandBufferPending {
+ return_receiver,
}
}
}
-impl CommandBufferPending for CpuCommandBuffer {
+pub struct CpuCommandBufferPending {
+ return_receiver: mpsc::Receiver<CpuCommandBufferCompleted>,
+}
+
+impl CommandBufferPending for CpuCommandBufferPending {
type CommandBuffer = CpuCommandBuffer;
- fn wait_until_completed(self) -> Result<CpuCommandBuffer, CpuError> {
- Ok(self)
+ fn wait_until_completed(self) -> Result<CpuCommandBufferCompleted, CpuError> {
+ self.return_receiver.recv().map_err(CpuError::CommandBufferExecutionFailed)
}
}
-impl CommandBufferCompleted for CpuCommandBuffer {
+pub struct CpuCommandBufferCompleted {
+ gpu_execution_time: Duration,
+}
+
+impl CommandBufferCompleted for CpuCommandBufferCompleted {
type CommandBuffer = CpuCommandBuffer;
fn gpu_execution_time(&self) -> Option<Duration> {
- self.gpu_execution_time.get().copied()
+ Some(self.gpu_execution_time)
}
}
diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rs
index e0d5e314d..8eff9ec0d 100644
--- a/crates/uzu/src/backends/cpu/context.rs
+++ b/crates/uzu/src/backends/cpu/context.rs
@@ -1,22 +1,37 @@
use std::{
- cell::{RefCell, UnsafeCell},
+ cell::UnsafeCell,
path::Path,
+ pin::Pin,
rc::Rc,
+ sync::{atomic::AtomicU64, mpsc},
+ thread,
};
-use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};
-use crate::backends::common::{Allocation, AllocationPool, AllocationType, Allocator, Context};
+use crate::backends::{
+ common::{Allocation, AllocationPool, AllocationType, Allocator, Context},
+ cpu::{Cpu, command_buffer::CpuCommandBufferInitial, error::CpuError},
+};
pub struct CpuContext {
allocator: Rc<Allocator<Cpu>>,
+ command_queue: mpsc::Sender<Box<dyn FnOnce() + Send>>,
}
impl Context for CpuContext {
type Backend = Cpu;
fn new() -> Result<Rc<Self>, CpuError> {
+ let (command_queue_sender, command_queue_receiever) = mpsc::channel::<Box<dyn FnOnce() + Send>>();
+
+ thread::spawn(|| {
+ for command_buffer in command_queue_receiever {
+ command_buffer();
+ }
+ });
+
Ok(Rc::new_cyclic(|weak_self| CpuContext {
allocator: Allocator::new(weak_self.clone()),
+ command_queue: command_queue_sender,
}))
}
@@ -38,8 +53,8 @@ impl Context for CpuContext {
fn create_buffer(
&self,
size: usize,
- ) -> Result<UnsafeCell<Box<[u8]>>, CpuError> {
- Ok(UnsafeCell::new(vec![0; size].into_boxed_slice()))
+ ) -> Result<UnsafeCell<Pin<Box<[u8]>>>, CpuError> {
+ Ok(UnsafeCell::new(Pin::new(vec![0; size].into_boxed_slice())))
}
fn create_allocation(
@@ -57,12 +72,12 @@ impl Context for CpuContext {
self.allocator.create_pool(reusable)
}
- fn create_command_buffer(&self) -> Result<CpuCommandBuffer, CpuError> {
- Ok(CpuCommandBuffer::new())
+ fn create_command_buffer(&self) -> Result<CpuCommandBufferInitial, CpuError> {
+ Ok(CpuCommandBufferInitial::new(self.command_queue.clone()))
}
- fn create_event(&self) -> Result<RefCell<u64>, CpuError> {
- Ok(RefCell::new(0))
+ fn create_event(&self) -> Result<Pin<Box<AtomicU64>>, CpuError> {
+ Ok(Box::pin(AtomicU64::new(0)))
}
fn enable_capture() {}
diff --git a/crates/uzu/src/backends/cpu/error.rs b/crates/uzu/src/backends/cpu/error.rs
index e221f160a..57e1ba306 100644
--- a/crates/uzu/src/backends/cpu/error.rs
+++ b/crates/uzu/src/backends/cpu/error.rs
@@ -1,7 +1,11 @@
+use std::sync::mpsc::RecvError;
+
use thiserror::Error;
#[derive(Debug, Error)]
pub enum CpuError {
- #[error("not supported")]
+ #[error("Not supported")]
NotSupported,
+ #[error("Command buffer execution failed")]
+ CommandBufferExecutionFailed(RecvError),
}
diff --git a/crates/uzu/src/backends/cpu/event.rs b/crates/uzu/src/backends/cpu/event.rs
index aa3309c3e..346f7af0f 100644
--- a/crates/uzu/src/backends/cpu/event.rs
+++ b/crates/uzu/src/backends/cpu/event.rs
@@ -1,8 +1,7 @@
-use std::cell::RefCell;
+use std::{pin::Pin, sync::atomic::AtomicU64};
-use super::Cpu;
-use crate::backends::common::Event;
+use crate::backends::{common::Event, cpu::Cpu};
-impl Event for RefCell<u64> {
+impl Event for Pin<Box<AtomicU64>> {
type Backend = Cpu;
}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 99213a5b7..33aa51c67 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -9,6 +9,7 @@ use crate::{
},
cpu::{Cpu, context::CpuContext},
},
+ utils::pointers::{SendPtr, SendPtrMut},
};
pub struct MatmulCpuKernel {
@@ -46,24 +47,24 @@ impl MatmulKernel for MatmulCpuKernel {
let a_offset = arguments.a_offset as usize;
let data_type = self.data_type;
- let a_ptr = unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset);
- let b_ptr = unsafe { &*arguments.b.get() }.as_ptr();
+ let a_ptr = SendPtr(unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset));
+ let b_ptr = SendPtr(unsafe { &*arguments.b.get() }.as_ptr());
let ab_scale = arguments.ab_scale;
- let d_ptr = arguments.d.get_mut().as_mut_ptr();
+ let d_ptr = SendPtrMut(arguments.d.get_mut().as_mut_ptr());
let (is_accumulate, bias_ptr) = match arguments.c {
MatmulArgumentC::Accumulate => (true, None),
- MatmulArgumentC::Bias(bias) => (false, Some(unsafe { &*bias.get() }.as_ptr())),
+ MatmulArgumentC::Bias(bias) => (false, Some(SendPtr(unsafe { &*bias.get() }.as_ptr()))),
MatmulArgumentC::None => (false, None),
};
command_buffer.push_command(move || match data_type {
DataType::F32 => {
super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f32>(
- a_ptr as *const f32,
- b_ptr as *const f32,
+ a_ptr.as_ptr() as *const f32,
+ b_ptr.as_ptr() as *const f32,
ab_scale,
- d_ptr as *mut f32,
+ d_ptr.as_ptr() as *mut f32,
m,
n,
k,
@@ -73,15 +74,15 @@ impl MatmulKernel for MatmulCpuKernel {
is_accumulate,
);
if let Some(bias) = bias_ptr {
- apply_bias::<f32>(d_ptr as *mut f32, bias as *const f32, m, n);
+ apply_bias::<f32>(d_ptr.as_ptr() as *mut f32, bias.as_ptr() as *const f32, m, n);
}
},
DataType::F16 => {
super::gemm::shaders::steel_gemm::matmul_gemm_impl::<f16>(
- a_ptr as *const f16,
- b_ptr as *const f16,
+ a_ptr.as_ptr() as *const f16,
+ b_ptr.as_ptr() as *const f16,
ab_scale,
- d_ptr as *mut f16,
+ d_ptr.as_ptr() as *mut f16,
m,
n,
k,
@@ -91,15 +92,15 @@ impl MatmulKernel for MatmulCpuKernel {
is_accumulate,
);
if let Some(bias) = bias_ptr {
- apply_bias::<f16>(d_ptr as *mut f16, bias as *const f16, m, n);
+ apply_bias::<f16>(d_ptr.as_ptr() as *mut f16, bias.as_ptr() as *const f16, m, n);
}
},
DataType::BF16 => {
super::gemm::shaders::steel_gemm::matmul_gemm_impl::<bf16>(
- a_ptr as *const bf16,
- b_ptr as *const bf16,
+ a_ptr.as_ptr() as *const bf16,
+ b_ptr.as_ptr() as *const bf16,
ab_scale,
- d_ptr as *mut bf16,
+ d_ptr.as_ptr() as *mut bf16,
m,
n,
k,
@@ -109,7 +110,7 @@ impl MatmulKernel for MatmulCpuKernel {
is_accumulate,
);
if let Some(bias) = bias_ptr {
- apply_bias::<bf16>(d_ptr as *mut bf16, bias as *const bf16, m, n);
+ apply_bias::<bf16>(d_ptr.as_ptr() as *mut bf16, bias.as_ptr() as *const bf16, m, n);
}
},
_ => unreachable!(),
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index aa3de6135..b2f294885 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -171,7 +171,7 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
fn add_completion_handler(
&mut self,
- handler: impl FnOnce(Result<&MetalCommandBufferCompleted, MetalError>) + 'static,
+ handler: impl FnOnce(Result<&MetalCommandBufferCompleted, MetalError>) + Send + 'static,
) {
let cell = Cell::new(Some(handler));
self.command_buffer.add_completed_handler(&MTLCommandBufferHandler::new(move |command_buffer| {
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 8b10b09ae..c0f01d1e0 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -34,6 +34,7 @@ use crate::{
types::Error,
},
trie::{TrieCreationConfig, TrieNode},
+ utils::pointers::SendPtr,
};
#[derive(Debug, Clone)]
@@ -573,11 +574,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.async_buffers.counter.set(next_counter);
// Add completion handler
- let results_buffer_ptr = results_buffer.borrow().cpu_ptr().as_ptr() as *const u32;
+ let results_buffer_ptr = SendPtr(results_buffer.borrow().cpu_ptr().as_ptr() as *const u32);
let handler = move |result: Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>| {
result.expect("async decoding forward pass completed with error");
- let token = { unsafe { *results_buffer_ptr.add(slot) as u64 } };
+ let token = { unsafe { *results_buffer_ptr.as_ptr().add(slot) as u64 } };
on_complete(token);
};
diff --git a/crates/uzu/src/utils/pointers.rs b/crates/uzu/src/utils/pointers.rs
index 9f623c59d..7b9b0556d 100644
--- a/crates/uzu/src/utils/pointers.rs
+++ b/crates/uzu/src/utils/pointers.rs
@@ -1,4 +1,9 @@
pub struct SendPtr<T>(pub *const T);
+impl<T> SendPtr<T> {
+ pub fn as_ptr(self) -> *const T {
+ self.0
+ }
+}
unsafe impl<T> Send for SendPtr<T> {}
unsafe impl<T> Sync for SendPtr<T> {}
impl<T> Clone for SendPtr<T> {
@@ -9,6 +14,11 @@ impl<T> Clone for SendPtr<T> {
impl<T> Copy for SendPtr<T> {}
pub struct SendPtrMut<T>(pub *mut T);
+impl<T> SendPtrMut<T> {
+ pub fn as_ptr(self) -> *mut T {
+ self.0
+ }
+}
unsafe impl<T> Send for SendPtrMut<T> {}
unsafe impl<T> Sync for SendPtrMut<T> {}
impl<T> Clone for SendPtrMut<T> {