Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

BufferRange (#393)

uuuvnc79d147b7a00
diff --git a/crates/backend-uzu/build/common/traitgen.rs b/crates/backend-uzu/build/common/traitgen.rs
index f3ca11283..7f8fe198f 100644
--- a/crates/backend-uzu/build/common/traitgen.rs
+++ b/crates/backend-uzu/build/common/traitgen.rs
@@ -109,62 +109,47 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>
}
let kernel_traits = quote! {
- pub trait BufferArg<'a, B: crate::backends::common::Buffer> {
+ use crate::backends::common::{
+ Buffer,
+ buffer::{AsBufferRangeMut, AsBufferRangeRef},
+ };
+
+ pub trait BufferArg<'a, B: Buffer> {
fn into_parts(self) -> (&'a B, usize, usize);
}
- impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for &'a B {
+ impl<'a, B: Buffer, T: AsBufferRangeRef<Buffer = B>> BufferArg<'a, B> for &'a T {
fn into_parts(self) -> (&'a B, usize, usize) {
- (self, 0, self.size())
- }
- }
-
- impl<'a, B: crate::backends::common::Backend> BufferArg<'a, B::DenseBuffer> for &'a crate::backends::common::Allocation<B> {
- fn into_parts(self) -> (&'a B::DenseBuffer, usize, usize) {
- let (buffer, range) = self.as_buffer_range();
+ let buffer_range = self.as_buffer_range_ref();
+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
(buffer, range.start, range.end - range.start)
}
}
- impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for (&'a B, usize) {
+ impl<'a, B: Buffer, T: AsBufferRangeRef<Buffer = B>> BufferArg<'a, B> for (&'a T, usize) {
fn into_parts(self) -> (&'a B, usize, usize) {
- (self.0, self.1, self.0.size() - self.1)
- }
- }
-
- impl<'a, B: crate::backends::common::Backend> BufferArg<'a, B::DenseBuffer> for (&'a crate::backends::common::Allocation<B>, usize) {
- fn into_parts(self) -> (&'a B::DenseBuffer, usize, usize) {
- let (buffer, range) = self.0.as_buffer_range();
+ let buffer_range = self.0.as_buffer_range_ref();
+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
(buffer, range.start + self.1, range.end - range.start - self.1)
}
}
- pub trait BufferArgMut<'a, B: crate::backends::common::Buffer> {
+ pub trait BufferArgMut<'a, B: Buffer> {
fn into_parts(self) -> (&'a B, usize, usize);
}
- impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for &'a mut B {
+ impl<'a, B: Buffer, T: AsBufferRangeMut<Buffer = B>> BufferArgMut<'a, B> for &'a mut T {
fn into_parts(self) -> (&'a B, usize, usize) {
- (self, 0, self.size())
- }
- }
-
- impl<'a, B: crate::backends::common::Backend> BufferArgMut<'a, B::DenseBuffer> for &'a mut crate::backends::common::Allocation<B> {
- fn into_parts(self) -> (&'a B::DenseBuffer, usize, usize) {
- let (buffer, range) = self.as_buffer_range();
+ let buffer_range = self.as_buffer_range_mut();
+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
(buffer, range.start, range.end - range.start)
}
}
- impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for (&'a mut B, usize) {
+ impl<'a, B: Buffer, T: AsBufferRangeMut<Buffer = B>> BufferArgMut<'a, B> for (&'a mut T, usize) {
fn into_parts(self) -> (&'a B, usize, usize) {
- (&*self.0, self.1, self.0.size() - self.1)
- }
- }
-
- impl<'a, B: crate::backends::common::Backend> BufferArgMut<'a, B::DenseBuffer> for (&'a mut crate::backends::common::Allocation<B>, usize) {
- fn into_parts(self) -> (&'a B::DenseBuffer, usize, usize) {
- let (buffer, range) = self.0.as_buffer_range();
+ let buffer_range = self.0.as_buffer_range_mut();
+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
(buffer, range.start + self.1, range.end - range.start - self.1)
}
}
diff --git a/crates/backend-uzu/src/array/array.rs b/crates/backend-uzu/src/array/array.rs
index 8f8cb3c13..77d169e1f 100644
--- a/crates/backend-uzu/src/array/array.rs
+++ b/crates/backend-uzu/src/array/array.rs
@@ -3,18 +3,18 @@ use std::{cell::RefCell, rc::Rc};
use crate::{
DataType,
array::size_for_shape,
- backends::common::{Backend, Buffer},
+ backends::common::{AsBufferRangeMut, Backend, Buffer},
};
#[derive(Debug)]
-pub struct Array<B: Backend, Buf: Buffer<Backend = B> = <B as Backend>::DenseBuffer> {
+pub struct Array<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>> = <B as Backend>::DenseBuffer> {
pub(super) buffer: Rc<RefCell<Buf>>,
pub(super) offset: usize,
pub(super) shape: Box<[usize]>,
pub(super) data_type: DataType,
}
-impl<B: Backend, Buf: Buffer<Backend = B>> Array<B, Buf> {
+impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Array<B, Buf> {
// Constructors
pub unsafe fn from_parts(
buffer: Rc<RefCell<Buf>>,
@@ -23,14 +23,17 @@ impl<B: Backend, Buf: Buffer<Backend = B>> Array<B, Buf> {
data_type: DataType,
) -> Self {
let required_bytes = size_for_shape(shape, data_type);
+ let buffer_range = buffer.borrow().as_buffer_range_ref().range();
+ let buffer_range_size = buffer_range.end - buffer_range.start;
+
assert!(
- offset + required_bytes <= buffer.borrow().size(),
+ offset + required_bytes <= buffer_range_size,
"Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",
shape,
data_type,
offset,
offset + required_bytes,
- buffer.borrow().size()
+ buffer_range_size
);
Self {
buffer: buffer.clone(),
@@ -73,7 +76,7 @@ impl<B: Backend, Buf: Buffer<Backend = B>> Array<B, Buf> {
}
}
-impl<B: Backend, Buf: Buffer<Backend = B>> Clone for Array<B, Buf> {
+impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Clone for Array<B, Buf> {
fn clone(&self) -> Self {
Self {
buffer: self.buffer.clone(),
diff --git a/crates/backend-uzu/src/array/dense.rs b/crates/backend-uzu/src/array/dense.rs
index de84b6b5a..1deda3906 100644
--- a/crates/backend-uzu/src/array/dense.rs
+++ b/crates/backend-uzu/src/array/dense.rs
@@ -5,12 +5,16 @@ use ndarray::{ArrayView, Dimension, IxDyn};
use crate::{
ArrayElement,
array::Array,
- backends::common::{Backend, DenseBuffer},
+ backends::common::{AsBufferRangeMut, Backend, DenseBuffer},
};
-impl<B: Backend, Buf: DenseBuffer<Backend = B>> Array<B, Buf> {
+impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<B, Buf> {
pub fn cpu_ptr(&self) -> NonNull<c_void> {
- unsafe { self.buffer.borrow().cpu_ptr().add(self.offset) }
+ let buffer = self.buffer.borrow();
+ let buffer_range = buffer.as_buffer_range_ref();
+ let range = buffer_range.range();
+
+ unsafe { buffer_range.buffer().cpu_ptr().add(range.start + self.offset) }
}
pub fn as_bytes(&self) -> &[u8] {
@@ -35,9 +39,12 @@ impl<B: Backend, Buf: DenseBuffer<Backend = B>> Array<B, Buf> {
ArrayView::from_shape(IxDyn(self.shape()), self.as_slice::<T>()).expect("Failed to create array view")
}
- pub fn copy_from_array<C: Backend>(
+ pub fn copy_from_array<
+ OtherBackend: Backend,
+ OtherBuf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,
+ >(
&mut self,
- other: &Array<C>,
+ other: &Array<OtherBackend, OtherBuf>,
) {
assert_eq!(self.shape, other.shape);
assert_eq!(self.data_type, other.data_type);
@@ -45,9 +52,12 @@ impl<B: Backend, Buf: DenseBuffer<Backend = B>> Array<B, Buf> {
self.as_bytes_mut().copy_from_slice(other.as_bytes());
}
- pub fn copy_slice<C: Backend>(
+ pub fn copy_slice<
+ OtherBackend: Backend,
+ OtherBuf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,
+ >(
&mut self,
- source: &Array<C>,
+ source: &Array<OtherBackend, OtherBuf>,
axis: usize,
src_range: Range<usize>,
dst_offset: usize,
diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
index 40fbeb63f..1d50b5374 100644
--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs
+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
@@ -6,7 +6,10 @@ use std::{
};
use super::{RangeAllocationType, RangeAllocator};
-use crate::backends::common::{Backend, Buffer, Context};
+use crate::backends::common::{
+ Backend, Buffer, Context,
+ buffer::{AsBufferRangeMut, AsBufferRangeRef, BufferRangeMut, BufferRangeRef},
+};
pub struct Allocation<B: Backend> {
allocator: Rc<Allocator<B>>,
@@ -14,9 +17,23 @@ pub struct Allocation<B: Backend> {
range: Range<usize>,
}
-impl<B: Backend> Allocation<B> {
- pub fn as_buffer_range<'a>(&'a self) -> (&'a B::DenseBuffer, Range<usize>) {
- (unsafe { &*self.buffer }, self.range.clone())
+impl<B: Backend> AsBufferRangeRef for Allocation<B> {
+ type Buffer = B::DenseBuffer;
+
+ fn as_buffer_range_ref<'a>(&'a self) -> BufferRangeRef<'a, B::DenseBuffer> {
+ // SAFETY: we keep a strong ref to the allocator that owns the bufs and won't deallocate them while we're alive
+ let buffer = unsafe { &*self.buffer };
+
+ BufferRangeRef::new(buffer, self.range.clone())
+ }
+}
+
+impl<B: Backend> AsBufferRangeMut for Allocation<B> {
+ fn as_buffer_range_mut<'a>(&'a mut self) -> BufferRangeMut<'a, B::DenseBuffer> {
+ // SAFETY: we keep a strong ref to the allocator that owns the bufs and won't deallocate them while we're alive
+ let buffer = unsafe { &*self.buffer };
+ // SAFETY: allocator algorithm (hopefully if there is no bugs) guarantees no two overlapping live allocations can exist (which is the contract of BufferRangeMut)
+ unsafe { BufferRangeMut::new_shared(buffer, self.range.clone()) }
}
}
diff --git a/crates/backend-uzu/src/backends/common/buffer.rs b/crates/backend-uzu/src/backends/common/buffer.rs
index 84298f668..caa0f32d7 100644
--- a/crates/backend-uzu/src/backends/common/buffer.rs
+++ b/crates/backend-uzu/src/backends/common/buffer.rs
@@ -31,3 +31,91 @@ pub trait BufferGpuAddressRangeExt: Buffer {
}
impl<B: Buffer> BufferGpuAddressRangeExt for B {}
+
+// NOTE: we treat the buffer as the entire buffer but we also give a "safe" way to get the ref to the whole buffer from a range, this is a fundamental limitation of the current design, but we can nicely fix this after migration to allocations
+
+pub struct BufferRangeRef<'a, B: Buffer> {
+ buffer: &'a B,
+ range: Range<usize>,
+}
+
+impl<'a, B: Buffer> BufferRangeRef<'a, B> {
+ pub(super) fn new(
+ buffer: &'a B,
+ range: Range<usize>,
+ ) -> Self {
+ Self {
+ buffer,
+ range,
+ }
+ }
+
+ pub fn buffer(&self) -> &'a B {
+ self.buffer
+ }
+
+ pub fn range(&self) -> Range<usize> {
+ self.range.clone()
+ }
+}
+
+pub trait AsBufferRangeRef {
+ type Buffer: Buffer;
+
+ fn as_buffer_range_ref<'a>(&'a self) -> BufferRangeRef<'a, Self::Buffer>;
+}
+
+pub struct BufferRangeMut<'a, B: Buffer> {
+ buffer: &'a B,
+ range: Range<usize>,
+}
+
+impl<'a, B: Buffer> BufferRangeMut<'a, B> {
+ pub(super) fn new_exclusive(
+ buffer: &'a mut B,
+ range: Range<usize>,
+ ) -> Self {
+ Self {
+ buffer,
+ range,
+ }
+ }
+
+ pub(super) unsafe fn new_shared(
+ buffer: &'a B,
+ range: Range<usize>,
+ ) -> Self {
+ Self {
+ buffer,
+ range,
+ }
+ }
+
+ pub fn buffer(&self) -> &'a B {
+ self.buffer
+ }
+
+ pub fn range(&self) -> Range<usize> {
+ self.range.clone()
+ }
+}
+
+pub trait AsBufferRangeMut: AsBufferRangeRef {
+ fn as_buffer_range_mut<'a>(&'a mut self) -> BufferRangeMut<'a, Self::Buffer>;
+}
+
+impl<B: Buffer> AsBufferRangeRef for B {
+ type Buffer = B;
+
+ fn as_buffer_range_ref<'a>(&'a self) -> BufferRangeRef<'a, B> {
+ BufferRangeRef::new(self, 0..self.size())
+ }
+}
+
+impl<B: Buffer> AsBufferRangeMut for B {
+ fn as_buffer_range_mut<'a>(&'a mut self) -> BufferRangeMut<'a, B> {
+ let size = self.size();
+
+ BufferRangeMut::new_exclusive(self, 0..size)
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rs
index 366f1cf21..848846f45 100644
--- a/crates/backend-uzu/src/backends/common/mod.rs
+++ b/crates/backend-uzu/src/backends/common/mod.rs
@@ -14,7 +14,9 @@ pub mod kernel;
pub use activation_config::ActivationConfig;
pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};
pub use backend::Backend;
-pub use buffer::{Buffer, BufferGpuAddressRangeExt};
+pub use buffer::{
+ AsBufferRangeMut, AsBufferRangeRef, Buffer, BufferGpuAddressRangeExt, BufferRangeMut, BufferRangeRef,
+};
pub use command_buffer::{
AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
CommandBufferInitial, CommandBufferPending,