A high-performance inference engine for AI models
BufferRange (#393)
uuuvnc79d147b7a00
diff --git a/crates/backend-uzu/build/common/traitgen.rs b/crates/backend-uzu/build/common/traitgen.rsindex f3ca11283..7f8fe198f 100644--- a/crates/backend-uzu/build/common/traitgen.rs+++ b/crates/backend-uzu/build/common/traitgen.rs@@ -109,62 +109,47 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>}let kernel_traits = quote! {- pub trait BufferArg<'a, B: crate::backends::common::Buffer> {+ use crate::backends::common::{+ Buffer,+ buffer::{AsBufferRangeMut, AsBufferRangeRef},+ };++ pub trait BufferArg<'a, B: Buffer> {fn into_parts(self) -> (&'a B, usize, usize);}- impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for &'a B {+ impl<'a, B: Buffer, T: AsBufferRangeRef<Buffer = B>> BufferArg<'a, B> for &'a T {fn into_parts(self) -> (&'a B, usize, usize) {- (self, 0, self.size())- }- }-- impl<'a, B: crate::backends::common::Backend> BufferArg<'a, B::DenseBuffer> for &'a crate::backends::common::Allocation<B> {- fn into_parts(self) -> (&'a B::DenseBuffer, usize, usize) {- let (buffer, range) = self.as_buffer_range();+ let buffer_range = self.as_buffer_range_ref();+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());(buffer, range.start, range.end - range.start)}}- impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for (&'a B, usize) {+ impl<'a, B: Buffer, T: AsBufferRangeRef<Buffer = B>> BufferArg<'a, B> for (&'a T, usize) {fn into_parts(self) -> (&'a B, usize, usize) {- (self.0, self.1, self.0.size() - self.1)- }- }-- impl<'a, B: crate::backends::common::Backend> BufferArg<'a, B::DenseBuffer> for (&'a crate::backends::common::Allocation<B>, usize) {- fn into_parts(self) -> (&'a B::DenseBuffer, usize, usize) {- let (buffer, range) = self.0.as_buffer_range();+ let buffer_range = self.0.as_buffer_range_ref();+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());(buffer, range.start + self.1, range.end - range.start - self.1)}}- pub trait BufferArgMut<'a, B: crate::backends::common::Buffer> {+ pub trait BufferArgMut<'a, B: Buffer> {fn into_parts(self) -> (&'a B, usize, usize);}- impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for &'a mut B {+ impl<'a, B: Buffer, T: AsBufferRangeMut<Buffer = B>> BufferArgMut<'a, B> for &'a mut T {fn into_parts(self) -> (&'a B, usize, usize) {- (self, 0, self.size())- }- }-- impl<'a, B: crate::backends::common::Backend> BufferArgMut<'a, B::DenseBuffer> for &'a mut crate::backends::common::Allocation<B> {- fn into_parts(self) -> (&'a B::DenseBuffer, usize, usize) {- let (buffer, range) = self.as_buffer_range();+ let buffer_range = self.as_buffer_range_mut();+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());(buffer, range.start, range.end - range.start)}}- impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for (&'a mut B, usize) {+ impl<'a, B: Buffer, T: AsBufferRangeMut<Buffer = B>> BufferArgMut<'a, B> for (&'a mut T, usize) {fn into_parts(self) -> (&'a B, usize, usize) {- (&*self.0, self.1, self.0.size() - self.1)- }- }-- impl<'a, B: crate::backends::common::Backend> BufferArgMut<'a, B::DenseBuffer> for (&'a mut crate::backends::common::Allocation<B>, usize) {- fn into_parts(self) -> (&'a B::DenseBuffer, usize, usize) {- let (buffer, range) = self.0.as_buffer_range();+ let buffer_range = self.0.as_buffer_range_mut();+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());(buffer, range.start + self.1, range.end - range.start - self.1)}}diff --git a/crates/backend-uzu/src/array/array.rs b/crates/backend-uzu/src/array/array.rsindex 8f8cb3c13..77d169e1f 100644--- a/crates/backend-uzu/src/array/array.rs+++ b/crates/backend-uzu/src/array/array.rs@@ -3,18 +3,18 @@ use std::{cell::RefCell, rc::Rc};use crate::{DataType,array::size_for_shape,- backends::common::{Backend, Buffer},+ backends::common::{AsBufferRangeMut, Backend, Buffer},};#[derive(Debug)]-pub struct Array<B: Backend, Buf: Buffer<Backend = B> = <B as Backend>::DenseBuffer> {+pub struct Array<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>> = <B as Backend>::DenseBuffer> {pub(super) buffer: Rc<RefCell<Buf>>,pub(super) offset: usize,pub(super) shape: Box<[usize]>,pub(super) data_type: DataType,}-impl<B: Backend, Buf: Buffer<Backend = B>> Array<B, Buf> {+impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Array<B, Buf> {// Constructorspub unsafe fn from_parts(buffer: Rc<RefCell<Buf>>,@@ -23,14 +23,17 @@ impl<B: Backend, Buf: Buffer<Backend = B>> Array<B, Buf> {data_type: DataType,) -> Self {let required_bytes = size_for_shape(shape, data_type);+ let buffer_range = buffer.borrow().as_buffer_range_ref().range();+ let buffer_range_size = buffer_range.end - buffer_range.start;+assert!(- offset + required_bytes <= buffer.borrow().size(),+ offset + required_bytes <= buffer_range_size,"Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",shape,data_type,offset,offset + required_bytes,- buffer.borrow().size()+ buffer_range_size);Self {buffer: buffer.clone(),@@ -73,7 +76,7 @@ impl<B: Backend, Buf: Buffer<Backend = B>> Array<B, Buf> {}}-impl<B: Backend, Buf: Buffer<Backend = B>> Clone for Array<B, Buf> {+impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Clone for Array<B, Buf> {fn clone(&self) -> Self {Self {buffer: self.buffer.clone(),diff --git a/crates/backend-uzu/src/array/dense.rs b/crates/backend-uzu/src/array/dense.rsindex de84b6b5a..1deda3906 100644--- a/crates/backend-uzu/src/array/dense.rs+++ b/crates/backend-uzu/src/array/dense.rs@@ -5,12 +5,16 @@ use ndarray::{ArrayView, Dimension, IxDyn};use crate::{ArrayElement,array::Array,- backends::common::{Backend, DenseBuffer},+ backends::common::{AsBufferRangeMut, Backend, DenseBuffer},};-impl<B: Backend, Buf: DenseBuffer<Backend = B>> Array<B, Buf> {+impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<B, Buf> {pub fn cpu_ptr(&self) -> NonNull<c_void> {- unsafe { self.buffer.borrow().cpu_ptr().add(self.offset) }+ let buffer = self.buffer.borrow();+ let buffer_range = buffer.as_buffer_range_ref();+ let range = buffer_range.range();++ unsafe { buffer_range.buffer().cpu_ptr().add(range.start + self.offset) }}pub fn as_bytes(&self) -> &[u8] {@@ -35,9 +39,12 @@ impl<B: Backend, Buf: DenseBuffer<Backend = B>> Array<B, Buf> {ArrayView::from_shape(IxDyn(self.shape()), self.as_slice::<T>()).expect("Failed to create array view")}- pub fn copy_from_array<C: Backend>(+ pub fn copy_from_array<+ OtherBackend: Backend,+ OtherBuf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,+ >(&mut self,- other: &Array<C>,+ other: &Array<OtherBackend, OtherBuf>,) {assert_eq!(self.shape, other.shape);assert_eq!(self.data_type, other.data_type);@@ -45,9 +52,12 @@ impl<B: Backend, Buf: DenseBuffer<Backend = B>> Array<B, Buf> {self.as_bytes_mut().copy_from_slice(other.as_bytes());}- pub fn copy_slice<C: Backend>(+ pub fn copy_slice<+ OtherBackend: Backend,+ OtherBuf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,+ >(&mut self,- source: &Array<C>,+ source: &Array<OtherBackend, OtherBuf>,axis: usize,src_range: Range<usize>,dst_offset: usize,diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rsindex 40fbeb63f..1d50b5374 100644--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs@@ -6,7 +6,10 @@ use std::{};use super::{RangeAllocationType, RangeAllocator};-use crate::backends::common::{Backend, Buffer, Context};+use crate::backends::common::{+ Backend, Buffer, Context,+ buffer::{AsBufferRangeMut, AsBufferRangeRef, BufferRangeMut, BufferRangeRef},+};pub struct Allocation<B: Backend> {allocator: Rc<Allocator<B>>,@@ -14,9 +17,23 @@ pub struct Allocation<B: Backend> {range: Range<usize>,}-impl<B: Backend> Allocation<B> {- pub fn as_buffer_range<'a>(&'a self) -> (&'a B::DenseBuffer, Range<usize>) {- (unsafe { &*self.buffer }, self.range.clone())+impl<B: Backend> AsBufferRangeRef for Allocation<B> {+ type Buffer = B::DenseBuffer;++ fn as_buffer_range_ref<'a>(&'a self) -> BufferRangeRef<'a, B::DenseBuffer> {+ // SAFETY: we keep a strong ref to the allocator that owns the bufs and won't deallocate them while we're alive+ let buffer = unsafe { &*self.buffer };++ BufferRangeRef::new(buffer, self.range.clone())+ }+}++impl<B: Backend> AsBufferRangeMut for Allocation<B> {+ fn as_buffer_range_mut<'a>(&'a mut self) -> BufferRangeMut<'a, B::DenseBuffer> {+ // SAFETY: we keep a strong ref to the allocator that owns the bufs and won't deallocate them while we're alive+ let buffer = unsafe { &*self.buffer };+ // SAFETY: allocator algorithm (hopefully if there is no bugs) guarantees no two overlapping live allocations can exist (which is the contract of BufferRangeMut)+ unsafe { BufferRangeMut::new_shared(buffer, self.range.clone()) }}}diff --git a/crates/backend-uzu/src/backends/common/buffer.rs b/crates/backend-uzu/src/backends/common/buffer.rsindex 84298f668..caa0f32d7 100644--- a/crates/backend-uzu/src/backends/common/buffer.rs+++ b/crates/backend-uzu/src/backends/common/buffer.rs@@ -31,3 +31,91 @@ pub trait BufferGpuAddressRangeExt: Buffer {}impl<B: Buffer> BufferGpuAddressRangeExt for B {}++// NOTE: we treat the buffer as the entire buffer but we also give a "safe" way to get the ref to the whole buffer from a range, this is a fundamental limitation of the current design, but we can nicely fix this after migration to allocations++pub struct BufferRangeRef<'a, B: Buffer> {+ buffer: &'a B,+ range: Range<usize>,+}++impl<'a, B: Buffer> BufferRangeRef<'a, B> {+ pub(super) fn new(+ buffer: &'a B,+ range: Range<usize>,+ ) -> Self {+ Self {+ buffer,+ range,+ }+ }++ pub fn buffer(&self) -> &'a B {+ self.buffer+ }++ pub fn range(&self) -> Range<usize> {+ self.range.clone()+ }+}++pub trait AsBufferRangeRef {+ type Buffer: Buffer;++ fn as_buffer_range_ref<'a>(&'a self) -> BufferRangeRef<'a, Self::Buffer>;+}++pub struct BufferRangeMut<'a, B: Buffer> {+ buffer: &'a B,+ range: Range<usize>,+}++impl<'a, B: Buffer> BufferRangeMut<'a, B> {+ pub(super) fn new_exclusive(+ buffer: &'a mut B,+ range: Range<usize>,+ ) -> Self {+ Self {+ buffer,+ range,+ }+ }++ pub(super) unsafe fn new_shared(+ buffer: &'a B,+ range: Range<usize>,+ ) -> Self {+ Self {+ buffer,+ range,+ }+ }++ pub fn buffer(&self) -> &'a B {+ self.buffer+ }++ pub fn range(&self) -> Range<usize> {+ self.range.clone()+ }+}++pub trait AsBufferRangeMut: AsBufferRangeRef {+ fn as_buffer_range_mut<'a>(&'a mut self) -> BufferRangeMut<'a, Self::Buffer>;+}++impl<B: Buffer> AsBufferRangeRef for B {+ type Buffer = B;++ fn as_buffer_range_ref<'a>(&'a self) -> BufferRangeRef<'a, B> {+ BufferRangeRef::new(self, 0..self.size())+ }+}++impl<B: Buffer> AsBufferRangeMut for B {+ fn as_buffer_range_mut<'a>(&'a mut self) -> BufferRangeMut<'a, B> {+ let size = self.size();++ BufferRangeMut::new_exclusive(self, 0..size)+ }+}diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rsindex 366f1cf21..848846f45 100644--- a/crates/backend-uzu/src/backends/common/mod.rs+++ b/crates/backend-uzu/src/backends/common/mod.rs@@ -14,7 +14,9 @@ pub mod kernel;pub use activation_config::ActivationConfig;pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};pub use backend::Backend;-pub use buffer::{Buffer, BufferGpuAddressRangeExt};+pub use buffer::{+ AsBufferRangeMut, AsBufferRangeRef, Buffer, BufferGpuAddressRangeExt, BufferRangeMut, BufferRangeRef,+};pub use command_buffer::{AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,CommandBufferInitial, CommandBufferPending,