A high-performance inference engine for AI models
Array buffer split (#386)
Aleksandr Golokozfd42bdd55ba0
diff --git a/crates/backend-uzu/src/array/array.rs b/crates/backend-uzu/src/array/array.rsnew file mode 100644index 000000000..8f8cb3c13--- /dev/null+++ b/crates/backend-uzu/src/array/array.rs@@ -0,0 +1,85 @@+use std::{cell::RefCell, rc::Rc};++use crate::{+ DataType,+ array::size_for_shape,+ backends::common::{Backend, Buffer},+};++#[derive(Debug)]+pub struct Array<B: Backend, Buf: Buffer<Backend = B> = <B as Backend>::DenseBuffer> {+ pub(super) buffer: Rc<RefCell<Buf>>,+ pub(super) offset: usize,+ pub(super) shape: Box<[usize]>,+ pub(super) data_type: DataType,+}++impl<B: Backend, Buf: Buffer<Backend = B>> Array<B, Buf> {+ // Constructors+ pub unsafe fn from_parts(+ buffer: Rc<RefCell<Buf>>,+ offset: usize,+ shape: &[usize],+ data_type: DataType,+ ) -> Self {+ let required_bytes = size_for_shape(shape, data_type);+ assert!(+ offset + required_bytes <= buffer.borrow().size(),+ "Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",+ shape,+ data_type,+ offset,+ offset + required_bytes,+ buffer.borrow().size()+ );+ Self {+ buffer: buffer.clone(),+ offset,+ shape: shape.into(),+ data_type,+ }+ }++ pub fn view(+ &self,+ shape: &[usize],+ ) -> Self {+ unsafe { Self::from_parts(self.buffer.clone(), self.offset, shape, self.data_type) }+ }++ // Getters+ pub fn buffer(&self) -> Rc<RefCell<Buf>> {+ self.buffer.clone()+ }++ pub fn offset(&self) -> usize {+ self.offset+ }++ pub fn shape(&self) -> &[usize] {+ &self.shape+ }++ pub fn data_type(&self) -> DataType {+ self.data_type+ }++ pub fn size(&self) -> usize {+ size_for_shape(&self.shape, self.data_type)+ }++ pub fn num_elements(&self) -> usize {+ self.shape.iter().product()+ }+}++impl<B: Backend, Buf: Buffer<Backend = B>> Clone for Array<B, Buf> {+ fn clone(&self) -> Self {+ Self {+ buffer: self.buffer.clone(),+ offset: self.offset,+ shape: self.shape.clone(),+ data_type: self.data_type,+ }+ }+}diff --git a/crates/backend-uzu/src/array/context_ext.rs b/crates/backend-uzu/src/array/context_ext.rsnew file mode 100644index 000000000..3b80227f0--- /dev/null+++ b/crates/backend-uzu/src/array/context_ext.rs@@ -0,0 +1,56 @@+use std::{cell::RefCell, rc::Rc};++use crate::{+ Array, ArrayElement, DataType,+ array::size_for_shape,+ backends::common::{Backend, Buffer, Context},+};++pub trait ArrayContextExt: Context {+ fn create_array_uninitialized(+ &self,+ shape: &[usize],+ data_type: DataType,+ label: &str,+ ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {+ let buffer_size_bytes = size_for_shape(shape, data_type);++ let mut buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");+ buffer.set_label(Some(label));++ unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, shape, data_type) }+ }++ fn create_array_zeros(+ &self,+ shape: &[usize],+ data_type: DataType,+ label: &str,+ ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {+ let mut array = self.create_array_uninitialized(shape, data_type, label);+ array.as_bytes_mut().fill(0);+ array+ }++ fn create_array_from<T: ArrayElement>(+ &self,+ shape: &[usize],+ data: &[T],+ label: &str,+ ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {+ let size_from_shape: usize = shape.iter().product();+ assert_eq!(+ data.len(),+ size_from_shape,+ "Shape size {} and data size {} are not equal",+ size_from_shape,+ data.len()+ );++ let mut array = self.create_array_uninitialized(shape, T::data_type(), label);+ array.as_slice_mut().copy_from_slice(data);+ array+ }+}++impl<C: Context> ArrayContextExt for C {}diff --git a/crates/backend-uzu/src/array.rs b/crates/backend-uzu/src/array/dense.rssimilarity index 50%rename from crates/backend-uzu/src/array.rsrename to crates/backend-uzu/src/array/dense.rsindex 642ded05b..de84b6b5a 100644--- a/crates/backend-uzu/src/array.rs+++ b/crates/backend-uzu/src/array/dense.rs@@ -1,93 +1,18 @@-use std::{cell::RefCell, ops::Range, os::raw::c_void, ptr::NonNull, rc::Rc};+use std::{ops::Range, os::raw::c_void, ptr::NonNull};use ndarray::{ArrayView, Dimension, IxDyn};use crate::{- ArrayElement, DataType,- backends::common::{Backend, Buffer, Context, DenseBuffer},+ ArrayElement,+ array::Array,+ backends::common::{Backend, DenseBuffer},};-#[derive(Debug)]-pub struct Array<B: Backend> {- buffer: Rc<RefCell<B::DenseBuffer>>,- offset: usize,- shape: Box<[usize]>,- data_type: DataType,-}--impl<B: Backend> Array<B> {- // Constructors- pub unsafe fn from_parts(- buffer: Rc<RefCell<B::DenseBuffer>>,- offset: usize,- shape: &[usize],- data_type: DataType,- ) -> Self {- let required_bytes = size_for_shape(shape, data_type);- assert!(- offset + required_bytes <= buffer.borrow().size(),- "Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",- shape,- data_type,- offset,- offset + required_bytes,- buffer.borrow().size()- );- Self {- buffer: buffer.clone(),- offset,- shape: shape.into(),- data_type,- }- }-- pub fn view(- &self,- shape: &[usize],- ) -> Self {- unsafe { Self::from_parts(self.buffer.clone(), self.offset, shape, self.data_type) }- }-- // Getters- pub fn buffer(&self) -> Rc<RefCell<B::DenseBuffer>> {- self.buffer.clone()- }-- pub fn offset(&self) -> usize {- self.offset- }-- pub fn shape(&self) -> &[usize] {- &self.shape- }-- pub fn data_type(&self) -> DataType {- self.data_type- }-- // Utility+impl<B: Backend, Buf: DenseBuffer<Backend = B>> Array<B, Buf> {pub fn cpu_ptr(&self) -> NonNull<c_void> {unsafe { self.buffer.borrow().cpu_ptr().add(self.offset) }}- pub fn size(&self) -> usize {- size_for_shape(&self.shape, self.data_type)- }-- pub fn num_elements(&self) -> usize {- self.shape.iter().product()- }-- fn validate_element_type<T: ArrayElement>(&self) {- assert_eq!(- T::data_type(),- self.data_type,- "Invalid data type, expected {:?}, actual {:?}",- T::data_type(),- self.data_type- );- }-pub fn as_bytes(&self) -> &[u8] {unsafe { std::slice::from_raw_parts(self.cpu_ptr().as_ptr() as *const u8, self.size()) }}@@ -187,93 +112,14 @@ impl<B: Backend> Array<B> {}}}-}--impl<B: Backend> Clone for Array<B> {- fn clone(&self) -> Self {- Self {- buffer: self.buffer.clone(),- offset: self.offset,- shape: self.shape.clone(),- data_type: self.data_type,- }- }-}--pub fn size_for_shape(- shape: &[usize],- data_type: DataType,-) -> usize {- let Some(last_dim) = shape.last() else {- return data_type.size_in_bytes();- };-- let bits_per_row = last_dim * data_type.size_in_bits();- let padded_bytes_per_row = bits_per_row.div_ceil(8);- let num_rows: usize = shape.iter().rev().skip(1).product();-- num_rows * padded_bytes_per_row-}--// Array extends Context with helper functions to create arrays from context--pub trait ArrayContextExt {- type Backend: Backend;-- fn create_array_uninitialized(- &self,- shape: &[usize],- data_type: DataType,- label: &str,- ) -> Array<Self::Backend>;-- fn create_array_zeros(- &self,- shape: &[usize],- data_type: DataType,- label: &str,- ) -> Array<Self::Backend> {- let mut array = self.create_array_uninitialized(shape, data_type, label);- array.as_bytes_mut().fill(0);- array- }-- fn create_array_from<T: ArrayElement>(- &self,- shape: &[usize],- data: &[T],- label: &str,- ) -> Array<Self::Backend> {- let size_from_shape: usize = shape.iter().product();+ fn validate_element_type<T: ArrayElement>(&self) {assert_eq!(- data.len(),- size_from_shape,- "Shape size {} and data size {} are not equal",- size_from_shape,- data.len()+ T::data_type(),+ self.data_type,+ "Invalid data type, expected {:?}, actual {:?}",+ T::data_type(),+ self.data_type);-- let mut array = self.create_array_uninitialized(shape, T::data_type(), label);- array.as_slice_mut().copy_from_slice(data);- array- }-}--impl<C: Context> ArrayContextExt for C {- type Backend = C::Backend;-- fn create_array_uninitialized(- &self,- shape: &[usize],- data_type: DataType,- label: &str,- ) -> Array<Self::Backend> {- let buffer_size_bytes = size_for_shape(shape, data_type);-- let mut buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");- buffer.set_label(Some(label));-- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, shape, data_type) }}}diff --git a/crates/backend-uzu/src/array/mod.rs b/crates/backend-uzu/src/array/mod.rsnew file mode 100644index 000000000..1a1e38ad7--- /dev/null+++ b/crates/backend-uzu/src/array/mod.rs@@ -0,0 +1,8 @@+mod array;+mod context_ext;+mod dense;+mod util;++pub use array::Array;+pub use context_ext::ArrayContextExt;+pub use util::size_for_shape;diff --git a/crates/backend-uzu/src/array/util.rs b/crates/backend-uzu/src/array/util.rsnew file mode 100644index 000000000..eaaddb9ac--- /dev/null+++ b/crates/backend-uzu/src/array/util.rs@@ -0,0 +1,17 @@+use crate::DataType;++pub fn size_for_shape(+ shape: &[usize],+ data_type: DataType,+) -> usize {+ let Some(last_dim) = shape.last() else {+ return data_type.size_in_bytes();+ };++ let bits_per_row = last_dim * data_type.size_in_bits();+ let padded_bytes_per_row = bits_per_row.div_ceil(8);++ let num_rows: usize = shape.iter().rev().skip(1).product();++ num_rows * padded_bytes_per_row+}