Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Array buffer split (#386)

Aleksandr Golokozfd42bdd55ba0
diff --git a/crates/backend-uzu/src/array/array.rs b/crates/backend-uzu/src/array/array.rs
new file mode 100644
index 000000000..8f8cb3c13
--- /dev/null
+++ b/crates/backend-uzu/src/array/array.rs
@@ -0,0 +1,85 @@
+use std::{cell::RefCell, rc::Rc};
+
+use crate::{
+ DataType,
+ array::size_for_shape,
+ backends::common::{Backend, Buffer},
+};
+
+#[derive(Debug)]
+pub struct Array<B: Backend, Buf: Buffer<Backend = B> = <B as Backend>::DenseBuffer> {
+ pub(super) buffer: Rc<RefCell<Buf>>,
+ pub(super) offset: usize,
+ pub(super) shape: Box<[usize]>,
+ pub(super) data_type: DataType,
+}
+
+impl<B: Backend, Buf: Buffer<Backend = B>> Array<B, Buf> {
+ // Constructors
+ pub unsafe fn from_parts(
+ buffer: Rc<RefCell<Buf>>,
+ offset: usize,
+ shape: &[usize],
+ data_type: DataType,
+ ) -> Self {
+ let required_bytes = size_for_shape(shape, data_type);
+ assert!(
+ offset + required_bytes <= buffer.borrow().size(),
+ "Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",
+ shape,
+ data_type,
+ offset,
+ offset + required_bytes,
+ buffer.borrow().size()
+ );
+ Self {
+ buffer: buffer.clone(),
+ offset,
+ shape: shape.into(),
+ data_type,
+ }
+ }
+
+ pub fn view(
+ &self,
+ shape: &[usize],
+ ) -> Self {
+ unsafe { Self::from_parts(self.buffer.clone(), self.offset, shape, self.data_type) }
+ }
+
+ // Getters
+ pub fn buffer(&self) -> Rc<RefCell<Buf>> {
+ self.buffer.clone()
+ }
+
+ pub fn offset(&self) -> usize {
+ self.offset
+ }
+
+ pub fn shape(&self) -> &[usize] {
+ &self.shape
+ }
+
+ pub fn data_type(&self) -> DataType {
+ self.data_type
+ }
+
+ pub fn size(&self) -> usize {
+ size_for_shape(&self.shape, self.data_type)
+ }
+
+ pub fn num_elements(&self) -> usize {
+ self.shape.iter().product()
+ }
+}
+
+impl<B: Backend, Buf: Buffer<Backend = B>> Clone for Array<B, Buf> {
+ fn clone(&self) -> Self {
+ Self {
+ buffer: self.buffer.clone(),
+ offset: self.offset,
+ shape: self.shape.clone(),
+ data_type: self.data_type,
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/array/context_ext.rs b/crates/backend-uzu/src/array/context_ext.rs
new file mode 100644
index 000000000..3b80227f0
--- /dev/null
+++ b/crates/backend-uzu/src/array/context_ext.rs
@@ -0,0 +1,56 @@
+use std::{cell::RefCell, rc::Rc};
+
+use crate::{
+ Array, ArrayElement, DataType,
+ array::size_for_shape,
+ backends::common::{Backend, Buffer, Context},
+};
+
+pub trait ArrayContextExt: Context {
+ fn create_array_uninitialized(
+ &self,
+ shape: &[usize],
+ data_type: DataType,
+ label: &str,
+ ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {
+ let buffer_size_bytes = size_for_shape(shape, data_type);
+
+ let mut buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");
+ buffer.set_label(Some(label));
+
+ unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, shape, data_type) }
+ }
+
+ fn create_array_zeros(
+ &self,
+ shape: &[usize],
+ data_type: DataType,
+ label: &str,
+ ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {
+ let mut array = self.create_array_uninitialized(shape, data_type, label);
+ array.as_bytes_mut().fill(0);
+ array
+ }
+
+ fn create_array_from<T: ArrayElement>(
+ &self,
+ shape: &[usize],
+ data: &[T],
+ label: &str,
+ ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {
+ let size_from_shape: usize = shape.iter().product();
+ assert_eq!(
+ data.len(),
+ size_from_shape,
+ "Shape size {} and data size {} are not equal",
+ size_from_shape,
+ data.len()
+ );
+
+ let mut array = self.create_array_uninitialized(shape, T::data_type(), label);
+ array.as_slice_mut().copy_from_slice(data);
+ array
+ }
+}
+
+impl<C: Context> ArrayContextExt for C {}
diff --git a/crates/backend-uzu/src/array.rs b/crates/backend-uzu/src/array/dense.rs
similarity index 50%
rename from crates/backend-uzu/src/array.rs
rename to crates/backend-uzu/src/array/dense.rs
index 642ded05b..de84b6b5a 100644
--- a/crates/backend-uzu/src/array.rs
+++ b/crates/backend-uzu/src/array/dense.rs
@@ -1,93 +1,18 @@
-use std::{cell::RefCell, ops::Range, os::raw::c_void, ptr::NonNull, rc::Rc};
+use std::{ops::Range, os::raw::c_void, ptr::NonNull};
use ndarray::{ArrayView, Dimension, IxDyn};
use crate::{
- ArrayElement, DataType,
- backends::common::{Backend, Buffer, Context, DenseBuffer},
+ ArrayElement,
+ array::Array,
+ backends::common::{Backend, DenseBuffer},
};
-#[derive(Debug)]
-pub struct Array<B: Backend> {
- buffer: Rc<RefCell<B::DenseBuffer>>,
- offset: usize,
- shape: Box<[usize]>,
- data_type: DataType,
-}
-
-impl<B: Backend> Array<B> {
- // Constructors
- pub unsafe fn from_parts(
- buffer: Rc<RefCell<B::DenseBuffer>>,
- offset: usize,
- shape: &[usize],
- data_type: DataType,
- ) -> Self {
- let required_bytes = size_for_shape(shape, data_type);
- assert!(
- offset + required_bytes <= buffer.borrow().size(),
- "Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",
- shape,
- data_type,
- offset,
- offset + required_bytes,
- buffer.borrow().size()
- );
- Self {
- buffer: buffer.clone(),
- offset,
- shape: shape.into(),
- data_type,
- }
- }
-
- pub fn view(
- &self,
- shape: &[usize],
- ) -> Self {
- unsafe { Self::from_parts(self.buffer.clone(), self.offset, shape, self.data_type) }
- }
-
- // Getters
- pub fn buffer(&self) -> Rc<RefCell<B::DenseBuffer>> {
- self.buffer.clone()
- }
-
- pub fn offset(&self) -> usize {
- self.offset
- }
-
- pub fn shape(&self) -> &[usize] {
- &self.shape
- }
-
- pub fn data_type(&self) -> DataType {
- self.data_type
- }
-
- // Utility
+impl<B: Backend, Buf: DenseBuffer<Backend = B>> Array<B, Buf> {
pub fn cpu_ptr(&self) -> NonNull<c_void> {
unsafe { self.buffer.borrow().cpu_ptr().add(self.offset) }
}
- pub fn size(&self) -> usize {
- size_for_shape(&self.shape, self.data_type)
- }
-
- pub fn num_elements(&self) -> usize {
- self.shape.iter().product()
- }
-
- fn validate_element_type<T: ArrayElement>(&self) {
- assert_eq!(
- T::data_type(),
- self.data_type,
- "Invalid data type, expected {:?}, actual {:?}",
- T::data_type(),
- self.data_type
- );
- }
-
pub fn as_bytes(&self) -> &[u8] {
unsafe { std::slice::from_raw_parts(self.cpu_ptr().as_ptr() as *const u8, self.size()) }
}
@@ -187,93 +112,14 @@ impl<B: Backend> Array<B> {
}
}
}
-}
-
-impl<B: Backend> Clone for Array<B> {
- fn clone(&self) -> Self {
- Self {
- buffer: self.buffer.clone(),
- offset: self.offset,
- shape: self.shape.clone(),
- data_type: self.data_type,
- }
- }
-}
-
-pub fn size_for_shape(
- shape: &[usize],
- data_type: DataType,
-) -> usize {
- let Some(last_dim) = shape.last() else {
- return data_type.size_in_bytes();
- };
-
- let bits_per_row = last_dim * data_type.size_in_bits();
- let padded_bytes_per_row = bits_per_row.div_ceil(8);
- let num_rows: usize = shape.iter().rev().skip(1).product();
-
- num_rows * padded_bytes_per_row
-}
-
-// Array extends Context with helper functions to create arrays from context
-
-pub trait ArrayContextExt {
- type Backend: Backend;
-
- fn create_array_uninitialized(
- &self,
- shape: &[usize],
- data_type: DataType,
- label: &str,
- ) -> Array<Self::Backend>;
-
- fn create_array_zeros(
- &self,
- shape: &[usize],
- data_type: DataType,
- label: &str,
- ) -> Array<Self::Backend> {
- let mut array = self.create_array_uninitialized(shape, data_type, label);
- array.as_bytes_mut().fill(0);
- array
- }
-
- fn create_array_from<T: ArrayElement>(
- &self,
- shape: &[usize],
- data: &[T],
- label: &str,
- ) -> Array<Self::Backend> {
- let size_from_shape: usize = shape.iter().product();
+ fn validate_element_type<T: ArrayElement>(&self) {
assert_eq!(
- data.len(),
- size_from_shape,
- "Shape size {} and data size {} are not equal",
- size_from_shape,
- data.len()
+ T::data_type(),
+ self.data_type,
+ "Invalid data type, expected {:?}, actual {:?}",
+ T::data_type(),
+ self.data_type
);
-
- let mut array = self.create_array_uninitialized(shape, T::data_type(), label);
- array.as_slice_mut().copy_from_slice(data);
- array
- }
-}
-
-impl<C: Context> ArrayContextExt for C {
- type Backend = C::Backend;
-
- fn create_array_uninitialized(
- &self,
- shape: &[usize],
- data_type: DataType,
- label: &str,
- ) -> Array<Self::Backend> {
- let buffer_size_bytes = size_for_shape(shape, data_type);
-
- let mut buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");
- buffer.set_label(Some(label));
-
- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, shape, data_type) }
}
}
diff --git a/crates/backend-uzu/src/array/mod.rs b/crates/backend-uzu/src/array/mod.rs
new file mode 100644
index 000000000..1a1e38ad7
--- /dev/null
+++ b/crates/backend-uzu/src/array/mod.rs
@@ -0,0 +1,8 @@
+mod array;
+mod context_ext;
+mod dense;
+mod util;
+
+pub use array::Array;
+pub use context_ext::ArrayContextExt;
+pub use util::size_for_shape;
diff --git a/crates/backend-uzu/src/array/util.rs b/crates/backend-uzu/src/array/util.rs
new file mode 100644
index 000000000..eaaddb9ac
--- /dev/null
+++ b/crates/backend-uzu/src/array/util.rs
@@ -0,0 +1,17 @@
+use crate::DataType;
+
+pub fn size_for_shape(
+ shape: &[usize],
+ data_type: DataType,
+) -> usize {
+ let Some(last_dim) = shape.last() else {
+ return data_type.size_in_bytes();
+ };
+
+ let bits_per_row = last_dim * data_type.size_in_bits();
+ let padded_bytes_per_row = bits_per_row.div_ceil(8);
+
+ let num_rows: usize = shape.iter().rev().skip(1).product();
+
+ num_rows * padded_bytes_per_row
+}