Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Parameters Cleanup (#873)

eugene0d8ce24d2c63
diff --git a/crates/uzu-engine/src/backends/common/allocator/allocator.rs b/crates/uzu-engine/src/backends/common/allocator/allocator.rs
index d38b810f5..8c23c2419 100644
--- a/crates/uzu-engine/src/backends/common/allocator/allocator.rs
+++ b/crates/uzu-engine/src/backends/common/allocator/allocator.rs
@@ -27,11 +27,8 @@ impl<B: Backend> Allocation<B> {
}
pub fn as_slice_mut<T: NoUninit + AnyBitPattern>(&mut self) -> &mut [T] {
- let buffer_range = self.as_buffer_range_mut();
- let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
- let bytes = unsafe {
- std::slice::from_raw_parts_mut((buffer.cpu_ptr().as_ptr() as *mut u8).add(range.start), range.len())
- };
+ let base = self.buffer.cpu_ptr().cast::<u8>().as_ptr();
+ let bytes = unsafe { std::slice::from_raw_parts_mut(base.add(self.range.start), self.range.len()) };
bytemuck::cast_slice_mut(bytes)
}
@@ -43,11 +40,8 @@ impl<B: Backend> Allocation<B> {
}
pub fn as_slice<T: AnyBitPattern>(&self) -> &[T] {
- let buffer_range = self.as_buffer_range_ref();
- let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
- let bytes = unsafe {
- std::slice::from_raw_parts((buffer.cpu_ptr().as_ptr() as *const u8).add(range.start), range.len())
- };
+ let base = self.buffer.cpu_ptr().cast::<u8>().as_ptr();
+ let bytes = unsafe { std::slice::from_raw_parts(base.add(self.range.start), self.range.len()) };
bytemuck::cast_slice(bytes)
}
diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rs
index 69c19a632..cd629b4a9 100644
--- a/crates/uzu-engine/src/engine/classifier_model.rs
+++ b/crates/uzu-engine/src/engine/classifier_model.rs
@@ -21,7 +21,7 @@ use crate::{
data_type::DataType,
encodable_block::classifier::{Classifier as ClassifierEncodable, ClassifierError as ClassifierEncodableError},
engine::Engine,
- parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
+ parameters::{ParameterLoader, ParameterLoaderError},
};
pub struct ClassifierModel<B: Backend> {
@@ -39,8 +39,6 @@ pub enum EngineLoadClassifierError<B: Backend> {
IO(#[from] io::Error),
#[error("Serde error: {0}")]
Serde(#[from] serde_json::Error),
- #[error("HeaderLoading error: {0}")]
- HeaderLoading(#[from] HeaderLoadingError),
#[error("ParameterLoader error: {0}")]
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("Classifier error: {0}")]
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index ba3486a08..20a78574a 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -15,7 +15,7 @@ use crate::{
sampling::{Sampling, SamplingMethod},
},
engine::Engine,
- parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
+ parameters::{ParameterLoader, ParameterLoaderError},
speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},
};
@@ -46,8 +46,6 @@ pub enum EngineLoadLanguageModelError<B: Backend> {
IO(#[from] io::Error),
#[error("Serde error: {0}")]
Serde(#[from] serde_json::Error),
- #[error("HeaderLoading error: {0}")]
- HeaderLoading(#[from] HeaderLoadingError),
#[error("ParameterLoader error: {0}")]
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("Backend error: {0}")]
diff --git a/crates/uzu-engine/src/parameters/error.rs b/crates/uzu-engine/src/parameters/error.rs
new file mode 100644
index 000000000..03c462a38
--- /dev/null
+++ b/crates/uzu-engine/src/parameters/error.rs
@@ -0,0 +1,37 @@
+use thiserror::Error;
+
+use super::safetensors_metadata::HeaderLoadingError;
+use crate::{backends::common::Backend, data_type::DataType};
+
+#[derive(Debug, Error)]
+pub enum ParameterLoaderError<B: Backend> {
+ #[error("Header loading error: {0}")]
+ HeaderLoadingError(#[from] HeaderLoadingError),
+ #[error("Array with key \"{0}\" not found.")]
+ KeyNotFound(String),
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Failed to read data")]
+ ArrayLoadingError(#[from] std::io::Error),
+ #[error("Failed to deserialize metadata")]
+ MetadataDeserializationError(#[from] serde_json::Error),
+ #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]
+ InvalidTensor {
+ shape: Box<[u32]>,
+ data_type: DataType,
+ expected_shape: Box<[u32]>,
+ expected_data_type: DataType,
+ },
+ #[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]
+ InvalidTensorSize {
+ shape: Box<[u32]>,
+ data_type: DataType,
+ size: usize,
+ expected_size: usize,
+ },
+ #[error("Unvalidated tensors under {prefix:?}: {keys:?}")]
+ UnvalidatedTensors {
+ prefix: Option<String>,
+ keys: Box<[String]>,
+ },
+}
diff --git a/crates/uzu-engine/src/parameters/loader.rs b/crates/uzu-engine/src/parameters/loader.rs
index 69190a35a..4de6a3190 100644
--- a/crates/uzu-engine/src/parameters/loader.rs
+++ b/crates/uzu-engine/src/parameters/loader.rs
@@ -8,9 +8,10 @@ use std::{
fs::File,
};
-use thiserror::Error;
-
-use super::safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata};
+use super::{
+ error::ParameterLoaderError,
+ safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata},
+};
use crate::{
array::{ArrayElement, size_for_shape},
backends::common::{Allocation, AllocationType, Backend, Context},
@@ -18,44 +19,13 @@ use crate::{
utils::strict_serde::DeserializeStrictOwned,
};
-pub struct ParameterMetadata {
+struct ParameterMetadata {
shape: Box<[u32]>,
data_type: DataType,
offset: usize,
size: usize,
}
-#[derive(Debug, Error)]
-pub enum ParameterLoaderError<B: Backend> {
- #[error("Array with key \"{0}\" not found.")]
- KeyNotFound(String),
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
- #[error("Failed to read data")]
- ArrayLoadingError(#[from] std::io::Error),
- #[error("Failed to deserialize metadata")]
- MetadataDeserializationError(#[from] serde_json::Error),
- #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]
- InvalidTensor {
- shape: Box<[u32]>,
- data_type: DataType,
- expected_shape: Box<[u32]>,
- expected_data_type: DataType,
- },
- #[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]
- InvalidTensorSize {
- shape: Box<[u32]>,
- data_type: DataType,
- size: usize,
- expected_size: usize,
- },
- #[error("Unvalidated tensors under {prefix:?}: {keys:?}")]
- UnvalidatedTensors {
- prefix: Option<String>,
- keys: Box<[String]>,
- },
-}
-
pub struct ParameterLoader<'a, B: Backend> {
context: &'a B::Context,
index: HashMap<String, ParameterMetadata>,
@@ -68,7 +38,7 @@ impl<'a, B: Backend> ParameterLoader<'a, B> {
pub fn new(
file: &'a File,
context: &'a B::Context,
- ) -> Result<Self, HeaderLoadingError> {
+ ) -> Result<Self, ParameterLoaderError<B>> {
let (global_offset, st_metadata) = read_st_metadata(file)?;
let index = st_metadata
.tensors
diff --git a/crates/uzu-engine/src/parameters/mod.rs b/crates/uzu-engine/src/parameters/mod.rs
index 420646214..b5d319300 100644
--- a/crates/uzu-engine/src/parameters/mod.rs
+++ b/crates/uzu-engine/src/parameters/mod.rs
@@ -1,7 +1,8 @@
// TODO: This is overdue for a complete rewrite
+mod error;
mod loader;
mod safetensors_metadata;
-pub use loader::{ParameterLoader, ParameterLoaderError, ParameterTree};
-pub use safetensors_metadata::HeaderLoadingError;
+pub use error::ParameterLoaderError;
+pub use loader::{ParameterLoader, ParameterTree};
diff --git a/crates/uzu-engine/src/parameters/safetensors_metadata.rs b/crates/uzu-engine/src/parameters/safetensors_metadata.rs
index fdd3e74bb..bb705e6d4 100644
--- a/crates/uzu-engine/src/parameters/safetensors_metadata.rs
+++ b/crates/uzu-engine/src/parameters/safetensors_metadata.rs
@@ -4,7 +4,7 @@
use std::os::unix::fs::FileExt;
#[cfg(target_family = "wasm")]
use std::os::wasi::fs::FileExt;
-use std::{collections::HashMap, fs::File, str::Utf8Error};
+use std::{collections::HashMap, fs::File};
use serde::{Deserialize, Serialize};
use thiserror::Error;
@@ -17,14 +17,10 @@ pub enum HeaderLoadingError {
UnableToReadHeader(#[source] std::io::Error),
#[error("Unable to read safetensors header JSON: {0}")]
UnableToReadHeaderJson(#[source] std::io::Error),
- #[error("The header is an invalid UTF-8 string and cannot be read: {0}")]
- InvalidHeader(#[from] Utf8Error),
- #[error("The header does contain a valid string, but it is not valid JSON: {0}")]
+ #[error("The header is not valid UTF-8 JSON: {0}")]
InvalidHeaderDeserialization(#[from] serde_json::Error),
#[error("The header is too large.")]
HeaderTooLarge,
- #[error("The header length is invalid.")]
- InvalidHeaderLength,
#[error("Unsupported safetensors dtype: {0:?}")]
UnsupportedDtype(Dtype),
#[error("Invalid data offsets for tensor {key}: begin={begin}, end={end}")]
@@ -60,7 +56,7 @@ pub struct TensorInfo {
#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]
#[non_exhaustive]
pub enum Dtype {
- /// Boolan type
+ /// Boolean type
BOOL,
/// Unsigned byte
U8,
@@ -119,15 +115,14 @@ pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoading
let metadata_size: usize =
u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;
if metadata_size > MAX_HEADER_SIZE {
- return Err(HeaderLoadingError::InvalidHeaderLength);
+ return Err(HeaderLoadingError::HeaderTooLarge);
}
- let stop = metadata_size.checked_add(8).ok_or(HeaderLoadingError::InvalidHeaderLength)?;
- let mut json_buffer: Box<[u8]> = core::iter::repeat_n(0, stop - size_of::<u64>()).collect();
- file.read_exact_at(&mut json_buffer, 8).map_err(HeaderLoadingError::UnableToReadHeaderJson)?;
- let string = core::str::from_utf8(&json_buffer)?;
- let metadata: HashMetadata = serde_json::from_str(string)?;
- Ok((stop, metadata))
+ let mut json_buffer = vec![0u8; metadata_size];
+ file.read_exact_at(&mut json_buffer, size_of::<u64>() as u64)
+ .map_err(HeaderLoadingError::UnableToReadHeaderJson)?;
+ let metadata: HashMetadata = serde_json::from_slice(&json_buffer)?;
+ Ok((size_of::<u64>() + metadata_size, metadata))
}
#[cfg(test)]
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index ecb16e409..49bf3aa72 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -25,7 +25,7 @@ use crate::{
sampling::{PRng, Sampling, SamplingMethod},
weaver::{ProposalNode, Weaver, WeaverEncodeError, WeaverNewError, WeaverTreeShape},
},
- parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
+ parameters::{ParameterLoader, ParameterLoaderError},
trie::TrieNode,
};
@@ -47,8 +47,6 @@ pub enum DFlashSpeculatorLoadError<B: Backend> {
IO(#[from] io::Error),
#[error("Serde error: {0}")]
Serde(#[from] serde_json::Error),
- #[error("HeaderLoading error: {0}")]
- HeaderLoading(#[from] HeaderLoadingError),
#[error("ParameterLoader error: {0}")]
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("DFlash error: {0}")]
diff --git a/crates/uzu-engine/unit/common/helpers.rs b/crates/uzu-engine/unit/common/helpers.rs
index 2725f6cdc..206784c62 100644
--- a/crates/uzu-engine/unit/common/helpers.rs
+++ b/crates/uzu-engine/unit/common/helpers.rs
@@ -2,10 +2,7 @@ use std::{mem::size_of, sync::Arc};
use crate::{
array::ArrayElement,
- backends::common::{
- Allocation, AllocationType, AsBufferRangeMut, Backend, Context, DenseBuffer, Encoder, SparseBuffer,
- SparseBufferExt,
- },
+ backends::common::{Allocation, AllocationType, Backend, Context, Encoder, SparseBuffer, SparseBufferExt},
};
/// Invokes `$body` once per available backend, with `$B` bound to each backend type.
@@ -61,7 +58,7 @@ pub fn alloc_allocation_with_data<B: Backend, T: ArrayElement>(
let mut allocation = context
.create_allocation(allocation_size_bytes::<T>(data.len()), AllocationType::Global)
.expect("Failed to create allocation");
- write_allocation(&mut allocation, data);
+ allocation.copyin(data);
allocation
}
@@ -78,23 +75,6 @@ pub fn allocation_prefix_to_vec<B: Backend, T: ArrayElement>(
values
}
-pub fn write_allocation<B: Backend, T: ArrayElement>(
- allocation: &mut Allocation<B>,
- data: &[T],
-) {
- let bytes = bytemuck::cast_slice(data);
- let buffer_range = allocation.as_buffer_range_mut();
- let range = buffer_range.range();
- assert!(bytes.len() <= range.len(), "source data is larger than destination allocation");
- let destination = unsafe {
- std::slice::from_raw_parts_mut(
- (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),
- range.len(),
- )
- };
- destination[..bytes.len()].copy_from_slice(bytes);
-}
-
pub fn create_context<B: Backend>() -> Arc<<B as Backend>::Context> {
B::Context::new().unwrap_or_else(|_| panic!("Failed to create context for {}", std::any::type_name::<B>()))
}