A high-performance inference engine for AI models
Parameters Cleanup (#873)
eugene0d8ce24d2c63
diff --git a/crates/uzu-engine/src/backends/common/allocator/allocator.rs b/crates/uzu-engine/src/backends/common/allocator/allocator.rsindex d38b810f5..8c23c2419 100644--- a/crates/uzu-engine/src/backends/common/allocator/allocator.rs+++ b/crates/uzu-engine/src/backends/common/allocator/allocator.rs@@ -27,11 +27,8 @@ impl<B: Backend> Allocation<B> {}pub fn as_slice_mut<T: NoUninit + AnyBitPattern>(&mut self) -> &mut [T] {- let buffer_range = self.as_buffer_range_mut();- let (buffer, range) = (buffer_range.buffer(), buffer_range.range());- let bytes = unsafe {- std::slice::from_raw_parts_mut((buffer.cpu_ptr().as_ptr() as *mut u8).add(range.start), range.len())- };+ let base = self.buffer.cpu_ptr().cast::<u8>().as_ptr();+ let bytes = unsafe { std::slice::from_raw_parts_mut(base.add(self.range.start), self.range.len()) };bytemuck::cast_slice_mut(bytes)}@@ -43,11 +40,8 @@ impl<B: Backend> Allocation<B> {}pub fn as_slice<T: AnyBitPattern>(&self) -> &[T] {- let buffer_range = self.as_buffer_range_ref();- let (buffer, range) = (buffer_range.buffer(), buffer_range.range());- let bytes = unsafe {- std::slice::from_raw_parts((buffer.cpu_ptr().as_ptr() as *const u8).add(range.start), range.len())- };+ let base = self.buffer.cpu_ptr().cast::<u8>().as_ptr();+ let bytes = unsafe { std::slice::from_raw_parts(base.add(self.range.start), self.range.len()) };bytemuck::cast_slice(bytes)}diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rsindex 69c19a632..cd629b4a9 100644--- a/crates/uzu-engine/src/engine/classifier_model.rs+++ b/crates/uzu-engine/src/engine/classifier_model.rs@@ -21,7 +21,7 @@ use crate::{data_type::DataType,encodable_block::classifier::{Classifier as ClassifierEncodable, ClassifierError as ClassifierEncodableError},engine::Engine,- parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},+ parameters::{ParameterLoader, ParameterLoaderError},};pub struct ClassifierModel<B: Backend> {@@ -39,8 +39,6 @@ pub enum EngineLoadClassifierError<B: Backend> {IO(#[from] io::Error),#[error("Serde error: {0}")]Serde(#[from] serde_json::Error),- #[error("HeaderLoading error: {0}")]- HeaderLoading(#[from] HeaderLoadingError),#[error("ParameterLoader error: {0}")]ParameterLoader(#[from] ParameterLoaderError<B>),#[error("Classifier error: {0}")]diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rsindex ba3486a08..20a78574a 100644--- a/crates/uzu-engine/src/engine/language_model/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/mod.rs@@ -15,7 +15,7 @@ use crate::{sampling::{Sampling, SamplingMethod},},engine::Engine,- parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},+ parameters::{ParameterLoader, ParameterLoaderError},speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},};@@ -46,8 +46,6 @@ pub enum EngineLoadLanguageModelError<B: Backend> {IO(#[from] io::Error),#[error("Serde error: {0}")]Serde(#[from] serde_json::Error),- #[error("HeaderLoading error: {0}")]- HeaderLoading(#[from] HeaderLoadingError),#[error("ParameterLoader error: {0}")]ParameterLoader(#[from] ParameterLoaderError<B>),#[error("Backend error: {0}")]diff --git a/crates/uzu-engine/src/parameters/error.rs b/crates/uzu-engine/src/parameters/error.rsnew file mode 100644index 000000000..03c462a38--- /dev/null+++ b/crates/uzu-engine/src/parameters/error.rs@@ -0,0 +1,37 @@+use thiserror::Error;++use super::safetensors_metadata::HeaderLoadingError;+use crate::{backends::common::Backend, data_type::DataType};++#[derive(Debug, Error)]+pub enum ParameterLoaderError<B: Backend> {+ #[error("Header loading error: {0}")]+ HeaderLoadingError(#[from] HeaderLoadingError),+ #[error("Array with key \"{0}\" not found.")]+ KeyNotFound(String),+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Failed to read data")]+ ArrayLoadingError(#[from] std::io::Error),+ #[error("Failed to deserialize metadata")]+ MetadataDeserializationError(#[from] serde_json::Error),+ #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]+ InvalidTensor {+ shape: Box<[u32]>,+ data_type: DataType,+ expected_shape: Box<[u32]>,+ expected_data_type: DataType,+ },+ #[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]+ InvalidTensorSize {+ shape: Box<[u32]>,+ data_type: DataType,+ size: usize,+ expected_size: usize,+ },+ #[error("Unvalidated tensors under {prefix:?}: {keys:?}")]+ UnvalidatedTensors {+ prefix: Option<String>,+ keys: Box<[String]>,+ },+}diff --git a/crates/uzu-engine/src/parameters/loader.rs b/crates/uzu-engine/src/parameters/loader.rsindex 69190a35a..4de6a3190 100644--- a/crates/uzu-engine/src/parameters/loader.rs+++ b/crates/uzu-engine/src/parameters/loader.rs@@ -8,9 +8,10 @@ use std::{fs::File,};-use thiserror::Error;--use super::safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata};+use super::{+ error::ParameterLoaderError,+ safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata},+};use crate::{array::{ArrayElement, size_for_shape},backends::common::{Allocation, AllocationType, Backend, Context},@@ -18,44 +19,13 @@ use crate::{utils::strict_serde::DeserializeStrictOwned,};-pub struct ParameterMetadata {+struct ParameterMetadata {shape: Box<[u32]>,data_type: DataType,offset: usize,size: usize,}-#[derive(Debug, Error)]-pub enum ParameterLoaderError<B: Backend> {- #[error("Array with key \"{0}\" not found.")]- KeyNotFound(String),- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),- #[error("Failed to read data")]- ArrayLoadingError(#[from] std::io::Error),- #[error("Failed to deserialize metadata")]- MetadataDeserializationError(#[from] serde_json::Error),- #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]- InvalidTensor {- shape: Box<[u32]>,- data_type: DataType,- expected_shape: Box<[u32]>,- expected_data_type: DataType,- },- #[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]- InvalidTensorSize {- shape: Box<[u32]>,- data_type: DataType,- size: usize,- expected_size: usize,- },- #[error("Unvalidated tensors under {prefix:?}: {keys:?}")]- UnvalidatedTensors {- prefix: Option<String>,- keys: Box<[String]>,- },-}-pub struct ParameterLoader<'a, B: Backend> {context: &'a B::Context,index: HashMap<String, ParameterMetadata>,@@ -68,7 +38,7 @@ impl<'a, B: Backend> ParameterLoader<'a, B> {pub fn new(file: &'a File,context: &'a B::Context,- ) -> Result<Self, HeaderLoadingError> {+ ) -> Result<Self, ParameterLoaderError<B>> {let (global_offset, st_metadata) = read_st_metadata(file)?;let index = st_metadata.tensorsdiff --git a/crates/uzu-engine/src/parameters/mod.rs b/crates/uzu-engine/src/parameters/mod.rsindex 420646214..b5d319300 100644--- a/crates/uzu-engine/src/parameters/mod.rs+++ b/crates/uzu-engine/src/parameters/mod.rs@@ -1,7 +1,8 @@// TODO: This is overdue for a complete rewrite+mod error;mod loader;mod safetensors_metadata;-pub use loader::{ParameterLoader, ParameterLoaderError, ParameterTree};-pub use safetensors_metadata::HeaderLoadingError;+pub use error::ParameterLoaderError;+pub use loader::{ParameterLoader, ParameterTree};diff --git a/crates/uzu-engine/src/parameters/safetensors_metadata.rs b/crates/uzu-engine/src/parameters/safetensors_metadata.rsindex fdd3e74bb..bb705e6d4 100644--- a/crates/uzu-engine/src/parameters/safetensors_metadata.rs+++ b/crates/uzu-engine/src/parameters/safetensors_metadata.rs@@ -4,7 +4,7 @@use std::os::unix::fs::FileExt;#[cfg(target_family = "wasm")]use std::os::wasi::fs::FileExt;-use std::{collections::HashMap, fs::File, str::Utf8Error};+use std::{collections::HashMap, fs::File};use serde::{Deserialize, Serialize};use thiserror::Error;@@ -17,14 +17,10 @@ pub enum HeaderLoadingError {UnableToReadHeader(#[source] std::io::Error),#[error("Unable to read safetensors header JSON: {0}")]UnableToReadHeaderJson(#[source] std::io::Error),- #[error("The header is an invalid UTF-8 string and cannot be read: {0}")]- InvalidHeader(#[from] Utf8Error),- #[error("The header does contain a valid string, but it is not valid JSON: {0}")]+ #[error("The header is not valid UTF-8 JSON: {0}")]InvalidHeaderDeserialization(#[from] serde_json::Error),#[error("The header is too large.")]HeaderTooLarge,- #[error("The header length is invalid.")]- InvalidHeaderLength,#[error("Unsupported safetensors dtype: {0:?}")]UnsupportedDtype(Dtype),#[error("Invalid data offsets for tensor {key}: begin={begin}, end={end}")]@@ -60,7 +56,7 @@ pub struct TensorInfo {#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]#[non_exhaustive]pub enum Dtype {- /// Boolan type+ /// Boolean typeBOOL,/// Unsigned byteU8,@@ -119,15 +115,14 @@ pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadinglet metadata_size: usize =u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;if metadata_size > MAX_HEADER_SIZE {- return Err(HeaderLoadingError::InvalidHeaderLength);+ return Err(HeaderLoadingError::HeaderTooLarge);}- let stop = metadata_size.checked_add(8).ok_or(HeaderLoadingError::InvalidHeaderLength)?;- let mut json_buffer: Box<[u8]> = core::iter::repeat_n(0, stop - size_of::<u64>()).collect();- file.read_exact_at(&mut json_buffer, 8).map_err(HeaderLoadingError::UnableToReadHeaderJson)?;- let string = core::str::from_utf8(&json_buffer)?;- let metadata: HashMetadata = serde_json::from_str(string)?;- Ok((stop, metadata))+ let mut json_buffer = vec![0u8; metadata_size];+ file.read_exact_at(&mut json_buffer, size_of::<u64>() as u64)+ .map_err(HeaderLoadingError::UnableToReadHeaderJson)?;+ let metadata: HashMetadata = serde_json::from_slice(&json_buffer)?;+ Ok((size_of::<u64>() + metadata_size, metadata))}#[cfg(test)]diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex ecb16e409..49bf3aa72 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -25,7 +25,7 @@ use crate::{sampling::{PRng, Sampling, SamplingMethod},weaver::{ProposalNode, Weaver, WeaverEncodeError, WeaverNewError, WeaverTreeShape},},- parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},+ parameters::{ParameterLoader, ParameterLoaderError},trie::TrieNode,};@@ -47,8 +47,6 @@ pub enum DFlashSpeculatorLoadError<B: Backend> {IO(#[from] io::Error),#[error("Serde error: {0}")]Serde(#[from] serde_json::Error),- #[error("HeaderLoading error: {0}")]- HeaderLoading(#[from] HeaderLoadingError),#[error("ParameterLoader error: {0}")]ParameterLoader(#[from] ParameterLoaderError<B>),#[error("DFlash error: {0}")]diff --git a/crates/uzu-engine/unit/common/helpers.rs b/crates/uzu-engine/unit/common/helpers.rsindex 2725f6cdc..206784c62 100644--- a/crates/uzu-engine/unit/common/helpers.rs+++ b/crates/uzu-engine/unit/common/helpers.rs@@ -2,10 +2,7 @@ use std::{mem::size_of, sync::Arc};use crate::{array::ArrayElement,- backends::common::{- Allocation, AllocationType, AsBufferRangeMut, Backend, Context, DenseBuffer, Encoder, SparseBuffer,- SparseBufferExt,- },+ backends::common::{Allocation, AllocationType, Backend, Context, Encoder, SparseBuffer, SparseBufferExt},};/// Invokes `$body` once per available backend, with `$B` bound to each backend type.@@ -61,7 +58,7 @@ pub fn alloc_allocation_with_data<B: Backend, T: ArrayElement>(let mut allocation = context.create_allocation(allocation_size_bytes::<T>(data.len()), AllocationType::Global).expect("Failed to create allocation");- write_allocation(&mut allocation, data);+ allocation.copyin(data);allocation}@@ -78,23 +75,6 @@ pub fn allocation_prefix_to_vec<B: Backend, T: ArrayElement>(values}-pub fn write_allocation<B: Backend, T: ArrayElement>(- allocation: &mut Allocation<B>,- data: &[T],-) {- let bytes = bytemuck::cast_slice(data);- let buffer_range = allocation.as_buffer_range_mut();- let range = buffer_range.range();- assert!(bytes.len() <= range.len(), "source data is larger than destination allocation");- let destination = unsafe {- std::slice::from_raw_parts_mut(- (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),- range.len(),- )- };- destination[..bytes.len()].copy_from_slice(bytes);-}-pub fn create_context<B: Backend>() -> Arc<<B as Backend>::Context> {B::Context::new().unwrap_or_else(|_| panic!("Failed to create context for {}", std::any::type_name::<B>()))}