Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Small cleanup part 5 (#956)

uuuvn16443dbad6bc
diff --git a/crates/uzu-engine/src/array/element.rs b/crates/uzu-engine/src/array/element.rs
deleted file mode 100644
index 2d59b779a..000000000
--- a/crates/uzu-engine/src/array/element.rs
+++ /dev/null
@@ -1,41 +0,0 @@
-use bytemuck::Pod;
-use half::{bf16, f16};
-use num_traits::NumCast;
-
-#[cfg(test)]
-use crate::data_type::DataType;
-
-// TODO: remove this
-
-pub trait ArrayElement: NumCast + Pod {
- #[cfg(test)]
- fn data_type() -> DataType;
-}
-
-macro_rules! impl_array_element {
- ($($type:ty => $variant:ident),+ $(,)?) => {
- $(
- impl ArrayElement for $type {
- #[cfg(test)]
- fn data_type() -> DataType {
- DataType::$variant
- }
- }
- )+
- };
-}
-
-impl_array_element! {
- f16 => F16,
- bf16 => BF16,
- f32 => F32,
- f64 => F64,
- i8 => I8,
- u8 => U8,
- i16 => I16,
- u16 => U16,
- i32 => I32,
- u32 => U32,
- i64 => I64,
- u64 => U64,
-}
diff --git a/crates/uzu-engine/src/array/mod.rs b/crates/uzu-engine/src/array/mod.rs
deleted file mode 100644
index 683c7be7a..000000000
--- a/crates/uzu-engine/src/array/mod.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-mod element;
-mod util;
-
-pub use element::ArrayElement;
-pub use util::size_for_shape;
diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rs
index c98712b8e..fcde34d18 100644
--- a/crates/uzu-engine/src/backends/common/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs
@@ -3,9 +3,9 @@ use std::{mem::size_of_val, time::Duration};
use bytemuck::{AnyBitPattern, NoUninit};
use crate::{
- array::size_for_shape,
backends::common::{Backend, BufferMut, BufferRef},
data_type::DataType,
+ utils::size_for_shape::size_for_shape,
};
pub trait CommandBuffer {
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation/activation.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation/activation.rs
index 190adffc9..14cb49406 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/activation/activation.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation/activation.rs
@@ -2,11 +2,11 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(Activation)]
#[variants(T, f16, f32, bf16)]
-pub fn activation<T: ArrayElement + Float>(
+pub fn activation<T: DataTypeRepr + Float>(
#[optional(!in_place)] input: Option<*const T>,
output: *mut T,
n: u32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
index aecabd67a..2a7e1bb54 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
@@ -4,11 +4,11 @@ use uzu_engine_macros::kernel;
use super::{hadamard_transform, min_max_symmetric_divisor, quantize_symmetric_i8};
use crate::{
- array::ArrayElement,
backends::common::{
gpu_types::{activation_transform::ActivationTransformOp, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},
kernel::matmul::Int8CodeLayout,
},
+ data_type::DataTypeRepr,
};
pub fn quantize_transformed_row(
@@ -48,7 +48,7 @@ pub fn quantize_transformed_row(
#[kernel(ActivationTransform)]
#[variants(T, f32, bf16)]
#[variants(BiasT, f32, bf16)]
-pub fn activation_transform<T: ArrayElement + Float, BiasT: ArrayElement + Float>(
+pub fn activation_transform<T: DataTypeRepr + Float, BiasT: DataTypeRepr + Float>(
#[optional(!in_place)] input: Option<*const T>,
#[optional(ops == ActivationTransformOp::InputRht || ops == ActivationTransformOp::OutputRht)] fp_out: Option<
*mut T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs
index be16009fd..cbd24aee9 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs
@@ -2,12 +2,12 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
-fn apply_rope<ElementT: ArrayElement + Float>(
+fn apply_rope<ElementT: DataTypeRepr + Float>(
head: *const ElementT,
- cosines: *const (impl ArrayElement + Float),
- sines: *const (impl ArrayElement + Float),
+ cosines: *const (impl DataTypeRepr + Float),
+ sines: *const (impl DataTypeRepr + Float),
batch_idx: usize,
head_dim_idx: usize,
rope_dim: usize,
@@ -34,7 +34,7 @@ fn apply_rope<ElementT: ArrayElement + Float>(
#[kernel(AttentionPrepare)]
#[variants(ElementT, bf16)]
#[variants(RopeT, f32)]
-pub fn attention_prepare<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(
+pub fn attention_prepare<ElementT: DataTypeRepr + Float, RopeT: DataTypeRepr + Float>(
qkvg: *const ElementT,
queries: *mut ElementT,
#[optional(has_kv)] keys: Option<*mut ElementT>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs
index dea58b22b..73a887dd8 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs
@@ -3,14 +3,14 @@ use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::{
- array::ArrayElement,
backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},
+ data_type::DataTypeRepr,
};
#[kernel(AttentionSinglePass)]
#[variants(T, f32, bf16)]
#[variants(HEAD_DIM, 64, 128, 256, 512)]
-pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(
+pub fn attention_single_pass<T: DataTypeRepr + Float, const HEAD_DIM: u32>(
queries: *const T,
keys: *const T,
values: *const T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs
index 736c515c3..3f0fea1e3 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs
@@ -2,11 +2,11 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(KVCacheUpdate)]
#[variants(T, f32, bf16)]
-pub fn kv_cache_update<T: ArrayElement + Float>(
+pub fn kv_cache_update<T: DataTypeRepr + Float>(
in_place_keys: *mut T,
in_place_values: *mut T,
copies: &[crate::backends::common::gpu_types::kv_cache_update::Copy],
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs
index cfb465b3d..8425198ad 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(QKVNorm)]
#[variants(InputT, f32, bf16)]
@@ -10,10 +10,10 @@ use crate::array::ArrayElement;
#[variants(OutputT, f32, bf16)]
#[variants(AccumT, f32)]
pub fn qkv_norm<
- InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
- OutputT: ArrayElement + Float,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ ScaleT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float,
+ AccumT: DataTypeRepr + Float,
>(
#[optional(!in_place)] qkvg_input: Option<*const InputT>,
#[optional(has_scales)] scales: Option<*const ScaleT>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs
index d10807ad1..c0cf351d3 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs
@@ -2,11 +2,11 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(SigmoidGate)]
#[variants(T, f32, bf16)]
-pub fn sigmoid_gate<T: ArrayElement + Float>(
+pub fn sigmoid_gate<T: DataTypeRepr + Float>(
gate: *const T,
output: *mut T,
gate_dim: u32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs b/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs
index aaa9a4190..3a2c8e7ac 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs
@@ -2,11 +2,11 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(SeparableCausalConv)]
#[variants(T, bf16)]
-pub fn separable_causal_conv<T: ArrayElement + Float>(
+pub fn separable_causal_conv<T: DataTypeRepr + Float>(
input: *const T,
coefficient_deltas: *const T,
weights: *const T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs
index 91ea779f4..21b36d47c 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs
@@ -3,7 +3,6 @@ use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::{
- array::ArrayElement,
backends::{
common::gpu_types::{
d4s4, embedding::EmbeddingTableKind, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
@@ -11,11 +10,12 @@ use crate::{
},
cpu::kernel::activation_transform::hadamard_transform,
},
+ data_type::DataTypeRepr,
};
#[kernel(InputEmbeddingLookup)]
#[variants(T, f32, bf16)]
-pub fn input_embedding_lookup<T: ArrayElement + Float>(
+pub fn input_embedding_lookup<T: DataTypeRepr + Float>(
token_ids: *const u32,
values: *const u8,
#[optional(table_kind != EmbeddingTableKind::Dense)] scales: Option<*const T>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
index 72ea5555f..95256b52c 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
@@ -3,7 +3,6 @@ use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::{
- array::ArrayElement,
backends::{
common::{
gpu_types::{
@@ -15,11 +14,12 @@ use crate::{
},
cpu::kernel::activation_transform::{hadamard_transform, quantize_transformed_row},
},
+ data_type::DataTypeRepr,
};
#[kernel(GatedActMul)]
#[variants(T, f32, bf16)]
-pub fn gated_act_mul<T: ArrayElement + Float>(
+pub fn gated_act_mul<T: DataTypeRepr + Float>(
act_operand: *const T,
#[optional(!interleaved)] value_operand: Option<*const T>,
#[optional(ops == GatedActMulOp::FullPrecision)] fp_out: Option<*mut T>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs
index 20fb61d10..991154674 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};
// Multi-token causal conv1d with SiLU for DeltaNet.
// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd values
@@ -15,7 +15,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::A
// state_out: [conv_dim, state_stride]
#[kernel(DeltaNetConvScan)]
#[variants(T, f32, bf16)]
-pub fn delta_net_conv_scan<T: ArrayElement + Float>(
+pub fn delta_net_conv_scan<T: DataTypeRepr + Float>(
conv_padded: *const f32,
conv_weight: *const f32,
#[optional(has_bias)] bias: Option<*const f32>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs
index 607b72505..0933f0b7f 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs
@@ -2,12 +2,12 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};
// Single-token causal conv1d with SiLU, in-place.
#[kernel(DeltaNetConvUpdate)]
#[variants(T, f32, bf16)]
-pub fn delta_net_conv_update<T: ArrayElement + Float>(
+pub fn delta_net_conv_update<T: DataTypeRepr + Float>(
conv_weight: *const f32,
#[optional(has_bias)] bias: Option<*const f32>,
in_out: *mut T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs
index ad61a8a20..22712385e 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};
// In-place RMSNorm + SiLU gate for DeltaNet prefill output.
//
@@ -15,7 +15,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::A
// and z is at in_proj[token * total_proj_dim + conv_dim + hv * head_v_dim + i]
#[kernel(DeltaNetNormGate)]
#[variants(T, f32, bf16)]
-pub fn delta_net_norm_gate<T: ArrayElement + Float>(
+pub fn delta_net_norm_gate<T: DataTypeRepr + Float>(
in_out: *mut T,
in_proj: *const T,
norm_weight: *const f32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill.rs
index 698adff98..97e80cb5b 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill.rs
@@ -2,12 +2,12 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(DeltaNetPrefill)]
#[variants(T, f32, bf16)]
#[variants(HEAD_K_DIM, 128)]
-pub fn delta_net_prefill<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
+pub fn delta_net_prefill<T: DataTypeRepr + Float, const HEAD_K_DIM: u32>(
q_norm: *const f32,
k_norm: *const f32,
beta_buf: *const f32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill_prep.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill_prep.rs
index ace6ffbe0..7549bc7db 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill_prep.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill_prep.rs
@@ -2,14 +2,14 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
// Pre-compute L2-normalized q/k, beta, and decay for all tokens.
#[kernel(DeltaNetPrefillPrep)]
#[variants(T, f32, bf16)]
#[variants(QKT, f32, bf16)]
#[variants(HEAD_K_DIM, 128)]
-pub fn delta_net_prefill_prep<T: ArrayElement + Float, QKT: ArrayElement + Float, const HEAD_K_DIM: u32>(
+pub fn delta_net_prefill_prep<T: DataTypeRepr + Float, QKT: DataTypeRepr + Float, const HEAD_K_DIM: u32>(
in_proj: *const T,
a_log: *const f32,
dt_bias: *const f32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs
index 28f6b3df7..b8fd85ecf 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs
@@ -2,11 +2,11 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};
#[kernel(ConvTreeScan)]
#[variants(T, f32, bf16)]
-pub fn conv_tree_scan<T: ArrayElement + Float>(
+pub fn conv_tree_scan<T: DataTypeRepr + Float>(
in_proj: *const T,
conv_weight: *const f32,
#[optional(has_bias)] bias: Option<*const f32>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs
index cbb490f06..fde31f84b 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs
@@ -2,12 +2,12 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(BuildTreeOut)]
#[variants(QKT, f32, bf16)]
#[variants(OutputT, f32, bf16)]
-pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(
+pub fn build_tree_out<QKT: DataTypeRepr + Float, OutputT: DataTypeRepr + Float>(
q: *const QKT,
prefix: *const f32,
qkd: *const f32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs
index b382fec31..5930c0ddc 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs
@@ -2,12 +2,12 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(StateAdvance)]
#[variants(T, f32, bf16)]
#[variants(HEAD_K_DIM, 128)]
-pub fn state_advance<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
+pub fn state_advance<T: DataTypeRepr + Float, const HEAD_K_DIM: u32>(
k_norm: *const T,
v: *const T,
log_decay_buf: *const f32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
index 9ac1eba10..8488e2ed7 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
@@ -2,14 +2,14 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::trie::TrieNode};
+use crate::{backends::common::gpu_types::trie::TrieNode, data_type::DataTypeRepr};
const BLOCK: usize = 16;
#[kernel(BuildTreeGram)]
#[variants(T, f32, bf16)]
#[variants(USE_MXU, false)]
-pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(
+pub fn build_tree_gram<T: DataTypeRepr + Float, const USE_MXU: bool>(
q: *const T,
k: *const T,
trie: *const TrieNode,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
index f243c1e38..fcc41335f 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
@@ -2,14 +2,14 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
/// Solves chunked GDN update coefficients: `(I + A) U = beta * (v - exp(prefix) * kh0)`.
/// `kh0 = k @ h0^T` is precomputed by BuildTreeGram.
#[kernel(TreeUpdateSolve)]
#[variants(T, f32, bf16)]
#[variants(BV, 16, 32)]
-pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(
+pub fn tree_update_solve<T: DataTypeRepr + Float, const BV: u32>(
#[optional(use_h0)] kh0: Option<*const f32>,
v: *const T,
prefix: *const f32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs
index 6c5ac3f3e..650f9cf17 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs
@@ -2,7 +2,7 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};
// Single-token delta net update: decay state, apply delta rule, RMSNorm + SiLU gate.
// Steps per v-head:
@@ -13,7 +13,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::A
#[kernel(DeltaNetUpdate)]
#[variants(T, f32, bf16)]
#[variants(HEAD_K_DIM, 128)]
-pub fn delta_net_update<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
+pub fn delta_net_update<T: DataTypeRepr + Float, const HEAD_K_DIM: u32>(
in_proj: *const T,
a_log: *const f32,
dt_bias: *const f32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/logit_transform/logit_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/logit_transform/logit_transform.rs
index 4343822e2..fe7fe3fdd 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/logit_transform/logit_transform.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/logit_transform/logit_transform.rs
@@ -2,11 +2,11 @@ use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(LogitTransform)]
#[variants(T, f32, bf16)]
-pub fn logit_transform<T: ArrayElement + Float>(
+pub fn logit_transform<T: DataTypeRepr + Float>(
logits: *mut T,
length: u32,
scale: f32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
index 0b5788925..cce269e1a 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
@@ -1,4 +1,4 @@
-use half::{bf16, f16};
+use num_traits::AsPrimitive;
use crate::{
backends::{
@@ -9,7 +9,7 @@ use crate::{
cpu::{Cpu, buffer::CpuBufferExt, kernel::matmul::MatmulCpuError},
},
data_type::DataType,
- utils::pointers::SendPtr,
+ utils::{dispatch_dtype::dispatch_dtype, pointers::SendPtr},
};
pub(super) enum WeightData {
@@ -82,14 +82,7 @@ pub(super) unsafe fn read_f32(
data_type: DataType,
index: usize,
) -> f32 {
- unsafe {
- match data_type {
- DataType::F32 => *(base as *const f32).add(index),
- DataType::F16 => (*(base as *const f16).add(index)).to_f32(),
- DataType::BF16 => (*(base as *const bf16).add(index)).to_f32(),
- _ => unreachable!(),
- }
- }
+ dispatch_dtype!(|(T: data_type)| unsafe { (*base.cast::<T>().add(index)).as_() })
}
#[inline]
@@ -99,12 +92,5 @@ pub(super) unsafe fn write_f32(
index: usize,
value: f32,
) {
- unsafe {
- match data_type {
- DataType::F32 => *(base as *mut f32).add(index) = value,
- DataType::F16 => *(base as *mut f16).add(index) = f16::from_f32(value),
- DataType::BF16 => *(base as *mut bf16).add(index) = bf16::from_f32(value),
- _ => unreachable!(),
- }
- }
+ dispatch_dtype!(|(T: data_type)| unsafe { *base.cast::<T>().add(index) = value.as_() })
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
index f87bb72ac..45d2fd7a7 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
@@ -3,11 +3,11 @@ use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::{
- array::ArrayElement,
backends::{
common::gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
cpu::kernel::activation_transform::hadamard_transform,
},
+ data_type::DataTypeRepr,
};
#[kernel(Normalization)]
@@ -16,10 +16,10 @@ use crate::{
#[variants(OutputT, f32, f16, bf16)]
#[variants(AccumT, f32)]
pub fn normalization<
- InputT: ArrayElement + Float,
- AffineT: ArrayElement + Float,
- OutputT: ArrayElement + Float,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ AffineT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float,
+ AccumT: DataTypeRepr + Float,
>(
#[optional(!in_place)] input: Option<*const InputT>,
#[optional(has_scales)] scales: Option<*const AffineT>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/pooling/pooling.rs b/crates/uzu-engine/src/backends/cpu/kernel/pooling/pooling.rs
index 80845c5af..534e6f896 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/pooling/pooling.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/pooling/pooling.rs
@@ -2,11 +2,11 @@ use half::{bf16, f16};
use num_traits::{Float, ToPrimitive};
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(PoolingMean)]
#[variants(T, f32, f16, bf16)]
-pub fn pooling_mean<T: ArrayElement + Float>(
+pub fn pooling_mean<T: DataTypeRepr + Float>(
input: *const T,
output: *mut T,
seq_len: u32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/sampling/repetition_penalty.rs b/crates/uzu-engine/src/backends/cpu/kernel/sampling/repetition_penalty.rs
index 1d9f059bf..660fe268c 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/sampling/repetition_penalty.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/sampling/repetition_penalty.rs
@@ -2,11 +2,11 @@ use half::bf16;
use num_traits::{Float, NumCast};
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::ring::RingParams};
+use crate::{backends::common::gpu_types::ring::RingParams, data_type::DataTypeRepr};
#[kernel(RepetitionPenalty)]
#[variants(T, f32, bf16)]
-pub fn repetition_penalty<T: ArrayElement + Float + NumCast>(
+pub fn repetition_penalty<T: DataTypeRepr + Float + NumCast>(
original_logits: *const T,
logits_copy: *mut T,
context_ring: *const u32,
@@ -18,8 +18,8 @@ pub fn repetition_penalty<T: ArrayElement + Float + NumCast>(
sampling_length: u32,
) {
fn apply(
- original_logits: *const (impl ArrayElement + Float),
- logits_copy: *mut (impl ArrayElement + Float),
+ original_logits: *const (impl DataTypeRepr + Float),
+ logits_copy: *mut (impl DataTypeRepr + Float),
vocab_size: usize,
sample_index: usize,
token_id: usize,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/sampling/unified_sampling.rs b/crates/uzu-engine/src/backends/cpu/kernel/sampling/unified_sampling.rs
index b0e7711f2..308724333 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/sampling/unified_sampling.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/sampling/unified_sampling.rs
@@ -5,14 +5,14 @@ use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::{
- array::ArrayElement,
+ data_type::DataTypeRepr,
encodable_block::sampling::{gumbel_float, revidx},
};
// NOTE: top_k + top_p combination is not exactly matching lalamo ("parallel" here, should be top-k then top-p)
#[kernel(UnifiedSampling)]
#[variants(T, f32, bf16)]
-pub fn unified_sampling<T: ArrayElement + Float>(
+pub fn unified_sampling<T: DataTypeRepr + Float>(
logits: *const T,
output: *mut u32,
#[optional(is_stochastic)] seeds: Option<*const u64>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/short_conv/short_conv.rs b/crates/uzu-engine/src/backends/cpu/kernel/short_conv/short_conv.rs
index 4fce0ea94..d6fdb85dd 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/short_conv/short_conv.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/short_conv/short_conv.rs
@@ -2,11 +2,11 @@ use half::{bf16, f16};
use num_traits::{Float, ToPrimitive};
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(ShortConvPack)]
#[variants(T, f32, f16, bf16)]
-pub fn short_conv_pack<T: ArrayElement + Float>(
+pub fn short_conv_pack<T: DataTypeRepr + Float>(
state_in: *const T,
in_proj: *const T,
padded: *mut T,
@@ -44,7 +44,7 @@ pub fn short_conv_pack<T: ArrayElement + Float>(
#[kernel(ShortConvPrefill)]
#[variants(T, f32, f16, bf16)]
#[variants(WeightT, f32)]
-pub fn short_conv_prefill<T: ArrayElement + Float, WeightT: ArrayElement + Float>(
+pub fn short_conv_prefill<T: DataTypeRepr + Float, WeightT: DataTypeRepr + Float>(
padded: *const T,
in_proj: *const T,
w: *const WeightT,
@@ -114,7 +114,7 @@ pub fn short_conv_prefill<T: ArrayElement + Float, WeightT: ArrayElement + Float
#[kernel(ShortConvDecode)]
#[variants(T, f32, f16, bf16)]
#[variants(WeightT, f32)]
-pub fn short_conv_decode<T: ArrayElement + Float, WeightT: ArrayElement + Float>(
+pub fn short_conv_decode<T: DataTypeRepr + Float, WeightT: DataTypeRepr + Float>(
in_proj: *const T,
w: *const WeightT,
#[optional(has_bias)] b: Option<*const WeightT>,
@@ -185,7 +185,7 @@ pub fn short_conv_decode<T: ArrayElement + Float, WeightT: ArrayElement + Float>
#[kernel(ShortConvTrie)]
#[variants(T, f32, f16, bf16)]
#[variants(WeightT, f32)]
-pub fn short_conv_trie<T: ArrayElement + Float, WeightT: ArrayElement + Float>(
+pub fn short_conv_trie<T: DataTypeRepr + Float, WeightT: DataTypeRepr + Float>(
in_proj: *const T,
w: *const WeightT,
#[optional(has_bias)] b: Option<*const WeightT>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs b/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs
index 295315768..2c888f85a 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs
@@ -2,11 +2,11 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(Softmax)]
#[variants(T, f32, f16, bf16)]
-pub fn softmax<T: ArrayElement + Float>(
+pub fn softmax<T: DataTypeRepr + Float>(
values: *mut T,
#[optional(has_sinks)] sinks: Option<*const T>,
row_length: u32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs
index a45ea849c..f16d261c7 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs
@@ -2,12 +2,12 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(Conv1dPack)]
#[variants(StateT, f32, bf16)]
#[variants(InputT, f32, bf16)]
-pub fn conv1d_pack<StateT: ArrayElement + Float, InputT: ArrayElement + Float>(
+pub fn conv1d_pack<StateT: DataTypeRepr + Float, InputT: DataTypeRepr + Float>(
state_in: *const StateT,
x: *const InputT,
padded: *mut StateT,
@@ -40,7 +40,7 @@ pub fn conv1d_pack<StateT: ArrayElement + Float, InputT: ArrayElement + Float>(
#[kernel(Conv1dDecode)]
#[variants(T, f32, f16, bf16)]
-pub fn conv1d_decode<T: ArrayElement + Float>(
+pub fn conv1d_decode<T: DataTypeRepr + Float>(
x: *const T,
w: *const T,
#[optional(has_bias)] b: Option<*const T>,
@@ -128,7 +128,7 @@ pub fn conv1d_decode<T: ArrayElement + Float>(
#[kernel(Conv1dScan)]
#[variants(T, f32, f16, bf16)]
-pub fn conv1d_scan<T: ArrayElement + Float>(
+pub fn conv1d_scan<T: DataTypeRepr + Float>(
padded: *const T,
w: *const T,
#[optional(has_bias)] b: Option<*const T>,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs
index 54d2bbe3d..e6268c9a4 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs
@@ -2,11 +2,11 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(SplitInProj)]
#[variants(T, f32, f16, bf16)]
-pub fn split_in_proj<T: ArrayElement + Float>(
+pub fn split_in_proj<T: DataTypeRepr + Float>(
input: *const T,
conv_out: *mut T,
z_out: *mut T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs
index 8fd765754..84ec1a5b2 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs
@@ -2,11 +2,11 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(SSDPrefill64)]
#[variants(T, f32, f16, bf16)]
-pub fn ssd_prefill64<T: ArrayElement + Float>(
+pub fn ssd_prefill64<T: DataTypeRepr + Float>(
#[allow(unused)] x: *const T,
#[allow(unused)] dt_raw: *const T,
#[allow(unused)] b: *const T,
@@ -30,7 +30,7 @@ pub fn ssd_prefill64<T: ArrayElement + Float>(
#[kernel(SSDPrefill)]
#[variants(T, f32, f16, bf16)]
-pub fn ssd_prefill<T: ArrayElement + Float>(
+pub fn ssd_prefill<T: DataTypeRepr + Float>(
#[allow(unused)] x: *const T,
#[allow(unused)] dt_raw: *const T,
#[allow(unused)] b: *const T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs
index 290953a07..c7669cb04 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs
@@ -2,11 +2,11 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};
+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};
#[kernel(SSDUpdate)]
#[variants(T, f32, f16, bf16)]
-pub fn ssd_update<T: ArrayElement + Float>(
+pub fn ssd_update<T: DataTypeRepr + Float>(
x: *const T,
dt_raw: *const T,
b: *const T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
index 6c68f1e82..07cdedbfd 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
@@ -2,12 +2,12 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(TensorAddBias)]
#[variants(T, f32, f16, bf16)]
#[variants(BiasT, f32, f16, bf16)]
-pub fn tensor_add_bias<T: ArrayElement + Float, BiasT: ArrayElement + Float>(
+pub fn tensor_add_bias<T: DataTypeRepr + Float, BiasT: DataTypeRepr + Float>(
#[optional(!in_place)] input: Option<*const T>,
bias: *const BiasT,
output: *mut T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs b/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
index 48e946c8d..e3bc2f778 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
@@ -2,11 +2,11 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
#[kernel(TensorAddScale)]
#[variants(T, f32, f16, bf16)]
-pub fn tensor_add_scale<T: ArrayElement + Float>(
+pub fn tensor_add_scale<T: DataTypeRepr + Float>(
#[optional(!in_place)] input: Option<*const T>,
bias: *const T,
output: *mut T,
diff --git a/crates/uzu-engine/src/data_type.rs b/crates/uzu-engine/src/data_type.rs
index 5ee66baf0..f58175327 100644
--- a/crates/uzu-engine/src/data_type.rs
+++ b/crates/uzu-engine/src/data_type.rs
@@ -1,38 +1,24 @@
+use bytemuck::Pod;
+use half::{bf16, f16};
+use num_traits::NumCast;
use uzu_engine_macros::uzu_config;
#[derive(Eq, PartialOrd, Ord, Copy, Hash)]
#[uzu_config]
pub enum DataType {
- // Floating point
- #[serde(rename = "bfloat16")]
BF16,
- #[serde(rename = "float16")]
F16,
- #[serde(rename = "float32")]
F32,
- #[serde(rename = "float64")]
F64,
- // Sub-byte integers
- #[serde(rename = "int4")]
I4,
- #[serde(rename = "uint4")]
U4,
- // Normal integers
- #[serde(rename = "int8")]
I8,
- #[serde(rename = "uint8")]
U8,
- #[serde(rename = "int16")]
I16,
- #[serde(rename = "uint16")]
U16,
- #[serde(rename = "int32")]
I32,
- #[serde(rename = "uint32")]
U32,
- #[serde(rename = "int64")]
I64,
- #[serde(rename = "uint64")]
U64,
}
@@ -52,3 +38,34 @@ impl DataType {
self.size_in_bits().div_ceil(8)
}
}
+
+pub trait DataTypeRepr: NumCast + Pod {
+ fn data_type() -> DataType;
+}
+
+macro_rules! impl_data_type_repr {
+ ($($type:ty => $variant:ident),+ $(,)?) => {
+ $(
+ impl DataTypeRepr for $type {
+ fn data_type() -> DataType {
+ DataType::$variant
+ }
+ }
+ )+
+ };
+}
+
+impl_data_type_repr! {
+ f16 => F16,
+ bf16 => BF16,
+ f32 => F32,
+ f64 => F64,
+ i8 => I8,
+ u8 => U8,
+ i16 => I16,
+ u16 => U16,
+ i32 => I32,
+ u32 => U32,
+ i64 => I64,
+ u64 => U64,
+}
diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rs
index bef35e2db..3225f1ac4 100644
--- a/crates/uzu-engine/src/encodable_block/classifier.rs
+++ b/crates/uzu-engine/src/encodable_block/classifier.rs
@@ -15,7 +15,7 @@ use crate::{
prediction_head::{PredictionHead, PredictionHeadError},
transformer::{Transformer, TransformerEncodeError, TransformerNewError},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
};
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rs
index 7fc633724..064c31cf8 100644
--- a/crates/uzu-engine/src/encodable_block/convolution.rs
+++ b/crates/uzu-engine/src/encodable_block/convolution.rs
@@ -2,14 +2,14 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Backend, BufferRef, CommandBuffer, CommandBufferEncoding,
kernel::{Kernels, SeparableCausalConvKernel},
},
config::token_mixer::convolutions::SeparableCausalConvConfig,
data_type::DataType,
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
+ utils::size_for_shape::size_for_shape,
};
#[derive(Debug, Error)]
@@ -43,12 +43,11 @@ impl<B: Backend> SeparableCausalConv<B> {
assert!(model_dim.is_multiple_of(group_size));
assert_eq!(data_type, DataType::BF16, "SeparableCausalConv only supports BF16");
- let weights =
- parameter_tree.leaf("weights")?.validate(&[model_dim, kernel_size], DataType::BF16)?.read_buffer()?;
+ let weights = parameter_tree.read_buffer("weights", &[model_dim, kernel_size], DataType::BF16)?;
let biases = config
.has_biases
- .then(|| parameter_tree.leaf("biases")?.validate(&[model_dim], DataType::BF16)?.read_buffer())
+ .then(|| parameter_tree.read_buffer("biases", &[model_dim], DataType::BF16))
.transpose()?;
let kernel = <B::Kernels as Kernels>::SeparableCausalConvKernel::new(
diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rs
index 44b462310..b35078f49 100644
--- a/crates/uzu-engine/src/encodable_block/decoder.rs
+++ b/crates/uzu-engine/src/encodable_block/decoder.rs
@@ -14,7 +14,7 @@ use crate::{
per_layer_embedding::{PerLayerEmbedding, PerLayerEmbeddingError},
transformer::{Transformer, TransformerEncodeError, TransformerNewError, TransformerState},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
};
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index 87743fc8a..3beac9005 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -4,7 +4,6 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, gpu_types::trie::TrieNode,
},
@@ -24,8 +23,8 @@ use crate::{
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
transformer_layer::{TransformerLayer, TransformerLayerError},
},
- parameters::ParameterTree,
- utils::maybe_mut::MaybeMut,
+ safetensors::ParameterTree,
+ utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},
};
pub struct DFlashState<B: Backend> {
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index 5eb3facee..434948441 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -16,7 +16,7 @@ use crate::{
embedding_table::{EmbeddingTable, EmbeddingTableError, read_output_signs},
linear::{Gather, LinearEncodeError, LinearMatmulError, UntiedReadout},
},
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
};
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rs
index ab6916574..b1970e563 100644
--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs
@@ -14,7 +14,7 @@ use crate::{
},
data_type::DataType,
encodable_block::weight_matrix::{WeightMatrix, WeightMatrixError},
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
};
type LookupKernel<B> = <<B as Backend>::Kernels as Kernels>::InputEmbeddingLookupKernel;
@@ -207,7 +207,7 @@ fn load_d4s4<B: Backend>(
"{spec:?} with {data_type:?} and embedding dim {embedding_dim}"
)));
}
- let read = |name: &str, shape: &[u32], data_type| tree.leaf(name)?.validate(shape, data_type)?.read_buffer();
+ let read = |name: &str, shape: &[u32], data_type| tree.read_buffer(name, shape, data_type);
let table = D4S4Table {
codes: read("codes", &[vocab_size, embedding_dim / d4s4::VALUES_PER_CODE], DataType::U8)?,
row_scales: read("row_scales", &[vocab_size], data_type)?,
@@ -228,5 +228,5 @@ pub(super) fn read_output_signs<B: Backend>(
embedding_dim: u32,
) -> Result<B::GlobalBuffer, EmbeddingTableError<B>> {
let signs = tree.subtree("incoherence_signs");
- Ok(signs.leaf("output_signs")?.validate(&[embedding_dim], DataType::I32)?.read_buffer()?)
+ Ok(signs.read_buffer("output_signs", &[embedding_dim], DataType::I32)?)
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
index 6e4853a6d..ce609b594 100644
--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
@@ -19,7 +19,7 @@ use crate::{
linear::{Gather, Linear, LinearEncodeError, LinearInput},
weight_matrix::{WeightMatrix, WeightMatrixError},
},
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
};
#[derive(Debug, Error)]
@@ -59,9 +59,7 @@ fn load_biases<B: Backend>(
"mixed precision linear with biases is not supported: weights={weights_data_type:?}, output={output_data_type:?}",
)));
}
- Ok(parameter_tree
- .map(|tree| tree.leaf("biases")?.validate(&[output_dim], weights_data_type)?.read_buffer())
- .transpose()?)
+ Ok(parameter_tree.map(|tree| tree.read_buffer("biases", &[output_dim], weights_data_type)).transpose()?)
}
impl<B: Backend> LinearMatmul<B> {
diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rs
index c151f8d5f..2e9615cad 100644
--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs
@@ -24,7 +24,7 @@ use crate::{
hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
},
data_type::DataType,
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
};
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index c97472e4c..4f8e5a9ef 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -14,7 +14,7 @@ use crate::{
Gather, Linear, LinearEncodeError, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError,
input_rht::InputRht,
},
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
};
#[derive(Debug, Error)]
@@ -136,14 +136,9 @@ impl<B: Backend> RHTLinearWrapper<B> {
let weights_tree = parameter_tree.subtree("weights");
let quantized_weights_tree = weights_tree.subtree("quantized");
let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
- let rht_signs = weights_tree
- .leaf("incoherence_signs.input_signs")?
- .validate(&[input_dimension], DataType::I32)?
- .read_buffer()?;
- let output_factors = weights_tree
- .leaf("incoherence_signs.output_signs")?
- .validate(&[output_dimension], DataType::I32)?
- .read_buffer()?;
+ let rht_signs = weights_tree.read_buffer("incoherence_signs.input_signs", &[input_dimension], DataType::I32)?;
+ let output_factors =
+ weights_tree.read_buffer("incoherence_signs.output_signs", &[output_dimension], DataType::I32)?;
let inner_linear = LinearMatmul::load(
input_dimension,
output_dimension,
diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
index 169a7a95a..1237d2233 100644
--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
@@ -12,7 +12,7 @@ use crate::{
encodable_block::linear::{
Gather, Linear, LinearEncodeError, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
};
pub struct UntiedReadout<B: Backend> {
@@ -56,11 +56,7 @@ impl<B: Backend> UntiedReadout<B> {
None,
context,
)?;
- let rht_signs = tree
- .subtree("incoherence_signs")
- .leaf("input_signs")?
- .validate(&[model_dim], DataType::I32)?
- .read_buffer()?;
+ let rht_signs = tree.subtree("incoherence_signs").read_buffer("input_signs", &[model_dim], DataType::I32)?;
let preparation = LinearInputPreparation {
rht_signs,
activation_quantization: linear.prepare_a8(context),
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
index 2acb217df..d076b308a 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
@@ -20,7 +20,7 @@ use crate::{
},
},
},
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
utils::maybe_mut::MaybeMut,
};
@@ -140,10 +140,8 @@ impl<B: Backend> Attention<B> {
rope_config.is_some(),
)
.map_err(AttentionNewError::Backend)?;
- let sinks = config
- .has_sinks
- .then(|| parameter_tree.leaf("sinks")?.validate(&[num_q_heads], data_type)?.read_buffer())
- .transpose()?;
+ let sinks =
+ config.has_sinks.then(|| parameter_tree.read_buffer("sinks", &[num_q_heads], data_type)).transpose()?;
assert!(sliding_window_size.is_none_or(|size| size > 0), "zero sliding window size");
let ring_capacity = sliding_window_size;
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
index 5fc53ef76..e2b6ce736 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
@@ -8,7 +8,7 @@ use crate::{
},
config::normalization::{NormalizationConfig, UpcastMode},
data_type::DataType,
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
};
#[derive(Debug, Error)]
@@ -52,16 +52,15 @@ impl<B: Backend> QKVNorm<B> {
let Some(config) = config else {
return Ok::<_, QKVNormError<B>>(None);
};
- let scales = config
- .has_scale
- .then(|| {
- parameter_tree
- .subtree(parameter_name.expect("scaled norm requires parameter tree"))
- .leaf("scales")?
- .validate(&[head_dim], DataType::F32)?
- .read_buffer()
- })
- .transpose()?;
+ let scales =
+ config
+ .has_scale
+ .then(|| {
+ parameter_tree
+ .subtree(parameter_name.expect("scaled norm requires parameter tree"))
+ .read_buffer("scales", &[head_dim], DataType::F32)
+ })
+ .transpose()?;
let kernel = <B::Kernels as Kernels>::QKVNormKernel::new(
context,
intermediate_data_type,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
index e43044655..dddbacb68 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
@@ -1,7 +1,6 @@
use std::any::Any;
use crate::{
- array::size_for_shape,
backends::common::{
Backend, Buffer, CommandBuffer, Context, DeviceCapabilities, Kernels, SparseBuffer,
gpu_types::{kv_cache_update::Copy, ring::RingParams},
@@ -9,6 +8,7 @@ use crate::{
},
data_type::DataType,
encodable_block::mixer::{MixerState, attention::Attention},
+ utils::size_for_shape::size_for_shape,
};
pub const ATTENTION_SUFFIX_CAPACITY: u32 = 1024; // TODO: remove hardcoded suffix capacity
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
index d7b9ec6ed..bbe6f5276 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
@@ -2,7 +2,6 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,
CommandBufferPending, Context, Kernels,
@@ -25,8 +24,8 @@ use crate::{
delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},
},
},
- parameters::{ParameterLoaderError, ParameterTree},
- utils::maybe_mut::MaybeMut,
+ safetensors::{ParameterLoaderError, ParameterTree},
+ utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},
};
pub(crate) mod tree_verify;
@@ -204,10 +203,9 @@ impl<B: Backend> DeltaNet<B> {
let conv_config = &config.conv_config;
let conv_tree = parameter_tree.subtree("conv");
- let conv_weight =
- conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], INNER_DATA_TYPE)?.read_buffer()?;
+ let conv_weight = conv_tree.read_buffer("weights", &[conv_dim, config.kernel_size], INNER_DATA_TYPE)?;
let conv_bias = if conv_config.has_biases {
- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], INNER_DATA_TYPE)?.read_buffer()?)
+ Some(conv_tree.read_buffer("biases", &[conv_dim], INNER_DATA_TYPE)?)
} else {
None
};
@@ -227,10 +225,9 @@ impl<B: Backend> DeltaNet<B> {
)
.map_err(DeltaNetNewError::Backend)?;
- let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], INNER_DATA_TYPE)?.read_buffer()?;
- let dt_bias = parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], INNER_DATA_TYPE)?.read_buffer()?;
- let norm_weight =
- parameter_tree.leaf("norm.scales")?.validate(&[config.value_head_dim], INNER_DATA_TYPE)?.read_buffer()?;
+ let a_log = parameter_tree.read_buffer("a_log", &[config.num_heads], INNER_DATA_TYPE)?;
+ let dt_bias = parameter_tree.read_buffer("dt_bias", &[config.num_heads], INNER_DATA_TYPE)?;
+ let norm_weight = parameter_tree.read_buffer("norm.scales", &[config.value_head_dim], INNER_DATA_TYPE)?;
let norm_epsilon = config.norm_config.epsilon;
let delta_net_update =
<B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
index 6bf011646..8605e17d0 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
@@ -2,7 +2,6 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
gpu_types::activation_type::ActivationType,
@@ -18,8 +17,8 @@ use crate::{
linear::{Linear, LinearBlockError},
mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},
},
- parameters::{ParameterLoaderError, ParameterTree},
- utils::maybe_mut::MaybeMut,
+ safetensors::{ParameterLoaderError, ParameterTree},
+ utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},
};
const INNER_DATA_TYPE: DataType = DataType::F32;
@@ -124,17 +123,16 @@ impl<B: Backend> Mamba2<B> {
context,
)?;
- let gate_bias = parameter_tree.leaf("gate_bias")?.validate(&[inner_dim], INNER_DATA_TYPE)?.read_buffer()?;
+ let gate_bias = parameter_tree.read_buffer("gate_bias", &[inner_dim], INNER_DATA_TYPE)?;
let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, INNER_DATA_TYPE)
.map_err(Mamba2NewError::Backend)?;
let conv_config = &config.conv_config;
let conv_tree = parameter_tree.subtree("conv");
- let conv_weight =
- conv_tree.leaf("weights")?.validate(&[conv_dim, kernel_size], INNER_DATA_TYPE)?.read_buffer()?;
+ let conv_weight = conv_tree.read_buffer("weights", &[conv_dim, kernel_size], INNER_DATA_TYPE)?;
let conv_bias = if conv_config.has_biases {
- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], INNER_DATA_TYPE)?.read_buffer()?)
+ Some(conv_tree.read_buffer("biases", &[conv_dim], INNER_DATA_TYPE)?)
} else {
None
};
@@ -148,7 +146,7 @@ impl<B: Backend> Mamba2<B> {
.map_err(Mamba2NewError::Backend)?;
let skip_connection_weight =
- parameter_tree.leaf("skip_connection_weight")?.validate(&[num_heads], INNER_DATA_TYPE)?.read_buffer()?;
+ parameter_tree.read_buffer("skip_connection_weight", &[num_heads], INNER_DATA_TYPE)?;
let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, INNER_DATA_TYPE, true)
.map_err(Mamba2NewError::Backend)?;
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
index 772c4b39e..dc1061e44 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
@@ -17,7 +17,7 @@ use crate::{
short_conv::{ShortConv, ShortConvNewError},
},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
utils::maybe_mut::MaybeMut,
};
diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
index 817d74e70..649026f80 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
@@ -2,7 +2,6 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,
CommandBufferPending, Context, Kernels,
@@ -15,8 +14,8 @@ use crate::{
linear::{Linear, LinearBlockError},
mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},
},
- parameters::{ParameterLoaderError, ParameterTree},
- utils::maybe_mut::MaybeMut,
+ safetensors::{ParameterLoaderError, ParameterTree},
+ utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},
};
enum ShortConvStateSuffixStatus<B: Backend> {
@@ -137,12 +136,11 @@ impl<B: Backend> ShortConv<B> {
let conv_tree = parameter_tree.subtree("conv");
let conv_data_type = DataType::F32;
- let conv_weight =
- conv_tree.leaf("weights")?.validate(&[hidden_dim, kernel_size], conv_data_type)?.read_buffer()?;
+ let conv_weight = conv_tree.read_buffer("weights", &[hidden_dim, kernel_size], conv_data_type)?;
let has_bias = config.conv_config.has_biases;
let conv_bias = if has_bias {
- Some(conv_tree.leaf("biases")?.validate(&[hidden_dim], conv_data_type)?.read_buffer()?)
+ Some(conv_tree.read_buffer("biases", &[hidden_dim], conv_data_type)?)
} else {
None
};
diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
index 2649f4a3a..8bb398930 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
@@ -11,7 +11,7 @@ use crate::{
linear::{Linear, LinearBlockError},
mlp::{Mlp, MlpEncodeError, gate_act_mul::MlpGateActMulEncodable},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
};
pub struct DenseMlp<B: Backend> {
diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
index 4dbaa9209..5944294e5 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
@@ -12,7 +12,7 @@ use crate::{
linear::LinearEncodeError,
mlp::dense::{DenseMlp, DenseMlpNewError},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
};
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rs
index 90c5fd109..04cfb0873 100644
--- a/crates/uzu-engine/src/encodable_block/normalization.rs
+++ b/crates/uzu-engine/src/encodable_block/normalization.rs
@@ -2,14 +2,14 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding,
kernel::{Kernels, NormalizationKernel},
},
config::normalization::{NormalizationConfig, UpcastMode},
data_type::DataType,
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
+ utils::size_for_shape::size_for_shape,
};
// TODO: clean up
@@ -67,11 +67,11 @@ impl<B: Backend> Normalization<B> {
let scales = config
.has_scale
- .then(|| parameter_tree.leaf("scales")?.validate(&[element_count], DataType::F32)?.read_buffer())
+ .then(|| parameter_tree.read_buffer("scales", &[element_count], DataType::F32))
.transpose()?;
let biases = config
.has_biases
- .then(|| parameter_tree.leaf("biases")?.validate(&[element_count], DataType::F32)?.read_buffer())
+ .then(|| parameter_tree.read_buffer("biases", &[element_count], DataType::F32))
.transpose()?;
let (scale_residual_sum, scale_output, post_layer_scalar_value) = match post_layer_scalar {
diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
index f3aa283f8..d7a9f31c1 100644
--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
@@ -16,7 +16,7 @@ use crate::{
linear::{Linear, LinearBlockError, LinearEncodeError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
};
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rs
index e28394451..b5d2e094d 100644
--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs
+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs
@@ -13,7 +13,7 @@ use crate::{
linear::{Linear, LinearBlockError, LinearEncodeError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
};
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rs
index a3a9a0ed3..0514d4065 100644
--- a/crates/uzu-engine/src/encodable_block/transformer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer.rs
@@ -15,7 +15,7 @@ use crate::{
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
transformer_layer::{TransformerLayer, TransformerLayerError},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
utils::maybe_mut::MaybeMut,
};
diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
index dff07cc0d..30d12166f 100644
--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
@@ -1,4 +1,5 @@
use derive_more::Debug;
+use num_traits::AsPrimitive;
use thiserror::Error;
use crate::{
@@ -16,8 +17,8 @@ use crate::{
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
per_layer_embedding::{PerLayerEmbeddingError, PerLayerEmbeddingProjection},
},
- parameters::{ParameterLoaderError, ParameterTree},
- utils::maybe_mut::MaybeMut,
+ safetensors::{ParameterLoaderError, ParameterTree},
+ utils::{dispatch_dtype::dispatch_dtype, maybe_mut::MaybeMut},
};
#[derive(Debug, Error)]
@@ -186,13 +187,8 @@ impl<B: Backend> TransformerLayer<B> {
layer_index,
});
}
- let leaf = parameter_tree.leaf("post_layer_scalar")?;
- let scalar = match data_type {
- DataType::F32 => leaf.validate(&[1], data_type)?.read_slice::<f32>()?[0],
- DataType::F16 => leaf.validate(&[1], data_type)?.read_slice::<half::f16>()?[0].to_f32(),
- DataType::BF16 => leaf.validate(&[1], data_type)?.read_slice::<half::bf16>()?[0].to_f32(),
- _ => unreachable!(),
- };
+ let scalar =
+ dispatch_dtype!(|(T: data_type)| parameter_tree.read_slice::<T>("post_layer_scalar", &[1])?[0].as_());
Some(scalar)
} else {
None
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index d90e3474b..4517ed87f 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -2,7 +2,6 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- array::size_for_shape,
backends::common::{
Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Context, Kernels,
gpu_types::weaver::{
@@ -23,7 +22,8 @@ use crate::{
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
weaver_layer::{PreparedPrefixAttention, WeaverLayer},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
+ utils::size_for_shape::size_for_shape,
};
pub const DATA_TYPE: DataType = DataType::BF16;
diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
index 6bfd7cf75..161bc17b8 100644
--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
@@ -1,5 +1,4 @@
use crate::{
- array::size_for_shape,
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,
kernel::{AncestorAttentionKernel, AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel},
@@ -16,7 +15,8 @@ use crate::{
normalization::{Normalization, PostLayerScalar, ShortcutMode},
weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverEncodeError, WeaverNewError},
},
- parameters::ParameterTree,
+ safetensors::ParameterTree,
+ utils::size_for_shape::size_for_shape,
};
pub struct PreparedPrefixAttention<B: Backend> {
diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
index 21731c4a4..a80036620 100644
--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs
+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
@@ -9,7 +9,7 @@ use crate::{
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
- parameters::{ParameterLoaderError, ParameterTree},
+ safetensors::{ParameterLoaderError, ParameterTree},
};
#[derive(Debug, Error)]
@@ -114,7 +114,7 @@ impl<B: Backend> WeightMatrix<B> {
let (rows, columns) = physical_shape(&layout, output_dim, input_dim);
let Some(info) = quantization_info else {
- let values = tree.leaf("weights")?.validate(&[rows, columns], data_type)?.read_buffer()?;
+ let values = tree.read_buffer("weights", &[rows, columns], data_type)?;
return Ok(Self {
values,
quantized: None,
@@ -136,12 +136,11 @@ impl<B: Backend> WeightMatrix<B> {
}
let groups = columns.div_ceil(group_size);
- let values =
- tree.leaf("weights")?.validate(&[rows, columns / packing_divisor], storage_data_type)?.read_buffer()?;
+ let values = tree.read_buffer("weights", &[rows, columns / packing_divisor], storage_data_type)?;
let params = QuantParams::new(params_layout, rows, groups);
let load_plane =
|name: &str, shape: [u32; 2], storage_type: DataType| -> Result<B::GlobalBuffer, WeightMatrixError<B>> {
- Ok(tree.leaf(name)?.validate(&shape, storage_type)?.read_buffer()?)
+ Ok(tree.read_buffer(name, &shape, storage_type)?)
};
let scales = load_plane("scales", params.scale_shape(), data_type)?;
let correction = match info.method {
diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rs
index 3c9832faa..557360353 100644
--- a/crates/uzu-engine/src/engine/classifier_model.rs
+++ b/crates/uzu-engine/src/engine/classifier_model.rs
@@ -24,7 +24,7 @@ use crate::{
data_type::DataType,
encodable_block::classifier::{Classifier as ClassifierEncodable, ClassifierError as ClassifierEncodableError},
engine::Engine,
- parameters::{ParameterLoader, ParameterLoaderError},
+ safetensors::{ParameterLoader, ParameterLoaderError},
};
pub struct ClassifierModel<B: Backend> {
@@ -79,7 +79,7 @@ impl<B: Backend> Engine<B> {
(0..config.classifier_config.num_labels).map(|index| format!("class_{index}")).collect()
};
- weight_loader.tree().assert_all_tensors_validated()?;
+ weight_loader.assert_all_tensors_read()?;
let tokenizer = Arc::new(Tokenizer::from_file(model_path.join("tokenizer.json"))?);
let token_codec_config = config.token_codec_config;
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index adc82fe58..91cc482b1 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -13,7 +13,7 @@ use crate::{
sampling::{Sampling, SamplingMethod},
},
engine::Engine,
- parameters::{ParameterLoader, ParameterLoaderError},
+ safetensors::{ParameterLoader, ParameterLoaderError},
speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},
};
pub use crate::{config::model::generation::GenerationConfig, utils::strict_serde::Unsupported};
@@ -97,7 +97,7 @@ impl<B: Backend> Engine<B> {
let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&self.context)
.map_err(EngineLoadLanguageModelError::Backend)?;
- weight_loader.tree().assert_all_tensors_validated()?;
+ weight_loader.assert_all_tensors_read()?;
let generation_config = config.generation_config;
let end_of_thinking_tag = match &config.token_codec_config {
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 706da8cd0..bf579c144 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -10,7 +10,6 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;
#[cfg(grammar)]
use crate::engine::language_model::grammar::Grammar;
use crate::{
- array::size_for_shape,
backends::common::{
Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,
CommandBufferPending, Context,
@@ -26,6 +25,7 @@ use crate::{
stream::{LanguageModelStreamError, LanguageModelStreamOptions},
},
trie::TrieNode,
+ utils::size_for_shape::size_for_shape,
};
enum ForwardPassChaining<B: Backend> {
diff --git a/crates/uzu-engine/src/lib.rs b/crates/uzu-engine/src/lib.rs
index d073d94a5..3a23a313c 100644
--- a/crates/uzu-engine/src/lib.rs
+++ b/crates/uzu-engine/src/lib.rs
@@ -2,11 +2,10 @@
#![cfg_attr(test, test_runner(crate::tests::harness::uzu_harness))]
#![cfg_attr(target_family = "wasm", feature(wasi_ext))]
-mod array;
mod config;
mod data_type;
mod encodable_block;
-mod parameters;
+mod safetensors;
mod speculators;
mod trie;
mod utils;
diff --git a/crates/uzu-engine/src/parameters/error.rs b/crates/uzu-engine/src/parameters/error.rs
deleted file mode 100644
index 6e042a546..000000000
--- a/crates/uzu-engine/src/parameters/error.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-use derive_more::Debug;
-use thiserror::Error;
-
-use super::safetensors_metadata::HeaderLoadingError;
-use crate::{backends::common::Backend, data_type::DataType};
-
-#[derive(Debug, Error)]
-pub enum ParameterLoaderError<B: Backend> {
- #[error("Header loading error: {0}")]
- HeaderLoadingError(#[from] HeaderLoadingError),
- #[error("Array with key \"{0}\" not found.")]
- KeyNotFound(String),
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
- #[error("Failed to read data")]
- ArrayLoadingError(#[from] std::io::Error),
- #[error("Failed to deserialize metadata")]
- MetadataDeserializationError(#[from] serde_json::Error),
- #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]
- InvalidTensor {
- shape: Box<[u32]>,
- data_type: DataType,
- expected_shape: Box<[u32]>,
- expected_data_type: DataType,
- },
- #[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]
- InvalidTensorSize {
- shape: Box<[u32]>,
- data_type: DataType,
- size: usize,
- expected_size: usize,
- },
- #[error("Unvalidated tensors under {prefix:?}: {keys:?}")]
- UnvalidatedTensors {
- prefix: Option<String>,
- keys: Box<[String]>,
- },
-}
diff --git a/crates/uzu-engine/src/parameters/loader.rs b/crates/uzu-engine/src/parameters/loader.rs
deleted file mode 100644
index 4c6a54e8d..000000000
--- a/crates/uzu-engine/src/parameters/loader.rs
+++ /dev/null
@@ -1,214 +0,0 @@
-#[cfg(unix)]
-use std::os::unix::fs::FileExt;
-#[cfg(target_family = "wasm")]
-use std::os::wasi::fs::FileExt;
-use std::{
- cell::RefCell,
- collections::{HashMap, HashSet},
- fs::File,
-};
-
-use super::{
- error::ParameterLoaderError,
- safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata},
-};
-use crate::{
- array::{ArrayElement, size_for_shape},
- backends::common::{Backend, BufferMut, Context},
- data_type::DataType,
- utils::strict_serde::DeserializeStrictOwned,
-};
-
-struct ParameterMetadata {
- shape: Box<[u32]>,
- data_type: DataType,
- offset: usize,
- size: usize,
-}
-
-pub struct ParameterLoader<'a, B: Backend> {
- context: &'a B::Context,
- index: HashMap<String, ParameterMetadata>,
- metadata: HashMap<String, String>,
- validated_tensors: RefCell<HashSet<String>>,
- file: &'a File,
-}
-
-impl<'a, B: Backend> ParameterLoader<'a, B> {
- pub fn new(
- file: &'a File,
- context: &'a B::Context,
- ) -> Result<Self, ParameterLoaderError<B>> {
- let (global_offset, st_metadata) = read_st_metadata(file)?;
- let index = st_metadata
- .tensors
- .into_iter()
- .map(|(key, value)| {
- let (local_begin, local_end) = value.data_offsets;
- let size =
- local_end.checked_sub(local_begin).ok_or_else(|| HeaderLoadingError::InvalidTensorOffsets {
- key: key.clone().into_boxed_str(),
- begin: local_begin,
- end: local_end,
- })?;
- let offset =
- global_offset.checked_add(local_begin).ok_or_else(|| HeaderLoadingError::TensorOffsetOverflow {
- key: key.clone().into_boxed_str(),
- global_offset,
- local_begin,
- })?;
- let weight_metadata = ParameterMetadata {
- shape: value.shape.into(),
- data_type: value.dtype.data_type()?,
- offset,
- size,
- };
- Ok((key, weight_metadata))
- })
- .collect::<Result<HashMap<_, _>, HeaderLoadingError>>()?;
- let metadata = st_metadata.metadata.unwrap_or_default();
- Ok(ParameterLoader {
- context,
- index,
- metadata,
- validated_tensors: RefCell::new(HashSet::new()),
- file,
- })
- }
-
- pub fn tree(&self) -> ParameterTree<'_, B> {
- ParameterTree {
- loader: self,
- prefix: None,
- }
- }
-}
-
-pub struct ParameterLeaf<'a, 'leaf, B: Backend, const VALIDATED: bool> {
- key: &'leaf str,
- metadata: &'leaf ParameterMetadata,
- loader: &'leaf ParameterLoader<'a, B>,
-}
-
-impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, false> {
- pub fn validate(
- self,
- expected_shape: &[u32],
- expected_data_type: DataType,
- ) -> Result<ParameterLeaf<'a, 'leaf, B, true>, ParameterLoaderError<B>> {
- let shape = self.metadata.shape.as_ref();
- let data_type = self.metadata.data_type;
- if (shape, data_type) != (expected_shape, expected_data_type) {
- return Err(ParameterLoaderError::InvalidTensor {
- shape: shape.into(),
- data_type,
- expected_shape: expected_shape.into(),
- expected_data_type,
- });
- }
- let expected_size = size_for_shape(expected_shape, expected_data_type);
- if self.metadata.size != expected_size {
- return Err(ParameterLoaderError::InvalidTensorSize {
- shape: shape.into(),
- data_type,
- size: self.metadata.size,
- expected_size,
- });
- }
- self.loader.validated_tensors.borrow_mut().insert(self.key.to_string());
- Ok(ParameterLeaf {
- key: self.key,
- metadata: self.metadata,
- loader: self.loader,
- })
- }
-}
-
-impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, true> {
- pub fn read_slice<T: ArrayElement>(&self) -> Result<Box<[T]>, ParameterLoaderError<B>> {
- let element_count = self.metadata.size / std::mem::size_of::<T>();
- let mut data = vec![T::zeroed(); element_count];
- let destination = bytemuck::cast_slice_mut(&mut data);
- self.loader.file.read_exact_at(destination, self.metadata.offset as u64)?;
- Ok(data.into_boxed_slice())
- }
-
- pub fn read_buffer(&self) -> Result<B::GlobalBuffer, ParameterLoaderError<B>> {
- let mut buffer =
- self.loader.context.create_buffer(self.metadata.size).map_err(ParameterLoaderError::BackendError)?;
- self.loader.file.read_exact_at(buffer.as_slice_mut::<u8>(), self.metadata.offset as u64)?;
- Ok(buffer)
- }
-}
-
-pub struct ParameterTree<'loader, B: Backend> {
- loader: &'loader ParameterLoader<'loader, B>,
- prefix: Option<String>,
-}
-
-impl<'loader, B: Backend> ParameterTree<'loader, B> {
- fn join_prefix(
- &self,
- name: &str,
- ) -> String {
- self.prefix.as_ref().map_or_else(|| name.to_string(), |p| format!("{p}.{name}"))
- }
-
- pub fn subtree(
- &self,
- name: &str,
- ) -> Self {
- Self {
- loader: self.loader,
- prefix: Some(self.join_prefix(name)),
- }
- }
-
- pub fn leaf<'leaf>(
- &'leaf self,
- name: &str,
- ) -> Result<ParameterLeaf<'loader, 'leaf, B, false>, ParameterLoaderError<B>> {
- let key = self.join_prefix(name);
- let Some((key, metadata)) = self.loader.index.get_key_value(&key) else {
- return Err(ParameterLoaderError::KeyNotFound(key));
- };
- Ok(ParameterLeaf {
- key,
- metadata,
- loader: self.loader,
- })
- }
-
- pub fn metadata<T: DeserializeStrictOwned>(
- &self,
- name: &str,
- ) -> Result<T, ParameterLoaderError<B>> {
- let new_prefix = self.join_prefix(name);
-
- Ok(serde_json::from_str(
- self.loader.metadata.get(&new_prefix).ok_or(ParameterLoaderError::KeyNotFound(new_prefix))?,
- )?)
- }
-
- pub fn assert_all_tensors_validated(&self) -> Result<(), ParameterLoaderError<B>> {
- let subtree_prefix = self.prefix.as_ref().map(|prefix| format!("{prefix}."));
- let validated_tensors = self.loader.validated_tensors.borrow();
- let mut unvalidated_tensors = self
- .loader
- .index
- .keys()
- .filter(|key| subtree_prefix.as_ref().is_none_or(|prefix| key.starts_with(prefix)))
- .filter(|key| !validated_tensors.contains(*key))
- .cloned()
- .collect::<Vec<_>>();
- unvalidated_tensors.sort();
- if unvalidated_tensors.is_empty() {
- Ok(())
- } else {
- Err(ParameterLoaderError::UnvalidatedTensors {
- prefix: self.prefix.clone(),
- keys: unvalidated_tensors.into_boxed_slice(),
- })
- }
- }
-}
diff --git a/crates/uzu-engine/src/parameters/mod.rs b/crates/uzu-engine/src/parameters/mod.rs
deleted file mode 100644
index b5d319300..000000000
--- a/crates/uzu-engine/src/parameters/mod.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-// TODO: This is overdue for a complete rewrite
-
-mod error;
-mod loader;
-mod safetensors_metadata;
-
-pub use error::ParameterLoaderError;
-pub use loader::{ParameterLoader, ParameterTree};
diff --git a/crates/uzu-engine/src/parameters/safetensors_metadata.rs b/crates/uzu-engine/src/parameters/safetensors_metadata.rs
deleted file mode 100644
index bb705e6d4..000000000
--- a/crates/uzu-engine/src/parameters/safetensors_metadata.rs
+++ /dev/null
@@ -1,130 +0,0 @@
-// This code is based on the safetensors implementation: https://docs.rs/safetensors/latest/src/safetensors/tensor.rs.html
-
-#[cfg(unix)]
-use std::os::unix::fs::FileExt;
-#[cfg(target_family = "wasm")]
-use std::os::wasi::fs::FileExt;
-use std::{collections::HashMap, fs::File};
-
-use serde::{Deserialize, Serialize};
-use thiserror::Error;
-
-use crate::data_type::DataType;
-
-#[derive(Debug, Error)]
-pub enum HeaderLoadingError {
- #[error("Unable to read safetensors header bytes: {0}")]
- UnableToReadHeader(#[source] std::io::Error),
- #[error("Unable to read safetensors header JSON: {0}")]
- UnableToReadHeaderJson(#[source] std::io::Error),
- #[error("The header is not valid UTF-8 JSON: {0}")]
- InvalidHeaderDeserialization(#[from] serde_json::Error),
- #[error("The header is too large.")]
- HeaderTooLarge,
- #[error("Unsupported safetensors dtype: {0:?}")]
- UnsupportedDtype(Dtype),
- #[error("Invalid data offsets for tensor {key}: begin={begin}, end={end}")]
- InvalidTensorOffsets {
- key: Box<str>,
- begin: usize,
- end: usize,
- },
- #[error("Tensor offset overflow for tensor {key}: global_offset={global_offset}, local_begin={local_begin}")]
- TensorOffsetOverflow {
- key: Box<str>,
- global_offset: usize,
- local_begin: usize,
- },
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct HashMetadata {
- #[serde(skip_serializing_if = "Option::is_none")]
- #[serde(rename = "__metadata__")]
- pub metadata: Option<HashMap<String, String>>,
- #[serde(flatten)]
- pub tensors: HashMap<String, TensorInfo>,
-}
-
-#[derive(Debug, Deserialize, Serialize, Clone)]
-pub struct TensorInfo {
- pub dtype: Dtype,
- pub shape: Vec<u32>,
- pub data_offsets: (usize, usize),
-}
-
-#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]
-#[non_exhaustive]
-pub enum Dtype {
- /// Boolean type
- BOOL,
- /// Unsigned byte
- U8,
- /// Signed byte
- I8,
- /// FP8 <https://arxiv.org/pdf/2209.05433.pdf>_
- #[allow(non_camel_case_types)]
- F8_E5M2,
- /// FP8 <https://arxiv.org/pdf/2209.05433.pdf>_
- #[allow(non_camel_case_types)]
- F8_E4M3,
- /// Signed integer (16-bit)
- I16,
- /// Unsigned integer (16-bit)
- U16,
- /// Half-precision floating point
- F16,
- /// Brain floating point
- BF16,
- /// Signed integer (32-bit)
- I32,
- /// Unsigned integer (32-bit)
- U32,
- /// Floating point (32-bit)
- F32,
- /// Floating point (64-bit)
- F64,
- /// Signed integer (64-bit)
- I64,
- /// Unsigned integer (64-bit)
- U64,
-}
-
-impl Dtype {
- pub fn data_type(self) -> Result<DataType, HeaderLoadingError> {
- Ok(match self {
- Self::F16 => DataType::F16,
- Self::BF16 => DataType::BF16,
- Self::F32 => DataType::F32,
- Self::I8 => DataType::I8,
- Self::U8 => DataType::U8,
- Self::I32 => DataType::I32,
- Self::U32 => DataType::U32,
- Self::I64 => DataType::I64,
- Self::U64 => DataType::U64,
- dtype => return Err(HeaderLoadingError::UnsupportedDtype(dtype)),
- })
- }
-}
-
-const MAX_HEADER_SIZE: usize = 100_000_000;
-
-pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadingError> {
- let mut header_buffer = [0u8; size_of::<u64>()];
- file.read_exact_at(&mut header_buffer, 0).map_err(HeaderLoadingError::UnableToReadHeader)?;
- let metadata_size: usize =
- u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;
- if metadata_size > MAX_HEADER_SIZE {
- return Err(HeaderLoadingError::HeaderTooLarge);
- }
-
- let mut json_buffer = vec![0u8; metadata_size];
- file.read_exact_at(&mut json_buffer, size_of::<u64>() as u64)
- .map_err(HeaderLoadingError::UnableToReadHeaderJson)?;
- let metadata: HashMetadata = serde_json::from_slice(&json_buffer)?;
- Ok((size_of::<u64>() + metadata_size, metadata))
-}
-
-#[cfg(test)]
-#[path = "../../unit/parameters/safetensors_metadata.rs"]
-mod tests;
diff --git a/crates/uzu-engine/src/safetensors.rs b/crates/uzu-engine/src/safetensors.rs
new file mode 100644
index 000000000..2bdc7a785
--- /dev/null
+++ b/crates/uzu-engine/src/safetensors.rs
@@ -0,0 +1,217 @@
+#[cfg(unix)]
+use std::os::unix::fs::FileExt;
+#[cfg(target_family = "wasm")]
+use std::os::wasi::fs::FileExt;
+use std::{cell::Cell, collections::HashMap, fs::File};
+
+use derive_more::Debug;
+use serde::Deserialize;
+use thiserror::Error;
+
+use crate::{
+ backends::common::{Backend, BufferMut, Context},
+ data_type::{DataType, DataTypeRepr},
+ utils::{size_for_shape::size_for_shape, strict_serde::DeserializeStrictOwned},
+};
+
+#[derive(Debug, Error)]
+pub enum ParameterLoaderError<B: Backend> {
+ #[error("IO error: {0}")]
+ IO(#[from] std::io::Error),
+ #[error("Unable to deserialize safetensors header JSON: {0}")]
+ HeaderDeserialization(#[from] serde_json::Error),
+ #[error("Key {0:?} not found")]
+ KeyNotFound(String),
+ #[error("Unable to deserialize metadata {key:?}: {source}")]
+ MetadataDeserializationError {
+ key: String,
+ #[source]
+ source: serde_json::Error,
+ },
+ #[error(
+ "Invalid tensor {key:?}: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}"
+ )]
+ InvalidTensor {
+ key: String,
+ shape: Box<[u32]>,
+ data_type: DataType,
+ expected_shape: Box<[u32]>,
+ expected_data_type: DataType,
+ },
+ #[error(
+ "Invalid representation for tensor {key:?}: {shape:?} @ {data_type:?} requires {expected_size} bytes, global_offset={global_offset}, begin={begin}, end={end}"
+ )]
+ InvalidTensorRepresentation {
+ key: String,
+ shape: Box<[u32]>,
+ data_type: DataType,
+ expected_size: usize,
+ global_offset: u64,
+ begin: u64,
+ end: u64,
+ },
+ #[error("Unread tensors: {0:?}")]
+ UnreadTensors(Vec<String>),
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+}
+
+#[derive(Deserialize)]
+struct Header {
+ #[serde(default, rename = "__metadata__")]
+ metadata: HashMap<String, String>,
+ #[serde(flatten)]
+ tensors: HashMap<String, TensorInfo>,
+}
+
+#[derive(Deserialize)]
+struct TensorInfo {
+ dtype: DataType,
+ shape: Box<[u32]>,
+ data_offsets: (u64, u64),
+ #[serde(skip)]
+ read: Cell<bool>,
+}
+
+pub struct ParameterLoader<'a, B: Backend> {
+ file: &'a File,
+ context: &'a B::Context,
+ header: Header,
+ data_offset: u64,
+}
+
+impl<'a, B: Backend> ParameterLoader<'a, B> {
+ pub fn new(
+ file: &'a File,
+ context: &'a B::Context,
+ ) -> Result<Self, ParameterLoaderError<B>> {
+ let mut header_size = [0; 8];
+ file.read_exact_at(&mut header_size, 0)?;
+ let header_size = u64::from_le_bytes(header_size);
+ let mut header = vec![0; header_size as usize];
+ file.read_exact_at(&mut header, 8)?;
+ let header: Header = serde_json::from_slice(&header)?;
+ Ok(Self {
+ file,
+ context,
+ header,
+ data_offset: 8 + header_size,
+ })
+ }
+
+ pub fn tree(&self) -> ParameterTree<'_, B> {
+ ParameterTree {
+ loader: self,
+ prefix: String::new(),
+ }
+ }
+
+ pub fn assert_all_tensors_read(&self) -> Result<(), ParameterLoaderError<B>> {
+ let mut unread = self
+ .header
+ .tensors
+ .iter()
+ .filter(|(_, tensor)| !tensor.read.get())
+ .map(|(key, _)| key.clone())
+ .collect::<Vec<_>>();
+ if unread.is_empty() {
+ Ok(())
+ } else {
+ unread.sort();
+ Err(ParameterLoaderError::UnreadTensors(unread))
+ }
+ }
+}
+
+pub struct ParameterTree<'a, B: Backend> {
+ loader: &'a ParameterLoader<'a, B>,
+ prefix: String,
+}
+
+impl<B: Backend> ParameterTree<'_, B> {
+ pub fn subtree(
+ &self,
+ name: &str,
+ ) -> Self {
+ Self {
+ loader: self.loader,
+ prefix: format!("{}{name}.", self.prefix),
+ }
+ }
+
+ pub fn metadata<T: DeserializeStrictOwned>(
+ &self,
+ name: &str,
+ ) -> Result<T, ParameterLoaderError<B>> {
+ let key = format!("{}{name}", self.prefix);
+ let json =
+ self.loader.header.metadata.get(&key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.clone()))?;
+ serde_json::from_str(json).map_err(|source| ParameterLoaderError::MetadataDeserializationError {
+ key,
+ source,
+ })
+ }
+
+ fn tensor(
+ &self,
+ name: &str,
+ shape: &[u32],
+ dtype: DataType,
+ ) -> Result<&TensorInfo, ParameterLoaderError<B>> {
+ let key = format!("{}{name}", self.prefix);
+ let tensor =
+ self.loader.header.tensors.get(&key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.clone()))?;
+ if (tensor.shape.as_ref(), tensor.dtype) != (shape, dtype) {
+ return Err(ParameterLoaderError::InvalidTensor {
+ key,
+ shape: tensor.shape.clone(),
+ data_type: tensor.dtype,
+ expected_shape: shape.into(),
+ expected_data_type: dtype,
+ });
+ }
+ let (start, end) = tensor.data_offsets;
+ let expected_size = size_for_shape(shape, dtype);
+ if end.checked_sub(start) != Some(expected_size as u64) || self.loader.data_offset.checked_add(end).is_none() {
+ return Err(ParameterLoaderError::InvalidTensorRepresentation {
+ key,
+ shape: tensor.shape.clone(),
+ data_type: tensor.dtype,
+ expected_size,
+ global_offset: self.loader.data_offset,
+ begin: start,
+ end,
+ });
+ }
+ Ok(tensor)
+ }
+
+ pub fn read_buffer(
+ &self,
+ name: &str,
+ shape: &[u32],
+ dtype: DataType,
+ ) -> Result<B::GlobalBuffer, ParameterLoaderError<B>> {
+ let tensor = self.tensor(name, shape, dtype)?;
+ let (start, end) = tensor.data_offsets;
+ let mut buffer =
+ self.loader.context.create_buffer((end - start) as usize).map_err(ParameterLoaderError::BackendError)?;
+ self.loader.file.read_exact_at(buffer.as_slice_mut::<u8>(), self.loader.data_offset + start)?;
+ tensor.read.set(true);
+ Ok(buffer)
+ }
+
+ pub fn read_slice<T: DataTypeRepr>(
+ &self,
+ name: &str,
+ shape: &[u32],
+ ) -> Result<Box<[T]>, ParameterLoaderError<B>> {
+ let tensor = self.tensor(name, shape, T::data_type())?;
+ let (start, end) = tensor.data_offsets;
+ let size = (end - start) as usize;
+ let mut data = vec![T::zeroed(); size / size_of::<T>()];
+ self.loader.file.read_exact_at(bytemuck::cast_slice_mut(&mut data), self.loader.data_offset + start)?;
+ tensor.read.set(true);
+ Ok(data.into_boxed_slice())
+ }
+}
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index b3c503b41..e29e7a831 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -27,7 +27,7 @@ use crate::{
sampling::{PRng, Sampling, SamplingMethod},
weaver::{ProposalNode, Weaver, WeaverEncodeError, WeaverNewError, WeaverTreeShape},
},
- parameters::{ParameterLoader, ParameterLoaderError},
+ safetensors::{ParameterLoader, ParameterLoaderError},
trie::TrieNode,
};
@@ -136,7 +136,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
})
.transpose()?;
- weight_loader.tree().assert_all_tensors_validated()?;
+ weight_loader.assert_all_tensors_read()?;
let sampling = Sampling::new(config.draft_config.vocab_size, data_type);
diff --git a/crates/uzu-engine/src/utils/dispatch_dtype.rs b/crates/uzu-engine/src/utils/dispatch_dtype.rs
new file mode 100644
index 000000000..38a6fc288
--- /dev/null
+++ b/crates/uzu-engine/src/utils/dispatch_dtype.rs
@@ -0,0 +1,12 @@
+macro_rules! dispatch_dtype {
+ (|()| $body:expr) => { $body };
+ (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {
+ match $dt {
+ $crate::data_type::DataType::F16 => { type $T = half::f16; $crate::utils::dispatch_dtype::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
+ $crate::data_type::DataType::BF16 => { type $T = half::bf16; $crate::utils::dispatch_dtype::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
+ $crate::data_type::DataType::F32 => { type $T = f32; $crate::utils::dispatch_dtype::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
+ other => panic!("Unsupported data type: {other:?}"),
+ }
+ };
+}
+pub(crate) use dispatch_dtype;
diff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rs
index e0f6473d3..0137b4219 100644
--- a/crates/uzu-engine/src/utils/mod.rs
+++ b/crates/uzu-engine/src/utils/mod.rs
@@ -1,6 +1,8 @@
+pub mod dispatch_dtype;
pub mod downcast;
pub mod maybe_mut;
#[cfg(backend = "cpu")]
pub mod pointers;
+pub mod size_for_shape;
pub mod strict_serde;
pub mod version;
diff --git a/crates/uzu-engine/src/array/util.rs b/crates/uzu-engine/src/utils/size_for_shape.rs
similarity index 100%
rename from crates/uzu-engine/src/array/util.rs
rename to crates/uzu-engine/src/utils/size_for_shape.rs
diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs
index 46ab80e16..7e6fba6e6 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,20 +12,20 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
data: Box<[T]>,
act_type: ActivationType,
in_place: bool,
}
-fn make_data<T: ArrayElement + Float>(n: usize) -> Vec<T> {
+fn make_data<T: DataTypeRepr + Float>(n: usize) -> Vec<T> {
let mut data = Vec::with_capacity(n);
for i in 0..n {
// Use a range of values including negative to exercise both branches of activations
@@ -35,7 +34,7 @@ fn make_data<T: ArrayElement + Float>(n: usize) -> Vec<T> {
data
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let kernel = <<B as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, T::data_type(), input.in_place)
.expect("Failed to create ActivationKernel");
@@ -58,7 +57,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
}
}
-fn get_test_data<T: ArrayElement + Float>(
+fn get_test_data<T: DataTypeRepr + Float>(
act_type: ActivationType,
in_place: bool,
) -> (Input<T>, Vec<T>) {
@@ -76,7 +75,7 @@ fn get_test_data<T: ArrayElement + Float>(
}
/// Large input to exercise more threads on GPU.
-fn get_test_data_large<T: ArrayElement + Float>(
+fn get_test_data_large<T: DataTypeRepr + Float>(
act_type: ActivationType,
in_place: bool,
) -> (Input<T>, Vec<T>) {
@@ -93,7 +92,7 @@ fn get_test_data_large<T: ArrayElement + Float>(
(input, expected)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected: &[T],
test_name: &str,
@@ -111,7 +110,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
});
}
-fn test_activation<T: ArrayElement + Float + Debug + Display>(
+fn test_activation<T: DataTypeRepr + Float + Debug + Display>(
act_type: ActivationType,
in_place: bool,
) {
@@ -128,7 +127,7 @@ fn test_activation<T: ArrayElement + Float + Debug + Display>(
test_internal::<T>(&input, &expected, &label);
}
-fn test_activation_large<T: ArrayElement + Float + Debug + Display>(
+fn test_activation_large<T: DataTypeRepr + Float + Debug + Display>(
act_type: ActivationType,
in_place: bool,
) {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
index e58fd30f8..13a5abb73 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
@@ -13,6 +12,7 @@ use crate::{
},
cpu::Cpu,
},
+ data_type::DataTypeRepr,
tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend},
};
@@ -22,7 +22,7 @@ enum TransformOrder {
Output,
}
-fn run<T: ArrayElement + Float + Debug, B: Backend>(
+fn run<T: DataTypeRepr + Float + Debug, B: Backend>(
data: &[T],
factors: &[i32],
channel_count: usize,
@@ -59,7 +59,7 @@ fn run<T: ArrayElement + Float + Debug, B: Backend>(
})
}
-fn check<T: ArrayElement + Float + Debug>(tolerance: f64) {
+fn check<T: DataTypeRepr + Float + Debug>(tolerance: f64) {
for (order, in_place) in [
(TransformOrder::Input, false),
(TransformOrder::Input, true),
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
index 4353c0c7a..8f6df0224 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,20 +12,21 @@ use crate::{
},
cpu::Cpu,
},
+ data_type::DataTypeRepr,
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
keys: Box<[T]>,
values: Box<[T]>,
copies: Vec<Copy>,
element_dim: u32,
}
-fn make_data<T: ArrayElement + Float>(
+fn make_data<T: DataTypeRepr + Float>(
max_sequence_length: usize,
element_dim: usize,
) -> (Vec<T>, Vec<T>) {
@@ -40,7 +40,7 @@ fn make_data<T: ArrayElement + Float>(
(keys, values)
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
let context = B::Context::new().expect("Failed to create Context");
let kernel = <<B as Backend>::Kernels as Kernels>::KVCacheUpdateKernel::new(&context, T::data_type())
.expect("Failed to create KVCacheUpdateKernel");
@@ -63,7 +63,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
}
/// Single copy between two different positions.
-fn get_test_data_single_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
+fn get_test_data_single_copy<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
let num_heads = 2usize;
let max_sequence_length = 4usize;
let head_dim = 3usize;
@@ -88,7 +88,7 @@ fn get_test_data_single_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Ve
}
/// Multiple copies.
-fn get_test_data_multi_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
+fn get_test_data_multi_copy<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
let num_heads = 2usize;
let max_sequence_length = 8usize;
let head_dim = 4usize;
@@ -123,7 +123,7 @@ fn get_test_data_multi_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec
}
/// No copies — data should remain unchanged.
-fn get_test_data_no_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
+fn get_test_data_no_copy<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
let num_heads = 2usize;
let max_sequence_length = 4usize;
let head_dim = 3usize;
@@ -145,7 +145,7 @@ fn get_test_data_no_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>
}
/// Larger dimensions to exercise more threads.
-fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
+fn get_test_data_large<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
let num_heads = 8usize;
let max_sequence_length = 64usize;
let head_dim = 128usize;
@@ -179,7 +179,7 @@ fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>)
(input, expected_keys, expected_values)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected_keys: &[T],
expected_values: &[T],
@@ -195,22 +195,22 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
});
}
-fn test_single_copy_internal<T: ArrayElement + Float + Debug + Display>() {
+fn test_single_copy_internal<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected_keys, expected_values) = get_test_data_single_copy::<T>();
test_internal::<T>(&input, &expected_keys, &expected_values, "single_copy");
}
-fn test_multi_copy_internal<T: ArrayElement + Float + Debug + Display>() {
+fn test_multi_copy_internal<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected_keys, expected_values) = get_test_data_multi_copy::<T>();
test_internal::<T>(&input, &expected_keys, &expected_values, "multi_copy");
}
-fn test_no_copy_internal<T: ArrayElement + Float + Debug + Display>() {
+fn test_no_copy_internal<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected_keys, expected_values) = get_test_data_no_copy::<T>();
test_internal::<T>(&input, &expected_keys, &expected_values, "no_copy");
}
-fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {
+fn test_large_internal<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected_keys, expected_values) = get_test_data_large::<T>();
test_internal::<T>(&input, &expected_keys, &expected_values, "large");
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs
index 0b5ef4e9a..cdaad5fb1 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<InputT: ArrayElement + Float, ScaleT: ArrayElement + Float, OutputT: ArrayElement + Float> {
+struct Input<InputT: DataTypeRepr + Float, ScaleT: DataTypeRepr + Float, OutputT: DataTypeRepr + Float> {
qkvg: Box<[OutputT]>,
scales: Box<[ScaleT]>,
batch_size: u32,
@@ -39,10 +38,10 @@ struct Input<InputT: ArrayElement + Float, ScaleT: ArrayElement + Float, OutputT
}
fn get_test_data<
- InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
- OutputT: ArrayElement + Float,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ ScaleT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float,
+ AccumT: DataTypeRepr + Float,
>(
head_offset: u32,
head_count: u32,
@@ -116,10 +115,10 @@ fn get_test_data<
fn get_output<
B: Backend,
- InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
- OutputT: ArrayElement + Float,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ ScaleT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float,
+ AccumT: DataTypeRepr + Float,
>(
input: &Input<InputT, ScaleT, OutputT>
) -> Vec<OutputT> {
@@ -160,10 +159,10 @@ fn get_output<
}
fn test_internal<
- InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
- OutputT: ArrayElement + Float + Debug + Display,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ ScaleT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float + Debug + Display,
+ AccumT: DataTypeRepr + Float,
>(
input: &Input<InputT, ScaleT, OutputT>,
expected: &[OutputT],
@@ -192,10 +191,10 @@ fn test_internal<
}
fn test_q_norm<
- InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
- OutputT: ArrayElement + Float + Debug + Display,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ ScaleT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float + Debug + Display,
+ AccumT: DataTypeRepr + Float,
>() {
for full_layer in [true, false] {
// Normalize Q heads: head_offset=0, head_count=num_q_heads(4)
@@ -205,10 +204,10 @@ fn test_q_norm<
}
fn test_k_norm<
- InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
- OutputT: ArrayElement + Float + Debug + Display,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ ScaleT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float + Debug + Display,
+ AccumT: DataTypeRepr + Float,
>() {
for full_layer in [true, false] {
// Normalize K heads: head_offset=num_q_heads(4), head_count=num_kv_heads(2)
@@ -218,20 +217,20 @@ fn test_k_norm<
}
fn test_v_norm_no_scales<
- InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
- OutputT: ArrayElement + Float + Debug + Display,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ ScaleT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float + Debug + Display,
+ AccumT: DataTypeRepr + Float,
>() {
let (input, expected) = get_test_data::<InputT, ScaleT, OutputT, AccumT>(6, 2, true, false);
test_internal::<InputT, ScaleT, OutputT, AccumT>(&input, &expected);
}
fn test_addressing<
- InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
- OutputT: ArrayElement + Float + Debug + Display,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ ScaleT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float + Debug + Display,
+ AccumT: DataTypeRepr + Float,
>() {
// Test that Q norm only modifies Q heads and leaves K/V untouched
let (input, _expected) = get_test_data::<InputT, ScaleT, OutputT, AccumT>(0, 4, false, true);
@@ -273,10 +272,10 @@ fn test_addressing<
}
fn test_v_addressing<
- InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
- OutputT: ArrayElement + Float + Debug + Display,
- AccumT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ ScaleT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float + Debug + Display,
+ AccumT: DataTypeRepr + Float,
>() {
// Weightless V norm must modify only V heads and leave Q/K untouched.
let (input, _expected) = get_test_data::<InputT, ScaleT, OutputT, AccumT>(6, 2, true, false);
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs
index fd13f9aff..b8beb7b9d 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,6 +12,7 @@ use crate::{
},
cpu::Cpu,
},
+ data_type::DataTypeRepr,
tests::helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},
};
@@ -23,7 +23,7 @@ struct Config {
suffix_length: u32,
}
-fn get_output<T: ArrayElement + Float, B: Backend>(
+fn get_output<T: DataTypeRepr + Float, B: Backend>(
qkvg_data: &[T],
output_data: &[T],
config: &Config,
@@ -53,7 +53,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
buffer_to_vec(&output)
}
-fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {
+fn run_test<T: DataTypeRepr + Float + Debug>(config: &Config) {
let size = (config.suffix_length * config.num_heads * config.head_dim) as usize;
let gate_dim = (config.num_heads * config.head_dim) as usize;
let gate_offset = ((config.num_heads + 2 * config.num_kv_heads) * config.head_dim) as usize;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs b/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs
index e2f45a15e..0746f403f 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs
@@ -6,7 +6,6 @@ use rstest::rstest;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, Context, Kernels,
@@ -20,6 +19,7 @@ use crate::{
},
cpu::Cpu,
},
+ data_type::DataTypeRepr,
tests::{
assert::assert_eq_float,
helpers::{
@@ -41,14 +41,14 @@ fn pattern(
(0..len).map(|index| (index * seed + 9) as u8).collect()
}
-fn floats<T: ArrayElement + Float>(
+fn floats<T: DataTypeRepr + Float>(
len: u32,
value: impl Fn(u32) -> f32,
) -> Vec<T> {
(0..len).map(|index| T::from(value(index)).unwrap()).collect()
}
-fn lookup<B: Backend, T: ArrayElement + Float>(
+fn lookup<B: Backend, T: DataTypeRepr + Float>(
table_kind: EmbeddingTableKind,
quantization: Option<Quantization>,
use_hadamard: bool,
@@ -139,7 +139,7 @@ fn lookup<B: Backend, T: ArrayElement + Float>(
buffer_to_vec::<B, T>(&output)
}
-fn check<T: ArrayElement + Float + Display>(
+fn check<T: DataTypeRepr + Float + Display>(
table_kind: EmbeddingTableKind,
quantization: Option<Quantization>,
use_hadamard: bool,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
index 2d7466022..0ba84c2c2 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
@@ -15,14 +14,14 @@ use crate::{
cpu::Cpu,
},
config::clipping::ClippingBounds,
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend, for_each_non_cpu_backend},
},
};
-struct InterleavedInput<T: ArrayElement + Float> {
+struct InterleavedInput<T: DataTypeRepr + Float> {
fused_up: Box<[T]>,
hadamard_factors: Box<[i32]>,
gated_dim: u32,
@@ -30,7 +29,7 @@ struct InterleavedInput<T: ArrayElement + Float> {
act_type: ActivationType,
}
-fn interleaved_input<T: ArrayElement + Float>(act_type: ActivationType) -> InterleavedInput<T> {
+fn interleaved_input<T: DataTypeRepr + Float>(act_type: ActivationType) -> InterleavedInput<T> {
let gated_dim = 128u32;
let batch_dim = 4u32;
let fused_length = (batch_dim * 2 * gated_dim) as usize;
@@ -47,7 +46,7 @@ fn interleaved_input<T: ArrayElement + Float>(act_type: ActivationType) -> Inter
}
}
-fn run_interleaved<T: ArrayElement + Float, B: Backend>(
+fn run_interleaved<T: DataTypeRepr + Float, B: Backend>(
input: &InterleavedInput<T>,
use_hadamard: bool,
) -> Vec<T> {
@@ -80,7 +79,7 @@ fn run_interleaved<T: ArrayElement + Float, B: Backend>(
buffer_to_vec::<B, T>(&output)
}
-fn interleaved_test<T: ArrayElement + Float + Debug + Display>(act_type: ActivationType) {
+fn interleaved_test<T: DataTypeRepr + Float + Debug + Display>(act_type: ActivationType) {
let eps = if matches!(T::data_type(), DataType::BF16) {
0.02f32
} else {
@@ -184,7 +183,7 @@ fn test_gated_act_mul_interleaved_gelu_exact_f32() {
interleaved_test::<f32>(ActivationType::GELUExact);
}
-struct SeparateInput<T: ArrayElement + Float> {
+struct SeparateInput<T: DataTypeRepr + Float> {
gate_out: Box<[T]>,
per_layer_input: Box<[T]>,
gated_dim: u32,
@@ -194,7 +193,7 @@ struct SeparateInput<T: ArrayElement + Float> {
act_type: ActivationType,
}
-fn separate_input<T: ArrayElement + Float>() -> (SeparateInput<T>, Vec<T>) {
+fn separate_input<T: DataTypeRepr + Float>() -> (SeparateInput<T>, Vec<T>) {
let gate_out = [1.0_f32, 2.0, 3.0, 4.0].into_iter().map(|value| T::from(value).unwrap()).collect::<Vec<_>>();
let per_layer_input = [0.0_f32, 0.0, 10.0, 20.0, 30.0, 40.0, 0.0, 0.0, 50.0, 60.0, 70.0, 80.0]
.into_iter()
@@ -217,7 +216,7 @@ fn separate_input<T: ArrayElement + Float>() -> (SeparateInput<T>, Vec<T>) {
)
}
-fn run_separate<T: ArrayElement + Float, B: Backend>(input: &SeparateInput<T>) -> Vec<T> {
+fn run_separate<T: DataTypeRepr + Float, B: Backend>(input: &SeparateInput<T>) -> Vec<T> {
let context = B::Context::new().expect("create context");
let kernel =
GatedActMul::<B>::full_precision(&context, T::data_type(), false, false, GatedActMulSettings::default())
@@ -245,7 +244,7 @@ fn run_separate<T: ArrayElement + Float, B: Backend>(input: &SeparateInput<T>) -
buffer_to_vec::<B, T>(&output)
}
-fn separate_test<T: ArrayElement + Float + Debug>() {
+fn separate_test<T: DataTypeRepr + Float + Debug>() {
let (input, expected) = separate_input::<T>();
for_each_backend!(|B| {
let output = run_separate::<T, B>(&input);
@@ -263,7 +262,7 @@ fn test_gated_act_mul_separate_bf16() {
separate_test::<bf16>();
}
-fn transformed_interleaved_test<T: ArrayElement + Float + Debug + Display>(
+fn transformed_interleaved_test<T: DataTypeRepr + Float + Debug + Display>(
gate_clipping: ClippingBounds,
value_clipping: ClippingBounds,
) {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs
index 573cc6eca..bd0c4bde2 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs
@@ -1,8 +1,15 @@
use half::bf16;
use uzu_engine_macros::uzu_test;
+#[cfg(backend = "metal")]
+use crate::{
+ backends::{
+ common::kernel::{Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel},
+ metal::Metal,
+ },
+ tests::helpers::buffer_prefix_to_vec,
+};
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -10,17 +17,9 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend},
};
-#[cfg(backend = "metal")]
-use crate::{
- backends::{
- common::kernel::{Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel},
- metal::Metal,
- },
- tests::helpers::buffer_prefix_to_vec,
-};
#[cfg(backend = "metal")]
fn run_conv_update<B: Backend>(
@@ -60,7 +59,7 @@ fn run_conv_update<B: Backend>(
(out, new_state)
}
-fn run_delta_net_update<B: Backend, T: ArrayElement>(
+fn run_delta_net_update<B: Backend, T: DataTypeRepr>(
in_proj: &[T],
a_log: &[f32],
dt_bias: &[f32],
@@ -108,7 +107,7 @@ fn run_delta_net_update<B: Backend, T: ArrayElement>(
(out, new_state)
}
-fn to_activation<T: ArrayElement>(values: &[f32]) -> Vec<T> {
+fn to_activation<T: DataTypeRepr>(values: &[f32]) -> Vec<T> {
values.iter().map(|&value| <T as num_traits::NumCast>::from(value).expect("input to activation dtype")).collect()
}
@@ -248,7 +247,7 @@ fn test_delta_net_conv_scan() {
// DeltaNetUpdate (decode)
// The reference is the CPU f32 kernel on the same rounded inputs; a_log, dt_bias, norm weight and state stay f32
-fn test_delta_net_update_impl<T: ArrayElement>(
+fn test_delta_net_update_impl<T: DataTypeRepr>(
num_v_heads: usize,
num_k_heads: usize,
head_k_dim: usize,
@@ -318,7 +317,7 @@ fn test_delta_net_update_qwen35_shapes_bf16() {
// DeltaNetPrefill + NormGate
-fn run_prefill_with_norm_gate_typed<B: Backend, T: ArrayElement>(
+fn run_prefill_with_norm_gate_typed<B: Backend, T: DataTypeRepr>(
in_proj: &[T],
a_log: &[f32],
dt_bias: &[f32],
@@ -418,7 +417,7 @@ fn run_prefill_with_norm_gate_typed<B: Backend, T: ArrayElement>(
(out, state)
}
-fn test_prefill_norm_gate_impl<T: ArrayElement>(
+fn test_prefill_norm_gate_impl<T: DataTypeRepr>(
num_v_heads: usize,
num_k_heads: usize,
head_k_dim: usize,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
index 52ff22aec..3b15fa683 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
@@ -8,11 +8,11 @@ use num_traits::Float;
use uzu_engine_macros::uzu_bench;
use crate::{
- array::ArrayElement,
backends::{
common::{Backend, Context, Kernels, kernel::BuildTreeOutKernel},
metal::Metal,
},
+ data_type::DataTypeRepr,
tests::{
helpers::{create_buffer, create_buffer_with_data},
matmul::iter_encode_loop_named,
@@ -44,7 +44,7 @@ struct TreeOutBuffers {
o: <Metal as Backend>::GlobalBuffer,
}
-fn make_buffers<T: ArrayElement + Float>(
+fn make_buffers<T: DataTypeRepr + Float>(
context: &<Metal as Backend>::Context,
batch_size: usize,
tree_size: usize,
@@ -79,7 +79,7 @@ fn bench_build_tree_out(c: &mut Criterion) {
bench_build_tree_out_type::<f32>(c, &context, "F32");
}
-fn bench_build_tree_out_type<T: ArrayElement + Float>(
+fn bench_build_tree_out_type<T: DataTypeRepr + Float>(
c: &mut Criterion,
context: &<Metal as Backend>::Context,
data_type_name: &str,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
index 698ab0b97..113f49cf8 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -14,6 +13,7 @@ use crate::{
cpu::Cpu,
metal::Metal,
},
+ data_type::DataTypeRepr,
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
@@ -44,7 +44,7 @@ const BUILD_TREE_OUT_PATHS: &[(&str, bool, bool)] = &[
("MXU/H0Direct/Rows16_VCols32_SG4", true, false),
];
-fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {
+fn make_inputs<T: DataTypeRepr + Float>(shape: Shape) -> Inputs<T> {
let q_len = shape.batch_size * shape.tree_size * shape.qk_heads * shape.head_k_dim;
let prefix_len = shape.batch_size * shape.tree_size * shape.value_heads;
let qkd_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.tree_size;
@@ -62,7 +62,7 @@ fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {
}
}
-fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(
+fn run_build_tree_out<B: Backend, T: DataTypeRepr + Float>(
shape: Shape,
inputs: &Inputs<T>,
use_h0: bool,
@@ -108,7 +108,7 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(
buffer_to_vec(&o)
}
-fn check_shape<T: ArrayElement + Float + std::fmt::Display>(
+fn check_shape<T: DataTypeRepr + Float + std::fmt::Display>(
shape: Shape,
eps: f32,
) {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs
index b6e37ef4c..45ac4d086 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs
@@ -9,8 +9,13 @@ use num_traits::Float;
use uzu_engine_macros::uzu_bench;
use uzu_engine_macros::uzu_test;
+#[cfg(backend = "metal")]
+use crate::{
+ backends::metal::Metal,
+ data_type::DataType,
+ tests::{cold_pool::ColdPool, helpers::buffer_size_bytes, matmul::iter_encode_loop_named},
+};
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -18,17 +23,12 @@ use crate::{
},
cpu::Cpu,
},
+ data_type::DataTypeRepr,
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-#[cfg(backend = "metal")]
-use crate::{
- backends::metal::Metal,
- data_type::DataType,
- tests::{cold_pool::ColdPool, helpers::buffer_size_bytes, matmul::iter_encode_loop_named},
-};
const TREE_SIZE: usize = 128;
const HEAD_DIM: usize = 128;
@@ -36,7 +36,7 @@ const PATH: &[u32] = &[0, 1, 3, 7, 15, 31, 47, 63, 79, 95, 111, 119, 123, 125, 1
const TEST_NUM_V_HEADS: usize = 6;
const TEST_NUM_K_HEADS: usize = 2;
-fn run<B: Backend, T: ArrayElement + Float>(accepted_indices: &[u32]) -> Vec<f32> {
+fn run<B: Backend, T: DataTypeRepr + Float>(accepted_indices: &[u32]) -> Vec<f32> {
let context = B::Context::new().expect("context");
let kernel = <<B as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(
&context,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
index 502c5a5da..0186e3d30 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -14,7 +13,7 @@ use crate::{
cpu::Cpu,
metal::Metal,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
@@ -53,7 +52,7 @@ fn make_inputs(tree_size: usize) -> Inputs {
}
}
-fn get_output<B: Backend, T: ArrayElement + Float>(
+fn get_output<B: Backend, T: DataTypeRepr + Float>(
q: &[T],
k: &[T],
trie: &[u32],
@@ -113,7 +112,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
(buffer_to_vec(&a_packed), buffer_to_vec(&qkd), buffer_to_vec(&a_inv), buffer_to_vec(&kh0))
}
-fn check_type<T: ArrayElement + Float>(
+fn check_type<T: DataTypeRepr + Float>(
inputs: &Inputs,
tree_size: usize,
scale: f32,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
index ffde218d7..a6530546e 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
@@ -4,7 +4,6 @@ use half::bf16;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -12,7 +11,7 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
@@ -86,7 +85,7 @@ const CASES: &[SolveCase] = &[
},
];
-fn run_case<B: Backend, T: ArrayElement + Copy>(
+fn run_case<B: Backend, T: DataTypeRepr + Copy>(
case: SolveCase,
bv: u32,
use_h0: bool,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/logit_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/logit_transform_test.rs
index 519858c70..5d1568df2 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/logit_transform_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/logit_transform_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-fn get_output<T: ArrayElement + Float, B: Backend>(
+fn get_output<T: DataTypeRepr + Float, B: Backend>(
logits: &[T],
scale: f32,
soft_cap: Option<f32>,
@@ -38,7 +37,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
buffer_to_vec::<B, T>(&logits_buffer)
}
-fn test_logit_transform<T: ArrayElement + Float + Display>(
+fn test_logit_transform<T: DataTypeRepr + Float + Display>(
scale: f32,
soft_cap: Option<f32>,
) {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
index 1eb047336..b8bf79104 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
@@ -5,8 +5,9 @@ use num_traits::Float;
use rstest::rstest;
use uzu_engine_macros::uzu_test;
+#[cfg(backend = "metal")]
+use crate::{backends::metal::Metal, tests::matmul::run_quant_cpu};
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
@@ -18,16 +19,15 @@ use crate::{
},
cpu::Cpu,
},
+ data_type::DataTypeRepr,
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
matmul::{QuantBuffers, QuantInput},
},
};
-#[cfg(backend = "metal")]
-use crate::{backends::metal::Metal, tests::matmul::run_quant_cpu};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
a: Box<[T]>,
b: Box<[T]>,
m: usize,
@@ -37,7 +37,7 @@ struct Input<T: ArrayElement + Float> {
soft_cap: Option<f32>,
}
-fn get_test_data<T: ArrayElement + Float>(
+fn get_test_data<T: DataTypeRepr + Float>(
m: usize,
k: usize,
n: usize,
@@ -60,7 +60,7 @@ fn get_test_data<T: ArrayElement + Float>(
}
// Encode one GEMV (dense, or a per-row B-row gather when `gather_indices` is set) and copy out.
-fn run_gemv<B: Backend, T: ArrayElement + Float>(
+fn run_gemv<B: Backend, T: DataTypeRepr + Float>(
context: &B::Context,
a: impl BufferRef<Backend = B>,
b: MatmulB<impl BufferRef<Backend = B>>,
@@ -104,7 +104,7 @@ fn run_gemv<B: Backend, T: ArrayElement + Float>(
buffer_to_vec::<B, T>(&d)
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let a = create_buffer_with_data::<B, T>(&context, &input.a);
let weights = create_buffer_with_data::<B, T>(&context, &input.b);
@@ -123,7 +123,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
)
}
-fn test<T: ArrayElement + Float + Debug + Display>(
+fn test<T: DataTypeRepr + Float + Debug + Display>(
m: usize,
k: usize,
n: usize,
@@ -198,7 +198,7 @@ fn gemv_f32(
test::<f32>(m, k, n, 0.01);
}
-fn assert_gather<T: ArrayElement + Float + Debug + Display>(
+fn assert_gather<T: DataTypeRepr + Float + Debug + Display>(
dense: &[T],
gather: &[T],
ids: &[u32],
@@ -232,7 +232,7 @@ macro_rules! check_gather {
}};
}
-fn fp_gather_case<T: ArrayElement + Float + Debug + Display>(
+fn fp_gather_case<T: DataTypeRepr + Float + Debug + Display>(
soft_cap: Option<f32>,
eps: f32,
) {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
index 3f03a49ab..0370f311d 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
@@ -4,12 +4,12 @@ use num_traits::Float;
use uzu_engine_macros::uzu_bench;
use crate::{
- array::ArrayElement,
backends::common::{
Backend, Context,
gpu_types::quantization_method::QuantizationMethod,
kernel::{Kernels, matmul::MatmulKernel},
},
+ data_type::DataTypeRepr,
tests::{
cold_pool::ColdPool,
helpers::for_each_backend,
@@ -18,7 +18,7 @@ use crate::{
},
};
-fn bench_gemv_typed<B: Backend, T: ArrayElement + Float>(
+fn bench_gemv_typed<B: Backend, T: DataTypeRepr + Float>(
c: &mut Criterion,
context: &B::Context,
label: &str,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs b/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs
index 5869a54cf..a9b2b88af 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,7 +12,7 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend, for_each_non_cpu_backend},
@@ -22,9 +21,9 @@ use crate::{
fn get_output<
B: Backend,
- InputT: ArrayElement + Float,
- AffineT: ArrayElement + Float,
- OutputT: ArrayElement + Float,
+ InputT: DataTypeRepr + Float,
+ AffineT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float,
>(
input: &[InputT],
scales: Option<&[AffineT]>,
@@ -81,9 +80,9 @@ fn get_output<
}
fn test_internal<
- InputT: ArrayElement + Float,
- AffineT: ArrayElement + Float,
- OutputT: ArrayElement + Float + Debug + Display,
+ InputT: DataTypeRepr + Float,
+ AffineT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float + Debug + Display,
>(
has_scales: bool,
full_layer: bool,
@@ -138,9 +137,9 @@ fn test_internal<
}
fn test_normalization<
- InputT: ArrayElement + Float,
- AffineT: ArrayElement + Float,
- OutputT: ArrayElement + Float + Debug + Display,
+ InputT: DataTypeRepr + Float,
+ AffineT: DataTypeRepr + Float,
+ OutputT: DataTypeRepr + Float + Debug + Display,
>() {
for has_scales in [true, false] {
for full_layer in [true, false] {
@@ -150,7 +149,7 @@ fn test_normalization<
}
// The fused transform has to match plain normalization followed by the standalone input RHT
-fn test_hadamard<T: ArrayElement + Float + Debug + Display>() {
+fn test_hadamard<T: DataTypeRepr + Float + Debug + Display>() {
let batch_size = 2u32;
let element_count = 64u32;
let epsilon = 1e-6f32;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/pooling/pooling_mean_test.rs b/crates/uzu-engine/unit/backends/common/kernel/pooling/pooling_mean_test.rs
index c883ec212..ecd4101ae 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/pooling/pooling_mean_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/pooling/pooling_mean_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,21 +12,21 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
input: Box<[T]>,
seq_len: u32,
hidden_dim: u32,
batch_size: u32,
}
-fn get_test_data<T: ArrayElement + Float>(
+fn get_test_data<T: DataTypeRepr + Float>(
batch_size: u32,
seq_len: u32,
hidden_dim: u32,
@@ -46,7 +45,7 @@ fn get_test_data<T: ArrayElement + Float>(
(input, expected)
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let kernel = <<B as Backend>::Kernels as Kernels>::PoolingMeanKernel::new(&context, T::data_type())
@@ -64,7 +63,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
buffer_to_vec(&output)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
batch_size: u32,
seq_len: u32,
hidden_dim: u32,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs
index 862b82951..87dbdd7ac 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs
@@ -2,7 +2,6 @@ use half::bf16;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -10,6 +9,7 @@ use crate::{
},
cpu::Cpu,
},
+ data_type::DataTypeRepr,
tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
};
diff --git a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_decode_test.rs b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_decode_test.rs
index a0ebeb5ac..3887ef5bc 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_decode_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_decode_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
in_proj: Box<[T]>,
w: Box<[f32]>,
b: Option<Box<[f32]>>,
@@ -32,7 +31,7 @@ struct Input<T: ArrayElement + Float> {
model_dim: u32,
}
-fn get_output<T: ArrayElement + Float, B: Backend>(
+fn get_output<T: DataTypeRepr + Float, B: Backend>(
input: &Input<T>,
state_in_place: bool,
) -> (Vec<T>, Vec<T>) {
@@ -88,7 +87,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
(buffer_to_vec(&out), buffer_to_vec(&next_state))
}
-fn get_test_data_basic<T: ArrayElement + Float>(
+fn get_test_data_basic<T: DataTypeRepr + Float>(
model_dim: usize,
kernel_size: usize,
has_bias: bool,
@@ -155,7 +154,7 @@ fn get_test_data_basic<T: ArrayElement + Float>(
(input, out, next_state)
}
-fn get_test_data_edge<T: ArrayElement + Float>(
+fn get_test_data_edge<T: DataTypeRepr + Float>(
model_dim: usize,
kernel_size: usize,
has_bias: bool,
@@ -216,7 +215,7 @@ fn get_test_data_edge<T: ArrayElement + Float>(
(input, out, next_state)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected_out: &[T],
expected_state: &[T],
@@ -252,7 +251,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
});
}
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
for state_in_place in [false, true] {
let (input, expected_out, expected_state) = get_test_data_basic::<T>(64, 4, has_bias);
@@ -261,7 +260,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {
}
}
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
for state_in_place in [false, true] {
let (input, expected_out, expected_state) = get_test_data_basic::<T>(256, 4, has_bias);
@@ -270,7 +269,7 @@ fn test_large<T: ArrayElement + Float + Debug + Display>() {
}
}
-fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {
+fn test_edge_small<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
for state_in_place in [false, true] {
let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 2, has_bias);
@@ -279,7 +278,7 @@ fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {
}
}
-fn test_edge_kernel<T: ArrayElement + Float + Debug + Display>() {
+fn test_edge_kernel<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
for state_in_place in [false, true] {
let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 1, has_bias);
diff --git a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_pack_test.rs b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_pack_test.rs
index 31f430b10..eb2324804 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_pack_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_pack_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
state_in: Box<[T]>,
in_proj: Box<[T]>,
state_stride: u32,
@@ -29,7 +28,7 @@ struct Input<T: ArrayElement + Float> {
model_dim: u32,
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPackKernel::new(&context, T::data_type())
@@ -66,7 +65,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
/// - columns [0..model_dim) = pre_gate
/// - columns [2*model_dim..3*model_dim) = x_in
/// padded (output): [(state_stride + suffix_len), model_dim]
-fn get_input<T: ArrayElement + Float>(
+fn get_input<T: DataTypeRepr + Float>(
model_dim: usize,
state_stride: usize,
suffix_len: usize,
@@ -106,7 +105,7 @@ fn get_input<T: ArrayElement + Float>(
(input, output)
}
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.02f32
} else {
@@ -124,7 +123,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {
});
}
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.02f32
} else {
@@ -142,7 +141,7 @@ fn test_large<T: ArrayElement + Float + Debug + Display>() {
});
}
-fn test_edge_single_token<T: ArrayElement + Float + Debug + Display>() {
+fn test_edge_single_token<T: DataTypeRepr + Float + Debug + Display>() {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.02f32
} else {
@@ -160,7 +159,7 @@ fn test_edge_single_token<T: ArrayElement + Float + Debug + Display>() {
});
}
-fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {
+fn test_edge_small<T: DataTypeRepr + Float + Debug + Display>() {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.02f32
} else {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_prefill_test.rs b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_prefill_test.rs
index ceaa191f2..dd9f5a192 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_prefill_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_prefill_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
padded: Box<[T]>,
in_proj: Box<[T]>,
w: Box<[f32]>,
@@ -32,7 +31,7 @@ struct Input<T: ArrayElement + Float> {
model_dim: u32,
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
let context = B::Context::new().expect("Failed to create Context");
let has_bias = input.b.is_some();
@@ -75,7 +74,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
(buffer_to_vec(&out), buffer_to_vec(&state_out))
}
-fn get_test_data_basic<T: ArrayElement + Float>(
+fn get_test_data_basic<T: DataTypeRepr + Float>(
model_dim: usize,
suffix_len: usize,
kernel_size: usize,
@@ -139,7 +138,7 @@ fn get_test_data_basic<T: ArrayElement + Float>(
(input, out, state_out)
}
-fn get_test_data_edge<T: ArrayElement + Float>(
+fn get_test_data_edge<T: DataTypeRepr + Float>(
model_dim: usize,
suffix_len: usize,
kernel_size: usize,
@@ -199,7 +198,7 @@ fn get_test_data_edge<T: ArrayElement + Float>(
(input, out, state_out)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected_out: &[T],
expected_state: &[T],
@@ -234,28 +233,28 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
});
}
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
let (input, expected_out, expected_state) = get_test_data_basic::<T>(64, 4, 4, has_bias);
test_internal(&input, &expected_out, &expected_state);
}
}
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
let (input, expected_out, expected_state) = get_test_data_basic::<T>(256, 16, 4, has_bias);
test_internal(&input, &expected_out, &expected_state);
}
}
-fn test_edge_single_token<T: ArrayElement + Float + Debug + Display>() {
+fn test_edge_single_token<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 1, 4, has_bias);
test_internal(&input, &expected_out, &expected_state);
}
}
-fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {
+fn test_edge_small<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 2, 2, has_bias);
test_internal(&input, &expected_out, &expected_state);
diff --git a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_trie_test.rs b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_trie_test.rs
index a7a51f846..0a9366c6d 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_trie_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_trie_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
in_proj: Box<[T]>,
w: Box<[f32]>,
b: Option<Box<[f32]>>,
@@ -33,7 +32,7 @@ struct Input<T: ArrayElement + Float> {
model_dim: u32,
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
let context = B::Context::new().expect("Failed to create Context");
let has_bias = input.b.is_some();
@@ -89,7 +88,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
}
/// Linear chain: node 0 has parent -1 (root), node i has parent i-1.
-fn get_test_data_basic<T: ArrayElement + Float>(
+fn get_test_data_basic<T: DataTypeRepr + Float>(
model_dim: usize,
kernel_size: usize,
suffix_len: usize,
@@ -161,7 +160,7 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
/// Branching trie: node 0 is root child (-1), nodes 1 and 2 both parent from 0.
-fn get_test_data_branching<T: ArrayElement + Float>(
+fn get_test_data_branching<T: DataTypeRepr + Float>(
model_dim: usize,
kernel_size: usize,
has_bias: bool,
@@ -229,7 +228,7 @@ fn get_test_data_branching<T: ArrayElement + Float>(
}
/// Edge case: single node with parent -1.
-fn get_test_data_edge<T: ArrayElement + Float>(
+fn get_test_data_edge<T: DataTypeRepr + Float>(
model_dim: usize,
kernel_size: usize,
has_bias: bool,
@@ -293,7 +292,7 @@ fn get_test_data_edge<T: ArrayElement + Float>(
(input, out, suffix_state)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected_out: &[T],
expected_state: &[T],
@@ -328,35 +327,35 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
});
}
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
let (input, expected_out, expected_state) = get_test_data_basic::<T>(64, 4, 3, has_bias);
test_internal(&input, &expected_out, &expected_state);
}
}
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
let (input, expected_out, expected_state) = get_test_data_basic::<T>(256, 4, 3, has_bias);
test_internal(&input, &expected_out, &expected_state);
}
}
-fn test_branching<T: ArrayElement + Float + Debug + Display>() {
+fn test_branching<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
let (input, expected_out, expected_state) = get_test_data_branching::<T>(64, 4, has_bias);
test_internal(&input, &expected_out, &expected_state);
}
}
-fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {
+fn test_edge_small<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 2, has_bias);
test_internal(&input, &expected_out, &expected_state);
}
}
-fn test_edge_kernel<T: ArrayElement + Float + Debug + Display>() {
+fn test_edge_kernel<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [false, true] {
let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 1, has_bias);
test_internal(&input, &expected_out, &expected_state);
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs
index e5a284ec2..5406e67ab 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
x: Box<[T]>,
w: Box<[T]>,
b: Option<Box<[T]>>,
@@ -36,14 +35,14 @@ struct Input<T: ArrayElement + Float> {
state_in_place: bool,
}
-struct Output<T: ArrayElement + Float> {
+struct Output<T: DataTypeRepr + Float> {
x_out: Vec<T>,
b_out: Vec<T>,
c_out: Vec<T>,
next_state: Vec<T>,
}
-fn get_input<T: ArrayElement + Float>(
+fn get_input<T: DataTypeRepr + Float>(
kernel_size: u32,
num_channels: u32,
inner_dim: u32,
@@ -90,7 +89,7 @@ fn get_input<T: ArrayElement + Float>(
}
}
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
let context = B::Context::new().expect("Failed to create Context");
let has_bias = input.b.is_some();
let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(
@@ -180,7 +179,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
}
}
-fn get_test_data<T: ArrayElement + Float>(
+fn get_test_data<T: DataTypeRepr + Float>(
kernel_size: u32,
num_channels: u32,
inner_dim: u32,
@@ -204,7 +203,7 @@ fn get_test_data<T: ArrayElement + Float>(
(input, expected)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected: &Output<T>,
label: &str,
@@ -248,7 +247,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
}
// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
for state_in_place in [true, false] {
for has_bias in [true, false] {
let label = format!("basic(bias={has_bias},in_place={state_in_place})");
@@ -259,23 +258,23 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {
}
// kernel_size=2 (state_stride=1, minimal state)
-fn test_small_kernel<T: ArrayElement + Float + Debug + Display>() {
+fn test_small_kernel<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 1, true, true, ActivationType::SILU);
test_internal(&input, &expected, "small_kernel");
}
// Larger dimensions closer to real usage
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 1, true, true, ActivationType::SILU);
test_internal(&input, &expected, "large");
}
-fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {
+fn test_identity_activation<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, false, true, ActivationType::IDENTITY);
test_internal(&input, &expected, "identity_activation");
}
-fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {
+fn test_gelu_activation<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELUApprox);
test_internal(&input, &expected, "gelu_activation");
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs
index f46798e6b..657d2c1d2 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
state_in: Box<[T]>,
x: Box<[T]>,
state_stride: u32,
@@ -29,7 +28,7 @@ struct Input<T: ArrayElement + Float> {
num_channels: u32,
}
-fn get_input<T: ArrayElement + Float>(
+fn get_input<T: DataTypeRepr + Float>(
state_stride: u32,
row_stride: u32,
suffix_len: u32,
@@ -51,7 +50,7 @@ fn get_input<T: ArrayElement + Float>(
}
}
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type(), T::data_type())
.expect("Failed to create Conv1dPackKernel");
@@ -79,7 +78,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
buffer_to_vec(&padded)
}
-fn get_test_data<T: ArrayElement + Float>(
+fn get_test_data<T: DataTypeRepr + Float>(
state_stride: u32,
row_stride: u32,
suffix_len: u32,
@@ -90,7 +89,7 @@ fn get_test_data<T: ArrayElement + Float>(
(input, expected)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected: &[T],
label: &str,
@@ -108,31 +107,31 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
});
}
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
// 4 channels, state_stride=3 (kernel_size=4), 2 tokens
let (input, expected) = get_test_data::<T>(3, 4, 2, 4);
test_internal(&input, &expected, "basic");
}
-fn test_single_token<T: ArrayElement + Float + Debug + Display>() {
+fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {
// 8 channels, state_stride=1, 1 token
let (input, expected) = get_test_data::<T>(1, 8, 1, 8);
test_internal(&input, &expected, "single_token");
}
-fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {
+fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {
// Minimal state (kernel_size=2), many tokens
let (input, expected) = get_test_data::<T>(1, 4, 8, 4);
test_internal(&input, &expected, "many_tokens");
}
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
// Larger dimensions closer to real usage
let (input, expected) = get_test_data::<T>(3, 128, 16, 128);
test_internal(&input, &expected, "large");
}
-fn test_channels_lt_row_stride<T: ArrayElement + Float + Debug + Display>() {
+fn test_channels_lt_row_stride<T: DataTypeRepr + Float + Debug + Display>() {
// num_channels < row_stride (padded channels that kernel doesn't write)
let (input, expected) = get_test_data::<T>(2, 8, 3, 6);
test_internal(&input, &expected, "channels_lt_row_stride");
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs
index 5706500ba..22f221edc 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
padded: Box<[T]>,
w: Box<[T]>,
b: Option<Box<[T]>>,
@@ -34,14 +33,14 @@ struct Input<T: ArrayElement + Float> {
activation_type: ActivationType,
}
-struct Output<T: ArrayElement + Float> {
+struct Output<T: DataTypeRepr + Float> {
x_out: Vec<T>,
b_out: Vec<T>,
c_out: Vec<T>,
state_out: Vec<T>,
}
-fn get_input<T: ArrayElement + Float>(
+fn get_input<T: DataTypeRepr + Float>(
kernel_size: u32,
num_channels: u32,
inner_dim: u32,
@@ -84,7 +83,7 @@ fn get_input<T: ArrayElement + Float>(
}
}
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
let context = B::Context::new().expect("Failed to create Context");
let has_bias = input.b.is_some();
let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&context, T::data_type(), has_bias)
@@ -133,7 +132,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
}
}
-fn get_test_data<T: ArrayElement + Float>(
+fn get_test_data<T: DataTypeRepr + Float>(
kernel_size: u32,
num_channels: u32,
inner_dim: u32,
@@ -147,7 +146,7 @@ fn get_test_data<T: ArrayElement + Float>(
(input, expected)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected: &Output<T>,
label: &str,
@@ -191,7 +190,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
}
// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4, suffix_len=2
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
for has_bias in [true, false] {
let label = format!("basic(bias={has_bias})");
let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, has_bias, ActivationType::SILU);
@@ -200,35 +199,35 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {
}
// kernel_size=2 (state_stride=1, minimal state)
-fn test_small_kernel<T: ArrayElement + Float + Debug + Display>() {
+fn test_small_kernel<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 3, true, ActivationType::SILU);
test_internal(&input, &expected, "small_kernel");
}
// Single token suffix
-fn test_single_token<T: ArrayElement + Float + Debug + Display>() {
+fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, ActivationType::SILU);
test_internal(&input, &expected, "single_token");
}
// Many tokens
-fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {
+fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 16, true, ActivationType::SILU);
test_internal(&input, &expected, "many_tokens");
}
// Larger dimensions closer to real usage
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 8, true, ActivationType::SILU);
test_internal(&input, &expected, "large");
}
-fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {
+fn test_identity_activation<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, false, ActivationType::IDENTITY);
test_internal(&input, &expected, "identity_activation");
}
-fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {
+fn test_gelu_activation<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELUApprox);
test_internal(&input, &expected, "gelu_activation");
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs
index e3e39e628..205b5df42 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs
@@ -5,19 +5,18 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
gpu_types::activation_type::ActivationType, kernel::SSDPrefillKernel,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
x: Box<[T]>,
dt: Box<[T]>,
b: Box<[T]>,
@@ -36,12 +35,12 @@ struct Input<T: ArrayElement + Float> {
state_strides: [usize; 3],
}
-struct Output<T: ArrayElement + Float> {
+struct Output<T: DataTypeRepr + Float> {
y: Vec<T>,
state: Vec<T>,
}
-fn get_input<T: ArrayElement + Float>(
+fn get_input<T: DataTypeRepr + Float>(
suffix_len: usize,
num_heads: usize,
head_dim: usize,
@@ -88,7 +87,7 @@ fn get_input<T: ArrayElement + Float>(
}
}
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
let context = B::Context::new().expect("Failed to create Context");
let total_x = input.suffix_len * input.num_heads * input.head_dim;
@@ -138,7 +137,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
}
}
-fn reference_output<T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+fn reference_output<T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
let safe_group = input.group_size.max(1) as usize;
let mut state = input.state.to_vec();
let mut y = vec![T::zero(); input.suffix_len * input.num_heads * input.head_dim];
@@ -177,7 +176,7 @@ fn reference_output<T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
}
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected: &Output<T>,
label: &str,
@@ -218,7 +217,7 @@ fn test_shape(
group_size: u32,
label: &str,
) {
- fn run<T: ArrayElement + Float + Debug + Display>(
+ fn run<T: DataTypeRepr + Float + Debug + Display>(
suffix_len: usize,
num_heads: usize,
head_dim: usize,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs
index e72fc14d5..6cd0d5d63 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
input: Box<[T]>,
z_bias: Box<[T]>,
suffix_length: u32,
@@ -30,13 +29,13 @@ struct Input<T: ArrayElement + Float> {
num_heads: u32,
}
-struct Output<T: ArrayElement + Float> {
+struct Output<T: DataTypeRepr + Float> {
conv_out: Vec<T>,
z_out: Vec<T>,
dt_out: Vec<T>,
}
-fn get_input<T: ArrayElement + Float>(
+fn get_input<T: DataTypeRepr + Float>(
suffix_length: u32,
conv_dim: u32,
inner_dim: u32,
@@ -59,7 +58,7 @@ fn get_input<T: ArrayElement + Float>(
}
}
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
let context = B::Context::new().expect("Failed to create Context");
let kernel = <<B as Backend>::Kernels as Kernels>::SplitInProjKernel::new(&context, T::data_type())
.expect("Failed to create SplitInProjKernel");
@@ -98,7 +97,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
}
}
-fn get_test_data<T: ArrayElement + Float>(
+fn get_test_data<T: DataTypeRepr + Float>(
suffix_length: u32,
conv_dim: u32,
inner_dim: u32,
@@ -109,7 +108,7 @@ fn get_test_data<T: ArrayElement + Float>(
(input, expected)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected: &Output<T>,
label: &str,
@@ -147,31 +146,31 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
}
// conv_dim=8, inner_dim=4, num_heads=2, suffix_length=2
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(2, 8, 4, 2);
test_internal(&input, &expected, "basic");
}
// Single token
-fn test_single_token<T: ArrayElement + Float + Debug + Display>() {
+fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(1, 8, 4, 2);
test_internal(&input, &expected, "single_token");
}
// Many tokens
-fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {
+fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(16, 8, 4, 2);
test_internal(&input, &expected, "many_tokens");
}
// Larger dimensions closer to real usage
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(8, 128, 64, 32);
test_internal(&input, &expected, "large");
}
// Minimal dimensions
-fn test_minimal<T: ArrayElement + Float + Debug + Display>() {
+fn test_minimal<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(1, 1, 1, 1);
test_internal(&input, &expected, "minimal");
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs
index f1cd7b71f..508e1ddb5 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
x: Box<[T]>,
dt: Box<[T]>,
b: Box<[T]>,
@@ -36,12 +35,12 @@ struct Input<T: ArrayElement + Float> {
state_in_place: bool,
}
-struct Output<T: ArrayElement + Float> {
+struct Output<T: DataTypeRepr + Float> {
y: Vec<T>,
next_state: Vec<T>,
}
-fn get_input<T: ArrayElement + Float>(
+fn get_input<T: DataTypeRepr + Float>(
bsz: u32,
h: u32,
dh: u32,
@@ -81,7 +80,7 @@ fn get_input<T: ArrayElement + Float>(
}
}
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {
+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {
let context = B::Context::new().expect("Failed to create Context");
let kernel =
<<B as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&context, T::data_type(), input.state_in_place)
@@ -176,7 +175,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
}
}
-fn get_test_data<T: ArrayElement + Float>(
+fn get_test_data<T: DataTypeRepr + Float>(
bsz: u32,
h: u32,
dh: u32,
@@ -189,7 +188,7 @@ fn get_test_data<T: ArrayElement + Float>(
(input, expected)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected: &Output<T>,
label: &str,
@@ -221,37 +220,37 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
}
// bsz=1, h=4, dh=3, g=2, n=8 (matches existing Metal test)
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, false);
test_internal(&input, &expected, "basic");
}
// bsz=2 multi-batch
-fn test_multi_batch<T: ArrayElement + Float + Debug + Display>() {
+fn test_multi_batch<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(2, 4, 3, 2, 8, false);
test_internal(&input, &expected, "multi_batch");
}
// state_in_place=true
-fn test_state_in_place<T: ArrayElement + Float + Debug + Display>() {
+fn test_state_in_place<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, true);
test_internal(&input, &expected, "state_in_place");
}
// Larger dimensions closer to real usage
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(1, 8, 4, 4, 16, false);
test_internal(&input, &expected, "large");
}
// Minimal: single head, single group, dh=1, n=1
-fn test_minimal<T: ArrayElement + Float + Debug + Display>() {
+fn test_minimal<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(1, 1, 1, 1, 1, false);
test_internal(&input, &expected, "minimal");
}
// g == h (each head is its own group)
-fn test_group_per_head<T: ArrayElement + Float + Debug + Display>() {
+fn test_group_per_head<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(1, 4, 2, 4, 8, false);
test_internal(&input, &expected, "group_per_head");
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/tensor_add_bias_test.rs b/crates/uzu-engine/unit/backends/common/kernel/tensor_add_bias_test.rs
index 4b8970125..f053c16aa 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/tensor_add_bias_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/tensor_add_bias_test.rs
@@ -5,15 +5,15 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
kernel::TensorAddBiasKernel,
},
+ data_type::DataTypeRepr,
tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
input: Box<[T]>,
bias: Box<[T]>,
output: Box<[T]>,
@@ -21,7 +21,7 @@ struct Input<T: ArrayElement + Float> {
length: u32,
}
-fn get_test_data<T: ArrayElement + Float>(in_place: bool) -> (Input<T>, Vec<T>) {
+fn get_test_data<T: DataTypeRepr + Float>(in_place: bool) -> (Input<T>, Vec<T>) {
let length = 1025usize;
let num_cols = 129usize;
@@ -55,7 +55,7 @@ fn get_test_data<T: ArrayElement + Float>(in_place: bool) -> (Input<T>, Vec<T>)
(input, expected)
}
-fn get_output<T: ArrayElement + Float, B: Backend>(
+fn get_output<T: DataTypeRepr + Float, B: Backend>(
input: &Input<T>,
in_place: bool,
) -> Vec<T> {
@@ -91,7 +91,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
buffer_to_vec::<B, T>(&output_buffer)
}
-fn test<T: ArrayElement + Float + Debug>(in_place: bool) {
+fn test<T: DataTypeRepr + Float + Debug>(in_place: bool) {
let (input, expected) = get_test_data::<T>(in_place);
for_each_backend!(|B| {
let output = get_output::<T, B>(&input, in_place);
diff --git a/crates/uzu-engine/unit/backends/common/kernel/tensor_add_scale_test.rs b/crates/uzu-engine/unit/backends/common/kernel/tensor_add_scale_test.rs
index 36b50dc28..bf59aae8c 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/tensor_add_scale_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/tensor_add_scale_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,14 +12,14 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
input: Box<[T]>,
bias: Box<[T]>,
num_cols: u32,
@@ -28,7 +27,7 @@ struct Input<T: ArrayElement + Float> {
scale: f32,
}
-fn get_input<T: ArrayElement + Float>(
+fn get_input<T: DataTypeRepr + Float>(
num_rows: usize,
num_cols: usize,
) -> (Vec<T>, Vec<T>) {
@@ -38,7 +37,7 @@ fn get_input<T: ArrayElement + Float>(
(input, bias)
}
-fn get_output<T: ArrayElement + Float, B: Backend>(
+fn get_output<T: DataTypeRepr + Float, B: Backend>(
input: &Input<T>,
in_place: bool,
) -> Vec<T> {
@@ -71,7 +70,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
buffer_to_vec::<B, T>(&output_buffer)
}
-fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+fn get_test_data_basic<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>) {
let num_rows = 4usize;
let num_cols = 32usize;
let (data, bias) = get_input::<T>(num_rows, num_cols);
@@ -88,7 +87,7 @@ fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
(input, expected)
}
-fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+fn get_test_data_large<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>) {
let num_rows = 64usize;
let num_cols = 128usize;
let (data, bias) = get_input::<T>(num_rows, num_cols);
@@ -105,7 +104,7 @@ fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
(input, expected)
}
-fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+fn get_test_data_edge<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>) {
// Single row, small dim, scale = 1.0
let num_rows = 1usize;
let num_cols = 4usize;
@@ -124,7 +123,7 @@ fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
(input, expected)
}
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.05f32
} else {
@@ -142,7 +141,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {
});
}
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.05f32
} else {
@@ -160,7 +159,7 @@ fn test_large<T: ArrayElement + Float + Debug + Display>() {
});
}
-fn test_edge<T: ArrayElement + Float + Debug + Display>() {
+fn test_edge<T: DataTypeRepr + Float + Debug + Display>() {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.05f32
} else {
@@ -178,7 +177,7 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>() {
});
}
-fn test_in_place<T: ArrayElement + Float + Debug + Display>() {
+fn test_in_place<T: DataTypeRepr + Float + Debug + Display>() {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.05f32
} else {
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
index a5f6b753c..b184ea37d 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
@@ -5,7 +5,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -14,6 +13,7 @@ use crate::{
cpu::Cpu,
metal::Metal,
},
+ data_type::DataTypeRepr,
encodable_block::mixer::attention::KVCacheView,
tests::{
assert::assert_eq_float,
@@ -21,7 +21,7 @@ use crate::{
},
};
-struct Input<T: ArrayElement + Float> {
+struct Input<T: DataTypeRepr + Float> {
queries: Box<[T]>,
keys: Box<[T]>,
values: Box<[T]>,
@@ -33,11 +33,11 @@ struct Input<T: ArrayElement + Float> {
do_causal: bool,
}
-fn attention_config<T: ArrayElement + Float>(input: &Input<T>) -> AttentionKernelConfig {
+fn attention_config<T: DataTypeRepr + Float>(input: &Input<T>) -> AttentionKernelConfig {
super::default_attention_config::<T>(input.head_dim, input.num_heads, input.num_kv_heads, input.do_causal)
}
-fn get_test_data<T: ArrayElement + Float>(
+fn get_test_data<T: DataTypeRepr + Float>(
num_heads: usize,
num_kv_heads: usize,
sequence_length: usize,
@@ -79,7 +79,7 @@ fn get_test_data<T: ArrayElement + Float>(
(input, expected)
}
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let config = attention_config(input);
@@ -114,7 +114,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
buffer_to_vec::<B, T>(&output_buffer)
}
-fn get_gemm_output<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+fn get_gemm_output<T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<T> {
let context = <Metal as Backend>::Context::new().expect("Failed to create Metal context");
let config = attention_config(input);
let queries = create_buffer_with_data::<Metal, T>(context.as_ref(), &input.queries);
@@ -146,7 +146,7 @@ fn get_gemm_output<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
buffer_to_vec::<Metal, T>(&output)
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
input: &Input<T>,
expected: &[T],
) {
@@ -165,7 +165,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
assert_eq_float::<T>(expected, &output, eps, &msg);
}
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 4, 8, 1, 64, false);
test_internal(&input, &expected);
@@ -173,7 +173,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {
test_internal(&input, &expected);
}
-fn test_causal<T: ArrayElement + Float + Debug + Display>() {
+fn test_causal<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 4, 16, 1, 64, true);
test_internal(&input, &expected);
@@ -181,7 +181,7 @@ fn test_causal<T: ArrayElement + Float + Debug + Display>() {
test_internal(&input, &expected);
}
-fn test_gqa<T: ArrayElement + Float + Debug + Display>() {
+fn test_gqa<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(8, 2, 8, 1, 64, false);
test_internal(&input, &expected);
@@ -189,12 +189,12 @@ fn test_gqa<T: ArrayElement + Float + Debug + Display>() {
test_internal(&input, &expected);
}
-fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: usize) {
+fn test_head_dim<T: DataTypeRepr + Float + Debug + Display>(head_dim: usize) {
let (input, expected) = get_test_data::<T>(4, 4, 8, 2, head_dim, true);
test_internal(&input, &expected);
}
-fn test_unaligned<T: ArrayElement + Float + Debug + Display>() {
+fn test_unaligned<T: DataTypeRepr + Float + Debug + Display>() {
let (input, expected) = get_test_data::<T>(4, 4, 40, 7, 64, true);
test_internal(&input, &expected);
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
index 9740132bd..7684edc83 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
@@ -3,7 +3,6 @@ use uzu_engine_macros::uzu_test;
use super::{gemm, single_pass, two_pass};
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -13,6 +12,7 @@ use crate::{
cpu::Cpu,
metal::Metal,
},
+ data_type::DataTypeRepr,
encodable_block::{mixer::attention::KVCacheView, sampling::PRng},
tests::{
assert::assert_eq_float,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs
index f13b664d7..164977a36 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs
@@ -1,12 +1,12 @@
#![cfg(backend = "metal")]
use super::{gemm, single_pass, two_pass};
-use crate::{array::ArrayElement, backends::common::kernel::AttentionKernelConfig};
+use crate::{backends::common::kernel::AttentionKernelConfig, data_type::DataTypeRepr};
mod gemm_test;
mod kernel_test;
-fn default_attention_config<T: ArrayElement>(
+fn default_attention_config<T: DataTypeRepr>(
head_dim: usize,
num_q_heads: usize,
num_groups: usize,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs
index 5be2ab32d..2c9a885e6 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs
@@ -4,7 +4,6 @@ use uzu_engine_macros::uzu_test;
use super::*;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, BufferMut, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
@@ -12,6 +11,7 @@ use crate::{
},
metal::Metal,
},
+ data_type::DataTypeRepr,
tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},
};
@@ -57,7 +57,7 @@ fn test_case(suffix_len: usize) -> TestCase {
}
}
-fn run_prefill<T: ArrayElement>(
+fn run_prefill<T: DataTypeRepr>(
context: &MetalContext,
case: &TestCase,
mode: PrefillMode,
@@ -172,7 +172,7 @@ fn chunked_prefill_matches_recurrent_prefill() {
let context = <Metal as Backend>::Context::new().expect("metal context");
if <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(
&context,
- <bf16 as ArrayElement>::data_type(),
+ <bf16 as DataTypeRepr>::data_type(),
HEAD_K_DIM as u32,
)
.expect("chunked")
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/dispatch_paths_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/dispatch_paths_test.rs
index f2cc84d72..121073d74 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/dispatch_paths_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/dispatch_paths_test.rs
@@ -6,7 +6,6 @@ use rstest::rstest;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, Context,
@@ -14,6 +13,7 @@ use crate::{
},
metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},
},
+ data_type::DataTypeRepr,
tests::{
assert::assert_eq_float,
matmul::{
@@ -23,7 +23,7 @@ use crate::{
},
};
-fn run_metal<T: ArrayElement + Float>(
+fn run_metal<T: DataTypeRepr + Float>(
context: &MetalContext,
kernel: &mut <<Metal as Backend>::Kernels as Kernels>::MatmulKernel,
input: &Input<T>,
@@ -46,7 +46,7 @@ fn gemm_engines_for_hw(context: &MetalContext) -> Vec<GemmEngine> {
engines
}
-fn check_case<T: ArrayElement + Float + Debug + Display>(
+fn check_case<T: DataTypeRepr + Float + Debug + Display>(
context: &MetalContext,
kernel: &mut <<Metal as Backend>::Kernels as Kernels>::MatmulKernel,
dispatch: Option<GemmEngine>,
@@ -64,7 +64,7 @@ fn check_case<T: ArrayElement + Float + Debug + Display>(
);
}
-fn run_matrix<T: ArrayElement + Float + Debug + Display>(
+fn run_matrix<T: DataTypeRepr + Float + Debug + Display>(
case_for_shape: impl Fn(crate::tests::matmul::Shape) -> Case,
tolerance: f32,
) {
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm_bench.rs
index 7e96c6cc0..48faf8722 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm_bench.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm_bench.rs
@@ -3,7 +3,6 @@ use half::bf16;
use uzu_engine_macros::uzu_bench;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend,
@@ -14,6 +13,7 @@ use crate::{
},
metal::{Metal, kernel::matmul::gemm::GemmEngine},
},
+ data_type::DataTypeRepr,
tests::{
helpers::create_buffer,
matmul::{bench_fp_gemm_shapes, iter_encode_loop},
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
index abcc83a66..860ed111a 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
@@ -8,7 +8,6 @@ use rstest::rstest;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
@@ -22,13 +21,14 @@ use crate::{
cpu::Cpu,
metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},
},
+ data_type::DataTypeRepr,
tests::{
helpers::buffer_to_vec,
matmul::{QuantBuffers, QuantInput, quant::run_quant_cpu, quant_arguments},
},
};
-fn run_quant_metal<T: ArrayElement + Float>(
+fn run_quant_metal<T: DataTypeRepr + Float>(
context: &MetalContext,
input: &QuantInput<T>,
dispatch: Option<GemmEngine>,
@@ -82,7 +82,7 @@ fn quant_params_layout_table() {
}
}
-fn assert_parity<T: ArrayElement + Float + Debug + Display>(
+fn assert_parity<T: DataTypeRepr + Float + Debug + Display>(
label: &str,
reference: &[T],
actual: &[T],
@@ -104,7 +104,7 @@ fn assert_parity<T: ArrayElement + Float + Debug + Display>(
assert_eq!(errors, 0, "{label}: {errors} mismatches");
}
-fn run_parity<T: ArrayElement + Float + Debug + Display>(
+fn run_parity<T: DataTypeRepr + Float + Debug + Display>(
m: u32,
k: u32,
n: u32,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs
index 7d664744e..47d46e237 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs
@@ -4,7 +4,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_bench;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend,
@@ -13,13 +12,14 @@ use crate::{
},
metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},
},
+ data_type::DataTypeRepr,
tests::{
matmul::{QuantBuffers, QuantInput, bench_quant_gemm_shapes, iter_encode_loop, quant_arguments},
util::type_short_name,
},
};
-fn bench_unified_quant_typed<T: ArrayElement + Float>(
+fn bench_unified_quant_typed<T: DataTypeRepr + Float>(
c: &mut Criterion,
context: &MetalContext,
label: &str,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs
index 6ade045f0..fd258eccc 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs
@@ -4,7 +4,6 @@ use num_traits::Float;
use uzu_engine_macros::uzu_bench;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend,
@@ -13,6 +12,7 @@ use crate::{
},
metal::{Metal, context::MetalContext},
},
+ data_type::DataTypeRepr,
tests::{
cold_pool::ColdPool,
matmul::{QuantBuffers, QuantInput, iter_encode_loop, quant_arguments, qwen3_layer_shapes},
@@ -20,7 +20,7 @@ use crate::{
},
};
-fn bench_qwen3_layers_typed<T: ArrayElement + Float>(
+fn bench_qwen3_layers_typed<T: DataTypeRepr + Float>(
c: &mut Criterion,
context: &MetalContext,
label: &str,
diff --git a/crates/uzu-engine/unit/common/assert.rs b/crates/uzu-engine/unit/common/assert.rs
index 569375579..1f41dfcb2 100644
--- a/crates/uzu-engine/unit/common/assert.rs
+++ b/crates/uzu-engine/unit/common/assert.rs
@@ -2,9 +2,9 @@ use std::fmt::Display;
use num_traits::Float;
-use crate::array::ArrayElement;
+use crate::data_type::DataTypeRepr;
-pub fn assert_eq_float<T: ArrayElement + Float + Display>(
+pub fn assert_eq_float<T: DataTypeRepr + Float + Display>(
expected: &[T],
actual: &[T],
eps: f32,
diff --git a/crates/uzu-engine/unit/common/helpers.rs b/crates/uzu-engine/unit/common/helpers.rs
index 6f9c90993..cc1a04068 100644
--- a/crates/uzu-engine/unit/common/helpers.rs
+++ b/crates/uzu-engine/unit/common/helpers.rs
@@ -1,11 +1,11 @@
use std::{mem::size_of, sync::Arc};
use crate::{
- array::ArrayElement,
backends::common::{
Backend, BufferCpuAccessible, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable,
CommandBufferPending, Context,
},
+ data_type::DataTypeRepr,
};
/// Invokes `$body` once per available backend, with `$B` bound to each backend type.
@@ -52,7 +52,7 @@ pub fn create_buffer<B: Backend, T>(
context.create_buffer(buffer_size_bytes::<T>(elements_count)).expect("Failed to create buffer")
}
-pub fn create_buffer_with_data<B: Backend, T: ArrayElement>(
+pub fn create_buffer_with_data<B: Backend, T: DataTypeRepr>(
context: &B::Context,
data: &[T],
) -> B::GlobalBuffer {
@@ -61,13 +61,13 @@ pub fn create_buffer_with_data<B: Backend, T: ArrayElement>(
buffer
}
-pub fn buffer_to_vec<B: Backend, T: ArrayElement>(
+pub fn buffer_to_vec<B: Backend, T: DataTypeRepr>(
buffer: impl BufferRef<Backend = B, Buffer: BufferCpuAccessible>
) -> Vec<T> {
buffer.copyout()
}
-pub fn buffer_prefix_to_vec<B: Backend, T: ArrayElement>(
+pub fn buffer_prefix_to_vec<B: Backend, T: DataTypeRepr>(
buffer: impl BufferRef<Backend = B, Buffer: BufferCpuAccessible>,
elements_count: usize,
) -> Vec<T> {
diff --git a/crates/uzu-engine/unit/common/matmul/harness.rs b/crates/uzu-engine/unit/common/matmul/harness.rs
index 55fb8e69d..a34da05a9 100644
--- a/crates/uzu-engine/unit/common/matmul/harness.rs
+++ b/crates/uzu-engine/unit/common/matmul/harness.rs
@@ -2,7 +2,6 @@ use num_traits::Float;
use super::Shape;
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
@@ -13,6 +12,7 @@ use crate::{
},
cpu::Cpu,
},
+ data_type::DataTypeRepr,
tests::helpers::{buffer_to_vec, create_buffer_with_data},
};
@@ -61,7 +61,7 @@ impl Case {
}
}
-pub struct Input<T: ArrayElement + Float> {
+pub struct Input<T: DataTypeRepr + Float> {
pub a: Box<[T]>,
pub b: Box<[T]>,
pub rht_factors: Option<Box<[i32]>>,
@@ -69,7 +69,7 @@ pub struct Input<T: ArrayElement + Float> {
pub case: Case,
}
-pub fn deterministic_input<T: ArrayElement + Float>(case: Case) -> Input<T> {
+pub fn deterministic_input<T: DataTypeRepr + Float>(case: Case) -> Input<T> {
let Shape {
m,
k,
@@ -101,7 +101,7 @@ pub fn deterministic_input<T: ArrayElement + Float>(case: Case) -> Input<T> {
}
}
-pub fn run<B: Backend, T: ArrayElement + Float>(
+pub fn run<B: Backend, T: DataTypeRepr + Float>(
context: &B::Context,
kernel: &mut <B::Kernels as Kernels>::MatmulKernel,
input: &Input<T>,
@@ -155,7 +155,7 @@ pub fn run<B: Backend, T: ArrayElement + Float>(
buffer_to_vec::<B, T>(&d_buffer)
}
-pub fn cpu_reference<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+pub fn cpu_reference<T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<T> {
let context = <Cpu as Backend>::Context::new().expect("CPU context");
let mut kernel = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
&context,
diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rs
index ef5ddacfc..083b34269 100644
--- a/crates/uzu-engine/unit/common/matmul/quant.rs
+++ b/crates/uzu-engine/unit/common/matmul/quant.rs
@@ -4,7 +4,6 @@ use num_traits::Float;
use rand::{RngExt, SeedableRng, rngs::SmallRng};
use crate::{
- array::ArrayElement,
backends::{
common::{
Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,
@@ -20,7 +19,7 @@ use crate::{
},
cpu::Cpu,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},
};
@@ -33,7 +32,7 @@ pub struct PreparedInt8A {
}
#[derive(Clone)]
-pub struct QuantInput<T: ArrayElement + Float> {
+pub struct QuantInput<T: DataTypeRepr + Float> {
pub w_packed: Vec<u32>,
pub scales: Vec<T>,
pub zero_points: Option<Vec<u8>>,
@@ -58,7 +57,7 @@ fn mode_for_bits(bits: u32) -> QuantizationMode {
}
}
-fn pad<T: ArrayElement>(
+fn pad<T: DataTypeRepr>(
values: &[T],
minimum_len: usize,
) -> Vec<T> {
@@ -121,7 +120,7 @@ pub fn transpose_metadata(
}
}
-impl<T: ArrayElement + Float> QuantInput<T> {
+impl<T: DataTypeRepr + Float> QuantInput<T> {
pub fn new(
m: u32,
k: u32,
@@ -284,7 +283,7 @@ impl<T: ArrayElement + Float> QuantInput<T> {
}
}
-pub struct QuantBuffers<B: Backend, T: ArrayElement + Float> {
+pub struct QuantBuffers<B: Backend, T: DataTypeRepr + Float> {
pub w: B::GlobalBuffer,
pub scales: B::GlobalBuffer,
pub zp: Option<B::GlobalBuffer>,
@@ -297,7 +296,7 @@ pub struct QuantBuffers<B: Backend, T: ArrayElement + Float> {
_t: std::marker::PhantomData<T>,
}
-impl<B: Backend, T: ArrayElement + Float> QuantBuffers<B, T> {
+impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {
pub fn allocate(
context: &B::Context,
input: &QuantInput<T>,
@@ -379,7 +378,7 @@ impl<B: Backend, T: ArrayElement + Float> QuantBuffers<B, T> {
}
}
-fn quant_b_variant<TB: BufferRef, T: ArrayElement + Float>(
+fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(
w: TB,
scales: TB,
zero_points: Option<TB>,
@@ -404,7 +403,7 @@ fn quant_b_variant<TB: BufferRef, T: ArrayElement + Float>(
})
}
-pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(
+pub fn quant_arguments<'a, B: Backend, T: DataTypeRepr + Float>(
buffers: &'a mut QuantBuffers<B, T>,
input: &QuantInput<T>,
) -> MatmulArguments<'a, B, &'a B::GlobalBuffer, &'a B::GlobalBuffer, &'a mut B::GlobalBuffer, &'a B::GlobalBuffer> {
@@ -449,7 +448,7 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(
}
}
-pub fn run_quant_cpu<T: ArrayElement + Float>(input: &QuantInput<T>) -> Vec<T> {
+pub fn run_quant_cpu<T: DataTypeRepr + Float>(input: &QuantInput<T>) -> Vec<T> {
let context = <Cpu as Backend>::Context::new().expect("Cpu context");
let mut buffers = QuantBuffers::<Cpu, T>::allocate(&context, input);
let mut matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
index fba8724f1..86d9321c1 100644
--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
@@ -7,31 +7,19 @@ use rand_distr::Normal;
use uzu_engine_macros::uzu_test;
use crate::{
- array::ArrayElement,
backends::common::{
Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
gpu_types::trie::TrieNode,
},
- data_type::DataType,
+ data_type::{DataType, DataTypeRepr},
encodable_block::{
batch_topology::BatchTopology,
sampling::{Sampling, SamplingMethod},
},
tests::proptest::{ComparableTestResults, TestContextes, for_each_context, kernel_data_type},
+ utils::dispatch_dtype::dispatch_dtype,
};
-macro_rules! dispatch_dtype {
- (|()| $body:expr) => { $body };
- (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {
- match $dt {
- $crate::data_type::DataType::F16 => { type $T = half::f16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
- $crate::data_type::DataType::BF16 => { type $T = half::bf16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
- $crate::data_type::DataType::F32 => { type $T = f32; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }
- other => panic!("unsupported dtype in test: {:?}", other),
- }
- };
-}
-
struct SamplingTestResults(Vec<u32>);
impl ComparableTestResults for SamplingTestResults {
@@ -46,7 +34,7 @@ impl ComparableTestResults for SamplingTestResults {
}
}
-fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(
+fn do_sampling_backend<B: Backend, T: DataTypeRepr + Float>(
context: &B::Context,
logits: &[T],
seeds: Option<&[u64]>,
@@ -94,7 +82,7 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(
Ok(SamplingTestResults(sampled))
}
-fn get_data<T: ArrayElement + Float>(
+fn get_data<T: DataTypeRepr + Float>(
seed: u64,
batch_size: usize,
vocab_size: usize,
diff --git a/crates/uzu-engine/unit/parameters/safetensors_metadata.rs b/crates/uzu-engine/unit/parameters/safetensors_metadata.rs
deleted file mode 100644
index 5ac50bd4d..000000000
--- a/crates/uzu-engine/unit/parameters/safetensors_metadata.rs
+++ /dev/null
@@ -1,14 +0,0 @@
-use std::fs::File;
-
-use uzu_engine_macros::uzu_test;
-
-use crate::{parameters::safetensors_metadata::read_metadata, tests::path::get_test_weights_path};
-
-#[uzu_test]
-#[ignore] // downloads the test model from the registry
-fn test_metadata_loading() {
- let path = get_test_weights_path();
- let file = File::open(&path).expect("weights not found");
- let (_offset, metadata) = read_metadata(&file).expect("read metadata");
- assert!(!metadata.tensors.is_empty());
-}