A high-performance inference engine for AI models
Small cleanup part 5 (#956)
uuuvn16443dbad6bc
diff --git a/crates/uzu-engine/src/array/element.rs b/crates/uzu-engine/src/array/element.rsdeleted file mode 100644index 2d59b779a..000000000--- a/crates/uzu-engine/src/array/element.rs+++ /dev/null@@ -1,41 +0,0 @@-use bytemuck::Pod;-use half::{bf16, f16};-use num_traits::NumCast;--#[cfg(test)]-use crate::data_type::DataType;--// TODO: remove this--pub trait ArrayElement: NumCast + Pod {- #[cfg(test)]- fn data_type() -> DataType;-}--macro_rules! impl_array_element {- ($($type:ty => $variant:ident),+ $(,)?) => {- $(- impl ArrayElement for $type {- #[cfg(test)]- fn data_type() -> DataType {- DataType::$variant- }- }- )+- };-}--impl_array_element! {- f16 => F16,- bf16 => BF16,- f32 => F32,- f64 => F64,- i8 => I8,- u8 => U8,- i16 => I16,- u16 => U16,- i32 => I32,- u32 => U32,- i64 => I64,- u64 => U64,-}diff --git a/crates/uzu-engine/src/array/mod.rs b/crates/uzu-engine/src/array/mod.rsdeleted file mode 100644index 683c7be7a..000000000--- a/crates/uzu-engine/src/array/mod.rs+++ /dev/null@@ -1,5 +0,0 @@-mod element;-mod util;--pub use element::ArrayElement;-pub use util::size_for_shape;diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rsindex c98712b8e..fcde34d18 100644--- a/crates/uzu-engine/src/backends/common/command_buffer.rs+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs@@ -3,9 +3,9 @@ use std::{mem::size_of_val, time::Duration};use bytemuck::{AnyBitPattern, NoUninit};use crate::{- array::size_for_shape,backends::common::{Backend, BufferMut, BufferRef},data_type::DataType,+ utils::size_for_shape::size_for_shape,};pub trait CommandBuffer {diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation/activation.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation/activation.rsindex 190adffc9..14cb49406 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/activation/activation.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation/activation.rs@@ -2,11 +2,11 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(Activation)]#[variants(T, f16, f32, bf16)]-pub fn activation<T: ArrayElement + Float>(+pub fn activation<T: DataTypeRepr + Float>(#[optional(!in_place)] input: Option<*const T>,output: *mut T,n: u32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rsindex aecabd67a..2a7e1bb54 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs@@ -4,11 +4,11 @@ use uzu_engine_macros::kernel;use super::{hadamard_transform, min_max_symmetric_divisor, quantize_symmetric_i8};use crate::{- array::ArrayElement,backends::common::{gpu_types::{activation_transform::ActivationTransformOp, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},kernel::matmul::Int8CodeLayout,},+ data_type::DataTypeRepr,};pub fn quantize_transformed_row(@@ -48,7 +48,7 @@ pub fn quantize_transformed_row(#[kernel(ActivationTransform)]#[variants(T, f32, bf16)]#[variants(BiasT, f32, bf16)]-pub fn activation_transform<T: ArrayElement + Float, BiasT: ArrayElement + Float>(+pub fn activation_transform<T: DataTypeRepr + Float, BiasT: DataTypeRepr + Float>(#[optional(!in_place)] input: Option<*const T>,#[optional(ops == ActivationTransformOp::InputRht || ops == ActivationTransformOp::OutputRht)] fp_out: Option<*mut T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rsindex be16009fd..cbd24aee9 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_prepare.rs@@ -2,12 +2,12 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;-fn apply_rope<ElementT: ArrayElement + Float>(+fn apply_rope<ElementT: DataTypeRepr + Float>(head: *const ElementT,- cosines: *const (impl ArrayElement + Float),- sines: *const (impl ArrayElement + Float),+ cosines: *const (impl DataTypeRepr + Float),+ sines: *const (impl DataTypeRepr + Float),batch_idx: usize,head_dim_idx: usize,rope_dim: usize,@@ -34,7 +34,7 @@ fn apply_rope<ElementT: ArrayElement + Float>(#[kernel(AttentionPrepare)]#[variants(ElementT, bf16)]#[variants(RopeT, f32)]-pub fn attention_prepare<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(+pub fn attention_prepare<ElementT: DataTypeRepr + Float, RopeT: DataTypeRepr + Float>(qkvg: *const ElementT,queries: *mut ElementT,#[optional(has_kv)] keys: Option<*mut ElementT>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rsindex dea58b22b..73a887dd8 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs@@ -3,14 +3,14 @@ use num_traits::Float;use uzu_engine_macros::kernel;use crate::{- array::ArrayElement,backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},+ data_type::DataTypeRepr,};#[kernel(AttentionSinglePass)]#[variants(T, f32, bf16)]#[variants(HEAD_DIM, 64, 128, 256, 512)]-pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(+pub fn attention_single_pass<T: DataTypeRepr + Float, const HEAD_DIM: u32>(queries: *const T,keys: *const T,values: *const T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rsindex 736c515c3..3f0fea1e3 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs@@ -2,11 +2,11 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(KVCacheUpdate)]#[variants(T, f32, bf16)]-pub fn kv_cache_update<T: ArrayElement + Float>(+pub fn kv_cache_update<T: DataTypeRepr + Float>(in_place_keys: *mut T,in_place_values: *mut T,copies: &[crate::backends::common::gpu_types::kv_cache_update::Copy],diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rsindex cfb465b3d..8425198ad 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(QKVNorm)]#[variants(InputT, f32, bf16)]@@ -10,10 +10,10 @@ use crate::array::ArrayElement;#[variants(OutputT, f32, bf16)]#[variants(AccumT, f32)]pub fn qkv_norm<- InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,- OutputT: ArrayElement + Float,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ ScaleT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float,+ AccumT: DataTypeRepr + Float,>(#[optional(!in_place)] qkvg_input: Option<*const InputT>,#[optional(has_scales)] scales: Option<*const ScaleT>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rsindex d10807ad1..c0cf351d3 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs@@ -2,11 +2,11 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(SigmoidGate)]#[variants(T, f32, bf16)]-pub fn sigmoid_gate<T: ArrayElement + Float>(+pub fn sigmoid_gate<T: DataTypeRepr + Float>(gate: *const T,output: *mut T,gate_dim: u32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs b/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rsindex aaa9a4190..3a2c8e7ac 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/convolution/separable_causal_conv.rs@@ -2,11 +2,11 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(SeparableCausalConv)]#[variants(T, bf16)]-pub fn separable_causal_conv<T: ArrayElement + Float>(+pub fn separable_causal_conv<T: DataTypeRepr + Float>(input: *const T,coefficient_deltas: *const T,weights: *const T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rsindex 91ea779f4..21b36d47c 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs@@ -3,7 +3,6 @@ use num_traits::Float;use uzu_engine_macros::kernel;use crate::{- array::ArrayElement,backends::{common::gpu_types::{d4s4, embedding::EmbeddingTableKind, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,@@ -11,11 +10,12 @@ use crate::{},cpu::kernel::activation_transform::hadamard_transform,},+ data_type::DataTypeRepr,};#[kernel(InputEmbeddingLookup)]#[variants(T, f32, bf16)]-pub fn input_embedding_lookup<T: ArrayElement + Float>(+pub fn input_embedding_lookup<T: DataTypeRepr + Float>(token_ids: *const u32,values: *const u8,#[optional(table_kind != EmbeddingTableKind::Dense)] scales: Option<*const T>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rsindex 72ea5555f..95256b52c 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs@@ -3,7 +3,6 @@ use num_traits::Float;use uzu_engine_macros::kernel;use crate::{- array::ArrayElement,backends::{common::{gpu_types::{@@ -15,11 +14,12 @@ use crate::{},cpu::kernel::activation_transform::{hadamard_transform, quantize_transformed_row},},+ data_type::DataTypeRepr,};#[kernel(GatedActMul)]#[variants(T, f32, bf16)]-pub fn gated_act_mul<T: ArrayElement + Float>(+pub fn gated_act_mul<T: DataTypeRepr + Float>(act_operand: *const T,#[optional(!interleaved)] value_operand: Option<*const T>,#[optional(ops == GatedActMulOp::FullPrecision)] fp_out: Option<*mut T>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rsindex 20fb61d10..991154674 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};// Multi-token causal conv1d with SiLU for DeltaNet.// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd values@@ -15,7 +15,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::A// state_out: [conv_dim, state_stride]#[kernel(DeltaNetConvScan)]#[variants(T, f32, bf16)]-pub fn delta_net_conv_scan<T: ArrayElement + Float>(+pub fn delta_net_conv_scan<T: DataTypeRepr + Float>(conv_padded: *const f32,conv_weight: *const f32,#[optional(has_bias)] bias: Option<*const f32>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rsindex 607b72505..0933f0b7f 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_update.rs@@ -2,12 +2,12 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};// Single-token causal conv1d with SiLU, in-place.#[kernel(DeltaNetConvUpdate)]#[variants(T, f32, bf16)]-pub fn delta_net_conv_update<T: ArrayElement + Float>(+pub fn delta_net_conv_update<T: DataTypeRepr + Float>(conv_weight: *const f32,#[optional(has_bias)] bias: Option<*const f32>,in_out: *mut T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rsindex ad61a8a20..22712385e 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/norm_gate.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};// In-place RMSNorm + SiLU gate for DeltaNet prefill output.//@@ -15,7 +15,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::A// and z is at in_proj[token * total_proj_dim + conv_dim + hv * head_v_dim + i]#[kernel(DeltaNetNormGate)]#[variants(T, f32, bf16)]-pub fn delta_net_norm_gate<T: ArrayElement + Float>(+pub fn delta_net_norm_gate<T: DataTypeRepr + Float>(in_out: *mut T,in_proj: *const T,norm_weight: *const f32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill.rsindex 698adff98..97e80cb5b 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill.rs@@ -2,12 +2,12 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(DeltaNetPrefill)]#[variants(T, f32, bf16)]#[variants(HEAD_K_DIM, 128)]-pub fn delta_net_prefill<T: ArrayElement + Float, const HEAD_K_DIM: u32>(+pub fn delta_net_prefill<T: DataTypeRepr + Float, const HEAD_K_DIM: u32>(q_norm: *const f32,k_norm: *const f32,beta_buf: *const f32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill_prep.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill_prep.rsindex ace6ffbe0..7549bc7db 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill_prep.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/prefill_prep.rs@@ -2,14 +2,14 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;// Pre-compute L2-normalized q/k, beta, and decay for all tokens.#[kernel(DeltaNetPrefillPrep)]#[variants(T, f32, bf16)]#[variants(QKT, f32, bf16)]#[variants(HEAD_K_DIM, 128)]-pub fn delta_net_prefill_prep<T: ArrayElement + Float, QKT: ArrayElement + Float, const HEAD_K_DIM: u32>(+pub fn delta_net_prefill_prep<T: DataTypeRepr + Float, QKT: DataTypeRepr + Float, const HEAD_K_DIM: u32>(in_proj: *const T,a_log: *const f32,dt_bias: *const f32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rsindex 28f6b3df7..b8fd85ecf 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs@@ -2,11 +2,11 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};#[kernel(ConvTreeScan)]#[variants(T, f32, bf16)]-pub fn conv_tree_scan<T: ArrayElement + Float>(+pub fn conv_tree_scan<T: DataTypeRepr + Float>(in_proj: *const T,conv_weight: *const f32,#[optional(has_bias)] bias: Option<*const f32>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rsindex cbb490f06..fde31f84b 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs@@ -2,12 +2,12 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(BuildTreeOut)]#[variants(QKT, f32, bf16)]#[variants(OutputT, f32, bf16)]-pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(+pub fn build_tree_out<QKT: DataTypeRepr + Float, OutputT: DataTypeRepr + Float>(q: *const QKT,prefix: *const f32,qkd: *const f32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rsindex b382fec31..5930c0ddc 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs@@ -2,12 +2,12 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(StateAdvance)]#[variants(T, f32, bf16)]#[variants(HEAD_K_DIM, 128)]-pub fn state_advance<T: ArrayElement + Float, const HEAD_K_DIM: u32>(+pub fn state_advance<T: DataTypeRepr + Float, const HEAD_K_DIM: u32>(k_norm: *const T,v: *const T,log_decay_buf: *const f32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rsindex 9ac1eba10..8488e2ed7 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs@@ -2,14 +2,14 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::trie::TrieNode};+use crate::{backends::common::gpu_types::trie::TrieNode, data_type::DataTypeRepr};const BLOCK: usize = 16;#[kernel(BuildTreeGram)]#[variants(T, f32, bf16)]#[variants(USE_MXU, false)]-pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(+pub fn build_tree_gram<T: DataTypeRepr + Float, const USE_MXU: bool>(q: *const T,k: *const T,trie: *const TrieNode,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rsindex f243c1e38..fcc41335f 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs@@ -2,14 +2,14 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;/// Solves chunked GDN update coefficients: `(I + A) U = beta * (v - exp(prefix) * kh0)`./// `kh0 = k @ h0^T` is precomputed by BuildTreeGram.#[kernel(TreeUpdateSolve)]#[variants(T, f32, bf16)]#[variants(BV, 16, 32)]-pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(+pub fn tree_update_solve<T: DataTypeRepr + Float, const BV: u32>(#[optional(use_h0)] kh0: Option<*const f32>,v: *const T,prefix: *const f32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rsindex 6c5ac3f3e..650f9cf17 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/update.rs@@ -2,7 +2,7 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};// Single-token delta net update: decay state, apply delta rule, RMSNorm + SiLU gate.// Steps per v-head:@@ -13,7 +13,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::A#[kernel(DeltaNetUpdate)]#[variants(T, f32, bf16)]#[variants(HEAD_K_DIM, 128)]-pub fn delta_net_update<T: ArrayElement + Float, const HEAD_K_DIM: u32>(+pub fn delta_net_update<T: DataTypeRepr + Float, const HEAD_K_DIM: u32>(in_proj: *const T,a_log: *const f32,dt_bias: *const f32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/logit_transform/logit_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/logit_transform/logit_transform.rsindex 4343822e2..fe7fe3fdd 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/logit_transform/logit_transform.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/logit_transform/logit_transform.rs@@ -2,11 +2,11 @@ use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(LogitTransform)]#[variants(T, f32, bf16)]-pub fn logit_transform<T: ArrayElement + Float>(+pub fn logit_transform<T: DataTypeRepr + Float>(logits: *mut T,length: u32,scale: f32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rsindex 0b5788925..cce269e1a 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs@@ -1,4 +1,4 @@-use half::{bf16, f16};+use num_traits::AsPrimitive;use crate::{backends::{@@ -9,7 +9,7 @@ use crate::{cpu::{Cpu, buffer::CpuBufferExt, kernel::matmul::MatmulCpuError},},data_type::DataType,- utils::pointers::SendPtr,+ utils::{dispatch_dtype::dispatch_dtype, pointers::SendPtr},};pub(super) enum WeightData {@@ -82,14 +82,7 @@ pub(super) unsafe fn read_f32(data_type: DataType,index: usize,) -> f32 {- unsafe {- match data_type {- DataType::F32 => *(base as *const f32).add(index),- DataType::F16 => (*(base as *const f16).add(index)).to_f32(),- DataType::BF16 => (*(base as *const bf16).add(index)).to_f32(),- _ => unreachable!(),- }- }+ dispatch_dtype!(|(T: data_type)| unsafe { (*base.cast::<T>().add(index)).as_() })}#[inline]@@ -99,12 +92,5 @@ pub(super) unsafe fn write_f32(index: usize,value: f32,) {- unsafe {- match data_type {- DataType::F32 => *(base as *mut f32).add(index) = value,- DataType::F16 => *(base as *mut f16).add(index) = f16::from_f32(value),- DataType::BF16 => *(base as *mut bf16).add(index) = bf16::from_f32(value),- _ => unreachable!(),- }- }+ dispatch_dtype!(|(T: data_type)| unsafe { *base.cast::<T>().add(index) = value.as_() })}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rsindex f87bb72ac..45d2fd7a7 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs@@ -3,11 +3,11 @@ use num_traits::Float;use uzu_engine_macros::kernel;use crate::{- array::ArrayElement,backends::{common::gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,cpu::kernel::activation_transform::hadamard_transform,},+ data_type::DataTypeRepr,};#[kernel(Normalization)]@@ -16,10 +16,10 @@ use crate::{#[variants(OutputT, f32, f16, bf16)]#[variants(AccumT, f32)]pub fn normalization<- InputT: ArrayElement + Float,- AffineT: ArrayElement + Float,- OutputT: ArrayElement + Float,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ AffineT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float,+ AccumT: DataTypeRepr + Float,>(#[optional(!in_place)] input: Option<*const InputT>,#[optional(has_scales)] scales: Option<*const AffineT>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/pooling/pooling.rs b/crates/uzu-engine/src/backends/cpu/kernel/pooling/pooling.rsindex 80845c5af..534e6f896 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/pooling/pooling.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/pooling/pooling.rs@@ -2,11 +2,11 @@ use half::{bf16, f16};use num_traits::{Float, ToPrimitive};use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(PoolingMean)]#[variants(T, f32, f16, bf16)]-pub fn pooling_mean<T: ArrayElement + Float>(+pub fn pooling_mean<T: DataTypeRepr + Float>(input: *const T,output: *mut T,seq_len: u32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/sampling/repetition_penalty.rs b/crates/uzu-engine/src/backends/cpu/kernel/sampling/repetition_penalty.rsindex 1d9f059bf..660fe268c 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/sampling/repetition_penalty.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/sampling/repetition_penalty.rs@@ -2,11 +2,11 @@ use half::bf16;use num_traits::{Float, NumCast};use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::ring::RingParams};+use crate::{backends::common::gpu_types::ring::RingParams, data_type::DataTypeRepr};#[kernel(RepetitionPenalty)]#[variants(T, f32, bf16)]-pub fn repetition_penalty<T: ArrayElement + Float + NumCast>(+pub fn repetition_penalty<T: DataTypeRepr + Float + NumCast>(original_logits: *const T,logits_copy: *mut T,context_ring: *const u32,@@ -18,8 +18,8 @@ pub fn repetition_penalty<T: ArrayElement + Float + NumCast>(sampling_length: u32,) {fn apply(- original_logits: *const (impl ArrayElement + Float),- logits_copy: *mut (impl ArrayElement + Float),+ original_logits: *const (impl DataTypeRepr + Float),+ logits_copy: *mut (impl DataTypeRepr + Float),vocab_size: usize,sample_index: usize,token_id: usize,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/sampling/unified_sampling.rs b/crates/uzu-engine/src/backends/cpu/kernel/sampling/unified_sampling.rsindex b0e7711f2..308724333 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/sampling/unified_sampling.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/sampling/unified_sampling.rs@@ -5,14 +5,14 @@ use num_traits::Float;use uzu_engine_macros::kernel;use crate::{- array::ArrayElement,+ data_type::DataTypeRepr,encodable_block::sampling::{gumbel_float, revidx},};// NOTE: top_k + top_p combination is not exactly matching lalamo ("parallel" here, should be top-k then top-p)#[kernel(UnifiedSampling)]#[variants(T, f32, bf16)]-pub fn unified_sampling<T: ArrayElement + Float>(+pub fn unified_sampling<T: DataTypeRepr + Float>(logits: *const T,output: *mut u32,#[optional(is_stochastic)] seeds: Option<*const u64>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/short_conv/short_conv.rs b/crates/uzu-engine/src/backends/cpu/kernel/short_conv/short_conv.rsindex 4fce0ea94..d6fdb85dd 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/short_conv/short_conv.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/short_conv/short_conv.rs@@ -2,11 +2,11 @@ use half::{bf16, f16};use num_traits::{Float, ToPrimitive};use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(ShortConvPack)]#[variants(T, f32, f16, bf16)]-pub fn short_conv_pack<T: ArrayElement + Float>(+pub fn short_conv_pack<T: DataTypeRepr + Float>(state_in: *const T,in_proj: *const T,padded: *mut T,@@ -44,7 +44,7 @@ pub fn short_conv_pack<T: ArrayElement + Float>(#[kernel(ShortConvPrefill)]#[variants(T, f32, f16, bf16)]#[variants(WeightT, f32)]-pub fn short_conv_prefill<T: ArrayElement + Float, WeightT: ArrayElement + Float>(+pub fn short_conv_prefill<T: DataTypeRepr + Float, WeightT: DataTypeRepr + Float>(padded: *const T,in_proj: *const T,w: *const WeightT,@@ -114,7 +114,7 @@ pub fn short_conv_prefill<T: ArrayElement + Float, WeightT: ArrayElement + Float#[kernel(ShortConvDecode)]#[variants(T, f32, f16, bf16)]#[variants(WeightT, f32)]-pub fn short_conv_decode<T: ArrayElement + Float, WeightT: ArrayElement + Float>(+pub fn short_conv_decode<T: DataTypeRepr + Float, WeightT: DataTypeRepr + Float>(in_proj: *const T,w: *const WeightT,#[optional(has_bias)] b: Option<*const WeightT>,@@ -185,7 +185,7 @@ pub fn short_conv_decode<T: ArrayElement + Float, WeightT: ArrayElement + Float>#[kernel(ShortConvTrie)]#[variants(T, f32, f16, bf16)]#[variants(WeightT, f32)]-pub fn short_conv_trie<T: ArrayElement + Float, WeightT: ArrayElement + Float>(+pub fn short_conv_trie<T: DataTypeRepr + Float, WeightT: DataTypeRepr + Float>(in_proj: *const T,w: *const WeightT,#[optional(has_bias)] b: Option<*const WeightT>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs b/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rsindex 295315768..2c888f85a 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs@@ -2,11 +2,11 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(Softmax)]#[variants(T, f32, f16, bf16)]-pub fn softmax<T: ArrayElement + Float>(+pub fn softmax<T: DataTypeRepr + Float>(values: *mut T,#[optional(has_sinks)] sinks: Option<*const T>,row_length: u32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rsindex a45ea849c..f16d261c7 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/conv1d.rs@@ -2,12 +2,12 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(Conv1dPack)]#[variants(StateT, f32, bf16)]#[variants(InputT, f32, bf16)]-pub fn conv1d_pack<StateT: ArrayElement + Float, InputT: ArrayElement + Float>(+pub fn conv1d_pack<StateT: DataTypeRepr + Float, InputT: DataTypeRepr + Float>(state_in: *const StateT,x: *const InputT,padded: *mut StateT,@@ -40,7 +40,7 @@ pub fn conv1d_pack<StateT: ArrayElement + Float, InputT: ArrayElement + Float>(#[kernel(Conv1dDecode)]#[variants(T, f32, f16, bf16)]-pub fn conv1d_decode<T: ArrayElement + Float>(+pub fn conv1d_decode<T: DataTypeRepr + Float>(x: *const T,w: *const T,#[optional(has_bias)] b: Option<*const T>,@@ -128,7 +128,7 @@ pub fn conv1d_decode<T: ArrayElement + Float>(#[kernel(Conv1dScan)]#[variants(T, f32, f16, bf16)]-pub fn conv1d_scan<T: ArrayElement + Float>(+pub fn conv1d_scan<T: DataTypeRepr + Float>(padded: *const T,w: *const T,#[optional(has_bias)] b: Option<*const T>,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rsindex 54d2bbe3d..e6268c9a4 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/split_inproj.rs@@ -2,11 +2,11 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(SplitInProj)]#[variants(T, f32, f16, bf16)]-pub fn split_in_proj<T: ArrayElement + Float>(+pub fn split_in_proj<T: DataTypeRepr + Float>(input: *const T,conv_out: *mut T,z_out: *mut T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rsindex 8fd765754..84ec1a5b2 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_prefill.rs@@ -2,11 +2,11 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(SSDPrefill64)]#[variants(T, f32, f16, bf16)]-pub fn ssd_prefill64<T: ArrayElement + Float>(+pub fn ssd_prefill64<T: DataTypeRepr + Float>(#[allow(unused)] x: *const T,#[allow(unused)] dt_raw: *const T,#[allow(unused)] b: *const T,@@ -30,7 +30,7 @@ pub fn ssd_prefill64<T: ArrayElement + Float>(#[kernel(SSDPrefill)]#[variants(T, f32, f16, bf16)]-pub fn ssd_prefill<T: ArrayElement + Float>(+pub fn ssd_prefill<T: DataTypeRepr + Float>(#[allow(unused)] x: *const T,#[allow(unused)] dt_raw: *const T,#[allow(unused)] b: *const T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rsindex 290953a07..c7669cb04 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/ssm/ssd_update.rs@@ -2,11 +2,11 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::{array::ArrayElement, backends::common::gpu_types::activation_type::ActivationType};+use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};#[kernel(SSDUpdate)]#[variants(T, f32, f16, bf16)]-pub fn ssd_update<T: ArrayElement + Float>(+pub fn ssd_update<T: DataTypeRepr + Float>(x: *const T,dt_raw: *const T,b: *const T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rsindex 6c68f1e82..07cdedbfd 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs@@ -2,12 +2,12 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(TensorAddBias)]#[variants(T, f32, f16, bf16)]#[variants(BiasT, f32, f16, bf16)]-pub fn tensor_add_bias<T: ArrayElement + Float, BiasT: ArrayElement + Float>(+pub fn tensor_add_bias<T: DataTypeRepr + Float, BiasT: DataTypeRepr + Float>(#[optional(!in_place)] input: Option<*const T>,bias: *const BiasT,output: *mut T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs b/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rsindex 48e946c8d..e3bc2f778 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs@@ -2,11 +2,11 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;#[kernel(TensorAddScale)]#[variants(T, f32, f16, bf16)]-pub fn tensor_add_scale<T: ArrayElement + Float>(+pub fn tensor_add_scale<T: DataTypeRepr + Float>(#[optional(!in_place)] input: Option<*const T>,bias: *const T,output: *mut T,diff --git a/crates/uzu-engine/src/data_type.rs b/crates/uzu-engine/src/data_type.rsindex 5ee66baf0..f58175327 100644--- a/crates/uzu-engine/src/data_type.rs+++ b/crates/uzu-engine/src/data_type.rs@@ -1,38 +1,24 @@+use bytemuck::Pod;+use half::{bf16, f16};+use num_traits::NumCast;use uzu_engine_macros::uzu_config;#[derive(Eq, PartialOrd, Ord, Copy, Hash)]#[uzu_config]pub enum DataType {- // Floating point- #[serde(rename = "bfloat16")]BF16,- #[serde(rename = "float16")]F16,- #[serde(rename = "float32")]F32,- #[serde(rename = "float64")]F64,- // Sub-byte integers- #[serde(rename = "int4")]I4,- #[serde(rename = "uint4")]U4,- // Normal integers- #[serde(rename = "int8")]I8,- #[serde(rename = "uint8")]U8,- #[serde(rename = "int16")]I16,- #[serde(rename = "uint16")]U16,- #[serde(rename = "int32")]I32,- #[serde(rename = "uint32")]U32,- #[serde(rename = "int64")]I64,- #[serde(rename = "uint64")]U64,}@@ -52,3 +38,34 @@ impl DataType {self.size_in_bits().div_ceil(8)}}++pub trait DataTypeRepr: NumCast + Pod {+ fn data_type() -> DataType;+}++macro_rules! impl_data_type_repr {+ ($($type:ty => $variant:ident),+ $(,)?) => {+ $(+ impl DataTypeRepr for $type {+ fn data_type() -> DataType {+ DataType::$variant+ }+ }+ )++ };+}++impl_data_type_repr! {+ f16 => F16,+ bf16 => BF16,+ f32 => F32,+ f64 => F64,+ i8 => I8,+ u8 => U8,+ i16 => I16,+ u16 => U16,+ i32 => I32,+ u32 => U32,+ i64 => I64,+ u64 => U64,+}diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rsindex bef35e2db..3225f1ac4 100644--- a/crates/uzu-engine/src/encodable_block/classifier.rs+++ b/crates/uzu-engine/src/encodable_block/classifier.rs@@ -15,7 +15,7 @@ use crate::{prediction_head::{PredictionHead, PredictionHeadError},transformer::{Transformer, TransformerEncodeError, TransformerNewError},},- parameters::ParameterTree,+ safetensors::ParameterTree,};#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rsindex 7fc633724..064c31cf8 100644--- a/crates/uzu-engine/src/encodable_block/convolution.rs+++ b/crates/uzu-engine/src/encodable_block/convolution.rs@@ -2,14 +2,14 @@ use derive_more::Debug;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding,kernel::{Kernels, SeparableCausalConvKernel},},config::token_mixer::convolutions::SeparableCausalConvConfig,data_type::DataType,- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},+ utils::size_for_shape::size_for_shape,};#[derive(Debug, Error)]@@ -43,12 +43,11 @@ impl<B: Backend> SeparableCausalConv<B> {assert!(model_dim.is_multiple_of(group_size));assert_eq!(data_type, DataType::BF16, "SeparableCausalConv only supports BF16");- let weights =- parameter_tree.leaf("weights")?.validate(&[model_dim, kernel_size], DataType::BF16)?.read_buffer()?;+ let weights = parameter_tree.read_buffer("weights", &[model_dim, kernel_size], DataType::BF16)?;let biases = config.has_biases- .then(|| parameter_tree.leaf("biases")?.validate(&[model_dim], DataType::BF16)?.read_buffer())+ .then(|| parameter_tree.read_buffer("biases", &[model_dim], DataType::BF16)).transpose()?;let kernel = <B::Kernels as Kernels>::SeparableCausalConvKernel::new(diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rsindex 44b462310..b35078f49 100644--- a/crates/uzu-engine/src/encodable_block/decoder.rs+++ b/crates/uzu-engine/src/encodable_block/decoder.rs@@ -14,7 +14,7 @@ use crate::{per_layer_embedding::{PerLayerEmbedding, PerLayerEmbeddingError},transformer::{Transformer, TransformerEncodeError, TransformerNewError, TransformerState},},- parameters::ParameterTree,+ safetensors::ParameterTree,};#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rsindex 87743fc8a..3beac9005 100644--- a/crates/uzu-engine/src/encodable_block/dflash.rs+++ b/crates/uzu-engine/src/encodable_block/dflash.rs@@ -4,7 +4,6 @@ use derive_more::Debug;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, gpu_types::trie::TrieNode,},@@ -24,8 +23,8 @@ use crate::{normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},transformer_layer::{TransformerLayer, TransformerLayerError},},- parameters::ParameterTree,- utils::maybe_mut::MaybeMut,+ safetensors::ParameterTree,+ utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},};pub struct DFlashState<B: Backend> {diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex 5eb3facee..434948441 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -16,7 +16,7 @@ use crate::{embedding_table::{EmbeddingTable, EmbeddingTableError, read_output_signs},linear::{Gather, LinearEncodeError, LinearMatmulError, UntiedReadout},},- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},};#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rsindex ab6916574..b1970e563 100644--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs@@ -14,7 +14,7 @@ use crate::{},data_type::DataType,encodable_block::weight_matrix::{WeightMatrix, WeightMatrixError},- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},};type LookupKernel<B> = <<B as Backend>::Kernels as Kernels>::InputEmbeddingLookupKernel;@@ -207,7 +207,7 @@ fn load_d4s4<B: Backend>("{spec:?} with {data_type:?} and embedding dim {embedding_dim}")));}- let read = |name: &str, shape: &[u32], data_type| tree.leaf(name)?.validate(shape, data_type)?.read_buffer();+ let read = |name: &str, shape: &[u32], data_type| tree.read_buffer(name, shape, data_type);let table = D4S4Table {codes: read("codes", &[vocab_size, embedding_dim / d4s4::VALUES_PER_CODE], DataType::U8)?,row_scales: read("row_scales", &[vocab_size], data_type)?,@@ -228,5 +228,5 @@ pub(super) fn read_output_signs<B: Backend>(embedding_dim: u32,) -> Result<B::GlobalBuffer, EmbeddingTableError<B>> {let signs = tree.subtree("incoherence_signs");- Ok(signs.leaf("output_signs")?.validate(&[embedding_dim], DataType::I32)?.read_buffer()?)+ Ok(signs.read_buffer("output_signs", &[embedding_dim], DataType::I32)?)}diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rsindex 6e4853a6d..ce609b594 100644--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs@@ -19,7 +19,7 @@ use crate::{linear::{Gather, Linear, LinearEncodeError, LinearInput},weight_matrix::{WeightMatrix, WeightMatrixError},},- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},};#[derive(Debug, Error)]@@ -59,9 +59,7 @@ fn load_biases<B: Backend>("mixed precision linear with biases is not supported: weights={weights_data_type:?}, output={output_data_type:?}",)));}- Ok(parameter_tree- .map(|tree| tree.leaf("biases")?.validate(&[output_dim], weights_data_type)?.read_buffer())- .transpose()?)+ Ok(parameter_tree.map(|tree| tree.read_buffer("biases", &[output_dim], weights_data_type)).transpose()?)}impl<B: Backend> LinearMatmul<B> {diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rsindex c151f8d5f..2e9615cad 100644--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs@@ -24,7 +24,7 @@ use crate::{hybrid_spec::{HybridSpec, IncoherenceProcessingMode},},data_type::DataType,- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},};#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex c97472e4c..4f8e5a9ef 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -14,7 +14,7 @@ use crate::{Gather, Linear, LinearEncodeError, LinearInput, LinearInputPreparation, LinearMatmul, LinearMatmulError,input_rht::InputRht,},- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},};#[derive(Debug, Error)]@@ -136,14 +136,9 @@ impl<B: Backend> RHTLinearWrapper<B> {let weights_tree = parameter_tree.subtree("weights");let quantized_weights_tree = weights_tree.subtree("quantized");let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;- let rht_signs = weights_tree- .leaf("incoherence_signs.input_signs")?- .validate(&[input_dimension], DataType::I32)?- .read_buffer()?;- let output_factors = weights_tree- .leaf("incoherence_signs.output_signs")?- .validate(&[output_dimension], DataType::I32)?- .read_buffer()?;+ let rht_signs = weights_tree.read_buffer("incoherence_signs.input_signs", &[input_dimension], DataType::I32)?;+ let output_factors =+ weights_tree.read_buffer("incoherence_signs.output_signs", &[output_dimension], DataType::I32)?;let inner_linear = LinearMatmul::load(input_dimension,output_dimension,diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rsindex 169a7a95a..1237d2233 100644--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs@@ -12,7 +12,7 @@ use crate::{encodable_block::linear::{Gather, Linear, LinearEncodeError, LinearInputPreparation, LinearMatmul, LinearMatmulError, input_rht::InputRht,},- parameters::ParameterTree,+ safetensors::ParameterTree,};pub struct UntiedReadout<B: Backend> {@@ -56,11 +56,7 @@ impl<B: Backend> UntiedReadout<B> {None,context,)?;- let rht_signs = tree- .subtree("incoherence_signs")- .leaf("input_signs")?- .validate(&[model_dim], DataType::I32)?- .read_buffer()?;+ let rht_signs = tree.subtree("incoherence_signs").read_buffer("input_signs", &[model_dim], DataType::I32)?;let preparation = LinearInputPreparation {rht_signs,activation_quantization: linear.prepare_a8(context),diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rsindex 2acb217df..d076b308a 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs@@ -20,7 +20,7 @@ use crate::{},},},- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},utils::maybe_mut::MaybeMut,};@@ -140,10 +140,8 @@ impl<B: Backend> Attention<B> {rope_config.is_some(),).map_err(AttentionNewError::Backend)?;- let sinks = config- .has_sinks- .then(|| parameter_tree.leaf("sinks")?.validate(&[num_q_heads], data_type)?.read_buffer())- .transpose()?;+ let sinks =+ config.has_sinks.then(|| parameter_tree.read_buffer("sinks", &[num_q_heads], data_type)).transpose()?;assert!(sliding_window_size.is_none_or(|size| size > 0), "zero sliding window size");let ring_capacity = sliding_window_size;diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rsindex 5fc53ef76..e2b6ce736 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs@@ -8,7 +8,7 @@ use crate::{},config::normalization::{NormalizationConfig, UpcastMode},data_type::DataType,- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},};#[derive(Debug, Error)]@@ -52,16 +52,15 @@ impl<B: Backend> QKVNorm<B> {let Some(config) = config else {return Ok::<_, QKVNormError<B>>(None);};- let scales = config- .has_scale- .then(|| {- parameter_tree- .subtree(parameter_name.expect("scaled norm requires parameter tree"))- .leaf("scales")?- .validate(&[head_dim], DataType::F32)?- .read_buffer()- })- .transpose()?;+ let scales =+ config+ .has_scale+ .then(|| {+ parameter_tree+ .subtree(parameter_name.expect("scaled norm requires parameter tree"))+ .read_buffer("scales", &[head_dim], DataType::F32)+ })+ .transpose()?;let kernel = <B::Kernels as Kernels>::QKVNormKernel::new(context,intermediate_data_type,diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rsindex e43044655..dddbacb68 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs@@ -1,7 +1,6 @@use std::any::Any;use crate::{- array::size_for_shape,backends::common::{Backend, Buffer, CommandBuffer, Context, DeviceCapabilities, Kernels, SparseBuffer,gpu_types::{kv_cache_update::Copy, ring::RingParams},@@ -9,6 +8,7 @@ use crate::{},data_type::DataType,encodable_block::mixer::{MixerState, attention::Attention},+ utils::size_for_shape::size_for_shape,};pub const ATTENTION_SUFFIX_CAPACITY: u32 = 1024; // TODO: remove hardcoded suffix capacitydiff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rsindex d7b9ec6ed..bbe6f5276 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs@@ -2,7 +2,6 @@ use derive_more::Debug;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,CommandBufferPending, Context, Kernels,@@ -25,8 +24,8 @@ use crate::{delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},},},- parameters::{ParameterLoaderError, ParameterTree},- utils::maybe_mut::MaybeMut,+ safetensors::{ParameterLoaderError, ParameterTree},+ utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},};pub(crate) mod tree_verify;@@ -204,10 +203,9 @@ impl<B: Backend> DeltaNet<B> {let conv_config = &config.conv_config;let conv_tree = parameter_tree.subtree("conv");- let conv_weight =- conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], INNER_DATA_TYPE)?.read_buffer()?;+ let conv_weight = conv_tree.read_buffer("weights", &[conv_dim, config.kernel_size], INNER_DATA_TYPE)?;let conv_bias = if conv_config.has_biases {- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], INNER_DATA_TYPE)?.read_buffer()?)+ Some(conv_tree.read_buffer("biases", &[conv_dim], INNER_DATA_TYPE)?)} else {None};@@ -227,10 +225,9 @@ impl<B: Backend> DeltaNet<B> {).map_err(DeltaNetNewError::Backend)?;- let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], INNER_DATA_TYPE)?.read_buffer()?;- let dt_bias = parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], INNER_DATA_TYPE)?.read_buffer()?;- let norm_weight =- parameter_tree.leaf("norm.scales")?.validate(&[config.value_head_dim], INNER_DATA_TYPE)?.read_buffer()?;+ let a_log = parameter_tree.read_buffer("a_log", &[config.num_heads], INNER_DATA_TYPE)?;+ let dt_bias = parameter_tree.read_buffer("dt_bias", &[config.num_heads], INNER_DATA_TYPE)?;+ let norm_weight = parameter_tree.read_buffer("norm.scales", &[config.value_head_dim], INNER_DATA_TYPE)?;let norm_epsilon = config.norm_config.epsilon;let delta_net_update =<B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim)diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rsindex 6bf011646..8605e17d0 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs@@ -2,7 +2,6 @@ use derive_more::Debug;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,gpu_types::activation_type::ActivationType,@@ -18,8 +17,8 @@ use crate::{linear::{Linear, LinearBlockError},mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},},- parameters::{ParameterLoaderError, ParameterTree},- utils::maybe_mut::MaybeMut,+ safetensors::{ParameterLoaderError, ParameterTree},+ utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},};const INNER_DATA_TYPE: DataType = DataType::F32;@@ -124,17 +123,16 @@ impl<B: Backend> Mamba2<B> {context,)?;- let gate_bias = parameter_tree.leaf("gate_bias")?.validate(&[inner_dim], INNER_DATA_TYPE)?.read_buffer()?;+ let gate_bias = parameter_tree.read_buffer("gate_bias", &[inner_dim], INNER_DATA_TYPE)?;let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, INNER_DATA_TYPE).map_err(Mamba2NewError::Backend)?;let conv_config = &config.conv_config;let conv_tree = parameter_tree.subtree("conv");- let conv_weight =- conv_tree.leaf("weights")?.validate(&[conv_dim, kernel_size], INNER_DATA_TYPE)?.read_buffer()?;+ let conv_weight = conv_tree.read_buffer("weights", &[conv_dim, kernel_size], INNER_DATA_TYPE)?;let conv_bias = if conv_config.has_biases {- Some(conv_tree.leaf("biases")?.validate(&[conv_dim], INNER_DATA_TYPE)?.read_buffer()?)+ Some(conv_tree.read_buffer("biases", &[conv_dim], INNER_DATA_TYPE)?)} else {None};@@ -148,7 +146,7 @@ impl<B: Backend> Mamba2<B> {.map_err(Mamba2NewError::Backend)?;let skip_connection_weight =- parameter_tree.leaf("skip_connection_weight")?.validate(&[num_heads], INNER_DATA_TYPE)?.read_buffer()?;+ parameter_tree.read_buffer("skip_connection_weight", &[num_heads], INNER_DATA_TYPE)?;let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, INNER_DATA_TYPE, true).map_err(Mamba2NewError::Backend)?;diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rsindex 772c4b39e..dc1061e44 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs@@ -17,7 +17,7 @@ use crate::{short_conv::{ShortConv, ShortConvNewError},},},- parameters::ParameterTree,+ safetensors::ParameterTree,utils::maybe_mut::MaybeMut,};diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rsindex 817d74e70..649026f80 100644--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs@@ -2,7 +2,6 @@ use derive_more::Debug;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,CommandBufferPending, Context, Kernels,@@ -15,8 +14,8 @@ use crate::{linear::{Linear, LinearBlockError},mixer::{Mixer, MixerEncodeError, MixerState, attention::rope::PrecalculatedRoPE},},- parameters::{ParameterLoaderError, ParameterTree},- utils::maybe_mut::MaybeMut,+ safetensors::{ParameterLoaderError, ParameterTree},+ utils::{maybe_mut::MaybeMut, size_for_shape::size_for_shape},};enum ShortConvStateSuffixStatus<B: Backend> {@@ -137,12 +136,11 @@ impl<B: Backend> ShortConv<B> {let conv_tree = parameter_tree.subtree("conv");let conv_data_type = DataType::F32;- let conv_weight =- conv_tree.leaf("weights")?.validate(&[hidden_dim, kernel_size], conv_data_type)?.read_buffer()?;+ let conv_weight = conv_tree.read_buffer("weights", &[hidden_dim, kernel_size], conv_data_type)?;let has_bias = config.conv_config.has_biases;let conv_bias = if has_bias {- Some(conv_tree.leaf("biases")?.validate(&[hidden_dim], conv_data_type)?.read_buffer()?)+ Some(conv_tree.read_buffer("biases", &[hidden_dim], conv_data_type)?)} else {None};diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rsindex 2649f4a3a..8bb398930 100644--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs@@ -11,7 +11,7 @@ use crate::{linear::{Linear, LinearBlockError},mlp::{Mlp, MlpEncodeError, gate_act_mul::MlpGateActMulEncodable},},- parameters::ParameterTree,+ safetensors::ParameterTree,};pub struct DenseMlp<B: Backend> {diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rsindex 4dbaa9209..5944294e5 100644--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs@@ -12,7 +12,7 @@ use crate::{linear::LinearEncodeError,mlp::dense::{DenseMlp, DenseMlpNewError},},- parameters::ParameterTree,+ safetensors::ParameterTree,};#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rsindex 90c5fd109..04cfb0873 100644--- a/crates/uzu-engine/src/encodable_block/normalization.rs+++ b/crates/uzu-engine/src/encodable_block/normalization.rs@@ -2,14 +2,14 @@ use derive_more::Debug;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding,kernel::{Kernels, NormalizationKernel},},config::normalization::{NormalizationConfig, UpcastMode},data_type::DataType,- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},+ utils::size_for_shape::size_for_shape,};// TODO: clean up@@ -67,11 +67,11 @@ impl<B: Backend> Normalization<B> {let scales = config.has_scale- .then(|| parameter_tree.leaf("scales")?.validate(&[element_count], DataType::F32)?.read_buffer())+ .then(|| parameter_tree.read_buffer("scales", &[element_count], DataType::F32)).transpose()?;let biases = config.has_biases- .then(|| parameter_tree.leaf("biases")?.validate(&[element_count], DataType::F32)?.read_buffer())+ .then(|| parameter_tree.read_buffer("biases", &[element_count], DataType::F32)).transpose()?;let (scale_residual_sum, scale_output, post_layer_scalar_value) = match post_layer_scalar {diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rsindex f3aa283f8..d7a9f31c1 100644--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs@@ -16,7 +16,7 @@ use crate::{linear::{Linear, LinearBlockError, LinearEncodeError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},},- parameters::ParameterTree,+ safetensors::ParameterTree,};#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rsindex e28394451..b5d2e094d 100644--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs@@ -13,7 +13,7 @@ use crate::{linear::{Linear, LinearBlockError, LinearEncodeError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},},- parameters::ParameterTree,+ safetensors::ParameterTree,};#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rsindex a3a9a0ed3..0514d4065 100644--- a/crates/uzu-engine/src/encodable_block/transformer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer.rs@@ -15,7 +15,7 @@ use crate::{normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},transformer_layer::{TransformerLayer, TransformerLayerError},},- parameters::ParameterTree,+ safetensors::ParameterTree,utils::maybe_mut::MaybeMut,};diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rsindex dff07cc0d..30d12166f 100644--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs@@ -1,4 +1,5 @@use derive_more::Debug;+use num_traits::AsPrimitive;use thiserror::Error;use crate::{@@ -16,8 +17,8 @@ use crate::{normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},per_layer_embedding::{PerLayerEmbeddingError, PerLayerEmbeddingProjection},},- parameters::{ParameterLoaderError, ParameterTree},- utils::maybe_mut::MaybeMut,+ safetensors::{ParameterLoaderError, ParameterTree},+ utils::{dispatch_dtype::dispatch_dtype, maybe_mut::MaybeMut},};#[derive(Debug, Error)]@@ -186,13 +187,8 @@ impl<B: Backend> TransformerLayer<B> {layer_index,});}- let leaf = parameter_tree.leaf("post_layer_scalar")?;- let scalar = match data_type {- DataType::F32 => leaf.validate(&[1], data_type)?.read_slice::<f32>()?[0],- DataType::F16 => leaf.validate(&[1], data_type)?.read_slice::<half::f16>()?[0].to_f32(),- DataType::BF16 => leaf.validate(&[1], data_type)?.read_slice::<half::bf16>()?[0].to_f32(),- _ => unreachable!(),- };+ let scalar =+ dispatch_dtype!(|(T: data_type)| parameter_tree.read_slice::<T>("post_layer_scalar", &[1])?[0].as_());Some(scalar)} else {Nonediff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex d90e3474b..4517ed87f 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -2,7 +2,6 @@ use derive_more::Debug;use thiserror::Error;use crate::{- array::size_for_shape,backends::common::{Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Context, Kernels,gpu_types::weaver::{@@ -23,7 +22,8 @@ use crate::{normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},weaver_layer::{PreparedPrefixAttention, WeaverLayer},},- parameters::ParameterTree,+ safetensors::ParameterTree,+ utils::size_for_shape::size_for_shape,};pub const DATA_TYPE: DataType = DataType::BF16;diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rsindex 6bfd7cf75..161bc17b8 100644--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs@@ -1,5 +1,4 @@use crate::{- array::size_for_shape,backends::common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,kernel::{AncestorAttentionKernel, AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel},@@ -16,7 +15,8 @@ use crate::{normalization::{Normalization, PostLayerScalar, ShortcutMode},weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverEncodeError, WeaverNewError},},- parameters::ParameterTree,+ safetensors::ParameterTree,+ utils::size_for_shape::size_for_shape,};pub struct PreparedPrefixAttention<B: Backend> {diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rsindex 21731c4a4..a80036620 100644--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs@@ -9,7 +9,7 @@ use crate::{},config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,- parameters::{ParameterLoaderError, ParameterTree},+ safetensors::{ParameterLoaderError, ParameterTree},};#[derive(Debug, Error)]@@ -114,7 +114,7 @@ impl<B: Backend> WeightMatrix<B> {let (rows, columns) = physical_shape(&layout, output_dim, input_dim);let Some(info) = quantization_info else {- let values = tree.leaf("weights")?.validate(&[rows, columns], data_type)?.read_buffer()?;+ let values = tree.read_buffer("weights", &[rows, columns], data_type)?;return Ok(Self {values,quantized: None,@@ -136,12 +136,11 @@ impl<B: Backend> WeightMatrix<B> {}let groups = columns.div_ceil(group_size);- let values =- tree.leaf("weights")?.validate(&[rows, columns / packing_divisor], storage_data_type)?.read_buffer()?;+ let values = tree.read_buffer("weights", &[rows, columns / packing_divisor], storage_data_type)?;let params = QuantParams::new(params_layout, rows, groups);let load_plane =|name: &str, shape: [u32; 2], storage_type: DataType| -> Result<B::GlobalBuffer, WeightMatrixError<B>> {- Ok(tree.leaf(name)?.validate(&shape, storage_type)?.read_buffer()?)+ Ok(tree.read_buffer(name, &shape, storage_type)?)};let scales = load_plane("scales", params.scale_shape(), data_type)?;let correction = match info.method {diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rsindex 3c9832faa..557360353 100644--- a/crates/uzu-engine/src/engine/classifier_model.rs+++ b/crates/uzu-engine/src/engine/classifier_model.rs@@ -24,7 +24,7 @@ use crate::{data_type::DataType,encodable_block::classifier::{Classifier as ClassifierEncodable, ClassifierError as ClassifierEncodableError},engine::Engine,- parameters::{ParameterLoader, ParameterLoaderError},+ safetensors::{ParameterLoader, ParameterLoaderError},};pub struct ClassifierModel<B: Backend> {@@ -79,7 +79,7 @@ impl<B: Backend> Engine<B> {(0..config.classifier_config.num_labels).map(|index| format!("class_{index}")).collect()};- weight_loader.tree().assert_all_tensors_validated()?;+ weight_loader.assert_all_tensors_read()?;let tokenizer = Arc::new(Tokenizer::from_file(model_path.join("tokenizer.json"))?);let token_codec_config = config.token_codec_config;diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rsindex adc82fe58..91cc482b1 100644--- a/crates/uzu-engine/src/engine/language_model/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/mod.rs@@ -13,7 +13,7 @@ use crate::{sampling::{Sampling, SamplingMethod},},engine::Engine,- parameters::{ParameterLoader, ParameterLoaderError},+ safetensors::{ParameterLoader, ParameterLoaderError},speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},};pub use crate::{config::model::generation::GenerationConfig, utils::strict_serde::Unsupported};@@ -97,7 +97,7 @@ impl<B: Backend> Engine<B> {let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&self.context).map_err(EngineLoadLanguageModelError::Backend)?;- weight_loader.tree().assert_all_tensors_validated()?;+ weight_loader.assert_all_tensors_read()?;let generation_config = config.generation_config;let end_of_thinking_tag = match &config.token_codec_config {diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 706da8cd0..bf579c144 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -10,7 +10,6 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;#[cfg(grammar)]use crate::engine::language_model::grammar::Grammar;use crate::{- array::size_for_shape,backends::common::{Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,CommandBufferPending, Context,@@ -26,6 +25,7 @@ use crate::{stream::{LanguageModelStreamError, LanguageModelStreamOptions},},trie::TrieNode,+ utils::size_for_shape::size_for_shape,};enum ForwardPassChaining<B: Backend> {diff --git a/crates/uzu-engine/src/lib.rs b/crates/uzu-engine/src/lib.rsindex d073d94a5..3a23a313c 100644--- a/crates/uzu-engine/src/lib.rs+++ b/crates/uzu-engine/src/lib.rs@@ -2,11 +2,10 @@#![cfg_attr(test, test_runner(crate::tests::harness::uzu_harness))]#![cfg_attr(target_family = "wasm", feature(wasi_ext))]-mod array;mod config;mod data_type;mod encodable_block;-mod parameters;+mod safetensors;mod speculators;mod trie;mod utils;diff --git a/crates/uzu-engine/src/parameters/error.rs b/crates/uzu-engine/src/parameters/error.rsdeleted file mode 100644index 6e042a546..000000000--- a/crates/uzu-engine/src/parameters/error.rs+++ /dev/null@@ -1,38 +0,0 @@-use derive_more::Debug;-use thiserror::Error;--use super::safetensors_metadata::HeaderLoadingError;-use crate::{backends::common::Backend, data_type::DataType};--#[derive(Debug, Error)]-pub enum ParameterLoaderError<B: Backend> {- #[error("Header loading error: {0}")]- HeaderLoadingError(#[from] HeaderLoadingError),- #[error("Array with key \"{0}\" not found.")]- KeyNotFound(String),- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),- #[error("Failed to read data")]- ArrayLoadingError(#[from] std::io::Error),- #[error("Failed to deserialize metadata")]- MetadataDeserializationError(#[from] serde_json::Error),- #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]- InvalidTensor {- shape: Box<[u32]>,- data_type: DataType,- expected_shape: Box<[u32]>,- expected_data_type: DataType,- },- #[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]- InvalidTensorSize {- shape: Box<[u32]>,- data_type: DataType,- size: usize,- expected_size: usize,- },- #[error("Unvalidated tensors under {prefix:?}: {keys:?}")]- UnvalidatedTensors {- prefix: Option<String>,- keys: Box<[String]>,- },-}diff --git a/crates/uzu-engine/src/parameters/loader.rs b/crates/uzu-engine/src/parameters/loader.rsdeleted file mode 100644index 4c6a54e8d..000000000--- a/crates/uzu-engine/src/parameters/loader.rs+++ /dev/null@@ -1,214 +0,0 @@-#[cfg(unix)]-use std::os::unix::fs::FileExt;-#[cfg(target_family = "wasm")]-use std::os::wasi::fs::FileExt;-use std::{- cell::RefCell,- collections::{HashMap, HashSet},- fs::File,-};--use super::{- error::ParameterLoaderError,- safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata},-};-use crate::{- array::{ArrayElement, size_for_shape},- backends::common::{Backend, BufferMut, Context},- data_type::DataType,- utils::strict_serde::DeserializeStrictOwned,-};--struct ParameterMetadata {- shape: Box<[u32]>,- data_type: DataType,- offset: usize,- size: usize,-}--pub struct ParameterLoader<'a, B: Backend> {- context: &'a B::Context,- index: HashMap<String, ParameterMetadata>,- metadata: HashMap<String, String>,- validated_tensors: RefCell<HashSet<String>>,- file: &'a File,-}--impl<'a, B: Backend> ParameterLoader<'a, B> {- pub fn new(- file: &'a File,- context: &'a B::Context,- ) -> Result<Self, ParameterLoaderError<B>> {- let (global_offset, st_metadata) = read_st_metadata(file)?;- let index = st_metadata- .tensors- .into_iter()- .map(|(key, value)| {- let (local_begin, local_end) = value.data_offsets;- let size =- local_end.checked_sub(local_begin).ok_or_else(|| HeaderLoadingError::InvalidTensorOffsets {- key: key.clone().into_boxed_str(),- begin: local_begin,- end: local_end,- })?;- let offset =- global_offset.checked_add(local_begin).ok_or_else(|| HeaderLoadingError::TensorOffsetOverflow {- key: key.clone().into_boxed_str(),- global_offset,- local_begin,- })?;- let weight_metadata = ParameterMetadata {- shape: value.shape.into(),- data_type: value.dtype.data_type()?,- offset,- size,- };- Ok((key, weight_metadata))- })- .collect::<Result<HashMap<_, _>, HeaderLoadingError>>()?;- let metadata = st_metadata.metadata.unwrap_or_default();- Ok(ParameterLoader {- context,- index,- metadata,- validated_tensors: RefCell::new(HashSet::new()),- file,- })- }-- pub fn tree(&self) -> ParameterTree<'_, B> {- ParameterTree {- loader: self,- prefix: None,- }- }-}--pub struct ParameterLeaf<'a, 'leaf, B: Backend, const VALIDATED: bool> {- key: &'leaf str,- metadata: &'leaf ParameterMetadata,- loader: &'leaf ParameterLoader<'a, B>,-}--impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, false> {- pub fn validate(- self,- expected_shape: &[u32],- expected_data_type: DataType,- ) -> Result<ParameterLeaf<'a, 'leaf, B, true>, ParameterLoaderError<B>> {- let shape = self.metadata.shape.as_ref();- let data_type = self.metadata.data_type;- if (shape, data_type) != (expected_shape, expected_data_type) {- return Err(ParameterLoaderError::InvalidTensor {- shape: shape.into(),- data_type,- expected_shape: expected_shape.into(),- expected_data_type,- });- }- let expected_size = size_for_shape(expected_shape, expected_data_type);- if self.metadata.size != expected_size {- return Err(ParameterLoaderError::InvalidTensorSize {- shape: shape.into(),- data_type,- size: self.metadata.size,- expected_size,- });- }- self.loader.validated_tensors.borrow_mut().insert(self.key.to_string());- Ok(ParameterLeaf {- key: self.key,- metadata: self.metadata,- loader: self.loader,- })- }-}--impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, true> {- pub fn read_slice<T: ArrayElement>(&self) -> Result<Box<[T]>, ParameterLoaderError<B>> {- let element_count = self.metadata.size / std::mem::size_of::<T>();- let mut data = vec![T::zeroed(); element_count];- let destination = bytemuck::cast_slice_mut(&mut data);- self.loader.file.read_exact_at(destination, self.metadata.offset as u64)?;- Ok(data.into_boxed_slice())- }-- pub fn read_buffer(&self) -> Result<B::GlobalBuffer, ParameterLoaderError<B>> {- let mut buffer =- self.loader.context.create_buffer(self.metadata.size).map_err(ParameterLoaderError::BackendError)?;- self.loader.file.read_exact_at(buffer.as_slice_mut::<u8>(), self.metadata.offset as u64)?;- Ok(buffer)- }-}--pub struct ParameterTree<'loader, B: Backend> {- loader: &'loader ParameterLoader<'loader, B>,- prefix: Option<String>,-}--impl<'loader, B: Backend> ParameterTree<'loader, B> {- fn join_prefix(- &self,- name: &str,- ) -> String {- self.prefix.as_ref().map_or_else(|| name.to_string(), |p| format!("{p}.{name}"))- }-- pub fn subtree(- &self,- name: &str,- ) -> Self {- Self {- loader: self.loader,- prefix: Some(self.join_prefix(name)),- }- }-- pub fn leaf<'leaf>(- &'leaf self,- name: &str,- ) -> Result<ParameterLeaf<'loader, 'leaf, B, false>, ParameterLoaderError<B>> {- let key = self.join_prefix(name);- let Some((key, metadata)) = self.loader.index.get_key_value(&key) else {- return Err(ParameterLoaderError::KeyNotFound(key));- };- Ok(ParameterLeaf {- key,- metadata,- loader: self.loader,- })- }-- pub fn metadata<T: DeserializeStrictOwned>(- &self,- name: &str,- ) -> Result<T, ParameterLoaderError<B>> {- let new_prefix = self.join_prefix(name);-- Ok(serde_json::from_str(- self.loader.metadata.get(&new_prefix).ok_or(ParameterLoaderError::KeyNotFound(new_prefix))?,- )?)- }-- pub fn assert_all_tensors_validated(&self) -> Result<(), ParameterLoaderError<B>> {- let subtree_prefix = self.prefix.as_ref().map(|prefix| format!("{prefix}."));- let validated_tensors = self.loader.validated_tensors.borrow();- let mut unvalidated_tensors = self- .loader- .index- .keys()- .filter(|key| subtree_prefix.as_ref().is_none_or(|prefix| key.starts_with(prefix)))- .filter(|key| !validated_tensors.contains(*key))- .cloned()- .collect::<Vec<_>>();- unvalidated_tensors.sort();- if unvalidated_tensors.is_empty() {- Ok(())- } else {- Err(ParameterLoaderError::UnvalidatedTensors {- prefix: self.prefix.clone(),- keys: unvalidated_tensors.into_boxed_slice(),- })- }- }-}diff --git a/crates/uzu-engine/src/parameters/mod.rs b/crates/uzu-engine/src/parameters/mod.rsdeleted file mode 100644index b5d319300..000000000--- a/crates/uzu-engine/src/parameters/mod.rs+++ /dev/null@@ -1,8 +0,0 @@-// TODO: This is overdue for a complete rewrite--mod error;-mod loader;-mod safetensors_metadata;--pub use error::ParameterLoaderError;-pub use loader::{ParameterLoader, ParameterTree};diff --git a/crates/uzu-engine/src/parameters/safetensors_metadata.rs b/crates/uzu-engine/src/parameters/safetensors_metadata.rsdeleted file mode 100644index bb705e6d4..000000000--- a/crates/uzu-engine/src/parameters/safetensors_metadata.rs+++ /dev/null@@ -1,130 +0,0 @@-// This code is based on the safetensors implementation: https://docs.rs/safetensors/latest/src/safetensors/tensor.rs.html--#[cfg(unix)]-use std::os::unix::fs::FileExt;-#[cfg(target_family = "wasm")]-use std::os::wasi::fs::FileExt;-use std::{collections::HashMap, fs::File};--use serde::{Deserialize, Serialize};-use thiserror::Error;--use crate::data_type::DataType;--#[derive(Debug, Error)]-pub enum HeaderLoadingError {- #[error("Unable to read safetensors header bytes: {0}")]- UnableToReadHeader(#[source] std::io::Error),- #[error("Unable to read safetensors header JSON: {0}")]- UnableToReadHeaderJson(#[source] std::io::Error),- #[error("The header is not valid UTF-8 JSON: {0}")]- InvalidHeaderDeserialization(#[from] serde_json::Error),- #[error("The header is too large.")]- HeaderTooLarge,- #[error("Unsupported safetensors dtype: {0:?}")]- UnsupportedDtype(Dtype),- #[error("Invalid data offsets for tensor {key}: begin={begin}, end={end}")]- InvalidTensorOffsets {- key: Box<str>,- begin: usize,- end: usize,- },- #[error("Tensor offset overflow for tensor {key}: global_offset={global_offset}, local_begin={local_begin}")]- TensorOffsetOverflow {- key: Box<str>,- global_offset: usize,- local_begin: usize,- },-}--#[derive(Debug, Serialize, Deserialize)]-pub struct HashMetadata {- #[serde(skip_serializing_if = "Option::is_none")]- #[serde(rename = "__metadata__")]- pub metadata: Option<HashMap<String, String>>,- #[serde(flatten)]- pub tensors: HashMap<String, TensorInfo>,-}--#[derive(Debug, Deserialize, Serialize, Clone)]-pub struct TensorInfo {- pub dtype: Dtype,- pub shape: Vec<u32>,- pub data_offsets: (usize, usize),-}--#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]-#[non_exhaustive]-pub enum Dtype {- /// Boolean type- BOOL,- /// Unsigned byte- U8,- /// Signed byte- I8,- /// FP8 <https://arxiv.org/pdf/2209.05433.pdf>_- #[allow(non_camel_case_types)]- F8_E5M2,- /// FP8 <https://arxiv.org/pdf/2209.05433.pdf>_- #[allow(non_camel_case_types)]- F8_E4M3,- /// Signed integer (16-bit)- I16,- /// Unsigned integer (16-bit)- U16,- /// Half-precision floating point- F16,- /// Brain floating point- BF16,- /// Signed integer (32-bit)- I32,- /// Unsigned integer (32-bit)- U32,- /// Floating point (32-bit)- F32,- /// Floating point (64-bit)- F64,- /// Signed integer (64-bit)- I64,- /// Unsigned integer (64-bit)- U64,-}--impl Dtype {- pub fn data_type(self) -> Result<DataType, HeaderLoadingError> {- Ok(match self {- Self::F16 => DataType::F16,- Self::BF16 => DataType::BF16,- Self::F32 => DataType::F32,- Self::I8 => DataType::I8,- Self::U8 => DataType::U8,- Self::I32 => DataType::I32,- Self::U32 => DataType::U32,- Self::I64 => DataType::I64,- Self::U64 => DataType::U64,- dtype => return Err(HeaderLoadingError::UnsupportedDtype(dtype)),- })- }-}--const MAX_HEADER_SIZE: usize = 100_000_000;--pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadingError> {- let mut header_buffer = [0u8; size_of::<u64>()];- file.read_exact_at(&mut header_buffer, 0).map_err(HeaderLoadingError::UnableToReadHeader)?;- let metadata_size: usize =- u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;- if metadata_size > MAX_HEADER_SIZE {- return Err(HeaderLoadingError::HeaderTooLarge);- }-- let mut json_buffer = vec![0u8; metadata_size];- file.read_exact_at(&mut json_buffer, size_of::<u64>() as u64)- .map_err(HeaderLoadingError::UnableToReadHeaderJson)?;- let metadata: HashMetadata = serde_json::from_slice(&json_buffer)?;- Ok((size_of::<u64>() + metadata_size, metadata))-}--#[cfg(test)]-#[path = "../../unit/parameters/safetensors_metadata.rs"]-mod tests;diff --git a/crates/uzu-engine/src/safetensors.rs b/crates/uzu-engine/src/safetensors.rsnew file mode 100644index 000000000..2bdc7a785--- /dev/null+++ b/crates/uzu-engine/src/safetensors.rs@@ -0,0 +1,217 @@+#[cfg(unix)]+use std::os::unix::fs::FileExt;+#[cfg(target_family = "wasm")]+use std::os::wasi::fs::FileExt;+use std::{cell::Cell, collections::HashMap, fs::File};++use derive_more::Debug;+use serde::Deserialize;+use thiserror::Error;++use crate::{+ backends::common::{Backend, BufferMut, Context},+ data_type::{DataType, DataTypeRepr},+ utils::{size_for_shape::size_for_shape, strict_serde::DeserializeStrictOwned},+};++#[derive(Debug, Error)]+pub enum ParameterLoaderError<B: Backend> {+ #[error("IO error: {0}")]+ IO(#[from] std::io::Error),+ #[error("Unable to deserialize safetensors header JSON: {0}")]+ HeaderDeserialization(#[from] serde_json::Error),+ #[error("Key {0:?} not found")]+ KeyNotFound(String),+ #[error("Unable to deserialize metadata {key:?}: {source}")]+ MetadataDeserializationError {+ key: String,+ #[source]+ source: serde_json::Error,+ },+ #[error(+ "Invalid tensor {key:?}: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}"+ )]+ InvalidTensor {+ key: String,+ shape: Box<[u32]>,+ data_type: DataType,+ expected_shape: Box<[u32]>,+ expected_data_type: DataType,+ },+ #[error(+ "Invalid representation for tensor {key:?}: {shape:?} @ {data_type:?} requires {expected_size} bytes, global_offset={global_offset}, begin={begin}, end={end}"+ )]+ InvalidTensorRepresentation {+ key: String,+ shape: Box<[u32]>,+ data_type: DataType,+ expected_size: usize,+ global_offset: u64,+ begin: u64,+ end: u64,+ },+ #[error("Unread tensors: {0:?}")]+ UnreadTensors(Vec<String>),+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+}++#[derive(Deserialize)]+struct Header {+ #[serde(default, rename = "__metadata__")]+ metadata: HashMap<String, String>,+ #[serde(flatten)]+ tensors: HashMap<String, TensorInfo>,+}++#[derive(Deserialize)]+struct TensorInfo {+ dtype: DataType,+ shape: Box<[u32]>,+ data_offsets: (u64, u64),+ #[serde(skip)]+ read: Cell<bool>,+}++pub struct ParameterLoader<'a, B: Backend> {+ file: &'a File,+ context: &'a B::Context,+ header: Header,+ data_offset: u64,+}++impl<'a, B: Backend> ParameterLoader<'a, B> {+ pub fn new(+ file: &'a File,+ context: &'a B::Context,+ ) -> Result<Self, ParameterLoaderError<B>> {+ let mut header_size = [0; 8];+ file.read_exact_at(&mut header_size, 0)?;+ let header_size = u64::from_le_bytes(header_size);+ let mut header = vec![0; header_size as usize];+ file.read_exact_at(&mut header, 8)?;+ let header: Header = serde_json::from_slice(&header)?;+ Ok(Self {+ file,+ context,+ header,+ data_offset: 8 + header_size,+ })+ }++ pub fn tree(&self) -> ParameterTree<'_, B> {+ ParameterTree {+ loader: self,+ prefix: String::new(),+ }+ }++ pub fn assert_all_tensors_read(&self) -> Result<(), ParameterLoaderError<B>> {+ let mut unread = self+ .header+ .tensors+ .iter()+ .filter(|(_, tensor)| !tensor.read.get())+ .map(|(key, _)| key.clone())+ .collect::<Vec<_>>();+ if unread.is_empty() {+ Ok(())+ } else {+ unread.sort();+ Err(ParameterLoaderError::UnreadTensors(unread))+ }+ }+}++pub struct ParameterTree<'a, B: Backend> {+ loader: &'a ParameterLoader<'a, B>,+ prefix: String,+}++impl<B: Backend> ParameterTree<'_, B> {+ pub fn subtree(+ &self,+ name: &str,+ ) -> Self {+ Self {+ loader: self.loader,+ prefix: format!("{}{name}.", self.prefix),+ }+ }++ pub fn metadata<T: DeserializeStrictOwned>(+ &self,+ name: &str,+ ) -> Result<T, ParameterLoaderError<B>> {+ let key = format!("{}{name}", self.prefix);+ let json =+ self.loader.header.metadata.get(&key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.clone()))?;+ serde_json::from_str(json).map_err(|source| ParameterLoaderError::MetadataDeserializationError {+ key,+ source,+ })+ }++ fn tensor(+ &self,+ name: &str,+ shape: &[u32],+ dtype: DataType,+ ) -> Result<&TensorInfo, ParameterLoaderError<B>> {+ let key = format!("{}{name}", self.prefix);+ let tensor =+ self.loader.header.tensors.get(&key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.clone()))?;+ if (tensor.shape.as_ref(), tensor.dtype) != (shape, dtype) {+ return Err(ParameterLoaderError::InvalidTensor {+ key,+ shape: tensor.shape.clone(),+ data_type: tensor.dtype,+ expected_shape: shape.into(),+ expected_data_type: dtype,+ });+ }+ let (start, end) = tensor.data_offsets;+ let expected_size = size_for_shape(shape, dtype);+ if end.checked_sub(start) != Some(expected_size as u64) || self.loader.data_offset.checked_add(end).is_none() {+ return Err(ParameterLoaderError::InvalidTensorRepresentation {+ key,+ shape: tensor.shape.clone(),+ data_type: tensor.dtype,+ expected_size,+ global_offset: self.loader.data_offset,+ begin: start,+ end,+ });+ }+ Ok(tensor)+ }++ pub fn read_buffer(+ &self,+ name: &str,+ shape: &[u32],+ dtype: DataType,+ ) -> Result<B::GlobalBuffer, ParameterLoaderError<B>> {+ let tensor = self.tensor(name, shape, dtype)?;+ let (start, end) = tensor.data_offsets;+ let mut buffer =+ self.loader.context.create_buffer((end - start) as usize).map_err(ParameterLoaderError::BackendError)?;+ self.loader.file.read_exact_at(buffer.as_slice_mut::<u8>(), self.loader.data_offset + start)?;+ tensor.read.set(true);+ Ok(buffer)+ }++ pub fn read_slice<T: DataTypeRepr>(+ &self,+ name: &str,+ shape: &[u32],+ ) -> Result<Box<[T]>, ParameterLoaderError<B>> {+ let tensor = self.tensor(name, shape, T::data_type())?;+ let (start, end) = tensor.data_offsets;+ let size = (end - start) as usize;+ let mut data = vec![T::zeroed(); size / size_of::<T>()];+ self.loader.file.read_exact_at(bytemuck::cast_slice_mut(&mut data), self.loader.data_offset + start)?;+ tensor.read.set(true);+ Ok(data.into_boxed_slice())+ }+}diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex b3c503b41..e29e7a831 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -27,7 +27,7 @@ use crate::{sampling::{PRng, Sampling, SamplingMethod},weaver::{ProposalNode, Weaver, WeaverEncodeError, WeaverNewError, WeaverTreeShape},},- parameters::{ParameterLoader, ParameterLoaderError},+ safetensors::{ParameterLoader, ParameterLoaderError},trie::TrieNode,};@@ -136,7 +136,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {}).transpose()?;- weight_loader.tree().assert_all_tensors_validated()?;+ weight_loader.assert_all_tensors_read()?;let sampling = Sampling::new(config.draft_config.vocab_size, data_type);diff --git a/crates/uzu-engine/src/utils/dispatch_dtype.rs b/crates/uzu-engine/src/utils/dispatch_dtype.rsnew file mode 100644index 000000000..38a6fc288--- /dev/null+++ b/crates/uzu-engine/src/utils/dispatch_dtype.rs@@ -0,0 +1,12 @@+macro_rules! dispatch_dtype {+ (|()| $body:expr) => { $body };+ (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {+ match $dt {+ $crate::data_type::DataType::F16 => { type $T = half::f16; $crate::utils::dispatch_dtype::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }+ $crate::data_type::DataType::BF16 => { type $T = half::bf16; $crate::utils::dispatch_dtype::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }+ $crate::data_type::DataType::F32 => { type $T = f32; $crate::utils::dispatch_dtype::dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }+ other => panic!("Unsupported data type: {other:?}"),+ }+ };+}+pub(crate) use dispatch_dtype;diff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rsindex e0f6473d3..0137b4219 100644--- a/crates/uzu-engine/src/utils/mod.rs+++ b/crates/uzu-engine/src/utils/mod.rs@@ -1,6 +1,8 @@+pub mod dispatch_dtype;pub mod downcast;pub mod maybe_mut;#[cfg(backend = "cpu")]pub mod pointers;+pub mod size_for_shape;pub mod strict_serde;pub mod version;diff --git a/crates/uzu-engine/src/array/util.rs b/crates/uzu-engine/src/utils/size_for_shape.rssimilarity index 100%rename from crates/uzu-engine/src/array/util.rsrename to crates/uzu-engine/src/utils/size_for_shape.rsdiff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_test.rsindex 46ab80e16..7e6fba6e6 100644--- a/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,20 +12,20 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {data: Box<[T]>,act_type: ActivationType,in_place: bool,}-fn make_data<T: ArrayElement + Float>(n: usize) -> Vec<T> {+fn make_data<T: DataTypeRepr + Float>(n: usize) -> Vec<T> {let mut data = Vec::with_capacity(n);for i in 0..n {// Use a range of values including negative to exercise both branches of activations@@ -35,7 +34,7 @@ fn make_data<T: ArrayElement + Float>(n: usize) -> Vec<T> {data}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let kernel = <<B as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, T::data_type(), input.in_place).expect("Failed to create ActivationKernel");@@ -58,7 +57,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {}}-fn get_test_data<T: ArrayElement + Float>(+fn get_test_data<T: DataTypeRepr + Float>(act_type: ActivationType,in_place: bool,) -> (Input<T>, Vec<T>) {@@ -76,7 +75,7 @@ fn get_test_data<T: ArrayElement + Float>(}/// Large input to exercise more threads on GPU.-fn get_test_data_large<T: ArrayElement + Float>(+fn get_test_data_large<T: DataTypeRepr + Float>(act_type: ActivationType,in_place: bool,) -> (Input<T>, Vec<T>) {@@ -93,7 +92,7 @@ fn get_test_data_large<T: ArrayElement + Float>((input, expected)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected: &[T],test_name: &str,@@ -111,7 +110,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(});}-fn test_activation<T: ArrayElement + Float + Debug + Display>(+fn test_activation<T: DataTypeRepr + Float + Debug + Display>(act_type: ActivationType,in_place: bool,) {@@ -128,7 +127,7 @@ fn test_activation<T: ArrayElement + Float + Debug + Display>(test_internal::<T>(&input, &expected, &label);}-fn test_activation_large<T: ArrayElement + Float + Debug + Display>(+fn test_activation_large<T: DataTypeRepr + Float + Debug + Display>(act_type: ActivationType,in_place: bool,) {diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rsindex e58fd30f8..13a5abb73 100644--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,@@ -13,6 +12,7 @@ use crate::{},cpu::Cpu,},+ data_type::DataTypeRepr,tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend},};@@ -22,7 +22,7 @@ enum TransformOrder {Output,}-fn run<T: ArrayElement + Float + Debug, B: Backend>(+fn run<T: DataTypeRepr + Float + Debug, B: Backend>(data: &[T],factors: &[i32],channel_count: usize,@@ -59,7 +59,7 @@ fn run<T: ArrayElement + Float + Debug, B: Backend>(})}-fn check<T: ArrayElement + Float + Debug>(tolerance: f64) {+fn check<T: DataTypeRepr + Float + Debug>(tolerance: f64) {for (order, in_place) in [(TransformOrder::Input, false),(TransformOrder::Input, true),diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rsindex 4353c0c7a..8f6df0224 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,20 +12,21 @@ use crate::{},cpu::Cpu,},+ data_type::DataTypeRepr,tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {keys: Box<[T]>,values: Box<[T]>,copies: Vec<Copy>,element_dim: u32,}-fn make_data<T: ArrayElement + Float>(+fn make_data<T: DataTypeRepr + Float>(max_sequence_length: usize,element_dim: usize,) -> (Vec<T>, Vec<T>) {@@ -40,7 +40,7 @@ fn make_data<T: ArrayElement + Float>((keys, values)}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {let context = B::Context::new().expect("Failed to create Context");let kernel = <<B as Backend>::Kernels as Kernels>::KVCacheUpdateKernel::new(&context, T::data_type()).expect("Failed to create KVCacheUpdateKernel");@@ -63,7 +63,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,}/// Single copy between two different positions.-fn get_test_data_single_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {+fn get_test_data_single_copy<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>, Vec<T>) {let num_heads = 2usize;let max_sequence_length = 4usize;let head_dim = 3usize;@@ -88,7 +88,7 @@ fn get_test_data_single_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Ve}/// Multiple copies.-fn get_test_data_multi_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {+fn get_test_data_multi_copy<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>, Vec<T>) {let num_heads = 2usize;let max_sequence_length = 8usize;let head_dim = 4usize;@@ -123,7 +123,7 @@ fn get_test_data_multi_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec}/// No copies — data should remain unchanged.-fn get_test_data_no_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {+fn get_test_data_no_copy<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>, Vec<T>) {let num_heads = 2usize;let max_sequence_length = 4usize;let head_dim = 3usize;@@ -145,7 +145,7 @@ fn get_test_data_no_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>}/// Larger dimensions to exercise more threads.-fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {+fn get_test_data_large<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>, Vec<T>) {let num_heads = 8usize;let max_sequence_length = 64usize;let head_dim = 128usize;@@ -179,7 +179,7 @@ fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>)(input, expected_keys, expected_values)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected_keys: &[T],expected_values: &[T],@@ -195,22 +195,22 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(});}-fn test_single_copy_internal<T: ArrayElement + Float + Debug + Display>() {+fn test_single_copy_internal<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected_keys, expected_values) = get_test_data_single_copy::<T>();test_internal::<T>(&input, &expected_keys, &expected_values, "single_copy");}-fn test_multi_copy_internal<T: ArrayElement + Float + Debug + Display>() {+fn test_multi_copy_internal<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected_keys, expected_values) = get_test_data_multi_copy::<T>();test_internal::<T>(&input, &expected_keys, &expected_values, "multi_copy");}-fn test_no_copy_internal<T: ArrayElement + Float + Debug + Display>() {+fn test_no_copy_internal<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected_keys, expected_values) = get_test_data_no_copy::<T>();test_internal::<T>(&input, &expected_keys, &expected_values, "no_copy");}-fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {+fn test_large_internal<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected_keys, expected_values) = get_test_data_large::<T>();test_internal::<T>(&input, &expected_keys, &expected_values, "large");}diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rsindex 0b5ef4e9a..cdaad5fb1 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<InputT: ArrayElement + Float, ScaleT: ArrayElement + Float, OutputT: ArrayElement + Float> {+struct Input<InputT: DataTypeRepr + Float, ScaleT: DataTypeRepr + Float, OutputT: DataTypeRepr + Float> {qkvg: Box<[OutputT]>,scales: Box<[ScaleT]>,batch_size: u32,@@ -39,10 +38,10 @@ struct Input<InputT: ArrayElement + Float, ScaleT: ArrayElement + Float, OutputT}fn get_test_data<- InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,- OutputT: ArrayElement + Float,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ ScaleT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float,+ AccumT: DataTypeRepr + Float,>(head_offset: u32,head_count: u32,@@ -116,10 +115,10 @@ fn get_test_data<fn get_output<B: Backend,- InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,- OutputT: ArrayElement + Float,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ ScaleT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float,+ AccumT: DataTypeRepr + Float,>(input: &Input<InputT, ScaleT, OutputT>) -> Vec<OutputT> {@@ -160,10 +159,10 @@ fn get_output<}fn test_internal<- InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,- OutputT: ArrayElement + Float + Debug + Display,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ ScaleT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float + Debug + Display,+ AccumT: DataTypeRepr + Float,>(input: &Input<InputT, ScaleT, OutputT>,expected: &[OutputT],@@ -192,10 +191,10 @@ fn test_internal<}fn test_q_norm<- InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,- OutputT: ArrayElement + Float + Debug + Display,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ ScaleT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float + Debug + Display,+ AccumT: DataTypeRepr + Float,>() {for full_layer in [true, false] {// Normalize Q heads: head_offset=0, head_count=num_q_heads(4)@@ -205,10 +204,10 @@ fn test_q_norm<}fn test_k_norm<- InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,- OutputT: ArrayElement + Float + Debug + Display,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ ScaleT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float + Debug + Display,+ AccumT: DataTypeRepr + Float,>() {for full_layer in [true, false] {// Normalize K heads: head_offset=num_q_heads(4), head_count=num_kv_heads(2)@@ -218,20 +217,20 @@ fn test_k_norm<}fn test_v_norm_no_scales<- InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,- OutputT: ArrayElement + Float + Debug + Display,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ ScaleT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float + Debug + Display,+ AccumT: DataTypeRepr + Float,>() {let (input, expected) = get_test_data::<InputT, ScaleT, OutputT, AccumT>(6, 2, true, false);test_internal::<InputT, ScaleT, OutputT, AccumT>(&input, &expected);}fn test_addressing<- InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,- OutputT: ArrayElement + Float + Debug + Display,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ ScaleT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float + Debug + Display,+ AccumT: DataTypeRepr + Float,>() {// Test that Q norm only modifies Q heads and leaves K/V untouchedlet (input, _expected) = get_test_data::<InputT, ScaleT, OutputT, AccumT>(0, 4, false, true);@@ -273,10 +272,10 @@ fn test_addressing<}fn test_v_addressing<- InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,- OutputT: ArrayElement + Float + Debug + Display,- AccumT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ ScaleT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float + Debug + Display,+ AccumT: DataTypeRepr + Float,>() {// Weightless V norm must modify only V heads and leave Q/K untouched.let (input, _expected) = get_test_data::<InputT, ScaleT, OutputT, AccumT>(6, 2, true, false);diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rsindex fd13f9aff..b8beb7b9d 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,6 +12,7 @@ use crate::{},cpu::Cpu,},+ data_type::DataTypeRepr,tests::helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},};@@ -23,7 +23,7 @@ struct Config {suffix_length: u32,}-fn get_output<T: ArrayElement + Float, B: Backend>(+fn get_output<T: DataTypeRepr + Float, B: Backend>(qkvg_data: &[T],output_data: &[T],config: &Config,@@ -53,7 +53,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(buffer_to_vec(&output)}-fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {+fn run_test<T: DataTypeRepr + Float + Debug>(config: &Config) {let size = (config.suffix_length * config.num_heads * config.head_dim) as usize;let gate_dim = (config.num_heads * config.head_dim) as usize;let gate_offset = ((config.num_heads + 2 * config.num_kv_heads) * config.head_dim) as usize;diff --git a/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs b/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rsindex e2f45a15e..0746f403f 100644--- a/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/embedding/input_embedding_lookup_test.rs@@ -6,7 +6,6 @@ use rstest::rstest;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, Context, Kernels,@@ -20,6 +19,7 @@ use crate::{},cpu::Cpu,},+ data_type::DataTypeRepr,tests::{assert::assert_eq_float,helpers::{@@ -41,14 +41,14 @@ fn pattern((0..len).map(|index| (index * seed + 9) as u8).collect()}-fn floats<T: ArrayElement + Float>(+fn floats<T: DataTypeRepr + Float>(len: u32,value: impl Fn(u32) -> f32,) -> Vec<T> {(0..len).map(|index| T::from(value(index)).unwrap()).collect()}-fn lookup<B: Backend, T: ArrayElement + Float>(+fn lookup<B: Backend, T: DataTypeRepr + Float>(table_kind: EmbeddingTableKind,quantization: Option<Quantization>,use_hadamard: bool,@@ -139,7 +139,7 @@ fn lookup<B: Backend, T: ArrayElement + Float>(buffer_to_vec::<B, T>(&output)}-fn check<T: ArrayElement + Float + Display>(+fn check<T: DataTypeRepr + Float + Display>(table_kind: EmbeddingTableKind,quantization: Option<Quantization>,use_hadamard: bool,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rsindex 2d7466022..0ba84c2c2 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,@@ -15,14 +14,14 @@ use crate::{cpu::Cpu,},config::clipping::ClippingBounds,- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend, for_each_non_cpu_backend},},};-struct InterleavedInput<T: ArrayElement + Float> {+struct InterleavedInput<T: DataTypeRepr + Float> {fused_up: Box<[T]>,hadamard_factors: Box<[i32]>,gated_dim: u32,@@ -30,7 +29,7 @@ struct InterleavedInput<T: ArrayElement + Float> {act_type: ActivationType,}-fn interleaved_input<T: ArrayElement + Float>(act_type: ActivationType) -> InterleavedInput<T> {+fn interleaved_input<T: DataTypeRepr + Float>(act_type: ActivationType) -> InterleavedInput<T> {let gated_dim = 128u32;let batch_dim = 4u32;let fused_length = (batch_dim * 2 * gated_dim) as usize;@@ -47,7 +46,7 @@ fn interleaved_input<T: ArrayElement + Float>(act_type: ActivationType) -> Inter}}-fn run_interleaved<T: ArrayElement + Float, B: Backend>(+fn run_interleaved<T: DataTypeRepr + Float, B: Backend>(input: &InterleavedInput<T>,use_hadamard: bool,) -> Vec<T> {@@ -80,7 +79,7 @@ fn run_interleaved<T: ArrayElement + Float, B: Backend>(buffer_to_vec::<B, T>(&output)}-fn interleaved_test<T: ArrayElement + Float + Debug + Display>(act_type: ActivationType) {+fn interleaved_test<T: DataTypeRepr + Float + Debug + Display>(act_type: ActivationType) {let eps = if matches!(T::data_type(), DataType::BF16) {0.02f32} else {@@ -184,7 +183,7 @@ fn test_gated_act_mul_interleaved_gelu_exact_f32() {interleaved_test::<f32>(ActivationType::GELUExact);}-struct SeparateInput<T: ArrayElement + Float> {+struct SeparateInput<T: DataTypeRepr + Float> {gate_out: Box<[T]>,per_layer_input: Box<[T]>,gated_dim: u32,@@ -194,7 +193,7 @@ struct SeparateInput<T: ArrayElement + Float> {act_type: ActivationType,}-fn separate_input<T: ArrayElement + Float>() -> (SeparateInput<T>, Vec<T>) {+fn separate_input<T: DataTypeRepr + Float>() -> (SeparateInput<T>, Vec<T>) {let gate_out = [1.0_f32, 2.0, 3.0, 4.0].into_iter().map(|value| T::from(value).unwrap()).collect::<Vec<_>>();let per_layer_input = [0.0_f32, 0.0, 10.0, 20.0, 30.0, 40.0, 0.0, 0.0, 50.0, 60.0, 70.0, 80.0].into_iter()@@ -217,7 +216,7 @@ fn separate_input<T: ArrayElement + Float>() -> (SeparateInput<T>, Vec<T>) {)}-fn run_separate<T: ArrayElement + Float, B: Backend>(input: &SeparateInput<T>) -> Vec<T> {+fn run_separate<T: DataTypeRepr + Float, B: Backend>(input: &SeparateInput<T>) -> Vec<T> {let context = B::Context::new().expect("create context");let kernel =GatedActMul::<B>::full_precision(&context, T::data_type(), false, false, GatedActMulSettings::default())@@ -245,7 +244,7 @@ fn run_separate<T: ArrayElement + Float, B: Backend>(input: &SeparateInput<T>) -buffer_to_vec::<B, T>(&output)}-fn separate_test<T: ArrayElement + Float + Debug>() {+fn separate_test<T: DataTypeRepr + Float + Debug>() {let (input, expected) = separate_input::<T>();for_each_backend!(|B| {let output = run_separate::<T, B>(&input);@@ -263,7 +262,7 @@ fn test_gated_act_mul_separate_bf16() {separate_test::<bf16>();}-fn transformed_interleaved_test<T: ArrayElement + Float + Debug + Display>(+fn transformed_interleaved_test<T: DataTypeRepr + Float + Debug + Display>(gate_clipping: ClippingBounds,value_clipping: ClippingBounds,) {diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rsindex 573cc6eca..bd0c4bde2 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs@@ -1,8 +1,15 @@use half::bf16;use uzu_engine_macros::uzu_test;+#[cfg(backend = "metal")]+use crate::{+ backends::{+ common::kernel::{Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel},+ metal::Metal,+ },+ tests::helpers::buffer_prefix_to_vec,+};use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -10,17 +17,9 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend},};-#[cfg(backend = "metal")]-use crate::{- backends::{- common::kernel::{Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel},- metal::Metal,- },- tests::helpers::buffer_prefix_to_vec,-};#[cfg(backend = "metal")]fn run_conv_update<B: Backend>(@@ -60,7 +59,7 @@ fn run_conv_update<B: Backend>((out, new_state)}-fn run_delta_net_update<B: Backend, T: ArrayElement>(+fn run_delta_net_update<B: Backend, T: DataTypeRepr>(in_proj: &[T],a_log: &[f32],dt_bias: &[f32],@@ -108,7 +107,7 @@ fn run_delta_net_update<B: Backend, T: ArrayElement>((out, new_state)}-fn to_activation<T: ArrayElement>(values: &[f32]) -> Vec<T> {+fn to_activation<T: DataTypeRepr>(values: &[f32]) -> Vec<T> {values.iter().map(|&value| <T as num_traits::NumCast>::from(value).expect("input to activation dtype")).collect()}@@ -248,7 +247,7 @@ fn test_delta_net_conv_scan() {// DeltaNetUpdate (decode)// The reference is the CPU f32 kernel on the same rounded inputs; a_log, dt_bias, norm weight and state stay f32-fn test_delta_net_update_impl<T: ArrayElement>(+fn test_delta_net_update_impl<T: DataTypeRepr>(num_v_heads: usize,num_k_heads: usize,head_k_dim: usize,@@ -318,7 +317,7 @@ fn test_delta_net_update_qwen35_shapes_bf16() {// DeltaNetPrefill + NormGate-fn run_prefill_with_norm_gate_typed<B: Backend, T: ArrayElement>(+fn run_prefill_with_norm_gate_typed<B: Backend, T: DataTypeRepr>(in_proj: &[T],a_log: &[f32],dt_bias: &[f32],@@ -418,7 +417,7 @@ fn run_prefill_with_norm_gate_typed<B: Backend, T: ArrayElement>((out, state)}-fn test_prefill_norm_gate_impl<T: ArrayElement>(+fn test_prefill_norm_gate_impl<T: DataTypeRepr>(num_v_heads: usize,num_k_heads: usize,head_k_dim: usize,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rsindex 52ff22aec..3b15fa683 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs@@ -8,11 +8,11 @@ use num_traits::Float;use uzu_engine_macros::uzu_bench;use crate::{- array::ArrayElement,backends::{common::{Backend, Context, Kernels, kernel::BuildTreeOutKernel},metal::Metal,},+ data_type::DataTypeRepr,tests::{helpers::{create_buffer, create_buffer_with_data},matmul::iter_encode_loop_named,@@ -44,7 +44,7 @@ struct TreeOutBuffers {o: <Metal as Backend>::GlobalBuffer,}-fn make_buffers<T: ArrayElement + Float>(+fn make_buffers<T: DataTypeRepr + Float>(context: &<Metal as Backend>::Context,batch_size: usize,tree_size: usize,@@ -79,7 +79,7 @@ fn bench_build_tree_out(c: &mut Criterion) {bench_build_tree_out_type::<f32>(c, &context, "F32");}-fn bench_build_tree_out_type<T: ArrayElement + Float>(+fn bench_build_tree_out_type<T: DataTypeRepr + Float>(c: &mut Criterion,context: &<Metal as Backend>::Context,data_type_name: &str,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rsindex 698ab0b97..113f49cf8 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -14,6 +13,7 @@ use crate::{cpu::Cpu,metal::Metal,},+ data_type::DataTypeRepr,tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},@@ -44,7 +44,7 @@ const BUILD_TREE_OUT_PATHS: &[(&str, bool, bool)] = &[("MXU/H0Direct/Rows16_VCols32_SG4", true, false),];-fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {+fn make_inputs<T: DataTypeRepr + Float>(shape: Shape) -> Inputs<T> {let q_len = shape.batch_size * shape.tree_size * shape.qk_heads * shape.head_k_dim;let prefix_len = shape.batch_size * shape.tree_size * shape.value_heads;let qkd_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.tree_size;@@ -62,7 +62,7 @@ fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {}}-fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(+fn run_build_tree_out<B: Backend, T: DataTypeRepr + Float>(shape: Shape,inputs: &Inputs<T>,use_h0: bool,@@ -108,7 +108,7 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(buffer_to_vec(&o)}-fn check_shape<T: ArrayElement + Float + std::fmt::Display>(+fn check_shape<T: DataTypeRepr + Float + std::fmt::Display>(shape: Shape,eps: f32,) {diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rsindex b6e37ef4c..45ac4d086 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs@@ -9,8 +9,13 @@ use num_traits::Float;use uzu_engine_macros::uzu_bench;use uzu_engine_macros::uzu_test;+#[cfg(backend = "metal")]+use crate::{+ backends::metal::Metal,+ data_type::DataType,+ tests::{cold_pool::ColdPool, helpers::buffer_size_bytes, matmul::iter_encode_loop_named},+};use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -18,17 +23,12 @@ use crate::{},cpu::Cpu,},+ data_type::DataTypeRepr,tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},},};-#[cfg(backend = "metal")]-use crate::{- backends::metal::Metal,- data_type::DataType,- tests::{cold_pool::ColdPool, helpers::buffer_size_bytes, matmul::iter_encode_loop_named},-};const TREE_SIZE: usize = 128;const HEAD_DIM: usize = 128;@@ -36,7 +36,7 @@ const PATH: &[u32] = &[0, 1, 3, 7, 15, 31, 47, 63, 79, 95, 111, 119, 123, 125, 1const TEST_NUM_V_HEADS: usize = 6;const TEST_NUM_K_HEADS: usize = 2;-fn run<B: Backend, T: ArrayElement + Float>(accepted_indices: &[u32]) -> Vec<f32> {+fn run<B: Backend, T: DataTypeRepr + Float>(accepted_indices: &[u32]) -> Vec<f32> {let context = B::Context::new().expect("context");let kernel = <<B as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(&context,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rsindex 502c5a5da..0186e3d30 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -14,7 +13,7 @@ use crate::{cpu::Cpu,metal::Metal,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},@@ -53,7 +52,7 @@ fn make_inputs(tree_size: usize) -> Inputs {}}-fn get_output<B: Backend, T: ArrayElement + Float>(+fn get_output<B: Backend, T: DataTypeRepr + Float>(q: &[T],k: &[T],trie: &[u32],@@ -113,7 +112,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>((buffer_to_vec(&a_packed), buffer_to_vec(&qkd), buffer_to_vec(&a_inv), buffer_to_vec(&kh0))}-fn check_type<T: ArrayElement + Float>(+fn check_type<T: DataTypeRepr + Float>(inputs: &Inputs,tree_size: usize,scale: f32,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rsindex ffde218d7..a6530546e 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs@@ -4,7 +4,6 @@ use half::bf16;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -12,7 +11,7 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},@@ -86,7 +85,7 @@ const CASES: &[SolveCase] = &[},];-fn run_case<B: Backend, T: ArrayElement + Copy>(+fn run_case<B: Backend, T: DataTypeRepr + Copy>(case: SolveCase,bv: u32,use_h0: bool,diff --git a/crates/uzu-engine/unit/backends/common/kernel/logit_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/logit_transform_test.rsindex 519858c70..5d1568df2 100644--- a/crates/uzu-engine/unit/backends/common/kernel/logit_transform_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/logit_transform_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},},};-fn get_output<T: ArrayElement + Float, B: Backend>(+fn get_output<T: DataTypeRepr + Float, B: Backend>(logits: &[T],scale: f32,soft_cap: Option<f32>,@@ -38,7 +37,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(buffer_to_vec::<B, T>(&logits_buffer)}-fn test_logit_transform<T: ArrayElement + Float + Display>(+fn test_logit_transform<T: DataTypeRepr + Float + Display>(scale: f32,soft_cap: Option<f32>,) {diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rsindex 1eb047336..b8bf79104 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs@@ -5,8 +5,9 @@ use num_traits::Float;use rstest::rstest;use uzu_engine_macros::uzu_test;+#[cfg(backend = "metal")]+use crate::{backends::metal::Metal, tests::matmul::run_quant_cpu};use crate::{- array::ArrayElement,backends::{common::{Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,@@ -18,16 +19,15 @@ use crate::{},cpu::Cpu,},+ data_type::DataTypeRepr,tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},matmul::{QuantBuffers, QuantInput},},};-#[cfg(backend = "metal")]-use crate::{backends::metal::Metal, tests::matmul::run_quant_cpu};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {a: Box<[T]>,b: Box<[T]>,m: usize,@@ -37,7 +37,7 @@ struct Input<T: ArrayElement + Float> {soft_cap: Option<f32>,}-fn get_test_data<T: ArrayElement + Float>(+fn get_test_data<T: DataTypeRepr + Float>(m: usize,k: usize,n: usize,@@ -60,7 +60,7 @@ fn get_test_data<T: ArrayElement + Float>(}// Encode one GEMV (dense, or a per-row B-row gather when `gather_indices` is set) and copy out.-fn run_gemv<B: Backend, T: ArrayElement + Float>(+fn run_gemv<B: Backend, T: DataTypeRepr + Float>(context: &B::Context,a: impl BufferRef<Backend = B>,b: MatmulB<impl BufferRef<Backend = B>>,@@ -104,7 +104,7 @@ fn run_gemv<B: Backend, T: ArrayElement + Float>(buffer_to_vec::<B, T>(&d)}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let a = create_buffer_with_data::<B, T>(&context, &input.a);let weights = create_buffer_with_data::<B, T>(&context, &input.b);@@ -123,7 +123,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {)}-fn test<T: ArrayElement + Float + Debug + Display>(+fn test<T: DataTypeRepr + Float + Debug + Display>(m: usize,k: usize,n: usize,@@ -198,7 +198,7 @@ fn gemv_f32(test::<f32>(m, k, n, 0.01);}-fn assert_gather<T: ArrayElement + Float + Debug + Display>(+fn assert_gather<T: DataTypeRepr + Float + Debug + Display>(dense: &[T],gather: &[T],ids: &[u32],@@ -232,7 +232,7 @@ macro_rules! check_gather {}};}-fn fp_gather_case<T: ArrayElement + Float + Debug + Display>(+fn fp_gather_case<T: DataTypeRepr + Float + Debug + Display>(soft_cap: Option<f32>,eps: f32,) {diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rsindex 3f03a49ab..0370f311d 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs@@ -4,12 +4,12 @@ use num_traits::Float;use uzu_engine_macros::uzu_bench;use crate::{- array::ArrayElement,backends::common::{Backend, Context,gpu_types::quantization_method::QuantizationMethod,kernel::{Kernels, matmul::MatmulKernel},},+ data_type::DataTypeRepr,tests::{cold_pool::ColdPool,helpers::for_each_backend,@@ -18,7 +18,7 @@ use crate::{},};-fn bench_gemv_typed<B: Backend, T: ArrayElement + Float>(+fn bench_gemv_typed<B: Backend, T: DataTypeRepr + Float>(c: &mut Criterion,context: &B::Context,label: &str,diff --git a/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs b/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rsindex 5869a54cf..a9b2b88af 100644--- a/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,7 +12,7 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend, for_each_non_cpu_backend},@@ -22,9 +21,9 @@ use crate::{fn get_output<B: Backend,- InputT: ArrayElement + Float,- AffineT: ArrayElement + Float,- OutputT: ArrayElement + Float,+ InputT: DataTypeRepr + Float,+ AffineT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float,>(input: &[InputT],scales: Option<&[AffineT]>,@@ -81,9 +80,9 @@ fn get_output<}fn test_internal<- InputT: ArrayElement + Float,- AffineT: ArrayElement + Float,- OutputT: ArrayElement + Float + Debug + Display,+ InputT: DataTypeRepr + Float,+ AffineT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float + Debug + Display,>(has_scales: bool,full_layer: bool,@@ -138,9 +137,9 @@ fn test_internal<}fn test_normalization<- InputT: ArrayElement + Float,- AffineT: ArrayElement + Float,- OutputT: ArrayElement + Float + Debug + Display,+ InputT: DataTypeRepr + Float,+ AffineT: DataTypeRepr + Float,+ OutputT: DataTypeRepr + Float + Debug + Display,>() {for has_scales in [true, false] {for full_layer in [true, false] {@@ -150,7 +149,7 @@ fn test_normalization<}// The fused transform has to match plain normalization followed by the standalone input RHT-fn test_hadamard<T: ArrayElement + Float + Debug + Display>() {+fn test_hadamard<T: DataTypeRepr + Float + Debug + Display>() {let batch_size = 2u32;let element_count = 64u32;let epsilon = 1e-6f32;diff --git a/crates/uzu-engine/unit/backends/common/kernel/pooling/pooling_mean_test.rs b/crates/uzu-engine/unit/backends/common/kernel/pooling/pooling_mean_test.rsindex c883ec212..ecd4101ae 100644--- a/crates/uzu-engine/unit/backends/common/kernel/pooling/pooling_mean_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/pooling/pooling_mean_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,21 +12,21 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {input: Box<[T]>,seq_len: u32,hidden_dim: u32,batch_size: u32,}-fn get_test_data<T: ArrayElement + Float>(+fn get_test_data<T: DataTypeRepr + Float>(batch_size: u32,seq_len: u32,hidden_dim: u32,@@ -46,7 +45,7 @@ fn get_test_data<T: ArrayElement + Float>((input, expected)}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let kernel = <<B as Backend>::Kernels as Kernels>::PoolingMeanKernel::new(&context, T::data_type())@@ -64,7 +63,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {buffer_to_vec(&output)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(batch_size: u32,seq_len: u32,hidden_dim: u32,diff --git a/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rsindex 862b82951..87dbdd7ac 100644--- a/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/separable_causal_conv_test.rs@@ -2,7 +2,6 @@ use half::bf16;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -10,6 +9,7 @@ use crate::{},cpu::Cpu,},+ data_type::DataTypeRepr,tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},};diff --git a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_decode_test.rs b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_decode_test.rsindex a0ebeb5ac..3887ef5bc 100644--- a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_decode_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_decode_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {in_proj: Box<[T]>,w: Box<[f32]>,b: Option<Box<[f32]>>,@@ -32,7 +31,7 @@ struct Input<T: ArrayElement + Float> {model_dim: u32,}-fn get_output<T: ArrayElement + Float, B: Backend>(+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>,state_in_place: bool,) -> (Vec<T>, Vec<T>) {@@ -88,7 +87,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>((buffer_to_vec(&out), buffer_to_vec(&next_state))}-fn get_test_data_basic<T: ArrayElement + Float>(+fn get_test_data_basic<T: DataTypeRepr + Float>(model_dim: usize,kernel_size: usize,has_bias: bool,@@ -155,7 +154,7 @@ fn get_test_data_basic<T: ArrayElement + Float>((input, out, next_state)}-fn get_test_data_edge<T: ArrayElement + Float>(+fn get_test_data_edge<T: DataTypeRepr + Float>(model_dim: usize,kernel_size: usize,has_bias: bool,@@ -216,7 +215,7 @@ fn get_test_data_edge<T: ArrayElement + Float>((input, out, next_state)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected_out: &[T],expected_state: &[T],@@ -252,7 +251,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(});}-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {for state_in_place in [false, true] {let (input, expected_out, expected_state) = get_test_data_basic::<T>(64, 4, has_bias);@@ -261,7 +260,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {}}-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {for state_in_place in [false, true] {let (input, expected_out, expected_state) = get_test_data_basic::<T>(256, 4, has_bias);@@ -270,7 +269,7 @@ fn test_large<T: ArrayElement + Float + Debug + Display>() {}}-fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {+fn test_edge_small<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {for state_in_place in [false, true] {let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 2, has_bias);@@ -279,7 +278,7 @@ fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {}}-fn test_edge_kernel<T: ArrayElement + Float + Debug + Display>() {+fn test_edge_kernel<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {for state_in_place in [false, true] {let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 1, has_bias);diff --git a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_pack_test.rs b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_pack_test.rsindex 31f430b10..eb2324804 100644--- a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_pack_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_pack_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {state_in: Box<[T]>,in_proj: Box<[T]>,state_stride: u32,@@ -29,7 +28,7 @@ struct Input<T: ArrayElement + Float> {model_dim: u32,}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPackKernel::new(&context, T::data_type())@@ -66,7 +65,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {/// - columns [0..model_dim) = pre_gate/// - columns [2*model_dim..3*model_dim) = x_in/// padded (output): [(state_stride + suffix_len), model_dim]-fn get_input<T: ArrayElement + Float>(+fn get_input<T: DataTypeRepr + Float>(model_dim: usize,state_stride: usize,suffix_len: usize,@@ -106,7 +105,7 @@ fn get_input<T: ArrayElement + Float>((input, output)}-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.02f32} else {@@ -124,7 +123,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {});}-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.02f32} else {@@ -142,7 +141,7 @@ fn test_large<T: ArrayElement + Float + Debug + Display>() {});}-fn test_edge_single_token<T: ArrayElement + Float + Debug + Display>() {+fn test_edge_single_token<T: DataTypeRepr + Float + Debug + Display>() {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.02f32} else {@@ -160,7 +159,7 @@ fn test_edge_single_token<T: ArrayElement + Float + Debug + Display>() {});}-fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {+fn test_edge_small<T: DataTypeRepr + Float + Debug + Display>() {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.02f32} else {diff --git a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_prefill_test.rs b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_prefill_test.rsindex ceaa191f2..dd9f5a192 100644--- a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_prefill_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_prefill_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {padded: Box<[T]>,in_proj: Box<[T]>,w: Box<[f32]>,@@ -32,7 +31,7 @@ struct Input<T: ArrayElement + Float> {model_dim: u32,}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {let context = B::Context::new().expect("Failed to create Context");let has_bias = input.b.is_some();@@ -75,7 +74,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,(buffer_to_vec(&out), buffer_to_vec(&state_out))}-fn get_test_data_basic<T: ArrayElement + Float>(+fn get_test_data_basic<T: DataTypeRepr + Float>(model_dim: usize,suffix_len: usize,kernel_size: usize,@@ -139,7 +138,7 @@ fn get_test_data_basic<T: ArrayElement + Float>((input, out, state_out)}-fn get_test_data_edge<T: ArrayElement + Float>(+fn get_test_data_edge<T: DataTypeRepr + Float>(model_dim: usize,suffix_len: usize,kernel_size: usize,@@ -199,7 +198,7 @@ fn get_test_data_edge<T: ArrayElement + Float>((input, out, state_out)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected_out: &[T],expected_state: &[T],@@ -234,28 +233,28 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(});}-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {let (input, expected_out, expected_state) = get_test_data_basic::<T>(64, 4, 4, has_bias);test_internal(&input, &expected_out, &expected_state);}}-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {let (input, expected_out, expected_state) = get_test_data_basic::<T>(256, 16, 4, has_bias);test_internal(&input, &expected_out, &expected_state);}}-fn test_edge_single_token<T: ArrayElement + Float + Debug + Display>() {+fn test_edge_single_token<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 1, 4, has_bias);test_internal(&input, &expected_out, &expected_state);}}-fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {+fn test_edge_small<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 2, 2, has_bias);test_internal(&input, &expected_out, &expected_state);diff --git a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_trie_test.rs b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_trie_test.rsindex a7a51f846..0a9366c6d 100644--- a/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_trie_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/short_conv/short_conv_trie_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {in_proj: Box<[T]>,w: Box<[f32]>,b: Option<Box<[f32]>>,@@ -33,7 +32,7 @@ struct Input<T: ArrayElement + Float> {model_dim: u32,}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {let context = B::Context::new().expect("Failed to create Context");let has_bias = input.b.is_some();@@ -89,7 +88,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,}/// Linear chain: node 0 has parent -1 (root), node i has parent i-1.-fn get_test_data_basic<T: ArrayElement + Float>(+fn get_test_data_basic<T: DataTypeRepr + Float>(model_dim: usize,kernel_size: usize,suffix_len: usize,@@ -161,7 +160,7 @@ fn get_test_data_basic<T: ArrayElement + Float>(}/// Branching trie: node 0 is root child (-1), nodes 1 and 2 both parent from 0.-fn get_test_data_branching<T: ArrayElement + Float>(+fn get_test_data_branching<T: DataTypeRepr + Float>(model_dim: usize,kernel_size: usize,has_bias: bool,@@ -229,7 +228,7 @@ fn get_test_data_branching<T: ArrayElement + Float>(}/// Edge case: single node with parent -1.-fn get_test_data_edge<T: ArrayElement + Float>(+fn get_test_data_edge<T: DataTypeRepr + Float>(model_dim: usize,kernel_size: usize,has_bias: bool,@@ -293,7 +292,7 @@ fn get_test_data_edge<T: ArrayElement + Float>((input, out, suffix_state)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected_out: &[T],expected_state: &[T],@@ -328,35 +327,35 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(});}-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {let (input, expected_out, expected_state) = get_test_data_basic::<T>(64, 4, 3, has_bias);test_internal(&input, &expected_out, &expected_state);}}-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {let (input, expected_out, expected_state) = get_test_data_basic::<T>(256, 4, 3, has_bias);test_internal(&input, &expected_out, &expected_state);}}-fn test_branching<T: ArrayElement + Float + Debug + Display>() {+fn test_branching<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {let (input, expected_out, expected_state) = get_test_data_branching::<T>(64, 4, has_bias);test_internal(&input, &expected_out, &expected_state);}}-fn test_edge_small<T: ArrayElement + Float + Debug + Display>() {+fn test_edge_small<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 2, has_bias);test_internal(&input, &expected_out, &expected_state);}}-fn test_edge_kernel<T: ArrayElement + Float + Debug + Display>() {+fn test_edge_kernel<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [false, true] {let (input, expected_out, expected_state) = get_test_data_edge::<T>(4, 1, has_bias);test_internal(&input, &expected_out, &expected_state);diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rsindex e5a284ec2..5406e67ab 100644--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_decode_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {x: Box<[T]>,w: Box<[T]>,b: Option<Box<[T]>>,@@ -36,14 +35,14 @@ struct Input<T: ArrayElement + Float> {state_in_place: bool,}-struct Output<T: ArrayElement + Float> {+struct Output<T: DataTypeRepr + Float> {x_out: Vec<T>,b_out: Vec<T>,c_out: Vec<T>,next_state: Vec<T>,}-fn get_input<T: ArrayElement + Float>(+fn get_input<T: DataTypeRepr + Float>(kernel_size: u32,num_channels: u32,inner_dim: u32,@@ -90,7 +89,7 @@ fn get_input<T: ArrayElement + Float>(}}-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {let context = B::Context::new().expect("Failed to create Context");let has_bias = input.b.is_some();let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(@@ -180,7 +179,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T}}-fn get_test_data<T: ArrayElement + Float>(+fn get_test_data<T: DataTypeRepr + Float>(kernel_size: u32,num_channels: u32,inner_dim: u32,@@ -204,7 +203,7 @@ fn get_test_data<T: ArrayElement + Float>((input, expected)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected: &Output<T>,label: &str,@@ -248,7 +247,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {for state_in_place in [true, false] {for has_bias in [true, false] {let label = format!("basic(bias={has_bias},in_place={state_in_place})");@@ -259,23 +258,23 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {}// kernel_size=2 (state_stride=1, minimal state)-fn test_small_kernel<T: ArrayElement + Float + Debug + Display>() {+fn test_small_kernel<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 1, true, true, ActivationType::SILU);test_internal(&input, &expected, "small_kernel");}// Larger dimensions closer to real usage-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 1, true, true, ActivationType::SILU);test_internal(&input, &expected, "large");}-fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {+fn test_identity_activation<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, false, true, ActivationType::IDENTITY);test_internal(&input, &expected, "identity_activation");}-fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {+fn test_gelu_activation<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELUApprox);test_internal(&input, &expected, "gelu_activation");}diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rsindex f46798e6b..657d2c1d2 100644--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_pack_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {state_in: Box<[T]>,x: Box<[T]>,state_stride: u32,@@ -29,7 +28,7 @@ struct Input<T: ArrayElement + Float> {num_channels: u32,}-fn get_input<T: ArrayElement + Float>(+fn get_input<T: DataTypeRepr + Float>(state_stride: u32,row_stride: u32,suffix_len: u32,@@ -51,7 +50,7 @@ fn get_input<T: ArrayElement + Float>(}}-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type(), T::data_type()).expect("Failed to create Conv1dPackKernel");@@ -79,7 +78,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {buffer_to_vec(&padded)}-fn get_test_data<T: ArrayElement + Float>(+fn get_test_data<T: DataTypeRepr + Float>(state_stride: u32,row_stride: u32,suffix_len: u32,@@ -90,7 +89,7 @@ fn get_test_data<T: ArrayElement + Float>((input, expected)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected: &[T],label: &str,@@ -108,31 +107,31 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(});}-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {// 4 channels, state_stride=3 (kernel_size=4), 2 tokenslet (input, expected) = get_test_data::<T>(3, 4, 2, 4);test_internal(&input, &expected, "basic");}-fn test_single_token<T: ArrayElement + Float + Debug + Display>() {+fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {// 8 channels, state_stride=1, 1 tokenlet (input, expected) = get_test_data::<T>(1, 8, 1, 8);test_internal(&input, &expected, "single_token");}-fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {+fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {// Minimal state (kernel_size=2), many tokenslet (input, expected) = get_test_data::<T>(1, 4, 8, 4);test_internal(&input, &expected, "many_tokens");}-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {// Larger dimensions closer to real usagelet (input, expected) = get_test_data::<T>(3, 128, 16, 128);test_internal(&input, &expected, "large");}-fn test_channels_lt_row_stride<T: ArrayElement + Float + Debug + Display>() {+fn test_channels_lt_row_stride<T: DataTypeRepr + Float + Debug + Display>() {// num_channels < row_stride (padded channels that kernel doesn't write)let (input, expected) = get_test_data::<T>(2, 8, 3, 6);test_internal(&input, &expected, "channels_lt_row_stride");diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rsindex 5706500ba..22f221edc 100644--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_conv1d_scan_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {padded: Box<[T]>,w: Box<[T]>,b: Option<Box<[T]>>,@@ -34,14 +33,14 @@ struct Input<T: ArrayElement + Float> {activation_type: ActivationType,}-struct Output<T: ArrayElement + Float> {+struct Output<T: DataTypeRepr + Float> {x_out: Vec<T>,b_out: Vec<T>,c_out: Vec<T>,state_out: Vec<T>,}-fn get_input<T: ArrayElement + Float>(+fn get_input<T: DataTypeRepr + Float>(kernel_size: u32,num_channels: u32,inner_dim: u32,@@ -84,7 +83,7 @@ fn get_input<T: ArrayElement + Float>(}}-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {let context = B::Context::new().expect("Failed to create Context");let has_bias = input.b.is_some();let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&context, T::data_type(), has_bias)@@ -133,7 +132,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T}}-fn get_test_data<T: ArrayElement + Float>(+fn get_test_data<T: DataTypeRepr + Float>(kernel_size: u32,num_channels: u32,inner_dim: u32,@@ -147,7 +146,7 @@ fn get_test_data<T: ArrayElement + Float>((input, expected)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected: &Output<T>,label: &str,@@ -191,7 +190,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// inner_dim=4, proj_dim=2 -> num_channels = 4 + 2*2 = 8, kernel_size=4, suffix_len=2-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {for has_bias in [true, false] {let label = format!("basic(bias={has_bias})");let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, has_bias, ActivationType::SILU);@@ -200,35 +199,35 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {}// kernel_size=2 (state_stride=1, minimal state)-fn test_small_kernel<T: ArrayElement + Float + Debug + Display>() {+fn test_small_kernel<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(2, 6, 4, 1, 3, true, ActivationType::SILU);test_internal(&input, &expected, "small_kernel");}// Single token suffix-fn test_single_token<T: ArrayElement + Float + Debug + Display>() {+fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, ActivationType::SILU);test_internal(&input, &expected, "single_token");}// Many tokens-fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {+fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 16, true, ActivationType::SILU);test_internal(&input, &expected, "many_tokens");}// Larger dimensions closer to real usage-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 128, 64, 32, 8, true, ActivationType::SILU);test_internal(&input, &expected, "large");}-fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {+fn test_identity_activation<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, false, ActivationType::IDENTITY);test_internal(&input, &expected, "identity_activation");}-fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {+fn test_gelu_activation<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELUApprox);test_internal(&input, &expected, "gelu_activation");}diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rsindex e3e39e628..205b5df42 100644--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_prefill_test.rs@@ -5,19 +5,18 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,gpu_types::activation_type::ActivationType, kernel::SSDPrefillKernel,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {x: Box<[T]>,dt: Box<[T]>,b: Box<[T]>,@@ -36,12 +35,12 @@ struct Input<T: ArrayElement + Float> {state_strides: [usize; 3],}-struct Output<T: ArrayElement + Float> {+struct Output<T: DataTypeRepr + Float> {y: Vec<T>,state: Vec<T>,}-fn get_input<T: ArrayElement + Float>(+fn get_input<T: DataTypeRepr + Float>(suffix_len: usize,num_heads: usize,head_dim: usize,@@ -88,7 +87,7 @@ fn get_input<T: ArrayElement + Float>(}}-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {let context = B::Context::new().expect("Failed to create Context");let total_x = input.suffix_len * input.num_heads * input.head_dim;@@ -138,7 +137,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T}}-fn reference_output<T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+fn reference_output<T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {let safe_group = input.group_size.max(1) as usize;let mut state = input.state.to_vec();let mut y = vec![T::zero(); input.suffix_len * input.num_heads * input.head_dim];@@ -177,7 +176,7 @@ fn reference_output<T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {}}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected: &Output<T>,label: &str,@@ -218,7 +217,7 @@ fn test_shape(group_size: u32,label: &str,) {- fn run<T: ArrayElement + Float + Debug + Display>(+ fn run<T: DataTypeRepr + Float + Debug + Display>(suffix_len: usize,num_heads: usize,head_dim: usize,diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rsindex e72fc14d5..6cd0d5d63 100644--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_split_inproj_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {input: Box<[T]>,z_bias: Box<[T]>,suffix_length: u32,@@ -30,13 +29,13 @@ struct Input<T: ArrayElement + Float> {num_heads: u32,}-struct Output<T: ArrayElement + Float> {+struct Output<T: DataTypeRepr + Float> {conv_out: Vec<T>,z_out: Vec<T>,dt_out: Vec<T>,}-fn get_input<T: ArrayElement + Float>(+fn get_input<T: DataTypeRepr + Float>(suffix_length: u32,conv_dim: u32,inner_dim: u32,@@ -59,7 +58,7 @@ fn get_input<T: ArrayElement + Float>(}}-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {let context = B::Context::new().expect("Failed to create Context");let kernel = <<B as Backend>::Kernels as Kernels>::SplitInProjKernel::new(&context, T::data_type()).expect("Failed to create SplitInProjKernel");@@ -98,7 +97,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T}}-fn get_test_data<T: ArrayElement + Float>(+fn get_test_data<T: DataTypeRepr + Float>(suffix_length: u32,conv_dim: u32,inner_dim: u32,@@ -109,7 +108,7 @@ fn get_test_data<T: ArrayElement + Float>((input, expected)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected: &Output<T>,label: &str,@@ -147,31 +146,31 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// conv_dim=8, inner_dim=4, num_heads=2, suffix_length=2-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(2, 8, 4, 2);test_internal(&input, &expected, "basic");}// Single token-fn test_single_token<T: ArrayElement + Float + Debug + Display>() {+fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(1, 8, 4, 2);test_internal(&input, &expected, "single_token");}// Many tokens-fn test_many_tokens<T: ArrayElement + Float + Debug + Display>() {+fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(16, 8, 4, 2);test_internal(&input, &expected, "many_tokens");}// Larger dimensions closer to real usage-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(8, 128, 64, 32);test_internal(&input, &expected, "large");}// Minimal dimensions-fn test_minimal<T: ArrayElement + Float + Debug + Display>() {+fn test_minimal<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(1, 1, 1, 1);test_internal(&input, &expected, "minimal");}diff --git a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rsindex f1cd7b71f..508e1ddb5 100644--- a/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/ssm/ssm_update_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {x: Box<[T]>,dt: Box<[T]>,b: Box<[T]>,@@ -36,12 +35,12 @@ struct Input<T: ArrayElement + Float> {state_in_place: bool,}-struct Output<T: ArrayElement + Float> {+struct Output<T: DataTypeRepr + Float> {y: Vec<T>,next_state: Vec<T>,}-fn get_input<T: ArrayElement + Float>(+fn get_input<T: DataTypeRepr + Float>(bsz: u32,h: u32,dh: u32,@@ -81,7 +80,7 @@ fn get_input<T: ArrayElement + Float>(}}-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T> {+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Output<T> {let context = B::Context::new().expect("Failed to create Context");let kernel =<<B as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&context, T::data_type(), input.state_in_place)@@ -176,7 +175,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T}}-fn get_test_data<T: ArrayElement + Float>(+fn get_test_data<T: DataTypeRepr + Float>(bsz: u32,h: u32,dh: u32,@@ -189,7 +188,7 @@ fn get_test_data<T: ArrayElement + Float>((input, expected)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected: &Output<T>,label: &str,@@ -221,37 +220,37 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}// bsz=1, h=4, dh=3, g=2, n=8 (matches existing Metal test)-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, false);test_internal(&input, &expected, "basic");}// bsz=2 multi-batch-fn test_multi_batch<T: ArrayElement + Float + Debug + Display>() {+fn test_multi_batch<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(2, 4, 3, 2, 8, false);test_internal(&input, &expected, "multi_batch");}// state_in_place=true-fn test_state_in_place<T: ArrayElement + Float + Debug + Display>() {+fn test_state_in_place<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(1, 4, 3, 2, 8, true);test_internal(&input, &expected, "state_in_place");}// Larger dimensions closer to real usage-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(1, 8, 4, 4, 16, false);test_internal(&input, &expected, "large");}// Minimal: single head, single group, dh=1, n=1-fn test_minimal<T: ArrayElement + Float + Debug + Display>() {+fn test_minimal<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(1, 1, 1, 1, 1, false);test_internal(&input, &expected, "minimal");}// g == h (each head is its own group)-fn test_group_per_head<T: ArrayElement + Float + Debug + Display>() {+fn test_group_per_head<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(1, 4, 2, 4, 8, false);test_internal(&input, &expected, "group_per_head");}diff --git a/crates/uzu-engine/unit/backends/common/kernel/tensor_add_bias_test.rs b/crates/uzu-engine/unit/backends/common/kernel/tensor_add_bias_test.rsindex 4b8970125..f053c16aa 100644--- a/crates/uzu-engine/unit/backends/common/kernel/tensor_add_bias_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/tensor_add_bias_test.rs@@ -5,15 +5,15 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,kernel::TensorAddBiasKernel,},+ data_type::DataTypeRepr,tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {input: Box<[T]>,bias: Box<[T]>,output: Box<[T]>,@@ -21,7 +21,7 @@ struct Input<T: ArrayElement + Float> {length: u32,}-fn get_test_data<T: ArrayElement + Float>(in_place: bool) -> (Input<T>, Vec<T>) {+fn get_test_data<T: DataTypeRepr + Float>(in_place: bool) -> (Input<T>, Vec<T>) {let length = 1025usize;let num_cols = 129usize;@@ -55,7 +55,7 @@ fn get_test_data<T: ArrayElement + Float>(in_place: bool) -> (Input<T>, Vec<T>)(input, expected)}-fn get_output<T: ArrayElement + Float, B: Backend>(+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>,in_place: bool,) -> Vec<T> {@@ -91,7 +91,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(buffer_to_vec::<B, T>(&output_buffer)}-fn test<T: ArrayElement + Float + Debug>(in_place: bool) {+fn test<T: DataTypeRepr + Float + Debug>(in_place: bool) {let (input, expected) = get_test_data::<T>(in_place);for_each_backend!(|B| {let output = get_output::<T, B>(&input, in_place);diff --git a/crates/uzu-engine/unit/backends/common/kernel/tensor_add_scale_test.rs b/crates/uzu-engine/unit/backends/common/kernel/tensor_add_scale_test.rsindex 36b50dc28..bf59aae8c 100644--- a/crates/uzu-engine/unit/backends/common/kernel/tensor_add_scale_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/tensor_add_scale_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,14 +12,14 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {input: Box<[T]>,bias: Box<[T]>,num_cols: u32,@@ -28,7 +27,7 @@ struct Input<T: ArrayElement + Float> {scale: f32,}-fn get_input<T: ArrayElement + Float>(+fn get_input<T: DataTypeRepr + Float>(num_rows: usize,num_cols: usize,) -> (Vec<T>, Vec<T>) {@@ -38,7 +37,7 @@ fn get_input<T: ArrayElement + Float>((input, bias)}-fn get_output<T: ArrayElement + Float, B: Backend>(+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>,in_place: bool,) -> Vec<T> {@@ -71,7 +70,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(buffer_to_vec::<B, T>(&output_buffer)}-fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+fn get_test_data_basic<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>) {let num_rows = 4usize;let num_cols = 32usize;let (data, bias) = get_input::<T>(num_rows, num_cols);@@ -88,7 +87,7 @@ fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {(input, expected)}-fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+fn get_test_data_large<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>) {let num_rows = 64usize;let num_cols = 128usize;let (data, bias) = get_input::<T>(num_rows, num_cols);@@ -105,7 +104,7 @@ fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {(input, expected)}-fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+fn get_test_data_edge<T: DataTypeRepr + Float>() -> (Input<T>, Vec<T>) {// Single row, small dim, scale = 1.0let num_rows = 1usize;let num_cols = 4usize;@@ -124,7 +123,7 @@ fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {(input, expected)}-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.05f32} else {@@ -142,7 +141,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {});}-fn test_large<T: ArrayElement + Float + Debug + Display>() {+fn test_large<T: DataTypeRepr + Float + Debug + Display>() {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.05f32} else {@@ -160,7 +159,7 @@ fn test_large<T: ArrayElement + Float + Debug + Display>() {});}-fn test_edge<T: ArrayElement + Float + Debug + Display>() {+fn test_edge<T: DataTypeRepr + Float + Debug + Display>() {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.05f32} else {@@ -178,7 +177,7 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>() {});}-fn test_in_place<T: ArrayElement + Float + Debug + Display>() {+fn test_in_place<T: DataTypeRepr + Float + Debug + Display>() {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.05f32} else {diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rsindex a5f6b753c..b184ea37d 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs@@ -5,7 +5,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -14,6 +13,7 @@ use crate::{cpu::Cpu,metal::Metal,},+ data_type::DataTypeRepr,encodable_block::mixer::attention::KVCacheView,tests::{assert::assert_eq_float,@@ -21,7 +21,7 @@ use crate::{},};-struct Input<T: ArrayElement + Float> {+struct Input<T: DataTypeRepr + Float> {queries: Box<[T]>,keys: Box<[T]>,values: Box<[T]>,@@ -33,11 +33,11 @@ struct Input<T: ArrayElement + Float> {do_causal: bool,}-fn attention_config<T: ArrayElement + Float>(input: &Input<T>) -> AttentionKernelConfig {+fn attention_config<T: DataTypeRepr + Float>(input: &Input<T>) -> AttentionKernelConfig {super::default_attention_config::<T>(input.head_dim, input.num_heads, input.num_kv_heads, input.do_causal)}-fn get_test_data<T: ArrayElement + Float>(+fn get_test_data<T: DataTypeRepr + Float>(num_heads: usize,num_kv_heads: usize,sequence_length: usize,@@ -79,7 +79,7 @@ fn get_test_data<T: ArrayElement + Float>((input, expected)}-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+fn get_output<T: DataTypeRepr + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let config = attention_config(input);@@ -114,7 +114,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {buffer_to_vec::<B, T>(&output_buffer)}-fn get_gemm_output<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+fn get_gemm_output<T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<T> {let context = <Metal as Backend>::Context::new().expect("Failed to create Metal context");let config = attention_config(input);let queries = create_buffer_with_data::<Metal, T>(context.as_ref(), &input.queries);@@ -146,7 +146,7 @@ fn get_gemm_output<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {buffer_to_vec::<Metal, T>(&output)}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn test_internal<T: DataTypeRepr + Float + Debug + Display>(input: &Input<T>,expected: &[T],) {@@ -165,7 +165,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(assert_eq_float::<T>(expected, &output, eps, &msg);}-fn test_basic<T: ArrayElement + Float + Debug + Display>() {+fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 4, 8, 1, 64, false);test_internal(&input, &expected);@@ -173,7 +173,7 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>() {test_internal(&input, &expected);}-fn test_causal<T: ArrayElement + Float + Debug + Display>() {+fn test_causal<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 4, 16, 1, 64, true);test_internal(&input, &expected);@@ -181,7 +181,7 @@ fn test_causal<T: ArrayElement + Float + Debug + Display>() {test_internal(&input, &expected);}-fn test_gqa<T: ArrayElement + Float + Debug + Display>() {+fn test_gqa<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(8, 2, 8, 1, 64, false);test_internal(&input, &expected);@@ -189,12 +189,12 @@ fn test_gqa<T: ArrayElement + Float + Debug + Display>() {test_internal(&input, &expected);}-fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: usize) {+fn test_head_dim<T: DataTypeRepr + Float + Debug + Display>(head_dim: usize) {let (input, expected) = get_test_data::<T>(4, 4, 8, 2, head_dim, true);test_internal(&input, &expected);}-fn test_unaligned<T: ArrayElement + Float + Debug + Display>() {+fn test_unaligned<T: DataTypeRepr + Float + Debug + Display>() {let (input, expected) = get_test_data::<T>(4, 4, 40, 7, 64, true);test_internal(&input, &expected);diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rsindex 9740132bd..7684edc83 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs@@ -3,7 +3,6 @@ use uzu_engine_macros::uzu_test;use super::{gemm, single_pass, two_pass};use crate::{- array::ArrayElement,backends::{common::{Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -13,6 +12,7 @@ use crate::{cpu::Cpu,metal::Metal,},+ data_type::DataTypeRepr,encodable_block::{mixer::attention::KVCacheView, sampling::PRng},tests::{assert::assert_eq_float,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rsindex f13b664d7..164977a36 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs@@ -1,12 +1,12 @@#![cfg(backend = "metal")]use super::{gemm, single_pass, two_pass};-use crate::{array::ArrayElement, backends::common::kernel::AttentionKernelConfig};+use crate::{backends::common::kernel::AttentionKernelConfig, data_type::DataTypeRepr};mod gemm_test;mod kernel_test;-fn default_attention_config<T: ArrayElement>(+fn default_attention_config<T: DataTypeRepr>(head_dim: usize,num_q_heads: usize,num_groups: usize,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rsindex 5be2ab32d..2c9a885e6 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs@@ -4,7 +4,6 @@ use uzu_engine_macros::uzu_test;use super::*;use crate::{- array::ArrayElement,backends::{common::{Backend, BufferMut, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,@@ -12,6 +11,7 @@ use crate::{},metal::Metal,},+ data_type::DataTypeRepr,tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},};@@ -57,7 +57,7 @@ fn test_case(suffix_len: usize) -> TestCase {}}-fn run_prefill<T: ArrayElement>(+fn run_prefill<T: DataTypeRepr>(context: &MetalContext,case: &TestCase,mode: PrefillMode,@@ -172,7 +172,7 @@ fn chunked_prefill_matches_recurrent_prefill() {let context = <Metal as Backend>::Context::new().expect("metal context");if <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(&context,- <bf16 as ArrayElement>::data_type(),+ <bf16 as DataTypeRepr>::data_type(),HEAD_K_DIM as u32,).expect("chunked")diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/dispatch_paths_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/dispatch_paths_test.rsindex f2cc84d72..121073d74 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/dispatch_paths_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/dispatch_paths_test.rs@@ -6,7 +6,6 @@ use rstest::rstest;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, Context,@@ -14,6 +13,7 @@ use crate::{},metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},},+ data_type::DataTypeRepr,tests::{assert::assert_eq_float,matmul::{@@ -23,7 +23,7 @@ use crate::{},};-fn run_metal<T: ArrayElement + Float>(+fn run_metal<T: DataTypeRepr + Float>(context: &MetalContext,kernel: &mut <<Metal as Backend>::Kernels as Kernels>::MatmulKernel,input: &Input<T>,@@ -46,7 +46,7 @@ fn gemm_engines_for_hw(context: &MetalContext) -> Vec<GemmEngine> {engines}-fn check_case<T: ArrayElement + Float + Debug + Display>(+fn check_case<T: DataTypeRepr + Float + Debug + Display>(context: &MetalContext,kernel: &mut <<Metal as Backend>::Kernels as Kernels>::MatmulKernel,dispatch: Option<GemmEngine>,@@ -64,7 +64,7 @@ fn check_case<T: ArrayElement + Float + Debug + Display>();}-fn run_matrix<T: ArrayElement + Float + Debug + Display>(+fn run_matrix<T: DataTypeRepr + Float + Debug + Display>(case_for_shape: impl Fn(crate::tests::matmul::Shape) -> Case,tolerance: f32,) {diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm_bench.rsindex 7e96c6cc0..48faf8722 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm_bench.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm_bench.rs@@ -3,7 +3,6 @@ use half::bf16;use uzu_engine_macros::uzu_bench;use crate::{- array::ArrayElement,backends::{common::{Backend,@@ -14,6 +13,7 @@ use crate::{},metal::{Metal, kernel::matmul::gemm::GemmEngine},},+ data_type::DataTypeRepr,tests::{helpers::create_buffer,matmul::{bench_fp_gemm_shapes, iter_encode_loop},diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rsindex abcc83a66..860ed111a 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs@@ -8,7 +8,6 @@ use rstest::rstest;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,@@ -22,13 +21,14 @@ use crate::{cpu::Cpu,metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},},+ data_type::DataTypeRepr,tests::{helpers::buffer_to_vec,matmul::{QuantBuffers, QuantInput, quant::run_quant_cpu, quant_arguments},},};-fn run_quant_metal<T: ArrayElement + Float>(+fn run_quant_metal<T: DataTypeRepr + Float>(context: &MetalContext,input: &QuantInput<T>,dispatch: Option<GemmEngine>,@@ -82,7 +82,7 @@ fn quant_params_layout_table() {}}-fn assert_parity<T: ArrayElement + Float + Debug + Display>(+fn assert_parity<T: DataTypeRepr + Float + Debug + Display>(label: &str,reference: &[T],actual: &[T],@@ -104,7 +104,7 @@ fn assert_parity<T: ArrayElement + Float + Debug + Display>(assert_eq!(errors, 0, "{label}: {errors} mismatches");}-fn run_parity<T: ArrayElement + Float + Debug + Display>(+fn run_parity<T: DataTypeRepr + Float + Debug + Display>(m: u32,k: u32,n: u32,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rsindex 7d664744e..47d46e237 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_gemm_bench.rs@@ -4,7 +4,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_bench;use crate::{- array::ArrayElement,backends::{common::{Backend,@@ -13,13 +12,14 @@ use crate::{},metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},},+ data_type::DataTypeRepr,tests::{matmul::{QuantBuffers, QuantInput, bench_quant_gemm_shapes, iter_encode_loop, quant_arguments},util::type_short_name,},};-fn bench_unified_quant_typed<T: ArrayElement + Float>(+fn bench_unified_quant_typed<T: DataTypeRepr + Float>(c: &mut Criterion,context: &MetalContext,label: &str,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rsindex 6ade045f0..fd258eccc 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qwen3_bench.rs@@ -4,7 +4,6 @@ use num_traits::Float;use uzu_engine_macros::uzu_bench;use crate::{- array::ArrayElement,backends::{common::{Backend,@@ -13,6 +12,7 @@ use crate::{},metal::{Metal, context::MetalContext},},+ data_type::DataTypeRepr,tests::{cold_pool::ColdPool,matmul::{QuantBuffers, QuantInput, iter_encode_loop, quant_arguments, qwen3_layer_shapes},@@ -20,7 +20,7 @@ use crate::{},};-fn bench_qwen3_layers_typed<T: ArrayElement + Float>(+fn bench_qwen3_layers_typed<T: DataTypeRepr + Float>(c: &mut Criterion,context: &MetalContext,label: &str,diff --git a/crates/uzu-engine/unit/common/assert.rs b/crates/uzu-engine/unit/common/assert.rsindex 569375579..1f41dfcb2 100644--- a/crates/uzu-engine/unit/common/assert.rs+++ b/crates/uzu-engine/unit/common/assert.rs@@ -2,9 +2,9 @@ use std::fmt::Display;use num_traits::Float;-use crate::array::ArrayElement;+use crate::data_type::DataTypeRepr;-pub fn assert_eq_float<T: ArrayElement + Float + Display>(+pub fn assert_eq_float<T: DataTypeRepr + Float + Display>(expected: &[T],actual: &[T],eps: f32,diff --git a/crates/uzu-engine/unit/common/helpers.rs b/crates/uzu-engine/unit/common/helpers.rsindex 6f9c90993..cc1a04068 100644--- a/crates/uzu-engine/unit/common/helpers.rs+++ b/crates/uzu-engine/unit/common/helpers.rs@@ -1,11 +1,11 @@use std::{mem::size_of, sync::Arc};use crate::{- array::ArrayElement,backends::common::{Backend, BufferCpuAccessible, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable,CommandBufferPending, Context,},+ data_type::DataTypeRepr,};/// Invokes `$body` once per available backend, with `$B` bound to each backend type.@@ -52,7 +52,7 @@ pub fn create_buffer<B: Backend, T>(context.create_buffer(buffer_size_bytes::<T>(elements_count)).expect("Failed to create buffer")}-pub fn create_buffer_with_data<B: Backend, T: ArrayElement>(+pub fn create_buffer_with_data<B: Backend, T: DataTypeRepr>(context: &B::Context,data: &[T],) -> B::GlobalBuffer {@@ -61,13 +61,13 @@ pub fn create_buffer_with_data<B: Backend, T: ArrayElement>(buffer}-pub fn buffer_to_vec<B: Backend, T: ArrayElement>(+pub fn buffer_to_vec<B: Backend, T: DataTypeRepr>(buffer: impl BufferRef<Backend = B, Buffer: BufferCpuAccessible>) -> Vec<T> {buffer.copyout()}-pub fn buffer_prefix_to_vec<B: Backend, T: ArrayElement>(+pub fn buffer_prefix_to_vec<B: Backend, T: DataTypeRepr>(buffer: impl BufferRef<Backend = B, Buffer: BufferCpuAccessible>,elements_count: usize,) -> Vec<T> {diff --git a/crates/uzu-engine/unit/common/matmul/harness.rs b/crates/uzu-engine/unit/common/matmul/harness.rsindex 55fb8e69d..a34da05a9 100644--- a/crates/uzu-engine/unit/common/matmul/harness.rs+++ b/crates/uzu-engine/unit/common/matmul/harness.rs@@ -2,7 +2,6 @@ use num_traits::Float;use super::Shape;use crate::{- array::ArrayElement,backends::{common::{Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,@@ -13,6 +12,7 @@ use crate::{},cpu::Cpu,},+ data_type::DataTypeRepr,tests::helpers::{buffer_to_vec, create_buffer_with_data},};@@ -61,7 +61,7 @@ impl Case {}}-pub struct Input<T: ArrayElement + Float> {+pub struct Input<T: DataTypeRepr + Float> {pub a: Box<[T]>,pub b: Box<[T]>,pub rht_factors: Option<Box<[i32]>>,@@ -69,7 +69,7 @@ pub struct Input<T: ArrayElement + Float> {pub case: Case,}-pub fn deterministic_input<T: ArrayElement + Float>(case: Case) -> Input<T> {+pub fn deterministic_input<T: DataTypeRepr + Float>(case: Case) -> Input<T> {let Shape {m,k,@@ -101,7 +101,7 @@ pub fn deterministic_input<T: ArrayElement + Float>(case: Case) -> Input<T> {}}-pub fn run<B: Backend, T: ArrayElement + Float>(+pub fn run<B: Backend, T: DataTypeRepr + Float>(context: &B::Context,kernel: &mut <B::Kernels as Kernels>::MatmulKernel,input: &Input<T>,@@ -155,7 +155,7 @@ pub fn run<B: Backend, T: ArrayElement + Float>(buffer_to_vec::<B, T>(&d_buffer)}-pub fn cpu_reference<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+pub fn cpu_reference<T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<T> {let context = <Cpu as Backend>::Context::new().expect("CPU context");let mut kernel = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(&context,diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rsindex ef5ddacfc..083b34269 100644--- a/crates/uzu-engine/unit/common/matmul/quant.rs+++ b/crates/uzu-engine/unit/common/matmul/quant.rs@@ -4,7 +4,6 @@ use num_traits::Float;use rand::{RngExt, SeedableRng, rngs::SmallRng};use crate::{- array::ArrayElement,backends::{common::{Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,@@ -20,7 +19,7 @@ use crate::{},cpu::Cpu,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},};@@ -33,7 +32,7 @@ pub struct PreparedInt8A {}#[derive(Clone)]-pub struct QuantInput<T: ArrayElement + Float> {+pub struct QuantInput<T: DataTypeRepr + Float> {pub w_packed: Vec<u32>,pub scales: Vec<T>,pub zero_points: Option<Vec<u8>>,@@ -58,7 +57,7 @@ fn mode_for_bits(bits: u32) -> QuantizationMode {}}-fn pad<T: ArrayElement>(+fn pad<T: DataTypeRepr>(values: &[T],minimum_len: usize,) -> Vec<T> {@@ -121,7 +120,7 @@ pub fn transpose_metadata(}}-impl<T: ArrayElement + Float> QuantInput<T> {+impl<T: DataTypeRepr + Float> QuantInput<T> {pub fn new(m: u32,k: u32,@@ -284,7 +283,7 @@ impl<T: ArrayElement + Float> QuantInput<T> {}}-pub struct QuantBuffers<B: Backend, T: ArrayElement + Float> {+pub struct QuantBuffers<B: Backend, T: DataTypeRepr + Float> {pub w: B::GlobalBuffer,pub scales: B::GlobalBuffer,pub zp: Option<B::GlobalBuffer>,@@ -297,7 +296,7 @@ pub struct QuantBuffers<B: Backend, T: ArrayElement + Float> {_t: std::marker::PhantomData<T>,}-impl<B: Backend, T: ArrayElement + Float> QuantBuffers<B, T> {+impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {pub fn allocate(context: &B::Context,input: &QuantInput<T>,@@ -379,7 +378,7 @@ impl<B: Backend, T: ArrayElement + Float> QuantBuffers<B, T> {}}-fn quant_b_variant<TB: BufferRef, T: ArrayElement + Float>(+fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(w: TB,scales: TB,zero_points: Option<TB>,@@ -404,7 +403,7 @@ fn quant_b_variant<TB: BufferRef, T: ArrayElement + Float>(})}-pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(+pub fn quant_arguments<'a, B: Backend, T: DataTypeRepr + Float>(buffers: &'a mut QuantBuffers<B, T>,input: &QuantInput<T>,) -> MatmulArguments<'a, B, &'a B::GlobalBuffer, &'a B::GlobalBuffer, &'a mut B::GlobalBuffer, &'a B::GlobalBuffer> {@@ -449,7 +448,7 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(}}-pub fn run_quant_cpu<T: ArrayElement + Float>(input: &QuantInput<T>) -> Vec<T> {+pub fn run_quant_cpu<T: DataTypeRepr + Float>(input: &QuantInput<T>) -> Vec<T> {let context = <Cpu as Backend>::Context::new().expect("Cpu context");let mut buffers = QuantBuffers::<Cpu, T>::allocate(&context, input);let mut matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rsindex fba8724f1..86d9321c1 100644--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs@@ -7,31 +7,19 @@ use rand_distr::Normal;use uzu_engine_macros::uzu_test;use crate::{- array::ArrayElement,backends::common::{Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,gpu_types::trie::TrieNode,},- data_type::DataType,+ data_type::{DataType, DataTypeRepr},encodable_block::{batch_topology::BatchTopology,sampling::{Sampling, SamplingMethod},},tests::proptest::{ComparableTestResults, TestContextes, for_each_context, kernel_data_type},+ utils::dispatch_dtype::dispatch_dtype,};-macro_rules! dispatch_dtype {- (|()| $body:expr) => { $body };- (|($T:ident : $dt:expr $(, $rest_T:ident : $rest_dt:expr)*)| $body:expr) => {- match $dt {- $crate::data_type::DataType::F16 => { type $T = half::f16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }- $crate::data_type::DataType::BF16 => { type $T = half::bf16; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }- $crate::data_type::DataType::F32 => { type $T = f32; dispatch_dtype!(|($($rest_T : $rest_dt),*)| $body) }- other => panic!("unsupported dtype in test: {:?}", other),- }- };-}-struct SamplingTestResults(Vec<u32>);impl ComparableTestResults for SamplingTestResults {@@ -46,7 +34,7 @@ impl ComparableTestResults for SamplingTestResults {}}-fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(+fn do_sampling_backend<B: Backend, T: DataTypeRepr + Float>(context: &B::Context,logits: &[T],seeds: Option<&[u64]>,@@ -94,7 +82,7 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(Ok(SamplingTestResults(sampled))}-fn get_data<T: ArrayElement + Float>(+fn get_data<T: DataTypeRepr + Float>(seed: u64,batch_size: usize,vocab_size: usize,diff --git a/crates/uzu-engine/unit/parameters/safetensors_metadata.rs b/crates/uzu-engine/unit/parameters/safetensors_metadata.rsdeleted file mode 100644index 5ac50bd4d..000000000--- a/crates/uzu-engine/unit/parameters/safetensors_metadata.rs+++ /dev/null@@ -1,14 +0,0 @@-use std::fs::File;--use uzu_engine_macros::uzu_test;--use crate::{parameters::safetensors_metadata::read_metadata, tests::path::get_test_weights_path};--#[uzu_test]-#[ignore] // downloads the test model from the registry-fn test_metadata_loading() {- let path = get_test_weights_path();- let file = File::open(&path).expect("weights not found");- let (_offset, metadata) = read_metadata(&file).expect("read metadata");- assert!(!metadata.tensors.is_empty());-}