A high-performance inference engine for AI models
New config dsl and lalamo 0.10.0 config format (#435)
uuuvn0d6b4c4459a7
diff --git a/Cargo.lock b/Cargo.lockindex ef6103615..eb2e2b091 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -576,6 +576,7 @@ dependencies = ["is_close","itertools 0.14.0","libc",+ "libm","memmap2","minijinja","minijinja-contrib",diff --git a/Cargo.toml b/Cargo.tomlindex 63e88a91e..7bf0c5855 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -80,6 +80,7 @@ num-traits = "0.2"regex = "1.12"fancy-regex = "0.17.0"libc = "0.2"+libm = "0.2.16"minijinja = { version = "2.17.0", features = ["json", "preserve_order"] }minijinja-contrib = { version = "2.17.0", features = ["pycompat"] }clap = { version = "4.6", features = ["derive"] }diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.tomlindex 1c340800d..083b4fe64 100644--- a/crates/backend-uzu/Cargo.toml+++ b/crates/backend-uzu/Cargo.toml@@ -29,6 +29,7 @@ memmap2.workspace = truenum-traits.workspace = trueregex.workspace = truelibc.workspace = true+libm.workspace = trueminijinja.workspace = trueminijinja-contrib.workspace = truerand.workspace = truediff --git a/crates/backend-uzu/src/audio/context.rs b/crates/backend-uzu/src/audio/context.rsindex 25c04ab0c..bb222682a 100644--- a/crates/backend-uzu/src/audio/context.rs+++ b/crates/backend-uzu/src/audio/context.rs@@ -1,7 +1,7 @@use std::{path::Path, rc::Rc};use super::{AudioResult, NanoCodecFsqRuntime};-use crate::{backends::common::Backend, config::TtsConfig};+use crate::{backends::common::Backend, config::tts::TTSConfig};#[derive(Clone)]pub struct AudioGenerationContext<B: Backend> {@@ -13,7 +13,7 @@ pub struct AudioGenerationContext<B: Backend> {impl<B: Backend> AudioGenerationContext<B> {pub fn from_tts_config_and_model_path(- tts_config: &TtsConfig,+ tts_config: &TTSConfig,model_path: &Path,) -> AudioResult<Self> {Self::with_runtime(NanoCodecFsqRuntime::from_tts_config_and_model_path(tts_config, model_path)?)diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime.rs b/crates/backend-uzu/src/audio/nanocodec/runtime.rsindex 8ad4a33c1..e3b4758f5 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime.rs@@ -22,8 +22,13 @@ use crate::{},},config::{- DecoderConfig, DescriptAudioCodecConfig, EmbeddingConfig, EmbeddingConfigCommon, NormalizationConfig,- TtsAudioDecoderConfig, TtsConfig,+ decoder::DecoderConfig,+ embedding::{AnyEmbeddingConfig, untied_embedding::UntiedEmbeddingConfig},+ normalization::NormalizationConfig,+ tts::{+ TTSConfig,+ audio_decoder::{AnyTTSAudioDecoderConfig, descript_audio_codec::DescriptAudioCodecConfig},+ },},encodable_block::{Decoder, LayerExecutables, RMSNorm},forward_pass::{model_shape::ModelShape, state::SharedBuffers},@@ -91,7 +96,7 @@ pub struct NanoCodecFsqRuntimeConfig {impl NanoCodecFsqRuntimeConfig {pub fn from_tts_config_and_model_path(- tts_config: &TtsConfig,+ tts_config: &TTSConfig,model_path: &Path,) -> AudioResult<Self> {let (runtime_config, decoder) = load_audio_runtime_from_tts_config(tts_config, model_path)?;@@ -233,7 +238,7 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {}pub fn from_tts_config_and_model_path(- tts_config: &TtsConfig,+ tts_config: &TTSConfig,model_path: &Path,) -> AudioResult<Self> {Ok(Self::new(NanoCodecFsqRuntimeConfig::from_tts_config_and_model_path(tts_config, model_path)?))diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rsindex e066650fe..36754723d 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs@@ -1,46 +1,26 @@use super::*;--pub(super) fn resolve_descript_audio_codec_vocoder_data_type(- top_level_precision: DataType,- config: &DescriptAudioCodecConfig,-) -> AudioResult<DataType> {- let resolved_precision = top_level_precision;- for (field_name, precision) in [- ("tts_config.audio_decoder_config.precision", config.precision),- ("tts_config.audio_decoder_config.quantizer_config.precision", config.quantizer_config.precision),- ] {- if resolved_precision != precision {- return Err(AudioError::Runtime(format!(- "conflicting DescriptAudioCodec precision in Lalamo export: {field_name}={precision:?} conflicts with {resolved_precision:?}"- )));- }- }-- let data_type: DataType = resolved_precision.into();- if !matches!(data_type, DataType::F32 | DataType::F16 | DataType::BF16) {- return Err(AudioError::Runtime(format!(- "unsupported DescriptAudioCodec vocoder precision in Lalamo export: {resolved_precision:?} (expected float32/float16/bfloat16)"- )));- }- Ok(data_type)-}+use crate::{+ config::{+ decoder::DecoderConfig,+ embedding::{AnyEmbeddingConfig, untied_embedding::UntiedEmbeddingConfig},+ },+ forward_pass::model_shape::ModelShape,+};pub(super) fn load_audio_runtime_from_tts_config(- tts_config: &TtsConfig,+ tts_config: &TTSConfig,model_path: &Path,) -> AudioResult<(RuntimeConfigJson, StructuredAudioCodecGraph)> {- let cfg = match &tts_config.audio_decoder_config {- TtsAudioDecoderConfig::DescriptAudioCodec(config) => config,+ let AnyTTSAudioDecoderConfig::DescriptAudioCodecConfig(cfg) = &tts_config.audio_decoder_config;+ let weights_path = model_path.join("model.safetensors");+ let post_module_decoder_config = DecoderConfig {+ embedding_config: AnyEmbeddingConfig::UntiedEmbeddingConfig(UntiedEmbeddingConfig::new(None, None)),+ transformer_config: cfg.quantizer_config.post_module_config.clone(),+ vocab_size: 1,+ pard_token: None,+ ple_model_config: None,};- let fishaudio_weights = model_path.join("model.safetensors");- if !fishaudio_weights.is_file() {- return Err(AudioError::Runtime(format!(- "missing exported FishAudio decoder weights '{}'",- fishaudio_weights.display()- )));- }-- let vocoder_data_type = resolve_descript_audio_codec_vocoder_data_type(tts_config.activation_precision, cfg)?;+ let vocoder_data_type = ModelShape::from_decoder_config(&post_module_decoder_config, DataType::F32).data_type;let total_codebooks = cfg.n_codebooks@@ -67,7 +47,7 @@ pub(super) fn load_audio_runtime_from_tts_config(};let decoder = StructuredAudioCodecGraph {config: cfg.clone(),- weights_path: fishaudio_weights.display().to_string(),+ weights_path: weights_path.display().to_string(),codebook_size: cfg.codebook_size,semantic_codebook_size: cfg.semantic_codebook_size,input_dim: cfg.input_dim,diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rsindex d6e718cfe..6408898ba 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs@@ -1,63 +1,5 @@use super::*;-use crate::array::{Array, ArrayContextExt};--pub(super) type BackendParameterTree<'loader, B> = crate::parameters::ParameterTree<'loader, <B as Backend>::Context>;--pub(super) fn read_float_array<B: Backend, const RANK: usize>(- tree: &BackendParameterTree<B>,- name: &str,- expected_data_type: DataType,-) -> AudioResult<([usize; RANK], Array<B>)> {- let array = tree.leaf_array(name)?;- if array.data_type() != expected_data_type {- return Err(AudioError::Runtime(format!(- "tensor '{name}' dtype mismatch: expected {expected_data_type:?}, got {:?}",- array.data_type()- )));- }- if array.shape().len() != RANK {- return Err(AudioError::Runtime(format!(- "expected rank-{RANK} tensor for '{name}', got rank {}",- array.shape().len()- )));- }- let mut dims = [0usize; RANK];- dims.copy_from_slice(array.shape());- Ok((dims, array))-}--pub(super) fn read_float_vector_exact<B: Backend>(- tree: &BackendParameterTree<B>,- name: &str,- expected_len: usize,- expected_data_type: DataType,-) -> AudioResult<Array<B>> {- let (shape, array) = read_float_array::<B, 1>(tree, name, expected_data_type)?;- if shape[0] != expected_len {- return Err(AudioError::Runtime(format!(- "tensor '{name}' shape mismatch: expected [{expected_len}], got {:?}",- shape- )));- }- Ok(array)-}--pub(super) fn read_float_matrix_exact<B: Backend>(- tree: &BackendParameterTree<B>,- name: &str,- expected_rows: usize,- expected_cols: usize,- expected_data_type: DataType,-) -> AudioResult<Array<B>> {- let (shape, array) = read_float_array::<B, 2>(tree, name, expected_data_type)?;- if shape != [expected_rows, expected_cols] {- return Err(AudioError::Runtime(format!(- "tensor '{name}' shape mismatch: expected [{expected_rows}, {expected_cols}], got {:?}",- shape- )));- }- Ok(array)-}+use crate::{array::Array, backends::common::gpu_types::ActivationType, parameters::ParameterTree};pub(super) fn copy_to_outer_axis_slice<B: Backend>(destination: &mut Array<B>,@@ -81,80 +23,71 @@ pub(super) fn copy_to_outer_axis_slice<B: Backend>(}pub(super) fn read_conv1d_layer<B: Backend>(- tree: &BackendParameterTree<B>,+ tree: &ParameterTree<B>,data_type: DataType,+ cin: usize,+ cout: usize,+ kernel_size: usize,dilation: usize,groups: usize,) -> AudioResult<StructuredAudioConv1d<B>> {- let (shape, weight) = read_float_array::<B, 3>(tree, "weights", data_type)?;- let bias = read_float_vector_exact::<B>(tree, "biases", shape[0], data_type)?;- if groups == 0 || shape[0] == 0 || shape[1] == 0 || shape[2] == 0 {+ if groups == 0 || cin == 0 || cout == 0 || kernel_size == 0 {return Err(AudioError::InvalidTokenCardinality);}- if shape[0] % groups != 0 {+ if cin % groups != 0 || cout % groups != 0 {return Err(AudioError::Runtime(format!(- "invalid grouped conv weights for 'weights': out_channels {} not divisible by groups {groups}",- shape[0]+ "invalid grouped conv dimensions: cin {cin}, cout {cout}, groups {groups}")));}+ let weight = tree.leaf("weights")?.validate(&[cout, cin / groups, kernel_size], data_type)?.read_array()?;+ let bias = tree.leaf("biases")?.validate(&[cout], data_type)?.read_array()?;Ok(StructuredAudioConv1d {weight,bias,- cin: shape[1].checked_mul(groups).ok_or(AudioError::Runtime("conv input channel overflow".to_string()))?,- cout: shape[0],- kernel_size: shape[2],+ cin,+ cout,+ kernel_size,dilation,groups,})}pub(super) fn read_conv_transpose1d_layer<B: Backend>(- tree: &BackendParameterTree<B>,+ tree: &ParameterTree<B>,data_type: DataType,+ cin: usize,+ cout: usize,+ kernel_size: usize,stride: usize,groups: usize,) -> AudioResult<StructuredAudioConvTranspose1d<B>> {- if stride == 0 {- return Err(AudioError::InvalidTokenCardinality);- }- let (shape, weight) = read_float_array::<B, 3>(tree, "weights", data_type)?;- if groups == 0 || shape[0] == 0 || shape[1] == 0 || shape[2] == 0 {+ if stride == 0 || groups == 0 || cin == 0 || cout == 0 || kernel_size == 0 {return Err(AudioError::InvalidTokenCardinality);}- let export_cout = shape[0];- let export_cin =- shape[1].checked_mul(groups).ok_or(AudioError::Runtime("transpose conv input channel overflow".to_string()))?;- if export_cout % groups != 0 {+ if cin % groups != 0 || cout % groups != 0 {return Err(AudioError::InvalidTokenCardinality);}- let bias = read_float_vector_exact::<B>(tree, "biases", export_cout, data_type)?;- let expected_weight_shape = [export_cout, export_cin / groups, shape[2]];- if shape != expected_weight_shape {- return Err(AudioError::Runtime(format!(- "transpose conv export-layout weight shape mismatch: expected {:?}, got {:?}",- expected_weight_shape, shape- )));- }-+ let weight = tree.leaf("weights")?.validate(&[cout, cin / groups, kernel_size], data_type)?.read_array()?;+ let bias = tree.leaf("biases")?.validate(&[cout], data_type)?.read_array()?;Ok(StructuredAudioConvTranspose1d {weight,bias,- cin: export_cin,- cout: export_cout,- kernel_size: shape[2],+ cin,+ cout,+ kernel_size,stride,groups,})}pub(super) fn read_pointwise_conv_layer<B: Backend>(- tree: &BackendParameterTree<B>,+ tree: &ParameterTree<B>,data_type: DataType,expected_out_dim: usize,expected_in_dim: usize,) -> AudioResult<StructuredAudioPointwiseConv<B>> {- let weight = read_float_matrix_exact::<B>(tree, "weights", expected_out_dim, expected_in_dim, data_type)?;- let bias = read_float_vector_exact::<B>(tree, "biases", expected_out_dim, data_type)?;+ let weight = tree.leaf("weights")?.validate(&[expected_out_dim, expected_in_dim], data_type)?.read_array()?;+ let bias = tree.leaf("biases")?.validate(&[expected_out_dim], data_type)?.read_array()?;Ok(StructuredAudioPointwiseConv {weight,bias,@@ -165,16 +98,16 @@ pub(super) fn read_pointwise_conv_layer<B: Backend>(pub(super) fn read_norm_layer<B: Backend>(context: &Rc<B::Context>,- tree: &BackendParameterTree<B>,+ tree: &ParameterTree<B>,data_type: DataType,channels: usize,epsilon: f32,subtract_mean: bool,use_bias: bool,) -> AudioResult<StructuredAudioNorm<B>> {- let scales = read_float_vector_exact::<B>(tree, "scales", channels, data_type)?;+ let scales = tree.leaf("scales")?.validate(&[channels], data_type)?.read_array()?;let bias = if use_bias {- read_float_vector_exact::<B>(tree, "biases", channels, data_type)?+ tree.leaf("biases")?.validate(&[channels], data_type)?.read_array()?} else {context.create_array_zeros(&[channels], data_type)};@@ -188,56 +121,27 @@ pub(super) fn read_norm_layer<B: Backend>(pub(super) fn read_convnext_layer<B: Backend>(context: &Rc<B::Context>,- tree: &BackendParameterTree<B>,+ tree: &ParameterTree<B>,data_type: DataType,+ activation_type: ActivationType,norm_config: &NormalizationConfig,+ dim: usize,) -> AudioResult<StructuredAudioConvNeXt<B>> {- let depthwise_tree = tree.subtree("dwconv")?;- let (depthwise_shape, depthwise_weight) = read_float_array::<B, 3>(&depthwise_tree, "weights", data_type)?;- if depthwise_shape[1] != 1 {- return Err(AudioError::Runtime(format!(- "ConvNeXt depthwise weight in_channels_per_group must be 1 at 'dwconv.weights', got {}",- depthwise_shape[1]- )));- }- let depthwise_conv = StructuredAudioConv1d {- weight: depthwise_weight,- bias: read_float_vector_exact::<B>(&depthwise_tree, "biases", depthwise_shape[0], data_type)?,- cin: depthwise_shape[0],- cout: depthwise_shape[0],- kernel_size: depthwise_shape[2],- dilation: 1,- groups: depthwise_shape[0],- };- if depthwise_conv.cout == 0 || depthwise_conv.kernel_size == 0 {- return Err(AudioError::InvalidTokenCardinality);- }- if depthwise_conv.cin != depthwise_conv.cout {- return Err(AudioError::Runtime(format!(- "ConvNeXt depthwise conv expects cin==cout, got {} vs {} at 'dwconv.weights'",- depthwise_conv.cin, depthwise_conv.cout- )));- }- let channels = depthwise_conv.cout;+ let hidden_dim = dim.checked_mul(4).ok_or(AudioError::Runtime("ConvNeXt hidden dimension overflow".to_string()))?;+ let depthwise_conv = read_conv1d_layer::<B>(&tree.subtree("dwconv")?, data_type, dim, dim, 7, 1, dim)?;let norm = read_norm_layer::<B>(context,&tree.subtree("norm")?,data_type,- channels,+ dim,norm_config.epsilon,norm_config.subtract_mean,- norm_config.use_bias,+ norm_config.has_biases,)?;- let (pwconv1_shape, _) = read_float_array::<B, 2>(&tree.subtree("pwconv1")?, "weights", data_type)?;- if pwconv1_shape[1] != channels {- return Err(AudioError::Runtime(format!(- "ConvNeXt pwconv1 input mismatch at 'pwconv1.weights': expected {}, got {}",- channels, pwconv1_shape[1]- )));- }- let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv1")?, data_type, pwconv1_shape[0], channels)?;- let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv2")?, data_type, channels, pwconv1_shape[0])?;+ let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv1")?, data_type, hidden_dim, dim)?;+ let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv2")?, data_type, dim, hidden_dim)?;Ok(StructuredAudioConvNeXt {+ activation_type,depthwise_conv,norm,pwconv1,@@ -246,31 +150,16 @@ pub(super) fn read_convnext_layer<B: Backend>(}pub(super) fn read_residual_unit_layer<B: Backend>(- tree: &BackendParameterTree<B>,+ tree: &ParameterTree<B>,data_type: DataType,dilation: usize,+ dim: usize,) -> AudioResult<StructuredAudioResidualUnit<B>> {- let snake1_alpha = {- let snake1_tree = tree.subtree("snake1")?;- let (shape, alpha) = read_float_array::<B, 1>(&snake1_tree, "alpha", data_type)?;- if shape[0] == 0 {- return Err(AudioError::InvalidTokenCardinality);- }- alpha- };- let conv1 = read_conv1d_layer::<B>(&tree.subtree("conv1")?, data_type, dilation, 1)?;- let snake2_alpha = {- let snake2_tree = tree.subtree("snake2")?;- let (shape, alpha) = read_float_array::<B, 1>(&snake2_tree, "alpha", data_type)?;- if shape[0] != conv1.cout {- return Err(AudioError::Runtime(format!(- "residual snake2 alpha mismatch: expected {}, got {}",- conv1.cout, shape[0]- )));- }- alpha- };- let conv2 = read_conv1d_layer::<B>(&tree.subtree("conv2")?, data_type, 1, 1)?;+ let alpha_shape = [dim];+ let snake1_alpha = tree.leaf("snake1.alpha")?.validate(&alpha_shape, data_type)?.read_array()?;+ let conv1 = read_conv1d_layer::<B>(&tree.subtree("conv1")?, data_type, dim, dim, 7, dilation, 1)?;+ let snake2_alpha = tree.leaf("snake2.alpha")?.validate(&alpha_shape, data_type)?.read_array()?;+ let conv2 = read_conv1d_layer::<B>(&tree.subtree("conv2")?, data_type, dim, dim, 1, 1, 1)?;Ok(StructuredAudioResidualUnit {snake1_alpha,conv1,@@ -281,7 +170,7 @@ pub(super) fn read_residual_unit_layer<B: Backend>(pub(super) fn build_vocoder_graph_from_tree<B: Backend>(context: &Rc<B::Context>,- root: &BackendParameterTree<B>,+ root: &ParameterTree<B>,config: &DescriptAudioCodecConfig,data_type: DataType,) -> AudioResult<StructuredAudioDecoderGraph<B>> {@@ -289,45 +178,62 @@ pub(super) fn build_vocoder_graph_from_tree<B: Backend>(let quantizer_tree = audio_decoder_tree.subtree("quantizer")?;let decoder_tree = audio_decoder_tree.subtree("decoder")?;- let first_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("first_conv")?, data_type, 1, 1)?;- let final_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("final_conv")?, data_type, 1, 1)?;- let final_snake_alpha =- read_float_vector_exact::<B>(&decoder_tree.subtree("final_snake")?, "alpha", final_conv.cin, data_type)?;+ let latent_dim = config.encoder_dim * (1usize << config.encoder_rates.len());+ let final_dim = config.decoder_dim / (1usize << config.decoder_rates.len());++ let first_conv = read_conv1d_layer::<B>(+ &decoder_tree.subtree("first_conv")?,+ data_type,+ latent_dim,+ config.decoder_dim,+ 7,+ 1,+ 1,+ )?;+ let final_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("final_conv")?, data_type, final_dim, 1, 7, 1, 1)?;+ let final_snake_alpha = decoder_tree.leaf("final_snake.alpha")?.validate(&[final_dim], data_type)?.read_array()?;let mut upsample_blocks = Vec::with_capacity(config.downsample_factor.len());for (index, &stride) in config.downsample_factor.iter().rev().enumerate() {let block_tree = quantizer_tree.subtree("upsampler")?.subtree("blocks")?.subtree(&index.to_string())?;- let trans_conv = read_conv_transpose1d_layer::<B>(&block_tree.subtree("trans_conv")?, data_type, stride, 1)?;+ let trans_conv = read_conv_transpose1d_layer::<B>(+ &block_tree.subtree("trans_conv")?,+ data_type,+ config.input_dim,+ config.input_dim,+ stride,+ stride,+ 1,+ )?;let convnext = read_convnext_layer::<B>(context,&block_tree.subtree("convnext")?,data_type,+ config.quantizer_config.upsampler_config.block_configs[index].convnext_config.activation.act_type(),&config.quantizer_config.upsampler_config.block_configs[index].convnext_config.norm_config,+ config.input_dim,)?;- if convnext.depthwise_conv.cin != trans_conv.cout {- return Err(AudioError::Runtime(format!(- "structured audio upsampler convnext channel mismatch at block {index}: trans_conv out {} vs convnext in {}",- trans_conv.cout, convnext.depthwise_conv.cin- )));- }upsample_blocks.push((trans_conv, convnext));}let mut decoder_blocks = Vec::with_capacity(config.decoder_rates.len());for (index, &stride) in config.decoder_rates.iter().enumerate() {let block_tree = decoder_tree.subtree("decoder_blocks")?.subtree(&index.to_string())?;- let trans_conv = read_conv_transpose1d_layer::<B>(&block_tree.subtree("trans_conv")?, data_type, stride, 1)?;- let snake_alpha =- read_float_vector_exact::<B>(&block_tree.subtree("snake")?, "alpha", trans_conv.cin, data_type)?;- let channels = trans_conv.cout;- let res_unit1 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit1")?, data_type, 1)?;- let res_unit2 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit2")?, data_type, 3)?;- let res_unit3 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit3")?, data_type, 9)?;- if res_unit1.conv1.cin != channels || res_unit2.conv1.cin != channels || res_unit3.conv1.cin != channels {- return Err(AudioError::Runtime(format!(- "structured audio decoder block {index} channel mismatch in exported weights"- )));- }+ let input_dim = config.decoder_dim / (1usize << index);+ let output_dim = config.decoder_dim / (1usize << (index + 1));+ let trans_conv = read_conv_transpose1d_layer::<B>(+ &block_tree.subtree("trans_conv")?,+ data_type,+ input_dim,+ output_dim,+ 2 * stride,+ stride,+ 1,+ )?;+ let snake_alpha = block_tree.leaf("snake.alpha")?.validate(&[input_dim], data_type)?.read_array()?;+ let res_unit1 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit1")?, data_type, 1, output_dim)?;+ let res_unit2 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit2")?, data_type, 3, output_dim)?;+ let res_unit3 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit3")?, data_type, 9, output_dim)?;decoder_blocks.push(StructuredAudioDecoderBlock {snake_alpha,trans_conv,diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rsindex 827917577..46838e677 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs@@ -401,18 +401,19 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}-pub(super) fn gelu_enqueue<B: Backend>(+pub(super) fn activation_enqueue<B: Backend>(encoder: &mut Encoder<B>,input: &Array<B>,output: Array<B>,+ activation_type: ActivationType,kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {let n_u32 = u32::try_from(input.num_elements())- .map_err(|_| AudioError::Runtime("gelu element count exceeds u32 range".to_string()))?;+ .map_err(|_| AudioError::Runtime("activation element count exceeds u32 range".to_string()))?;let output_shape = output.shape().to_vec();let output_data_type = output.data_type();let mut output = output.into_allocation();- kernels.activation.encode(Some(input.allocation()), &mut output, n_u32, ActivationType::GELU, encoder);+ kernels.activation.encode(Some(input.allocation()), &mut output, n_u32, activation_type, encoder);Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rsindex 4b3293b3e..34a20a188 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs@@ -63,8 +63,9 @@ impl StructuredAudioCodecGraph {self.vocoder_data_type,kernels,)?;- let cnxt_gelu = ws.next_scratch(encoder, &[batch_size, layer.pwconv1.cout, seq_len], self.vocoder_data_type);- let x = gelu_enqueue(encoder, &x, cnxt_gelu, kernels)?;+ let cnxt_activation =+ ws.next_scratch(encoder, &[batch_size, layer.pwconv1.cout, seq_len], self.vocoder_data_type);+ let x = activation_enqueue(encoder, &x, cnxt_activation, layer.activation_type, kernels)?;let cnxt_pw2 = ws.next_scratch(encoder, &[batch_size, layer.pwconv2.cout, seq_len], self.vocoder_data_type);let x = conv1d_pointwise_ncs_enqueue(encoder,@@ -88,19 +89,13 @@ impl StructuredAudioCodecGraph {required_sequence_length: usize,) -> AudioResult<StructuredAudioPostModuleRuntime<B>> {let decoder_config = Rc::new(DecoderConfig {- embedding_config: EmbeddingConfig::Untied {- common: EmbeddingConfigCommon {- input_scale: None,- logit_soft_cap: None,- },- precision: self.vocoder_data_type.into(),- },+ embedding_config: AnyEmbeddingConfig::UntiedEmbeddingConfig(UntiedEmbeddingConfig::new(None, None)),transformer_config: self.config.quantizer_config.post_module_config.clone(),vocab_size: 1,pard_token: None,ple_model_config: None,});- let model_shape = ModelShape::from_decoder_config(&decoder_config);+ let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::F32);let weights_file = File::open(self.weights_path.as_str()).map_err(|err| {AudioError::Runtime(format!("failed to open post_module weights '{}': {err}", self.weights_path))@@ -110,12 +105,9 @@ impl StructuredAudioCodecGraph {})?;let root_loader_view = loader.tree();let transformer_subtree_name = "audio_decoder.quantizer.post_module";- root_loader_view- .subtree(transformer_subtree_name)- .map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;let max_sequence_length = decoder_config.transformer_config.context_length.max(required_sequence_length.max(1));- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config);+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);{let transformer_tree = root_loader_view.subtree(transformer_subtree_name)@@ -130,7 +122,10 @@ impl StructuredAudioCodecGraph {&decoder_config,&root_loader_view,transformer_subtree_name,- );+ None,+ &model_shape,+ )+ .map_err(|error| AudioError::Runtime(format!("failed to build structured audio post_module: {error}")))?;Ok(StructuredAudioPostModuleRuntime {context,@@ -180,23 +175,17 @@ impl StructuredAudioCodecGraph {let audio_decoder_tree = root.subtree("audio_decoder")?;let quantizer_tree = audio_decoder_tree.subtree("quantizer")?;let semantic_tree = quantizer_tree.subtree("semantic_quantizer")?.subtree("quantizers")?.subtree("0")?;- let semantic_codebook = read_float_matrix_exact::<B>(- &semantic_tree.subtree("codebook")?,- "weights",- self.semantic_codebook_size,- self.config.codebook_dim,- data_type,- )?;let codebook_dim = self.config.codebook_dim;- let semantic_out_proj = read_float_matrix_exact::<B>(- &semantic_tree.subtree("out_proj")?,- "weights",- self.input_dim,- codebook_dim,- data_type,- )?;+ let semantic_codebook = semantic_tree+ .leaf("codebook.weights")?+ .validate(&[self.semantic_codebook_size, codebook_dim], data_type)?+ .read_array()?;+ let semantic_out_proj = semantic_tree+ .leaf("out_proj.weights")?+ .validate(&[self.input_dim, codebook_dim], data_type)?+ .read_array()?;let semantic_out_bias =- read_float_vector_exact::<B>(&semantic_tree.subtree("out_proj")?, "biases", self.input_dim, data_type)?;+ semantic_tree.leaf("out_proj.biases")?.validate(&[self.input_dim], data_type)?.read_array()?;let residual_quantizers = self.config.n_codebooks;let residual_count_for_shape = residual_quantizers.max(1);@@ -210,26 +199,16 @@ impl StructuredAudioCodecGraph {let residual_root = quantizer_tree.subtree("quantizer")?.subtree("quantizers")?;for index in 0..residual_quantizers {let quantizer_tree = residual_root.subtree(&index.to_string())?;- let codebook = read_float_matrix_exact::<B>(- &quantizer_tree.subtree("codebook")?,- "weights",- self.codebook_size,- codebook_dim,- data_type,- )?;- let out_proj = read_float_matrix_exact::<B>(- &quantizer_tree.subtree("out_proj")?,- "weights",- self.input_dim,- codebook_dim,- data_type,- )?;- let out_bias = read_float_vector_exact::<B>(- &quantizer_tree.subtree("out_proj")?,- "biases",- self.input_dim,- data_type,- )?;+ let codebook = quantizer_tree+ .leaf("codebook.weights")?+ .validate(&[self.codebook_size, codebook_dim], data_type)?+ .read_array()?;+ let out_proj = quantizer_tree+ .leaf("out_proj.weights")?+ .validate(&[self.input_dim, codebook_dim], data_type)?+ .read_array()?;+ let out_bias =+ quantizer_tree.leaf("out_proj.biases")?.validate(&[self.input_dim], data_type)?.read_array()?;copy_to_outer_axis_slice(&mut residual_codebooks, index, &codebook, &[self.codebook_size, codebook_dim])?;copy_to_outer_axis_slice(&mut residual_out_proj, index, &out_proj, &[self.input_dim, codebook_dim])?;@@ -310,7 +289,6 @@ impl StructuredAudioCodecGraph {token_positions: token_inputs.token_positions(),token_parents: token_inputs.token_parents(),token_subtrie_ranges: None,- attention_sinks: runtime.shared_buffers.attention_sinks(layer.layer_index),rope_buffers,sampling_start: 0,sampling_length: batch_dim,@@ -431,11 +409,10 @@ impl StructuredAudioCodecGraph {self.input_dim, main_shape[0], main_shape[1])));}- if runtime.model_shape.activation_data_type() != self.vocoder_data_type {+ if runtime.model_shape.data_type != self.vocoder_data_type {return Err(AudioError::Runtime(format!("post_module dtype mismatch: main={:?}, latent={:?}",- runtime.model_shape.activation_data_type(),- self.vocoder_data_type+ runtime.model_shape.data_type, self.vocoder_data_type)));}diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rsindex 5677cdaec..a37a2830d 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs@@ -1,7 +1,7 @@use std::ops::Range;use super::*;-use crate::array::Array;+use crate::{array::Array, backends::common::gpu_types::ActivationType};pub(super) fn array_batch_byte_range<B: Backend>(array: &Array<B>,@@ -91,6 +91,7 @@ pub(super) struct StructuredAudioNorm<B: Backend> {#[derive(Debug)]pub(super) struct StructuredAudioConvNeXt<B: Backend> {+ pub(super) activation_type: ActivationType,pub(super) depthwise_conv: StructuredAudioConv1d<B>,pub(super) norm: StructuredAudioNorm<B>,pub(super) pwconv1: StructuredAudioPointwiseConv<B>,diff --git a/crates/backend-uzu/src/backends/common/activation_config.rs b/crates/backend-uzu/src/backends/common/activation_config.rsindex 499ddab07..3903d2609 100644--- a/crates/backend-uzu/src/backends/common/activation_config.rs+++ b/crates/backend-uzu/src/backends/common/activation_config.rs@@ -22,11 +22,14 @@ impl ActivationConfig {pub fn act_type(&self) -> ActivationType {match self {ActivationConfig::SILU {- ..+ alpha: _,} => ActivationType::SILU,ActivationConfig::GELU {- ..- } => ActivationType::GELU,+ approximate: true,+ } => ActivationType::GELUApprox,+ ActivationConfig::GELU {+ approximate: false,+ } => ActivationType::GELUExact,ActivationConfig::IDENTITY => ActivationType::IDENTITY,}}@@ -37,7 +40,7 @@ impl ActivationConfig {alpha,} => *alpha,ActivationConfig::GELU {- ..+ approximate: _,} => 1.0,ActivationConfig::IDENTITY => 1.0,}diff --git a/crates/backend-uzu/src/backends/common/command_buffer.rs b/crates/backend-uzu/src/backends/common/command_buffer.rsindex a4190b047..1f8a84648 100644--- a/crates/backend-uzu/src/backends/common/command_buffer.rs+++ b/crates/backend-uzu/src/backends/common/command_buffer.rs@@ -75,13 +75,13 @@ pub trait CommandBufferEncoding {Dst: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>,>(&mut self,- src: BufferRangeRef<'_, Src>,- dst: BufferRangeMut<'_, Dst>,+ src: BufferRangeRef<Src>,+ dst: BufferRangeMut<Dst>,);fn encode_fill<Dst: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>>(&mut self,- dst: BufferRangeMut<'_, Dst>,+ dst: BufferRangeMut<Dst>,value: u8,);diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rsindex 9e429ea7e..466fc3414 100644--- a/crates/backend-uzu/src/backends/common/context.rs+++ b/crates/backend-uzu/src/backends/common/context.rs@@ -10,7 +10,7 @@ pub trait Context: Sized {fn recommended_async_batch_size(&self,model_path: &Path,- ) -> usize;+ ) -> Result<usize, <Self::Backend as Backend>::Error>;fn is_high_performance(&self) -> bool;diff --git a/crates/backend-uzu/src/backends/common/gpu_types/activation_type.rs b/crates/backend-uzu/src/backends/common/gpu_types/activation_type.rsindex 7524bff9b..654236249 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/activation_type.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/activation_type.rs@@ -7,7 +7,8 @@ const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)#[derive(Debug, Copy, Clone, PartialEq)]pub enum ActivationType {SILU,- GELU,+ GELUApprox,+ GELUExact,TANH,IDENTITY,SOFTPLUS,@@ -19,10 +20,9 @@ impl ActivationType {x: T,) -> T {match self {- ActivationType::SILU {- ..- } => silu(x),- ActivationType::GELU => gelu(x),+ ActivationType::SILU => silu(x),+ ActivationType::GELUApprox => gelu_approx(x),+ ActivationType::GELUExact => gelu_exact(x),ActivationType::TANH => tanh_activation(x),ActivationType::IDENTITY => x,ActivationType::SOFTPLUS => softplus(x),@@ -43,13 +43,19 @@ fn silu<T: Float>(x: T) -> T {activation_silu_alpha(x, 1.0f32)}-fn gelu<T: Float>(x: T) -> T {+fn gelu_approx<T: Float>(x: T) -> T {let x_float = x.to_f32().unwrap();let tan_arg = GELU_K1 * (x_float + GELU_K0 * x_float * x_float * x_float);let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());T::from(y_float).unwrap()}+fn gelu_exact<T: Float>(x: T) -> T {+ let x_float = x.to_f32().unwrap();+ let y_float = 0.5f32 * x_float * (1.0f32 + libm::erff(x_float * std::f32::consts::FRAC_1_SQRT_2));+ T::from(y_float).unwrap()+}+fn tanh_activation<T: Float>(x: T) -> T {let x_float = x.to_f32().unwrap();T::from(x_float.tanh()).unwrap()diff --git a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rsnew file mode 100644index 000000000..d29a63a54--- /dev/null+++ b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs@@ -0,0 +1,6 @@+#[repr(C)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum HadamardTransformOrder {+ Input,+ Output,+}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs b/crates/backend-uzu/src/backends/common/gpu_types/mod.rsindex 8d39a391d..4be07fa23 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs@@ -7,6 +7,7 @@ pub mod activation_type;pub mod argmax;pub mod attention;pub mod gemm;+pub mod hadamard_order;pub mod kv_cache_update;pub mod matmul;pub mod quantization;@@ -17,6 +18,7 @@ pub mod trie;pub use activation_type::*;pub use argmax::*;pub use attention::*;+pub use hadamard_order::*;pub use kv_cache_update::*;pub use matmul::*;pub use quantization::*;diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rsindex 7cb79d790..0e81bcd14 100644--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs@@ -70,8 +70,8 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {fn encode_copy<Src: Buffer<Backend = Cpu>, Dst: Buffer<Backend = Cpu>>(&mut self,- src: BufferRangeRef<'_, Src>,- dst: BufferRangeMut<'_, Dst>,+ src: BufferRangeRef<Src>,+ dst: BufferRangeMut<Dst>,) {let src_range = src.range();let dst_range = dst.range();@@ -90,7 +90,7 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {fn encode_fill<Dst: Buffer<Backend = Cpu>>(&mut self,- dst: BufferRangeMut<'_, Dst>,+ dst: BufferRangeMut<Dst>,value: u8,) {let range = dst.range();@@ -172,7 +172,7 @@ impl CommandBufferPending for CpuCommandBufferPending {type CommandBuffer = CpuCommandBuffer;fn wait_until_completed(self) -> Result<CpuCommandBufferCompleted, CpuError> {- self.return_receiver.recv().map_err(CpuError::CommandBufferExecutionFailed)+ Ok(self.return_receiver.recv()?)}}diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rsindex deeb3edfc..b252bc06c 100644--- a/crates/backend-uzu/src/backends/cpu/context.rs+++ b/crates/backend-uzu/src/backends/cpu/context.rs@@ -30,10 +30,9 @@ impl Context for CpuContext {fn recommended_async_batch_size(&self,- model_path: &Path,- ) -> usize {- let _ = model_path;- 1+ _model_path: &Path,+ ) -> Result<usize, CpuError> {+ Ok(1)}fn is_high_performance(&self) -> bool {diff --git a/crates/backend-uzu/src/backends/cpu/error.rs b/crates/backend-uzu/src/backends/cpu/error.rsindex 5b5ca0821..581d30aa0 100644--- a/crates/backend-uzu/src/backends/cpu/error.rs+++ b/crates/backend-uzu/src/backends/cpu/error.rs@@ -9,7 +9,7 @@ pub enum CpuError {#[error("Not supported")]NotSupported,#[error("Command buffer execution failed")]- CommandBufferExecutionFailed(RecvError),+ CommandBufferExecutionFailed(#[from] RecvError),#[error("Kernel dispatch failed: {0}")]KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rsindex a6c599203..7b5d9fe33 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs@@ -20,7 +20,7 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(#[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,#[optional(is_trie)] trie: Option<*const TrieNode>,#[optional(is_sliding_window)] sliding_window_size: Option<u32>,- #[optional(has_sinks)] sinks: Option<*const f32>,+ #[optional(has_sinks)] sinks: Option<*const T>,num_heads: u32,suffix_length: u32,#[specialize] align_q: bool,@@ -72,7 +72,7 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(// Initialize with attention sinks if presentif has_sinks {- max_score = unsafe { *sinks.unwrap().add(head_idx) };+ max_score = unsafe { *sinks.unwrap().add(head_idx) }.to_f32().unwrap();sum_exp = 1.0;}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rsindex bf9af43be..b1e77d62f 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs@@ -25,7 +25,7 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(scale: f32,#[optional(is_trie)] trie: Option<*const TrieNode>,#[optional(is_sliding_window)] sliding_window_size: Option<u32>,- #[optional(has_sinks)] sinks: Option<*const f32>,+ #[optional(has_sinks)] sinks: Option<*const T>,num_heads: u32,suffix_length: u32,#[specialize] has_sinks: bool,@@ -72,7 +72,7 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(let mut sum_exp_score = 0.0f32;if has_sinks {let q_head_idx = head_idx % num_heads;- max_score = unsafe { *sinks.unwrap().add(q_head_idx as usize) };+ max_score = unsafe { *sinks.unwrap().add(q_head_idx as usize) }.to_f32().unwrap();sum_exp_score = 1.0;}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rsindex 8ca28bf61..1f8d0b250 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs@@ -31,7 +31,7 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(suffix_length: u32,#[optional(is_trie)] trie: Option<*const TrieNode>,#[optional(is_sliding_window)] sliding_window_size: Option<u32>,- #[optional(has_sinks)] sinks: Option<*const f32>,+ #[optional(has_sinks)] sinks: Option<*const T>,#[specialize] has_sinks: bool,#[specialize] is_kv_cache_ring: bool,#[specialize] is_causal: bool,@@ -79,7 +79,7 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(let mut sum_exp_score = 0.0f32;if has_sinks && block_idx == 0 {- max_score = unsafe { *sinks.unwrap().add(head_idx as usize) };+ max_score = unsafe { *sinks.unwrap().add(head_idx as usize) }.to_f32().unwrap();sum_exp_score = 1.0;}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsindex f63a9dae9..97e257a8a 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -2,7 +2,10 @@ use half::{bf16, f16};use num_traits::Float;use proc_macros::kernel;-use crate::{ArrayElement, backends::common::gpu_types::QuantizationMode};+use crate::{+ ArrayElement,+ backends::common::gpu_types::{QuantizationMethod, QuantizationMode},+};#[kernel(QuantizedEmbeddingLookup)]#[variants(T, f32, f16, bf16)]@@ -10,8 +13,16 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(#[allow(unused)] token_ids: *const u64,#[allow(unused)] weights: *const u8,#[allow(unused)] scales: *const T,- #[allow(unused)] biases: *const T,+ #[allow(unused)]+ #[optional(quantization_method == QuantizationMethod::ScaleZeroPoint)]+ zero_points: Option<*const u8>,+ #[allow(unused)]+ #[optional(quantization_method == QuantizationMethod::ScaleBias)]+ biases: Option<*const T>,#[allow(unused)] output: *mut T,+ #[allow(unused)]+ #[optional(use_hadamard)]+ output_hadamard_factors: Option<*const i32>,#[allow(unused)] batch_size: u32,#[allow(unused)] vocab_size: u32,#[allow(unused)] model_dim: u32,@@ -22,10 +33,24 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(#[allow(unused)]#[specialize]quantization_mode: QuantizationMode,+ #[allow(unused)]+ #[specialize]+ quantization_method: QuantizationMethod,+ #[allow(unused)]+ #[specialize]+ use_hadamard: bool,) {+ if use_hadamard {+ unimplemented!("not supported yet");+ }+let packing_divisor = quantization_mode.packing_divisor() as u32;let weights_stride = model_dim / packing_divisor;let num_groups = (model_dim + group_size - 1) / group_size;+ let zero_points_stride = match quantization_mode {+ QuantizationMode::U4 => num_groups.div_ceil(2),+ QuantizationMode::I8 | QuantizationMode::U8 => num_groups,+ };unsafe {for batch_idx in 0..batch_size {@@ -41,7 +66,6 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(let group_idx = dim_idx / group_size;let scale = *scales.add((token_id as u32 * num_groups + group_idx) as usize);- let bias = *biases.add((token_id as u32 * num_groups + group_idx) as usize);let quantized_value: i32 = match quantization_mode {QuantizationMode::U4 => {@@ -64,7 +88,34 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(},};- let out_f = scale.to_f32().unwrap() * quantized_value as f32 + bias.to_f32().unwrap();+ let bias = match quantization_method {+ QuantizationMethod::ScaleZeroPoint => {+ let zero_points = zero_points.expect("ScaleZeroPoint quantized embedding requires zero_points");+ let zero_point = match quantization_mode {+ QuantizationMode::U4 => {+ let byte_idx = (token_id as u32 * zero_points_stride + group_idx / 2) as usize;+ let packed = *zero_points.add(byte_idx);+ if (group_idx & 1) == 0 {+ packed & 0x0F+ } else {+ (packed >> 4) & 0x0F+ }+ },+ QuantizationMode::I8 | QuantizationMode::U8 => {+ *zero_points.add((token_id as u32 * zero_points_stride + group_idx) as usize)+ },+ };+ -scale.to_f32().unwrap() * zero_point as f32+ },+ QuantizationMethod::ScaleBias => biases+ .expect("ScaleBias quantized embedding requires biases")+ .add((token_id as u32 * num_groups + group_idx) as usize)+ .read()+ .to_f32()+ .unwrap(),+ };++ let out_f = scale.to_f32().unwrap() * quantized_value as f32 + bias;let out_f = out_f * input_scale;*output.add(out_idx) = T::from(out_f).unwrap();}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rsindex 53a0f82d1..94345bdd5 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs@@ -2,7 +2,7 @@ use half::{bf16, f16};use num_traits::Float;use proc_macros::kernel;-use crate::ArrayElement;+use crate::{ArrayElement, backends::common::gpu_types::HadamardTransformOrder};#[kernel(HadamardTransform)]#[variants(T, f32, f16, bf16)]@@ -11,6 +11,9 @@ pub fn hadamard_transform_mul<T: ArrayElement + Float>(#[allow(unused)] factors: *const i32,#[allow(unused)] hidden_dim: u32,#[allow(unused)] batch_size: u32,+ #[allow(unused)]+ #[specialize]+ transform_order: HadamardTransformOrder,) {todo!()}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 58094e700..178acc315 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -5,7 +5,7 @@ use crate::{backends::{common::{Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,- gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},+ gpu_types::{HadamardTransformOrder, QuantizationMethod, QuantizationMode, gemm::GemmDTransform},kernel::{HadamardTransformKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, TensorAddBiasKernel,matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel},@@ -34,10 +34,12 @@ impl MatmulKernel for MatmulCpuKernel {if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());}- let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)- .map_err(CpuError::from)?;- let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, true)- .map_err(CpuError::from)?;+ let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(+ context,+ data_type,+ HadamardTransformOrder::Output,+ )?;+ let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, true)?;Ok(Self {data_type,hadamard,@@ -53,13 +55,13 @@ impl MatmulKernel for MatmulCpuKernel {match arguments.b {MatmulB::FullPrecision {..- } => self.encode_fp(arguments, encoder).map_err(CpuError::from),+ } => Ok(self.encode_fp(arguments, encoder)?),MatmulB::ScaleBiasDequant {..}| MatmulB::ScaleZeroPointDequant {..- } => self.encode_quant(arguments, encoder).map_err(CpuError::from),+ } => Ok(self.encode_quant(arguments, encoder)?),}}}@@ -241,6 +243,10 @@ impl MatmulCpuKernel {let post_bias = arguments.d_transform.bias;if arguments.m >= 5 && arguments.n > 1 {+ assert!(+ !(post_bias.is_some() && post_rht.is_some()),+ "MatmulCpuKernel/Quant GEMM with both output bias and output RHT is not supported: bias must be applied after RHT",+ );let MatmulArguments {a,a_offset,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/moe/experts_single.rs b/crates/backend-uzu/src/backends/cpu/kernel/moe/experts_single.rsindex 40948e2f9..261ed6a4d 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/moe/experts_single.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/moe/experts_single.rs@@ -65,7 +65,7 @@ pub fn moe_experts_decode_single_pass_a<T: ArrayElement + Float>(let up_val = (acc_up + unsafe { *biases.add(bias_base + h_idx) }.to_f32().unwrap()).clamp(up_clip_min, up_clip_max);if gating_sel == 0 {- ActivationType::GELU.activate(up_val)+ ActivationType::GELUApprox.activate(up_val)} else {activation_silu_alpha(up_val, silu_alpha)}@@ -77,7 +77,7 @@ pub fn moe_experts_decode_single_pass_a<T: ArrayElement + Float>(let gate_act = if gating_sel == 2 {activation_silu_alpha(gate_val, silu_alpha)} else {- ActivationType::GELU.activate(gate_val)+ ActivationType::GELUApprox.activate(gate_val)};gate_act * up_val};diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs b/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rsindex a57068d9f..76a125e9a 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs@@ -4,33 +4,38 @@ use proc_macros::kernel;use crate::ArrayElement;-fn get_paired_val<T: ArrayElement + Float>(- values: *const T,+fn get_paired_val<ElementT: ArrayElement + Float>(+ values: *const ElementT,dim_index: usize,half_dim: usize,token_index: usize,total_heads: usize,head_dim: usize,head_index: usize,-) -> T {+) -> f32 {if dim_index < half_dim {let v =unsafe { *values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index + half_dim) };- T::zero() - v+ -v.to_f32().unwrap()} else {- unsafe { *values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index - half_dim) }+ unsafe {+ (*values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index - half_dim))+ .to_f32()+ .unwrap()+ }}}#[kernel(Rope)]-#[variants(T, f32, f16, bf16)]-pub fn rope<T: ArrayElement + Float>(- qkv: *const T,- cosines: *const T,- sines: *const T,+#[variants(ElementT, f32, f16, bf16)]+#[variants(RopeT, f32, f16, bf16)]+pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(+ qkv: *const ElementT,+ cosines: *const RopeT,+ sines: *const RopeT,token_positions: *const i32,- rotated_queries: *mut T,- rotated_keys: *mut T,+ rotated_queries: *mut ElementT,+ rotated_keys: *mut ElementT,head_dim: u32,rope_dim: u32,num_heads: u32,@@ -66,25 +71,30 @@ pub fn rope<T: ArrayElement + Float>(// Rotated dimensions: apply RoPE to dims 0..rope_dimfor dim_index in 0..rope_dim {- let cos_val = unsafe { *cosines.add(absolute_position * rope_dim + dim_index) };- let sin_val = unsafe { *sines.add(absolute_position * rope_dim + dim_index) };+ let cos_val = unsafe { (*cosines.add(absolute_position * rope_dim + dim_index)).to_f32().unwrap() };+ let sin_val = unsafe { (*sines.add(absolute_position * rope_dim + dim_index)).to_f32().unwrap() };// Query rotation- let q_val =- unsafe { *qkv.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index) };+ let q_val = unsafe {+ (*qkv.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index))+ .to_f32()+ .unwrap()+ };let q_paired =get_paired_val(qkv, dim_index, half_rope_dim, token_index, total_heads, head_dim, head_index);let q_result = q_val * cos_val + q_paired * sin_val;unsafe {*rotated_queries.add(head_index * suffix_length * head_dim + token_index * head_dim + dim_index) =- q_result;+ ElementT::from(q_result).unwrap();}// Key rotation (only first head in each group)if head_index == group_index * heads_per_group {let key_head_index = num_heads + group_index;let k_val = unsafe {- *qkv.add(token_index * total_heads * head_dim + key_head_index * head_dim + dim_index)+ (*qkv.add(token_index * total_heads * head_dim + key_head_index * head_dim + dim_index))+ .to_f32()+ .unwrap()};let k_paired = get_paired_val(qkv,@@ -99,7 +109,7 @@ pub fn rope<T: ArrayElement + Float>(unsafe {*rotated_keys.add(group_index * suffix_length * head_dim + token_index * head_dim + dim_index) =- k_result;+ ElementT::from(k_result).unwrap();}}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs b/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rsindex 0d3947ddc..d484b2a57 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs@@ -8,7 +8,7 @@ use crate::ArrayElement;#[variants(T, f32, f16, bf16)]pub fn softmax<T: ArrayElement + Float>(values: *mut T,- #[optional(has_sinks)] sinks: Option<*const f32>,+ #[optional(has_sinks)] sinks: Option<*const T>,row_length: u32,outer_dim: u32,batch_dim: u32,@@ -18,7 +18,7 @@ pub fn softmax<T: ArrayElement + Float>(for batch_index in 0..batch_dim {let row_offset = ((outer_index * batch_dim + batch_index) * row_length) as usize;let sink = if has_sinks {- unsafe { *sinks.unwrap().add(outer_index as usize) }+ unsafe { *sinks.unwrap().add(outer_index as usize) }.to_f32().unwrap()} else {f32::NEG_INFINITY};diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rsindex a55e6dc1a..3b939927c 100644--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs@@ -130,8 +130,8 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {fn encode_copy<Src: Buffer<Backend = Metal>, Dst: Buffer<Backend = Metal>>(&mut self,- src: BufferRangeRef<'_, Src>,- dst: BufferRangeMut<'_, Dst>,+ src: BufferRangeRef<Src>,+ dst: BufferRangeMut<Dst>,) {let src_range = src.range();let dst_range = dst.range();@@ -148,7 +148,7 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {fn encode_fill<Dst: Buffer<Backend = Metal>>(&mut self,- dst: BufferRangeMut<'_, Dst>,+ dst: BufferRangeMut<Dst>,value: u8,) {let range = dst.range();diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex ef90189b1..c4d9af74c 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -160,17 +160,17 @@ impl Context for MetalContext {fn recommended_async_batch_size(&self,model_path: &Path,- ) -> usize {+ ) -> Result<usize, MetalError> {let cores = self.device_capabilities.gpu_core_count;- let model_size = ModelSize::from_path(model_path);- match (model_size, cores) {+ let model_size = ModelSize::from_path(model_path)?;+ Ok(match (model_size, cores) {(ModelSize::Large, c) if c > 20 => 32,(ModelSize::Large, c) if c > 10 => 16,(ModelSize::Large, _) => 8,(ModelSize::Small, c) if c > 20 => 256,(ModelSize::Small, c) if c > 10 => 128,(ModelSize::Small, _) => 64,- }+ })}fn debug_active(&self) -> bool {diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rsindex 2d3c1e4ed..099eb48ef 100644--- a/crates/backend-uzu/src/backends/metal/error.rs+++ b/crates/backend-uzu/src/backends/metal/error.rs@@ -19,6 +19,8 @@ pub enum MetalError {CannotCreateCommandQueue,#[error("Cannot create command Metal 4 queue")]CannotCreateCommandQueueMtl4,+ #[error("Cannot read model weights metadata: {0}")]+ CannotReadModelWeightsMetadata(#[from] std::io::Error),#[error("Cannot create buffer")]CannotCreateBuffer,#[error("Cannot create command buffer")]diff --git a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.hindex b05b638c4..ede7a07cd 100644--- a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h+++ b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h@@ -28,6 +28,40 @@ inline T activate_gelu(T x) {return T(yf);}+inline float erf_precise(float a) {+ // Faithfully rounded erff polynomial from Norbert Juffa, also used by MLX.+ float t = metal::abs(a);+ float s = a * a;++ if (t > 0.927734375f) {+ float r = metal::fma(-1.72853470e-5f, t, 3.83197126e-4f);+ float u = metal::fma(-3.88396438e-3f, t, 2.42546219e-2f);+ r = metal::fma(r, s, u);+ r = metal::fma(r, t, -1.06777877e-1f);+ r = metal::fma(r, t, -6.34846687e-1f);+ r = metal::fma(r, t, -1.28717512e-1f);+ r = metal::fma(r, t, -t);+ r = 1.0f - metal::precise::exp(r);+ return metal::copysign(r, a);+ }++ float r = -5.96761703e-4f;+ r = metal::fma(r, s, 4.99119423e-3f);+ r = metal::fma(r, s, -2.67681349e-2f);+ r = metal::fma(r, s, 1.12819925e-1f);+ r = metal::fma(r, s, -3.76125336e-1f);+ r = metal::fma(r, s, 1.28379166e-1f);+ return metal::fma(r, a, a);+}++template <typename T>+inline T activate_gelu_exact(T x) {+ constexpr float inv_sqrt_2 = 0.7071067811865475f;+ float xf = float(x);+ float yf = 0.5f * xf * (1.0f + erf_precise(inv_sqrt_2 * xf));+ return T(yf);+}+template <typename T>inline T activate_leaky_relu(T x, float alpha) {const float xf = float(x);@@ -56,8 +90,10 @@ inline T activate(T x, ActivationType type) {switch (type) {case ActivationType::SILU:return activate_silu(x);- case ActivationType::GELU:+ case ActivationType::GELUApprox:return activate_gelu(x);+ case ActivationType::GELUExact:+ return activate_gelu_exact(x);case ActivationType::TANH:return activate_tanh(x);case ActivationType::IDENTITY:diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex fd40c931f..33d1ed248 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -54,7 +54,7 @@ PUBLIC KERNEL(AttentionGemm)(const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),const device TrieNode* trie OPTIONAL(is_trie),const constant uint& sliding_window_size OPTIONAL(is_sliding_window),- const device float* sinks OPTIONAL(has_sinks),+ const device T* sinks OPTIONAL(has_sinks),const constant uint& num_heads,const constant uint& suffix_length,const bool align_q SPECIALIZE,diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex e9c89c2f7..221d1ed3d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -30,7 +30,7 @@ PUBLIC KERNEL(AttentionSinglePass)(const constant float& scale,const device TrieNode* trie OPTIONAL(is_trie),const constant uint& sliding_window_size OPTIONAL(is_sliding_window),- const device float* sinks OPTIONAL(has_sinks),+ const device T* sinks OPTIONAL(has_sinks),const constant uint& num_heads,const constant uint& suffix_length,threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex 713629426..5f4b95300 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -39,7 +39,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(const constant uint& suffix_length,const device TrieNode* trie OPTIONAL(is_trie),const constant uint& sliding_window_size OPTIONAL(is_sliding_window),- const device float* sinks OPTIONAL(has_sinks),+ const device T* sinks OPTIONAL(has_sinks),const bool has_sinks SPECIALIZE,const bool is_kv_cache_ring SPECIALIZE,const bool is_causal SPECIALIZE,diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex d0852fb1d..14a6b81fc 100644--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -1,7 +1,10 @@#include <metal_stdlib>#include "../common/dsl.h"+#include "../generated/quantization_method.h"+#include "../hadamard_transform/hadamard_transform.h"#include "quantization.h"+using namespace uzu::quantization_method;using namespace uzu::quantization;template <typename T>@@ -10,16 +13,20 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (const device uint64_t* token_ids, // [batch_size]const device uint8_t* weights, // [vocab_size, model_dim/packing_divisor] packedconst device T* scales, // [vocab_size, num_groups]- const device T* biases, // [vocab_size, num_groups]+ const device uint8_t* zero_points OPTIONAL(quantization_method == QuantizationMethod::ScaleZeroPoint),+ const device T* biases OPTIONAL(quantization_method == QuantizationMethod::ScaleBias),device T* output, // [batch_size, model_dim]+ const device int32_t* output_hadamard_factors OPTIONAL(use_hadamard),constant uint32_t& batch_size,constant uint32_t& vocab_size,constant uint32_t& model_dim,constant float& input_scale,const uint32_t group_size SPECIALIZE,const QuantizationMode quantization_mode SPECIALIZE,- const uint dim_idx AXIS(model_dim, 16),- const uint batch_idx AXIS(batch_size, 16)+ const QuantizationMethod quantization_method SPECIALIZE,+ const bool use_hadamard SPECIALIZE,+ const uint dim_idx AXIS(model_dim, 32),+ const uint batch_idx AXIS(batch_size, 1)) {const uint thread_position_in_grid = batch_idx * model_dim + dim_idx;const uint64_t token_id = token_ids[batch_idx];@@ -30,9 +37,11 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (const uint group_idx = dim_idx / group_size;const uint num_groups = (model_dim + group_size - 1) / group_size;+ const uint zp_stride = quantization_mode == QuantizationMode::U4+ ? (num_groups + 1) / 2+ : num_groups;const T scale = scales[token_id * num_groups + group_idx];- const T bias = biases[token_id * num_groups + group_idx];const uint packing_divisor =quantization_mode == QuantizationMode::U4 ? 2 : 1;@@ -64,7 +73,30 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (};}- float out_f = float(scale) * float(quantized_value) + float(bias);+ float bias = 0.0f;+ if (quantization_method == QuantizationMethod::ScaleBias) {+ bias = float(biases[token_id * num_groups + group_idx]);+ } else {+ uint zero_point = 0;+ if (quantization_mode == QuantizationMode::U4) {+ const uint8_t packed = zero_points[token_id * zp_stride + group_idx / 2];+ zero_point =+ (group_idx & 1) == 0 ? (packed & 0x0F) : ((packed >> 4) & 0x0F);+ } else {+ zero_point = zero_points[token_id * zp_stride + group_idx];+ }+ bias = -float(scale) * float(zero_point);+ }++ float out_f = float(scale) * float(quantized_value) + bias;out_f *= input_scale;- output[thread_position_in_grid] = T(out_f);+ T out = T(out_f);+ if (use_hadamard) {+ out = simdgroup_output_random_hadamard_transform(+ static_cast<ushort>(dim_idx % METAL_SIMD_SIZE),+ out,+ output_hadamard_factors[dim_idx]+ );+ }+ output[thread_position_in_grid] = out;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.hindex eda99e0a2..60b21b5de 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h@@ -7,9 +7,10 @@ using namespace metal;namespace uzu::activation_type {enum class ActivationType : uint32_t {SILU = 0,- GELU = 1,- TANH = 2,- IDENTITY = 3,- SOFTPLUS = 4,+ GELUApprox = 1,+ GELUExact = 2,+ TANH = 3,+ IDENTITY = 4,+ SOFTPLUS = 5,};} // namespace uzu::activation_typediff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.hnew file mode 100644index 000000000..e9f7bd50c--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h@@ -0,0 +1,12 @@+// Auto-generated from gpu_types/hadamard_order - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::hadamard_order {+enum class HadamardTransformOrder : uint32_t {+ Input = 0,+ Output = 1,+};+} // namespace uzu::hadamard_orderdiff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.hindex d5f62b642..314628029 100644--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h@@ -18,7 +18,7 @@ static METAL_FUNC float simdgroup_hadamard_transform(}template <typename T>-static METAL_FUNC T simdgroup_random_hadamard_transform(+static METAL_FUNC T simdgroup_input_random_hadamard_transform(ushort lane_index,T lane_value,int32_t lane_factor@@ -28,3 +28,15 @@ static METAL_FUNC T simdgroup_random_hadamard_transform(float(lane_value) * float(lane_factor)));}++template <typename T>+static METAL_FUNC T simdgroup_output_random_hadamard_transform(+ ushort lane_index,+ T lane_value,+ int32_t lane_factor+) {+ return T(+ simdgroup_hadamard_transform(lane_index, float(lane_value)) *+ float(lane_factor)+ );+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metalindex 4056b73b5..78b038b02 100644--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal@@ -1,9 +1,11 @@#include <metal_stdlib>#include "../common/defines.h"#include "../common/dsl.h"+#include "../generated/hadamard_order.h"#include "hadamard_transform.h"using namespace metal;+using namespace uzu::hadamard_order;template <typename T>VARIANTS(T, half, bfloat, float)@@ -12,6 +14,7 @@ PUBLIC KERNEL(HadamardTransform)(const device int32_t* factors,constant uint& hidden_dim,constant uint& batch_size,+ const HadamardTransformOrder transform_order SPECIALIZE,uint block_index GROUPS(hidden_dim.div_ceil(METAL_SIMD_SIZE)),uint batch_index GROUPS(batch_size),uint lane_index THREADS(METAL_SIMD_SIZE)@@ -19,9 +22,17 @@ PUBLIC KERNEL(HadamardTransform)(uint factor_index = block_index * METAL_SIMD_SIZE + lane_index;uint element_index = batch_index * hidden_dim + factor_index;- data[element_index] = simdgroup_random_hadamard_transform(- lane_index,- data[element_index],- factors[factor_index]- );+ if (transform_order == HadamardTransformOrder::Input) {+ data[element_index] = simdgroup_input_random_hadamard_transform(+ lane_index,+ data[element_index],+ factors[factor_index]+ );+ } else {+ data[element_index] = simdgroup_output_random_hadamard_transform(+ lane_index,+ data[element_index],+ factors[factor_index]+ );+ }}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex cc5650f78..d56741462 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -7,7 +7,7 @@ use crate::{common::{Allocation, AsBufferRangeRef, Backend, Buffer, Encoder,gpu_types::{- GemmParams,+ GemmParams, HadamardTransformOrder,gemm::{GemmAlignment, GemmDTransform, GemmTiling},},kernel::{@@ -44,7 +44,11 @@ impl GemmKernel {data_type: DataType,) -> Result<Self, MetalError> {let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true)?;- let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)?;+ let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(+ context,+ data_type,+ HadamardTransformOrder::Output,+ )?;let mut kernel = Self {data_type,kernels: HashMap::new(),@@ -137,13 +141,18 @@ impl GemmKernel {let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });if is_quant {- if arguments.d_transform.mask().contains(GemmDTransform::ACCUMULATE) {+ let d_mask = arguments.d_transform.mask();+ if d_mask.contains(GemmDTransform::ACCUMULATE) {return Err(MatmulError::UnsupportedDOp {bit: GemmDTransform::ACCUMULATE,path: "QuantGemm",}.into());}+ assert!(+ !d_mask.contains(GemmDTransform::BIAS | GemmDTransform::RHT),+ "QuantGemm with both output bias and output RHT is not supported: bias must be applied after RHT",+ );if !arguments.b_transpose || arguments.b_leading_dimension.is_some() || arguments.b_offset != 0 {return Err(MatmulError::UnsupportedLayout {path: "QuantGemm",@@ -255,7 +264,7 @@ impl GemmKernel {bits_per_weight,group_size,};- specialization.validate().map_err(MetalError::from)?;+ specialization.validate()?;let kernel = self.get_or_create(encoder.context(), specialization)?;kernel.encode((a, a_offset),@@ -311,7 +320,7 @@ impl GemmKernel {bits_per_weight,group_size,};- specialization.validate().map_err(MetalError::from)?;+ specialization.validate()?;let kernel = self.get_or_create(encoder.context(), specialization)?;kernel.encode((a, a_offset),diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metalindex a27f1591c..8d0fbfb2c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal@@ -177,7 +177,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(if (simd_group == 0) {uint global_out_idx = out_block_idx * 32 + simd_lane;if (global_out_idx < out_vec_size) {- output[simd_lane] = simdgroup_random_hadamard_transform(+ output[simd_lane] = simdgroup_output_random_hadamard_transform(static_cast<ushort>(simd_lane),T(shared_results[simd_lane]),hadamard_factors[global_out_idx]diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 7756cc248..04e1db85d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -147,7 +147,7 @@ impl MatmulKernel for MatmulMetalKernel {}let gemm = GemmKernel::new(context, data_type)?;- let gemv = GemvKernel::new(context, data_type).map_err(MetalError::from)?;+ let gemv = GemvKernel::new(context, data_type)?;let quant_gemv = QuantGemvKernel::new(context, data_type);Ok(Self {@@ -174,10 +174,11 @@ impl MatmulKernel for MatmulMetalKernel {if gemv_eligible {if is_quant {- self.dispatch_quant_gemv(arguments, encoder).map_err(MetalError::from)+ self.dispatch_quant_gemv(arguments, encoder)?;} else {- self.dispatch_gemv(arguments, encoder).map_err(MetalError::from)+ self.dispatch_gemv(arguments, encoder)?;}+ Ok(())} else {self.gemm.encode(arguments, encoder)}diff --git a/crates/backend-uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/backend-uzu/src/backends/metal/kernel/mlp/activation_mul.metalindex 157527261..3aabfcb80 100644--- a/crates/backend-uzu/src/backends/metal/kernel/mlp/activation_mul.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -25,7 +25,7 @@ PUBLIC KERNEL(MlpGateActMul) (T result = T(float(up) * float(g));if (use_hadamard) {- result = simdgroup_random_hadamard_transform(+ result = simdgroup_input_random_hadamard_transform(static_cast<ushort>(j % METAL_SIMD_SIZE),result,hadamard_factors[j]diff --git a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex bc8afaa78..9915a5c20 100644--- a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -109,7 +109,7 @@ PUBLIC KERNEL(RMSNorm)(}if (use_hadamard) {- val = static_cast<OutputT>(simdgroup_random_hadamard_transform(+ val = static_cast<OutputT>(simdgroup_input_random_hadamard_transform(static_cast<ushort>(thread_in_row % METAL_SIMD_SIZE),val,hadamard_factors[i]diff --git a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metalindex 0bee9eb34..f3bb1f7ce 100644--- a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal@@ -2,32 +2,34 @@#include "../common/dsl.h"#include "../common/tensor_view.h"-template <typename T>-inline T applyRopeTransform(- TensorView3D<const T> qkv_tensor_view,+template <typename ElementT>+inline ElementT applyRopeTransform(+ TensorView3D<const ElementT> qkv_tensor_view,uint token_idx,uint head_idx,uint dim_idx,uint half_dim,- T cos_val,- T sin_val+ float cos_val,+ float sin_val) {- T inputVal = qkv_tensor_view(token_idx, head_idx, dim_idx);- T pairedVal = (dim_idx < half_dim)- ? -qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim)- : qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim);- return inputVal * cos_val + pairedVal * sin_val;+ float inputVal = float(qkv_tensor_view(token_idx, head_idx, dim_idx));+ float pairedVal =+ (dim_idx < half_dim)+ ? -float(qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim))+ : float(qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim));+ return static_cast<ElementT>(inputVal * cos_val + pairedVal * sin_val);}-template <typename T>-VARIANTS(T, float, half, bfloat)+template <typename ElementT, typename RopeT>+VARIANTS(ElementT, float, half, bfloat)+VARIANTS(RopeT, float, half, bfloat)PUBLIC KERNEL(Rope)(- device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]- device const T* cosines, // [max_seq_len, rope_dim]- device const T* sines, // [max_seq_len, rope_dim]+ device const ElementT* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]+ device const RopeT* cosines, // [max_seq_len, rope_dim]+ device const RopeT* sines, // [max_seq_len, rope_dim]device const int* token_positions, // [suffix_len] - actual token positions- device T* rotated_queries, // [num_heads, suffix_len, head_dim]- device T* rotated_keys, // [num_groups, suffix_len, head_dim]+ device ElementT* rotated_queries, // [num_heads, suffix_len, head_dim]+ device ElementT* rotated_keys, // [num_groups, suffix_len, head_dim]constant uint& head_dim,constant uint& rope_dim,constant uint& num_heads,@@ -62,26 +64,30 @@ PUBLIC KERNEL(Rope)(raw_position > max_sequence_length ? 0 : raw_position;const uint half_rope_dim = rope_dim / 2;- TensorView3D<const T> qkv_tensor_view =- TensorView3D<const T>(qkv).shaped(suffix_length, total_heads, head_dim);- TensorView2D<const T> cosines_tensor_view =- TensorView2D<const T>(cosines).shaped(max_sequence_length, rope_dim);- TensorView2D<const T> sines_tensor_view =- TensorView2D<const T>(sines).shaped(max_sequence_length, rope_dim);- TensorView3D<T> rotated_queries_tensor_view =- TensorView3D<T>(rotated_queries)+ TensorView3D<const ElementT> qkv_tensor_view =+ TensorView3D<const ElementT>(qkv)+ .shaped(suffix_length, total_heads, head_dim);+ TensorView2D<const RopeT> cosines_tensor_view =+ TensorView2D<const RopeT>(cosines).shaped(max_sequence_length, rope_dim);+ TensorView2D<const RopeT> sines_tensor_view =+ TensorView2D<const RopeT>(sines).shaped(max_sequence_length, rope_dim);+ TensorView3D<ElementT> rotated_queries_tensor_view =+ TensorView3D<ElementT>(rotated_queries).shaped(num_heads, suffix_length, head_dim);- TensorView3D<T> rotated_keys_tensor_view =- TensorView3D<T>(rotated_keys).shaped(num_groups, suffix_length, head_dim);+ TensorView3D<ElementT> rotated_keys_tensor_view =+ TensorView3D<ElementT>(rotated_keys)+ .shaped(num_groups, suffix_length, head_dim);uint first_head_in_group = group_index * (num_heads / num_groups);/* ---------- QUERIES ---------- */if (dimension_index < rope_dim) {- const T cos_val = cosines_tensor_view(absolutePosition, dimension_index);- const T sin_val = sines_tensor_view(absolutePosition, dimension_index);+ const float cos_val =+ float(cosines_tensor_view(absolutePosition, dimension_index));+ const float sin_val =+ float(sines_tensor_view(absolutePosition, dimension_index));- T queryResult = applyRopeTransform(+ ElementT queryResult = applyRopeTransform(qkv_tensor_view,token_index,head_index,@@ -97,7 +103,7 @@ PUBLIC KERNEL(Rope)(if (head_index == first_head_in_group) {uint key_head_index =num_heads + group_index; // Keys start after all query heads- T keyResult = applyRopeTransform(+ ElementT keyResult = applyRopeTransform(qkv_tensor_view,token_index,key_head_index,@@ -120,4 +126,4 @@ PUBLIC KERNEL(Rope)(qkv_tensor_view(token_index, key_head_index, dimension_index);}}-}\ No newline at end of file+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metalindex d53fdd198..a716e7c92 100644--- a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal@@ -16,7 +16,7 @@ template <typename T>VARIANTS(T, float, half, bfloat)PUBLIC KERNEL(Softmax)(device T* values,- const device float* sinks OPTIONAL(has_sinks),+ const device T* sinks OPTIONAL(has_sinks),const constant uint& row_length,const constant uint& outer_dim,const constant uint& batch_dim,diff --git a/crates/backend-uzu/src/classifier/classifier.rs b/crates/backend-uzu/src/classifier/classifier.rsindex f9ed6ab53..179571fcb 100644--- a/crates/backend-uzu/src/classifier/classifier.rs+++ b/crates/backend-uzu/src/classifier/classifier.rs@@ -6,7 +6,7 @@ use super::{ClassificationOutput, ClassificationStats, ClassifierContext};use crate::{DataType, allocation_to_vec,backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},- config::{ClassifierModelConfig, ModelMetadata},+ config::model::classifier_model::ClassifierModelConfig,encodable_block::LayerArguments,forward_pass::token_inputs::TokenInputs,session::types::Error,@@ -67,9 +67,9 @@ impl<B: Backend> ClassifierTrait for Classifier<B> {impl<B: Backend> Classifier<B> {pub fn new(model_path: &Path,- model_metadata: &ModelMetadata<ClassifierModelConfig>,+ model_config: &ClassifierModelConfig,) -> Result<Self, Error> {- let context = ClassifierContext::new(model_path, model_metadata)?;+ let context = ClassifierContext::new(model_path, model_config)?;Ok(Self {context,})@@ -81,7 +81,7 @@ impl<B: Backend> Classifier<B> {token_ids: &[u64],token_positions: &[usize],) -> Result<(Box<[f32]>, ActivationTrace<B>), Error> {- let num_labels = self.context.model_config.model_config.num_labels;+ let num_labels = self.context.model_config.classifier_config.num_labels;let mut traces = ActivationTrace::new_classifier(self.context.context.as_ref(),&self.context.model_shape,@@ -143,7 +143,6 @@ impl<B: Backend> Classifier<B> {token_positions: token_inputs.token_positions(),token_parents: token_inputs.token_parents(),token_subtrie_ranges: None,- attention_sinks: self.context.shared_buffers.attention_sinks(layer_index),rope_buffers,sampling_start: 0,sampling_length: batch_dim,@@ -211,10 +210,7 @@ impl<B: Backend> Classifier<B> {&self,logits: &Allocation<B>,) -> Result<Box<[f32]>, Error> {- let logits_data_type: DataType =- self.context.model_config.model_config.prediction_head_config.readout_config.activation_precision().into();-- match logits_data_type {+ match self.context.logits_data_type {DataType::F32 => Ok(allocation_to_vec::<B, f32>(logits).into()),DataType::F16 => {Ok(allocation_to_vec::<B, half::f16>(logits).into_iter().map(|x| x.to_f32()).collect::<Box<[_]>>())@@ -222,7 +218,9 @@ impl<B: Backend> Classifier<B> {DataType::BF16 => {Ok(allocation_to_vec::<B, half::bf16>(logits).into_iter().map(|x| x.to_f32()).collect::<Box<[_]>>())},- _ => Err(Error::UnableToDecodeText),+ data_type => {+ Err(Error::InvalidModelConfig(format!("unsupported classifier logits data type {data_type:?}")))+ },}}@@ -230,7 +228,7 @@ impl<B: Backend> Classifier<B> {&self,logits: &[f32],) -> Result<HashMap<String, f32>, Error> {- let output_labels = &self.context.model_config.model_config.output_labels;+ let output_labels = &self.context.model_config.classifier_config.output_labels;let mut probabilities = HashMap::new();for (idx, &logit) in logits.iter().enumerate() {diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rsindex f40c01f28..decd22795 100644--- a/crates/backend-uzu/src/classifier/classifier_context.rs+++ b/crates/backend-uzu/src/classifier/classifier_context.rs@@ -1,16 +1,14 @@use std::{fs::File, path::Path, rc::Rc};-#[cfg(feature = "tracing")]-use crate::forward_pass::model_shape::ModelShape;use crate::{DataType,backends::common::{Backend, Context},classifier::ClassifierError,- config::{ClassifierModelConfig, ModelMetadata},+ config::{decoder::DecoderConfig, model::classifier_model::ClassifierModelConfig},encodable_block::{ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, QkUnpack, Rope,},- forward_pass::state::SharedBuffers,+ forward_pass::{model_shape::ModelShape, state::SharedBuffers},parameters::ParameterLoader,session::types::Error,};@@ -31,145 +29,138 @@ pub struct ClassifierContext<B: Backend> {pub pooling: Pooling<B>,pub prediction_head: ClassifierPredictionHead<B>,+ pub logits_data_type: DataType,}impl<B: Backend> ClassifierContext<B> {pub fn new(model_path: &Path,- model_metadata: &ModelMetadata<ClassifierModelConfig>,+ model_config: &ClassifierModelConfig,) -> Result<Self, Error> {let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;+ let classifier_config = &model_config.classifier_config;- let decoder_config = Rc::new(crate::config::DecoderConfig {- embedding_config: model_metadata.model_config.model_config.embedding_config.clone(),- transformer_config: model_metadata.model_config.model_config.transformer_config.clone(),- vocab_size: model_metadata.model_config.model_config.vocab_size,+ let decoder_config = Rc::new(DecoderConfig {+ embedding_config: classifier_config.embedding_config.clone(),+ transformer_config: classifier_config.transformer_config.clone(),+ vocab_size: classifier_config.vocab_size,pard_token: None,ple_model_config: None,});- #[cfg(feature = "tracing")]- let model_shape = ModelShape::from_decoder_config(&decoder_config);let weights_path = model_path.join("model.safetensors");- if !weights_path.exists() {- return Err(Error::UnableToLoadWeights);- }- let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;- let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;- let root_loader_view = loader.tree();+ let weights_file = File::open(&weights_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let loader = ParameterLoader::new(&weights_file, context.as_ref())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let root_loader_view =+ loader.tree().subtree("classifier").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;++ let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::BF16);- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config);+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);shared_buffers.update_data(&root_loader_view)?;let shared_buffers = Rc::new(shared_buffers);- let transformer_tree = root_loader_view- .subtree("transformer")- .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("transformer".to_string())))?;+ let transformer_tree =+ root_loader_view.subtree("transformer").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- let data_type = decoder_config- .first_attention()- .ok_or(Error::Classifier(ClassifierError::NonAttentionMixer))?- .qkv_projection_config- .activation_precision()- .into();+ let output_norm_tree =+ transformer_tree.subtree("output_norm").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- let embed = Embedding::new(+ let (embed, readout_input_hadamard_factors) = Embedding::new(context.as_ref(),decoder_config.vocab_size as u32,decoder_config.transformer_config.model_dim as u32,&decoder_config.embedding_config,- &root_loader_view.subtree("embedding").expect("Failed to get embedding subtree"),+ &root_loader_view.subtree("embedding").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?,+ &model_shape,)- .expect("Failed to create embedding");+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ if readout_input_hadamard_factors.is_some() {+ return Err(Error::Classifier(ClassifierError::Custom(+ "classifier does not support embedding readout hadamard".to_string(),+ )));+ }let rope = Rc::new(- Rope::<B>::new(context.as_ref(), data_type)+ Rope::<B>::new(context.as_ref(), &model_shape).map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e))))?,);let qk_unpack = Rc::new(- QkUnpack::<B>::new(context.as_ref(), data_type)+ QkUnpack::<B>::new(context.as_ref(), model_shape.data_type).map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("QkUnpack: {:?}", e))))?,);- let layers = model_metadata- .model_config- .model_config+ let layers = classifier_config.transformer_config.layer_configs.iter().enumerate().map(|(layer_index, layer_config)| {- layer_config.mixer_config.as_attention().ok_or(ClassifierError::NonAttentionMixer)?;-let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_index))- .map_err(|_| ClassifierError::WeightSubtreeNotFound(format!("layers.{}", layer_index)))?;+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- Ok(ClassifierLayer::new(+ ClassifierLayer::new(context.as_ref(),- &model_metadata.model_config.model_config.transformer_config,+ &classifier_config.transformer_config,layer_config,- layer_index,&layer_tree,rope.clone(),qk_unpack.clone(),- ))+ model_shape.data_type,+ )+ .map_err(|error| Error::UnableToCreateClassifierLayer(Box::new(error)))})- .collect::<Result<Vec<_>, ClassifierError>>()- .map_err(Error::Classifier)?+ .collect::<Result<Vec<_>, Error>>()?.into_boxed_slice();- let output_norm_tree = transformer_tree- .subtree("output_norm")- .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("output_norm".to_string())))?;let output_norm = Normalization::new(context.as_ref(),- data_type,- model_metadata.model_config.model_config.transformer_config.output_norm_config.clone(),+ model_shape.data_type,+ classifier_config.transformer_config.model_dim,+ classifier_config.transformer_config.output_norm_config.clone(),&output_norm_tree,).map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("output norm: {:?}", e))))?;- let embedding_norm_tree = root_loader_view- .subtree("embedding_norm")- .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("embedding_norm".to_string())))?;+ let embedding_norm_tree =+ root_loader_view.subtree("embedding_norm").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let embedding_norm = Normalization::new(context.as_ref(),- data_type,- model_metadata.model_config.model_config.embedding_norm_config.clone(),+ model_shape.data_type,+ classifier_config.transformer_config.model_dim,+ classifier_config.embedding_norm_config.clone(),&embedding_norm_tree,).map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("embedding norm: {:?}", e))))?;- let model_dim = model_metadata.model_config.model_config.model_dim;- let num_labels = model_metadata.model_config.model_config.num_labels;- let prediction_head_config = &model_metadata.model_config.model_config.prediction_head_config;- let prediction_head_tree = root_loader_view- .subtree("prediction_head")- .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head".to_string())))?;-- let prediction_head_data_type: DataType = prediction_head_config.dense_config.activation_precision().into();+ let model_dim = classifier_config.model_dim;+ let num_labels = classifier_config.num_labels;+ let prediction_head_config = &classifier_config.prediction_head_config;+ let prediction_head_tree =+ root_loader_view.subtree("prediction_head").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))- })?;+ let prediction_head_dense_tree =+ prediction_head_tree.subtree("dense").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let prediction_head_dense = <dyn Linear<B>>::new::<1>(- &prediction_head_config.dense_config,model_dim,[model_dim],+ prediction_head_config.use_dense_bias,context.as_ref(),+ model_shape.data_type,&prediction_head_dense_tree,).map_err(|e| {Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))})?;- let prediction_head_norm_tree = prediction_head_tree.subtree("norm").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.norm".to_string()))- })?;+ let prediction_head_norm_tree =+ prediction_head_tree.subtree("norm").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let prediction_head_norm = Normalization::new(context.as_ref(),- prediction_head_data_type,+ model_shape.data_type,+ model_dim,prediction_head_config.normalization_config.clone(),&prediction_head_norm_tree,)@@ -177,14 +168,14 @@ impl<B: Backend> ClassifierContext<B> {Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head norm: {:?}", e)))})?;- let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {- Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))- })?;+ let prediction_head_readout_tree =+ prediction_head_tree.subtree("readout").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let prediction_head_final_linear = <dyn Linear<B>>::new::<1>(- &prediction_head_config.readout_config,model_dim,[num_labels],+ true,context.as_ref(),+ model_shape.data_type,&prediction_head_readout_tree,).map_err(|e| {@@ -193,20 +184,17 @@ impl<B: Backend> ClassifierContext<B> {let pooling = Pooling::<B>::new(context.as_ref(),- data_type,- model_metadata.model_config.model_config.classifier_pooling.clone(),+ model_shape.data_type,+ classifier_config.classifier_pooling.clone(),model_dim,)- .map_err(|e| {- eprintln!("Failed to create pooling: {:?}", e);- Error::UnableToCreateContext(e.into())- })?;+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;let prediction_head = ClassifierPredictionHead::new(context.as_ref(),prediction_head_dense,- prediction_head_config.activation.clone(),- prediction_head_data_type,+ prediction_head_config.activation.clone().into(),+ model_shape.data_type,prediction_head_norm,prediction_head_final_linear,model_dim,@@ -215,10 +203,13 @@ impl<B: Backend> ClassifierContext<B> {Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))})?;+ loader.tree().assert_all_tensors_validated().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+Ok(Self {context,shared_buffers,- model_config: model_metadata.model_config.clone(),+ model_config: model_config.clone(),+ logits_data_type: model_shape.data_type,#[cfg(feature = "tracing")]model_shape,embed,diff --git a/crates/backend-uzu/src/classifier/error.rs b/crates/backend-uzu/src/classifier/error.rsindex c650bcb58..1afd24f83 100644--- a/crates/backend-uzu/src/classifier/error.rs+++ b/crates/backend-uzu/src/classifier/error.rs@@ -7,8 +7,6 @@ pub enum ClassifierError {MissingConfigField(String),#[error("Model requires attention layers but found non-attention mixer")]NonAttentionMixer,- #[error("Weight subtree not found: {0}")]- WeightSubtreeNotFound(String),#[error("Kernel creation failed: {0}")]KernelCreationFailed(String),#[error("{0}")]diff --git a/crates/backend-uzu/src/config/activation/gelu.rs b/crates/backend-uzu/src/config/activation/gelu.rsnew file mode 100644index 000000000..73222e6ae--- /dev/null+++ b/crates/backend-uzu/src/config/activation/gelu.rs@@ -0,0 +1,6 @@+use proc_macros::uzu_config;++#[uzu_config(super::Activation)]+pub struct GELU {+ pub approximate: bool,+}diff --git a/crates/backend-uzu/src/config/activation/identity.rs b/crates/backend-uzu/src/config/activation/identity.rsnew file mode 100644index 000000000..e5c2c84e6--- /dev/null+++ b/crates/backend-uzu/src/config/activation/identity.rs@@ -0,0 +1,4 @@+use proc_macros::uzu_config;++#[uzu_config(super::Activation)]+pub struct Identity;diff --git a/crates/backend-uzu/src/config/activation/mod.rs b/crates/backend-uzu/src/config/activation/mod.rsnew file mode 100644index 000000000..66dfea909--- /dev/null+++ b/crates/backend-uzu/src/config/activation/mod.rs@@ -0,0 +1,48 @@+use proc_macros::uzu_config_abstract;++use crate::backends::common::{ActivationConfig, gpu_types::ActivationType};++pub mod gelu;+pub mod identity;+pub mod silu;++#[uzu_config_abstract(silu::SiLU, gelu::GELU, identity::Identity)]+pub struct Activation;++impl AnyActivation {+ pub fn act_type(&self) -> ActivationType {+ match self {+ AnyActivation::SiLU(_) => ActivationType::SILU,+ AnyActivation::GELU(gelu::GELU {+ approximate: true,+ ..+ }) => ActivationType::GELUApprox,+ AnyActivation::GELU(gelu::GELU {+ approximate: false,+ ..+ }) => ActivationType::GELUExact,+ AnyActivation::Identity(_) => ActivationType::IDENTITY,+ }+ }++ pub fn alpha(&self) -> f32 {+ match self {+ AnyActivation::SiLU(config) => config.alpha,+ AnyActivation::GELU(_) | AnyActivation::Identity(_) => 1.0,+ }+ }+}++impl From<AnyActivation> for ActivationConfig {+ fn from(value: AnyActivation) -> Self {+ match value {+ AnyActivation::SiLU(config) => ActivationConfig::SILU {+ alpha: config.alpha,+ },+ AnyActivation::GELU(config) => ActivationConfig::GELU {+ approximate: config.approximate,+ },+ AnyActivation::Identity(_) => ActivationConfig::IDENTITY,+ }+ }+}diff --git a/crates/backend-uzu/src/config/activation/silu.rs b/crates/backend-uzu/src/config/activation/silu.rsnew file mode 100644index 000000000..4dbf2ba15--- /dev/null+++ b/crates/backend-uzu/src/config/activation/silu.rs@@ -0,0 +1,6 @@+use proc_macros::uzu_config;++#[uzu_config(super::Activation)]+pub struct SiLU {+ pub alpha: f32,+}diff --git a/crates/backend-uzu/src/config/classifier.rs b/crates/backend-uzu/src/config/classifier.rsnew file mode 100644index 000000000..7a4ec8be7--- /dev/null+++ b/crates/backend-uzu/src/config/classifier.rs@@ -0,0 +1,42 @@+use proc_macros::uzu_config;++use crate::config::{+ activation::AnyActivation, embedding::AnyEmbeddingConfig, linear::LinearConfig, normalization::NormalizationConfig,+ transformer::TransformerConfig,+};++#[uzu_config]+pub enum PoolingType {+ #[serde(rename = "cls")]+ CLS,+ #[serde(rename = "mean")]+ Mean,+}++#[uzu_config]+pub struct PredictionHeadConfig {+ pub dense_config: LinearConfig,+ pub activation: AnyActivation,+ pub normalization_config: NormalizationConfig,+ pub readout_config: LinearConfig,+ pub use_dense_bias: bool,+}++#[uzu_config]+pub struct ClassifierConfig {+ pub embedding_config: AnyEmbeddingConfig,+ pub embedding_norm_config: NormalizationConfig,+ pub transformer_config: TransformerConfig,+ pub prediction_head_config: PredictionHeadConfig,+ pub readout_config: LinearConfig,++ pub vocab_size: usize,+ pub model_dim: usize,+ pub hidden_dim: usize,+ pub attention_scale: Option<f32>,+ pub num_layers: usize,+ pub context_length: usize,+ pub num_labels: usize,+ pub classifier_pooling: PoolingType,+ pub output_labels: Option<Box<[String]>>,+}diff --git a/crates/backend-uzu/src/config/classifier/classifier_config.rs b/crates/backend-uzu/src/config/classifier/classifier_config.rsdeleted file mode 100644index 407a354f2..000000000--- a/crates/backend-uzu/src/config/classifier/classifier_config.rs+++ /dev/null@@ -1,22 +0,0 @@-use proc_macros::uzu_config;--use super::{PoolingType, PredictionHeadConfig};-use crate::config::{EmbeddingConfig, LinearConfig, NormalizationConfig, TransformerConfig};--#[uzu_config]-pub struct ClassifierConfig {- pub embedding_config: EmbeddingConfig,- pub embedding_norm_config: NormalizationConfig,- pub transformer_config: TransformerConfig,- pub prediction_head_config: PredictionHeadConfig,- pub readout_config: LinearConfig,- pub vocab_size: usize,- pub model_dim: usize,- pub hidden_dim: usize,- pub attention_scale: Option<f32>,- pub num_layers: usize,- pub context_length: usize,- pub num_labels: usize,- pub classifier_pooling: PoolingType,- pub output_labels: Option<Vec<String>>,-}diff --git a/crates/backend-uzu/src/config/classifier/mod.rs b/crates/backend-uzu/src/config/classifier/mod.rsdeleted file mode 100644index 7084c1314..000000000--- a/crates/backend-uzu/src/config/classifier/mod.rs+++ /dev/null@@ -1,7 +0,0 @@-mod classifier_config;-mod pooling_type;-mod prediction_head_config;--pub use classifier_config::ClassifierConfig;-pub use pooling_type::PoolingType;-pub use prediction_head_config::PredictionHeadConfig;diff --git a/crates/backend-uzu/src/config/classifier/pooling_type.rs b/crates/backend-uzu/src/config/classifier/pooling_type.rsdeleted file mode 100644index b67357da2..000000000--- a/crates/backend-uzu/src/config/classifier/pooling_type.rs+++ /dev/null@@ -1,8 +0,0 @@-use proc_macros::uzu_config;--#[uzu_config]-#[serde(rename_all = "lowercase")]-pub enum PoolingType {- Cls,- Mean,-}diff --git a/crates/backend-uzu/src/config/classifier/prediction_head_config.rs b/crates/backend-uzu/src/config/classifier/prediction_head_config.rsdeleted file mode 100644index 5fdc92a4c..000000000--- a/crates/backend-uzu/src/config/classifier/prediction_head_config.rs+++ /dev/null@@ -1,15 +0,0 @@-use proc_macros::uzu_config;--use crate::{- backends::common::ActivationConfig,- config::{LinearConfig, NormalizationConfig},-};--#[uzu_config]-pub struct PredictionHeadConfig {- pub dense_config: LinearConfig,- pub activation: ActivationConfig,- pub normalization_config: NormalizationConfig,- pub readout_config: LinearConfig,- pub use_dense_bias: bool,-}diff --git a/crates/backend-uzu/src/config/classifier_model.rs b/crates/backend-uzu/src/config/classifier_model.rsdeleted file mode 100644index 30a5348ae..000000000--- a/crates/backend-uzu/src/config/classifier_model.rs+++ /dev/null@@ -1,9 +0,0 @@-use proc_macros::uzu_config;--use crate::config::{ClassifierConfig, MessageProcessorConfig};--#[uzu_config]-pub struct ClassifierModelConfig {- pub model_config: ClassifierConfig,- pub message_processor_config: MessageProcessorConfig,-}diff --git a/crates/backend-uzu/src/config/decoder.rs b/crates/backend-uzu/src/config/decoder.rsindex 5992ac3f4..b622f8632 100644--- a/crates/backend-uzu/src/config/decoder.rs+++ b/crates/backend-uzu/src/config/decoder.rs@@ -1,19 +1,22 @@use proc_macros::uzu_config;-use super::{AttentionConfig, EmbeddingConfig, TransformerConfig};-use crate::utils::strict_serde::Unsupported;+use crate::{+ config::{embedding::AnyEmbeddingConfig, token_mixer::attention::AttentionConfig, transformer::TransformerConfig},+ utils::strict_serde::Unsupported,+};#[uzu_config]pub struct DecoderConfig {- pub embedding_config: EmbeddingConfig,+ pub embedding_config: AnyEmbeddingConfig,pub transformer_config: TransformerConfig,+pub vocab_size: usize,- pub pard_token: Option<usize>,+ pub pard_token: Option<u64>,pub ple_model_config: Option<Unsupported>,}impl DecoderConfig {pub fn first_attention(&self) -> Option<&AttentionConfig> {- self.transformer_config.layer_configs.iter().find_map(|l| l.mixer_config.as_attention())+ self.transformer_config.layer_configs.iter().find_map(|layer| layer.mixer_config.as_attention())}}diff --git a/crates/backend-uzu/src/config/embedding.rs b/crates/backend-uzu/src/config/embedding.rsdeleted file mode 100644index 69e89a3ff..000000000--- a/crates/backend-uzu/src/config/embedding.rs+++ /dev/null@@ -1,105 +0,0 @@-use proc_macros::uzu_config;--use crate::{DataType, backends::common::gpu_types::QuantizationMode};--#[uzu_config]-pub struct EmbeddingConfigCommon {- pub input_scale: Option<f32>,- pub logit_soft_cap: Option<f32>,-}--#[uzu_config]-#[serde(tag = "type")]-pub enum EmbeddingConfig {- #[serde(rename = "TiedEmbeddingConfig")]- Tied {- #[serde(flatten)]- common: EmbeddingConfigCommon,- precision: DataType,- },- #[serde(rename = "UntiedEmbeddingConfig")]- Untied {- #[serde(flatten)]- common: EmbeddingConfigCommon,- precision: DataType,- },- #[serde(rename = "MLXQuantizedTiedEmbeddingConfig")]- ScaleBiasQuantizedTied {- #[serde(flatten)]- common: EmbeddingConfigCommon,- group_size: usize,- embedding_quantization_mode: QuantizationMode,- activation_quantization_mode: Option<QuantizationMode>,- activation_precision: DataType,- },- #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]- ScaleBiasQuantizedUntied {- #[serde(flatten)]- common: EmbeddingConfigCommon,- group_size: usize,- embedding_quantization_mode: QuantizationMode,- activation_quantization_mode: Option<QuantizationMode>,- activation_precision: DataType,- },- #[serde(rename = "MLXSemiQuantizedUntiedEmbeddingConfig")]- ScaleBiasSemiQuantizedUntied {- #[serde(flatten)]- common: EmbeddingConfigCommon,- group_size: usize,- embedding_quantization_mode: QuantizationMode,- activation_quantization_mode: Option<QuantizationMode>,- activation_precision: DataType,- },-}--impl EmbeddingConfig {- pub fn common(&self) -> &EmbeddingConfigCommon {- match self {- EmbeddingConfig::Tied {- common,- ..- }- | EmbeddingConfig::Untied {- common,- ..- }- | EmbeddingConfig::ScaleBiasQuantizedTied {- common,- ..- }- | EmbeddingConfig::ScaleBiasQuantizedUntied {- common,- ..- }- | EmbeddingConfig::ScaleBiasSemiQuantizedUntied {- common,- ..- } => common,- }- }-- pub fn activation_precision(&self) -> DataType {- match self {- EmbeddingConfig::Tied {- precision,- ..- }- | EmbeddingConfig::Untied {- precision,- ..- } => *precision,- EmbeddingConfig::ScaleBiasQuantizedTied {- activation_precision,- ..- }- | EmbeddingConfig::ScaleBiasQuantizedUntied {- activation_precision,- ..- }- | EmbeddingConfig::ScaleBiasSemiQuantizedUntied {- activation_precision,- ..- } => *activation_precision,- }- }-}diff --git a/crates/backend-uzu/src/config/embedding/mod.rs b/crates/backend-uzu/src/config/embedding/mod.rsnew file mode 100644index 000000000..d4b968c52--- /dev/null+++ b/crates/backend-uzu/src/config/embedding/mod.rs@@ -0,0 +1,10 @@+use proc_macros::uzu_config_abstract;++pub mod tied_embedding;+pub mod untied_embedding;++#[uzu_config_abstract(tied_embedding::TiedEmbeddingConfig, untied_embedding::UntiedEmbeddingConfig)]+pub struct EmbeddingConfig {+ pub input_scale: Option<f32>,+ pub logit_soft_cap: Option<f32>,+}diff --git a/crates/backend-uzu/src/config/embedding/tied_embedding.rs b/crates/backend-uzu/src/config/embedding/tied_embedding.rsnew file mode 100644index 000000000..720896674--- /dev/null+++ b/crates/backend-uzu/src/config/embedding/tied_embedding.rs@@ -0,0 +1,4 @@+use proc_macros::uzu_config;++#[uzu_config(super::EmbeddingConfig)]+pub struct TiedEmbeddingConfig;diff --git a/crates/backend-uzu/src/config/embedding/untied_embedding.rs b/crates/backend-uzu/src/config/embedding/untied_embedding.rsnew file mode 100644index 000000000..0376e7cf2--- /dev/null+++ b/crates/backend-uzu/src/config/embedding/untied_embedding.rs@@ -0,0 +1,18 @@+use monostate::MustBe;+use proc_macros::uzu_config;++#[uzu_config(super::EmbeddingConfig)]+pub struct UntiedEmbeddingConfig;++impl UntiedEmbeddingConfig {+ pub fn new(+ input_scale: Option<f32>,+ logit_soft_cap: Option<f32>,+ ) -> Self {+ Self {+ ty: MustBe!("UntiedEmbeddingConfig"),+ input_scale,+ logit_soft_cap,+ }+ }+}diff --git a/crates/backend-uzu/src/config/language_model.rs b/crates/backend-uzu/src/config/language_model.rsdeleted file mode 100644index 90e1f1561..000000000--- a/crates/backend-uzu/src/config/language_model.rs+++ /dev/null@@ -1,10 +0,0 @@-use proc_macros::uzu_config;--use crate::config::{DecoderConfig, GenerationConfig, MessageProcessorConfig};--#[uzu_config]-pub struct LanguageModelConfig {- pub model_config: DecoderConfig,- pub message_processor_config: MessageProcessorConfig,- pub generation_config: GenerationConfig,-}diff --git a/crates/backend-uzu/src/config/linear.rs b/crates/backend-uzu/src/config/linear.rsindex 4c2e4948e..0364b538d 100644--- a/crates/backend-uzu/src/config/linear.rs+++ b/crates/backend-uzu/src/config/linear.rs@@ -1,56 +1,4 @@use proc_macros::uzu_config;-use crate::{DataType, backends::common::gpu_types::QuantizationMode};--#[uzu_config]-pub struct QuantizationConfig {- pub group_size: usize,- pub weight_quantization_mode: QuantizationMode,- pub activation_quantization_mode: Option<QuantizationMode>,- pub activation_precision: DataType,-}-#[uzu_config]-#[serde(tag = "type")]-pub enum LinearConfig {- #[serde(rename = "FullPrecisionLinearConfig")]- FullPrecision {- precision: DataType,- },- #[serde(rename = "GroupQuantizedLinearConfig")]- Quantized(QuantizationConfig),- #[serde(rename = "MLXQuantizedLinearConfig")]- ScaleBiasQuantized(QuantizationConfig),- #[serde(rename = "QLoRALinearConfig")]- QLoRA {- #[serde(flatten)]- quantization: QuantizationConfig,- lora_rank: usize,- lora_scale: f32,- },- #[serde(rename = "RHTLinearWrapperConfig")]- RHTLinearWrapper {- block_size: usize,- inner_config: Box<LinearConfig>,- },-}--impl LinearConfig {- pub fn activation_precision(&self) -> DataType {- match self {- LinearConfig::FullPrecision {- precision,- } => *precision,- LinearConfig::Quantized(quantization) => quantization.activation_precision,- LinearConfig::ScaleBiasQuantized(quantization) => quantization.activation_precision,- LinearConfig::QLoRA {- quantization,- ..- } => quantization.activation_precision,- LinearConfig::RHTLinearWrapper {- inner_config,- ..- } => inner_config.activation_precision(),- }- }-}+pub struct LinearConfig {}diff --git a/crates/backend-uzu/src/config/metadata.rs b/crates/backend-uzu/src/config/metadata.rsdeleted file mode 100644index e6b407185..000000000--- a/crates/backend-uzu/src/config/metadata.rs+++ /dev/null@@ -1,30 +0,0 @@-use proc_macros::uzu_config;--#[cfg(metal_backend)]-use super::TtsModelConfig;-use super::{ClassifierModelConfig, LanguageModelConfig, ModelType};-use crate::{backends::common::gpu_types::QuantizationMode, utils::strict_serde::DeserializeStrictOwned};--#[uzu_config]-#[serde(untagged)]-pub enum ModelConfig {- LanguageModel(LanguageModelConfig),- ClassifierModel(ClassifierModelConfig),- #[cfg(metal_backend)]- TtsModel(TtsModelConfig),-}--#[uzu_config]-pub struct ModelMetadata<T: DeserializeStrictOwned> {- pub toolchain_version: String,- pub vendor: String,- pub family: String,- pub name: String,- pub size: String,- pub quantization: Option<QuantizationMode>,- pub repo: String,- pub use_cases: Vec<String>,- pub model_type: ModelType,- pub model_config: T,- pub grammar_start_tokens: Vec<String>,-}diff --git a/crates/backend-uzu/src/config/mixer.rs b/crates/backend-uzu/src/config/mixer.rsdeleted file mode 100644index 56fc990f1..000000000--- a/crates/backend-uzu/src/config/mixer.rs+++ /dev/null@@ -1,56 +0,0 @@-use proc_macros::uzu_config;--use super::{AttentionConfig, DeltaNetAttentionConfig, Mamba2Config, ShortConvConfig};-use crate::DataType;--#[uzu_config]-#[serde(tag = "type")]-pub enum MixerConfig {- #[serde(rename = "AttentionConfig")]- Attention(AttentionConfig),- #[serde(rename = "Mamba2Config")]- Mamba(Mamba2Config),- #[serde(rename = "ShortConvConfig")]- ShortConv(ShortConvConfig),- #[serde(rename = "DeltaNetAttentionConfig")]- DeltaNet(DeltaNetAttentionConfig),-}--impl MixerConfig {- pub fn as_attention(&self) -> Option<&AttentionConfig> {- match self {- MixerConfig::Attention(config) => Some(config),- _ => None,- }- }-- pub fn as_short_conv(&self) -> Option<&ShortConvConfig> {- match self {- MixerConfig::ShortConv(config) => Some(config),- _ => None,- }- }-- pub fn as_delta_net(&self) -> Option<&DeltaNetAttentionConfig> {- match self {- MixerConfig::DeltaNet(config) => Some(config),- _ => None,- }- }-- pub fn sliding_window_size(&self) -> Option<usize> {- match self {- MixerConfig::Attention(config) => config.sliding_window_size,- MixerConfig::Mamba(_) | MixerConfig::ShortConv(_) | MixerConfig::DeltaNet(_) => None,- }- }-- pub fn activation_precision(&self) -> DataType {- match self {- MixerConfig::Attention(c) => c.qkv_projection_config.activation_precision(),- MixerConfig::Mamba(c) => c.in_projection_config.activation_precision(),- MixerConfig::ShortConv(c) => c.in_projection_config.activation_precision(),- MixerConfig::DeltaNet(c) => c.in_proj_config.activation_precision(),- }- }-}diff --git a/crates/backend-uzu/src/config/mlp.rs b/crates/backend-uzu/src/config/mlp.rsdeleted file mode 100644index c3b25a7d9..000000000--- a/crates/backend-uzu/src/config/mlp.rs+++ /dev/null@@ -1,42 +0,0 @@-use proc_macros::uzu_config;--use crate::{backends::common::ActivationConfig, config::LinearConfig};--#[uzu_config]-#[serde(tag = "type")]-pub enum MLPConfig {- #[serde(rename = "DenseMLPConfig")]- Dense(DenseMLPConfig),- #[serde(rename = "MixtureOfExpertsConfig")]- MixtureOfExperts(MixtureOfExpertsConfig),-}--#[uzu_config]-pub struct DenseMLPConfig {- pub linear_config: LinearConfig,- pub activation: ActivationConfig,- pub has_up_biases: bool,- pub has_down_biases: bool,- pub gate_clipping: Option<(Option<f32>, Option<f32>)>,- pub up_clipping: Option<(Option<f32>, Option<f32>)>,-}--#[uzu_config]-pub struct MixtureOfExpertsConfig {- pub expert_config: DenseMLPConfig,- pub router_config: LinearConfig,- pub routing_function: RoutingFunctionConfig,- pub num_routed_experts: usize,- pub num_active_routed_experts: usize,- pub router_has_biases: bool,- pub num_shared_experts: usize,- pub expert_hidden_dim: usize,- pub gate_config: Option<LinearConfig>,-}--#[uzu_config]-#[serde(tag = "type")]-pub enum RoutingFunctionConfig {- #[serde(rename = "SoftmaxRouting")]- SoftmaxRouting,-}diff --git a/crates/backend-uzu/src/config/mlp/dense_mlp.rs b/crates/backend-uzu/src/config/mlp/dense_mlp.rsnew file mode 100644index 000000000..8b85f1871--- /dev/null+++ b/crates/backend-uzu/src/config/mlp/dense_mlp.rs@@ -0,0 +1,13 @@+use proc_macros::uzu_config;++use crate::config::{activation::AnyActivation, linear::LinearConfig};++#[uzu_config(super::MLPConfig)]+pub struct DenseMLPConfig {+ pub linear_config: LinearConfig,+ pub activation: AnyActivation,+ pub has_up_biases: bool,+ pub has_down_biases: bool,+ pub gate_clipping: Option<(Option<f32>, Option<f32>)>,+ pub up_clipping: Option<(Option<f32>, Option<f32>)>,+}diff --git a/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs b/crates/backend-uzu/src/config/mlp/mixture_of_experts.rsnew file mode 100644index 000000000..8678c539b--- /dev/null+++ b/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs@@ -0,0 +1,21 @@+use proc_macros::uzu_config;++use crate::config::{+ linear::LinearConfig,+ mlp::{dense_mlp::DenseMLPConfig, routing_function::AnyRoutingFunction},+};++#[uzu_config(super::MLPConfig)]+pub struct MixtureOfExpertsConfig {+ pub expert_config: DenseMLPConfig,+ pub router_config: LinearConfig,+ pub routing_function: AnyRoutingFunction,++ pub num_routed_experts: usize,+ pub num_active_routed_experts: usize,+ pub router_has_biases: bool,++ pub num_shared_experts: usize,+ pub expert_hidden_dim: usize,+ pub gate_config: Option<LinearConfig>,+}diff --git a/crates/backend-uzu/src/config/mlp/mod.rs b/crates/backend-uzu/src/config/mlp/mod.rsnew file mode 100644index 000000000..551cf4330--- /dev/null+++ b/crates/backend-uzu/src/config/mlp/mod.rs@@ -0,0 +1,8 @@+use proc_macros::uzu_config_abstract;++pub mod dense_mlp;+pub mod mixture_of_experts;+pub mod routing_function;++#[uzu_config_abstract(dense_mlp::DenseMLPConfig, mixture_of_experts::MixtureOfExpertsConfig)]+pub struct MLPConfig;diff --git a/crates/backend-uzu/src/config/mlp/routing_function/mod.rs b/crates/backend-uzu/src/config/mlp/routing_function/mod.rsnew file mode 100644index 000000000..2464c814c--- /dev/null+++ b/crates/backend-uzu/src/config/mlp/routing_function/mod.rs@@ -0,0 +1,6 @@+use proc_macros::uzu_config_abstract;++pub mod softmax_routing;++#[uzu_config_abstract(softmax_routing::SoftmaxRouting)]+pub struct RoutingFunction;diff --git a/crates/backend-uzu/src/config/mlp/routing_function/softmax_routing.rs b/crates/backend-uzu/src/config/mlp/routing_function/softmax_routing.rsnew file mode 100644index 000000000..08db7bda9--- /dev/null+++ b/crates/backend-uzu/src/config/mlp/routing_function/softmax_routing.rs@@ -0,0 +1,4 @@+use proc_macros::uzu_config;++#[uzu_config(super::RoutingFunction)]+pub struct SoftmaxRouting;diff --git a/crates/backend-uzu/src/config/mod.rs b/crates/backend-uzu/src/config/mod.rsindex 65b81eb82..a2deaa02d 100644--- a/crates/backend-uzu/src/config/mod.rs+++ b/crates/backend-uzu/src/config/mod.rs@@ -1,45 +1,15 @@-mod attention;-mod classifier;-mod classifier_model;-mod decoder;-mod delta_net;-mod embedding;-mod generation;-mod language_model;-mod linear;-mod mamba;-mod message_processor;-mod metadata;-mod mixer;-mod mlp;-mod model_type;-mod normalization;-mod rope;-mod short_conv;-mod transformer;-mod transformer_layer;-#[cfg(metal_backend)]-mod tts_model;--pub use attention::*;-pub use classifier::*;-pub use classifier_model::*;-pub use decoder::*;-pub use delta_net::*;-pub use embedding::*;-pub use generation::*;-pub use language_model::*;-pub use linear::*;-pub use mamba::*;-pub use message_processor::*;-pub use metadata::*;-pub use mixer::*;-pub use mlp::*;-pub use model_type::*;-pub use normalization::*;-pub use rope::*;-pub use short_conv::*;-pub use transformer::*;-pub use transformer_layer::*;-#[cfg(metal_backend)]-pub use tts_model::*;+pub mod activation;+pub mod classifier;+pub mod decoder;+pub mod embedding;+pub mod linear;+pub mod mlp;+pub mod model;+pub mod normalization;+pub mod rope;+pub mod token_codec;+pub mod token_mixer;+pub mod transformer;+pub mod transformer_layer;+pub mod tts;+pub mod weight_matrix;diff --git a/crates/backend-uzu/src/config/model/classifier_model.rs b/crates/backend-uzu/src/config/model/classifier_model.rsnew file mode 100644index 000000000..5769a3f15--- /dev/null+++ b/crates/backend-uzu/src/config/model/classifier_model.rs@@ -0,0 +1,8 @@+use proc_macros::uzu_config;++use crate::config::classifier::ClassifierConfig;++#[uzu_config(super::ModelConfig)]+pub struct ClassifierModelConfig {+ pub classifier_config: ClassifierConfig,+}diff --git a/crates/backend-uzu/src/config/generation.rs b/crates/backend-uzu/src/config/model/generation.rssimilarity index 73%rename from crates/backend-uzu/src/config/generation.rsrename to crates/backend-uzu/src/config/model/generation.rsindex 300d07bb5..feddaab5e 100644--- a/crates/backend-uzu/src/config/generation.rs+++ b/crates/backend-uzu/src/config/model/generation.rs@@ -4,13 +4,14 @@ use crate::utils::strict_serde::Unsupported;#[uzu_config]pub struct GenerationConfig {- pub stop_token_ids: Vec<u32>,+ pub stop_token_ids: Box<[u64]>,pub temperature: Option<f32>,pub top_k: Option<u32>,pub top_p: Option<f32>,pub min_p: Option<f32>,- pub banned_tokens: Option<Vec<u32>>,+ pub banned_tokens: Option<Unsupported>,pub repetition_penalty: Option<Unsupported>,pub presence_penalty: Option<Unsupported>,pub frequency_penalty: Option<Unsupported>,+ pub suffix_repetition_length: Option<Unsupported>,}diff --git a/crates/backend-uzu/src/config/model/language_model.rs b/crates/backend-uzu/src/config/model/language_model.rsnew file mode 100644index 000000000..23b607ac7--- /dev/null+++ b/crates/backend-uzu/src/config/model/language_model.rs@@ -0,0 +1,9 @@+use proc_macros::uzu_config;++use crate::config::{decoder::DecoderConfig, model::generation::GenerationConfig};++#[uzu_config(super::ModelConfig)]+pub struct LanguageModelConfig {+ pub decoder_config: DecoderConfig,+ pub generation_config: GenerationConfig,+}diff --git a/crates/backend-uzu/src/config/model/mod.rs b/crates/backend-uzu/src/config/model/mod.rsnew file mode 100644index 000000000..0156e345e--- /dev/null+++ b/crates/backend-uzu/src/config/model/mod.rs@@ -0,0 +1,17 @@+use proc_macros::uzu_config_abstract;++use crate::config::token_codec::AnyTokenCodecConfig;++pub mod classifier_model;+pub mod generation;+pub mod language_model;+pub mod tts_model;++#[uzu_config_abstract(+ language_model::LanguageModelConfig,+ classifier_model::ClassifierModelConfig,+ tts_model::TTSModelConfig+)]+pub struct ModelConfig {+ pub token_codec_config: AnyTokenCodecConfig,+}diff --git a/crates/backend-uzu/src/config/model/tts_model.rs b/crates/backend-uzu/src/config/model/tts_model.rsnew file mode 100644index 000000000..853ce9f22--- /dev/null+++ b/crates/backend-uzu/src/config/model/tts_model.rs@@ -0,0 +1,8 @@+use proc_macros::uzu_config;++use crate::config::tts::TTSConfig;++#[uzu_config(super::ModelConfig)]+pub struct TTSModelConfig {+ pub tts_config: TTSConfig,+}diff --git a/crates/backend-uzu/src/config/model_type.rs b/crates/backend-uzu/src/config/model_type.rsdeleted file mode 100644index 89a96a278..000000000--- a/crates/backend-uzu/src/config/model_type.rs+++ /dev/null@@ -1,10 +0,0 @@-use proc_macros::uzu_config;--#[derive(Eq)]-#[uzu_config]-#[serde(rename_all = "snake_case")]-pub enum ModelType {- LanguageModel,- ClassifierModel,- TtsModel,-}diff --git a/crates/backend-uzu/src/config/normalization/normalization_config.rs b/crates/backend-uzu/src/config/normalization.rssimilarity index 57%rename from crates/backend-uzu/src/config/normalization/normalization_config.rsrename to crates/backend-uzu/src/config/normalization.rsindex 212e5a1a9..7a6af42bc 100644--- a/crates/backend-uzu/src/config/normalization/normalization_config.rs+++ b/crates/backend-uzu/src/config/normalization.rs@@ -1,15 +1,17 @@use proc_macros::uzu_config;-use super::UpcastMode;-use crate::DataType;+#[uzu_config]+#[serde(rename_all = "snake_case")]+pub enum UpcastMode {+ OnlyNormalization,+ FullLayer,+}#[uzu_config]pub struct NormalizationConfig {- pub scale_precision: DataType,- pub accumulation_precision: DataType,pub epsilon: f32,pub scale_offset: Option<f32>,pub upcast_mode: UpcastMode,pub subtract_mean: bool,- pub use_bias: bool,+ pub has_biases: bool,}diff --git a/crates/backend-uzu/src/config/normalization/mod.rs b/crates/backend-uzu/src/config/normalization/mod.rsdeleted file mode 100644index 7dcb34734..000000000--- a/crates/backend-uzu/src/config/normalization/mod.rs+++ /dev/null@@ -1,5 +0,0 @@-mod normalization_config;-mod upcast_mode;--pub use normalization_config::*;-pub use upcast_mode::*;diff --git a/crates/backend-uzu/src/config/normalization/upcast_mode.rs b/crates/backend-uzu/src/config/normalization/upcast_mode.rsdeleted file mode 100644index 0d923dd2b..000000000--- a/crates/backend-uzu/src/config/normalization/upcast_mode.rs+++ /dev/null@@ -1,8 +0,0 @@-use proc_macros::uzu_config;--#[uzu_config]-#[serde(rename_all = "snake_case")]-pub enum UpcastMode {- OnlyNormalization,- FullLayer,-}diff --git a/crates/backend-uzu/src/config/rope.rs b/crates/backend-uzu/src/config/rope.rsdeleted file mode 100644index 31a1a4375..000000000--- a/crates/backend-uzu/src/config/rope.rs+++ /dev/null@@ -1,64 +0,0 @@-use proc_macros::uzu_config;--use crate::DataType;--#[uzu_config]-pub struct RopeConfigCommon {- pub precision: DataType,- pub base: f32,- pub max_sequence_length: usize,- pub head_dim: usize,- pub partial_rotary_dim: Option<usize>,-}--#[uzu_config]-#[serde(tag = "type")]-pub enum RoPEConfig {- #[serde(rename = "UnscaledRoPEConfig")]- Unscaled(RopeConfigCommon),- #[serde(rename = "LlamaRoPEConfig")]- Llama {- #[serde(flatten)]- common: RopeConfigCommon,- scaling_factor: f32,- original_context_length: usize,- low_frequency_factor: f32,- high_frequency_factor: f32,- },- #[serde(rename = "YARNRoPEConfig")]- YARN {- #[serde(flatten)]- common: RopeConfigCommon,- scaling_factor: f32,- original_context_length: usize,- beta_fast: f32,- beta_slow: f32,- truncate: bool,- },- #[serde(rename = "LinearScalingRoPEConfig")]- LinearScaling {- #[serde(flatten)]- common: RopeConfigCommon,- scaling_factor: f32,- },-}--impl RoPEConfig {- pub fn common(&self) -> &RopeConfigCommon {- match self {- RoPEConfig::Unscaled(config) => config,- RoPEConfig::Llama {- common,- ..- } => common,- RoPEConfig::YARN {- common,- ..- } => common,- RoPEConfig::LinearScaling {- common,- ..- } => common,- }- }-}diff --git a/crates/backend-uzu/src/config/rope/linear_scaling_rope.rs b/crates/backend-uzu/src/config/rope/linear_scaling_rope.rsnew file mode 100644index 000000000..cc28d9a4a--- /dev/null+++ b/crates/backend-uzu/src/config/rope/linear_scaling_rope.rs@@ -0,0 +1,6 @@+use proc_macros::uzu_config;++#[uzu_config(super::RoPEConfig)]+pub struct LinearScalingRoPEConfig {+ pub scaling_factor: f32,+}diff --git a/crates/backend-uzu/src/config/rope/llama_rope.rs b/crates/backend-uzu/src/config/rope/llama_rope.rsnew file mode 100644index 000000000..8e6ac47ba--- /dev/null+++ b/crates/backend-uzu/src/config/rope/llama_rope.rs@@ -0,0 +1,9 @@+use proc_macros::uzu_config;++#[uzu_config(super::RoPEConfig)]+pub struct LlamaRoPEConfig {+ pub scaling_factor: f32,+ pub original_context_length: usize,+ pub low_frequency_factor: f32,+ pub high_frequency_factor: f32,+}diff --git a/crates/backend-uzu/src/config/rope/mod.rs b/crates/backend-uzu/src/config/rope/mod.rsnew file mode 100644index 000000000..49b9da09f--- /dev/null+++ b/crates/backend-uzu/src/config/rope/mod.rs@@ -0,0 +1,19 @@+use proc_macros::uzu_config_abstract;++pub mod linear_scaling_rope;+pub mod llama_rope;+pub mod unscaled_rope;+pub mod yarn_rope;++#[uzu_config_abstract(+ unscaled_rope::UnscaledRoPEConfig,+ llama_rope::LlamaRoPEConfig,+ yarn_rope::YARNRoPEConfig,+ linear_scaling_rope::LinearScalingRoPEConfig+)]+pub struct RoPEConfig {+ pub base: f32,+ pub max_sequence_length: usize,+ pub head_dim: Option<usize>,+ pub partial_rotary_dim: Option<usize>,+}diff --git a/crates/backend-uzu/src/config/rope/unscaled_rope.rs b/crates/backend-uzu/src/config/rope/unscaled_rope.rsnew file mode 100644index 000000000..6ff16c708--- /dev/null+++ b/crates/backend-uzu/src/config/rope/unscaled_rope.rs@@ -0,0 +1,4 @@+use proc_macros::uzu_config;++#[uzu_config(super::RoPEConfig)]+pub struct UnscaledRoPEConfig;diff --git a/crates/backend-uzu/src/config/rope/yarn_rope.rs b/crates/backend-uzu/src/config/rope/yarn_rope.rsnew file mode 100644index 000000000..491b10707--- /dev/null+++ b/crates/backend-uzu/src/config/rope/yarn_rope.rs@@ -0,0 +1,10 @@+use proc_macros::uzu_config;++#[uzu_config(super::RoPEConfig)]+pub struct YARNRoPEConfig {+ pub scaling_factor: f32,+ pub original_context_length: usize,+ pub beta_fast: f32,+ pub beta_slow: f32,+ pub truncate: bool,+}diff --git a/crates/backend-uzu/src/config/message_processor.rs b/crates/backend-uzu/src/config/token_codec/chat_codec.rssimilarity index 82%rename from crates/backend-uzu/src/config/message_processor.rsrename to crates/backend-uzu/src/config/token_codec/chat_codec.rsindex 542ff8727..3881185fe 100644--- a/crates/backend-uzu/src/config/message_processor.rs+++ b/crates/backend-uzu/src/config/token_codec/chat_codec.rs@@ -1,7 +1,7 @@use proc_macros::uzu_config;-#[uzu_config]-pub struct MessageProcessorConfig {+#[uzu_config(super::TokenCodecConfig)]+pub struct ChatCodecConfig {pub prompt_template: String,pub output_parser_regex: Option<String>,pub system_role_name: String,diff --git a/crates/backend-uzu/src/config/token_codec/mod.rs b/crates/backend-uzu/src/config/token_codec/mod.rsnew file mode 100644index 000000000..8999d3886--- /dev/null+++ b/crates/backend-uzu/src/config/token_codec/mod.rs@@ -0,0 +1,7 @@+use proc_macros::uzu_config_abstract;++pub mod chat_codec;+pub mod tts_codec;++#[uzu_config_abstract(chat_codec::ChatCodecConfig, tts_codec::TTSCodecConfig)]+pub struct TokenCodecConfig;diff --git a/crates/backend-uzu/src/config/token_codec/tts_codec.rs b/crates/backend-uzu/src/config/token_codec/tts_codec.rsnew file mode 100644index 000000000..17a6555f5--- /dev/null+++ b/crates/backend-uzu/src/config/token_codec/tts_codec.rs@@ -0,0 +1,7 @@+use proc_macros::uzu_config;++#[uzu_config(super::TokenCodecConfig)]+pub struct TTSCodecConfig {+ pub prompt_template: String,+ pub drop_initial_newline: bool,+}diff --git a/crates/backend-uzu/src/config/attention.rs b/crates/backend-uzu/src/config/token_mixer/attention.rssimilarity index 66%rename from crates/backend-uzu/src/config/attention.rsrename to crates/backend-uzu/src/config/token_mixer/attention.rsindex 1326dc828..86c999116 100644--- a/crates/backend-uzu/src/config/attention.rs+++ b/crates/backend-uzu/src/config/token_mixer/attention.rs@@ -1,18 +1,17 @@use proc_macros::uzu_config;-use super::{LinearConfig, NormalizationConfig, UpcastMode};-use crate::DataType;+use crate::config::{+ linear::LinearConfig,+ normalization::{NormalizationConfig, UpcastMode},+};-pub const VALUE_NORM_EPSILON: f32 = 1e-6;--#[uzu_config]+#[uzu_config(super::TokenMixerConfig)]pub struct AttentionConfig {pub qkv_projection_config: LinearConfig,pub out_projection_config: LinearConfig,pub query_norm_config: Option<NormalizationConfig>,pub key_norm_config: Option<NormalizationConfig>,- pub normalize_values: bool,pub num_heads: usize,pub num_groups: usize,@@ -26,21 +25,17 @@ pub struct AttentionConfig {pub has_qkv_biases: bool,pub has_out_biases: bool,pub gate_projection_config: Option<LinearConfig>,+ pub normalize_values: bool,}impl AttentionConfig {pub fn value_norm_config(&self) -> Option<NormalizationConfig> {- if !self.normalize_values {- return None;- }- Some(NormalizationConfig {- scale_precision: self.qkv_projection_config.activation_precision(),- accumulation_precision: DataType::F32,- epsilon: VALUE_NORM_EPSILON,+ self.normalize_values.then_some(NormalizationConfig {+ epsilon: 1e-6,scale_offset: None,upcast_mode: UpcastMode::FullLayer,subtract_mean: false,- use_bias: false,+ has_biases: false,})}}diff --git a/crates/backend-uzu/src/config/token_mixer/convolutions.rs b/crates/backend-uzu/src/config/token_mixer/convolutions.rsnew file mode 100644index 000000000..04c0ce49f--- /dev/null+++ b/crates/backend-uzu/src/config/token_mixer/convolutions.rs@@ -0,0 +1,6 @@+use proc_macros::uzu_config;++#[uzu_config]+pub struct SeparableCausalConvConfig {+ pub has_biases: bool,+}diff --git a/crates/backend-uzu/src/config/delta_net.rs b/crates/backend-uzu/src/config/token_mixer/delta_net.rssimilarity index 76%rename from crates/backend-uzu/src/config/delta_net.rsrename to crates/backend-uzu/src/config/token_mixer/delta_net.rsindex dc8253813..b46f9d63c 100644--- a/crates/backend-uzu/src/config/delta_net.rs+++ b/crates/backend-uzu/src/config/token_mixer/delta_net.rs@@ -1,13 +1,16 @@use proc_macros::uzu_config;-use super::{linear::LinearConfig, normalization::NormalizationConfig, short_conv::SeparableCausalConvConfig};+use crate::config::{+ linear::LinearConfig, normalization::NormalizationConfig, token_mixer::convolutions::SeparableCausalConvConfig,+};-#[uzu_config]-pub struct DeltaNetAttentionConfig {+#[uzu_config(super::TokenMixerConfig)]+pub struct DeltaNetConfig {pub in_proj_config: LinearConfig,pub conv_config: SeparableCausalConvConfig,pub out_proj_config: LinearConfig,pub norm_config: NormalizationConfig,+pub num_heads: usize,pub num_groups: usize,pub head_dim: usize,@@ -15,7 +18,7 @@ pub struct DeltaNetAttentionConfig {pub kernel_size: usize,}-impl DeltaNetAttentionConfig {+impl DeltaNetConfig {pub fn key_dim(&self) -> usize {self.num_groups * self.head_dim}diff --git a/crates/backend-uzu/src/config/mamba.rs b/crates/backend-uzu/src/config/token_mixer/mamba2.rssimilarity index 75%rename from crates/backend-uzu/src/config/mamba.rsrename to crates/backend-uzu/src/config/token_mixer/mamba2.rsindex 41b406d1f..cdedbabb9 100644--- a/crates/backend-uzu/src/config/mamba.rs+++ b/crates/backend-uzu/src/config/token_mixer/mamba2.rs@@ -1,23 +1,25 @@use proc_macros::uzu_config;-use super::{linear::LinearConfig, short_conv::SeparableCausalConvConfig};-use crate::backends::common::ActivationConfig;+use crate::config::{+ activation::AnyActivation, linear::LinearConfig, token_mixer::convolutions::SeparableCausalConvConfig,+};-#[uzu_config]+#[uzu_config(super::TokenMixerConfig)]pub struct Mamba2Config {pub in_projection_config: LinearConfig,pub out_projection_config: LinearConfig,pub conv_config: SeparableCausalConvConfig,- pub activation: ActivationConfig,+ pub activation: AnyActivation,+pub kernel_size: usize,pub num_heads: usize,pub num_groups: usize,pub head_dim: usize,pub state_dim: usize,pub expansion_factor: usize,+pub has_in_biases: bool,pub has_out_biases: bool,- pub chunk_size: usize,}impl Mamba2Config {diff --git a/crates/backend-uzu/src/config/token_mixer/mod.rs b/crates/backend-uzu/src/config/token_mixer/mod.rsnew file mode 100644index 000000000..fdc0cce81--- /dev/null+++ b/crates/backend-uzu/src/config/token_mixer/mod.rs@@ -0,0 +1,28 @@+use proc_macros::uzu_config_abstract;++pub mod attention;+pub mod convolutions;+pub mod delta_net;+pub mod mamba2;+pub mod short_conv;++#[uzu_config_abstract(+ attention::AttentionConfig,+ delta_net::DeltaNetConfig,+ mamba2::Mamba2Config,+ short_conv::ShortConvConfig+)]+pub struct TokenMixerConfig;++impl AnyTokenMixerConfig {+ pub fn as_attention(&self) -> Option<&attention::AttentionConfig> {+ match self {+ AnyTokenMixerConfig::AttentionConfig(config) => Some(config),+ _ => None,+ }+ }++ pub fn sliding_window_size(&self) -> Option<usize> {+ self.as_attention().and_then(|config| config.sliding_window_size)+ }+}diff --git a/crates/backend-uzu/src/config/short_conv.rs b/crates/backend-uzu/src/config/token_mixer/short_conv.rssimilarity index 56%rename from crates/backend-uzu/src/config/short_conv.rsrename to crates/backend-uzu/src/config/token_mixer/short_conv.rsindex d12b53555..aa0c11f9c 100644--- a/crates/backend-uzu/src/config/short_conv.rs+++ b/crates/backend-uzu/src/config/token_mixer/short_conv.rs@@ -1,18 +1,12 @@use proc_macros::uzu_config;-use super::linear::LinearConfig;-use crate::DataType;+use crate::config::{linear::LinearConfig, token_mixer::convolutions::SeparableCausalConvConfig};-#[uzu_config]-pub struct SeparableCausalConvConfig {- pub precision: DataType,- pub has_biases: bool,-}--#[uzu_config]+#[uzu_config(super::TokenMixerConfig)]pub struct ShortConvConfig {pub in_projection_config: LinearConfig,pub conv_config: SeparableCausalConvConfig,pub out_projection_config: LinearConfig,+pub kernel_size: usize,}diff --git a/crates/backend-uzu/src/config/transformer.rs b/crates/backend-uzu/src/config/transformer.rsindex e293ebe5d..b968c2f93 100644--- a/crates/backend-uzu/src/config/transformer.rs+++ b/crates/backend-uzu/src/config/transformer.rs@@ -1,10 +1,10 @@use proc_macros::uzu_config;-use super::{NormalizationConfig, TransformerLayerConfig};+use crate::config::{normalization::NormalizationConfig, transformer_layer::TransformerLayerConfig};#[uzu_config]pub struct TransformerConfig {- pub layer_configs: Vec<TransformerLayerConfig>,+ pub layer_configs: Box<[TransformerLayerConfig]>,pub output_norm_config: NormalizationConfig,pub model_dim: usize,pub hidden_dim: usize,diff --git a/crates/backend-uzu/src/config/transformer_layer.rs b/crates/backend-uzu/src/config/transformer_layer.rsindex 12c88ffcb..cc929063e 100644--- a/crates/backend-uzu/src/config/transformer_layer.rs+++ b/crates/backend-uzu/src/config/transformer_layer.rs@@ -1,19 +1,23 @@use proc_macros::uzu_config;-use super::{MLPConfig, MixerConfig, NormalizationConfig, RoPEConfig};-use crate::utils::strict_serde::Unsupported;+use crate::{+ config::{+ mlp::AnyMLPConfig, normalization::NormalizationConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig,+ },+ utils::strict_serde::Unsupported,+};#[uzu_config]pub struct TransformerLayerConfig {pub pre_mixer_norm_config: Option<NormalizationConfig>,- pub mixer_config: MixerConfig,+ pub mixer_config: AnyTokenMixerConfig,pub post_mixer_norm_config: Option<NormalizationConfig>,pub pre_mlp_norm_config: NormalizationConfig,- pub mlp_config: MLPConfig,+ pub mlp_config: AnyMLPConfig,pub post_mlp_norm_config: Option<NormalizationConfig>,- pub rope_config: Option<RoPEConfig>,pub hidden_dim: Option<usize>,pub ple_config: Option<Unsupported>,pub has_post_layer_scalar: bool,- pub kv_source_layer: Option<usize>,+ pub kv_source_layer_index: Option<usize>,+ pub rope_config: Option<AnyRoPEConfig>,}diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/common.rs b/crates/backend-uzu/src/config/tts/audio_decoder/common.rsnew file mode 100644index 000000000..c9e3719fd--- /dev/null+++ b/crates/backend-uzu/src/config/tts/audio_decoder/common.rs@@ -0,0 +1,15 @@+use monostate::MustBe;+use proc_macros::uzu_config;++#[uzu_config]+pub struct CausalConv1dConfig {+ pub has_biases: MustBe!(true),+}++#[uzu_config]+pub struct CausalTransposeConv1dConfig {+ pub has_biases: MustBe!(true),+}++#[uzu_config]+pub struct Snake1dConfig {}diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/descript_audio_codec.rs b/crates/backend-uzu/src/config/tts/audio_decoder/descript_audio_codec.rsnew file mode 100644index 000000000..c1448ee49--- /dev/null+++ b/crates/backend-uzu/src/config/tts/audio_decoder/descript_audio_codec.rs@@ -0,0 +1,21 @@+use proc_macros::uzu_config;++use crate::config::tts::audio_decoder::fish_audio_modules::{DACDecoderConfig, DownsampleResidualVectorQuantizeConfig};++#[uzu_config(super::TTSAudioDecoderConfig)]+pub struct DescriptAudioCodecConfig {+ pub quantizer_config: DownsampleResidualVectorQuantizeConfig,+ pub decoder_config: DACDecoderConfig,+ pub samplerate: u32,++ pub encoder_dim: usize,+ pub encoder_rates: Box<[usize]>,+ pub decoder_dim: usize,+ pub decoder_rates: Box<[usize]>,+ pub input_dim: usize,+ pub n_codebooks: usize,+ pub codebook_dim: usize,+ pub downsample_factor: Box<[usize]>,+ pub codebook_size: usize,+ pub semantic_codebook_size: usize,+}diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs b/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rsnew file mode 100644index 000000000..0907943b0--- /dev/null+++ b/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs@@ -0,0 +1,71 @@+use proc_macros::uzu_config;++use crate::config::{+ activation::AnyActivation,+ embedding::tied_embedding::TiedEmbeddingConfig,+ linear::LinearConfig,+ normalization::NormalizationConfig,+ transformer::TransformerConfig,+ tts::audio_decoder::common::{CausalConv1dConfig, CausalTransposeConv1dConfig, Snake1dConfig},+};++#[uzu_config]+pub struct ConvNeXtBlockConfig {+ pub activation: AnyActivation,+ pub dwconv_config: CausalConv1dConfig,+ pub norm_config: NormalizationConfig,+ pub pwconv_config: LinearConfig,+}++#[uzu_config]+pub struct UpsamplingBlockConfig {+ pub trans_conv_config: CausalTransposeConv1dConfig,+ pub convnext_config: ConvNeXtBlockConfig,+}++#[uzu_config]+pub struct UpsamplerConfig {+ pub block_configs: Box<[UpsamplingBlockConfig]>,+}++#[uzu_config]+pub struct VectorQuantizeConfig {+ pub codebook_config: TiedEmbeddingConfig,+ pub out_proj_config: LinearConfig,+}++#[uzu_config]+pub struct ResidualVectorQuantizeConfig {+ pub vq_config: VectorQuantizeConfig,+}++#[uzu_config]+pub struct DownsampleResidualVectorQuantizeConfig {+ pub semantic_quantizer_config: ResidualVectorQuantizeConfig,+ pub quantizer_config: ResidualVectorQuantizeConfig,+ pub post_module_config: TransformerConfig,+ pub upsampler_config: UpsamplerConfig,+}++#[uzu_config]+pub struct ResidualUnitConfig {+ pub snake_config: Snake1dConfig,+ pub conv_config: CausalConv1dConfig,+ pub causal: bool,+}++#[uzu_config]+pub struct DACDecoderBlockConfig {+ pub snake_config: Snake1dConfig,+ pub trans_conv_config: CausalTransposeConv1dConfig,+ pub res_unit_config: ResidualUnitConfig,+ pub causal: bool,+}++#[uzu_config]+pub struct DACDecoderConfig {+ pub conv_config: CausalConv1dConfig,+ pub snake_config: Snake1dConfig,+ pub decoder_block_config: DACDecoderBlockConfig,+ pub causal: bool,+}diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/mod.rs b/crates/backend-uzu/src/config/tts/audio_decoder/mod.rsnew file mode 100644index 000000000..20b2759ea--- /dev/null+++ b/crates/backend-uzu/src/config/tts/audio_decoder/mod.rs@@ -0,0 +1,8 @@+use proc_macros::uzu_config_abstract;++pub mod common;+pub mod descript_audio_codec;+pub mod fish_audio_modules;++#[uzu_config_abstract(descript_audio_codec::DescriptAudioCodecConfig)]+pub struct TTSAudioDecoderConfig;diff --git a/crates/backend-uzu/src/config/tts/mod.rs b/crates/backend-uzu/src/config/tts/mod.rsnew file mode 100644index 000000000..a5dd1a3c9--- /dev/null+++ b/crates/backend-uzu/src/config/tts/mod.rs@@ -0,0 +1,16 @@+use proc_macros::uzu_config;++use crate::config::tts::{+ audio_decoder::AnyTTSAudioDecoderConfig, text_decoder::AnyTTSTextDecoderConfig, vocoder::AnyVocoderConfig,+};++pub mod audio_decoder;+pub mod text_decoder;+pub mod vocoder;++#[uzu_config]+pub struct TTSConfig {+ pub text_decoder_config: AnyTTSTextDecoderConfig,+ pub audio_decoder_config: AnyTTSAudioDecoderConfig,+ pub vocoder_config: AnyVocoderConfig,+}diff --git a/crates/backend-uzu/src/config/tts/text_decoder/fish_audio_text_decoder.rs b/crates/backend-uzu/src/config/tts/text_decoder/fish_audio_text_decoder.rsnew file mode 100644index 000000000..c2de107ee--- /dev/null+++ b/crates/backend-uzu/src/config/tts/text_decoder/fish_audio_text_decoder.rs@@ -0,0 +1,34 @@+use proc_macros::uzu_config;++use crate::config::{+ embedding::tied_embedding::TiedEmbeddingConfig, linear::LinearConfig, transformer::TransformerConfig,+};++#[uzu_config(super::TTSTextDecoderConfig)]+pub struct FishAudioTextDecoderConfig {+ pub slow_embeddings_config: TiedEmbeddingConfig,+ pub slow_model_config: TransformerConfig,+ pub slow_readout_config: LinearConfig,++ pub fast_embeddings_config: TiedEmbeddingConfig,+ pub fast_model_config: TransformerConfig,+ pub fast_readout_config: LinearConfig,++ pub codebook_embeddings_config: TiedEmbeddingConfig,+ pub fast_model_projection_config: Option<LinearConfig>,++ pub semantic_token_begin_id: u64,+ pub semantic_token_end_id: u64,+ pub im_end_token_id: u64,+ pub codebook_size: usize,+ pub vocab_size: usize,+ pub slow_model_dim: usize,+ pub fast_model_dim: usize,+ pub num_codebooks: usize,+ pub max_seq_len: usize,++ pub scale_codebook_embeddings: bool,++ pub short_logits_size: usize,+ pub repeat_window_size: usize,+}diff --git a/crates/backend-uzu/src/config/tts/text_decoder/mod.rs b/crates/backend-uzu/src/config/tts/text_decoder/mod.rsnew file mode 100644index 000000000..3398fb588--- /dev/null+++ b/crates/backend-uzu/src/config/tts/text_decoder/mod.rs@@ -0,0 +1,6 @@+use proc_macros::uzu_config_abstract;++pub mod fish_audio_text_decoder;++#[uzu_config_abstract(fish_audio_text_decoder::FishAudioTextDecoderConfig)]+pub struct TTSTextDecoderConfig;diff --git a/crates/backend-uzu/src/config/tts/vocoder/mod.rs b/crates/backend-uzu/src/config/tts/vocoder/mod.rsnew file mode 100644index 000000000..aec73f953--- /dev/null+++ b/crates/backend-uzu/src/config/tts/vocoder/mod.rs@@ -0,0 +1,6 @@+use proc_macros::uzu_config_abstract;++pub mod noop_vocoder;++#[uzu_config_abstract(noop_vocoder::NoopVocoderConfig)]+pub struct VocoderConfig;diff --git a/crates/backend-uzu/src/config/tts/vocoder/noop_vocoder.rs b/crates/backend-uzu/src/config/tts/vocoder/noop_vocoder.rsnew file mode 100644index 000000000..380caba0f--- /dev/null+++ b/crates/backend-uzu/src/config/tts/vocoder/noop_vocoder.rs@@ -0,0 +1,4 @@+use proc_macros::uzu_config;++#[uzu_config(super::VocoderConfig)]+pub struct NoopVocoderConfig;diff --git a/crates/backend-uzu/src/config/tts_model.rs b/crates/backend-uzu/src/config/tts_model.rsdeleted file mode 100644index 4851cb7b3..000000000--- a/crates/backend-uzu/src/config/tts_model.rs+++ /dev/null@@ -1,205 +0,0 @@-#[cfg(metal_backend)]-use std::path::Path;--use proc_macros::uzu_config;--use super::{EmbeddingConfig, EmbeddingConfigCommon, NormalizationConfig, TransformerConfig};-use crate::{DataType, backends::common::ActivationConfig};--#[uzu_config]-pub struct TtsMessageProcessorConfig {- pub prompt_template: String,- pub drop_initial_newline: bool,-}--#[uzu_config]-pub struct TtsConfig {- pub text_decoder_config: TtsTextDecoderConfig,- pub audio_decoder_config: TtsAudioDecoderConfig,- pub vocoder_config: NoopVocoderConfig,- pub activation_precision: DataType,-}--#[uzu_config]-pub struct NoopVocoderConfig {}--#[uzu_config]-#[serde(tag = "type")]-pub enum TtsTextDecoderConfig {- #[serde(rename = "FishAudioTextDecoderConfig")]- FishAudio(FishAudioTextDecoderConfig),-}--#[uzu_config]-pub struct FishAudioTextDecoderConfig {- pub slow_embeddings_config: TiedEmbeddingConfig,- pub slow_model_config: TransformerConfig,- pub slow_readout_config: FullPrecisionLinearConfig,- pub fast_embeddings_config: TiedEmbeddingConfig,- pub fast_model_config: TransformerConfig,- pub fast_readout_config: FullPrecisionLinearConfig,- pub codebook_embeddings_config: TiedEmbeddingConfig,- pub fast_model_projection_config: Option<FullPrecisionLinearConfig>,- pub semantic_token_begin_id: i64,- pub semantic_token_end_id: i64,- pub im_end_token_id: i64,- pub codebook_size: usize,- pub vocab_size: usize,- pub slow_model_dim: usize,- pub fast_model_dim: usize,- pub num_codebooks: usize,- pub max_seq_len: usize,- pub scale_codebook_embeddings: bool,- pub precision: DataType,- pub short_logits_size: usize,- pub repeat_window_size: usize,-}--#[uzu_config]-pub struct FullPrecisionLinearConfig {- pub precision: DataType,-}--#[uzu_config]-pub struct TiedEmbeddingConfig {- #[serde(flatten)]- pub common: EmbeddingConfigCommon,- pub precision: DataType,-}--impl TiedEmbeddingConfig {- #[cfg(metal_backend)]- pub(crate) fn to_text_decoder_embedding_config(&self) -> EmbeddingConfig {- EmbeddingConfig::Untied {- common: self.common.clone(),- precision: self.precision,- }- }-}--#[uzu_config]-#[serde(tag = "type")]-pub enum TtsAudioDecoderConfig {- #[serde(rename = "DescriptAudioCodecConfig")]- DescriptAudioCodec(DescriptAudioCodecConfig),-}--#[uzu_config]-pub struct DescriptAudioCodecConfig {- pub precision: DataType,- pub quantizer_config: DescriptAudioQuantizerConfig,- pub decoder_config: DescriptAudioDacDecoderConfig,- pub samplerate: u32,- pub encoder_dim: usize,- pub encoder_rates: Vec<usize>,- pub decoder_dim: usize,- pub decoder_rates: Vec<usize>,- pub input_dim: usize,- pub n_codebooks: usize,- pub codebook_dim: usize,- pub downsample_factor: Vec<usize>,- pub codebook_size: usize,- pub semantic_codebook_size: usize,-}--#[uzu_config]-pub struct DescriptAudioDacDecoderConfig {- pub precision: DataType,- pub conv_config: DescriptAudioCausalConv1dConfig,- pub snake_config: DescriptAudioSnake1dConfig,- pub decoder_block_config: DescriptAudioDacDecoderBlockConfig,- pub causal: bool,-}--#[uzu_config]-pub struct DescriptAudioDacDecoderBlockConfig {- pub precision: DataType,- pub snake_config: DescriptAudioSnake1dConfig,- pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,- pub res_unit_config: DescriptAudioResidualUnitConfig,- pub causal: bool,-}--#[uzu_config]-pub struct DescriptAudioResidualUnitConfig {- pub precision: DataType,- pub snake_config: DescriptAudioSnake1dConfig,- pub conv_config: DescriptAudioCausalConv1dConfig,- pub causal: bool,-}--#[uzu_config]-pub struct DescriptAudioCausalConv1dConfig {- pub precision: DataType,- pub has_biases: bool,-}--#[uzu_config]-pub struct DescriptAudioCausalTransposeConv1dConfig {- pub precision: DataType,- pub has_biases: bool,-}--#[uzu_config]-pub struct DescriptAudioSnake1dConfig {- pub precision: DataType,-}--#[uzu_config]-pub struct DescriptAudioQuantizerConfig {- pub precision: DataType,- pub semantic_quantizer_config: DescriptAudioResidualVectorQuantizeConfig,- pub quantizer_config: DescriptAudioResidualVectorQuantizeConfig,- pub post_module_config: TransformerConfig,- pub upsampler_config: DescriptAudioUpsamplerConfig,-}--#[uzu_config]-pub struct DescriptAudioResidualVectorQuantizeConfig {- pub precision: DataType,- pub vq_config: DescriptAudioVectorQuantizeConfig,-}--#[uzu_config]-pub struct DescriptAudioVectorQuantizeConfig {- pub precision: DataType,- pub codebook_config: TiedEmbeddingConfig,- pub out_proj_config: FullPrecisionLinearConfig,-}--#[uzu_config]-pub struct DescriptAudioUpsamplerConfig {- pub block_configs: Vec<DescriptAudioUpsamplingBlockConfig>,-}--#[uzu_config]-pub struct DescriptAudioUpsamplingBlockConfig {- pub precision: DataType,- pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,- pub convnext_config: DescriptAudioConvNeXtConfig,-}--#[uzu_config]-pub struct DescriptAudioConvNeXtConfig {- pub precision: DataType,- pub activation: ActivationConfig,- pub dwconv_config: DescriptAudioCausalConv1dConfig,- pub norm_config: NormalizationConfig,- pub pwconv_config: FullPrecisionLinearConfig,-}--#[uzu_config]-pub struct TtsModelConfig {- pub tts_config: TtsConfig,- pub message_processor_config: TtsMessageProcessorConfig,-}--#[cfg(metal_backend)]-impl TtsModelConfig {- pub fn create_audio_generation_context_with_model_path<B: crate::backends::common::Backend>(- &self,- model_path: &Path,- ) -> crate::audio::AudioResult<crate::audio::AudioGenerationContext<B>> {- crate::audio::AudioGenerationContext::from_tts_config_and_model_path(&self.tts_config, model_path)- }-}diff --git a/crates/backend-uzu/src/config/weight_matrix/awq_spec.rs b/crates/backend-uzu/src/config/weight_matrix/awq_spec.rsnew file mode 100644index 000000000..914c4eded--- /dev/null+++ b/crates/backend-uzu/src/config/weight_matrix/awq_spec.rs@@ -0,0 +1,11 @@+use proc_macros::uzu_config;++use crate::config::weight_matrix::Layout;++#[uzu_config(super::WeightMatrixSpec)]+pub struct AWQSpec {+ pub bits: u32,+ pub group_size: usize,+ pub is_symmetric: bool,+ pub layout: Layout,+}diff --git a/crates/backend-uzu/src/config/weight_matrix/full_precision_spec.rs b/crates/backend-uzu/src/config/weight_matrix/full_precision_spec.rsnew file mode 100644index 000000000..41dc4e03b--- /dev/null+++ b/crates/backend-uzu/src/config/weight_matrix/full_precision_spec.rs@@ -0,0 +1,8 @@+use proc_macros::uzu_config;++use crate::config::weight_matrix::Layout;++#[uzu_config(super::WeightMatrixSpec)]+pub struct FullPrecisionSpec {+ pub layout: Layout,+}diff --git a/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs b/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rsnew file mode 100644index 000000000..7c9e08009--- /dev/null+++ b/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs@@ -0,0 +1,19 @@+use proc_macros::uzu_config;++use crate::config::weight_matrix::AnyWeightMatrixSpec;++#[uzu_config]+#[serde(rename_all = "snake_case")]+pub enum IncoherenceProcessingMode {+ Input,+ Output,+ InputOutput,+}++#[uzu_config(super::WeightMatrixSpec)]+pub struct HybridSpec {+ pub quantization_spec: Box<AnyWeightMatrixSpec>,+ pub adapter_spec: Option<Box<AnyWeightMatrixSpec>>,+ pub incoherence_block_size: Option<usize>,+ pub incoherence_processing_mode: IncoherenceProcessingMode,+}diff --git a/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs b/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rsnew file mode 100644index 000000000..d056781e4--- /dev/null+++ b/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs@@ -0,0 +1,6 @@+use proc_macros::uzu_config;++#[uzu_config(super::WeightMatrixSpec)]+pub struct LowRankSpec {+ pub rank: usize,+}diff --git a/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs b/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rsnew file mode 100644index 000000000..eda4bf6c5--- /dev/null+++ b/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs@@ -0,0 +1,10 @@+use proc_macros::uzu_config;++use crate::config::weight_matrix::Layout;++#[uzu_config(super::WeightMatrixSpec)]+pub struct MLXSpec {+ pub bits: u32,+ pub group_size: usize,+ pub layout: Layout,+}diff --git a/crates/backend-uzu/src/config/weight_matrix/mod.rs b/crates/backend-uzu/src/config/weight_matrix/mod.rsnew file mode 100644index 000000000..bd87da016--- /dev/null+++ b/crates/backend-uzu/src/config/weight_matrix/mod.rs@@ -0,0 +1,23 @@+use proc_macros::{uzu_config, uzu_config_abstract};++pub mod awq_spec;+pub mod full_precision_spec;+pub mod hybrid_spec;+pub mod low_rank_spec;+pub mod mlx_spec;++#[uzu_config]+#[serde(rename_all = "snake_case")]+pub enum Layout {+ OutputInput,+ InputOutput,+}++#[uzu_config_abstract(+ full_precision_spec::FullPrecisionSpec,+ low_rank_spec::LowRankSpec,+ hybrid_spec::HybridSpec,+ awq_spec::AWQSpec,+ mlx_spec::MLXSpec+)]+pub struct WeightMatrixSpec;diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rsindex 63d530590..ce5e3df68 100644--- a/crates/backend-uzu/src/encodable_block/attention.rs+++ b/crates/backend-uzu/src/encodable_block/attention.rs@@ -1,9 +1,11 @@//! Attention kernel encodable.-use std::{cell::RefCell, collections::HashMap};+use std::{cell::RefCell, collections::HashMap, rc::Rc};use itertools::iproduct;+use thiserror::Error;+use super::{Linear, QKVNorm, QkUnpack, Rope, linear::LinearBlockError, qkv_norm::QKVNormError};use crate::{DataType,array::size_for_shape,@@ -18,11 +20,13 @@ use crate::{matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},- config::AttentionConfig,+ config::token_mixer::attention::AttentionConfig,forward_pass::{cache_layers::LayerCacheAccess,kv_cache_layer::{KVCacheLayer, KVCacheLayerState},+ state::RopeBuffers,},+ parameters::{ParameterLoaderError, ParameterTree},};fn env_gemm_attention_enabled() -> bool {@@ -42,6 +46,13 @@ fn env_gemm_attention_enabled() -> bool {}pub struct Attention<B: Backend> {+ qkv_projection: Box<dyn Linear<B>>,+ gate_projection: Option<Box<dyn Linear<B>>>,+ qkv_norm: Option<QKVNorm<B>>,+ rope: Rc<Rope<B>>,+ qk_unpack: Rc<QkUnpack<B>>,+ out_projection: Box<dyn Linear<B>>,+ sinks: Option<Allocation<B>>,single_pass_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionSinglePassKernel>,two_pass_1_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionTwoPass1Kernel>,two_pass_2_kernels: HashMap<u32, <B::Kernels as Kernels>::AttentionTwoPass2Kernel>,@@ -58,21 +69,100 @@ pub struct Attention<B: Backend> {attention_scale: Option<f32>,is_causal: bool,sliding_window_size: Option<usize>,+ model_dim: usize,+ num_heads: usize,+ num_groups: usize,+ head_dim: usize,}-pub struct AttentionArguments<'a, B: Backend> {- pub token_subtrie_ranges: Option<&'a Allocation<B>>,- pub attention_sinks: Option<&'a Allocation<B>>,- pub cache_access: Option<LayerCacheAccess<'a, B>>,+#[derive(Debug, Error)]+pub enum AttentionError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Linear block error: {0}")]+ LinearBlockError(#[from] LinearBlockError<B>),+ #[error("QKV norm error: {0}")]+ QKVNormError(#[from] QKVNormError<B>),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[from] ParameterLoaderError<B>),}impl<B: Backend> Attention<B> {+ #[allow(clippy::too_many_arguments)]pub fn new(context: &B::Context,+ model_dim: usize,data_type: DataType,config: &AttentionConfig,- has_gate: bool,- ) -> Result<Self, B::Error> {+ parameter_tree: &ParameterTree<B>,+ rope: Rc<Rope<B>>,+ qk_unpack: Rc<QkUnpack<B>>,+ extract_input_hadamard: bool,+ ) -> Result<(Self, Option<Allocation<B>>), AttentionError<B>> {+ let q_dim = config.num_heads * config.head_dim;+ let kv_dim = config.num_groups * config.head_dim;++ let qkv_projection_tree = parameter_tree.subtree("qkv_projection")?;+ let (qkv_projection, input_hadamard_factors) = if extract_input_hadamard {+ <dyn Linear<B>>::new_extracting_input_hadamard(+ model_dim,+ [q_dim, kv_dim, kv_dim],+ config.has_qkv_biases,+ context,+ data_type,+ &qkv_projection_tree,+ )?+ } else {+ (+ <dyn Linear<B>>::new(+ model_dim,+ [q_dim, kv_dim, kv_dim],+ config.has_qkv_biases,+ context,+ data_type,+ &qkv_projection_tree,+ )?,+ None,+ )+ };++ let gate_projection = config+ .gate_projection_config+ .as_ref()+ .map(|_| {+ let gate_projection_tree = parameter_tree.subtree("gate_projection")?;+ <dyn Linear<B>>::new(model_dim, [q_dim], false, context, data_type, &gate_projection_tree)+ })+ .transpose()?;++ let value_norm_config = config.value_norm_config();+ let qkv_norm =+ if config.query_norm_config.is_some() || config.key_norm_config.is_some() || value_norm_config.is_some() {+ Some(QKVNorm::new(+ context,+ data_type,+ config.query_norm_config.clone(),+ config.key_norm_config.clone(),+ value_norm_config,+ parameter_tree,+ config.num_heads,+ config.num_groups,+ config.head_dim,+ )?)+ } else {+ None+ };++ let out_projection_tree = parameter_tree.subtree("out_projection")?;+ let out_projection =+ <dyn Linear<B>>::new(q_dim, [model_dim], config.has_out_biases, context, data_type, &out_projection_tree)?;++ let sinks = if config.has_sinks {+ Some(parameter_tree.leaf("sinks")?.validate(&[config.num_heads], data_type)?.read_allocation()?)+ } else {+ None+ };+let mut single_pass_kernels = HashMap::new();let mut two_pass_1_kernels = HashMap::new();let mut two_pass_2_kernels = HashMap::new();@@ -96,7 +186,8 @@ impl<B: Backend> Attention<B> {config.is_causal,is_trie,config.sliding_window_size.is_some(),- )?;+ )+ .map_err(AttentionError::BackendError)?;single_pass_kernels.insert(key, sp_kernel);let tp1_kernel = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(@@ -108,10 +199,12 @@ impl<B: Backend> Attention<B> {config.is_causal,is_trie,config.sliding_window_size.is_some(),- )?;+ )+ .map_err(AttentionError::BackendError)?;two_pass_1_kernels.insert(key, tp1_kernel);- let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)?;+ let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)+ .map_err(AttentionError::BackendError)?;two_pass_2_kernels.insert(head_dim, tp2_kernel);}@@ -123,49 +216,74 @@ impl<B: Backend> Attention<B> {config.is_causal,false,config.sliding_window_size.is_some(),- )?;+ )+ .map_err(AttentionError::BackendError)?;fallback_scatter_scores_kernels.insert(is_kv_cache_ring, scatter);}- let softmax_kernel = <B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, false)?;+ let softmax_kernel = <B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, false)+ .map_err(AttentionError::BackendError)?;let softmax_sinks_kernel = if config.has_sinks {- Some(<B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, true)?)+ Some(+ <B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, true)+ .map_err(AttentionError::BackendError)?,+ )} else {None};let fallback_scatter_values_kernel =- <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, data_type)?;+ <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, data_type)+ .map_err(AttentionError::BackendError)?;let matmul_kernel =<<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).ok().map(RefCell::new);let update_kv_cache_kernel =- <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false)?;+ <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false)+ .map_err(AttentionError::BackendError)?;let update_kv_cache_inplace_kernel =- <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, true)?;- let gate_kernel = if has_gate {- Some(<B::Kernels as Kernels>::SigmoidGateKernel::new(context, data_type)?)+ <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, true)+ .map_err(AttentionError::BackendError)?;+ let gate_kernel = if gate_projection.is_some() {+ Some(+ <B::Kernels as Kernels>::SigmoidGateKernel::new(context, data_type)+ .map_err(AttentionError::BackendError)?,+ )} else {None};let gemm_block = AttentionGemmBlock::new(data_type);- Ok(Self {- single_pass_kernels,- two_pass_1_kernels,- two_pass_2_kernels,- softmax_kernel,- softmax_sinks_kernel,- fallback_scatter_scores_kernels,- fallback_scatter_values_kernel,- matmul_kernel,- update_kv_cache_kernel,- update_kv_cache_inplace_kernel,- gate_kernel,- gemm_block,- data_type,- attention_scale: config.scale,- is_causal: config.is_causal,- sliding_window_size: config.sliding_window_size,- })+ Ok((+ Self {+ qkv_projection,+ gate_projection,+ qkv_norm,+ rope,+ qk_unpack,+ out_projection,+ sinks,+ single_pass_kernels,+ two_pass_1_kernels,+ two_pass_2_kernels,+ softmax_kernel,+ softmax_sinks_kernel,+ fallback_scatter_scores_kernels,+ fallback_scatter_values_kernel,+ matmul_kernel,+ update_kv_cache_kernel,+ update_kv_cache_inplace_kernel,+ gate_kernel,+ gemm_block,+ data_type,+ attention_scale: config.scale,+ is_causal: config.is_causal,+ sliding_window_size: config.sliding_window_size,+ model_dim,+ num_heads: config.num_heads,+ num_groups: config.num_groups,+ head_dim: config.head_dim,+ },+ input_hadamard_factors,+ ))}fn select_variant(@@ -201,22 +319,76 @@ impl<B: Backend> Attention<B> {KernelVariant::SinglePass}+ #[allow(clippy::too_many_arguments)]pub fn encode(&self,- args: AttentionArguments<B>,+ token_positions: &Allocation<B>,+ token_subtrie_ranges: Option<&Allocation<B>>,+ rope_buffers: Option<&RopeBuffers<B>>,+ cache_access: Option<LayerCacheAccess<B>>,+ hidden: Allocation<B>,+ suffix_length: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let (qkv_input, gate) = if let Some(gate_projection) = &self.gate_projection {+ let mut qkv_input =+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.model_dim], self.data_type))?;+ encoder.encode_copy(&hidden, .., &mut qkv_input, ..);+ let gate = gate_projection.encode(hidden, suffix_length, encoder)?;+ (qkv_input, Some(gate))+ } else {+ (hidden, None)+ };++ let mut qkv = self.qkv_projection.encode(qkv_input, suffix_length, encoder)?;+ if let Some(qkv_norm) = &self.qkv_norm {+ qkv_norm.encode(&mut qkv, suffix_length, encoder)?;+ }+ let (queries, rotated_keys) = match rope_buffers {+ Some(rope_buffers) => self.rope.encode(+ &qkv,+ token_positions,+ &rope_buffers.cosines,+ &rope_buffers.sines,+ suffix_length,+ self.num_heads,+ self.num_groups,+ self.head_dim,+ rope_buffers.max_sequence_length(),+ rope_buffers.dim(),+ encoder,+ )?,+ None => {+ self.qk_unpack.encode(&qkv, suffix_length, self.num_heads, self.num_groups, self.head_dim, encoder)?+ },+ };+ let attention_output = self.encode_core(+ token_subtrie_ranges,+ cache_access,+ &qkv,+ &queries,+ rotated_keys,+ gate.as_ref(),+ suffix_length,+ encoder,+ )?;+ self.out_projection.encode(attention_output, suffix_length, encoder)+ }++ fn encode_core(+ &self,+ token_subtrie_ranges: Option<&Allocation<B>>,+ cache_access: Option<LayerCacheAccess<B>>,qkv: &Allocation<B>,queries: &Allocation<B>,mut rotated_keys: Allocation<B>,gate: Option<&Allocation<B>>,suffix_length: usize,- num_heads: usize,- num_groups: usize,- head_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let is_trie = args.token_subtrie_ranges.is_some();+ let is_trie = token_subtrie_ranges.is_some();- let cache_layer = args.cache_access.as_ref().map(|a| {+ let cache_layer = cache_access.as_ref().map(|a| {match a {LayerCacheAccess::Owned {entry,@@ -253,8 +425,8 @@ impl<B: Backend> Attention<B> {let sequence_length = segment_prefix_length + suffix_length;- let gqa_factor = num_heads / num_groups;- let scale = self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());+ let gqa_factor = self.num_heads / self.num_groups;+ let scale = self.attention_scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt());let gemm_enabled = env_gemm_attention_enabled();if !gemm_enabled {@@ -264,24 +436,26 @@ impl<B: Backend> Attention<B> {});}let variant =- self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, is_trie, is_kv_cache_ring);+ self.select_variant(gemm_enabled, suffix_length, self.head_dim, sequence_length, is_trie, is_kv_cache_ring);- let sinks_allocation = args.attention_sinks;- let trie_allocation = args.token_subtrie_ranges;+ let sinks_allocation = self.sinks.as_ref();+ let trie_allocation = token_subtrie_ranges;- let has_kv_cache = args.cache_access.is_some();+ let has_kv_cache = cache_access.is_some();let extracted_values = (!has_kv_cache).then(|| {- let mut values =- encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?;+ let mut values = encoder.allocate_scratch(size_for_shape(+ &[self.num_groups, suffix_length, self.head_dim],+ self.data_type,+ ))?;self.update_kv_cache_inplace_kernel.encode(None::<&Allocation<B>>,qkv,&mut rotated_keys,&mut values,- num_groups as u32,- num_heads as u32,- head_dim as u32,+ self.num_groups as u32,+ self.num_heads as u32,+ self.head_dim as u32,suffix_length as u32,0u32,max_sequence_length as u32,@@ -295,18 +469,23 @@ impl<B: Backend> Attention<B> {// For classifier mode (no KV cache) keys/values still come in group-major layout// [num_groups, suffix_length, head_dim].let (k_head_stride, k_seq_stride, v_head_stride, v_seq_stride) = if has_kv_cache {- (head_dim as u32, (num_groups * head_dim) as u32, head_dim as u32, (num_groups * head_dim) as u32)+ (+ self.head_dim as u32,+ (self.num_groups * self.head_dim) as u32,+ self.head_dim as u32,+ (self.num_groups * self.head_dim) as u32,+ )} else {(- (max_sequence_length * head_dim) as u32,- head_dim as u32,- (max_sequence_length * head_dim) as u32,- head_dim as u32,+ (max_sequence_length * self.head_dim) as u32,+ self.head_dim as u32,+ (max_sequence_length * self.head_dim) as u32,+ self.head_dim as u32,)};let kernel_key = KernelKey {- head_dim: head_dim as u32,+ head_dim: self.head_dim as u32,is_trie,is_kv_cache_ring,};@@ -329,18 +508,18 @@ impl<B: Backend> Attention<B> {v_seq_stride,ring_params,scale,- num_heads,- num_groups,+ self.num_heads,+ self.num_groups,suffix_length,segment_prefix_length,max_sequence_length,- head_dim,+ self.head_dim,encoder,)?};}- let mut attention_output = if let Some(access) = args.cache_access {+ let mut attention_output = if let Some(access) = cache_access {match access {LayerCacheAccess::Owned {entry,@@ -352,9 +531,9 @@ impl<B: Backend> Attention<B> {qkv,&mut layer.keys,&mut layer.values,- num_groups as u32,- num_heads as u32,- head_dim as u32,+ self.num_groups as u32,+ self.num_heads as u32,+ self.head_dim as u32,suffix_length as u32,segment_prefix_length as u32,max_sequence_length as u32,@@ -367,9 +546,9 @@ impl<B: Backend> Attention<B> {qkv,&mut layer.keys,&mut layer.values,- num_groups as u32,- num_heads as u32,- head_dim as u32,+ self.num_groups as u32,+ self.num_heads as u32,+ self.head_dim as u32,suffix_length as u32,segment_prefix_length as u32,max_sequence_length as u32,@@ -399,7 +578,7 @@ impl<B: Backend> Attention<B> {};if let Some(gate_kernel) = &self.gate_kernel {- let total_elements = (suffix_length * num_heads * head_dim) as u32;+ let total_elements = (suffix_length * self.num_heads * self.head_dim) as u32;let gate = gate.expect("Gate allocation not initialized");gate_kernel.encode(gate, &mut attention_output, total_elements, encoder);}diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rsindex 8697ac102..99cdbd23d 100644--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs@@ -1,23 +1,41 @@use std::rc::Rc;+use thiserror::Error;+use crate::{DataType,backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder, Kernels, kernel::TensorAddSwapKernel},- config::{TransformerConfig, TransformerLayerConfig},+ config::{transformer::TransformerConfig, transformer_layer::TransformerLayerConfig},encodable_block::{- Attention, AttentionArguments, LayerArguments, Linear, Mlp, Normalization, QKVNorm, QkUnpack, Rope,+ Attention, AttentionError, LayerArguments, Mlp, MlpBlockError, Normalization, NormalizationError, QkUnpack,+ Rope,},- parameters::ParameterTree,+ parameters::{ParameterLoaderError, ParameterTree},};+#[derive(Debug, Error)]+pub enum ClassifierLayerError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[from] ParameterLoaderError<B>),+ #[error("Attention error: {0}")]+ AttentionError(#[from] AttentionError<B>),+ #[error("MLP error: {0}")]+ MlpBlockError(#[from] MlpBlockError<B>),+ #[error("Normalization error: {0}")]+ NormalizationError(#[from] NormalizationError<B>),+ #[error("classifier layers must use attention")]+ NonAttentionMixer,+ #[error("classifier does not support attention hadamard")]+ AttentionHadamardUnsupported,+ #[error("classifier does not support MLP hadamard")]+ MlpHadamardUnsupported,+}+pub struct ClassifierLayer<B: Backend> {pre_attention_norm: Option<Normalization<B>>,- qkv_projection: Box<dyn Linear<B>>,- qkv_norm: Option<QKVNorm<B>>,- rope: Rc<Rope<B>>,- qk_unpack: Rc<QkUnpack<B>>,attention: Attention<B>,- out_projection: Box<dyn Linear<B>>,post_attention_norm: Option<Normalization<B>>,mixer_residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,pre_mlp_norm: Normalization<B>,@@ -25,9 +43,6 @@ pub struct ClassifierLayer<B: Backend> {post_mlp_norm: Option<Normalization<B>>,mlp_residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,model_dim: usize,- num_heads: usize,- num_groups: usize,- head_dim: usize,}impl<B: Backend> ClassifierLayer<B> {@@ -35,125 +50,94 @@ impl<B: Backend> ClassifierLayer<B> {context: &B::Context,transformer_config: &TransformerConfig,layer_config: &TransformerLayerConfig,- layer_index: usize,- layer_loader: &ParameterTree<B::Context>,+ layer_loader: &ParameterTree<B>,rope: Rc<Rope<B>>,qk_unpack: Rc<QkUnpack<B>>,- ) -> Self {- let attention_config = layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");- let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();+ data_type: DataType,+ ) -> Result<Self, ClassifierLayerError<B>> {+ let attention_config =+ layer_config.mixer_config.as_attention().ok_or(ClassifierLayerError::NonAttentionMixer)?;- let pre_attention_norm = layer_config.pre_mixer_norm_config.as_ref().map(|norm_config| {- Normalization::new(+ let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_mixer_norm_config {+ Some(Normalization::new(context,- intermediate_data_type,+ data_type,+ transformer_config.model_dim,norm_config.clone(),- &layer_loader.subtree("pre_mixer_norm").unwrap(),- )- .expect("Failed to create pre-attention norm kernel")- });-- let qkv_projection = <dyn Linear<B>>::new(- &attention_config.qkv_projection_config,- transformer_config.model_dim,- [- attention_config.num_heads * attention_config.head_dim,- attention_config.num_groups * attention_config.head_dim,- attention_config.num_groups * attention_config.head_dim,- ],- context,- &layer_loader.subtree("mixer.qkv_projection").unwrap(),- )- .expect("Failed to create qkv projection");-- let value_norm_config = attention_config.value_norm_config();- let qkv_norm = if attention_config.query_norm_config.is_some()- || attention_config.key_norm_config.is_some()- || value_norm_config.is_some()- {- match QKVNorm::new(- context,- intermediate_data_type,- attention_config.query_norm_config.clone(),- attention_config.key_norm_config.clone(),- value_norm_config,- &layer_loader.subtree("mixer").unwrap(),- attention_config.num_heads,- attention_config.num_groups,- attention_config.head_dim,- ) {- Ok(qkv_norm) => Some(qkv_norm),- Err(error) => panic!("Failed to create QKV norm kernel for layer {}: {:?}", layer_index, error),- }+ &layer_loader.subtree("pre_mixer_norm")?,+ )?)} else {None};- let out_projection = <dyn Linear<B>>::new(- &attention_config.out_projection_config,- attention_config.num_heads * attention_config.head_dim,- [transformer_config.model_dim],+ let (attention, attention_hadamard_factors) = Attention::new(context,- &layer_loader.subtree("mixer.out_projection").unwrap(),- )- .expect("Failed to create out projection");+ transformer_config.model_dim,+ data_type,+ attention_config,+ &layer_loader.subtree("mixer")?,+ rope,+ qk_unpack,+ false,+ )?;+ if attention_hadamard_factors.is_some() {+ return Err(ClassifierLayerError::AttentionHadamardUnsupported);+ }- let post_attention_norm = layer_config.post_mixer_norm_config.as_ref().map(|norm_config| {- Normalization::new(+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {+ Some(Normalization::new(context,- intermediate_data_type,+ data_type,+ transformer_config.model_dim,norm_config.clone(),- &layer_loader.subtree("post_mixer_norm").unwrap(),- )- .expect("Failed to create post-attention norm kernel")- });+ &layer_loader.subtree("post_mixer_norm")?,+ )?)+ } else {+ None+ };- let mixer_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, intermediate_data_type)- .expect("Failed to create mixer residual add kernel");+ let mixer_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type)+ .map_err(ClassifierLayerError::BackendError)?;let pre_mlp_norm = Normalization::new(context,- intermediate_data_type,+ data_type,+ transformer_config.model_dim,layer_config.pre_mlp_norm_config.clone(),- &layer_loader.subtree("pre_mlp_norm").unwrap(),- )- .expect("Failed to create pre-MLP norm kernel");+ &layer_loader.subtree("pre_mlp_norm")?,+ )?;let (mlp, mlp_hadamard_factors) = <dyn Mlp<B>>::new(&layer_config.mlp_config,transformer_config.model_dim,layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),context,- &layer_loader.subtree("mlp").unwrap(),- )- .expect("Failed to create mlp block");+ &layer_loader.subtree("mlp")?,+ data_type,+ )?;- assert!(mlp_hadamard_factors.is_none(), "classifier doesn't support hadamard");+ if mlp_hadamard_factors.is_some() {+ return Err(ClassifierLayerError::MlpHadamardUnsupported);+ }- let post_mlp_norm = layer_config.post_mlp_norm_config.as_ref().map(|norm_config| {- Normalization::new(+ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {+ Some(Normalization::new(context,- intermediate_data_type,+ data_type,+ transformer_config.model_dim,norm_config.clone(),- &layer_loader.subtree("post_mlp_norm").unwrap(),- )- .expect("Failed to create post-MLP norm kernel")- });-- let attention = Attention::new(context, intermediate_data_type, attention_config, false)- .expect("Failed to create attention kernel");+ &layer_loader.subtree("post_mlp_norm")?,+ )?)+ } else {+ None+ };- let mlp_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, intermediate_data_type)- .expect("Failed to create mlp residual add kernel");+ let mlp_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type)+ .map_err(ClassifierLayerError::BackendError)?;- Self {+ Ok(Self {pre_attention_norm,- qkv_projection,- qkv_norm,- rope,- qk_unpack,attention,- out_projection,post_attention_norm,mixer_residual_add,pre_mlp_norm,@@ -161,10 +145,7 @@ impl<B: Backend> ClassifierLayer<B> {post_mlp_norm,mlp_residual_add,model_dim: transformer_config.model_dim,- num_heads: attention_config.num_heads,- num_groups: attention_config.num_groups,- head_dim: attention_config.head_dim,- }+ })}pub fn encode(@@ -178,7 +159,6 @@ impl<B: Backend> ClassifierLayer<B> {batch_dim,token_positions,token_subtrie_ranges,- attention_sinks,rope_buffers,#[cfg(feature = "tracing")]trace,@@ -206,43 +186,15 @@ impl<B: Backend> ClassifierLayer<B> {encoder.encode_copy(&main, .., layer_traces.pre_attention_norm.allocation_mut(), ..);}- let mut qkv = self.qkv_projection.encode(main, batch_dim, encoder)?;- if let Some(ref qkv_norm) = self.qkv_norm {- qkv_norm.encode(&mut qkv, batch_dim, encoder)?;- }- let (queries, rotated_keys) = match rope_buffers {- Some(rope_buffers) => self.rope.encode(- &qkv,- token_positions,- &rope_buffers.cosines,- &rope_buffers.sines,- batch_dim,- self.num_heads,- self.num_groups,- self.head_dim,- rope_buffers.max_sequence_length(),- rope_buffers.dim(),- encoder,- )?,- None => self.qk_unpack.encode(&qkv, batch_dim, self.num_heads, self.num_groups, self.head_dim, encoder)?,- };- let attention_output = self.attention.encode(- AttentionArguments {- token_subtrie_ranges,- attention_sinks,- cache_access: None,- },- &qkv,- &queries,- rotated_keys,+ main = self.attention.encode(+ token_positions,+ token_subtrie_ranges,+ rope_buffers,None,+ main,batch_dim,- self.num_heads,- self.num_groups,- self.head_dim,encoder,)?;- main = self.out_projection.encode(attention_output, batch_dim, encoder)?;#[cfg(feature = "tracing")]if let Some(layer_traces) = layer_traces.as_deref_mut() {encoder.encode_copy(&main, .., layer_traces.attention.allocation_mut(), ..);diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex e951c5068..223ddc0d2 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -7,15 +7,14 @@ use thiserror::Error;#[cfg(feature = "tracing")]use crate::forward_pass::traces::ActivationTrace;use crate::{- DataType,backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},- config::DecoderConfig,+ config::decoder::DecoderConfig,encodable_block::{- Embedding, LayerArguments, LayerExecutables, PostLayerScalar, QkUnpack, RMSNorm, Rope,- embedding::EmbeddingError,+ Embedding, LayerArguments, LayerExecutables, PostLayerScalar, QkUnpack, RMSNorm, RMSNormError, Rope,+ embedding::EmbeddingError, layer::LayerExecutablesError,},- forward_pass::{cache_layers::CacheLayers, state::SharedBuffers},- parameters::ParameterTree,+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},+ parameters::{ParameterLoaderError, ParameterTree},};#[derive(Debug, Error)]@@ -24,6 +23,12 @@ pub enum DecoderError<B: Backend> {BackendError(#[source] B::Error),#[error("Embedding error: {0}")]EmbeddingError(#[from] EmbeddingError<B>),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[from] ParameterLoaderError<B>),+ #[error("Layer error: {0}")]+ LayerError(#[from] LayerExecutablesError<B>),+ #[error("RMSNorm error: {0}")]+ RMSNormError(#[from] RMSNormError<B>),}/// Full decoder executable with all layers and components.@@ -56,27 +61,34 @@ impl<B: Backend> Decoder<B> {pub fn new(context: &B::Context,decoder_config: &DecoderConfig,- root_weight_loader: &ParameterTree<B::Context>,- ) -> Self {- let embedding_weight_loader = root_weight_loader.subtree("embedding").expect("Failed to get embedding subtree");+ root_weight_loader: &ParameterTree<B>,+ model_shape: &ModelShape,+ ) -> Result<Self, DecoderError<B>> {+ let embedding_weight_loader = root_weight_loader.subtree("embedding")?;- let embed = Embedding::new(+ let (embed, readout_input_hadamard_factors) = Embedding::new(context,decoder_config.vocab_size as u32,decoder_config.transformer_config.model_dim as u32,&decoder_config.embedding_config,&embedding_weight_loader,- )- .expect("Failed to create embedding");+ model_shape,+ )?;- let (layers, norm) =- Self::build_transformer_layers_and_norm(context, decoder_config, root_weight_loader, "transformer");+ let (layers, norm) = Self::build_transformer_layers_and_norm(+ context,+ decoder_config,+ root_weight_loader,+ "transformer",+ readout_input_hadamard_factors,+ model_shape,+ )?;- Self {+ Ok(Self {embed,layers,norm,- }+ })}/// Used by models whose token lookup weights and logits readout weights@@ -84,73 +96,90 @@ impl<B: Backend> Decoder<B> {pub fn new_with_embedding_and_readout_subtrees(context: &B::Context,decoder_config: &DecoderConfig,- root_weight_loader: &ParameterTree<B::Context>,+ root_weight_loader: &ParameterTree<B>,transformer_subtree: &str,embedding_subtree: &str,readout_subtree: &str,- ) -> Self {- let embedding_weight_loader =- root_weight_loader.subtree(embedding_subtree).expect("Failed to get embedding subtree");- let readout_weight_loader = root_weight_loader.subtree(readout_subtree).expect("Failed to get readout subtree");+ model_shape: &ModelShape,+ ) -> Result<Self, DecoderError<B>> {+ let embedding_weight_loader = root_weight_loader.subtree(embedding_subtree)?;+ let readout_weight_loader = root_weight_loader.subtree(readout_subtree)?;- let embed = Embedding::new_with_lookup_and_readout_trees(+ let (embed, readout_input_hadamard_factors) = Embedding::new_with_lookup_and_readout_trees(context,decoder_config.vocab_size as u32,decoder_config.transformer_config.model_dim as u32,&decoder_config.embedding_config,&embedding_weight_loader,&readout_weight_loader,- )- .expect("Failed to create embedding");+ model_shape,+ )?;- let (layers, norm) =- Self::build_transformer_layers_and_norm(context, decoder_config, root_weight_loader, transformer_subtree);+ let (layers, norm) = Self::build_transformer_layers_and_norm(+ context,+ decoder_config,+ root_weight_loader,+ transformer_subtree,+ readout_input_hadamard_factors,+ model_shape,+ )?;- Self {+ Ok(Self {embed,layers,norm,- }+ })}pub(crate) fn build_transformer_layers_and_norm(context: &B::Context,decoder_config: &DecoderConfig,- root_weight_loader: &ParameterTree<B::Context>,+ root_weight_loader: &ParameterTree<B>,transformer_subtree: &str,- ) -> (Box<[LayerExecutables<B>]>, RMSNorm<B>) {- let decoder_weight_loader =- root_weight_loader.subtree(transformer_subtree).expect("transformer subtree not found");+ output_norm_hadamard_factors: Option<Allocation<B>>,+ model_shape: &ModelShape,+ ) -> Result<(Box<[LayerExecutables<B>]>, RMSNorm<B>), DecoderError<B>> {+ let decoder_weight_loader = root_weight_loader.subtree(transformer_subtree)?;let tf = &decoder_config.transformer_config;- let norm_data_type: DataType = tf.layer_configs[0].mixer_config.activation_precision().into();- let rope = Rc::new(Rope::<B>::new(context, norm_data_type).expect("Failed to create Rope"));- let qk_unpack = Rc::new(QkUnpack::<B>::new(context, norm_data_type).expect("Failed to create QkUnpack"));+ let rope = Rc::new(Rope::<B>::new(context, model_shape).map_err(DecoderError::BackendError)?);+ let qk_unpack =+ Rc::new(QkUnpack::<B>::new(context, model_shape.data_type).map_err(DecoderError::BackendError)?);let layers = tf.layer_configs.iter().enumerate().map(|(layer_index, layer_config)| {- let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap();+ let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index))?;- LayerExecutables::new(context, tf, layer_config, layer_index, &layer_loader, &rope, &qk_unpack)+ LayerExecutables::new(+ context,+ tf,+ layer_config,+ layer_index,+ &layer_loader,+ &rope,+ &qk_unpack,+ model_shape.data_type,+ )+ .map_err(DecoderError::LayerError)})- .collect::<Vec<_>>();+ .collect::<Result<Vec<_>, DecoderError<B>>>()?;let norm_block = RMSNorm::new(context,- norm_data_type,+ model_shape.data_type,+ model_shape.model_dim(),tf.output_norm_config.clone(),- &decoder_weight_loader.subtree("output_norm").unwrap(),- None,+ &decoder_weight_loader.subtree("output_norm")?,+ output_norm_hadamard_factors,true,true,PostLayerScalar::None,- )- .expect("Failed to create output RMS norm kernel");+ )?;- (layers.into_boxed_slice(), norm_block)+ Ok((layers.into_boxed_slice(), norm_block))}fn run_layers(@@ -179,7 +208,6 @@ impl<B: Backend> Decoder<B> {for layer in self.layers.iter() {let rope_buffers = shared_buffers.rope_buffers_for_layer(layer.layer_index);- let attention_sinks = shared_buffers.attention_sinks(layer.layer_index);#[cfg(feature = "tracing")]let layer_trace = trace.as_deref_mut().map(|trace| &mut trace.layer_results[layer.layer_index]);@@ -192,7 +220,6 @@ impl<B: Backend> Decoder<B> {token_positions,token_parents,token_subtrie_ranges,- attention_sinks,rope_buffers,sampling_start,sampling_length,diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rsindex 004560627..497d78629 100644--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs@@ -10,7 +10,7 @@ use crate::{DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,},},- config::DeltaNetAttentionConfig,+ config::token_mixer::delta_net::DeltaNetConfig,encodable_block::linear::{Linear, LinearBlockError},forward_pass::delta_net_layer::DeltaNetLayer,parameters::{ParameterLoaderError, ParameterTree},@@ -23,13 +23,21 @@ pub enum DeltaNetMixerError<B: Backend> {#[error("Unsupported configuration: {0}")]UnsupportedConfiguration(String),#[error("Linear error: {0}")]- InnerLinearError(#[from] Box<LinearBlockError<B>>),+ InnerLinearError(#[from] LinearBlockError<B>),#[error("Parameter loader error: {0}")]ParameterLoaderError(#[from] ParameterLoaderError<B>),}pub(crate) struct DeltaNetMixer<B: Backend> {- config: DeltaNetAttentionConfig,+ kernel_size: usize,+ num_heads: usize,+ num_groups: usize,+ value_head_dim: usize,+ key_dim: usize,+ value_dim: usize,+ conv_dim: usize,+ total_proj_dim: usize,+ norm_epsilon: f32,in_projection: Box<dyn Linear<B>>,out_projection: Box<dyn Linear<B>>,// Decode kernels@@ -58,9 +66,10 @@ pub(crate) struct DeltaNetArguments<'a, B: Backend> {impl<B: Backend> DeltaNetMixer<B> {pub(crate) fn new(context: &B::Context,- config: DeltaNetAttentionConfig,+ config: &DeltaNetConfig,model_dim: usize,- decoder_layer_loader: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,+ data_type: DataType,) -> Result<(Self, Option<Allocation<B>>), DeltaNetMixerError<B>> {if config.kernel_size < 2 {return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(@@ -81,42 +90,45 @@ impl<B: Backend> DeltaNetMixer<B> {)));}- let data_type: DataType = config.in_proj_config.activation_precision().into();let has_bias = config.conv_config.has_biases;+ let key_dim = config.key_dim();+ let value_dim = config.value_dim();+ let conv_dim = config.conv_dim();+ let total_proj_dim = config.total_proj_dim();// Load weights- let mixer_tree = decoder_layer_loader.subtree("mixer")?;- let conv_tree = mixer_tree.subtree("conv")?;+ let conv_tree = parameter_tree.subtree("conv")?;let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(- &config.in_proj_config,model_dim,- [config.total_proj_dim()],+ [total_proj_dim],+ false,context,- &decoder_layer_loader.subtree("mixer.in_proj")?,- )- .map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;+ data_type,+ ¶meter_tree.subtree("in_proj")?,+ )?;let out_projection = <dyn Linear<B>>::new(- &config.out_proj_config,- config.value_dim(),+ value_dim,[model_dim],+ false,context,- &decoder_layer_loader.subtree("mixer.out_proj")?,- )- .map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;+ data_type,+ ¶meter_tree.subtree("out_proj")?,+ )?;- let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;+ let conv_weight =+ conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], data_type)?.read_allocation()?;let conv_bias = if has_bias {- Some(conv_tree.leaf("biases")?.read_allocation()?)+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], data_type)?.read_allocation()?)} else {None};- let a_log = mixer_tree.leaf("a_log")?.read_allocation()?;- let dt_bias = mixer_tree.leaf("dt_bias")?.read_allocation()?;- let norm_tree = mixer_tree.subtree("norm")?;- let norm_weight = norm_tree.leaf("scales")?.read_allocation()?;+ let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], data_type)?.read_allocation()?;+ let dt_bias = parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], data_type)?.read_allocation()?;+ let norm_weight =+ parameter_tree.leaf("norm.scales")?.validate(&[config.value_head_dim], data_type)?.read_allocation()?;// Create kernelslet conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, data_type, has_bias)@@ -139,7 +151,15 @@ impl<B: Backend> DeltaNetMixer<B> {Ok((Self {- config,+ kernel_size: config.kernel_size,+ num_heads: config.num_heads,+ num_groups: config.num_groups,+ value_head_dim: config.value_head_dim,+ key_dim,+ value_dim,+ conv_dim,+ total_proj_dim,+ norm_epsilon: config.norm_config.epsilon,in_projection,out_projection,conv_update,@@ -166,14 +186,14 @@ impl<B: Backend> DeltaNetMixer<B> {in_proj: &mut Allocation<B>,encoder: &mut Encoder<B>,) {- let kernel_size = self.config.kernel_size;+ let kernel_size = self.kernel_size;self.conv_update.encode(&self.conv_weight,self.conv_bias.as_ref(),in_proj,&mut layer.conv_state,kernel_size as u32,- self.config.conv_dim() as u32,+ self.conv_dim as u32,(kernel_size - 1) as u32,encoder,);@@ -186,10 +206,10 @@ impl<B: Backend> DeltaNetMixer<B> {encoder: &mut Encoder<B>,suffix_length: usize,) -> Result<(), B::Error> {- let kernel_size = self.config.kernel_size;+ let kernel_size = self.kernel_size;let state_stride = kernel_size - 1;- let conv_dim = self.config.conv_dim();- let total_proj_dim = self.config.total_proj_dim();+ let conv_dim = self.conv_dim;+ let total_proj_dim = self.total_proj_dim;let mut padded = encoder.allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.data_type))?;@@ -228,8 +248,7 @@ impl<B: Backend> DeltaNetMixer<B> {active_row_count: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let mut out =- encoder.allocate_scratch(size_for_shape(&[active_row_count, self.config.value_dim()], self.data_type))?;+ let mut out = encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.data_type))?;self.delta_net_update.encode(in_proj,&self.a_log,@@ -237,12 +256,12 @@ impl<B: Backend> DeltaNetMixer<B> {&self.norm_weight,&mut layer.ssm_state,&mut out,- self.config.num_heads as u32,- self.config.num_groups as u32,- self.config.value_head_dim as u32,- self.config.key_dim() as u32,- self.config.value_dim() as u32,- self.config.norm_config.epsilon,+ self.num_heads as u32,+ self.num_groups as u32,+ self.value_head_dim as u32,+ self.key_dim as u32,+ self.value_dim as u32,+ self.norm_epsilon,encoder,);Ok(out)@@ -255,15 +274,15 @@ impl<B: Backend> DeltaNetMixer<B> {suffix_length: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let num_dv_groups = ((self.config.value_head_dim + 7) / 8) as u32;+ let num_dv_groups = ((self.value_head_dim + 7) / 8) as u32;let mut prep_q_norm =- encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.key_dim()], DataType::F32))?;+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.key_dim], DataType::F32))?;let mut prep_k_norm =- encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.key_dim()], DataType::F32))?;+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.key_dim], DataType::F32))?;let mut prep_beta =- encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.num_heads], DataType::F32))?;let mut prep_decay =- encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.num_heads], DataType::F32))?;self.prefill_prep.encode(in_proj,@@ -273,16 +292,15 @@ impl<B: Backend> DeltaNetMixer<B> {&mut prep_k_norm,&mut prep_beta,&mut prep_decay,- self.config.num_heads as u32,- self.config.num_groups as u32,- self.config.key_dim() as u32,- self.config.value_dim() as u32,+ self.num_heads as u32,+ self.num_groups as u32,+ self.key_dim as u32,+ self.value_dim as u32,suffix_length as u32,encoder,);- let mut out =- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.value_dim()], self.data_type))?;+ let mut out = encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.data_type))?;self.delta_net_prefill.encode(&prep_q_norm,&prep_k_norm,@@ -291,11 +309,11 @@ impl<B: Backend> DeltaNetMixer<B> {in_proj,&mut layer.ssm_state,&mut out,- self.config.num_heads as u32,- self.config.num_groups as u32,- self.config.value_head_dim as u32,- self.config.key_dim() as u32,- self.config.value_dim() as u32,+ self.num_heads as u32,+ self.num_groups as u32,+ self.value_head_dim as u32,+ self.key_dim as u32,+ self.value_dim as u32,suffix_length as u32,num_dv_groups,encoder,@@ -305,12 +323,12 @@ impl<B: Backend> DeltaNetMixer<B> {&mut out,in_proj,&self.norm_weight,- self.config.num_heads as u32,- self.config.value_head_dim as u32,- self.config.value_dim() as u32,- self.config.conv_dim() as u32,- self.config.total_proj_dim() as u32,- self.config.norm_config.epsilon,+ self.num_heads as u32,+ self.value_head_dim as u32,+ self.value_dim as u32,+ self.conv_dim as u32,+ self.total_proj_dim as u32,+ self.norm_epsilon,suffix_length as u32,encoder,);diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 365eb3ff4..1f5a917e0 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -4,6 +4,7 @@ use thiserror::Error;use crate::{DataType,+ array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,gpu_types::{QuantizationMethod, QuantizationMode},@@ -12,7 +13,17 @@ use crate::{matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},},},- config::EmbeddingConfig,+ config::{+ embedding::AnyEmbeddingConfig,+ weight_matrix::{+ AnyWeightMatrixSpec, Layout,+ awq_spec::AWQSpec,+ full_precision_spec::FullPrecisionSpec,+ hybrid_spec::{HybridSpec, IncoherenceProcessingMode},+ mlx_spec::MLXSpec,+ },+ },+ forward_pass::model_shape::ModelShape,parameters::{ParameterLoaderError, ParameterTree},};@@ -27,6 +38,7 @@ pub enum EmbeddingError<B: Backend> {}struct ReadoutQuantConfig {+ method: QuantizationMethod,mode: QuantizationMode,group_size: u32,}@@ -40,7 +52,9 @@ enum TiedEmbeddingType<B: Backend> {Quantized {weights: Allocation<B>,scales: Allocation<B>,- biases: Allocation<B>,+ zero_points_or_biases: Allocation<B>,+ quantization_method: QuantizationMethod,+ output_hadamard_factors: Option<Allocation<B>>,lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,readout_config: ReadoutQuantConfig,@@ -55,7 +69,9 @@ enum UntiedEmbeddingLookupType<B: Backend> {Quantized {weights: Allocation<B>,scales: Allocation<B>,- biases: Allocation<B>,+ zero_points_or_biases: Allocation<B>,+ quantization_method: QuantizationMethod,+ output_hadamard_factors: Option<Allocation<B>>,lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,},}@@ -68,7 +84,7 @@ enum UntiedEmbeddingReadoutType<B: Backend> {Quantized {weights: Allocation<B>,scales: Allocation<B>,- biases: Allocation<B>,+ zero_points_or_biases: Allocation<B>,readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,readout_config: ReadoutQuantConfig,},@@ -87,387 +103,330 @@ enum EmbeddingTying<B: Backend> {pub struct Embedding<B: Backend> {tying: EmbeddingTying<B>,input_scale: f32,- activation_data_type: DataType,+ data_type: DataType,vocab_size: u32,model_dim: u32,}-struct UntiedWeightKeys {- lookup_weights: &'static str,- lookup_scales: &'static str,- lookup_biases: &'static str,- readout_weights: &'static str,- readout_scales: &'static str,- readout_biases: &'static str,-}--const SHARED_TREE_UNTIED_KEYS: UntiedWeightKeys = UntiedWeightKeys {- lookup_weights: "input_weights",- lookup_scales: "input_scales",- lookup_biases: "input_biases",- readout_weights: "output_weights",- readout_scales: "output_scales",- readout_biases: "output_biases",-};--#[cfg(metal_backend)]-const SPLIT_TREE_UNTIED_KEYS: UntiedWeightKeys = UntiedWeightKeys {- lookup_weights: "weights",- lookup_scales: "scales",- lookup_biases: "biases",- readout_weights: "weights",- readout_scales: "scales",- readout_biases: "biases",-};-impl<B: Backend> Embedding<B> {pub fn new(context: &B::Context,vocab_size: u32,model_dim: u32,- config: &EmbeddingConfig,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, EmbeddingError<B>> {- Self::new_with_weight_trees(+ config: &AnyEmbeddingConfig,+ parameter_tree: &ParameterTree<B>,+ model_shape: &ModelShape,+ ) -> Result<(Self, Option<Allocation<B>>), EmbeddingError<B>> {+ Self::new_with_lookup_and_readout_trees(context,vocab_size,model_dim,config,parameter_tree,parameter_tree,- &SHARED_TREE_UNTIED_KEYS,- )- }-- #[cfg(metal_backend)]- pub(crate) fn new_with_lookup_and_readout_trees(- context: &B::Context,- vocab_size: u32,- model_dim: u32,- config: &EmbeddingConfig,- lookup_tree: &ParameterTree<B::Context>,- readout_tree: &ParameterTree<B::Context>,- ) -> Result<Self, EmbeddingError<B>> {- Self::new_with_weight_trees(- context,- vocab_size,- model_dim,- config,- lookup_tree,- readout_tree,- &SPLIT_TREE_UNTIED_KEYS,+ model_shape,)}- fn new_with_weight_trees(+ pub fn new_with_lookup_and_readout_trees(context: &B::Context,vocab_size: u32,model_dim: u32,- config: &EmbeddingConfig,- lookup_tree: &ParameterTree<B::Context>,- readout_tree: &ParameterTree<B::Context>,- untied_keys: &UntiedWeightKeys,- ) -> Result<Self, EmbeddingError<B>> {- let common = config.common();-- let tying = match config {- EmbeddingConfig::Tied {- common: _,- precision,- } => {- let data_type = (*precision).into();-- let weights_leaf = lookup_tree.leaf("weights")?;- weights_leaf.validate_shape(&[vocab_size as usize, model_dim as usize], data_type)?;- let weights = weights_leaf.read_allocation()?;-- let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)- .map_err(EmbeddingError::BackendError)?;- let readout = RefCell::new(- <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)- .map_err(EmbeddingError::BackendError)?,- );-- EmbeddingTying::Tied {- ty: TiedEmbeddingType::FullPrecision {- weights,- lookup,- readout,+ config: &AnyEmbeddingConfig,+ lookup_tree: &ParameterTree<B>,+ readout_tree: &ParameterTree<B>,+ model_shape: &ModelShape,+ ) -> Result<(Self, Option<Allocation<B>>), EmbeddingError<B>> {+ let (tying, readout_input_hadamard_factors) = match config {+ AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {+ let embedding_tree = lookup_tree.subtree("embedding")?;+ let embedding_spec = embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;++ let (ty, readout_input_hadamard_factors) = match embedding_spec {+ AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {+ layout: Layout::InputOutput,+ ..+ }) => {+ let weights = embedding_tree+ .leaf("weights")?+ .validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?+ .read_allocation()?;++ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(+ context,+ model_shape.data_type,+ )+ .map_err(EmbeddingError::BackendError)?;+ let readout = RefCell::new(+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, model_shape.data_type)+ .map_err(EmbeddingError::BackendError)?,+ );++ (+ TiedEmbeddingType::FullPrecision {+ weights,+ lookup,+ readout,+ },+ None,+ )},- }- },- EmbeddingConfig::Untied {- common: _,- precision,- } => {- let data_type = (*precision).into();-- let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;- input_weights_leaf.validate_shape(&[vocab_size as usize, model_dim as usize], data_type)?;- let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;- output_weights_leaf.validate_shape(&[vocab_size as usize, model_dim as usize], data_type)?;-- let input_weights = input_weights_leaf.read_allocation()?;- let output_weights = output_weights_leaf.read_allocation()?;-- let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)- .map_err(EmbeddingError::BackendError)?;- let readout = RefCell::new(- <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)- .map_err(EmbeddingError::BackendError)?,- );-- EmbeddingTying::Untied {- input_ty: UntiedEmbeddingLookupType::FullPrecision {- weights: input_weights,- lookup,+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ let (embedding_quantization_mode, group_size, quantization_method) =+ input_quantization_from_spec(spec)?;+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(+ &embedding_tree,+ vocab_size as usize,+ model_dim as usize,+ model_shape.data_type,+ embedding_quantization_mode,+ quantization_method,+ group_size,+ )?;++ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(+ context,+ model_shape.data_type,+ group_size as u32,+ embedding_quantization_mode,+ quantization_method,+ false,+ )+ .map_err(EmbeddingError::BackendError)?;+ let (readout, readout_config) = quantized_readout(+ context,+ model_shape.data_type,+ embedding_quantization_mode,+ quantization_method,+ group_size,+ )?;++ (+ TiedEmbeddingType::Quantized {+ weights,+ scales,+ zero_points_or_biases,+ quantization_method,+ output_hadamard_factors: None,+ lookup,+ readout,+ readout_config,+ },+ None,+ )},- output_ty: UntiedEmbeddingReadoutType::FullPrecision {- weights: output_weights,- readout,+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {+ quantization_spec,+ adapter_spec: None,+ incoherence_block_size: Some(32),+ incoherence_processing_mode: IncoherenceProcessingMode::Output,+ ..+ }) => {+ let (embedding_quantization_mode, group_size, quantization_method) =+ input_quantization_from_spec(*quantization_spec)?;+ let quantized_tree = embedding_tree.subtree("quantized")?;+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(+ &quantized_tree,+ vocab_size as usize,+ model_dim as usize,+ model_shape.data_type,+ embedding_quantization_mode,+ quantization_method,+ group_size,+ )?;++ let incoherence_signs_tree = embedding_tree.subtree("incoherence_signs")?;+ let output_hadamard_factors = Some(+ incoherence_signs_tree+ .leaf("output_signs")?+ .validate(&[model_dim as usize], DataType::I32)?+ .read_allocation()?,+ );+ let readout_input_hadamard_factors = Some(+ incoherence_signs_tree+ .leaf("output_signs")?+ .validate(&[model_dim as usize], DataType::I32)?+ .read_allocation()?,+ );+ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(+ context,+ model_shape.data_type,+ group_size as u32,+ embedding_quantization_mode,+ quantization_method,+ true,+ )+ .map_err(EmbeddingError::BackendError)?;+ let (readout, readout_config) = quantized_readout(+ context,+ model_shape.data_type,+ embedding_quantization_mode,+ quantization_method,+ group_size,+ )?;++ (+ TiedEmbeddingType::Quantized {+ weights,+ scales,+ zero_points_or_biases,+ quantization_method,+ output_hadamard_factors,+ lookup,+ readout,+ readout_config,+ },+ readout_input_hadamard_factors,+ )},- }- },- EmbeddingConfig::ScaleBiasQuantizedTied {- common: _,- group_size,- embedding_quantization_mode,- activation_quantization_mode,- activation_precision,- } => {- let data_type = (*activation_precision).into();-- let packing_divisor = embedding_quantization_mode.packing_divisor();- let storage_data_type = embedding_quantization_mode.storage_type();-- let num_groups = (model_dim as usize).div_ceil(*group_size);-- let weights_leaf = lookup_tree.leaf("weights")?;- weights_leaf- .validate_shape(&[vocab_size as usize, model_dim as usize / packing_divisor], storage_data_type)?;- let scales_leaf = lookup_tree.leaf("scales")?;- scales_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;- let biases_leaf = lookup_tree.leaf("biases")?;- biases_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;-- let weights = weights_leaf.read_allocation()?;- let scales = scales_leaf.read_allocation()?;- let biases = biases_leaf.read_allocation()?;-- let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(- context,- data_type,- *group_size as u32,- (*embedding_quantization_mode).into(),- )- .map_err(EmbeddingError::BackendError)?;- let readout_config = ReadoutQuantConfig {- mode: *embedding_quantization_mode,- group_size: *group_size as u32,+ spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),};- let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)- .map_err(EmbeddingError::BackendError)?;- readout_kernel- .preheat_quant_combo(- context,- MatmulQuantCombo {- method: QuantizationMethod::ScaleBias,- mode: *embedding_quantization_mode,- group_size: *group_size as u32,- },- )- .map_err(EmbeddingError::BackendError)?;- let readout = RefCell::new(readout_kernel);- if let Some(activation_quantization_mode) = activation_quantization_mode {- return Err(EmbeddingError::UnsupportedConfiguration(format!(- "activation_quantization_mode={activation_quantization_mode:?}"- )));- }-- EmbeddingTying::Tied {- ty: TiedEmbeddingType::Quantized {- weights,- scales,- biases,- lookup,- readout,- readout_config,+ (+ EmbeddingTying::Tied {+ ty,},- }- },- EmbeddingConfig::ScaleBiasQuantizedUntied {- common: _,- group_size,- embedding_quantization_mode,- activation_quantization_mode,- activation_precision,- } => {- let data_type = (*activation_precision).into();-- let packing_divisor = embedding_quantization_mode.packing_divisor();- let storage_data_type = embedding_quantization_mode.storage_type();-- let num_groups = (model_dim as usize).div_ceil(*group_size);-- let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;- input_weights_leaf- .validate_shape(&[vocab_size as usize, model_dim as usize / packing_divisor], storage_data_type)?;- let input_scales_leaf = lookup_tree.leaf(untied_keys.lookup_scales)?;- input_scales_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;- let input_biases_leaf = lookup_tree.leaf(untied_keys.lookup_biases)?;- input_biases_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;-- let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;- output_weights_leaf- .validate_shape(&[vocab_size as usize, model_dim as usize / packing_divisor], storage_data_type)?;- let output_scales_leaf = readout_tree.leaf(untied_keys.readout_scales)?;- output_scales_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;- let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;- output_biases_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;-- let input_weights = input_weights_leaf.read_allocation()?;- let input_scales = input_scales_leaf.read_allocation()?;- let input_biases = input_biases_leaf.read_allocation()?;-- let output_weights = output_weights_leaf.read_allocation()?;- let output_scales = output_scales_leaf.read_allocation()?;- let output_biases = output_biases_leaf.read_allocation()?;-- let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(- context,- data_type,- *group_size as u32,- (*embedding_quantization_mode).into(),+ readout_input_hadamard_factors,)- .map_err(EmbeddingError::BackendError)?;- let readout_config = ReadoutQuantConfig {- mode: *embedding_quantization_mode,- group_size: *group_size as u32,- };- let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)- .map_err(EmbeddingError::BackendError)?;- readout_kernel- .preheat_quant_combo(- context,- MatmulQuantCombo {- method: QuantizationMethod::ScaleBias,- mode: *embedding_quantization_mode,- group_size: *group_size as u32,- },- )- .map_err(EmbeddingError::BackendError)?;- let readout = RefCell::new(readout_kernel);-- if let Some(activation_quantization_mode) = activation_quantization_mode {- return Err(EmbeddingError::UnsupportedConfiguration(format!(- "activation_quantization_mode={activation_quantization_mode:?}"- )));- }-- EmbeddingTying::Untied {- input_ty: UntiedEmbeddingLookupType::Quantized {- weights: input_weights,- scales: input_scales,- biases: input_biases,- lookup,+ },+ AnyEmbeddingConfig::UntiedEmbeddingConfig(_) => {+ let input_embedding_tree = lookup_tree.subtree("input_embedding")?;+ let input_embedding_spec = input_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;++ let input_ty = match input_embedding_spec {+ AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {+ layout: Layout::InputOutput,+ ..+ }) => {+ let weights = input_embedding_tree+ .leaf("weights")?+ .validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?+ .read_allocation()?;++ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(+ context,+ model_shape.data_type,+ )+ .map_err(EmbeddingError::BackendError)?;++ UntiedEmbeddingLookupType::FullPrecision {+ weights,+ lookup,+ }},- output_ty: UntiedEmbeddingReadoutType::Quantized {- weights: output_weights,- scales: output_scales,- biases: output_biases,- readout,- readout_config,+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ let (embedding_quantization_mode, group_size, quantization_method) =+ input_quantization_from_spec(spec)?;+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(+ &input_embedding_tree,+ vocab_size as usize,+ model_dim as usize,+ model_shape.data_type,+ embedding_quantization_mode,+ quantization_method,+ group_size,+ )?;++ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(+ context,+ model_shape.data_type,+ group_size as u32,+ embedding_quantization_mode,+ quantization_method,+ false,+ )+ .map_err(EmbeddingError::BackendError)?;++ UntiedEmbeddingLookupType::Quantized {+ weights,+ scales,+ zero_points_or_biases,+ quantization_method,+ output_hadamard_factors: None,+ lookup,+ }},- }- },- EmbeddingConfig::ScaleBiasSemiQuantizedUntied {- common: _,- group_size,- embedding_quantization_mode,- activation_quantization_mode,- activation_precision,- } => {- let data_type = (*activation_precision).into();-- let packing_divisor = embedding_quantization_mode.packing_divisor();- let storage_data_type = embedding_quantization_mode.storage_type();-- let num_groups = (model_dim as usize).div_ceil(*group_size);-- let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;- input_weights_leaf.validate_shape(&[vocab_size as usize, model_dim as usize], data_type)?;-- let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;- output_weights_leaf- .validate_shape(&[vocab_size as usize, model_dim as usize / packing_divisor], storage_data_type)?;- let output_scales_leaf = readout_tree.leaf(untied_keys.readout_scales)?;- output_scales_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;- let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;- output_biases_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;-- let input_weights = input_weights_leaf.read_allocation()?;-- let output_weights = output_weights_leaf.read_allocation()?;- let output_scales = output_scales_leaf.read_allocation()?;- let output_biases = output_biases_leaf.read_allocation()?;-- let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)- .map_err(EmbeddingError::BackendError)?;- let readout_config = ReadoutQuantConfig {- mode: *embedding_quantization_mode,- group_size: *group_size as u32,+ spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),};- let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)- .map_err(EmbeddingError::BackendError)?;- readout_kernel- .preheat_quant_combo(- context,- MatmulQuantCombo {- method: QuantizationMethod::ScaleBias,- mode: *embedding_quantization_mode,- group_size: *group_size as u32,- },- )- .map_err(EmbeddingError::BackendError)?;- let readout = RefCell::new(readout_kernel);-- if let Some(activation_quantization_mode) = activation_quantization_mode {- return Err(EmbeddingError::UnsupportedConfiguration(format!(- "activation_quantization_mode={activation_quantization_mode:?}"- )));- }- EmbeddingTying::Untied {- input_ty: UntiedEmbeddingLookupType::FullPrecision {- weights: input_weights,- lookup,+ let output_embedding_tree = readout_tree.subtree("output_embedding")?;+ let output_embedding_spec = output_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;++ let output_ty = match output_embedding_spec {+ AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {+ layout: Layout::OutputInput,+ ..+ }) => {+ let weights = output_embedding_tree+ .leaf("weights")?+ .validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?+ .read_allocation()?;+ let readout = RefCell::new(+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, model_shape.data_type)+ .map_err(EmbeddingError::BackendError)?,+ );++ UntiedEmbeddingReadoutType::FullPrecision {+ weights,+ readout,+ }},- output_ty: UntiedEmbeddingReadoutType::Quantized {- weights: output_weights,- scales: output_scales,- biases: output_biases,- readout,- readout_config,+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ let (embedding_quantization_mode, group_size, quantization_method) =+ output_quantization_from_spec(spec)?;+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(+ &output_embedding_tree,+ vocab_size as usize,+ model_dim as usize,+ model_shape.data_type,+ embedding_quantization_mode,+ quantization_method,+ group_size,+ )?;+ let (readout, readout_config) = quantized_readout(+ context,+ model_shape.data_type,+ embedding_quantization_mode,+ quantization_method,+ group_size,+ )?;++ UntiedEmbeddingReadoutType::Quantized {+ weights,+ scales,+ zero_points_or_biases,+ readout,+ readout_config,+ }},- }+ spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),+ };++ (+ EmbeddingTying::Untied {+ input_ty,+ output_ty,+ },+ None,+ )},};- let input_scale = common.input_scale.unwrap_or(1.0);- let activation_data_type: DataType = config.activation_precision().into();-- if let Some(logit_soft_cap) = common.logit_soft_cap {+ let input_scale = config.input_scale().unwrap_or(1.0);+ if let Some(logit_soft_cap) = config.logit_soft_cap() {return Err(EmbeddingError::UnsupportedConfiguration(format!("logit_soft_cap={logit_soft_cap:?}")));}- Ok(Self {- tying,- input_scale,- activation_data_type,- vocab_size,- model_dim,- })+ Ok((+ Self {+ tying,+ input_scale,+ data_type: model_shape.data_type,+ vocab_size,+ model_dim,+ },+ readout_input_hadamard_factors,+ ))}pub fn encode_lookup(@@ -477,10 +436,7 @@ impl<B: Backend> Embedding<B> {encoder: &mut Encoder<B>,) -> Result<Allocation<B>, EmbeddingError<B>> {let mut output = encoder- .allocate_scratch(crate::array::size_for_shape(- &[batch_dim, self.model_dim as usize],- self.activation_data_type,- ))+ .allocate_scratch(size_for_shape(&[batch_dim, self.model_dim as usize], self.data_type)).map_err(EmbeddingError::BackendError)?;let batch_dim = batch_dim as u32;@@ -515,7 +471,9 @@ impl<B: Backend> Embedding<B> {TiedEmbeddingType::Quantized {weights,scales,- biases,+ zero_points_or_biases,+ quantization_method,+ output_hadamard_factors,lookup,readout: _,readout_config: _,@@ -526,17 +484,25 @@ impl<B: Backend> Embedding<B> {UntiedEmbeddingLookupType::Quantized {weights,scales,- biases,+ zero_points_or_biases,+ quantization_method,+ output_hadamard_factors,lookup,},output_ty: _,} => {+ let (zero_points, biases) = match quantization_method {+ QuantizationMethod::ScaleZeroPoint => (Some(zero_points_or_biases), None),+ QuantizationMethod::ScaleBias => (None, Some(zero_points_or_biases)),+ };lookup.encode(token_ids,weights,scales,+ zero_points,biases,&mut output,+ output_hadamard_factors.as_ref(),batch_dim,self.vocab_size,self.model_dim,@@ -557,10 +523,7 @@ impl<B: Backend> Embedding<B> {) -> Result<Allocation<B>, EmbeddingError<B>> {assert!(batch_dim > 0, "Embedding readout requires at least one row");let mut output_allocation = encoder- .allocate_scratch(crate::array::size_for_shape(- &[batch_dim, self.vocab_size as usize],- self.activation_data_type,- ))+ .allocate_scratch(size_for_shape(&[batch_dim, self.vocab_size as usize], self.data_type)).map_err(EmbeddingError::BackendError)?;match &self.tying {@@ -580,8 +543,6 @@ impl<B: Backend> Embedding<B> {readout,},} => {- let input_dim = self.model_dim as usize;- let output_dim = self.vocab_size as usize;readout.borrow_mut().encode(@@ -597,8 +558,8 @@ impl<B: Backend> Embedding<B> {d: &mut output_allocation,d_transform: MatmulDOps::none(),m: batch_dim as u32,- n: output_dim as u32,- k: input_dim as u32,+ n: self.vocab_size,+ k: self.model_dim,},encoder,)@@ -609,7 +570,9 @@ impl<B: Backend> Embedding<B> {TiedEmbeddingType::Quantized {weights,scales,- biases,+ zero_points_or_biases,+ quantization_method: _,+ output_hadamard_factors: _,lookup: _,readout,readout_config,@@ -621,17 +584,26 @@ impl<B: Backend> Embedding<B> {UntiedEmbeddingReadoutType::Quantized {weights,scales,- biases,+ zero_points_or_biases,readout,readout_config,},} => {- let b_variant: MatmulB<'_, B> = MatmulB::ScaleBiasDequant {- b: weights,- scales,- biases,- mode: readout_config.mode,- group_size: readout_config.group_size,+ let b: MatmulB<'_, B> = match readout_config.method {+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {+ b: weights,+ scales,+ biases: zero_points_or_biases,+ mode: readout_config.mode,+ group_size: readout_config.group_size,+ },+ QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {+ b: weights,+ scales,+ zero_points: zero_points_or_biases,+ mode: readout_config.mode,+ group_size: readout_config.group_size,+ },};readout.borrow_mut()@@ -639,7 +611,7 @@ impl<B: Backend> Embedding<B> {MatmulArguments {a: input_allocation,a_offset: 0,- b: b_variant,+ b,b_offset: 0,b_leading_dimension: None,b_transpose: true,@@ -658,3 +630,124 @@ impl<B: Backend> Embedding<B> {Ok(output_allocation)}}++fn input_quantization_from_spec<B: Backend>(+ spec: AnyWeightMatrixSpec+) -> Result<(QuantizationMode, usize, QuantizationMethod), EmbeddingError<B>> {+ match spec {+ AnyWeightMatrixSpec::MLXSpec(MLXSpec {+ bits,+ group_size,+ layout: Layout::InputOutput,+ ..+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleBias),+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ bits,+ group_size,+ is_symmetric: false,+ layout: Layout::InputOutput,+ ..+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),+ spec => Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),+ }+}++fn output_quantization_from_spec<B: Backend>(+ spec: AnyWeightMatrixSpec+) -> Result<(QuantizationMode, usize, QuantizationMethod), EmbeddingError<B>> {+ match spec {+ AnyWeightMatrixSpec::MLXSpec(MLXSpec {+ bits,+ group_size,+ layout: Layout::OutputInput,+ ..+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleBias),+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ bits,+ group_size,+ is_symmetric: false,+ layout: Layout::OutputInput,+ ..+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),+ spec => Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),+ }+}++fn quantization_mode<B: Backend>(+ bits: u32,+ group_size: usize,+ method: QuantizationMethod,+) -> Result<(QuantizationMode, usize, QuantizationMethod), EmbeddingError<B>> {+ let mode = match bits {+ 4 => QuantizationMode::U4,+ 8 => QuantizationMode::U8,+ _ => {+ return Err(EmbeddingError::UnsupportedConfiguration(format!(+ "{method:?} bits={bits}, group_size={group_size}"+ )));+ },+ };+ Ok((mode, group_size, method))+}++fn load_quantized_embedding_parts<B: Backend>(+ tree: &ParameterTree<B>,+ vocab_size: usize,+ model_dim: usize,+ data_type: DataType,+ quantization_mode: QuantizationMode,+ quantization_method: QuantizationMethod,+ group_size: usize,+) -> Result<(Allocation<B>, Allocation<B>, Allocation<B>), EmbeddingError<B>> {+ let packing_divisor = quantization_mode.packing_divisor();+ let storage_data_type = quantization_mode.storage_type();+ let num_groups = model_dim.div_ceil(group_size);++ let weights = tree+ .leaf("weights")?+ .validate(&[vocab_size, model_dim / packing_divisor], storage_data_type)?+ .read_allocation()?;+ let scales = tree.leaf("scales")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?;+ let zero_points_or_biases = match quantization_method {+ QuantizationMethod::ScaleBias => {+ tree.leaf("biases")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?+ },+ QuantizationMethod::ScaleZeroPoint => {+ let expected_zero_points_entries = num_groups.div_ceil(packing_divisor);+ tree.leaf("zero_points")?+ .validate(&[vocab_size, expected_zero_points_entries], storage_data_type)?+ .read_allocation()?+ },+ };++ Ok((weights, scales, zero_points_or_biases))+}++fn quantized_readout<B: Backend>(+ context: &B::Context,+ data_type: DataType,+ mode: QuantizationMode,+ method: QuantizationMethod,+ group_size: usize,+) -> Result<(RefCell<<B::Kernels as ManualKernels>::MatmulKernel>, ReadoutQuantConfig), EmbeddingError<B>> {+ let mut readout =+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;+ readout+ .preheat_quant_combo(+ context,+ MatmulQuantCombo {+ method,+ mode,+ group_size: group_size as u32,+ },+ )+ .map_err(EmbeddingError::BackendError)?;+ Ok((+ RefCell::new(readout),+ ReadoutQuantConfig {+ method,+ mode,+ group_size: group_size as u32,+ },+ ))+}diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rsindex b03a76811..2d3bfe7b6 100644--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs@@ -2,25 +2,59 @@use std::rc::Rc;-use half::{bf16, f16};+use thiserror::Error;-use super::MixerExecutables;-#[cfg(feature = "tracing")]-use crate::backends::common::{Kernels, kernel::TensorAddBiasKernel};-#[cfg(feature = "tracing")]-use crate::forward_pass::traces::LayerActivationTrace;use crate::{DataType,- backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},- config::{MixerConfig, TransformerConfig, TransformerLayerConfig},+ backends::common::{Allocation, Backend, Encoder},+ config::{+ token_mixer::AnyTokenMixerConfig, transformer::TransformerConfig, transformer_layer::TransformerLayerConfig,+ },encodable_block::{- Attention, AttentionArguments, DeltaNetArguments, DeltaNetMixer, Linear, MambaArguments, MambaMixer, Mlp,- PostLayerScalar, QKVNorm, QkUnpack, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,+ Attention, AttentionError, DeltaNetArguments, DeltaNetMixer, DeltaNetMixerError, MambaArguments, MambaMixer,+ MambaMixerError, Mlp, MlpBlockError, PostLayerScalar, QkUnpack, RMSNorm, RMSNormError, Rope,+ ShortConvArguments, ShortConvMixer, ShortConvMixerError, layer::MixerExecutables,},forward_pass::{cache_layers::LayerCacheAccess, state::RopeBuffers},- parameters::ParameterTree,+ parameters::{ParameterLoaderError, ParameterTree},+};+#[cfg(feature = "tracing")]+use crate::{+ backends::common::{Kernels, kernel::TensorAddBiasKernel},+ forward_pass::traces::LayerActivationTrace,};+#[derive(Debug, Error)]+pub enum LayerExecutablesError<B: Backend> {+ #[cfg(feature = "tracing")]+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[from] ParameterLoaderError<B>),+ #[error("Attention error: {0}")]+ AttentionError(#[from] AttentionError<B>),+ #[error("Mamba mixer error: {0}")]+ MambaMixerError(#[from] MambaMixerError<B>),+ #[error("ShortConv mixer error: {0}")]+ ShortConvMixerError(#[from] ShortConvMixerError<B>),+ #[error("DeltaNet mixer error: {0}")]+ DeltaNetMixerError(#[from] DeltaNetMixerError<B>),+ #[error("MLP error: {0}")]+ MlpBlockError(#[from] MlpBlockError<B>),+ #[error("RMSNorm error: {0}")]+ RMSNormError(#[from] RMSNormError<B>),+ #[error("layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]+ PostLayerScalarWithoutPostMlpNorm {+ layer_index: usize,+ },+ #[error("Unsupported post_layer_scalar data type: {data_type:?}")]+ UnsupportedPostLayerScalarDataType {+ data_type: DataType,+ },+ #[error("decoder layers require pre_mixer_norm_config")]+ MissingPreMixerNormConfig,+}+/// A single decoder layer with all its components.pub struct LayerExecutables<B: Backend> {pub layer_index: usize,@@ -43,25 +77,27 @@ impl<B: Backend> LayerExecutables<B> {transformer_config: &TransformerConfig,layer_config: &TransformerLayerConfig,layer_index: usize,- decoder_layer_loader: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,rope: &Rc<Rope<B>>,qk_unpack: &Rc<QkUnpack<B>>,- ) -> Self {- let intermediate_data_type: DataType = layer_config.mixer_config.activation_precision().into();-+ data_type: DataType,+ ) -> Result<Self, LayerExecutablesError<B>> {let (residual_sum_scalar, output_scalar) = if layer_config.has_post_layer_scalar {- assert!(- layer_config.post_mlp_norm_config.is_some(),- "layer {layer_index} sets post_layer_scalar but has no post_mlp_norm"- );- let leaf = decoder_layer_loader.leaf("post_layer_scalar").expect("Failed to read post_layer_scalar weight");- let scalar = match leaf.data_type() {- DataType::BF16 => {- leaf.read_slice::<bf16>().expect("Failed to read post_layer_scalar weight")[0].to_f32()+ if layer_config.post_mlp_norm_config.is_none() {+ return Err(LayerExecutablesError::PostLayerScalarWithoutPostMlpNorm {+ layer_index,+ });+ }+ let leaf = parameter_tree.leaf("post_layer_scalar")?;+ let scalar = match data_type {+ DataType::F32 => leaf.validate(&[1], data_type)?.read_slice::<f32>()?[0],+ DataType::F16 => leaf.validate(&[1], data_type)?.read_slice::<half::f16>()?[0].to_f32(),+ DataType::BF16 => leaf.validate(&[1], data_type)?.read_slice::<half::bf16>()?[0].to_f32(),+ data_type => {+ return Err(LayerExecutablesError::UnsupportedPostLayerScalarDataType {+ data_type,+ });},- DataType::F16 => leaf.read_slice::<f16>().expect("Failed to read post_layer_scalar weight")[0].to_f32(),- DataType::F32 => leaf.read_slice::<f32>().expect("Failed to read post_layer_scalar weight")[0],- other => panic!("post_layer_scalar must be a float dtype, got {other:?}"),};(PostLayerScalar::ScaleResidualSum(scalar), PostLayerScalar::ScaleOutput(scalar))} else {@@ -69,128 +105,34 @@ impl<B: Backend> LayerExecutables<B> {};#[cfg(feature = "tracing")]- let tensor_add = TensorAddBiasKernel::new(context, intermediate_data_type, false)- .expect("Failed to create TensorAddBiasKernel kernel"); // TODO: this function return Result-- let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {- MixerConfig::Attention(attention_config) => {- let q_dim = attention_config.num_heads * attention_config.head_dim;- let kv_dim = attention_config.num_groups * attention_config.head_dim;-- let (qkv_projection, input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(- &attention_config.qkv_projection_config,- transformer_config.model_dim,- [q_dim, kv_dim, kv_dim],- context,- &decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),- )- .expect("Failed to create qkv projection");+ let tensor_add =+ TensorAddBiasKernel::new(context, data_type, false).map_err(LayerExecutablesError::BackendError)?;- let gate_projection = attention_config.gate_projection_config.as_ref().map(|gate_config| {- let gate_tree = decoder_layer_loader.subtree("mixer.gate_projection").unwrap();- match (input_hadamard_factors.is_some(), gate_config) {- (- true,- crate::config::LinearConfig::RHTLinearWrapper {- inner_config,- ..- },- ) => {- let output_factors = gate_tree- .leaf("output_factors")- .expect("Failed to get gate projection output_factors")- .read_allocation()- .expect("Failed to read gate projection output_factors");- let inner_tree = gate_tree- .subtree("inner_linear")- .expect("Failed to get gate projection inner_linear subtree");- <dyn Linear<B>>::new_with_output_hadamard(- context,- inner_config,- &inner_tree,- output_factors,- transformer_config.model_dim,- q_dim,- )- },- (- false,- crate::config::LinearConfig::RHTLinearWrapper {- ..- },- )- | (true, _) => {- panic!("attention qkv/gate projections must share input hadamard")- },- (false, _) => <dyn Linear<B>>::new(- gate_config,- transformer_config.model_dim,- [q_dim],- context,- &gate_tree,- ),- }- .expect("Failed to create gate projection")- });-- let value_norm_config = attention_config.value_norm_config();- let qkv_norm = if attention_config.query_norm_config.is_some()- || attention_config.key_norm_config.is_some()- || value_norm_config.is_some()- {- match QKVNorm::new(- context,- intermediate_data_type,- attention_config.query_norm_config.clone(),- attention_config.key_norm_config.clone(),- value_norm_config,- &decoder_layer_loader.subtree("mixer").unwrap(),- attention_config.num_heads,- attention_config.num_groups,- attention_config.head_dim,- ) {- Ok(qkv_norm) => Some(qkv_norm),- Err(error) => {- panic!("Failed to create QKV norm kernel for layer {}: {:?}", layer_index, error)- },- }- } else {- None- };+ let mixer_subtree = parameter_tree.subtree("mixer")?;- let out_projection = <dyn Linear<B>>::new(- &attention_config.out_projection_config,- q_dim,- [transformer_config.model_dim],+ let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {+ AnyTokenMixerConfig::AttentionConfig(attention_config) => {+ let (attention, input_hadamard_factors) = Attention::new(context,- &decoder_layer_loader.subtree("mixer.out_projection").unwrap(),- )- .expect("Failed to create out projection");-- let attention =- Attention::new(context, intermediate_data_type, attention_config, gate_projection.is_some())- .expect("Failed to create AttentionWrapper kernel");+ transformer_config.model_dim,+ data_type,+ attention_config,+ &mixer_subtree,+ rope.clone(),+ qk_unpack.clone(),+ attention_config.gate_projection_config.is_none(),+ )?;(MixerExecutables::Attention {- qkv_projection,- gate_projection,- qkv_norm,- rope: rope.clone(),- qk_unpack: qk_unpack.clone(),attention,- out_projection,- num_heads: attention_config.num_heads,- num_groups: attention_config.num_groups,- head_dim: attention_config.head_dim,},input_hadamard_factors,)},- MixerConfig::Mamba(mamba_config) => {+ AnyTokenMixerConfig::Mamba2Config(mamba_config) => {let (mixer, input_hadamard_factors) =- MambaMixer::new(context, mamba_config.clone(), transformer_config.model_dim, decoder_layer_loader)- .expect("Failed to create Mamba mixer");+ MambaMixer::new(context, mamba_config, transformer_config.model_dim, &mixer_subtree, data_type)?;(MixerExecutables::StateSpace {mixer,@@ -198,14 +140,14 @@ impl<B: Backend> LayerExecutables<B> {input_hadamard_factors,)},- MixerConfig::ShortConv(short_conv_config) => {+ AnyTokenMixerConfig::ShortConvConfig(short_conv_config) => {let (mixer, input_hadamard_factors) = ShortConvMixer::new(context,- short_conv_config.clone(),+ short_conv_config,transformer_config.model_dim,- decoder_layer_loader,- )- .expect("Failed to create ShortConv mixer");+ &mixer_subtree,+ data_type,+ )?;(MixerExecutables::ShortConv {mixer,@@ -213,14 +155,14 @@ impl<B: Backend> LayerExecutables<B> {input_hadamard_factors,)},- MixerConfig::DeltaNet(delta_net_config) => {+ AnyTokenMixerConfig::DeltaNetConfig(delta_net_config) => {let (mixer, input_hadamard_factors) = DeltaNetMixer::new(context,- delta_net_config.clone(),+ delta_net_config,transformer_config.model_dim,- decoder_layer_loader,- )- .expect("Failed to create DeltaNet mixer");+ &mixer_subtree,+ data_type,+ )?;(MixerExecutables::DeltaNet {mixer,@@ -232,30 +174,28 @@ impl<B: Backend> LayerExecutables<B> {let pre_mixer_norm = RMSNorm::new(context,- intermediate_data_type,- layer_config.pre_mixer_norm_config.clone().expect("decoder layers require pre_mixer_norm_config"),- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),+ data_type,+ transformer_config.model_dim,+ layer_config.pre_mixer_norm_config.clone().ok_or(LayerExecutablesError::MissingPreMixerNormConfig)?,+ ¶meter_tree.subtree("pre_mixer_norm")?,mixer_hadamard_factors,true,layer_index > 0,PostLayerScalar::None,- )- .expect("Failed to create RMS norm kernel");+ )?;let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {- Some(- RMSNorm::new(- context,- intermediate_data_type,- norm_config.clone(),- &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),- None,- false,- false,- PostLayerScalar::None,- )- .expect("Failed to create RMS norm kernel"),- )+ Some(RMSNorm::new(+ context,+ data_type,+ transformer_config.model_dim,+ norm_config.clone(),+ ¶meter_tree.subtree("post_mixer_norm")?,+ None,+ false,+ false,+ PostLayerScalar::None,+ )?)} else {None};@@ -265,41 +205,39 @@ impl<B: Backend> LayerExecutables<B> {transformer_config.model_dim,layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),context,- &decoder_layer_loader.subtree("mlp").unwrap(),- )- .expect("Failed to create mlp block");+ ¶meter_tree.subtree("mlp")?,+ data_type,+ )?;let pre_mlp_norm = RMSNorm::new(context,- intermediate_data_type,+ data_type,+ transformer_config.model_dim,layer_config.pre_mlp_norm_config.clone(),- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),+ ¶meter_tree.subtree("pre_mlp_norm")?,mlp_input_hadamard_factors,true,true,residual_sum_scalar,- )- .expect("Failed to create RMS norm kernel");+ )?;let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {- Some(- RMSNorm::new(- context,- intermediate_data_type,- norm_config.clone(),- &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),- None,- false,- false,- output_scalar,- )- .expect("Failed to create RMS norm kernel"),- )+ Some(RMSNorm::new(+ context,+ data_type,+ transformer_config.model_dim,+ norm_config.clone(),+ ¶meter_tree.subtree("post_mlp_norm")?,+ None,+ false,+ false,+ output_scalar,+ )?)} else {None};- Self {+ Ok(Self {layer_index,#[cfg(feature = "tracing")]tensor_add,@@ -311,7 +249,7 @@ impl<B: Backend> LayerExecutables<B> {post_mlp_norm,#[cfg(feature = "tracing")]model_dim: transformer_config.model_dim,- }+ })}pub fn encode(@@ -326,7 +264,6 @@ impl<B: Backend> LayerExecutables<B> {token_positions,token_parents,token_subtrie_ranges,- attention_sinks,rope_buffers,sampling_start,sampling_length,@@ -346,67 +283,16 @@ impl<B: Backend> LayerExecutables<B> {hidden = match &self.mixer {MixerExecutables::Attention {- qkv_projection,- gate_projection,- qkv_norm,- rope,- qk_unpack,attention,- out_projection,- num_heads,- num_groups,- head_dim,- } => {- let gate_input = if gate_projection.is_some() {- let hidden_len = hidden.as_buffer_range_ref().range().len();- let mut gate_input = encoder.allocate_scratch(hidden_len)?;- encoder.encode_copy(&hidden, .., &mut gate_input, ..);- Some(gate_input)- } else {- None- };- let mut qkv = qkv_projection.encode(hidden, batch_dim, encoder)?;- let gate = match (gate_projection, gate_input) {- (Some(gate_proj), Some(gate_input)) => Some(gate_proj.encode(gate_input, batch_dim, encoder)?),- _ => None,- };- if let Some(norm) = qkv_norm {- norm.encode(&mut qkv, batch_dim, encoder)?;- }- let (queries, rotated_keys) = match rope_buffers {- Some(rope_buffers) => rope.encode(- &qkv,- token_positions,- &rope_buffers.cosines,- &rope_buffers.sines,- batch_dim,- *num_heads,- *num_groups,- *head_dim,- rope_buffers.max_sequence_length(),- rope_buffers.dim(),- encoder,- )?,- None => qk_unpack.encode(&qkv, batch_dim, *num_heads, *num_groups, *head_dim, encoder)?,- };- let attention_output = attention.encode(- AttentionArguments {- token_subtrie_ranges,- attention_sinks,- cache_access,- },- &qkv,- &queries,- rotated_keys,- gate.as_ref(),- batch_dim,- *num_heads,- *num_groups,- *head_dim,- encoder,- )?;- out_projection.encode(attention_output, batch_dim, encoder)?- },+ } => attention.encode(+ token_positions,+ token_subtrie_ranges,+ rope_buffers,+ cache_access,+ hidden,+ batch_dim,+ encoder,+ )?,MixerExecutables::StateSpace {mixer,} => {@@ -524,7 +410,6 @@ pub struct LayerArguments<'a, B: Backend> {pub token_positions: &'a Allocation<B>,pub token_parents: &'a Allocation<B>,pub token_subtrie_ranges: Option<&'a Allocation<B>>,- pub attention_sinks: Option<&'a Allocation<B>>,pub rope_buffers: Option<&'a RopeBuffers<B>>,pub sampling_start: usize,pub sampling_length: usize,diff --git a/crates/backend-uzu/src/encodable_block/layer/mixer.rs b/crates/backend-uzu/src/encodable_block/layer/mixer.rsindex c3ef46033..2acf6117e 100644--- a/crates/backend-uzu/src/encodable_block/layer/mixer.rs+++ b/crates/backend-uzu/src/encodable_block/layer/mixer.rs@@ -1,25 +1,14 @@//! Mixer component - either attention or state space.-use std::rc::Rc;-use crate::{backends::common::Backend,- encodable_block::{Attention, DeltaNetMixer, MambaMixer, QKVNorm, QkUnpack, Rope, ShortConvMixer, linear::Linear},+ encodable_block::{Attention, DeltaNetMixer, MambaMixer, ShortConvMixer},};/// Mixer component - either attention, state space, short conv, or delta net.pub(crate) enum MixerExecutables<B: Backend> {Attention {- qkv_projection: Box<dyn Linear<B>>,- gate_projection: Option<Box<dyn Linear<B>>>,- qkv_norm: Option<QKVNorm<B>>,- rope: Rc<Rope<B>>,- qk_unpack: Rc<QkUnpack<B>>,attention: Attention<B>,- out_projection: Box<dyn Linear<B>>,- num_heads: usize,- num_groups: usize,- head_dim: usize,},StateSpace {mixer: MambaMixer<B>,diff --git a/crates/backend-uzu/src/encodable_block/layer/mod.rs b/crates/backend-uzu/src/encodable_block/layer/mod.rsindex 70b5019cb..99ea4fb08 100644--- a/crates/backend-uzu/src/encodable_block/layer/mod.rs+++ b/crates/backend-uzu/src/encodable_block/layer/mod.rs@@ -3,5 +3,5 @@mod executables;mod mixer;-pub use executables::{LayerArguments, LayerExecutables};+pub use executables::{LayerArguments, LayerExecutables, LayerExecutablesError};pub(crate) use mixer::MixerExecutables;diff --git a/crates/backend-uzu/src/encodable_block/layer_norm.rs b/crates/backend-uzu/src/encodable_block/layer_norm.rsindex 90a08cd16..31647d0f2 100644--- a/crates/backend-uzu/src/encodable_block/layer_norm.rs+++ b/crates/backend-uzu/src/encodable_block/layer_norm.rs@@ -9,7 +9,7 @@ use crate::{Allocation, Backend, Encoder,kernel::{Kernels, LayerNormKernel},},- config::{NormalizationConfig, UpcastMode},+ config::normalization::{NormalizationConfig, UpcastMode},parameters::{ParameterLoaderError, ParameterTree},};@@ -18,7 +18,7 @@ pub enum LayerNormError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),#[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError<B>),+ ParameterError(#[from] ParameterLoaderError<B>),}pub struct LayerNorm<B: Backend> {@@ -34,20 +34,20 @@ impl<B: Backend> LayerNorm<B> {pub fn new(context: &B::Context,intermediate_data_type: DataType,+ element_count: usize,config: NormalizationConfig,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,) -> Result<Self, LayerNormError<B>> {- let scales_leaf = parameter_tree.leaf("scales").map_err(LayerNormError::ParameterError)?;- let element_count = scales_leaf.shape()[0];- let scales = scales_leaf.read_allocation().map_err(LayerNormError::ParameterError)?;+ let scale_data_type = super::normalization::NORMALIZATION_SCALE_DATA_TYPE;+ let scales = parameter_tree.leaf("scales")?.validate(&[element_count], scale_data_type)?.read_allocation()?;- let accumulation_data_type: DataType = config.accumulation_precision.into();- let scale_data_type: DataType = config.scale_precision.into();+ let accumulation_data_type = super::normalization::NORMALIZATION_ACCUMULATION_DATA_TYPE;+ let output_data_type = intermediate_data_type;let kernel = <B::Kernels as Kernels>::LayerNormKernel::new(context,intermediate_data_type,scale_data_type,- scale_data_type,+ output_data_type,accumulation_data_type,false,)@@ -59,7 +59,7 @@ impl<B: Backend> LayerNorm<B> {scales,element_count,input_data_type: intermediate_data_type,- output_data_type: scale_data_type,+ output_data_type,})}diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex 8524b3e7f..4fccea9ee 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -14,7 +14,7 @@ use crate::{matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},},},- config::QuantizationConfig,+ config::weight_matrix::{AnyWeightMatrixSpec, Layout, awq_spec::AWQSpec, mlx_spec::MLXSpec},parameters::{ParameterLoaderError, ParameterTree},};@@ -26,61 +26,8 @@ pub enum LinearMatmulError<B: Backend> {ParameterError(#[from] ParameterLoaderError<B>),#[error("Unsupported data type: {0:?}")]UnsupportedDataType(DataType),- #[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]- InvalidFpWeightsShape {- got: Box<[usize]>,- expected_output_dim: usize,- expected_input_dim: usize,- },- #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidWeightsDataType {- expected: DataType,- got: DataType,- },- #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidScalesDataType {- expected: DataType,- got: DataType,- },- #[error(- "Unexpected scale-bias shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"- )]- InvalidScaleBiasShapes {- weights: Box<[usize]>,- scales: Box<[usize]>,- deq_biases: Box<[usize]>,- packing_divisor: usize,- },- #[error("deq_biases dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidDeqBiasesDataType {- expected: DataType,- got: DataType,- },- #[error(- "Unexpected scale-zero-point shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"- )]- InvalidScaleZeroPointShapes {- weights: Box<[usize]>,- scales: Box<[usize]>,- zero_points: Box<[usize]>,- packing_divisor: usize,- packing_minus_one: usize,- },- #[error("Zero-points dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidZeroPointsDataType {- expected: DataType,- got: DataType,- },- #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]- InvalidBiasShape {- got: Box<[usize]>,- expected_output_dim: usize,- },- #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidBiasDataType {- expected: DataType,- got: DataType,- },+ #[error("Unsupported linear matmul configuration: {0}")]+ UnsupportedConfiguration(String),}enum Mode<B: Backend> {@@ -108,36 +55,22 @@ pub struct LinearMatmul<B: Backend> {impl<B: Backend> LinearMatmul<B> {pub fn full_precision(context: &B::Context,- precision: DataType,input_dim: usize,output_dim: usize,- parameter_tree: &ParameterTree<B::Context>,+ has_biases: bool,+ data_type: DataType,+ parameter_tree: &ParameterTree<B>,) -> Result<Self, LinearMatmulError<B>> {- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(LinearMatmulError::UnsupportedDataType(precision));- }-- let weights_leaf = parameter_tree.leaf("weights")?;- let weights_shape = weights_leaf.shape().to_vec();- if weights_shape != [output_dim, input_dim] {- return Err(LinearMatmulError::InvalidFpWeightsShape {- got: weights_shape.into_boxed_slice(),- expected_output_dim: output_dim,- expected_input_dim: input_dim,- });- }- if weights_leaf.data_type() != precision {- return Err(LinearMatmulError::InvalidWeightsDataType {- expected: precision,- got: weights_leaf.data_type(),- });+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(LinearMatmulError::UnsupportedDataType(data_type));}- let biases = load_biases(precision, output_dim, parameter_tree)?;+ let weights =+ parameter_tree.leaf("weights.weights")?.validate(&[output_dim, input_dim], data_type)?.read_allocation()?;+ let biases = load_biases(data_type, output_dim, has_biases.then_some(parameter_tree))?;- let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, precision)+ let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).map_err(LinearMatmulError::BackendError)?;- let weights = weights_leaf.read_allocation()?;Ok(Self {kernel: RefCell::new(kernel),@@ -145,96 +78,75 @@ impl<B: Backend> LinearMatmul<B> {biases,input_dim,output_dim,- data_type: precision,+ data_type,mode: Mode::FullPrecision,})}pub fn quantized(context: &B::Context,- config: &QuantizationConfig,+ spec: AnyWeightMatrixSpec,input_dim: usize,output_dim: usize,- parameter_tree: &ParameterTree<B::Context>,+ data_type: DataType,+ weights_tree: &ParameterTree<B>,+ bias_tree: Option<&ParameterTree<B>>,output_hadamard_factors: Option<Allocation<B>>,) -> Result<Self, LinearMatmulError<B>> {- let data_type: DataType = config.activation_precision.into();- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(LinearMatmulError::UnsupportedDataType(data_type));- }-- let packing_divisor = config.weight_quantization_mode.packing_divisor();- let storage_type = config.weight_quantization_mode.storage_type();+ let (bits, group_size, quantization_method) = match spec {+ AnyWeightMatrixSpec::MLXSpec(MLXSpec {+ bits,+ group_size,+ layout: Layout::OutputInput,+ ..+ }) => (bits, group_size, QuantizationMethod::ScaleBias),+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ bits,+ group_size,+ is_symmetric: false,+ layout: Layout::OutputInput,+ ..+ }) => (bits, group_size, QuantizationMethod::ScaleZeroPoint),+ spec => return Err(LinearMatmulError::UnsupportedConfiguration(format!("{spec:?}"))),+ };- let weights_leaf = parameter_tree.leaf("weights")?;- if weights_leaf.data_type() != storage_type {- return Err(LinearMatmulError::InvalidWeightsDataType {- expected: storage_type,- got: weights_leaf.data_type(),- });- }+ let weight_quantization_mode = match bits {+ 4 => QuantizationMode::U4,+ 8 => QuantizationMode::U8,+ _ => {+ return Err(LinearMatmulError::UnsupportedConfiguration(format!(+ "{quantization_method} bits={bits}, group_size={group_size}"+ )));+ },+ };- let scales_leaf = parameter_tree.leaf("scales")?;- if scales_leaf.data_type() != data_type {- return Err(LinearMatmulError::InvalidScalesDataType {- expected: data_type,- got: scales_leaf.data_type(),- });+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(LinearMatmulError::UnsupportedDataType(data_type));}- let k_g = input_dim.div_ceil(config.group_size);- let weights_shape = weights_leaf.shape().to_vec();- let scales_shape = scales_leaf.shape().to_vec();+ let packing_divisor = weight_quantization_mode.packing_divisor();+ let storage_type = weight_quantization_mode.storage_type();+ let k_g = input_dim.div_ceil(group_size);- let (quantization_method, zero_points_or_biases) = match parameter_tree.leaf("deq_biases") {- Ok(deq_biases) => {- let deq_biases_shape = deq_biases.shape().to_vec();- if !(weights_shape == [output_dim, input_dim / packing_divisor]- && scales_shape == [output_dim, k_g]- && deq_biases_shape == [output_dim, k_g])- {- return Err(LinearMatmulError::InvalidScaleBiasShapes {- weights: weights_shape.into_boxed_slice(),- scales: scales_shape.into_boxed_slice(),- deq_biases: deq_biases_shape.into_boxed_slice(),- packing_divisor,- });- }- if deq_biases.data_type() != data_type {- return Err(LinearMatmulError::InvalidDeqBiasesDataType {- expected: data_type,- got: deq_biases.data_type(),- });- }- (QuantizationMethod::ScaleBias, deq_biases.read_allocation()?)+ let weights = weights_tree+ .leaf("weights")?+ .validate(&[output_dim, input_dim / packing_divisor], storage_type)?+ .read_allocation()?;+ let scales = weights_tree.leaf("scales")?.validate(&[output_dim, k_g], data_type)?.read_allocation()?;+ let zero_points_or_biases = match quantization_method {+ QuantizationMethod::ScaleBias => {+ weights_tree.leaf("biases")?.validate(&[output_dim, k_g], data_type)?.read_allocation()?},- Err(_) => {- let zero_points_leaf = parameter_tree.leaf("zero_points")?;- let zero_points_shape = zero_points_leaf.shape().to_vec();+ QuantizationMethod::ScaleZeroPoint => {let expected_zero_points_entries = k_g.div_ceil(packing_divisor);- if !(weights_shape == [output_dim, input_dim / packing_divisor]- && scales_shape == [output_dim, k_g]- && zero_points_shape == [output_dim, expected_zero_points_entries])- {- return Err(LinearMatmulError::InvalidScaleZeroPointShapes {- weights: weights_shape.into_boxed_slice(),- scales: scales_shape.into_boxed_slice(),- zero_points: zero_points_shape.into_boxed_slice(),- packing_divisor,- packing_minus_one: packing_divisor - 1,- });- }- if zero_points_leaf.data_type() != storage_type {- return Err(LinearMatmulError::InvalidZeroPointsDataType {- expected: storage_type,- got: zero_points_leaf.data_type(),- });- }- (QuantizationMethod::ScaleZeroPoint, zero_points_leaf.read_allocation()?)+ weights_tree+ .leaf("zero_points")?+ .validate(&[output_dim, expected_zero_points_entries], storage_type)?+ .read_allocation()?},};- let biases = load_biases(data_type, output_dim, parameter_tree)?;+ let biases = load_biases(data_type, output_dim, bias_tree)?;let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).map_err(LinearMatmulError::BackendError)?;@@ -243,24 +155,24 @@ impl<B: Backend> LinearMatmul<B> {context,MatmulQuantCombo {method: quantization_method,- mode: config.weight_quantization_mode,- group_size: config.group_size as u32,+ mode: weight_quantization_mode,+ group_size: group_size as u32,},).map_err(LinearMatmulError::BackendError)?;Ok(Self {kernel: RefCell::new(kernel),- weights: weights_leaf.read_allocation()?,+ weights,biases,input_dim,output_dim,data_type,mode: Mode::Quantized {method: quantization_method,- mode: config.weight_quantization_mode,- group_size: config.group_size as u32,- scales: scales_leaf.read_allocation()?,+ mode: weight_quantization_mode,+ group_size: group_size as u32,+ scales,zero_points_or_biases,output_hadamard_factors,},@@ -271,27 +183,11 @@ impl<B: Backend> LinearMatmul<B> {fn load_biases<B: Backend>(data_type: DataType,output_dim: usize,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: Option<&ParameterTree<B>>,) -> Result<Option<Allocation<B>>, LinearMatmulError<B>> {- match parameter_tree.leaf("biases") {- Ok(biases_leaf) => {- let bias_shape = biases_leaf.shape().to_vec();- if bias_shape != [output_dim] {- return Err(LinearMatmulError::InvalidBiasShape {- got: bias_shape.into_boxed_slice(),- expected_output_dim: output_dim,- });- }- if biases_leaf.data_type() != data_type {- return Err(LinearMatmulError::InvalidBiasDataType {- expected: data_type,- got: biases_leaf.data_type(),- });- }- Ok(Some(biases_leaf.read_allocation()?))- },- Err(_) => Ok(None),- }+ Ok(parameter_tree+ .map(|tree| tree.leaf("biases")?.validate(&[output_dim], data_type)?.read_allocation())+ .transpose()?)}impl<B: Backend> Linear<B> for LinearMatmul<B> {diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rsindex a7056b36c..178f45d0b 100644--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs@@ -8,8 +8,13 @@ pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};use thiserror::Error;use crate::{+ DataType,backends::common::{Allocation, Backend, Encoder},- config::LinearConfig,+ config::weight_matrix::{+ AnyWeightMatrixSpec, Layout,+ full_precision_spec::FullPrecisionSpec,+ hybrid_spec::{HybridSpec, IncoherenceProcessingMode},+ },parameters::{ParameterLoaderError, ParameterTree},};@@ -26,138 +31,178 @@ pub trait Linear<B: Backend> {pub enum LinearBlockError<B: Backend> {#[error("LinearMatmul error: {0}")]LinearMatmulError(#[from] LinearMatmulError<B>),+ #[error("Output hadamard linear error: {0}")]+ OutputHadamardLinearError(#[from] OutputHadamardLinearError<B>),#[error("QLoRALinearWrapper error: {0}")]QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),#[error("RHTLinearWrapper error: {0}")]RHTLinearWrapperError(#[from] RHTLinearWrapperError<B>),#[error("Parameter loading error: {0}")]ParameterError(#[from] ParameterLoaderError<B>),+ #[error("Unsupported linear configuration: {0}")]+ UnsupportedConfiguration(String),+}++#[derive(Debug, Error)]+pub enum OutputHadamardLinearError<B: Backend> {+ #[error("LinearMatmul error: {0}")]+ LinearMatmulError(#[from] LinearMatmulError<B>),+ #[error("Parameter loading error: {0}")]+ ParameterError(#[from] ParameterLoaderError<B>),+ #[error("Unsupported linear configuration: {0}")]+ UnsupportedConfiguration(String),}impl<B: Backend> dyn Linear<B> {pub fn new<const N: usize>(- config: &LinearConfig,input_dimension: usize,output_dimensions: [usize; N],+ has_biases: bool,context: &B::Context,- parameter_tree: &ParameterTree<B::Context>,+ data_type: DataType,+ parameter_tree: &ParameterTree<B>,) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {let output_dimension_sum: usize = output_dimensions.iter().sum();- match config {- LinearConfig::Quantized(quantization_config) | LinearConfig::ScaleBiasQuantized(quantization_config) => {- let block = LinearMatmul::quantized(+ let weights_tree = parameter_tree.subtree("weights")?;+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;+ match spec {+ AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {+ layout: Layout::OutputInput,+ ..+ }) => {+ let block = LinearMatmul::full_precision(context,- quantization_config,input_dimension,output_dimension_sum,+ has_biases,+ data_type,parameter_tree,- None,)?;Ok(Box::new(block))},- LinearConfig::FullPrecision {- precision,- } => {- let block = LinearMatmul::full_precision(+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {+ adapter_spec: None,+ incoherence_block_size: Some(32),+ incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,+ ..+ }) => Ok(Box::new(RHTLinearWrapper::new(+ context,+ input_dimension,+ output_dimension_sum,+ has_biases,+ data_type,+ parameter_tree,+ )?)),+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {+ quantization_spec,+ adapter_spec: Some(adapter_spec),+ incoherence_block_size,+ incoherence_processing_mode,+ ..+ }) => {+ assert!(!has_biases, "QLoRA linear with biases is not supported");+ let adapter_spec = *adapter_spec;+ let AnyWeightMatrixSpec::LowRankSpec(adapter_spec) = adapter_spec else {+ return Err(LinearBlockError::UnsupportedConfiguration(format!("{adapter_spec:?}")));+ };+ Ok(Box::new(QLoRALinearWrapper::new(context,- (*precision).into(),+ *quantization_spec,+ adapter_spec,+ incoherence_block_size,+ incoherence_processing_mode,input_dimension,output_dimension_sum,- parameter_tree,- )?;- Ok(Box::new(block))+ data_type,+ &weights_tree,+ )?))},- LinearConfig::QLoRA {- quantization,- lora_rank,- lora_scale,- } => {- let block = QLoRALinearWrapper::new(+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ let block = LinearMatmul::quantized(context,- quantization,- *lora_rank,- *lora_scale,+ spec,input_dimension,output_dimension_sum,- parameter_tree,+ data_type,+ &weights_tree,+ has_biases.then_some(parameter_tree),None,)?;Ok(Box::new(block))},- LinearConfig::RHTLinearWrapper {- block_size,- inner_config,- } => Ok(Box::new(RHTLinearWrapper::new(- context,- *block_size,- inner_config,- input_dimension,- output_dimension_sum,- parameter_tree,- )?)),+ spec => Err(LinearBlockError::UnsupportedConfiguration(format!("{spec:?}"))),}}pub fn new_with_output_hadamard(context: &B::Context,- config: &LinearConfig,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,output_factors: Allocation<B>,input_dim: usize,output_dim: usize,- ) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {- match config {- LinearConfig::Quantized(config) | LinearConfig::ScaleBiasQuantized(config) => Ok(Box::new(- LinearMatmul::quantized(context, config, input_dim, output_dim, parameter_tree, Some(output_factors))?,- )),- LinearConfig::QLoRA {- quantization,- lora_rank,- lora_scale,- } => Ok(Box::new(QLoRALinearWrapper::new(- context,- quantization,- *lora_rank,- *lora_scale,- input_dim,- output_dim,- parameter_tree,- Some(output_factors),- )?)),- inner_config => unimplemented!("{inner_config:?} doesn't support fused output hadamard"),+ has_biases: bool,+ data_type: DataType,+ ) -> Result<Box<dyn Linear<B>>, OutputHadamardLinearError<B>> {+ let weights_tree = parameter_tree.subtree("weights")?.subtree("quantized")?;+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;+ match spec {+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ Ok(Box::new(LinearMatmul::quantized(+ context,+ spec,+ input_dim,+ output_dim,+ data_type,+ &weights_tree,+ has_biases.then_some(parameter_tree),+ Some(output_factors),+ )?))+ },+ spec => Err(OutputHadamardLinearError::UnsupportedConfiguration(format!(+ "{spec:?} doesn't support fused output hadamard"+ ))),}}pub fn new_extracting_input_hadamard<const N: usize>(- config: &LinearConfig,input_dimension: usize,output_dimensions: [usize; N],+ has_biases: bool,context: &B::Context,- parameter_tree: &ParameterTree<B::Context>,+ data_type: DataType,+ parameter_tree: &ParameterTree<B>,) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {let output_dimension_sum: usize = output_dimensions.iter().sum();- match config {- LinearConfig::RHTLinearWrapper {- inner_config,- ..- } => {- let input_factors = parameter_tree.leaf("input_factors")?.read_allocation()?;- let output_factors = parameter_tree.leaf("output_factors")?.read_allocation()?;- let inner_tree = parameter_tree.subtree("inner_linear")?;- let inner_linear = Self::new_with_output_hadamard(- context,- inner_config,- &inner_tree,- output_factors,- input_dimension,- output_dimension_sum,- )?;- Ok((inner_linear, Some(input_factors)))- },- other => {- let linear = Self::new(other, input_dimension, output_dimensions, context, parameter_tree)?;- Ok((linear, None))- },+ let weights_tree = parameter_tree.subtree("weights")?;+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;+ if let AnyWeightMatrixSpec::HybridSpec(HybridSpec {+ adapter_spec: None,+ incoherence_block_size: Some(32),+ incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,+ ..+ }) = spec+ {+ let input_factors = weights_tree+ .leaf("incoherence_signs.input_signs")?+ .validate(&[input_dimension], DataType::I32)?+ .read_allocation()?;+ let output_factors = weights_tree+ .leaf("incoherence_signs.output_signs")?+ .validate(&[output_dimension_sum], DataType::I32)?+ .read_allocation()?;+ let inner_linear = Self::new_with_output_hadamard(+ context,+ parameter_tree,+ output_factors,+ input_dimension,+ output_dimension_sum,+ has_biases,+ data_type,+ )?;+ Ok((inner_linear, Some(input_factors)))+ } else {+ let linear = Self::new(input_dimension, output_dimensions, has_biases, context, data_type, parameter_tree)?;+ Ok((linear, None))}}}diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex c3fae59ec..08bc3aab4 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -7,86 +7,137 @@ use crate::{array::size_for_shape,backends::common::{Allocation, Backend, Encoder,+ gpu_types::HadamardTransformOrder,kernel::{- ManualKernels,+ HadamardTransformKernel, Kernels, ManualKernels,matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},- config::QuantizationConfig,+ config::weight_matrix::{AnyWeightMatrixSpec, hybrid_spec::IncoherenceProcessingMode, low_rank_spec::LowRankSpec},encodable_block::{Linear,- linear::{LinearBlockError, LinearMatmul, LinearMatmulError},+ linear::{LinearMatmul, LinearMatmulError},},prelude::{ParameterLoaderError, ParameterTree},};#[derive(Debug, Error)]pub enum QLoRALinearWrapperError<B: Backend> {- #[error("Inner linear error: {0}")]- InnerLinearError(#[from] Box<LinearBlockError<B>>),- #[error("Linear matmul error: {0}")]+ #[error("LinearMatmul error: {0}")]LinearMatmulError(#[from] LinearMatmulError<B>),#[error("Parameter loader error: {0}")]ParameterLoaderError(#[from] ParameterLoaderError<B>),#[error("Backend error: {0}")]BackendError(#[source] B::Error),+ #[error("Unsupported QLoRA linear configuration: {0}")]+ UnsupportedConfiguration(String),}pub struct QLoRALinearWrapper<B: Backend> {base_linear: LinearMatmul<B>,+ input_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,+ output_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,adapter_down: Allocation<B>,adapter_up: Allocation<B>,input_dim: usize,output_dim: usize,lora_rank: usize,- lora_scale: f32,data_type: DataType,}impl<B: Backend> QLoRALinearWrapper<B> {pub fn new(context: &B::Context,- quantization: &QuantizationConfig,- lora_rank: usize,- lora_scale: f32,+ quantization_spec: AnyWeightMatrixSpec,+ adapter_spec: LowRankSpec,+ incoherence_block_size: Option<usize>,+ incoherence_processing_mode: IncoherenceProcessingMode,input_dim: usize,output_dim: usize,- parameter_tree: &ParameterTree<B::Context>,- output_quantized_hadamard_factors: Option<Allocation<B>>,+ data_type: DataType,+ weights_tree: &ParameterTree<B>,) -> Result<Self, QLoRALinearWrapperError<B>> {- let data_type = quantization.activation_precision.into();+ let use_incoherence_signs = match (incoherence_block_size, incoherence_processing_mode) {+ (None, _) => false,+ (Some(32), IncoherenceProcessingMode::InputOutput) => true,+ (incoherence_block_size, incoherence_processing_mode) => {+ return Err(QLoRALinearWrapperError::UnsupportedConfiguration(format!(+ "incoherence block_size={incoherence_block_size:?}, processing_mode={incoherence_processing_mode:?}"+ )));+ },+ };+ let quantized_tree = weights_tree.subtree("quantized")?;let base_linear = LinearMatmul::quantized(context,- quantization,+ quantization_spec,input_dim,output_dim,- parameter_tree,- output_quantized_hadamard_factors,+ data_type,+ &quantized_tree,+ None,+ None,)?;++ let (input_hadamard, output_hadamard) = if use_incoherence_signs {+ let input_factors = weights_tree+ .leaf("incoherence_signs.input_signs")?+ .validate(&[input_dim], DataType::I32)?+ .read_allocation()?;+ let output_factors = weights_tree+ .leaf("incoherence_signs.output_signs")?+ .validate(&[output_dim], DataType::I32)?+ .read_allocation()?;+ (+ Some((+ <B::Kernels as Kernels>::HadamardTransformKernel::new(+ context,+ data_type,+ HadamardTransformOrder::Input,+ )+ .map_err(QLoRALinearWrapperError::BackendError)?,+ input_factors,+ )),+ Some((+ <B::Kernels as Kernels>::HadamardTransformKernel::new(+ context,+ data_type,+ HadamardTransformOrder::Output,+ )+ .map_err(QLoRALinearWrapperError::BackendError)?,+ output_factors,+ )),+ )+ } else {+ (None, None)+ };+let adapter_kernel = RefCell::new(<<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type).map_err(QLoRALinearWrapperError::BackendError)?,);- let adapter_down_leaf = parameter_tree.leaf("down_weights")?;- adapter_down_leaf.validate_shape(&[lora_rank, input_dim], data_type)?;- let adapter_down = adapter_down_leaf.read_allocation()?;+ let adapter_down = weights_tree+ .leaf("adapter.down_projection")?+ .validate(&[adapter_spec.rank, input_dim], data_type)?+ .read_allocation()?;- let adapter_up_leaf = parameter_tree.leaf("up_weights")?;- adapter_up_leaf.validate_shape(&[output_dim, lora_rank], data_type)?;- let adapter_up = adapter_up_leaf.read_allocation()?;+ let adapter_up = weights_tree+ .leaf("adapter.up_projection")?+ .validate(&[output_dim, adapter_spec.rank], data_type)?+ .read_allocation()?;Ok(Self {base_linear,+ input_hadamard,+ output_hadamard,adapter_kernel,adapter_down,adapter_up,input_dim,output_dim,- lora_rank,- lora_scale,+ lora_rank: adapter_spec.rank,data_type,})}@@ -98,55 +149,86 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {input: Allocation<B>,batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, <B as Backend>::Error> {- let mut adapter_kernel = self.adapter_kernel.borrow_mut();+ ) -> Result<Allocation<B>, B::Error> {let mut intermediate =encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.data_type))?;- adapter_kernel.encode(- MatmulArguments {- a: &input,- a_offset: 0,- b: MatmulB::FullPrecision {- b: &self.adapter_down,+ {+ let mut adapter_kernel = self.adapter_kernel.borrow_mut();+ adapter_kernel.encode(+ MatmulArguments {+ a: &input,+ a_offset: 0,+ b: MatmulB::FullPrecision {+ b: &self.adapter_down,+ },+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut intermediate,+ d_transform: MatmulDOps::none(),+ m: batch_dim as u32,+ n: self.lora_rank as u32,+ k: self.input_dim as u32,},- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- d: &mut intermediate,- d_transform: MatmulDOps::none(),- m: batch_dim as u32,- n: self.lora_rank as u32,- k: self.input_dim as u32,- },- encoder,- )?;+ encoder,+ )?;+ }- let mut output = self.base_linear.encode(input, batch_dim, encoder)?;+ let base_input = if let Some((input_hadamard_kernel, input_factors)) = &self.input_hadamard {+ let mut base_input =+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dim], self.data_type))?;+ encoder.encode_copy(&input, .., &mut base_input, ..);+ input_hadamard_kernel.encode(+ &mut base_input,+ input_factors,+ self.input_dim as u32,+ batch_dim as u32,+ encoder,+ );+ base_input+ } else {+ input+ };- adapter_kernel.encode(- MatmulArguments {- a: &intermediate,- a_offset: 0,- b: MatmulB::FullPrecision {- b: &self.adapter_up,- },- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- d: &mut output,- d_transform: MatmulDOps {- ab_scale: self.lora_scale,- accumulate: true,- bias: None,- rht_factors: None,+ let mut output = self.base_linear.encode(base_input, batch_dim, encoder)?;++ {+ let mut adapter_kernel = self.adapter_kernel.borrow_mut();+ adapter_kernel.encode(+ MatmulArguments {+ a: &intermediate,+ a_offset: 0,+ b: MatmulB::FullPrecision {+ b: &self.adapter_up,+ },+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut output,+ d_transform: MatmulDOps {+ ab_scale: 1.0,+ accumulate: true,+ bias: None,+ rht_factors: None,+ },+ m: batch_dim as u32,+ n: self.output_dim as u32,+ k: self.lora_rank as u32,},- m: batch_dim as u32,- n: self.output_dim as u32,- k: self.lora_rank as u32,- },- encoder,- )?;+ encoder,+ )?;+ }++ if let Some((output_hadamard_kernel, output_factors)) = &self.output_hadamard {+ output_hadamard_kernel.encode(+ &mut output,+ output_factors,+ self.output_dim as u32,+ batch_dim as u32,+ encoder,+ );+ }Ok(output)}diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rsindex 2714551f4..ddcd964ea 100644--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs@@ -1,23 +1,30 @@use thiserror::Error;-use super::{Linear, LinearBlockError};+use super::{Linear, OutputHadamardLinearError};use crate::{+ DataType,backends::common::{Allocation, Backend, Encoder,+ gpu_types::HadamardTransformOrder,kernel::{HadamardTransformKernel, Kernels},},- config::LinearConfig,+ config::weight_matrix::{+ AnyWeightMatrixSpec,+ hybrid_spec::{HybridSpec, IncoherenceProcessingMode},+ },parameters::{ParameterLoaderError, ParameterTree},};#[derive(Debug, Error)]pub enum RHTLinearWrapperError<B: Backend> {#[error("Inner linear error: {0}")]- InnerLinearError(#[from] Box<LinearBlockError<B>>),+ InnerLinearError(#[from] OutputHadamardLinearError<B>),#[error("Parameter loading error: {0}")]ParameterError(#[from] ParameterLoaderError<B>),#[error("Backend error: {0}")]BackendError(#[source] B::Error),+ #[error("Unsupported RHT linear configuration: {0}")]+ UnsupportedConfiguration(String),}pub struct RHTLinearWrapper<B: Backend> {@@ -30,29 +37,44 @@ pub struct RHTLinearWrapper<B: Backend> {impl<B: Backend> RHTLinearWrapper<B> {pub fn new(context: &B::Context,- block_size: usize,- inner_config: &LinearConfig,input_dimension: usize,output_dimension: usize,- parameter_tree: &ParameterTree<B::Context>,+ has_biases: bool,+ data_type: DataType,+ parameter_tree: &ParameterTree<B>,) -> Result<Self, RHTLinearWrapperError<B>> {- assert!(block_size == 32, "only block size 32 hadamard is supported");+ let weights_tree = parameter_tree.subtree("weights")?;+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;+ let AnyWeightMatrixSpec::HybridSpec(HybridSpec {+ adapter_spec: None,+ incoherence_block_size: Some(32),+ incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,+ ..+ }) = &spec+ else {+ return Err(RHTLinearWrapperError::UnsupportedConfiguration(format!("{spec:?}")));+ };- let data_type = inner_config.activation_precision().into();- let input_factors = parameter_tree.leaf("input_factors")?.read_allocation()?;- let output_factors = parameter_tree.leaf("output_factors")?.read_allocation()?;- let input_hadamard_kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)- .map_err(RHTLinearWrapperError::BackendError)?;- let inner_linear_tree = parameter_tree.subtree("inner_linear")?;+ let input_factors = weights_tree+ .leaf("incoherence_signs.input_signs")?+ .validate(&[input_dimension], DataType::I32)?+ .read_allocation()?;+ let output_factors = weights_tree+ .leaf("incoherence_signs.output_signs")?+ .validate(&[output_dimension], DataType::I32)?+ .read_allocation()?;+ let input_hadamard_kernel =+ <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type, HadamardTransformOrder::Input)+ .map_err(RHTLinearWrapperError::BackendError)?;let inner_linear = <dyn Linear<B>>::new_with_output_hadamard(context,- inner_config,- &inner_linear_tree,+ parameter_tree,output_factors,input_dimension,output_dimension,- )- .map_err(|error| RHTLinearWrapperError::InnerLinearError(Box::new(error)))?;+ has_biases,+ data_type,+ )?;Ok(Self {input_hadamard_kernel,diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer.rsindex a6af9f11e..f5a616e3c 100644--- a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs@@ -9,12 +9,13 @@ use crate::{array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,+ gpu_types::ActivationType,kernel::{Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},},- config::Mamba2Config,+ config::token_mixer::mamba2::Mamba2Config,encodable_block::linear::{Linear, LinearBlockError},forward_pass::ssm_layer::SSMLayer,parameters::{ParameterLoaderError, ParameterTree},@@ -25,13 +26,20 @@ pub enum MambaMixerError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),#[error("Linear error: {0}")]- LinearError(#[from] Box<LinearBlockError<B>>),+ LinearError(#[from] LinearBlockError<B>),#[error("Parameter loader error: {0}")]ParameterLoaderError(#[from] ParameterLoaderError<B>),}pub(crate) struct MambaMixer<B: Backend> {- config: Mamba2Config,+ conv_dim: usize,+ inner_dim: usize,+ num_heads: usize,+ num_groups: usize,+ head_dim: usize,+ state_dim: usize,+ kernel_size: usize,+ activation_type: ActivationType,in_projection: Box<dyn Linear<B>>,out_projection: Box<dyn Linear<B>>,split_inproj: <B::Kernels as Kernels>::SplitInProjKernel,@@ -79,41 +87,45 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {impl<B: Backend> MambaMixer<B> {pub(crate) fn new(context: &B::Context,- mamba_config: Mamba2Config,+ mamba_config: &Mamba2Config,model_dim: usize,- decoder_layer_loader: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,+ data_type: DataType,) -> Result<(Self, Option<Allocation<B>>), MambaMixerError<B>> {- let split_tree = decoder_layer_loader.subtree("mixer")?;- let conv_tree = split_tree.subtree("conv")?;-- let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();+ let conv_tree = parameter_tree.subtree("conv")?;+ let conv_dim = mamba_config.conv_dim();+ let inner_dim = mamba_config.inner_dim();let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(- &mamba_config.in_projection_config,model_dim,- [mamba_config.conv_dim(), mamba_config.inner_dim(), mamba_config.num_heads],+ [conv_dim, inner_dim, mamba_config.num_heads],+ mamba_config.has_in_biases,context,- &decoder_layer_loader.subtree("mixer.in_projection")?,- )- .map_err(|err| MambaMixerError::LinearError(Box::new(err)))?;+ data_type,+ ¶meter_tree.subtree("in_projection")?,+ )?;let out_projection = <dyn Linear<B>>::new(- &mamba_config.out_projection_config,- mamba_config.inner_dim(),+ inner_dim,[model_dim],+ mamba_config.has_out_biases,context,- &decoder_layer_loader.subtree("mixer.out_projection")?,- )- .map_err(|err| MambaMixerError::LinearError(Box::new(err)))?;+ data_type,+ ¶meter_tree.subtree("out_projection")?,+ )?;- let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;+ let conv_weight =+ conv_tree.leaf("weights")?.validate(&[conv_dim, mamba_config.kernel_size], data_type)?.read_allocation()?;let conv_bias = if mamba_config.conv_config.has_biases {- Some(conv_tree.leaf("biases")?.read_allocation()?)+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], data_type)?.read_allocation()?)} else {None};- let gate_bias = split_tree.leaf("gate_bias")?.read_allocation()?;- let skip_connection_weight = split_tree.leaf("skip_connection_weight")?.read_allocation()?;+ let gate_bias = parameter_tree.leaf("gate_bias")?.validate(&[inner_dim], data_type)?.read_allocation()?;+ let skip_connection_weight = parameter_tree+ .leaf("skip_connection_weight")?+ .validate(&[mamba_config.num_heads], data_type)?+ .read_allocation()?;let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type).map_err(MambaMixerError::BackendError)?;@@ -136,7 +148,14 @@ impl<B: Backend> MambaMixer<B> {Ok((Self {- config: mamba_config,+ conv_dim,+ inner_dim,+ num_heads: mamba_config.num_heads,+ num_groups: mamba_config.num_groups,+ head_dim: mamba_config.head_dim,+ state_dim: mamba_config.state_dim,+ kernel_size: mamba_config.kernel_size,+ activation_type: mamba_config.activation.act_type(),in_projection,out_projection,split_inproj,@@ -162,13 +181,13 @@ impl<B: Backend> MambaMixer<B> {suffix_length: usize,encoder: &mut Encoder<B>,) -> Result<SplitInProjectionOutput<B>, B::Error> {- let conv_dim = self.config.conv_dim();- let inner_dim = self.config.inner_dim();- let num_heads = self.config.num_heads;+ let conv_dim = self.conv_dim;+ let inner_dim = self.inner_dim;+ let num_heads = self.num_heads;let total_dim = conv_dim + inner_dim + num_heads;let mut conv_inputs = encoder.allocate_scratch(size_for_shape(&[suffix_length, conv_dim], self.data_type))?;- let mut gate = encoder- .allocate_scratch(size_for_shape(&[suffix_length, num_heads, self.config.head_dim], self.data_type))?;+ let mut gate =+ encoder.allocate_scratch(size_for_shape(&[suffix_length, num_heads, self.head_dim], self.data_type))?;let mut time_step = encoder.allocate_scratch(size_for_shape(&[suffix_length, num_heads], self.data_type))?;self.split_inproj.encode(in_proj,@@ -197,20 +216,18 @@ impl<B: Backend> MambaMixer<B> {encoder: &mut Encoder<B>,suffix_length: usize,) -> Result<ConvScanOutput<B>, B::Error> {- let conv_dim = self.config.conv_dim();- let inner_dim = self.config.inner_dim();- let proj_dim = self.config.num_groups * self.config.state_dim;- let state_stride = self.config.kernel_size.saturating_sub(1);- let mut conv_x = encoder.allocate_scratch(size_for_shape(- &[suffix_length, self.config.num_heads, self.config.head_dim],- self.data_type,- ))?;- let state_shape = [suffix_length, self.config.num_groups, self.config.state_dim];+ let conv_dim = self.conv_dim;+ let inner_dim = self.inner_dim;+ let proj_dim = self.num_groups * self.state_dim;+ let state_stride = self.kernel_size.saturating_sub(1);+ let mut conv_x = encoder+ .allocate_scratch(size_for_shape(&[suffix_length, self.num_heads, self.head_dim], self.data_type))?;+ let state_shape = [suffix_length, self.num_groups, self.state_dim];let mut state_b = encoder.allocate_scratch(size_for_shape(&state_shape, self.data_type))?;let mut state_c = encoder.allocate_scratch(size_for_shape(&state_shape, self.data_type))?;if suffix_length == 1 {- if conv_dim > 0 && self.config.kernel_size > 0 {+ if conv_dim > 0 && self.kernel_size > 0 {let next_state = layer.conv_state.as_mut().unwrap_or(&mut layer.ssm_state);self.conv_decode.encode(conv_inputs,@@ -221,14 +238,14 @@ impl<B: Backend> MambaMixer<B> {&mut state_b,&mut state_c,next_state,- self.config.kernel_size as u32,+ self.kernel_size as u32,conv_dim as u32,state_stride as u32,conv_dim as u32,suffix_length as u32,inner_dim as u32,proj_dim as u32,- self.config.activation.act_type(),+ self.activation_type,encoder,);}@@ -252,7 +269,7 @@ impl<B: Backend> MambaMixer<B> {);}- if conv_dim > 0 && self.config.kernel_size > 0 {+ if conv_dim > 0 && self.kernel_size > 0 {let conv_source = match &padded {Some(padded) => padded,None => conv_inputs,@@ -267,13 +284,13 @@ impl<B: Backend> MambaMixer<B> {&mut state_c,next_state,suffix_length as u32,- self.config.kernel_size as u32,+ self.kernel_size as u32,conv_dim as u32,state_stride as u32,conv_dim as u32,inner_dim as u32,proj_dim as u32,- self.config.activation.act_type(),+ self.activation_type,encoder,);}@@ -307,14 +324,14 @@ impl<B: Backend> MambaMixer<B> {z: gate,state: &mut layer.ssm_state,suffix_len: suffix_length,- group_size: (self.config.num_heads / self.config.num_groups) as u32,- state_size: self.config.state_dim as u32,- x_strides: [self.config.num_heads * self.config.head_dim, self.config.head_dim, 1],- dt_strides: [self.config.num_heads, 1],- cb_strides: [self.config.num_groups * self.config.state_dim, self.config.state_dim, 1],- state_strides: [self.config.head_dim * self.config.state_dim, self.config.state_dim, 1],- channels: self.config.num_heads,- head_dim: self.config.head_dim,+ group_size: (self.num_heads / self.num_groups) as u32,+ state_size: self.state_dim as u32,+ x_strides: [self.num_heads * self.head_dim, self.head_dim, 1],+ dt_strides: [self.num_heads, 1],+ cb_strides: [self.num_groups * self.state_dim, self.state_dim, 1],+ state_strides: [self.head_dim * self.state_dim, self.state_dim, 1],+ channels: self.num_heads,+ head_dim: self.head_dim,},self.prefill_mode,)@@ -331,12 +348,11 @@ impl<B: Backend> MambaMixer<B> {encoder: &mut Encoder<B>,suffix_length: usize,) -> Result<Allocation<B>, B::Error> {- let mut out =- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.inner_dim()], self.data_type))?;- let h = self.config.num_heads as u32;- let g = self.config.num_groups as u32;- let dh = self.config.head_dim as u32;- let n = self.config.state_dim as u32;+ let mut out = encoder.allocate_scratch(size_for_shape(&[suffix_length, self.inner_dim], self.data_type))?;+ let h = self.num_heads as u32;+ let g = self.num_groups as u32;+ let dh = self.head_dim as u32;+ let n = self.state_dim as u32;let x_strides = [h * dh, dh, 1u32];let dt_strides = [h, 1u32];diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rsindex 464f58a20..c360dde6c 100644--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs@@ -9,7 +9,7 @@ use super::linear::{Linear, LinearBlockError};use crate::{DataType,backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},- config::MLPConfig,+ config::mlp::AnyMLPConfig,parameters::{ParameterLoaderError, ParameterTree},};@@ -36,36 +36,37 @@ pub enum MlpBlockError<B: Backend> {impl<B: Backend> dyn Mlp<B> {pub fn new(- config: &MLPConfig,+ config: &AnyMLPConfig,model_dimension: usize,hidden_dimension: usize,context: &B::Context,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,+ data_type: DataType,) -> Result<(Box<dyn Mlp<B>>, Option<Allocation<B>>), MlpBlockError<B>> {match config {- MLPConfig::Dense(dense_config) => {- let data_type: DataType = dense_config.linear_config.activation_precision().into();-+ AnyMLPConfig::DenseMLPConfig(dense_config) => {let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(- &dense_config.linear_config,model_dimension,[2 * hidden_dimension],+ dense_config.has_up_biases,context,+ data_type,¶meter_tree.subtree("up_projection")?,)?;let (down_projection, down_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(- &dense_config.linear_config,hidden_dimension,[model_dimension],+ dense_config.has_down_biases,context,+ data_type,¶meter_tree.subtree("down_projection")?,)?;let gate = MlpGateActMulEncodable::new(context,data_type,- dense_config.activation.clone(),+ dense_config.activation.clone().into(),hidden_dimension,down_input_hadamard_factors,)@@ -73,8 +74,14 @@ impl<B: Backend> dyn Mlp<B> {Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))},- MLPConfig::MixtureOfExperts(mixture_of_experts_config) => Ok((- Box::new(MoeBlock::new(context, mixture_of_experts_config, model_dimension, parameter_tree)?),+ AnyMLPConfig::MixtureOfExpertsConfig(mixture_of_experts_config) => Ok((+ Box::new(MoeBlock::new(+ context,+ mixture_of_experts_config,+ model_dimension,+ data_type,+ parameter_tree,+ )?),None,)),}diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe.rs b/crates/backend-uzu/src/encodable_block/mlp/moe.rsindex b118420a7..33e18c6b1 100644--- a/crates/backend-uzu/src/encodable_block/mlp/moe.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/moe.rs@@ -17,7 +17,10 @@ use crate::{},},},- config::{LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig},+ config::{+ mlp::{mixture_of_experts::MixtureOfExpertsConfig, routing_function::AnyRoutingFunction},+ weight_matrix::{AnyWeightMatrixSpec, Layout, full_precision_spec::FullPrecisionSpec},+ },encodable_block::mlp::Mlp,parameters::{ParameterLoaderError, ParameterTree},};@@ -66,6 +69,12 @@ pub enum MoeBlockError<B: Backend> {UnsupportedSharedExperts,#[error("MoE expert gate is not supported")]UnsupportedExpertGate,+ #[error("MoE without router, up, and down biases is not supported")]+ UnsupportedNoBiases,+ #[error("Unsupported MoE router configuration: {0}")]+ UnsupportedRouterConfiguration(String),+ #[error("Unsupported MoE expert activation: {0:?}")]+ UnsupportedExpertActivation(ActivationType),}impl<B: Backend> MoeBlock<B> {@@ -73,7 +82,8 @@ impl<B: Backend> MoeBlock<B> {context: &B::Context,moe_config: &MixtureOfExpertsConfig,model_dim: usize,- parameter_tree: &ParameterTree<B::Context>,+ data_type: DataType,+ parameter_tree: &ParameterTree<B>,) -> Result<Self, MoeBlockError<B>> {if model_dim % 4 != 0 {return Err(MoeBlockError::InvalidModelDim);@@ -90,30 +100,62 @@ impl<B: Backend> MoeBlock<B> {if moe_config.gate_config.is_some() {return Err(MoeBlockError::UnsupportedExpertGate);}+ if !moe_config.router_has_biases+ || !moe_config.expert_config.has_up_biases+ || !moe_config.expert_config.has_down_biases+ {+ return Err(MoeBlockError::UnsupportedNoBiases);+ }- let data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();let gating_code = match moe_config.expert_config.activation.act_type() {- ActivationType::GELU => 3,- ActivationType::SILU {- ..- } => 2,- _ => {- panic!("{:?} is not supported for MoE kernels", moe_config.expert_config.activation.act_type())- },+ ActivationType::GELUApprox => 3,+ ActivationType::SILU => 2,+ activation_type => return Err(MoeBlockError::UnsupportedExpertActivation(activation_type)),};- let router_data_type: DataType = moe_config.router_config.activation_precision().into();-- let router_renorm = matches!(moe_config.routing_function, RoutingFunctionConfig::SoftmaxRouting);+ let router_renorm = matches!(moe_config.routing_function, AnyRoutingFunction::SoftmaxRouting(_));let router_tree = parameter_tree.subtree("router")?;- if !matches!(moe_config.router_config, LinearConfig::FullPrecision { .. }) {- unimplemented!("Only FullPrecision MoE router is supported; got {:?}", moe_config.router_config);- }- let router_weights = router_tree.leaf_allocation("weights")?;- let router_biases = router_tree.leaf_allocation("biases")?;+ let router_weights_tree = router_tree.subtree("weights")?;+ let router_spec = router_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;+ let AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {+ layout: Layout::OutputInput,+ ..+ }) = &router_spec+ else {+ return Err(MoeBlockError::UnsupportedRouterConfiguration(format!("{router_spec:?}")));+ };+ let router_weights = router_weights_tree+ .leaf("weights")?+ .validate(&[moe_config.num_routed_experts, model_dim], data_type)?+ .read_allocation()?;+ let router_biases =+ router_tree.leaf("biases")?.validate(&[moe_config.num_routed_experts], data_type)?.read_allocation()?;++ let experts_tree = parameter_tree.subtree("experts")?;+ let up_tree = experts_tree.subtree("up_projection")?;+ let down_tree = experts_tree.subtree("down_projection")?;+ let up_weights_tree = up_tree.subtree("weights")?;+ let down_weights_tree = down_tree.subtree("weights")?;- let router_topk_kernel = <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, router_data_type)+ let w13 = up_weights_tree+ .leaf("weights")?+ .validate(&[moe_config.num_routed_experts, moe_config.expert_hidden_dim * 2, model_dim], data_type)?+ .read_allocation()?;+ let w2 = down_weights_tree+ .leaf("weights")?+ .validate(&[moe_config.num_routed_experts, model_dim, moe_config.expert_hidden_dim], data_type)?+ .read_allocation()?;+ let up_biases = up_tree+ .leaf("biases")?+ .validate(&[moe_config.num_routed_experts, moe_config.expert_hidden_dim * 2], data_type)?+ .read_allocation()?;+ let down_biases = down_tree+ .leaf("biases")?+ .validate(&[moe_config.num_routed_experts, model_dim], data_type)?+ .read_allocation()?;++ let router_topk_kernel = <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, data_type).map_err(MoeBlockError::BackendError)?;let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(MoeBlockError::BackendError)?;@@ -130,15 +172,6 @@ impl<B: Backend> MoeBlock<B> {let finalize_kernel =<B::Kernels as Kernels>::MoeFinalizeKernel::new(context, data_type).map_err(MoeBlockError::BackendError)?;- let experts_tree = parameter_tree.subtree("experts")?;- let up_tree = experts_tree.subtree("up_projection")?;- let down_tree = experts_tree.subtree("down_projection")?;-- let w13 = up_tree.leaf_allocation("weights")?;- let w2 = down_tree.leaf_allocation("weights")?;- let up_biases = up_tree.leaf_allocation("biases")?;- let down_biases = down_tree.leaf_allocation("biases")?;-let (gate_lo, gate_hi) = moe_config.expert_config.gate_clipping.unwrap_or_default();let (up_lo, up_hi) = moe_config.expert_config.up_clipping.unwrap_or_default();diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rsindex 21676ce4c..26942ab18 100644--- a/crates/backend-uzu/src/encodable_block/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mod.rs@@ -18,17 +18,17 @@ mod rope;mod sampling;mod short_conv_mixer;-pub use attention::{Attention, AttentionArguments};+pub use attention::{Attention, AttentionError};pub use classifier_layer::ClassifierLayer;pub use decoder::{Decoder, DecoderArguments, DecoderDecodeInput};-pub(crate) use delta_net_mixer::{DeltaNetArguments, DeltaNetMixer};+pub(crate) use delta_net_mixer::{DeltaNetArguments, DeltaNetMixer, DeltaNetMixerError};pub use embedding::Embedding;pub use layer::{LayerArguments, LayerExecutables};pub use layer_norm::{LayerNorm, LayerNormError};pub use linear::Linear;-pub(crate) use mamba_mixer::{MambaArguments, MambaMixer};-pub use mlp::Mlp;-pub use normalization::Normalization;+pub(crate) use mamba_mixer::{MambaArguments, MambaMixer, MambaMixerError};+pub use mlp::{Mlp, MlpBlockError};+pub use normalization::{Normalization, NormalizationError};pub use pooling::Pooling;pub use prediction_head::ClassifierPredictionHead;pub use qk_unpack::QkUnpack;@@ -38,4 +38,4 @@ pub use rope::Rope;pub(crate) use sampling::SamplingInputs;pub use sampling::{Sampling, SamplingArguments};pub(crate) use short_conv_mixer::ShortConvArguments;-pub use short_conv_mixer::ShortConvMixer;+pub use short_conv_mixer::{ShortConvMixer, ShortConvMixerError};diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rsindex 829cbadf5..db6aaeb16 100644--- a/crates/backend-uzu/src/encodable_block/normalization.rs+++ b/crates/backend-uzu/src/encodable_block/normalization.rs@@ -6,10 +6,13 @@ use super::{LayerNorm, LayerNormError, PostLayerScalar, RMSNorm, RMSNormError};use crate::{DataType,backends::common::{Allocation, Backend, Encoder},- config::NormalizationConfig,+ config::normalization::NormalizationConfig,parameters::ParameterTree,};+pub(super) const NORMALIZATION_ACCUMULATION_DATA_TYPE: DataType = DataType::F32;+pub(super) const NORMALIZATION_SCALE_DATA_TYPE: DataType = DataType::F32;+#[derive(Debug, Error)]pub enum NormalizationError<B: Backend> {#[error("LayerNorm error: {0}")]@@ -30,15 +33,17 @@ impl<B: Backend> Normalization<B> {pub fn new(context: &B::Context,intermediate_data_type: DataType,+ element_count: usize,config: NormalizationConfig,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,) -> Result<Self, NormalizationError<B>> {if config.subtract_mean {- Ok(Self::LayerNorm(LayerNorm::new(context, intermediate_data_type, config, parameter_tree)?))+ Ok(Self::LayerNorm(LayerNorm::new(context, intermediate_data_type, element_count, config, parameter_tree)?))} else {Ok(Self::RMSNorm(RMSNorm::new(context,intermediate_data_type,+ element_count,config,parameter_tree,None,diff --git a/crates/backend-uzu/src/encodable_block/pooling.rs b/crates/backend-uzu/src/encodable_block/pooling.rsindex ced65d036..0dadda424 100644--- a/crates/backend-uzu/src/encodable_block/pooling.rs+++ b/crates/backend-uzu/src/encodable_block/pooling.rs@@ -7,7 +7,7 @@ use crate::{Allocation, Backend, Encoder,kernel::{Kernels, PoolingClsKernel, PoolingMeanKernel},},- config::PoolingType,+ config::classifier::PoolingType,};enum PoolingKernel<B: Backend> {@@ -46,7 +46,7 @@ impl<B: Backend> Pooling<B> {model_dim: usize,) -> Result<Self, B::Error> {let pooling_kernel = match pooling_type {- PoolingType::Cls => PoolingKernel::Cls(<B::Kernels as Kernels>::PoolingClsKernel::new(context, data_type)?),+ PoolingType::CLS => PoolingKernel::Cls(<B::Kernels as Kernels>::PoolingClsKernel::new(context, data_type)?),PoolingType::Mean => {PoolingKernel::Mean(<B::Kernels as Kernels>::PoolingMeanKernel::new(context, data_type)?)},diff --git a/crates/backend-uzu/src/encodable_block/qkv_norm.rs b/crates/backend-uzu/src/encodable_block/qkv_norm.rsindex 574d81a22..0ff40018f 100644--- a/crates/backend-uzu/src/encodable_block/qkv_norm.rs+++ b/crates/backend-uzu/src/encodable_block/qkv_norm.rs@@ -6,7 +6,7 @@ use crate::{Allocation, Backend, Encoder,kernel::{Kernels, QKVNormKernel},},- config::{NormalizationConfig, UpcastMode},+ config::normalization::{NormalizationConfig, UpcastMode},parameters::{ParameterLoaderError, ParameterTree},};@@ -40,21 +40,37 @@ impl<B: Backend> QKVNorm<B> {query_config: Option<NormalizationConfig>,key_config: Option<NormalizationConfig>,value_config: Option<NormalizationConfig>,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,num_q_heads: usize,num_kv_heads: usize,head_dim: usize,) -> Result<Self, QKVNormError<B>> {let query = query_config.map(|cfg| {- Self::build_head(context, intermediate_data_type, cfg, parameter_tree, Some("query_norm.scales"))+ Self::build_head(+ context,+ intermediate_data_type,+ cfg,+ parameter_tree,+ Some("query_norm.scales"),+ head_dim,+ )}).transpose()?;let key = key_config- .map(|cfg| Self::build_head(context, intermediate_data_type, cfg, parameter_tree, Some("key_norm.scales")))+ .map(|cfg| {+ Self::build_head(+ context,+ intermediate_data_type,+ cfg,+ parameter_tree,+ Some("key_norm.scales"),+ head_dim,+ )+ }).transpose()?;let value = value_config- .map(|cfg| Self::build_head(context, intermediate_data_type, cfg, parameter_tree, None))+ .map(|cfg| Self::build_head(context, intermediate_data_type, cfg, parameter_tree, None, head_dim)).transpose()?;Ok(Self {@@ -71,18 +87,27 @@ impl<B: Backend> QKVNorm<B> {context: &B::Context,intermediate_data_type: DataType,config: NormalizationConfig,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,scales_leaf: Option<&str>,+ head_dim: usize,) -> Result<Head<B>, QKVNormError<B>> {- let scales = scales_leaf.map(|leaf| parameter_tree.leaf_allocation(leaf)).transpose()?;+ let (scales, scale_data_type) = if let Some(scales_leaf) = scales_leaf {+ let scale_data_type = super::normalization::NORMALIZATION_SCALE_DATA_TYPE;+ (+ Some(parameter_tree.leaf(scales_leaf)?.validate(&[head_dim], scale_data_type)?.read_allocation()?),+ scale_data_type,+ )+ } else {+ (None, intermediate_data_type)+ };let scale_free = scales.is_none();- let scale_data_type: DataType = config.scale_precision.into();- let accumulation_data_type: DataType = config.accumulation_precision.into();+ let accumulation_data_type = super::normalization::NORMALIZATION_ACCUMULATION_DATA_TYPE;+ let output_data_type = intermediate_data_type;let kernel = <B::Kernels as Kernels>::QKVNormKernel::new(context,intermediate_data_type,scale_data_type,- scale_data_type,+ output_data_type,accumulation_data_type,true,scale_free,diff --git a/crates/backend-uzu/src/encodable_block/rms_norm.rs b/crates/backend-uzu/src/encodable_block/rms_norm.rsindex 4b1d83134..6c770bf4a 100644--- a/crates/backend-uzu/src/encodable_block/rms_norm.rs+++ b/crates/backend-uzu/src/encodable_block/rms_norm.rs@@ -9,7 +9,7 @@ use crate::{Allocation, Backend, Encoder,kernel::{Kernels, RMSNormKernel},},- config::{NormalizationConfig, UpcastMode},+ config::normalization::{NormalizationConfig, UpcastMode},parameters::{ParameterLoaderError, ParameterTree},};@@ -25,7 +25,7 @@ pub enum RMSNormError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),#[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError<B>),+ ParameterError(#[from] ParameterLoaderError<B>),}pub struct RMSNorm<B: Backend> {@@ -43,19 +43,19 @@ impl<B: Backend> RMSNorm<B> {pub fn new(context: &B::Context,intermediate_data_type: DataType,+ element_count: usize,config: NormalizationConfig,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,hadamard_factors: Option<Allocation<B>>,use_shortcut: bool,residual_add: bool,post_layer_scalar: PostLayerScalar,) -> Result<Self, RMSNormError<B>> {- let scales_leaf = parameter_tree.leaf("scales").map_err(RMSNormError::ParameterError)?;- let element_count = scales_leaf.shape()[0];- let scales = scales_leaf.read_allocation().map_err(RMSNormError::ParameterError)?;+ let scale_data_type = super::normalization::NORMALIZATION_SCALE_DATA_TYPE;+ let scales = parameter_tree.leaf("scales")?.validate(&[element_count], scale_data_type)?.read_allocation()?;- let accumulation_data_type: DataType = config.accumulation_precision.into();- let scale_data_type: DataType = config.scale_precision.into();+ let accumulation_data_type = super::normalization::NORMALIZATION_ACCUMULATION_DATA_TYPE;+ let output_data_type = intermediate_data_type;let (scale_residual_sum, scale_output, post_layer_scalar_value) = match post_layer_scalar {PostLayerScalar::None => (false, false, 1.0),@@ -67,7 +67,7 @@ impl<B: Backend> RMSNorm<B> {context,intermediate_data_type,scale_data_type,- scale_data_type,+ output_data_type,accumulation_data_type,false,config.upcast_mode == UpcastMode::FullLayer,@@ -85,7 +85,7 @@ impl<B: Backend> RMSNorm<B> {scales,element_count,input_data_type: intermediate_data_type,- output_data_type: scale_data_type,+ output_data_type,hadamard_factors,post_layer_scalar_value,})diff --git a/crates/backend-uzu/src/encodable_block/rope.rs b/crates/backend-uzu/src/encodable_block/rope.rsindex 5d603aa80..ef52eadee 100644--- a/crates/backend-uzu/src/encodable_block/rope.rs+++ b/crates/backend-uzu/src/encodable_block/rope.rs@@ -7,6 +7,7 @@ use crate::{Allocation, Backend, Encoder,kernel::{Kernels, RopeKernel},},+ forward_pass::model_shape::ModelShape,};pub struct Rope<B: Backend> {@@ -17,11 +18,15 @@ pub struct Rope<B: Backend> {impl<B: Backend> Rope<B> {pub fn new(context: &B::Context,- data_type: DataType,+ model_shape: &ModelShape,) -> Result<Self, B::Error> {Ok(Self {- kernel: <B::Kernels as Kernels>::RopeKernel::new(context, data_type)?,- data_type,+ kernel: <B::Kernels as Kernels>::RopeKernel::new(+ context,+ model_shape.data_type,+ model_shape.rope_data_type,+ )?,+ data_type: model_shape.data_type,})}diff --git a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rsindex 6545e6050..f39e7d3f6 100644--- a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs@@ -7,7 +7,7 @@ use crate::{Allocation, Backend, Encoder, Kernels,kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},},- config::ShortConvConfig,+ config::token_mixer::short_conv::ShortConvConfig,encodable_block::linear::{Linear, LinearBlockError},forward_pass::short_conv_layer::ShortConvLayer,parameters::{ParameterLoaderError, ParameterTree},@@ -20,14 +20,14 @@ pub enum ShortConvMixerError<B: Backend> {#[error("Unsupported configuration: {0}")]UnsupportedConfiguration(String),#[error("Linear error: {0}")]- LinearError(#[from] Box<LinearBlockError<B>>),+ LinearError(#[from] LinearBlockError<B>),#[error("Parameter loader error: {0}")]ParameterLoaderError(#[from] ParameterLoaderError<B>),}pub struct ShortConvMixer<B: Backend> {- config: ShortConvConfig,model_dim: usize,+ kernel_size: usize,in_projection: Box<dyn Linear<B>>,out_projection: Box<dyn Linear<B>>,short_conv_pack: <B::Kernels as Kernels>::ShortConvPackKernel,@@ -50,9 +50,10 @@ pub struct ShortConvArguments<'a, B: Backend> {impl<B: Backend> ShortConvMixer<B> {pub fn new(context: &B::Context,- short_conv_config: ShortConvConfig,+ short_conv_config: &ShortConvConfig,model_dim: usize,- decoder_layer_loader: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,+ data_type: DataType,) -> Result<(Self, Option<Allocation<B>>), ShortConvMixerError<B>> {if short_conv_config.kernel_size < 2 {return Err(ShortConvMixerError::UnsupportedConfiguration(format!(@@ -61,32 +62,32 @@ impl<B: Backend> ShortConvMixer<B> {)));}- let mixer_tree = decoder_layer_loader.subtree("mixer")?;- let conv_tree = mixer_tree.subtree("conv")?;-- let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();+ let conv_tree = parameter_tree.subtree("conv")?;let (in_projection, in_proj_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(- &short_conv_config.in_projection_config,model_dim,[model_dim * 3],+ false,context,- &mixer_tree.subtree("in_projection")?,- )- .map_err(|err| ShortConvMixerError::LinearError(Box::new(err)))?;+ data_type,+ ¶meter_tree.subtree("in_projection")?,+ )?;let out_projection = <dyn Linear<B>>::new(- &short_conv_config.out_projection_config,model_dim,[model_dim],+ false,context,- &mixer_tree.subtree("out_projection")?,- )- .map_err(|err| ShortConvMixerError::LinearError(Box::new(err)))?;-- let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;+ data_type,+ ¶meter_tree.subtree("out_projection")?,+ )?;++ let conv_weight = conv_tree+ .leaf("weights")?+ .validate(&[model_dim, short_conv_config.kernel_size], data_type)?+ .read_allocation()?;let conv_bias = if short_conv_config.conv_config.has_biases {- Some(conv_tree.leaf("biases")?.read_allocation()?)+ Some(conv_tree.leaf("biases")?.validate(&[model_dim], data_type)?.read_allocation()?)} else {None};@@ -103,8 +104,8 @@ impl<B: Backend> ShortConvMixer<B> {Ok((Self {- config: short_conv_config,model_dim,+ kernel_size: short_conv_config.kernel_size,in_projection,out_projection,short_conv_pack,@@ -174,7 +175,7 @@ impl<B: Backend> ShortConvMixer<B> {return Ok(());}- let kernel_size = self.config.kernel_size;+ let kernel_size = self.kernel_size;let state_stride = kernel_size - 1;let padded_rows = state_stride + suffix_length;@@ -220,9 +221,9 @@ impl<B: Backend> ShortConvMixer<B> {return Ok(());}- let elem_bytes = DataType::from(self.config.in_projection_config.activation_precision()).size_in_bytes();+ let elem_bytes = self.data_type.size_in_bytes();- let kernel_size = self.config.kernel_size;+ let kernel_size = self.kernel_size;let state_stride = kernel_size - 1;let in_proj_stride = self.model_dim * 3;@@ -259,7 +260,7 @@ impl<B: Backend> ShortConvMixer<B> {return Ok(());}- let kernel_size = self.config.kernel_size;+ let kernel_size = self.kernel_size;let state_stride = kernel_size - 1;self.short_conv_decode.encode(diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rsindex de84ab441..8c63187dc 100644--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs@@ -3,7 +3,7 @@ use std::cell::Cell;use crate::{array::size_for_shape,backends::common::{AllocationType, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},- config::MixerConfig,+ config::token_mixer::AnyTokenMixerConfig,forward_pass::{delta_net_layer::DeltaNetLayer,kv_cache_layer::{KVCacheLayerState, KVCacheLayerTrait, KVSlice},@@ -139,10 +139,11 @@ impl<B: Backend> CacheLayers<B> {entry: source,} = bindings[source_layer_index]else {- panic!("kv_source_layer {source_layer_index} (layer {layer_index}) must be a prior owner");+ panic!("kv_source_layer_index {source_layer_index} (layer {layer_index}) must be a prior owner");+ };+ let geometry = |m: &AnyTokenMixerConfig| {+ m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size))};- let geometry =- |m: &MixerConfig| m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size));assert_eq!(geometry(&model_shape.layer_mixers()[source_layer_index]),geometry(mixer),@@ -155,7 +156,7 @@ impl<B: Backend> CacheLayers<B> {}let layer = match mixer {- MixerConfig::Attention(attn) => {+ AnyTokenMixerConfig::AttentionConfig(attn) => {let sliding_window = attn.sliding_window_size;let length = sliding_window.unwrap_or(max_prefix_length);let shape = [length + max_suffix_length, attn.num_groups, attn.head_dim];@@ -172,18 +173,15 @@ impl<B: Backend> CacheLayers<B> {prefix_len: 0,}};-- let kv_layer =- <dyn KVCacheLayerTrait<B>>::new(context, &state, shape, model_shape.kv_cache_data_type())- .expect("Failed to create KVCacheLayer");+ let kv_layer = <dyn KVCacheLayerTrait<B>>::new(context, &state, shape, model_shape.data_type)+ .expect("Failed to create KVCacheLayer");CacheLayer::Transformer(kv_layer)},- MixerConfig::Mamba(c) => {+ AnyTokenMixerConfig::Mamba2Config(c) => {let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];let ssm_shape = [c.num_heads, c.head_dim, c.state_dim];- let dtype = model_shape.activation_data_type();- let conv_bytes = size_for_shape(&conv_shape, dtype);- let ssm_bytes = size_for_shape(&ssm_shape, dtype);+ let conv_bytes = size_for_shape(&conv_shape, model_shape.data_type);+ let ssm_bytes = size_for_shape(&ssm_shape, model_shape.data_type);CacheLayer::StateSpace(SSMLayer {conv_state: (conv_bytes > 0).then(|| {@@ -196,16 +194,15 @@ impl<B: Backend> CacheLayers<B> {.create_allocation(ssm_bytes, AllocationType::Global).expect("Failed to create ssm state allocation"),ssm_shape,- data_type: dtype,+ data_type: model_shape.data_type,})},- MixerConfig::ShortConv(c) => {+ AnyTokenMixerConfig::ShortConvConfig(c) => {assert!(c.kernel_size >= 2, "ShortConv kernel_size must be >= 2, got {}", c.kernel_size);let conv_shape = [model_shape.model_dim(), c.kernel_size - 1];let suffix_state_shape = [max_suffix_length, model_shape.model_dim(), c.kernel_size - 1];- let dtype = model_shape.activation_data_type();- let conv_bytes = size_for_shape(&conv_shape, dtype);- let suffix_bytes = size_for_shape(&suffix_state_shape, dtype);+ let conv_bytes = size_for_shape(&conv_shape, model_shape.data_type);+ let suffix_bytes = size_for_shape(&suffix_state_shape, model_shape.data_type);CacheLayer::ShortConv(ShortConvLayer {conv_state: context@@ -216,17 +213,16 @@ impl<B: Backend> CacheLayers<B> {.create_allocation(suffix_bytes, AllocationType::Global).expect("Failed to create short conv suffix allocation"),suffix_shape: suffix_state_shape,- data_type: dtype,+ data_type: model_shape.data_type,suffix_state_valid_start: Cell::new(0),suffix_state_valid_len: Cell::new(0),})},- MixerConfig::DeltaNet(c) => {+ AnyTokenMixerConfig::DeltaNetConfig(c) => {let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];let ssm_shape = [c.num_heads, c.value_head_dim, c.head_dim];- let dtype = model_shape.activation_data_type();- let conv_bytes = size_for_shape(&conv_shape, dtype);- let ssm_bytes = size_for_shape(&ssm_shape, dtype);+ let conv_bytes = size_for_shape(&conv_shape, model_shape.data_type);+ let ssm_bytes = size_for_shape(&ssm_shape, model_shape.data_type);CacheLayer::DeltaNet(DeltaNetLayer {conv_state: context@@ -237,7 +233,7 @@ impl<B: Backend> CacheLayers<B> {.create_allocation(ssm_bytes, AllocationType::Global).expect("Failed to create delta net ssm allocation"),ssm_shape,- data_type: dtype,+ data_type: model_shape.data_type,})},};diff --git a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rsindex dbd1fb688..5a769a090 100644--- a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs+++ b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs@@ -52,7 +52,7 @@ pub trait KVCacheLayerTrait<B: Backend> {fn encode_zero(&mut self,- encoder: &mut Encoder<'_, B>,+ encoder: &mut Encoder<B>,);fn state(&self) -> KVCacheLayerState;@@ -267,7 +267,7 @@ impl<B: Backend, Buf: Buffer<Backend = B>> KVCacheLayerTrait<B> for KVCacheLayerfn encode_zero(&mut self,- encoder: &mut Encoder<'_, B>,+ encoder: &mut Encoder<B>,) {if self.as_any().is::<KVCacheLayer<B, B::SparseBuffer>>() {return;diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rsindex 8efdfe4fc..57b50f9d9 100644--- a/crates/backend-uzu/src/forward_pass/model_shape.rs+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs@@ -1,12 +1,12 @@use crate::{DataType,- config::{DecoderConfig, MLPConfig, MixerConfig},+ config::{decoder::DecoderConfig, token_mixer::AnyTokenMixerConfig},};#[derive(Debug, Clone)]pub struct ModelShape {- activation_type: DataType,- kv_cache_type: DataType,+ pub data_type: DataType,+ pub rope_data_type: DataType,vocabulary_size: usize,model_dim: usize,@@ -14,12 +14,15 @@ pub struct ModelShape {num_groups: usize,pub num_layers: usize,- layer_mixers: Box<[MixerConfig]>,+ layer_mixers: Box<[AnyTokenMixerConfig]>,kv_source_layers: Box<[Option<usize>]>,}impl ModelShape {- pub fn from_decoder_config(decoder_config: &DecoderConfig) -> Self {+ pub fn from_decoder_config(+ decoder_config: &DecoderConfig,+ data_type: DataType,+ ) -> Self {let tf = &decoder_config.transformer_config;let layer_configs = &tf.layer_configs;let num_layers = layer_configs.len();@@ -31,17 +34,12 @@ impl ModelShape {assert_eq!(attn.num_groups, num_groups, "attention layers must share num_groups");}- let activation_type: DataType = match &layer_configs[0].mlp_config {- MLPConfig::Dense(d) => d.linear_config.activation_precision().into(),- MLPConfig::MixtureOfExperts(m) => m.expert_config.linear_config.activation_precision().into(),- };-- let layer_mixers: Box<[MixerConfig]> = layer_configs.iter().map(|l| l.mixer_config.clone()).collect();- let kv_source_layers: Box<[Option<usize>]> = layer_configs.iter().map(|l| l.kv_source_layer).collect();+ let layer_mixers: Box<[AnyTokenMixerConfig]> = layer_configs.iter().map(|l| l.mixer_config.clone()).collect();+ let kv_source_layers: Box<[Option<usize>]> = layer_configs.iter().map(|l| l.kv_source_layer_index).collect();Self {- activation_type,- kv_cache_type: activation_type,+ data_type,+ rope_data_type: DataType::F32,vocabulary_size: decoder_config.vocab_size,model_dim: tf.model_dim,context_length: tf.context_length,@@ -52,14 +50,6 @@ impl ModelShape {}}- pub fn activation_data_type(&self) -> DataType {- self.activation_type- }-- pub fn kv_cache_data_type(&self) -> DataType {- self.kv_cache_type- }-pub fn num_groups(&self) -> usize {self.num_groups}@@ -72,7 +62,7 @@ impl ModelShape {&self.kv_source_layers}- pub fn layer_mixers(&self) -> &[MixerConfig] {+ pub fn layer_mixers(&self) -> &[AnyTokenMixerConfig] {&self.layer_mixers}diff --git a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rsindex 678e3c8a7..0a7e470bb 100644--- a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs+++ b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs@@ -13,6 +13,7 @@ pub struct RopeBuffers<B: Backend> {pub sines: Allocation<B>,max_sequence_length: usize,dim: usize,+ data_type: DataType,}impl<B: Backend> RopeBuffers<B> {@@ -28,18 +29,29 @@ impl<B: Backend> RopeBuffers<B> {sines: context.create_array_uninitialized(&shape, data_type).into_allocation(),max_sequence_length,dim: head_dim,+ data_type,}}pub fn update_data(&mut self,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,rope_index: usize,) -> Result<(), Error> {- let rope_tree =- parameter_tree.subtree(&format!("ropes.{}", rope_index)).map_err(|_| Error::UnableToLoadWeights)?;- self.cosines = rope_tree.leaf_allocation("cosines").map_err(|_| Error::UnableToLoadWeights)?;- self.sines = rope_tree.leaf_allocation("sines").map_err(|_| Error::UnableToLoadWeights)?;+ let rope_tree = parameter_tree+ .subtree(&format!("ropes.{}", rope_index))+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let cosines_leaf = rope_tree.leaf("cosines").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let cosines_leaf = cosines_leaf+ .validate(&[self.max_sequence_length, self.dim], self.data_type)+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ self.cosines = cosines_leaf.read_allocation().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;++ let sines_leaf = rope_tree.leaf("sines").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let sines_leaf = sines_leaf+ .validate(&[self.max_sequence_length, self.dim], self.data_type)+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ self.sines = sines_leaf.read_allocation().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;Ok(())}diff --git a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rsindex 37652bd64..de58f52a8 100644--- a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs+++ b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs@@ -1,11 +1,8 @@-use half::{bf16, f16};-use super::RopeBuffers;use crate::{- DataType,- array::ArrayContextExt,- backends::common::{Allocation, AsBufferRangeMut, Backend, DenseBuffer},- config::{DecoderConfig, RoPEConfig},+ backends::common::Backend,+ config::{decoder::DecoderConfig, rope::AnyRoPEConfig},+ forward_pass::model_shape::ModelShape,parameters::ParameterTree,session::types::Error,};@@ -19,114 +16,71 @@ enum LayerRopeKind {pub struct SharedBuffers<B: Backend> {pub rope_buffers: Box<[RopeBuffers<B>]>,layer_rope_kinds: Box<[LayerRopeKind]>,- pub attention_sinks: Box<[Option<Allocation<B>>]>,}impl<B: Backend> SharedBuffers<B> {pub fn new(context: &B::Context,decoder_config: &DecoderConfig,+ model_shape: &ModelShape,) -> Self {let tf = &decoder_config.transformer_config;- let mut configs = Vec::<RoPEConfig>::new();+ let mut configs = Vec::<(AnyRoPEConfig, usize)>::new();let layer_rope_kinds: Box<[LayerRopeKind]> = tf.layer_configs.iter().map(|layer_config| {- if layer_config.mixer_config.as_attention().is_none() {+ let Some(attention_config) = layer_config.mixer_config.as_attention() else {return LayerRopeKind::NoKernel;- }+ };let Some(rope_config) = &layer_config.rope_config else {return LayerRopeKind::NoKernel;};- let index = configs.iter().position(|existing| existing == rope_config).unwrap_or_else(|| {- configs.push(rope_config.clone());- configs.len() - 1- });+ let head_dim = rope_config.head_dim().unwrap_or(attention_config.head_dim);+ let index = configs+ .iter()+ .position(|(existing_config, existing_head_dim)| {+ existing_config == rope_config && *existing_head_dim == head_dim+ })+ .unwrap_or_else(|| {+ configs.push((rope_config.clone(), head_dim));+ configs.len() - 1+ });LayerRopeKind::Indexed(index)}).collect();let rope_buffers: Box<[RopeBuffers<B>]> = configs.iter()- .map(|config| {- let common = config.common();- RopeBuffers::new(context, common.max_sequence_length, common.head_dim, common.precision.into())- })- .collect();-- let attention_sinks = tf- .layer_configs- .iter()- .map(|layer| {- let attn = layer.mixer_config.as_attention()?;- attn.has_sinks- .then(|| context.create_array_uninitialized(&[attn.num_heads], DataType::F32).into_allocation())+ .map(|(config, head_dim)| {+ RopeBuffers::new(context, *config.max_sequence_length(), *head_dim, model_shape.rope_data_type)}).collect();Self {rope_buffers,layer_rope_kinds,- attention_sinks,}}pub fn update_data(&mut self,- parameter_tree: &ParameterTree<B::Context>,+ parameter_tree: &ParameterTree<B>,) -> Result<(), Error> {- let transformer_tree = parameter_tree.subtree("transformer").map_err(|_| Error::UnableToLoadWeights)?;+ let transformer_tree =+ parameter_tree.subtree("transformer").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;self.update_data_from_transformer_tree(&transformer_tree)?;Ok(())}pub fn update_data_from_transformer_tree(&mut self,- transformer_tree: &ParameterTree<B::Context>,+ transformer_tree: &ParameterTree<B>,) -> Result<(), Error> {for (rope_index, rope_buffers) in self.rope_buffers.iter_mut().enumerate() {rope_buffers.update_data(transformer_tree, rope_index)?;}- for (layer_idx, sink_cell) in self.attention_sinks.iter_mut().enumerate() {- let Some(sink_cell) = sink_cell.as_mut() else {- continue;- };- let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_idx)).unwrap();- let attn_tree = layer_tree.subtree("mixer").unwrap();- let sinks_arr = attn_tree.leaf_array("sinks").unwrap();- let dst_slice = unsafe {- let buffer_range = sink_cell.as_buffer_range_mut();- let range = buffer_range.range();- std::slice::from_raw_parts_mut(- (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start) as *mut f32,- range.len() / std::mem::size_of::<f32>(),- )- };-- match sinks_arr.data_type() {- DataType::F32 => {- let src = sinks_arr.as_slice::<f32>();- dst_slice.copy_from_slice(src);- },- DataType::BF16 => {- let src = sinks_arr.as_slice::<bf16>();- for (dst_val, src_val) in dst_slice.iter_mut().zip(src.iter()) {- *dst_val = f32::from(*src_val);- }- },- DataType::F16 => {- let src = sinks_arr.as_slice::<f16>();- for (dst_val, src_val) in dst_slice.iter_mut().zip(src.iter()) {- *dst_val = f32::from(*src_val);- }- },- other => {- panic!("Unsupported attention sink data type: {:?}", other);- },- }- }Ok(())}@@ -139,11 +93,4 @@ impl<B: Backend> SharedBuffers<B> {LayerRopeKind::Indexed(index) => Some(&self.rope_buffers[index]),}}-- pub fn attention_sinks(- &self,- layer_index: usize,- ) -> Option<&Allocation<B>> {- self.attention_sinks.get(layer_index)?.as_ref()- }}diff --git a/crates/backend-uzu/src/forward_pass/traces.rs b/crates/backend-uzu/src/forward_pass/traces.rsindex 820073ff1..4b3474955 100644--- a/crates/backend-uzu/src/forward_pass/traces.rs+++ b/crates/backend-uzu/src/forward_pass/traces.rs@@ -31,8 +31,7 @@ impl<B: Backend> LayerActivationTrace<B> {suffix_length: usize,) -> Self {let main_shape = model_shape.main_shape(suffix_length);- let activation_data_type = model_shape.activation_data_type();- let main = || context.create_array_uninitialized(&main_shape, activation_data_type);+ let main = || context.create_array_uninitialized(&main_shape, model_shape.data_type);Self {inputs: main(),@@ -62,16 +61,15 @@ impl<B: Backend> ActivationTrace<B> {model_shape: &ModelShape,suffix_length: usize,) -> Self {- let activation_data_type = model_shape.activation_data_type();let main_shape = model_shape.main_shape(suffix_length);let layer_results = create_layer_results(context, model_shape, suffix_length);Self {embedding_norm: None,layer_results,- output_norm: context.create_array_uninitialized(&main_shape, activation_data_type),+ output_norm: context.create_array_uninitialized(&main_shape, model_shape.data_type),output_pooling: None,- logits: context.create_array_uninitialized(&model_shape.logits_shape(suffix_length), activation_data_type),+ logits: context.create_array_uninitialized(&model_shape.logits_shape(suffix_length), model_shape.data_type),}}@@ -81,17 +79,16 @@ impl<B: Backend> ActivationTrace<B> {suffix_length: usize,num_labels: usize,) -> Self {- let activation_data_type = model_shape.activation_data_type();let main_shape = model_shape.main_shape(suffix_length);let layer_results = create_layer_results(context, model_shape, suffix_length);let model_dim = model_shape.main_shape(1)[1];Self {- embedding_norm: Some(context.create_array_uninitialized(&main_shape, activation_data_type)),+ embedding_norm: Some(context.create_array_uninitialized(&main_shape, model_shape.data_type)),layer_results,- output_norm: context.create_array_uninitialized(&main_shape, activation_data_type),- output_pooling: Some(context.create_array_uninitialized(&[1, model_dim], activation_data_type)),- logits: context.create_array_uninitialized(&[1, num_labels], activation_data_type),+ output_norm: context.create_array_uninitialized(&main_shape, model_shape.data_type),+ output_pooling: Some(context.create_array_uninitialized(&[1, model_dim], model_shape.data_type)),+ logits: context.create_array_uninitialized(&[1, num_labels], model_shape.data_type),}}diff --git a/crates/backend-uzu/src/inference/chat/bridging.rs b/crates/backend-uzu/src/inference/chat/bridging.rsindex 6cec0db3d..036e4e254 100644--- a/crates/backend-uzu/src/inference/chat/bridging.rs+++ b/crates/backend-uzu/src/inference/chat/bridging.rs@@ -58,8 +58,7 @@ pub fn build_decoding_config(Some(ShojiSpeculationPreset::Classification {feature,}) => {- let tokenizer =- Tokenizer::from_file(model_path.join("tokenizer.json")).map_err(|_| Error::UnableToLoadTokenizer)?;+ let tokenizer = Tokenizer::from_file(model_path.join("tokenizer.json"))?;let proposals: Vec<Vec<u64>> = feature.values.iter()diff --git a/crates/backend-uzu/src/inference/chat/mod.rs b/crates/backend-uzu/src/inference/chat/mod.rsindex 072f40d04..473451ce6 100644--- a/crates/backend-uzu/src/inference/chat/mod.rs+++ b/crates/backend-uzu/src/inference/chat/mod.rs@@ -41,7 +41,7 @@ impl Instance {) -> Result<Self, Error> {let model_path = PathBuf::from(reference.clone());let decoding_config = build_decoding_config(&config, &model_path)?;- let session = ChatSession::new(model_path, decoding_config).map_err(Error::from)?;+ let session = ChatSession::new(model_path, decoding_config)?;let container = Container::new(session);Ok(Self {session_container: container,@@ -62,9 +62,12 @@ impl InstanceTrait for Instance {.map_err(|error| Error::Runtime {message: error.to_string(),})- .and_then(|mut session| session.reset().map_err(Error::from));+ .and_then(|mut session| {+ session.reset()?;+ Ok(())+ });Box::pin(async move {- result.map_err(|error| Box::new(error) as BackendError)?;+ result?;Ok(Box::new(State) as Box<dyn StateTrait>)})}diff --git a/crates/backend-uzu/src/inference/classification/mod.rs b/crates/backend-uzu/src/inference/classification/mod.rsindex 073045065..496d540a8 100644--- a/crates/backend-uzu/src/inference/classification/mod.rs+++ b/crates/backend-uzu/src/inference/classification/mod.rs@@ -33,7 +33,7 @@ pub struct Instance {impl Instance {pub fn new(reference: String) -> Result<Self, Error> {let model_path = PathBuf::from(reference);- let session = ClassificationSession::new(model_path).map_err(Error::from)?;+ let session = ClassificationSession::new(model_path)?;let container = Container::new(session);Ok(Self {session_container: container,diff --git a/crates/backend-uzu/src/inference/error.rs b/crates/backend-uzu/src/inference/error.rsindex dfc8f2b21..50100b5a4 100644--- a/crates/backend-uzu/src/inference/error.rs+++ b/crates/backend-uzu/src/inference/error.rs@@ -10,8 +10,8 @@ pub enum Error {Session {message: String,},- #[error("Unable to load tokenizer")]- UnableToLoadTokenizer,+ #[error("Unable to load tokenizer: {0}")]+ UnableToLoadTokenizer(#[from] tokenizers::Error),}impl From<ChatSessionError> for Error {diff --git a/crates/backend-uzu/src/inference/mod.rs b/crates/backend-uzu/src/inference/mod.rsindex 87d6bea05..d334ebf2e 100644--- a/crates/backend-uzu/src/inference/mod.rs+++ b/crates/backend-uzu/src/inference/mod.rs@@ -11,7 +11,7 @@ pub use chat::Instance as ChatInstance;pub use classification::Instance as ClassificationInstance;pub use container::Container;pub use error::Error;-pub use model_metadata::{ModelMetadata, resolve_model_metadata};+pub use model_metadata::{ModelMetadataError, resolve_model_specialization};use shoji::{traits::{Backend as BackendTrait,@@ -63,7 +63,7 @@ impl chat_message_trait::Backend for Backend {config: ShojiChatConfig,) -> Pin<Box<dyn Future<Output = Result<Box<dyn chat_message_trait::Instance>, BackendError>> + Send + '_>> {Box::pin(async move {- let instance = ChatInstance::new(reference, config).map_err(|error| Box::new(error) as BackendError)?;+ let instance = ChatInstance::new(reference, config)?;Ok(Box::new(instance) as Box<dyn chat_message_trait::Instance>)})}@@ -76,7 +76,7 @@ impl classification_trait::Backend for Backend {_config: classification_trait::Config,) -> Pin<Box<dyn Future<Output = Result<Box<dyn classification_trait::Instance>, BackendError>> + Send + '_>> {Box::pin(async move {- let instance = ClassificationInstance::new(reference).map_err(|error| Box::new(error) as BackendError)?;+ let instance = ClassificationInstance::new(reference)?;Ok(Box::new(instance) as Box<dyn classification_trait::Instance>)})}@@ -89,7 +89,7 @@ impl text_to_speech_trait::Backend for Backend {_config: text_to_speech_trait::Config,) -> Pin<Box<dyn Future<Output = Result<Box<dyn text_to_speech_trait::Instance>, BackendError>> + Send + '_>> {Box::pin(async move {- let instance = TextToSpeechInstance::new(reference).map_err(|error| Box::new(error) as BackendError)?;+ let instance = TextToSpeechInstance::new(reference)?;Ok(Box::new(instance) as Box<dyn text_to_speech_trait::Instance>)})}diff --git a/crates/backend-uzu/src/inference/model_metadata.rs b/crates/backend-uzu/src/inference/model_metadata.rsindex a71afb779..adf2f5030 100644--- a/crates/backend-uzu/src/inference/model_metadata.rs+++ b/crates/backend-uzu/src/inference/model_metadata.rs@@ -2,24 +2,23 @@ use std::{fs::File, io::BufReader, path::Path};use shoji::types::model::ModelSpecialization;-use crate::config::{ModelConfig, ModelMetadata as InnerModelMetadata, ModelType};+use crate::config::model::AnyModelConfig;-pub struct ModelMetadata {- pub toolchain_version: String,- pub specialization: ModelSpecialization,+#[derive(Debug, thiserror::Error)]+pub enum ModelMetadataError {+ #[error("Unable to open model configuration: {0}")]+ UnableToOpenConfig(#[from] std::io::Error),+ #[error("Unable to deserialize model configuration: {0}")]+ UnableToDeserializeConfig(#[from] serde_json::Error),}-pub fn resolve_model_metadata(model_path: &Path) -> Option<ModelMetadata> {+pub fn resolve_model_specialization(model_path: &Path) -> Result<ModelSpecialization, ModelMetadataError> {let config_path = model_path.join("config.json");- let file = File::open(&config_path).ok()?;- let metadata: InnerModelMetadata<ModelConfig> = serde_json::from_reader(BufReader::new(file)).ok()?;- let specialization = match metadata.model_type {- ModelType::LanguageModel => ModelSpecialization::Chat {},- ModelType::ClassifierModel => ModelSpecialization::Classification {},- ModelType::TtsModel => ModelSpecialization::TextToSpeech {},- };- Some(ModelMetadata {- toolchain_version: metadata.toolchain_version,- specialization,+ let file = File::open(&config_path)?;+ let config: AnyModelConfig = serde_json::from_reader(BufReader::new(file))?;+ Ok(match config {+ AnyModelConfig::LanguageModelConfig(_) => ModelSpecialization::Chat {},+ AnyModelConfig::ClassifierModelConfig(_) => ModelSpecialization::Classification {},+ AnyModelConfig::TTSModelConfig(_) => ModelSpecialization::TextToSpeech {},})}diff --git a/crates/backend-uzu/src/inference/text_to_speech/mod.rs b/crates/backend-uzu/src/inference/text_to_speech/mod.rsindex f46d3a466..33c7e1f66 100644--- a/crates/backend-uzu/src/inference/text_to_speech/mod.rs+++ b/crates/backend-uzu/src/inference/text_to_speech/mod.rs@@ -36,19 +36,16 @@ use crate::{session::{TtsSession, config::TtsRunConfig, types::Input},};-#[cfg(metal_backend)]-type UzuTtsSession = TtsSession<Metal>;-pub struct Instance {#[cfg(metal_backend)]- session_container: Container<UzuTtsSession>,+ session_container: Container<TtsSession<Metal>>,}impl Instance {#[cfg(metal_backend)]pub fn new(reference: String) -> Result<Self, Error> {let model_path = std::path::PathBuf::from(reference);- let session = UzuTtsSession::new(model_path).map_err(Error::from)?;+ let session = TtsSession::<Metal>::new(model_path)?;Ok(Self {session_container: Container::new(session),})@@ -109,7 +106,7 @@ impl InstanceTrait for Instance {#[cfg(metal_backend)]fn run(- session_container: Container<UzuTtsSession>,+ session_container: Container<TtsSession<Metal>>,input: Input,_cancel_token: CancellationToken,sender: UnboundedSender<Result<ShojiPcmBatch, BackendError>>,diff --git a/crates/backend-uzu/src/language_model/gpu_capture.rs b/crates/backend-uzu/src/language_model/gpu_capture.rsindex d7e13f5ef..645d5508d 100644--- a/crates/backend-uzu/src/language_model/gpu_capture.rs+++ b/crates/backend-uzu/src/language_model/gpu_capture.rs@@ -9,6 +9,16 @@ use crate::{utils::env_utils::EnvVar,};+#[derive(Debug, thiserror::Error)]+pub enum GpuCaptureError<B: Backend> {+ #[error("Unable to resolve capture timestamp: {0}")]+ Timestamp(#[from] std::time::SystemTimeError),+ #[error("Unable to get current directory: {0}")]+ CurrentDirectory(#[from] std::io::Error),+ #[error("Backend capture failed: {0}")]+ Backend(#[source] B::Error),+}+pub struct GpuCaptureManager<B: Backend> {capture_prefill_enabled: bool,capture_decode_enabled: bool,@@ -54,15 +64,12 @@ impl<B: Backend> GpuCaptureManager<B> {&self,context: &B::Context,capture_type: &str,- ) -> Result<PathBuf, String> {- let timestamp =- SystemTime::now().duration_since(UNIX_EPOCH).map_err(|e| format!("Time error: {}", e))?.as_secs();+ ) -> Result<PathBuf, GpuCaptureError<B>> {+ let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();- let trace_path = std::env::current_dir()- .unwrap_or_else(|_| PathBuf::from("."))- .join(format!("uzu_first_{}-{}.gputrace", capture_type, timestamp));+ let trace_path = std::env::current_dir()?.join(format!("uzu_first_{}-{}.gputrace", capture_type, timestamp));- context.start_capture(&trace_path).map_err(|err| err.to_string())?;+ context.start_capture(&trace_path).map_err(GpuCaptureError::Backend)?;println!("GPU capture started for first {}: {:?}", capture_type, trace_path);@@ -73,8 +80,8 @@ impl<B: Backend> GpuCaptureManager<B> {&mut self,context: &B::Context,capture_type: &str,- ) -> Result<(), String> {- context.stop_capture().map_err(|err| err.to_string())?;+ ) -> Result<(), GpuCaptureError<B>> {+ context.stop_capture().map_err(GpuCaptureError::Backend)?;println!("GPU capture stopped for {}", capture_type);match capture_type {diff --git a/crates/backend-uzu/src/language_model/grammar/mod.rs b/crates/backend-uzu/src/language_model/grammar/mod.rsindex bf464d854..4082cc2d2 100644--- a/crates/backend-uzu/src/language_model/grammar/mod.rs+++ b/crates/backend-uzu/src/language_model/grammar/mod.rs@@ -29,8 +29,8 @@ pub fn create_compiled_grammar(#[cfg(grammar_xgrammar)]{use xgrammar::TokenizerInfo;- let tokenizer_info = TokenizerInfo::from_huggingface(&tokenizer, None, stop_token_ids)- .map_err(|msg| Error::GrammarError(msg))?;+ let tokenizer_info =+ TokenizerInfo::from_huggingface(&tokenizer, None, stop_token_ids).map_err(Error::GrammarError)?;use xgram::CompiledXGrammar;let grammar = CompiledXGrammar::from_config(config, None, &tokenizer_info)?;diff --git a/crates/backend-uzu/src/language_model/grammar/xgram.rs b/crates/backend-uzu/src/language_model/grammar/xgram.rsindex 93cafc8dc..41334a79a 100644--- a/crates/backend-uzu/src/language_model/grammar/xgram.rs+++ b/crates/backend-uzu/src/language_model/grammar/xgram.rs@@ -89,21 +89,17 @@ impl CompiledXGrammar {} => {let separators_ref = separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));Grammar::from_json_schema(schema, *any_whitespace, *indent, separators_ref, *strict_mode, None, false)- .map_err(|error_message| Error::GrammarError(error_message))?+ .map_err(Error::GrammarError)?},GrammarConfig::Regex {pattern,print_converted_ebnf,- } => Grammar::from_regex(pattern, *print_converted_ebnf)- .map_err(|error_message| Error::GrammarError(error_message))?,+ } => Grammar::from_regex(pattern, *print_converted_ebnf).map_err(Error::GrammarError)?,GrammarConfig::BuiltinJson => Grammar::builtin_json_grammar(),};- let mut compiler = GrammarCompiler::new(tokenizer_info, 8, true, -1)- .map_err(|error_message| Error::GrammarError(error_message))?;- let compiled =- compiler.compile_grammar(&grammar).map_err(|error_message| Error::GrammarError(error_message))?;- let matcher = GrammarMatcher::new(&compiled, None, true, -1)- .map_err(|error_message| Error::GrammarError(error_message))?;+ let mut compiler = GrammarCompiler::new(tokenizer_info, 8, true, -1).map_err(Error::GrammarError)?;+ let compiled = compiler.compile_grammar(&grammar).map_err(Error::GrammarError)?;+ let matcher = GrammarMatcher::new(&compiled, None, true, -1).map_err(Error::GrammarError)?;let engagement_state = if let Some(trigger_token_id) = trigger_token_id {CompiledGrammarEngagementState::Triggered {diff --git a/crates/backend-uzu/src/language_model/gumbel.rs b/crates/backend-uzu/src/language_model/gumbel.rsindex b2fcb18c6..651c8dcbb 100644--- a/crates/backend-uzu/src/language_model/gumbel.rs+++ b/crates/backend-uzu/src/language_model/gumbel.rs@@ -83,6 +83,6 @@ pub fn speculator_sample(speculator_probs.iter().map(|(&k, &v)| (k, f32::ln(v) + gumbel_float(seed, revidx(k as u32))))- .max_by(|(_ak, av), (_bk, bv)| f32::total_cmp(av, bv))- .map(|(k, _v)| k)+ .max_by(|(_, av), (_, bv)| f32::total_cmp(av, bv))+ .map(|(k, _)| k)}diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rsindex 111f6f5f3..85747dc7f 100644--- a/crates/backend-uzu/src/language_model/language_model_generator.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs@@ -9,12 +9,13 @@ use super::{rng::PRng,};use crate::{+ DataType,array::{Array, ArrayContextExt},backends::common::{Allocation, AsBufferRangeRef, Backend, CommandBuffer, Context, DenseBuffer, Encoder, Pending,kernel::TokenCopySampledKernel,},- config::{LanguageModelConfig, ModelMetadata},+ config::model::language_model::LanguageModelConfig,encodable_block::{DecoderDecodeInput, SamplingArguments, SamplingInputs},forward_pass::{cache_layers::CacheLayersSlice, kv_cache_layer::INVALID_POSITION, token_inputs::TokenInputs},language_model::grammar::CompiledGrammar,@@ -97,6 +98,7 @@ pub trait LanguageModelGeneratorTrait {fn reset_state(&mut self);fn peak_memory_usage(&self) -> Option<usize>;+ fn activation_data_type(&self) -> DataType;fn tokens_len(&self) -> usize;fn tokens_push(@@ -107,7 +109,7 @@ pub trait LanguageModelGeneratorTrait {fn async_batch_size(&self,model_path: &Path,- ) -> usize;+ ) -> Result<usize, Error>;fn get_slice(&self,@@ -262,7 +264,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let should_capture = self.gpu_capture.should_capture_prefill(step == 0);if should_capture {- let _ = self.gpu_capture.start_capture(&self.context.context, "prefill");+ self.gpu_capture+ .start_capture(&self.context.context, "prefill")+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;}let _ = last_sampling_output.take();@@ -282,7 +286,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let (sampling_output, run_time) = self.run_model(task, sampling_method)?;if should_capture {- self.gpu_capture.stop_capture(&self.context.context, "prefill").map_err(|_| Error::CaptureFailed)?;+ self.gpu_capture+ .stop_capture(&self.context.context, "prefill")+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;}// Register the accepted prompt tokens from this step.@@ -492,7 +498,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let is_first_decode = !is_continuation;let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);if should_capture {- let _ = self.gpu_capture.start_capture(&self.context.context, "decode");+ self.gpu_capture+ .start_capture(&self.context.context, "decode")+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;}let batch_dim = task.active_row_count;@@ -558,7 +566,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let completed = pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;drop(resources);drop(completed);- self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;+ self.gpu_capture+ .stop_capture(&self.context.context, "decode")+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;} else {assert!(self.async_in_flight[slot].is_none(), "async slot {slot} still holds an in-flight state");self.async_in_flight[slot] = Some(InFlightForwardPass {@@ -601,6 +611,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.context.peak_memory_usage()}+ fn activation_data_type(&self) -> DataType {+ self.context.model_shape.data_type+ }+fn tokens_len(&self) -> usize {self.tokens.len()}@@ -616,8 +630,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {fn async_batch_size(&self,model_path: &Path,- ) -> usize {- self.decoding_config.async_batch_size.resolve::<B>(model_path, self.context.context.as_ref())+ ) -> Result<usize, Error> {+ self.decoding_config+ .async_batch_size+ .resolve::<B>(model_path, self.context.context.as_ref())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))}fn get_slice(@@ -659,11 +676,11 @@ impl<B: Backend> LanguageModelGenerator<B> {pub fn new(model_path: &Path,decoding_config: DecodingConfig,- model_metadata: &ModelMetadata<LanguageModelConfig>,+ model_config: &LanguageModelConfig,) -> Result<Self, Error> {let gpu_capture = GpuCaptureManager::new();- let context = LanguageModelGeneratorContext::new(model_path, &decoding_config, model_metadata)?;+ let context = LanguageModelGeneratorContext::new(model_path, &decoding_config, model_config)?;Ok(Self {decoding_config,@@ -696,7 +713,9 @@ impl<B: Backend> LanguageModelGenerator<B> {let token_inputs = self.build_token_inputs(task, None, None);if should_capture {- self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;+ self.gpu_capture+ .start_capture(&self.context.context, "decode")+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;}let mut encoder = Encoder::<B>::new(self.context.context.as_ref())@@ -728,7 +747,9 @@ impl<B: Backend> LanguageModelGenerator<B> {let run_time = run_start.elapsed().as_secs_f64();if should_capture {- self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;+ self.gpu_capture+ .stop_capture(&self.context.context, "decode")+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;}Ok((sampling_output, run_time))@@ -791,7 +812,7 @@ impl<B: Backend> LanguageModelGenerator<B> {output: sampling_output.as_mut().expect("Sampling requires output allocation"),sampling_method: sampling_method.expect("Sampling requires method"),batch_size: sampling_length,- vocab_size: context.model_config.model_config.vocab_size,+ vocab_size: context.model_config.decoder_config.vocab_size,},encoder,);@@ -809,7 +830,7 @@ impl<B: Backend> LanguageModelGenerator<B> {fn build_token_inputs(&self,- task: Task<'_>,+ task: Task,token_ids_array: Option<Array<B>>,token_positions_array: Option<Array<B>>,) -> TokenInputs<B> {diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rsindex d7f7e75e6..2c9028212 100644--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs@@ -13,7 +13,7 @@ use crate::{Backend, Context, Kernels,kernel::{TokenCopySampledKernel, kv_cache_update::KVCacheUpdate},},- config::{DecoderConfig, LanguageModelConfig, ModelMetadata},+ config::{decoder::DecoderConfig, model::language_model::LanguageModelConfig},encodable_block::{Decoder, Sampling},forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},language_model::rng::PRng,@@ -118,31 +118,30 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {pub fn new(model_path: &Path,decoding_config: &DecodingConfig,- model_metadata: &ModelMetadata<LanguageModelConfig>,+ model_config: &LanguageModelConfig,) -> Result<Self, Error> {let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;- let model_shape = ModelShape::from_decoder_config(&model_metadata.model_config.model_config);-- let prefill_step_size = decoding_config.prefill_step_size.resolve(&model_metadata.model_config);+ let prefill_step_size = decoding_config.prefill_step_size.resolve(model_config);let generate_suffix_length = decoding_config.generate_suffix_length();let max_prefix_length: usize =- Self::get_context_length_internal(&context, &model_metadata.model_config.model_config, decoding_config);+ Self::get_context_length_internal(&context, &model_config.decoder_config, decoding_config);let max_suffix_length: usize = std::cmp::max(prefill_step_size, generate_suffix_length);let weights_path = model_path.join("model.safetensors");- if !weights_path.exists() {- return Err(Error::UnableToLoadWeights);- }- let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;- let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;- let root_loader_view = loader.tree();-- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &model_metadata.model_config.model_config);+ let weights_file = File::open(&weights_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let loader = ParameterLoader::new(&weights_file, context.as_ref())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let root_loader_view =+ loader.tree().subtree("decoder").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let model_shape = ModelShape::from_decoder_config(&model_config.decoder_config, DataType::BF16);++ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &model_config.decoder_config, &model_shape);shared_buffers.update_data(&root_loader_view)?;let shared_buffers = Rc::new(shared_buffers);- let executables = Decoder::new(context.as_ref(), &model_metadata.model_config.model_config, &root_loader_view);+ let executables = Decoder::new(context.as_ref(), &model_config.decoder_config, &root_loader_view, &model_shape)+ .map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;let cache_layers = Rc::new(RefCell::new(CacheLayers::new(context.as_ref(),@@ -151,29 +150,32 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {max_suffix_length,)));- let intermediate_data_type: DataType =- model_metadata.model_config.model_config.transformer_config.output_norm_config.scale_precision.into();let kv_cache_update = Box::new(- KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)+ KVCacheUpdate::new(context.as_ref(), model_shape.data_type, max_prefix_length).map_err(|e| Error::UnableToCreateContext(e.into()))?,);let gpu_sampler =- Sampling::<B>::new(&context, intermediate_data_type).map_err(|e| Error::UnableToCreateContext(e.into()))?;+ Sampling::<B>::new(&context, model_shape.data_type).map_err(|e| Error::UnableToCreateContext(e.into()))?;let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context).map_err(|e| Error::UnableToCreateContext(e.into()))?;- let async_batch_size = decoding_config.async_batch_size.resolve::<B>(model_path, context.as_ref());+ let async_batch_size = decoding_config+ .async_batch_size+ .resolve::<B>(model_path, context.as_ref())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);+ loader.tree().assert_all_tensors_validated().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+let seed = PRng::new(decoding_config.sampling_seed.resolve());Ok(Self {context,cache_layers,shared_buffers,- model_config: model_metadata.model_config.clone(),+ model_config: model_config.clone(),model_shape,executables,kv_cache_update,@@ -188,7 +190,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {&self,decoding_config: &DecodingConfig,) -> usize {- Self::get_context_length_internal(&self.context, &self.model_config.model_config, decoding_config)+ Self::get_context_length_internal(&self.context, &self.model_config.decoder_config, decoding_config)}fn get_context_length_internal(diff --git a/crates/backend-uzu/src/lib.rs b/crates/backend-uzu/src/lib.rsindex 5daa9b751..06185d865 100644--- a/crates/backend-uzu/src/lib.rs+++ b/crates/backend-uzu/src/lib.rs@@ -33,7 +33,7 @@ pub use utils::{TOOLCHAIN_VERSION, VERSION};pub mod _private {pub use crate::{classifier::Classifier,- config::{ModelConfig, ModelMetadata, ModelType},+ config::model::AnyModelConfig,encodable_block::{DecoderDecodeInput, Sampling},forward_pass::{cache_layers::CacheLayers, kv_cache_layer::KVCacheLayer, token_inputs::TokenInputs, traces::ActivationTrace,@@ -42,6 +42,6 @@ pub mod _private {language_model_generator_context::LanguageModelGeneratorContext,sampler::{ArgmaxSampler, LogitsSampler},},- parameters::ParameterTree,+ parameters::{ParameterLoaderError, ParameterTree},};}diff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rsindex d79c7ebd2..806492323 100644--- a/crates/backend-uzu/src/parameters/loader.rs+++ b/crates/backend-uzu/src/parameters/loader.rs@@ -1,16 +1,17 @@use std::{- collections::{HashMap, hash_map::Keys},+ cell::RefCell,+ collections::{HashMap, HashSet},fs::File,};use thiserror::Error;-use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingError, read_metadata as read_st_metadata};+use super::safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata};use crate::{- ArrayElement, DataType,- array::{Array, ArrayContextExt},+ Array, ArrayElement, DataType,+ array::size_for_shape,backends::common::{Allocation, AllocationType, AsBufferRangeRef, Backend, Context, DenseBuffer},- utils::fs::file_read_exact_at,+ utils::{fs::file_read_exact_at, strict_serde::DeserializeStrictOwned},};pub struct ParameterMetadata {@@ -20,48 +21,18 @@ pub struct ParameterMetadata {size: usize,}-fn st_metadata_into_index(- global_offset: usize,- st_metadata: STMetadata,-) -> HashMap<String, ParameterMetadata> {- st_metadata- .tensors- .into_iter()- .map(|(key, value)| {- let (local_begin, local_end) = value.data_offsets;- let actual_local_offset = local_begin;- let actual_size = local_end - local_begin;- let weight_metadata = ParameterMetadata {- shape: value.shape.into(),- data_type: value.dtype.into(),- offset: global_offset + actual_local_offset,- size: actual_size,- };- (key, weight_metadata)- })- .collect()-}-#[derive(Debug, Error)]pub enum ParameterLoaderError<B: Backend> {#[error("Array with key \"{0}\" not found.")]KeyNotFound(String),#[error("Couldn't find any arrays with prefix \"{0}\".")]SubtreeNotFound(String),- #[error(- "Size mismatch: array of shape {shape:?} and data type \- {data_type:?} expected to be {expected_size} bytes, got {actual_size} bytes."- )]- SizeMismatch {- data_type: DataType,- shape: Box<[usize]>,- expected_size: usize,- actual_size: usize,- },#[error("Backend error: {0}")]BackendError(#[source] B::Error),#[error("Failed to read data")]ArrayLoadingError(#[from] std::io::Error),+ #[error("Failed to deserialize metadata")]+ MetadataDeserializationError(#[from] serde_json::Error),#[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]InvalidTensor {shape: Box<[usize]>,@@ -69,83 +40,71 @@ pub enum ParameterLoaderError<B: Backend> {expected_shape: Box<[usize]>,expected_data_type: DataType,},+ #[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]+ InvalidTensorSize {+ shape: Box<[usize]>,+ data_type: DataType,+ size: usize,+ expected_size: usize,+ },+ #[error("Unvalidated tensors under {prefix:?}: {keys:?}")]+ UnvalidatedTensors {+ prefix: Option<String>,+ keys: Box<[String]>,+ },}-pub struct ParameterLoader<'context, 'file, C: Context>-where- 'file: 'context,-{- context: &'context C,+pub struct ParameterLoader<'a, B: Backend> {+ context: &'a B::Context,index: HashMap<String, ParameterMetadata>,- file: &'file File,+ metadata: HashMap<String, String>,+ validated_tensors: RefCell<HashSet<String>>,+ file: &'a File,}-impl<'file, 'context, C: Context> ParameterLoader<'file, 'context, C>-where- 'file: 'context,-{+impl<'a, B: Backend> ParameterLoader<'a, B> {pub fn new(- file: &'file File,- context: &'context C,+ file: &'a File,+ context: &'a B::Context,) -> Result<Self, HeaderLoadingError> {let (global_offset, st_metadata) = read_st_metadata(file)?;- let index = st_metadata_into_index(global_offset, st_metadata);+ let index = st_metadata+ .tensors+ .into_iter()+ .map(|(key, value)| {+ let (local_begin, local_end) = value.data_offsets;+ let size =+ local_end.checked_sub(local_begin).ok_or_else(|| HeaderLoadingError::InvalidTensorOffsets {+ key: key.clone().into_boxed_str(),+ begin: local_begin,+ end: local_end,+ })?;+ let offset =+ global_offset.checked_add(local_begin).ok_or_else(|| HeaderLoadingError::TensorOffsetOverflow {+ key: key.clone().into_boxed_str(),+ global_offset,+ local_begin,+ })?;+ let weight_metadata = ParameterMetadata {+ shape: value.shape.into(),+ data_type: value.dtype.data_type()?,+ offset,+ size,+ };+ Ok((key, weight_metadata))+ })+ .collect::<Result<HashMap<_, _>, HeaderLoadingError>>()?;+ let metadata = st_metadata.metadata.unwrap_or_default();Ok(ParameterLoader {context,- file,index,+ metadata,+ validated_tensors: RefCell::new(HashSet::new()),+ file,})}- pub fn keys(&self) -> Keys<'_, String, ParameterMetadata> {- self.index.keys()- }-- fn get_leaf<'leaf>(- &'leaf self,- key: &str,- ) -> Result<ParameterLeaf<'file, 'context, 'leaf, C>, ParameterLoaderError<C::Backend>> {- Ok(ParameterLeaf {- metadata: self.index.get(key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.to_string()))?,- loader: self,- })- }-- fn get(- &self,- key: &str,- ) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {- let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;- let (offset, size) = (metadata_entry.offset, metadata_entry.size);- let mut array = self.context.create_array_uninitialized(&metadata_entry.shape, metadata_entry.data_type);- if array.size() != size {- return Err(ParameterLoaderError::SizeMismatch {- data_type: metadata_entry.data_type,- shape: metadata_entry.shape.to_owned(),- expected_size: array.size(),- actual_size: size,- });- }-- file_read_exact_at(self.file, array.as_bytes_mut(), offset as u64)?;- Ok(array)- }-- pub fn read_extract_at(- &self,- key: &str,- buf: &mut [u8],- shape: &mut Box<[usize]>,- data_type: &mut DataType,- ) -> Result<(), ParameterLoaderError<C::Backend>> {- let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;- file_read_exact_at(self.file, buf, metadata_entry.offset as u64)?;- *shape = metadata_entry.shape.to_owned();- *data_type = metadata_entry.data_type;- Ok(())- }-- pub fn tree<'loader>(&'loader self) -> ParameterTree<'loader, C> {+ pub fn tree(&self) -> ParameterTree<'_, B> {ParameterTree {loader: self,prefix: None,@@ -153,31 +112,20 @@ where}}-pub struct ParameterLeaf<'file, 'context, 'leaf, C: Context> {+pub struct ParameterLeaf<'a, 'leaf, B: Backend, const VALIDATED: bool> {+ key: &'leaf str,metadata: &'leaf ParameterMetadata,- loader: &'leaf ParameterLoader<'context, 'file, C>,+ loader: &'leaf ParameterLoader<'a, B>,}-impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C> {- pub fn shape(&self) -> &[usize] {- &self.metadata.shape- }-- pub fn data_type(&self) -> DataType {- self.metadata.data_type- }-- pub fn size(&self) -> usize {- self.metadata.size- }-- pub fn validate_shape(- &self,+impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, false> {+ pub fn validate(+ self,expected_shape: &[usize],expected_data_type: DataType,- ) -> Result<(), ParameterLoaderError<C::Backend>> {- let shape = self.shape();- let data_type = self.data_type();+ ) -> Result<ParameterLeaf<'a, 'leaf, B, true>, ParameterLoaderError<B>> {+ let shape = self.metadata.shape.as_ref();+ let data_type = self.metadata.data_type;if (shape, data_type) != (expected_shape, expected_data_type) {return Err(ParameterLoaderError::InvalidTensor {shape: shape.into(),@@ -186,17 +134,42 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, Cexpected_data_type,});}- Ok(())+ let expected_size = size_for_shape(expected_shape, expected_data_type);+ if self.metadata.size != expected_size {+ return Err(ParameterLoaderError::InvalidTensorSize {+ shape: shape.into(),+ data_type,+ size: self.metadata.size,+ expected_size,+ });+ }+ self.loader.validated_tensors.borrow_mut().insert(self.key.to_string());+ Ok(ParameterLeaf {+ key: self.key,+ metadata: self.metadata,+ loader: self.loader,+ })+ }++ #[cfg(test)]+ pub fn unvalidated(self) -> ParameterLeaf<'a, 'leaf, B, true> {+ ParameterLeaf {+ key: self.key,+ metadata: self.metadata,+ loader: self.loader,+ }}+}- pub fn read_slice<T: ArrayElement>(&self) -> Result<Box<[T]>, ParameterLoaderError<C::Backend>> {+impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, true> {+ pub fn read_slice<T: ArrayElement>(&self) -> Result<Box<[T]>, ParameterLoaderError<B>> {let element_count = self.metadata.size / std::mem::size_of::<T>();let mut data = vec![T::zeroed(); element_count];file_read_exact_at(self.loader.file, bytemuck::cast_slice_mut(&mut data), self.metadata.offset as u64)?;Ok(data.into_boxed_slice())}- pub fn read_allocation(&self) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {+ pub fn read_allocation(&self) -> Result<Allocation<B>, ParameterLoaderError<B>> {let allocation = self.loader.context@@ -216,18 +189,19 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C)?;Ok(allocation)}++ pub fn read_array(&self) -> Result<Array<B>, ParameterLoaderError<B>> {+ let allocation = self.read_allocation()?;+ Ok(unsafe { Array::from_allocation(allocation, 0, self.metadata.shape.as_ref(), self.metadata.data_type) })+ }}-pub struct ParameterTree<'loader, C: Context> {- loader: &'loader ParameterLoader<'loader, 'loader, C>,+pub struct ParameterTree<'loader, B: Backend> {+ loader: &'loader ParameterLoader<'loader, B>,prefix: Option<String>,}-impl<'loader, C: Context> ParameterTree<'loader, C> {- pub fn path_prefix(&self) -> Option<&str> {- self.prefix.as_deref()- }-+impl<'loader, B: Backend> ParameterTree<'loader, B> {fn join_prefix(&self,name: &str,@@ -238,47 +212,64 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {pub fn subtree(&self,name: &str,- ) -> Result<Self, ParameterLoaderError<C::Backend>> {+ ) -> Result<Self, ParameterLoaderError<B>> {let new_prefix = self.join_prefix(name);- let num_suffixes = self.loader.keys().filter_map(|suffix| suffix.strip_prefix(&new_prefix)).count();- if num_suffixes > 0 {+ let subtree_prefix = format!("{new_prefix}.");+ if self.loader.index.keys().any(|key| key.starts_with(&subtree_prefix)) {Ok(Self {loader: self.loader,prefix: Some(new_prefix),})} else {- Err(ParameterLoaderError::SubtreeNotFound(name.to_string()))+ Err(ParameterLoaderError::SubtreeNotFound(new_prefix))}}- pub fn leaf_array(- &self,- name: &str,- ) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {- self.loader.get(&self.join_prefix(name))- }-pub fn leaf<'leaf>(&'leaf self,name: &str,- ) -> Result<ParameterLeaf<'loader, 'loader, 'leaf, C>, ParameterLoaderError<C::Backend>> {- self.loader.get_leaf(&self.join_prefix(name))+ ) -> Result<ParameterLeaf<'loader, 'leaf, B, false>, ParameterLoaderError<B>> {+ let key = self.join_prefix(name);+ let Some((key, metadata)) = self.loader.index.get_key_value(&key) else {+ return Err(ParameterLoaderError::KeyNotFound(key));+ };+ Ok(ParameterLeaf {+ key,+ metadata,+ loader: self.loader,+ })}- pub fn leaf_allocation(+ pub fn metadata<T: DeserializeStrictOwned>(&self,name: &str,- ) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {- self.leaf(name)?.read_allocation()+ ) -> Result<T, ParameterLoaderError<B>> {+ let new_prefix = self.join_prefix(name);++ Ok(serde_json::from_str(+ self.loader.metadata.get(&new_prefix).ok_or(ParameterLoaderError::KeyNotFound(new_prefix))?,+ )?)}- pub fn read_extract_at(- &self,- name: &str,- buf: &mut [u8],- shape: &mut Box<[usize]>,- data_type: &mut DataType,- ) -> Result<(), ParameterLoaderError<C::Backend>> {- self.loader.read_extract_at(&self.join_prefix(name), buf, shape, data_type)+ pub fn assert_all_tensors_validated(&self) -> Result<(), ParameterLoaderError<B>> {+ let subtree_prefix = self.prefix.as_ref().map(|prefix| format!("{prefix}."));+ let validated_tensors = self.loader.validated_tensors.borrow();+ let mut unvalidated_tensors = self+ .loader+ .index+ .keys()+ .filter(|key| subtree_prefix.as_ref().is_none_or(|prefix| key.starts_with(prefix)))+ .filter(|key| !validated_tensors.contains(*key))+ .cloned()+ .collect::<Vec<_>>();+ unvalidated_tensors.sort();+ if unvalidated_tensors.is_empty() {+ Ok(())+ } else {+ Err(ParameterLoaderError::UnvalidatedTensors {+ prefix: self.prefix.clone(),+ keys: unvalidated_tensors.into_boxed_slice(),+ })+ }}}diff --git a/crates/backend-uzu/src/parameters/safetensors_metadata.rs b/crates/backend-uzu/src/parameters/safetensors_metadata.rsindex 2a027a470..cffd40b33 100644--- a/crates/backend-uzu/src/parameters/safetensors_metadata.rs+++ b/crates/backend-uzu/src/parameters/safetensors_metadata.rs@@ -1,6 +1,6 @@// This code is based on the safetensors implementation: https://docs.rs/safetensors/latest/src/safetensors/tensor.rs.html-use std::{collections::HashMap, fs::File};+use std::{collections::HashMap, fs::File, str::Utf8Error};use serde::{Deserialize, Serialize};use thiserror::Error;@@ -9,18 +9,32 @@ use crate::{DataType, utils::fs::file_read_exact_at};#[derive(Debug, Error)]pub enum HeaderLoadingError {- #[error("The header is an invalid UTF-8 string and cannot be read.")]- InvalidHeader,- #[error("The header does contain a valid string, but it is not valid JSON.")]- InvalidHeaderDeserialization,- #[error("The header is large than 100Mo which is considered too large.")]+ #[error("Unable to read safetensors header bytes: {0}")]+ UnableToReadHeader(#[source] std::io::Error),+ #[error("Unable to read safetensors header JSON: {0}")]+ UnableToReadHeaderJson(#[source] std::io::Error),+ #[error("The header is an invalid UTF-8 string and cannot be read: {0}")]+ InvalidHeader(#[from] Utf8Error),+ #[error("The header does contain a valid string, but it is not valid JSON: {0}")]+ InvalidHeaderDeserialization(#[from] serde_json::Error),+ #[error("The header is too large.")]HeaderTooLarge,- #[error("The header is smaller than 8 bytes.")]- HeaderTooSmall,#[error("The header length is invalid.")]InvalidHeaderLength,- #[error("Couldn't deserialize the header JSON.")]- JsonError(#[from] serde_json::Error),+ #[error("Unsupported safetensors dtype: {0:?}")]+ UnsupportedDtype(Dtype),+ #[error("Invalid data offsets for tensor {key}: begin={begin}, end={end}")]+ InvalidTensorOffsets {+ key: Box<str>,+ begin: usize,+ end: usize,+ },+ #[error("Tensor offset overflow for tensor {key}: global_offset={global_offset}, local_begin={local_begin}")]+ TensorOffsetOverflow {+ key: Box<str>,+ global_offset: usize,+ local_begin: usize,+ },}#[derive(Debug, Serialize, Deserialize)]@@ -76,37 +90,20 @@ pub enum Dtype {U64,}-impl From<Dtype> for DataType {- fn from(val: Dtype) -> Self {- match val {- Dtype::F16 => DataType::F16,- Dtype::BF16 => DataType::BF16,- Dtype::F32 => DataType::F32,- Dtype::I8 => DataType::I8,- Dtype::U8 => DataType::U8,- Dtype::I32 => DataType::I32,- Dtype::U32 => DataType::U32,- Dtype::I64 => DataType::I64,- Dtype::U64 => DataType::U64,- Dtype::BOOL => DataType::I8,- // Add mappings for other types as needed- _ => panic!("Unsupported dtype: {:?}", val),- }- }-}--impl From<DataType> for Dtype {- fn from(dtype: DataType) -> Self {- match dtype {- DataType::F16 => Dtype::F16,- DataType::BF16 => Dtype::BF16,- DataType::F32 => Dtype::F32,- DataType::I8 => Dtype::I8,- DataType::I32 => Dtype::I32,- DataType::I64 => Dtype::I64,- DataType::U64 => Dtype::U64,- _ => panic!("Unsupported dtype: {:?}", dtype),- }+impl Dtype {+ pub fn data_type(self) -> Result<DataType, HeaderLoadingError> {+ Ok(match self {+ Self::F16 => DataType::F16,+ Self::BF16 => DataType::BF16,+ Self::F32 => DataType::F32,+ Self::I8 => DataType::I8,+ Self::U8 => DataType::U8,+ Self::I32 => DataType::I32,+ Self::U32 => DataType::U32,+ Self::I64 => DataType::I64,+ Self::U64 => DataType::U64,+ dtype => return Err(HeaderLoadingError::UnsupportedDtype(dtype)),+ })}}@@ -114,7 +111,7 @@ const MAX_HEADER_SIZE: usize = 100_000_000;pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadingError> {let mut header_buffer = [0u8; size_of::<u64>()];- file_read_exact_at(file, &mut header_buffer, 0).map_err(|_| HeaderLoadingError::HeaderTooSmall)?;+ file_read_exact_at(file, &mut header_buffer, 0).map_err(HeaderLoadingError::UnableToReadHeader)?;let metadata_size: usize =u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;if metadata_size > MAX_HEADER_SIZE {@@ -123,9 +120,8 @@ pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadinglet stop = metadata_size.checked_add(8).ok_or(HeaderLoadingError::InvalidHeaderLength)?;let mut json_buffer: Box<[u8]> = core::iter::repeat(0).take(stop - size_of::<u64>()).collect();- file_read_exact_at(file, &mut json_buffer, 8).map_err(|_| HeaderLoadingError::InvalidHeader)?;- let string = core::str::from_utf8(&json_buffer).map_err(|_| HeaderLoadingError::InvalidHeader)?;- let metadata: HashMetadata =- serde_json::from_str(string).map_err(|_| HeaderLoadingError::InvalidHeaderDeserialization)?;+ file_read_exact_at(file, &mut json_buffer, 8).map_err(HeaderLoadingError::UnableToReadHeaderJson)?;+ let string = core::str::from_utf8(&json_buffer)?;+ let metadata: HashMetadata = serde_json::from_str(string)?;Ok((stop, metadata))}diff --git a/crates/backend-uzu/src/session/chat_session.rs b/crates/backend-uzu/src/session/chat_session.rsindex 9987ba81a..f6745d209 100644--- a/crates/backend-uzu/src/session/chat_session.rs+++ b/crates/backend-uzu/src/session/chat_session.rs@@ -14,8 +14,11 @@ use std::{use tokenizers::Tokenizer;use crate::{+ DataType,backends::{common::Backend, select_backend},- config::{LanguageModelConfig, MixerConfig, ModelMetadata},+ config::{+ model::language_model::LanguageModelConfig, token_codec::AnyTokenCodecConfig, token_mixer::AnyTokenMixerConfig,+ },language_model::{LanguageModelGenerator, LanguageModelGeneratorTrait,grammar::{CompiledGrammar, create_compiled_grammar},@@ -24,7 +27,7 @@ use crate::{session::{config::{DecodingConfig, RunConfig},helpers::{InputProcessor, InputProcessorDefault, OutputParser, is_directory_fits_ram},- parameter::{ConfigResolvableValue, ContextMode},+ parameter::{ConfigResolvableValue, ContextMode, SamplingMethod},types::{Error, FinishReason, Input, Output, RunStats, Stats, StepStats, TotalStats},},};@@ -47,14 +50,14 @@ struct RunContext {pub struct ChatSession {pub model_path: PathBuf,- pub model_metadata: ModelMetadata<LanguageModelConfig>,+ pub model_config: LanguageModelConfig,tokenizer: Tokenizer,stop_token_ids: Vec<i32>,input_processor: Box<dyn InputProcessor>,output_parser: OutputParser,decoding_config: DecodingConfig,- llm: Option<Box<dyn LanguageModelGeneratorTrait>>,+ llm: Box<dyn LanguageModelGeneratorTrait>,static_context: Option<Box<dyn Any>>,}@@ -79,20 +82,14 @@ impl ChatSession {}let config_path = model_path.join("config.json");- if !config_path.exists() {- return Err(Error::UnableToLoadConfig);- }- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;- let model_metadata: ModelMetadata<LanguageModelConfig> = serde_json::from_reader(BufReader::new(config_file))- .map_err(|err| {- eprintln!("Failed to parse config.json: {err}");- Error::UnableToLoadConfig- })?;-- let layers = &model_metadata.model_config.model_config.transformer_config.layer_configs;+ let config_file = File::open(&config_path)?;+ let model_config: LanguageModelConfig = serde_json::from_reader(BufReader::new(config_file))?;++ let layers = &model_config.decoder_config.transformer_config.layer_configs;let has_non_attention_mixer =- layers.iter().any(|layer| !matches!(layer.mixer_config, MixerConfig::Attention(_)));- let has_mamba_mixer = layers.iter().any(|layer| matches!(layer.mixer_config, MixerConfig::Mamba(_)));+ layers.iter().any(|layer| !matches!(layer.mixer_config, AnyTokenMixerConfig::AttentionConfig(_)));+ let has_mamba_mixer =+ layers.iter().any(|layer| matches!(layer.mixer_config, AnyTokenMixerConfig::Mamba2Config(_)));if has_non_attention_mixer {match decoding_config.context_mode {ContextMode::None => {},@@ -112,50 +109,47 @@ impl ChatSession {}let tokenizer_path = model_path.join("tokenizer.json");- if !tokenizer_path.exists() {- return Err(Error::UnableToLoadTokenizer);- }- let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(Error::UnableToLoadTokenizer)?;++ let AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) = &model_config.token_codec_config else {+ return Err(Error::InvalidModelConfig("expected ChatCodecConfig token codec".to_string()));+ };let stop_token_ids: Vec<i32> =- model_metadata.model_config.generation_config.stop_token_ids.iter().map(|&x| x as i32).collect();+ model_config.generation_config.stop_token_ids.iter().map(|&x| x as i32).collect();- let input_processor = InputProcessorDefault::new(model_metadata.model_config.message_processor_config.clone());+ let input_processor = InputProcessorDefault::new(token_codec_config.clone());- let output_parser =- OutputParser::new(model_metadata.model_config.message_processor_config.output_parser_regex.clone())?;+ let output_parser = OutputParser::new(token_codec_config.output_parser_regex.clone())?;- let llm = Box::new(- LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone(), &model_metadata)- .map_err(Error::from)?,- );+ let llm = Box::new(LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone(), &model_config)?);Ok(Self {model_path,- model_metadata,+ model_config,tokenizer,stop_token_ids,input_processor: Box::new(input_processor),output_parser,decoding_config,- llm: Some(llm),+ llm,static_context: None,})}- pub fn run<F>(+ pub fn activation_data_type(&self) -> DataType {+ self.llm.activation_data_type()+ }++ pub fn run(&mut self,input: Input,config: RunConfig,- progress: Option<F>,- ) -> Result<Output, Error>- where- F: Fn(Output) -> bool,- {+ progress: Option<impl Fn(Output) -> bool>,+ ) -> Result<Output, Error> {match &self.decoding_config.context_mode {ContextMode::None => {- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- language_model_generator.reset_state();+ self.llm.reset_state();},ContextMode::Static {input,@@ -163,12 +157,13 @@ impl ChatSession {if self.static_context.is_none() {let mut prefill_config = config.clone();prefill_config.tokens_limit = 0;- let (_out, ctx) = self.extend(input.clone(), None, prefill_config)?;+ let (_, ctx) = self.extend(input.clone(), None, prefill_config)?;self.static_context = Some(ctx);}let tmp = self.static_context.take();+ self.llm.reset_state();if let Some(ref ctx) = tmp {- self.reconfigure_language_model_generator(Some(ctx))?;+ self.llm.reconfigure_from_context(ctx.as_ref());}self.static_context = tmp;},@@ -182,8 +177,7 @@ impl ChatSession {| ContextMode::Static {..} => {- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- language_model_generator.reset_state();+ self.llm.reset_state();},ContextMode::Dynamic => {},}@@ -191,8 +185,7 @@ impl ChatSession {}pub fn reset(&mut self) -> Result<(), Error> {- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- language_model_generator.reset_state();+ self.llm.reset_state();Ok(())}@@ -202,35 +195,34 @@ impl ChatSession {context: Option<&Box<dyn Any>>,config: RunConfig,) -> Result<(Output, Box<dyn Any>), Error> {- self.reconfigure_language_model_generator(context)?;+ self.llm.reset_state();+ if let Some(ctx) = context {+ self.llm.reconfigure_from_context(ctx.as_ref());+ }let output = self.run_internal(input, config, None::<fn(Output) -> bool>)?;- let new_context = self.build_context_from_language_model_generator()?;- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- language_model_generator.reset_state();+ let new_context = self.llm.build_llm_context();+ self.llm.reset_state();Ok((output, new_context))}- fn run_internal<F>(+ fn run_internal(&mut self,input: Input,config: RunConfig,- progress: Option<F>,- ) -> Result<Output, Error>- where- F: Fn(Output) -> bool,- {+ progress: Option<impl Fn(Output) -> bool>,+ ) -> Result<Output, Error> {let run_start = Instant::now();let text = self.input_processor.process(&input, config.enable_thinking, config.tokens_limit > 0)?;let tokens: Vec<u64> = self.tokenizer.encode(text.as_str(), false)- .map_err(|_| Error::UnableToEncodeText)?+ .map_err(Error::UnableToEncodeText)?.get_ids().iter().map(|&id| id as u64).collect();- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;+ let language_model_generator = self.llm.as_mut();let context_length = language_model_generator.get_context_length();if tokens.len() >= context_length {return Err(Error::ContextLengthExceeded);@@ -239,10 +231,9 @@ impl ChatSession {let prefix_offset = language_model_generator.tokens_len();let prefix_len_before = prefix_offset;- let eos_tokens: Vec<u64> =- self.model_metadata.model_config.generation_config.stop_token_ids.iter().map(|&x| x as u64).collect();+ let eos_tokens: Vec<u64> = self.model_config.generation_config.stop_token_ids.iter().copied().collect();- let sampling_method = config.sampling_policy.resolve(&self.model_metadata.model_config);+ let sampling_method = config.sampling_policy.resolve(&self.model_config);let mut compiled_grammar: Option<Box<dyn CompiledGrammar>> = if let Some(ref config) = config.grammar_config {Some(create_compiled_grammar(config, &self.tokenizer, Some(&self.stop_token_ids))?)@@ -262,7 +253,7 @@ impl ChatSession {let prefill_tokens = prefill_result.tokens.clone();let prefill_duration = prefill_start.elapsed().as_secs_f64();- let prefill_suffix_length = self.decoding_config.prefill_step_size.resolve(&self.model_metadata.model_config);+ let prefill_suffix_length = self.decoding_config.prefill_step_size.resolve(&self.model_config);let run_context = RunContext {eos_tokens,@@ -281,13 +272,13 @@ impl ChatSession {let prefill_finish_reason = if grammar_terminated {Some(FinishReason::Stop)} else {- Self::check_finish_reason(&run_context, language_model_generator.as_ref(), &prefill_tokens)+ Self::check_finish_reason(&run_context, &*language_model_generator, &prefill_tokens)};let prefill_output = Self::build_output(&self.tokenizer,&self.output_parser,&run_context,- language_model_generator.as_ref(),+ &*language_model_generator,&[],&[],prefill_finish_reason.clone(),@@ -310,12 +301,12 @@ impl ChatSession {let can_use_async = language_model_generator.generate_suffix_length() == 1 && compiled_grammar.is_none();let generate_output = if can_use_async {- let batch_size = language_model_generator.async_batch_size(&self.model_path);+ let batch_size = language_model_generator.async_batch_size(&self.model_path)?;Self::run_async_batch(&self.tokenizer,&self.output_parser,&run_context,- language_model_generator.as_mut(),+ language_model_generator,sampling_method,&progress,batch_size,@@ -325,7 +316,7 @@ impl ChatSession {&self.tokenizer,&self.output_parser,&run_context,- language_model_generator.as_mut(),+ language_model_generator,compiled_grammar.as_deref_mut(),sampling_method,&progress,@@ -335,18 +326,15 @@ impl ChatSession {Ok(generate_output.clone_with_duration(run_start.elapsed().as_secs_f64()))}- fn run_sync_generate<F>(+ fn run_sync_generate(tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,language_model_generator: &mut dyn LanguageModelGeneratorTrait,compiled_grammar: Option<&mut (dyn CompiledGrammar + 'static)>,- sampling_method: super::parameter::SamplingMethod,- progress: &Option<F>,- ) -> Result<Output, Error>- where- F: Fn(Output) -> bool,- {+ sampling_method: SamplingMethod,+ progress: &Option<impl Fn(Output) -> bool>,+ ) -> Result<Output, Error> {let mut generate_results: Vec<GenerateResult> = Vec::new();let mut generate_durations: Vec<f64> = Vec::new();let mut compiled_grammar_mut = compiled_grammar;@@ -393,18 +381,15 @@ impl ChatSession {}}- fn run_async_batch<F>(+ fn run_async_batch(tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,llm: &mut dyn LanguageModelGeneratorTrait,- sampling_method: super::parameter::SamplingMethod,- progress: &Option<F>,+ sampling_method: SamplingMethod,+ progress: &Option<impl Fn(Output) -> bool>,batch_size: usize,- ) -> Result<Output, Error>- where- F: Fn(Output) -> bool,- {+ ) -> Result<Output, Error> {let remaining_by_limit = run_context.tokens_limit.saturating_sub(run_context.prefill_result.tokens.len());let remaining_by_context = run_context.context_length.saturating_sub(llm.tokens_len());let tokens_to_generate = remaining_by_limit.min(remaining_by_context);@@ -507,23 +492,6 @@ impl ChatSession {Self::build_output(tokenizer, output_parser, run_context, llm, &results, &durations, Some(finish_reason))}- fn reconfigure_language_model_generator(- &mut self,- context: Option<&Box<dyn Any>>,- ) -> Result<(), Error> {- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- language_model_generator.reset_state();- if let Some(ctx) = context {- language_model_generator.reconfigure_from_context(ctx.as_ref());- }- Ok(())- }-- fn build_context_from_language_model_generator(&self) -> Result<Box<dyn Any>, Error> {- let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- Ok(language_model_generator.build_llm_context())- }-fn check_finish_reason(run_context: &RunContext,language_model_generator: &dyn LanguageModelGeneratorTrait,@@ -555,7 +523,7 @@ impl ChatSession {.chain(generate_results.iter().flat_map(|r| r.tokens.iter())).map(|&v| v as u32).collect();- tokenizer.decode(&generated_tokens, true).map_err(|_| Error::UnableToDecodeText)+ tokenizer.decode(&generated_tokens, true).map_err(Error::UnableToDecodeText)}fn build_output(@@ -680,12 +648,6 @@ impl ChatSession {}pub fn peak_memory_usage(&self) -> Option<usize> {- self.llm.as_ref().unwrap().peak_memory_usage()- }-}--impl Drop for ChatSession {- fn drop(&mut self) {- self.llm = None+ self.llm.peak_memory_usage()}}diff --git a/crates/backend-uzu/src/session/classification_session.rs b/crates/backend-uzu/src/session/classification_session.rsindex 942650be4..d9cb15a37 100644--- a/crates/backend-uzu/src/session/classification_session.rs+++ b/crates/backend-uzu/src/session/classification_session.rs@@ -1,11 +1,11 @@-use std::path::PathBuf;+use std::{fs::File, io::BufReader, path::PathBuf};use tokenizers::Tokenizer;use crate::{backends::{common::Backend, select_backend},classifier::{ClassificationOutput, Classifier, ClassifierTrait},- config::{ClassifierModelConfig, ModelMetadata},+ config::{model::classifier_model::ClassifierModelConfig, token_codec::AnyTokenCodecConfig},session::{helpers::{InputProcessor, InputProcessorDefault},types::{Error, Input},@@ -24,25 +24,23 @@ impl ClassificationSession {}pub fn new_with_backend<B: Backend>(model_path: PathBuf) -> Result<Self, Error> {- let config_path = model_path.join("config.json");- if !config_path.exists() {+ if !model_path.exists() {return Err(Error::ModelFolderNotFound);}- let config_file = std::fs::File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;- let model_metadata: ModelMetadata<ClassifierModelConfig> =- serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;+ let config_path = model_path.join("config.json");+ let config_file = File::open(&config_path)?;+ let model_config: ClassifierModelConfig = serde_json::from_reader(BufReader::new(config_file))?;let tokenizer_path = model_path.join("tokenizer.json");- if !tokenizer_path.exists() {- return Err(Error::UnableToLoadTokenizer);- }- let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(Error::UnableToLoadTokenizer)?;+ let AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) = &model_config.token_codec_config else {+ return Err(Error::InvalidModelConfig("expected ChatCodecConfig token codec".to_string()));+ };let input_processor =- Box::new(InputProcessorDefault::new(model_metadata.model_config.message_processor_config.clone()))- as Box<dyn InputProcessor>;- let classifier = Box::new(Classifier::<B>::new(&model_path, &model_metadata)?);+ Box::new(InputProcessorDefault::new(token_codec_config.clone())) as Box<dyn InputProcessor>;+ let classifier = Box::new(Classifier::<B>::new(&model_path, &model_config)?);Ok(Self {tokenizer,@@ -60,7 +58,7 @@ impl ClassificationSession {let tokens: Vec<u64> = self.tokenizer.encode(text.as_str(), false)- .map_err(|_| Error::UnableToEncodeText)?+ .map_err(Error::UnableToEncodeText)?.get_ids().iter().map(|&id| id as u64)diff --git a/crates/backend-uzu/src/session/config/grammar_config.rs b/crates/backend-uzu/src/session/config/grammar_config.rsindex 054dc7952..19eed4fa4 100644--- a/crates/backend-uzu/src/session/config/grammar_config.rs+++ b/crates/backend-uzu/src/session/config/grammar_config.rs@@ -58,10 +58,7 @@ impl GrammarConfig {Self::BuiltinJson}- pub fn from_json_schema_type<T>() -> Result<Self, String>- where- T: JsonSchema,- {+ pub fn from_json_schema_type<T: JsonSchema>() -> Result<Self, String> {let schema_root = schemars::schema_for!(T);let schema_str =serde_json::to_string(&schema_root).map_err(|e| format!("Failed to serialize schema: {}", e))?;@@ -75,15 +72,12 @@ impl GrammarConfig {})}- pub fn from_json_schema_type_with_config<T>(+ pub fn from_json_schema_type_with_config<T: JsonSchema>(any_whitespace: bool,indent: Option<i32>,separators: Option<(String, String)>,strict_mode: bool,- ) -> Result<Self, String>- where- T: JsonSchema,- {+ ) -> Result<Self, String> {let schema_root = schemars::schema_for!(T);let schema_str =serde_json::to_string(&schema_root).map_err(|e| format!("Failed to serialize schema: {}", e))?;diff --git a/crates/backend-uzu/src/session/helpers/input_processor.rs b/crates/backend-uzu/src/session/helpers/input_processor.rsindex d55887bff..04e392c7b 100644--- a/crates/backend-uzu/src/session/helpers/input_processor.rs+++ b/crates/backend-uzu/src/session/helpers/input_processor.rs@@ -4,10 +4,10 @@ use minijinja::{Environment, context};use minijinja_contrib::pycompat::unknown_method_callback;use crate::{- config::MessageProcessorConfig,+ config::token_codec::chat_codec::ChatCodecConfig,session::{parameter::ConfigResolvableValue,- types::{Error, Input, Role},+ types::{Error, Input, Message, Role},},};@@ -21,13 +21,13 @@ pub trait InputProcessor: Send + Sync {}pub struct InputProcessorDefault {- message_processing_config: MessageProcessorConfig,+ token_codec_config: ChatCodecConfig,}impl InputProcessorDefault {- pub fn new(message_processing_config: MessageProcessorConfig) -> Self {+ pub fn new(token_codec_config: ChatCodecConfig) -> Self {Self {- message_processing_config,+ token_codec_config,}}}@@ -78,33 +78,40 @@ impl InputProcessor for InputProcessorDefault {enable_thinking: bool,add_generation_prompt: bool,) -> Result<String, Error> {- let messages = input.get_messages();+ let mut messages = input.get_messages();for message in &messages {if message.role != Role::Assistant && message.reasoning_content.is_some() {return Err(Error::UnexpectedReasoningContent);}}+ if let Some(default_system_prompt) = self.token_codec_config.default_system_prompt.clone() {+ if messages.first().is_none_or(|message| message.role != Role::System) {+ messages.insert(0, Message::system(default_system_prompt));+ }+ }let messages =- messages.into_iter().map(|message| message.resolve(&self.message_processing_config)).collect::<Vec<_>>();- let template = self.message_processing_config.prompt_template.clone();- let bos_token = self.message_processing_config.bos_token.clone();+ messages.into_iter().map(|message| message.resolve(&self.token_codec_config)).collect::<Vec<_>>();+ let template = self.token_codec_config.prompt_template.clone();+ let bos_token = self.token_codec_config.bos_token.clone();+ let eos_token = self.token_codec_config.eos_token.clone();let template_name = "chat_template";let mut environment = Environment::new();environment.set_unknown_method_callback(unknown_method_callback);environment.add_function("strftime_now", strftime_now);- environment.add_template(template_name, template.as_str()).map_err(|_| Error::UnableToLoadPromptTemplate)?;- let template = environment.get_template(template_name).map_err(|_| Error::UnableToLoadPromptTemplate)?;+ environment.add_template(template_name, template.as_str()).map_err(Error::UnableToLoadPromptTemplate)?;+ let template = environment.get_template(template_name).map_err(Error::UnableToLoadPromptTemplate)?;let result = template.render(context!(messages => messages,add_generation_prompt => add_generation_prompt,bos_token => bos_token,+ eos_token => eos_token,enable_thinking => enable_thinking))- .map_err(|_| Error::UnableToRenderPromptTemplate)?;+ .map_err(Error::UnableToRenderPromptTemplate)?;Ok(result)}}diff --git a/crates/backend-uzu/src/session/helpers/output_parser.rs b/crates/backend-uzu/src/session/helpers/output_parser.rsindex c4dbeed0e..8f4dd7900 100644--- a/crates/backend-uzu/src/session/helpers/output_parser.rs+++ b/crates/backend-uzu/src/session/helpers/output_parser.rs@@ -11,10 +11,7 @@ impl OutputParser {let regex = match regex_str {Some(regex_str) => {let regex_str = Self::preprocess_regex_string(regex_str);- let re = RegexBuilder::new(®ex_str)- .dot_matches_new_line(true)- .build()- .map_err(|_| Error::UnableToBuildOutputParserRegex)?;+ let re = RegexBuilder::new(®ex_str).dot_matches_new_line(true).build()?;Some(re)},None => None,diff --git a/crates/backend-uzu/src/session/parameter/async_batch_size.rs b/crates/backend-uzu/src/session/parameter/async_batch_size.rsindex b46a48f78..5e728fc2c 100644--- a/crates/backend-uzu/src/session/parameter/async_batch_size.rs+++ b/crates/backend-uzu/src/session/parameter/async_batch_size.rs@@ -19,10 +19,10 @@ impl AsyncBatchSize {&self,model_path: &Path,context: &B::Context,- ) -> usize {+ ) -> Result<usize, B::Error> {match self {AsyncBatchSize::Default => context.recommended_async_batch_size(model_path),- AsyncBatchSize::Custom(value) => *value,+ AsyncBatchSize::Custom(value) => Ok(*value),}}}diff --git a/crates/backend-uzu/src/session/parameter/prefill_step_size.rs b/crates/backend-uzu/src/session/parameter/prefill_step_size.rsindex 6bd145fda..027c909a0 100644--- a/crates/backend-uzu/src/session/parameter/prefill_step_size.rs+++ b/crates/backend-uzu/src/session/parameter/prefill_step_size.rs@@ -1,4 +1,4 @@-use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};+use crate::{config::model::language_model::LanguageModelConfig, session::parameter::ConfigResolvableValue};fn env_prefill_step_size_default_override() -> Option<usize> {static OVERRIDE: std::sync::OnceLock<Option<usize>> = std::sync::OnceLock::new();@@ -31,10 +31,10 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {config: &LanguageModelConfig,) -> usize {let default_limit: usize = 4096;- let model_context_length = config.model_config.transformer_config.context_length;+ let model_context_length = config.decoder_config.transformer_config.context_length;let minimal_sliding_window_size = config- .model_config+ .decoder_config.transformer_config.layer_configs.iter()diff --git a/crates/backend-uzu/src/session/parameter/sampling_policy.rs b/crates/backend-uzu/src/session/parameter/sampling_policy.rsindex e60c6e1e3..86f76b7a4 100644--- a/crates/backend-uzu/src/session/parameter/sampling_policy.rs+++ b/crates/backend-uzu/src/session/parameter/sampling_policy.rs@@ -1,4 +1,4 @@-use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};+use crate::{config::model::language_model::LanguageModelConfig, session::parameter::ConfigResolvableValue};#[derive(Debug, Copy, Clone, PartialEq, Eq)]pub enum SamplingProcessingOrder {diff --git a/crates/backend-uzu/src/session/tts_session.rs b/crates/backend-uzu/src/session/tts_session.rsindex 7f3246ece..09f2a0b32 100644--- a/crates/backend-uzu/src/session/tts_session.rs+++ b/crates/backend-uzu/src/session/tts_session.rs@@ -11,6 +11,7 @@ use std::{cell::RefCell,collections::HashMap,fs::File,+ io::BufReader,path::{Path, PathBuf},rc::Rc,time::Instant,@@ -37,7 +38,7 @@ use crate::{matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},- config::{ModelMetadata, TtsMessageProcessorConfig, TtsModelConfig},+ config::{model::tts_model::TTSModelConfig, token_codec::tts_codec::TTSCodecConfig},encodable_block::{Decoder, Sampling as GpuSampling},forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},parameters::ParameterLoader,@@ -85,7 +86,7 @@ pub struct TtsSession<B: Backend> {tokenizer: Tokenizer,audio: AudioGenerationContext<B>,audio_decoder: Box<dyn AudioDecoderBackend>,- message_processor_config: TtsMessageProcessorConfig,+ token_codec_config: TTSCodecConfig,text_decoder: Box<dyn SemanticDecoderBackend>,last_execution_stats: Option<TtsExecutionStats>,}@@ -212,12 +213,12 @@ impl<B: Backend> PendingAudioChunkBackend for NanoCodecPendingAudioChunk<B> {}let pending = self.inner.take().ok_or(Error::GenerateFailed)?;- pending.resolve().map(Some).map_err(Error::from)+ Ok(Some(pending.resolve()?))}fn resolve(mut self: Box<Self>) -> Result<AudioPcmBatch, Error> {let pending = self.inner.take().ok_or(Error::GenerateFailed)?;- pending.resolve().map_err(Error::from)+ Ok(pending.resolve()?)}}diff --git a/crates/backend-uzu/src/session/tts_session/audio_backend.rs b/crates/backend-uzu/src/session/tts_session/audio_backend.rsindex 5e3ea6663..6d1b6908e 100644--- a/crates/backend-uzu/src/session/tts_session/audio_backend.rs+++ b/crates/backend-uzu/src/session/tts_session/audio_backend.rs@@ -37,7 +37,7 @@ impl<B: Backend + Send + Sync> AudioDecoderBackend for NanoCodecAudioDecoderBack&self,tokens: &AudioTokenGrid,) -> Result<AudioPcmBatch, Error> {- self.audio.runtime().decode(tokens).map_err(Error::from)+ Ok(self.audio.runtime().decode(tokens)?)}fn begin_stream(@@ -46,11 +46,8 @@ impl<B: Backend + Send + Sync> AudioDecoderBackend for NanoCodecAudioDecoderBackcodebooks: usize,max_workspace_frames: usize,) -> Result<Box<dyn AudioDecoderStreamBackend>, Error> {- let state = self- .audio- .runtime()- .begin_decode_stream_with_options(batch_size, codebooks, max_workspace_frames)- .map_err(Error::from)?;+ let state =+ self.audio.runtime().begin_decode_stream_with_options(batch_size, codebooks, max_workspace_frames)?;Ok(Box::new(NanoCodecAudioDecoderStream {audio: self.audio.clone(),state: Some(state),@@ -65,8 +62,8 @@ impl<B: Backend> AudioDecoderStreamBackend for NanoCodecAudioDecoderStream<B> {is_final: bool,) -> Result<AudioPcmBatch, Error> {let state = self.state.as_mut().ok_or(Error::GenerateFailed)?;- let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?;- self.audio.runtime().decoded_padded_to_pcm_batch(&decoded).map_err(Error::from)+ let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final)?;+ Ok(self.audio.runtime().decoded_padded_to_pcm_batch(&decoded)?)}fn decode_step_pending(@@ -75,16 +72,14 @@ impl<B: Backend> AudioDecoderStreamBackend for NanoCodecAudioDecoderStream<B> {is_final: bool,) -> Result<Box<dyn PendingAudioChunkBackend>, Error> {let state = self.state.as_mut().ok_or(Error::GenerateFailed)?;- if let Some(pending) =- self.audio.runtime().submit_decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?- {+ if let Some(pending) = self.audio.runtime().submit_decode_stream_step(state, new_tokens, is_final)? {return Ok(Box::new(NanoCodecPendingAudioChunk {inner: Some(pending),}));}- let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?;- let pcm = self.audio.runtime().decoded_padded_to_pcm_batch(&decoded).map_err(Error::from)?;+ let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final)?;+ let pcm = self.audio.runtime().decoded_padded_to_pcm_batch(&decoded)?;Ok(Box::new(ImmediatePendingAudioChunk {pcm: Some(pcm),step_stats: Some(state.last_step_stats()),@@ -97,6 +92,6 @@ impl<B: Backend> AudioDecoderStreamBackend for NanoCodecAudioDecoderStream<B> {fn finish(mut self: Box<Self>) -> Result<(), Error> {let state = self.state.take().ok_or(Error::GenerateFailed)?;- self.audio.runtime().end_decode_stream(state).map_err(Error::from)+ Ok(self.audio.runtime().end_decode_stream(state)?)}}diff --git a/crates/backend-uzu/src/session/tts_session/backend_factory.rs b/crates/backend-uzu/src/session/tts_session/backend_factory.rsindex 2732d11eb..07a01f392 100644--- a/crates/backend-uzu/src/session/tts_session/backend_factory.rs+++ b/crates/backend-uzu/src/session/tts_session/backend_factory.rs@@ -1,14 +1,16 @@use super::*;use crate::{- config::{ModelType, TtsModelConfig, TtsTextDecoderConfig},- session::types::TtsModelConfigError,+ audio::AudioGenerationContext,+ config::{+ model::tts_model::TTSModelConfig, token_codec::AnyTokenCodecConfig, tts::text_decoder::AnyTTSTextDecoderConfig,+ },};pub(super) struct LoadedTtsRuntime<B: Backend> {pub(super) audio: AudioGenerationContext<B>,pub(super) audio_decoder: Box<dyn AudioDecoderBackend>,pub(super) text_decoder: Box<dyn SemanticDecoderBackend>,- pub(super) message_processor_config: TtsMessageProcessorConfig,+ pub(super) token_codec_config: TTSCodecConfig,}pub(super) fn build_audio_decoder_backend<B: Backend + Send + Sync>(@@ -19,39 +21,33 @@ pub(super) fn build_audio_decoder_backend<B: Backend + Send + Sync>(pub(super) fn load_tts_runtime<B: Backend + Send + Sync>(model_path: &Path,- model_metadata: &ModelMetadata<TtsModelConfig>,+ model_config: &TTSModelConfig,options: &TtsSessionOptions,) -> Result<LoadedTtsRuntime<B>, Error> {- if model_metadata.model_type != ModelType::TtsModel {- return Err(TtsModelConfigError::ModelTypeMismatch {- expected: ModelType::TtsModel,- actual: model_metadata.model_type.clone(),- }- .into());- }-- let audio: AudioGenerationContext<B> =- model_metadata.model_config.create_audio_generation_context_with_model_path(model_path)?;- let text_decoder = build_text_decoder_backend(&model_metadata.model_config, &audio, model_path, options)?;+ let audio = AudioGenerationContext::from_tts_config_and_model_path(&model_config.tts_config, model_path)?;+ let text_decoder = build_text_decoder_backend(model_config, &audio, model_path, options)?;let audio_decoder = build_audio_decoder_backend(&audio)?;- let message_processor_config = model_metadata.model_config.message_processor_config.clone();+ let AnyTokenCodecConfig::TTSCodecConfig(token_codec_config) = &model_config.token_codec_config else {+ return Err(Error::InvalidModelConfig("expected TTSCodecConfig token codec".to_string()));+ };+ let token_codec_config = token_codec_config.clone();Ok(LoadedTtsRuntime {audio,audio_decoder,text_decoder,- message_processor_config,+ token_codec_config,})}pub(super) fn build_text_decoder_backend<B: Backend>(- tts_model_config: &TtsModelConfig,+ tts_model_config: &TTSModelConfig,audio: &AudioGenerationContext<B>,model_path: &Path,options: &TtsSessionOptions,) -> Result<Box<dyn SemanticDecoderBackend>, Error> {match &tts_model_config.tts_config.text_decoder_config {- TtsTextDecoderConfig::FishAudio(config) => {+ AnyTTSTextDecoderConfig::FishAudioTextDecoderConfig(config) => {fishaudio::build_fishaudio_text_decoder_runtime(config, audio, model_path, &options.text_decoder)},}diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rsindex 76b73cc51..cbb0280ed 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs@@ -2,6 +2,7 @@ use super::{decoder_support::*, *};use crate::{array::{Array, ArrayContextExt},backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},+ config::decoder::DecoderConfig,encodable_block::{DecoderArguments, DecoderDecodeInput, SamplingInputs},forward_pass::token_inputs::TokenInputs,session::types::TtsModelConfigError,@@ -17,34 +18,31 @@ struct TokenDecoderLoadedModel<B: Backend> {impl<B: Backend> TokenDecoderLoadedModel<B> {fn load(context: &Rc<B::Context>,- model_path: &Path,- decoder_config: &Rc<crate::config::DecoderConfig>,+ decoder_config: &Rc<DecoderConfig>,model_shape: &ModelShape,+ root_loader_view: &crate::parameters::ParameterTree<B>,transformer_subtree: &str,embedding_subtree: &str,readout_subtree: &str,) -> Result<Self, Error> {- let weights_path = model_path.join("model.safetensors");- let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;- let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;- let root_loader_view = loader.tree();-let shared_buffers = TokenDecoderContext::<B>::build_shared_buffers(context,decoder_config,- &root_loader_view,+ model_shape,+ root_loader_view,transformer_subtree,)?;let executables = Decoder::new_with_embedding_and_readout_subtrees(context.as_ref(),&decoder_config,- &root_loader_view,+ root_loader_view,transformer_subtree,embedding_subtree,readout_subtree,- );- let logits_data_type = model_shape.activation_data_type();- let sampler = GpuSampling::new(context.as_ref(), logits_data_type).map_err(unable_to_create_context)?;+ model_shape,+ )+ .map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;+ let sampler = GpuSampling::new(context.as_ref(), model_shape.data_type).map_err(unable_to_create_context)?;let token_copy_sampled =<B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;@@ -62,7 +60,7 @@ struct TokenDecoderContext<B: Backend> {cache_layers: Rc<RefCell<CacheLayers<B>>>,shared_buffers: Rc<SharedBuffers<B>>,model_shape: ModelShape,- decoder_config: Rc<crate::config::DecoderConfig>,+ decoder_config: Rc<DecoderConfig>,runtime_config: TextDecoderRuntimeConfig,executables: Decoder<B>,sampler: GpuSampling<B>,@@ -83,22 +81,27 @@ impl<B: Backend> TokenDecoderContext<B> {fn new(context: Rc<B::Context>,model_path: &Path,- decoder_config: Rc<crate::config::DecoderConfig>,+ decoder_config: Rc<DecoderConfig>,transformer_subtree: &str,embedding_subtree: &str,readout_subtree: &str,runtime_config: &TextDecoderRuntimeConfig,- ) -> Result<(Self, DataType), Error> {- let model_shape = ModelShape::from_decoder_config(&decoder_config);+ ) -> Result<Self, Error> {+ let weights_path = model_path.join("model.safetensors");+ let weights_file = File::open(&weights_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let loader = ParameterLoader::new(&weights_file, context.as_ref())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let root_loader_view = loader.tree();++ let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::F32);let max_suffix_length =text_decoder_prefill_step_size(runtime_config, decoder_config.transformer_config.context_length).max(32);let max_prefix_length = max_suffix_length;- let activation_data_type = model_shape.activation_data_type();let loaded_model = TokenDecoderLoadedModel::<B>::load(&context,- model_path,&decoder_config,&model_shape,+ &root_loader_view,transformer_subtree,embedding_subtree,readout_subtree,@@ -112,41 +115,39 @@ impl<B: Backend> TokenDecoderContext<B> {max_prefix_length,max_suffix_length,)));- let intermediate_data_type: DataType =- decoder_config.transformer_config.output_norm_config.scale_precision.into();- let kv_cache_update = KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)+ let kv_cache_update = KVCacheUpdate::new(context.as_ref(), model_shape.data_type, max_prefix_length).map_err(unable_to_create_context)?;- Ok((- Self {- context,- cache_layers,- shared_buffers: loaded_model.shared_buffers,- model_shape,- decoder_config,- runtime_config: runtime_config.clone(),- executables: loaded_model.executables,- sampler: loaded_model.sampler,- kv_cache_update,- token_copy_sampled: loaded_model.token_copy_sampled,- async_chain_positions,- async_chain_seeds,- async_chain_results,- async_chain_capacity,- current_max_prefix_length: max_prefix_length,- },- activation_data_type,- ))+ Ok(Self {+ context,+ cache_layers,+ shared_buffers: loaded_model.shared_buffers,+ model_shape,+ decoder_config,+ runtime_config: runtime_config.clone(),+ executables: loaded_model.executables,+ sampler: loaded_model.sampler,+ kv_cache_update,+ token_copy_sampled: loaded_model.token_copy_sampled,+ async_chain_positions,+ async_chain_seeds,+ async_chain_results,+ async_chain_capacity,+ current_max_prefix_length: max_prefix_length,+ })}fn build_shared_buffers(context: &Rc<B::Context>,- decoder_config: &Rc<crate::config::DecoderConfig>,- root_loader_view: &crate::parameters::ParameterTree<B::Context>,+ decoder_config: &Rc<DecoderConfig>,+ model_shape: &ModelShape,+ root_loader_view: &crate::parameters::ParameterTree<B>,transformer_subtree: &str,) -> Result<Rc<SharedBuffers<B>>, Error> {- let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config);- let transformer_tree = root_loader_view.subtree(transformer_subtree).map_err(|_| Error::UnableToLoadWeights)?;+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config, model_shape);+ let transformer_tree = root_loader_view+ .subtree(transformer_subtree)+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;shared_buffers.update_data_from_transformer_tree(&transformer_tree)?;Ok(Rc::new(shared_buffers))}@@ -176,7 +177,7 @@ pub(super) struct TokenDecoderRunner<B: Backend> {impl<B: Backend> TokenDecoderRunner<B> {pub(super) fn activation_data_type(&self) -> DataType {- self.ctx.model_shape.activation_data_type()+ self.ctx.model_shape.data_type}pub(super) fn context(&self) -> &Rc<B::Context> {@@ -186,13 +187,13 @@ impl<B: Backend> TokenDecoderRunner<B> {pub(super) fn new_with_context(context: Rc<B::Context>,model_path: &Path,- decoder_config: Rc<crate::config::DecoderConfig>,+ decoder_config: Rc<DecoderConfig>,transformer_subtree: &str,embedding_subtree: &str,readout_subtree: &str,runtime_config: &TextDecoderRuntimeConfig,) -> Result<Self, Error> {- let (ctx, activation_data_type) = TokenDecoderContext::new(+ let ctx = TokenDecoderContext::new(context,model_path,decoder_config,@@ -204,11 +205,12 @@ impl<B: Backend> TokenDecoderRunner<B> {let context: Rc<B::Context> = Rc::clone(ctx.context());let model_dim = ctx.decoder_config.transformer_config.model_dim;let tensor_add_scale =- <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), activation_data_type)+ <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), ctx.model_shape.data_type).map_err(unable_to_create_context)?;- let single_hidden_capture = context.create_array_zeros(&[1, model_dim], activation_data_type).into_allocation();+ let single_hidden_capture =+ context.create_array_zeros(&[1, model_dim], ctx.model_shape.data_type).into_allocation();let single_override_embedding =- context.create_array_zeros(&[1, model_dim], activation_data_type).into_allocation();+ context.create_array_zeros(&[1, model_dim], ctx.model_shape.data_type).into_allocation();Ok(Self {ctx,@@ -337,10 +339,8 @@ impl<B: Backend> TokenDecoderRunner<B> {let context = self.ctx.context.as_ref();*self.ctx.cache_layers.borrow_mut() =CacheLayers::new(context, &self.ctx.model_shape, max_prefix_length, max_suffix_length);- let intermediate_data_type: DataType =- self.ctx.decoder_config.transformer_config.output_norm_config.scale_precision.into();- self.ctx.kv_cache_update =- KVCacheUpdate::new(context, intermediate_data_type, max_prefix_length).map_err(unable_to_create_context)?;+ self.ctx.kv_cache_update = KVCacheUpdate::new(context, self.ctx.model_shape.data_type, max_prefix_length)+ .map_err(unable_to_create_context)?;self.ctx.current_max_prefix_length = max_prefix_length;Ok(())}diff --git a/crates/backend-uzu/src/session/tts_session/decoder_support.rs b/crates/backend-uzu/src/session/tts_session/decoder_support.rsindex a04ccb40c..2dcbfcda9 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_support.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_support.rs@@ -187,7 +187,7 @@ pub(super) struct StreamingTokenAccumulator {impl StreamingTokenAccumulator {pub(super) fn new(num_codebooks: usize) -> Result<Self, Error> {if num_codebooks == 0 {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig("TTS num_codebooks must be greater than 0".to_string()));}Ok(Self {by_codebook: vec![Vec::new(); num_codebooks],@@ -229,14 +229,13 @@ impl StreamingTokenAccumulator {tokens.extend_from_slice(&codebook[frame_start..frame_end]);}- AudioTokenGrid::new(+ Ok(AudioTokenGrid::new(tokens.into_boxed_slice(),1,self.by_codebook.len(),range_frames,vec![range_frames].into_boxed_slice(),- )- .map_err(Error::from)+ )?)}}@@ -273,8 +272,7 @@ pub(super) fn slice_grid_range(lengths.push(length.saturating_sub(frame_start).min(range_frames));}- AudioTokenGrid::new(tokens.into_boxed_slice(), batch_size, codebooks, range_frames, lengths.into_boxed_slice())- .map_err(Error::from)+ Ok(AudioTokenGrid::new(tokens.into_boxed_slice(), batch_size, codebooks, range_frames, lengths.into_boxed_slice())?)}pub(super) fn accumulate_audio_decode_step_stats(@@ -441,14 +439,10 @@ impl<B: Backend + Send + Sync> TtsSession<B> {}let config_path = model_path.join("config.json");- if !config_path.exists() {- return Err(Error::UnableToLoadConfig);- }- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;- let model_metadata: ModelMetadata<TtsModelConfig> =- serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;+ let config_file = File::open(&config_path)?;+ let model_config: TTSModelConfig = serde_json::from_reader(BufReader::new(config_file))?;- Self::from_model_metadata_with_options(model_path, model_metadata, options)+ Self::from_model_config_with_options(model_path, model_config, options)}pub fn last_execution_stats(&self) -> Option<TtsExecutionStats> {@@ -459,24 +453,21 @@ impl<B: Backend + Send + Sync> TtsSession<B> {self.audio.sample_rate()}- fn from_model_metadata_with_options(+ fn from_model_config_with_options(model_path: PathBuf,- model_metadata: ModelMetadata<TtsModelConfig>,+ model_config: TTSModelConfig,options: TtsSessionOptions,) -> Result<Self, Error> {let tokenizer_path = model_path.join("tokenizer.json");- if !tokenizer_path.exists() {- return Err(Error::UnableToLoadTokenizer);- }- let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(Error::UnableToLoadTokenizer)?;- let loaded_runtime = load_tts_runtime(&model_path, &model_metadata, &options)?;+ let loaded_runtime = load_tts_runtime(&model_path, &model_config, &options)?;Ok(Self {tokenizer,audio: loaded_runtime.audio,audio_decoder: loaded_runtime.audio_decoder,- message_processor_config: loaded_runtime.message_processor_config,+ token_codec_config: loaded_runtime.token_codec_config,text_decoder: loaded_runtime.text_decoder,last_execution_stats: None,})@@ -489,73 +480,81 @@ impl<B: Backend + Send + Sync> TtsSession<B> {let messages = input.get_messages().into_iter()- .map(|message| message.resolve(&self.message_processor_config))+ .map(|message| message.resolve(&self.token_codec_config)).collect::<Vec<_>>();let template_name = "tts_prompt_template";let mut environment = Environment::new();environment- .add_template(template_name, self.message_processor_config.prompt_template.as_str())- .map_err(|_| Error::UnableToLoadPromptTemplate)?;- let template = environment.get_template(template_name).map_err(|_| Error::UnableToLoadPromptTemplate)?;+ .add_template(template_name, self.token_codec_config.prompt_template.as_str())+ .map_err(Error::UnableToLoadPromptTemplate)?;+ let template = environment.get_template(template_name).map_err(Error::UnableToLoadPromptTemplate)?;let result = template.render(context!(messages => messages))- .map_err(|_| Error::UnableToRenderPromptTemplate)?;+ .map_err(Error::UnableToRenderPromptTemplate)?;Ok(normalize_rendered_prompt(result,- self.message_processor_config.prompt_template.as_str(),- self.message_processor_config.drop_initial_newline,+ self.token_codec_config.prompt_template.as_str(),+ self.token_codec_config.drop_initial_newline,))}}pub(super) fn semantic_token_to_code(semantic_token: u64,- semantic_begin: i64,- semantic_end: i64,+ semantic_begin: u64,+ semantic_end: u64,token_upper_bound: usize,) -> u32 {if semantic_begin > semantic_end || token_upper_bound == 0 {return 0;}- let semantic = semantic_token as i64;- if semantic < semantic_begin || semantic > semantic_end {+ if semantic_token < semantic_begin || semantic_token > semantic_end {return 0;}- let relative = usize::try_from(semantic - semantic_begin).unwrap_or(0);+ let relative = usize::try_from(semantic_token - semantic_begin).unwrap_or(0);let clamped = relative.min(token_upper_bound.saturating_sub(1));u32::try_from(clamped).unwrap_or(0)}pub(super) fn build_semantic_sampling_mask_row(vocab_size: usize,- semantic_begin: i64,- semantic_end: i64,- im_end: i64,+ semantic_begin: u64,+ semantic_end: u64,+ im_end: u64,) -> Result<Box<[u32]>, Error> {if vocab_size == 0 || semantic_begin > semantic_end {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig(format!(+ "invalid semantic sampling mask range: vocab_size={vocab_size}, semantic_begin={semantic_begin}, semantic_end={semantic_end}",+ )));}- let max_token_id = i64::try_from(vocab_size.saturating_sub(1)).map_err(|_| Error::UnableToLoadConfig)?;- if semantic_begin < 0 || semantic_end < 0 || semantic_end > max_token_id || im_end < 0 || im_end > max_token_id {- return Err(Error::UnableToLoadConfig);+ let max_token_id = u64::try_from(vocab_size.saturating_sub(1)).map_err(|_| {+ Error::InvalidModelConfig(format!("vocab_size={vocab_size} cannot be represented as a token id"))+ })?;+ if semantic_end > max_token_id || im_end > max_token_id {+ return Err(Error::InvalidModelConfig(format!(+ "semantic sampling mask token id exceeds vocab: semantic_end={semantic_end}, im_end={im_end}, max_token_id={max_token_id}",+ )));}let row_words = vocab_size.div_ceil(32);let mut mask = vec![0_u32; row_words];for token_index in semantic_begin..=semantic_end {- let token = usize::try_from(token_index).map_err(|_| Error::UnableToLoadConfig)?;+ let token = usize::try_from(token_index).map_err(|_| {+ Error::InvalidModelConfig(format!("semantic token id {token_index} cannot be represented as usize"))+ })?;let word = token / 32;mask[word] |= 2_u32.pow((token % 32) as u32);}- let im_end_token = usize::try_from(im_end).map_err(|_| Error::UnableToLoadConfig)?;+ let im_end_token = usize::try_from(im_end)+ .map_err(|_| Error::InvalidModelConfig(format!("im_end token id {im_end} cannot be represented as usize")))?;let word = im_end_token / 32;mask[word] |= 2_u32.pow((im_end_token % 32) as u32);Ok(mask.into_boxed_slice())@@ -563,15 +562,16 @@ pub(super) fn build_semantic_sampling_mask_row(pub(super) fn clear_token_in_sampling_mask(mask: &mut [u32],- token: i64,+ token: u64,) -> Result<(), Error> {- if token < 0 {- return Err(Error::UnableToLoadConfig);- }- let token = usize::try_from(token).map_err(|_| Error::UnableToLoadConfig)?;+ let token = usize::try_from(token)+ .map_err(|_| Error::InvalidModelConfig(format!("token id {token} cannot be represented as usize")))?;let word = token / 32;if word >= mask.len() {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig(format!(+ "token id {token} is outside sampling mask with {} words",+ mask.len(),+ )));}mask[word] &= !2_u32.pow((token % 32) as u32);Ok(())diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rsindex d4371fbe7..fe17cdac3 100644--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs@@ -2,7 +2,10 @@ use super::*;use crate::{array::ArrayContextExt,backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},- config::DecoderConfig,+ config::{+ decoder::DecoderConfig, embedding::AnyEmbeddingConfig,+ tts::text_decoder::fish_audio_text_decoder::FishAudioTextDecoderConfig,+ },encodable_block::SamplingInputs,forward_pass::token_inputs::TokenInputs,session::types::TtsModelConfigError,@@ -13,10 +16,10 @@ pub(super) struct FishAudioTextDecoderRuntime<B: Backend> {fast_runner: TokenDecoderRunner<B>,semantic_bridge: FishAudioSemanticBridge<B>,runtime_config: TextDecoderRuntimeConfig,- semantic_token_begin_id: i64,- semantic_token_end_id: i64,+ semantic_token_begin_id: u64,+ semantic_token_end_id: u64,semantic_cardinality: usize,- im_end_token_id: i64,+ im_end_token_id: u64,codebook_size: usize,num_codebooks: usize,slow_model_dim: usize,@@ -46,26 +49,29 @@ struct FishAudioSemanticBridge<B: Backend> {impl<B: Backend> FishAudioSemanticBridge<B> {fn load(context: &B::Context,- parameter_tree: &crate::parameters::ParameterTree<B::Context>,- config: &crate::config::FishAudioTextDecoderConfig,+ parameter_tree: &crate::parameters::ParameterTree<B>,+ config: &FishAudioTextDecoderConfig,data_type: DataType,) -> Result<Self, Error> {- let codebook_embeddings = load_float_tensor_allocation(- parameter_tree,- "text_decoder.codebook_embeddings.weights",- [- config.codebook_size.checked_mul(config.num_codebooks).ok_or(Error::UnableToLoadConfig)?,- config.slow_model_dim,- ],- data_type,- )?;+ let codebook_embedding_rows = config.codebook_size.checked_mul(config.num_codebooks).ok_or_else(|| {+ Error::InvalidModelConfig(format!(+ "FishAudio codebook embedding rows overflow: codebook_size={}, num_codebooks={}",+ config.codebook_size, config.num_codebooks,+ ))+ })?;+ let codebook_embeddings = parameter_tree+ .leaf("text_decoder.codebook_embeddings.embedding.weights")+ .and_then(|leaf| leaf.validate(&[codebook_embedding_rows, config.slow_model_dim], data_type))+ .and_then(|leaf| leaf.read_allocation())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let fast_model_projection = if config.fast_model_projection_config.is_some() {- Some(load_float_tensor_allocation(- parameter_tree,- "text_decoder.fast_model_projection.weights",- [config.fast_model_dim, config.slow_model_dim],- data_type,- )?)+ Some(+ parameter_tree+ .leaf("text_decoder.fast_model_projection.weights.weights")+ .and_then(|leaf| leaf.validate(&[config.fast_model_dim, config.slow_model_dim], data_type))+ .and_then(|leaf| leaf.read_allocation())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?,+ )} else {None};@@ -112,33 +118,12 @@ impl<B: Backend> FishAudioSemanticBridge<B> {}}-fn load_float_tensor_allocation<B: Backend>(- parameter_tree: &crate::parameters::ParameterTree<B::Context>,- key: &str,- expected_shape: [usize; 2],- target_data_type: DataType,-) -> Result<Allocation<B>, Error> {- let leaf = parameter_tree.leaf(key).map_err(|_| Error::UnableToLoadWeights)?;- if leaf.shape() != expected_shape {- return Err(Error::UnableToLoadConfig);- }- if leaf.data_type() != target_data_type {- return Err(TtsModelConfigError::FishAudioTensorDataTypeMismatch {- key: key.into(),- expected: target_data_type,- actual: leaf.data_type(),- }- .into());- }- leaf.read_allocation().map_err(|_| Error::UnableToLoadWeights)-}-fn validate_fishaudio_decoder_contract(num_codebooks: usize,codebook_size: usize,max_seq_len: usize,- semantic_token_begin_id: i64,- semantic_token_end_id: i64,+ semantic_token_begin_id: u64,+ semantic_token_end_id: u64,audio_num_codebooks: usize,audio_codec_cardinality: usize,audio_semantic_cardinality: usize,@@ -173,12 +158,14 @@ fn validate_fishaudio_decoder_contract(.into());}- let semantic_cardinality = usize::try_from(semantic_token_end_id - semantic_token_begin_id + 1).map_err(|_| {- TtsModelConfigError::FishAudioSemanticTokenRangeOverflow {+ let semantic_cardinality = semantic_token_end_id+ .checked_sub(semantic_token_begin_id)+ .and_then(|value| value.checked_add(1))+ .and_then(|value| usize::try_from(value).ok())+ .ok_or(TtsModelConfigError::FishAudioSemanticTokenRangeOverflow {begin: semantic_token_begin_id,end: semantic_token_end_id,- }- })?;+ })?;if semantic_cardinality == 0 || semantic_cardinality != audio_semantic_cardinality {return Err(TtsModelConfigError::FishAudioSemanticCodecCardinalityMismatch {semantic_cardinality,@@ -191,13 +178,14 @@ fn validate_fishaudio_decoder_contract(}pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(- config: &crate::config::FishAudioTextDecoderConfig,+ config: &FishAudioTextDecoderConfig,audio: &AudioGenerationContext<B>,model_path: &Path,runtime_config: &TextDecoderRuntimeConfig,) -> Result<Box<dyn SemanticDecoderBackend>, Error> {- let audio_semantic_cardinality =- audio.runtime().config().semantic_codec_cardinality().ok_or(Error::UnableToLoadConfig)?;+ let audio_semantic_cardinality = audio.runtime().config().semantic_codec_cardinality().ok_or_else(|| {+ Error::InvalidModelConfig("audio decoder does not expose semantic codec cardinality".to_string())+ })?;let semantic_cardinality = validate_fishaudio_decoder_contract(config.num_codebooks,config.codebook_size,@@ -209,13 +197,19 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(audio_semantic_cardinality,)?;if config.slow_model_config.model_dim != config.slow_model_dim {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig(format!(+ "FishAudio slow model_dim mismatch: transformer={}, configured={}",+ config.slow_model_config.model_dim, config.slow_model_dim,+ )));}if config.fast_model_config.model_dim != config.fast_model_dim {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig(format!(+ "FishAudio fast model_dim mismatch: transformer={}, configured={}",+ config.fast_model_config.model_dim, config.fast_model_dim,+ )));}if config.short_logits_size == 0 {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig("FishAudio short_logits_size must be greater than 0".to_string()));}let semantic_sampling_mask_row = build_semantic_sampling_mask_row(config.vocab_size,@@ -230,14 +224,14 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(let fast_transformer_config = config.fast_model_config.clone();let slow_decoder_config = Rc::new(DecoderConfig {- embedding_config: config.slow_embeddings_config.to_text_decoder_embedding_config(),+ embedding_config: AnyEmbeddingConfig::TiedEmbeddingConfig(config.slow_embeddings_config.clone()),transformer_config: slow_transformer_config,vocab_size: config.vocab_size,pard_token: None,ple_model_config: None,});let fast_decoder_config = Rc::new(DecoderConfig {- embedding_config: config.fast_embeddings_config.to_text_decoder_embedding_config(),+ embedding_config: AnyEmbeddingConfig::TiedEmbeddingConfig(config.fast_embeddings_config.clone()),transformer_config: fast_transformer_config,vocab_size: config.codebook_size,pard_token: None,@@ -268,13 +262,15 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(let activation_data_type = slow_runner.activation_data_type();let fast_data_type = fast_runner.activation_data_type();if fast_data_type != activation_data_type {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig(format!(+ "FishAudio slow/fast activation data type mismatch: slow={activation_data_type:?}, fast={fast_data_type:?}",+ )));}let weights_path = model_path.join("model.safetensors");- let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;- let loader =- ParameterLoader::new(&weights_file, text_decoder_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;+ let weights_file = File::open(&weights_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let loader = ParameterLoader::new(&weights_file, text_decoder_context.as_ref())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let root_weights = loader.tree();let apply_semantic_sampling_mask =@@ -317,26 +313,30 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {mut on_frame: Option<&mut dyn FnMut(&[u32]) -> Result<(), Error>>,) -> Result<AudioTokenGrid, Error> {if text_tokens.is_empty() {- return AudioTokenGrid::new(+ return Ok(AudioTokenGrid::new(Vec::new().into_boxed_slice(),1,self.num_codebooks,0,vec![0].into_boxed_slice(),- )- .map_err(Error::from);+ )?);}if text_tokens.len() >= self.max_seq_len {return Err(Error::GenerateFailed);}if codec_cardinality == 0 || codec_cardinality > self.codebook_size {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig(format!(+ "FishAudio codec cardinality {codec_cardinality} must be in 1..={}",+ self.codebook_size,+ )));}let semantic_token_upper_bound = self.semantic_cardinality;let residual_token_upper_bound = codec_cardinality;if semantic_token_upper_bound == 0 || residual_token_upper_bound == 0 {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig(format!(+ "FishAudio token upper bounds must be positive: semantic={semantic_token_upper_bound}, residual={residual_token_upper_bound}",+ )));}self.reset_generation_state();@@ -365,7 +365,7 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {self.prepare_fast_runner_masks(residual_token_upper_bound)?;for _step in 0..max_new_tokens {- if current_semantic_token as i64 == self.im_end_token_id {+ if current_semantic_token == self.im_end_token_id {break;}let first_code = semantic_token_to_code(@@ -448,7 +448,10 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {) -> Result<(), Error> {let fast_vocab_limit = self.fast_vocab_limit.min(residual_token_upper_bound);if fast_vocab_limit == 0 {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig(format!(+ "FishAudio fast vocab limit is zero: configured={}, residual={residual_token_upper_bound}",+ self.fast_vocab_limit,+ )));}self.fast_runner.prepare_single_token_vocab_mask(fast_vocab_limit)?;self.fast_runner.prepare_two_token_vocab_mask(fast_vocab_limit)?;@@ -467,8 +470,13 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {}tokens.extend_from_slice(codebook_tokens);}- AudioTokenGrid::new(tokens.into_boxed_slice(), 1, self.num_codebooks, frames, vec![frames].into_boxed_slice())- .map_err(Error::from)+ Ok(AudioTokenGrid::new(+ tokens.into_boxed_slice(),+ 1,+ self.num_codebooks,+ frames,+ vec![frames].into_boxed_slice(),+ )?)}fn encode_project_slow_hidden_to_fast_on(@@ -508,7 +516,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {}if slow_model_dim != fast_model_dim {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig(format!(+ "FishAudio slow_model_dim={slow_model_dim} must equal fast_model_dim={fast_model_dim} without projection weights",+ )));}debug_assert_eq!(@@ -537,7 +547,7 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {encoder: &mut Encoder<B>,) -> Result<(), Error> {if num_codebooks == 0 {- return Err(Error::UnableToLoadConfig);+ return Err(Error::InvalidModelConfig("FishAudio num_codebooks must be greater than 0".to_string()));}// Write first_code at position 0 from CPU.@@ -566,11 +576,10 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {)?;}- let total_vocab = num_codebooks.checked_mul(codebook_size).ok_or(Error::GenerateFailed)?;- let total_vocab_u32 = u32::try_from(total_vocab).map_err(|_| Error::GenerateFailed)?;- let num_codebooks_u32 = u32::try_from(num_codebooks).map_err(|_| Error::GenerateFailed)?;- let slow_model_dim_u32 = u32::try_from(slow_model_dim).map_err(|_| Error::GenerateFailed)?;- let codebook_size_u32 = u32::try_from(codebook_size).map_err(|_| Error::GenerateFailed)?;+ let total_vocab_u32 = (num_codebooks * codebook_size) as u32;+ let num_codebooks_u32 = num_codebooks as u32;+ let slow_model_dim_u32 = slow_model_dim as u32;+ let codebook_size_u32 = codebook_size as u32;let codebook_token_indices = &semantic_bridge.codebook_token_indices;let codebook_embeddings = &semantic_bridge.codebook_embeddings;@@ -646,7 +655,7 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {let sampled = self.fast_runner.read_async_chain_result(pass)?;let codebook_index = pass + 1;let clamped = (sampled as usize).min(residual_token_upper_bound.saturating_sub(1));- let clamped = u32::try_from(clamped).map_err(|_| Error::GenerateFailed)?;+ let clamped = clamped as u32;by_codebook[codebook_index].push(clamped);self.current_codes_scratch[codebook_index] = clamped;}diff --git a/crates/backend-uzu/src/session/tts_session/generator.rs b/crates/backend-uzu/src/session/tts_session/generator.rsindex 690b21fe2..cc282b849 100644--- a/crates/backend-uzu/src/session/tts_session/generator.rs+++ b/crates/backend-uzu/src/session/tts_session/generator.rs@@ -36,7 +36,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {let text_tokens: Vec<u64> = self.tokenizer.encode(prompt.as_str(), false)- .map_err(|_| Error::UnableToEncodeText)?+ .map_err(Error::UnableToEncodeText)?.get_ids().iter().map(|&token| token as u64)@@ -117,8 +117,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {sample_rate,channels,accumulated_lengths.into_boxed_slice(),- )- .map_err(Error::from)?+ )?}},};@@ -162,7 +161,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {let text_tokens: Vec<u64> = self.tokenizer.encode(prompt.as_str(), false)- .map_err(|_| Error::UnableToEncodeText)?+ .map_err(Error::UnableToEncodeText)?.get_ids().iter().map(|&token| token as u64)@@ -231,7 +230,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {let text_tokens: Vec<u64> = self.tokenizer.encode(prompt.as_str(), false)- .map_err(|_| Error::UnableToEncodeText)?+ .map_err(Error::UnableToEncodeText)?.get_ids().iter().map(|&token| token as u64)@@ -370,8 +369,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {ss.output_sample_rate,ss.output_channels,vec![ss.output_frames].into_boxed_slice(),- )- .map_err(Error::from)?;+ )?;let instrumentation = self.take_text_decoder_instrumentation();self.record_last_execution_stats(TtsExecutionStats {diff --git a/crates/backend-uzu/src/session/types/error.rs b/crates/backend-uzu/src/session/types/error.rsindex 794454017..a7de58b2c 100644--- a/crates/backend-uzu/src/session/types/error.rs+++ b/crates/backend-uzu/src/session/types/error.rs@@ -1,12 +1,7 @@-use crate::{DataType, classifier::ClassifierError, config::ModelType, session::config::TtsRunConfigError};+use crate::{DataType, classifier::ClassifierError, session::config::TtsRunConfigError};#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]pub enum TtsModelConfigError {- #[error("expected model_type={expected:?}, got {actual:?}")]- ModelTypeMismatch {- expected: ModelType,- actual: ModelType,- },#[error("missing TTS model config")]MissingTtsModelConfig,#[error("FishAudio tensor {key} dtype mismatch: expected {expected:?}, got {actual:?}")]@@ -39,13 +34,13 @@ pub enum TtsModelConfigError {},#[error("FishAudio semantic token range is invalid: begin={begin}, end={end}")]FishAudioSemanticTokenRangeInvalid {- begin: i64,- end: i64,+ begin: u64,+ end: u64,},#[error("FishAudio semantic token range overflow: begin={begin}, end={end}")]FishAudioSemanticTokenRangeOverflow {- begin: i64,- end: i64,+ begin: u64,+ end: u64,},#[error("FishAudio semantic codec cardinality={semantic_cardinality} does not match audio semantic codec cardinality={audio_semantic_cardinality}"@@ -170,16 +165,24 @@ pub enum Error {UnableToCreateContext(Box<dyn std::error::Error>),#[error("Unable to create CommandBuffer: {0}")]UnableToCreateCommandBuffer(Box<dyn std::error::Error>),- #[error("Unable to load model configuration")]- UnableToLoadConfig,+ #[error("Unable to open model configuration: {0}")]+ UnableToOpenConfig(#[from] std::io::Error),+ #[error("Unable to deserialize model configuration: {0}")]+ UnableToDeserializeConfig(#[from] serde_json::Error),+ #[error("Invalid model configuration: {0}")]+ InvalidModelConfig(String),#[error("Invalid TTS model config: {0}")]InvalidTtsModelConfig(#[from] TtsModelConfigError),#[error("Invalid TTS run config: {0}")]InvalidTtsRunConfig(#[from] TtsRunConfigError),- #[error("Unable to load model weights")]- UnableToLoadWeights,- #[error("Unable to load tokenizer")]- UnableToLoadTokenizer,+ #[error("Unable to load model weights: {0}")]+ UnableToLoadWeights(#[source] Box<dyn std::error::Error>),+ #[error("Unable to create decoder: {0}")]+ UnableToCreateDecoder(#[source] Box<dyn std::error::Error>),+ #[error("Unable to create classifier layer: {0}")]+ UnableToCreateClassifierLayer(#[source] Box<dyn std::error::Error>),+ #[error("Unable to load tokenizer: {0}")]+ UnableToLoadTokenizer(#[source] tokenizers::Error),#[error("Model is too large to fit into available RAM")]NotEnoughMemory,#[error("Command buffer failed: {0}")]@@ -190,20 +193,18 @@ pub enum Error {UnsupportedContextModeForModel,#[error("Unsupported speculator config for model")]UnsupportedSpeculatorConfigForModel,- #[error("Language model generator not loaded")]- LanguageModelGeneratorNotLoaded,- #[error("Unable to load prompt template")]- UnableToLoadPromptTemplate,- #[error("Unable to render prompt template")]- UnableToRenderPromptTemplate,+ #[error("Unable to load prompt template: {0}")]+ UnableToLoadPromptTemplate(#[source] minijinja::Error),+ #[error("Unable to render prompt template: {0}")]+ UnableToRenderPromptTemplate(#[source] minijinja::Error),#[error("Only assistant messages can have reasoning content")]UnexpectedReasoningContent,- #[error("Unable to build output parser regex")]- UnableToBuildOutputParserRegex,- #[error("Unable to encode text")]- UnableToEncodeText,- #[error("Unable to decode text")]- UnableToDecodeText,+ #[error("Unable to build output parser regex: {0}")]+ UnableToBuildOutputParserRegex(#[from] regex::Error),+ #[error("Unable to encode text: {0}")]+ UnableToEncodeText(#[source] tokenizers::Error),+ #[error("Unable to decode text: {0}")]+ UnableToDecodeText(#[source] tokenizers::Error),#[error("Context length exceeded")]ContextLengthExceeded,#[error("Prefill failed")]@@ -220,8 +221,8 @@ pub enum Error {GrammarReject,#[error("Token {0} out of grammar vocabulary range (0..{1})")]TokenOutOfGrammarRange(u64, usize),- #[error("Capture failed")]- CaptureFailed,+ #[error("Capture failed: {0}")]+ CaptureFailed(#[source] Box<dyn std::error::Error>),#[error("Classifier error: {0}")]Classifier(#[from] ClassifierError),#[error("Audio codec error: {0}")]diff --git a/crates/backend-uzu/src/session/types/message.rs b/crates/backend-uzu/src/session/types/message.rsindex 134ead89c..8d9bfc890 100644--- a/crates/backend-uzu/src/session/types/message.rs+++ b/crates/backend-uzu/src/session/types/message.rs@@ -3,9 +3,9 @@ use std::collections::HashMap;use serde::{Deserialize, Serialize};#[cfg(metal_backend)]-use crate::config::TtsMessageProcessorConfig;+use crate::config::token_codec::tts_codec::TTSCodecConfig;use crate::{- config::MessageProcessorConfig,+ config::token_codec::chat_codec::ChatCodecConfig,session::{parameter::ConfigResolvableValue, types::Role},};@@ -50,10 +50,10 @@ impl Message {}}-impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>> for Message {+impl ConfigResolvableValue<ChatCodecConfig, HashMap<String, String>> for Message {fn resolve(&self,- config: &MessageProcessorConfig,+ config: &ChatCodecConfig,) -> HashMap<String, String> {let role = match self.role {Role::System => config.system_role_name.clone(),@@ -70,10 +70,10 @@ impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>> for}#[cfg(metal_backend)]-impl ConfigResolvableValue<TtsMessageProcessorConfig, HashMap<String, String>> for Message {+impl ConfigResolvableValue<TTSCodecConfig, HashMap<String, String>> for Message {fn resolve(&self,- _config: &TtsMessageProcessorConfig,+ _config: &TTSCodecConfig,) -> HashMap<String, String> {HashMap::from([(String::from("content"), self.content.clone()),diff --git a/crates/backend-uzu/src/utils/model_size.rs b/crates/backend-uzu/src/utils/model_size.rsindex b95719b4b..22f42c479 100644--- a/crates/backend-uzu/src/utils/model_size.rs+++ b/crates/backend-uzu/src/utils/model_size.rs@@ -9,11 +9,11 @@ pub enum ModelSize {}impl ModelSize {- pub fn from_path(model_path: &Path) -> Self {+ pub fn from_path(model_path: &Path) -> std::io::Result<Self> {let weights_path = model_path.join("model.safetensors");- let size_bytes = std::fs::metadata(&weights_path).map(|m| m.len()).unwrap_or(0);+ let size_bytes = std::fs::metadata(&weights_path)?.len();- Self::from_bytes(size_bytes)+ Ok(Self::from_bytes(size_bytes))}pub fn from_bytes(size_bytes: u64) -> Self {diff --git a/crates/backend-uzu/src/utils/strict_serde.rs b/crates/backend-uzu/src/utils/strict_serde.rsindex bc2ec29d6..9cca71654 100644--- a/crates/backend-uzu/src/utils/strict_serde.rs+++ b/crates/backend-uzu/src/utils/strict_serde.rs@@ -1,4 +1,4 @@-use monostate::MustBeBool;+use monostate::{ConstStr, MustBeBool, MustBeStr};use serde::{Deserialize, Deserializer, Serialize};#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]@@ -6,7 +6,7 @@ pub enum Unsupported {}pub trait DeserializeStrict<'de>: Deserialize<'de> {}pub trait DeserializeStrictOwned: for<'de> DeserializeStrict<'de> {}-impl<T> DeserializeStrictOwned for T where T: for<'de> DeserializeStrict<'de> {}+impl<T: for<'de> DeserializeStrict<'de>> DeserializeStrictOwned for T {}macro_rules! impl_strict {($($ty:ty),* $(,)?) => {@@ -14,13 +14,14 @@ macro_rules! impl_strict {};}-impl_strict!(Unsupported, String, f32, i64, u32, usize, bool);+impl_strict!(Unsupported, String, f32, u32, u64, usize, bool);impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Box<T> {}-impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Vec<T> {}+impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Box<[T]> {}impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Option<T> {}impl<'de, A: DeserializeStrict<'de>, B: DeserializeStrict<'de>> DeserializeStrict<'de> for (A, B) {}impl<'de, const V: bool> DeserializeStrict<'de> for MustBeBool<V> {}+impl<'de, V: ConstStr> DeserializeStrict<'de> for MustBeStr<V> {}pub fn required<'de, D: Deserializer<'de>, T: DeserializeStrict<'de>>(deserializer: D) -> Result<T, D::Error> {T::deserialize(deserializer)diff --git a/crates/backend-uzu/src/utils/version.rs b/crates/backend-uzu/src/utils/version.rsindex bafe1a60c..8888ef48b 100644--- a/crates/backend-uzu/src/utils/version.rs+++ b/crates/backend-uzu/src/utils/version.rs@@ -1,2 +1,2 @@pub const VERSION: &str = env!("CARGO_PKG_VERSION");-pub const TOOLCHAIN_VERSION: &str = "0.8.0";+pub const TOOLCHAIN_VERSION: &str = "0.10.0";diff --git a/crates/backend-uzu/tests/integration/parameters/loader_test.rs b/crates/backend-uzu/tests/integration/parameters/loader_test.rsindex 43de028b7..27ceb1d23 100644--- a/crates/backend-uzu/tests/integration/parameters/loader_test.rs+++ b/crates/backend-uzu/tests/integration/parameters/loader_test.rs@@ -2,32 +2,41 @@// New integration test for ParameterLoaderuse backend_uzu::{- ParameterLoader,+ DataType, ParameterLoader,backends::{common::{Backend, Context},metal::Metal,},+ read_safetensors_metadata,};use half::bf16;-use is_close::is_close;use test_tag::tag;use crate::common::path::get_test_weights_path;+const EMBEDDING_PATH: &str = "decoder.embedding.embedding.weights";+const EMBEDDING_TREE_PATH: &str = "decoder.embedding.embedding";+#[tag(heavy)]#[test]fn test_parameter_loader_basic() {let weights_path = get_test_weights_path();let context = <Metal as Backend>::Context::new().expect("Failed to create MetalContext");let file = std::fs::File::open(&weights_path).expect("Weights file not found; run download script");+ let (_header_len, metadata) = read_safetensors_metadata(&file).expect("read weights metadata");+ let embedding_shape = metadata.tensors.get(EMBEDDING_PATH).expect("weights embeddings metadata").shape.clone();+ assert_eq!(embedding_shape.len(), 2);- let loader = ParameterLoader::new(&file, context.as_ref()).expect("create loader");- let embeddings = loader.tree().leaf_array("embedding.weights").expect("weights embeddings");+ let loader = ParameterLoader::<Metal>::new(&file, context.as_ref()).expect("create loader");+ let tree = loader.tree();+ let embeddings_leaf =+ tree.leaf(EMBEDDING_PATH).expect("weights embeddings").validate(&embedding_shape, DataType::BF16).unwrap();+ let embeddings = embeddings_leaf.read_array().unwrap();let view = embeddings.as_view::<bf16>();- assert!(is_close!(view[[5usize, 3usize]], bf16::from_f32(-0.01819)));// tree API check- let subtree = loader.tree().subtree("embedding").unwrap();- let same = subtree.leaf_array("weights").unwrap();+ let subtree = tree.subtree(EMBEDDING_TREE_PATH).unwrap();+ let weights_leaf = subtree.leaf("weights").unwrap().validate(&embedding_shape, DataType::BF16).unwrap();+ let same = weights_leaf.read_array().unwrap();assert_eq!(view, same.as_view::<bf16>());}diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rsindex eb1f32bcd..f5a774f2a 100644--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs@@ -6,19 +6,20 @@use std::{cell::RefCell,+ collections::HashMap,fs::File,+ io::BufReader,path::{Path, PathBuf},rc::Rc,};use backend_uzu::{_private::{- ActivationTrace, ArgmaxSampler, CacheLayers, Classifier, DecoderDecodeInput, KVCacheLayer,- LanguageModelGeneratorContext, LogitsSampler, ModelConfig, ModelMetadata, ModelType, ParameterTree, Sampling,- TokenInputs,+ ActivationTrace, AnyModelConfig, ArgmaxSampler, CacheLayers, Classifier, DecoderDecodeInput, KVCacheLayer,+ LanguageModelGeneratorContext, LogitsSampler, ParameterLoaderError, ParameterTree, Sampling, TokenInputs,},Array, ArrayElement, DataType, ParameterLoader, allocation_to_vec,- backends::common::{Allocation, Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},+ backends::common::{Allocation, AllocationType, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},read_safetensors_metadata,session::{config::{DecodingConfig, SpeculatorConfig},@@ -140,9 +141,21 @@ pub enum ArrayTransform {// Model Context (internal)// ============================================================================+#[derive(Clone, Copy)]+struct LayerTraceRequirements {+ post_mixer_norm: bool,+ post_mlp_norm: bool,+}+enum ModelContext<B: Backend> {- LanguageModelGenerator(LanguageModelGeneratorContext<B>),- Classifier(Classifier<B>),+ LanguageModelGenerator {+ context: LanguageModelGeneratorContext<B>,+ layer_trace_requirements: Box<[LayerTraceRequirements]>,+ },+ Classifier {+ classifier: Classifier<B>,+ layer_trace_requirements: Box<[LayerTraceRequirements]>,+ },}// ============================================================================@@ -163,53 +176,43 @@ impl<B: Backend> TraceValidator<B> {////// Automatically detects the model type from config.json.pub fn new(model_path: &Path) -> Result<Self, Error> {- let config_path = model_path.join("config.json");- if !config_path.exists() {+ if !model_path.exists() {return Err(Error::ModelFolderNotFound);}- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;- let raw_metadata: ModelMetadata<ModelConfig> =- serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;-- let context = match raw_metadata.model_type.clone() {- ModelType::ClassifierModel => {- let ModelConfig::ClassifierModel(model_config) = raw_metadata.model_config.clone() else {- return Err(Error::UnableToLoadConfig);- };- let metadata = ModelMetadata {- toolchain_version: raw_metadata.toolchain_version,- vendor: raw_metadata.vendor,- family: raw_metadata.family,- name: raw_metadata.name,- size: raw_metadata.size,- quantization: raw_metadata.quantization,- repo: raw_metadata.repo,- use_cases: raw_metadata.use_cases,- model_type: raw_metadata.model_type,- model_config,- grammar_start_tokens: raw_metadata.grammar_start_tokens,- };- ModelContext::Classifier(Classifier::new(model_path, &metadata)?)+ let config_path = model_path.join("config.json");+ let config_file = File::open(&config_path)?;+ let model_config: AnyModelConfig = serde_json::from_reader(BufReader::new(config_file))?;++ let context = match model_config {+ AnyModelConfig::ClassifierModelConfig(model_config) => {+ let layer_trace_requirements = model_config+ .classifier_config+ .transformer_config+ .layer_configs+ .iter()+ .map(|layer_config| LayerTraceRequirements {+ post_mixer_norm: layer_config.post_mixer_norm_config.is_some(),+ post_mlp_norm: layer_config.post_mlp_norm_config.is_some(),+ })+ .collect();+ ModelContext::Classifier {+ classifier: Classifier::new(model_path, &model_config)?,+ layer_trace_requirements,+ }},- ModelType::LanguageModel => {- let ModelConfig::LanguageModel(model_config) = raw_metadata.model_config.clone() else {- return Err(Error::UnableToLoadConfig);- };- let metadata = ModelMetadata {- toolchain_version: raw_metadata.toolchain_version,- vendor: raw_metadata.vendor,- family: raw_metadata.family,- name: raw_metadata.name,- size: raw_metadata.size,- quantization: raw_metadata.quantization,- repo: raw_metadata.repo,- use_cases: raw_metadata.use_cases,- model_type: raw_metadata.model_type,- model_config,- grammar_start_tokens: raw_metadata.grammar_start_tokens,- };- let prefill_step_size = Self::determine_prefill_step_size(model_path);+ AnyModelConfig::LanguageModelConfig(model_config) => {+ let layer_trace_requirements = model_config+ .decoder_config+ .transformer_config+ .layer_configs+ .iter()+ .map(|layer_config| LayerTraceRequirements {+ post_mixer_norm: layer_config.post_mixer_norm_config.is_some(),+ post_mlp_norm: layer_config.post_mlp_norm_config.is_some(),+ })+ .collect();+ let prefill_step_size = Self::determine_prefill_step_size(model_path)?;let decoding_config = DecodingConfig::new(ContextMode::default(),ContextLength::default(),@@ -218,12 +221,17 @@ impl<B: Backend> TraceValidator<B> {SamplingSeed::default(),AsyncBatchSize::default(),);- let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config, &metadata)?;+ let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config, &model_config)?;let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);- ModelContext::LanguageModelGenerator(llm_context)+ ModelContext::LanguageModelGenerator {+ context: llm_context,+ layer_trace_requirements,+ }+ },+ AnyModelConfig::TTSModelConfig(_) => {+ return Err(Error::InvalidModelConfig("TTS trace validation is not supported".to_string()));},- ModelType::TtsModel => return Err(Error::UnableToLoadConfig),};Ok(Self {@@ -235,13 +243,15 @@ impl<B: Backend> TraceValidator<B> {/// Run trace validation and return results.pub fn run(&mut self) -> Result<TracerValidationResults, Error> {let traces_path = self.model_path.join("traces.safetensors");- if !traces_path.exists() {- return Err(Error::UnableToLoadWeights);- }-match &mut self.context {- ModelContext::LanguageModelGenerator(ctx) => Self::run_llm_validation(ctx, &traces_path),- ModelContext::Classifier(classifier) => Self::run_classifier_validation(classifier, &traces_path),+ ModelContext::LanguageModelGenerator {+ context,+ layer_trace_requirements,+ } => Self::run_llm_validation(context, layer_trace_requirements, &traces_path),+ ModelContext::Classifier {+ classifier,+ layer_trace_requirements,+ } => Self::run_classifier_validation(classifier, layer_trace_requirements, &traces_path),}}@@ -251,15 +261,30 @@ impl<B: Backend> TraceValidator<B> {fn run_llm_validation(ctx: &LanguageModelGeneratorContext<B>,+ layer_trace_requirements: &[LayerTraceRequirements],traces_path: &Path,) -> Result<TracerValidationResults, Error> {- let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;- let traces_loader =- ParameterLoader::new(&traces_file, ctx.context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;+ let traces_file = File::open(traces_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let (_header_len, traces_metadata) =+ read_safetensors_metadata(&traces_file).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let trace_shapes: HashMap<String, Vec<usize>> =+ traces_metadata.tensors.iter().map(|(name, tensor)| (name.clone(), tensor.shape.clone())).collect();+ let token_shape = Self::trace_shape(&trace_shapes, "activation_trace.token_ids");+ let position_shape = Self::trace_shape(&trace_shapes, "activation_trace.token_positions");+ let suffix_length = Self::trace_token_count("activation_trace.token_ids", token_shape);+ assert_eq!(+ Self::trace_token_count("activation_trace.token_positions", position_shape),+ suffix_length,+ "trace token ids and token positions must have the same suffix length"+ );++ let traces_loader = ParameterLoader::new(&traces_file, ctx.context.as_ref())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let traces_view = traces_loader.tree();- let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");- let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");+ let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, &trace_shapes, "activation_trace.token_ids");+ let token_positions =+ Self::load_array_as_vec::<i32, usize>(&traces_view, &trace_shapes, "activation_trace.token_positions");let token_inputs = TokenInputs::new_llm(ctx.context.as_ref(),&ctx.model_shape,@@ -269,9 +294,9 @@ impl<B: Backend> TraceValidator<B> {None,None,/*sampling_start=*/ 0,- /*sampling_length=*/ token_ids.len(),+ /*sampling_length=*/ suffix_length,);- let mut traces = ActivationTrace::new_llm(ctx.context.as_ref(), &ctx.model_shape, token_ids.len());+ let mut traces = ActivationTrace::new_llm(ctx.context.as_ref(), &ctx.model_shape, suffix_length);let mut encoder =Encoder::<B>::new(ctx.context.as_ref()).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;@@ -280,9 +305,9 @@ impl<B: Backend> TraceValidator<B> {let decoder_arguments = token_inputs.decoder_arguments(ctx.shared_buffers.as_ref(),Some(&mut *cache_layers),- token_ids.len(),+ suffix_length,/*sampling_start=*/ 0,- /*sampling_length=*/ token_ids.len(),+ /*sampling_length=*/ suffix_length,#[cfg(feature = "tracing")]Some(&mut traces),);@@ -298,159 +323,178 @@ impl<B: Backend> TraceValidator<B> {let pending = encoder.end_encoding().submit();pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;- let data_type = ctx.model_shape.activation_data_type();-// Common layer validation- let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);+ let mut results = Self::validate_layer_traces(+ &traces,+ &traces_view,+ &trace_shapes,+ ctx.model_shape.data_type,+ layer_trace_requirements,+ );- // LLM-specific: KV cache validation+ // LLM-specific state validationlet cache = ctx.cache_layers.borrow();for (index, layer) in cache.iter_layers() {- let Some(kv) = layer.as_transformer() else {- continue;- };-- if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {- let size = kv.shape().iter().product::<usize>() * data_type.size_in_bytes();+ if let Some(kv) = layer.as_transformer() {+ let shape = kv.shape();+ let keys_path = format!("activation_trace.layer_results.{}.updated_state.keys", index);+ let expected =+ Self::read_required_array(&traces_view, &trace_shapes, &keys_path, ctx.model_shape.data_type);+ let size = shape.iter().product::<usize>() * ctx.model_shape.data_type.size_in_bytes();let keys = if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {common::helpers::sparse_buffer_read_allocation(ctx.context.as_ref(), &layer.keys, size)+ } else if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::DenseBuffer>>() {+ let mut allocation = ctx+ .context+ .create_allocation(size, AllocationType::Global)+ .expect("Failed to create KV cache keys read allocation");+ let mut encoder =+ Encoder::<B>::new(ctx.context.as_ref()).expect("Failed to create KV cache keys read encoder");+ encoder.encode_copy(&layer.keys, 0..size, &mut allocation, 0..size);+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to read KV cache keys");+ allocation} else {panic!("Wrong keys type")};results.push(TracerValidationResult {- name: format!("updated_kv_cache.{}.keys", index),+ name: keys_path,metrics: Self::validate_allocation(- data_type,+ ctx.model_shape.data_type,&expected,&keys,- &kv.shape(),+ &shape,Some(ArrayTransform::KVCacheSlice),),});- }- if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {- let size = kv.shape().iter().product::<usize>() * data_type.size_in_bytes();+ let values_path = format!("activation_trace.layer_results.{}.updated_state.values", index);+ let expected =+ Self::read_required_array(&traces_view, &trace_shapes, &values_path, ctx.model_shape.data_type);let values = if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {common::helpers::sparse_buffer_read_allocation(ctx.context.as_ref(), &layer.values, size)+ } else if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::DenseBuffer>>() {+ let mut allocation = ctx+ .context+ .create_allocation(size, AllocationType::Global)+ .expect("Failed to create KV cache values read allocation");+ let mut encoder =+ Encoder::<B>::new(ctx.context.as_ref()).expect("Failed to create KV cache values read encoder");+ encoder.encode_copy(&layer.values, 0..size, &mut allocation, 0..size);+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to read KV cache values");+ allocation} else {panic!("Wrong values type")};results.push(TracerValidationResult {- name: format!("updated_kv_cache.{}.values", index),+ name: values_path,metrics: Self::validate_allocation(- data_type,+ ctx.model_shape.data_type,&expected,&values,- &kv.shape(),+ &shape,Some(ArrayTransform::KVCacheSlice),),});- }- }-- // LLM-specific: SSM state validation- for (index, layer) in cache.iter_layers() {- let Some(ssm) = layer.as_state_space() else {- continue;- };+ } else if let Some(ssm) = layer.as_state_space() {+ let conv_path = format!("activation_trace.layer_results.{}.updated_state.conv_state", index);+ let expected =+ Self::read_required_array(&traces_view, &trace_shapes, &conv_path, ctx.model_shape.data_type);+ let conv_state = ssm.conv_state.as_ref().expect("SSM trace requires a produced conv state");+ results.push(TracerValidationResult {+ name: conv_path,+ metrics: Self::validate_allocation(+ ctx.model_shape.data_type,+ &expected,+ conv_state,+ &ssm.conv_shape,+ Some(ArrayTransform::SsmConvState),+ ),+ });- for path in [- format!("updated_state.{}.conv_state", index),- format!("activation_trace.layer_results.{}.updated_state.conv_state", index),- ] {- if let Ok(expected) = traces_view.leaf_array(&path) {- results.push(TracerValidationResult {- name: path,- metrics: Self::validate_optional_allocation(- data_type,- &expected,- ssm.conv_state.as_ref(),- &ssm.conv_shape,- Some(ArrayTransform::SsmConvState),- ),- });- }- }+ let ssm_path = format!("activation_trace.layer_results.{}.updated_state.ssm_state", index);+ let expected =+ Self::read_required_array(&traces_view, &trace_shapes, &ssm_path, ctx.model_shape.data_type);+ results.push(TracerValidationResult {+ name: ssm_path,+ metrics: Self::validate_allocation(+ ctx.model_shape.data_type,+ &expected,+ &ssm.ssm_state,+ &ssm.ssm_shape,+ None,+ ),+ });+ } else if let Some(delta) = layer.as_delta_net() {+ let conv_path = format!("activation_trace.layer_results.{}.updated_state.conv_state", index);+ let expected =+ Self::read_required_array(&traces_view, &trace_shapes, &conv_path, ctx.model_shape.data_type);+ results.push(TracerValidationResult {+ name: conv_path,+ metrics: Self::validate_allocation(+ ctx.model_shape.data_type,+ &expected,+ &delta.conv_state,+ &delta.conv_shape,+ Some(ArrayTransform::SsmConvState),+ ),+ });- for path in [- format!("updated_state.{}.ssm_state", index),- format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),- ] {- if let Ok(expected) = traces_view.leaf_array(&path) {- results.push(TracerValidationResult {- name: path,- metrics: Self::validate_allocation(data_type, &expected, &ssm.ssm_state, &ssm.ssm_shape, None),- });- }+ let ssm_path = format!("activation_trace.layer_results.{}.updated_state.ssm_state", index);+ let expected =+ Self::read_required_array(&traces_view, &trace_shapes, &ssm_path, ctx.model_shape.data_type);+ results.push(TracerValidationResult {+ name: ssm_path,+ metrics: Self::validate_allocation(+ ctx.model_shape.data_type,+ &expected,+ &delta.ssm_state,+ &delta.ssm_shape,+ None,+ ),+ });+ } else if let Some(short_conv) = layer.as_short_conv() {+ let conv_path = format!("activation_trace.layer_results.{}.updated_state.conv_state", index);+ let expected =+ Self::read_required_array(&traces_view, &trace_shapes, &conv_path, ctx.model_shape.data_type);+ results.push(TracerValidationResult {+ name: conv_path,+ metrics: Self::validate_allocation(+ ctx.model_shape.data_type,+ &expected,+ &short_conv.conv_state,+ &short_conv.conv_shape,+ Some(ArrayTransform::SsmConvState),+ ),+ });+ } else {+ panic!("Unsupported cache layer type at layer {index}");}}- // LLM-specific: DeltaNet state validation- for (index, layer) in cache.iter_layers() {- let Some(delta) = layer.as_delta_net() else {- continue;- };-- for path in [- format!("updated_state.{}.conv_state", index),- format!("activation_trace.layer_results.{}.updated_state.conv_state", index),- ] {- if let Ok(expected) = traces_view.leaf_array(&path) {- results.push(TracerValidationResult {- name: path,- metrics: Self::validate_allocation(- data_type,- &expected,- &delta.conv_state,- &delta.conv_shape,- Some(ArrayTransform::SsmConvState),- ),- });- }- }-- for path in [- format!("updated_state.{}.ssm_state", index),- format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),- ] {- if let Ok(expected) = traces_view.leaf_array(&path) {- results.push(TracerValidationResult {- name: path,- metrics: Self::validate_allocation(- data_type,- &expected,- &delta.ssm_state,- &delta.ssm_shape,- None,- ),- });+ // LLM-specific: Token comparison+ let expected_logits =+ Self::read_required_array(&traces_view, &trace_shapes, "logits", ctx.model_shape.data_type);+ let expected_tokens = Self::get_tokens_from_logits(&expected_logits);+ let produced_tokens = Self::get_tokens_from_logits(&traces.logits);+ let tokens_violation_indices = expected_tokens+ .iter()+ .zip(produced_tokens.iter())+ .enumerate()+ .filter_map(|(i, (a, b))| {+ if a != b {+ Some(i)+ } else {+ None}- }- }+ })+ .collect();- // LLM-specific: Token comparison- let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf_array("logits") {- let expected_tokens = Self::get_tokens_from_logits(&expected_logits);- let produced_tokens = Self::get_tokens_from_logits(&traces.logits);- expected_tokens- .iter()- .zip(produced_tokens.iter())- .enumerate()- .filter_map(|(i, (a, b))| {- if a != b {- Some(i)- } else {- None- }- })- .collect()- } else {- Vec::new()- };+ traces_view.assert_all_tensors_validated().unwrap_or_else(|error| {+ panic!("Trace contains tensors that Uzu tracer did not validate: {error}");+ });Ok(TracerValidationResults {- suffix_length: token_ids.len(),+ suffix_length,results,tokens_violation_indices,})@@ -462,37 +506,54 @@ impl<B: Backend> TraceValidator<B> {fn run_classifier_validation(classifier: &mut Classifier<B>,+ layer_trace_requirements: &[LayerTraceRequirements],traces_path: &Path,) -> Result<TracerValidationResults, Error> {- let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;+ let traces_file = File::open(traces_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let (_header_len, traces_metadata) =+ read_safetensors_metadata(&traces_file).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let trace_shapes: HashMap<String, Vec<usize>> =+ traces_metadata.tensors.iter().map(|(name, tensor)| (name.clone(), tensor.shape.clone())).collect();let context = classifier.context.context.clone();- let traces_loader =- ParameterLoader::new(&traces_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;+ let traces_loader = ParameterLoader::new(&traces_file, context.as_ref())+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let traces_view = traces_loader.tree();- let has_token_ids = traces_view.leaf_array("activation_trace.token_ids").is_ok();- let has_token_positions = traces_view.leaf_array("activation_trace.token_positions").is_ok();-- if !has_token_ids || !has_token_positions {- return Ok(Self::handle_missing_tokens(&traces_view));- }-- let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");- let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");-- let suffix_length = token_ids.len();+ let token_shape = Self::trace_shape(&trace_shapes, "activation_trace.token_ids");+ let position_shape = Self::trace_shape(&trace_shapes, "activation_trace.token_positions");+ let suffix_length = Self::trace_token_count("activation_trace.token_ids", token_shape);+ assert_eq!(+ Self::trace_token_count("activation_trace.token_positions", position_shape),+ suffix_length,+ "trace token ids and token positions must have the same suffix length"+ );- let (_logits, traces) =- classifier.forward_pass_with_traces(&token_ids, &token_positions).map_err(|_| Error::GenerateFailed)?;+ let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, &trace_shapes, "activation_trace.token_ids");+ let token_positions =+ Self::load_array_as_vec::<i32, usize>(&traces_view, &trace_shapes, "activation_trace.token_positions");- let data_type = classifier.context.model_shape.activation_data_type();+ let (_logits, traces) = classifier.forward_pass_with_traces(&token_ids, &token_positions)?;// Common layer validation- let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);+ let mut results = Self::validate_layer_traces(+ &traces,+ &traces_view,+ &trace_shapes,+ classifier.context.model_shape.data_type,+ layer_trace_requirements,+ );// Classifier-specific: embedding_norm, output_pooling- let classifier_results = Self::validate_classifier_traces(&traces, &traces_view, data_type);+ let classifier_results = Self::validate_classifier_traces(+ &traces,+ &traces_view,+ &trace_shapes,+ classifier.context.model_shape.data_type,+ );results.extend(classifier_results);+ traces_view.assert_all_tensors_validated().unwrap_or_else(|error| {+ panic!("Trace contains tensors that Uzu tracer did not validate: {error}");+ });Ok(TracerValidationResults {suffix_length,@@ -501,141 +562,104 @@ impl<B: Backend> TraceValidator<B> {})}- fn handle_missing_tokens(traces_view: &ParameterTree<B::Context>) -> TracerValidationResults {- if let Ok(expected_logits) = traces_view.leaf_array("logits") {- let reference_shape = expected_logits.shape().to_vec();- let metrics = TracerValidationMetrics {- atol: 0.0,- rtol: 0.0,- fraction_of_allowed_violations: 0.0,- reference_shape: reference_shape.clone(),- result_shape: reference_shape,- num_violations: 0,- max_allowed_violations: 0,- max_err_idx: 0,- max_err: 0.0,- max_err_rel: 0.0,- max_err_reference_value: 0.0,- rms_diff: 0.0,- rms_result: 0.0,- rms_reference: 0.0,- rel_rms_reference: 0.0,- diff_max: 0.0,- diff_avg: 0.0,- result_nan: false,- };- return TracerValidationResults {- suffix_length: 1,- results: vec![TracerValidationResult {- name: "activation_trace.logits".to_string(),- metrics,- }],- tokens_violation_indices: Vec::new(),- };- }-- TracerValidationResults {- suffix_length: 1,- results: Vec::new(),- tokens_violation_indices: Vec::new(),- }- }-// ========================================================================// Common Validation Helpers// ========================================================================fn validate_layer_traces(traces: &ActivationTrace<B>,- traces_view: &ParameterTree<B::Context>,+ traces_view: &ParameterTree<B>,+ trace_shapes: &HashMap<String, Vec<usize>>,data_type: DataType,+ layer_trace_requirements: &[LayerTraceRequirements],) -> Vec<TracerValidationResult> {let mut results = Vec::new();- let validate = |path: &str, array: &Array<B>| -> Option<TracerValidationResult> {- if let Ok(expected) = traces_view.leaf_array(path) {- Some(TracerValidationResult {- name: path.to_string(),- metrics: Self::validate_allocation(data_type, &expected, array.allocation(), array.shape(), None),- })- } else {- None+ assert_eq!(+ traces.layer_results.len(),+ layer_trace_requirements.len(),+ "trace layer count must match model layer count"+ );++ let validate = |path: &str, array: &Array<B>| -> TracerValidationResult {+ let expected = Self::read_required_array(traces_view, trace_shapes, path, data_type);+ TracerValidationResult {+ name: path.to_string(),+ metrics: Self::validate_allocation(data_type, &expected, array.allocation(), array.shape(), None),}};for (index, layer_traces) in traces.layer_results.iter().enumerate() {+ let requirements = layer_trace_requirements[index];let path = |suffix: &str| -> String {format!("activation_trace.layer_results.{}.activation_trace.{}", index, suffix)};- if let Some(r) = validate(&path("inputs"), &layer_traces.inputs) {- results.push(r);- }- if let Some(r) = validate(&path("pre_mixer_norm"), &layer_traces.pre_attention_norm) {- results.push(r);- }- if let Some(r) = validate(&path("mixer"), &layer_traces.attention) {- results.push(r);- }- if let Some(r) = validate(&path("post_mixer_norm"), &layer_traces.post_attention_norm) {- results.push(r);- }- if let Some(r) = validate(&path("mlp_inputs"), &layer_traces.mlp_inputs) {- results.push(r);- }- if let Some(r) = validate(&path("pre_mlp_norm"), &layer_traces.pre_mlp_norm) {- results.push(r);+ results.push(validate(&path("inputs"), &layer_traces.inputs));+ results.push(validate(&path("pre_mixer_norm"), &layer_traces.pre_attention_norm));+ results.push(validate(&path("mixer"), &layer_traces.attention));+ if requirements.post_mixer_norm {+ results.push(validate(&path("post_mixer_norm"), &layer_traces.post_attention_norm));}- if let Some(r) = validate(&path("mlp"), &layer_traces.mlp) {- results.push(r);- }- if let Some(r) = validate(&path("post_mlp_norm"), &layer_traces.post_mlp_norm) {- results.push(r);+ results.push(validate(&path("mlp_inputs"), &layer_traces.mlp_inputs));+ results.push(validate(&path("pre_mlp_norm"), &layer_traces.pre_mlp_norm));+ results.push(validate(&path("mlp"), &layer_traces.mlp));+ if requirements.post_mlp_norm {+ results.push(validate(&path("post_mlp_norm"), &layer_traces.post_mlp_norm));}let outputs_path = format!("activation_trace.layer_results.{}.outputs", index);- if let Some(r) = validate(&outputs_path, &layer_traces.outputs) {- results.push(r);- }+ results.push(validate(&outputs_path, &layer_traces.outputs));}// Output norm (common to all models)- if let Some(r) = validate("activation_trace.output_norm", &traces.output_norm) {- results.push(r);- }+ results.push(validate("activation_trace.output_norm", &traces.output_norm));- // Logits (common to all models, but path may vary)- if let Some(r) = validate("activation_trace.logits", &traces.logits) {- results.push(r);- } else if let Some(r) = validate("logits", &traces.logits) {- results.push(r);- }+ // Logits live on DecoderResult/ClassifierResult, not inside activation_trace.+ results.push(validate("logits", &traces.logits));results}fn validate_classifier_traces(traces: &ActivationTrace<B>,- traces_view: &ParameterTree<B::Context>,+ traces_view: &ParameterTree<B>,+ trace_shapes: &HashMap<String, Vec<usize>>,data_type: DataType,) -> Vec<TracerValidationResult> {let mut results = Vec::new();- // Output pooling (classifier-specific)+ let embedding_norm =+ traces.embedding_norm.as_ref().expect("classifier traces must include produced embedding norm output");+ let expected =+ Self::read_required_array(traces_view, trace_shapes, "activation_trace.embedding_norm_output", data_type);+ results.push(TracerValidationResult {+ name: "activation_trace.embedding_norm_output".to_string(),+ metrics: Self::validate_allocation(+ data_type,+ &expected,+ embedding_norm.allocation(),+ embedding_norm.shape(),+ None,+ ),+ });+if let Some(output_pooling) = &traces.output_pooling {- if let Ok(expected) = traces_view.leaf_array("activation_trace.output_pooling") {- results.push(TracerValidationResult {- name: "activation_trace.output_pooling".to_string(),- metrics: Self::validate_allocation(- data_type,- &expected,- output_pooling.allocation(),- output_pooling.shape(),- None,- ),- });- }- }+ let expected =+ Self::read_required_array(traces_view, trace_shapes, "activation_trace.output_pooling", data_type);+ results.push(TracerValidationResult {+ name: "activation_trace.output_pooling".to_string(),+ metrics: Self::validate_allocation(+ data_type,+ &expected,+ output_pooling.allocation(),+ output_pooling.shape(),+ None,+ ),+ });+ } else {+ panic!("classifier traces must include produced output pooling");+ };results}@@ -668,32 +692,6 @@ impl<B: Backend> TraceValidator<B> {}}- fn validate_optional_allocation(- data_type: DataType,- expected_array: &Array<B>,- produced_allocation: Option<&Allocation<B>>,- produced_shape: &[usize],- transform: Option<ArrayTransform>,- ) -> TracerValidationMetrics {- match produced_allocation {- Some(produced_allocation) => {- Self::validate_allocation(data_type, expected_array, produced_allocation, produced_shape, transform)- },- None => match data_type {- DataType::F16 => {- Self::validate_allocation_data_of_type::<f16>(expected_array, &[], produced_shape, transform)- },- DataType::BF16 => {- Self::validate_allocation_data_of_type::<bf16>(expected_array, &[], produced_shape, transform)- },- DataType::F32 => {- Self::validate_allocation_data_of_type::<f32>(expected_array, &[], produced_shape, transform)- },- _ => panic!("Unsupported data type: {:?}", data_type),- },- }- }-fn validate_allocation_of_type<Precision: ArrayElement>(expected_array: &Array<B>,produced_allocation: &Allocation<B>,@@ -714,16 +712,14 @@ impl<B: Backend> TraceValidator<B> {let produced_view = ndarray::ArrayView::from_shape(IxDyn(produced_shape), produced_slice).expect("Failed to reshape allocation");- let (mut expected_data, mut produced_data) = match transform {+ let (expected_data, mut produced_data) = match transform {Some(ArrayTransform::KVCacheSlice) => {- let permuted = produced_view.permuted_axes(IxDyn(&[1, 0, 2]));- let total_tokens = permuted.shape()[0];+ let expected_shape = expected_view.shape();let expected_tokens = expected_view.shape()[1];- let start = total_tokens.saturating_sub(expected_tokens);- let sliced = permuted.slice(s![start.., .., ..]);+ let sliced = produced_view.slice(s![..expected_tokens, .., ..]);let reshaped = sliced.into_owned()- .to_shape(IxDyn(&[1, expected_tokens, permuted.shape()[1], permuted.shape()[2]]))+ .to_shape(IxDyn(expected_shape)).expect("Failed to reshape KV cache slice").to_owned();(expected_view.to_owned(), reshaped)@@ -757,14 +753,14 @@ impl<B: Backend> TraceValidator<B> {&& expected_shape.get(0) == Some(&1)&& expected_shape[1..] == produced_shape[..]{- expected_data =- expected_data.to_shape(IxDyn(&produced_shape)).expect("Failed to reshape expected data").to_owned();- } else if produced_shape.len() == expected_shape.len() + 1- && produced_shape.get(0) == Some(&1)- && produced_shape[1..] == expected_shape[..]- {- produced_data =- produced_data.to_shape(IxDyn(&expected_shape)).expect("Failed to reshape produced data").to_owned();+ produced_data = produced_data+ .to_shape(IxDyn(&expected_shape))+ .expect("Failed to reshape produced data to trace shape")+ .to_owned();+ } else {+ panic!(+ "Shape mismatch: expected trace shape {expected_shape:?}, produced Uzu shape {produced_shape:?}"+ );}}@@ -776,8 +772,14 @@ impl<B: Backend> TraceValidator<B> {);}- let reference: Vec<f32> = expected_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();- let result: Vec<f32> = produced_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();+ let reference: Vec<f32> = expected_data+ .iter()+ .map(|value| NumCast::from(*value).expect("Failed to cast reference trace value"))+ .collect();+ let result: Vec<f32> = produced_data+ .iter()+ .map(|value| NumCast::from(*value).expect("Failed to cast produced trace value"))+ .collect();let (atol, rtol, allowed_voilations_tol) = match expected_array.data_type() {DataType::BF16 => (0.04, 0.06, 0.03),@@ -910,26 +912,72 @@ impl<B: Backend> TraceValidator<B> {// ========================================================================fn load_array_as_vec<SourcePrecision: ArrayElement, TargetPrecision: NumCast>(- traces_view: &ParameterTree<B::Context>,+ traces_view: &ParameterTree<B>,+ trace_shapes: &HashMap<String, Vec<usize>>,name: &str,) -> Vec<TargetPrecision> {- let array = traces_view.leaf_array(name).unwrap();- let slice = array.as_slice::<SourcePrecision>();- slice.iter().map(|x| NumCast::from(*x).unwrap()).collect()+ let expected_shape = Self::trace_shape(trace_shapes, name);+ let leaf = traces_view+ .leaf(name)+ .unwrap_or_else(|error| panic!("Missing required trace tensor {name}: {error}"))+ .validate(expected_shape, SourcePrecision::data_type())+ .unwrap_or_else(|error| panic!("Invalid trace tensor {name}: {error}"));+ let slice = leaf.read_slice::<SourcePrecision>().unwrap_or_else(|error| {+ panic!("Failed to read trace tensor {name}: {error}");+ });+ slice.iter().map(|x| NumCast::from(*x).expect("Failed to cast trace tensor value")).collect()+ }++ fn trace_shape<'shape>(+ trace_shapes: &'shape HashMap<String, Vec<usize>>,+ name: &str,+ ) -> &'shape [usize] {+ trace_shapes.get(name).unwrap_or_else(|| panic!("Missing required trace tensor {name}"))+ }++ fn trace_token_count(+ name: &str,+ shape: &[usize],+ ) -> usize {+ let [batch_size, suffix_length] = shape else {+ panic!("Trace tensor {name} must have shape [1, suffix_tokens], got {shape:?}");+ };+ assert_eq!(*batch_size, 1, "Trace tensor {name} must have batch size 1, got {shape:?}");+ *suffix_length+ }++ fn read_required_array(+ traces_view: &ParameterTree<B>,+ trace_shapes: &HashMap<String, Vec<usize>>,+ name: &str,+ data_type: DataType,+ ) -> Array<B> {+ let expected_shape = Self::trace_shape(trace_shapes, name);+ Self::read_array(traces_view, name, expected_shape, data_type)+ .unwrap_or_else(|error| panic!("Invalid trace tensor {name}: {error}"))+ }++ fn read_array(+ traces_view: &ParameterTree<B>,+ name: &str,+ expected_shape: &[usize],+ data_type: DataType,+ ) -> Result<Array<B>, ParameterLoaderError<B>> {+ traces_view.leaf(name)?.validate(expected_shape, data_type)?.read_array()}- fn determine_prefill_step_size(model_path: &Path) -> usize {+ fn determine_prefill_step_size(model_path: &Path) -> Result<usize, Error> {let traces_path = model_path.join("traces.safetensors");- if let Ok(file) = File::open(&traces_path) {- if let Ok((_header_len, metadata)) = read_safetensors_metadata(&file) {- if let Some(tensor) = metadata.tensors.get("activation_trace.token_ids") {- if let Some(&length) = tensor.shape.first() {- return tensor.shape.iter().copied().max().unwrap_or(length).max(1);- }- }- }- }- 1+ let file = File::open(&traces_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let (_header_len, metadata) =+ read_safetensors_metadata(&file).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let token_shape =+ metadata.tensors.get("activation_trace.token_ids").map(|tensor| tensor.shape.as_slice()).ok_or_else(+ || Error::InvalidModelConfig("missing required trace tensor activation_trace.token_ids".to_string()),+ )?;+ let suffix_length = Self::trace_token_count("activation_trace.token_ids", token_shape);+ assert!(suffix_length > 0, "trace must contain at least one token");+ Ok(suffix_length)}fn ensure_llm_context_capacity(@@ -954,16 +1002,13 @@ impl<B: Backend> TraceValidator<B> {desired_suffix_length,)));- let intermediate_dtype: DataType =- context.model_config.model_config.transformer_config.output_norm_config.scale_precision.into();-context.kv_cache_update = Box::new(- KVCacheUpdate::new(context.context.as_ref(), intermediate_dtype, resolved_prefix_length)+ KVCacheUpdate::new(context.context.as_ref(), context.model_shape.data_type, resolved_prefix_length).expect("Failed to create KV cache update kernel"),);- context.gpu_sampler =- Sampling::new(context.context.as_ref(), intermediate_dtype).expect("Failed to create sampling kernel");+ context.gpu_sampler = Sampling::new(context.context.as_ref(), context.model_shape.data_type)+ .expect("Failed to create sampling kernel");}fn get_tokens_from_logits(logits: &Array<B>) -> Vec<u64> {diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator_test.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator_test.rsindex 6983c8d2e..5871f9327 100644--- a/crates/backend-uzu/tests/integration/tracer/trace_validator_test.rs+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator_test.rs@@ -13,10 +13,6 @@ fn test_tracer_internal<B: Backend>() {let mut tracer = TraceValidator::<B>::new(&model_path).expect("Failed to create TraceValidator");let results = tracer.run().expect("Failed to run tracer");for result in results.results.iter() {- // this layers contains too many errors- if result.name == "activation_trace.output_norm" || result.name == "logits" {- continue;- }assert!(result.metrics.is_valid(), "{} error:\n{}", result.name, result.metrics.message().as_str());}@@ -32,6 +28,7 @@ fn test_tracer_internal<B: Backend>() {}#[test]+#[ignore = "Lalamo 0.10.0 doesn't support exporting traces"] // TODO: this is horrible, should be resolved asapfn test_tracer() {let traces_path = get_traces_path();assert!(traces_path.exists(), "Traces file missing at {:?}", traces_path);diff --git a/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs b/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rsindex 9484154b5..3d752bbbf 100644--- a/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs+++ b/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs@@ -707,7 +707,7 @@ fn audio_kernel_perf() {{let i = input.allocation();let o = &mut output;- k_act.encode(Some(i), o, n as u32, ActivationType::GELU, &mut encoder);+ k_act.encode(Some(i), o, n as u32, ActivationType::GELUApprox, &mut encoder);}encoder.end_encoding().submit().wait_until_completed().unwrap();});@@ -973,7 +973,13 @@ fn audio_kernel_perf() {);}{- k_act.encode(Some(&scratch_a), &mut scratch_b, (c0_ch * f_up0) as u32, ActivationType::GELU, &mut encoder);+ k_act.encode(+ Some(&scratch_a),+ &mut scratch_b,+ (c0_ch * f_up0) as u32,+ ActivationType::GELUApprox,+ &mut encoder,+ );}{let w = c0_p2_w.allocation();@@ -1089,7 +1095,13 @@ fn audio_kernel_perf() {);}{- k_act.encode(Some(&scratch_b), &mut scratch_a, (c1_ch * f_up1) as u32, ActivationType::GELU, &mut encoder);+ k_act.encode(+ Some(&scratch_b),+ &mut scratch_a,+ (c1_ch * f_up1) as u32,+ ActivationType::GELUApprox,+ &mut encoder,+ );}{let w = c1_p2_w.allocation();diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rsindex 37d8c8b42..40538f4af 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs@@ -106,10 +106,10 @@ fn moe_cpu_reference(let mut hidden = vec![0.0f32; d_ff];for i in 0..d_ff {hidden[i] = match gating_code {- 0 => ActivationType::GELU.activate(up_out[i]),+ 0 => ActivationType::GELUApprox.activate(up_out[i]),1 => activation_silu_alpha(up_out[i], silu_alpha),2 => activation_silu_alpha(gate_out[i], silu_alpha) * up_out[i],- 3 => ActivationType::GELU.activate(gate_out[i]) * up_out[i],+ 3 => ActivationType::GELUApprox.activate(gate_out[i]) * up_out[i],_ => activation_silu_alpha(gate_out[i], silu_alpha) * up_out[i], // fallback to SwiGLU};}diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rsindex 2240c5a41..1bb46fca3 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs@@ -223,7 +223,7 @@ fn cpu_moe_reference(// GELU or SiLU on up (with clipping)let up_val = acc_up.clamp(up_clip_min, up_clip_max);if gating_code == 0 {- ActivationType::GELU.activate(up_val)+ ActivationType::GELUApprox.activate(up_val)} else {activation_silu_alpha(up_val, silu_alpha)}@@ -243,7 +243,7 @@ fn cpu_moe_reference(let gate_act = if gating_code == 2 {activation_silu_alpha(gate_val, silu_alpha)} else {- ActivationType::GELU.activate(gate_val)+ ActivationType::GELUApprox.activate(gate_val)};gate_act * up_val};diff --git a/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs b/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rsindex db5e5ea56..be814eea0 100644--- a/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs@@ -105,15 +105,30 @@ fn test_silu_bf16() {#[uzu_test]fn test_gelu_f32() {- test::<f32>(ActivationType::GELU);+ test::<f32>(ActivationType::GELUApprox);}#[uzu_test]fn test_gelu_f16() {- test::<f16>(ActivationType::GELU);+ test::<f16>(ActivationType::GELUApprox);}#[uzu_test]fn test_gelu_bf16() {- test::<bf16>(ActivationType::GELU);+ test::<bf16>(ActivationType::GELUApprox);+}++#[uzu_test]+fn test_gelu_exact_f32() {+ test::<f32>(ActivationType::GELUExact);+}++#[uzu_test]+fn test_gelu_exact_f16() {+ test::<f16>(ActivationType::GELUExact);+}++#[uzu_test]+fn test_gelu_exact_bf16() {+ test::<bf16>(ActivationType::GELUExact);}diff --git a/crates/backend-uzu/tests/unit/kernel/activation_test.rs b/crates/backend-uzu/tests/unit/kernel/activation_test.rsindex 32466262e..2f6c83fde 100644--- a/crates/backend-uzu/tests/unit/kernel/activation_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/activation_test.rs@@ -184,33 +184,49 @@ fn test_silu_in_place_bf16() {// GELU out-of-place tests#[uzu_test]fn test_gelu_f32() {- test_activation::<f32>(ActivationType::GELU, false);+ test_activation::<f32>(ActivationType::GELUApprox, false);}#[uzu_test]fn test_gelu_f16() {- test_activation::<f16>(ActivationType::GELU, false);+ test_activation::<f16>(ActivationType::GELUApprox, false);}#[uzu_test]fn test_gelu_bf16() {- test_activation::<bf16>(ActivationType::GELU, false);+ test_activation::<bf16>(ActivationType::GELUApprox, false);+}++// Exact GELU out-of-place tests+#[uzu_test]+fn test_gelu_exact_f32() {+ test_activation::<f32>(ActivationType::GELUExact, false);+}++#[uzu_test]+fn test_gelu_exact_f16() {+ test_activation::<f16>(ActivationType::GELUExact, false);+}++#[uzu_test]+fn test_gelu_exact_bf16() {+ test_activation::<bf16>(ActivationType::GELUExact, false);}// GELU in-place tests#[uzu_test]fn test_gelu_in_place_f32() {- test_activation::<f32>(ActivationType::GELU, true);+ test_activation::<f32>(ActivationType::GELUApprox, true);}#[uzu_test]fn test_gelu_in_place_f16() {- test_activation::<f16>(ActivationType::GELU, true);+ test_activation::<f16>(ActivationType::GELUApprox, true);}#[uzu_test]fn test_gelu_in_place_bf16() {- test_activation::<bf16>(ActivationType::GELU, true);+ test_activation::<bf16>(ActivationType::GELUApprox, true);}// Large SILU tests@@ -232,17 +248,17 @@ fn test_silu_large_bf16() {// Large GELU tests#[uzu_test]fn test_gelu_large_f32() {- test_activation_large::<f32>(ActivationType::GELU, false);+ test_activation_large::<f32>(ActivationType::GELUApprox, false);}#[uzu_test]fn test_gelu_large_f16() {- test_activation_large::<f16>(ActivationType::GELU, false);+ test_activation_large::<f16>(ActivationType::GELUApprox, false);}#[uzu_test]fn test_gelu_large_bf16() {- test_activation_large::<bf16>(ActivationType::GELU, false);+ test_activation_large::<bf16>(ActivationType::GELUApprox, false);}// Large in-place tests@@ -253,5 +269,5 @@ fn test_silu_in_place_large_f32() {#[uzu_test]fn test_gelu_in_place_large_f32() {- test_activation_large::<f32>(ActivationType::GELU, true);+ test_activation_large::<f32>(ActivationType::GELUApprox, true);}diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rsindex 38fd72538..1d2507b18 100644--- a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs@@ -6,7 +6,9 @@ use backend_uzu::{ArrayContextExt, ArrayElement, DataType,backends::{common::{- Backend, Context, Encoder, Kernels, gpu_types::QuantizationMode, kernel::QuantizedEmbeddingLookupKernel,+ Backend, Context, Encoder, Kernels,+ gpu_types::{QuantizationMethod, QuantizationMode},+ kernel::QuantizedEmbeddingLookupKernel,},cpu::Cpu,},@@ -112,6 +114,8 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {T::data_type(),input.group_size,input.quant_mode,+ QuantizationMethod::ScaleBias,+ false,).expect("Failed to create QuantizedEmbeddingLookupKernel");@@ -132,8 +136,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {token_ids_array.allocation(),weights_array.allocation(),scales_array.allocation(),- biases_array.allocation(),+ None::<&backend_uzu::backends::common::Allocation<B>>,+ Some(biases_array.allocation()),&mut output,+ None::<&backend_uzu::backends::common::Allocation<B>>,input.batch_size,input.vocab_size,input.model_dim,diff --git a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rsindex c67f7353b..e4a455737 100644--- a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs@@ -4,7 +4,11 @@ use std::{use backend_uzu::{ArrayContextExt, ArrayElement,- backends::common::{Backend, Context, Encoder, Kernels, kernel::HadamardTransformKernel},+ backends::common::{+ Backend, Context, Encoder, Kernels,+ gpu_types::HadamardTransformOrder,+ kernel::HadamardTransformKernel,+ },};use half::{bf16, f16};use num_traits::Float;@@ -100,8 +104,12 @@ fn generate_test_input<T: ArrayElement + Float>(fn run_kernel<T: ArrayElement + Float, B: Backend>(input: &TestInput<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create context");- let kernel = <<B as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(&context, T::data_type())- .expect("Failed to create HadamardTransformKernel");+ let kernel = <<B as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(+ &context,+ T::data_type(),+ HadamardTransformOrder::Input,+ )+ .expect("Failed to create HadamardTransformKernel");let total_elements = input.batch_count * input.channel_count;let mut data = contextdiff --git a/crates/backend-uzu/tests/unit/kernel/rope_test.rs b/crates/backend-uzu/tests/unit/kernel/rope_test.rsindex e30d780f6..4a3460c53 100644--- a/crates/backend-uzu/tests/unit/kernel/rope_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/rope_test.rs@@ -1,6 +1,4 @@-use std::{- fmt::{Debug, Display},-};+use std::fmt::{Debug, Display};use backend_uzu::{ArrayContextExt, ArrayElement, DataType,@@ -16,8 +14,8 @@ use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {qkv: Box<[T]>,- cosines: Box<[T]>,- sines: Box<[T]>,+ cosines: Box<[f32]>,+ sines: Box<[f32]>,token_positions: Box<[i32]>,head_dim: u32,rope_dim: u32,@@ -44,11 +42,11 @@ fn get_test_data<T: ArrayElement + Float>(qkv[i] = T::from(((i as f32) * 0.1).sin() * 2.0).unwrap();}- let mut cosines = vec![T::zero(); cos_sin_size];- let mut sines = vec![T::zero(); cos_sin_size];+ let mut cosines = vec![0.0f32; cos_sin_size];+ let mut sines = vec![0.0f32; cos_sin_size];for i in 0..cos_sin_size {- cosines[i] = T::from(((i as f32) * 0.05).cos()).unwrap();- sines[i] = T::from(((i as f32) * 0.05).sin()).unwrap();+ cosines[i] = ((i as f32) * 0.05).cos();+ sines[i] = ((i as f32) * 0.05).sin();}let mut token_positions = vec![0i32; suffix_length as usize];@@ -73,7 +71,9 @@ fn get_test_data<T: ArrayElement + Float>(fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::RopeKernel::new(&context, T::data_type())+ let element_data_type = T::data_type();+ let rope_data_type = DataType::F32;+ let kernel = <<B as Backend>::Kernels as Kernels>::RopeKernel::new(&context, element_data_type, rope_data_type).expect("Failed to create RopeKernel");let total_heads = (input.num_heads + 2 * input.num_groups) as usize;@@ -86,12 +86,8 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines);let sines_array = context.create_array_from(&[cos_sin_len], &input.sines);let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions);- let mut rotated_queries = context- .create_array_uninitialized(&[queries_len], T::data_type())- .into_allocation();- let mut rotated_keys = context- .create_array_uninitialized(&[keys_len], T::data_type())- .into_allocation();+ let mut rotated_queries = context.create_array_uninitialized(&[queries_len], T::data_type()).into_allocation();+ let mut rotated_keys = context.create_array_uninitialized(&[keys_len], T::data_type()).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rsindex 8bad6a293..0825c539e 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs@@ -271,7 +271,7 @@ fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {}fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELU);+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELUApprox);test_internal(&input, &expected, "gelu_activation");}diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rsindex 6be3cc6b2..8f3d56f8f 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs@@ -224,7 +224,7 @@ fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {}fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELU);+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELUApprox);test_internal(&input, &expected, "gelu_activation");}diff --git a/crates/backend-uzu/tests/unit/utils/model_size_test.rs b/crates/backend-uzu/tests/unit/utils/model_size_test.rsindex 2f66dd25a..207eadad2 100644--- a/crates/backend-uzu/tests/unit/utils/model_size_test.rs+++ b/crates/backend-uzu/tests/unit/utils/model_size_test.rs@@ -23,7 +23,7 @@ fn test_model_size_from_path() {let model_path = get_test_model_path();if model_path.exists() {- let size = ModelSize::from_path(&model_path);+ let size = ModelSize::from_path(&model_path).expect("load model size");println!("Model size: {:?}", size);assert_eq!(size, ModelSize::Small);}diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rsindex 7b4b716dd..cefc7c7dc 100644--- a/crates/benchmarks/src/runner/runner.rs+++ b/crates/benchmarks/src/runner/runner.rs@@ -51,16 +51,14 @@ impl Runner {}let mut session = ChatSession::new(PathBuf::from(self.model_path.clone()), decoding_config)?;-- let precision =- session.model_metadata.model_config.model_config.transformer_config.output_norm_config.scale_precision;+ let data_type = session.activation_data_type();let device = self.get_device_info();let input = Input::Messages(self.task.messages.clone());let warmup_config = RunConfig::default().tokens_limit(1);- session.run(input.clone(), warmup_config, Some(|_: Output| true))?;+ session.run(input.clone(), warmup_config, Option::<fn(Output) -> bool>::None)?;let mut results: Vec<TaskResult> = Vec::new();for run_index in 0..self.task.number_of_runs {@@ -72,13 +70,7 @@ impl Runner {value: SamplingMethod::Greedy,});}- let output = session.run(- input.clone(),- run_config,- Some(|_: Output| {- return true;- }),- )?;+ let output = session.run(input.clone(), run_config, Option::<fn(Output) -> bool>::None)?;let memory_used = get_memory_usage();@@ -87,7 +79,7 @@ impl Runner {device: device.clone(),engine_version: VERSION.to_string(),timestamp,- precision,+ data_type,memory_used,tokens_count_input: output.stats.total_stats.tokens_count_input,tokens_count_output: output.stats.total_stats.tokens_count_output,diff --git a/crates/benchmarks/src/runner/types/result.rs b/crates/benchmarks/src/runner/types/result.rsindex 7b3980ff2..7c14b3647 100644--- a/crates/benchmarks/src/runner/types/result.rs+++ b/crates/benchmarks/src/runner/types/result.rs@@ -9,7 +9,7 @@ pub struct Result {pub device: Device,pub engine_version: String,pub timestamp: u64,- pub precision: DataType,+ pub data_type: DataType,pub memory_used: Option<u64>,pub tokens_count_input: u64,pub tokens_count_output: u64,diff --git a/crates/proc-macros/src/lib.rs b/crates/proc-macros/src/lib.rsindex d7df287d1..b1d3a0be2 100644--- a/crates/proc-macros/src/lib.rs+++ b/crates/proc-macros/src/lib.rs@@ -12,6 +12,14 @@ pub fn kernel(dsl::kernel(args, input)}+#[proc_macro_attribute]+pub fn uzu_config_abstract(+ args: TokenStream,+ input: TokenStream,+) -> TokenStream {+ uzu_config::uzu_config_abstract(args, input)+}+#[proc_macro_attribute]pub fn uzu_config(args: TokenStream,diff --git a/crates/proc-macros/src/uzu_config.rs b/crates/proc-macros/src/uzu_config.rsindex d2eff31cc..d69aa542f 100644--- a/crates/proc-macros/src/uzu_config.rs+++ b/crates/proc-macros/src/uzu_config.rs@@ -1,13 +1,167 @@use proc_macro::TokenStream;-use quote::{ToTokens, quote};-use syn::{Data, DeriveInput, Field, Generics, parse_macro_input, parse_quote};+use quote::{ToTokens, format_ident, quote};+use syn::{+ Data, DataStruct, DeriveInput, Field, Fields, FieldsNamed, Generics, ItemStruct, Path, Token, Type,+ parse_macro_input, parse_quote, punctuated::Punctuated, spanned::Spanned,+};++// TODO: This is very hacky, there has to be a better way to do all of this stuff++pub fn uzu_config_abstract(+ args: TokenStream,+ input: TokenStream,+) -> TokenStream {+ let concrete_configs = parse_macro_input!(args with Punctuated::<Path, Token![,]>::parse_terminated);+ let input = parse_macro_input!(input as ItemStruct);++ let struct_fields = match input.fields {+ Fields::Named(FieldsNamed {+ named,+ ..+ }) => named,+ Fields::Unit => Punctuated::new(),+ fields => {+ return syn::Error::new(fields.span(), "uzu_config_abstract requires a named-field or unit struct")+ .to_compile_error()+ .into();+ },+ };++ let input_vis = input.vis;+ let struct_ident = input.ident;+ let enum_ident = format_ident!("Any{}", struct_ident);+ let (alias_struct_fields, callback_struct_fields, enum_field_getters) = struct_fields+ .into_iter()+ .map(|mut struct_field| {+ let field_ident = struct_field.ident.as_ref().unwrap();+ let typealias_ident = format_ident!("__{}_{}", struct_ident, field_ident);+ let field_type = std::mem::replace(+ &mut struct_field.ty,+ Type::Verbatim(quote! { $($typealias_base)::*::#typealias_ident }),+ );+ let match_arms = concrete_configs.iter().map(|concrete_config_path| {+ let concrete_config_ident = &concrete_config_path.segments.last().unwrap().ident;+ quote! { Self::#concrete_config_ident(__concrete) => &__concrete.#field_ident, }+ });+ (+ quote! {+ #[allow(non_camel_case_types)]+ type #typealias_ident = #field_type;+ },+ quote! {+ #struct_field+ },+ quote! {+ #input_vis fn #field_ident(&self) -> &#field_type {+ match self {+ #(#match_arms)*+ }+ }+ },+ )+ })+ .collect::<(Vec<_>, Vec<_>, Vec<_>)>();+ let enum_variants = concrete_configs.into_iter().map(|concrete_config_path| {+ let concrete_config_ident = &concrete_config_path.segments.last().unwrap().ident;+ quote! { #concrete_config_ident(#concrete_config_path) }+ });++ quote! {+ #(#alias_struct_fields)*++ macro_rules! #struct_ident {+ ($($typealias_base:ident)::+, $callback:path) => {+ $callback! {+ #(#callback_struct_fields,)*+ }+ };+ }++ use #struct_ident;++ #[proc_macros::uzu_config]+ #[serde(untagged)]+ #input_vis enum #enum_ident {+ #(#enum_variants,)*+ }++ impl #enum_ident {+ #(#enum_field_getters)*+ }+ }+ .into()+}pub fn uzu_config(- _args: TokenStream,+ args: TokenStream,input: TokenStream,) -> TokenStream {let mut input = parse_macro_input!(input as DeriveInput);+ if !args.is_empty() {+ let fields_macro_path = parse_macro_input!(args as Path);+ let fields_macro_base_path = {+ let mut fields_macro_base = fields_macro_path.clone();+ fields_macro_base.segments.pop();+ fields_macro_base.segments.pop_punct();+ fields_macro_base+ };+ let fields_macro_base = if fields_macro_base_path.segments.is_empty() {+ quote! { self }+ } else {+ quote! { #fields_macro_base_path }+ };++ let struct_fields = match input.data {+ Data::Struct(DataStruct {+ fields: Fields::Named(FieldsNamed {+ named,+ ..+ }),+ ..+ }) => named,+ Data::Struct(DataStruct {+ fields: Fields::Unit,+ ..+ }) => Punctuated::new(),+ _ => {+ return syn::Error::new(+ input.ident.span(),+ "uzu_config(Abstract) requires a named-field or unit struct",+ )+ .to_compile_error()+ .into();+ },+ };++ let input_attrs = input.attrs;+ let input_vis = input.vis;+ let input_ident = input.ident;+ let input_ident_str = input_ident.to_string();+ let input_generics = input.generics;+ let struct_fields = struct_fields.into_iter().collect::<Vec<_>>();+ let callback_ident = format_ident!("__emit_{}", input_ident);++ // Re-entry with super fields inserted+ return quote! {+ macro_rules! #callback_ident {+ ($($super_fields:tt)*) => {+ #[proc_macros::uzu_config]+ #(#input_attrs)*+ #input_vis struct #input_ident #input_generics {+ #[serde(rename = "type")]+ ty: monostate::MustBe!(#input_ident_str),+ $($super_fields)*+ #(#struct_fields,)*+ }+ }+ }++ #fields_macro_path!(#fields_macro_base, #callback_ident);+ }+ .into();+ };+let fields: Vec<&mut Field> = match &mut input.data {Data::Struct(data_struct) => data_struct.fields.iter_mut().collect(),Data::Enum(data_enum) => data_enum.variants.iter_mut().flat_map(|variant| variant.fields.iter_mut()).collect(),diff --git a/crates/uzu/src/registry/local/config.rs b/crates/uzu/src/registry/local/config.rsindex 3773d8d89..9ffadcdc6 100644--- a/crates/uzu/src/registry/local/config.rs+++ b/crates/uzu/src/registry/local/config.rs@@ -1,9 +1,11 @@use std::{path::PathBuf, sync::Arc};-use backend_uzu::inference::resolve_model_metadata;+use backend_uzu::inference::resolve_model_specialization;use shoji::types::model::Model;-pub type ModelResolver = Arc<dyn Fn(Model) -> Option<Model> + Send + Sync>;+use crate::registry::RegistryError;++pub type ModelResolver = Arc<dyn Fn(Model) -> Result<Model, RegistryError> + Send + Sync>;#[derive(Clone)]pub struct Config {@@ -40,18 +42,18 @@ impl Config {path: String,) -> Self {let models_path = PathBuf::from(&path).join("models");- let resolver: ModelResolver = {- let backend_version = backend_version.clone();- Arc::new(move |mut model: Model| -> Option<Model> {- let model_path = PathBuf::from(model.local_external_path()?);- let metadata = resolve_model_metadata(&model_path)?;- if !is_lalamo_version_supported(&metadata.toolchain_version, &backend_version) {- return None;- }- model.specializations = vec![metadata.specialization];- Some(model)- })- };+ let resolver: ModelResolver = Arc::new(move |mut model: Model| -> Result<Model, RegistryError> {+ let model_path = model.local_external_path().ok_or_else(|| RegistryError::UnableToGetModels {+ message: format!("Local model {} has no local path", model.identifier),+ })?;+ let model_path = PathBuf::from(model_path);+ let specialization =+ resolve_model_specialization(&model_path).map_err(|error| RegistryError::UnableToGetModels {+ message: format!("Unable to resolve specialization for {}: {error}", model_path.display()),+ })?;+ model.specializations = vec![specialization];+ Ok(model)+ });Self::new("lalamo".to_string(),backend_identifier,@@ -62,23 +64,3 @@ impl Config {)}}--fn is_lalamo_version_supported(- model_version: &str,- backend_version: &str,-) -> bool {- let parse_version = |value: &str| -> Option<(u32, u32, u32)> {- let mut parts = value.split('.').map(|part| part.parse::<u32>().ok());- let major = parts.next().flatten()?;- let minor = parts.next().flatten()?;- let patch = parts.next().flatten()?;- Some((major, minor, patch))- };- let Some((model_major, model_minor, model_patch)) = parse_version(model_version) else {- return false;- };- let Some((backend_major, backend_minor, backend_patch)) = parse_version(backend_version) else {- return false;- };- model_major == backend_major && model_minor == backend_minor && model_patch <= backend_patch-}diff --git a/crates/uzu/src/registry/local/mod.rs b/crates/uzu/src/registry/local/mod.rsindex b3fcbbdbc..4a9de94a3 100644--- a/crates/uzu/src/registry/local/mod.rs+++ b/crates/uzu/src/registry/local/mod.rs@@ -43,26 +43,29 @@ impl RegistryTrait for Registry {})?;let mut models = Vec::new();- for entry in entries.flatten() {+ for entry in entries {+ let entry = entry.map_err(|error| RegistryError::UnableToGetModels {+ message: error.to_string(),+ })?;let path = entry.path();- let Ok(file_type) = entry.file_type() else {- continue;- };+ let file_type = entry.file_type().map_err(|error| RegistryError::UnableToGetModels {+ message: error.to_string(),+ })?;if !file_type.is_dir() {continue;}- let Some(name) = path.file_name().and_then(|name| name.to_str()) else {- continue;- };+ let name = path.file_name().and_then(|name| name.to_str()).ok_or_else(|| {+ RegistryError::UnableToGetModels {+ message: format!("Invalid local model path: {}", path.display()),+ }+ })?;let model = self.model(name);let model = match self.config.resolver.as_ref() {- Some(resolver) => resolver(model),- None => Some(model),+ Some(resolver) => resolver(model)?,+ None => model,};- if let Some(model) = model {- models.push(model);- }+ models.push(model);}Ok(models)diff --git a/flake.nix b/flake.nixindex e1ab398ae..f4345dc15 100644--- a/flake.nix+++ b/flake.nix@@ -72,6 +72,7 @@rustToolchaincargo-denycargo-nextest+ cargo-expandcargo-flamegraphcargo-show-asmcritcmp