Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

New config dsl and lalamo 0.10.0 config format (#435)

uuuvn0d6b4c4459a7
diff --git a/Cargo.lock b/Cargo.lock
index ef6103615..eb2e2b091 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -576,6 +576,7 @@ dependencies = [
"is_close",
"itertools 0.14.0",
"libc",
+ "libm",
"memmap2",
"minijinja",
"minijinja-contrib",
diff --git a/Cargo.toml b/Cargo.toml
index 63e88a91e..7bf0c5855 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -80,6 +80,7 @@ num-traits = "0.2"
regex = "1.12"
fancy-regex = "0.17.0"
libc = "0.2"
+libm = "0.2.16"
minijinja = { version = "2.17.0", features = ["json", "preserve_order"] }
minijinja-contrib = { version = "2.17.0", features = ["pycompat"] }
clap = { version = "4.6", features = ["derive"] }
diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.toml
index 1c340800d..083b4fe64 100644
--- a/crates/backend-uzu/Cargo.toml
+++ b/crates/backend-uzu/Cargo.toml
@@ -29,6 +29,7 @@ memmap2.workspace = true
num-traits.workspace = true
regex.workspace = true
libc.workspace = true
+libm.workspace = true
minijinja.workspace = true
minijinja-contrib.workspace = true
rand.workspace = true
diff --git a/crates/backend-uzu/src/audio/context.rs b/crates/backend-uzu/src/audio/context.rs
index 25c04ab0c..bb222682a 100644
--- a/crates/backend-uzu/src/audio/context.rs
+++ b/crates/backend-uzu/src/audio/context.rs
@@ -1,7 +1,7 @@
use std::{path::Path, rc::Rc};
use super::{AudioResult, NanoCodecFsqRuntime};
-use crate::{backends::common::Backend, config::TtsConfig};
+use crate::{backends::common::Backend, config::tts::TTSConfig};
#[derive(Clone)]
pub struct AudioGenerationContext<B: Backend> {
@@ -13,7 +13,7 @@ pub struct AudioGenerationContext<B: Backend> {
impl<B: Backend> AudioGenerationContext<B> {
pub fn from_tts_config_and_model_path(
- tts_config: &TtsConfig,
+ tts_config: &TTSConfig,
model_path: &Path,
) -> AudioResult<Self> {
Self::with_runtime(NanoCodecFsqRuntime::from_tts_config_and_model_path(tts_config, model_path)?)
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime.rs b/crates/backend-uzu/src/audio/nanocodec/runtime.rs
index 8ad4a33c1..e3b4758f5 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime.rs
@@ -22,8 +22,13 @@ use crate::{
},
},
config::{
- DecoderConfig, DescriptAudioCodecConfig, EmbeddingConfig, EmbeddingConfigCommon, NormalizationConfig,
- TtsAudioDecoderConfig, TtsConfig,
+ decoder::DecoderConfig,
+ embedding::{AnyEmbeddingConfig, untied_embedding::UntiedEmbeddingConfig},
+ normalization::NormalizationConfig,
+ tts::{
+ TTSConfig,
+ audio_decoder::{AnyTTSAudioDecoderConfig, descript_audio_codec::DescriptAudioCodecConfig},
+ },
},
encodable_block::{Decoder, LayerExecutables, RMSNorm},
forward_pass::{model_shape::ModelShape, state::SharedBuffers},
@@ -91,7 +96,7 @@ pub struct NanoCodecFsqRuntimeConfig {
impl NanoCodecFsqRuntimeConfig {
pub fn from_tts_config_and_model_path(
- tts_config: &TtsConfig,
+ tts_config: &TTSConfig,
model_path: &Path,
) -> AudioResult<Self> {
let (runtime_config, decoder) = load_audio_runtime_from_tts_config(tts_config, model_path)?;
@@ -233,7 +238,7 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
}
pub fn from_tts_config_and_model_path(
- tts_config: &TtsConfig,
+ tts_config: &TTSConfig,
model_path: &Path,
) -> AudioResult<Self> {
Ok(Self::new(NanoCodecFsqRuntimeConfig::from_tts_config_and_model_path(tts_config, model_path)?))
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs
index e066650fe..36754723d 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs
@@ -1,46 +1,26 @@
use super::*;
-
-pub(super) fn resolve_descript_audio_codec_vocoder_data_type(
- top_level_precision: DataType,
- config: &DescriptAudioCodecConfig,
-) -> AudioResult<DataType> {
- let resolved_precision = top_level_precision;
- for (field_name, precision) in [
- ("tts_config.audio_decoder_config.precision", config.precision),
- ("tts_config.audio_decoder_config.quantizer_config.precision", config.quantizer_config.precision),
- ] {
- if resolved_precision != precision {
- return Err(AudioError::Runtime(format!(
- "conflicting DescriptAudioCodec precision in Lalamo export: {field_name}={precision:?} conflicts with {resolved_precision:?}"
- )));
- }
- }
-
- let data_type: DataType = resolved_precision.into();
- if !matches!(data_type, DataType::F32 | DataType::F16 | DataType::BF16) {
- return Err(AudioError::Runtime(format!(
- "unsupported DescriptAudioCodec vocoder precision in Lalamo export: {resolved_precision:?} (expected float32/float16/bfloat16)"
- )));
- }
- Ok(data_type)
-}
+use crate::{
+ config::{
+ decoder::DecoderConfig,
+ embedding::{AnyEmbeddingConfig, untied_embedding::UntiedEmbeddingConfig},
+ },
+ forward_pass::model_shape::ModelShape,
+};
pub(super) fn load_audio_runtime_from_tts_config(
- tts_config: &TtsConfig,
+ tts_config: &TTSConfig,
model_path: &Path,
) -> AudioResult<(RuntimeConfigJson, StructuredAudioCodecGraph)> {
- let cfg = match &tts_config.audio_decoder_config {
- TtsAudioDecoderConfig::DescriptAudioCodec(config) => config,
+ let AnyTTSAudioDecoderConfig::DescriptAudioCodecConfig(cfg) = &tts_config.audio_decoder_config;
+ let weights_path = model_path.join("model.safetensors");
+ let post_module_decoder_config = DecoderConfig {
+ embedding_config: AnyEmbeddingConfig::UntiedEmbeddingConfig(UntiedEmbeddingConfig::new(None, None)),
+ transformer_config: cfg.quantizer_config.post_module_config.clone(),
+ vocab_size: 1,
+ pard_token: None,
+ ple_model_config: None,
};
- let fishaudio_weights = model_path.join("model.safetensors");
- if !fishaudio_weights.is_file() {
- return Err(AudioError::Runtime(format!(
- "missing exported FishAudio decoder weights '{}'",
- fishaudio_weights.display()
- )));
- }
-
- let vocoder_data_type = resolve_descript_audio_codec_vocoder_data_type(tts_config.activation_precision, cfg)?;
+ let vocoder_data_type = ModelShape::from_decoder_config(&post_module_decoder_config, DataType::F32).data_type;
let total_codebooks = cfg
.n_codebooks
@@ -67,7 +47,7 @@ pub(super) fn load_audio_runtime_from_tts_config(
};
let decoder = StructuredAudioCodecGraph {
config: cfg.clone(),
- weights_path: fishaudio_weights.display().to_string(),
+ weights_path: weights_path.display().to_string(),
codebook_size: cfg.codebook_size,
semantic_codebook_size: cfg.semantic_codebook_size,
input_dim: cfg.input_dim,
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs
index d6e718cfe..6408898ba 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs
@@ -1,63 +1,5 @@
use super::*;
-use crate::array::{Array, ArrayContextExt};
-
-pub(super) type BackendParameterTree<'loader, B> = crate::parameters::ParameterTree<'loader, <B as Backend>::Context>;
-
-pub(super) fn read_float_array<B: Backend, const RANK: usize>(
- tree: &BackendParameterTree<B>,
- name: &str,
- expected_data_type: DataType,
-) -> AudioResult<([usize; RANK], Array<B>)> {
- let array = tree.leaf_array(name)?;
- if array.data_type() != expected_data_type {
- return Err(AudioError::Runtime(format!(
- "tensor '{name}' dtype mismatch: expected {expected_data_type:?}, got {:?}",
- array.data_type()
- )));
- }
- if array.shape().len() != RANK {
- return Err(AudioError::Runtime(format!(
- "expected rank-{RANK} tensor for '{name}', got rank {}",
- array.shape().len()
- )));
- }
- let mut dims = [0usize; RANK];
- dims.copy_from_slice(array.shape());
- Ok((dims, array))
-}
-
-pub(super) fn read_float_vector_exact<B: Backend>(
- tree: &BackendParameterTree<B>,
- name: &str,
- expected_len: usize,
- expected_data_type: DataType,
-) -> AudioResult<Array<B>> {
- let (shape, array) = read_float_array::<B, 1>(tree, name, expected_data_type)?;
- if shape[0] != expected_len {
- return Err(AudioError::Runtime(format!(
- "tensor '{name}' shape mismatch: expected [{expected_len}], got {:?}",
- shape
- )));
- }
- Ok(array)
-}
-
-pub(super) fn read_float_matrix_exact<B: Backend>(
- tree: &BackendParameterTree<B>,
- name: &str,
- expected_rows: usize,
- expected_cols: usize,
- expected_data_type: DataType,
-) -> AudioResult<Array<B>> {
- let (shape, array) = read_float_array::<B, 2>(tree, name, expected_data_type)?;
- if shape != [expected_rows, expected_cols] {
- return Err(AudioError::Runtime(format!(
- "tensor '{name}' shape mismatch: expected [{expected_rows}, {expected_cols}], got {:?}",
- shape
- )));
- }
- Ok(array)
-}
+use crate::{array::Array, backends::common::gpu_types::ActivationType, parameters::ParameterTree};
pub(super) fn copy_to_outer_axis_slice<B: Backend>(
destination: &mut Array<B>,
@@ -81,80 +23,71 @@ pub(super) fn copy_to_outer_axis_slice<B: Backend>(
}
pub(super) fn read_conv1d_layer<B: Backend>(
- tree: &BackendParameterTree<B>,
+ tree: &ParameterTree<B>,
data_type: DataType,
+ cin: usize,
+ cout: usize,
+ kernel_size: usize,
dilation: usize,
groups: usize,
) -> AudioResult<StructuredAudioConv1d<B>> {
- let (shape, weight) = read_float_array::<B, 3>(tree, "weights", data_type)?;
- let bias = read_float_vector_exact::<B>(tree, "biases", shape[0], data_type)?;
- if groups == 0 || shape[0] == 0 || shape[1] == 0 || shape[2] == 0 {
+ if groups == 0 || cin == 0 || cout == 0 || kernel_size == 0 {
return Err(AudioError::InvalidTokenCardinality);
}
- if shape[0] % groups != 0 {
+ if cin % groups != 0 || cout % groups != 0 {
return Err(AudioError::Runtime(format!(
- "invalid grouped conv weights for 'weights': out_channels {} not divisible by groups {groups}",
- shape[0]
+ "invalid grouped conv dimensions: cin {cin}, cout {cout}, groups {groups}"
)));
}
+ let weight = tree.leaf("weights")?.validate(&[cout, cin / groups, kernel_size], data_type)?.read_array()?;
+ let bias = tree.leaf("biases")?.validate(&[cout], data_type)?.read_array()?;
Ok(StructuredAudioConv1d {
weight,
bias,
- cin: shape[1].checked_mul(groups).ok_or(AudioError::Runtime("conv input channel overflow".to_string()))?,
- cout: shape[0],
- kernel_size: shape[2],
+ cin,
+ cout,
+ kernel_size,
dilation,
groups,
})
}
pub(super) fn read_conv_transpose1d_layer<B: Backend>(
- tree: &BackendParameterTree<B>,
+ tree: &ParameterTree<B>,
data_type: DataType,
+ cin: usize,
+ cout: usize,
+ kernel_size: usize,
stride: usize,
groups: usize,
) -> AudioResult<StructuredAudioConvTranspose1d<B>> {
- if stride == 0 {
- return Err(AudioError::InvalidTokenCardinality);
- }
- let (shape, weight) = read_float_array::<B, 3>(tree, "weights", data_type)?;
- if groups == 0 || shape[0] == 0 || shape[1] == 0 || shape[2] == 0 {
+ if stride == 0 || groups == 0 || cin == 0 || cout == 0 || kernel_size == 0 {
return Err(AudioError::InvalidTokenCardinality);
}
- let export_cout = shape[0];
- let export_cin =
- shape[1].checked_mul(groups).ok_or(AudioError::Runtime("transpose conv input channel overflow".to_string()))?;
- if export_cout % groups != 0 {
+ if cin % groups != 0 || cout % groups != 0 {
return Err(AudioError::InvalidTokenCardinality);
}
- let bias = read_float_vector_exact::<B>(tree, "biases", export_cout, data_type)?;
- let expected_weight_shape = [export_cout, export_cin / groups, shape[2]];
- if shape != expected_weight_shape {
- return Err(AudioError::Runtime(format!(
- "transpose conv export-layout weight shape mismatch: expected {:?}, got {:?}",
- expected_weight_shape, shape
- )));
- }
-
+ let weight = tree.leaf("weights")?.validate(&[cout, cin / groups, kernel_size], data_type)?.read_array()?;
+ let bias = tree.leaf("biases")?.validate(&[cout], data_type)?.read_array()?;
Ok(StructuredAudioConvTranspose1d {
weight,
bias,
- cin: export_cin,
- cout: export_cout,
- kernel_size: shape[2],
+ cin,
+ cout,
+ kernel_size,
stride,
groups,
})
}
pub(super) fn read_pointwise_conv_layer<B: Backend>(
- tree: &BackendParameterTree<B>,
+ tree: &ParameterTree<B>,
data_type: DataType,
expected_out_dim: usize,
expected_in_dim: usize,
) -> AudioResult<StructuredAudioPointwiseConv<B>> {
- let weight = read_float_matrix_exact::<B>(tree, "weights", expected_out_dim, expected_in_dim, data_type)?;
- let bias = read_float_vector_exact::<B>(tree, "biases", expected_out_dim, data_type)?;
+ let weight = tree.leaf("weights")?.validate(&[expected_out_dim, expected_in_dim], data_type)?.read_array()?;
+ let bias = tree.leaf("biases")?.validate(&[expected_out_dim], data_type)?.read_array()?;
Ok(StructuredAudioPointwiseConv {
weight,
bias,
@@ -165,16 +98,16 @@ pub(super) fn read_pointwise_conv_layer<B: Backend>(
pub(super) fn read_norm_layer<B: Backend>(
context: &Rc<B::Context>,
- tree: &BackendParameterTree<B>,
+ tree: &ParameterTree<B>,
data_type: DataType,
channels: usize,
epsilon: f32,
subtract_mean: bool,
use_bias: bool,
) -> AudioResult<StructuredAudioNorm<B>> {
- let scales = read_float_vector_exact::<B>(tree, "scales", channels, data_type)?;
+ let scales = tree.leaf("scales")?.validate(&[channels], data_type)?.read_array()?;
let bias = if use_bias {
- read_float_vector_exact::<B>(tree, "biases", channels, data_type)?
+ tree.leaf("biases")?.validate(&[channels], data_type)?.read_array()?
} else {
context.create_array_zeros(&[channels], data_type)
};
@@ -188,56 +121,27 @@ pub(super) fn read_norm_layer<B: Backend>(
pub(super) fn read_convnext_layer<B: Backend>(
context: &Rc<B::Context>,
- tree: &BackendParameterTree<B>,
+ tree: &ParameterTree<B>,
data_type: DataType,
+ activation_type: ActivationType,
norm_config: &NormalizationConfig,
+ dim: usize,
) -> AudioResult<StructuredAudioConvNeXt<B>> {
- let depthwise_tree = tree.subtree("dwconv")?;
- let (depthwise_shape, depthwise_weight) = read_float_array::<B, 3>(&depthwise_tree, "weights", data_type)?;
- if depthwise_shape[1] != 1 {
- return Err(AudioError::Runtime(format!(
- "ConvNeXt depthwise weight in_channels_per_group must be 1 at 'dwconv.weights', got {}",
- depthwise_shape[1]
- )));
- }
- let depthwise_conv = StructuredAudioConv1d {
- weight: depthwise_weight,
- bias: read_float_vector_exact::<B>(&depthwise_tree, "biases", depthwise_shape[0], data_type)?,
- cin: depthwise_shape[0],
- cout: depthwise_shape[0],
- kernel_size: depthwise_shape[2],
- dilation: 1,
- groups: depthwise_shape[0],
- };
- if depthwise_conv.cout == 0 || depthwise_conv.kernel_size == 0 {
- return Err(AudioError::InvalidTokenCardinality);
- }
- if depthwise_conv.cin != depthwise_conv.cout {
- return Err(AudioError::Runtime(format!(
- "ConvNeXt depthwise conv expects cin==cout, got {} vs {} at 'dwconv.weights'",
- depthwise_conv.cin, depthwise_conv.cout
- )));
- }
- let channels = depthwise_conv.cout;
+ let hidden_dim = dim.checked_mul(4).ok_or(AudioError::Runtime("ConvNeXt hidden dimension overflow".to_string()))?;
+ let depthwise_conv = read_conv1d_layer::<B>(&tree.subtree("dwconv")?, data_type, dim, dim, 7, 1, dim)?;
let norm = read_norm_layer::<B>(
context,
&tree.subtree("norm")?,
data_type,
- channels,
+ dim,
norm_config.epsilon,
norm_config.subtract_mean,
- norm_config.use_bias,
+ norm_config.has_biases,
)?;
- let (pwconv1_shape, _) = read_float_array::<B, 2>(&tree.subtree("pwconv1")?, "weights", data_type)?;
- if pwconv1_shape[1] != channels {
- return Err(AudioError::Runtime(format!(
- "ConvNeXt pwconv1 input mismatch at 'pwconv1.weights': expected {}, got {}",
- channels, pwconv1_shape[1]
- )));
- }
- let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv1")?, data_type, pwconv1_shape[0], channels)?;
- let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv2")?, data_type, channels, pwconv1_shape[0])?;
+ let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv1")?, data_type, hidden_dim, dim)?;
+ let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv2")?, data_type, dim, hidden_dim)?;
Ok(StructuredAudioConvNeXt {
+ activation_type,
depthwise_conv,
norm,
pwconv1,
@@ -246,31 +150,16 @@ pub(super) fn read_convnext_layer<B: Backend>(
}
pub(super) fn read_residual_unit_layer<B: Backend>(
- tree: &BackendParameterTree<B>,
+ tree: &ParameterTree<B>,
data_type: DataType,
dilation: usize,
+ dim: usize,
) -> AudioResult<StructuredAudioResidualUnit<B>> {
- let snake1_alpha = {
- let snake1_tree = tree.subtree("snake1")?;
- let (shape, alpha) = read_float_array::<B, 1>(&snake1_tree, "alpha", data_type)?;
- if shape[0] == 0 {
- return Err(AudioError::InvalidTokenCardinality);
- }
- alpha
- };
- let conv1 = read_conv1d_layer::<B>(&tree.subtree("conv1")?, data_type, dilation, 1)?;
- let snake2_alpha = {
- let snake2_tree = tree.subtree("snake2")?;
- let (shape, alpha) = read_float_array::<B, 1>(&snake2_tree, "alpha", data_type)?;
- if shape[0] != conv1.cout {
- return Err(AudioError::Runtime(format!(
- "residual snake2 alpha mismatch: expected {}, got {}",
- conv1.cout, shape[0]
- )));
- }
- alpha
- };
- let conv2 = read_conv1d_layer::<B>(&tree.subtree("conv2")?, data_type, 1, 1)?;
+ let alpha_shape = [dim];
+ let snake1_alpha = tree.leaf("snake1.alpha")?.validate(&alpha_shape, data_type)?.read_array()?;
+ let conv1 = read_conv1d_layer::<B>(&tree.subtree("conv1")?, data_type, dim, dim, 7, dilation, 1)?;
+ let snake2_alpha = tree.leaf("snake2.alpha")?.validate(&alpha_shape, data_type)?.read_array()?;
+ let conv2 = read_conv1d_layer::<B>(&tree.subtree("conv2")?, data_type, dim, dim, 1, 1, 1)?;
Ok(StructuredAudioResidualUnit {
snake1_alpha,
conv1,
@@ -281,7 +170,7 @@ pub(super) fn read_residual_unit_layer<B: Backend>(
pub(super) fn build_vocoder_graph_from_tree<B: Backend>(
context: &Rc<B::Context>,
- root: &BackendParameterTree<B>,
+ root: &ParameterTree<B>,
config: &DescriptAudioCodecConfig,
data_type: DataType,
) -> AudioResult<StructuredAudioDecoderGraph<B>> {
@@ -289,45 +178,62 @@ pub(super) fn build_vocoder_graph_from_tree<B: Backend>(
let quantizer_tree = audio_decoder_tree.subtree("quantizer")?;
let decoder_tree = audio_decoder_tree.subtree("decoder")?;
- let first_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("first_conv")?, data_type, 1, 1)?;
- let final_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("final_conv")?, data_type, 1, 1)?;
- let final_snake_alpha =
- read_float_vector_exact::<B>(&decoder_tree.subtree("final_snake")?, "alpha", final_conv.cin, data_type)?;
+ let latent_dim = config.encoder_dim * (1usize << config.encoder_rates.len());
+ let final_dim = config.decoder_dim / (1usize << config.decoder_rates.len());
+
+ let first_conv = read_conv1d_layer::<B>(
+ &decoder_tree.subtree("first_conv")?,
+ data_type,
+ latent_dim,
+ config.decoder_dim,
+ 7,
+ 1,
+ 1,
+ )?;
+ let final_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("final_conv")?, data_type, final_dim, 1, 7, 1, 1)?;
+ let final_snake_alpha = decoder_tree.leaf("final_snake.alpha")?.validate(&[final_dim], data_type)?.read_array()?;
let mut upsample_blocks = Vec::with_capacity(config.downsample_factor.len());
for (index, &stride) in config.downsample_factor.iter().rev().enumerate() {
let block_tree = quantizer_tree.subtree("upsampler")?.subtree("blocks")?.subtree(&index.to_string())?;
- let trans_conv = read_conv_transpose1d_layer::<B>(&block_tree.subtree("trans_conv")?, data_type, stride, 1)?;
+ let trans_conv = read_conv_transpose1d_layer::<B>(
+ &block_tree.subtree("trans_conv")?,
+ data_type,
+ config.input_dim,
+ config.input_dim,
+ stride,
+ stride,
+ 1,
+ )?;
let convnext = read_convnext_layer::<B>(
context,
&block_tree.subtree("convnext")?,
data_type,
+ config.quantizer_config.upsampler_config.block_configs[index].convnext_config.activation.act_type(),
&config.quantizer_config.upsampler_config.block_configs[index].convnext_config.norm_config,
+ config.input_dim,
)?;
- if convnext.depthwise_conv.cin != trans_conv.cout {
- return Err(AudioError::Runtime(format!(
- "structured audio upsampler convnext channel mismatch at block {index}: trans_conv out {} vs convnext in {}",
- trans_conv.cout, convnext.depthwise_conv.cin
- )));
- }
upsample_blocks.push((trans_conv, convnext));
}
let mut decoder_blocks = Vec::with_capacity(config.decoder_rates.len());
for (index, &stride) in config.decoder_rates.iter().enumerate() {
let block_tree = decoder_tree.subtree("decoder_blocks")?.subtree(&index.to_string())?;
- let trans_conv = read_conv_transpose1d_layer::<B>(&block_tree.subtree("trans_conv")?, data_type, stride, 1)?;
- let snake_alpha =
- read_float_vector_exact::<B>(&block_tree.subtree("snake")?, "alpha", trans_conv.cin, data_type)?;
- let channels = trans_conv.cout;
- let res_unit1 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit1")?, data_type, 1)?;
- let res_unit2 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit2")?, data_type, 3)?;
- let res_unit3 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit3")?, data_type, 9)?;
- if res_unit1.conv1.cin != channels || res_unit2.conv1.cin != channels || res_unit3.conv1.cin != channels {
- return Err(AudioError::Runtime(format!(
- "structured audio decoder block {index} channel mismatch in exported weights"
- )));
- }
+ let input_dim = config.decoder_dim / (1usize << index);
+ let output_dim = config.decoder_dim / (1usize << (index + 1));
+ let trans_conv = read_conv_transpose1d_layer::<B>(
+ &block_tree.subtree("trans_conv")?,
+ data_type,
+ input_dim,
+ output_dim,
+ 2 * stride,
+ stride,
+ 1,
+ )?;
+ let snake_alpha = block_tree.leaf("snake.alpha")?.validate(&[input_dim], data_type)?.read_array()?;
+ let res_unit1 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit1")?, data_type, 1, output_dim)?;
+ let res_unit2 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit2")?, data_type, 3, output_dim)?;
+ let res_unit3 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit3")?, data_type, 9, output_dim)?;
decoder_blocks.push(StructuredAudioDecoderBlock {
snake_alpha,
trans_conv,
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs
index 827917577..46838e677 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs
@@ -401,18 +401,19 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(
Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
-pub(super) fn gelu_enqueue<B: Backend>(
+pub(super) fn activation_enqueue<B: Backend>(
encoder: &mut Encoder<B>,
input: &Array<B>,
output: Array<B>,
+ activation_type: ActivationType,
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
let n_u32 = u32::try_from(input.num_elements())
- .map_err(|_| AudioError::Runtime("gelu element count exceeds u32 range".to_string()))?;
+ .map_err(|_| AudioError::Runtime("activation element count exceeds u32 range".to_string()))?;
let output_shape = output.shape().to_vec();
let output_data_type = output.data_type();
let mut output = output.into_allocation();
- kernels.activation.encode(Some(input.allocation()), &mut output, n_u32, ActivationType::GELU, encoder);
+ kernels.activation.encode(Some(input.allocation()), &mut output, n_u32, activation_type, encoder);
Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index 4b3293b3e..34a20a188 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -63,8 +63,9 @@ impl StructuredAudioCodecGraph {
self.vocoder_data_type,
kernels,
)?;
- let cnxt_gelu = ws.next_scratch(encoder, &[batch_size, layer.pwconv1.cout, seq_len], self.vocoder_data_type);
- let x = gelu_enqueue(encoder, &x, cnxt_gelu, kernels)?;
+ let cnxt_activation =
+ ws.next_scratch(encoder, &[batch_size, layer.pwconv1.cout, seq_len], self.vocoder_data_type);
+ let x = activation_enqueue(encoder, &x, cnxt_activation, layer.activation_type, kernels)?;
let cnxt_pw2 = ws.next_scratch(encoder, &[batch_size, layer.pwconv2.cout, seq_len], self.vocoder_data_type);
let x = conv1d_pointwise_ncs_enqueue(
encoder,
@@ -88,19 +89,13 @@ impl StructuredAudioCodecGraph {
required_sequence_length: usize,
) -> AudioResult<StructuredAudioPostModuleRuntime<B>> {
let decoder_config = Rc::new(DecoderConfig {
- embedding_config: EmbeddingConfig::Untied {
- common: EmbeddingConfigCommon {
- input_scale: None,
- logit_soft_cap: None,
- },
- precision: self.vocoder_data_type.into(),
- },
+ embedding_config: AnyEmbeddingConfig::UntiedEmbeddingConfig(UntiedEmbeddingConfig::new(None, None)),
transformer_config: self.config.quantizer_config.post_module_config.clone(),
vocab_size: 1,
pard_token: None,
ple_model_config: None,
});
- let model_shape = ModelShape::from_decoder_config(&decoder_config);
+ let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::F32);
let weights_file = File::open(self.weights_path.as_str()).map_err(|err| {
AudioError::Runtime(format!("failed to open post_module weights '{}': {err}", self.weights_path))
@@ -110,12 +105,9 @@ impl StructuredAudioCodecGraph {
})?;
let root_loader_view = loader.tree();
let transformer_subtree_name = "audio_decoder.quantizer.post_module";
- root_loader_view
- .subtree(transformer_subtree_name)
- .map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;
let max_sequence_length = decoder_config.transformer_config.context_length.max(required_sequence_length.max(1));
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config);
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
{
let transformer_tree = root_loader_view
.subtree(transformer_subtree_name)
@@ -130,7 +122,10 @@ impl StructuredAudioCodecGraph {
&decoder_config,
&root_loader_view,
transformer_subtree_name,
- );
+ None,
+ &model_shape,
+ )
+ .map_err(|error| AudioError::Runtime(format!("failed to build structured audio post_module: {error}")))?;
Ok(StructuredAudioPostModuleRuntime {
context,
@@ -180,23 +175,17 @@ impl StructuredAudioCodecGraph {
let audio_decoder_tree = root.subtree("audio_decoder")?;
let quantizer_tree = audio_decoder_tree.subtree("quantizer")?;
let semantic_tree = quantizer_tree.subtree("semantic_quantizer")?.subtree("quantizers")?.subtree("0")?;
- let semantic_codebook = read_float_matrix_exact::<B>(
- &semantic_tree.subtree("codebook")?,
- "weights",
- self.semantic_codebook_size,
- self.config.codebook_dim,
- data_type,
- )?;
let codebook_dim = self.config.codebook_dim;
- let semantic_out_proj = read_float_matrix_exact::<B>(
- &semantic_tree.subtree("out_proj")?,
- "weights",
- self.input_dim,
- codebook_dim,
- data_type,
- )?;
+ let semantic_codebook = semantic_tree
+ .leaf("codebook.weights")?
+ .validate(&[self.semantic_codebook_size, codebook_dim], data_type)?
+ .read_array()?;
+ let semantic_out_proj = semantic_tree
+ .leaf("out_proj.weights")?
+ .validate(&[self.input_dim, codebook_dim], data_type)?
+ .read_array()?;
let semantic_out_bias =
- read_float_vector_exact::<B>(&semantic_tree.subtree("out_proj")?, "biases", self.input_dim, data_type)?;
+ semantic_tree.leaf("out_proj.biases")?.validate(&[self.input_dim], data_type)?.read_array()?;
let residual_quantizers = self.config.n_codebooks;
let residual_count_for_shape = residual_quantizers.max(1);
@@ -210,26 +199,16 @@ impl StructuredAudioCodecGraph {
let residual_root = quantizer_tree.subtree("quantizer")?.subtree("quantizers")?;
for index in 0..residual_quantizers {
let quantizer_tree = residual_root.subtree(&index.to_string())?;
- let codebook = read_float_matrix_exact::<B>(
- &quantizer_tree.subtree("codebook")?,
- "weights",
- self.codebook_size,
- codebook_dim,
- data_type,
- )?;
- let out_proj = read_float_matrix_exact::<B>(
- &quantizer_tree.subtree("out_proj")?,
- "weights",
- self.input_dim,
- codebook_dim,
- data_type,
- )?;
- let out_bias = read_float_vector_exact::<B>(
- &quantizer_tree.subtree("out_proj")?,
- "biases",
- self.input_dim,
- data_type,
- )?;
+ let codebook = quantizer_tree
+ .leaf("codebook.weights")?
+ .validate(&[self.codebook_size, codebook_dim], data_type)?
+ .read_array()?;
+ let out_proj = quantizer_tree
+ .leaf("out_proj.weights")?
+ .validate(&[self.input_dim, codebook_dim], data_type)?
+ .read_array()?;
+ let out_bias =
+ quantizer_tree.leaf("out_proj.biases")?.validate(&[self.input_dim], data_type)?.read_array()?;
copy_to_outer_axis_slice(&mut residual_codebooks, index, &codebook, &[self.codebook_size, codebook_dim])?;
copy_to_outer_axis_slice(&mut residual_out_proj, index, &out_proj, &[self.input_dim, codebook_dim])?;
@@ -310,7 +289,6 @@ impl StructuredAudioCodecGraph {
token_positions: token_inputs.token_positions(),
token_parents: token_inputs.token_parents(),
token_subtrie_ranges: None,
- attention_sinks: runtime.shared_buffers.attention_sinks(layer.layer_index),
rope_buffers,
sampling_start: 0,
sampling_length: batch_dim,
@@ -431,11 +409,10 @@ impl StructuredAudioCodecGraph {
self.input_dim, main_shape[0], main_shape[1]
)));
}
- if runtime.model_shape.activation_data_type() != self.vocoder_data_type {
+ if runtime.model_shape.data_type != self.vocoder_data_type {
return Err(AudioError::Runtime(format!(
"post_module dtype mismatch: main={:?}, latent={:?}",
- runtime.model_shape.activation_data_type(),
- self.vocoder_data_type
+ runtime.model_shape.data_type, self.vocoder_data_type
)));
}
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs
index 5677cdaec..a37a2830d 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs
@@ -1,7 +1,7 @@
use std::ops::Range;
use super::*;
-use crate::array::Array;
+use crate::{array::Array, backends::common::gpu_types::ActivationType};
pub(super) fn array_batch_byte_range<B: Backend>(
array: &Array<B>,
@@ -91,6 +91,7 @@ pub(super) struct StructuredAudioNorm<B: Backend> {
#[derive(Debug)]
pub(super) struct StructuredAudioConvNeXt<B: Backend> {
+ pub(super) activation_type: ActivationType,
pub(super) depthwise_conv: StructuredAudioConv1d<B>,
pub(super) norm: StructuredAudioNorm<B>,
pub(super) pwconv1: StructuredAudioPointwiseConv<B>,
diff --git a/crates/backend-uzu/src/backends/common/activation_config.rs b/crates/backend-uzu/src/backends/common/activation_config.rs
index 499ddab07..3903d2609 100644
--- a/crates/backend-uzu/src/backends/common/activation_config.rs
+++ b/crates/backend-uzu/src/backends/common/activation_config.rs
@@ -22,11 +22,14 @@ impl ActivationConfig {
pub fn act_type(&self) -> ActivationType {
match self {
ActivationConfig::SILU {
- ..
+ alpha: _,
} => ActivationType::SILU,
ActivationConfig::GELU {
- ..
- } => ActivationType::GELU,
+ approximate: true,
+ } => ActivationType::GELUApprox,
+ ActivationConfig::GELU {
+ approximate: false,
+ } => ActivationType::GELUExact,
ActivationConfig::IDENTITY => ActivationType::IDENTITY,
}
}
@@ -37,7 +40,7 @@ impl ActivationConfig {
alpha,
} => *alpha,
ActivationConfig::GELU {
- ..
+ approximate: _,
} => 1.0,
ActivationConfig::IDENTITY => 1.0,
}
diff --git a/crates/backend-uzu/src/backends/common/command_buffer.rs b/crates/backend-uzu/src/backends/common/command_buffer.rs
index a4190b047..1f8a84648 100644
--- a/crates/backend-uzu/src/backends/common/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/common/command_buffer.rs
@@ -75,13 +75,13 @@ pub trait CommandBufferEncoding {
Dst: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>,
>(
&mut self,
- src: BufferRangeRef<'_, Src>,
- dst: BufferRangeMut<'_, Dst>,
+ src: BufferRangeRef<Src>,
+ dst: BufferRangeMut<Dst>,
);
fn encode_fill<Dst: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>>(
&mut self,
- dst: BufferRangeMut<'_, Dst>,
+ dst: BufferRangeMut<Dst>,
value: u8,
);
diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rs
index 9e429ea7e..466fc3414 100644
--- a/crates/backend-uzu/src/backends/common/context.rs
+++ b/crates/backend-uzu/src/backends/common/context.rs
@@ -10,7 +10,7 @@ pub trait Context: Sized {
fn recommended_async_batch_size(
&self,
model_path: &Path,
- ) -> usize;
+ ) -> Result<usize, <Self::Backend as Backend>::Error>;
fn is_high_performance(&self) -> bool;
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/activation_type.rs b/crates/backend-uzu/src/backends/common/gpu_types/activation_type.rs
index 7524bff9b..654236249 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/activation_type.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/activation_type.rs
@@ -7,7 +7,8 @@ const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum ActivationType {
SILU,
- GELU,
+ GELUApprox,
+ GELUExact,
TANH,
IDENTITY,
SOFTPLUS,
@@ -19,10 +20,9 @@ impl ActivationType {
x: T,
) -> T {
match self {
- ActivationType::SILU {
- ..
- } => silu(x),
- ActivationType::GELU => gelu(x),
+ ActivationType::SILU => silu(x),
+ ActivationType::GELUApprox => gelu_approx(x),
+ ActivationType::GELUExact => gelu_exact(x),
ActivationType::TANH => tanh_activation(x),
ActivationType::IDENTITY => x,
ActivationType::SOFTPLUS => softplus(x),
@@ -43,13 +43,19 @@ fn silu<T: Float>(x: T) -> T {
activation_silu_alpha(x, 1.0f32)
}
-fn gelu<T: Float>(x: T) -> T {
+fn gelu_approx<T: Float>(x: T) -> T {
let x_float = x.to_f32().unwrap();
let tan_arg = GELU_K1 * (x_float + GELU_K0 * x_float * x_float * x_float);
let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());
T::from(y_float).unwrap()
}
+fn gelu_exact<T: Float>(x: T) -> T {
+ let x_float = x.to_f32().unwrap();
+ let y_float = 0.5f32 * x_float * (1.0f32 + libm::erff(x_float * std::f32::consts::FRAC_1_SQRT_2));
+ T::from(y_float).unwrap()
+}
+
fn tanh_activation<T: Float>(x: T) -> T {
let x_float = x.to_f32().unwrap();
T::from(x_float.tanh()).unwrap()
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs
new file mode 100644
index 000000000..d29a63a54
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs
@@ -0,0 +1,6 @@
+#[repr(C)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum HadamardTransformOrder {
+ Input,
+ Output,
+}
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs
index 8d39a391d..4be07fa23 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs
@@ -7,6 +7,7 @@ pub mod activation_type;
pub mod argmax;
pub mod attention;
pub mod gemm;
+pub mod hadamard_order;
pub mod kv_cache_update;
pub mod matmul;
pub mod quantization;
@@ -17,6 +18,7 @@ pub mod trie;
pub use activation_type::*;
pub use argmax::*;
pub use attention::*;
+pub use hadamard_order::*;
pub use kv_cache_update::*;
pub use matmul::*;
pub use quantization::*;
diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
index 7cb79d790..0e81bcd14 100644
--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
@@ -70,8 +70,8 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
fn encode_copy<Src: Buffer<Backend = Cpu>, Dst: Buffer<Backend = Cpu>>(
&mut self,
- src: BufferRangeRef<'_, Src>,
- dst: BufferRangeMut<'_, Dst>,
+ src: BufferRangeRef<Src>,
+ dst: BufferRangeMut<Dst>,
) {
let src_range = src.range();
let dst_range = dst.range();
@@ -90,7 +90,7 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
fn encode_fill<Dst: Buffer<Backend = Cpu>>(
&mut self,
- dst: BufferRangeMut<'_, Dst>,
+ dst: BufferRangeMut<Dst>,
value: u8,
) {
let range = dst.range();
@@ -172,7 +172,7 @@ impl CommandBufferPending for CpuCommandBufferPending {
type CommandBuffer = CpuCommandBuffer;
fn wait_until_completed(self) -> Result<CpuCommandBufferCompleted, CpuError> {
- self.return_receiver.recv().map_err(CpuError::CommandBufferExecutionFailed)
+ Ok(self.return_receiver.recv()?)
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rs
index deeb3edfc..b252bc06c 100644
--- a/crates/backend-uzu/src/backends/cpu/context.rs
+++ b/crates/backend-uzu/src/backends/cpu/context.rs
@@ -30,10 +30,9 @@ impl Context for CpuContext {
fn recommended_async_batch_size(
&self,
- model_path: &Path,
- ) -> usize {
- let _ = model_path;
- 1
+ _model_path: &Path,
+ ) -> Result<usize, CpuError> {
+ Ok(1)
}
fn is_high_performance(&self) -> bool {
diff --git a/crates/backend-uzu/src/backends/cpu/error.rs b/crates/backend-uzu/src/backends/cpu/error.rs
index 5b5ca0821..581d30aa0 100644
--- a/crates/backend-uzu/src/backends/cpu/error.rs
+++ b/crates/backend-uzu/src/backends/cpu/error.rs
@@ -9,7 +9,7 @@ pub enum CpuError {
#[error("Not supported")]
NotSupported,
#[error("Command buffer execution failed")]
- CommandBufferExecutionFailed(RecvError),
+ CommandBufferExecutionFailed(#[from] RecvError),
#[error("Kernel dispatch failed: {0}")]
KernelDispatchFailed(#[source] Box<dyn StdError + Send + Sync + 'static>),
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
index a6c599203..7b5d9fe33 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
@@ -20,7 +20,7 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(
#[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,
#[optional(is_trie)] trie: Option<*const TrieNode>,
#[optional(is_sliding_window)] sliding_window_size: Option<u32>,
- #[optional(has_sinks)] sinks: Option<*const f32>,
+ #[optional(has_sinks)] sinks: Option<*const T>,
num_heads: u32,
suffix_length: u32,
#[specialize] align_q: bool,
@@ -72,7 +72,7 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(
// Initialize with attention sinks if present
if has_sinks {
- max_score = unsafe { *sinks.unwrap().add(head_idx) };
+ max_score = unsafe { *sinks.unwrap().add(head_idx) }.to_f32().unwrap();
sum_exp = 1.0;
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
index bf9af43be..b1e77d62f 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
@@ -25,7 +25,7 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(
scale: f32,
#[optional(is_trie)] trie: Option<*const TrieNode>,
#[optional(is_sliding_window)] sliding_window_size: Option<u32>,
- #[optional(has_sinks)] sinks: Option<*const f32>,
+ #[optional(has_sinks)] sinks: Option<*const T>,
num_heads: u32,
suffix_length: u32,
#[specialize] has_sinks: bool,
@@ -72,7 +72,7 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(
let mut sum_exp_score = 0.0f32;
if has_sinks {
let q_head_idx = head_idx % num_heads;
- max_score = unsafe { *sinks.unwrap().add(q_head_idx as usize) };
+ max_score = unsafe { *sinks.unwrap().add(q_head_idx as usize) }.to_f32().unwrap();
sum_exp_score = 1.0;
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
index 8ca28bf61..1f8d0b250 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
@@ -31,7 +31,7 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
suffix_length: u32,
#[optional(is_trie)] trie: Option<*const TrieNode>,
#[optional(is_sliding_window)] sliding_window_size: Option<u32>,
- #[optional(has_sinks)] sinks: Option<*const f32>,
+ #[optional(has_sinks)] sinks: Option<*const T>,
#[specialize] has_sinks: bool,
#[specialize] is_kv_cache_ring: bool,
#[specialize] is_causal: bool,
@@ -79,7 +79,7 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
let mut sum_exp_score = 0.0f32;
if has_sinks && block_idx == 0 {
- max_score = unsafe { *sinks.unwrap().add(head_idx as usize) };
+ max_score = unsafe { *sinks.unwrap().add(head_idx as usize) }.to_f32().unwrap();
sum_exp_score = 1.0;
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
index f63a9dae9..97e257a8a 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -2,7 +2,10 @@ use half::{bf16, f16};
use num_traits::Float;
use proc_macros::kernel;
-use crate::{ArrayElement, backends::common::gpu_types::QuantizationMode};
+use crate::{
+ ArrayElement,
+ backends::common::gpu_types::{QuantizationMethod, QuantizationMode},
+};
#[kernel(QuantizedEmbeddingLookup)]
#[variants(T, f32, f16, bf16)]
@@ -10,8 +13,16 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
#[allow(unused)] token_ids: *const u64,
#[allow(unused)] weights: *const u8,
#[allow(unused)] scales: *const T,
- #[allow(unused)] biases: *const T,
+ #[allow(unused)]
+ #[optional(quantization_method == QuantizationMethod::ScaleZeroPoint)]
+ zero_points: Option<*const u8>,
+ #[allow(unused)]
+ #[optional(quantization_method == QuantizationMethod::ScaleBias)]
+ biases: Option<*const T>,
#[allow(unused)] output: *mut T,
+ #[allow(unused)]
+ #[optional(use_hadamard)]
+ output_hadamard_factors: Option<*const i32>,
#[allow(unused)] batch_size: u32,
#[allow(unused)] vocab_size: u32,
#[allow(unused)] model_dim: u32,
@@ -22,10 +33,24 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
#[allow(unused)]
#[specialize]
quantization_mode: QuantizationMode,
+ #[allow(unused)]
+ #[specialize]
+ quantization_method: QuantizationMethod,
+ #[allow(unused)]
+ #[specialize]
+ use_hadamard: bool,
) {
+ if use_hadamard {
+ unimplemented!("not supported yet");
+ }
+
let packing_divisor = quantization_mode.packing_divisor() as u32;
let weights_stride = model_dim / packing_divisor;
let num_groups = (model_dim + group_size - 1) / group_size;
+ let zero_points_stride = match quantization_mode {
+ QuantizationMode::U4 => num_groups.div_ceil(2),
+ QuantizationMode::I8 | QuantizationMode::U8 => num_groups,
+ };
unsafe {
for batch_idx in 0..batch_size {
@@ -41,7 +66,6 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
let group_idx = dim_idx / group_size;
let scale = *scales.add((token_id as u32 * num_groups + group_idx) as usize);
- let bias = *biases.add((token_id as u32 * num_groups + group_idx) as usize);
let quantized_value: i32 = match quantization_mode {
QuantizationMode::U4 => {
@@ -64,7 +88,34 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
},
};
- let out_f = scale.to_f32().unwrap() * quantized_value as f32 + bias.to_f32().unwrap();
+ let bias = match quantization_method {
+ QuantizationMethod::ScaleZeroPoint => {
+ let zero_points = zero_points.expect("ScaleZeroPoint quantized embedding requires zero_points");
+ let zero_point = match quantization_mode {
+ QuantizationMode::U4 => {
+ let byte_idx = (token_id as u32 * zero_points_stride + group_idx / 2) as usize;
+ let packed = *zero_points.add(byte_idx);
+ if (group_idx & 1) == 0 {
+ packed & 0x0F
+ } else {
+ (packed >> 4) & 0x0F
+ }
+ },
+ QuantizationMode::I8 | QuantizationMode::U8 => {
+ *zero_points.add((token_id as u32 * zero_points_stride + group_idx) as usize)
+ },
+ };
+ -scale.to_f32().unwrap() * zero_point as f32
+ },
+ QuantizationMethod::ScaleBias => biases
+ .expect("ScaleBias quantized embedding requires biases")
+ .add((token_id as u32 * num_groups + group_idx) as usize)
+ .read()
+ .to_f32()
+ .unwrap(),
+ };
+
+ let out_f = scale.to_f32().unwrap() * quantized_value as f32 + bias;
let out_f = out_f * input_scale;
*output.add(out_idx) = T::from(out_f).unwrap();
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
index 53a0f82d1..94345bdd5 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
@@ -2,7 +2,7 @@ use half::{bf16, f16};
use num_traits::Float;
use proc_macros::kernel;
-use crate::ArrayElement;
+use crate::{ArrayElement, backends::common::gpu_types::HadamardTransformOrder};
#[kernel(HadamardTransform)]
#[variants(T, f32, f16, bf16)]
@@ -11,6 +11,9 @@ pub fn hadamard_transform_mul<T: ArrayElement + Float>(
#[allow(unused)] factors: *const i32,
#[allow(unused)] hidden_dim: u32,
#[allow(unused)] batch_size: u32,
+ #[allow(unused)]
+ #[specialize]
+ transform_order: HadamardTransformOrder,
) {
todo!()
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 58094e700..178acc315 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -5,7 +5,7 @@ use crate::{
backends::{
common::{
Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,
- gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},
+ gpu_types::{HadamardTransformOrder, QuantizationMethod, QuantizationMode, gemm::GemmDTransform},
kernel::{
HadamardTransformKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, TensorAddBiasKernel,
matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel},
@@ -34,10 +34,12 @@ impl MatmulKernel for MatmulCpuKernel {
if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());
}
- let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)
- .map_err(CpuError::from)?;
- let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, true)
- .map_err(CpuError::from)?;
+ let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(
+ context,
+ data_type,
+ HadamardTransformOrder::Output,
+ )?;
+ let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, true)?;
Ok(Self {
data_type,
hadamard,
@@ -53,13 +55,13 @@ impl MatmulKernel for MatmulCpuKernel {
match arguments.b {
MatmulB::FullPrecision {
..
- } => self.encode_fp(arguments, encoder).map_err(CpuError::from),
+ } => Ok(self.encode_fp(arguments, encoder)?),
MatmulB::ScaleBiasDequant {
..
}
| MatmulB::ScaleZeroPointDequant {
..
- } => self.encode_quant(arguments, encoder).map_err(CpuError::from),
+ } => Ok(self.encode_quant(arguments, encoder)?),
}
}
}
@@ -241,6 +243,10 @@ impl MatmulCpuKernel {
let post_bias = arguments.d_transform.bias;
if arguments.m >= 5 && arguments.n > 1 {
+ assert!(
+ !(post_bias.is_some() && post_rht.is_some()),
+ "MatmulCpuKernel/Quant GEMM with both output bias and output RHT is not supported: bias must be applied after RHT",
+ );
let MatmulArguments {
a,
a_offset,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/moe/experts_single.rs b/crates/backend-uzu/src/backends/cpu/kernel/moe/experts_single.rs
index 40948e2f9..261ed6a4d 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/moe/experts_single.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/moe/experts_single.rs
@@ -65,7 +65,7 @@ pub fn moe_experts_decode_single_pass_a<T: ArrayElement + Float>(
let up_val = (acc_up + unsafe { *biases.add(bias_base + h_idx) }.to_f32().unwrap())
.clamp(up_clip_min, up_clip_max);
if gating_sel == 0 {
- ActivationType::GELU.activate(up_val)
+ ActivationType::GELUApprox.activate(up_val)
} else {
activation_silu_alpha(up_val, silu_alpha)
}
@@ -77,7 +77,7 @@ pub fn moe_experts_decode_single_pass_a<T: ArrayElement + Float>(
let gate_act = if gating_sel == 2 {
activation_silu_alpha(gate_val, silu_alpha)
} else {
- ActivationType::GELU.activate(gate_val)
+ ActivationType::GELUApprox.activate(gate_val)
};
gate_act * up_val
};
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs b/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs
index a57068d9f..76a125e9a 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs
@@ -4,33 +4,38 @@ use proc_macros::kernel;
use crate::ArrayElement;
-fn get_paired_val<T: ArrayElement + Float>(
- values: *const T,
+fn get_paired_val<ElementT: ArrayElement + Float>(
+ values: *const ElementT,
dim_index: usize,
half_dim: usize,
token_index: usize,
total_heads: usize,
head_dim: usize,
head_index: usize,
-) -> T {
+) -> f32 {
if dim_index < half_dim {
let v =
unsafe { *values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index + half_dim) };
- T::zero() - v
+ -v.to_f32().unwrap()
} else {
- unsafe { *values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index - half_dim) }
+ unsafe {
+ (*values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index - half_dim))
+ .to_f32()
+ .unwrap()
+ }
}
}
#[kernel(Rope)]
-#[variants(T, f32, f16, bf16)]
-pub fn rope<T: ArrayElement + Float>(
- qkv: *const T,
- cosines: *const T,
- sines: *const T,
+#[variants(ElementT, f32, f16, bf16)]
+#[variants(RopeT, f32, f16, bf16)]
+pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(
+ qkv: *const ElementT,
+ cosines: *const RopeT,
+ sines: *const RopeT,
token_positions: *const i32,
- rotated_queries: *mut T,
- rotated_keys: *mut T,
+ rotated_queries: *mut ElementT,
+ rotated_keys: *mut ElementT,
head_dim: u32,
rope_dim: u32,
num_heads: u32,
@@ -66,25 +71,30 @@ pub fn rope<T: ArrayElement + Float>(
// Rotated dimensions: apply RoPE to dims 0..rope_dim
for dim_index in 0..rope_dim {
- let cos_val = unsafe { *cosines.add(absolute_position * rope_dim + dim_index) };
- let sin_val = unsafe { *sines.add(absolute_position * rope_dim + dim_index) };
+ let cos_val = unsafe { (*cosines.add(absolute_position * rope_dim + dim_index)).to_f32().unwrap() };
+ let sin_val = unsafe { (*sines.add(absolute_position * rope_dim + dim_index)).to_f32().unwrap() };
// Query rotation
- let q_val =
- unsafe { *qkv.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index) };
+ let q_val = unsafe {
+ (*qkv.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index))
+ .to_f32()
+ .unwrap()
+ };
let q_paired =
get_paired_val(qkv, dim_index, half_rope_dim, token_index, total_heads, head_dim, head_index);
let q_result = q_val * cos_val + q_paired * sin_val;
unsafe {
*rotated_queries.add(head_index * suffix_length * head_dim + token_index * head_dim + dim_index) =
- q_result;
+ ElementT::from(q_result).unwrap();
}
// Key rotation (only first head in each group)
if head_index == group_index * heads_per_group {
let key_head_index = num_heads + group_index;
let k_val = unsafe {
- *qkv.add(token_index * total_heads * head_dim + key_head_index * head_dim + dim_index)
+ (*qkv.add(token_index * total_heads * head_dim + key_head_index * head_dim + dim_index))
+ .to_f32()
+ .unwrap()
};
let k_paired = get_paired_val(
qkv,
@@ -99,7 +109,7 @@ pub fn rope<T: ArrayElement + Float>(
unsafe {
*rotated_keys
.add(group_index * suffix_length * head_dim + token_index * head_dim + dim_index) =
- k_result;
+ ElementT::from(k_result).unwrap();
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs b/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs
index 0d3947ddc..d484b2a57 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs
@@ -8,7 +8,7 @@ use crate::ArrayElement;
#[variants(T, f32, f16, bf16)]
pub fn softmax<T: ArrayElement + Float>(
values: *mut T,
- #[optional(has_sinks)] sinks: Option<*const f32>,
+ #[optional(has_sinks)] sinks: Option<*const T>,
row_length: u32,
outer_dim: u32,
batch_dim: u32,
@@ -18,7 +18,7 @@ pub fn softmax<T: ArrayElement + Float>(
for batch_index in 0..batch_dim {
let row_offset = ((outer_index * batch_dim + batch_index) * row_length) as usize;
let sink = if has_sinks {
- unsafe { *sinks.unwrap().add(outer_index as usize) }
+ unsafe { *sinks.unwrap().add(outer_index as usize) }.to_f32().unwrap()
} else {
f32::NEG_INFINITY
};
diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rs
index a55e6dc1a..3b939927c 100644
--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs
@@ -130,8 +130,8 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
fn encode_copy<Src: Buffer<Backend = Metal>, Dst: Buffer<Backend = Metal>>(
&mut self,
- src: BufferRangeRef<'_, Src>,
- dst: BufferRangeMut<'_, Dst>,
+ src: BufferRangeRef<Src>,
+ dst: BufferRangeMut<Dst>,
) {
let src_range = src.range();
let dst_range = dst.range();
@@ -148,7 +148,7 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
fn encode_fill<Dst: Buffer<Backend = Metal>>(
&mut self,
- dst: BufferRangeMut<'_, Dst>,
+ dst: BufferRangeMut<Dst>,
value: u8,
) {
let range = dst.range();
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index ef90189b1..c4d9af74c 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -160,17 +160,17 @@ impl Context for MetalContext {
fn recommended_async_batch_size(
&self,
model_path: &Path,
- ) -> usize {
+ ) -> Result<usize, MetalError> {
let cores = self.device_capabilities.gpu_core_count;
- let model_size = ModelSize::from_path(model_path);
- match (model_size, cores) {
+ let model_size = ModelSize::from_path(model_path)?;
+ Ok(match (model_size, cores) {
(ModelSize::Large, c) if c > 20 => 32,
(ModelSize::Large, c) if c > 10 => 16,
(ModelSize::Large, _) => 8,
(ModelSize::Small, c) if c > 20 => 256,
(ModelSize::Small, c) if c > 10 => 128,
(ModelSize::Small, _) => 64,
- }
+ })
}
fn debug_active(&self) -> bool {
diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rs
index 2d3c1e4ed..099eb48ef 100644
--- a/crates/backend-uzu/src/backends/metal/error.rs
+++ b/crates/backend-uzu/src/backends/metal/error.rs
@@ -19,6 +19,8 @@ pub enum MetalError {
CannotCreateCommandQueue,
#[error("Cannot create command Metal 4 queue")]
CannotCreateCommandQueueMtl4,
+ #[error("Cannot read model weights metadata: {0}")]
+ CannotReadModelWeightsMetadata(#[from] std::io::Error),
#[error("Cannot create buffer")]
CannotCreateBuffer,
#[error("Cannot create command buffer")]
diff --git a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h
index b05b638c4..ede7a07cd 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h
@@ -28,6 +28,40 @@ inline T activate_gelu(T x) {
return T(yf);
}
+inline float erf_precise(float a) {
+ // Faithfully rounded erff polynomial from Norbert Juffa, also used by MLX.
+ float t = metal::abs(a);
+ float s = a * a;
+
+ if (t > 0.927734375f) {
+ float r = metal::fma(-1.72853470e-5f, t, 3.83197126e-4f);
+ float u = metal::fma(-3.88396438e-3f, t, 2.42546219e-2f);
+ r = metal::fma(r, s, u);
+ r = metal::fma(r, t, -1.06777877e-1f);
+ r = metal::fma(r, t, -6.34846687e-1f);
+ r = metal::fma(r, t, -1.28717512e-1f);
+ r = metal::fma(r, t, -t);
+ r = 1.0f - metal::precise::exp(r);
+ return metal::copysign(r, a);
+ }
+
+ float r = -5.96761703e-4f;
+ r = metal::fma(r, s, 4.99119423e-3f);
+ r = metal::fma(r, s, -2.67681349e-2f);
+ r = metal::fma(r, s, 1.12819925e-1f);
+ r = metal::fma(r, s, -3.76125336e-1f);
+ r = metal::fma(r, s, 1.28379166e-1f);
+ return metal::fma(r, a, a);
+}
+
+template <typename T>
+inline T activate_gelu_exact(T x) {
+ constexpr float inv_sqrt_2 = 0.7071067811865475f;
+ float xf = float(x);
+ float yf = 0.5f * xf * (1.0f + erf_precise(inv_sqrt_2 * xf));
+ return T(yf);
+}
+
template <typename T>
inline T activate_leaky_relu(T x, float alpha) {
const float xf = float(x);
@@ -56,8 +90,10 @@ inline T activate(T x, ActivationType type) {
switch (type) {
case ActivationType::SILU:
return activate_silu(x);
- case ActivationType::GELU:
+ case ActivationType::GELUApprox:
return activate_gelu(x);
+ case ActivationType::GELUExact:
+ return activate_gelu_exact(x);
case ActivationType::TANH:
return activate_tanh(x);
case ActivationType::IDENTITY:
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index fd40c931f..33d1ed248 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -54,7 +54,7 @@ PUBLIC KERNEL(AttentionGemm)(
const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),
const device TrieNode* trie OPTIONAL(is_trie),
const constant uint& sliding_window_size OPTIONAL(is_sliding_window),
- const device float* sinks OPTIONAL(has_sinks),
+ const device T* sinks OPTIONAL(has_sinks),
const constant uint& num_heads,
const constant uint& suffix_length,
const bool align_q SPECIALIZE,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index e9c89c2f7..221d1ed3d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -30,7 +30,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
const constant float& scale,
const device TrieNode* trie OPTIONAL(is_trie),
const constant uint& sliding_window_size OPTIONAL(is_sliding_window),
- const device float* sinks OPTIONAL(has_sinks),
+ const device T* sinks OPTIONAL(has_sinks),
const constant uint& num_heads,
const constant uint& suffix_length,
threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index 713629426..5f4b95300 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -39,7 +39,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
const constant uint& suffix_length,
const device TrieNode* trie OPTIONAL(is_trie),
const constant uint& sliding_window_size OPTIONAL(is_sliding_window),
- const device float* sinks OPTIONAL(has_sinks),
+ const device T* sinks OPTIONAL(has_sinks),
const bool has_sinks SPECIALIZE,
const bool is_kv_cache_ring SPECIALIZE,
const bool is_causal SPECIALIZE,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index d0852fb1d..14a6b81fc 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -1,7 +1,10 @@
#include <metal_stdlib>
#include "../common/dsl.h"
+#include "../generated/quantization_method.h"
+#include "../hadamard_transform/hadamard_transform.h"
#include "quantization.h"
+using namespace uzu::quantization_method;
using namespace uzu::quantization;
template <typename T>
@@ -10,16 +13,20 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
const device uint64_t* token_ids, // [batch_size]
const device uint8_t* weights, // [vocab_size, model_dim/packing_divisor] packed
const device T* scales, // [vocab_size, num_groups]
- const device T* biases, // [vocab_size, num_groups]
+ const device uint8_t* zero_points OPTIONAL(quantization_method == QuantizationMethod::ScaleZeroPoint),
+ const device T* biases OPTIONAL(quantization_method == QuantizationMethod::ScaleBias),
device T* output, // [batch_size, model_dim]
+ const device int32_t* output_hadamard_factors OPTIONAL(use_hadamard),
constant uint32_t& batch_size,
constant uint32_t& vocab_size,
constant uint32_t& model_dim,
constant float& input_scale,
const uint32_t group_size SPECIALIZE,
const QuantizationMode quantization_mode SPECIALIZE,
- const uint dim_idx AXIS(model_dim, 16),
- const uint batch_idx AXIS(batch_size, 16)
+ const QuantizationMethod quantization_method SPECIALIZE,
+ const bool use_hadamard SPECIALIZE,
+ const uint dim_idx AXIS(model_dim, 32),
+ const uint batch_idx AXIS(batch_size, 1)
) {
const uint thread_position_in_grid = batch_idx * model_dim + dim_idx;
const uint64_t token_id = token_ids[batch_idx];
@@ -30,9 +37,11 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
const uint group_idx = dim_idx / group_size;
const uint num_groups = (model_dim + group_size - 1) / group_size;
+ const uint zp_stride = quantization_mode == QuantizationMode::U4
+ ? (num_groups + 1) / 2
+ : num_groups;
const T scale = scales[token_id * num_groups + group_idx];
- const T bias = biases[token_id * num_groups + group_idx];
const uint packing_divisor =
quantization_mode == QuantizationMode::U4 ? 2 : 1;
@@ -64,7 +73,30 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
};
}
- float out_f = float(scale) * float(quantized_value) + float(bias);
+ float bias = 0.0f;
+ if (quantization_method == QuantizationMethod::ScaleBias) {
+ bias = float(biases[token_id * num_groups + group_idx]);
+ } else {
+ uint zero_point = 0;
+ if (quantization_mode == QuantizationMode::U4) {
+ const uint8_t packed = zero_points[token_id * zp_stride + group_idx / 2];
+ zero_point =
+ (group_idx & 1) == 0 ? (packed & 0x0F) : ((packed >> 4) & 0x0F);
+ } else {
+ zero_point = zero_points[token_id * zp_stride + group_idx];
+ }
+ bias = -float(scale) * float(zero_point);
+ }
+
+ float out_f = float(scale) * float(quantized_value) + bias;
out_f *= input_scale;
- output[thread_position_in_grid] = T(out_f);
+ T out = T(out_f);
+ if (use_hadamard) {
+ out = simdgroup_output_random_hadamard_transform(
+ static_cast<ushort>(dim_idx % METAL_SIMD_SIZE),
+ out,
+ output_hadamard_factors[dim_idx]
+ );
+ }
+ output[thread_position_in_grid] = out;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h
index eda99e0a2..60b21b5de 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_type.h
@@ -7,9 +7,10 @@ using namespace metal;
namespace uzu::activation_type {
enum class ActivationType : uint32_t {
SILU = 0,
- GELU = 1,
- TANH = 2,
- IDENTITY = 3,
- SOFTPLUS = 4,
+ GELUApprox = 1,
+ GELUExact = 2,
+ TANH = 3,
+ IDENTITY = 4,
+ SOFTPLUS = 5,
};
} // namespace uzu::activation_type
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h
new file mode 100644
index 000000000..e9f7bd50c
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h
@@ -0,0 +1,12 @@
+// Auto-generated from gpu_types/hadamard_order - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::hadamard_order {
+enum class HadamardTransformOrder : uint32_t {
+ Input = 0,
+ Output = 1,
+};
+} // namespace uzu::hadamard_order
diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
index d5f62b642..314628029 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
@@ -18,7 +18,7 @@ static METAL_FUNC float simdgroup_hadamard_transform(
}
template <typename T>
-static METAL_FUNC T simdgroup_random_hadamard_transform(
+static METAL_FUNC T simdgroup_input_random_hadamard_transform(
ushort lane_index,
T lane_value,
int32_t lane_factor
@@ -28,3 +28,15 @@ static METAL_FUNC T simdgroup_random_hadamard_transform(
float(lane_value) * float(lane_factor)
));
}
+
+template <typename T>
+static METAL_FUNC T simdgroup_output_random_hadamard_transform(
+ ushort lane_index,
+ T lane_value,
+ int32_t lane_factor
+) {
+ return T(
+ simdgroup_hadamard_transform(lane_index, float(lane_value)) *
+ float(lane_factor)
+ );
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
index 4056b73b5..78b038b02 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
@@ -1,9 +1,11 @@
#include <metal_stdlib>
#include "../common/defines.h"
#include "../common/dsl.h"
+#include "../generated/hadamard_order.h"
#include "hadamard_transform.h"
using namespace metal;
+using namespace uzu::hadamard_order;
template <typename T>
VARIANTS(T, half, bfloat, float)
@@ -12,6 +14,7 @@ PUBLIC KERNEL(HadamardTransform)(
const device int32_t* factors,
constant uint& hidden_dim,
constant uint& batch_size,
+ const HadamardTransformOrder transform_order SPECIALIZE,
uint block_index GROUPS(hidden_dim.div_ceil(METAL_SIMD_SIZE)),
uint batch_index GROUPS(batch_size),
uint lane_index THREADS(METAL_SIMD_SIZE)
@@ -19,9 +22,17 @@ PUBLIC KERNEL(HadamardTransform)(
uint factor_index = block_index * METAL_SIMD_SIZE + lane_index;
uint element_index = batch_index * hidden_dim + factor_index;
- data[element_index] = simdgroup_random_hadamard_transform(
- lane_index,
- data[element_index],
- factors[factor_index]
- );
+ if (transform_order == HadamardTransformOrder::Input) {
+ data[element_index] = simdgroup_input_random_hadamard_transform(
+ lane_index,
+ data[element_index],
+ factors[factor_index]
+ );
+ } else {
+ data[element_index] = simdgroup_output_random_hadamard_transform(
+ lane_index,
+ data[element_index],
+ factors[factor_index]
+ );
+ }
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index cc5650f78..d56741462 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -7,7 +7,7 @@ use crate::{
common::{
Allocation, AsBufferRangeRef, Backend, Buffer, Encoder,
gpu_types::{
- GemmParams,
+ GemmParams, HadamardTransformOrder,
gemm::{GemmAlignment, GemmDTransform, GemmTiling},
},
kernel::{
@@ -44,7 +44,11 @@ impl GemmKernel {
data_type: DataType,
) -> Result<Self, MetalError> {
let bias_add = TensorAddBiasMetalKernel::new(context, data_type, true)?;
- let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)?;
+ let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(
+ context,
+ data_type,
+ HadamardTransformOrder::Output,
+ )?;
let mut kernel = Self {
data_type,
kernels: HashMap::new(),
@@ -137,13 +141,18 @@ impl GemmKernel {
let is_quant = matches!(arguments.b, MatmulB::ScaleBiasDequant { .. } | MatmulB::ScaleZeroPointDequant { .. });
if is_quant {
- if arguments.d_transform.mask().contains(GemmDTransform::ACCUMULATE) {
+ let d_mask = arguments.d_transform.mask();
+ if d_mask.contains(GemmDTransform::ACCUMULATE) {
return Err(MatmulError::UnsupportedDOp {
bit: GemmDTransform::ACCUMULATE,
path: "QuantGemm",
}
.into());
}
+ assert!(
+ !d_mask.contains(GemmDTransform::BIAS | GemmDTransform::RHT),
+ "QuantGemm with both output bias and output RHT is not supported: bias must be applied after RHT",
+ );
if !arguments.b_transpose || arguments.b_leading_dimension.is_some() || arguments.b_offset != 0 {
return Err(MatmulError::UnsupportedLayout {
path: "QuantGemm",
@@ -255,7 +264,7 @@ impl GemmKernel {
bits_per_weight,
group_size,
};
- specialization.validate().map_err(MetalError::from)?;
+ specialization.validate()?;
let kernel = self.get_or_create(encoder.context(), specialization)?;
kernel.encode(
(a, a_offset),
@@ -311,7 +320,7 @@ impl GemmKernel {
bits_per_weight,
group_size,
};
- specialization.validate().map_err(MetalError::from)?;
+ specialization.validate()?;
let kernel = self.get_or_create(encoder.context(), specialization)?;
kernel.encode(
(a, a_offset),
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
index a27f1591c..8d0fbfb2c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
@@ -177,7 +177,7 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
if (simd_group == 0) {
uint global_out_idx = out_block_idx * 32 + simd_lane;
if (global_out_idx < out_vec_size) {
- output[simd_lane] = simdgroup_random_hadamard_transform(
+ output[simd_lane] = simdgroup_output_random_hadamard_transform(
static_cast<ushort>(simd_lane),
T(shared_results[simd_lane]),
hadamard_factors[global_out_idx]
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 7756cc248..04e1db85d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -147,7 +147,7 @@ impl MatmulKernel for MatmulMetalKernel {
}
let gemm = GemmKernel::new(context, data_type)?;
- let gemv = GemvKernel::new(context, data_type).map_err(MetalError::from)?;
+ let gemv = GemvKernel::new(context, data_type)?;
let quant_gemv = QuantGemvKernel::new(context, data_type);
Ok(Self {
@@ -174,10 +174,11 @@ impl MatmulKernel for MatmulMetalKernel {
if gemv_eligible {
if is_quant {
- self.dispatch_quant_gemv(arguments, encoder).map_err(MetalError::from)
+ self.dispatch_quant_gemv(arguments, encoder)?;
} else {
- self.dispatch_gemv(arguments, encoder).map_err(MetalError::from)
+ self.dispatch_gemv(arguments, encoder)?;
}
+ Ok(())
} else {
self.gemm.encode(arguments, encoder)
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/backend-uzu/src/backends/metal/kernel/mlp/activation_mul.metal
index 157527261..3aabfcb80 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/mlp/activation_mul.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -25,7 +25,7 @@ PUBLIC KERNEL(MlpGateActMul) (
T result = T(float(up) * float(g));
if (use_hadamard) {
- result = simdgroup_random_hadamard_transform(
+ result = simdgroup_input_random_hadamard_transform(
static_cast<ushort>(j % METAL_SIMD_SIZE),
result,
hadamard_factors[j]
diff --git a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index bc8afaa78..9915a5c20 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -109,7 +109,7 @@ PUBLIC KERNEL(RMSNorm)(
}
if (use_hadamard) {
- val = static_cast<OutputT>(simdgroup_random_hadamard_transform(
+ val = static_cast<OutputT>(simdgroup_input_random_hadamard_transform(
static_cast<ushort>(thread_in_row % METAL_SIMD_SIZE),
val,
hadamard_factors[i]
diff --git a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal
index 0bee9eb34..f3bb1f7ce 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal
@@ -2,32 +2,34 @@
#include "../common/dsl.h"
#include "../common/tensor_view.h"
-template <typename T>
-inline T applyRopeTransform(
- TensorView3D<const T> qkv_tensor_view,
+template <typename ElementT>
+inline ElementT applyRopeTransform(
+ TensorView3D<const ElementT> qkv_tensor_view,
uint token_idx,
uint head_idx,
uint dim_idx,
uint half_dim,
- T cos_val,
- T sin_val
+ float cos_val,
+ float sin_val
) {
- T inputVal = qkv_tensor_view(token_idx, head_idx, dim_idx);
- T pairedVal = (dim_idx < half_dim)
- ? -qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim)
- : qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim);
- return inputVal * cos_val + pairedVal * sin_val;
+ float inputVal = float(qkv_tensor_view(token_idx, head_idx, dim_idx));
+ float pairedVal =
+ (dim_idx < half_dim)
+ ? -float(qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim))
+ : float(qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim));
+ return static_cast<ElementT>(inputVal * cos_val + pairedVal * sin_val);
}
-template <typename T>
-VARIANTS(T, float, half, bfloat)
+template <typename ElementT, typename RopeT>
+VARIANTS(ElementT, float, half, bfloat)
+VARIANTS(RopeT, float, half, bfloat)
PUBLIC KERNEL(Rope)(
- device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]
- device const T* cosines, // [max_seq_len, rope_dim]
- device const T* sines, // [max_seq_len, rope_dim]
+ device const ElementT* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]
+ device const RopeT* cosines, // [max_seq_len, rope_dim]
+ device const RopeT* sines, // [max_seq_len, rope_dim]
device const int* token_positions, // [suffix_len] - actual token positions
- device T* rotated_queries, // [num_heads, suffix_len, head_dim]
- device T* rotated_keys, // [num_groups, suffix_len, head_dim]
+ device ElementT* rotated_queries, // [num_heads, suffix_len, head_dim]
+ device ElementT* rotated_keys, // [num_groups, suffix_len, head_dim]
constant uint& head_dim,
constant uint& rope_dim,
constant uint& num_heads,
@@ -62,26 +64,30 @@ PUBLIC KERNEL(Rope)(
raw_position > max_sequence_length ? 0 : raw_position;
const uint half_rope_dim = rope_dim / 2;
- TensorView3D<const T> qkv_tensor_view =
- TensorView3D<const T>(qkv).shaped(suffix_length, total_heads, head_dim);
- TensorView2D<const T> cosines_tensor_view =
- TensorView2D<const T>(cosines).shaped(max_sequence_length, rope_dim);
- TensorView2D<const T> sines_tensor_view =
- TensorView2D<const T>(sines).shaped(max_sequence_length, rope_dim);
- TensorView3D<T> rotated_queries_tensor_view =
- TensorView3D<T>(rotated_queries)
+ TensorView3D<const ElementT> qkv_tensor_view =
+ TensorView3D<const ElementT>(qkv)
+ .shaped(suffix_length, total_heads, head_dim);
+ TensorView2D<const RopeT> cosines_tensor_view =
+ TensorView2D<const RopeT>(cosines).shaped(max_sequence_length, rope_dim);
+ TensorView2D<const RopeT> sines_tensor_view =
+ TensorView2D<const RopeT>(sines).shaped(max_sequence_length, rope_dim);
+ TensorView3D<ElementT> rotated_queries_tensor_view =
+ TensorView3D<ElementT>(rotated_queries)
.shaped(num_heads, suffix_length, head_dim);
- TensorView3D<T> rotated_keys_tensor_view =
- TensorView3D<T>(rotated_keys).shaped(num_groups, suffix_length, head_dim);
+ TensorView3D<ElementT> rotated_keys_tensor_view =
+ TensorView3D<ElementT>(rotated_keys)
+ .shaped(num_groups, suffix_length, head_dim);
uint first_head_in_group = group_index * (num_heads / num_groups);
/* ---------- QUERIES ---------- */
if (dimension_index < rope_dim) {
- const T cos_val = cosines_tensor_view(absolutePosition, dimension_index);
- const T sin_val = sines_tensor_view(absolutePosition, dimension_index);
+ const float cos_val =
+ float(cosines_tensor_view(absolutePosition, dimension_index));
+ const float sin_val =
+ float(sines_tensor_view(absolutePosition, dimension_index));
- T queryResult = applyRopeTransform(
+ ElementT queryResult = applyRopeTransform(
qkv_tensor_view,
token_index,
head_index,
@@ -97,7 +103,7 @@ PUBLIC KERNEL(Rope)(
if (head_index == first_head_in_group) {
uint key_head_index =
num_heads + group_index; // Keys start after all query heads
- T keyResult = applyRopeTransform(
+ ElementT keyResult = applyRopeTransform(
qkv_tensor_view,
token_index,
key_head_index,
@@ -120,4 +126,4 @@ PUBLIC KERNEL(Rope)(
qkv_tensor_view(token_index, key_head_index, dimension_index);
}
}
-}
\ No newline at end of file
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal
index d53fdd198..a716e7c92 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal
@@ -16,7 +16,7 @@ template <typename T>
VARIANTS(T, float, half, bfloat)
PUBLIC KERNEL(Softmax)(
device T* values,
- const device float* sinks OPTIONAL(has_sinks),
+ const device T* sinks OPTIONAL(has_sinks),
const constant uint& row_length,
const constant uint& outer_dim,
const constant uint& batch_dim,
diff --git a/crates/backend-uzu/src/classifier/classifier.rs b/crates/backend-uzu/src/classifier/classifier.rs
index f9ed6ab53..179571fcb 100644
--- a/crates/backend-uzu/src/classifier/classifier.rs
+++ b/crates/backend-uzu/src/classifier/classifier.rs
@@ -6,7 +6,7 @@ use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
use crate::{
DataType, allocation_to_vec,
backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
- config::{ClassifierModelConfig, ModelMetadata},
+ config::model::classifier_model::ClassifierModelConfig,
encodable_block::LayerArguments,
forward_pass::token_inputs::TokenInputs,
session::types::Error,
@@ -67,9 +67,9 @@ impl<B: Backend> ClassifierTrait for Classifier<B> {
impl<B: Backend> Classifier<B> {
pub fn new(
model_path: &Path,
- model_metadata: &ModelMetadata<ClassifierModelConfig>,
+ model_config: &ClassifierModelConfig,
) -> Result<Self, Error> {
- let context = ClassifierContext::new(model_path, model_metadata)?;
+ let context = ClassifierContext::new(model_path, model_config)?;
Ok(Self {
context,
})
@@ -81,7 +81,7 @@ impl<B: Backend> Classifier<B> {
token_ids: &[u64],
token_positions: &[usize],
) -> Result<(Box<[f32]>, ActivationTrace<B>), Error> {
- let num_labels = self.context.model_config.model_config.num_labels;
+ let num_labels = self.context.model_config.classifier_config.num_labels;
let mut traces = ActivationTrace::new_classifier(
self.context.context.as_ref(),
&self.context.model_shape,
@@ -143,7 +143,6 @@ impl<B: Backend> Classifier<B> {
token_positions: token_inputs.token_positions(),
token_parents: token_inputs.token_parents(),
token_subtrie_ranges: None,
- attention_sinks: self.context.shared_buffers.attention_sinks(layer_index),
rope_buffers,
sampling_start: 0,
sampling_length: batch_dim,
@@ -211,10 +210,7 @@ impl<B: Backend> Classifier<B> {
&self,
logits: &Allocation<B>,
) -> Result<Box<[f32]>, Error> {
- let logits_data_type: DataType =
- self.context.model_config.model_config.prediction_head_config.readout_config.activation_precision().into();
-
- match logits_data_type {
+ match self.context.logits_data_type {
DataType::F32 => Ok(allocation_to_vec::<B, f32>(logits).into()),
DataType::F16 => {
Ok(allocation_to_vec::<B, half::f16>(logits).into_iter().map(|x| x.to_f32()).collect::<Box<[_]>>())
@@ -222,7 +218,9 @@ impl<B: Backend> Classifier<B> {
DataType::BF16 => {
Ok(allocation_to_vec::<B, half::bf16>(logits).into_iter().map(|x| x.to_f32()).collect::<Box<[_]>>())
},
- _ => Err(Error::UnableToDecodeText),
+ data_type => {
+ Err(Error::InvalidModelConfig(format!("unsupported classifier logits data type {data_type:?}")))
+ },
}
}
@@ -230,7 +228,7 @@ impl<B: Backend> Classifier<B> {
&self,
logits: &[f32],
) -> Result<HashMap<String, f32>, Error> {
- let output_labels = &self.context.model_config.model_config.output_labels;
+ let output_labels = &self.context.model_config.classifier_config.output_labels;
let mut probabilities = HashMap::new();
for (idx, &logit) in logits.iter().enumerate() {
diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rs
index f40c01f28..decd22795 100644
--- a/crates/backend-uzu/src/classifier/classifier_context.rs
+++ b/crates/backend-uzu/src/classifier/classifier_context.rs
@@ -1,16 +1,14 @@
use std::{fs::File, path::Path, rc::Rc};
-#[cfg(feature = "tracing")]
-use crate::forward_pass::model_shape::ModelShape;
use crate::{
DataType,
backends::common::{Backend, Context},
classifier::ClassifierError,
- config::{ClassifierModelConfig, ModelMetadata},
+ config::{decoder::DecoderConfig, model::classifier_model::ClassifierModelConfig},
encodable_block::{
ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, QkUnpack, Rope,
},
- forward_pass::state::SharedBuffers,
+ forward_pass::{model_shape::ModelShape, state::SharedBuffers},
parameters::ParameterLoader,
session::types::Error,
};
@@ -31,145 +29,138 @@ pub struct ClassifierContext<B: Backend> {
pub pooling: Pooling<B>,
pub prediction_head: ClassifierPredictionHead<B>,
+ pub logits_data_type: DataType,
}
impl<B: Backend> ClassifierContext<B> {
pub fn new(
model_path: &Path,
- model_metadata: &ModelMetadata<ClassifierModelConfig>,
+ model_config: &ClassifierModelConfig,
) -> Result<Self, Error> {
let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;
+ let classifier_config = &model_config.classifier_config;
- let decoder_config = Rc::new(crate::config::DecoderConfig {
- embedding_config: model_metadata.model_config.model_config.embedding_config.clone(),
- transformer_config: model_metadata.model_config.model_config.transformer_config.clone(),
- vocab_size: model_metadata.model_config.model_config.vocab_size,
+ let decoder_config = Rc::new(DecoderConfig {
+ embedding_config: classifier_config.embedding_config.clone(),
+ transformer_config: classifier_config.transformer_config.clone(),
+ vocab_size: classifier_config.vocab_size,
pard_token: None,
ple_model_config: None,
});
- #[cfg(feature = "tracing")]
- let model_shape = ModelShape::from_decoder_config(&decoder_config);
let weights_path = model_path.join("model.safetensors");
- if !weights_path.exists() {
- return Err(Error::UnableToLoadWeights);
- }
- let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;
- let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
- let root_loader_view = loader.tree();
+ let weights_file = File::open(&weights_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let loader = ParameterLoader::new(&weights_file, context.as_ref())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let root_loader_view =
+ loader.tree().subtree("classifier").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+
+ let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::BF16);
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config);
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
shared_buffers.update_data(&root_loader_view)?;
let shared_buffers = Rc::new(shared_buffers);
- let transformer_tree = root_loader_view
- .subtree("transformer")
- .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("transformer".to_string())))?;
+ let transformer_tree =
+ root_loader_view.subtree("transformer").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- let data_type = decoder_config
- .first_attention()
- .ok_or(Error::Classifier(ClassifierError::NonAttentionMixer))?
- .qkv_projection_config
- .activation_precision()
- .into();
+ let output_norm_tree =
+ transformer_tree.subtree("output_norm").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- let embed = Embedding::new(
+ let (embed, readout_input_hadamard_factors) = Embedding::new(
context.as_ref(),
decoder_config.vocab_size as u32,
decoder_config.transformer_config.model_dim as u32,
&decoder_config.embedding_config,
- &root_loader_view.subtree("embedding").expect("Failed to get embedding subtree"),
+ &root_loader_view.subtree("embedding").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?,
+ &model_shape,
)
- .expect("Failed to create embedding");
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ if readout_input_hadamard_factors.is_some() {
+ return Err(Error::Classifier(ClassifierError::Custom(
+ "classifier does not support embedding readout hadamard".to_string(),
+ )));
+ }
let rope = Rc::new(
- Rope::<B>::new(context.as_ref(), data_type)
+ Rope::<B>::new(context.as_ref(), &model_shape)
.map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e))))?,
);
let qk_unpack = Rc::new(
- QkUnpack::<B>::new(context.as_ref(), data_type)
+ QkUnpack::<B>::new(context.as_ref(), model_shape.data_type)
.map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("QkUnpack: {:?}", e))))?,
);
- let layers = model_metadata
- .model_config
- .model_config
+ let layers = classifier_config
.transformer_config
.layer_configs
.iter()
.enumerate()
.map(|(layer_index, layer_config)| {
- layer_config.mixer_config.as_attention().ok_or(ClassifierError::NonAttentionMixer)?;
-
let layer_tree = transformer_tree
.subtree(&format!("layers.{}", layer_index))
- .map_err(|_| ClassifierError::WeightSubtreeNotFound(format!("layers.{}", layer_index)))?;
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- Ok(ClassifierLayer::new(
+ ClassifierLayer::new(
context.as_ref(),
- &model_metadata.model_config.model_config.transformer_config,
+ &classifier_config.transformer_config,
layer_config,
- layer_index,
&layer_tree,
rope.clone(),
qk_unpack.clone(),
- ))
+ model_shape.data_type,
+ )
+ .map_err(|error| Error::UnableToCreateClassifierLayer(Box::new(error)))
})
- .collect::<Result<Vec<_>, ClassifierError>>()
- .map_err(Error::Classifier)?
+ .collect::<Result<Vec<_>, Error>>()?
.into_boxed_slice();
- let output_norm_tree = transformer_tree
- .subtree("output_norm")
- .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("output_norm".to_string())))?;
let output_norm = Normalization::new(
context.as_ref(),
- data_type,
- model_metadata.model_config.model_config.transformer_config.output_norm_config.clone(),
+ model_shape.data_type,
+ classifier_config.transformer_config.model_dim,
+ classifier_config.transformer_config.output_norm_config.clone(),
&output_norm_tree,
)
.map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("output norm: {:?}", e))))?;
- let embedding_norm_tree = root_loader_view
- .subtree("embedding_norm")
- .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("embedding_norm".to_string())))?;
+ let embedding_norm_tree =
+ root_loader_view.subtree("embedding_norm").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let embedding_norm = Normalization::new(
context.as_ref(),
- data_type,
- model_metadata.model_config.model_config.embedding_norm_config.clone(),
+ model_shape.data_type,
+ classifier_config.transformer_config.model_dim,
+ classifier_config.embedding_norm_config.clone(),
&embedding_norm_tree,
)
.map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("embedding norm: {:?}", e))))?;
- let model_dim = model_metadata.model_config.model_config.model_dim;
- let num_labels = model_metadata.model_config.model_config.num_labels;
- let prediction_head_config = &model_metadata.model_config.model_config.prediction_head_config;
- let prediction_head_tree = root_loader_view
- .subtree("prediction_head")
- .map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head".to_string())))?;
-
- let prediction_head_data_type: DataType = prediction_head_config.dense_config.activation_precision().into();
+ let model_dim = classifier_config.model_dim;
+ let num_labels = classifier_config.num_labels;
+ let prediction_head_config = &classifier_config.prediction_head_config;
+ let prediction_head_tree =
+ root_loader_view.subtree("prediction_head").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))
- })?;
+ let prediction_head_dense_tree =
+ prediction_head_tree.subtree("dense").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let prediction_head_dense = <dyn Linear<B>>::new::<1>(
- &prediction_head_config.dense_config,
model_dim,
[model_dim],
+ prediction_head_config.use_dense_bias,
context.as_ref(),
+ model_shape.data_type,
&prediction_head_dense_tree,
)
.map_err(|e| {
Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))
})?;
- let prediction_head_norm_tree = prediction_head_tree.subtree("norm").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.norm".to_string()))
- })?;
+ let prediction_head_norm_tree =
+ prediction_head_tree.subtree("norm").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let prediction_head_norm = Normalization::new(
context.as_ref(),
- prediction_head_data_type,
+ model_shape.data_type,
+ model_dim,
prediction_head_config.normalization_config.clone(),
&prediction_head_norm_tree,
)
@@ -177,14 +168,14 @@ impl<B: Backend> ClassifierContext<B> {
Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head norm: {:?}", e)))
})?;
- let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {
- Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))
- })?;
+ let prediction_head_readout_tree =
+ prediction_head_tree.subtree("readout").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let prediction_head_final_linear = <dyn Linear<B>>::new::<1>(
- &prediction_head_config.readout_config,
model_dim,
[num_labels],
+ true,
context.as_ref(),
+ model_shape.data_type,
&prediction_head_readout_tree,
)
.map_err(|e| {
@@ -193,20 +184,17 @@ impl<B: Backend> ClassifierContext<B> {
let pooling = Pooling::<B>::new(
context.as_ref(),
- data_type,
- model_metadata.model_config.model_config.classifier_pooling.clone(),
+ model_shape.data_type,
+ classifier_config.classifier_pooling.clone(),
model_dim,
)
- .map_err(|e| {
- eprintln!("Failed to create pooling: {:?}", e);
- Error::UnableToCreateContext(e.into())
- })?;
+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;
let prediction_head = ClassifierPredictionHead::new(
context.as_ref(),
prediction_head_dense,
- prediction_head_config.activation.clone(),
- prediction_head_data_type,
+ prediction_head_config.activation.clone().into(),
+ model_shape.data_type,
prediction_head_norm,
prediction_head_final_linear,
model_dim,
@@ -215,10 +203,13 @@ impl<B: Backend> ClassifierContext<B> {
Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))
})?;
+ loader.tree().assert_all_tensors_validated().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+
Ok(Self {
context,
shared_buffers,
- model_config: model_metadata.model_config.clone(),
+ model_config: model_config.clone(),
+ logits_data_type: model_shape.data_type,
#[cfg(feature = "tracing")]
model_shape,
embed,
diff --git a/crates/backend-uzu/src/classifier/error.rs b/crates/backend-uzu/src/classifier/error.rs
index c650bcb58..1afd24f83 100644
--- a/crates/backend-uzu/src/classifier/error.rs
+++ b/crates/backend-uzu/src/classifier/error.rs
@@ -7,8 +7,6 @@ pub enum ClassifierError {
MissingConfigField(String),
#[error("Model requires attention layers but found non-attention mixer")]
NonAttentionMixer,
- #[error("Weight subtree not found: {0}")]
- WeightSubtreeNotFound(String),
#[error("Kernel creation failed: {0}")]
KernelCreationFailed(String),
#[error("{0}")]
diff --git a/crates/backend-uzu/src/config/activation/gelu.rs b/crates/backend-uzu/src/config/activation/gelu.rs
new file mode 100644
index 000000000..73222e6ae
--- /dev/null
+++ b/crates/backend-uzu/src/config/activation/gelu.rs
@@ -0,0 +1,6 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::Activation)]
+pub struct GELU {
+ pub approximate: bool,
+}
diff --git a/crates/backend-uzu/src/config/activation/identity.rs b/crates/backend-uzu/src/config/activation/identity.rs
new file mode 100644
index 000000000..e5c2c84e6
--- /dev/null
+++ b/crates/backend-uzu/src/config/activation/identity.rs
@@ -0,0 +1,4 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::Activation)]
+pub struct Identity;
diff --git a/crates/backend-uzu/src/config/activation/mod.rs b/crates/backend-uzu/src/config/activation/mod.rs
new file mode 100644
index 000000000..66dfea909
--- /dev/null
+++ b/crates/backend-uzu/src/config/activation/mod.rs
@@ -0,0 +1,48 @@
+use proc_macros::uzu_config_abstract;
+
+use crate::backends::common::{ActivationConfig, gpu_types::ActivationType};
+
+pub mod gelu;
+pub mod identity;
+pub mod silu;
+
+#[uzu_config_abstract(silu::SiLU, gelu::GELU, identity::Identity)]
+pub struct Activation;
+
+impl AnyActivation {
+ pub fn act_type(&self) -> ActivationType {
+ match self {
+ AnyActivation::SiLU(_) => ActivationType::SILU,
+ AnyActivation::GELU(gelu::GELU {
+ approximate: true,
+ ..
+ }) => ActivationType::GELUApprox,
+ AnyActivation::GELU(gelu::GELU {
+ approximate: false,
+ ..
+ }) => ActivationType::GELUExact,
+ AnyActivation::Identity(_) => ActivationType::IDENTITY,
+ }
+ }
+
+ pub fn alpha(&self) -> f32 {
+ match self {
+ AnyActivation::SiLU(config) => config.alpha,
+ AnyActivation::GELU(_) | AnyActivation::Identity(_) => 1.0,
+ }
+ }
+}
+
+impl From<AnyActivation> for ActivationConfig {
+ fn from(value: AnyActivation) -> Self {
+ match value {
+ AnyActivation::SiLU(config) => ActivationConfig::SILU {
+ alpha: config.alpha,
+ },
+ AnyActivation::GELU(config) => ActivationConfig::GELU {
+ approximate: config.approximate,
+ },
+ AnyActivation::Identity(_) => ActivationConfig::IDENTITY,
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/config/activation/silu.rs b/crates/backend-uzu/src/config/activation/silu.rs
new file mode 100644
index 000000000..4dbf2ba15
--- /dev/null
+++ b/crates/backend-uzu/src/config/activation/silu.rs
@@ -0,0 +1,6 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::Activation)]
+pub struct SiLU {
+ pub alpha: f32,
+}
diff --git a/crates/backend-uzu/src/config/classifier.rs b/crates/backend-uzu/src/config/classifier.rs
new file mode 100644
index 000000000..7a4ec8be7
--- /dev/null
+++ b/crates/backend-uzu/src/config/classifier.rs
@@ -0,0 +1,42 @@
+use proc_macros::uzu_config;
+
+use crate::config::{
+ activation::AnyActivation, embedding::AnyEmbeddingConfig, linear::LinearConfig, normalization::NormalizationConfig,
+ transformer::TransformerConfig,
+};
+
+#[uzu_config]
+pub enum PoolingType {
+ #[serde(rename = "cls")]
+ CLS,
+ #[serde(rename = "mean")]
+ Mean,
+}
+
+#[uzu_config]
+pub struct PredictionHeadConfig {
+ pub dense_config: LinearConfig,
+ pub activation: AnyActivation,
+ pub normalization_config: NormalizationConfig,
+ pub readout_config: LinearConfig,
+ pub use_dense_bias: bool,
+}
+
+#[uzu_config]
+pub struct ClassifierConfig {
+ pub embedding_config: AnyEmbeddingConfig,
+ pub embedding_norm_config: NormalizationConfig,
+ pub transformer_config: TransformerConfig,
+ pub prediction_head_config: PredictionHeadConfig,
+ pub readout_config: LinearConfig,
+
+ pub vocab_size: usize,
+ pub model_dim: usize,
+ pub hidden_dim: usize,
+ pub attention_scale: Option<f32>,
+ pub num_layers: usize,
+ pub context_length: usize,
+ pub num_labels: usize,
+ pub classifier_pooling: PoolingType,
+ pub output_labels: Option<Box<[String]>>,
+}
diff --git a/crates/backend-uzu/src/config/classifier/classifier_config.rs b/crates/backend-uzu/src/config/classifier/classifier_config.rs
deleted file mode 100644
index 407a354f2..000000000
--- a/crates/backend-uzu/src/config/classifier/classifier_config.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-use proc_macros::uzu_config;
-
-use super::{PoolingType, PredictionHeadConfig};
-use crate::config::{EmbeddingConfig, LinearConfig, NormalizationConfig, TransformerConfig};
-
-#[uzu_config]
-pub struct ClassifierConfig {
- pub embedding_config: EmbeddingConfig,
- pub embedding_norm_config: NormalizationConfig,
- pub transformer_config: TransformerConfig,
- pub prediction_head_config: PredictionHeadConfig,
- pub readout_config: LinearConfig,
- pub vocab_size: usize,
- pub model_dim: usize,
- pub hidden_dim: usize,
- pub attention_scale: Option<f32>,
- pub num_layers: usize,
- pub context_length: usize,
- pub num_labels: usize,
- pub classifier_pooling: PoolingType,
- pub output_labels: Option<Vec<String>>,
-}
diff --git a/crates/backend-uzu/src/config/classifier/mod.rs b/crates/backend-uzu/src/config/classifier/mod.rs
deleted file mode 100644
index 7084c1314..000000000
--- a/crates/backend-uzu/src/config/classifier/mod.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-mod classifier_config;
-mod pooling_type;
-mod prediction_head_config;
-
-pub use classifier_config::ClassifierConfig;
-pub use pooling_type::PoolingType;
-pub use prediction_head_config::PredictionHeadConfig;
diff --git a/crates/backend-uzu/src/config/classifier/pooling_type.rs b/crates/backend-uzu/src/config/classifier/pooling_type.rs
deleted file mode 100644
index b67357da2..000000000
--- a/crates/backend-uzu/src/config/classifier/pooling_type.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-use proc_macros::uzu_config;
-
-#[uzu_config]
-#[serde(rename_all = "lowercase")]
-pub enum PoolingType {
- Cls,
- Mean,
-}
diff --git a/crates/backend-uzu/src/config/classifier/prediction_head_config.rs b/crates/backend-uzu/src/config/classifier/prediction_head_config.rs
deleted file mode 100644
index 5fdc92a4c..000000000
--- a/crates/backend-uzu/src/config/classifier/prediction_head_config.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-use proc_macros::uzu_config;
-
-use crate::{
- backends::common::ActivationConfig,
- config::{LinearConfig, NormalizationConfig},
-};
-
-#[uzu_config]
-pub struct PredictionHeadConfig {
- pub dense_config: LinearConfig,
- pub activation: ActivationConfig,
- pub normalization_config: NormalizationConfig,
- pub readout_config: LinearConfig,
- pub use_dense_bias: bool,
-}
diff --git a/crates/backend-uzu/src/config/classifier_model.rs b/crates/backend-uzu/src/config/classifier_model.rs
deleted file mode 100644
index 30a5348ae..000000000
--- a/crates/backend-uzu/src/config/classifier_model.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-use proc_macros::uzu_config;
-
-use crate::config::{ClassifierConfig, MessageProcessorConfig};
-
-#[uzu_config]
-pub struct ClassifierModelConfig {
- pub model_config: ClassifierConfig,
- pub message_processor_config: MessageProcessorConfig,
-}
diff --git a/crates/backend-uzu/src/config/decoder.rs b/crates/backend-uzu/src/config/decoder.rs
index 5992ac3f4..b622f8632 100644
--- a/crates/backend-uzu/src/config/decoder.rs
+++ b/crates/backend-uzu/src/config/decoder.rs
@@ -1,19 +1,22 @@
use proc_macros::uzu_config;
-use super::{AttentionConfig, EmbeddingConfig, TransformerConfig};
-use crate::utils::strict_serde::Unsupported;
+use crate::{
+ config::{embedding::AnyEmbeddingConfig, token_mixer::attention::AttentionConfig, transformer::TransformerConfig},
+ utils::strict_serde::Unsupported,
+};
#[uzu_config]
pub struct DecoderConfig {
- pub embedding_config: EmbeddingConfig,
+ pub embedding_config: AnyEmbeddingConfig,
pub transformer_config: TransformerConfig,
+
pub vocab_size: usize,
- pub pard_token: Option<usize>,
+ pub pard_token: Option<u64>,
pub ple_model_config: Option<Unsupported>,
}
impl DecoderConfig {
pub fn first_attention(&self) -> Option<&AttentionConfig> {
- self.transformer_config.layer_configs.iter().find_map(|l| l.mixer_config.as_attention())
+ self.transformer_config.layer_configs.iter().find_map(|layer| layer.mixer_config.as_attention())
}
}
diff --git a/crates/backend-uzu/src/config/embedding.rs b/crates/backend-uzu/src/config/embedding.rs
deleted file mode 100644
index 69e89a3ff..000000000
--- a/crates/backend-uzu/src/config/embedding.rs
+++ /dev/null
@@ -1,105 +0,0 @@
-use proc_macros::uzu_config;
-
-use crate::{DataType, backends::common::gpu_types::QuantizationMode};
-
-#[uzu_config]
-pub struct EmbeddingConfigCommon {
- pub input_scale: Option<f32>,
- pub logit_soft_cap: Option<f32>,
-}
-
-#[uzu_config]
-#[serde(tag = "type")]
-pub enum EmbeddingConfig {
- #[serde(rename = "TiedEmbeddingConfig")]
- Tied {
- #[serde(flatten)]
- common: EmbeddingConfigCommon,
- precision: DataType,
- },
- #[serde(rename = "UntiedEmbeddingConfig")]
- Untied {
- #[serde(flatten)]
- common: EmbeddingConfigCommon,
- precision: DataType,
- },
- #[serde(rename = "MLXQuantizedTiedEmbeddingConfig")]
- ScaleBiasQuantizedTied {
- #[serde(flatten)]
- common: EmbeddingConfigCommon,
- group_size: usize,
- embedding_quantization_mode: QuantizationMode,
- activation_quantization_mode: Option<QuantizationMode>,
- activation_precision: DataType,
- },
- #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]
- ScaleBiasQuantizedUntied {
- #[serde(flatten)]
- common: EmbeddingConfigCommon,
- group_size: usize,
- embedding_quantization_mode: QuantizationMode,
- activation_quantization_mode: Option<QuantizationMode>,
- activation_precision: DataType,
- },
- #[serde(rename = "MLXSemiQuantizedUntiedEmbeddingConfig")]
- ScaleBiasSemiQuantizedUntied {
- #[serde(flatten)]
- common: EmbeddingConfigCommon,
- group_size: usize,
- embedding_quantization_mode: QuantizationMode,
- activation_quantization_mode: Option<QuantizationMode>,
- activation_precision: DataType,
- },
-}
-
-impl EmbeddingConfig {
- pub fn common(&self) -> &EmbeddingConfigCommon {
- match self {
- EmbeddingConfig::Tied {
- common,
- ..
- }
- | EmbeddingConfig::Untied {
- common,
- ..
- }
- | EmbeddingConfig::ScaleBiasQuantizedTied {
- common,
- ..
- }
- | EmbeddingConfig::ScaleBiasQuantizedUntied {
- common,
- ..
- }
- | EmbeddingConfig::ScaleBiasSemiQuantizedUntied {
- common,
- ..
- } => common,
- }
- }
-
- pub fn activation_precision(&self) -> DataType {
- match self {
- EmbeddingConfig::Tied {
- precision,
- ..
- }
- | EmbeddingConfig::Untied {
- precision,
- ..
- } => *precision,
- EmbeddingConfig::ScaleBiasQuantizedTied {
- activation_precision,
- ..
- }
- | EmbeddingConfig::ScaleBiasQuantizedUntied {
- activation_precision,
- ..
- }
- | EmbeddingConfig::ScaleBiasSemiQuantizedUntied {
- activation_precision,
- ..
- } => *activation_precision,
- }
- }
-}
diff --git a/crates/backend-uzu/src/config/embedding/mod.rs b/crates/backend-uzu/src/config/embedding/mod.rs
new file mode 100644
index 000000000..d4b968c52
--- /dev/null
+++ b/crates/backend-uzu/src/config/embedding/mod.rs
@@ -0,0 +1,10 @@
+use proc_macros::uzu_config_abstract;
+
+pub mod tied_embedding;
+pub mod untied_embedding;
+
+#[uzu_config_abstract(tied_embedding::TiedEmbeddingConfig, untied_embedding::UntiedEmbeddingConfig)]
+pub struct EmbeddingConfig {
+ pub input_scale: Option<f32>,
+ pub logit_soft_cap: Option<f32>,
+}
diff --git a/crates/backend-uzu/src/config/embedding/tied_embedding.rs b/crates/backend-uzu/src/config/embedding/tied_embedding.rs
new file mode 100644
index 000000000..720896674
--- /dev/null
+++ b/crates/backend-uzu/src/config/embedding/tied_embedding.rs
@@ -0,0 +1,4 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::EmbeddingConfig)]
+pub struct TiedEmbeddingConfig;
diff --git a/crates/backend-uzu/src/config/embedding/untied_embedding.rs b/crates/backend-uzu/src/config/embedding/untied_embedding.rs
new file mode 100644
index 000000000..0376e7cf2
--- /dev/null
+++ b/crates/backend-uzu/src/config/embedding/untied_embedding.rs
@@ -0,0 +1,18 @@
+use monostate::MustBe;
+use proc_macros::uzu_config;
+
+#[uzu_config(super::EmbeddingConfig)]
+pub struct UntiedEmbeddingConfig;
+
+impl UntiedEmbeddingConfig {
+ pub fn new(
+ input_scale: Option<f32>,
+ logit_soft_cap: Option<f32>,
+ ) -> Self {
+ Self {
+ ty: MustBe!("UntiedEmbeddingConfig"),
+ input_scale,
+ logit_soft_cap,
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/config/language_model.rs b/crates/backend-uzu/src/config/language_model.rs
deleted file mode 100644
index 90e1f1561..000000000
--- a/crates/backend-uzu/src/config/language_model.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-use proc_macros::uzu_config;
-
-use crate::config::{DecoderConfig, GenerationConfig, MessageProcessorConfig};
-
-#[uzu_config]
-pub struct LanguageModelConfig {
- pub model_config: DecoderConfig,
- pub message_processor_config: MessageProcessorConfig,
- pub generation_config: GenerationConfig,
-}
diff --git a/crates/backend-uzu/src/config/linear.rs b/crates/backend-uzu/src/config/linear.rs
index 4c2e4948e..0364b538d 100644
--- a/crates/backend-uzu/src/config/linear.rs
+++ b/crates/backend-uzu/src/config/linear.rs
@@ -1,56 +1,4 @@
use proc_macros::uzu_config;
-use crate::{DataType, backends::common::gpu_types::QuantizationMode};
-
-#[uzu_config]
-pub struct QuantizationConfig {
- pub group_size: usize,
- pub weight_quantization_mode: QuantizationMode,
- pub activation_quantization_mode: Option<QuantizationMode>,
- pub activation_precision: DataType,
-}
-
#[uzu_config]
-#[serde(tag = "type")]
-pub enum LinearConfig {
- #[serde(rename = "FullPrecisionLinearConfig")]
- FullPrecision {
- precision: DataType,
- },
- #[serde(rename = "GroupQuantizedLinearConfig")]
- Quantized(QuantizationConfig),
- #[serde(rename = "MLXQuantizedLinearConfig")]
- ScaleBiasQuantized(QuantizationConfig),
- #[serde(rename = "QLoRALinearConfig")]
- QLoRA {
- #[serde(flatten)]
- quantization: QuantizationConfig,
- lora_rank: usize,
- lora_scale: f32,
- },
- #[serde(rename = "RHTLinearWrapperConfig")]
- RHTLinearWrapper {
- block_size: usize,
- inner_config: Box<LinearConfig>,
- },
-}
-
-impl LinearConfig {
- pub fn activation_precision(&self) -> DataType {
- match self {
- LinearConfig::FullPrecision {
- precision,
- } => *precision,
- LinearConfig::Quantized(quantization) => quantization.activation_precision,
- LinearConfig::ScaleBiasQuantized(quantization) => quantization.activation_precision,
- LinearConfig::QLoRA {
- quantization,
- ..
- } => quantization.activation_precision,
- LinearConfig::RHTLinearWrapper {
- inner_config,
- ..
- } => inner_config.activation_precision(),
- }
- }
-}
+pub struct LinearConfig {}
diff --git a/crates/backend-uzu/src/config/metadata.rs b/crates/backend-uzu/src/config/metadata.rs
deleted file mode 100644
index e6b407185..000000000
--- a/crates/backend-uzu/src/config/metadata.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-use proc_macros::uzu_config;
-
-#[cfg(metal_backend)]
-use super::TtsModelConfig;
-use super::{ClassifierModelConfig, LanguageModelConfig, ModelType};
-use crate::{backends::common::gpu_types::QuantizationMode, utils::strict_serde::DeserializeStrictOwned};
-
-#[uzu_config]
-#[serde(untagged)]
-pub enum ModelConfig {
- LanguageModel(LanguageModelConfig),
- ClassifierModel(ClassifierModelConfig),
- #[cfg(metal_backend)]
- TtsModel(TtsModelConfig),
-}
-
-#[uzu_config]
-pub struct ModelMetadata<T: DeserializeStrictOwned> {
- pub toolchain_version: String,
- pub vendor: String,
- pub family: String,
- pub name: String,
- pub size: String,
- pub quantization: Option<QuantizationMode>,
- pub repo: String,
- pub use_cases: Vec<String>,
- pub model_type: ModelType,
- pub model_config: T,
- pub grammar_start_tokens: Vec<String>,
-}
diff --git a/crates/backend-uzu/src/config/mixer.rs b/crates/backend-uzu/src/config/mixer.rs
deleted file mode 100644
index 56fc990f1..000000000
--- a/crates/backend-uzu/src/config/mixer.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-use proc_macros::uzu_config;
-
-use super::{AttentionConfig, DeltaNetAttentionConfig, Mamba2Config, ShortConvConfig};
-use crate::DataType;
-
-#[uzu_config]
-#[serde(tag = "type")]
-pub enum MixerConfig {
- #[serde(rename = "AttentionConfig")]
- Attention(AttentionConfig),
- #[serde(rename = "Mamba2Config")]
- Mamba(Mamba2Config),
- #[serde(rename = "ShortConvConfig")]
- ShortConv(ShortConvConfig),
- #[serde(rename = "DeltaNetAttentionConfig")]
- DeltaNet(DeltaNetAttentionConfig),
-}
-
-impl MixerConfig {
- pub fn as_attention(&self) -> Option<&AttentionConfig> {
- match self {
- MixerConfig::Attention(config) => Some(config),
- _ => None,
- }
- }
-
- pub fn as_short_conv(&self) -> Option<&ShortConvConfig> {
- match self {
- MixerConfig::ShortConv(config) => Some(config),
- _ => None,
- }
- }
-
- pub fn as_delta_net(&self) -> Option<&DeltaNetAttentionConfig> {
- match self {
- MixerConfig::DeltaNet(config) => Some(config),
- _ => None,
- }
- }
-
- pub fn sliding_window_size(&self) -> Option<usize> {
- match self {
- MixerConfig::Attention(config) => config.sliding_window_size,
- MixerConfig::Mamba(_) | MixerConfig::ShortConv(_) | MixerConfig::DeltaNet(_) => None,
- }
- }
-
- pub fn activation_precision(&self) -> DataType {
- match self {
- MixerConfig::Attention(c) => c.qkv_projection_config.activation_precision(),
- MixerConfig::Mamba(c) => c.in_projection_config.activation_precision(),
- MixerConfig::ShortConv(c) => c.in_projection_config.activation_precision(),
- MixerConfig::DeltaNet(c) => c.in_proj_config.activation_precision(),
- }
- }
-}
diff --git a/crates/backend-uzu/src/config/mlp.rs b/crates/backend-uzu/src/config/mlp.rs
deleted file mode 100644
index c3b25a7d9..000000000
--- a/crates/backend-uzu/src/config/mlp.rs
+++ /dev/null
@@ -1,42 +0,0 @@
-use proc_macros::uzu_config;
-
-use crate::{backends::common::ActivationConfig, config::LinearConfig};
-
-#[uzu_config]
-#[serde(tag = "type")]
-pub enum MLPConfig {
- #[serde(rename = "DenseMLPConfig")]
- Dense(DenseMLPConfig),
- #[serde(rename = "MixtureOfExpertsConfig")]
- MixtureOfExperts(MixtureOfExpertsConfig),
-}
-
-#[uzu_config]
-pub struct DenseMLPConfig {
- pub linear_config: LinearConfig,
- pub activation: ActivationConfig,
- pub has_up_biases: bool,
- pub has_down_biases: bool,
- pub gate_clipping: Option<(Option<f32>, Option<f32>)>,
- pub up_clipping: Option<(Option<f32>, Option<f32>)>,
-}
-
-#[uzu_config]
-pub struct MixtureOfExpertsConfig {
- pub expert_config: DenseMLPConfig,
- pub router_config: LinearConfig,
- pub routing_function: RoutingFunctionConfig,
- pub num_routed_experts: usize,
- pub num_active_routed_experts: usize,
- pub router_has_biases: bool,
- pub num_shared_experts: usize,
- pub expert_hidden_dim: usize,
- pub gate_config: Option<LinearConfig>,
-}
-
-#[uzu_config]
-#[serde(tag = "type")]
-pub enum RoutingFunctionConfig {
- #[serde(rename = "SoftmaxRouting")]
- SoftmaxRouting,
-}
diff --git a/crates/backend-uzu/src/config/mlp/dense_mlp.rs b/crates/backend-uzu/src/config/mlp/dense_mlp.rs
new file mode 100644
index 000000000..8b85f1871
--- /dev/null
+++ b/crates/backend-uzu/src/config/mlp/dense_mlp.rs
@@ -0,0 +1,13 @@
+use proc_macros::uzu_config;
+
+use crate::config::{activation::AnyActivation, linear::LinearConfig};
+
+#[uzu_config(super::MLPConfig)]
+pub struct DenseMLPConfig {
+ pub linear_config: LinearConfig,
+ pub activation: AnyActivation,
+ pub has_up_biases: bool,
+ pub has_down_biases: bool,
+ pub gate_clipping: Option<(Option<f32>, Option<f32>)>,
+ pub up_clipping: Option<(Option<f32>, Option<f32>)>,
+}
diff --git a/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs b/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs
new file mode 100644
index 000000000..8678c539b
--- /dev/null
+++ b/crates/backend-uzu/src/config/mlp/mixture_of_experts.rs
@@ -0,0 +1,21 @@
+use proc_macros::uzu_config;
+
+use crate::config::{
+ linear::LinearConfig,
+ mlp::{dense_mlp::DenseMLPConfig, routing_function::AnyRoutingFunction},
+};
+
+#[uzu_config(super::MLPConfig)]
+pub struct MixtureOfExpertsConfig {
+ pub expert_config: DenseMLPConfig,
+ pub router_config: LinearConfig,
+ pub routing_function: AnyRoutingFunction,
+
+ pub num_routed_experts: usize,
+ pub num_active_routed_experts: usize,
+ pub router_has_biases: bool,
+
+ pub num_shared_experts: usize,
+ pub expert_hidden_dim: usize,
+ pub gate_config: Option<LinearConfig>,
+}
diff --git a/crates/backend-uzu/src/config/mlp/mod.rs b/crates/backend-uzu/src/config/mlp/mod.rs
new file mode 100644
index 000000000..551cf4330
--- /dev/null
+++ b/crates/backend-uzu/src/config/mlp/mod.rs
@@ -0,0 +1,8 @@
+use proc_macros::uzu_config_abstract;
+
+pub mod dense_mlp;
+pub mod mixture_of_experts;
+pub mod routing_function;
+
+#[uzu_config_abstract(dense_mlp::DenseMLPConfig, mixture_of_experts::MixtureOfExpertsConfig)]
+pub struct MLPConfig;
diff --git a/crates/backend-uzu/src/config/mlp/routing_function/mod.rs b/crates/backend-uzu/src/config/mlp/routing_function/mod.rs
new file mode 100644
index 000000000..2464c814c
--- /dev/null
+++ b/crates/backend-uzu/src/config/mlp/routing_function/mod.rs
@@ -0,0 +1,6 @@
+use proc_macros::uzu_config_abstract;
+
+pub mod softmax_routing;
+
+#[uzu_config_abstract(softmax_routing::SoftmaxRouting)]
+pub struct RoutingFunction;
diff --git a/crates/backend-uzu/src/config/mlp/routing_function/softmax_routing.rs b/crates/backend-uzu/src/config/mlp/routing_function/softmax_routing.rs
new file mode 100644
index 000000000..08db7bda9
--- /dev/null
+++ b/crates/backend-uzu/src/config/mlp/routing_function/softmax_routing.rs
@@ -0,0 +1,4 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::RoutingFunction)]
+pub struct SoftmaxRouting;
diff --git a/crates/backend-uzu/src/config/mod.rs b/crates/backend-uzu/src/config/mod.rs
index 65b81eb82..a2deaa02d 100644
--- a/crates/backend-uzu/src/config/mod.rs
+++ b/crates/backend-uzu/src/config/mod.rs
@@ -1,45 +1,15 @@
-mod attention;
-mod classifier;
-mod classifier_model;
-mod decoder;
-mod delta_net;
-mod embedding;
-mod generation;
-mod language_model;
-mod linear;
-mod mamba;
-mod message_processor;
-mod metadata;
-mod mixer;
-mod mlp;
-mod model_type;
-mod normalization;
-mod rope;
-mod short_conv;
-mod transformer;
-mod transformer_layer;
-#[cfg(metal_backend)]
-mod tts_model;
-
-pub use attention::*;
-pub use classifier::*;
-pub use classifier_model::*;
-pub use decoder::*;
-pub use delta_net::*;
-pub use embedding::*;
-pub use generation::*;
-pub use language_model::*;
-pub use linear::*;
-pub use mamba::*;
-pub use message_processor::*;
-pub use metadata::*;
-pub use mixer::*;
-pub use mlp::*;
-pub use model_type::*;
-pub use normalization::*;
-pub use rope::*;
-pub use short_conv::*;
-pub use transformer::*;
-pub use transformer_layer::*;
-#[cfg(metal_backend)]
-pub use tts_model::*;
+pub mod activation;
+pub mod classifier;
+pub mod decoder;
+pub mod embedding;
+pub mod linear;
+pub mod mlp;
+pub mod model;
+pub mod normalization;
+pub mod rope;
+pub mod token_codec;
+pub mod token_mixer;
+pub mod transformer;
+pub mod transformer_layer;
+pub mod tts;
+pub mod weight_matrix;
diff --git a/crates/backend-uzu/src/config/model/classifier_model.rs b/crates/backend-uzu/src/config/model/classifier_model.rs
new file mode 100644
index 000000000..5769a3f15
--- /dev/null
+++ b/crates/backend-uzu/src/config/model/classifier_model.rs
@@ -0,0 +1,8 @@
+use proc_macros::uzu_config;
+
+use crate::config::classifier::ClassifierConfig;
+
+#[uzu_config(super::ModelConfig)]
+pub struct ClassifierModelConfig {
+ pub classifier_config: ClassifierConfig,
+}
diff --git a/crates/backend-uzu/src/config/generation.rs b/crates/backend-uzu/src/config/model/generation.rs
similarity index 73%
rename from crates/backend-uzu/src/config/generation.rs
rename to crates/backend-uzu/src/config/model/generation.rs
index 300d07bb5..feddaab5e 100644
--- a/crates/backend-uzu/src/config/generation.rs
+++ b/crates/backend-uzu/src/config/model/generation.rs
@@ -4,13 +4,14 @@ use crate::utils::strict_serde::Unsupported;
#[uzu_config]
pub struct GenerationConfig {
- pub stop_token_ids: Vec<u32>,
+ pub stop_token_ids: Box<[u64]>,
pub temperature: Option<f32>,
pub top_k: Option<u32>,
pub top_p: Option<f32>,
pub min_p: Option<f32>,
- pub banned_tokens: Option<Vec<u32>>,
+ pub banned_tokens: Option<Unsupported>,
pub repetition_penalty: Option<Unsupported>,
pub presence_penalty: Option<Unsupported>,
pub frequency_penalty: Option<Unsupported>,
+ pub suffix_repetition_length: Option<Unsupported>,
}
diff --git a/crates/backend-uzu/src/config/model/language_model.rs b/crates/backend-uzu/src/config/model/language_model.rs
new file mode 100644
index 000000000..23b607ac7
--- /dev/null
+++ b/crates/backend-uzu/src/config/model/language_model.rs
@@ -0,0 +1,9 @@
+use proc_macros::uzu_config;
+
+use crate::config::{decoder::DecoderConfig, model::generation::GenerationConfig};
+
+#[uzu_config(super::ModelConfig)]
+pub struct LanguageModelConfig {
+ pub decoder_config: DecoderConfig,
+ pub generation_config: GenerationConfig,
+}
diff --git a/crates/backend-uzu/src/config/model/mod.rs b/crates/backend-uzu/src/config/model/mod.rs
new file mode 100644
index 000000000..0156e345e
--- /dev/null
+++ b/crates/backend-uzu/src/config/model/mod.rs
@@ -0,0 +1,17 @@
+use proc_macros::uzu_config_abstract;
+
+use crate::config::token_codec::AnyTokenCodecConfig;
+
+pub mod classifier_model;
+pub mod generation;
+pub mod language_model;
+pub mod tts_model;
+
+#[uzu_config_abstract(
+ language_model::LanguageModelConfig,
+ classifier_model::ClassifierModelConfig,
+ tts_model::TTSModelConfig
+)]
+pub struct ModelConfig {
+ pub token_codec_config: AnyTokenCodecConfig,
+}
diff --git a/crates/backend-uzu/src/config/model/tts_model.rs b/crates/backend-uzu/src/config/model/tts_model.rs
new file mode 100644
index 000000000..853ce9f22
--- /dev/null
+++ b/crates/backend-uzu/src/config/model/tts_model.rs
@@ -0,0 +1,8 @@
+use proc_macros::uzu_config;
+
+use crate::config::tts::TTSConfig;
+
+#[uzu_config(super::ModelConfig)]
+pub struct TTSModelConfig {
+ pub tts_config: TTSConfig,
+}
diff --git a/crates/backend-uzu/src/config/model_type.rs b/crates/backend-uzu/src/config/model_type.rs
deleted file mode 100644
index 89a96a278..000000000
--- a/crates/backend-uzu/src/config/model_type.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-use proc_macros::uzu_config;
-
-#[derive(Eq)]
-#[uzu_config]
-#[serde(rename_all = "snake_case")]
-pub enum ModelType {
- LanguageModel,
- ClassifierModel,
- TtsModel,
-}
diff --git a/crates/backend-uzu/src/config/normalization/normalization_config.rs b/crates/backend-uzu/src/config/normalization.rs
similarity index 57%
rename from crates/backend-uzu/src/config/normalization/normalization_config.rs
rename to crates/backend-uzu/src/config/normalization.rs
index 212e5a1a9..7a6af42bc 100644
--- a/crates/backend-uzu/src/config/normalization/normalization_config.rs
+++ b/crates/backend-uzu/src/config/normalization.rs
@@ -1,15 +1,17 @@
use proc_macros::uzu_config;
-use super::UpcastMode;
-use crate::DataType;
+#[uzu_config]
+#[serde(rename_all = "snake_case")]
+pub enum UpcastMode {
+ OnlyNormalization,
+ FullLayer,
+}
#[uzu_config]
pub struct NormalizationConfig {
- pub scale_precision: DataType,
- pub accumulation_precision: DataType,
pub epsilon: f32,
pub scale_offset: Option<f32>,
pub upcast_mode: UpcastMode,
pub subtract_mean: bool,
- pub use_bias: bool,
+ pub has_biases: bool,
}
diff --git a/crates/backend-uzu/src/config/normalization/mod.rs b/crates/backend-uzu/src/config/normalization/mod.rs
deleted file mode 100644
index 7dcb34734..000000000
--- a/crates/backend-uzu/src/config/normalization/mod.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-mod normalization_config;
-mod upcast_mode;
-
-pub use normalization_config::*;
-pub use upcast_mode::*;
diff --git a/crates/backend-uzu/src/config/normalization/upcast_mode.rs b/crates/backend-uzu/src/config/normalization/upcast_mode.rs
deleted file mode 100644
index 0d923dd2b..000000000
--- a/crates/backend-uzu/src/config/normalization/upcast_mode.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-use proc_macros::uzu_config;
-
-#[uzu_config]
-#[serde(rename_all = "snake_case")]
-pub enum UpcastMode {
- OnlyNormalization,
- FullLayer,
-}
diff --git a/crates/backend-uzu/src/config/rope.rs b/crates/backend-uzu/src/config/rope.rs
deleted file mode 100644
index 31a1a4375..000000000
--- a/crates/backend-uzu/src/config/rope.rs
+++ /dev/null
@@ -1,64 +0,0 @@
-use proc_macros::uzu_config;
-
-use crate::DataType;
-
-#[uzu_config]
-pub struct RopeConfigCommon {
- pub precision: DataType,
- pub base: f32,
- pub max_sequence_length: usize,
- pub head_dim: usize,
- pub partial_rotary_dim: Option<usize>,
-}
-
-#[uzu_config]
-#[serde(tag = "type")]
-pub enum RoPEConfig {
- #[serde(rename = "UnscaledRoPEConfig")]
- Unscaled(RopeConfigCommon),
- #[serde(rename = "LlamaRoPEConfig")]
- Llama {
- #[serde(flatten)]
- common: RopeConfigCommon,
- scaling_factor: f32,
- original_context_length: usize,
- low_frequency_factor: f32,
- high_frequency_factor: f32,
- },
- #[serde(rename = "YARNRoPEConfig")]
- YARN {
- #[serde(flatten)]
- common: RopeConfigCommon,
- scaling_factor: f32,
- original_context_length: usize,
- beta_fast: f32,
- beta_slow: f32,
- truncate: bool,
- },
- #[serde(rename = "LinearScalingRoPEConfig")]
- LinearScaling {
- #[serde(flatten)]
- common: RopeConfigCommon,
- scaling_factor: f32,
- },
-}
-
-impl RoPEConfig {
- pub fn common(&self) -> &RopeConfigCommon {
- match self {
- RoPEConfig::Unscaled(config) => config,
- RoPEConfig::Llama {
- common,
- ..
- } => common,
- RoPEConfig::YARN {
- common,
- ..
- } => common,
- RoPEConfig::LinearScaling {
- common,
- ..
- } => common,
- }
- }
-}
diff --git a/crates/backend-uzu/src/config/rope/linear_scaling_rope.rs b/crates/backend-uzu/src/config/rope/linear_scaling_rope.rs
new file mode 100644
index 000000000..cc28d9a4a
--- /dev/null
+++ b/crates/backend-uzu/src/config/rope/linear_scaling_rope.rs
@@ -0,0 +1,6 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::RoPEConfig)]
+pub struct LinearScalingRoPEConfig {
+ pub scaling_factor: f32,
+}
diff --git a/crates/backend-uzu/src/config/rope/llama_rope.rs b/crates/backend-uzu/src/config/rope/llama_rope.rs
new file mode 100644
index 000000000..8e6ac47ba
--- /dev/null
+++ b/crates/backend-uzu/src/config/rope/llama_rope.rs
@@ -0,0 +1,9 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::RoPEConfig)]
+pub struct LlamaRoPEConfig {
+ pub scaling_factor: f32,
+ pub original_context_length: usize,
+ pub low_frequency_factor: f32,
+ pub high_frequency_factor: f32,
+}
diff --git a/crates/backend-uzu/src/config/rope/mod.rs b/crates/backend-uzu/src/config/rope/mod.rs
new file mode 100644
index 000000000..49b9da09f
--- /dev/null
+++ b/crates/backend-uzu/src/config/rope/mod.rs
@@ -0,0 +1,19 @@
+use proc_macros::uzu_config_abstract;
+
+pub mod linear_scaling_rope;
+pub mod llama_rope;
+pub mod unscaled_rope;
+pub mod yarn_rope;
+
+#[uzu_config_abstract(
+ unscaled_rope::UnscaledRoPEConfig,
+ llama_rope::LlamaRoPEConfig,
+ yarn_rope::YARNRoPEConfig,
+ linear_scaling_rope::LinearScalingRoPEConfig
+)]
+pub struct RoPEConfig {
+ pub base: f32,
+ pub max_sequence_length: usize,
+ pub head_dim: Option<usize>,
+ pub partial_rotary_dim: Option<usize>,
+}
diff --git a/crates/backend-uzu/src/config/rope/unscaled_rope.rs b/crates/backend-uzu/src/config/rope/unscaled_rope.rs
new file mode 100644
index 000000000..6ff16c708
--- /dev/null
+++ b/crates/backend-uzu/src/config/rope/unscaled_rope.rs
@@ -0,0 +1,4 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::RoPEConfig)]
+pub struct UnscaledRoPEConfig;
diff --git a/crates/backend-uzu/src/config/rope/yarn_rope.rs b/crates/backend-uzu/src/config/rope/yarn_rope.rs
new file mode 100644
index 000000000..491b10707
--- /dev/null
+++ b/crates/backend-uzu/src/config/rope/yarn_rope.rs
@@ -0,0 +1,10 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::RoPEConfig)]
+pub struct YARNRoPEConfig {
+ pub scaling_factor: f32,
+ pub original_context_length: usize,
+ pub beta_fast: f32,
+ pub beta_slow: f32,
+ pub truncate: bool,
+}
diff --git a/crates/backend-uzu/src/config/message_processor.rs b/crates/backend-uzu/src/config/token_codec/chat_codec.rs
similarity index 82%
rename from crates/backend-uzu/src/config/message_processor.rs
rename to crates/backend-uzu/src/config/token_codec/chat_codec.rs
index 542ff8727..3881185fe 100644
--- a/crates/backend-uzu/src/config/message_processor.rs
+++ b/crates/backend-uzu/src/config/token_codec/chat_codec.rs
@@ -1,7 +1,7 @@
use proc_macros::uzu_config;
-#[uzu_config]
-pub struct MessageProcessorConfig {
+#[uzu_config(super::TokenCodecConfig)]
+pub struct ChatCodecConfig {
pub prompt_template: String,
pub output_parser_regex: Option<String>,
pub system_role_name: String,
diff --git a/crates/backend-uzu/src/config/token_codec/mod.rs b/crates/backend-uzu/src/config/token_codec/mod.rs
new file mode 100644
index 000000000..8999d3886
--- /dev/null
+++ b/crates/backend-uzu/src/config/token_codec/mod.rs
@@ -0,0 +1,7 @@
+use proc_macros::uzu_config_abstract;
+
+pub mod chat_codec;
+pub mod tts_codec;
+
+#[uzu_config_abstract(chat_codec::ChatCodecConfig, tts_codec::TTSCodecConfig)]
+pub struct TokenCodecConfig;
diff --git a/crates/backend-uzu/src/config/token_codec/tts_codec.rs b/crates/backend-uzu/src/config/token_codec/tts_codec.rs
new file mode 100644
index 000000000..17a6555f5
--- /dev/null
+++ b/crates/backend-uzu/src/config/token_codec/tts_codec.rs
@@ -0,0 +1,7 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::TokenCodecConfig)]
+pub struct TTSCodecConfig {
+ pub prompt_template: String,
+ pub drop_initial_newline: bool,
+}
diff --git a/crates/backend-uzu/src/config/attention.rs b/crates/backend-uzu/src/config/token_mixer/attention.rs
similarity index 66%
rename from crates/backend-uzu/src/config/attention.rs
rename to crates/backend-uzu/src/config/token_mixer/attention.rs
index 1326dc828..86c999116 100644
--- a/crates/backend-uzu/src/config/attention.rs
+++ b/crates/backend-uzu/src/config/token_mixer/attention.rs
@@ -1,18 +1,17 @@
use proc_macros::uzu_config;
-use super::{LinearConfig, NormalizationConfig, UpcastMode};
-use crate::DataType;
+use crate::config::{
+ linear::LinearConfig,
+ normalization::{NormalizationConfig, UpcastMode},
+};
-pub const VALUE_NORM_EPSILON: f32 = 1e-6;
-
-#[uzu_config]
+#[uzu_config(super::TokenMixerConfig)]
pub struct AttentionConfig {
pub qkv_projection_config: LinearConfig,
pub out_projection_config: LinearConfig,
pub query_norm_config: Option<NormalizationConfig>,
pub key_norm_config: Option<NormalizationConfig>,
- pub normalize_values: bool,
pub num_heads: usize,
pub num_groups: usize,
@@ -26,21 +25,17 @@ pub struct AttentionConfig {
pub has_qkv_biases: bool,
pub has_out_biases: bool,
pub gate_projection_config: Option<LinearConfig>,
+ pub normalize_values: bool,
}
impl AttentionConfig {
pub fn value_norm_config(&self) -> Option<NormalizationConfig> {
- if !self.normalize_values {
- return None;
- }
- Some(NormalizationConfig {
- scale_precision: self.qkv_projection_config.activation_precision(),
- accumulation_precision: DataType::F32,
- epsilon: VALUE_NORM_EPSILON,
+ self.normalize_values.then_some(NormalizationConfig {
+ epsilon: 1e-6,
scale_offset: None,
upcast_mode: UpcastMode::FullLayer,
subtract_mean: false,
- use_bias: false,
+ has_biases: false,
})
}
}
diff --git a/crates/backend-uzu/src/config/token_mixer/convolutions.rs b/crates/backend-uzu/src/config/token_mixer/convolutions.rs
new file mode 100644
index 000000000..04c0ce49f
--- /dev/null
+++ b/crates/backend-uzu/src/config/token_mixer/convolutions.rs
@@ -0,0 +1,6 @@
+use proc_macros::uzu_config;
+
+#[uzu_config]
+pub struct SeparableCausalConvConfig {
+ pub has_biases: bool,
+}
diff --git a/crates/backend-uzu/src/config/delta_net.rs b/crates/backend-uzu/src/config/token_mixer/delta_net.rs
similarity index 76%
rename from crates/backend-uzu/src/config/delta_net.rs
rename to crates/backend-uzu/src/config/token_mixer/delta_net.rs
index dc8253813..b46f9d63c 100644
--- a/crates/backend-uzu/src/config/delta_net.rs
+++ b/crates/backend-uzu/src/config/token_mixer/delta_net.rs
@@ -1,13 +1,16 @@
use proc_macros::uzu_config;
-use super::{linear::LinearConfig, normalization::NormalizationConfig, short_conv::SeparableCausalConvConfig};
+use crate::config::{
+ linear::LinearConfig, normalization::NormalizationConfig, token_mixer::convolutions::SeparableCausalConvConfig,
+};
-#[uzu_config]
-pub struct DeltaNetAttentionConfig {
+#[uzu_config(super::TokenMixerConfig)]
+pub struct DeltaNetConfig {
pub in_proj_config: LinearConfig,
pub conv_config: SeparableCausalConvConfig,
pub out_proj_config: LinearConfig,
pub norm_config: NormalizationConfig,
+
pub num_heads: usize,
pub num_groups: usize,
pub head_dim: usize,
@@ -15,7 +18,7 @@ pub struct DeltaNetAttentionConfig {
pub kernel_size: usize,
}
-impl DeltaNetAttentionConfig {
+impl DeltaNetConfig {
pub fn key_dim(&self) -> usize {
self.num_groups * self.head_dim
}
diff --git a/crates/backend-uzu/src/config/mamba.rs b/crates/backend-uzu/src/config/token_mixer/mamba2.rs
similarity index 75%
rename from crates/backend-uzu/src/config/mamba.rs
rename to crates/backend-uzu/src/config/token_mixer/mamba2.rs
index 41b406d1f..cdedbabb9 100644
--- a/crates/backend-uzu/src/config/mamba.rs
+++ b/crates/backend-uzu/src/config/token_mixer/mamba2.rs
@@ -1,23 +1,25 @@
use proc_macros::uzu_config;
-use super::{linear::LinearConfig, short_conv::SeparableCausalConvConfig};
-use crate::backends::common::ActivationConfig;
+use crate::config::{
+ activation::AnyActivation, linear::LinearConfig, token_mixer::convolutions::SeparableCausalConvConfig,
+};
-#[uzu_config]
+#[uzu_config(super::TokenMixerConfig)]
pub struct Mamba2Config {
pub in_projection_config: LinearConfig,
pub out_projection_config: LinearConfig,
pub conv_config: SeparableCausalConvConfig,
- pub activation: ActivationConfig,
+ pub activation: AnyActivation,
+
pub kernel_size: usize,
pub num_heads: usize,
pub num_groups: usize,
pub head_dim: usize,
pub state_dim: usize,
pub expansion_factor: usize,
+
pub has_in_biases: bool,
pub has_out_biases: bool,
- pub chunk_size: usize,
}
impl Mamba2Config {
diff --git a/crates/backend-uzu/src/config/token_mixer/mod.rs b/crates/backend-uzu/src/config/token_mixer/mod.rs
new file mode 100644
index 000000000..fdc0cce81
--- /dev/null
+++ b/crates/backend-uzu/src/config/token_mixer/mod.rs
@@ -0,0 +1,28 @@
+use proc_macros::uzu_config_abstract;
+
+pub mod attention;
+pub mod convolutions;
+pub mod delta_net;
+pub mod mamba2;
+pub mod short_conv;
+
+#[uzu_config_abstract(
+ attention::AttentionConfig,
+ delta_net::DeltaNetConfig,
+ mamba2::Mamba2Config,
+ short_conv::ShortConvConfig
+)]
+pub struct TokenMixerConfig;
+
+impl AnyTokenMixerConfig {
+ pub fn as_attention(&self) -> Option<&attention::AttentionConfig> {
+ match self {
+ AnyTokenMixerConfig::AttentionConfig(config) => Some(config),
+ _ => None,
+ }
+ }
+
+ pub fn sliding_window_size(&self) -> Option<usize> {
+ self.as_attention().and_then(|config| config.sliding_window_size)
+ }
+}
diff --git a/crates/backend-uzu/src/config/short_conv.rs b/crates/backend-uzu/src/config/token_mixer/short_conv.rs
similarity index 56%
rename from crates/backend-uzu/src/config/short_conv.rs
rename to crates/backend-uzu/src/config/token_mixer/short_conv.rs
index d12b53555..aa0c11f9c 100644
--- a/crates/backend-uzu/src/config/short_conv.rs
+++ b/crates/backend-uzu/src/config/token_mixer/short_conv.rs
@@ -1,18 +1,12 @@
use proc_macros::uzu_config;
-use super::linear::LinearConfig;
-use crate::DataType;
+use crate::config::{linear::LinearConfig, token_mixer::convolutions::SeparableCausalConvConfig};
-#[uzu_config]
-pub struct SeparableCausalConvConfig {
- pub precision: DataType,
- pub has_biases: bool,
-}
-
-#[uzu_config]
+#[uzu_config(super::TokenMixerConfig)]
pub struct ShortConvConfig {
pub in_projection_config: LinearConfig,
pub conv_config: SeparableCausalConvConfig,
pub out_projection_config: LinearConfig,
+
pub kernel_size: usize,
}
diff --git a/crates/backend-uzu/src/config/transformer.rs b/crates/backend-uzu/src/config/transformer.rs
index e293ebe5d..b968c2f93 100644
--- a/crates/backend-uzu/src/config/transformer.rs
+++ b/crates/backend-uzu/src/config/transformer.rs
@@ -1,10 +1,10 @@
use proc_macros::uzu_config;
-use super::{NormalizationConfig, TransformerLayerConfig};
+use crate::config::{normalization::NormalizationConfig, transformer_layer::TransformerLayerConfig};
#[uzu_config]
pub struct TransformerConfig {
- pub layer_configs: Vec<TransformerLayerConfig>,
+ pub layer_configs: Box<[TransformerLayerConfig]>,
pub output_norm_config: NormalizationConfig,
pub model_dim: usize,
pub hidden_dim: usize,
diff --git a/crates/backend-uzu/src/config/transformer_layer.rs b/crates/backend-uzu/src/config/transformer_layer.rs
index 12c88ffcb..cc929063e 100644
--- a/crates/backend-uzu/src/config/transformer_layer.rs
+++ b/crates/backend-uzu/src/config/transformer_layer.rs
@@ -1,19 +1,23 @@
use proc_macros::uzu_config;
-use super::{MLPConfig, MixerConfig, NormalizationConfig, RoPEConfig};
-use crate::utils::strict_serde::Unsupported;
+use crate::{
+ config::{
+ mlp::AnyMLPConfig, normalization::NormalizationConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig,
+ },
+ utils::strict_serde::Unsupported,
+};
#[uzu_config]
pub struct TransformerLayerConfig {
pub pre_mixer_norm_config: Option<NormalizationConfig>,
- pub mixer_config: MixerConfig,
+ pub mixer_config: AnyTokenMixerConfig,
pub post_mixer_norm_config: Option<NormalizationConfig>,
pub pre_mlp_norm_config: NormalizationConfig,
- pub mlp_config: MLPConfig,
+ pub mlp_config: AnyMLPConfig,
pub post_mlp_norm_config: Option<NormalizationConfig>,
- pub rope_config: Option<RoPEConfig>,
pub hidden_dim: Option<usize>,
pub ple_config: Option<Unsupported>,
pub has_post_layer_scalar: bool,
- pub kv_source_layer: Option<usize>,
+ pub kv_source_layer_index: Option<usize>,
+ pub rope_config: Option<AnyRoPEConfig>,
}
diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/common.rs b/crates/backend-uzu/src/config/tts/audio_decoder/common.rs
new file mode 100644
index 000000000..c9e3719fd
--- /dev/null
+++ b/crates/backend-uzu/src/config/tts/audio_decoder/common.rs
@@ -0,0 +1,15 @@
+use monostate::MustBe;
+use proc_macros::uzu_config;
+
+#[uzu_config]
+pub struct CausalConv1dConfig {
+ pub has_biases: MustBe!(true),
+}
+
+#[uzu_config]
+pub struct CausalTransposeConv1dConfig {
+ pub has_biases: MustBe!(true),
+}
+
+#[uzu_config]
+pub struct Snake1dConfig {}
diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/descript_audio_codec.rs b/crates/backend-uzu/src/config/tts/audio_decoder/descript_audio_codec.rs
new file mode 100644
index 000000000..c1448ee49
--- /dev/null
+++ b/crates/backend-uzu/src/config/tts/audio_decoder/descript_audio_codec.rs
@@ -0,0 +1,21 @@
+use proc_macros::uzu_config;
+
+use crate::config::tts::audio_decoder::fish_audio_modules::{DACDecoderConfig, DownsampleResidualVectorQuantizeConfig};
+
+#[uzu_config(super::TTSAudioDecoderConfig)]
+pub struct DescriptAudioCodecConfig {
+ pub quantizer_config: DownsampleResidualVectorQuantizeConfig,
+ pub decoder_config: DACDecoderConfig,
+ pub samplerate: u32,
+
+ pub encoder_dim: usize,
+ pub encoder_rates: Box<[usize]>,
+ pub decoder_dim: usize,
+ pub decoder_rates: Box<[usize]>,
+ pub input_dim: usize,
+ pub n_codebooks: usize,
+ pub codebook_dim: usize,
+ pub downsample_factor: Box<[usize]>,
+ pub codebook_size: usize,
+ pub semantic_codebook_size: usize,
+}
diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs b/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs
new file mode 100644
index 000000000..0907943b0
--- /dev/null
+++ b/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs
@@ -0,0 +1,71 @@
+use proc_macros::uzu_config;
+
+use crate::config::{
+ activation::AnyActivation,
+ embedding::tied_embedding::TiedEmbeddingConfig,
+ linear::LinearConfig,
+ normalization::NormalizationConfig,
+ transformer::TransformerConfig,
+ tts::audio_decoder::common::{CausalConv1dConfig, CausalTransposeConv1dConfig, Snake1dConfig},
+};
+
+#[uzu_config]
+pub struct ConvNeXtBlockConfig {
+ pub activation: AnyActivation,
+ pub dwconv_config: CausalConv1dConfig,
+ pub norm_config: NormalizationConfig,
+ pub pwconv_config: LinearConfig,
+}
+
+#[uzu_config]
+pub struct UpsamplingBlockConfig {
+ pub trans_conv_config: CausalTransposeConv1dConfig,
+ pub convnext_config: ConvNeXtBlockConfig,
+}
+
+#[uzu_config]
+pub struct UpsamplerConfig {
+ pub block_configs: Box<[UpsamplingBlockConfig]>,
+}
+
+#[uzu_config]
+pub struct VectorQuantizeConfig {
+ pub codebook_config: TiedEmbeddingConfig,
+ pub out_proj_config: LinearConfig,
+}
+
+#[uzu_config]
+pub struct ResidualVectorQuantizeConfig {
+ pub vq_config: VectorQuantizeConfig,
+}
+
+#[uzu_config]
+pub struct DownsampleResidualVectorQuantizeConfig {
+ pub semantic_quantizer_config: ResidualVectorQuantizeConfig,
+ pub quantizer_config: ResidualVectorQuantizeConfig,
+ pub post_module_config: TransformerConfig,
+ pub upsampler_config: UpsamplerConfig,
+}
+
+#[uzu_config]
+pub struct ResidualUnitConfig {
+ pub snake_config: Snake1dConfig,
+ pub conv_config: CausalConv1dConfig,
+ pub causal: bool,
+}
+
+#[uzu_config]
+pub struct DACDecoderBlockConfig {
+ pub snake_config: Snake1dConfig,
+ pub trans_conv_config: CausalTransposeConv1dConfig,
+ pub res_unit_config: ResidualUnitConfig,
+ pub causal: bool,
+}
+
+#[uzu_config]
+pub struct DACDecoderConfig {
+ pub conv_config: CausalConv1dConfig,
+ pub snake_config: Snake1dConfig,
+ pub decoder_block_config: DACDecoderBlockConfig,
+ pub causal: bool,
+}
diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/mod.rs b/crates/backend-uzu/src/config/tts/audio_decoder/mod.rs
new file mode 100644
index 000000000..20b2759ea
--- /dev/null
+++ b/crates/backend-uzu/src/config/tts/audio_decoder/mod.rs
@@ -0,0 +1,8 @@
+use proc_macros::uzu_config_abstract;
+
+pub mod common;
+pub mod descript_audio_codec;
+pub mod fish_audio_modules;
+
+#[uzu_config_abstract(descript_audio_codec::DescriptAudioCodecConfig)]
+pub struct TTSAudioDecoderConfig;
diff --git a/crates/backend-uzu/src/config/tts/mod.rs b/crates/backend-uzu/src/config/tts/mod.rs
new file mode 100644
index 000000000..a5dd1a3c9
--- /dev/null
+++ b/crates/backend-uzu/src/config/tts/mod.rs
@@ -0,0 +1,16 @@
+use proc_macros::uzu_config;
+
+use crate::config::tts::{
+ audio_decoder::AnyTTSAudioDecoderConfig, text_decoder::AnyTTSTextDecoderConfig, vocoder::AnyVocoderConfig,
+};
+
+pub mod audio_decoder;
+pub mod text_decoder;
+pub mod vocoder;
+
+#[uzu_config]
+pub struct TTSConfig {
+ pub text_decoder_config: AnyTTSTextDecoderConfig,
+ pub audio_decoder_config: AnyTTSAudioDecoderConfig,
+ pub vocoder_config: AnyVocoderConfig,
+}
diff --git a/crates/backend-uzu/src/config/tts/text_decoder/fish_audio_text_decoder.rs b/crates/backend-uzu/src/config/tts/text_decoder/fish_audio_text_decoder.rs
new file mode 100644
index 000000000..c2de107ee
--- /dev/null
+++ b/crates/backend-uzu/src/config/tts/text_decoder/fish_audio_text_decoder.rs
@@ -0,0 +1,34 @@
+use proc_macros::uzu_config;
+
+use crate::config::{
+ embedding::tied_embedding::TiedEmbeddingConfig, linear::LinearConfig, transformer::TransformerConfig,
+};
+
+#[uzu_config(super::TTSTextDecoderConfig)]
+pub struct FishAudioTextDecoderConfig {
+ pub slow_embeddings_config: TiedEmbeddingConfig,
+ pub slow_model_config: TransformerConfig,
+ pub slow_readout_config: LinearConfig,
+
+ pub fast_embeddings_config: TiedEmbeddingConfig,
+ pub fast_model_config: TransformerConfig,
+ pub fast_readout_config: LinearConfig,
+
+ pub codebook_embeddings_config: TiedEmbeddingConfig,
+ pub fast_model_projection_config: Option<LinearConfig>,
+
+ pub semantic_token_begin_id: u64,
+ pub semantic_token_end_id: u64,
+ pub im_end_token_id: u64,
+ pub codebook_size: usize,
+ pub vocab_size: usize,
+ pub slow_model_dim: usize,
+ pub fast_model_dim: usize,
+ pub num_codebooks: usize,
+ pub max_seq_len: usize,
+
+ pub scale_codebook_embeddings: bool,
+
+ pub short_logits_size: usize,
+ pub repeat_window_size: usize,
+}
diff --git a/crates/backend-uzu/src/config/tts/text_decoder/mod.rs b/crates/backend-uzu/src/config/tts/text_decoder/mod.rs
new file mode 100644
index 000000000..3398fb588
--- /dev/null
+++ b/crates/backend-uzu/src/config/tts/text_decoder/mod.rs
@@ -0,0 +1,6 @@
+use proc_macros::uzu_config_abstract;
+
+pub mod fish_audio_text_decoder;
+
+#[uzu_config_abstract(fish_audio_text_decoder::FishAudioTextDecoderConfig)]
+pub struct TTSTextDecoderConfig;
diff --git a/crates/backend-uzu/src/config/tts/vocoder/mod.rs b/crates/backend-uzu/src/config/tts/vocoder/mod.rs
new file mode 100644
index 000000000..aec73f953
--- /dev/null
+++ b/crates/backend-uzu/src/config/tts/vocoder/mod.rs
@@ -0,0 +1,6 @@
+use proc_macros::uzu_config_abstract;
+
+pub mod noop_vocoder;
+
+#[uzu_config_abstract(noop_vocoder::NoopVocoderConfig)]
+pub struct VocoderConfig;
diff --git a/crates/backend-uzu/src/config/tts/vocoder/noop_vocoder.rs b/crates/backend-uzu/src/config/tts/vocoder/noop_vocoder.rs
new file mode 100644
index 000000000..380caba0f
--- /dev/null
+++ b/crates/backend-uzu/src/config/tts/vocoder/noop_vocoder.rs
@@ -0,0 +1,4 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::VocoderConfig)]
+pub struct NoopVocoderConfig;
diff --git a/crates/backend-uzu/src/config/tts_model.rs b/crates/backend-uzu/src/config/tts_model.rs
deleted file mode 100644
index 4851cb7b3..000000000
--- a/crates/backend-uzu/src/config/tts_model.rs
+++ /dev/null
@@ -1,205 +0,0 @@
-#[cfg(metal_backend)]
-use std::path::Path;
-
-use proc_macros::uzu_config;
-
-use super::{EmbeddingConfig, EmbeddingConfigCommon, NormalizationConfig, TransformerConfig};
-use crate::{DataType, backends::common::ActivationConfig};
-
-#[uzu_config]
-pub struct TtsMessageProcessorConfig {
- pub prompt_template: String,
- pub drop_initial_newline: bool,
-}
-
-#[uzu_config]
-pub struct TtsConfig {
- pub text_decoder_config: TtsTextDecoderConfig,
- pub audio_decoder_config: TtsAudioDecoderConfig,
- pub vocoder_config: NoopVocoderConfig,
- pub activation_precision: DataType,
-}
-
-#[uzu_config]
-pub struct NoopVocoderConfig {}
-
-#[uzu_config]
-#[serde(tag = "type")]
-pub enum TtsTextDecoderConfig {
- #[serde(rename = "FishAudioTextDecoderConfig")]
- FishAudio(FishAudioTextDecoderConfig),
-}
-
-#[uzu_config]
-pub struct FishAudioTextDecoderConfig {
- pub slow_embeddings_config: TiedEmbeddingConfig,
- pub slow_model_config: TransformerConfig,
- pub slow_readout_config: FullPrecisionLinearConfig,
- pub fast_embeddings_config: TiedEmbeddingConfig,
- pub fast_model_config: TransformerConfig,
- pub fast_readout_config: FullPrecisionLinearConfig,
- pub codebook_embeddings_config: TiedEmbeddingConfig,
- pub fast_model_projection_config: Option<FullPrecisionLinearConfig>,
- pub semantic_token_begin_id: i64,
- pub semantic_token_end_id: i64,
- pub im_end_token_id: i64,
- pub codebook_size: usize,
- pub vocab_size: usize,
- pub slow_model_dim: usize,
- pub fast_model_dim: usize,
- pub num_codebooks: usize,
- pub max_seq_len: usize,
- pub scale_codebook_embeddings: bool,
- pub precision: DataType,
- pub short_logits_size: usize,
- pub repeat_window_size: usize,
-}
-
-#[uzu_config]
-pub struct FullPrecisionLinearConfig {
- pub precision: DataType,
-}
-
-#[uzu_config]
-pub struct TiedEmbeddingConfig {
- #[serde(flatten)]
- pub common: EmbeddingConfigCommon,
- pub precision: DataType,
-}
-
-impl TiedEmbeddingConfig {
- #[cfg(metal_backend)]
- pub(crate) fn to_text_decoder_embedding_config(&self) -> EmbeddingConfig {
- EmbeddingConfig::Untied {
- common: self.common.clone(),
- precision: self.precision,
- }
- }
-}
-
-#[uzu_config]
-#[serde(tag = "type")]
-pub enum TtsAudioDecoderConfig {
- #[serde(rename = "DescriptAudioCodecConfig")]
- DescriptAudioCodec(DescriptAudioCodecConfig),
-}
-
-#[uzu_config]
-pub struct DescriptAudioCodecConfig {
- pub precision: DataType,
- pub quantizer_config: DescriptAudioQuantizerConfig,
- pub decoder_config: DescriptAudioDacDecoderConfig,
- pub samplerate: u32,
- pub encoder_dim: usize,
- pub encoder_rates: Vec<usize>,
- pub decoder_dim: usize,
- pub decoder_rates: Vec<usize>,
- pub input_dim: usize,
- pub n_codebooks: usize,
- pub codebook_dim: usize,
- pub downsample_factor: Vec<usize>,
- pub codebook_size: usize,
- pub semantic_codebook_size: usize,
-}
-
-#[uzu_config]
-pub struct DescriptAudioDacDecoderConfig {
- pub precision: DataType,
- pub conv_config: DescriptAudioCausalConv1dConfig,
- pub snake_config: DescriptAudioSnake1dConfig,
- pub decoder_block_config: DescriptAudioDacDecoderBlockConfig,
- pub causal: bool,
-}
-
-#[uzu_config]
-pub struct DescriptAudioDacDecoderBlockConfig {
- pub precision: DataType,
- pub snake_config: DescriptAudioSnake1dConfig,
- pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,
- pub res_unit_config: DescriptAudioResidualUnitConfig,
- pub causal: bool,
-}
-
-#[uzu_config]
-pub struct DescriptAudioResidualUnitConfig {
- pub precision: DataType,
- pub snake_config: DescriptAudioSnake1dConfig,
- pub conv_config: DescriptAudioCausalConv1dConfig,
- pub causal: bool,
-}
-
-#[uzu_config]
-pub struct DescriptAudioCausalConv1dConfig {
- pub precision: DataType,
- pub has_biases: bool,
-}
-
-#[uzu_config]
-pub struct DescriptAudioCausalTransposeConv1dConfig {
- pub precision: DataType,
- pub has_biases: bool,
-}
-
-#[uzu_config]
-pub struct DescriptAudioSnake1dConfig {
- pub precision: DataType,
-}
-
-#[uzu_config]
-pub struct DescriptAudioQuantizerConfig {
- pub precision: DataType,
- pub semantic_quantizer_config: DescriptAudioResidualVectorQuantizeConfig,
- pub quantizer_config: DescriptAudioResidualVectorQuantizeConfig,
- pub post_module_config: TransformerConfig,
- pub upsampler_config: DescriptAudioUpsamplerConfig,
-}
-
-#[uzu_config]
-pub struct DescriptAudioResidualVectorQuantizeConfig {
- pub precision: DataType,
- pub vq_config: DescriptAudioVectorQuantizeConfig,
-}
-
-#[uzu_config]
-pub struct DescriptAudioVectorQuantizeConfig {
- pub precision: DataType,
- pub codebook_config: TiedEmbeddingConfig,
- pub out_proj_config: FullPrecisionLinearConfig,
-}
-
-#[uzu_config]
-pub struct DescriptAudioUpsamplerConfig {
- pub block_configs: Vec<DescriptAudioUpsamplingBlockConfig>,
-}
-
-#[uzu_config]
-pub struct DescriptAudioUpsamplingBlockConfig {
- pub precision: DataType,
- pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,
- pub convnext_config: DescriptAudioConvNeXtConfig,
-}
-
-#[uzu_config]
-pub struct DescriptAudioConvNeXtConfig {
- pub precision: DataType,
- pub activation: ActivationConfig,
- pub dwconv_config: DescriptAudioCausalConv1dConfig,
- pub norm_config: NormalizationConfig,
- pub pwconv_config: FullPrecisionLinearConfig,
-}
-
-#[uzu_config]
-pub struct TtsModelConfig {
- pub tts_config: TtsConfig,
- pub message_processor_config: TtsMessageProcessorConfig,
-}
-
-#[cfg(metal_backend)]
-impl TtsModelConfig {
- pub fn create_audio_generation_context_with_model_path<B: crate::backends::common::Backend>(
- &self,
- model_path: &Path,
- ) -> crate::audio::AudioResult<crate::audio::AudioGenerationContext<B>> {
- crate::audio::AudioGenerationContext::from_tts_config_and_model_path(&self.tts_config, model_path)
- }
-}
diff --git a/crates/backend-uzu/src/config/weight_matrix/awq_spec.rs b/crates/backend-uzu/src/config/weight_matrix/awq_spec.rs
new file mode 100644
index 000000000..914c4eded
--- /dev/null
+++ b/crates/backend-uzu/src/config/weight_matrix/awq_spec.rs
@@ -0,0 +1,11 @@
+use proc_macros::uzu_config;
+
+use crate::config::weight_matrix::Layout;
+
+#[uzu_config(super::WeightMatrixSpec)]
+pub struct AWQSpec {
+ pub bits: u32,
+ pub group_size: usize,
+ pub is_symmetric: bool,
+ pub layout: Layout,
+}
diff --git a/crates/backend-uzu/src/config/weight_matrix/full_precision_spec.rs b/crates/backend-uzu/src/config/weight_matrix/full_precision_spec.rs
new file mode 100644
index 000000000..41dc4e03b
--- /dev/null
+++ b/crates/backend-uzu/src/config/weight_matrix/full_precision_spec.rs
@@ -0,0 +1,8 @@
+use proc_macros::uzu_config;
+
+use crate::config::weight_matrix::Layout;
+
+#[uzu_config(super::WeightMatrixSpec)]
+pub struct FullPrecisionSpec {
+ pub layout: Layout,
+}
diff --git a/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs b/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs
new file mode 100644
index 000000000..7c9e08009
--- /dev/null
+++ b/crates/backend-uzu/src/config/weight_matrix/hybrid_spec.rs
@@ -0,0 +1,19 @@
+use proc_macros::uzu_config;
+
+use crate::config::weight_matrix::AnyWeightMatrixSpec;
+
+#[uzu_config]
+#[serde(rename_all = "snake_case")]
+pub enum IncoherenceProcessingMode {
+ Input,
+ Output,
+ InputOutput,
+}
+
+#[uzu_config(super::WeightMatrixSpec)]
+pub struct HybridSpec {
+ pub quantization_spec: Box<AnyWeightMatrixSpec>,
+ pub adapter_spec: Option<Box<AnyWeightMatrixSpec>>,
+ pub incoherence_block_size: Option<usize>,
+ pub incoherence_processing_mode: IncoherenceProcessingMode,
+}
diff --git a/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs b/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs
new file mode 100644
index 000000000..d056781e4
--- /dev/null
+++ b/crates/backend-uzu/src/config/weight_matrix/low_rank_spec.rs
@@ -0,0 +1,6 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::WeightMatrixSpec)]
+pub struct LowRankSpec {
+ pub rank: usize,
+}
diff --git a/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs b/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs
new file mode 100644
index 000000000..eda4bf6c5
--- /dev/null
+++ b/crates/backend-uzu/src/config/weight_matrix/mlx_spec.rs
@@ -0,0 +1,10 @@
+use proc_macros::uzu_config;
+
+use crate::config::weight_matrix::Layout;
+
+#[uzu_config(super::WeightMatrixSpec)]
+pub struct MLXSpec {
+ pub bits: u32,
+ pub group_size: usize,
+ pub layout: Layout,
+}
diff --git a/crates/backend-uzu/src/config/weight_matrix/mod.rs b/crates/backend-uzu/src/config/weight_matrix/mod.rs
new file mode 100644
index 000000000..bd87da016
--- /dev/null
+++ b/crates/backend-uzu/src/config/weight_matrix/mod.rs
@@ -0,0 +1,23 @@
+use proc_macros::{uzu_config, uzu_config_abstract};
+
+pub mod awq_spec;
+pub mod full_precision_spec;
+pub mod hybrid_spec;
+pub mod low_rank_spec;
+pub mod mlx_spec;
+
+#[uzu_config]
+#[serde(rename_all = "snake_case")]
+pub enum Layout {
+ OutputInput,
+ InputOutput,
+}
+
+#[uzu_config_abstract(
+ full_precision_spec::FullPrecisionSpec,
+ low_rank_spec::LowRankSpec,
+ hybrid_spec::HybridSpec,
+ awq_spec::AWQSpec,
+ mlx_spec::MLXSpec
+)]
+pub struct WeightMatrixSpec;
diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rs
index 63d530590..ce5e3df68 100644
--- a/crates/backend-uzu/src/encodable_block/attention.rs
+++ b/crates/backend-uzu/src/encodable_block/attention.rs
@@ -1,9 +1,11 @@
//! Attention kernel encodable.
-use std::{cell::RefCell, collections::HashMap};
+use std::{cell::RefCell, collections::HashMap, rc::Rc};
use itertools::iproduct;
+use thiserror::Error;
+use super::{Linear, QKVNorm, QkUnpack, Rope, linear::LinearBlockError, qkv_norm::QKVNormError};
use crate::{
DataType,
array::size_for_shape,
@@ -18,11 +20,13 @@ use crate::{
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
- config::AttentionConfig,
+ config::token_mixer::attention::AttentionConfig,
forward_pass::{
cache_layers::LayerCacheAccess,
kv_cache_layer::{KVCacheLayer, KVCacheLayerState},
+ state::RopeBuffers,
},
+ parameters::{ParameterLoaderError, ParameterTree},
};
fn env_gemm_attention_enabled() -> bool {
@@ -42,6 +46,13 @@ fn env_gemm_attention_enabled() -> bool {
}
pub struct Attention<B: Backend> {
+ qkv_projection: Box<dyn Linear<B>>,
+ gate_projection: Option<Box<dyn Linear<B>>>,
+ qkv_norm: Option<QKVNorm<B>>,
+ rope: Rc<Rope<B>>,
+ qk_unpack: Rc<QkUnpack<B>>,
+ out_projection: Box<dyn Linear<B>>,
+ sinks: Option<Allocation<B>>,
single_pass_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionSinglePassKernel>,
two_pass_1_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionTwoPass1Kernel>,
two_pass_2_kernels: HashMap<u32, <B::Kernels as Kernels>::AttentionTwoPass2Kernel>,
@@ -58,21 +69,100 @@ pub struct Attention<B: Backend> {
attention_scale: Option<f32>,
is_causal: bool,
sliding_window_size: Option<usize>,
+ model_dim: usize,
+ num_heads: usize,
+ num_groups: usize,
+ head_dim: usize,
}
-pub struct AttentionArguments<'a, B: Backend> {
- pub token_subtrie_ranges: Option<&'a Allocation<B>>,
- pub attention_sinks: Option<&'a Allocation<B>>,
- pub cache_access: Option<LayerCacheAccess<'a, B>>,
+#[derive(Debug, Error)]
+pub enum AttentionError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Linear block error: {0}")]
+ LinearBlockError(#[from] LinearBlockError<B>),
+ #[error("QKV norm error: {0}")]
+ QKVNormError(#[from] QKVNormError<B>),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
}
impl<B: Backend> Attention<B> {
+ #[allow(clippy::too_many_arguments)]
pub fn new(
context: &B::Context,
+ model_dim: usize,
data_type: DataType,
config: &AttentionConfig,
- has_gate: bool,
- ) -> Result<Self, B::Error> {
+ parameter_tree: &ParameterTree<B>,
+ rope: Rc<Rope<B>>,
+ qk_unpack: Rc<QkUnpack<B>>,
+ extract_input_hadamard: bool,
+ ) -> Result<(Self, Option<Allocation<B>>), AttentionError<B>> {
+ let q_dim = config.num_heads * config.head_dim;
+ let kv_dim = config.num_groups * config.head_dim;
+
+ let qkv_projection_tree = parameter_tree.subtree("qkv_projection")?;
+ let (qkv_projection, input_hadamard_factors) = if extract_input_hadamard {
+ <dyn Linear<B>>::new_extracting_input_hadamard(
+ model_dim,
+ [q_dim, kv_dim, kv_dim],
+ config.has_qkv_biases,
+ context,
+ data_type,
+ &qkv_projection_tree,
+ )?
+ } else {
+ (
+ <dyn Linear<B>>::new(
+ model_dim,
+ [q_dim, kv_dim, kv_dim],
+ config.has_qkv_biases,
+ context,
+ data_type,
+ &qkv_projection_tree,
+ )?,
+ None,
+ )
+ };
+
+ let gate_projection = config
+ .gate_projection_config
+ .as_ref()
+ .map(|_| {
+ let gate_projection_tree = parameter_tree.subtree("gate_projection")?;
+ <dyn Linear<B>>::new(model_dim, [q_dim], false, context, data_type, &gate_projection_tree)
+ })
+ .transpose()?;
+
+ let value_norm_config = config.value_norm_config();
+ let qkv_norm =
+ if config.query_norm_config.is_some() || config.key_norm_config.is_some() || value_norm_config.is_some() {
+ Some(QKVNorm::new(
+ context,
+ data_type,
+ config.query_norm_config.clone(),
+ config.key_norm_config.clone(),
+ value_norm_config,
+ parameter_tree,
+ config.num_heads,
+ config.num_groups,
+ config.head_dim,
+ )?)
+ } else {
+ None
+ };
+
+ let out_projection_tree = parameter_tree.subtree("out_projection")?;
+ let out_projection =
+ <dyn Linear<B>>::new(q_dim, [model_dim], config.has_out_biases, context, data_type, &out_projection_tree)?;
+
+ let sinks = if config.has_sinks {
+ Some(parameter_tree.leaf("sinks")?.validate(&[config.num_heads], data_type)?.read_allocation()?)
+ } else {
+ None
+ };
+
let mut single_pass_kernels = HashMap::new();
let mut two_pass_1_kernels = HashMap::new();
let mut two_pass_2_kernels = HashMap::new();
@@ -96,7 +186,8 @@ impl<B: Backend> Attention<B> {
config.is_causal,
is_trie,
config.sliding_window_size.is_some(),
- )?;
+ )
+ .map_err(AttentionError::BackendError)?;
single_pass_kernels.insert(key, sp_kernel);
let tp1_kernel = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
@@ -108,10 +199,12 @@ impl<B: Backend> Attention<B> {
config.is_causal,
is_trie,
config.sliding_window_size.is_some(),
- )?;
+ )
+ .map_err(AttentionError::BackendError)?;
two_pass_1_kernels.insert(key, tp1_kernel);
- let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)?;
+ let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)
+ .map_err(AttentionError::BackendError)?;
two_pass_2_kernels.insert(head_dim, tp2_kernel);
}
@@ -123,49 +216,74 @@ impl<B: Backend> Attention<B> {
config.is_causal,
false,
config.sliding_window_size.is_some(),
- )?;
+ )
+ .map_err(AttentionError::BackendError)?;
fallback_scatter_scores_kernels.insert(is_kv_cache_ring, scatter);
}
- let softmax_kernel = <B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, false)?;
+ let softmax_kernel = <B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, false)
+ .map_err(AttentionError::BackendError)?;
let softmax_sinks_kernel = if config.has_sinks {
- Some(<B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, true)?)
+ Some(
+ <B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, true)
+ .map_err(AttentionError::BackendError)?,
+ )
} else {
None
};
let fallback_scatter_values_kernel =
- <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, data_type)?;
+ <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, data_type)
+ .map_err(AttentionError::BackendError)?;
let matmul_kernel =
<<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).ok().map(RefCell::new);
let update_kv_cache_kernel =
- <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false)?;
+ <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false)
+ .map_err(AttentionError::BackendError)?;
let update_kv_cache_inplace_kernel =
- <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, true)?;
- let gate_kernel = if has_gate {
- Some(<B::Kernels as Kernels>::SigmoidGateKernel::new(context, data_type)?)
+ <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, true)
+ .map_err(AttentionError::BackendError)?;
+ let gate_kernel = if gate_projection.is_some() {
+ Some(
+ <B::Kernels as Kernels>::SigmoidGateKernel::new(context, data_type)
+ .map_err(AttentionError::BackendError)?,
+ )
} else {
None
};
let gemm_block = AttentionGemmBlock::new(data_type);
- Ok(Self {
- single_pass_kernels,
- two_pass_1_kernels,
- two_pass_2_kernels,
- softmax_kernel,
- softmax_sinks_kernel,
- fallback_scatter_scores_kernels,
- fallback_scatter_values_kernel,
- matmul_kernel,
- update_kv_cache_kernel,
- update_kv_cache_inplace_kernel,
- gate_kernel,
- gemm_block,
- data_type,
- attention_scale: config.scale,
- is_causal: config.is_causal,
- sliding_window_size: config.sliding_window_size,
- })
+ Ok((
+ Self {
+ qkv_projection,
+ gate_projection,
+ qkv_norm,
+ rope,
+ qk_unpack,
+ out_projection,
+ sinks,
+ single_pass_kernels,
+ two_pass_1_kernels,
+ two_pass_2_kernels,
+ softmax_kernel,
+ softmax_sinks_kernel,
+ fallback_scatter_scores_kernels,
+ fallback_scatter_values_kernel,
+ matmul_kernel,
+ update_kv_cache_kernel,
+ update_kv_cache_inplace_kernel,
+ gate_kernel,
+ gemm_block,
+ data_type,
+ attention_scale: config.scale,
+ is_causal: config.is_causal,
+ sliding_window_size: config.sliding_window_size,
+ model_dim,
+ num_heads: config.num_heads,
+ num_groups: config.num_groups,
+ head_dim: config.head_dim,
+ },
+ input_hadamard_factors,
+ ))
}
fn select_variant(
@@ -201,22 +319,76 @@ impl<B: Backend> Attention<B> {
KernelVariant::SinglePass
}
+ #[allow(clippy::too_many_arguments)]
pub fn encode(
&self,
- args: AttentionArguments<B>,
+ token_positions: &Allocation<B>,
+ token_subtrie_ranges: Option<&Allocation<B>>,
+ rope_buffers: Option<&RopeBuffers<B>>,
+ cache_access: Option<LayerCacheAccess<B>>,
+ hidden: Allocation<B>,
+ suffix_length: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let (qkv_input, gate) = if let Some(gate_projection) = &self.gate_projection {
+ let mut qkv_input =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.model_dim], self.data_type))?;
+ encoder.encode_copy(&hidden, .., &mut qkv_input, ..);
+ let gate = gate_projection.encode(hidden, suffix_length, encoder)?;
+ (qkv_input, Some(gate))
+ } else {
+ (hidden, None)
+ };
+
+ let mut qkv = self.qkv_projection.encode(qkv_input, suffix_length, encoder)?;
+ if let Some(qkv_norm) = &self.qkv_norm {
+ qkv_norm.encode(&mut qkv, suffix_length, encoder)?;
+ }
+ let (queries, rotated_keys) = match rope_buffers {
+ Some(rope_buffers) => self.rope.encode(
+ &qkv,
+ token_positions,
+ &rope_buffers.cosines,
+ &rope_buffers.sines,
+ suffix_length,
+ self.num_heads,
+ self.num_groups,
+ self.head_dim,
+ rope_buffers.max_sequence_length(),
+ rope_buffers.dim(),
+ encoder,
+ )?,
+ None => {
+ self.qk_unpack.encode(&qkv, suffix_length, self.num_heads, self.num_groups, self.head_dim, encoder)?
+ },
+ };
+ let attention_output = self.encode_core(
+ token_subtrie_ranges,
+ cache_access,
+ &qkv,
+ &queries,
+ rotated_keys,
+ gate.as_ref(),
+ suffix_length,
+ encoder,
+ )?;
+ self.out_projection.encode(attention_output, suffix_length, encoder)
+ }
+
+ fn encode_core(
+ &self,
+ token_subtrie_ranges: Option<&Allocation<B>>,
+ cache_access: Option<LayerCacheAccess<B>>,
qkv: &Allocation<B>,
queries: &Allocation<B>,
mut rotated_keys: Allocation<B>,
gate: Option<&Allocation<B>>,
suffix_length: usize,
- num_heads: usize,
- num_groups: usize,
- head_dim: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let is_trie = args.token_subtrie_ranges.is_some();
+ let is_trie = token_subtrie_ranges.is_some();
- let cache_layer = args.cache_access.as_ref().map(|a| {
+ let cache_layer = cache_access.as_ref().map(|a| {
match a {
LayerCacheAccess::Owned {
entry,
@@ -253,8 +425,8 @@ impl<B: Backend> Attention<B> {
let sequence_length = segment_prefix_length + suffix_length;
- let gqa_factor = num_heads / num_groups;
- let scale = self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());
+ let gqa_factor = self.num_heads / self.num_groups;
+ let scale = self.attention_scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt());
let gemm_enabled = env_gemm_attention_enabled();
if !gemm_enabled {
@@ -264,24 +436,26 @@ impl<B: Backend> Attention<B> {
});
}
let variant =
- self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, is_trie, is_kv_cache_ring);
+ self.select_variant(gemm_enabled, suffix_length, self.head_dim, sequence_length, is_trie, is_kv_cache_ring);
- let sinks_allocation = args.attention_sinks;
- let trie_allocation = args.token_subtrie_ranges;
+ let sinks_allocation = self.sinks.as_ref();
+ let trie_allocation = token_subtrie_ranges;
- let has_kv_cache = args.cache_access.is_some();
+ let has_kv_cache = cache_access.is_some();
let extracted_values = (!has_kv_cache)
.then(|| {
- let mut values =
- encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?;
+ let mut values = encoder.allocate_scratch(size_for_shape(
+ &[self.num_groups, suffix_length, self.head_dim],
+ self.data_type,
+ ))?;
self.update_kv_cache_inplace_kernel.encode(
None::<&Allocation<B>>,
qkv,
&mut rotated_keys,
&mut values,
- num_groups as u32,
- num_heads as u32,
- head_dim as u32,
+ self.num_groups as u32,
+ self.num_heads as u32,
+ self.head_dim as u32,
suffix_length as u32,
0u32,
max_sequence_length as u32,
@@ -295,18 +469,23 @@ impl<B: Backend> Attention<B> {
// For classifier mode (no KV cache) keys/values still come in group-major layout
// [num_groups, suffix_length, head_dim].
let (k_head_stride, k_seq_stride, v_head_stride, v_seq_stride) = if has_kv_cache {
- (head_dim as u32, (num_groups * head_dim) as u32, head_dim as u32, (num_groups * head_dim) as u32)
+ (
+ self.head_dim as u32,
+ (self.num_groups * self.head_dim) as u32,
+ self.head_dim as u32,
+ (self.num_groups * self.head_dim) as u32,
+ )
} else {
(
- (max_sequence_length * head_dim) as u32,
- head_dim as u32,
- (max_sequence_length * head_dim) as u32,
- head_dim as u32,
+ (max_sequence_length * self.head_dim) as u32,
+ self.head_dim as u32,
+ (max_sequence_length * self.head_dim) as u32,
+ self.head_dim as u32,
)
};
let kernel_key = KernelKey {
- head_dim: head_dim as u32,
+ head_dim: self.head_dim as u32,
is_trie,
is_kv_cache_ring,
};
@@ -329,18 +508,18 @@ impl<B: Backend> Attention<B> {
v_seq_stride,
ring_params,
scale,
- num_heads,
- num_groups,
+ self.num_heads,
+ self.num_groups,
suffix_length,
segment_prefix_length,
max_sequence_length,
- head_dim,
+ self.head_dim,
encoder,
)?
};
}
- let mut attention_output = if let Some(access) = args.cache_access {
+ let mut attention_output = if let Some(access) = cache_access {
match access {
LayerCacheAccess::Owned {
entry,
@@ -352,9 +531,9 @@ impl<B: Backend> Attention<B> {
qkv,
&mut layer.keys,
&mut layer.values,
- num_groups as u32,
- num_heads as u32,
- head_dim as u32,
+ self.num_groups as u32,
+ self.num_heads as u32,
+ self.head_dim as u32,
suffix_length as u32,
segment_prefix_length as u32,
max_sequence_length as u32,
@@ -367,9 +546,9 @@ impl<B: Backend> Attention<B> {
qkv,
&mut layer.keys,
&mut layer.values,
- num_groups as u32,
- num_heads as u32,
- head_dim as u32,
+ self.num_groups as u32,
+ self.num_heads as u32,
+ self.head_dim as u32,
suffix_length as u32,
segment_prefix_length as u32,
max_sequence_length as u32,
@@ -399,7 +578,7 @@ impl<B: Backend> Attention<B> {
};
if let Some(gate_kernel) = &self.gate_kernel {
- let total_elements = (suffix_length * num_heads * head_dim) as u32;
+ let total_elements = (suffix_length * self.num_heads * self.head_dim) as u32;
let gate = gate.expect("Gate allocation not initialized");
gate_kernel.encode(gate, &mut attention_output, total_elements, encoder);
}
diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
index 8697ac102..99cdbd23d 100644
--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
@@ -1,23 +1,41 @@
use std::rc::Rc;
+use thiserror::Error;
+
use crate::{
DataType,
backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder, Kernels, kernel::TensorAddSwapKernel},
- config::{TransformerConfig, TransformerLayerConfig},
+ config::{transformer::TransformerConfig, transformer_layer::TransformerLayerConfig},
encodable_block::{
- Attention, AttentionArguments, LayerArguments, Linear, Mlp, Normalization, QKVNorm, QkUnpack, Rope,
+ Attention, AttentionError, LayerArguments, Mlp, MlpBlockError, Normalization, NormalizationError, QkUnpack,
+ Rope,
},
- parameters::ParameterTree,
+ parameters::{ParameterLoaderError, ParameterTree},
};
+#[derive(Debug, Error)]
+pub enum ClassifierLayerError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
+ #[error("Attention error: {0}")]
+ AttentionError(#[from] AttentionError<B>),
+ #[error("MLP error: {0}")]
+ MlpBlockError(#[from] MlpBlockError<B>),
+ #[error("Normalization error: {0}")]
+ NormalizationError(#[from] NormalizationError<B>),
+ #[error("classifier layers must use attention")]
+ NonAttentionMixer,
+ #[error("classifier does not support attention hadamard")]
+ AttentionHadamardUnsupported,
+ #[error("classifier does not support MLP hadamard")]
+ MlpHadamardUnsupported,
+}
+
pub struct ClassifierLayer<B: Backend> {
pre_attention_norm: Option<Normalization<B>>,
- qkv_projection: Box<dyn Linear<B>>,
- qkv_norm: Option<QKVNorm<B>>,
- rope: Rc<Rope<B>>,
- qk_unpack: Rc<QkUnpack<B>>,
attention: Attention<B>,
- out_projection: Box<dyn Linear<B>>,
post_attention_norm: Option<Normalization<B>>,
mixer_residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,
pre_mlp_norm: Normalization<B>,
@@ -25,9 +43,6 @@ pub struct ClassifierLayer<B: Backend> {
post_mlp_norm: Option<Normalization<B>>,
mlp_residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,
model_dim: usize,
- num_heads: usize,
- num_groups: usize,
- head_dim: usize,
}
impl<B: Backend> ClassifierLayer<B> {
@@ -35,125 +50,94 @@ impl<B: Backend> ClassifierLayer<B> {
context: &B::Context,
transformer_config: &TransformerConfig,
layer_config: &TransformerLayerConfig,
- layer_index: usize,
- layer_loader: &ParameterTree<B::Context>,
+ layer_loader: &ParameterTree<B>,
rope: Rc<Rope<B>>,
qk_unpack: Rc<QkUnpack<B>>,
- ) -> Self {
- let attention_config = layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
- let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
+ data_type: DataType,
+ ) -> Result<Self, ClassifierLayerError<B>> {
+ let attention_config =
+ layer_config.mixer_config.as_attention().ok_or(ClassifierLayerError::NonAttentionMixer)?;
- let pre_attention_norm = layer_config.pre_mixer_norm_config.as_ref().map(|norm_config| {
- Normalization::new(
+ let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_mixer_norm_config {
+ Some(Normalization::new(
context,
- intermediate_data_type,
+ data_type,
+ transformer_config.model_dim,
norm_config.clone(),
- &layer_loader.subtree("pre_mixer_norm").unwrap(),
- )
- .expect("Failed to create pre-attention norm kernel")
- });
-
- let qkv_projection = <dyn Linear<B>>::new(
- &attention_config.qkv_projection_config,
- transformer_config.model_dim,
- [
- attention_config.num_heads * attention_config.head_dim,
- attention_config.num_groups * attention_config.head_dim,
- attention_config.num_groups * attention_config.head_dim,
- ],
- context,
- &layer_loader.subtree("mixer.qkv_projection").unwrap(),
- )
- .expect("Failed to create qkv projection");
-
- let value_norm_config = attention_config.value_norm_config();
- let qkv_norm = if attention_config.query_norm_config.is_some()
- || attention_config.key_norm_config.is_some()
- || value_norm_config.is_some()
- {
- match QKVNorm::new(
- context,
- intermediate_data_type,
- attention_config.query_norm_config.clone(),
- attention_config.key_norm_config.clone(),
- value_norm_config,
- &layer_loader.subtree("mixer").unwrap(),
- attention_config.num_heads,
- attention_config.num_groups,
- attention_config.head_dim,
- ) {
- Ok(qkv_norm) => Some(qkv_norm),
- Err(error) => panic!("Failed to create QKV norm kernel for layer {}: {:?}", layer_index, error),
- }
+ &layer_loader.subtree("pre_mixer_norm")?,
+ )?)
} else {
None
};
- let out_projection = <dyn Linear<B>>::new(
- &attention_config.out_projection_config,
- attention_config.num_heads * attention_config.head_dim,
- [transformer_config.model_dim],
+ let (attention, attention_hadamard_factors) = Attention::new(
context,
- &layer_loader.subtree("mixer.out_projection").unwrap(),
- )
- .expect("Failed to create out projection");
+ transformer_config.model_dim,
+ data_type,
+ attention_config,
+ &layer_loader.subtree("mixer")?,
+ rope,
+ qk_unpack,
+ false,
+ )?;
+ if attention_hadamard_factors.is_some() {
+ return Err(ClassifierLayerError::AttentionHadamardUnsupported);
+ }
- let post_attention_norm = layer_config.post_mixer_norm_config.as_ref().map(|norm_config| {
- Normalization::new(
+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {
+ Some(Normalization::new(
context,
- intermediate_data_type,
+ data_type,
+ transformer_config.model_dim,
norm_config.clone(),
- &layer_loader.subtree("post_mixer_norm").unwrap(),
- )
- .expect("Failed to create post-attention norm kernel")
- });
+ &layer_loader.subtree("post_mixer_norm")?,
+ )?)
+ } else {
+ None
+ };
- let mixer_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, intermediate_data_type)
- .expect("Failed to create mixer residual add kernel");
+ let mixer_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type)
+ .map_err(ClassifierLayerError::BackendError)?;
let pre_mlp_norm = Normalization::new(
context,
- intermediate_data_type,
+ data_type,
+ transformer_config.model_dim,
layer_config.pre_mlp_norm_config.clone(),
- &layer_loader.subtree("pre_mlp_norm").unwrap(),
- )
- .expect("Failed to create pre-MLP norm kernel");
+ &layer_loader.subtree("pre_mlp_norm")?,
+ )?;
let (mlp, mlp_hadamard_factors) = <dyn Mlp<B>>::new(
&layer_config.mlp_config,
transformer_config.model_dim,
layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),
context,
- &layer_loader.subtree("mlp").unwrap(),
- )
- .expect("Failed to create mlp block");
+ &layer_loader.subtree("mlp")?,
+ data_type,
+ )?;
- assert!(mlp_hadamard_factors.is_none(), "classifier doesn't support hadamard");
+ if mlp_hadamard_factors.is_some() {
+ return Err(ClassifierLayerError::MlpHadamardUnsupported);
+ }
- let post_mlp_norm = layer_config.post_mlp_norm_config.as_ref().map(|norm_config| {
- Normalization::new(
+ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
+ Some(Normalization::new(
context,
- intermediate_data_type,
+ data_type,
+ transformer_config.model_dim,
norm_config.clone(),
- &layer_loader.subtree("post_mlp_norm").unwrap(),
- )
- .expect("Failed to create post-MLP norm kernel")
- });
-
- let attention = Attention::new(context, intermediate_data_type, attention_config, false)
- .expect("Failed to create attention kernel");
+ &layer_loader.subtree("post_mlp_norm")?,
+ )?)
+ } else {
+ None
+ };
- let mlp_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, intermediate_data_type)
- .expect("Failed to create mlp residual add kernel");
+ let mlp_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type)
+ .map_err(ClassifierLayerError::BackendError)?;
- Self {
+ Ok(Self {
pre_attention_norm,
- qkv_projection,
- qkv_norm,
- rope,
- qk_unpack,
attention,
- out_projection,
post_attention_norm,
mixer_residual_add,
pre_mlp_norm,
@@ -161,10 +145,7 @@ impl<B: Backend> ClassifierLayer<B> {
post_mlp_norm,
mlp_residual_add,
model_dim: transformer_config.model_dim,
- num_heads: attention_config.num_heads,
- num_groups: attention_config.num_groups,
- head_dim: attention_config.head_dim,
- }
+ })
}
pub fn encode(
@@ -178,7 +159,6 @@ impl<B: Backend> ClassifierLayer<B> {
batch_dim,
token_positions,
token_subtrie_ranges,
- attention_sinks,
rope_buffers,
#[cfg(feature = "tracing")]
trace,
@@ -206,43 +186,15 @@ impl<B: Backend> ClassifierLayer<B> {
encoder.encode_copy(&main, .., layer_traces.pre_attention_norm.allocation_mut(), ..);
}
- let mut qkv = self.qkv_projection.encode(main, batch_dim, encoder)?;
- if let Some(ref qkv_norm) = self.qkv_norm {
- qkv_norm.encode(&mut qkv, batch_dim, encoder)?;
- }
- let (queries, rotated_keys) = match rope_buffers {
- Some(rope_buffers) => self.rope.encode(
- &qkv,
- token_positions,
- &rope_buffers.cosines,
- &rope_buffers.sines,
- batch_dim,
- self.num_heads,
- self.num_groups,
- self.head_dim,
- rope_buffers.max_sequence_length(),
- rope_buffers.dim(),
- encoder,
- )?,
- None => self.qk_unpack.encode(&qkv, batch_dim, self.num_heads, self.num_groups, self.head_dim, encoder)?,
- };
- let attention_output = self.attention.encode(
- AttentionArguments {
- token_subtrie_ranges,
- attention_sinks,
- cache_access: None,
- },
- &qkv,
- &queries,
- rotated_keys,
+ main = self.attention.encode(
+ token_positions,
+ token_subtrie_ranges,
+ rope_buffers,
None,
+ main,
batch_dim,
- self.num_heads,
- self.num_groups,
- self.head_dim,
encoder,
)?;
- main = self.out_projection.encode(attention_output, batch_dim, encoder)?;
#[cfg(feature = "tracing")]
if let Some(layer_traces) = layer_traces.as_deref_mut() {
encoder.encode_copy(&main, .., layer_traces.attention.allocation_mut(), ..);
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index e951c5068..223ddc0d2 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -7,15 +7,14 @@ use thiserror::Error;
#[cfg(feature = "tracing")]
use crate::forward_pass::traces::ActivationTrace;
use crate::{
- DataType,
backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
- config::DecoderConfig,
+ config::decoder::DecoderConfig,
encodable_block::{
- Embedding, LayerArguments, LayerExecutables, PostLayerScalar, QkUnpack, RMSNorm, Rope,
- embedding::EmbeddingError,
+ Embedding, LayerArguments, LayerExecutables, PostLayerScalar, QkUnpack, RMSNorm, RMSNormError, Rope,
+ embedding::EmbeddingError, layer::LayerExecutablesError,
},
- forward_pass::{cache_layers::CacheLayers, state::SharedBuffers},
- parameters::ParameterTree,
+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
+ parameters::{ParameterLoaderError, ParameterTree},
};
#[derive(Debug, Error)]
@@ -24,6 +23,12 @@ pub enum DecoderError<B: Backend> {
BackendError(#[source] B::Error),
#[error("Embedding error: {0}")]
EmbeddingError(#[from] EmbeddingError<B>),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
+ #[error("Layer error: {0}")]
+ LayerError(#[from] LayerExecutablesError<B>),
+ #[error("RMSNorm error: {0}")]
+ RMSNormError(#[from] RMSNormError<B>),
}
/// Full decoder executable with all layers and components.
@@ -56,27 +61,34 @@ impl<B: Backend> Decoder<B> {
pub fn new(
context: &B::Context,
decoder_config: &DecoderConfig,
- root_weight_loader: &ParameterTree<B::Context>,
- ) -> Self {
- let embedding_weight_loader = root_weight_loader.subtree("embedding").expect("Failed to get embedding subtree");
+ root_weight_loader: &ParameterTree<B>,
+ model_shape: &ModelShape,
+ ) -> Result<Self, DecoderError<B>> {
+ let embedding_weight_loader = root_weight_loader.subtree("embedding")?;
- let embed = Embedding::new(
+ let (embed, readout_input_hadamard_factors) = Embedding::new(
context,
decoder_config.vocab_size as u32,
decoder_config.transformer_config.model_dim as u32,
&decoder_config.embedding_config,
&embedding_weight_loader,
- )
- .expect("Failed to create embedding");
+ model_shape,
+ )?;
- let (layers, norm) =
- Self::build_transformer_layers_and_norm(context, decoder_config, root_weight_loader, "transformer");
+ let (layers, norm) = Self::build_transformer_layers_and_norm(
+ context,
+ decoder_config,
+ root_weight_loader,
+ "transformer",
+ readout_input_hadamard_factors,
+ model_shape,
+ )?;
- Self {
+ Ok(Self {
embed,
layers,
norm,
- }
+ })
}
/// Used by models whose token lookup weights and logits readout weights
@@ -84,73 +96,90 @@ impl<B: Backend> Decoder<B> {
pub fn new_with_embedding_and_readout_subtrees(
context: &B::Context,
decoder_config: &DecoderConfig,
- root_weight_loader: &ParameterTree<B::Context>,
+ root_weight_loader: &ParameterTree<B>,
transformer_subtree: &str,
embedding_subtree: &str,
readout_subtree: &str,
- ) -> Self {
- let embedding_weight_loader =
- root_weight_loader.subtree(embedding_subtree).expect("Failed to get embedding subtree");
- let readout_weight_loader = root_weight_loader.subtree(readout_subtree).expect("Failed to get readout subtree");
+ model_shape: &ModelShape,
+ ) -> Result<Self, DecoderError<B>> {
+ let embedding_weight_loader = root_weight_loader.subtree(embedding_subtree)?;
+ let readout_weight_loader = root_weight_loader.subtree(readout_subtree)?;
- let embed = Embedding::new_with_lookup_and_readout_trees(
+ let (embed, readout_input_hadamard_factors) = Embedding::new_with_lookup_and_readout_trees(
context,
decoder_config.vocab_size as u32,
decoder_config.transformer_config.model_dim as u32,
&decoder_config.embedding_config,
&embedding_weight_loader,
&readout_weight_loader,
- )
- .expect("Failed to create embedding");
+ model_shape,
+ )?;
- let (layers, norm) =
- Self::build_transformer_layers_and_norm(context, decoder_config, root_weight_loader, transformer_subtree);
+ let (layers, norm) = Self::build_transformer_layers_and_norm(
+ context,
+ decoder_config,
+ root_weight_loader,
+ transformer_subtree,
+ readout_input_hadamard_factors,
+ model_shape,
+ )?;
- Self {
+ Ok(Self {
embed,
layers,
norm,
- }
+ })
}
pub(crate) fn build_transformer_layers_and_norm(
context: &B::Context,
decoder_config: &DecoderConfig,
- root_weight_loader: &ParameterTree<B::Context>,
+ root_weight_loader: &ParameterTree<B>,
transformer_subtree: &str,
- ) -> (Box<[LayerExecutables<B>]>, RMSNorm<B>) {
- let decoder_weight_loader =
- root_weight_loader.subtree(transformer_subtree).expect("transformer subtree not found");
+ output_norm_hadamard_factors: Option<Allocation<B>>,
+ model_shape: &ModelShape,
+ ) -> Result<(Box<[LayerExecutables<B>]>, RMSNorm<B>), DecoderError<B>> {
+ let decoder_weight_loader = root_weight_loader.subtree(transformer_subtree)?;
let tf = &decoder_config.transformer_config;
- let norm_data_type: DataType = tf.layer_configs[0].mixer_config.activation_precision().into();
- let rope = Rc::new(Rope::<B>::new(context, norm_data_type).expect("Failed to create Rope"));
- let qk_unpack = Rc::new(QkUnpack::<B>::new(context, norm_data_type).expect("Failed to create QkUnpack"));
+ let rope = Rc::new(Rope::<B>::new(context, model_shape).map_err(DecoderError::BackendError)?);
+ let qk_unpack =
+ Rc::new(QkUnpack::<B>::new(context, model_shape.data_type).map_err(DecoderError::BackendError)?);
let layers = tf
.layer_configs
.iter()
.enumerate()
.map(|(layer_index, layer_config)| {
- let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap();
+ let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index))?;
- LayerExecutables::new(context, tf, layer_config, layer_index, &layer_loader, &rope, &qk_unpack)
+ LayerExecutables::new(
+ context,
+ tf,
+ layer_config,
+ layer_index,
+ &layer_loader,
+ &rope,
+ &qk_unpack,
+ model_shape.data_type,
+ )
+ .map_err(DecoderError::LayerError)
})
- .collect::<Vec<_>>();
+ .collect::<Result<Vec<_>, DecoderError<B>>>()?;
let norm_block = RMSNorm::new(
context,
- norm_data_type,
+ model_shape.data_type,
+ model_shape.model_dim(),
tf.output_norm_config.clone(),
- &decoder_weight_loader.subtree("output_norm").unwrap(),
- None,
+ &decoder_weight_loader.subtree("output_norm")?,
+ output_norm_hadamard_factors,
true,
true,
PostLayerScalar::None,
- )
- .expect("Failed to create output RMS norm kernel");
+ )?;
- (layers.into_boxed_slice(), norm_block)
+ Ok((layers.into_boxed_slice(), norm_block))
}
fn run_layers(
@@ -179,7 +208,6 @@ impl<B: Backend> Decoder<B> {
for layer in self.layers.iter() {
let rope_buffers = shared_buffers.rope_buffers_for_layer(layer.layer_index);
- let attention_sinks = shared_buffers.attention_sinks(layer.layer_index);
#[cfg(feature = "tracing")]
let layer_trace = trace.as_deref_mut().map(|trace| &mut trace.layer_results[layer.layer_index]);
@@ -192,7 +220,6 @@ impl<B: Backend> Decoder<B> {
token_positions,
token_parents,
token_subtrie_ranges,
- attention_sinks,
rope_buffers,
sampling_start,
sampling_length,
diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
index 004560627..497d78629 100644
--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
@@ -10,7 +10,7 @@ use crate::{
DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
},
},
- config::DeltaNetAttentionConfig,
+ config::token_mixer::delta_net::DeltaNetConfig,
encodable_block::linear::{Linear, LinearBlockError},
forward_pass::delta_net_layer::DeltaNetLayer,
parameters::{ParameterLoaderError, ParameterTree},
@@ -23,13 +23,21 @@ pub enum DeltaNetMixerError<B: Backend> {
#[error("Unsupported configuration: {0}")]
UnsupportedConfiguration(String),
#[error("Linear error: {0}")]
- InnerLinearError(#[from] Box<LinearBlockError<B>>),
+ InnerLinearError(#[from] LinearBlockError<B>),
#[error("Parameter loader error: {0}")]
ParameterLoaderError(#[from] ParameterLoaderError<B>),
}
pub(crate) struct DeltaNetMixer<B: Backend> {
- config: DeltaNetAttentionConfig,
+ kernel_size: usize,
+ num_heads: usize,
+ num_groups: usize,
+ value_head_dim: usize,
+ key_dim: usize,
+ value_dim: usize,
+ conv_dim: usize,
+ total_proj_dim: usize,
+ norm_epsilon: f32,
in_projection: Box<dyn Linear<B>>,
out_projection: Box<dyn Linear<B>>,
// Decode kernels
@@ -58,9 +66,10 @@ pub(crate) struct DeltaNetArguments<'a, B: Backend> {
impl<B: Backend> DeltaNetMixer<B> {
pub(crate) fn new(
context: &B::Context,
- config: DeltaNetAttentionConfig,
+ config: &DeltaNetConfig,
model_dim: usize,
- decoder_layer_loader: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
+ data_type: DataType,
) -> Result<(Self, Option<Allocation<B>>), DeltaNetMixerError<B>> {
if config.kernel_size < 2 {
return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(
@@ -81,42 +90,45 @@ impl<B: Backend> DeltaNetMixer<B> {
)));
}
- let data_type: DataType = config.in_proj_config.activation_precision().into();
let has_bias = config.conv_config.has_biases;
+ let key_dim = config.key_dim();
+ let value_dim = config.value_dim();
+ let conv_dim = config.conv_dim();
+ let total_proj_dim = config.total_proj_dim();
// Load weights
- let mixer_tree = decoder_layer_loader.subtree("mixer")?;
- let conv_tree = mixer_tree.subtree("conv")?;
+ let conv_tree = parameter_tree.subtree("conv")?;
let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
- &config.in_proj_config,
model_dim,
- [config.total_proj_dim()],
+ [total_proj_dim],
+ false,
context,
- &decoder_layer_loader.subtree("mixer.in_proj")?,
- )
- .map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;
+ data_type,
+ &parameter_tree.subtree("in_proj")?,
+ )?;
let out_projection = <dyn Linear<B>>::new(
- &config.out_proj_config,
- config.value_dim(),
+ value_dim,
[model_dim],
+ false,
context,
- &decoder_layer_loader.subtree("mixer.out_proj")?,
- )
- .map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;
+ data_type,
+ &parameter_tree.subtree("out_proj")?,
+ )?;
- let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;
+ let conv_weight =
+ conv_tree.leaf("weights")?.validate(&[conv_dim, config.kernel_size], data_type)?.read_allocation()?;
let conv_bias = if has_bias {
- Some(conv_tree.leaf("biases")?.read_allocation()?)
+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], data_type)?.read_allocation()?)
} else {
None
};
- let a_log = mixer_tree.leaf("a_log")?.read_allocation()?;
- let dt_bias = mixer_tree.leaf("dt_bias")?.read_allocation()?;
- let norm_tree = mixer_tree.subtree("norm")?;
- let norm_weight = norm_tree.leaf("scales")?.read_allocation()?;
+ let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], data_type)?.read_allocation()?;
+ let dt_bias = parameter_tree.leaf("dt_bias")?.validate(&[config.num_heads], data_type)?.read_allocation()?;
+ let norm_weight =
+ parameter_tree.leaf("norm.scales")?.validate(&[config.value_head_dim], data_type)?.read_allocation()?;
// Create kernels
let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, data_type, has_bias)
@@ -139,7 +151,15 @@ impl<B: Backend> DeltaNetMixer<B> {
Ok((
Self {
- config,
+ kernel_size: config.kernel_size,
+ num_heads: config.num_heads,
+ num_groups: config.num_groups,
+ value_head_dim: config.value_head_dim,
+ key_dim,
+ value_dim,
+ conv_dim,
+ total_proj_dim,
+ norm_epsilon: config.norm_config.epsilon,
in_projection,
out_projection,
conv_update,
@@ -166,14 +186,14 @@ impl<B: Backend> DeltaNetMixer<B> {
in_proj: &mut Allocation<B>,
encoder: &mut Encoder<B>,
) {
- let kernel_size = self.config.kernel_size;
+ let kernel_size = self.kernel_size;
self.conv_update.encode(
&self.conv_weight,
self.conv_bias.as_ref(),
in_proj,
&mut layer.conv_state,
kernel_size as u32,
- self.config.conv_dim() as u32,
+ self.conv_dim as u32,
(kernel_size - 1) as u32,
encoder,
);
@@ -186,10 +206,10 @@ impl<B: Backend> DeltaNetMixer<B> {
encoder: &mut Encoder<B>,
suffix_length: usize,
) -> Result<(), B::Error> {
- let kernel_size = self.config.kernel_size;
+ let kernel_size = self.kernel_size;
let state_stride = kernel_size - 1;
- let conv_dim = self.config.conv_dim();
- let total_proj_dim = self.config.total_proj_dim();
+ let conv_dim = self.conv_dim;
+ let total_proj_dim = self.total_proj_dim;
let mut padded = encoder
.allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.data_type))?;
@@ -228,8 +248,7 @@ impl<B: Backend> DeltaNetMixer<B> {
active_row_count: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut out =
- encoder.allocate_scratch(size_for_shape(&[active_row_count, self.config.value_dim()], self.data_type))?;
+ let mut out = encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.data_type))?;
self.delta_net_update.encode(
in_proj,
&self.a_log,
@@ -237,12 +256,12 @@ impl<B: Backend> DeltaNetMixer<B> {
&self.norm_weight,
&mut layer.ssm_state,
&mut out,
- self.config.num_heads as u32,
- self.config.num_groups as u32,
- self.config.value_head_dim as u32,
- self.config.key_dim() as u32,
- self.config.value_dim() as u32,
- self.config.norm_config.epsilon,
+ self.num_heads as u32,
+ self.num_groups as u32,
+ self.value_head_dim as u32,
+ self.key_dim as u32,
+ self.value_dim as u32,
+ self.norm_epsilon,
encoder,
);
Ok(out)
@@ -255,15 +274,15 @@ impl<B: Backend> DeltaNetMixer<B> {
suffix_length: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let num_dv_groups = ((self.config.value_head_dim + 7) / 8) as u32;
+ let num_dv_groups = ((self.value_head_dim + 7) / 8) as u32;
let mut prep_q_norm =
- encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.key_dim()], DataType::F32))?;
+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.key_dim], DataType::F32))?;
let mut prep_k_norm =
- encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.key_dim()], DataType::F32))?;
+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.key_dim], DataType::F32))?;
let mut prep_beta =
- encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;
+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.num_heads], DataType::F32))?;
let mut prep_decay =
- encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;
+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.num_heads], DataType::F32))?;
self.prefill_prep.encode(
in_proj,
@@ -273,16 +292,15 @@ impl<B: Backend> DeltaNetMixer<B> {
&mut prep_k_norm,
&mut prep_beta,
&mut prep_decay,
- self.config.num_heads as u32,
- self.config.num_groups as u32,
- self.config.key_dim() as u32,
- self.config.value_dim() as u32,
+ self.num_heads as u32,
+ self.num_groups as u32,
+ self.key_dim as u32,
+ self.value_dim as u32,
suffix_length as u32,
encoder,
);
- let mut out =
- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.value_dim()], self.data_type))?;
+ let mut out = encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.data_type))?;
self.delta_net_prefill.encode(
&prep_q_norm,
&prep_k_norm,
@@ -291,11 +309,11 @@ impl<B: Backend> DeltaNetMixer<B> {
in_proj,
&mut layer.ssm_state,
&mut out,
- self.config.num_heads as u32,
- self.config.num_groups as u32,
- self.config.value_head_dim as u32,
- self.config.key_dim() as u32,
- self.config.value_dim() as u32,
+ self.num_heads as u32,
+ self.num_groups as u32,
+ self.value_head_dim as u32,
+ self.key_dim as u32,
+ self.value_dim as u32,
suffix_length as u32,
num_dv_groups,
encoder,
@@ -305,12 +323,12 @@ impl<B: Backend> DeltaNetMixer<B> {
&mut out,
in_proj,
&self.norm_weight,
- self.config.num_heads as u32,
- self.config.value_head_dim as u32,
- self.config.value_dim() as u32,
- self.config.conv_dim() as u32,
- self.config.total_proj_dim() as u32,
- self.config.norm_config.epsilon,
+ self.num_heads as u32,
+ self.value_head_dim as u32,
+ self.value_dim as u32,
+ self.conv_dim as u32,
+ self.total_proj_dim as u32,
+ self.norm_epsilon,
suffix_length as u32,
encoder,
);
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 365eb3ff4..1f5a917e0 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -4,6 +4,7 @@ use thiserror::Error;
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
gpu_types::{QuantizationMethod, QuantizationMode},
@@ -12,7 +13,17 @@ use crate::{
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},
},
},
- config::EmbeddingConfig,
+ config::{
+ embedding::AnyEmbeddingConfig,
+ weight_matrix::{
+ AnyWeightMatrixSpec, Layout,
+ awq_spec::AWQSpec,
+ full_precision_spec::FullPrecisionSpec,
+ hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
+ mlx_spec::MLXSpec,
+ },
+ },
+ forward_pass::model_shape::ModelShape,
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -27,6 +38,7 @@ pub enum EmbeddingError<B: Backend> {
}
struct ReadoutQuantConfig {
+ method: QuantizationMethod,
mode: QuantizationMode,
group_size: u32,
}
@@ -40,7 +52,9 @@ enum TiedEmbeddingType<B: Backend> {
Quantized {
weights: Allocation<B>,
scales: Allocation<B>,
- biases: Allocation<B>,
+ zero_points_or_biases: Allocation<B>,
+ quantization_method: QuantizationMethod,
+ output_hadamard_factors: Option<Allocation<B>>,
lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
readout_config: ReadoutQuantConfig,
@@ -55,7 +69,9 @@ enum UntiedEmbeddingLookupType<B: Backend> {
Quantized {
weights: Allocation<B>,
scales: Allocation<B>,
- biases: Allocation<B>,
+ zero_points_or_biases: Allocation<B>,
+ quantization_method: QuantizationMethod,
+ output_hadamard_factors: Option<Allocation<B>>,
lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
},
}
@@ -68,7 +84,7 @@ enum UntiedEmbeddingReadoutType<B: Backend> {
Quantized {
weights: Allocation<B>,
scales: Allocation<B>,
- biases: Allocation<B>,
+ zero_points_or_biases: Allocation<B>,
readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
readout_config: ReadoutQuantConfig,
},
@@ -87,387 +103,330 @@ enum EmbeddingTying<B: Backend> {
pub struct Embedding<B: Backend> {
tying: EmbeddingTying<B>,
input_scale: f32,
- activation_data_type: DataType,
+ data_type: DataType,
vocab_size: u32,
model_dim: u32,
}
-struct UntiedWeightKeys {
- lookup_weights: &'static str,
- lookup_scales: &'static str,
- lookup_biases: &'static str,
- readout_weights: &'static str,
- readout_scales: &'static str,
- readout_biases: &'static str,
-}
-
-const SHARED_TREE_UNTIED_KEYS: UntiedWeightKeys = UntiedWeightKeys {
- lookup_weights: "input_weights",
- lookup_scales: "input_scales",
- lookup_biases: "input_biases",
- readout_weights: "output_weights",
- readout_scales: "output_scales",
- readout_biases: "output_biases",
-};
-
-#[cfg(metal_backend)]
-const SPLIT_TREE_UNTIED_KEYS: UntiedWeightKeys = UntiedWeightKeys {
- lookup_weights: "weights",
- lookup_scales: "scales",
- lookup_biases: "biases",
- readout_weights: "weights",
- readout_scales: "scales",
- readout_biases: "biases",
-};
-
impl<B: Backend> Embedding<B> {
pub fn new(
context: &B::Context,
vocab_size: u32,
model_dim: u32,
- config: &EmbeddingConfig,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, EmbeddingError<B>> {
- Self::new_with_weight_trees(
+ config: &AnyEmbeddingConfig,
+ parameter_tree: &ParameterTree<B>,
+ model_shape: &ModelShape,
+ ) -> Result<(Self, Option<Allocation<B>>), EmbeddingError<B>> {
+ Self::new_with_lookup_and_readout_trees(
context,
vocab_size,
model_dim,
config,
parameter_tree,
parameter_tree,
- &SHARED_TREE_UNTIED_KEYS,
- )
- }
-
- #[cfg(metal_backend)]
- pub(crate) fn new_with_lookup_and_readout_trees(
- context: &B::Context,
- vocab_size: u32,
- model_dim: u32,
- config: &EmbeddingConfig,
- lookup_tree: &ParameterTree<B::Context>,
- readout_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, EmbeddingError<B>> {
- Self::new_with_weight_trees(
- context,
- vocab_size,
- model_dim,
- config,
- lookup_tree,
- readout_tree,
- &SPLIT_TREE_UNTIED_KEYS,
+ model_shape,
)
}
- fn new_with_weight_trees(
+ pub fn new_with_lookup_and_readout_trees(
context: &B::Context,
vocab_size: u32,
model_dim: u32,
- config: &EmbeddingConfig,
- lookup_tree: &ParameterTree<B::Context>,
- readout_tree: &ParameterTree<B::Context>,
- untied_keys: &UntiedWeightKeys,
- ) -> Result<Self, EmbeddingError<B>> {
- let common = config.common();
-
- let tying = match config {
- EmbeddingConfig::Tied {
- common: _,
- precision,
- } => {
- let data_type = (*precision).into();
-
- let weights_leaf = lookup_tree.leaf("weights")?;
- weights_leaf.validate_shape(&[vocab_size as usize, model_dim as usize], data_type)?;
- let weights = weights_leaf.read_allocation()?;
-
- let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
- .map_err(EmbeddingError::BackendError)?;
- let readout = RefCell::new(
- <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
- .map_err(EmbeddingError::BackendError)?,
- );
-
- EmbeddingTying::Tied {
- ty: TiedEmbeddingType::FullPrecision {
- weights,
- lookup,
- readout,
+ config: &AnyEmbeddingConfig,
+ lookup_tree: &ParameterTree<B>,
+ readout_tree: &ParameterTree<B>,
+ model_shape: &ModelShape,
+ ) -> Result<(Self, Option<Allocation<B>>), EmbeddingError<B>> {
+ let (tying, readout_input_hadamard_factors) = match config {
+ AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {
+ let embedding_tree = lookup_tree.subtree("embedding")?;
+ let embedding_spec = embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
+
+ let (ty, readout_input_hadamard_factors) = match embedding_spec {
+ AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {
+ layout: Layout::InputOutput,
+ ..
+ }) => {
+ let weights = embedding_tree
+ .leaf("weights")?
+ .validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?
+ .read_allocation()?;
+
+ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(
+ context,
+ model_shape.data_type,
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ let readout = RefCell::new(
+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, model_shape.data_type)
+ .map_err(EmbeddingError::BackendError)?,
+ );
+
+ (
+ TiedEmbeddingType::FullPrecision {
+ weights,
+ lookup,
+ readout,
+ },
+ None,
+ )
},
- }
- },
- EmbeddingConfig::Untied {
- common: _,
- precision,
- } => {
- let data_type = (*precision).into();
-
- let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;
- input_weights_leaf.validate_shape(&[vocab_size as usize, model_dim as usize], data_type)?;
- let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;
- output_weights_leaf.validate_shape(&[vocab_size as usize, model_dim as usize], data_type)?;
-
- let input_weights = input_weights_leaf.read_allocation()?;
- let output_weights = output_weights_leaf.read_allocation()?;
-
- let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
- .map_err(EmbeddingError::BackendError)?;
- let readout = RefCell::new(
- <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
- .map_err(EmbeddingError::BackendError)?,
- );
-
- EmbeddingTying::Untied {
- input_ty: UntiedEmbeddingLookupType::FullPrecision {
- weights: input_weights,
- lookup,
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ let (embedding_quantization_mode, group_size, quantization_method) =
+ input_quantization_from_spec(spec)?;
+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(
+ &embedding_tree,
+ vocab_size as usize,
+ model_dim as usize,
+ model_shape.data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+
+ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ context,
+ model_shape.data_type,
+ group_size as u32,
+ embedding_quantization_mode,
+ quantization_method,
+ false,
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ let (readout, readout_config) = quantized_readout(
+ context,
+ model_shape.data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+
+ (
+ TiedEmbeddingType::Quantized {
+ weights,
+ scales,
+ zero_points_or_biases,
+ quantization_method,
+ output_hadamard_factors: None,
+ lookup,
+ readout,
+ readout_config,
+ },
+ None,
+ )
},
- output_ty: UntiedEmbeddingReadoutType::FullPrecision {
- weights: output_weights,
- readout,
+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {
+ quantization_spec,
+ adapter_spec: None,
+ incoherence_block_size: Some(32),
+ incoherence_processing_mode: IncoherenceProcessingMode::Output,
+ ..
+ }) => {
+ let (embedding_quantization_mode, group_size, quantization_method) =
+ input_quantization_from_spec(*quantization_spec)?;
+ let quantized_tree = embedding_tree.subtree("quantized")?;
+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(
+ &quantized_tree,
+ vocab_size as usize,
+ model_dim as usize,
+ model_shape.data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+
+ let incoherence_signs_tree = embedding_tree.subtree("incoherence_signs")?;
+ let output_hadamard_factors = Some(
+ incoherence_signs_tree
+ .leaf("output_signs")?
+ .validate(&[model_dim as usize], DataType::I32)?
+ .read_allocation()?,
+ );
+ let readout_input_hadamard_factors = Some(
+ incoherence_signs_tree
+ .leaf("output_signs")?
+ .validate(&[model_dim as usize], DataType::I32)?
+ .read_allocation()?,
+ );
+ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ context,
+ model_shape.data_type,
+ group_size as u32,
+ embedding_quantization_mode,
+ quantization_method,
+ true,
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ let (readout, readout_config) = quantized_readout(
+ context,
+ model_shape.data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+
+ (
+ TiedEmbeddingType::Quantized {
+ weights,
+ scales,
+ zero_points_or_biases,
+ quantization_method,
+ output_hadamard_factors,
+ lookup,
+ readout,
+ readout_config,
+ },
+ readout_input_hadamard_factors,
+ )
},
- }
- },
- EmbeddingConfig::ScaleBiasQuantizedTied {
- common: _,
- group_size,
- embedding_quantization_mode,
- activation_quantization_mode,
- activation_precision,
- } => {
- let data_type = (*activation_precision).into();
-
- let packing_divisor = embedding_quantization_mode.packing_divisor();
- let storage_data_type = embedding_quantization_mode.storage_type();
-
- let num_groups = (model_dim as usize).div_ceil(*group_size);
-
- let weights_leaf = lookup_tree.leaf("weights")?;
- weights_leaf
- .validate_shape(&[vocab_size as usize, model_dim as usize / packing_divisor], storage_data_type)?;
- let scales_leaf = lookup_tree.leaf("scales")?;
- scales_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;
- let biases_leaf = lookup_tree.leaf("biases")?;
- biases_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;
-
- let weights = weights_leaf.read_allocation()?;
- let scales = scales_leaf.read_allocation()?;
- let biases = biases_leaf.read_allocation()?;
-
- let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
- context,
- data_type,
- *group_size as u32,
- (*embedding_quantization_mode).into(),
- )
- .map_err(EmbeddingError::BackendError)?;
- let readout_config = ReadoutQuantConfig {
- mode: *embedding_quantization_mode,
- group_size: *group_size as u32,
+ spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
};
- let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
- .map_err(EmbeddingError::BackendError)?;
- readout_kernel
- .preheat_quant_combo(
- context,
- MatmulQuantCombo {
- method: QuantizationMethod::ScaleBias,
- mode: *embedding_quantization_mode,
- group_size: *group_size as u32,
- },
- )
- .map_err(EmbeddingError::BackendError)?;
- let readout = RefCell::new(readout_kernel);
- if let Some(activation_quantization_mode) = activation_quantization_mode {
- return Err(EmbeddingError::UnsupportedConfiguration(format!(
- "activation_quantization_mode={activation_quantization_mode:?}"
- )));
- }
-
- EmbeddingTying::Tied {
- ty: TiedEmbeddingType::Quantized {
- weights,
- scales,
- biases,
- lookup,
- readout,
- readout_config,
+ (
+ EmbeddingTying::Tied {
+ ty,
},
- }
- },
- EmbeddingConfig::ScaleBiasQuantizedUntied {
- common: _,
- group_size,
- embedding_quantization_mode,
- activation_quantization_mode,
- activation_precision,
- } => {
- let data_type = (*activation_precision).into();
-
- let packing_divisor = embedding_quantization_mode.packing_divisor();
- let storage_data_type = embedding_quantization_mode.storage_type();
-
- let num_groups = (model_dim as usize).div_ceil(*group_size);
-
- let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;
- input_weights_leaf
- .validate_shape(&[vocab_size as usize, model_dim as usize / packing_divisor], storage_data_type)?;
- let input_scales_leaf = lookup_tree.leaf(untied_keys.lookup_scales)?;
- input_scales_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;
- let input_biases_leaf = lookup_tree.leaf(untied_keys.lookup_biases)?;
- input_biases_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;
-
- let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;
- output_weights_leaf
- .validate_shape(&[vocab_size as usize, model_dim as usize / packing_divisor], storage_data_type)?;
- let output_scales_leaf = readout_tree.leaf(untied_keys.readout_scales)?;
- output_scales_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;
- let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;
- output_biases_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;
-
- let input_weights = input_weights_leaf.read_allocation()?;
- let input_scales = input_scales_leaf.read_allocation()?;
- let input_biases = input_biases_leaf.read_allocation()?;
-
- let output_weights = output_weights_leaf.read_allocation()?;
- let output_scales = output_scales_leaf.read_allocation()?;
- let output_biases = output_biases_leaf.read_allocation()?;
-
- let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
- context,
- data_type,
- *group_size as u32,
- (*embedding_quantization_mode).into(),
+ readout_input_hadamard_factors,
)
- .map_err(EmbeddingError::BackendError)?;
- let readout_config = ReadoutQuantConfig {
- mode: *embedding_quantization_mode,
- group_size: *group_size as u32,
- };
- let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
- .map_err(EmbeddingError::BackendError)?;
- readout_kernel
- .preheat_quant_combo(
- context,
- MatmulQuantCombo {
- method: QuantizationMethod::ScaleBias,
- mode: *embedding_quantization_mode,
- group_size: *group_size as u32,
- },
- )
- .map_err(EmbeddingError::BackendError)?;
- let readout = RefCell::new(readout_kernel);
-
- if let Some(activation_quantization_mode) = activation_quantization_mode {
- return Err(EmbeddingError::UnsupportedConfiguration(format!(
- "activation_quantization_mode={activation_quantization_mode:?}"
- )));
- }
-
- EmbeddingTying::Untied {
- input_ty: UntiedEmbeddingLookupType::Quantized {
- weights: input_weights,
- scales: input_scales,
- biases: input_biases,
- lookup,
+ },
+ AnyEmbeddingConfig::UntiedEmbeddingConfig(_) => {
+ let input_embedding_tree = lookup_tree.subtree("input_embedding")?;
+ let input_embedding_spec = input_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
+
+ let input_ty = match input_embedding_spec {
+ AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {
+ layout: Layout::InputOutput,
+ ..
+ }) => {
+ let weights = input_embedding_tree
+ .leaf("weights")?
+ .validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?
+ .read_allocation()?;
+
+ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(
+ context,
+ model_shape.data_type,
+ )
+ .map_err(EmbeddingError::BackendError)?;
+
+ UntiedEmbeddingLookupType::FullPrecision {
+ weights,
+ lookup,
+ }
},
- output_ty: UntiedEmbeddingReadoutType::Quantized {
- weights: output_weights,
- scales: output_scales,
- biases: output_biases,
- readout,
- readout_config,
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ let (embedding_quantization_mode, group_size, quantization_method) =
+ input_quantization_from_spec(spec)?;
+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(
+ &input_embedding_tree,
+ vocab_size as usize,
+ model_dim as usize,
+ model_shape.data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+
+ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ context,
+ model_shape.data_type,
+ group_size as u32,
+ embedding_quantization_mode,
+ quantization_method,
+ false,
+ )
+ .map_err(EmbeddingError::BackendError)?;
+
+ UntiedEmbeddingLookupType::Quantized {
+ weights,
+ scales,
+ zero_points_or_biases,
+ quantization_method,
+ output_hadamard_factors: None,
+ lookup,
+ }
},
- }
- },
- EmbeddingConfig::ScaleBiasSemiQuantizedUntied {
- common: _,
- group_size,
- embedding_quantization_mode,
- activation_quantization_mode,
- activation_precision,
- } => {
- let data_type = (*activation_precision).into();
-
- let packing_divisor = embedding_quantization_mode.packing_divisor();
- let storage_data_type = embedding_quantization_mode.storage_type();
-
- let num_groups = (model_dim as usize).div_ceil(*group_size);
-
- let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;
- input_weights_leaf.validate_shape(&[vocab_size as usize, model_dim as usize], data_type)?;
-
- let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;
- output_weights_leaf
- .validate_shape(&[vocab_size as usize, model_dim as usize / packing_divisor], storage_data_type)?;
- let output_scales_leaf = readout_tree.leaf(untied_keys.readout_scales)?;
- output_scales_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;
- let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;
- output_biases_leaf.validate_shape(&[vocab_size as usize, num_groups], data_type)?;
-
- let input_weights = input_weights_leaf.read_allocation()?;
-
- let output_weights = output_weights_leaf.read_allocation()?;
- let output_scales = output_scales_leaf.read_allocation()?;
- let output_biases = output_biases_leaf.read_allocation()?;
-
- let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
- .map_err(EmbeddingError::BackendError)?;
- let readout_config = ReadoutQuantConfig {
- mode: *embedding_quantization_mode,
- group_size: *group_size as u32,
+ spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
};
- let mut readout_kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
- .map_err(EmbeddingError::BackendError)?;
- readout_kernel
- .preheat_quant_combo(
- context,
- MatmulQuantCombo {
- method: QuantizationMethod::ScaleBias,
- mode: *embedding_quantization_mode,
- group_size: *group_size as u32,
- },
- )
- .map_err(EmbeddingError::BackendError)?;
- let readout = RefCell::new(readout_kernel);
-
- if let Some(activation_quantization_mode) = activation_quantization_mode {
- return Err(EmbeddingError::UnsupportedConfiguration(format!(
- "activation_quantization_mode={activation_quantization_mode:?}"
- )));
- }
- EmbeddingTying::Untied {
- input_ty: UntiedEmbeddingLookupType::FullPrecision {
- weights: input_weights,
- lookup,
+ let output_embedding_tree = readout_tree.subtree("output_embedding")?;
+ let output_embedding_spec = output_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
+
+ let output_ty = match output_embedding_spec {
+ AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {
+ layout: Layout::OutputInput,
+ ..
+ }) => {
+ let weights = output_embedding_tree
+ .leaf("weights")?
+ .validate(&[vocab_size as usize, model_dim as usize], model_shape.data_type)?
+ .read_allocation()?;
+ let readout = RefCell::new(
+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, model_shape.data_type)
+ .map_err(EmbeddingError::BackendError)?,
+ );
+
+ UntiedEmbeddingReadoutType::FullPrecision {
+ weights,
+ readout,
+ }
},
- output_ty: UntiedEmbeddingReadoutType::Quantized {
- weights: output_weights,
- scales: output_scales,
- biases: output_biases,
- readout,
- readout_config,
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ let (embedding_quantization_mode, group_size, quantization_method) =
+ output_quantization_from_spec(spec)?;
+ let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(
+ &output_embedding_tree,
+ vocab_size as usize,
+ model_dim as usize,
+ model_shape.data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+ let (readout, readout_config) = quantized_readout(
+ context,
+ model_shape.data_type,
+ embedding_quantization_mode,
+ quantization_method,
+ group_size,
+ )?;
+
+ UntiedEmbeddingReadoutType::Quantized {
+ weights,
+ scales,
+ zero_points_or_biases,
+ readout,
+ readout_config,
+ }
},
- }
+ spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
+ };
+
+ (
+ EmbeddingTying::Untied {
+ input_ty,
+ output_ty,
+ },
+ None,
+ )
},
};
- let input_scale = common.input_scale.unwrap_or(1.0);
- let activation_data_type: DataType = config.activation_precision().into();
-
- if let Some(logit_soft_cap) = common.logit_soft_cap {
+ let input_scale = config.input_scale().unwrap_or(1.0);
+ if let Some(logit_soft_cap) = config.logit_soft_cap() {
return Err(EmbeddingError::UnsupportedConfiguration(format!("logit_soft_cap={logit_soft_cap:?}")));
}
- Ok(Self {
- tying,
- input_scale,
- activation_data_type,
- vocab_size,
- model_dim,
- })
+ Ok((
+ Self {
+ tying,
+ input_scale,
+ data_type: model_shape.data_type,
+ vocab_size,
+ model_dim,
+ },
+ readout_input_hadamard_factors,
+ ))
}
pub fn encode_lookup(
@@ -477,10 +436,7 @@ impl<B: Backend> Embedding<B> {
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, EmbeddingError<B>> {
let mut output = encoder
- .allocate_scratch(crate::array::size_for_shape(
- &[batch_dim, self.model_dim as usize],
- self.activation_data_type,
- ))
+ .allocate_scratch(size_for_shape(&[batch_dim, self.model_dim as usize], self.data_type))
.map_err(EmbeddingError::BackendError)?;
let batch_dim = batch_dim as u32;
@@ -515,7 +471,9 @@ impl<B: Backend> Embedding<B> {
TiedEmbeddingType::Quantized {
weights,
scales,
- biases,
+ zero_points_or_biases,
+ quantization_method,
+ output_hadamard_factors,
lookup,
readout: _,
readout_config: _,
@@ -526,17 +484,25 @@ impl<B: Backend> Embedding<B> {
UntiedEmbeddingLookupType::Quantized {
weights,
scales,
- biases,
+ zero_points_or_biases,
+ quantization_method,
+ output_hadamard_factors,
lookup,
},
output_ty: _,
} => {
+ let (zero_points, biases) = match quantization_method {
+ QuantizationMethod::ScaleZeroPoint => (Some(zero_points_or_biases), None),
+ QuantizationMethod::ScaleBias => (None, Some(zero_points_or_biases)),
+ };
lookup.encode(
token_ids,
weights,
scales,
+ zero_points,
biases,
&mut output,
+ output_hadamard_factors.as_ref(),
batch_dim,
self.vocab_size,
self.model_dim,
@@ -557,10 +523,7 @@ impl<B: Backend> Embedding<B> {
) -> Result<Allocation<B>, EmbeddingError<B>> {
assert!(batch_dim > 0, "Embedding readout requires at least one row");
let mut output_allocation = encoder
- .allocate_scratch(crate::array::size_for_shape(
- &[batch_dim, self.vocab_size as usize],
- self.activation_data_type,
- ))
+ .allocate_scratch(size_for_shape(&[batch_dim, self.vocab_size as usize], self.data_type))
.map_err(EmbeddingError::BackendError)?;
match &self.tying {
@@ -580,8 +543,6 @@ impl<B: Backend> Embedding<B> {
readout,
},
} => {
- let input_dim = self.model_dim as usize;
- let output_dim = self.vocab_size as usize;
readout
.borrow_mut()
.encode(
@@ -597,8 +558,8 @@ impl<B: Backend> Embedding<B> {
d: &mut output_allocation,
d_transform: MatmulDOps::none(),
m: batch_dim as u32,
- n: output_dim as u32,
- k: input_dim as u32,
+ n: self.vocab_size,
+ k: self.model_dim,
},
encoder,
)
@@ -609,7 +570,9 @@ impl<B: Backend> Embedding<B> {
TiedEmbeddingType::Quantized {
weights,
scales,
- biases,
+ zero_points_or_biases,
+ quantization_method: _,
+ output_hadamard_factors: _,
lookup: _,
readout,
readout_config,
@@ -621,17 +584,26 @@ impl<B: Backend> Embedding<B> {
UntiedEmbeddingReadoutType::Quantized {
weights,
scales,
- biases,
+ zero_points_or_biases,
readout,
readout_config,
},
} => {
- let b_variant: MatmulB<'_, B> = MatmulB::ScaleBiasDequant {
- b: weights,
- scales,
- biases,
- mode: readout_config.mode,
- group_size: readout_config.group_size,
+ let b: MatmulB<'_, B> = match readout_config.method {
+ QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {
+ b: weights,
+ scales,
+ biases: zero_points_or_biases,
+ mode: readout_config.mode,
+ group_size: readout_config.group_size,
+ },
+ QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {
+ b: weights,
+ scales,
+ zero_points: zero_points_or_biases,
+ mode: readout_config.mode,
+ group_size: readout_config.group_size,
+ },
};
readout
.borrow_mut()
@@ -639,7 +611,7 @@ impl<B: Backend> Embedding<B> {
MatmulArguments {
a: input_allocation,
a_offset: 0,
- b: b_variant,
+ b,
b_offset: 0,
b_leading_dimension: None,
b_transpose: true,
@@ -658,3 +630,124 @@ impl<B: Backend> Embedding<B> {
Ok(output_allocation)
}
}
+
+fn input_quantization_from_spec<B: Backend>(
+ spec: AnyWeightMatrixSpec
+) -> Result<(QuantizationMode, usize, QuantizationMethod), EmbeddingError<B>> {
+ match spec {
+ AnyWeightMatrixSpec::MLXSpec(MLXSpec {
+ bits,
+ group_size,
+ layout: Layout::InputOutput,
+ ..
+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleBias),
+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ bits,
+ group_size,
+ is_symmetric: false,
+ layout: Layout::InputOutput,
+ ..
+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),
+ spec => Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
+ }
+}
+
+fn output_quantization_from_spec<B: Backend>(
+ spec: AnyWeightMatrixSpec
+) -> Result<(QuantizationMode, usize, QuantizationMethod), EmbeddingError<B>> {
+ match spec {
+ AnyWeightMatrixSpec::MLXSpec(MLXSpec {
+ bits,
+ group_size,
+ layout: Layout::OutputInput,
+ ..
+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleBias),
+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ bits,
+ group_size,
+ is_symmetric: false,
+ layout: Layout::OutputInput,
+ ..
+ }) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),
+ spec => Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
+ }
+}
+
+fn quantization_mode<B: Backend>(
+ bits: u32,
+ group_size: usize,
+ method: QuantizationMethod,
+) -> Result<(QuantizationMode, usize, QuantizationMethod), EmbeddingError<B>> {
+ let mode = match bits {
+ 4 => QuantizationMode::U4,
+ 8 => QuantizationMode::U8,
+ _ => {
+ return Err(EmbeddingError::UnsupportedConfiguration(format!(
+ "{method:?} bits={bits}, group_size={group_size}"
+ )));
+ },
+ };
+ Ok((mode, group_size, method))
+}
+
+fn load_quantized_embedding_parts<B: Backend>(
+ tree: &ParameterTree<B>,
+ vocab_size: usize,
+ model_dim: usize,
+ data_type: DataType,
+ quantization_mode: QuantizationMode,
+ quantization_method: QuantizationMethod,
+ group_size: usize,
+) -> Result<(Allocation<B>, Allocation<B>, Allocation<B>), EmbeddingError<B>> {
+ let packing_divisor = quantization_mode.packing_divisor();
+ let storage_data_type = quantization_mode.storage_type();
+ let num_groups = model_dim.div_ceil(group_size);
+
+ let weights = tree
+ .leaf("weights")?
+ .validate(&[vocab_size, model_dim / packing_divisor], storage_data_type)?
+ .read_allocation()?;
+ let scales = tree.leaf("scales")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?;
+ let zero_points_or_biases = match quantization_method {
+ QuantizationMethod::ScaleBias => {
+ tree.leaf("biases")?.validate(&[vocab_size, num_groups], data_type)?.read_allocation()?
+ },
+ QuantizationMethod::ScaleZeroPoint => {
+ let expected_zero_points_entries = num_groups.div_ceil(packing_divisor);
+ tree.leaf("zero_points")?
+ .validate(&[vocab_size, expected_zero_points_entries], storage_data_type)?
+ .read_allocation()?
+ },
+ };
+
+ Ok((weights, scales, zero_points_or_biases))
+}
+
+fn quantized_readout<B: Backend>(
+ context: &B::Context,
+ data_type: DataType,
+ mode: QuantizationMode,
+ method: QuantizationMethod,
+ group_size: usize,
+) -> Result<(RefCell<<B::Kernels as ManualKernels>::MatmulKernel>, ReadoutQuantConfig), EmbeddingError<B>> {
+ let mut readout =
+ <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;
+ readout
+ .preheat_quant_combo(
+ context,
+ MatmulQuantCombo {
+ method,
+ mode,
+ group_size: group_size as u32,
+ },
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ Ok((
+ RefCell::new(readout),
+ ReadoutQuantConfig {
+ method,
+ mode,
+ group_size: group_size as u32,
+ },
+ ))
+}
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index b03a76811..2d3bfe7b6 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -2,25 +2,59 @@
use std::rc::Rc;
-use half::{bf16, f16};
+use thiserror::Error;
-use super::MixerExecutables;
-#[cfg(feature = "tracing")]
-use crate::backends::common::{Kernels, kernel::TensorAddBiasKernel};
-#[cfg(feature = "tracing")]
-use crate::forward_pass::traces::LayerActivationTrace;
use crate::{
DataType,
- backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
- config::{MixerConfig, TransformerConfig, TransformerLayerConfig},
+ backends::common::{Allocation, Backend, Encoder},
+ config::{
+ token_mixer::AnyTokenMixerConfig, transformer::TransformerConfig, transformer_layer::TransformerLayerConfig,
+ },
encodable_block::{
- Attention, AttentionArguments, DeltaNetArguments, DeltaNetMixer, Linear, MambaArguments, MambaMixer, Mlp,
- PostLayerScalar, QKVNorm, QkUnpack, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,
+ Attention, AttentionError, DeltaNetArguments, DeltaNetMixer, DeltaNetMixerError, MambaArguments, MambaMixer,
+ MambaMixerError, Mlp, MlpBlockError, PostLayerScalar, QkUnpack, RMSNorm, RMSNormError, Rope,
+ ShortConvArguments, ShortConvMixer, ShortConvMixerError, layer::MixerExecutables,
},
forward_pass::{cache_layers::LayerCacheAccess, state::RopeBuffers},
- parameters::ParameterTree,
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+#[cfg(feature = "tracing")]
+use crate::{
+ backends::common::{Kernels, kernel::TensorAddBiasKernel},
+ forward_pass::traces::LayerActivationTrace,
};
+#[derive(Debug, Error)]
+pub enum LayerExecutablesError<B: Backend> {
+ #[cfg(feature = "tracing")]
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
+ #[error("Attention error: {0}")]
+ AttentionError(#[from] AttentionError<B>),
+ #[error("Mamba mixer error: {0}")]
+ MambaMixerError(#[from] MambaMixerError<B>),
+ #[error("ShortConv mixer error: {0}")]
+ ShortConvMixerError(#[from] ShortConvMixerError<B>),
+ #[error("DeltaNet mixer error: {0}")]
+ DeltaNetMixerError(#[from] DeltaNetMixerError<B>),
+ #[error("MLP error: {0}")]
+ MlpBlockError(#[from] MlpBlockError<B>),
+ #[error("RMSNorm error: {0}")]
+ RMSNormError(#[from] RMSNormError<B>),
+ #[error("layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]
+ PostLayerScalarWithoutPostMlpNorm {
+ layer_index: usize,
+ },
+ #[error("Unsupported post_layer_scalar data type: {data_type:?}")]
+ UnsupportedPostLayerScalarDataType {
+ data_type: DataType,
+ },
+ #[error("decoder layers require pre_mixer_norm_config")]
+ MissingPreMixerNormConfig,
+}
+
/// A single decoder layer with all its components.
pub struct LayerExecutables<B: Backend> {
pub layer_index: usize,
@@ -43,25 +77,27 @@ impl<B: Backend> LayerExecutables<B> {
transformer_config: &TransformerConfig,
layer_config: &TransformerLayerConfig,
layer_index: usize,
- decoder_layer_loader: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
rope: &Rc<Rope<B>>,
qk_unpack: &Rc<QkUnpack<B>>,
- ) -> Self {
- let intermediate_data_type: DataType = layer_config.mixer_config.activation_precision().into();
-
+ data_type: DataType,
+ ) -> Result<Self, LayerExecutablesError<B>> {
let (residual_sum_scalar, output_scalar) = if layer_config.has_post_layer_scalar {
- assert!(
- layer_config.post_mlp_norm_config.is_some(),
- "layer {layer_index} sets post_layer_scalar but has no post_mlp_norm"
- );
- let leaf = decoder_layer_loader.leaf("post_layer_scalar").expect("Failed to read post_layer_scalar weight");
- let scalar = match leaf.data_type() {
- DataType::BF16 => {
- leaf.read_slice::<bf16>().expect("Failed to read post_layer_scalar weight")[0].to_f32()
+ if layer_config.post_mlp_norm_config.is_none() {
+ return Err(LayerExecutablesError::PostLayerScalarWithoutPostMlpNorm {
+ layer_index,
+ });
+ }
+ let leaf = parameter_tree.leaf("post_layer_scalar")?;
+ let scalar = match data_type {
+ DataType::F32 => leaf.validate(&[1], data_type)?.read_slice::<f32>()?[0],
+ DataType::F16 => leaf.validate(&[1], data_type)?.read_slice::<half::f16>()?[0].to_f32(),
+ DataType::BF16 => leaf.validate(&[1], data_type)?.read_slice::<half::bf16>()?[0].to_f32(),
+ data_type => {
+ return Err(LayerExecutablesError::UnsupportedPostLayerScalarDataType {
+ data_type,
+ });
},
- DataType::F16 => leaf.read_slice::<f16>().expect("Failed to read post_layer_scalar weight")[0].to_f32(),
- DataType::F32 => leaf.read_slice::<f32>().expect("Failed to read post_layer_scalar weight")[0],
- other => panic!("post_layer_scalar must be a float dtype, got {other:?}"),
};
(PostLayerScalar::ScaleResidualSum(scalar), PostLayerScalar::ScaleOutput(scalar))
} else {
@@ -69,128 +105,34 @@ impl<B: Backend> LayerExecutables<B> {
};
#[cfg(feature = "tracing")]
- let tensor_add = TensorAddBiasKernel::new(context, intermediate_data_type, false)
- .expect("Failed to create TensorAddBiasKernel kernel"); // TODO: this function return Result
-
- let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {
- MixerConfig::Attention(attention_config) => {
- let q_dim = attention_config.num_heads * attention_config.head_dim;
- let kv_dim = attention_config.num_groups * attention_config.head_dim;
-
- let (qkv_projection, input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
- &attention_config.qkv_projection_config,
- transformer_config.model_dim,
- [q_dim, kv_dim, kv_dim],
- context,
- &decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),
- )
- .expect("Failed to create qkv projection");
+ let tensor_add =
+ TensorAddBiasKernel::new(context, data_type, false).map_err(LayerExecutablesError::BackendError)?;
- let gate_projection = attention_config.gate_projection_config.as_ref().map(|gate_config| {
- let gate_tree = decoder_layer_loader.subtree("mixer.gate_projection").unwrap();
- match (input_hadamard_factors.is_some(), gate_config) {
- (
- true,
- crate::config::LinearConfig::RHTLinearWrapper {
- inner_config,
- ..
- },
- ) => {
- let output_factors = gate_tree
- .leaf("output_factors")
- .expect("Failed to get gate projection output_factors")
- .read_allocation()
- .expect("Failed to read gate projection output_factors");
- let inner_tree = gate_tree
- .subtree("inner_linear")
- .expect("Failed to get gate projection inner_linear subtree");
- <dyn Linear<B>>::new_with_output_hadamard(
- context,
- inner_config,
- &inner_tree,
- output_factors,
- transformer_config.model_dim,
- q_dim,
- )
- },
- (
- false,
- crate::config::LinearConfig::RHTLinearWrapper {
- ..
- },
- )
- | (true, _) => {
- panic!("attention qkv/gate projections must share input hadamard")
- },
- (false, _) => <dyn Linear<B>>::new(
- gate_config,
- transformer_config.model_dim,
- [q_dim],
- context,
- &gate_tree,
- ),
- }
- .expect("Failed to create gate projection")
- });
-
- let value_norm_config = attention_config.value_norm_config();
- let qkv_norm = if attention_config.query_norm_config.is_some()
- || attention_config.key_norm_config.is_some()
- || value_norm_config.is_some()
- {
- match QKVNorm::new(
- context,
- intermediate_data_type,
- attention_config.query_norm_config.clone(),
- attention_config.key_norm_config.clone(),
- value_norm_config,
- &decoder_layer_loader.subtree("mixer").unwrap(),
- attention_config.num_heads,
- attention_config.num_groups,
- attention_config.head_dim,
- ) {
- Ok(qkv_norm) => Some(qkv_norm),
- Err(error) => {
- panic!("Failed to create QKV norm kernel for layer {}: {:?}", layer_index, error)
- },
- }
- } else {
- None
- };
+ let mixer_subtree = parameter_tree.subtree("mixer")?;
- let out_projection = <dyn Linear<B>>::new(
- &attention_config.out_projection_config,
- q_dim,
- [transformer_config.model_dim],
+ let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {
+ AnyTokenMixerConfig::AttentionConfig(attention_config) => {
+ let (attention, input_hadamard_factors) = Attention::new(
context,
- &decoder_layer_loader.subtree("mixer.out_projection").unwrap(),
- )
- .expect("Failed to create out projection");
-
- let attention =
- Attention::new(context, intermediate_data_type, attention_config, gate_projection.is_some())
- .expect("Failed to create AttentionWrapper kernel");
+ transformer_config.model_dim,
+ data_type,
+ attention_config,
+ &mixer_subtree,
+ rope.clone(),
+ qk_unpack.clone(),
+ attention_config.gate_projection_config.is_none(),
+ )?;
(
MixerExecutables::Attention {
- qkv_projection,
- gate_projection,
- qkv_norm,
- rope: rope.clone(),
- qk_unpack: qk_unpack.clone(),
attention,
- out_projection,
- num_heads: attention_config.num_heads,
- num_groups: attention_config.num_groups,
- head_dim: attention_config.head_dim,
},
input_hadamard_factors,
)
},
- MixerConfig::Mamba(mamba_config) => {
+ AnyTokenMixerConfig::Mamba2Config(mamba_config) => {
let (mixer, input_hadamard_factors) =
- MambaMixer::new(context, mamba_config.clone(), transformer_config.model_dim, decoder_layer_loader)
- .expect("Failed to create Mamba mixer");
+ MambaMixer::new(context, mamba_config, transformer_config.model_dim, &mixer_subtree, data_type)?;
(
MixerExecutables::StateSpace {
mixer,
@@ -198,14 +140,14 @@ impl<B: Backend> LayerExecutables<B> {
input_hadamard_factors,
)
},
- MixerConfig::ShortConv(short_conv_config) => {
+ AnyTokenMixerConfig::ShortConvConfig(short_conv_config) => {
let (mixer, input_hadamard_factors) = ShortConvMixer::new(
context,
- short_conv_config.clone(),
+ short_conv_config,
transformer_config.model_dim,
- decoder_layer_loader,
- )
- .expect("Failed to create ShortConv mixer");
+ &mixer_subtree,
+ data_type,
+ )?;
(
MixerExecutables::ShortConv {
mixer,
@@ -213,14 +155,14 @@ impl<B: Backend> LayerExecutables<B> {
input_hadamard_factors,
)
},
- MixerConfig::DeltaNet(delta_net_config) => {
+ AnyTokenMixerConfig::DeltaNetConfig(delta_net_config) => {
let (mixer, input_hadamard_factors) = DeltaNetMixer::new(
context,
- delta_net_config.clone(),
+ delta_net_config,
transformer_config.model_dim,
- decoder_layer_loader,
- )
- .expect("Failed to create DeltaNet mixer");
+ &mixer_subtree,
+ data_type,
+ )?;
(
MixerExecutables::DeltaNet {
mixer,
@@ -232,30 +174,28 @@ impl<B: Backend> LayerExecutables<B> {
let pre_mixer_norm = RMSNorm::new(
context,
- intermediate_data_type,
- layer_config.pre_mixer_norm_config.clone().expect("decoder layers require pre_mixer_norm_config"),
- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
+ data_type,
+ transformer_config.model_dim,
+ layer_config.pre_mixer_norm_config.clone().ok_or(LayerExecutablesError::MissingPreMixerNormConfig)?,
+ &parameter_tree.subtree("pre_mixer_norm")?,
mixer_hadamard_factors,
true,
layer_index > 0,
PostLayerScalar::None,
- )
- .expect("Failed to create RMS norm kernel");
+ )?;
let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {
- Some(
- RMSNorm::new(
- context,
- intermediate_data_type,
- norm_config.clone(),
- &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
- None,
- false,
- false,
- PostLayerScalar::None,
- )
- .expect("Failed to create RMS norm kernel"),
- )
+ Some(RMSNorm::new(
+ context,
+ data_type,
+ transformer_config.model_dim,
+ norm_config.clone(),
+ &parameter_tree.subtree("post_mixer_norm")?,
+ None,
+ false,
+ false,
+ PostLayerScalar::None,
+ )?)
} else {
None
};
@@ -265,41 +205,39 @@ impl<B: Backend> LayerExecutables<B> {
transformer_config.model_dim,
layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),
context,
- &decoder_layer_loader.subtree("mlp").unwrap(),
- )
- .expect("Failed to create mlp block");
+ &parameter_tree.subtree("mlp")?,
+ data_type,
+ )?;
let pre_mlp_norm = RMSNorm::new(
context,
- intermediate_data_type,
+ data_type,
+ transformer_config.model_dim,
layer_config.pre_mlp_norm_config.clone(),
- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
+ &parameter_tree.subtree("pre_mlp_norm")?,
mlp_input_hadamard_factors,
true,
true,
residual_sum_scalar,
- )
- .expect("Failed to create RMS norm kernel");
+ )?;
let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
- Some(
- RMSNorm::new(
- context,
- intermediate_data_type,
- norm_config.clone(),
- &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
- None,
- false,
- false,
- output_scalar,
- )
- .expect("Failed to create RMS norm kernel"),
- )
+ Some(RMSNorm::new(
+ context,
+ data_type,
+ transformer_config.model_dim,
+ norm_config.clone(),
+ &parameter_tree.subtree("post_mlp_norm")?,
+ None,
+ false,
+ false,
+ output_scalar,
+ )?)
} else {
None
};
- Self {
+ Ok(Self {
layer_index,
#[cfg(feature = "tracing")]
tensor_add,
@@ -311,7 +249,7 @@ impl<B: Backend> LayerExecutables<B> {
post_mlp_norm,
#[cfg(feature = "tracing")]
model_dim: transformer_config.model_dim,
- }
+ })
}
pub fn encode(
@@ -326,7 +264,6 @@ impl<B: Backend> LayerExecutables<B> {
token_positions,
token_parents,
token_subtrie_ranges,
- attention_sinks,
rope_buffers,
sampling_start,
sampling_length,
@@ -346,67 +283,16 @@ impl<B: Backend> LayerExecutables<B> {
hidden = match &self.mixer {
MixerExecutables::Attention {
- qkv_projection,
- gate_projection,
- qkv_norm,
- rope,
- qk_unpack,
attention,
- out_projection,
- num_heads,
- num_groups,
- head_dim,
- } => {
- let gate_input = if gate_projection.is_some() {
- let hidden_len = hidden.as_buffer_range_ref().range().len();
- let mut gate_input = encoder.allocate_scratch(hidden_len)?;
- encoder.encode_copy(&hidden, .., &mut gate_input, ..);
- Some(gate_input)
- } else {
- None
- };
- let mut qkv = qkv_projection.encode(hidden, batch_dim, encoder)?;
- let gate = match (gate_projection, gate_input) {
- (Some(gate_proj), Some(gate_input)) => Some(gate_proj.encode(gate_input, batch_dim, encoder)?),
- _ => None,
- };
- if let Some(norm) = qkv_norm {
- norm.encode(&mut qkv, batch_dim, encoder)?;
- }
- let (queries, rotated_keys) = match rope_buffers {
- Some(rope_buffers) => rope.encode(
- &qkv,
- token_positions,
- &rope_buffers.cosines,
- &rope_buffers.sines,
- batch_dim,
- *num_heads,
- *num_groups,
- *head_dim,
- rope_buffers.max_sequence_length(),
- rope_buffers.dim(),
- encoder,
- )?,
- None => qk_unpack.encode(&qkv, batch_dim, *num_heads, *num_groups, *head_dim, encoder)?,
- };
- let attention_output = attention.encode(
- AttentionArguments {
- token_subtrie_ranges,
- attention_sinks,
- cache_access,
- },
- &qkv,
- &queries,
- rotated_keys,
- gate.as_ref(),
- batch_dim,
- *num_heads,
- *num_groups,
- *head_dim,
- encoder,
- )?;
- out_projection.encode(attention_output, batch_dim, encoder)?
- },
+ } => attention.encode(
+ token_positions,
+ token_subtrie_ranges,
+ rope_buffers,
+ cache_access,
+ hidden,
+ batch_dim,
+ encoder,
+ )?,
MixerExecutables::StateSpace {
mixer,
} => {
@@ -524,7 +410,6 @@ pub struct LayerArguments<'a, B: Backend> {
pub token_positions: &'a Allocation<B>,
pub token_parents: &'a Allocation<B>,
pub token_subtrie_ranges: Option<&'a Allocation<B>>,
- pub attention_sinks: Option<&'a Allocation<B>>,
pub rope_buffers: Option<&'a RopeBuffers<B>>,
pub sampling_start: usize,
pub sampling_length: usize,
diff --git a/crates/backend-uzu/src/encodable_block/layer/mixer.rs b/crates/backend-uzu/src/encodable_block/layer/mixer.rs
index c3ef46033..2acf6117e 100644
--- a/crates/backend-uzu/src/encodable_block/layer/mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/mixer.rs
@@ -1,25 +1,14 @@
//! Mixer component - either attention or state space.
-use std::rc::Rc;
-
use crate::{
backends::common::Backend,
- encodable_block::{Attention, DeltaNetMixer, MambaMixer, QKVNorm, QkUnpack, Rope, ShortConvMixer, linear::Linear},
+ encodable_block::{Attention, DeltaNetMixer, MambaMixer, ShortConvMixer},
};
/// Mixer component - either attention, state space, short conv, or delta net.
pub(crate) enum MixerExecutables<B: Backend> {
Attention {
- qkv_projection: Box<dyn Linear<B>>,
- gate_projection: Option<Box<dyn Linear<B>>>,
- qkv_norm: Option<QKVNorm<B>>,
- rope: Rc<Rope<B>>,
- qk_unpack: Rc<QkUnpack<B>>,
attention: Attention<B>,
- out_projection: Box<dyn Linear<B>>,
- num_heads: usize,
- num_groups: usize,
- head_dim: usize,
},
StateSpace {
mixer: MambaMixer<B>,
diff --git a/crates/backend-uzu/src/encodable_block/layer/mod.rs b/crates/backend-uzu/src/encodable_block/layer/mod.rs
index 70b5019cb..99ea4fb08 100644
--- a/crates/backend-uzu/src/encodable_block/layer/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/mod.rs
@@ -3,5 +3,5 @@
mod executables;
mod mixer;
-pub use executables::{LayerArguments, LayerExecutables};
+pub use executables::{LayerArguments, LayerExecutables, LayerExecutablesError};
pub(crate) use mixer::MixerExecutables;
diff --git a/crates/backend-uzu/src/encodable_block/layer_norm.rs b/crates/backend-uzu/src/encodable_block/layer_norm.rs
index 90a08cd16..31647d0f2 100644
--- a/crates/backend-uzu/src/encodable_block/layer_norm.rs
+++ b/crates/backend-uzu/src/encodable_block/layer_norm.rs
@@ -9,7 +9,7 @@ use crate::{
Allocation, Backend, Encoder,
kernel::{Kernels, LayerNormKernel},
},
- config::{NormalizationConfig, UpcastMode},
+ config::normalization::{NormalizationConfig, UpcastMode},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -18,7 +18,7 @@ pub enum LayerNormError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
#[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError<B>),
+ ParameterError(#[from] ParameterLoaderError<B>),
}
pub struct LayerNorm<B: Backend> {
@@ -34,20 +34,20 @@ impl<B: Backend> LayerNorm<B> {
pub fn new(
context: &B::Context,
intermediate_data_type: DataType,
+ element_count: usize,
config: NormalizationConfig,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
) -> Result<Self, LayerNormError<B>> {
- let scales_leaf = parameter_tree.leaf("scales").map_err(LayerNormError::ParameterError)?;
- let element_count = scales_leaf.shape()[0];
- let scales = scales_leaf.read_allocation().map_err(LayerNormError::ParameterError)?;
+ let scale_data_type = super::normalization::NORMALIZATION_SCALE_DATA_TYPE;
+ let scales = parameter_tree.leaf("scales")?.validate(&[element_count], scale_data_type)?.read_allocation()?;
- let accumulation_data_type: DataType = config.accumulation_precision.into();
- let scale_data_type: DataType = config.scale_precision.into();
+ let accumulation_data_type = super::normalization::NORMALIZATION_ACCUMULATION_DATA_TYPE;
+ let output_data_type = intermediate_data_type;
let kernel = <B::Kernels as Kernels>::LayerNormKernel::new(
context,
intermediate_data_type,
scale_data_type,
- scale_data_type,
+ output_data_type,
accumulation_data_type,
false,
)
@@ -59,7 +59,7 @@ impl<B: Backend> LayerNorm<B> {
scales,
element_count,
input_data_type: intermediate_data_type,
- output_data_type: scale_data_type,
+ output_data_type,
})
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
index 8524b3e7f..4fccea9ee 100644
--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
@@ -14,7 +14,7 @@ use crate::{
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},
},
},
- config::QuantizationConfig,
+ config::weight_matrix::{AnyWeightMatrixSpec, Layout, awq_spec::AWQSpec, mlx_spec::MLXSpec},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -26,61 +26,8 @@ pub enum LinearMatmulError<B: Backend> {
ParameterError(#[from] ParameterLoaderError<B>),
#[error("Unsupported data type: {0:?}")]
UnsupportedDataType(DataType),
- #[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]
- InvalidFpWeightsShape {
- got: Box<[usize]>,
- expected_output_dim: usize,
- expected_input_dim: usize,
- },
- #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidWeightsDataType {
- expected: DataType,
- got: DataType,
- },
- #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidScalesDataType {
- expected: DataType,
- got: DataType,
- },
- #[error(
- "Unexpected scale-bias shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"
- )]
- InvalidScaleBiasShapes {
- weights: Box<[usize]>,
- scales: Box<[usize]>,
- deq_biases: Box<[usize]>,
- packing_divisor: usize,
- },
- #[error("deq_biases dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidDeqBiasesDataType {
- expected: DataType,
- got: DataType,
- },
- #[error(
- "Unexpected scale-zero-point shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"
- )]
- InvalidScaleZeroPointShapes {
- weights: Box<[usize]>,
- scales: Box<[usize]>,
- zero_points: Box<[usize]>,
- packing_divisor: usize,
- packing_minus_one: usize,
- },
- #[error("Zero-points dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidZeroPointsDataType {
- expected: DataType,
- got: DataType,
- },
- #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]
- InvalidBiasShape {
- got: Box<[usize]>,
- expected_output_dim: usize,
- },
- #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidBiasDataType {
- expected: DataType,
- got: DataType,
- },
+ #[error("Unsupported linear matmul configuration: {0}")]
+ UnsupportedConfiguration(String),
}
enum Mode<B: Backend> {
@@ -108,36 +55,22 @@ pub struct LinearMatmul<B: Backend> {
impl<B: Backend> LinearMatmul<B> {
pub fn full_precision(
context: &B::Context,
- precision: DataType,
input_dim: usize,
output_dim: usize,
- parameter_tree: &ParameterTree<B::Context>,
+ has_biases: bool,
+ data_type: DataType,
+ parameter_tree: &ParameterTree<B>,
) -> Result<Self, LinearMatmulError<B>> {
- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(LinearMatmulError::UnsupportedDataType(precision));
- }
-
- let weights_leaf = parameter_tree.leaf("weights")?;
- let weights_shape = weights_leaf.shape().to_vec();
- if weights_shape != [output_dim, input_dim] {
- return Err(LinearMatmulError::InvalidFpWeightsShape {
- got: weights_shape.into_boxed_slice(),
- expected_output_dim: output_dim,
- expected_input_dim: input_dim,
- });
- }
- if weights_leaf.data_type() != precision {
- return Err(LinearMatmulError::InvalidWeightsDataType {
- expected: precision,
- got: weights_leaf.data_type(),
- });
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(LinearMatmulError::UnsupportedDataType(data_type));
}
- let biases = load_biases(precision, output_dim, parameter_tree)?;
+ let weights =
+ parameter_tree.leaf("weights.weights")?.validate(&[output_dim, input_dim], data_type)?.read_allocation()?;
+ let biases = load_biases(data_type, output_dim, has_biases.then_some(parameter_tree))?;
- let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, precision)
+ let kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
.map_err(LinearMatmulError::BackendError)?;
- let weights = weights_leaf.read_allocation()?;
Ok(Self {
kernel: RefCell::new(kernel),
@@ -145,96 +78,75 @@ impl<B: Backend> LinearMatmul<B> {
biases,
input_dim,
output_dim,
- data_type: precision,
+ data_type,
mode: Mode::FullPrecision,
})
}
pub fn quantized(
context: &B::Context,
- config: &QuantizationConfig,
+ spec: AnyWeightMatrixSpec,
input_dim: usize,
output_dim: usize,
- parameter_tree: &ParameterTree<B::Context>,
+ data_type: DataType,
+ weights_tree: &ParameterTree<B>,
+ bias_tree: Option<&ParameterTree<B>>,
output_hadamard_factors: Option<Allocation<B>>,
) -> Result<Self, LinearMatmulError<B>> {
- let data_type: DataType = config.activation_precision.into();
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(LinearMatmulError::UnsupportedDataType(data_type));
- }
-
- let packing_divisor = config.weight_quantization_mode.packing_divisor();
- let storage_type = config.weight_quantization_mode.storage_type();
+ let (bits, group_size, quantization_method) = match spec {
+ AnyWeightMatrixSpec::MLXSpec(MLXSpec {
+ bits,
+ group_size,
+ layout: Layout::OutputInput,
+ ..
+ }) => (bits, group_size, QuantizationMethod::ScaleBias),
+ AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ bits,
+ group_size,
+ is_symmetric: false,
+ layout: Layout::OutputInput,
+ ..
+ }) => (bits, group_size, QuantizationMethod::ScaleZeroPoint),
+ spec => return Err(LinearMatmulError::UnsupportedConfiguration(format!("{spec:?}"))),
+ };
- let weights_leaf = parameter_tree.leaf("weights")?;
- if weights_leaf.data_type() != storage_type {
- return Err(LinearMatmulError::InvalidWeightsDataType {
- expected: storage_type,
- got: weights_leaf.data_type(),
- });
- }
+ let weight_quantization_mode = match bits {
+ 4 => QuantizationMode::U4,
+ 8 => QuantizationMode::U8,
+ _ => {
+ return Err(LinearMatmulError::UnsupportedConfiguration(format!(
+ "{quantization_method} bits={bits}, group_size={group_size}"
+ )));
+ },
+ };
- let scales_leaf = parameter_tree.leaf("scales")?;
- if scales_leaf.data_type() != data_type {
- return Err(LinearMatmulError::InvalidScalesDataType {
- expected: data_type,
- got: scales_leaf.data_type(),
- });
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(LinearMatmulError::UnsupportedDataType(data_type));
}
- let k_g = input_dim.div_ceil(config.group_size);
- let weights_shape = weights_leaf.shape().to_vec();
- let scales_shape = scales_leaf.shape().to_vec();
+ let packing_divisor = weight_quantization_mode.packing_divisor();
+ let storage_type = weight_quantization_mode.storage_type();
+ let k_g = input_dim.div_ceil(group_size);
- let (quantization_method, zero_points_or_biases) = match parameter_tree.leaf("deq_biases") {
- Ok(deq_biases) => {
- let deq_biases_shape = deq_biases.shape().to_vec();
- if !(weights_shape == [output_dim, input_dim / packing_divisor]
- && scales_shape == [output_dim, k_g]
- && deq_biases_shape == [output_dim, k_g])
- {
- return Err(LinearMatmulError::InvalidScaleBiasShapes {
- weights: weights_shape.into_boxed_slice(),
- scales: scales_shape.into_boxed_slice(),
- deq_biases: deq_biases_shape.into_boxed_slice(),
- packing_divisor,
- });
- }
- if deq_biases.data_type() != data_type {
- return Err(LinearMatmulError::InvalidDeqBiasesDataType {
- expected: data_type,
- got: deq_biases.data_type(),
- });
- }
- (QuantizationMethod::ScaleBias, deq_biases.read_allocation()?)
+ let weights = weights_tree
+ .leaf("weights")?
+ .validate(&[output_dim, input_dim / packing_divisor], storage_type)?
+ .read_allocation()?;
+ let scales = weights_tree.leaf("scales")?.validate(&[output_dim, k_g], data_type)?.read_allocation()?;
+ let zero_points_or_biases = match quantization_method {
+ QuantizationMethod::ScaleBias => {
+ weights_tree.leaf("biases")?.validate(&[output_dim, k_g], data_type)?.read_allocation()?
},
- Err(_) => {
- let zero_points_leaf = parameter_tree.leaf("zero_points")?;
- let zero_points_shape = zero_points_leaf.shape().to_vec();
+ QuantizationMethod::ScaleZeroPoint => {
let expected_zero_points_entries = k_g.div_ceil(packing_divisor);
- if !(weights_shape == [output_dim, input_dim / packing_divisor]
- && scales_shape == [output_dim, k_g]
- && zero_points_shape == [output_dim, expected_zero_points_entries])
- {
- return Err(LinearMatmulError::InvalidScaleZeroPointShapes {
- weights: weights_shape.into_boxed_slice(),
- scales: scales_shape.into_boxed_slice(),
- zero_points: zero_points_shape.into_boxed_slice(),
- packing_divisor,
- packing_minus_one: packing_divisor - 1,
- });
- }
- if zero_points_leaf.data_type() != storage_type {
- return Err(LinearMatmulError::InvalidZeroPointsDataType {
- expected: storage_type,
- got: zero_points_leaf.data_type(),
- });
- }
- (QuantizationMethod::ScaleZeroPoint, zero_points_leaf.read_allocation()?)
+ weights_tree
+ .leaf("zero_points")?
+ .validate(&[output_dim, expected_zero_points_entries], storage_type)?
+ .read_allocation()?
},
};
- let biases = load_biases(data_type, output_dim, parameter_tree)?;
+ let biases = load_biases(data_type, output_dim, bias_tree)?;
let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(context, data_type)
.map_err(LinearMatmulError::BackendError)?;
@@ -243,24 +155,24 @@ impl<B: Backend> LinearMatmul<B> {
context,
MatmulQuantCombo {
method: quantization_method,
- mode: config.weight_quantization_mode,
- group_size: config.group_size as u32,
+ mode: weight_quantization_mode,
+ group_size: group_size as u32,
},
)
.map_err(LinearMatmulError::BackendError)?;
Ok(Self {
kernel: RefCell::new(kernel),
- weights: weights_leaf.read_allocation()?,
+ weights,
biases,
input_dim,
output_dim,
data_type,
mode: Mode::Quantized {
method: quantization_method,
- mode: config.weight_quantization_mode,
- group_size: config.group_size as u32,
- scales: scales_leaf.read_allocation()?,
+ mode: weight_quantization_mode,
+ group_size: group_size as u32,
+ scales,
zero_points_or_biases,
output_hadamard_factors,
},
@@ -271,27 +183,11 @@ impl<B: Backend> LinearMatmul<B> {
fn load_biases<B: Backend>(
data_type: DataType,
output_dim: usize,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: Option<&ParameterTree<B>>,
) -> Result<Option<Allocation<B>>, LinearMatmulError<B>> {
- match parameter_tree.leaf("biases") {
- Ok(biases_leaf) => {
- let bias_shape = biases_leaf.shape().to_vec();
- if bias_shape != [output_dim] {
- return Err(LinearMatmulError::InvalidBiasShape {
- got: bias_shape.into_boxed_slice(),
- expected_output_dim: output_dim,
- });
- }
- if biases_leaf.data_type() != data_type {
- return Err(LinearMatmulError::InvalidBiasDataType {
- expected: data_type,
- got: biases_leaf.data_type(),
- });
- }
- Ok(Some(biases_leaf.read_allocation()?))
- },
- Err(_) => Ok(None),
- }
+ Ok(parameter_tree
+ .map(|tree| tree.leaf("biases")?.validate(&[output_dim], data_type)?.read_allocation())
+ .transpose()?)
}
impl<B: Backend> Linear<B> for LinearMatmul<B> {
diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rs
index a7056b36c..178f45d0b 100644
--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs
@@ -8,8 +8,13 @@ pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};
use thiserror::Error;
use crate::{
+ DataType,
backends::common::{Allocation, Backend, Encoder},
- config::LinearConfig,
+ config::weight_matrix::{
+ AnyWeightMatrixSpec, Layout,
+ full_precision_spec::FullPrecisionSpec,
+ hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
+ },
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -26,138 +31,178 @@ pub trait Linear<B: Backend> {
pub enum LinearBlockError<B: Backend> {
#[error("LinearMatmul error: {0}")]
LinearMatmulError(#[from] LinearMatmulError<B>),
+ #[error("Output hadamard linear error: {0}")]
+ OutputHadamardLinearError(#[from] OutputHadamardLinearError<B>),
#[error("QLoRALinearWrapper error: {0}")]
QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),
#[error("RHTLinearWrapper error: {0}")]
RHTLinearWrapperError(#[from] RHTLinearWrapperError<B>),
#[error("Parameter loading error: {0}")]
ParameterError(#[from] ParameterLoaderError<B>),
+ #[error("Unsupported linear configuration: {0}")]
+ UnsupportedConfiguration(String),
+}
+
+#[derive(Debug, Error)]
+pub enum OutputHadamardLinearError<B: Backend> {
+ #[error("LinearMatmul error: {0}")]
+ LinearMatmulError(#[from] LinearMatmulError<B>),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(#[from] ParameterLoaderError<B>),
+ #[error("Unsupported linear configuration: {0}")]
+ UnsupportedConfiguration(String),
}
impl<B: Backend> dyn Linear<B> {
pub fn new<const N: usize>(
- config: &LinearConfig,
input_dimension: usize,
output_dimensions: [usize; N],
+ has_biases: bool,
context: &B::Context,
- parameter_tree: &ParameterTree<B::Context>,
+ data_type: DataType,
+ parameter_tree: &ParameterTree<B>,
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
- match config {
- LinearConfig::Quantized(quantization_config) | LinearConfig::ScaleBiasQuantized(quantization_config) => {
- let block = LinearMatmul::quantized(
+ let weights_tree = parameter_tree.subtree("weights")?;
+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
+ match spec {
+ AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {
+ layout: Layout::OutputInput,
+ ..
+ }) => {
+ let block = LinearMatmul::full_precision(
context,
- quantization_config,
input_dimension,
output_dimension_sum,
+ has_biases,
+ data_type,
parameter_tree,
- None,
)?;
Ok(Box::new(block))
},
- LinearConfig::FullPrecision {
- precision,
- } => {
- let block = LinearMatmul::full_precision(
+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {
+ adapter_spec: None,
+ incoherence_block_size: Some(32),
+ incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
+ ..
+ }) => Ok(Box::new(RHTLinearWrapper::new(
+ context,
+ input_dimension,
+ output_dimension_sum,
+ has_biases,
+ data_type,
+ parameter_tree,
+ )?)),
+ AnyWeightMatrixSpec::HybridSpec(HybridSpec {
+ quantization_spec,
+ adapter_spec: Some(adapter_spec),
+ incoherence_block_size,
+ incoherence_processing_mode,
+ ..
+ }) => {
+ assert!(!has_biases, "QLoRA linear with biases is not supported");
+ let adapter_spec = *adapter_spec;
+ let AnyWeightMatrixSpec::LowRankSpec(adapter_spec) = adapter_spec else {
+ return Err(LinearBlockError::UnsupportedConfiguration(format!("{adapter_spec:?}")));
+ };
+ Ok(Box::new(QLoRALinearWrapper::new(
context,
- (*precision).into(),
+ *quantization_spec,
+ adapter_spec,
+ incoherence_block_size,
+ incoherence_processing_mode,
input_dimension,
output_dimension_sum,
- parameter_tree,
- )?;
- Ok(Box::new(block))
+ data_type,
+ &weights_tree,
+ )?))
},
- LinearConfig::QLoRA {
- quantization,
- lora_rank,
- lora_scale,
- } => {
- let block = QLoRALinearWrapper::new(
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ let block = LinearMatmul::quantized(
context,
- quantization,
- *lora_rank,
- *lora_scale,
+ spec,
input_dimension,
output_dimension_sum,
- parameter_tree,
+ data_type,
+ &weights_tree,
+ has_biases.then_some(parameter_tree),
None,
)?;
Ok(Box::new(block))
},
- LinearConfig::RHTLinearWrapper {
- block_size,
- inner_config,
- } => Ok(Box::new(RHTLinearWrapper::new(
- context,
- *block_size,
- inner_config,
- input_dimension,
- output_dimension_sum,
- parameter_tree,
- )?)),
+ spec => Err(LinearBlockError::UnsupportedConfiguration(format!("{spec:?}"))),
}
}
pub fn new_with_output_hadamard(
context: &B::Context,
- config: &LinearConfig,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
output_factors: Allocation<B>,
input_dim: usize,
output_dim: usize,
- ) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
- match config {
- LinearConfig::Quantized(config) | LinearConfig::ScaleBiasQuantized(config) => Ok(Box::new(
- LinearMatmul::quantized(context, config, input_dim, output_dim, parameter_tree, Some(output_factors))?,
- )),
- LinearConfig::QLoRA {
- quantization,
- lora_rank,
- lora_scale,
- } => Ok(Box::new(QLoRALinearWrapper::new(
- context,
- quantization,
- *lora_rank,
- *lora_scale,
- input_dim,
- output_dim,
- parameter_tree,
- Some(output_factors),
- )?)),
- inner_config => unimplemented!("{inner_config:?} doesn't support fused output hadamard"),
+ has_biases: bool,
+ data_type: DataType,
+ ) -> Result<Box<dyn Linear<B>>, OutputHadamardLinearError<B>> {
+ let weights_tree = parameter_tree.subtree("weights")?.subtree("quantized")?;
+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
+ match spec {
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ Ok(Box::new(LinearMatmul::quantized(
+ context,
+ spec,
+ input_dim,
+ output_dim,
+ data_type,
+ &weights_tree,
+ has_biases.then_some(parameter_tree),
+ Some(output_factors),
+ )?))
+ },
+ spec => Err(OutputHadamardLinearError::UnsupportedConfiguration(format!(
+ "{spec:?} doesn't support fused output hadamard"
+ ))),
}
}
pub fn new_extracting_input_hadamard<const N: usize>(
- config: &LinearConfig,
input_dimension: usize,
output_dimensions: [usize; N],
+ has_biases: bool,
context: &B::Context,
- parameter_tree: &ParameterTree<B::Context>,
+ data_type: DataType,
+ parameter_tree: &ParameterTree<B>,
) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
- match config {
- LinearConfig::RHTLinearWrapper {
- inner_config,
- ..
- } => {
- let input_factors = parameter_tree.leaf("input_factors")?.read_allocation()?;
- let output_factors = parameter_tree.leaf("output_factors")?.read_allocation()?;
- let inner_tree = parameter_tree.subtree("inner_linear")?;
- let inner_linear = Self::new_with_output_hadamard(
- context,
- inner_config,
- &inner_tree,
- output_factors,
- input_dimension,
- output_dimension_sum,
- )?;
- Ok((inner_linear, Some(input_factors)))
- },
- other => {
- let linear = Self::new(other, input_dimension, output_dimensions, context, parameter_tree)?;
- Ok((linear, None))
- },
+ let weights_tree = parameter_tree.subtree("weights")?;
+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
+ if let AnyWeightMatrixSpec::HybridSpec(HybridSpec {
+ adapter_spec: None,
+ incoherence_block_size: Some(32),
+ incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
+ ..
+ }) = spec
+ {
+ let input_factors = weights_tree
+ .leaf("incoherence_signs.input_signs")?
+ .validate(&[input_dimension], DataType::I32)?
+ .read_allocation()?;
+ let output_factors = weights_tree
+ .leaf("incoherence_signs.output_signs")?
+ .validate(&[output_dimension_sum], DataType::I32)?
+ .read_allocation()?;
+ let inner_linear = Self::new_with_output_hadamard(
+ context,
+ parameter_tree,
+ output_factors,
+ input_dimension,
+ output_dimension_sum,
+ has_biases,
+ data_type,
+ )?;
+ Ok((inner_linear, Some(input_factors)))
+ } else {
+ let linear = Self::new(input_dimension, output_dimensions, has_biases, context, data_type, parameter_tree)?;
+ Ok((linear, None))
}
}
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
index c3fae59ec..08bc3aab4 100644
--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -7,86 +7,137 @@ use crate::{
array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder,
+ gpu_types::HadamardTransformOrder,
kernel::{
- ManualKernels,
+ HadamardTransformKernel, Kernels, ManualKernels,
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
- config::QuantizationConfig,
+ config::weight_matrix::{AnyWeightMatrixSpec, hybrid_spec::IncoherenceProcessingMode, low_rank_spec::LowRankSpec},
encodable_block::{
Linear,
- linear::{LinearBlockError, LinearMatmul, LinearMatmulError},
+ linear::{LinearMatmul, LinearMatmulError},
},
prelude::{ParameterLoaderError, ParameterTree},
};
#[derive(Debug, Error)]
pub enum QLoRALinearWrapperError<B: Backend> {
- #[error("Inner linear error: {0}")]
- InnerLinearError(#[from] Box<LinearBlockError<B>>),
- #[error("Linear matmul error: {0}")]
+ #[error("LinearMatmul error: {0}")]
LinearMatmulError(#[from] LinearMatmulError<B>),
#[error("Parameter loader error: {0}")]
ParameterLoaderError(#[from] ParameterLoaderError<B>),
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
+ #[error("Unsupported QLoRA linear configuration: {0}")]
+ UnsupportedConfiguration(String),
}
pub struct QLoRALinearWrapper<B: Backend> {
base_linear: LinearMatmul<B>,
+ input_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,
+ output_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,
adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
adapter_down: Allocation<B>,
adapter_up: Allocation<B>,
input_dim: usize,
output_dim: usize,
lora_rank: usize,
- lora_scale: f32,
data_type: DataType,
}
impl<B: Backend> QLoRALinearWrapper<B> {
pub fn new(
context: &B::Context,
- quantization: &QuantizationConfig,
- lora_rank: usize,
- lora_scale: f32,
+ quantization_spec: AnyWeightMatrixSpec,
+ adapter_spec: LowRankSpec,
+ incoherence_block_size: Option<usize>,
+ incoherence_processing_mode: IncoherenceProcessingMode,
input_dim: usize,
output_dim: usize,
- parameter_tree: &ParameterTree<B::Context>,
- output_quantized_hadamard_factors: Option<Allocation<B>>,
+ data_type: DataType,
+ weights_tree: &ParameterTree<B>,
) -> Result<Self, QLoRALinearWrapperError<B>> {
- let data_type = quantization.activation_precision.into();
+ let use_incoherence_signs = match (incoherence_block_size, incoherence_processing_mode) {
+ (None, _) => false,
+ (Some(32), IncoherenceProcessingMode::InputOutput) => true,
+ (incoherence_block_size, incoherence_processing_mode) => {
+ return Err(QLoRALinearWrapperError::UnsupportedConfiguration(format!(
+ "incoherence block_size={incoherence_block_size:?}, processing_mode={incoherence_processing_mode:?}"
+ )));
+ },
+ };
+ let quantized_tree = weights_tree.subtree("quantized")?;
let base_linear = LinearMatmul::quantized(
context,
- quantization,
+ quantization_spec,
input_dim,
output_dim,
- parameter_tree,
- output_quantized_hadamard_factors,
+ data_type,
+ &quantized_tree,
+ None,
+ None,
)?;
+
+ let (input_hadamard, output_hadamard) = if use_incoherence_signs {
+ let input_factors = weights_tree
+ .leaf("incoherence_signs.input_signs")?
+ .validate(&[input_dim], DataType::I32)?
+ .read_allocation()?;
+ let output_factors = weights_tree
+ .leaf("incoherence_signs.output_signs")?
+ .validate(&[output_dim], DataType::I32)?
+ .read_allocation()?;
+ (
+ Some((
+ <B::Kernels as Kernels>::HadamardTransformKernel::new(
+ context,
+ data_type,
+ HadamardTransformOrder::Input,
+ )
+ .map_err(QLoRALinearWrapperError::BackendError)?,
+ input_factors,
+ )),
+ Some((
+ <B::Kernels as Kernels>::HadamardTransformKernel::new(
+ context,
+ data_type,
+ HadamardTransformOrder::Output,
+ )
+ .map_err(QLoRALinearWrapperError::BackendError)?,
+ output_factors,
+ )),
+ )
+ } else {
+ (None, None)
+ };
+
let adapter_kernel = RefCell::new(
<<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)
.map_err(QLoRALinearWrapperError::BackendError)?,
);
- let adapter_down_leaf = parameter_tree.leaf("down_weights")?;
- adapter_down_leaf.validate_shape(&[lora_rank, input_dim], data_type)?;
- let adapter_down = adapter_down_leaf.read_allocation()?;
+ let adapter_down = weights_tree
+ .leaf("adapter.down_projection")?
+ .validate(&[adapter_spec.rank, input_dim], data_type)?
+ .read_allocation()?;
- let adapter_up_leaf = parameter_tree.leaf("up_weights")?;
- adapter_up_leaf.validate_shape(&[output_dim, lora_rank], data_type)?;
- let adapter_up = adapter_up_leaf.read_allocation()?;
+ let adapter_up = weights_tree
+ .leaf("adapter.up_projection")?
+ .validate(&[output_dim, adapter_spec.rank], data_type)?
+ .read_allocation()?;
Ok(Self {
base_linear,
+ input_hadamard,
+ output_hadamard,
adapter_kernel,
adapter_down,
adapter_up,
input_dim,
output_dim,
- lora_rank,
- lora_scale,
+ lora_rank: adapter_spec.rank,
data_type,
})
}
@@ -98,55 +149,86 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
input: Allocation<B>,
batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, <B as Backend>::Error> {
- let mut adapter_kernel = self.adapter_kernel.borrow_mut();
+ ) -> Result<Allocation<B>, B::Error> {
let mut intermediate =
encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.data_type))?;
- adapter_kernel.encode(
- MatmulArguments {
- a: &input,
- a_offset: 0,
- b: MatmulB::FullPrecision {
- b: &self.adapter_down,
+ {
+ let mut adapter_kernel = self.adapter_kernel.borrow_mut();
+ adapter_kernel.encode(
+ MatmulArguments {
+ a: &input,
+ a_offset: 0,
+ b: MatmulB::FullPrecision {
+ b: &self.adapter_down,
+ },
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut intermediate,
+ d_transform: MatmulDOps::none(),
+ m: batch_dim as u32,
+ n: self.lora_rank as u32,
+ k: self.input_dim as u32,
},
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- d: &mut intermediate,
- d_transform: MatmulDOps::none(),
- m: batch_dim as u32,
- n: self.lora_rank as u32,
- k: self.input_dim as u32,
- },
- encoder,
- )?;
+ encoder,
+ )?;
+ }
- let mut output = self.base_linear.encode(input, batch_dim, encoder)?;
+ let base_input = if let Some((input_hadamard_kernel, input_factors)) = &self.input_hadamard {
+ let mut base_input =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dim], self.data_type))?;
+ encoder.encode_copy(&input, .., &mut base_input, ..);
+ input_hadamard_kernel.encode(
+ &mut base_input,
+ input_factors,
+ self.input_dim as u32,
+ batch_dim as u32,
+ encoder,
+ );
+ base_input
+ } else {
+ input
+ };
- adapter_kernel.encode(
- MatmulArguments {
- a: &intermediate,
- a_offset: 0,
- b: MatmulB::FullPrecision {
- b: &self.adapter_up,
- },
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- d: &mut output,
- d_transform: MatmulDOps {
- ab_scale: self.lora_scale,
- accumulate: true,
- bias: None,
- rht_factors: None,
+ let mut output = self.base_linear.encode(base_input, batch_dim, encoder)?;
+
+ {
+ let mut adapter_kernel = self.adapter_kernel.borrow_mut();
+ adapter_kernel.encode(
+ MatmulArguments {
+ a: &intermediate,
+ a_offset: 0,
+ b: MatmulB::FullPrecision {
+ b: &self.adapter_up,
+ },
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut output,
+ d_transform: MatmulDOps {
+ ab_scale: 1.0,
+ accumulate: true,
+ bias: None,
+ rht_factors: None,
+ },
+ m: batch_dim as u32,
+ n: self.output_dim as u32,
+ k: self.lora_rank as u32,
},
- m: batch_dim as u32,
- n: self.output_dim as u32,
- k: self.lora_rank as u32,
- },
- encoder,
- )?;
+ encoder,
+ )?;
+ }
+
+ if let Some((output_hadamard_kernel, output_factors)) = &self.output_hadamard {
+ output_hadamard_kernel.encode(
+ &mut output,
+ output_factors,
+ self.output_dim as u32,
+ batch_dim as u32,
+ encoder,
+ );
+ }
Ok(output)
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
index 2714551f4..ddcd964ea 100644
--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
@@ -1,23 +1,30 @@
use thiserror::Error;
-use super::{Linear, LinearBlockError};
+use super::{Linear, OutputHadamardLinearError};
use crate::{
+ DataType,
backends::common::{
Allocation, Backend, Encoder,
+ gpu_types::HadamardTransformOrder,
kernel::{HadamardTransformKernel, Kernels},
},
- config::LinearConfig,
+ config::weight_matrix::{
+ AnyWeightMatrixSpec,
+ hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
+ },
parameters::{ParameterLoaderError, ParameterTree},
};
#[derive(Debug, Error)]
pub enum RHTLinearWrapperError<B: Backend> {
#[error("Inner linear error: {0}")]
- InnerLinearError(#[from] Box<LinearBlockError<B>>),
+ InnerLinearError(#[from] OutputHadamardLinearError<B>),
#[error("Parameter loading error: {0}")]
ParameterError(#[from] ParameterLoaderError<B>),
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
+ #[error("Unsupported RHT linear configuration: {0}")]
+ UnsupportedConfiguration(String),
}
pub struct RHTLinearWrapper<B: Backend> {
@@ -30,29 +37,44 @@ pub struct RHTLinearWrapper<B: Backend> {
impl<B: Backend> RHTLinearWrapper<B> {
pub fn new(
context: &B::Context,
- block_size: usize,
- inner_config: &LinearConfig,
input_dimension: usize,
output_dimension: usize,
- parameter_tree: &ParameterTree<B::Context>,
+ has_biases: bool,
+ data_type: DataType,
+ parameter_tree: &ParameterTree<B>,
) -> Result<Self, RHTLinearWrapperError<B>> {
- assert!(block_size == 32, "only block size 32 hadamard is supported");
+ let weights_tree = parameter_tree.subtree("weights")?;
+ let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
+ let AnyWeightMatrixSpec::HybridSpec(HybridSpec {
+ adapter_spec: None,
+ incoherence_block_size: Some(32),
+ incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
+ ..
+ }) = &spec
+ else {
+ return Err(RHTLinearWrapperError::UnsupportedConfiguration(format!("{spec:?}")));
+ };
- let data_type = inner_config.activation_precision().into();
- let input_factors = parameter_tree.leaf("input_factors")?.read_allocation()?;
- let output_factors = parameter_tree.leaf("output_factors")?.read_allocation()?;
- let input_hadamard_kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)
- .map_err(RHTLinearWrapperError::BackendError)?;
- let inner_linear_tree = parameter_tree.subtree("inner_linear")?;
+ let input_factors = weights_tree
+ .leaf("incoherence_signs.input_signs")?
+ .validate(&[input_dimension], DataType::I32)?
+ .read_allocation()?;
+ let output_factors = weights_tree
+ .leaf("incoherence_signs.output_signs")?
+ .validate(&[output_dimension], DataType::I32)?
+ .read_allocation()?;
+ let input_hadamard_kernel =
+ <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type, HadamardTransformOrder::Input)
+ .map_err(RHTLinearWrapperError::BackendError)?;
let inner_linear = <dyn Linear<B>>::new_with_output_hadamard(
context,
- inner_config,
- &inner_linear_tree,
+ parameter_tree,
output_factors,
input_dimension,
output_dimension,
- )
- .map_err(|error| RHTLinearWrapperError::InnerLinearError(Box::new(error)))?;
+ has_biases,
+ data_type,
+ )?;
Ok(Self {
input_hadamard_kernel,
diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
index a6af9f11e..f5a616e3c 100644
--- a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
@@ -9,12 +9,13 @@ use crate::{
array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
+ gpu_types::ActivationType,
kernel::{
Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,
ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
},
- config::Mamba2Config,
+ config::token_mixer::mamba2::Mamba2Config,
encodable_block::linear::{Linear, LinearBlockError},
forward_pass::ssm_layer::SSMLayer,
parameters::{ParameterLoaderError, ParameterTree},
@@ -25,13 +26,20 @@ pub enum MambaMixerError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
#[error("Linear error: {0}")]
- LinearError(#[from] Box<LinearBlockError<B>>),
+ LinearError(#[from] LinearBlockError<B>),
#[error("Parameter loader error: {0}")]
ParameterLoaderError(#[from] ParameterLoaderError<B>),
}
pub(crate) struct MambaMixer<B: Backend> {
- config: Mamba2Config,
+ conv_dim: usize,
+ inner_dim: usize,
+ num_heads: usize,
+ num_groups: usize,
+ head_dim: usize,
+ state_dim: usize,
+ kernel_size: usize,
+ activation_type: ActivationType,
in_projection: Box<dyn Linear<B>>,
out_projection: Box<dyn Linear<B>>,
split_inproj: <B::Kernels as Kernels>::SplitInProjKernel,
@@ -79,41 +87,45 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {
impl<B: Backend> MambaMixer<B> {
pub(crate) fn new(
context: &B::Context,
- mamba_config: Mamba2Config,
+ mamba_config: &Mamba2Config,
model_dim: usize,
- decoder_layer_loader: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
+ data_type: DataType,
) -> Result<(Self, Option<Allocation<B>>), MambaMixerError<B>> {
- let split_tree = decoder_layer_loader.subtree("mixer")?;
- let conv_tree = split_tree.subtree("conv")?;
-
- let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();
+ let conv_tree = parameter_tree.subtree("conv")?;
+ let conv_dim = mamba_config.conv_dim();
+ let inner_dim = mamba_config.inner_dim();
let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
- &mamba_config.in_projection_config,
model_dim,
- [mamba_config.conv_dim(), mamba_config.inner_dim(), mamba_config.num_heads],
+ [conv_dim, inner_dim, mamba_config.num_heads],
+ mamba_config.has_in_biases,
context,
- &decoder_layer_loader.subtree("mixer.in_projection")?,
- )
- .map_err(|err| MambaMixerError::LinearError(Box::new(err)))?;
+ data_type,
+ &parameter_tree.subtree("in_projection")?,
+ )?;
let out_projection = <dyn Linear<B>>::new(
- &mamba_config.out_projection_config,
- mamba_config.inner_dim(),
+ inner_dim,
[model_dim],
+ mamba_config.has_out_biases,
context,
- &decoder_layer_loader.subtree("mixer.out_projection")?,
- )
- .map_err(|err| MambaMixerError::LinearError(Box::new(err)))?;
+ data_type,
+ &parameter_tree.subtree("out_projection")?,
+ )?;
- let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;
+ let conv_weight =
+ conv_tree.leaf("weights")?.validate(&[conv_dim, mamba_config.kernel_size], data_type)?.read_allocation()?;
let conv_bias = if mamba_config.conv_config.has_biases {
- Some(conv_tree.leaf("biases")?.read_allocation()?)
+ Some(conv_tree.leaf("biases")?.validate(&[conv_dim], data_type)?.read_allocation()?)
} else {
None
};
- let gate_bias = split_tree.leaf("gate_bias")?.read_allocation()?;
- let skip_connection_weight = split_tree.leaf("skip_connection_weight")?.read_allocation()?;
+ let gate_bias = parameter_tree.leaf("gate_bias")?.validate(&[inner_dim], data_type)?.read_allocation()?;
+ let skip_connection_weight = parameter_tree
+ .leaf("skip_connection_weight")?
+ .validate(&[mamba_config.num_heads], data_type)?
+ .read_allocation()?;
let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)
.map_err(MambaMixerError::BackendError)?;
@@ -136,7 +148,14 @@ impl<B: Backend> MambaMixer<B> {
Ok((
Self {
- config: mamba_config,
+ conv_dim,
+ inner_dim,
+ num_heads: mamba_config.num_heads,
+ num_groups: mamba_config.num_groups,
+ head_dim: mamba_config.head_dim,
+ state_dim: mamba_config.state_dim,
+ kernel_size: mamba_config.kernel_size,
+ activation_type: mamba_config.activation.act_type(),
in_projection,
out_projection,
split_inproj,
@@ -162,13 +181,13 @@ impl<B: Backend> MambaMixer<B> {
suffix_length: usize,
encoder: &mut Encoder<B>,
) -> Result<SplitInProjectionOutput<B>, B::Error> {
- let conv_dim = self.config.conv_dim();
- let inner_dim = self.config.inner_dim();
- let num_heads = self.config.num_heads;
+ let conv_dim = self.conv_dim;
+ let inner_dim = self.inner_dim;
+ let num_heads = self.num_heads;
let total_dim = conv_dim + inner_dim + num_heads;
let mut conv_inputs = encoder.allocate_scratch(size_for_shape(&[suffix_length, conv_dim], self.data_type))?;
- let mut gate = encoder
- .allocate_scratch(size_for_shape(&[suffix_length, num_heads, self.config.head_dim], self.data_type))?;
+ let mut gate =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length, num_heads, self.head_dim], self.data_type))?;
let mut time_step = encoder.allocate_scratch(size_for_shape(&[suffix_length, num_heads], self.data_type))?;
self.split_inproj.encode(
in_proj,
@@ -197,20 +216,18 @@ impl<B: Backend> MambaMixer<B> {
encoder: &mut Encoder<B>,
suffix_length: usize,
) -> Result<ConvScanOutput<B>, B::Error> {
- let conv_dim = self.config.conv_dim();
- let inner_dim = self.config.inner_dim();
- let proj_dim = self.config.num_groups * self.config.state_dim;
- let state_stride = self.config.kernel_size.saturating_sub(1);
- let mut conv_x = encoder.allocate_scratch(size_for_shape(
- &[suffix_length, self.config.num_heads, self.config.head_dim],
- self.data_type,
- ))?;
- let state_shape = [suffix_length, self.config.num_groups, self.config.state_dim];
+ let conv_dim = self.conv_dim;
+ let inner_dim = self.inner_dim;
+ let proj_dim = self.num_groups * self.state_dim;
+ let state_stride = self.kernel_size.saturating_sub(1);
+ let mut conv_x = encoder
+ .allocate_scratch(size_for_shape(&[suffix_length, self.num_heads, self.head_dim], self.data_type))?;
+ let state_shape = [suffix_length, self.num_groups, self.state_dim];
let mut state_b = encoder.allocate_scratch(size_for_shape(&state_shape, self.data_type))?;
let mut state_c = encoder.allocate_scratch(size_for_shape(&state_shape, self.data_type))?;
if suffix_length == 1 {
- if conv_dim > 0 && self.config.kernel_size > 0 {
+ if conv_dim > 0 && self.kernel_size > 0 {
let next_state = layer.conv_state.as_mut().unwrap_or(&mut layer.ssm_state);
self.conv_decode.encode(
conv_inputs,
@@ -221,14 +238,14 @@ impl<B: Backend> MambaMixer<B> {
&mut state_b,
&mut state_c,
next_state,
- self.config.kernel_size as u32,
+ self.kernel_size as u32,
conv_dim as u32,
state_stride as u32,
conv_dim as u32,
suffix_length as u32,
inner_dim as u32,
proj_dim as u32,
- self.config.activation.act_type(),
+ self.activation_type,
encoder,
);
}
@@ -252,7 +269,7 @@ impl<B: Backend> MambaMixer<B> {
);
}
- if conv_dim > 0 && self.config.kernel_size > 0 {
+ if conv_dim > 0 && self.kernel_size > 0 {
let conv_source = match &padded {
Some(padded) => padded,
None => conv_inputs,
@@ -267,13 +284,13 @@ impl<B: Backend> MambaMixer<B> {
&mut state_c,
next_state,
suffix_length as u32,
- self.config.kernel_size as u32,
+ self.kernel_size as u32,
conv_dim as u32,
state_stride as u32,
conv_dim as u32,
inner_dim as u32,
proj_dim as u32,
- self.config.activation.act_type(),
+ self.activation_type,
encoder,
);
}
@@ -307,14 +324,14 @@ impl<B: Backend> MambaMixer<B> {
z: gate,
state: &mut layer.ssm_state,
suffix_len: suffix_length,
- group_size: (self.config.num_heads / self.config.num_groups) as u32,
- state_size: self.config.state_dim as u32,
- x_strides: [self.config.num_heads * self.config.head_dim, self.config.head_dim, 1],
- dt_strides: [self.config.num_heads, 1],
- cb_strides: [self.config.num_groups * self.config.state_dim, self.config.state_dim, 1],
- state_strides: [self.config.head_dim * self.config.state_dim, self.config.state_dim, 1],
- channels: self.config.num_heads,
- head_dim: self.config.head_dim,
+ group_size: (self.num_heads / self.num_groups) as u32,
+ state_size: self.state_dim as u32,
+ x_strides: [self.num_heads * self.head_dim, self.head_dim, 1],
+ dt_strides: [self.num_heads, 1],
+ cb_strides: [self.num_groups * self.state_dim, self.state_dim, 1],
+ state_strides: [self.head_dim * self.state_dim, self.state_dim, 1],
+ channels: self.num_heads,
+ head_dim: self.head_dim,
},
self.prefill_mode,
)
@@ -331,12 +348,11 @@ impl<B: Backend> MambaMixer<B> {
encoder: &mut Encoder<B>,
suffix_length: usize,
) -> Result<Allocation<B>, B::Error> {
- let mut out =
- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.inner_dim()], self.data_type))?;
- let h = self.config.num_heads as u32;
- let g = self.config.num_groups as u32;
- let dh = self.config.head_dim as u32;
- let n = self.config.state_dim as u32;
+ let mut out = encoder.allocate_scratch(size_for_shape(&[suffix_length, self.inner_dim], self.data_type))?;
+ let h = self.num_heads as u32;
+ let g = self.num_groups as u32;
+ let dh = self.head_dim as u32;
+ let n = self.state_dim as u32;
let x_strides = [h * dh, dh, 1u32];
let dt_strides = [h, 1u32];
diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
index 464f58a20..c360dde6c 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
@@ -9,7 +9,7 @@ use super::linear::{Linear, LinearBlockError};
use crate::{
DataType,
backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
- config::MLPConfig,
+ config::mlp::AnyMLPConfig,
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -36,36 +36,37 @@ pub enum MlpBlockError<B: Backend> {
impl<B: Backend> dyn Mlp<B> {
pub fn new(
- config: &MLPConfig,
+ config: &AnyMLPConfig,
model_dimension: usize,
hidden_dimension: usize,
context: &B::Context,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
+ data_type: DataType,
) -> Result<(Box<dyn Mlp<B>>, Option<Allocation<B>>), MlpBlockError<B>> {
match config {
- MLPConfig::Dense(dense_config) => {
- let data_type: DataType = dense_config.linear_config.activation_precision().into();
-
+ AnyMLPConfig::DenseMLPConfig(dense_config) => {
let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
- &dense_config.linear_config,
model_dimension,
[2 * hidden_dimension],
+ dense_config.has_up_biases,
context,
+ data_type,
&parameter_tree.subtree("up_projection")?,
)?;
let (down_projection, down_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
- &dense_config.linear_config,
hidden_dimension,
[model_dimension],
+ dense_config.has_down_biases,
context,
+ data_type,
&parameter_tree.subtree("down_projection")?,
)?;
let gate = MlpGateActMulEncodable::new(
context,
data_type,
- dense_config.activation.clone(),
+ dense_config.activation.clone().into(),
hidden_dimension,
down_input_hadamard_factors,
)
@@ -73,8 +74,14 @@ impl<B: Backend> dyn Mlp<B> {
Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))
},
- MLPConfig::MixtureOfExperts(mixture_of_experts_config) => Ok((
- Box::new(MoeBlock::new(context, mixture_of_experts_config, model_dimension, parameter_tree)?),
+ AnyMLPConfig::MixtureOfExpertsConfig(mixture_of_experts_config) => Ok((
+ Box::new(MoeBlock::new(
+ context,
+ mixture_of_experts_config,
+ model_dimension,
+ data_type,
+ parameter_tree,
+ )?),
None,
)),
}
diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe.rs b/crates/backend-uzu/src/encodable_block/mlp/moe.rs
index b118420a7..33e18c6b1 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/moe.rs
@@ -17,7 +17,10 @@ use crate::{
},
},
},
- config::{LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig},
+ config::{
+ mlp::{mixture_of_experts::MixtureOfExpertsConfig, routing_function::AnyRoutingFunction},
+ weight_matrix::{AnyWeightMatrixSpec, Layout, full_precision_spec::FullPrecisionSpec},
+ },
encodable_block::mlp::Mlp,
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -66,6 +69,12 @@ pub enum MoeBlockError<B: Backend> {
UnsupportedSharedExperts,
#[error("MoE expert gate is not supported")]
UnsupportedExpertGate,
+ #[error("MoE without router, up, and down biases is not supported")]
+ UnsupportedNoBiases,
+ #[error("Unsupported MoE router configuration: {0}")]
+ UnsupportedRouterConfiguration(String),
+ #[error("Unsupported MoE expert activation: {0:?}")]
+ UnsupportedExpertActivation(ActivationType),
}
impl<B: Backend> MoeBlock<B> {
@@ -73,7 +82,8 @@ impl<B: Backend> MoeBlock<B> {
context: &B::Context,
moe_config: &MixtureOfExpertsConfig,
model_dim: usize,
- parameter_tree: &ParameterTree<B::Context>,
+ data_type: DataType,
+ parameter_tree: &ParameterTree<B>,
) -> Result<Self, MoeBlockError<B>> {
if model_dim % 4 != 0 {
return Err(MoeBlockError::InvalidModelDim);
@@ -90,30 +100,62 @@ impl<B: Backend> MoeBlock<B> {
if moe_config.gate_config.is_some() {
return Err(MoeBlockError::UnsupportedExpertGate);
}
+ if !moe_config.router_has_biases
+ || !moe_config.expert_config.has_up_biases
+ || !moe_config.expert_config.has_down_biases
+ {
+ return Err(MoeBlockError::UnsupportedNoBiases);
+ }
- let data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();
let gating_code = match moe_config.expert_config.activation.act_type() {
- ActivationType::GELU => 3,
- ActivationType::SILU {
- ..
- } => 2,
- _ => {
- panic!("{:?} is not supported for MoE kernels", moe_config.expert_config.activation.act_type())
- },
+ ActivationType::GELUApprox => 3,
+ ActivationType::SILU => 2,
+ activation_type => return Err(MoeBlockError::UnsupportedExpertActivation(activation_type)),
};
- let router_data_type: DataType = moe_config.router_config.activation_precision().into();
-
- let router_renorm = matches!(moe_config.routing_function, RoutingFunctionConfig::SoftmaxRouting);
+ let router_renorm = matches!(moe_config.routing_function, AnyRoutingFunction::SoftmaxRouting(_));
let router_tree = parameter_tree.subtree("router")?;
- if !matches!(moe_config.router_config, LinearConfig::FullPrecision { .. }) {
- unimplemented!("Only FullPrecision MoE router is supported; got {:?}", moe_config.router_config);
- }
- let router_weights = router_tree.leaf_allocation("weights")?;
- let router_biases = router_tree.leaf_allocation("biases")?;
+ let router_weights_tree = router_tree.subtree("weights")?;
+ let router_spec = router_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
+ let AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {
+ layout: Layout::OutputInput,
+ ..
+ }) = &router_spec
+ else {
+ return Err(MoeBlockError::UnsupportedRouterConfiguration(format!("{router_spec:?}")));
+ };
+ let router_weights = router_weights_tree
+ .leaf("weights")?
+ .validate(&[moe_config.num_routed_experts, model_dim], data_type)?
+ .read_allocation()?;
+ let router_biases =
+ router_tree.leaf("biases")?.validate(&[moe_config.num_routed_experts], data_type)?.read_allocation()?;
+
+ let experts_tree = parameter_tree.subtree("experts")?;
+ let up_tree = experts_tree.subtree("up_projection")?;
+ let down_tree = experts_tree.subtree("down_projection")?;
+ let up_weights_tree = up_tree.subtree("weights")?;
+ let down_weights_tree = down_tree.subtree("weights")?;
- let router_topk_kernel = <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, router_data_type)
+ let w13 = up_weights_tree
+ .leaf("weights")?
+ .validate(&[moe_config.num_routed_experts, moe_config.expert_hidden_dim * 2, model_dim], data_type)?
+ .read_allocation()?;
+ let w2 = down_weights_tree
+ .leaf("weights")?
+ .validate(&[moe_config.num_routed_experts, model_dim, moe_config.expert_hidden_dim], data_type)?
+ .read_allocation()?;
+ let up_biases = up_tree
+ .leaf("biases")?
+ .validate(&[moe_config.num_routed_experts, moe_config.expert_hidden_dim * 2], data_type)?
+ .read_allocation()?;
+ let down_biases = down_tree
+ .leaf("biases")?
+ .validate(&[moe_config.num_routed_experts, model_dim], data_type)?
+ .read_allocation()?;
+
+ let router_topk_kernel = <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, data_type)
.map_err(MoeBlockError::BackendError)?;
let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(MoeBlockError::BackendError)?;
@@ -130,15 +172,6 @@ impl<B: Backend> MoeBlock<B> {
let finalize_kernel =
<B::Kernels as Kernels>::MoeFinalizeKernel::new(context, data_type).map_err(MoeBlockError::BackendError)?;
- let experts_tree = parameter_tree.subtree("experts")?;
- let up_tree = experts_tree.subtree("up_projection")?;
- let down_tree = experts_tree.subtree("down_projection")?;
-
- let w13 = up_tree.leaf_allocation("weights")?;
- let w2 = down_tree.leaf_allocation("weights")?;
- let up_biases = up_tree.leaf_allocation("biases")?;
- let down_biases = down_tree.leaf_allocation("biases")?;
-
let (gate_lo, gate_hi) = moe_config.expert_config.gate_clipping.unwrap_or_default();
let (up_lo, up_hi) = moe_config.expert_config.up_clipping.unwrap_or_default();
diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rs
index 21676ce4c..26942ab18 100644
--- a/crates/backend-uzu/src/encodable_block/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mod.rs
@@ -18,17 +18,17 @@ mod rope;
mod sampling;
mod short_conv_mixer;
-pub use attention::{Attention, AttentionArguments};
+pub use attention::{Attention, AttentionError};
pub use classifier_layer::ClassifierLayer;
pub use decoder::{Decoder, DecoderArguments, DecoderDecodeInput};
-pub(crate) use delta_net_mixer::{DeltaNetArguments, DeltaNetMixer};
+pub(crate) use delta_net_mixer::{DeltaNetArguments, DeltaNetMixer, DeltaNetMixerError};
pub use embedding::Embedding;
pub use layer::{LayerArguments, LayerExecutables};
pub use layer_norm::{LayerNorm, LayerNormError};
pub use linear::Linear;
-pub(crate) use mamba_mixer::{MambaArguments, MambaMixer};
-pub use mlp::Mlp;
-pub use normalization::Normalization;
+pub(crate) use mamba_mixer::{MambaArguments, MambaMixer, MambaMixerError};
+pub use mlp::{Mlp, MlpBlockError};
+pub use normalization::{Normalization, NormalizationError};
pub use pooling::Pooling;
pub use prediction_head::ClassifierPredictionHead;
pub use qk_unpack::QkUnpack;
@@ -38,4 +38,4 @@ pub use rope::Rope;
pub(crate) use sampling::SamplingInputs;
pub use sampling::{Sampling, SamplingArguments};
pub(crate) use short_conv_mixer::ShortConvArguments;
-pub use short_conv_mixer::ShortConvMixer;
+pub use short_conv_mixer::{ShortConvMixer, ShortConvMixerError};
diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rs
index 829cbadf5..db6aaeb16 100644
--- a/crates/backend-uzu/src/encodable_block/normalization.rs
+++ b/crates/backend-uzu/src/encodable_block/normalization.rs
@@ -6,10 +6,13 @@ use super::{LayerNorm, LayerNormError, PostLayerScalar, RMSNorm, RMSNormError};
use crate::{
DataType,
backends::common::{Allocation, Backend, Encoder},
- config::NormalizationConfig,
+ config::normalization::NormalizationConfig,
parameters::ParameterTree,
};
+pub(super) const NORMALIZATION_ACCUMULATION_DATA_TYPE: DataType = DataType::F32;
+pub(super) const NORMALIZATION_SCALE_DATA_TYPE: DataType = DataType::F32;
+
#[derive(Debug, Error)]
pub enum NormalizationError<B: Backend> {
#[error("LayerNorm error: {0}")]
@@ -30,15 +33,17 @@ impl<B: Backend> Normalization<B> {
pub fn new(
context: &B::Context,
intermediate_data_type: DataType,
+ element_count: usize,
config: NormalizationConfig,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
) -> Result<Self, NormalizationError<B>> {
if config.subtract_mean {
- Ok(Self::LayerNorm(LayerNorm::new(context, intermediate_data_type, config, parameter_tree)?))
+ Ok(Self::LayerNorm(LayerNorm::new(context, intermediate_data_type, element_count, config, parameter_tree)?))
} else {
Ok(Self::RMSNorm(RMSNorm::new(
context,
intermediate_data_type,
+ element_count,
config,
parameter_tree,
None,
diff --git a/crates/backend-uzu/src/encodable_block/pooling.rs b/crates/backend-uzu/src/encodable_block/pooling.rs
index ced65d036..0dadda424 100644
--- a/crates/backend-uzu/src/encodable_block/pooling.rs
+++ b/crates/backend-uzu/src/encodable_block/pooling.rs
@@ -7,7 +7,7 @@ use crate::{
Allocation, Backend, Encoder,
kernel::{Kernels, PoolingClsKernel, PoolingMeanKernel},
},
- config::PoolingType,
+ config::classifier::PoolingType,
};
enum PoolingKernel<B: Backend> {
@@ -46,7 +46,7 @@ impl<B: Backend> Pooling<B> {
model_dim: usize,
) -> Result<Self, B::Error> {
let pooling_kernel = match pooling_type {
- PoolingType::Cls => PoolingKernel::Cls(<B::Kernels as Kernels>::PoolingClsKernel::new(context, data_type)?),
+ PoolingType::CLS => PoolingKernel::Cls(<B::Kernels as Kernels>::PoolingClsKernel::new(context, data_type)?),
PoolingType::Mean => {
PoolingKernel::Mean(<B::Kernels as Kernels>::PoolingMeanKernel::new(context, data_type)?)
},
diff --git a/crates/backend-uzu/src/encodable_block/qkv_norm.rs b/crates/backend-uzu/src/encodable_block/qkv_norm.rs
index 574d81a22..0ff40018f 100644
--- a/crates/backend-uzu/src/encodable_block/qkv_norm.rs
+++ b/crates/backend-uzu/src/encodable_block/qkv_norm.rs
@@ -6,7 +6,7 @@ use crate::{
Allocation, Backend, Encoder,
kernel::{Kernels, QKVNormKernel},
},
- config::{NormalizationConfig, UpcastMode},
+ config::normalization::{NormalizationConfig, UpcastMode},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -40,21 +40,37 @@ impl<B: Backend> QKVNorm<B> {
query_config: Option<NormalizationConfig>,
key_config: Option<NormalizationConfig>,
value_config: Option<NormalizationConfig>,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
num_q_heads: usize,
num_kv_heads: usize,
head_dim: usize,
) -> Result<Self, QKVNormError<B>> {
let query = query_config
.map(|cfg| {
- Self::build_head(context, intermediate_data_type, cfg, parameter_tree, Some("query_norm.scales"))
+ Self::build_head(
+ context,
+ intermediate_data_type,
+ cfg,
+ parameter_tree,
+ Some("query_norm.scales"),
+ head_dim,
+ )
})
.transpose()?;
let key = key_config
- .map(|cfg| Self::build_head(context, intermediate_data_type, cfg, parameter_tree, Some("key_norm.scales")))
+ .map(|cfg| {
+ Self::build_head(
+ context,
+ intermediate_data_type,
+ cfg,
+ parameter_tree,
+ Some("key_norm.scales"),
+ head_dim,
+ )
+ })
.transpose()?;
let value = value_config
- .map(|cfg| Self::build_head(context, intermediate_data_type, cfg, parameter_tree, None))
+ .map(|cfg| Self::build_head(context, intermediate_data_type, cfg, parameter_tree, None, head_dim))
.transpose()?;
Ok(Self {
@@ -71,18 +87,27 @@ impl<B: Backend> QKVNorm<B> {
context: &B::Context,
intermediate_data_type: DataType,
config: NormalizationConfig,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
scales_leaf: Option<&str>,
+ head_dim: usize,
) -> Result<Head<B>, QKVNormError<B>> {
- let scales = scales_leaf.map(|leaf| parameter_tree.leaf_allocation(leaf)).transpose()?;
+ let (scales, scale_data_type) = if let Some(scales_leaf) = scales_leaf {
+ let scale_data_type = super::normalization::NORMALIZATION_SCALE_DATA_TYPE;
+ (
+ Some(parameter_tree.leaf(scales_leaf)?.validate(&[head_dim], scale_data_type)?.read_allocation()?),
+ scale_data_type,
+ )
+ } else {
+ (None, intermediate_data_type)
+ };
let scale_free = scales.is_none();
- let scale_data_type: DataType = config.scale_precision.into();
- let accumulation_data_type: DataType = config.accumulation_precision.into();
+ let accumulation_data_type = super::normalization::NORMALIZATION_ACCUMULATION_DATA_TYPE;
+ let output_data_type = intermediate_data_type;
let kernel = <B::Kernels as Kernels>::QKVNormKernel::new(
context,
intermediate_data_type,
scale_data_type,
- scale_data_type,
+ output_data_type,
accumulation_data_type,
true,
scale_free,
diff --git a/crates/backend-uzu/src/encodable_block/rms_norm.rs b/crates/backend-uzu/src/encodable_block/rms_norm.rs
index 4b1d83134..6c770bf4a 100644
--- a/crates/backend-uzu/src/encodable_block/rms_norm.rs
+++ b/crates/backend-uzu/src/encodable_block/rms_norm.rs
@@ -9,7 +9,7 @@ use crate::{
Allocation, Backend, Encoder,
kernel::{Kernels, RMSNormKernel},
},
- config::{NormalizationConfig, UpcastMode},
+ config::normalization::{NormalizationConfig, UpcastMode},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -25,7 +25,7 @@ pub enum RMSNormError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
#[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError<B>),
+ ParameterError(#[from] ParameterLoaderError<B>),
}
pub struct RMSNorm<B: Backend> {
@@ -43,19 +43,19 @@ impl<B: Backend> RMSNorm<B> {
pub fn new(
context: &B::Context,
intermediate_data_type: DataType,
+ element_count: usize,
config: NormalizationConfig,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
hadamard_factors: Option<Allocation<B>>,
use_shortcut: bool,
residual_add: bool,
post_layer_scalar: PostLayerScalar,
) -> Result<Self, RMSNormError<B>> {
- let scales_leaf = parameter_tree.leaf("scales").map_err(RMSNormError::ParameterError)?;
- let element_count = scales_leaf.shape()[0];
- let scales = scales_leaf.read_allocation().map_err(RMSNormError::ParameterError)?;
+ let scale_data_type = super::normalization::NORMALIZATION_SCALE_DATA_TYPE;
+ let scales = parameter_tree.leaf("scales")?.validate(&[element_count], scale_data_type)?.read_allocation()?;
- let accumulation_data_type: DataType = config.accumulation_precision.into();
- let scale_data_type: DataType = config.scale_precision.into();
+ let accumulation_data_type = super::normalization::NORMALIZATION_ACCUMULATION_DATA_TYPE;
+ let output_data_type = intermediate_data_type;
let (scale_residual_sum, scale_output, post_layer_scalar_value) = match post_layer_scalar {
PostLayerScalar::None => (false, false, 1.0),
@@ -67,7 +67,7 @@ impl<B: Backend> RMSNorm<B> {
context,
intermediate_data_type,
scale_data_type,
- scale_data_type,
+ output_data_type,
accumulation_data_type,
false,
config.upcast_mode == UpcastMode::FullLayer,
@@ -85,7 +85,7 @@ impl<B: Backend> RMSNorm<B> {
scales,
element_count,
input_data_type: intermediate_data_type,
- output_data_type: scale_data_type,
+ output_data_type,
hadamard_factors,
post_layer_scalar_value,
})
diff --git a/crates/backend-uzu/src/encodable_block/rope.rs b/crates/backend-uzu/src/encodable_block/rope.rs
index 5d603aa80..ef52eadee 100644
--- a/crates/backend-uzu/src/encodable_block/rope.rs
+++ b/crates/backend-uzu/src/encodable_block/rope.rs
@@ -7,6 +7,7 @@ use crate::{
Allocation, Backend, Encoder,
kernel::{Kernels, RopeKernel},
},
+ forward_pass::model_shape::ModelShape,
};
pub struct Rope<B: Backend> {
@@ -17,11 +18,15 @@ pub struct Rope<B: Backend> {
impl<B: Backend> Rope<B> {
pub fn new(
context: &B::Context,
- data_type: DataType,
+ model_shape: &ModelShape,
) -> Result<Self, B::Error> {
Ok(Self {
- kernel: <B::Kernels as Kernels>::RopeKernel::new(context, data_type)?,
- data_type,
+ kernel: <B::Kernels as Kernels>::RopeKernel::new(
+ context,
+ model_shape.data_type,
+ model_shape.rope_data_type,
+ )?,
+ data_type: model_shape.data_type,
})
}
diff --git a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs
index 6545e6050..f39e7d3f6 100644
--- a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs
@@ -7,7 +7,7 @@ use crate::{
Allocation, Backend, Encoder, Kernels,
kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},
},
- config::ShortConvConfig,
+ config::token_mixer::short_conv::ShortConvConfig,
encodable_block::linear::{Linear, LinearBlockError},
forward_pass::short_conv_layer::ShortConvLayer,
parameters::{ParameterLoaderError, ParameterTree},
@@ -20,14 +20,14 @@ pub enum ShortConvMixerError<B: Backend> {
#[error("Unsupported configuration: {0}")]
UnsupportedConfiguration(String),
#[error("Linear error: {0}")]
- LinearError(#[from] Box<LinearBlockError<B>>),
+ LinearError(#[from] LinearBlockError<B>),
#[error("Parameter loader error: {0}")]
ParameterLoaderError(#[from] ParameterLoaderError<B>),
}
pub struct ShortConvMixer<B: Backend> {
- config: ShortConvConfig,
model_dim: usize,
+ kernel_size: usize,
in_projection: Box<dyn Linear<B>>,
out_projection: Box<dyn Linear<B>>,
short_conv_pack: <B::Kernels as Kernels>::ShortConvPackKernel,
@@ -50,9 +50,10 @@ pub struct ShortConvArguments<'a, B: Backend> {
impl<B: Backend> ShortConvMixer<B> {
pub fn new(
context: &B::Context,
- short_conv_config: ShortConvConfig,
+ short_conv_config: &ShortConvConfig,
model_dim: usize,
- decoder_layer_loader: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
+ data_type: DataType,
) -> Result<(Self, Option<Allocation<B>>), ShortConvMixerError<B>> {
if short_conv_config.kernel_size < 2 {
return Err(ShortConvMixerError::UnsupportedConfiguration(format!(
@@ -61,32 +62,32 @@ impl<B: Backend> ShortConvMixer<B> {
)));
}
- let mixer_tree = decoder_layer_loader.subtree("mixer")?;
- let conv_tree = mixer_tree.subtree("conv")?;
-
- let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();
+ let conv_tree = parameter_tree.subtree("conv")?;
let (in_projection, in_proj_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
- &short_conv_config.in_projection_config,
model_dim,
[model_dim * 3],
+ false,
context,
- &mixer_tree.subtree("in_projection")?,
- )
- .map_err(|err| ShortConvMixerError::LinearError(Box::new(err)))?;
+ data_type,
+ &parameter_tree.subtree("in_projection")?,
+ )?;
let out_projection = <dyn Linear<B>>::new(
- &short_conv_config.out_projection_config,
model_dim,
[model_dim],
+ false,
context,
- &mixer_tree.subtree("out_projection")?,
- )
- .map_err(|err| ShortConvMixerError::LinearError(Box::new(err)))?;
-
- let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;
+ data_type,
+ &parameter_tree.subtree("out_projection")?,
+ )?;
+
+ let conv_weight = conv_tree
+ .leaf("weights")?
+ .validate(&[model_dim, short_conv_config.kernel_size], data_type)?
+ .read_allocation()?;
let conv_bias = if short_conv_config.conv_config.has_biases {
- Some(conv_tree.leaf("biases")?.read_allocation()?)
+ Some(conv_tree.leaf("biases")?.validate(&[model_dim], data_type)?.read_allocation()?)
} else {
None
};
@@ -103,8 +104,8 @@ impl<B: Backend> ShortConvMixer<B> {
Ok((
Self {
- config: short_conv_config,
model_dim,
+ kernel_size: short_conv_config.kernel_size,
in_projection,
out_projection,
short_conv_pack,
@@ -174,7 +175,7 @@ impl<B: Backend> ShortConvMixer<B> {
return Ok(());
}
- let kernel_size = self.config.kernel_size;
+ let kernel_size = self.kernel_size;
let state_stride = kernel_size - 1;
let padded_rows = state_stride + suffix_length;
@@ -220,9 +221,9 @@ impl<B: Backend> ShortConvMixer<B> {
return Ok(());
}
- let elem_bytes = DataType::from(self.config.in_projection_config.activation_precision()).size_in_bytes();
+ let elem_bytes = self.data_type.size_in_bytes();
- let kernel_size = self.config.kernel_size;
+ let kernel_size = self.kernel_size;
let state_stride = kernel_size - 1;
let in_proj_stride = self.model_dim * 3;
@@ -259,7 +260,7 @@ impl<B: Backend> ShortConvMixer<B> {
return Ok(());
}
- let kernel_size = self.config.kernel_size;
+ let kernel_size = self.kernel_size;
let state_stride = kernel_size - 1;
self.short_conv_decode.encode(
diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rs
index de84ab441..8c63187dc 100644
--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs
+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs
@@ -3,7 +3,7 @@ use std::cell::Cell;
use crate::{
array::size_for_shape,
backends::common::{AllocationType, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
- config::MixerConfig,
+ config::token_mixer::AnyTokenMixerConfig,
forward_pass::{
delta_net_layer::DeltaNetLayer,
kv_cache_layer::{KVCacheLayerState, KVCacheLayerTrait, KVSlice},
@@ -139,10 +139,11 @@ impl<B: Backend> CacheLayers<B> {
entry: source,
} = bindings[source_layer_index]
else {
- panic!("kv_source_layer {source_layer_index} (layer {layer_index}) must be a prior owner");
+ panic!("kv_source_layer_index {source_layer_index} (layer {layer_index}) must be a prior owner");
+ };
+ let geometry = |m: &AnyTokenMixerConfig| {
+ m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size))
};
- let geometry =
- |m: &MixerConfig| m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size));
assert_eq!(
geometry(&model_shape.layer_mixers()[source_layer_index]),
geometry(mixer),
@@ -155,7 +156,7 @@ impl<B: Backend> CacheLayers<B> {
}
let layer = match mixer {
- MixerConfig::Attention(attn) => {
+ AnyTokenMixerConfig::AttentionConfig(attn) => {
let sliding_window = attn.sliding_window_size;
let length = sliding_window.unwrap_or(max_prefix_length);
let shape = [length + max_suffix_length, attn.num_groups, attn.head_dim];
@@ -172,18 +173,15 @@ impl<B: Backend> CacheLayers<B> {
prefix_len: 0,
}
};
-
- let kv_layer =
- <dyn KVCacheLayerTrait<B>>::new(context, &state, shape, model_shape.kv_cache_data_type())
- .expect("Failed to create KVCacheLayer");
+ let kv_layer = <dyn KVCacheLayerTrait<B>>::new(context, &state, shape, model_shape.data_type)
+ .expect("Failed to create KVCacheLayer");
CacheLayer::Transformer(kv_layer)
},
- MixerConfig::Mamba(c) => {
+ AnyTokenMixerConfig::Mamba2Config(c) => {
let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];
let ssm_shape = [c.num_heads, c.head_dim, c.state_dim];
- let dtype = model_shape.activation_data_type();
- let conv_bytes = size_for_shape(&conv_shape, dtype);
- let ssm_bytes = size_for_shape(&ssm_shape, dtype);
+ let conv_bytes = size_for_shape(&conv_shape, model_shape.data_type);
+ let ssm_bytes = size_for_shape(&ssm_shape, model_shape.data_type);
CacheLayer::StateSpace(SSMLayer {
conv_state: (conv_bytes > 0).then(|| {
@@ -196,16 +194,15 @@ impl<B: Backend> CacheLayers<B> {
.create_allocation(ssm_bytes, AllocationType::Global)
.expect("Failed to create ssm state allocation"),
ssm_shape,
- data_type: dtype,
+ data_type: model_shape.data_type,
})
},
- MixerConfig::ShortConv(c) => {
+ AnyTokenMixerConfig::ShortConvConfig(c) => {
assert!(c.kernel_size >= 2, "ShortConv kernel_size must be >= 2, got {}", c.kernel_size);
let conv_shape = [model_shape.model_dim(), c.kernel_size - 1];
let suffix_state_shape = [max_suffix_length, model_shape.model_dim(), c.kernel_size - 1];
- let dtype = model_shape.activation_data_type();
- let conv_bytes = size_for_shape(&conv_shape, dtype);
- let suffix_bytes = size_for_shape(&suffix_state_shape, dtype);
+ let conv_bytes = size_for_shape(&conv_shape, model_shape.data_type);
+ let suffix_bytes = size_for_shape(&suffix_state_shape, model_shape.data_type);
CacheLayer::ShortConv(ShortConvLayer {
conv_state: context
@@ -216,17 +213,16 @@ impl<B: Backend> CacheLayers<B> {
.create_allocation(suffix_bytes, AllocationType::Global)
.expect("Failed to create short conv suffix allocation"),
suffix_shape: suffix_state_shape,
- data_type: dtype,
+ data_type: model_shape.data_type,
suffix_state_valid_start: Cell::new(0),
suffix_state_valid_len: Cell::new(0),
})
},
- MixerConfig::DeltaNet(c) => {
+ AnyTokenMixerConfig::DeltaNetConfig(c) => {
let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];
let ssm_shape = [c.num_heads, c.value_head_dim, c.head_dim];
- let dtype = model_shape.activation_data_type();
- let conv_bytes = size_for_shape(&conv_shape, dtype);
- let ssm_bytes = size_for_shape(&ssm_shape, dtype);
+ let conv_bytes = size_for_shape(&conv_shape, model_shape.data_type);
+ let ssm_bytes = size_for_shape(&ssm_shape, model_shape.data_type);
CacheLayer::DeltaNet(DeltaNetLayer {
conv_state: context
@@ -237,7 +233,7 @@ impl<B: Backend> CacheLayers<B> {
.create_allocation(ssm_bytes, AllocationType::Global)
.expect("Failed to create delta net ssm allocation"),
ssm_shape,
- data_type: dtype,
+ data_type: model_shape.data_type,
})
},
};
diff --git a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs
index dbd1fb688..5a769a090 100644
--- a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs
+++ b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs
@@ -52,7 +52,7 @@ pub trait KVCacheLayerTrait<B: Backend> {
fn encode_zero(
&mut self,
- encoder: &mut Encoder<'_, B>,
+ encoder: &mut Encoder<B>,
);
fn state(&self) -> KVCacheLayerState;
@@ -267,7 +267,7 @@ impl<B: Backend, Buf: Buffer<Backend = B>> KVCacheLayerTrait<B> for KVCacheLayer
fn encode_zero(
&mut self,
- encoder: &mut Encoder<'_, B>,
+ encoder: &mut Encoder<B>,
) {
if self.as_any().is::<KVCacheLayer<B, B::SparseBuffer>>() {
return;
diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rs
index 8efdfe4fc..57b50f9d9 100644
--- a/crates/backend-uzu/src/forward_pass/model_shape.rs
+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs
@@ -1,12 +1,12 @@
use crate::{
DataType,
- config::{DecoderConfig, MLPConfig, MixerConfig},
+ config::{decoder::DecoderConfig, token_mixer::AnyTokenMixerConfig},
};
#[derive(Debug, Clone)]
pub struct ModelShape {
- activation_type: DataType,
- kv_cache_type: DataType,
+ pub data_type: DataType,
+ pub rope_data_type: DataType,
vocabulary_size: usize,
model_dim: usize,
@@ -14,12 +14,15 @@ pub struct ModelShape {
num_groups: usize,
pub num_layers: usize,
- layer_mixers: Box<[MixerConfig]>,
+ layer_mixers: Box<[AnyTokenMixerConfig]>,
kv_source_layers: Box<[Option<usize>]>,
}
impl ModelShape {
- pub fn from_decoder_config(decoder_config: &DecoderConfig) -> Self {
+ pub fn from_decoder_config(
+ decoder_config: &DecoderConfig,
+ data_type: DataType,
+ ) -> Self {
let tf = &decoder_config.transformer_config;
let layer_configs = &tf.layer_configs;
let num_layers = layer_configs.len();
@@ -31,17 +34,12 @@ impl ModelShape {
assert_eq!(attn.num_groups, num_groups, "attention layers must share num_groups");
}
- let activation_type: DataType = match &layer_configs[0].mlp_config {
- MLPConfig::Dense(d) => d.linear_config.activation_precision().into(),
- MLPConfig::MixtureOfExperts(m) => m.expert_config.linear_config.activation_precision().into(),
- };
-
- let layer_mixers: Box<[MixerConfig]> = layer_configs.iter().map(|l| l.mixer_config.clone()).collect();
- let kv_source_layers: Box<[Option<usize>]> = layer_configs.iter().map(|l| l.kv_source_layer).collect();
+ let layer_mixers: Box<[AnyTokenMixerConfig]> = layer_configs.iter().map(|l| l.mixer_config.clone()).collect();
+ let kv_source_layers: Box<[Option<usize>]> = layer_configs.iter().map(|l| l.kv_source_layer_index).collect();
Self {
- activation_type,
- kv_cache_type: activation_type,
+ data_type,
+ rope_data_type: DataType::F32,
vocabulary_size: decoder_config.vocab_size,
model_dim: tf.model_dim,
context_length: tf.context_length,
@@ -52,14 +50,6 @@ impl ModelShape {
}
}
- pub fn activation_data_type(&self) -> DataType {
- self.activation_type
- }
-
- pub fn kv_cache_data_type(&self) -> DataType {
- self.kv_cache_type
- }
-
pub fn num_groups(&self) -> usize {
self.num_groups
}
@@ -72,7 +62,7 @@ impl ModelShape {
&self.kv_source_layers
}
- pub fn layer_mixers(&self) -> &[MixerConfig] {
+ pub fn layer_mixers(&self) -> &[AnyTokenMixerConfig] {
&self.layer_mixers
}
diff --git a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
index 678e3c8a7..0a7e470bb 100644
--- a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
+++ b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
@@ -13,6 +13,7 @@ pub struct RopeBuffers<B: Backend> {
pub sines: Allocation<B>,
max_sequence_length: usize,
dim: usize,
+ data_type: DataType,
}
impl<B: Backend> RopeBuffers<B> {
@@ -28,18 +29,29 @@ impl<B: Backend> RopeBuffers<B> {
sines: context.create_array_uninitialized(&shape, data_type).into_allocation(),
max_sequence_length,
dim: head_dim,
+ data_type,
}
}
pub fn update_data(
&mut self,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
rope_index: usize,
) -> Result<(), Error> {
- let rope_tree =
- parameter_tree.subtree(&format!("ropes.{}", rope_index)).map_err(|_| Error::UnableToLoadWeights)?;
- self.cosines = rope_tree.leaf_allocation("cosines").map_err(|_| Error::UnableToLoadWeights)?;
- self.sines = rope_tree.leaf_allocation("sines").map_err(|_| Error::UnableToLoadWeights)?;
+ let rope_tree = parameter_tree
+ .subtree(&format!("ropes.{}", rope_index))
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let cosines_leaf = rope_tree.leaf("cosines").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let cosines_leaf = cosines_leaf
+ .validate(&[self.max_sequence_length, self.dim], self.data_type)
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ self.cosines = cosines_leaf.read_allocation().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+
+ let sines_leaf = rope_tree.leaf("sines").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let sines_leaf = sines_leaf
+ .validate(&[self.max_sequence_length, self.dim], self.data_type)
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ self.sines = sines_leaf.read_allocation().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
Ok(())
}
diff --git a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
index 37652bd64..de58f52a8 100644
--- a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
+++ b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
@@ -1,11 +1,8 @@
-use half::{bf16, f16};
-
use super::RopeBuffers;
use crate::{
- DataType,
- array::ArrayContextExt,
- backends::common::{Allocation, AsBufferRangeMut, Backend, DenseBuffer},
- config::{DecoderConfig, RoPEConfig},
+ backends::common::Backend,
+ config::{decoder::DecoderConfig, rope::AnyRoPEConfig},
+ forward_pass::model_shape::ModelShape,
parameters::ParameterTree,
session::types::Error,
};
@@ -19,114 +16,71 @@ enum LayerRopeKind {
pub struct SharedBuffers<B: Backend> {
pub rope_buffers: Box<[RopeBuffers<B>]>,
layer_rope_kinds: Box<[LayerRopeKind]>,
- pub attention_sinks: Box<[Option<Allocation<B>>]>,
}
impl<B: Backend> SharedBuffers<B> {
pub fn new(
context: &B::Context,
decoder_config: &DecoderConfig,
+ model_shape: &ModelShape,
) -> Self {
let tf = &decoder_config.transformer_config;
- let mut configs = Vec::<RoPEConfig>::new();
+ let mut configs = Vec::<(AnyRoPEConfig, usize)>::new();
let layer_rope_kinds: Box<[LayerRopeKind]> = tf
.layer_configs
.iter()
.map(|layer_config| {
- if layer_config.mixer_config.as_attention().is_none() {
+ let Some(attention_config) = layer_config.mixer_config.as_attention() else {
return LayerRopeKind::NoKernel;
- }
+ };
let Some(rope_config) = &layer_config.rope_config else {
return LayerRopeKind::NoKernel;
};
- let index = configs.iter().position(|existing| existing == rope_config).unwrap_or_else(|| {
- configs.push(rope_config.clone());
- configs.len() - 1
- });
+ let head_dim = rope_config.head_dim().unwrap_or(attention_config.head_dim);
+ let index = configs
+ .iter()
+ .position(|(existing_config, existing_head_dim)| {
+ existing_config == rope_config && *existing_head_dim == head_dim
+ })
+ .unwrap_or_else(|| {
+ configs.push((rope_config.clone(), head_dim));
+ configs.len() - 1
+ });
LayerRopeKind::Indexed(index)
})
.collect();
let rope_buffers: Box<[RopeBuffers<B>]> = configs
.iter()
- .map(|config| {
- let common = config.common();
- RopeBuffers::new(context, common.max_sequence_length, common.head_dim, common.precision.into())
- })
- .collect();
-
- let attention_sinks = tf
- .layer_configs
- .iter()
- .map(|layer| {
- let attn = layer.mixer_config.as_attention()?;
- attn.has_sinks
- .then(|| context.create_array_uninitialized(&[attn.num_heads], DataType::F32).into_allocation())
+ .map(|(config, head_dim)| {
+ RopeBuffers::new(context, *config.max_sequence_length(), *head_dim, model_shape.rope_data_type)
})
.collect();
Self {
rope_buffers,
layer_rope_kinds,
- attention_sinks,
}
}
pub fn update_data(
&mut self,
- parameter_tree: &ParameterTree<B::Context>,
+ parameter_tree: &ParameterTree<B>,
) -> Result<(), Error> {
- let transformer_tree = parameter_tree.subtree("transformer").map_err(|_| Error::UnableToLoadWeights)?;
+ let transformer_tree =
+ parameter_tree.subtree("transformer").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
self.update_data_from_transformer_tree(&transformer_tree)?;
Ok(())
}
pub fn update_data_from_transformer_tree(
&mut self,
- transformer_tree: &ParameterTree<B::Context>,
+ transformer_tree: &ParameterTree<B>,
) -> Result<(), Error> {
for (rope_index, rope_buffers) in self.rope_buffers.iter_mut().enumerate() {
rope_buffers.update_data(transformer_tree, rope_index)?;
}
- for (layer_idx, sink_cell) in self.attention_sinks.iter_mut().enumerate() {
- let Some(sink_cell) = sink_cell.as_mut() else {
- continue;
- };
- let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_idx)).unwrap();
- let attn_tree = layer_tree.subtree("mixer").unwrap();
- let sinks_arr = attn_tree.leaf_array("sinks").unwrap();
- let dst_slice = unsafe {
- let buffer_range = sink_cell.as_buffer_range_mut();
- let range = buffer_range.range();
- std::slice::from_raw_parts_mut(
- (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start) as *mut f32,
- range.len() / std::mem::size_of::<f32>(),
- )
- };
-
- match sinks_arr.data_type() {
- DataType::F32 => {
- let src = sinks_arr.as_slice::<f32>();
- dst_slice.copy_from_slice(src);
- },
- DataType::BF16 => {
- let src = sinks_arr.as_slice::<bf16>();
- for (dst_val, src_val) in dst_slice.iter_mut().zip(src.iter()) {
- *dst_val = f32::from(*src_val);
- }
- },
- DataType::F16 => {
- let src = sinks_arr.as_slice::<f16>();
- for (dst_val, src_val) in dst_slice.iter_mut().zip(src.iter()) {
- *dst_val = f32::from(*src_val);
- }
- },
- other => {
- panic!("Unsupported attention sink data type: {:?}", other);
- },
- }
- }
Ok(())
}
@@ -139,11 +93,4 @@ impl<B: Backend> SharedBuffers<B> {
LayerRopeKind::Indexed(index) => Some(&self.rope_buffers[index]),
}
}
-
- pub fn attention_sinks(
- &self,
- layer_index: usize,
- ) -> Option<&Allocation<B>> {
- self.attention_sinks.get(layer_index)?.as_ref()
- }
}
diff --git a/crates/backend-uzu/src/forward_pass/traces.rs b/crates/backend-uzu/src/forward_pass/traces.rs
index 820073ff1..4b3474955 100644
--- a/crates/backend-uzu/src/forward_pass/traces.rs
+++ b/crates/backend-uzu/src/forward_pass/traces.rs
@@ -31,8 +31,7 @@ impl<B: Backend> LayerActivationTrace<B> {
suffix_length: usize,
) -> Self {
let main_shape = model_shape.main_shape(suffix_length);
- let activation_data_type = model_shape.activation_data_type();
- let main = || context.create_array_uninitialized(&main_shape, activation_data_type);
+ let main = || context.create_array_uninitialized(&main_shape, model_shape.data_type);
Self {
inputs: main(),
@@ -62,16 +61,15 @@ impl<B: Backend> ActivationTrace<B> {
model_shape: &ModelShape,
suffix_length: usize,
) -> Self {
- let activation_data_type = model_shape.activation_data_type();
let main_shape = model_shape.main_shape(suffix_length);
let layer_results = create_layer_results(context, model_shape, suffix_length);
Self {
embedding_norm: None,
layer_results,
- output_norm: context.create_array_uninitialized(&main_shape, activation_data_type),
+ output_norm: context.create_array_uninitialized(&main_shape, model_shape.data_type),
output_pooling: None,
- logits: context.create_array_uninitialized(&model_shape.logits_shape(suffix_length), activation_data_type),
+ logits: context.create_array_uninitialized(&model_shape.logits_shape(suffix_length), model_shape.data_type),
}
}
@@ -81,17 +79,16 @@ impl<B: Backend> ActivationTrace<B> {
suffix_length: usize,
num_labels: usize,
) -> Self {
- let activation_data_type = model_shape.activation_data_type();
let main_shape = model_shape.main_shape(suffix_length);
let layer_results = create_layer_results(context, model_shape, suffix_length);
let model_dim = model_shape.main_shape(1)[1];
Self {
- embedding_norm: Some(context.create_array_uninitialized(&main_shape, activation_data_type)),
+ embedding_norm: Some(context.create_array_uninitialized(&main_shape, model_shape.data_type)),
layer_results,
- output_norm: context.create_array_uninitialized(&main_shape, activation_data_type),
- output_pooling: Some(context.create_array_uninitialized(&[1, model_dim], activation_data_type)),
- logits: context.create_array_uninitialized(&[1, num_labels], activation_data_type),
+ output_norm: context.create_array_uninitialized(&main_shape, model_shape.data_type),
+ output_pooling: Some(context.create_array_uninitialized(&[1, model_dim], model_shape.data_type)),
+ logits: context.create_array_uninitialized(&[1, num_labels], model_shape.data_type),
}
}
diff --git a/crates/backend-uzu/src/inference/chat/bridging.rs b/crates/backend-uzu/src/inference/chat/bridging.rs
index 6cec0db3d..036e4e254 100644
--- a/crates/backend-uzu/src/inference/chat/bridging.rs
+++ b/crates/backend-uzu/src/inference/chat/bridging.rs
@@ -58,8 +58,7 @@ pub fn build_decoding_config(
Some(ShojiSpeculationPreset::Classification {
feature,
}) => {
- let tokenizer =
- Tokenizer::from_file(model_path.join("tokenizer.json")).map_err(|_| Error::UnableToLoadTokenizer)?;
+ let tokenizer = Tokenizer::from_file(model_path.join("tokenizer.json"))?;
let proposals: Vec<Vec<u64>> = feature
.values
.iter()
diff --git a/crates/backend-uzu/src/inference/chat/mod.rs b/crates/backend-uzu/src/inference/chat/mod.rs
index 072f40d04..473451ce6 100644
--- a/crates/backend-uzu/src/inference/chat/mod.rs
+++ b/crates/backend-uzu/src/inference/chat/mod.rs
@@ -41,7 +41,7 @@ impl Instance {
) -> Result<Self, Error> {
let model_path = PathBuf::from(reference.clone());
let decoding_config = build_decoding_config(&config, &model_path)?;
- let session = ChatSession::new(model_path, decoding_config).map_err(Error::from)?;
+ let session = ChatSession::new(model_path, decoding_config)?;
let container = Container::new(session);
Ok(Self {
session_container: container,
@@ -62,9 +62,12 @@ impl InstanceTrait for Instance {
.map_err(|error| Error::Runtime {
message: error.to_string(),
})
- .and_then(|mut session| session.reset().map_err(Error::from));
+ .and_then(|mut session| {
+ session.reset()?;
+ Ok(())
+ });
Box::pin(async move {
- result.map_err(|error| Box::new(error) as BackendError)?;
+ result?;
Ok(Box::new(State) as Box<dyn StateTrait>)
})
}
diff --git a/crates/backend-uzu/src/inference/classification/mod.rs b/crates/backend-uzu/src/inference/classification/mod.rs
index 073045065..496d540a8 100644
--- a/crates/backend-uzu/src/inference/classification/mod.rs
+++ b/crates/backend-uzu/src/inference/classification/mod.rs
@@ -33,7 +33,7 @@ pub struct Instance {
impl Instance {
pub fn new(reference: String) -> Result<Self, Error> {
let model_path = PathBuf::from(reference);
- let session = ClassificationSession::new(model_path).map_err(Error::from)?;
+ let session = ClassificationSession::new(model_path)?;
let container = Container::new(session);
Ok(Self {
session_container: container,
diff --git a/crates/backend-uzu/src/inference/error.rs b/crates/backend-uzu/src/inference/error.rs
index dfc8f2b21..50100b5a4 100644
--- a/crates/backend-uzu/src/inference/error.rs
+++ b/crates/backend-uzu/src/inference/error.rs
@@ -10,8 +10,8 @@ pub enum Error {
Session {
message: String,
},
- #[error("Unable to load tokenizer")]
- UnableToLoadTokenizer,
+ #[error("Unable to load tokenizer: {0}")]
+ UnableToLoadTokenizer(#[from] tokenizers::Error),
}
impl From<ChatSessionError> for Error {
diff --git a/crates/backend-uzu/src/inference/mod.rs b/crates/backend-uzu/src/inference/mod.rs
index 87d6bea05..d334ebf2e 100644
--- a/crates/backend-uzu/src/inference/mod.rs
+++ b/crates/backend-uzu/src/inference/mod.rs
@@ -11,7 +11,7 @@ pub use chat::Instance as ChatInstance;
pub use classification::Instance as ClassificationInstance;
pub use container::Container;
pub use error::Error;
-pub use model_metadata::{ModelMetadata, resolve_model_metadata};
+pub use model_metadata::{ModelMetadataError, resolve_model_specialization};
use shoji::{
traits::{
Backend as BackendTrait,
@@ -63,7 +63,7 @@ impl chat_message_trait::Backend for Backend {
config: ShojiChatConfig,
) -> Pin<Box<dyn Future<Output = Result<Box<dyn chat_message_trait::Instance>, BackendError>> + Send + '_>> {
Box::pin(async move {
- let instance = ChatInstance::new(reference, config).map_err(|error| Box::new(error) as BackendError)?;
+ let instance = ChatInstance::new(reference, config)?;
Ok(Box::new(instance) as Box<dyn chat_message_trait::Instance>)
})
}
@@ -76,7 +76,7 @@ impl classification_trait::Backend for Backend {
_config: classification_trait::Config,
) -> Pin<Box<dyn Future<Output = Result<Box<dyn classification_trait::Instance>, BackendError>> + Send + '_>> {
Box::pin(async move {
- let instance = ClassificationInstance::new(reference).map_err(|error| Box::new(error) as BackendError)?;
+ let instance = ClassificationInstance::new(reference)?;
Ok(Box::new(instance) as Box<dyn classification_trait::Instance>)
})
}
@@ -89,7 +89,7 @@ impl text_to_speech_trait::Backend for Backend {
_config: text_to_speech_trait::Config,
) -> Pin<Box<dyn Future<Output = Result<Box<dyn text_to_speech_trait::Instance>, BackendError>> + Send + '_>> {
Box::pin(async move {
- let instance = TextToSpeechInstance::new(reference).map_err(|error| Box::new(error) as BackendError)?;
+ let instance = TextToSpeechInstance::new(reference)?;
Ok(Box::new(instance) as Box<dyn text_to_speech_trait::Instance>)
})
}
diff --git a/crates/backend-uzu/src/inference/model_metadata.rs b/crates/backend-uzu/src/inference/model_metadata.rs
index a71afb779..adf2f5030 100644
--- a/crates/backend-uzu/src/inference/model_metadata.rs
+++ b/crates/backend-uzu/src/inference/model_metadata.rs
@@ -2,24 +2,23 @@ use std::{fs::File, io::BufReader, path::Path};
use shoji::types::model::ModelSpecialization;
-use crate::config::{ModelConfig, ModelMetadata as InnerModelMetadata, ModelType};
+use crate::config::model::AnyModelConfig;
-pub struct ModelMetadata {
- pub toolchain_version: String,
- pub specialization: ModelSpecialization,
+#[derive(Debug, thiserror::Error)]
+pub enum ModelMetadataError {
+ #[error("Unable to open model configuration: {0}")]
+ UnableToOpenConfig(#[from] std::io::Error),
+ #[error("Unable to deserialize model configuration: {0}")]
+ UnableToDeserializeConfig(#[from] serde_json::Error),
}
-pub fn resolve_model_metadata(model_path: &Path) -> Option<ModelMetadata> {
+pub fn resolve_model_specialization(model_path: &Path) -> Result<ModelSpecialization, ModelMetadataError> {
let config_path = model_path.join("config.json");
- let file = File::open(&config_path).ok()?;
- let metadata: InnerModelMetadata<ModelConfig> = serde_json::from_reader(BufReader::new(file)).ok()?;
- let specialization = match metadata.model_type {
- ModelType::LanguageModel => ModelSpecialization::Chat {},
- ModelType::ClassifierModel => ModelSpecialization::Classification {},
- ModelType::TtsModel => ModelSpecialization::TextToSpeech {},
- };
- Some(ModelMetadata {
- toolchain_version: metadata.toolchain_version,
- specialization,
+ let file = File::open(&config_path)?;
+ let config: AnyModelConfig = serde_json::from_reader(BufReader::new(file))?;
+ Ok(match config {
+ AnyModelConfig::LanguageModelConfig(_) => ModelSpecialization::Chat {},
+ AnyModelConfig::ClassifierModelConfig(_) => ModelSpecialization::Classification {},
+ AnyModelConfig::TTSModelConfig(_) => ModelSpecialization::TextToSpeech {},
})
}
diff --git a/crates/backend-uzu/src/inference/text_to_speech/mod.rs b/crates/backend-uzu/src/inference/text_to_speech/mod.rs
index f46d3a466..33c7e1f66 100644
--- a/crates/backend-uzu/src/inference/text_to_speech/mod.rs
+++ b/crates/backend-uzu/src/inference/text_to_speech/mod.rs
@@ -36,19 +36,16 @@ use crate::{
session::{TtsSession, config::TtsRunConfig, types::Input},
};
-#[cfg(metal_backend)]
-type UzuTtsSession = TtsSession<Metal>;
-
pub struct Instance {
#[cfg(metal_backend)]
- session_container: Container<UzuTtsSession>,
+ session_container: Container<TtsSession<Metal>>,
}
impl Instance {
#[cfg(metal_backend)]
pub fn new(reference: String) -> Result<Self, Error> {
let model_path = std::path::PathBuf::from(reference);
- let session = UzuTtsSession::new(model_path).map_err(Error::from)?;
+ let session = TtsSession::<Metal>::new(model_path)?;
Ok(Self {
session_container: Container::new(session),
})
@@ -109,7 +106,7 @@ impl InstanceTrait for Instance {
#[cfg(metal_backend)]
fn run(
- session_container: Container<UzuTtsSession>,
+ session_container: Container<TtsSession<Metal>>,
input: Input,
_cancel_token: CancellationToken,
sender: UnboundedSender<Result<ShojiPcmBatch, BackendError>>,
diff --git a/crates/backend-uzu/src/language_model/gpu_capture.rs b/crates/backend-uzu/src/language_model/gpu_capture.rs
index d7e13f5ef..645d5508d 100644
--- a/crates/backend-uzu/src/language_model/gpu_capture.rs
+++ b/crates/backend-uzu/src/language_model/gpu_capture.rs
@@ -9,6 +9,16 @@ use crate::{
utils::env_utils::EnvVar,
};
+#[derive(Debug, thiserror::Error)]
+pub enum GpuCaptureError<B: Backend> {
+ #[error("Unable to resolve capture timestamp: {0}")]
+ Timestamp(#[from] std::time::SystemTimeError),
+ #[error("Unable to get current directory: {0}")]
+ CurrentDirectory(#[from] std::io::Error),
+ #[error("Backend capture failed: {0}")]
+ Backend(#[source] B::Error),
+}
+
pub struct GpuCaptureManager<B: Backend> {
capture_prefill_enabled: bool,
capture_decode_enabled: bool,
@@ -54,15 +64,12 @@ impl<B: Backend> GpuCaptureManager<B> {
&self,
context: &B::Context,
capture_type: &str,
- ) -> Result<PathBuf, String> {
- let timestamp =
- SystemTime::now().duration_since(UNIX_EPOCH).map_err(|e| format!("Time error: {}", e))?.as_secs();
+ ) -> Result<PathBuf, GpuCaptureError<B>> {
+ let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
- let trace_path = std::env::current_dir()
- .unwrap_or_else(|_| PathBuf::from("."))
- .join(format!("uzu_first_{}-{}.gputrace", capture_type, timestamp));
+ let trace_path = std::env::current_dir()?.join(format!("uzu_first_{}-{}.gputrace", capture_type, timestamp));
- context.start_capture(&trace_path).map_err(|err| err.to_string())?;
+ context.start_capture(&trace_path).map_err(GpuCaptureError::Backend)?;
println!("GPU capture started for first {}: {:?}", capture_type, trace_path);
@@ -73,8 +80,8 @@ impl<B: Backend> GpuCaptureManager<B> {
&mut self,
context: &B::Context,
capture_type: &str,
- ) -> Result<(), String> {
- context.stop_capture().map_err(|err| err.to_string())?;
+ ) -> Result<(), GpuCaptureError<B>> {
+ context.stop_capture().map_err(GpuCaptureError::Backend)?;
println!("GPU capture stopped for {}", capture_type);
match capture_type {
diff --git a/crates/backend-uzu/src/language_model/grammar/mod.rs b/crates/backend-uzu/src/language_model/grammar/mod.rs
index bf464d854..4082cc2d2 100644
--- a/crates/backend-uzu/src/language_model/grammar/mod.rs
+++ b/crates/backend-uzu/src/language_model/grammar/mod.rs
@@ -29,8 +29,8 @@ pub fn create_compiled_grammar(
#[cfg(grammar_xgrammar)]
{
use xgrammar::TokenizerInfo;
- let tokenizer_info = TokenizerInfo::from_huggingface(&tokenizer, None, stop_token_ids)
- .map_err(|msg| Error::GrammarError(msg))?;
+ let tokenizer_info =
+ TokenizerInfo::from_huggingface(&tokenizer, None, stop_token_ids).map_err(Error::GrammarError)?;
use xgram::CompiledXGrammar;
let grammar = CompiledXGrammar::from_config(config, None, &tokenizer_info)?;
diff --git a/crates/backend-uzu/src/language_model/grammar/xgram.rs b/crates/backend-uzu/src/language_model/grammar/xgram.rs
index 93cafc8dc..41334a79a 100644
--- a/crates/backend-uzu/src/language_model/grammar/xgram.rs
+++ b/crates/backend-uzu/src/language_model/grammar/xgram.rs
@@ -89,21 +89,17 @@ impl CompiledXGrammar {
} => {
let separators_ref = separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));
Grammar::from_json_schema(schema, *any_whitespace, *indent, separators_ref, *strict_mode, None, false)
- .map_err(|error_message| Error::GrammarError(error_message))?
+ .map_err(Error::GrammarError)?
},
GrammarConfig::Regex {
pattern,
print_converted_ebnf,
- } => Grammar::from_regex(pattern, *print_converted_ebnf)
- .map_err(|error_message| Error::GrammarError(error_message))?,
+ } => Grammar::from_regex(pattern, *print_converted_ebnf).map_err(Error::GrammarError)?,
GrammarConfig::BuiltinJson => Grammar::builtin_json_grammar(),
};
- let mut compiler = GrammarCompiler::new(tokenizer_info, 8, true, -1)
- .map_err(|error_message| Error::GrammarError(error_message))?;
- let compiled =
- compiler.compile_grammar(&grammar).map_err(|error_message| Error::GrammarError(error_message))?;
- let matcher = GrammarMatcher::new(&compiled, None, true, -1)
- .map_err(|error_message| Error::GrammarError(error_message))?;
+ let mut compiler = GrammarCompiler::new(tokenizer_info, 8, true, -1).map_err(Error::GrammarError)?;
+ let compiled = compiler.compile_grammar(&grammar).map_err(Error::GrammarError)?;
+ let matcher = GrammarMatcher::new(&compiled, None, true, -1).map_err(Error::GrammarError)?;
let engagement_state = if let Some(trigger_token_id) = trigger_token_id {
CompiledGrammarEngagementState::Triggered {
diff --git a/crates/backend-uzu/src/language_model/gumbel.rs b/crates/backend-uzu/src/language_model/gumbel.rs
index b2fcb18c6..651c8dcbb 100644
--- a/crates/backend-uzu/src/language_model/gumbel.rs
+++ b/crates/backend-uzu/src/language_model/gumbel.rs
@@ -83,6 +83,6 @@ pub fn speculator_sample(
speculator_probs
.iter()
.map(|(&k, &v)| (k, f32::ln(v) + gumbel_float(seed, revidx(k as u32))))
- .max_by(|(_ak, av), (_bk, bv)| f32::total_cmp(av, bv))
- .map(|(k, _v)| k)
+ .max_by(|(_, av), (_, bv)| f32::total_cmp(av, bv))
+ .map(|(k, _)| k)
}
diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rs
index 111f6f5f3..85747dc7f 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs
@@ -9,12 +9,13 @@ use super::{
rng::PRng,
};
use crate::{
+ DataType,
array::{Array, ArrayContextExt},
backends::common::{
Allocation, AsBufferRangeRef, Backend, CommandBuffer, Context, DenseBuffer, Encoder, Pending,
kernel::TokenCopySampledKernel,
},
- config::{LanguageModelConfig, ModelMetadata},
+ config::model::language_model::LanguageModelConfig,
encodable_block::{DecoderDecodeInput, SamplingArguments, SamplingInputs},
forward_pass::{cache_layers::CacheLayersSlice, kv_cache_layer::INVALID_POSITION, token_inputs::TokenInputs},
language_model::grammar::CompiledGrammar,
@@ -97,6 +98,7 @@ pub trait LanguageModelGeneratorTrait {
fn reset_state(&mut self);
fn peak_memory_usage(&self) -> Option<usize>;
+ fn activation_data_type(&self) -> DataType;
fn tokens_len(&self) -> usize;
fn tokens_push(
@@ -107,7 +109,7 @@ pub trait LanguageModelGeneratorTrait {
fn async_batch_size(
&self,
model_path: &Path,
- ) -> usize;
+ ) -> Result<usize, Error>;
fn get_slice(
&self,
@@ -262,7 +264,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let should_capture = self.gpu_capture.should_capture_prefill(step == 0);
if should_capture {
- let _ = self.gpu_capture.start_capture(&self.context.context, "prefill");
+ self.gpu_capture
+ .start_capture(&self.context.context, "prefill")
+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;
}
let _ = last_sampling_output.take();
@@ -282,7 +286,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let (sampling_output, run_time) = self.run_model(task, sampling_method)?;
if should_capture {
- self.gpu_capture.stop_capture(&self.context.context, "prefill").map_err(|_| Error::CaptureFailed)?;
+ self.gpu_capture
+ .stop_capture(&self.context.context, "prefill")
+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;
}
// Register the accepted prompt tokens from this step.
@@ -492,7 +498,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let is_first_decode = !is_continuation;
let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
if should_capture {
- let _ = self.gpu_capture.start_capture(&self.context.context, "decode");
+ self.gpu_capture
+ .start_capture(&self.context.context, "decode")
+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;
}
let batch_dim = task.active_row_count;
@@ -558,7 +566,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let completed = pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
drop(resources);
drop(completed);
- self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
+ self.gpu_capture
+ .stop_capture(&self.context.context, "decode")
+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;
} else {
assert!(self.async_in_flight[slot].is_none(), "async slot {slot} still holds an in-flight state");
self.async_in_flight[slot] = Some(InFlightForwardPass {
@@ -601,6 +611,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.context.peak_memory_usage()
}
+ fn activation_data_type(&self) -> DataType {
+ self.context.model_shape.data_type
+ }
+
fn tokens_len(&self) -> usize {
self.tokens.len()
}
@@ -616,8 +630,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
fn async_batch_size(
&self,
model_path: &Path,
- ) -> usize {
- self.decoding_config.async_batch_size.resolve::<B>(model_path, self.context.context.as_ref())
+ ) -> Result<usize, Error> {
+ self.decoding_config
+ .async_batch_size
+ .resolve::<B>(model_path, self.context.context.as_ref())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))
}
fn get_slice(
@@ -659,11 +676,11 @@ impl<B: Backend> LanguageModelGenerator<B> {
pub fn new(
model_path: &Path,
decoding_config: DecodingConfig,
- model_metadata: &ModelMetadata<LanguageModelConfig>,
+ model_config: &LanguageModelConfig,
) -> Result<Self, Error> {
let gpu_capture = GpuCaptureManager::new();
- let context = LanguageModelGeneratorContext::new(model_path, &decoding_config, model_metadata)?;
+ let context = LanguageModelGeneratorContext::new(model_path, &decoding_config, model_config)?;
Ok(Self {
decoding_config,
@@ -696,7 +713,9 @@ impl<B: Backend> LanguageModelGenerator<B> {
let token_inputs = self.build_token_inputs(task, None, None);
if should_capture {
- self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
+ self.gpu_capture
+ .start_capture(&self.context.context, "decode")
+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;
}
let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
@@ -728,7 +747,9 @@ impl<B: Backend> LanguageModelGenerator<B> {
let run_time = run_start.elapsed().as_secs_f64();
if should_capture {
- self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
+ self.gpu_capture
+ .stop_capture(&self.context.context, "decode")
+ .map_err(|error| Error::CaptureFailed(Box::new(error)))?;
}
Ok((sampling_output, run_time))
@@ -791,7 +812,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
output: sampling_output.as_mut().expect("Sampling requires output allocation"),
sampling_method: sampling_method.expect("Sampling requires method"),
batch_size: sampling_length,
- vocab_size: context.model_config.model_config.vocab_size,
+ vocab_size: context.model_config.decoder_config.vocab_size,
},
encoder,
);
@@ -809,7 +830,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
fn build_token_inputs(
&self,
- task: Task<'_>,
+ task: Task,
token_ids_array: Option<Array<B>>,
token_positions_array: Option<Array<B>>,
) -> TokenInputs<B> {
diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
index d7f7e75e6..2c9028212 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
@@ -13,7 +13,7 @@ use crate::{
Backend, Context, Kernels,
kernel::{TokenCopySampledKernel, kv_cache_update::KVCacheUpdate},
},
- config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
+ config::{decoder::DecoderConfig, model::language_model::LanguageModelConfig},
encodable_block::{Decoder, Sampling},
forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
language_model::rng::PRng,
@@ -118,31 +118,30 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
pub fn new(
model_path: &Path,
decoding_config: &DecodingConfig,
- model_metadata: &ModelMetadata<LanguageModelConfig>,
+ model_config: &LanguageModelConfig,
) -> Result<Self, Error> {
let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;
- let model_shape = ModelShape::from_decoder_config(&model_metadata.model_config.model_config);
-
- let prefill_step_size = decoding_config.prefill_step_size.resolve(&model_metadata.model_config);
+ let prefill_step_size = decoding_config.prefill_step_size.resolve(model_config);
let generate_suffix_length = decoding_config.generate_suffix_length();
let max_prefix_length: usize =
- Self::get_context_length_internal(&context, &model_metadata.model_config.model_config, decoding_config);
+ Self::get_context_length_internal(&context, &model_config.decoder_config, decoding_config);
let max_suffix_length: usize = std::cmp::max(prefill_step_size, generate_suffix_length);
let weights_path = model_path.join("model.safetensors");
- if !weights_path.exists() {
- return Err(Error::UnableToLoadWeights);
- }
- let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;
- let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
- let root_loader_view = loader.tree();
-
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &model_metadata.model_config.model_config);
+ let weights_file = File::open(&weights_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let loader = ParameterLoader::new(&weights_file, context.as_ref())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let root_loader_view =
+ loader.tree().subtree("decoder").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let model_shape = ModelShape::from_decoder_config(&model_config.decoder_config, DataType::BF16);
+
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &model_config.decoder_config, &model_shape);
shared_buffers.update_data(&root_loader_view)?;
let shared_buffers = Rc::new(shared_buffers);
- let executables = Decoder::new(context.as_ref(), &model_metadata.model_config.model_config, &root_loader_view);
+ let executables = Decoder::new(context.as_ref(), &model_config.decoder_config, &root_loader_view, &model_shape)
+ .map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;
let cache_layers = Rc::new(RefCell::new(CacheLayers::new(
context.as_ref(),
@@ -151,29 +150,32 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
max_suffix_length,
)));
- let intermediate_data_type: DataType =
- model_metadata.model_config.model_config.transformer_config.output_norm_config.scale_precision.into();
let kv_cache_update = Box::new(
- KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)
+ KVCacheUpdate::new(context.as_ref(), model_shape.data_type, max_prefix_length)
.map_err(|e| Error::UnableToCreateContext(e.into()))?,
);
let gpu_sampler =
- Sampling::<B>::new(&context, intermediate_data_type).map_err(|e| Error::UnableToCreateContext(e.into()))?;
+ Sampling::<B>::new(&context, model_shape.data_type).map_err(|e| Error::UnableToCreateContext(e.into()))?;
let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
.map_err(|e| Error::UnableToCreateContext(e.into()))?;
- let async_batch_size = decoding_config.async_batch_size.resolve::<B>(model_path, context.as_ref());
+ let async_batch_size = decoding_config
+ .async_batch_size
+ .resolve::<B>(model_path, context.as_ref())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);
+ loader.tree().assert_all_tensors_validated().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+
let seed = PRng::new(decoding_config.sampling_seed.resolve());
Ok(Self {
context,
cache_layers,
shared_buffers,
- model_config: model_metadata.model_config.clone(),
+ model_config: model_config.clone(),
model_shape,
executables,
kv_cache_update,
@@ -188,7 +190,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
&self,
decoding_config: &DecodingConfig,
) -> usize {
- Self::get_context_length_internal(&self.context, &self.model_config.model_config, decoding_config)
+ Self::get_context_length_internal(&self.context, &self.model_config.decoder_config, decoding_config)
}
fn get_context_length_internal(
diff --git a/crates/backend-uzu/src/lib.rs b/crates/backend-uzu/src/lib.rs
index 5daa9b751..06185d865 100644
--- a/crates/backend-uzu/src/lib.rs
+++ b/crates/backend-uzu/src/lib.rs
@@ -33,7 +33,7 @@ pub use utils::{TOOLCHAIN_VERSION, VERSION};
pub mod _private {
pub use crate::{
classifier::Classifier,
- config::{ModelConfig, ModelMetadata, ModelType},
+ config::model::AnyModelConfig,
encodable_block::{DecoderDecodeInput, Sampling},
forward_pass::{
cache_layers::CacheLayers, kv_cache_layer::KVCacheLayer, token_inputs::TokenInputs, traces::ActivationTrace,
@@ -42,6 +42,6 @@ pub mod _private {
language_model_generator_context::LanguageModelGeneratorContext,
sampler::{ArgmaxSampler, LogitsSampler},
},
- parameters::ParameterTree,
+ parameters::{ParameterLoaderError, ParameterTree},
};
}
diff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rs
index d79c7ebd2..806492323 100644
--- a/crates/backend-uzu/src/parameters/loader.rs
+++ b/crates/backend-uzu/src/parameters/loader.rs
@@ -1,16 +1,17 @@
use std::{
- collections::{HashMap, hash_map::Keys},
+ cell::RefCell,
+ collections::{HashMap, HashSet},
fs::File,
};
use thiserror::Error;
-use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingError, read_metadata as read_st_metadata};
+use super::safetensors_metadata::{HeaderLoadingError, read_metadata as read_st_metadata};
use crate::{
- ArrayElement, DataType,
- array::{Array, ArrayContextExt},
+ Array, ArrayElement, DataType,
+ array::size_for_shape,
backends::common::{Allocation, AllocationType, AsBufferRangeRef, Backend, Context, DenseBuffer},
- utils::fs::file_read_exact_at,
+ utils::{fs::file_read_exact_at, strict_serde::DeserializeStrictOwned},
};
pub struct ParameterMetadata {
@@ -20,48 +21,18 @@ pub struct ParameterMetadata {
size: usize,
}
-fn st_metadata_into_index(
- global_offset: usize,
- st_metadata: STMetadata,
-) -> HashMap<String, ParameterMetadata> {
- st_metadata
- .tensors
- .into_iter()
- .map(|(key, value)| {
- let (local_begin, local_end) = value.data_offsets;
- let actual_local_offset = local_begin;
- let actual_size = local_end - local_begin;
- let weight_metadata = ParameterMetadata {
- shape: value.shape.into(),
- data_type: value.dtype.into(),
- offset: global_offset + actual_local_offset,
- size: actual_size,
- };
- (key, weight_metadata)
- })
- .collect()
-}
-
#[derive(Debug, Error)]
pub enum ParameterLoaderError<B: Backend> {
#[error("Array with key \"{0}\" not found.")]
KeyNotFound(String),
#[error("Couldn't find any arrays with prefix \"{0}\".")]
SubtreeNotFound(String),
- #[error(
- "Size mismatch: array of shape {shape:?} and data type \
- {data_type:?} expected to be {expected_size} bytes, got {actual_size} bytes."
- )]
- SizeMismatch {
- data_type: DataType,
- shape: Box<[usize]>,
- expected_size: usize,
- actual_size: usize,
- },
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
#[error("Failed to read data")]
ArrayLoadingError(#[from] std::io::Error),
+ #[error("Failed to deserialize metadata")]
+ MetadataDeserializationError(#[from] serde_json::Error),
#[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]
InvalidTensor {
shape: Box<[usize]>,
@@ -69,83 +40,71 @@ pub enum ParameterLoaderError<B: Backend> {
expected_shape: Box<[usize]>,
expected_data_type: DataType,
},
+ #[error("Invalid tensor byte size: got {size} bytes for {shape:?} @ {data_type:?}, expected {expected_size} bytes")]
+ InvalidTensorSize {
+ shape: Box<[usize]>,
+ data_type: DataType,
+ size: usize,
+ expected_size: usize,
+ },
+ #[error("Unvalidated tensors under {prefix:?}: {keys:?}")]
+ UnvalidatedTensors {
+ prefix: Option<String>,
+ keys: Box<[String]>,
+ },
}
-pub struct ParameterLoader<'context, 'file, C: Context>
-where
- 'file: 'context,
-{
- context: &'context C,
+pub struct ParameterLoader<'a, B: Backend> {
+ context: &'a B::Context,
index: HashMap<String, ParameterMetadata>,
- file: &'file File,
+ metadata: HashMap<String, String>,
+ validated_tensors: RefCell<HashSet<String>>,
+ file: &'a File,
}
-impl<'file, 'context, C: Context> ParameterLoader<'file, 'context, C>
-where
- 'file: 'context,
-{
+impl<'a, B: Backend> ParameterLoader<'a, B> {
pub fn new(
- file: &'file File,
- context: &'context C,
+ file: &'a File,
+ context: &'a B::Context,
) -> Result<Self, HeaderLoadingError> {
let (global_offset, st_metadata) = read_st_metadata(file)?;
- let index = st_metadata_into_index(global_offset, st_metadata);
+ let index = st_metadata
+ .tensors
+ .into_iter()
+ .map(|(key, value)| {
+ let (local_begin, local_end) = value.data_offsets;
+ let size =
+ local_end.checked_sub(local_begin).ok_or_else(|| HeaderLoadingError::InvalidTensorOffsets {
+ key: key.clone().into_boxed_str(),
+ begin: local_begin,
+ end: local_end,
+ })?;
+ let offset =
+ global_offset.checked_add(local_begin).ok_or_else(|| HeaderLoadingError::TensorOffsetOverflow {
+ key: key.clone().into_boxed_str(),
+ global_offset,
+ local_begin,
+ })?;
+ let weight_metadata = ParameterMetadata {
+ shape: value.shape.into(),
+ data_type: value.dtype.data_type()?,
+ offset,
+ size,
+ };
+ Ok((key, weight_metadata))
+ })
+ .collect::<Result<HashMap<_, _>, HeaderLoadingError>>()?;
+ let metadata = st_metadata.metadata.unwrap_or_default();
Ok(ParameterLoader {
context,
- file,
index,
+ metadata,
+ validated_tensors: RefCell::new(HashSet::new()),
+ file,
})
}
- pub fn keys(&self) -> Keys<'_, String, ParameterMetadata> {
- self.index.keys()
- }
-
- fn get_leaf<'leaf>(
- &'leaf self,
- key: &str,
- ) -> Result<ParameterLeaf<'file, 'context, 'leaf, C>, ParameterLoaderError<C::Backend>> {
- Ok(ParameterLeaf {
- metadata: self.index.get(key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.to_string()))?,
- loader: self,
- })
- }
-
- fn get(
- &self,
- key: &str,
- ) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {
- let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
- let (offset, size) = (metadata_entry.offset, metadata_entry.size);
- let mut array = self.context.create_array_uninitialized(&metadata_entry.shape, metadata_entry.data_type);
- if array.size() != size {
- return Err(ParameterLoaderError::SizeMismatch {
- data_type: metadata_entry.data_type,
- shape: metadata_entry.shape.to_owned(),
- expected_size: array.size(),
- actual_size: size,
- });
- }
-
- file_read_exact_at(self.file, array.as_bytes_mut(), offset as u64)?;
- Ok(array)
- }
-
- pub fn read_extract_at(
- &self,
- key: &str,
- buf: &mut [u8],
- shape: &mut Box<[usize]>,
- data_type: &mut DataType,
- ) -> Result<(), ParameterLoaderError<C::Backend>> {
- let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
- file_read_exact_at(self.file, buf, metadata_entry.offset as u64)?;
- *shape = metadata_entry.shape.to_owned();
- *data_type = metadata_entry.data_type;
- Ok(())
- }
-
- pub fn tree<'loader>(&'loader self) -> ParameterTree<'loader, C> {
+ pub fn tree(&self) -> ParameterTree<'_, B> {
ParameterTree {
loader: self,
prefix: None,
@@ -153,31 +112,20 @@ where
}
}
-pub struct ParameterLeaf<'file, 'context, 'leaf, C: Context> {
+pub struct ParameterLeaf<'a, 'leaf, B: Backend, const VALIDATED: bool> {
+ key: &'leaf str,
metadata: &'leaf ParameterMetadata,
- loader: &'leaf ParameterLoader<'context, 'file, C>,
+ loader: &'leaf ParameterLoader<'a, B>,
}
-impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C> {
- pub fn shape(&self) -> &[usize] {
- &self.metadata.shape
- }
-
- pub fn data_type(&self) -> DataType {
- self.metadata.data_type
- }
-
- pub fn size(&self) -> usize {
- self.metadata.size
- }
-
- pub fn validate_shape(
- &self,
+impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, false> {
+ pub fn validate(
+ self,
expected_shape: &[usize],
expected_data_type: DataType,
- ) -> Result<(), ParameterLoaderError<C::Backend>> {
- let shape = self.shape();
- let data_type = self.data_type();
+ ) -> Result<ParameterLeaf<'a, 'leaf, B, true>, ParameterLoaderError<B>> {
+ let shape = self.metadata.shape.as_ref();
+ let data_type = self.metadata.data_type;
if (shape, data_type) != (expected_shape, expected_data_type) {
return Err(ParameterLoaderError::InvalidTensor {
shape: shape.into(),
@@ -186,17 +134,42 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C
expected_data_type,
});
}
- Ok(())
+ let expected_size = size_for_shape(expected_shape, expected_data_type);
+ if self.metadata.size != expected_size {
+ return Err(ParameterLoaderError::InvalidTensorSize {
+ shape: shape.into(),
+ data_type,
+ size: self.metadata.size,
+ expected_size,
+ });
+ }
+ self.loader.validated_tensors.borrow_mut().insert(self.key.to_string());
+ Ok(ParameterLeaf {
+ key: self.key,
+ metadata: self.metadata,
+ loader: self.loader,
+ })
+ }
+
+ #[cfg(test)]
+ pub fn unvalidated(self) -> ParameterLeaf<'a, 'leaf, B, true> {
+ ParameterLeaf {
+ key: self.key,
+ metadata: self.metadata,
+ loader: self.loader,
+ }
}
+}
- pub fn read_slice<T: ArrayElement>(&self) -> Result<Box<[T]>, ParameterLoaderError<C::Backend>> {
+impl<'a, 'leaf, B: Backend> ParameterLeaf<'a, 'leaf, B, true> {
+ pub fn read_slice<T: ArrayElement>(&self) -> Result<Box<[T]>, ParameterLoaderError<B>> {
let element_count = self.metadata.size / std::mem::size_of::<T>();
let mut data = vec![T::zeroed(); element_count];
file_read_exact_at(self.loader.file, bytemuck::cast_slice_mut(&mut data), self.metadata.offset as u64)?;
Ok(data.into_boxed_slice())
}
- pub fn read_allocation(&self) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {
+ pub fn read_allocation(&self) -> Result<Allocation<B>, ParameterLoaderError<B>> {
let allocation = self
.loader
.context
@@ -216,18 +189,19 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C
)?;
Ok(allocation)
}
+
+ pub fn read_array(&self) -> Result<Array<B>, ParameterLoaderError<B>> {
+ let allocation = self.read_allocation()?;
+ Ok(unsafe { Array::from_allocation(allocation, 0, self.metadata.shape.as_ref(), self.metadata.data_type) })
+ }
}
-pub struct ParameterTree<'loader, C: Context> {
- loader: &'loader ParameterLoader<'loader, 'loader, C>,
+pub struct ParameterTree<'loader, B: Backend> {
+ loader: &'loader ParameterLoader<'loader, B>,
prefix: Option<String>,
}
-impl<'loader, C: Context> ParameterTree<'loader, C> {
- pub fn path_prefix(&self) -> Option<&str> {
- self.prefix.as_deref()
- }
-
+impl<'loader, B: Backend> ParameterTree<'loader, B> {
fn join_prefix(
&self,
name: &str,
@@ -238,47 +212,64 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {
pub fn subtree(
&self,
name: &str,
- ) -> Result<Self, ParameterLoaderError<C::Backend>> {
+ ) -> Result<Self, ParameterLoaderError<B>> {
let new_prefix = self.join_prefix(name);
- let num_suffixes = self.loader.keys().filter_map(|suffix| suffix.strip_prefix(&new_prefix)).count();
- if num_suffixes > 0 {
+ let subtree_prefix = format!("{new_prefix}.");
+ if self.loader.index.keys().any(|key| key.starts_with(&subtree_prefix)) {
Ok(Self {
loader: self.loader,
prefix: Some(new_prefix),
})
} else {
- Err(ParameterLoaderError::SubtreeNotFound(name.to_string()))
+ Err(ParameterLoaderError::SubtreeNotFound(new_prefix))
}
}
- pub fn leaf_array(
- &self,
- name: &str,
- ) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {
- self.loader.get(&self.join_prefix(name))
- }
-
pub fn leaf<'leaf>(
&'leaf self,
name: &str,
- ) -> Result<ParameterLeaf<'loader, 'loader, 'leaf, C>, ParameterLoaderError<C::Backend>> {
- self.loader.get_leaf(&self.join_prefix(name))
+ ) -> Result<ParameterLeaf<'loader, 'leaf, B, false>, ParameterLoaderError<B>> {
+ let key = self.join_prefix(name);
+ let Some((key, metadata)) = self.loader.index.get_key_value(&key) else {
+ return Err(ParameterLoaderError::KeyNotFound(key));
+ };
+ Ok(ParameterLeaf {
+ key,
+ metadata,
+ loader: self.loader,
+ })
}
- pub fn leaf_allocation(
+ pub fn metadata<T: DeserializeStrictOwned>(
&self,
name: &str,
- ) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {
- self.leaf(name)?.read_allocation()
+ ) -> Result<T, ParameterLoaderError<B>> {
+ let new_prefix = self.join_prefix(name);
+
+ Ok(serde_json::from_str(
+ self.loader.metadata.get(&new_prefix).ok_or(ParameterLoaderError::KeyNotFound(new_prefix))?,
+ )?)
}
- pub fn read_extract_at(
- &self,
- name: &str,
- buf: &mut [u8],
- shape: &mut Box<[usize]>,
- data_type: &mut DataType,
- ) -> Result<(), ParameterLoaderError<C::Backend>> {
- self.loader.read_extract_at(&self.join_prefix(name), buf, shape, data_type)
+ pub fn assert_all_tensors_validated(&self) -> Result<(), ParameterLoaderError<B>> {
+ let subtree_prefix = self.prefix.as_ref().map(|prefix| format!("{prefix}."));
+ let validated_tensors = self.loader.validated_tensors.borrow();
+ let mut unvalidated_tensors = self
+ .loader
+ .index
+ .keys()
+ .filter(|key| subtree_prefix.as_ref().is_none_or(|prefix| key.starts_with(prefix)))
+ .filter(|key| !validated_tensors.contains(*key))
+ .cloned()
+ .collect::<Vec<_>>();
+ unvalidated_tensors.sort();
+ if unvalidated_tensors.is_empty() {
+ Ok(())
+ } else {
+ Err(ParameterLoaderError::UnvalidatedTensors {
+ prefix: self.prefix.clone(),
+ keys: unvalidated_tensors.into_boxed_slice(),
+ })
+ }
}
}
diff --git a/crates/backend-uzu/src/parameters/safetensors_metadata.rs b/crates/backend-uzu/src/parameters/safetensors_metadata.rs
index 2a027a470..cffd40b33 100644
--- a/crates/backend-uzu/src/parameters/safetensors_metadata.rs
+++ b/crates/backend-uzu/src/parameters/safetensors_metadata.rs
@@ -1,6 +1,6 @@
// This code is based on the safetensors implementation: https://docs.rs/safetensors/latest/src/safetensors/tensor.rs.html
-use std::{collections::HashMap, fs::File};
+use std::{collections::HashMap, fs::File, str::Utf8Error};
use serde::{Deserialize, Serialize};
use thiserror::Error;
@@ -9,18 +9,32 @@ use crate::{DataType, utils::fs::file_read_exact_at};
#[derive(Debug, Error)]
pub enum HeaderLoadingError {
- #[error("The header is an invalid UTF-8 string and cannot be read.")]
- InvalidHeader,
- #[error("The header does contain a valid string, but it is not valid JSON.")]
- InvalidHeaderDeserialization,
- #[error("The header is large than 100Mo which is considered too large.")]
+ #[error("Unable to read safetensors header bytes: {0}")]
+ UnableToReadHeader(#[source] std::io::Error),
+ #[error("Unable to read safetensors header JSON: {0}")]
+ UnableToReadHeaderJson(#[source] std::io::Error),
+ #[error("The header is an invalid UTF-8 string and cannot be read: {0}")]
+ InvalidHeader(#[from] Utf8Error),
+ #[error("The header does contain a valid string, but it is not valid JSON: {0}")]
+ InvalidHeaderDeserialization(#[from] serde_json::Error),
+ #[error("The header is too large.")]
HeaderTooLarge,
- #[error("The header is smaller than 8 bytes.")]
- HeaderTooSmall,
#[error("The header length is invalid.")]
InvalidHeaderLength,
- #[error("Couldn't deserialize the header JSON.")]
- JsonError(#[from] serde_json::Error),
+ #[error("Unsupported safetensors dtype: {0:?}")]
+ UnsupportedDtype(Dtype),
+ #[error("Invalid data offsets for tensor {key}: begin={begin}, end={end}")]
+ InvalidTensorOffsets {
+ key: Box<str>,
+ begin: usize,
+ end: usize,
+ },
+ #[error("Tensor offset overflow for tensor {key}: global_offset={global_offset}, local_begin={local_begin}")]
+ TensorOffsetOverflow {
+ key: Box<str>,
+ global_offset: usize,
+ local_begin: usize,
+ },
}
#[derive(Debug, Serialize, Deserialize)]
@@ -76,37 +90,20 @@ pub enum Dtype {
U64,
}
-impl From<Dtype> for DataType {
- fn from(val: Dtype) -> Self {
- match val {
- Dtype::F16 => DataType::F16,
- Dtype::BF16 => DataType::BF16,
- Dtype::F32 => DataType::F32,
- Dtype::I8 => DataType::I8,
- Dtype::U8 => DataType::U8,
- Dtype::I32 => DataType::I32,
- Dtype::U32 => DataType::U32,
- Dtype::I64 => DataType::I64,
- Dtype::U64 => DataType::U64,
- Dtype::BOOL => DataType::I8,
- // Add mappings for other types as needed
- _ => panic!("Unsupported dtype: {:?}", val),
- }
- }
-}
-
-impl From<DataType> for Dtype {
- fn from(dtype: DataType) -> Self {
- match dtype {
- DataType::F16 => Dtype::F16,
- DataType::BF16 => Dtype::BF16,
- DataType::F32 => Dtype::F32,
- DataType::I8 => Dtype::I8,
- DataType::I32 => Dtype::I32,
- DataType::I64 => Dtype::I64,
- DataType::U64 => Dtype::U64,
- _ => panic!("Unsupported dtype: {:?}", dtype),
- }
+impl Dtype {
+ pub fn data_type(self) -> Result<DataType, HeaderLoadingError> {
+ Ok(match self {
+ Self::F16 => DataType::F16,
+ Self::BF16 => DataType::BF16,
+ Self::F32 => DataType::F32,
+ Self::I8 => DataType::I8,
+ Self::U8 => DataType::U8,
+ Self::I32 => DataType::I32,
+ Self::U32 => DataType::U32,
+ Self::I64 => DataType::I64,
+ Self::U64 => DataType::U64,
+ dtype => return Err(HeaderLoadingError::UnsupportedDtype(dtype)),
+ })
}
}
@@ -114,7 +111,7 @@ const MAX_HEADER_SIZE: usize = 100_000_000;
pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoadingError> {
let mut header_buffer = [0u8; size_of::<u64>()];
- file_read_exact_at(file, &mut header_buffer, 0).map_err(|_| HeaderLoadingError::HeaderTooSmall)?;
+ file_read_exact_at(file, &mut header_buffer, 0).map_err(HeaderLoadingError::UnableToReadHeader)?;
let metadata_size: usize =
u64::from_le_bytes(header_buffer).try_into().map_err(|_| HeaderLoadingError::HeaderTooLarge)?;
if metadata_size > MAX_HEADER_SIZE {
@@ -123,9 +120,8 @@ pub fn read_metadata(file: &File) -> Result<(usize, HashMetadata), HeaderLoading
let stop = metadata_size.checked_add(8).ok_or(HeaderLoadingError::InvalidHeaderLength)?;
let mut json_buffer: Box<[u8]> = core::iter::repeat(0).take(stop - size_of::<u64>()).collect();
- file_read_exact_at(file, &mut json_buffer, 8).map_err(|_| HeaderLoadingError::InvalidHeader)?;
- let string = core::str::from_utf8(&json_buffer).map_err(|_| HeaderLoadingError::InvalidHeader)?;
- let metadata: HashMetadata =
- serde_json::from_str(string).map_err(|_| HeaderLoadingError::InvalidHeaderDeserialization)?;
+ file_read_exact_at(file, &mut json_buffer, 8).map_err(HeaderLoadingError::UnableToReadHeaderJson)?;
+ let string = core::str::from_utf8(&json_buffer)?;
+ let metadata: HashMetadata = serde_json::from_str(string)?;
Ok((stop, metadata))
}
diff --git a/crates/backend-uzu/src/session/chat_session.rs b/crates/backend-uzu/src/session/chat_session.rs
index 9987ba81a..f6745d209 100644
--- a/crates/backend-uzu/src/session/chat_session.rs
+++ b/crates/backend-uzu/src/session/chat_session.rs
@@ -14,8 +14,11 @@ use std::{
use tokenizers::Tokenizer;
use crate::{
+ DataType,
backends::{common::Backend, select_backend},
- config::{LanguageModelConfig, MixerConfig, ModelMetadata},
+ config::{
+ model::language_model::LanguageModelConfig, token_codec::AnyTokenCodecConfig, token_mixer::AnyTokenMixerConfig,
+ },
language_model::{
LanguageModelGenerator, LanguageModelGeneratorTrait,
grammar::{CompiledGrammar, create_compiled_grammar},
@@ -24,7 +27,7 @@ use crate::{
session::{
config::{DecodingConfig, RunConfig},
helpers::{InputProcessor, InputProcessorDefault, OutputParser, is_directory_fits_ram},
- parameter::{ConfigResolvableValue, ContextMode},
+ parameter::{ConfigResolvableValue, ContextMode, SamplingMethod},
types::{Error, FinishReason, Input, Output, RunStats, Stats, StepStats, TotalStats},
},
};
@@ -47,14 +50,14 @@ struct RunContext {
pub struct ChatSession {
pub model_path: PathBuf,
- pub model_metadata: ModelMetadata<LanguageModelConfig>,
+ pub model_config: LanguageModelConfig,
tokenizer: Tokenizer,
stop_token_ids: Vec<i32>,
input_processor: Box<dyn InputProcessor>,
output_parser: OutputParser,
decoding_config: DecodingConfig,
- llm: Option<Box<dyn LanguageModelGeneratorTrait>>,
+ llm: Box<dyn LanguageModelGeneratorTrait>,
static_context: Option<Box<dyn Any>>,
}
@@ -79,20 +82,14 @@ impl ChatSession {
}
let config_path = model_path.join("config.json");
- if !config_path.exists() {
- return Err(Error::UnableToLoadConfig);
- }
- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
- let model_metadata: ModelMetadata<LanguageModelConfig> = serde_json::from_reader(BufReader::new(config_file))
- .map_err(|err| {
- eprintln!("Failed to parse config.json: {err}");
- Error::UnableToLoadConfig
- })?;
-
- let layers = &model_metadata.model_config.model_config.transformer_config.layer_configs;
+ let config_file = File::open(&config_path)?;
+ let model_config: LanguageModelConfig = serde_json::from_reader(BufReader::new(config_file))?;
+
+ let layers = &model_config.decoder_config.transformer_config.layer_configs;
let has_non_attention_mixer =
- layers.iter().any(|layer| !matches!(layer.mixer_config, MixerConfig::Attention(_)));
- let has_mamba_mixer = layers.iter().any(|layer| matches!(layer.mixer_config, MixerConfig::Mamba(_)));
+ layers.iter().any(|layer| !matches!(layer.mixer_config, AnyTokenMixerConfig::AttentionConfig(_)));
+ let has_mamba_mixer =
+ layers.iter().any(|layer| matches!(layer.mixer_config, AnyTokenMixerConfig::Mamba2Config(_)));
if has_non_attention_mixer {
match decoding_config.context_mode {
ContextMode::None => {},
@@ -112,50 +109,47 @@ impl ChatSession {
}
let tokenizer_path = model_path.join("tokenizer.json");
- if !tokenizer_path.exists() {
- return Err(Error::UnableToLoadTokenizer);
- }
- let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;
+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(Error::UnableToLoadTokenizer)?;
+
+ let AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) = &model_config.token_codec_config else {
+ return Err(Error::InvalidModelConfig("expected ChatCodecConfig token codec".to_string()));
+ };
let stop_token_ids: Vec<i32> =
- model_metadata.model_config.generation_config.stop_token_ids.iter().map(|&x| x as i32).collect();
+ model_config.generation_config.stop_token_ids.iter().map(|&x| x as i32).collect();
- let input_processor = InputProcessorDefault::new(model_metadata.model_config.message_processor_config.clone());
+ let input_processor = InputProcessorDefault::new(token_codec_config.clone());
- let output_parser =
- OutputParser::new(model_metadata.model_config.message_processor_config.output_parser_regex.clone())?;
+ let output_parser = OutputParser::new(token_codec_config.output_parser_regex.clone())?;
- let llm = Box::new(
- LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone(), &model_metadata)
- .map_err(Error::from)?,
- );
+ let llm = Box::new(LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone(), &model_config)?);
Ok(Self {
model_path,
- model_metadata,
+ model_config,
tokenizer,
stop_token_ids,
input_processor: Box::new(input_processor),
output_parser,
decoding_config,
- llm: Some(llm),
+ llm,
static_context: None,
})
}
- pub fn run<F>(
+ pub fn activation_data_type(&self) -> DataType {
+ self.llm.activation_data_type()
+ }
+
+ pub fn run(
&mut self,
input: Input,
config: RunConfig,
- progress: Option<F>,
- ) -> Result<Output, Error>
- where
- F: Fn(Output) -> bool,
- {
+ progress: Option<impl Fn(Output) -> bool>,
+ ) -> Result<Output, Error> {
match &self.decoding_config.context_mode {
ContextMode::None => {
- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- language_model_generator.reset_state();
+ self.llm.reset_state();
},
ContextMode::Static {
input,
@@ -163,12 +157,13 @@ impl ChatSession {
if self.static_context.is_none() {
let mut prefill_config = config.clone();
prefill_config.tokens_limit = 0;
- let (_out, ctx) = self.extend(input.clone(), None, prefill_config)?;
+ let (_, ctx) = self.extend(input.clone(), None, prefill_config)?;
self.static_context = Some(ctx);
}
let tmp = self.static_context.take();
+ self.llm.reset_state();
if let Some(ref ctx) = tmp {
- self.reconfigure_language_model_generator(Some(ctx))?;
+ self.llm.reconfigure_from_context(ctx.as_ref());
}
self.static_context = tmp;
},
@@ -182,8 +177,7 @@ impl ChatSession {
| ContextMode::Static {
..
} => {
- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- language_model_generator.reset_state();
+ self.llm.reset_state();
},
ContextMode::Dynamic => {},
}
@@ -191,8 +185,7 @@ impl ChatSession {
}
pub fn reset(&mut self) -> Result<(), Error> {
- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- language_model_generator.reset_state();
+ self.llm.reset_state();
Ok(())
}
@@ -202,35 +195,34 @@ impl ChatSession {
context: Option<&Box<dyn Any>>,
config: RunConfig,
) -> Result<(Output, Box<dyn Any>), Error> {
- self.reconfigure_language_model_generator(context)?;
+ self.llm.reset_state();
+ if let Some(ctx) = context {
+ self.llm.reconfigure_from_context(ctx.as_ref());
+ }
let output = self.run_internal(input, config, None::<fn(Output) -> bool>)?;
- let new_context = self.build_context_from_language_model_generator()?;
- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- language_model_generator.reset_state();
+ let new_context = self.llm.build_llm_context();
+ self.llm.reset_state();
Ok((output, new_context))
}
- fn run_internal<F>(
+ fn run_internal(
&mut self,
input: Input,
config: RunConfig,
- progress: Option<F>,
- ) -> Result<Output, Error>
- where
- F: Fn(Output) -> bool,
- {
+ progress: Option<impl Fn(Output) -> bool>,
+ ) -> Result<Output, Error> {
let run_start = Instant::now();
let text = self.input_processor.process(&input, config.enable_thinking, config.tokens_limit > 0)?;
let tokens: Vec<u64> = self
.tokenizer
.encode(text.as_str(), false)
- .map_err(|_| Error::UnableToEncodeText)?
+ .map_err(Error::UnableToEncodeText)?
.get_ids()
.iter()
.map(|&id| id as u64)
.collect();
- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
+ let language_model_generator = self.llm.as_mut();
let context_length = language_model_generator.get_context_length();
if tokens.len() >= context_length {
return Err(Error::ContextLengthExceeded);
@@ -239,10 +231,9 @@ impl ChatSession {
let prefix_offset = language_model_generator.tokens_len();
let prefix_len_before = prefix_offset;
- let eos_tokens: Vec<u64> =
- self.model_metadata.model_config.generation_config.stop_token_ids.iter().map(|&x| x as u64).collect();
+ let eos_tokens: Vec<u64> = self.model_config.generation_config.stop_token_ids.iter().copied().collect();
- let sampling_method = config.sampling_policy.resolve(&self.model_metadata.model_config);
+ let sampling_method = config.sampling_policy.resolve(&self.model_config);
let mut compiled_grammar: Option<Box<dyn CompiledGrammar>> = if let Some(ref config) = config.grammar_config {
Some(create_compiled_grammar(config, &self.tokenizer, Some(&self.stop_token_ids))?)
@@ -262,7 +253,7 @@ impl ChatSession {
let prefill_tokens = prefill_result.tokens.clone();
let prefill_duration = prefill_start.elapsed().as_secs_f64();
- let prefill_suffix_length = self.decoding_config.prefill_step_size.resolve(&self.model_metadata.model_config);
+ let prefill_suffix_length = self.decoding_config.prefill_step_size.resolve(&self.model_config);
let run_context = RunContext {
eos_tokens,
@@ -281,13 +272,13 @@ impl ChatSession {
let prefill_finish_reason = if grammar_terminated {
Some(FinishReason::Stop)
} else {
- Self::check_finish_reason(&run_context, language_model_generator.as_ref(), &prefill_tokens)
+ Self::check_finish_reason(&run_context, &*language_model_generator, &prefill_tokens)
};
let prefill_output = Self::build_output(
&self.tokenizer,
&self.output_parser,
&run_context,
- language_model_generator.as_ref(),
+ &*language_model_generator,
&[],
&[],
prefill_finish_reason.clone(),
@@ -310,12 +301,12 @@ impl ChatSession {
let can_use_async = language_model_generator.generate_suffix_length() == 1 && compiled_grammar.is_none();
let generate_output = if can_use_async {
- let batch_size = language_model_generator.async_batch_size(&self.model_path);
+ let batch_size = language_model_generator.async_batch_size(&self.model_path)?;
Self::run_async_batch(
&self.tokenizer,
&self.output_parser,
&run_context,
- language_model_generator.as_mut(),
+ language_model_generator,
sampling_method,
&progress,
batch_size,
@@ -325,7 +316,7 @@ impl ChatSession {
&self.tokenizer,
&self.output_parser,
&run_context,
- language_model_generator.as_mut(),
+ language_model_generator,
compiled_grammar.as_deref_mut(),
sampling_method,
&progress,
@@ -335,18 +326,15 @@ impl ChatSession {
Ok(generate_output.clone_with_duration(run_start.elapsed().as_secs_f64()))
}
- fn run_sync_generate<F>(
+ fn run_sync_generate(
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
language_model_generator: &mut dyn LanguageModelGeneratorTrait,
compiled_grammar: Option<&mut (dyn CompiledGrammar + 'static)>,
- sampling_method: super::parameter::SamplingMethod,
- progress: &Option<F>,
- ) -> Result<Output, Error>
- where
- F: Fn(Output) -> bool,
- {
+ sampling_method: SamplingMethod,
+ progress: &Option<impl Fn(Output) -> bool>,
+ ) -> Result<Output, Error> {
let mut generate_results: Vec<GenerateResult> = Vec::new();
let mut generate_durations: Vec<f64> = Vec::new();
let mut compiled_grammar_mut = compiled_grammar;
@@ -393,18 +381,15 @@ impl ChatSession {
}
}
- fn run_async_batch<F>(
+ fn run_async_batch(
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
llm: &mut dyn LanguageModelGeneratorTrait,
- sampling_method: super::parameter::SamplingMethod,
- progress: &Option<F>,
+ sampling_method: SamplingMethod,
+ progress: &Option<impl Fn(Output) -> bool>,
batch_size: usize,
- ) -> Result<Output, Error>
- where
- F: Fn(Output) -> bool,
- {
+ ) -> Result<Output, Error> {
let remaining_by_limit = run_context.tokens_limit.saturating_sub(run_context.prefill_result.tokens.len());
let remaining_by_context = run_context.context_length.saturating_sub(llm.tokens_len());
let tokens_to_generate = remaining_by_limit.min(remaining_by_context);
@@ -507,23 +492,6 @@ impl ChatSession {
Self::build_output(tokenizer, output_parser, run_context, llm, &results, &durations, Some(finish_reason))
}
- fn reconfigure_language_model_generator(
- &mut self,
- context: Option<&Box<dyn Any>>,
- ) -> Result<(), Error> {
- let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- language_model_generator.reset_state();
- if let Some(ctx) = context {
- language_model_generator.reconfigure_from_context(ctx.as_ref());
- }
- Ok(())
- }
-
- fn build_context_from_language_model_generator(&self) -> Result<Box<dyn Any>, Error> {
- let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- Ok(language_model_generator.build_llm_context())
- }
-
fn check_finish_reason(
run_context: &RunContext,
language_model_generator: &dyn LanguageModelGeneratorTrait,
@@ -555,7 +523,7 @@ impl ChatSession {
.chain(generate_results.iter().flat_map(|r| r.tokens.iter()))
.map(|&v| v as u32)
.collect();
- tokenizer.decode(&generated_tokens, true).map_err(|_| Error::UnableToDecodeText)
+ tokenizer.decode(&generated_tokens, true).map_err(Error::UnableToDecodeText)
}
fn build_output(
@@ -680,12 +648,6 @@ impl ChatSession {
}
pub fn peak_memory_usage(&self) -> Option<usize> {
- self.llm.as_ref().unwrap().peak_memory_usage()
- }
-}
-
-impl Drop for ChatSession {
- fn drop(&mut self) {
- self.llm = None
+ self.llm.peak_memory_usage()
}
}
diff --git a/crates/backend-uzu/src/session/classification_session.rs b/crates/backend-uzu/src/session/classification_session.rs
index 942650be4..d9cb15a37 100644
--- a/crates/backend-uzu/src/session/classification_session.rs
+++ b/crates/backend-uzu/src/session/classification_session.rs
@@ -1,11 +1,11 @@
-use std::path::PathBuf;
+use std::{fs::File, io::BufReader, path::PathBuf};
use tokenizers::Tokenizer;
use crate::{
backends::{common::Backend, select_backend},
classifier::{ClassificationOutput, Classifier, ClassifierTrait},
- config::{ClassifierModelConfig, ModelMetadata},
+ config::{model::classifier_model::ClassifierModelConfig, token_codec::AnyTokenCodecConfig},
session::{
helpers::{InputProcessor, InputProcessorDefault},
types::{Error, Input},
@@ -24,25 +24,23 @@ impl ClassificationSession {
}
pub fn new_with_backend<B: Backend>(model_path: PathBuf) -> Result<Self, Error> {
- let config_path = model_path.join("config.json");
- if !config_path.exists() {
+ if !model_path.exists() {
return Err(Error::ModelFolderNotFound);
}
- let config_file = std::fs::File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
- let model_metadata: ModelMetadata<ClassifierModelConfig> =
- serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
+ let config_path = model_path.join("config.json");
+ let config_file = File::open(&config_path)?;
+ let model_config: ClassifierModelConfig = serde_json::from_reader(BufReader::new(config_file))?;
let tokenizer_path = model_path.join("tokenizer.json");
- if !tokenizer_path.exists() {
- return Err(Error::UnableToLoadTokenizer);
- }
- let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;
+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(Error::UnableToLoadTokenizer)?;
+ let AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) = &model_config.token_codec_config else {
+ return Err(Error::InvalidModelConfig("expected ChatCodecConfig token codec".to_string()));
+ };
let input_processor =
- Box::new(InputProcessorDefault::new(model_metadata.model_config.message_processor_config.clone()))
- as Box<dyn InputProcessor>;
- let classifier = Box::new(Classifier::<B>::new(&model_path, &model_metadata)?);
+ Box::new(InputProcessorDefault::new(token_codec_config.clone())) as Box<dyn InputProcessor>;
+ let classifier = Box::new(Classifier::<B>::new(&model_path, &model_config)?);
Ok(Self {
tokenizer,
@@ -60,7 +58,7 @@ impl ClassificationSession {
let tokens: Vec<u64> = self
.tokenizer
.encode(text.as_str(), false)
- .map_err(|_| Error::UnableToEncodeText)?
+ .map_err(Error::UnableToEncodeText)?
.get_ids()
.iter()
.map(|&id| id as u64)
diff --git a/crates/backend-uzu/src/session/config/grammar_config.rs b/crates/backend-uzu/src/session/config/grammar_config.rs
index 054dc7952..19eed4fa4 100644
--- a/crates/backend-uzu/src/session/config/grammar_config.rs
+++ b/crates/backend-uzu/src/session/config/grammar_config.rs
@@ -58,10 +58,7 @@ impl GrammarConfig {
Self::BuiltinJson
}
- pub fn from_json_schema_type<T>() -> Result<Self, String>
- where
- T: JsonSchema,
- {
+ pub fn from_json_schema_type<T: JsonSchema>() -> Result<Self, String> {
let schema_root = schemars::schema_for!(T);
let schema_str =
serde_json::to_string(&schema_root).map_err(|e| format!("Failed to serialize schema: {}", e))?;
@@ -75,15 +72,12 @@ impl GrammarConfig {
})
}
- pub fn from_json_schema_type_with_config<T>(
+ pub fn from_json_schema_type_with_config<T: JsonSchema>(
any_whitespace: bool,
indent: Option<i32>,
separators: Option<(String, String)>,
strict_mode: bool,
- ) -> Result<Self, String>
- where
- T: JsonSchema,
- {
+ ) -> Result<Self, String> {
let schema_root = schemars::schema_for!(T);
let schema_str =
serde_json::to_string(&schema_root).map_err(|e| format!("Failed to serialize schema: {}", e))?;
diff --git a/crates/backend-uzu/src/session/helpers/input_processor.rs b/crates/backend-uzu/src/session/helpers/input_processor.rs
index d55887bff..04e392c7b 100644
--- a/crates/backend-uzu/src/session/helpers/input_processor.rs
+++ b/crates/backend-uzu/src/session/helpers/input_processor.rs
@@ -4,10 +4,10 @@ use minijinja::{Environment, context};
use minijinja_contrib::pycompat::unknown_method_callback;
use crate::{
- config::MessageProcessorConfig,
+ config::token_codec::chat_codec::ChatCodecConfig,
session::{
parameter::ConfigResolvableValue,
- types::{Error, Input, Role},
+ types::{Error, Input, Message, Role},
},
};
@@ -21,13 +21,13 @@ pub trait InputProcessor: Send + Sync {
}
pub struct InputProcessorDefault {
- message_processing_config: MessageProcessorConfig,
+ token_codec_config: ChatCodecConfig,
}
impl InputProcessorDefault {
- pub fn new(message_processing_config: MessageProcessorConfig) -> Self {
+ pub fn new(token_codec_config: ChatCodecConfig) -> Self {
Self {
- message_processing_config,
+ token_codec_config,
}
}
}
@@ -78,33 +78,40 @@ impl InputProcessor for InputProcessorDefault {
enable_thinking: bool,
add_generation_prompt: bool,
) -> Result<String, Error> {
- let messages = input.get_messages();
+ let mut messages = input.get_messages();
for message in &messages {
if message.role != Role::Assistant && message.reasoning_content.is_some() {
return Err(Error::UnexpectedReasoningContent);
}
}
+ if let Some(default_system_prompt) = self.token_codec_config.default_system_prompt.clone() {
+ if messages.first().is_none_or(|message| message.role != Role::System) {
+ messages.insert(0, Message::system(default_system_prompt));
+ }
+ }
let messages =
- messages.into_iter().map(|message| message.resolve(&self.message_processing_config)).collect::<Vec<_>>();
- let template = self.message_processing_config.prompt_template.clone();
- let bos_token = self.message_processing_config.bos_token.clone();
+ messages.into_iter().map(|message| message.resolve(&self.token_codec_config)).collect::<Vec<_>>();
+ let template = self.token_codec_config.prompt_template.clone();
+ let bos_token = self.token_codec_config.bos_token.clone();
+ let eos_token = self.token_codec_config.eos_token.clone();
let template_name = "chat_template";
let mut environment = Environment::new();
environment.set_unknown_method_callback(unknown_method_callback);
environment.add_function("strftime_now", strftime_now);
- environment.add_template(template_name, template.as_str()).map_err(|_| Error::UnableToLoadPromptTemplate)?;
- let template = environment.get_template(template_name).map_err(|_| Error::UnableToLoadPromptTemplate)?;
+ environment.add_template(template_name, template.as_str()).map_err(Error::UnableToLoadPromptTemplate)?;
+ let template = environment.get_template(template_name).map_err(Error::UnableToLoadPromptTemplate)?;
let result = template
.render(context!(
messages => messages,
add_generation_prompt => add_generation_prompt,
bos_token => bos_token,
+ eos_token => eos_token,
enable_thinking => enable_thinking
))
- .map_err(|_| Error::UnableToRenderPromptTemplate)?;
+ .map_err(Error::UnableToRenderPromptTemplate)?;
Ok(result)
}
}
diff --git a/crates/backend-uzu/src/session/helpers/output_parser.rs b/crates/backend-uzu/src/session/helpers/output_parser.rs
index c4dbeed0e..8f4dd7900 100644
--- a/crates/backend-uzu/src/session/helpers/output_parser.rs
+++ b/crates/backend-uzu/src/session/helpers/output_parser.rs
@@ -11,10 +11,7 @@ impl OutputParser {
let regex = match regex_str {
Some(regex_str) => {
let regex_str = Self::preprocess_regex_string(regex_str);
- let re = RegexBuilder::new(&regex_str)
- .dot_matches_new_line(true)
- .build()
- .map_err(|_| Error::UnableToBuildOutputParserRegex)?;
+ let re = RegexBuilder::new(&regex_str).dot_matches_new_line(true).build()?;
Some(re)
},
None => None,
diff --git a/crates/backend-uzu/src/session/parameter/async_batch_size.rs b/crates/backend-uzu/src/session/parameter/async_batch_size.rs
index b46a48f78..5e728fc2c 100644
--- a/crates/backend-uzu/src/session/parameter/async_batch_size.rs
+++ b/crates/backend-uzu/src/session/parameter/async_batch_size.rs
@@ -19,10 +19,10 @@ impl AsyncBatchSize {
&self,
model_path: &Path,
context: &B::Context,
- ) -> usize {
+ ) -> Result<usize, B::Error> {
match self {
AsyncBatchSize::Default => context.recommended_async_batch_size(model_path),
- AsyncBatchSize::Custom(value) => *value,
+ AsyncBatchSize::Custom(value) => Ok(*value),
}
}
}
diff --git a/crates/backend-uzu/src/session/parameter/prefill_step_size.rs b/crates/backend-uzu/src/session/parameter/prefill_step_size.rs
index 6bd145fda..027c909a0 100644
--- a/crates/backend-uzu/src/session/parameter/prefill_step_size.rs
+++ b/crates/backend-uzu/src/session/parameter/prefill_step_size.rs
@@ -1,4 +1,4 @@
-use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};
+use crate::{config::model::language_model::LanguageModelConfig, session::parameter::ConfigResolvableValue};
fn env_prefill_step_size_default_override() -> Option<usize> {
static OVERRIDE: std::sync::OnceLock<Option<usize>> = std::sync::OnceLock::new();
@@ -31,10 +31,10 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
config: &LanguageModelConfig,
) -> usize {
let default_limit: usize = 4096;
- let model_context_length = config.model_config.transformer_config.context_length;
+ let model_context_length = config.decoder_config.transformer_config.context_length;
let minimal_sliding_window_size = config
- .model_config
+ .decoder_config
.transformer_config
.layer_configs
.iter()
diff --git a/crates/backend-uzu/src/session/parameter/sampling_policy.rs b/crates/backend-uzu/src/session/parameter/sampling_policy.rs
index e60c6e1e3..86f76b7a4 100644
--- a/crates/backend-uzu/src/session/parameter/sampling_policy.rs
+++ b/crates/backend-uzu/src/session/parameter/sampling_policy.rs
@@ -1,4 +1,4 @@
-use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};
+use crate::{config::model::language_model::LanguageModelConfig, session::parameter::ConfigResolvableValue};
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum SamplingProcessingOrder {
diff --git a/crates/backend-uzu/src/session/tts_session.rs b/crates/backend-uzu/src/session/tts_session.rs
index 7f3246ece..09f2a0b32 100644
--- a/crates/backend-uzu/src/session/tts_session.rs
+++ b/crates/backend-uzu/src/session/tts_session.rs
@@ -11,6 +11,7 @@ use std::{
cell::RefCell,
collections::HashMap,
fs::File,
+ io::BufReader,
path::{Path, PathBuf},
rc::Rc,
time::Instant,
@@ -37,7 +38,7 @@ use crate::{
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
- config::{ModelMetadata, TtsMessageProcessorConfig, TtsModelConfig},
+ config::{model::tts_model::TTSModelConfig, token_codec::tts_codec::TTSCodecConfig},
encodable_block::{Decoder, Sampling as GpuSampling},
forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
parameters::ParameterLoader,
@@ -85,7 +86,7 @@ pub struct TtsSession<B: Backend> {
tokenizer: Tokenizer,
audio: AudioGenerationContext<B>,
audio_decoder: Box<dyn AudioDecoderBackend>,
- message_processor_config: TtsMessageProcessorConfig,
+ token_codec_config: TTSCodecConfig,
text_decoder: Box<dyn SemanticDecoderBackend>,
last_execution_stats: Option<TtsExecutionStats>,
}
@@ -212,12 +213,12 @@ impl<B: Backend> PendingAudioChunkBackend for NanoCodecPendingAudioChunk<B> {
}
let pending = self.inner.take().ok_or(Error::GenerateFailed)?;
- pending.resolve().map(Some).map_err(Error::from)
+ Ok(Some(pending.resolve()?))
}
fn resolve(mut self: Box<Self>) -> Result<AudioPcmBatch, Error> {
let pending = self.inner.take().ok_or(Error::GenerateFailed)?;
- pending.resolve().map_err(Error::from)
+ Ok(pending.resolve()?)
}
}
diff --git a/crates/backend-uzu/src/session/tts_session/audio_backend.rs b/crates/backend-uzu/src/session/tts_session/audio_backend.rs
index 5e3ea6663..6d1b6908e 100644
--- a/crates/backend-uzu/src/session/tts_session/audio_backend.rs
+++ b/crates/backend-uzu/src/session/tts_session/audio_backend.rs
@@ -37,7 +37,7 @@ impl<B: Backend + Send + Sync> AudioDecoderBackend for NanoCodecAudioDecoderBack
&self,
tokens: &AudioTokenGrid,
) -> Result<AudioPcmBatch, Error> {
- self.audio.runtime().decode(tokens).map_err(Error::from)
+ Ok(self.audio.runtime().decode(tokens)?)
}
fn begin_stream(
@@ -46,11 +46,8 @@ impl<B: Backend + Send + Sync> AudioDecoderBackend for NanoCodecAudioDecoderBack
codebooks: usize,
max_workspace_frames: usize,
) -> Result<Box<dyn AudioDecoderStreamBackend>, Error> {
- let state = self
- .audio
- .runtime()
- .begin_decode_stream_with_options(batch_size, codebooks, max_workspace_frames)
- .map_err(Error::from)?;
+ let state =
+ self.audio.runtime().begin_decode_stream_with_options(batch_size, codebooks, max_workspace_frames)?;
Ok(Box::new(NanoCodecAudioDecoderStream {
audio: self.audio.clone(),
state: Some(state),
@@ -65,8 +62,8 @@ impl<B: Backend> AudioDecoderStreamBackend for NanoCodecAudioDecoderStream<B> {
is_final: bool,
) -> Result<AudioPcmBatch, Error> {
let state = self.state.as_mut().ok_or(Error::GenerateFailed)?;
- let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?;
- self.audio.runtime().decoded_padded_to_pcm_batch(&decoded).map_err(Error::from)
+ let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final)?;
+ Ok(self.audio.runtime().decoded_padded_to_pcm_batch(&decoded)?)
}
fn decode_step_pending(
@@ -75,16 +72,14 @@ impl<B: Backend> AudioDecoderStreamBackend for NanoCodecAudioDecoderStream<B> {
is_final: bool,
) -> Result<Box<dyn PendingAudioChunkBackend>, Error> {
let state = self.state.as_mut().ok_or(Error::GenerateFailed)?;
- if let Some(pending) =
- self.audio.runtime().submit_decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?
- {
+ if let Some(pending) = self.audio.runtime().submit_decode_stream_step(state, new_tokens, is_final)? {
return Ok(Box::new(NanoCodecPendingAudioChunk {
inner: Some(pending),
}));
}
- let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?;
- let pcm = self.audio.runtime().decoded_padded_to_pcm_batch(&decoded).map_err(Error::from)?;
+ let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final)?;
+ let pcm = self.audio.runtime().decoded_padded_to_pcm_batch(&decoded)?;
Ok(Box::new(ImmediatePendingAudioChunk {
pcm: Some(pcm),
step_stats: Some(state.last_step_stats()),
@@ -97,6 +92,6 @@ impl<B: Backend> AudioDecoderStreamBackend for NanoCodecAudioDecoderStream<B> {
fn finish(mut self: Box<Self>) -> Result<(), Error> {
let state = self.state.take().ok_or(Error::GenerateFailed)?;
- self.audio.runtime().end_decode_stream(state).map_err(Error::from)
+ Ok(self.audio.runtime().end_decode_stream(state)?)
}
}
diff --git a/crates/backend-uzu/src/session/tts_session/backend_factory.rs b/crates/backend-uzu/src/session/tts_session/backend_factory.rs
index 2732d11eb..07a01f392 100644
--- a/crates/backend-uzu/src/session/tts_session/backend_factory.rs
+++ b/crates/backend-uzu/src/session/tts_session/backend_factory.rs
@@ -1,14 +1,16 @@
use super::*;
use crate::{
- config::{ModelType, TtsModelConfig, TtsTextDecoderConfig},
- session::types::TtsModelConfigError,
+ audio::AudioGenerationContext,
+ config::{
+ model::tts_model::TTSModelConfig, token_codec::AnyTokenCodecConfig, tts::text_decoder::AnyTTSTextDecoderConfig,
+ },
};
pub(super) struct LoadedTtsRuntime<B: Backend> {
pub(super) audio: AudioGenerationContext<B>,
pub(super) audio_decoder: Box<dyn AudioDecoderBackend>,
pub(super) text_decoder: Box<dyn SemanticDecoderBackend>,
- pub(super) message_processor_config: TtsMessageProcessorConfig,
+ pub(super) token_codec_config: TTSCodecConfig,
}
pub(super) fn build_audio_decoder_backend<B: Backend + Send + Sync>(
@@ -19,39 +21,33 @@ pub(super) fn build_audio_decoder_backend<B: Backend + Send + Sync>(
pub(super) fn load_tts_runtime<B: Backend + Send + Sync>(
model_path: &Path,
- model_metadata: &ModelMetadata<TtsModelConfig>,
+ model_config: &TTSModelConfig,
options: &TtsSessionOptions,
) -> Result<LoadedTtsRuntime<B>, Error> {
- if model_metadata.model_type != ModelType::TtsModel {
- return Err(TtsModelConfigError::ModelTypeMismatch {
- expected: ModelType::TtsModel,
- actual: model_metadata.model_type.clone(),
- }
- .into());
- }
-
- let audio: AudioGenerationContext<B> =
- model_metadata.model_config.create_audio_generation_context_with_model_path(model_path)?;
- let text_decoder = build_text_decoder_backend(&model_metadata.model_config, &audio, model_path, options)?;
+ let audio = AudioGenerationContext::from_tts_config_and_model_path(&model_config.tts_config, model_path)?;
+ let text_decoder = build_text_decoder_backend(model_config, &audio, model_path, options)?;
let audio_decoder = build_audio_decoder_backend(&audio)?;
- let message_processor_config = model_metadata.model_config.message_processor_config.clone();
+ let AnyTokenCodecConfig::TTSCodecConfig(token_codec_config) = &model_config.token_codec_config else {
+ return Err(Error::InvalidModelConfig("expected TTSCodecConfig token codec".to_string()));
+ };
+ let token_codec_config = token_codec_config.clone();
Ok(LoadedTtsRuntime {
audio,
audio_decoder,
text_decoder,
- message_processor_config,
+ token_codec_config,
})
}
pub(super) fn build_text_decoder_backend<B: Backend>(
- tts_model_config: &TtsModelConfig,
+ tts_model_config: &TTSModelConfig,
audio: &AudioGenerationContext<B>,
model_path: &Path,
options: &TtsSessionOptions,
) -> Result<Box<dyn SemanticDecoderBackend>, Error> {
match &tts_model_config.tts_config.text_decoder_config {
- TtsTextDecoderConfig::FishAudio(config) => {
+ AnyTTSTextDecoderConfig::FishAudioTextDecoderConfig(config) => {
fishaudio::build_fishaudio_text_decoder_runtime(config, audio, model_path, &options.text_decoder)
},
}
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
index 76b73cc51..cbb0280ed 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
@@ -2,6 +2,7 @@ use super::{decoder_support::*, *};
use crate::{
array::{Array, ArrayContextExt},
backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},
+ config::decoder::DecoderConfig,
encodable_block::{DecoderArguments, DecoderDecodeInput, SamplingInputs},
forward_pass::token_inputs::TokenInputs,
session::types::TtsModelConfigError,
@@ -17,34 +18,31 @@ struct TokenDecoderLoadedModel<B: Backend> {
impl<B: Backend> TokenDecoderLoadedModel<B> {
fn load(
context: &Rc<B::Context>,
- model_path: &Path,
- decoder_config: &Rc<crate::config::DecoderConfig>,
+ decoder_config: &Rc<DecoderConfig>,
model_shape: &ModelShape,
+ root_loader_view: &crate::parameters::ParameterTree<B>,
transformer_subtree: &str,
embedding_subtree: &str,
readout_subtree: &str,
) -> Result<Self, Error> {
- let weights_path = model_path.join("model.safetensors");
- let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;
- let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
- let root_loader_view = loader.tree();
-
let shared_buffers = TokenDecoderContext::<B>::build_shared_buffers(
context,
decoder_config,
- &root_loader_view,
+ model_shape,
+ root_loader_view,
transformer_subtree,
)?;
let executables = Decoder::new_with_embedding_and_readout_subtrees(
context.as_ref(),
&decoder_config,
- &root_loader_view,
+ root_loader_view,
transformer_subtree,
embedding_subtree,
readout_subtree,
- );
- let logits_data_type = model_shape.activation_data_type();
- let sampler = GpuSampling::new(context.as_ref(), logits_data_type).map_err(unable_to_create_context)?;
+ model_shape,
+ )
+ .map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;
+ let sampler = GpuSampling::new(context.as_ref(), model_shape.data_type).map_err(unable_to_create_context)?;
let token_copy_sampled =
<B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;
@@ -62,7 +60,7 @@ struct TokenDecoderContext<B: Backend> {
cache_layers: Rc<RefCell<CacheLayers<B>>>,
shared_buffers: Rc<SharedBuffers<B>>,
model_shape: ModelShape,
- decoder_config: Rc<crate::config::DecoderConfig>,
+ decoder_config: Rc<DecoderConfig>,
runtime_config: TextDecoderRuntimeConfig,
executables: Decoder<B>,
sampler: GpuSampling<B>,
@@ -83,22 +81,27 @@ impl<B: Backend> TokenDecoderContext<B> {
fn new(
context: Rc<B::Context>,
model_path: &Path,
- decoder_config: Rc<crate::config::DecoderConfig>,
+ decoder_config: Rc<DecoderConfig>,
transformer_subtree: &str,
embedding_subtree: &str,
readout_subtree: &str,
runtime_config: &TextDecoderRuntimeConfig,
- ) -> Result<(Self, DataType), Error> {
- let model_shape = ModelShape::from_decoder_config(&decoder_config);
+ ) -> Result<Self, Error> {
+ let weights_path = model_path.join("model.safetensors");
+ let weights_file = File::open(&weights_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let loader = ParameterLoader::new(&weights_file, context.as_ref())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let root_loader_view = loader.tree();
+
+ let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::F32);
let max_suffix_length =
text_decoder_prefill_step_size(runtime_config, decoder_config.transformer_config.context_length).max(32);
let max_prefix_length = max_suffix_length;
- let activation_data_type = model_shape.activation_data_type();
let loaded_model = TokenDecoderLoadedModel::<B>::load(
&context,
- model_path,
&decoder_config,
&model_shape,
+ &root_loader_view,
transformer_subtree,
embedding_subtree,
readout_subtree,
@@ -112,41 +115,39 @@ impl<B: Backend> TokenDecoderContext<B> {
max_prefix_length,
max_suffix_length,
)));
- let intermediate_data_type: DataType =
- decoder_config.transformer_config.output_norm_config.scale_precision.into();
- let kv_cache_update = KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)
+ let kv_cache_update = KVCacheUpdate::new(context.as_ref(), model_shape.data_type, max_prefix_length)
.map_err(unable_to_create_context)?;
- Ok((
- Self {
- context,
- cache_layers,
- shared_buffers: loaded_model.shared_buffers,
- model_shape,
- decoder_config,
- runtime_config: runtime_config.clone(),
- executables: loaded_model.executables,
- sampler: loaded_model.sampler,
- kv_cache_update,
- token_copy_sampled: loaded_model.token_copy_sampled,
- async_chain_positions,
- async_chain_seeds,
- async_chain_results,
- async_chain_capacity,
- current_max_prefix_length: max_prefix_length,
- },
- activation_data_type,
- ))
+ Ok(Self {
+ context,
+ cache_layers,
+ shared_buffers: loaded_model.shared_buffers,
+ model_shape,
+ decoder_config,
+ runtime_config: runtime_config.clone(),
+ executables: loaded_model.executables,
+ sampler: loaded_model.sampler,
+ kv_cache_update,
+ token_copy_sampled: loaded_model.token_copy_sampled,
+ async_chain_positions,
+ async_chain_seeds,
+ async_chain_results,
+ async_chain_capacity,
+ current_max_prefix_length: max_prefix_length,
+ })
}
fn build_shared_buffers(
context: &Rc<B::Context>,
- decoder_config: &Rc<crate::config::DecoderConfig>,
- root_loader_view: &crate::parameters::ParameterTree<B::Context>,
+ decoder_config: &Rc<DecoderConfig>,
+ model_shape: &ModelShape,
+ root_loader_view: &crate::parameters::ParameterTree<B>,
transformer_subtree: &str,
) -> Result<Rc<SharedBuffers<B>>, Error> {
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config);
- let transformer_tree = root_loader_view.subtree(transformer_subtree).map_err(|_| Error::UnableToLoadWeights)?;
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config, model_shape);
+ let transformer_tree = root_loader_view
+ .subtree(transformer_subtree)
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
shared_buffers.update_data_from_transformer_tree(&transformer_tree)?;
Ok(Rc::new(shared_buffers))
}
@@ -176,7 +177,7 @@ pub(super) struct TokenDecoderRunner<B: Backend> {
impl<B: Backend> TokenDecoderRunner<B> {
pub(super) fn activation_data_type(&self) -> DataType {
- self.ctx.model_shape.activation_data_type()
+ self.ctx.model_shape.data_type
}
pub(super) fn context(&self) -> &Rc<B::Context> {
@@ -186,13 +187,13 @@ impl<B: Backend> TokenDecoderRunner<B> {
pub(super) fn new_with_context(
context: Rc<B::Context>,
model_path: &Path,
- decoder_config: Rc<crate::config::DecoderConfig>,
+ decoder_config: Rc<DecoderConfig>,
transformer_subtree: &str,
embedding_subtree: &str,
readout_subtree: &str,
runtime_config: &TextDecoderRuntimeConfig,
) -> Result<Self, Error> {
- let (ctx, activation_data_type) = TokenDecoderContext::new(
+ let ctx = TokenDecoderContext::new(
context,
model_path,
decoder_config,
@@ -204,11 +205,12 @@ impl<B: Backend> TokenDecoderRunner<B> {
let context: Rc<B::Context> = Rc::clone(ctx.context());
let model_dim = ctx.decoder_config.transformer_config.model_dim;
let tensor_add_scale =
- <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), activation_data_type)
+ <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), ctx.model_shape.data_type)
.map_err(unable_to_create_context)?;
- let single_hidden_capture = context.create_array_zeros(&[1, model_dim], activation_data_type).into_allocation();
+ let single_hidden_capture =
+ context.create_array_zeros(&[1, model_dim], ctx.model_shape.data_type).into_allocation();
let single_override_embedding =
- context.create_array_zeros(&[1, model_dim], activation_data_type).into_allocation();
+ context.create_array_zeros(&[1, model_dim], ctx.model_shape.data_type).into_allocation();
Ok(Self {
ctx,
@@ -337,10 +339,8 @@ impl<B: Backend> TokenDecoderRunner<B> {
let context = self.ctx.context.as_ref();
*self.ctx.cache_layers.borrow_mut() =
CacheLayers::new(context, &self.ctx.model_shape, max_prefix_length, max_suffix_length);
- let intermediate_data_type: DataType =
- self.ctx.decoder_config.transformer_config.output_norm_config.scale_precision.into();
- self.ctx.kv_cache_update =
- KVCacheUpdate::new(context, intermediate_data_type, max_prefix_length).map_err(unable_to_create_context)?;
+ self.ctx.kv_cache_update = KVCacheUpdate::new(context, self.ctx.model_shape.data_type, max_prefix_length)
+ .map_err(unable_to_create_context)?;
self.ctx.current_max_prefix_length = max_prefix_length;
Ok(())
}
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_support.rs b/crates/backend-uzu/src/session/tts_session/decoder_support.rs
index a04ccb40c..2dcbfcda9 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_support.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_support.rs
@@ -187,7 +187,7 @@ pub(super) struct StreamingTokenAccumulator {
impl StreamingTokenAccumulator {
pub(super) fn new(num_codebooks: usize) -> Result<Self, Error> {
if num_codebooks == 0 {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig("TTS num_codebooks must be greater than 0".to_string()));
}
Ok(Self {
by_codebook: vec![Vec::new(); num_codebooks],
@@ -229,14 +229,13 @@ impl StreamingTokenAccumulator {
tokens.extend_from_slice(&codebook[frame_start..frame_end]);
}
- AudioTokenGrid::new(
+ Ok(AudioTokenGrid::new(
tokens.into_boxed_slice(),
1,
self.by_codebook.len(),
range_frames,
vec![range_frames].into_boxed_slice(),
- )
- .map_err(Error::from)
+ )?)
}
}
@@ -273,8 +272,7 @@ pub(super) fn slice_grid_range(
lengths.push(length.saturating_sub(frame_start).min(range_frames));
}
- AudioTokenGrid::new(tokens.into_boxed_slice(), batch_size, codebooks, range_frames, lengths.into_boxed_slice())
- .map_err(Error::from)
+ Ok(AudioTokenGrid::new(tokens.into_boxed_slice(), batch_size, codebooks, range_frames, lengths.into_boxed_slice())?)
}
pub(super) fn accumulate_audio_decode_step_stats(
@@ -441,14 +439,10 @@ impl<B: Backend + Send + Sync> TtsSession<B> {
}
let config_path = model_path.join("config.json");
- if !config_path.exists() {
- return Err(Error::UnableToLoadConfig);
- }
- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
- let model_metadata: ModelMetadata<TtsModelConfig> =
- serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
+ let config_file = File::open(&config_path)?;
+ let model_config: TTSModelConfig = serde_json::from_reader(BufReader::new(config_file))?;
- Self::from_model_metadata_with_options(model_path, model_metadata, options)
+ Self::from_model_config_with_options(model_path, model_config, options)
}
pub fn last_execution_stats(&self) -> Option<TtsExecutionStats> {
@@ -459,24 +453,21 @@ impl<B: Backend + Send + Sync> TtsSession<B> {
self.audio.sample_rate()
}
- fn from_model_metadata_with_options(
+ fn from_model_config_with_options(
model_path: PathBuf,
- model_metadata: ModelMetadata<TtsModelConfig>,
+ model_config: TTSModelConfig,
options: TtsSessionOptions,
) -> Result<Self, Error> {
let tokenizer_path = model_path.join("tokenizer.json");
- if !tokenizer_path.exists() {
- return Err(Error::UnableToLoadTokenizer);
- }
- let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;
+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(Error::UnableToLoadTokenizer)?;
- let loaded_runtime = load_tts_runtime(&model_path, &model_metadata, &options)?;
+ let loaded_runtime = load_tts_runtime(&model_path, &model_config, &options)?;
Ok(Self {
tokenizer,
audio: loaded_runtime.audio,
audio_decoder: loaded_runtime.audio_decoder,
- message_processor_config: loaded_runtime.message_processor_config,
+ token_codec_config: loaded_runtime.token_codec_config,
text_decoder: loaded_runtime.text_decoder,
last_execution_stats: None,
})
@@ -489,73 +480,81 @@ impl<B: Backend + Send + Sync> TtsSession<B> {
let messages = input
.get_messages()
.into_iter()
- .map(|message| message.resolve(&self.message_processor_config))
+ .map(|message| message.resolve(&self.token_codec_config))
.collect::<Vec<_>>();
let template_name = "tts_prompt_template";
let mut environment = Environment::new();
environment
- .add_template(template_name, self.message_processor_config.prompt_template.as_str())
- .map_err(|_| Error::UnableToLoadPromptTemplate)?;
- let template = environment.get_template(template_name).map_err(|_| Error::UnableToLoadPromptTemplate)?;
+ .add_template(template_name, self.token_codec_config.prompt_template.as_str())
+ .map_err(Error::UnableToLoadPromptTemplate)?;
+ let template = environment.get_template(template_name).map_err(Error::UnableToLoadPromptTemplate)?;
let result = template
.render(context!(
messages => messages
))
- .map_err(|_| Error::UnableToRenderPromptTemplate)?;
+ .map_err(Error::UnableToRenderPromptTemplate)?;
Ok(normalize_rendered_prompt(
result,
- self.message_processor_config.prompt_template.as_str(),
- self.message_processor_config.drop_initial_newline,
+ self.token_codec_config.prompt_template.as_str(),
+ self.token_codec_config.drop_initial_newline,
))
}
}
pub(super) fn semantic_token_to_code(
semantic_token: u64,
- semantic_begin: i64,
- semantic_end: i64,
+ semantic_begin: u64,
+ semantic_end: u64,
token_upper_bound: usize,
) -> u32 {
if semantic_begin > semantic_end || token_upper_bound == 0 {
return 0;
}
- let semantic = semantic_token as i64;
- if semantic < semantic_begin || semantic > semantic_end {
+ if semantic_token < semantic_begin || semantic_token > semantic_end {
return 0;
}
- let relative = usize::try_from(semantic - semantic_begin).unwrap_or(0);
+ let relative = usize::try_from(semantic_token - semantic_begin).unwrap_or(0);
let clamped = relative.min(token_upper_bound.saturating_sub(1));
u32::try_from(clamped).unwrap_or(0)
}
pub(super) fn build_semantic_sampling_mask_row(
vocab_size: usize,
- semantic_begin: i64,
- semantic_end: i64,
- im_end: i64,
+ semantic_begin: u64,
+ semantic_end: u64,
+ im_end: u64,
) -> Result<Box<[u32]>, Error> {
if vocab_size == 0 || semantic_begin > semantic_end {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig(format!(
+ "invalid semantic sampling mask range: vocab_size={vocab_size}, semantic_begin={semantic_begin}, semantic_end={semantic_end}",
+ )));
}
- let max_token_id = i64::try_from(vocab_size.saturating_sub(1)).map_err(|_| Error::UnableToLoadConfig)?;
- if semantic_begin < 0 || semantic_end < 0 || semantic_end > max_token_id || im_end < 0 || im_end > max_token_id {
- return Err(Error::UnableToLoadConfig);
+ let max_token_id = u64::try_from(vocab_size.saturating_sub(1)).map_err(|_| {
+ Error::InvalidModelConfig(format!("vocab_size={vocab_size} cannot be represented as a token id"))
+ })?;
+ if semantic_end > max_token_id || im_end > max_token_id {
+ return Err(Error::InvalidModelConfig(format!(
+ "semantic sampling mask token id exceeds vocab: semantic_end={semantic_end}, im_end={im_end}, max_token_id={max_token_id}",
+ )));
}
let row_words = vocab_size.div_ceil(32);
let mut mask = vec![0_u32; row_words];
for token_index in semantic_begin..=semantic_end {
- let token = usize::try_from(token_index).map_err(|_| Error::UnableToLoadConfig)?;
+ let token = usize::try_from(token_index).map_err(|_| {
+ Error::InvalidModelConfig(format!("semantic token id {token_index} cannot be represented as usize"))
+ })?;
let word = token / 32;
mask[word] |= 2_u32.pow((token % 32) as u32);
}
- let im_end_token = usize::try_from(im_end).map_err(|_| Error::UnableToLoadConfig)?;
+ let im_end_token = usize::try_from(im_end)
+ .map_err(|_| Error::InvalidModelConfig(format!("im_end token id {im_end} cannot be represented as usize")))?;
let word = im_end_token / 32;
mask[word] |= 2_u32.pow((im_end_token % 32) as u32);
Ok(mask.into_boxed_slice())
@@ -563,15 +562,16 @@ pub(super) fn build_semantic_sampling_mask_row(
pub(super) fn clear_token_in_sampling_mask(
mask: &mut [u32],
- token: i64,
+ token: u64,
) -> Result<(), Error> {
- if token < 0 {
- return Err(Error::UnableToLoadConfig);
- }
- let token = usize::try_from(token).map_err(|_| Error::UnableToLoadConfig)?;
+ let token = usize::try_from(token)
+ .map_err(|_| Error::InvalidModelConfig(format!("token id {token} cannot be represented as usize")))?;
let word = token / 32;
if word >= mask.len() {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig(format!(
+ "token id {token} is outside sampling mask with {} words",
+ mask.len(),
+ )));
}
mask[word] &= !2_u32.pow((token % 32) as u32);
Ok(())
diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
index d4371fbe7..fe17cdac3 100644
--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs
+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
@@ -2,7 +2,10 @@ use super::*;
use crate::{
array::ArrayContextExt,
backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},
- config::DecoderConfig,
+ config::{
+ decoder::DecoderConfig, embedding::AnyEmbeddingConfig,
+ tts::text_decoder::fish_audio_text_decoder::FishAudioTextDecoderConfig,
+ },
encodable_block::SamplingInputs,
forward_pass::token_inputs::TokenInputs,
session::types::TtsModelConfigError,
@@ -13,10 +16,10 @@ pub(super) struct FishAudioTextDecoderRuntime<B: Backend> {
fast_runner: TokenDecoderRunner<B>,
semantic_bridge: FishAudioSemanticBridge<B>,
runtime_config: TextDecoderRuntimeConfig,
- semantic_token_begin_id: i64,
- semantic_token_end_id: i64,
+ semantic_token_begin_id: u64,
+ semantic_token_end_id: u64,
semantic_cardinality: usize,
- im_end_token_id: i64,
+ im_end_token_id: u64,
codebook_size: usize,
num_codebooks: usize,
slow_model_dim: usize,
@@ -46,26 +49,29 @@ struct FishAudioSemanticBridge<B: Backend> {
impl<B: Backend> FishAudioSemanticBridge<B> {
fn load(
context: &B::Context,
- parameter_tree: &crate::parameters::ParameterTree<B::Context>,
- config: &crate::config::FishAudioTextDecoderConfig,
+ parameter_tree: &crate::parameters::ParameterTree<B>,
+ config: &FishAudioTextDecoderConfig,
data_type: DataType,
) -> Result<Self, Error> {
- let codebook_embeddings = load_float_tensor_allocation(
- parameter_tree,
- "text_decoder.codebook_embeddings.weights",
- [
- config.codebook_size.checked_mul(config.num_codebooks).ok_or(Error::UnableToLoadConfig)?,
- config.slow_model_dim,
- ],
- data_type,
- )?;
+ let codebook_embedding_rows = config.codebook_size.checked_mul(config.num_codebooks).ok_or_else(|| {
+ Error::InvalidModelConfig(format!(
+ "FishAudio codebook embedding rows overflow: codebook_size={}, num_codebooks={}",
+ config.codebook_size, config.num_codebooks,
+ ))
+ })?;
+ let codebook_embeddings = parameter_tree
+ .leaf("text_decoder.codebook_embeddings.embedding.weights")
+ .and_then(|leaf| leaf.validate(&[codebook_embedding_rows, config.slow_model_dim], data_type))
+ .and_then(|leaf| leaf.read_allocation())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let fast_model_projection = if config.fast_model_projection_config.is_some() {
- Some(load_float_tensor_allocation(
- parameter_tree,
- "text_decoder.fast_model_projection.weights",
- [config.fast_model_dim, config.slow_model_dim],
- data_type,
- )?)
+ Some(
+ parameter_tree
+ .leaf("text_decoder.fast_model_projection.weights.weights")
+ .and_then(|leaf| leaf.validate(&[config.fast_model_dim, config.slow_model_dim], data_type))
+ .and_then(|leaf| leaf.read_allocation())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?,
+ )
} else {
None
};
@@ -112,33 +118,12 @@ impl<B: Backend> FishAudioSemanticBridge<B> {
}
}
-fn load_float_tensor_allocation<B: Backend>(
- parameter_tree: &crate::parameters::ParameterTree<B::Context>,
- key: &str,
- expected_shape: [usize; 2],
- target_data_type: DataType,
-) -> Result<Allocation<B>, Error> {
- let leaf = parameter_tree.leaf(key).map_err(|_| Error::UnableToLoadWeights)?;
- if leaf.shape() != expected_shape {
- return Err(Error::UnableToLoadConfig);
- }
- if leaf.data_type() != target_data_type {
- return Err(TtsModelConfigError::FishAudioTensorDataTypeMismatch {
- key: key.into(),
- expected: target_data_type,
- actual: leaf.data_type(),
- }
- .into());
- }
- leaf.read_allocation().map_err(|_| Error::UnableToLoadWeights)
-}
-
fn validate_fishaudio_decoder_contract(
num_codebooks: usize,
codebook_size: usize,
max_seq_len: usize,
- semantic_token_begin_id: i64,
- semantic_token_end_id: i64,
+ semantic_token_begin_id: u64,
+ semantic_token_end_id: u64,
audio_num_codebooks: usize,
audio_codec_cardinality: usize,
audio_semantic_cardinality: usize,
@@ -173,12 +158,14 @@ fn validate_fishaudio_decoder_contract(
.into());
}
- let semantic_cardinality = usize::try_from(semantic_token_end_id - semantic_token_begin_id + 1).map_err(|_| {
- TtsModelConfigError::FishAudioSemanticTokenRangeOverflow {
+ let semantic_cardinality = semantic_token_end_id
+ .checked_sub(semantic_token_begin_id)
+ .and_then(|value| value.checked_add(1))
+ .and_then(|value| usize::try_from(value).ok())
+ .ok_or(TtsModelConfigError::FishAudioSemanticTokenRangeOverflow {
begin: semantic_token_begin_id,
end: semantic_token_end_id,
- }
- })?;
+ })?;
if semantic_cardinality == 0 || semantic_cardinality != audio_semantic_cardinality {
return Err(TtsModelConfigError::FishAudioSemanticCodecCardinalityMismatch {
semantic_cardinality,
@@ -191,13 +178,14 @@ fn validate_fishaudio_decoder_contract(
}
pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(
- config: &crate::config::FishAudioTextDecoderConfig,
+ config: &FishAudioTextDecoderConfig,
audio: &AudioGenerationContext<B>,
model_path: &Path,
runtime_config: &TextDecoderRuntimeConfig,
) -> Result<Box<dyn SemanticDecoderBackend>, Error> {
- let audio_semantic_cardinality =
- audio.runtime().config().semantic_codec_cardinality().ok_or(Error::UnableToLoadConfig)?;
+ let audio_semantic_cardinality = audio.runtime().config().semantic_codec_cardinality().ok_or_else(|| {
+ Error::InvalidModelConfig("audio decoder does not expose semantic codec cardinality".to_string())
+ })?;
let semantic_cardinality = validate_fishaudio_decoder_contract(
config.num_codebooks,
config.codebook_size,
@@ -209,13 +197,19 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(
audio_semantic_cardinality,
)?;
if config.slow_model_config.model_dim != config.slow_model_dim {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig(format!(
+ "FishAudio slow model_dim mismatch: transformer={}, configured={}",
+ config.slow_model_config.model_dim, config.slow_model_dim,
+ )));
}
if config.fast_model_config.model_dim != config.fast_model_dim {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig(format!(
+ "FishAudio fast model_dim mismatch: transformer={}, configured={}",
+ config.fast_model_config.model_dim, config.fast_model_dim,
+ )));
}
if config.short_logits_size == 0 {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig("FishAudio short_logits_size must be greater than 0".to_string()));
}
let semantic_sampling_mask_row = build_semantic_sampling_mask_row(
config.vocab_size,
@@ -230,14 +224,14 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(
let fast_transformer_config = config.fast_model_config.clone();
let slow_decoder_config = Rc::new(DecoderConfig {
- embedding_config: config.slow_embeddings_config.to_text_decoder_embedding_config(),
+ embedding_config: AnyEmbeddingConfig::TiedEmbeddingConfig(config.slow_embeddings_config.clone()),
transformer_config: slow_transformer_config,
vocab_size: config.vocab_size,
pard_token: None,
ple_model_config: None,
});
let fast_decoder_config = Rc::new(DecoderConfig {
- embedding_config: config.fast_embeddings_config.to_text_decoder_embedding_config(),
+ embedding_config: AnyEmbeddingConfig::TiedEmbeddingConfig(config.fast_embeddings_config.clone()),
transformer_config: fast_transformer_config,
vocab_size: config.codebook_size,
pard_token: None,
@@ -268,13 +262,15 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(
let activation_data_type = slow_runner.activation_data_type();
let fast_data_type = fast_runner.activation_data_type();
if fast_data_type != activation_data_type {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig(format!(
+ "FishAudio slow/fast activation data type mismatch: slow={activation_data_type:?}, fast={fast_data_type:?}",
+ )));
}
let weights_path = model_path.join("model.safetensors");
- let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;
- let loader =
- ParameterLoader::new(&weights_file, text_decoder_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
+ let weights_file = File::open(&weights_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let loader = ParameterLoader::new(&weights_file, text_decoder_context.as_ref())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let root_weights = loader.tree();
let apply_semantic_sampling_mask =
@@ -317,26 +313,30 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
mut on_frame: Option<&mut dyn FnMut(&[u32]) -> Result<(), Error>>,
) -> Result<AudioTokenGrid, Error> {
if text_tokens.is_empty() {
- return AudioTokenGrid::new(
+ return Ok(AudioTokenGrid::new(
Vec::new().into_boxed_slice(),
1,
self.num_codebooks,
0,
vec![0].into_boxed_slice(),
- )
- .map_err(Error::from);
+ )?);
}
if text_tokens.len() >= self.max_seq_len {
return Err(Error::GenerateFailed);
}
if codec_cardinality == 0 || codec_cardinality > self.codebook_size {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig(format!(
+ "FishAudio codec cardinality {codec_cardinality} must be in 1..={}",
+ self.codebook_size,
+ )));
}
let semantic_token_upper_bound = self.semantic_cardinality;
let residual_token_upper_bound = codec_cardinality;
if semantic_token_upper_bound == 0 || residual_token_upper_bound == 0 {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig(format!(
+ "FishAudio token upper bounds must be positive: semantic={semantic_token_upper_bound}, residual={residual_token_upper_bound}",
+ )));
}
self.reset_generation_state();
@@ -365,7 +365,7 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
self.prepare_fast_runner_masks(residual_token_upper_bound)?;
for _step in 0..max_new_tokens {
- if current_semantic_token as i64 == self.im_end_token_id {
+ if current_semantic_token == self.im_end_token_id {
break;
}
let first_code = semantic_token_to_code(
@@ -448,7 +448,10 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
) -> Result<(), Error> {
let fast_vocab_limit = self.fast_vocab_limit.min(residual_token_upper_bound);
if fast_vocab_limit == 0 {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig(format!(
+ "FishAudio fast vocab limit is zero: configured={}, residual={residual_token_upper_bound}",
+ self.fast_vocab_limit,
+ )));
}
self.fast_runner.prepare_single_token_vocab_mask(fast_vocab_limit)?;
self.fast_runner.prepare_two_token_vocab_mask(fast_vocab_limit)?;
@@ -467,8 +470,13 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
}
tokens.extend_from_slice(codebook_tokens);
}
- AudioTokenGrid::new(tokens.into_boxed_slice(), 1, self.num_codebooks, frames, vec![frames].into_boxed_slice())
- .map_err(Error::from)
+ Ok(AudioTokenGrid::new(
+ tokens.into_boxed_slice(),
+ 1,
+ self.num_codebooks,
+ frames,
+ vec![frames].into_boxed_slice(),
+ )?)
}
fn encode_project_slow_hidden_to_fast_on(
@@ -508,7 +516,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
}
if slow_model_dim != fast_model_dim {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig(format!(
+ "FishAudio slow_model_dim={slow_model_dim} must equal fast_model_dim={fast_model_dim} without projection weights",
+ )));
}
debug_assert_eq!(
@@ -537,7 +547,7 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
encoder: &mut Encoder<B>,
) -> Result<(), Error> {
if num_codebooks == 0 {
- return Err(Error::UnableToLoadConfig);
+ return Err(Error::InvalidModelConfig("FishAudio num_codebooks must be greater than 0".to_string()));
}
// Write first_code at position 0 from CPU.
@@ -566,11 +576,10 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
)?;
}
- let total_vocab = num_codebooks.checked_mul(codebook_size).ok_or(Error::GenerateFailed)?;
- let total_vocab_u32 = u32::try_from(total_vocab).map_err(|_| Error::GenerateFailed)?;
- let num_codebooks_u32 = u32::try_from(num_codebooks).map_err(|_| Error::GenerateFailed)?;
- let slow_model_dim_u32 = u32::try_from(slow_model_dim).map_err(|_| Error::GenerateFailed)?;
- let codebook_size_u32 = u32::try_from(codebook_size).map_err(|_| Error::GenerateFailed)?;
+ let total_vocab_u32 = (num_codebooks * codebook_size) as u32;
+ let num_codebooks_u32 = num_codebooks as u32;
+ let slow_model_dim_u32 = slow_model_dim as u32;
+ let codebook_size_u32 = codebook_size as u32;
let codebook_token_indices = &semantic_bridge.codebook_token_indices;
let codebook_embeddings = &semantic_bridge.codebook_embeddings;
@@ -646,7 +655,7 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
let sampled = self.fast_runner.read_async_chain_result(pass)?;
let codebook_index = pass + 1;
let clamped = (sampled as usize).min(residual_token_upper_bound.saturating_sub(1));
- let clamped = u32::try_from(clamped).map_err(|_| Error::GenerateFailed)?;
+ let clamped = clamped as u32;
by_codebook[codebook_index].push(clamped);
self.current_codes_scratch[codebook_index] = clamped;
}
diff --git a/crates/backend-uzu/src/session/tts_session/generator.rs b/crates/backend-uzu/src/session/tts_session/generator.rs
index 690b21fe2..cc282b849 100644
--- a/crates/backend-uzu/src/session/tts_session/generator.rs
+++ b/crates/backend-uzu/src/session/tts_session/generator.rs
@@ -36,7 +36,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {
let text_tokens: Vec<u64> = self
.tokenizer
.encode(prompt.as_str(), false)
- .map_err(|_| Error::UnableToEncodeText)?
+ .map_err(Error::UnableToEncodeText)?
.get_ids()
.iter()
.map(|&token| token as u64)
@@ -117,8 +117,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {
sample_rate,
channels,
accumulated_lengths.into_boxed_slice(),
- )
- .map_err(Error::from)?
+ )?
}
},
};
@@ -162,7 +161,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {
let text_tokens: Vec<u64> = self
.tokenizer
.encode(prompt.as_str(), false)
- .map_err(|_| Error::UnableToEncodeText)?
+ .map_err(Error::UnableToEncodeText)?
.get_ids()
.iter()
.map(|&token| token as u64)
@@ -231,7 +230,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {
let text_tokens: Vec<u64> = self
.tokenizer
.encode(prompt.as_str(), false)
- .map_err(|_| Error::UnableToEncodeText)?
+ .map_err(Error::UnableToEncodeText)?
.get_ids()
.iter()
.map(|&token| token as u64)
@@ -370,8 +369,7 @@ impl<B: Backend + Send + Sync> TtsSession<B> {
ss.output_sample_rate,
ss.output_channels,
vec![ss.output_frames].into_boxed_slice(),
- )
- .map_err(Error::from)?;
+ )?;
let instrumentation = self.take_text_decoder_instrumentation();
self.record_last_execution_stats(TtsExecutionStats {
diff --git a/crates/backend-uzu/src/session/types/error.rs b/crates/backend-uzu/src/session/types/error.rs
index 794454017..a7de58b2c 100644
--- a/crates/backend-uzu/src/session/types/error.rs
+++ b/crates/backend-uzu/src/session/types/error.rs
@@ -1,12 +1,7 @@
-use crate::{DataType, classifier::ClassifierError, config::ModelType, session::config::TtsRunConfigError};
+use crate::{DataType, classifier::ClassifierError, session::config::TtsRunConfigError};
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum TtsModelConfigError {
- #[error("expected model_type={expected:?}, got {actual:?}")]
- ModelTypeMismatch {
- expected: ModelType,
- actual: ModelType,
- },
#[error("missing TTS model config")]
MissingTtsModelConfig,
#[error("FishAudio tensor {key} dtype mismatch: expected {expected:?}, got {actual:?}")]
@@ -39,13 +34,13 @@ pub enum TtsModelConfigError {
},
#[error("FishAudio semantic token range is invalid: begin={begin}, end={end}")]
FishAudioSemanticTokenRangeInvalid {
- begin: i64,
- end: i64,
+ begin: u64,
+ end: u64,
},
#[error("FishAudio semantic token range overflow: begin={begin}, end={end}")]
FishAudioSemanticTokenRangeOverflow {
- begin: i64,
- end: i64,
+ begin: u64,
+ end: u64,
},
#[error(
"FishAudio semantic codec cardinality={semantic_cardinality} does not match audio semantic codec cardinality={audio_semantic_cardinality}"
@@ -170,16 +165,24 @@ pub enum Error {
UnableToCreateContext(Box<dyn std::error::Error>),
#[error("Unable to create CommandBuffer: {0}")]
UnableToCreateCommandBuffer(Box<dyn std::error::Error>),
- #[error("Unable to load model configuration")]
- UnableToLoadConfig,
+ #[error("Unable to open model configuration: {0}")]
+ UnableToOpenConfig(#[from] std::io::Error),
+ #[error("Unable to deserialize model configuration: {0}")]
+ UnableToDeserializeConfig(#[from] serde_json::Error),
+ #[error("Invalid model configuration: {0}")]
+ InvalidModelConfig(String),
#[error("Invalid TTS model config: {0}")]
InvalidTtsModelConfig(#[from] TtsModelConfigError),
#[error("Invalid TTS run config: {0}")]
InvalidTtsRunConfig(#[from] TtsRunConfigError),
- #[error("Unable to load model weights")]
- UnableToLoadWeights,
- #[error("Unable to load tokenizer")]
- UnableToLoadTokenizer,
+ #[error("Unable to load model weights: {0}")]
+ UnableToLoadWeights(#[source] Box<dyn std::error::Error>),
+ #[error("Unable to create decoder: {0}")]
+ UnableToCreateDecoder(#[source] Box<dyn std::error::Error>),
+ #[error("Unable to create classifier layer: {0}")]
+ UnableToCreateClassifierLayer(#[source] Box<dyn std::error::Error>),
+ #[error("Unable to load tokenizer: {0}")]
+ UnableToLoadTokenizer(#[source] tokenizers::Error),
#[error("Model is too large to fit into available RAM")]
NotEnoughMemory,
#[error("Command buffer failed: {0}")]
@@ -190,20 +193,18 @@ pub enum Error {
UnsupportedContextModeForModel,
#[error("Unsupported speculator config for model")]
UnsupportedSpeculatorConfigForModel,
- #[error("Language model generator not loaded")]
- LanguageModelGeneratorNotLoaded,
- #[error("Unable to load prompt template")]
- UnableToLoadPromptTemplate,
- #[error("Unable to render prompt template")]
- UnableToRenderPromptTemplate,
+ #[error("Unable to load prompt template: {0}")]
+ UnableToLoadPromptTemplate(#[source] minijinja::Error),
+ #[error("Unable to render prompt template: {0}")]
+ UnableToRenderPromptTemplate(#[source] minijinja::Error),
#[error("Only assistant messages can have reasoning content")]
UnexpectedReasoningContent,
- #[error("Unable to build output parser regex")]
- UnableToBuildOutputParserRegex,
- #[error("Unable to encode text")]
- UnableToEncodeText,
- #[error("Unable to decode text")]
- UnableToDecodeText,
+ #[error("Unable to build output parser regex: {0}")]
+ UnableToBuildOutputParserRegex(#[from] regex::Error),
+ #[error("Unable to encode text: {0}")]
+ UnableToEncodeText(#[source] tokenizers::Error),
+ #[error("Unable to decode text: {0}")]
+ UnableToDecodeText(#[source] tokenizers::Error),
#[error("Context length exceeded")]
ContextLengthExceeded,
#[error("Prefill failed")]
@@ -220,8 +221,8 @@ pub enum Error {
GrammarReject,
#[error("Token {0} out of grammar vocabulary range (0..{1})")]
TokenOutOfGrammarRange(u64, usize),
- #[error("Capture failed")]
- CaptureFailed,
+ #[error("Capture failed: {0}")]
+ CaptureFailed(#[source] Box<dyn std::error::Error>),
#[error("Classifier error: {0}")]
Classifier(#[from] ClassifierError),
#[error("Audio codec error: {0}")]
diff --git a/crates/backend-uzu/src/session/types/message.rs b/crates/backend-uzu/src/session/types/message.rs
index 134ead89c..8d9bfc890 100644
--- a/crates/backend-uzu/src/session/types/message.rs
+++ b/crates/backend-uzu/src/session/types/message.rs
@@ -3,9 +3,9 @@ use std::collections::HashMap;
use serde::{Deserialize, Serialize};
#[cfg(metal_backend)]
-use crate::config::TtsMessageProcessorConfig;
+use crate::config::token_codec::tts_codec::TTSCodecConfig;
use crate::{
- config::MessageProcessorConfig,
+ config::token_codec::chat_codec::ChatCodecConfig,
session::{parameter::ConfigResolvableValue, types::Role},
};
@@ -50,10 +50,10 @@ impl Message {
}
}
-impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>> for Message {
+impl ConfigResolvableValue<ChatCodecConfig, HashMap<String, String>> for Message {
fn resolve(
&self,
- config: &MessageProcessorConfig,
+ config: &ChatCodecConfig,
) -> HashMap<String, String> {
let role = match self.role {
Role::System => config.system_role_name.clone(),
@@ -70,10 +70,10 @@ impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>> for
}
#[cfg(metal_backend)]
-impl ConfigResolvableValue<TtsMessageProcessorConfig, HashMap<String, String>> for Message {
+impl ConfigResolvableValue<TTSCodecConfig, HashMap<String, String>> for Message {
fn resolve(
&self,
- _config: &TtsMessageProcessorConfig,
+ _config: &TTSCodecConfig,
) -> HashMap<String, String> {
HashMap::from([
(String::from("content"), self.content.clone()),
diff --git a/crates/backend-uzu/src/utils/model_size.rs b/crates/backend-uzu/src/utils/model_size.rs
index b95719b4b..22f42c479 100644
--- a/crates/backend-uzu/src/utils/model_size.rs
+++ b/crates/backend-uzu/src/utils/model_size.rs
@@ -9,11 +9,11 @@ pub enum ModelSize {
}
impl ModelSize {
- pub fn from_path(model_path: &Path) -> Self {
+ pub fn from_path(model_path: &Path) -> std::io::Result<Self> {
let weights_path = model_path.join("model.safetensors");
- let size_bytes = std::fs::metadata(&weights_path).map(|m| m.len()).unwrap_or(0);
+ let size_bytes = std::fs::metadata(&weights_path)?.len();
- Self::from_bytes(size_bytes)
+ Ok(Self::from_bytes(size_bytes))
}
pub fn from_bytes(size_bytes: u64) -> Self {
diff --git a/crates/backend-uzu/src/utils/strict_serde.rs b/crates/backend-uzu/src/utils/strict_serde.rs
index bc2ec29d6..9cca71654 100644
--- a/crates/backend-uzu/src/utils/strict_serde.rs
+++ b/crates/backend-uzu/src/utils/strict_serde.rs
@@ -1,4 +1,4 @@
-use monostate::MustBeBool;
+use monostate::{ConstStr, MustBeBool, MustBeStr};
use serde::{Deserialize, Deserializer, Serialize};
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
@@ -6,7 +6,7 @@ pub enum Unsupported {}
pub trait DeserializeStrict<'de>: Deserialize<'de> {}
pub trait DeserializeStrictOwned: for<'de> DeserializeStrict<'de> {}
-impl<T> DeserializeStrictOwned for T where T: for<'de> DeserializeStrict<'de> {}
+impl<T: for<'de> DeserializeStrict<'de>> DeserializeStrictOwned for T {}
macro_rules! impl_strict {
($($ty:ty),* $(,)?) => {
@@ -14,13 +14,14 @@ macro_rules! impl_strict {
};
}
-impl_strict!(Unsupported, String, f32, i64, u32, usize, bool);
+impl_strict!(Unsupported, String, f32, u32, u64, usize, bool);
impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Box<T> {}
-impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Vec<T> {}
+impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Box<[T]> {}
impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Option<T> {}
impl<'de, A: DeserializeStrict<'de>, B: DeserializeStrict<'de>> DeserializeStrict<'de> for (A, B) {}
impl<'de, const V: bool> DeserializeStrict<'de> for MustBeBool<V> {}
+impl<'de, V: ConstStr> DeserializeStrict<'de> for MustBeStr<V> {}
pub fn required<'de, D: Deserializer<'de>, T: DeserializeStrict<'de>>(deserializer: D) -> Result<T, D::Error> {
T::deserialize(deserializer)
diff --git a/crates/backend-uzu/src/utils/version.rs b/crates/backend-uzu/src/utils/version.rs
index bafe1a60c..8888ef48b 100644
--- a/crates/backend-uzu/src/utils/version.rs
+++ b/crates/backend-uzu/src/utils/version.rs
@@ -1,2 +1,2 @@
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-pub const TOOLCHAIN_VERSION: &str = "0.8.0";
+pub const TOOLCHAIN_VERSION: &str = "0.10.0";
diff --git a/crates/backend-uzu/tests/integration/parameters/loader_test.rs b/crates/backend-uzu/tests/integration/parameters/loader_test.rs
index 43de028b7..27ceb1d23 100644
--- a/crates/backend-uzu/tests/integration/parameters/loader_test.rs
+++ b/crates/backend-uzu/tests/integration/parameters/loader_test.rs
@@ -2,32 +2,41 @@
// New integration test for ParameterLoader
use backend_uzu::{
- ParameterLoader,
+ DataType, ParameterLoader,
backends::{
common::{Backend, Context},
metal::Metal,
},
+ read_safetensors_metadata,
};
use half::bf16;
-use is_close::is_close;
use test_tag::tag;
use crate::common::path::get_test_weights_path;
+const EMBEDDING_PATH: &str = "decoder.embedding.embedding.weights";
+const EMBEDDING_TREE_PATH: &str = "decoder.embedding.embedding";
+
#[tag(heavy)]
#[test]
fn test_parameter_loader_basic() {
let weights_path = get_test_weights_path();
let context = <Metal as Backend>::Context::new().expect("Failed to create MetalContext");
let file = std::fs::File::open(&weights_path).expect("Weights file not found; run download script");
+ let (_header_len, metadata) = read_safetensors_metadata(&file).expect("read weights metadata");
+ let embedding_shape = metadata.tensors.get(EMBEDDING_PATH).expect("weights embeddings metadata").shape.clone();
+ assert_eq!(embedding_shape.len(), 2);
- let loader = ParameterLoader::new(&file, context.as_ref()).expect("create loader");
- let embeddings = loader.tree().leaf_array("embedding.weights").expect("weights embeddings");
+ let loader = ParameterLoader::<Metal>::new(&file, context.as_ref()).expect("create loader");
+ let tree = loader.tree();
+ let embeddings_leaf =
+ tree.leaf(EMBEDDING_PATH).expect("weights embeddings").validate(&embedding_shape, DataType::BF16).unwrap();
+ let embeddings = embeddings_leaf.read_array().unwrap();
let view = embeddings.as_view::<bf16>();
- assert!(is_close!(view[[5usize, 3usize]], bf16::from_f32(-0.01819)));
// tree API check
- let subtree = loader.tree().subtree("embedding").unwrap();
- let same = subtree.leaf_array("weights").unwrap();
+ let subtree = tree.subtree(EMBEDDING_TREE_PATH).unwrap();
+ let weights_leaf = subtree.leaf("weights").unwrap().validate(&embedding_shape, DataType::BF16).unwrap();
+ let same = weights_leaf.read_array().unwrap();
assert_eq!(view, same.as_view::<bf16>());
}
diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
index eb1f32bcd..f5a774f2a 100644
--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
@@ -6,19 +6,20 @@
use std::{
cell::RefCell,
+ collections::HashMap,
fs::File,
+ io::BufReader,
path::{Path, PathBuf},
rc::Rc,
};
use backend_uzu::{
_private::{
- ActivationTrace, ArgmaxSampler, CacheLayers, Classifier, DecoderDecodeInput, KVCacheLayer,
- LanguageModelGeneratorContext, LogitsSampler, ModelConfig, ModelMetadata, ModelType, ParameterTree, Sampling,
- TokenInputs,
+ ActivationTrace, AnyModelConfig, ArgmaxSampler, CacheLayers, Classifier, DecoderDecodeInput, KVCacheLayer,
+ LanguageModelGeneratorContext, LogitsSampler, ParameterLoaderError, ParameterTree, Sampling, TokenInputs,
},
Array, ArrayElement, DataType, ParameterLoader, allocation_to_vec,
- backends::common::{Allocation, Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},
+ backends::common::{Allocation, AllocationType, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
read_safetensors_metadata,
session::{
config::{DecodingConfig, SpeculatorConfig},
@@ -140,9 +141,21 @@ pub enum ArrayTransform {
// Model Context (internal)
// ============================================================================
+#[derive(Clone, Copy)]
+struct LayerTraceRequirements {
+ post_mixer_norm: bool,
+ post_mlp_norm: bool,
+}
+
enum ModelContext<B: Backend> {
- LanguageModelGenerator(LanguageModelGeneratorContext<B>),
- Classifier(Classifier<B>),
+ LanguageModelGenerator {
+ context: LanguageModelGeneratorContext<B>,
+ layer_trace_requirements: Box<[LayerTraceRequirements]>,
+ },
+ Classifier {
+ classifier: Classifier<B>,
+ layer_trace_requirements: Box<[LayerTraceRequirements]>,
+ },
}
// ============================================================================
@@ -163,53 +176,43 @@ impl<B: Backend> TraceValidator<B> {
///
/// Automatically detects the model type from config.json.
pub fn new(model_path: &Path) -> Result<Self, Error> {
- let config_path = model_path.join("config.json");
- if !config_path.exists() {
+ if !model_path.exists() {
return Err(Error::ModelFolderNotFound);
}
- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
- let raw_metadata: ModelMetadata<ModelConfig> =
- serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
-
- let context = match raw_metadata.model_type.clone() {
- ModelType::ClassifierModel => {
- let ModelConfig::ClassifierModel(model_config) = raw_metadata.model_config.clone() else {
- return Err(Error::UnableToLoadConfig);
- };
- let metadata = ModelMetadata {
- toolchain_version: raw_metadata.toolchain_version,
- vendor: raw_metadata.vendor,
- family: raw_metadata.family,
- name: raw_metadata.name,
- size: raw_metadata.size,
- quantization: raw_metadata.quantization,
- repo: raw_metadata.repo,
- use_cases: raw_metadata.use_cases,
- model_type: raw_metadata.model_type,
- model_config,
- grammar_start_tokens: raw_metadata.grammar_start_tokens,
- };
- ModelContext::Classifier(Classifier::new(model_path, &metadata)?)
+ let config_path = model_path.join("config.json");
+ let config_file = File::open(&config_path)?;
+ let model_config: AnyModelConfig = serde_json::from_reader(BufReader::new(config_file))?;
+
+ let context = match model_config {
+ AnyModelConfig::ClassifierModelConfig(model_config) => {
+ let layer_trace_requirements = model_config
+ .classifier_config
+ .transformer_config
+ .layer_configs
+ .iter()
+ .map(|layer_config| LayerTraceRequirements {
+ post_mixer_norm: layer_config.post_mixer_norm_config.is_some(),
+ post_mlp_norm: layer_config.post_mlp_norm_config.is_some(),
+ })
+ .collect();
+ ModelContext::Classifier {
+ classifier: Classifier::new(model_path, &model_config)?,
+ layer_trace_requirements,
+ }
},
- ModelType::LanguageModel => {
- let ModelConfig::LanguageModel(model_config) = raw_metadata.model_config.clone() else {
- return Err(Error::UnableToLoadConfig);
- };
- let metadata = ModelMetadata {
- toolchain_version: raw_metadata.toolchain_version,
- vendor: raw_metadata.vendor,
- family: raw_metadata.family,
- name: raw_metadata.name,
- size: raw_metadata.size,
- quantization: raw_metadata.quantization,
- repo: raw_metadata.repo,
- use_cases: raw_metadata.use_cases,
- model_type: raw_metadata.model_type,
- model_config,
- grammar_start_tokens: raw_metadata.grammar_start_tokens,
- };
- let prefill_step_size = Self::determine_prefill_step_size(model_path);
+ AnyModelConfig::LanguageModelConfig(model_config) => {
+ let layer_trace_requirements = model_config
+ .decoder_config
+ .transformer_config
+ .layer_configs
+ .iter()
+ .map(|layer_config| LayerTraceRequirements {
+ post_mixer_norm: layer_config.post_mixer_norm_config.is_some(),
+ post_mlp_norm: layer_config.post_mlp_norm_config.is_some(),
+ })
+ .collect();
+ let prefill_step_size = Self::determine_prefill_step_size(model_path)?;
let decoding_config = DecodingConfig::new(
ContextMode::default(),
ContextLength::default(),
@@ -218,12 +221,17 @@ impl<B: Backend> TraceValidator<B> {
SamplingSeed::default(),
AsyncBatchSize::default(),
);
- let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config, &metadata)?;
+ let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config, &model_config)?;
let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());
Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);
- ModelContext::LanguageModelGenerator(llm_context)
+ ModelContext::LanguageModelGenerator {
+ context: llm_context,
+ layer_trace_requirements,
+ }
+ },
+ AnyModelConfig::TTSModelConfig(_) => {
+ return Err(Error::InvalidModelConfig("TTS trace validation is not supported".to_string()));
},
- ModelType::TtsModel => return Err(Error::UnableToLoadConfig),
};
Ok(Self {
@@ -235,13 +243,15 @@ impl<B: Backend> TraceValidator<B> {
/// Run trace validation and return results.
pub fn run(&mut self) -> Result<TracerValidationResults, Error> {
let traces_path = self.model_path.join("traces.safetensors");
- if !traces_path.exists() {
- return Err(Error::UnableToLoadWeights);
- }
-
match &mut self.context {
- ModelContext::LanguageModelGenerator(ctx) => Self::run_llm_validation(ctx, &traces_path),
- ModelContext::Classifier(classifier) => Self::run_classifier_validation(classifier, &traces_path),
+ ModelContext::LanguageModelGenerator {
+ context,
+ layer_trace_requirements,
+ } => Self::run_llm_validation(context, layer_trace_requirements, &traces_path),
+ ModelContext::Classifier {
+ classifier,
+ layer_trace_requirements,
+ } => Self::run_classifier_validation(classifier, layer_trace_requirements, &traces_path),
}
}
@@ -251,15 +261,30 @@ impl<B: Backend> TraceValidator<B> {
fn run_llm_validation(
ctx: &LanguageModelGeneratorContext<B>,
+ layer_trace_requirements: &[LayerTraceRequirements],
traces_path: &Path,
) -> Result<TracerValidationResults, Error> {
- let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
- let traces_loader =
- ParameterLoader::new(&traces_file, ctx.context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
+ let traces_file = File::open(traces_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let (_header_len, traces_metadata) =
+ read_safetensors_metadata(&traces_file).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let trace_shapes: HashMap<String, Vec<usize>> =
+ traces_metadata.tensors.iter().map(|(name, tensor)| (name.clone(), tensor.shape.clone())).collect();
+ let token_shape = Self::trace_shape(&trace_shapes, "activation_trace.token_ids");
+ let position_shape = Self::trace_shape(&trace_shapes, "activation_trace.token_positions");
+ let suffix_length = Self::trace_token_count("activation_trace.token_ids", token_shape);
+ assert_eq!(
+ Self::trace_token_count("activation_trace.token_positions", position_shape),
+ suffix_length,
+ "trace token ids and token positions must have the same suffix length"
+ );
+
+ let traces_loader = ParameterLoader::new(&traces_file, ctx.context.as_ref())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let traces_view = traces_loader.tree();
- let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");
- let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");
+ let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, &trace_shapes, "activation_trace.token_ids");
+ let token_positions =
+ Self::load_array_as_vec::<i32, usize>(&traces_view, &trace_shapes, "activation_trace.token_positions");
let token_inputs = TokenInputs::new_llm(
ctx.context.as_ref(),
&ctx.model_shape,
@@ -269,9 +294,9 @@ impl<B: Backend> TraceValidator<B> {
None,
None,
/*sampling_start=*/ 0,
- /*sampling_length=*/ token_ids.len(),
+ /*sampling_length=*/ suffix_length,
);
- let mut traces = ActivationTrace::new_llm(ctx.context.as_ref(), &ctx.model_shape, token_ids.len());
+ let mut traces = ActivationTrace::new_llm(ctx.context.as_ref(), &ctx.model_shape, suffix_length);
let mut encoder =
Encoder::<B>::new(ctx.context.as_ref()).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
@@ -280,9 +305,9 @@ impl<B: Backend> TraceValidator<B> {
let decoder_arguments = token_inputs.decoder_arguments(
ctx.shared_buffers.as_ref(),
Some(&mut *cache_layers),
- token_ids.len(),
+ suffix_length,
/*sampling_start=*/ 0,
- /*sampling_length=*/ token_ids.len(),
+ /*sampling_length=*/ suffix_length,
#[cfg(feature = "tracing")]
Some(&mut traces),
);
@@ -298,159 +323,178 @@ impl<B: Backend> TraceValidator<B> {
let pending = encoder.end_encoding().submit();
pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
- let data_type = ctx.model_shape.activation_data_type();
-
// Common layer validation
- let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);
+ let mut results = Self::validate_layer_traces(
+ &traces,
+ &traces_view,
+ &trace_shapes,
+ ctx.model_shape.data_type,
+ layer_trace_requirements,
+ );
- // LLM-specific: KV cache validation
+ // LLM-specific state validation
let cache = ctx.cache_layers.borrow();
for (index, layer) in cache.iter_layers() {
- let Some(kv) = layer.as_transformer() else {
- continue;
- };
-
- if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {
- let size = kv.shape().iter().product::<usize>() * data_type.size_in_bytes();
+ if let Some(kv) = layer.as_transformer() {
+ let shape = kv.shape();
+ let keys_path = format!("activation_trace.layer_results.{}.updated_state.keys", index);
+ let expected =
+ Self::read_required_array(&traces_view, &trace_shapes, &keys_path, ctx.model_shape.data_type);
+ let size = shape.iter().product::<usize>() * ctx.model_shape.data_type.size_in_bytes();
let keys = if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {
common::helpers::sparse_buffer_read_allocation(ctx.context.as_ref(), &layer.keys, size)
+ } else if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::DenseBuffer>>() {
+ let mut allocation = ctx
+ .context
+ .create_allocation(size, AllocationType::Global)
+ .expect("Failed to create KV cache keys read allocation");
+ let mut encoder =
+ Encoder::<B>::new(ctx.context.as_ref()).expect("Failed to create KV cache keys read encoder");
+ encoder.encode_copy(&layer.keys, 0..size, &mut allocation, 0..size);
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to read KV cache keys");
+ allocation
} else {
panic!("Wrong keys type")
};
results.push(TracerValidationResult {
- name: format!("updated_kv_cache.{}.keys", index),
+ name: keys_path,
metrics: Self::validate_allocation(
- data_type,
+ ctx.model_shape.data_type,
&expected,
&keys,
- &kv.shape(),
+ &shape,
Some(ArrayTransform::KVCacheSlice),
),
});
- }
- if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {
- let size = kv.shape().iter().product::<usize>() * data_type.size_in_bytes();
+ let values_path = format!("activation_trace.layer_results.{}.updated_state.values", index);
+ let expected =
+ Self::read_required_array(&traces_view, &trace_shapes, &values_path, ctx.model_shape.data_type);
let values = if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {
common::helpers::sparse_buffer_read_allocation(ctx.context.as_ref(), &layer.values, size)
+ } else if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::DenseBuffer>>() {
+ let mut allocation = ctx
+ .context
+ .create_allocation(size, AllocationType::Global)
+ .expect("Failed to create KV cache values read allocation");
+ let mut encoder =
+ Encoder::<B>::new(ctx.context.as_ref()).expect("Failed to create KV cache values read encoder");
+ encoder.encode_copy(&layer.values, 0..size, &mut allocation, 0..size);
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to read KV cache values");
+ allocation
} else {
panic!("Wrong values type")
};
results.push(TracerValidationResult {
- name: format!("updated_kv_cache.{}.values", index),
+ name: values_path,
metrics: Self::validate_allocation(
- data_type,
+ ctx.model_shape.data_type,
&expected,
&values,
- &kv.shape(),
+ &shape,
Some(ArrayTransform::KVCacheSlice),
),
});
- }
- }
-
- // LLM-specific: SSM state validation
- for (index, layer) in cache.iter_layers() {
- let Some(ssm) = layer.as_state_space() else {
- continue;
- };
+ } else if let Some(ssm) = layer.as_state_space() {
+ let conv_path = format!("activation_trace.layer_results.{}.updated_state.conv_state", index);
+ let expected =
+ Self::read_required_array(&traces_view, &trace_shapes, &conv_path, ctx.model_shape.data_type);
+ let conv_state = ssm.conv_state.as_ref().expect("SSM trace requires a produced conv state");
+ results.push(TracerValidationResult {
+ name: conv_path,
+ metrics: Self::validate_allocation(
+ ctx.model_shape.data_type,
+ &expected,
+ conv_state,
+ &ssm.conv_shape,
+ Some(ArrayTransform::SsmConvState),
+ ),
+ });
- for path in [
- format!("updated_state.{}.conv_state", index),
- format!("activation_trace.layer_results.{}.updated_state.conv_state", index),
- ] {
- if let Ok(expected) = traces_view.leaf_array(&path) {
- results.push(TracerValidationResult {
- name: path,
- metrics: Self::validate_optional_allocation(
- data_type,
- &expected,
- ssm.conv_state.as_ref(),
- &ssm.conv_shape,
- Some(ArrayTransform::SsmConvState),
- ),
- });
- }
- }
+ let ssm_path = format!("activation_trace.layer_results.{}.updated_state.ssm_state", index);
+ let expected =
+ Self::read_required_array(&traces_view, &trace_shapes, &ssm_path, ctx.model_shape.data_type);
+ results.push(TracerValidationResult {
+ name: ssm_path,
+ metrics: Self::validate_allocation(
+ ctx.model_shape.data_type,
+ &expected,
+ &ssm.ssm_state,
+ &ssm.ssm_shape,
+ None,
+ ),
+ });
+ } else if let Some(delta) = layer.as_delta_net() {
+ let conv_path = format!("activation_trace.layer_results.{}.updated_state.conv_state", index);
+ let expected =
+ Self::read_required_array(&traces_view, &trace_shapes, &conv_path, ctx.model_shape.data_type);
+ results.push(TracerValidationResult {
+ name: conv_path,
+ metrics: Self::validate_allocation(
+ ctx.model_shape.data_type,
+ &expected,
+ &delta.conv_state,
+ &delta.conv_shape,
+ Some(ArrayTransform::SsmConvState),
+ ),
+ });
- for path in [
- format!("updated_state.{}.ssm_state", index),
- format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),
- ] {
- if let Ok(expected) = traces_view.leaf_array(&path) {
- results.push(TracerValidationResult {
- name: path,
- metrics: Self::validate_allocation(data_type, &expected, &ssm.ssm_state, &ssm.ssm_shape, None),
- });
- }
+ let ssm_path = format!("activation_trace.layer_results.{}.updated_state.ssm_state", index);
+ let expected =
+ Self::read_required_array(&traces_view, &trace_shapes, &ssm_path, ctx.model_shape.data_type);
+ results.push(TracerValidationResult {
+ name: ssm_path,
+ metrics: Self::validate_allocation(
+ ctx.model_shape.data_type,
+ &expected,
+ &delta.ssm_state,
+ &delta.ssm_shape,
+ None,
+ ),
+ });
+ } else if let Some(short_conv) = layer.as_short_conv() {
+ let conv_path = format!("activation_trace.layer_results.{}.updated_state.conv_state", index);
+ let expected =
+ Self::read_required_array(&traces_view, &trace_shapes, &conv_path, ctx.model_shape.data_type);
+ results.push(TracerValidationResult {
+ name: conv_path,
+ metrics: Self::validate_allocation(
+ ctx.model_shape.data_type,
+ &expected,
+ &short_conv.conv_state,
+ &short_conv.conv_shape,
+ Some(ArrayTransform::SsmConvState),
+ ),
+ });
+ } else {
+ panic!("Unsupported cache layer type at layer {index}");
}
}
- // LLM-specific: DeltaNet state validation
- for (index, layer) in cache.iter_layers() {
- let Some(delta) = layer.as_delta_net() else {
- continue;
- };
-
- for path in [
- format!("updated_state.{}.conv_state", index),
- format!("activation_trace.layer_results.{}.updated_state.conv_state", index),
- ] {
- if let Ok(expected) = traces_view.leaf_array(&path) {
- results.push(TracerValidationResult {
- name: path,
- metrics: Self::validate_allocation(
- data_type,
- &expected,
- &delta.conv_state,
- &delta.conv_shape,
- Some(ArrayTransform::SsmConvState),
- ),
- });
- }
- }
-
- for path in [
- format!("updated_state.{}.ssm_state", index),
- format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),
- ] {
- if let Ok(expected) = traces_view.leaf_array(&path) {
- results.push(TracerValidationResult {
- name: path,
- metrics: Self::validate_allocation(
- data_type,
- &expected,
- &delta.ssm_state,
- &delta.ssm_shape,
- None,
- ),
- });
+ // LLM-specific: Token comparison
+ let expected_logits =
+ Self::read_required_array(&traces_view, &trace_shapes, "logits", ctx.model_shape.data_type);
+ let expected_tokens = Self::get_tokens_from_logits(&expected_logits);
+ let produced_tokens = Self::get_tokens_from_logits(&traces.logits);
+ let tokens_violation_indices = expected_tokens
+ .iter()
+ .zip(produced_tokens.iter())
+ .enumerate()
+ .filter_map(|(i, (a, b))| {
+ if a != b {
+ Some(i)
+ } else {
+ None
}
- }
- }
+ })
+ .collect();
- // LLM-specific: Token comparison
- let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf_array("logits") {
- let expected_tokens = Self::get_tokens_from_logits(&expected_logits);
- let produced_tokens = Self::get_tokens_from_logits(&traces.logits);
- expected_tokens
- .iter()
- .zip(produced_tokens.iter())
- .enumerate()
- .filter_map(|(i, (a, b))| {
- if a != b {
- Some(i)
- } else {
- None
- }
- })
- .collect()
- } else {
- Vec::new()
- };
+ traces_view.assert_all_tensors_validated().unwrap_or_else(|error| {
+ panic!("Trace contains tensors that Uzu tracer did not validate: {error}");
+ });
Ok(TracerValidationResults {
- suffix_length: token_ids.len(),
+ suffix_length,
results,
tokens_violation_indices,
})
@@ -462,37 +506,54 @@ impl<B: Backend> TraceValidator<B> {
fn run_classifier_validation(
classifier: &mut Classifier<B>,
+ layer_trace_requirements: &[LayerTraceRequirements],
traces_path: &Path,
) -> Result<TracerValidationResults, Error> {
- let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
+ let traces_file = File::open(traces_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let (_header_len, traces_metadata) =
+ read_safetensors_metadata(&traces_file).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let trace_shapes: HashMap<String, Vec<usize>> =
+ traces_metadata.tensors.iter().map(|(name, tensor)| (name.clone(), tensor.shape.clone())).collect();
let context = classifier.context.context.clone();
- let traces_loader =
- ParameterLoader::new(&traces_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
+ let traces_loader = ParameterLoader::new(&traces_file, context.as_ref())
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let traces_view = traces_loader.tree();
- let has_token_ids = traces_view.leaf_array("activation_trace.token_ids").is_ok();
- let has_token_positions = traces_view.leaf_array("activation_trace.token_positions").is_ok();
-
- if !has_token_ids || !has_token_positions {
- return Ok(Self::handle_missing_tokens(&traces_view));
- }
-
- let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");
- let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");
-
- let suffix_length = token_ids.len();
+ let token_shape = Self::trace_shape(&trace_shapes, "activation_trace.token_ids");
+ let position_shape = Self::trace_shape(&trace_shapes, "activation_trace.token_positions");
+ let suffix_length = Self::trace_token_count("activation_trace.token_ids", token_shape);
+ assert_eq!(
+ Self::trace_token_count("activation_trace.token_positions", position_shape),
+ suffix_length,
+ "trace token ids and token positions must have the same suffix length"
+ );
- let (_logits, traces) =
- classifier.forward_pass_with_traces(&token_ids, &token_positions).map_err(|_| Error::GenerateFailed)?;
+ let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, &trace_shapes, "activation_trace.token_ids");
+ let token_positions =
+ Self::load_array_as_vec::<i32, usize>(&traces_view, &trace_shapes, "activation_trace.token_positions");
- let data_type = classifier.context.model_shape.activation_data_type();
+ let (_logits, traces) = classifier.forward_pass_with_traces(&token_ids, &token_positions)?;
// Common layer validation
- let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);
+ let mut results = Self::validate_layer_traces(
+ &traces,
+ &traces_view,
+ &trace_shapes,
+ classifier.context.model_shape.data_type,
+ layer_trace_requirements,
+ );
// Classifier-specific: embedding_norm, output_pooling
- let classifier_results = Self::validate_classifier_traces(&traces, &traces_view, data_type);
+ let classifier_results = Self::validate_classifier_traces(
+ &traces,
+ &traces_view,
+ &trace_shapes,
+ classifier.context.model_shape.data_type,
+ );
results.extend(classifier_results);
+ traces_view.assert_all_tensors_validated().unwrap_or_else(|error| {
+ panic!("Trace contains tensors that Uzu tracer did not validate: {error}");
+ });
Ok(TracerValidationResults {
suffix_length,
@@ -501,141 +562,104 @@ impl<B: Backend> TraceValidator<B> {
})
}
- fn handle_missing_tokens(traces_view: &ParameterTree<B::Context>) -> TracerValidationResults {
- if let Ok(expected_logits) = traces_view.leaf_array("logits") {
- let reference_shape = expected_logits.shape().to_vec();
- let metrics = TracerValidationMetrics {
- atol: 0.0,
- rtol: 0.0,
- fraction_of_allowed_violations: 0.0,
- reference_shape: reference_shape.clone(),
- result_shape: reference_shape,
- num_violations: 0,
- max_allowed_violations: 0,
- max_err_idx: 0,
- max_err: 0.0,
- max_err_rel: 0.0,
- max_err_reference_value: 0.0,
- rms_diff: 0.0,
- rms_result: 0.0,
- rms_reference: 0.0,
- rel_rms_reference: 0.0,
- diff_max: 0.0,
- diff_avg: 0.0,
- result_nan: false,
- };
- return TracerValidationResults {
- suffix_length: 1,
- results: vec![TracerValidationResult {
- name: "activation_trace.logits".to_string(),
- metrics,
- }],
- tokens_violation_indices: Vec::new(),
- };
- }
-
- TracerValidationResults {
- suffix_length: 1,
- results: Vec::new(),
- tokens_violation_indices: Vec::new(),
- }
- }
-
// ========================================================================
// Common Validation Helpers
// ========================================================================
fn validate_layer_traces(
traces: &ActivationTrace<B>,
- traces_view: &ParameterTree<B::Context>,
+ traces_view: &ParameterTree<B>,
+ trace_shapes: &HashMap<String, Vec<usize>>,
data_type: DataType,
+ layer_trace_requirements: &[LayerTraceRequirements],
) -> Vec<TracerValidationResult> {
let mut results = Vec::new();
- let validate = |path: &str, array: &Array<B>| -> Option<TracerValidationResult> {
- if let Ok(expected) = traces_view.leaf_array(path) {
- Some(TracerValidationResult {
- name: path.to_string(),
- metrics: Self::validate_allocation(data_type, &expected, array.allocation(), array.shape(), None),
- })
- } else {
- None
+ assert_eq!(
+ traces.layer_results.len(),
+ layer_trace_requirements.len(),
+ "trace layer count must match model layer count"
+ );
+
+ let validate = |path: &str, array: &Array<B>| -> TracerValidationResult {
+ let expected = Self::read_required_array(traces_view, trace_shapes, path, data_type);
+ TracerValidationResult {
+ name: path.to_string(),
+ metrics: Self::validate_allocation(data_type, &expected, array.allocation(), array.shape(), None),
}
};
for (index, layer_traces) in traces.layer_results.iter().enumerate() {
+ let requirements = layer_trace_requirements[index];
let path = |suffix: &str| -> String {
format!("activation_trace.layer_results.{}.activation_trace.{}", index, suffix)
};
- if let Some(r) = validate(&path("inputs"), &layer_traces.inputs) {
- results.push(r);
- }
- if let Some(r) = validate(&path("pre_mixer_norm"), &layer_traces.pre_attention_norm) {
- results.push(r);
- }
- if let Some(r) = validate(&path("mixer"), &layer_traces.attention) {
- results.push(r);
- }
- if let Some(r) = validate(&path("post_mixer_norm"), &layer_traces.post_attention_norm) {
- results.push(r);
- }
- if let Some(r) = validate(&path("mlp_inputs"), &layer_traces.mlp_inputs) {
- results.push(r);
- }
- if let Some(r) = validate(&path("pre_mlp_norm"), &layer_traces.pre_mlp_norm) {
- results.push(r);
+ results.push(validate(&path("inputs"), &layer_traces.inputs));
+ results.push(validate(&path("pre_mixer_norm"), &layer_traces.pre_attention_norm));
+ results.push(validate(&path("mixer"), &layer_traces.attention));
+ if requirements.post_mixer_norm {
+ results.push(validate(&path("post_mixer_norm"), &layer_traces.post_attention_norm));
}
- if let Some(r) = validate(&path("mlp"), &layer_traces.mlp) {
- results.push(r);
- }
- if let Some(r) = validate(&path("post_mlp_norm"), &layer_traces.post_mlp_norm) {
- results.push(r);
+ results.push(validate(&path("mlp_inputs"), &layer_traces.mlp_inputs));
+ results.push(validate(&path("pre_mlp_norm"), &layer_traces.pre_mlp_norm));
+ results.push(validate(&path("mlp"), &layer_traces.mlp));
+ if requirements.post_mlp_norm {
+ results.push(validate(&path("post_mlp_norm"), &layer_traces.post_mlp_norm));
}
let outputs_path = format!("activation_trace.layer_results.{}.outputs", index);
- if let Some(r) = validate(&outputs_path, &layer_traces.outputs) {
- results.push(r);
- }
+ results.push(validate(&outputs_path, &layer_traces.outputs));
}
// Output norm (common to all models)
- if let Some(r) = validate("activation_trace.output_norm", &traces.output_norm) {
- results.push(r);
- }
+ results.push(validate("activation_trace.output_norm", &traces.output_norm));
- // Logits (common to all models, but path may vary)
- if let Some(r) = validate("activation_trace.logits", &traces.logits) {
- results.push(r);
- } else if let Some(r) = validate("logits", &traces.logits) {
- results.push(r);
- }
+ // Logits live on DecoderResult/ClassifierResult, not inside activation_trace.
+ results.push(validate("logits", &traces.logits));
results
}
fn validate_classifier_traces(
traces: &ActivationTrace<B>,
- traces_view: &ParameterTree<B::Context>,
+ traces_view: &ParameterTree<B>,
+ trace_shapes: &HashMap<String, Vec<usize>>,
data_type: DataType,
) -> Vec<TracerValidationResult> {
let mut results = Vec::new();
- // Output pooling (classifier-specific)
+ let embedding_norm =
+ traces.embedding_norm.as_ref().expect("classifier traces must include produced embedding norm output");
+ let expected =
+ Self::read_required_array(traces_view, trace_shapes, "activation_trace.embedding_norm_output", data_type);
+ results.push(TracerValidationResult {
+ name: "activation_trace.embedding_norm_output".to_string(),
+ metrics: Self::validate_allocation(
+ data_type,
+ &expected,
+ embedding_norm.allocation(),
+ embedding_norm.shape(),
+ None,
+ ),
+ });
+
if let Some(output_pooling) = &traces.output_pooling {
- if let Ok(expected) = traces_view.leaf_array("activation_trace.output_pooling") {
- results.push(TracerValidationResult {
- name: "activation_trace.output_pooling".to_string(),
- metrics: Self::validate_allocation(
- data_type,
- &expected,
- output_pooling.allocation(),
- output_pooling.shape(),
- None,
- ),
- });
- }
- }
+ let expected =
+ Self::read_required_array(traces_view, trace_shapes, "activation_trace.output_pooling", data_type);
+ results.push(TracerValidationResult {
+ name: "activation_trace.output_pooling".to_string(),
+ metrics: Self::validate_allocation(
+ data_type,
+ &expected,
+ output_pooling.allocation(),
+ output_pooling.shape(),
+ None,
+ ),
+ });
+ } else {
+ panic!("classifier traces must include produced output pooling");
+ };
results
}
@@ -668,32 +692,6 @@ impl<B: Backend> TraceValidator<B> {
}
}
- fn validate_optional_allocation(
- data_type: DataType,
- expected_array: &Array<B>,
- produced_allocation: Option<&Allocation<B>>,
- produced_shape: &[usize],
- transform: Option<ArrayTransform>,
- ) -> TracerValidationMetrics {
- match produced_allocation {
- Some(produced_allocation) => {
- Self::validate_allocation(data_type, expected_array, produced_allocation, produced_shape, transform)
- },
- None => match data_type {
- DataType::F16 => {
- Self::validate_allocation_data_of_type::<f16>(expected_array, &[], produced_shape, transform)
- },
- DataType::BF16 => {
- Self::validate_allocation_data_of_type::<bf16>(expected_array, &[], produced_shape, transform)
- },
- DataType::F32 => {
- Self::validate_allocation_data_of_type::<f32>(expected_array, &[], produced_shape, transform)
- },
- _ => panic!("Unsupported data type: {:?}", data_type),
- },
- }
- }
-
fn validate_allocation_of_type<Precision: ArrayElement>(
expected_array: &Array<B>,
produced_allocation: &Allocation<B>,
@@ -714,16 +712,14 @@ impl<B: Backend> TraceValidator<B> {
let produced_view = ndarray::ArrayView::from_shape(IxDyn(produced_shape), produced_slice)
.expect("Failed to reshape allocation");
- let (mut expected_data, mut produced_data) = match transform {
+ let (expected_data, mut produced_data) = match transform {
Some(ArrayTransform::KVCacheSlice) => {
- let permuted = produced_view.permuted_axes(IxDyn(&[1, 0, 2]));
- let total_tokens = permuted.shape()[0];
+ let expected_shape = expected_view.shape();
let expected_tokens = expected_view.shape()[1];
- let start = total_tokens.saturating_sub(expected_tokens);
- let sliced = permuted.slice(s![start.., .., ..]);
+ let sliced = produced_view.slice(s![..expected_tokens, .., ..]);
let reshaped = sliced
.into_owned()
- .to_shape(IxDyn(&[1, expected_tokens, permuted.shape()[1], permuted.shape()[2]]))
+ .to_shape(IxDyn(expected_shape))
.expect("Failed to reshape KV cache slice")
.to_owned();
(expected_view.to_owned(), reshaped)
@@ -757,14 +753,14 @@ impl<B: Backend> TraceValidator<B> {
&& expected_shape.get(0) == Some(&1)
&& expected_shape[1..] == produced_shape[..]
{
- expected_data =
- expected_data.to_shape(IxDyn(&produced_shape)).expect("Failed to reshape expected data").to_owned();
- } else if produced_shape.len() == expected_shape.len() + 1
- && produced_shape.get(0) == Some(&1)
- && produced_shape[1..] == expected_shape[..]
- {
- produced_data =
- produced_data.to_shape(IxDyn(&expected_shape)).expect("Failed to reshape produced data").to_owned();
+ produced_data = produced_data
+ .to_shape(IxDyn(&expected_shape))
+ .expect("Failed to reshape produced data to trace shape")
+ .to_owned();
+ } else {
+ panic!(
+ "Shape mismatch: expected trace shape {expected_shape:?}, produced Uzu shape {produced_shape:?}"
+ );
}
}
@@ -776,8 +772,14 @@ impl<B: Backend> TraceValidator<B> {
);
}
- let reference: Vec<f32> = expected_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();
- let result: Vec<f32> = produced_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();
+ let reference: Vec<f32> = expected_data
+ .iter()
+ .map(|value| NumCast::from(*value).expect("Failed to cast reference trace value"))
+ .collect();
+ let result: Vec<f32> = produced_data
+ .iter()
+ .map(|value| NumCast::from(*value).expect("Failed to cast produced trace value"))
+ .collect();
let (atol, rtol, allowed_voilations_tol) = match expected_array.data_type() {
DataType::BF16 => (0.04, 0.06, 0.03),
@@ -910,26 +912,72 @@ impl<B: Backend> TraceValidator<B> {
// ========================================================================
fn load_array_as_vec<SourcePrecision: ArrayElement, TargetPrecision: NumCast>(
- traces_view: &ParameterTree<B::Context>,
+ traces_view: &ParameterTree<B>,
+ trace_shapes: &HashMap<String, Vec<usize>>,
name: &str,
) -> Vec<TargetPrecision> {
- let array = traces_view.leaf_array(name).unwrap();
- let slice = array.as_slice::<SourcePrecision>();
- slice.iter().map(|x| NumCast::from(*x).unwrap()).collect()
+ let expected_shape = Self::trace_shape(trace_shapes, name);
+ let leaf = traces_view
+ .leaf(name)
+ .unwrap_or_else(|error| panic!("Missing required trace tensor {name}: {error}"))
+ .validate(expected_shape, SourcePrecision::data_type())
+ .unwrap_or_else(|error| panic!("Invalid trace tensor {name}: {error}"));
+ let slice = leaf.read_slice::<SourcePrecision>().unwrap_or_else(|error| {
+ panic!("Failed to read trace tensor {name}: {error}");
+ });
+ slice.iter().map(|x| NumCast::from(*x).expect("Failed to cast trace tensor value")).collect()
+ }
+
+ fn trace_shape<'shape>(
+ trace_shapes: &'shape HashMap<String, Vec<usize>>,
+ name: &str,
+ ) -> &'shape [usize] {
+ trace_shapes.get(name).unwrap_or_else(|| panic!("Missing required trace tensor {name}"))
+ }
+
+ fn trace_token_count(
+ name: &str,
+ shape: &[usize],
+ ) -> usize {
+ let [batch_size, suffix_length] = shape else {
+ panic!("Trace tensor {name} must have shape [1, suffix_tokens], got {shape:?}");
+ };
+ assert_eq!(*batch_size, 1, "Trace tensor {name} must have batch size 1, got {shape:?}");
+ *suffix_length
+ }
+
+ fn read_required_array(
+ traces_view: &ParameterTree<B>,
+ trace_shapes: &HashMap<String, Vec<usize>>,
+ name: &str,
+ data_type: DataType,
+ ) -> Array<B> {
+ let expected_shape = Self::trace_shape(trace_shapes, name);
+ Self::read_array(traces_view, name, expected_shape, data_type)
+ .unwrap_or_else(|error| panic!("Invalid trace tensor {name}: {error}"))
+ }
+
+ fn read_array(
+ traces_view: &ParameterTree<B>,
+ name: &str,
+ expected_shape: &[usize],
+ data_type: DataType,
+ ) -> Result<Array<B>, ParameterLoaderError<B>> {
+ traces_view.leaf(name)?.validate(expected_shape, data_type)?.read_array()
}
- fn determine_prefill_step_size(model_path: &Path) -> usize {
+ fn determine_prefill_step_size(model_path: &Path) -> Result<usize, Error> {
let traces_path = model_path.join("traces.safetensors");
- if let Ok(file) = File::open(&traces_path) {
- if let Ok((_header_len, metadata)) = read_safetensors_metadata(&file) {
- if let Some(tensor) = metadata.tensors.get("activation_trace.token_ids") {
- if let Some(&length) = tensor.shape.first() {
- return tensor.shape.iter().copied().max().unwrap_or(length).max(1);
- }
- }
- }
- }
- 1
+ let file = File::open(&traces_path).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let (_header_len, metadata) =
+ read_safetensors_metadata(&file).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let token_shape =
+ metadata.tensors.get("activation_trace.token_ids").map(|tensor| tensor.shape.as_slice()).ok_or_else(
+ || Error::InvalidModelConfig("missing required trace tensor activation_trace.token_ids".to_string()),
+ )?;
+ let suffix_length = Self::trace_token_count("activation_trace.token_ids", token_shape);
+ assert!(suffix_length > 0, "trace must contain at least one token");
+ Ok(suffix_length)
}
fn ensure_llm_context_capacity(
@@ -954,16 +1002,13 @@ impl<B: Backend> TraceValidator<B> {
desired_suffix_length,
)));
- let intermediate_dtype: DataType =
- context.model_config.model_config.transformer_config.output_norm_config.scale_precision.into();
-
context.kv_cache_update = Box::new(
- KVCacheUpdate::new(context.context.as_ref(), intermediate_dtype, resolved_prefix_length)
+ KVCacheUpdate::new(context.context.as_ref(), context.model_shape.data_type, resolved_prefix_length)
.expect("Failed to create KV cache update kernel"),
);
- context.gpu_sampler =
- Sampling::new(context.context.as_ref(), intermediate_dtype).expect("Failed to create sampling kernel");
+ context.gpu_sampler = Sampling::new(context.context.as_ref(), context.model_shape.data_type)
+ .expect("Failed to create sampling kernel");
}
fn get_tokens_from_logits(logits: &Array<B>) -> Vec<u64> {
diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator_test.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator_test.rs
index 6983c8d2e..5871f9327 100644
--- a/crates/backend-uzu/tests/integration/tracer/trace_validator_test.rs
+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator_test.rs
@@ -13,10 +13,6 @@ fn test_tracer_internal<B: Backend>() {
let mut tracer = TraceValidator::<B>::new(&model_path).expect("Failed to create TraceValidator");
let results = tracer.run().expect("Failed to run tracer");
for result in results.results.iter() {
- // this layers contains too many errors
- if result.name == "activation_trace.output_norm" || result.name == "logits" {
- continue;
- }
assert!(result.metrics.is_valid(), "{} error:\n{}", result.name, result.metrics.message().as_str());
}
@@ -32,6 +28,7 @@ fn test_tracer_internal<B: Backend>() {
}
#[test]
+#[ignore = "Lalamo 0.10.0 doesn't support exporting traces"] // TODO: this is horrible, should be resolved asap
fn test_tracer() {
let traces_path = get_traces_path();
assert!(traces_path.exists(), "Traces file missing at {:?}", traces_path);
diff --git a/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs b/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs
index 9484154b5..3d752bbbf 100644
--- a/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs
+++ b/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs
@@ -707,7 +707,7 @@ fn audio_kernel_perf() {
{
let i = input.allocation();
let o = &mut output;
- k_act.encode(Some(i), o, n as u32, ActivationType::GELU, &mut encoder);
+ k_act.encode(Some(i), o, n as u32, ActivationType::GELUApprox, &mut encoder);
}
encoder.end_encoding().submit().wait_until_completed().unwrap();
});
@@ -973,7 +973,13 @@ fn audio_kernel_perf() {
);
}
{
- k_act.encode(Some(&scratch_a), &mut scratch_b, (c0_ch * f_up0) as u32, ActivationType::GELU, &mut encoder);
+ k_act.encode(
+ Some(&scratch_a),
+ &mut scratch_b,
+ (c0_ch * f_up0) as u32,
+ ActivationType::GELUApprox,
+ &mut encoder,
+ );
}
{
let w = c0_p2_w.allocation();
@@ -1089,7 +1095,13 @@ fn audio_kernel_perf() {
);
}
{
- k_act.encode(Some(&scratch_b), &mut scratch_a, (c1_ch * f_up1) as u32, ActivationType::GELU, &mut encoder);
+ k_act.encode(
+ Some(&scratch_b),
+ &mut scratch_a,
+ (c1_ch * f_up1) as u32,
+ ActivationType::GELUApprox,
+ &mut encoder,
+ );
}
{
let w = c1_p2_w.allocation();
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
index 37d8c8b42..40538f4af 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
@@ -106,10 +106,10 @@ fn moe_cpu_reference(
let mut hidden = vec![0.0f32; d_ff];
for i in 0..d_ff {
hidden[i] = match gating_code {
- 0 => ActivationType::GELU.activate(up_out[i]),
+ 0 => ActivationType::GELUApprox.activate(up_out[i]),
1 => activation_silu_alpha(up_out[i], silu_alpha),
2 => activation_silu_alpha(gate_out[i], silu_alpha) * up_out[i],
- 3 => ActivationType::GELU.activate(gate_out[i]) * up_out[i],
+ 3 => ActivationType::GELUApprox.activate(gate_out[i]) * up_out[i],
_ => activation_silu_alpha(gate_out[i], silu_alpha) * up_out[i], // fallback to SwiGLU
};
}
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
index 2240c5a41..1bb46fca3 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
@@ -223,7 +223,7 @@ fn cpu_moe_reference(
// GELU or SiLU on up (with clipping)
let up_val = acc_up.clamp(up_clip_min, up_clip_max);
if gating_code == 0 {
- ActivationType::GELU.activate(up_val)
+ ActivationType::GELUApprox.activate(up_val)
} else {
activation_silu_alpha(up_val, silu_alpha)
}
@@ -243,7 +243,7 @@ fn cpu_moe_reference(
let gate_act = if gating_code == 2 {
activation_silu_alpha(gate_val, silu_alpha)
} else {
- ActivationType::GELU.activate(gate_val)
+ ActivationType::GELUApprox.activate(gate_val)
};
gate_act * up_val
};
diff --git a/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs b/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs
index db5e5ea56..be814eea0 100644
--- a/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs
@@ -105,15 +105,30 @@ fn test_silu_bf16() {
#[uzu_test]
fn test_gelu_f32() {
- test::<f32>(ActivationType::GELU);
+ test::<f32>(ActivationType::GELUApprox);
}
#[uzu_test]
fn test_gelu_f16() {
- test::<f16>(ActivationType::GELU);
+ test::<f16>(ActivationType::GELUApprox);
}
#[uzu_test]
fn test_gelu_bf16() {
- test::<bf16>(ActivationType::GELU);
+ test::<bf16>(ActivationType::GELUApprox);
+}
+
+#[uzu_test]
+fn test_gelu_exact_f32() {
+ test::<f32>(ActivationType::GELUExact);
+}
+
+#[uzu_test]
+fn test_gelu_exact_f16() {
+ test::<f16>(ActivationType::GELUExact);
+}
+
+#[uzu_test]
+fn test_gelu_exact_bf16() {
+ test::<bf16>(ActivationType::GELUExact);
}
diff --git a/crates/backend-uzu/tests/unit/kernel/activation_test.rs b/crates/backend-uzu/tests/unit/kernel/activation_test.rs
index 32466262e..2f6c83fde 100644
--- a/crates/backend-uzu/tests/unit/kernel/activation_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/activation_test.rs
@@ -184,33 +184,49 @@ fn test_silu_in_place_bf16() {
// GELU out-of-place tests
#[uzu_test]
fn test_gelu_f32() {
- test_activation::<f32>(ActivationType::GELU, false);
+ test_activation::<f32>(ActivationType::GELUApprox, false);
}
#[uzu_test]
fn test_gelu_f16() {
- test_activation::<f16>(ActivationType::GELU, false);
+ test_activation::<f16>(ActivationType::GELUApprox, false);
}
#[uzu_test]
fn test_gelu_bf16() {
- test_activation::<bf16>(ActivationType::GELU, false);
+ test_activation::<bf16>(ActivationType::GELUApprox, false);
+}
+
+// Exact GELU out-of-place tests
+#[uzu_test]
+fn test_gelu_exact_f32() {
+ test_activation::<f32>(ActivationType::GELUExact, false);
+}
+
+#[uzu_test]
+fn test_gelu_exact_f16() {
+ test_activation::<f16>(ActivationType::GELUExact, false);
+}
+
+#[uzu_test]
+fn test_gelu_exact_bf16() {
+ test_activation::<bf16>(ActivationType::GELUExact, false);
}
// GELU in-place tests
#[uzu_test]
fn test_gelu_in_place_f32() {
- test_activation::<f32>(ActivationType::GELU, true);
+ test_activation::<f32>(ActivationType::GELUApprox, true);
}
#[uzu_test]
fn test_gelu_in_place_f16() {
- test_activation::<f16>(ActivationType::GELU, true);
+ test_activation::<f16>(ActivationType::GELUApprox, true);
}
#[uzu_test]
fn test_gelu_in_place_bf16() {
- test_activation::<bf16>(ActivationType::GELU, true);
+ test_activation::<bf16>(ActivationType::GELUApprox, true);
}
// Large SILU tests
@@ -232,17 +248,17 @@ fn test_silu_large_bf16() {
// Large GELU tests
#[uzu_test]
fn test_gelu_large_f32() {
- test_activation_large::<f32>(ActivationType::GELU, false);
+ test_activation_large::<f32>(ActivationType::GELUApprox, false);
}
#[uzu_test]
fn test_gelu_large_f16() {
- test_activation_large::<f16>(ActivationType::GELU, false);
+ test_activation_large::<f16>(ActivationType::GELUApprox, false);
}
#[uzu_test]
fn test_gelu_large_bf16() {
- test_activation_large::<bf16>(ActivationType::GELU, false);
+ test_activation_large::<bf16>(ActivationType::GELUApprox, false);
}
// Large in-place tests
@@ -253,5 +269,5 @@ fn test_silu_in_place_large_f32() {
#[uzu_test]
fn test_gelu_in_place_large_f32() {
- test_activation_large::<f32>(ActivationType::GELU, true);
+ test_activation_large::<f32>(ActivationType::GELUApprox, true);
}
diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
index 38fd72538..1d2507b18 100644
--- a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
@@ -6,7 +6,9 @@ use backend_uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
common::{
- Backend, Context, Encoder, Kernels, gpu_types::QuantizationMode, kernel::QuantizedEmbeddingLookupKernel,
+ Backend, Context, Encoder, Kernels,
+ gpu_types::{QuantizationMethod, QuantizationMode},
+ kernel::QuantizedEmbeddingLookupKernel,
},
cpu::Cpu,
},
@@ -112,6 +114,8 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
T::data_type(),
input.group_size,
input.quant_mode,
+ QuantizationMethod::ScaleBias,
+ false,
)
.expect("Failed to create QuantizedEmbeddingLookupKernel");
@@ -132,8 +136,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
token_ids_array.allocation(),
weights_array.allocation(),
scales_array.allocation(),
- biases_array.allocation(),
+ None::<&backend_uzu::backends::common::Allocation<B>>,
+ Some(biases_array.allocation()),
&mut output,
+ None::<&backend_uzu::backends::common::Allocation<B>>,
input.batch_size,
input.vocab_size,
input.model_dim,
diff --git a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs
index c67f7353b..e4a455737 100644
--- a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs
@@ -4,7 +4,11 @@ use std::{
use backend_uzu::{
ArrayContextExt, ArrayElement,
- backends::common::{Backend, Context, Encoder, Kernels, kernel::HadamardTransformKernel},
+ backends::common::{
+ Backend, Context, Encoder, Kernels,
+ gpu_types::HadamardTransformOrder,
+ kernel::HadamardTransformKernel,
+ },
};
use half::{bf16, f16};
use num_traits::Float;
@@ -100,8 +104,12 @@ fn generate_test_input<T: ArrayElement + Float>(
fn run_kernel<T: ArrayElement + Float, B: Backend>(input: &TestInput<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create context");
- let kernel = <<B as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(&context, T::data_type())
- .expect("Failed to create HadamardTransformKernel");
+ let kernel = <<B as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(
+ &context,
+ T::data_type(),
+ HadamardTransformOrder::Input,
+ )
+ .expect("Failed to create HadamardTransformKernel");
let total_elements = input.batch_count * input.channel_count;
let mut data = context
diff --git a/crates/backend-uzu/tests/unit/kernel/rope_test.rs b/crates/backend-uzu/tests/unit/kernel/rope_test.rs
index e30d780f6..4a3460c53 100644
--- a/crates/backend-uzu/tests/unit/kernel/rope_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/rope_test.rs
@@ -1,6 +1,4 @@
-use std::{
- fmt::{Debug, Display},
-};
+use std::fmt::{Debug, Display};
use backend_uzu::{
ArrayContextExt, ArrayElement, DataType,
@@ -16,8 +14,8 @@ use crate::{common::assert::assert_eq_float, uzu_test};
struct Input<T: ArrayElement + Float> {
qkv: Box<[T]>,
- cosines: Box<[T]>,
- sines: Box<[T]>,
+ cosines: Box<[f32]>,
+ sines: Box<[f32]>,
token_positions: Box<[i32]>,
head_dim: u32,
rope_dim: u32,
@@ -44,11 +42,11 @@ fn get_test_data<T: ArrayElement + Float>(
qkv[i] = T::from(((i as f32) * 0.1).sin() * 2.0).unwrap();
}
- let mut cosines = vec![T::zero(); cos_sin_size];
- let mut sines = vec![T::zero(); cos_sin_size];
+ let mut cosines = vec![0.0f32; cos_sin_size];
+ let mut sines = vec![0.0f32; cos_sin_size];
for i in 0..cos_sin_size {
- cosines[i] = T::from(((i as f32) * 0.05).cos()).unwrap();
- sines[i] = T::from(((i as f32) * 0.05).sin()).unwrap();
+ cosines[i] = ((i as f32) * 0.05).cos();
+ sines[i] = ((i as f32) * 0.05).sin();
}
let mut token_positions = vec![0i32; suffix_length as usize];
@@ -73,7 +71,9 @@ fn get_test_data<T: ArrayElement + Float>(
fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::RopeKernel::new(&context, T::data_type())
+ let element_data_type = T::data_type();
+ let rope_data_type = DataType::F32;
+ let kernel = <<B as Backend>::Kernels as Kernels>::RopeKernel::new(&context, element_data_type, rope_data_type)
.expect("Failed to create RopeKernel");
let total_heads = (input.num_heads + 2 * input.num_groups) as usize;
@@ -86,12 +86,8 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines);
let sines_array = context.create_array_from(&[cos_sin_len], &input.sines);
let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions);
- let mut rotated_queries = context
- .create_array_uninitialized(&[queries_len], T::data_type())
- .into_allocation();
- let mut rotated_keys = context
- .create_array_uninitialized(&[keys_len], T::data_type())
- .into_allocation();
+ let mut rotated_queries = context.create_array_uninitialized(&[queries_len], T::data_type()).into_allocation();
+ let mut rotated_keys = context.create_array_uninitialized(&[keys_len], T::data_type()).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs
index 8bad6a293..0825c539e 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs
@@ -271,7 +271,7 @@ fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {
}
fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELU);
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 1, true, true, ActivationType::GELUApprox);
test_internal(&input, &expected, "gelu_activation");
}
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs
index 6be3cc6b2..8f3d56f8f 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs
@@ -224,7 +224,7 @@ fn test_identity_activation<T: ArrayElement + Float + Debug + Display>() {
}
fn test_gelu_activation<T: ArrayElement + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELU);
+ let (input, expected) = get_test_data::<T>(4, 8, 4, 2, 2, true, ActivationType::GELUApprox);
test_internal(&input, &expected, "gelu_activation");
}
diff --git a/crates/backend-uzu/tests/unit/utils/model_size_test.rs b/crates/backend-uzu/tests/unit/utils/model_size_test.rs
index 2f66dd25a..207eadad2 100644
--- a/crates/backend-uzu/tests/unit/utils/model_size_test.rs
+++ b/crates/backend-uzu/tests/unit/utils/model_size_test.rs
@@ -23,7 +23,7 @@ fn test_model_size_from_path() {
let model_path = get_test_model_path();
if model_path.exists() {
- let size = ModelSize::from_path(&model_path);
+ let size = ModelSize::from_path(&model_path).expect("load model size");
println!("Model size: {:?}", size);
assert_eq!(size, ModelSize::Small);
}
diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rs
index 7b4b716dd..cefc7c7dc 100644
--- a/crates/benchmarks/src/runner/runner.rs
+++ b/crates/benchmarks/src/runner/runner.rs
@@ -51,16 +51,14 @@ impl Runner {
}
let mut session = ChatSession::new(PathBuf::from(self.model_path.clone()), decoding_config)?;
-
- let precision =
- session.model_metadata.model_config.model_config.transformer_config.output_norm_config.scale_precision;
+ let data_type = session.activation_data_type();
let device = self.get_device_info();
let input = Input::Messages(self.task.messages.clone());
let warmup_config = RunConfig::default().tokens_limit(1);
- session.run(input.clone(), warmup_config, Some(|_: Output| true))?;
+ session.run(input.clone(), warmup_config, Option::<fn(Output) -> bool>::None)?;
let mut results: Vec<TaskResult> = Vec::new();
for run_index in 0..self.task.number_of_runs {
@@ -72,13 +70,7 @@ impl Runner {
value: SamplingMethod::Greedy,
});
}
- let output = session.run(
- input.clone(),
- run_config,
- Some(|_: Output| {
- return true;
- }),
- )?;
+ let output = session.run(input.clone(), run_config, Option::<fn(Output) -> bool>::None)?;
let memory_used = get_memory_usage();
@@ -87,7 +79,7 @@ impl Runner {
device: device.clone(),
engine_version: VERSION.to_string(),
timestamp,
- precision,
+ data_type,
memory_used,
tokens_count_input: output.stats.total_stats.tokens_count_input,
tokens_count_output: output.stats.total_stats.tokens_count_output,
diff --git a/crates/benchmarks/src/runner/types/result.rs b/crates/benchmarks/src/runner/types/result.rs
index 7b3980ff2..7c14b3647 100644
--- a/crates/benchmarks/src/runner/types/result.rs
+++ b/crates/benchmarks/src/runner/types/result.rs
@@ -9,7 +9,7 @@ pub struct Result {
pub device: Device,
pub engine_version: String,
pub timestamp: u64,
- pub precision: DataType,
+ pub data_type: DataType,
pub memory_used: Option<u64>,
pub tokens_count_input: u64,
pub tokens_count_output: u64,
diff --git a/crates/proc-macros/src/lib.rs b/crates/proc-macros/src/lib.rs
index d7df287d1..b1d3a0be2 100644
--- a/crates/proc-macros/src/lib.rs
+++ b/crates/proc-macros/src/lib.rs
@@ -12,6 +12,14 @@ pub fn kernel(
dsl::kernel(args, input)
}
+#[proc_macro_attribute]
+pub fn uzu_config_abstract(
+ args: TokenStream,
+ input: TokenStream,
+) -> TokenStream {
+ uzu_config::uzu_config_abstract(args, input)
+}
+
#[proc_macro_attribute]
pub fn uzu_config(
args: TokenStream,
diff --git a/crates/proc-macros/src/uzu_config.rs b/crates/proc-macros/src/uzu_config.rs
index d2eff31cc..d69aa542f 100644
--- a/crates/proc-macros/src/uzu_config.rs
+++ b/crates/proc-macros/src/uzu_config.rs
@@ -1,13 +1,167 @@
use proc_macro::TokenStream;
-use quote::{ToTokens, quote};
-use syn::{Data, DeriveInput, Field, Generics, parse_macro_input, parse_quote};
+use quote::{ToTokens, format_ident, quote};
+use syn::{
+ Data, DataStruct, DeriveInput, Field, Fields, FieldsNamed, Generics, ItemStruct, Path, Token, Type,
+ parse_macro_input, parse_quote, punctuated::Punctuated, spanned::Spanned,
+};
+
+// TODO: This is very hacky, there has to be a better way to do all of this stuff
+
+pub fn uzu_config_abstract(
+ args: TokenStream,
+ input: TokenStream,
+) -> TokenStream {
+ let concrete_configs = parse_macro_input!(args with Punctuated::<Path, Token![,]>::parse_terminated);
+ let input = parse_macro_input!(input as ItemStruct);
+
+ let struct_fields = match input.fields {
+ Fields::Named(FieldsNamed {
+ named,
+ ..
+ }) => named,
+ Fields::Unit => Punctuated::new(),
+ fields => {
+ return syn::Error::new(fields.span(), "uzu_config_abstract requires a named-field or unit struct")
+ .to_compile_error()
+ .into();
+ },
+ };
+
+ let input_vis = input.vis;
+ let struct_ident = input.ident;
+ let enum_ident = format_ident!("Any{}", struct_ident);
+ let (alias_struct_fields, callback_struct_fields, enum_field_getters) = struct_fields
+ .into_iter()
+ .map(|mut struct_field| {
+ let field_ident = struct_field.ident.as_ref().unwrap();
+ let typealias_ident = format_ident!("__{}_{}", struct_ident, field_ident);
+ let field_type = std::mem::replace(
+ &mut struct_field.ty,
+ Type::Verbatim(quote! { $($typealias_base)::*::#typealias_ident }),
+ );
+ let match_arms = concrete_configs.iter().map(|concrete_config_path| {
+ let concrete_config_ident = &concrete_config_path.segments.last().unwrap().ident;
+ quote! { Self::#concrete_config_ident(__concrete) => &__concrete.#field_ident, }
+ });
+ (
+ quote! {
+ #[allow(non_camel_case_types)]
+ type #typealias_ident = #field_type;
+ },
+ quote! {
+ #struct_field
+ },
+ quote! {
+ #input_vis fn #field_ident(&self) -> &#field_type {
+ match self {
+ #(#match_arms)*
+ }
+ }
+ },
+ )
+ })
+ .collect::<(Vec<_>, Vec<_>, Vec<_>)>();
+ let enum_variants = concrete_configs.into_iter().map(|concrete_config_path| {
+ let concrete_config_ident = &concrete_config_path.segments.last().unwrap().ident;
+ quote! { #concrete_config_ident(#concrete_config_path) }
+ });
+
+ quote! {
+ #(#alias_struct_fields)*
+
+ macro_rules! #struct_ident {
+ ($($typealias_base:ident)::+, $callback:path) => {
+ $callback! {
+ #(#callback_struct_fields,)*
+ }
+ };
+ }
+
+ use #struct_ident;
+
+ #[proc_macros::uzu_config]
+ #[serde(untagged)]
+ #input_vis enum #enum_ident {
+ #(#enum_variants,)*
+ }
+
+ impl #enum_ident {
+ #(#enum_field_getters)*
+ }
+ }
+ .into()
+}
pub fn uzu_config(
- _args: TokenStream,
+ args: TokenStream,
input: TokenStream,
) -> TokenStream {
let mut input = parse_macro_input!(input as DeriveInput);
+ if !args.is_empty() {
+ let fields_macro_path = parse_macro_input!(args as Path);
+ let fields_macro_base_path = {
+ let mut fields_macro_base = fields_macro_path.clone();
+ fields_macro_base.segments.pop();
+ fields_macro_base.segments.pop_punct();
+ fields_macro_base
+ };
+ let fields_macro_base = if fields_macro_base_path.segments.is_empty() {
+ quote! { self }
+ } else {
+ quote! { #fields_macro_base_path }
+ };
+
+ let struct_fields = match input.data {
+ Data::Struct(DataStruct {
+ fields: Fields::Named(FieldsNamed {
+ named,
+ ..
+ }),
+ ..
+ }) => named,
+ Data::Struct(DataStruct {
+ fields: Fields::Unit,
+ ..
+ }) => Punctuated::new(),
+ _ => {
+ return syn::Error::new(
+ input.ident.span(),
+ "uzu_config(Abstract) requires a named-field or unit struct",
+ )
+ .to_compile_error()
+ .into();
+ },
+ };
+
+ let input_attrs = input.attrs;
+ let input_vis = input.vis;
+ let input_ident = input.ident;
+ let input_ident_str = input_ident.to_string();
+ let input_generics = input.generics;
+ let struct_fields = struct_fields.into_iter().collect::<Vec<_>>();
+ let callback_ident = format_ident!("__emit_{}", input_ident);
+
+ // Re-entry with super fields inserted
+ return quote! {
+ macro_rules! #callback_ident {
+ ($($super_fields:tt)*) => {
+ #[proc_macros::uzu_config]
+ #(#input_attrs)*
+ #input_vis struct #input_ident #input_generics {
+ #[serde(rename = "type")]
+ ty: monostate::MustBe!(#input_ident_str),
+ $($super_fields)*
+ #(#struct_fields,)*
+ }
+ }
+ }
+
+ #fields_macro_path!(#fields_macro_base, #callback_ident);
+ }
+ .into();
+ };
+
let fields: Vec<&mut Field> = match &mut input.data {
Data::Struct(data_struct) => data_struct.fields.iter_mut().collect(),
Data::Enum(data_enum) => data_enum.variants.iter_mut().flat_map(|variant| variant.fields.iter_mut()).collect(),
diff --git a/crates/uzu/src/registry/local/config.rs b/crates/uzu/src/registry/local/config.rs
index 3773d8d89..9ffadcdc6 100644
--- a/crates/uzu/src/registry/local/config.rs
+++ b/crates/uzu/src/registry/local/config.rs
@@ -1,9 +1,11 @@
use std::{path::PathBuf, sync::Arc};
-use backend_uzu::inference::resolve_model_metadata;
+use backend_uzu::inference::resolve_model_specialization;
use shoji::types::model::Model;
-pub type ModelResolver = Arc<dyn Fn(Model) -> Option<Model> + Send + Sync>;
+use crate::registry::RegistryError;
+
+pub type ModelResolver = Arc<dyn Fn(Model) -> Result<Model, RegistryError> + Send + Sync>;
#[derive(Clone)]
pub struct Config {
@@ -40,18 +42,18 @@ impl Config {
path: String,
) -> Self {
let models_path = PathBuf::from(&path).join("models");
- let resolver: ModelResolver = {
- let backend_version = backend_version.clone();
- Arc::new(move |mut model: Model| -> Option<Model> {
- let model_path = PathBuf::from(model.local_external_path()?);
- let metadata = resolve_model_metadata(&model_path)?;
- if !is_lalamo_version_supported(&metadata.toolchain_version, &backend_version) {
- return None;
- }
- model.specializations = vec![metadata.specialization];
- Some(model)
- })
- };
+ let resolver: ModelResolver = Arc::new(move |mut model: Model| -> Result<Model, RegistryError> {
+ let model_path = model.local_external_path().ok_or_else(|| RegistryError::UnableToGetModels {
+ message: format!("Local model {} has no local path", model.identifier),
+ })?;
+ let model_path = PathBuf::from(model_path);
+ let specialization =
+ resolve_model_specialization(&model_path).map_err(|error| RegistryError::UnableToGetModels {
+ message: format!("Unable to resolve specialization for {}: {error}", model_path.display()),
+ })?;
+ model.specializations = vec![specialization];
+ Ok(model)
+ });
Self::new(
"lalamo".to_string(),
backend_identifier,
@@ -62,23 +64,3 @@ impl Config {
)
}
}
-
-fn is_lalamo_version_supported(
- model_version: &str,
- backend_version: &str,
-) -> bool {
- let parse_version = |value: &str| -> Option<(u32, u32, u32)> {
- let mut parts = value.split('.').map(|part| part.parse::<u32>().ok());
- let major = parts.next().flatten()?;
- let minor = parts.next().flatten()?;
- let patch = parts.next().flatten()?;
- Some((major, minor, patch))
- };
- let Some((model_major, model_minor, model_patch)) = parse_version(model_version) else {
- return false;
- };
- let Some((backend_major, backend_minor, backend_patch)) = parse_version(backend_version) else {
- return false;
- };
- model_major == backend_major && model_minor == backend_minor && model_patch <= backend_patch
-}
diff --git a/crates/uzu/src/registry/local/mod.rs b/crates/uzu/src/registry/local/mod.rs
index b3fcbbdbc..4a9de94a3 100644
--- a/crates/uzu/src/registry/local/mod.rs
+++ b/crates/uzu/src/registry/local/mod.rs
@@ -43,26 +43,29 @@ impl RegistryTrait for Registry {
})?;
let mut models = Vec::new();
- for entry in entries.flatten() {
+ for entry in entries {
+ let entry = entry.map_err(|error| RegistryError::UnableToGetModels {
+ message: error.to_string(),
+ })?;
let path = entry.path();
- let Ok(file_type) = entry.file_type() else {
- continue;
- };
+ let file_type = entry.file_type().map_err(|error| RegistryError::UnableToGetModels {
+ message: error.to_string(),
+ })?;
if !file_type.is_dir() {
continue;
}
- let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
- continue;
- };
+ let name = path.file_name().and_then(|name| name.to_str()).ok_or_else(|| {
+ RegistryError::UnableToGetModels {
+ message: format!("Invalid local model path: {}", path.display()),
+ }
+ })?;
let model = self.model(name);
let model = match self.config.resolver.as_ref() {
- Some(resolver) => resolver(model),
- None => Some(model),
+ Some(resolver) => resolver(model)?,
+ None => model,
};
- if let Some(model) = model {
- models.push(model);
- }
+ models.push(model);
}
Ok(models)
diff --git a/flake.nix b/flake.nix
index e1ab398ae..f4345dc15 100644
--- a/flake.nix
+++ b/flake.nix
@@ -72,6 +72,7 @@
rustToolchain
cargo-deny
cargo-nextest
+ cargo-expand
cargo-flamegraph
cargo-show-asm
critcmp