Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Toolchain 0.12.1 (#492)

uuuvnacb69572a5b6
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs
index 36754723d..b252d295b 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/loaders.rs
@@ -17,7 +17,6 @@ pub(super) fn load_audio_runtime_from_tts_config(
embedding_config: AnyEmbeddingConfig::UntiedEmbeddingConfig(UntiedEmbeddingConfig::new(None, None)),
transformer_config: cfg.quantizer_config.post_module_config.clone(),
vocab_size: 1,
- pard_token: None,
ple_model_config: None,
};
let vocoder_data_type = ModelShape::from_decoder_config(&post_module_decoder_config, DataType::F32).data_type;
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index 351756762..b7bbbbd2c 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -92,7 +92,6 @@ impl StructuredAudioCodecGraph {
embedding_config: AnyEmbeddingConfig::UntiedEmbeddingConfig(UntiedEmbeddingConfig::new(None, None)),
transformer_config: self.config.quantizer_config.post_module_config.clone(),
vocab_size: 1,
- pard_token: None,
ple_model_config: None,
});
let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::F32);
diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rs
index 6896a1d75..78e64f39c 100644
--- a/crates/backend-uzu/src/classifier/classifier_context.rs
+++ b/crates/backend-uzu/src/classifier/classifier_context.rs
@@ -44,7 +44,6 @@ impl<B: Backend> ClassifierContext<B> {
embedding_config: classifier_config.embedding_config.clone(),
transformer_config: classifier_config.transformer_config.clone(),
vocab_size: classifier_config.vocab_size,
- pard_token: None,
ple_model_config: None,
});
@@ -82,21 +81,6 @@ impl<B: Backend> ClassifierContext<B> {
})
.collect();
- for (rope_index, (rope_config, head_dim)) in rope_configs.iter().enumerate() {
- let rope_tree = transformer_tree
- .subtree(&format!("ropes.{}", rope_index))
- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- let shape = [*rope_config.max_sequence_length(), *head_dim];
- rope_tree
- .leaf("cosines")
- .and_then(|leaf| leaf.validate(&shape, model_shape.rope_data_type))
- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- rope_tree
- .leaf("sines")
- .and_then(|leaf| leaf.validate(&shape, model_shape.rope_data_type))
- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- }
-
let output_norm_tree =
transformer_tree.subtree("output_norm").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
diff --git a/crates/backend-uzu/src/config/classifier.rs b/crates/backend-uzu/src/config/classifier.rs
index 7a4ec8be7..827eab624 100644
--- a/crates/backend-uzu/src/config/classifier.rs
+++ b/crates/backend-uzu/src/config/classifier.rs
@@ -28,14 +28,10 @@ pub struct ClassifierConfig {
pub embedding_norm_config: NormalizationConfig,
pub transformer_config: TransformerConfig,
pub prediction_head_config: PredictionHeadConfig,
- pub readout_config: LinearConfig,
pub vocab_size: usize,
pub model_dim: usize,
pub hidden_dim: usize,
- pub attention_scale: Option<f32>,
- pub num_layers: usize,
- pub context_length: usize,
pub num_labels: usize,
pub classifier_pooling: PoolingType,
pub output_labels: Option<Box<[String]>>,
diff --git a/crates/backend-uzu/src/config/decoder.rs b/crates/backend-uzu/src/config/decoder.rs
index 2c2e3b845..4096e7605 100644
--- a/crates/backend-uzu/src/config/decoder.rs
+++ b/crates/backend-uzu/src/config/decoder.rs
@@ -11,7 +11,6 @@ pub struct DecoderConfig {
pub transformer_config: TransformerConfig,
pub vocab_size: usize,
- pub pard_token: Option<u64>,
pub ple_model_config: Option<PLEModelConfig>,
}
diff --git a/crates/backend-uzu/src/config/token_mixer/attention.rs b/crates/backend-uzu/src/config/token_mixer/attention.rs
index 86c999116..b3804de36 100644
--- a/crates/backend-uzu/src/config/token_mixer/attention.rs
+++ b/crates/backend-uzu/src/config/token_mixer/attention.rs
@@ -26,6 +26,7 @@ pub struct AttentionConfig {
pub has_out_biases: bool,
pub gate_projection_config: Option<LinearConfig>,
pub normalize_values: bool,
+ pub is_kv_sharing: bool,
}
impl AttentionConfig {
diff --git a/crates/backend-uzu/src/config/token_mixer/mamba2.rs b/crates/backend-uzu/src/config/token_mixer/mamba2.rs
index cdedbabb9..3a3be17c0 100644
--- a/crates/backend-uzu/src/config/token_mixer/mamba2.rs
+++ b/crates/backend-uzu/src/config/token_mixer/mamba2.rs
@@ -16,7 +16,6 @@ pub struct Mamba2Config {
pub num_groups: usize,
pub head_dim: usize,
pub state_dim: usize,
- pub expansion_factor: usize,
pub has_in_biases: bool,
pub has_out_biases: bool,
diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
index fa9fa3823..1811b2ab3 100644
--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
@@ -29,6 +29,8 @@ pub enum ClassifierLayerError<B: Backend> {
NonAttentionMixer,
#[error("classifier does not support attention hadamard")]
AttentionHadamardUnsupported,
+ #[error("classifier does not support KV-sharing attention")]
+ KvSharingAttentionUnsupported,
#[error("classifier does not support MLP hadamard")]
MlpHadamardUnsupported,
}
@@ -62,6 +64,9 @@ impl<B: Backend> ClassifierLayer<B> {
) -> Result<Self, ClassifierLayerError<B>> {
let attention_config =
layer_config.mixer_config.as_attention().ok_or(ClassifierLayerError::NonAttentionMixer)?;
+ if attention_config.is_kv_sharing || layer_config.kv_source_layer_index.is_some() {
+ return Err(ClassifierLayerError::KvSharingAttentionUnsupported);
+ }
let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_mixer_norm_config {
Some(Normalization::new(
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index d99f02637..7650e32f2 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -180,13 +180,6 @@ impl<B: Backend> Decoder<B> {
})
.collect();
- for (rope_index, (rope_config, head_dim)) in rope_configs.iter().enumerate() {
- let rope_tree = decoder_weight_loader.subtree(&format!("ropes.{}", rope_index))?;
- let shape = [*rope_config.max_sequence_length(), *head_dim];
- rope_tree.leaf("cosines")?.validate(&shape, model_shape.rope_data_type)?;
- rope_tree.leaf("sines")?.validate(&shape, model_shape.rope_data_type)?;
- }
-
let rope = Rc::new(Rope::<B>::new(context, model_shape, false).map_err(DecoderError::BackendError)?);
// Built only when some layer projects queries only (reuses an earlier layer's KV cache).
let rope_query_only = if tf.layer_configs.iter().any(|l| l.kv_source_layer_index.is_some()) {
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index 633273c49..d39c3b135 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -51,6 +51,14 @@ pub enum LayerExecutablesError<B: Backend> {
UnsupportedPostLayerScalarDataType {
data_type: DataType,
},
+ #[error(
+ "layer {layer_index} AttentionConfig.is_kv_sharing={attention_is_kv_sharing} does not match kv_source_layer_index presence={has_kv_source_layer}"
+ )]
+ AttentionKvSharingMismatch {
+ layer_index: usize,
+ attention_is_kv_sharing: bool,
+ has_kv_source_layer: bool,
+ },
#[error("decoder layers require pre_mixer_norm_config")]
MissingPreMixerNormConfig,
}
@@ -139,6 +147,15 @@ impl<B: Backend> LayerExecutables<B> {
let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {
AnyTokenMixerConfig::AttentionConfig(attention_config) => {
+ let has_kv_source_layer = layer_config.kv_source_layer_index.is_some();
+ if attention_config.is_kv_sharing != has_kv_source_layer {
+ return Err(LayerExecutablesError::AttentionKvSharingMismatch {
+ layer_index,
+ attention_is_kv_sharing: attention_config.is_kv_sharing,
+ has_kv_source_layer,
+ });
+ }
+
let (attention, input_hadamard_factors) = Attention::new(
context,
transformer_config.model_dim,
@@ -148,7 +165,7 @@ impl<B: Backend> LayerExecutables<B> {
rope.clone(),
qk_unpack.clone(),
attention_config.gate_projection_config.is_none(),
- layer_config.kv_source_layer_index.is_some(),
+ has_kv_source_layer,
)?;
(
diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
index 144ad3293..a4fd27d7a 100644
--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs
+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
@@ -228,14 +228,12 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(
embedding_config: AnyEmbeddingConfig::TiedEmbeddingConfig(config.slow_embeddings_config.clone()),
transformer_config: slow_transformer_config,
vocab_size: config.vocab_size,
- pard_token: None,
ple_model_config: None,
});
let fast_decoder_config = Rc::new(DecoderConfig {
embedding_config: AnyEmbeddingConfig::TiedEmbeddingConfig(config.fast_embeddings_config.clone()),
transformer_config: fast_transformer_config,
vocab_size: config.codebook_size,
- pard_token: None,
ple_model_config: None,
});
let text_decoder_context = B::Context::new().map_err(unable_to_create_context)?;
diff --git a/crates/backend-uzu/src/utils/version.rs b/crates/backend-uzu/src/utils/version.rs
index 0bea2c31a..2c39d1571 100644
--- a/crates/backend-uzu/src/utils/version.rs
+++ b/crates/backend-uzu/src/utils/version.rs
@@ -1,2 +1,2 @@
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-pub const TOOLCHAIN_VERSION: &str = "0.11.0";
+pub const TOOLCHAIN_VERSION: &str = "0.12.1";