Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support lalamo 0.19 config (#1092)

odilf7568c26bec67
diff --git a/crates/legacy/uzu/src/engine/catalog_tests.rs b/crates/legacy/uzu/src/engine/catalog_tests.rs
index 505632bf5..72cfa4d43 100644
--- a/crates/legacy/uzu/src/engine/catalog_tests.rs
+++ b/crates/legacy/uzu/src/engine/catalog_tests.rs
@@ -131,51 +131,13 @@ async fn refresh_models_waits_for_in_flight_catalog_publication() {
}
#[tokio::test]
-async fn chat_model_info_reads_capabilities_and_context_without_weights() {
+async fn chat_model_info_derives_capabilities_from_the_catalog_encoding() {
use shoji::types::{
basic::Value,
model::{ModelSource, ModelSpecialization},
};
- let (engine, directory) = engine().await;
- let norm = serde_json::json!({
- "epsilon": 0.00001, "scale_offset": null, "upcast_mode": "only_normalization",
- "subtract_mean": false, "has_scale": true, "has_biases": false,
- });
- let layer = |length| {
- serde_json::json!({
- "pre_mixer_norm_config": null, "post_mixer_norm_config": null,
- "mixer_conv_config": null, "mlp_conv_config": null, "post_mlp_norm_config": null,
- "hidden_dim": null, "ple_config": null, "has_post_layer_scalar": false,
- "kv_source_layer_index": null, "pre_mlp_norm_config": norm,
- "rope_config": { "type": "UnscaledRoPEConfig", "base": 10000.0, "max_sequence_length": length, "head_dim": 4 },
- "mixer_config": {
- "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {},
- "query_norm_config": null, "key_norm_config": null, "num_heads": 1, "num_groups": 1,
- "head_dim": 4, "is_causal": true, "scale": null, "sliding_window_size": null,
- "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false,
- "has_out_biases": false, "has_gate": false, "normalize_values": false, "is_kv_sharing": false,
- },
- "mlp_config": {
- "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 },
- "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null,
- },
- })
- };
- let config = serde_json::json!({
- "type": "LanguageModelConfig", "token_codec_config": { "type": "RawTextCodecConfig" },
- "decoder_config": {
- "embedding_config": { "type": "TiedEmbeddingConfig", "input_scale": null, "logit_soft_cap": null, "logit_scale": null },
- "transformer_config": { "layer_configs": [layer(32768), layer(16384)], "model_dim": 4, "hidden_dim": 4, "output_norm_config": norm },
- "vocab_size": 4, "ple_model_config": null, "embedding_norm_config": null,
- },
- "generation_config": {
- "stop_token_ids": [], "temperature": null, "top_k": null, "top_p": null, "min_p": null,
- "banned_tokens": null, "repetition_penalty": null, "presence_penalty": null,
- "frequency_penalty": null, "suffix_repetition_length": null,
- },
- });
- std::fs::write(directory.path().join("config.json"), config.to_string()).unwrap();
+ let (engine, _directory) = engine().await;
let model = Model::external(
"metadata-test".into(),
"local".into(),
@@ -185,45 +147,64 @@ async fn chat_model_info_reads_capabilities_and_context_without_weights() {
"1".into(),
vec![ModelSpecialization::Chat {}],
ModelAccessibility::OnDevice {
- source: ModelSource::Filesystem {
- path: directory.path().to_string_lossy().into_owned(),
+ source: ModelSource::Registry {
+ toolchain_version: "1".into(),
+ repository: None,
+ source_repository: None,
+ files: vec![],
},
},
Some(Value::from(serde_json::json!({ "type": "hanashi", "name": "qwen3.5" }))),
);
let info = engine.chat_model_info(&model).await.unwrap().unwrap();
- assert_eq!(info.context_length, Some(16384));
+ assert_eq!(info.context_length, None);
assert!(info.capabilities.supports_tools);
assert!(info.capabilities.supports_reasoning);
assert!(info.capabilities.supports_disable_reasoning);
- let mut unlimited = config;
- unlimited["decoder_config"]["transformer_config"]["layer_configs"] = serde_json::json!([]);
- std::fs::write(directory.path().join("config.json"), unlimited.to_string()).unwrap();
- assert_eq!(engine.chat_model_info(&model).await.unwrap().unwrap().context_length, None);
+}
- let downloadable = Model {
- identifier: "not-downloaded".into(),
- accessibility: ModelAccessibility::OnDevice {
- source: ModelSource::Registry {
- toolchain_version: "1".into(),
- repository: None,
- source_repository: None,
- files: vec![],
+#[tokio::test]
+async fn chat_model_info_returns_none_for_models_without_a_local_uzu_chat_config() {
+ use shoji::types::{
+ basic::Value,
+ model::{ModelSource, ModelSpecialization},
+ };
+
+ let (engine, _directory) = engine().await;
+ let local = |specializations: Vec<ModelSpecialization>, backend: &str| {
+ Model::external(
+ "metadata-test".into(),
+ "local".into(),
+ "Local".into(),
+ backend.into(),
+ "Uzu".into(),
+ "1".into(),
+ specializations,
+ ModelAccessibility::OnDevice {
+ source: ModelSource::Registry {
+ toolchain_version: "1".into(),
+ repository: None,
+ source_repository: None,
+ files: vec![],
+ },
},
- },
- ..model.clone()
+ Some(Value::from(serde_json::json!({ "type": "hanashi", "name": "qwen3.5" }))),
+ )
};
- let pending = engine.chat_model_info(&downloadable).await.unwrap().unwrap();
- assert_eq!(pending.context_length, None);
- assert_eq!(pending.capabilities, info.capabilities);
- assert_eq!(pending.default_reasoning_effort, info.default_reasoning_effort);
let remote = Model {
accessibility: ModelAccessibility::Remote {
repository: None,
},
- ..model
+ ..local(vec![ModelSpecialization::Chat {}], "uzu")
};
assert!(engine.chat_model_info(&remote).await.unwrap().is_none());
+ assert!(
+ engine.chat_model_info(&local(vec![ModelSpecialization::Classification {}], "uzu")).await.unwrap().is_none()
+ );
+ assert!(engine.chat_model_info(&local(vec![ModelSpecialization::Chat {}], "other")).await.unwrap().is_none());
+ let mut unencoded = local(vec![ModelSpecialization::Chat {}], "uzu");
+ unencoded.encoding = None;
+ assert!(engine.chat_model_info(&unencoded).await.is_err());
}
#[tokio::test]
diff --git a/crates/legacy/uzu/src/engine/sampling.rs b/crates/legacy/uzu/src/engine/sampling.rs
index 765a4b7d6..9782685ec 100644
--- a/crates/legacy/uzu/src/engine/sampling.rs
+++ b/crates/legacy/uzu/src/engine/sampling.rs
@@ -58,43 +58,43 @@ mod tests {
use super::*;
#[test]
- fn reads_sampling_defaults_without_weights_and_preserves_disabled_filters() {
- let directory = tempfile::tempdir().unwrap();
- let config = serde_json::json!({
- "type": "LanguageModelConfig",
- "token_codec_config": { "type": "RawTextCodecConfig" },
- "decoder_config": {
- "embedding_config": {
- "type": "TiedEmbeddingConfig", "input_scale": null,
- "logit_soft_cap": null, "logit_scale": null,
- },
- "transformer_config": {
- "layer_configs": [], "model_dim": 4, "hidden_dim": 4,
- "output_norm_config": {
- "epsilon": 0.00001, "scale_offset": null,
- "upcast_mode": "only_normalization", "subtract_mean": false,
- "has_scale": true, "has_biases": false,
- },
- },
- "vocab_size": 4, "ple_model_config": null, "embedding_norm_config": null,
- },
- "generation_config": {
- "stop_token_ids": [], "temperature": null, "top_k": 20, "top_p": null, "min_p": null,
- "banned_tokens": null, "repetition_penalty": null, "presence_penalty": null,
- "frequency_penalty": null, "suffix_repetition_length": null,
- },
- });
- std::fs::write(directory.path().join("config.json"), serde_json::to_vec(&config).unwrap()).unwrap();
+ fn sampling_method_maps_all_fields() {
+ assert_eq!(sampling_method(UzuSamplingMethod::Greedy), SamplingMethod::Greedy {});
assert_eq!(
- futures::executor::block_on(read_sampling_defaults(directory.path())).unwrap(),
- Some(SamplingMethod::Stochastic {
- temperature: None,
+ sampling_method(UzuSamplingMethod::Stochastic {
+ temperature: Some(0.6),
+ top_k: Some(20),
+ top_p: Some(0.95),
+ min_p: Some(0.05),
+ repetition_penalty: Some(1.1),
+ suffix_repetition_length: Some(64),
+ }),
+ SamplingMethod::Stochastic {
+ temperature: Some(0.6),
top_k: Some(20),
+ top_p: Some(0.95),
+ min_p: Some(0.05),
+ repetition_penalty: Some(1.1),
+ suffix_repetition_length: Some(64),
+ }
+ );
+ assert_eq!(
+ sampling_method(UzuSamplingMethod::Stochastic {
+ temperature: None,
+ top_k: None,
+ top_p: None,
+ min_p: None,
+ repetition_penalty: None,
+ suffix_repetition_length: None,
+ }),
+ SamplingMethod::Stochastic {
+ temperature: None,
+ top_k: None,
top_p: None,
min_p: None,
repetition_penalty: None,
suffix_repetition_length: None,
- })
+ }
);
}
diff --git a/crates/uzu-engine/src/config/embedding/mod.rs b/crates/uzu-engine/src/config/embedding/mod.rs
index 893c790ff..2d49a398f 100644
--- a/crates/uzu-engine/src/config/embedding/mod.rs
+++ b/crates/uzu-engine/src/config/embedding/mod.rs
@@ -8,4 +8,5 @@ pub struct EmbeddingConfig {
pub input_scale: Option<f32>,
pub logit_soft_cap: Option<f32>,
pub logit_scale: Option<f32>,
+ pub permute_indices: bool,
}
diff --git a/crates/uzu-engine/src/config/mod.rs b/crates/uzu-engine/src/config/mod.rs
index 7d60d3c95..cbafb9a77 100644
--- a/crates/uzu-engine/src/config/mod.rs
+++ b/crates/uzu-engine/src/config/mod.rs
@@ -11,8 +11,10 @@ pub mod mlp;
pub mod model;
pub mod normalization;
pub mod per_layer_embedding;
+pub mod residual;
pub mod rope;
pub mod speculator;
+pub mod stream_op;
pub mod token_codec;
pub mod token_mixer;
pub mod transformer;
diff --git a/crates/uzu-engine/src/config/per_layer_embedding.rs b/crates/uzu-engine/src/config/per_layer_embedding.rs
index f97f5866b..b4469ad96 100644
--- a/crates/uzu-engine/src/config/per_layer_embedding.rs
+++ b/crates/uzu-engine/src/config/per_layer_embedding.rs
@@ -1,6 +1,6 @@
use uzu_engine_macros::uzu_config;
-use super::{activation::AnyActivation, linear::LinearConfig, normalization::NormalizationConfig};
+use super::{linear::LinearConfig, normalization::NormalizationConfig};
#[uzu_config]
pub struct PLEModelConfig {
@@ -13,11 +13,3 @@ pub struct PLEModelConfig {
pub linear_config: LinearConfig,
pub norm_config: NormalizationConfig,
}
-
-#[uzu_config]
-pub struct PLELayerConfig {
- pub linear_config: LinearConfig,
- pub norm_config: NormalizationConfig,
- pub ple_dim: u32,
- pub activation: AnyActivation,
-}
diff --git a/crates/uzu-engine/src/config/residual/additive.rs b/crates/uzu-engine/src/config/residual/additive.rs
new file mode 100644
index 000000000..8a5731aca
--- /dev/null
+++ b/crates/uzu-engine/src/config/residual/additive.rs
@@ -0,0 +1,5 @@
+use uzu_engine_macros::uzu_config;
+
+#[uzu_config(super::ResidualConfig)]
+#[derive(Default)]
+pub struct AdditiveResidualConfig {}
diff --git a/crates/uzu-engine/src/config/residual/gated.rs b/crates/uzu-engine/src/config/residual/gated.rs
new file mode 100644
index 000000000..1a8e8e745
--- /dev/null
+++ b/crates/uzu-engine/src/config/residual/gated.rs
@@ -0,0 +1,11 @@
+use uzu_engine_macros::uzu_config;
+
+use crate::config::{linear::LinearConfig, normalization::NormalizationConfig};
+
+#[uzu_config(super::ResidualConfig)]
+pub struct GatedResidualConfig {
+ pub stream_count: u32,
+ pub low_rank_dim: u32,
+ pub norm_config: NormalizationConfig,
+ pub linear_config: LinearConfig,
+}
diff --git a/crates/uzu-engine/src/config/residual/mod.rs b/crates/uzu-engine/src/config/residual/mod.rs
new file mode 100644
index 000000000..fea8efdff
--- /dev/null
+++ b/crates/uzu-engine/src/config/residual/mod.rs
@@ -0,0 +1,7 @@
+use uzu_engine_macros::uzu_config_abstract;
+
+pub mod additive;
+pub mod gated;
+
+#[uzu_config_abstract(additive::AdditiveResidualConfig, gated::GatedResidualConfig)]
+pub struct ResidualConfig;
diff --git a/crates/uzu-engine/src/config/speculator/mod.rs b/crates/uzu-engine/src/config/speculator/mod.rs
index e723adc14..51a6914c2 100644
--- a/crates/uzu-engine/src/config/speculator/mod.rs
+++ b/crates/uzu-engine/src/config/speculator/mod.rs
@@ -1,7 +1,6 @@
use uzu_engine_macros::uzu_config_abstract;
pub mod dflash;
-pub mod model;
#[uzu_config_abstract(dflash::DFlashSpeculatorConfig)]
pub struct SpeculatorConfig;
diff --git a/crates/uzu-engine/src/config/speculator/model.rs b/crates/uzu-engine/src/config/speculator/model.rs
deleted file mode 100644
index 4aeddedb9..000000000
--- a/crates/uzu-engine/src/config/speculator/model.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-use uzu_engine_macros::uzu_config;
-
-use crate::config::speculator::AnySpeculatorConfig;
-
-#[uzu_config(super::SpeculatorConfig)]
-pub struct SpeculatorModelConfig {
- pub speculator_config: AnySpeculatorConfig,
-}
diff --git a/crates/uzu-engine/src/config/stream_op/mod.rs b/crates/uzu-engine/src/config/stream_op/mod.rs
new file mode 100644
index 000000000..fb51790be
--- /dev/null
+++ b/crates/uzu-engine/src/config/stream_op/mod.rs
@@ -0,0 +1,12 @@
+use uzu_engine_macros::uzu_config_abstract;
+
+pub mod ngram_injection;
+pub mod per_layer_embedding_injection;
+pub mod residual_scale;
+
+#[uzu_config_abstract(
+ ngram_injection::NGramInjectionConfig,
+ per_layer_embedding_injection::PerLayerEmbeddingInjectionConfig,
+ residual_scale::ResidualScaleConfig
+)]
+pub struct StreamOpConfig;
diff --git a/crates/uzu-engine/src/config/stream_op/ngram_injection.rs b/crates/uzu-engine/src/config/stream_op/ngram_injection.rs
new file mode 100644
index 000000000..4f5f88ac7
--- /dev/null
+++ b/crates/uzu-engine/src/config/stream_op/ngram_injection.rs
@@ -0,0 +1,25 @@
+use uzu_engine_macros::uzu_config;
+
+use crate::config::{
+ linear::LinearConfig, normalization::NormalizationConfig, token_mixer::convolutions::SeparableCausalConvConfig,
+};
+
+#[uzu_config]
+pub struct NGramHashConfig {
+ pub token_vocab_size: u32,
+ pub ngram_size: u32,
+ pub heads_per_ngram: u32,
+ pub eos_token_id: u32,
+ pub multipliers: Box<[u64]>,
+ pub head_vocab_sizes: Box<[u32]>,
+}
+
+#[uzu_config(super::StreamOpConfig)]
+pub struct NGramInjectionConfig {
+ pub hash_config: NGramHashConfig,
+ pub linear_config: LinearConfig,
+ pub norm_config: NormalizationConfig,
+ pub conv_config: SeparableCausalConvConfig,
+ pub embedding_dim: u32,
+ pub conv_kernel_size: u32,
+}
diff --git a/crates/uzu-engine/src/config/stream_op/per_layer_embedding_injection.rs b/crates/uzu-engine/src/config/stream_op/per_layer_embedding_injection.rs
new file mode 100644
index 000000000..c4fae76ee
--- /dev/null
+++ b/crates/uzu-engine/src/config/stream_op/per_layer_embedding_injection.rs
@@ -0,0 +1,11 @@
+use uzu_engine_macros::uzu_config;
+
+use crate::config::{activation::AnyActivation, linear::LinearConfig, normalization::NormalizationConfig};
+
+#[uzu_config(super::StreamOpConfig)]
+pub struct PerLayerEmbeddingInjectionConfig {
+ pub linear_config: LinearConfig,
+ pub norm_config: NormalizationConfig,
+ pub ple_dim: u32,
+ pub activation: AnyActivation,
+}
diff --git a/crates/uzu-engine/src/config/stream_op/residual_scale.rs b/crates/uzu-engine/src/config/stream_op/residual_scale.rs
new file mode 100644
index 000000000..6c505e793
--- /dev/null
+++ b/crates/uzu-engine/src/config/stream_op/residual_scale.rs
@@ -0,0 +1,4 @@
+use uzu_engine_macros::uzu_config;
+
+#[uzu_config(super::StreamOpConfig)]
+pub struct ResidualScaleConfig {}
diff --git a/crates/uzu-engine/src/config/token_codec/chat_codec.rs b/crates/uzu-engine/src/config/token_codec/chat_codec.rs
index 8a90b1bb9..7f66df493 100644
--- a/crates/uzu-engine/src/config/token_codec/chat_codec.rs
+++ b/crates/uzu-engine/src/config/token_codec/chat_codec.rs
@@ -7,17 +7,33 @@ pub struct ReasoningConfig {
pub reasoning_effort_to_field_value: serde_json::Map<String, serde_json::Value>,
}
+#[uzu_config]
+pub struct ResponseFormatConfig {
+ pub reasoning_pattern: Option<String>,
+ pub prefixes: Box<[String]>,
+ turn_pattern: Option<String>,
+ // TODO: This is not respected, we use our own configs (see
+ // `hanashi/configs`). It's just here to parse the format. This should be
+ // unified eventually.
+ tool_calls: serde_json::Value,
+}
+
#[uzu_config(super::TokenCodecConfig)]
pub struct ChatCodecConfig {
pub prompt_template: String,
- pub output_parser_regex: Option<String>,
+ pub response_format: Option<ResponseFormatConfig>,
pub system_role_name: String,
pub user_role_name: String,
pub assistant_role_name: String,
pub eos_token: Option<String>,
pub bos_token: Option<String>,
- pub end_of_thinking_tag: Option<String>,
pub default_system_prompt: Option<String>,
#[serde(default)] // TODO: ??? inconsistency in converted models in registry workaround, remove when resolved
pub reasoning_config: Option<ReasoningConfig>,
}
+
+impl ChatCodecConfig {
+ pub fn end_of_thinking_tag(&self) -> Option<&str> {
+ self.response_format.as_ref()?.prefixes.last().map(String::as_str)
+ }
+}
diff --git a/crates/uzu-engine/src/config/token_codec/mod.rs b/crates/uzu-engine/src/config/token_codec/mod.rs
index a1d48f6b7..2a0ed693f 100644
--- a/crates/uzu-engine/src/config/token_codec/mod.rs
+++ b/crates/uzu-engine/src/config/token_codec/mod.rs
@@ -12,13 +12,16 @@ impl From<&AnyTokenCodecConfig> for ShojiTokenCodecConfig {
match config {
AnyTokenCodecConfig::ChatCodecConfig(config) => ShojiTokenCodecConfig::Chat(ChatTokenCodecConfig {
prompt_template: config.prompt_template.clone(),
- output_parser_regex: config.output_parser_regex.clone(),
+ output_parser_regex: config
+ .response_format
+ .as_ref()
+ .and_then(|format| format.reasoning_pattern.clone()),
system_role_name: config.system_role_name.clone(),
user_role_name: config.user_role_name.clone(),
assistant_role_name: config.assistant_role_name.clone(),
eos_token: config.eos_token.clone(),
bos_token: config.bos_token.clone(),
- end_of_thinking_tag: config.end_of_thinking_tag.clone(),
+ end_of_thinking_tag: config.end_of_thinking_tag().map(str::to_owned),
default_system_prompt: config.default_system_prompt.clone(),
}),
AnyTokenCodecConfig::RawTextCodecConfig(_) => ShojiTokenCodecConfig::RawText,
diff --git a/crates/uzu-engine/src/config/token_mixer/delta_net.rs b/crates/uzu-engine/src/config/token_mixer/delta_net.rs
index cddf1314f..55ac744e1 100644
--- a/crates/uzu-engine/src/config/token_mixer/delta_net.rs
+++ b/crates/uzu-engine/src/config/token_mixer/delta_net.rs
@@ -1,7 +1,8 @@
use uzu_engine_macros::uzu_config;
use crate::config::{
- linear::LinearConfig, normalization::NormalizationConfig, token_mixer::convolutions::SeparableCausalConvConfig,
+ activation::AnyActivation, linear::LinearConfig, normalization::NormalizationConfig,
+ token_mixer::convolutions::SeparableCausalConvConfig,
};
#[uzu_config(super::TokenMixerConfig)]
@@ -16,4 +17,5 @@ pub struct DeltaNetConfig {
pub head_dim: u32,
pub value_head_dim: u32,
pub kernel_size: u32,
+ pub output_gate: AnyActivation,
}
diff --git a/crates/uzu-engine/src/config/token_mixer/mod.rs b/crates/uzu-engine/src/config/token_mixer/mod.rs
index fff58ebd6..71c5fe360 100644
--- a/crates/uzu-engine/src/config/token_mixer/mod.rs
+++ b/crates/uzu-engine/src/config/token_mixer/mod.rs
@@ -4,6 +4,12 @@ pub mod attention;
pub mod convolutions;
pub mod delta_net;
pub mod short_conv;
+pub mod sparse_attention;
-#[uzu_config_abstract(attention::AttentionConfig, delta_net::DeltaNetConfig, short_conv::ShortConvConfig)]
+#[uzu_config_abstract(
+ attention::AttentionConfig,
+ delta_net::DeltaNetConfig,
+ short_conv::ShortConvConfig,
+ sparse_attention::SparseAttentionConfig
+)]
pub struct TokenMixerConfig;
diff --git a/crates/uzu-engine/src/config/token_mixer/sparse_attention.rs b/crates/uzu-engine/src/config/token_mixer/sparse_attention.rs
new file mode 100644
index 000000000..81313de67
--- /dev/null
+++ b/crates/uzu-engine/src/config/token_mixer/sparse_attention.rs
@@ -0,0 +1,22 @@
+use uzu_engine_macros::uzu_config;
+
+use crate::config::{
+ linear::LinearConfig, normalization::NormalizationConfig, token_mixer::attention::AttentionConfig,
+};
+
+#[uzu_config]
+pub struct SparseAttentionIndexerConfig {
+ pub projection_config: LinearConfig,
+ pub query_norm_config: NormalizationConfig,
+ pub key_norm_config: NormalizationConfig,
+ pub num_heads: u32,
+ pub head_dim: u32,
+ pub compression_ratio: u32,
+ pub token_budget: u32,
+}
+
+#[uzu_config(super::TokenMixerConfig)]
+pub struct SparseAttentionConfig {
+ pub attention_config: AttentionConfig,
+ pub indexer_config: SparseAttentionIndexerConfig,
+}
diff --git a/crates/uzu-engine/src/config/transformer.rs b/crates/uzu-engine/src/config/transformer.rs
index 205df98f9..90b176c8d 100644
--- a/crates/uzu-engine/src/config/transformer.rs
+++ b/crates/uzu-engine/src/config/transformer.rs
@@ -1,11 +1,14 @@
use uzu_engine_macros::uzu_config;
-use crate::config::{normalization::NormalizationConfig, transformer_layer::TransformerLayerConfig};
+use crate::config::{
+ normalization::NormalizationConfig, residual::AnyResidualConfig, transformer_layer::TransformerLayerConfig,
+};
#[uzu_config]
pub struct TransformerConfig {
+ pub residual_config: AnyResidualConfig,
pub layer_configs: Box<[TransformerLayerConfig]>,
- pub output_norm_config: NormalizationConfig,
+ pub output_norm_config: Option<NormalizationConfig>,
pub model_dim: u32,
pub hidden_dim: u32,
}
diff --git a/crates/uzu-engine/src/config/transformer_layer.rs b/crates/uzu-engine/src/config/transformer_layer.rs
index 3944b8b95..7ab571697 100644
--- a/crates/uzu-engine/src/config/transformer_layer.rs
+++ b/crates/uzu-engine/src/config/transformer_layer.rs
@@ -4,8 +4,8 @@ use crate::config::{
linear::LinearConfig,
mlp::AnyMLPConfig,
normalization::NormalizationConfig,
- per_layer_embedding::PLELayerConfig,
rope::AnyRoPEConfig,
+ stream_op::AnyStreamOpConfig,
token_mixer::{AnyTokenMixerConfig, convolutions::SeparableCausalConvConfig},
};
@@ -17,19 +17,34 @@ pub struct TransformerLayerConvConfig {
pub conv_group_size: u32,
}
+#[uzu_config]
+pub struct SublayerConfigCommon {
+ pub pre_norm_config: Option<NormalizationConfig>,
+ pub conv_config: Option<TransformerLayerConvConfig>,
+ pub post_norm_config: Option<NormalizationConfig>,
+}
+
+#[uzu_config]
+pub struct MixerSublayerConfig {
+ #[serde(flatten)]
+ pub common: SublayerConfigCommon,
+ pub body_config: AnyTokenMixerConfig,
+}
+
+#[uzu_config]
+pub struct MlpSublayerConfig {
+ #[serde(flatten)]
+ pub common: SublayerConfigCommon,
+ pub body_config: AnyMLPConfig,
+}
+
#[uzu_config]
pub struct TransformerLayerConfig {
- pub pre_mixer_norm_config: Option<NormalizationConfig>,
- pub mixer_config: AnyTokenMixerConfig,
- pub mixer_conv_config: Option<TransformerLayerConvConfig>,
- pub post_mixer_norm_config: Option<NormalizationConfig>,
- pub pre_mlp_norm_config: NormalizationConfig,
- pub mlp_config: AnyMLPConfig,
- pub mlp_conv_config: Option<TransformerLayerConvConfig>,
- pub post_mlp_norm_config: Option<NormalizationConfig>,
+ pub prologue_configs: Box<[AnyStreamOpConfig]>,
+ pub mixer: MixerSublayerConfig,
+ pub mlp: MlpSublayerConfig,
+ pub epilogue_configs: Box<[AnyStreamOpConfig]>,
pub hidden_dim: Option<u32>,
- pub ple_config: Option<PLELayerConfig>,
- pub has_post_layer_scalar: bool,
pub kv_source_layer_index: Option<u32>,
pub rope_config: Option<AnyRoPEConfig>,
}
diff --git a/crates/uzu-engine/src/config/weight_matrix/hybrid_spec.rs b/crates/uzu-engine/src/config/weight_matrix/hybrid_spec.rs
index 332eda59c..631c9056b 100644
--- a/crates/uzu-engine/src/config/weight_matrix/hybrid_spec.rs
+++ b/crates/uzu-engine/src/config/weight_matrix/hybrid_spec.rs
@@ -10,10 +10,17 @@ pub enum IncoherenceProcessingMode {
InputOutput,
}
+#[uzu_config]
+pub struct HadamardSpec {
+ #[serde(rename = "type")]
+ ty: monostate::MustBe!("HadamardSpec"),
+ pub block_size: u32,
+ pub processing_mode: IncoherenceProcessingMode,
+}
+
#[uzu_config(super::WeightMatrixSpec)]
pub struct HybridSpec {
pub quantization_spec: Box<AnyWeightMatrixSpec>,
pub adapter_spec: Option<Unsupported>,
- pub incoherence_block_size: Option<u32>,
- pub incoherence_processing_mode: IncoherenceProcessingMode,
+ pub incoherence_spec: Option<HadamardSpec>,
}
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index ff4b42bd7..09de58301 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -5,7 +5,12 @@ use thiserror::Error;
use crate::{
backends::common::{Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding},
- config::{dflash::DFlashDraftConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig},
+ config::{
+ dflash::DFlashDraftConfig,
+ residual::{AnyResidualConfig, additive::AdditiveResidualConfig},
+ rope::AnyRoPEConfig,
+ token_mixer::AnyTokenMixerConfig,
+ },
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
@@ -121,7 +126,7 @@ impl<B: Backend> DFlash<B> {
.await?;
let layers_tree = parameter_tree.subtree("layers");
let first_layer_config = config.layer_configs.first().expect("DFlash draft model must have at least one layer");
- let AnyTokenMixerConfig::AttentionConfig(attention_config) = &first_layer_config.mixer_config else {
+ let AnyTokenMixerConfig::AttentionConfig(attention_config) = &first_layer_config.mixer.body_config else {
return Err(DFlashNewError::InvalidAttentionConfig("DFlash layers must use attention mixers"));
};
let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;
@@ -135,6 +140,8 @@ impl<B: Backend> DFlash<B> {
context,
)
.await?;
+ // DFlash draft models are hard-coded with an additive residual.
+ let residual_config = AnyResidualConfig::AdditiveResidualConfig(AdditiveResidualConfig::default());
let mut layers = Vec::with_capacity(num_layers as usize);
for (layer_config, index) in config.layer_configs.iter().zip(0u32..) {
layers.push(
@@ -145,6 +152,7 @@ impl<B: Backend> DFlash<B> {
index,
data_type,
layer_config,
+ &residual_config,
&layers_tree.subtree(&index.to_string()),
context,
)
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index 473dfee45..412c1fe32 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -87,6 +87,12 @@ impl<B: Backend> Embedding<B> {
parameter_tree: &ParameterTree<'_, B>,
context: &B::Context,
) -> Result<(Self, Option<Arc<B::GlobalGpuBuffer>>), EmbeddingError<B>> {
+ if *config.permute_indices() {
+ return Err(EmbeddingError::UnsupportedConfiguration(
+ "embeddings with permuted vocabulary indices are not supported".into(),
+ ));
+ }
+
let (tying, readout_input_hadamard_factors) = match config {
AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {
let embedding_tree = parameter_tree.subtree("embedding");
diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rs
index 2ccbba2d7..ef662893c 100644
--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs
@@ -10,7 +10,7 @@ use crate::{
config::weight_matrix::{
AnyWeightMatrixSpec, Layout,
d4s4_spec::D4S4Spec,
- hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
+ hybrid_spec::{HadamardSpec, HybridSpec, IncoherenceProcessingMode},
},
data_type::DataType,
encodable_block::weight_matrix::{WeightMatrix, WeightMatrixError},
@@ -112,8 +112,12 @@ impl<B: Backend> EmbeddingTable<B> {
},
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
quantization_spec,
- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
- incoherence_processing_mode: IncoherenceProcessingMode::Output,
+ incoherence_spec:
+ Some(HadamardSpec {
+ block_size: HADAMARD_TRANSFORM_BLOCK_SIZE,
+ processing_mode: IncoherenceProcessingMode::Output,
+ ..
+ }),
..
}) if embedding_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) => {
let matrix = load_matrix(&tree.subtree("quantized"), *quantization_spec).await?;
diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rs
index ad96f26b5..c5afbe7e2 100644
--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs
@@ -18,7 +18,7 @@ use crate::{
},
config::weight_matrix::{
AnyWeightMatrixSpec,
- hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
+ hybrid_spec::{HadamardSpec, HybridSpec, IncoherenceProcessingMode},
},
data_type::DataType,
safetensors::{ParameterLoaderError, ParameterTree},
@@ -115,10 +115,14 @@ impl<B: Backend> dyn Linear<B> {
Ok(Box::new(block))
},
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
- incoherence_block_size: Some(block_size),
- incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
+ incoherence_spec:
+ Some(HadamardSpec {
+ block_size: HADAMARD_TRANSFORM_BLOCK_SIZE,
+ processing_mode: IncoherenceProcessingMode::InputOutput,
+ ..
+ }),
..
- }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => Ok(Box::new(
+ }) => Ok(Box::new(
RHTLinearWrapper::new(
input_dimension,
output_dimension,
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index fc13ef196..c17059acf 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -9,7 +9,7 @@ use crate::{
},
config::weight_matrix::{
AnyWeightMatrixSpec,
- hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
+ hybrid_spec::{HadamardSpec, HybridSpec, IncoherenceProcessingMode},
},
data_type::DataType,
encodable_block::linear::{
@@ -40,10 +40,13 @@ fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {
matches!(
spec,
AnyWeightMatrixSpec::HybridSpec(HybridSpec {
- incoherence_block_size: Some(block_size),
- incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
+ incoherence_spec: Some(HadamardSpec {
+ block_size: HADAMARD_TRANSFORM_BLOCK_SIZE,
+ processing_mode: IncoherenceProcessingMode::InputOutput,
+ ..
+ }),
..
- }) if *block_size == HADAMARD_TRANSFORM_BLOCK_SIZE
+ })
)
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
index e474d2c10..28e2e4ac7 100644
--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
@@ -6,7 +6,7 @@ use crate::{
},
config::weight_matrix::{
AnyWeightMatrixSpec,
- hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
+ hybrid_spec::{HadamardSpec, HybridSpec, IncoherenceProcessingMode},
},
data_type::DataType,
encodable_block::linear::{
@@ -31,8 +31,12 @@ impl<B: Backend> UntiedReadout<B> {
) -> Result<Self, LinearMatmulError<B>> {
let AnyWeightMatrixSpec::HybridSpec(HybridSpec {
quantization_spec,
- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),
- incoherence_processing_mode: IncoherenceProcessingMode::Input,
+ incoherence_spec:
+ Some(HadamardSpec {
+ block_size: HADAMARD_TRANSFORM_BLOCK_SIZE,
+ processing_mode: IncoherenceProcessingMode::Input,
+ ..
+ }),
..
}) = spec
else {
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
index cd4e1d02a..b618c97e9 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
@@ -12,7 +12,10 @@ use crate::{
delta_net_tree_verify::DeltaNetTreeVerify as DeltaNetTreeVerifyTrait,
},
},
- config::token_mixer::delta_net::DeltaNetConfig,
+ config::{
+ activation::{AnyActivation, silu::SiLU},
+ token_mixer::delta_net::DeltaNetConfig,
+ },
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
@@ -186,6 +189,14 @@ impl<B: Backend> DeltaNet<B> {
)));
}
+ // TODO: Don't hardcode SiLU
+ if config.output_gate != AnyActivation::SiLU(SiLU::default()) {
+ return Err(DeltaNetNewError::UnsupportedConfiguration(format!(
+ "output_gate must be SiLU, got {:?}",
+ config.output_gate.act_type()
+ )));
+ }
+
let key_dim = config.num_groups * config.head_dim;
let value_dim = config.num_heads * config.value_head_dim;
let conv_dim = key_dim * 2 + value_dim;
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
index 7f4deeb7f..2f79474e8 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
@@ -86,6 +86,8 @@ pub enum MixerNewError<B: Backend> {
DeltaNet(#[from] DeltaNetNewError<B>),
#[error("Short conv mixer error: {0}")]
ShortConv(#[from] ShortConvNewError<B>),
+ #[error("Sparse attention mixers are not supported")]
+ SparseAttentionUnsupported,
}
impl<B: Backend> dyn Mixer<B> {
@@ -120,6 +122,7 @@ impl<B: Backend> dyn Mixer<B> {
Ok((Box::new(short_conv), in_projection_input_hadamard_factors))
},
+ AnyTokenMixerConfig::SparseAttentionConfig(_) => Err(MixerNewError::SparseAttentionUnsupported),
}
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rs
index baa5ab656..1d7e1efb5 100644
--- a/crates/uzu-engine/src/encodable_block/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mod.rs
@@ -13,6 +13,7 @@ pub mod mlp;
pub mod normalization;
pub mod per_layer_embedding;
pub mod prediction_head;
+pub mod residual;
pub mod sampling;
pub mod transformer;
pub mod transformer_layer;
diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
index b8f8afd99..563cccc37 100644
--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
@@ -7,8 +7,8 @@ use crate::{
kernel::{GatedActMul, GatedActMulSettings, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},
},
config::{
- activation::AnyActivation,
- per_layer_embedding::{PLELayerConfig, PLEModelConfig},
+ activation::AnyActivation, per_layer_embedding::PLEModelConfig,
+ stream_op::per_layer_embedding_injection::PerLayerEmbeddingInjectionConfig,
},
data_type::DataType,
encodable_block::{
@@ -182,7 +182,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
num_layers: u32,
data_type: DataType,
post_layer_scalar: f32,
- config: &PLELayerConfig,
+ config: &PerLayerEmbeddingInjectionConfig,
parameter_tree: &ParameterTree<'_, B>,
context: &B::Context,
) -> Result<Self, PerLayerEmbeddingError<B>> {
diff --git a/crates/uzu-engine/src/encodable_block/residual.rs b/crates/uzu-engine/src/encodable_block/residual.rs
new file mode 100644
index 000000000..0215ab27d
--- /dev/null
+++ b/crates/uzu-engine/src/encodable_block/residual.rs
@@ -0,0 +1,117 @@
+use thiserror::Error;
+
+use crate::{
+ backends::common::{Backend, BufferRef, CommandBuffer, Kernels, kernel::TensorAddScaleKernel},
+ config::residual::AnyResidualConfig,
+ data_type::DataType,
+ safetensors::ParameterTree,
+};
+
+#[derive(Debug, Error)]
+pub enum ResidualNewError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("Gated residual connections are not supported yet")]
+ GatedUnsupported,
+}
+
+pub enum ResidualConnection<B: Backend> {
+ Additive(AdditiveResidualConnection<B>),
+ // TODO: Gated(GatedResidualConnection<B>)
+}
+
+pub struct AdditiveResidualConnection<B: Backend> {
+ residual_add: <B::Kernels as Kernels>::TensorAddScaleKernel,
+}
+
+impl<B: Backend> AdditiveResidualConnection<B> {
+ fn new(
+ context: &B::Context,
+ data_type: DataType,
+ ) -> Result<Self, B::Error> {
+ Ok(Self {
+ residual_add: <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, data_type, false)?,
+ })
+ }
+
+ fn encode_update(
+ &self,
+ stream: impl BufferRef<Backend = B>,
+ update: impl BufferRef<Backend = B>,
+ output: &mut B::ScratchBuffer,
+ elements: u32,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
+ self.residual_add.encode(Some(stream), update, output, elements, elements, 1.0, command_buffer);
+ Ok(())
+ }
+}
+
+impl<B: Backend> ResidualConnection<B> {
+ pub fn new(
+ config: &AnyResidualConfig,
+ data_type: DataType,
+ parameter_tree: &ParameterTree<'_, B>,
+ context: &B::Context,
+ ) -> Result<Self, ResidualNewError<B>> {
+ match config {
+ AnyResidualConfig::AdditiveResidualConfig(_) => {
+ // TODO: Load weights for gated residuals.
+ let _ = parameter_tree;
+ Ok(Self::Additive(
+ AdditiveResidualConnection::new(context, data_type).map_err(ResidualNewError::Backend)?,
+ ))
+ },
+ AnyResidualConfig::GatedResidualConfig(_) => Err(ResidualNewError::GatedUnsupported),
+ }
+ }
+
+ pub fn encode_update(
+ &self,
+ stream: impl BufferRef<Backend = B>,
+ update: impl BufferRef<Backend = B>,
+ output: &mut B::ScratchBuffer,
+ elements: u32,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
+ match self {
+ Self::Additive(connection) => connection.encode_update(stream, update, output, elements, command_buffer),
+ }
+ }
+}
+
+pub struct ResidualHead<B: Backend> {
+ connection: ResidualConnection<B>,
+}
+
+impl<B: Backend> ResidualHead<B> {
+ pub fn new(
+ config: &AnyResidualConfig,
+ data_type: DataType,
+ parameter_tree: &ParameterTree<'_, B>,
+ context: &B::Context,
+ ) -> Result<Self, ResidualNewError<B>> {
+ Ok(Self {
+ connection: ResidualConnection::new(config, data_type, parameter_tree, context)?,
+ })
+ }
+
+ // TODO: gated repeats the embeddings `stream_count` times
+ pub fn expand(
+ &self,
+ embeddings: B::ScratchBuffer,
+ ) -> B::ScratchBuffer {
+ embeddings
+ }
+
+ pub fn encode_update(
+ &self,
+ stream: impl BufferRef<Backend = B>,
+ update: impl BufferRef<Backend = B>,
+ output: &mut B::ScratchBuffer,
+ elements: u32,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
+ self.connection.encode_update(stream, update, output, elements, command_buffer)
+ }
+}
diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rs
index 0966c0a05..dc04f29c5 100644
--- a/crates/uzu-engine/src/encodable_block/transformer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer.rs
@@ -4,15 +4,14 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- backends::common::{
- Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, kernel::TensorAddScaleKernel,
- },
+ backends::common::{Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding},
config::{rope::AnyRoPEConfig, transformer::TransformerConfig},
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
mixer::{MixerState, attention::rope::PrecalculatedRoPE},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
+ residual::{ResidualHead, ResidualNewError},
transformer_layer::{TransformerLayer, TransformerLayerError},
},
safetensors::ParameterTree,
@@ -88,6 +87,10 @@ pub enum TransformerNewError<B: Backend> {
Layer(#[from] TransformerLayerError<B>),
#[error("Normalization error: {0}")]
Normalization(#[from] NormalizationNewError<B>),
+ #[error("Residual error: {0}")]
+ Residual(#[from] ResidualNewError<B>),
+ #[error("Readout expects RHT-rotated input but there is no output norm to apply it")]
+ UnappliedReadoutRht,
}
#[derive(Debug, Error)]
@@ -101,9 +104,9 @@ pub enum TransformerEncodeError<B: Backend> {
pub struct Transformer<B: Backend> {
ropes: Box<[AnyRoPEConfig]>,
layers: Box<[(TransformerLayer<B>, Option<usize>)]>,
- output_norm: Normalization<B>,
+ output_norm: Option<Normalization<B>>,
model_dim: u32,
- residual_add: <B::Kernels as Kernels>::TensorAddScaleKernel,
+ residual_head: ResidualHead<B>,
}
impl<B: Backend> Transformer<B> {
@@ -117,6 +120,10 @@ impl<B: Backend> Transformer<B> {
) -> Result<Self, TransformerNewError<B>> {
let mut ropes: Vec<AnyRoPEConfig> = Vec::new();
+ let residual_config = transformer_config.residual_config.clone();
+ let residual_head =
+ ResidualHead::new(&residual_config, input_data_type, &parameter_tree.subtree("residual_head"), context)?;
+
let num_layers = transformer_config.layer_configs.len() as u32;
let mut layers = Vec::with_capacity(num_layers as usize);
for (layer_config, layer_index) in transformer_config.layer_configs.iter().zip(0u32..) {
@@ -136,6 +143,7 @@ impl<B: Backend> Transformer<B> {
layer_index,
input_data_type,
layer_config,
+ &residual_config,
&layer_loader,
context,
)
@@ -144,28 +152,31 @@ impl<B: Backend> Transformer<B> {
layers.push((layer, rope));
}
- let output_norm = Normalization::new(
- transformer_config.model_dim,
- input_data_type,
- output_data_type,
- output_norm_hadamard_factors,
- ShortcutMode::Add,
- PostLayerScalar::None,
- &transformer_config.output_norm_config,
- &parameter_tree.subtree("output_norm"),
- context,
- )
- .await?;
-
- let residual_add = <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, input_data_type, false)
- .map_err(TransformerNewError::Backend)?;
+ let output_norm = match (&transformer_config.output_norm_config, output_norm_hadamard_factors) {
+ (Some(output_norm_config), hadamard_factors) => Some(
+ Normalization::new(
+ transformer_config.model_dim,
+ input_data_type,
+ output_data_type,
+ hadamard_factors,
+ ShortcutMode::Add,
+ PostLayerScalar::None,
+ output_norm_config,
+ &parameter_tree.subtree("output_norm"),
+ context,
+ )
+ .await?,
+ ),
+ (None, Some(_)) => return Err(TransformerNewError::UnappliedReadoutRht),
+ (None, None) => None,
+ };
Ok(Self {
ropes: ropes.into_boxed_slice(),
layers: layers.into_boxed_slice(),
output_norm,
model_dim: transformer_config.model_dim,
- residual_add,
+ residual_head,
})
}
@@ -178,7 +189,7 @@ impl<B: Backend> Transformer<B> {
) -> Result<B::ScratchBuffer, B::Error> {
let mut output = command_buffer.allocate_scratch(hidden.size())?;
let elements = batch_size * self.model_dim;
- self.residual_add.encode(Some(shortcut), hidden, &mut output, elements, elements, 1.0, command_buffer);
+ self.residual_head.encode_update(shortcut, hidden, &mut output, elements, command_buffer)?;
Ok(output)
}
@@ -249,7 +260,8 @@ impl<B: Backend> Transformer<B> {
mut state: Option<&mut TransformerState<B>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<TransformerEncodeOutput<B>, TransformerEncodeError<B>> {
- let mut hidden = input;
+ // TODO: change shortcut for residual
+ let mut hidden = self.residual_head.expand(input);
let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_none() {
self.prefill_cache_layer_count()
} else {
@@ -333,19 +345,25 @@ impl<B: Backend> Transformer<B> {
});
};
- let output_normalized = self
- .output_norm
- .encode(
- &hidden,
- output_range.start,
- output_range.end - output_range.start,
- Some(&mut shortcut),
- command_buffer,
- )
- .map_err(TransformerEncodeError::Backend)?;
+ // TODO: route the final combine through `residual_head.readout` (stream + last update)
+ // instead of the ShortcutMode::Add fused into the output norm.
+ let output = match &self.output_norm {
+ Some(output_norm) => output_norm
+ .encode(
+ &hidden,
+ output_range.start,
+ output_range.end - output_range.start,
+ Some(&mut shortcut),
+ command_buffer,
+ )
+ .map_err(TransformerEncodeError::Backend)?,
+ None => self
+ .capture_residual(&shortcut, &hidden, batch_dim.size(), command_buffer)
+ .map_err(TransformerEncodeError::Backend)?,
+ };
Ok(TransformerEncodeOutput {
- output: Some(output_normalized),
+ output: Some(output),
hidden_features,
})
}
diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
index 6016db49c..1fa7cc133 100644
--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
@@ -8,7 +8,11 @@ use crate::{
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, kernel::ActivationTransform,
},
- config::transformer_layer::{TransformerLayerConfig, TransformerLayerConvConfig},
+ config::{
+ residual::AnyResidualConfig,
+ stream_op::AnyStreamOpConfig,
+ transformer_layer::{TransformerLayerConfig, TransformerLayerConvConfig},
+ },
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
@@ -18,6 +22,7 @@ use crate::{
mlp::{Mlp, MlpBlockNewError, MlpEncodeError},
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
per_layer_embedding::{PerLayerEmbeddingError, PerLayerEmbeddingProjection},
+ residual::{ResidualConnection, ResidualNewError},
},
safetensors::{ParameterLoaderError, ParameterTree},
utils::{dispatch_dtype::dispatch_dtype, maybe_mut::MaybeMut},
@@ -47,12 +52,34 @@ pub enum TransformerLayerError<B: Backend> {
LinearEncode(#[from] LinearEncodeError<B>),
#[error("Per-layer embedding error: {0}")]
PerLayerEmbedding(#[from] PerLayerEmbeddingError<B>),
- #[error("Layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]
- PostLayerScalarWithoutPostMlpNorm {
+ #[error("Residual error: {0}")]
+ Residual(#[from] ResidualNewError<B>),
+ #[error("Layer {layer_index} sets a residual scale op but has no mlp post_norm")]
+ ResidualScaleWithoutPostMlpNorm {
+ layer_index: u32,
+ },
+ #[error("Layer {layer_index} uses prologue stream ops, which are not supported")]
+ UnsupportedPrologueOps {
+ layer_index: u32,
+ },
+ #[error("Layer {layer_index} uses an n-gram injection op, which is not supported")]
+ UnsupportedNGramInjection {
layer_index: u32,
},
- #[error("Transformer layers except the first one if it doesn't have rht in mixer require pre_mixer_norm_config")]
+ #[error("Layer {layer_index} has multiple per-layer embedding injection ops, but only one is applied")]
+ MultiplePerLayerEmbeddingInjectionOps {
+ layer_index: u32,
+ },
+ #[error("Layer {layer_index} has multiple residual scale ops, but only one is applied")]
+ MultipleResidualScaleOps {
+ layer_index: u32,
+ },
+ #[error("Transformer layers except the first one if it doesn't have rht in mixer require a mixer pre_norm")]
MissingPreMixerNormConfig,
+ #[error("Layer {layer_index} requires an mlp pre_norm")]
+ MissingPreMlpNormConfig {
+ layer_index: u32,
+ },
}
struct TransformerLayerConv<B: Backend> {
@@ -64,8 +91,6 @@ struct TransformerLayerConv<B: Backend> {
coefficient_count: u32,
}
-// TODO: saner shortcut
-
pub struct TransformerLayer<B: Backend> {
pub layer_index: u32,
pub kv_source_layer_index: Option<u32>,
@@ -78,6 +103,12 @@ pub struct TransformerLayer<B: Backend> {
pub mlp: Box<dyn Mlp<B>>,
pub post_mlp_norm: Option<Normalization<B>>,
pub ple_projection: Option<PerLayerEmbeddingProjection<B>>,
+ // TODO: read these in `encode` once norm kernels stop fusing the residual add (ShortcutMode):
+ // the stream update becomes `self.mixer_connection.encode_update(...)` / `mlp_connection`.
+ #[allow(dead_code)]
+ mixer_connection: ResidualConnection<B>,
+ #[allow(dead_code)]
+ mlp_connection: ResidualConnection<B>,
}
impl<B: Backend> TransformerLayerConv<B> {
@@ -183,16 +214,55 @@ impl<B: Backend> TransformerLayer<B> {
layer_index: u32,
data_type: DataType,
layer_config: &TransformerLayerConfig,
+ residual_config: &AnyResidualConfig,
parameter_tree: &ParameterTree<'_, B>,
context: &B::Context,
) -> Result<Self, TransformerLayerError<B>> {
- let post_layer_scalar = if layer_config.has_post_layer_scalar {
- if layer_config.post_mlp_norm_config.is_none() {
- return Err(TransformerLayerError::PostLayerScalarWithoutPostMlpNorm {
+ if !layer_config.prologue_configs.is_empty() {
+ return Err(TransformerLayerError::UnsupportedPrologueOps {
+ layer_index,
+ });
+ }
+
+ // TODO: Apply epilogue ops as a sequence properly.
+ let mut ple_op = None;
+ let mut residual_scale_op_index = None;
+ for (index, op) in layer_config.epilogue_configs.iter().enumerate() {
+ match op {
+ AnyStreamOpConfig::NGramInjectionConfig(_) => {
+ return Err(TransformerLayerError::UnsupportedNGramInjection {
+ layer_index,
+ });
+ },
+ AnyStreamOpConfig::PerLayerEmbeddingInjectionConfig(config) => {
+ if ple_op.replace((index, config)).is_some() {
+ return Err(TransformerLayerError::MultiplePerLayerEmbeddingInjectionOps {
+ layer_index,
+ });
+ }
+ },
+ AnyStreamOpConfig::ResidualScaleConfig(_) => {
+ if residual_scale_op_index.replace(index).is_some() {
+ return Err(TransformerLayerError::MultipleResidualScaleOps {
+ layer_index,
+ });
+ }
+ },
+ }
+ }
+
+ let post_layer_scalar = if let Some(op_index) = residual_scale_op_index {
+ if layer_config.mlp.common.post_norm_config.is_none() {
+ return Err(TransformerLayerError::ResidualScaleWithoutPostMlpNorm {
layer_index,
});
}
- let scalar = dispatch_dtype!(|(T: data_type)| parameter_tree.read_slice::<T>("post_layer_scalar", &[1]).await?[0].as_());
+ let scalar = dispatch_dtype!(|(T: data_type)| parameter_tree
+ .subtree("epilogue")
+ .subtree(&op_index.to_string())
+ .read_slice::<T>("scale", &[1])
+ .await?[0]
+ .as_());
Some(scalar)
} else {
None
@@ -200,7 +270,7 @@ impl<B: Backend> TransformerLayer<B> {
// When a PLE projection is present it owns the post-layer scalar (applied
// to the combined residual), so the norms must not also apply it.
- let (residual_sum_scalar, output_scalar) = match (post_layer_scalar, layer_config.ple_config.is_none()) {
+ let (residual_sum_scalar, output_scalar) = match (post_layer_scalar, ple_op.is_none()) {
(Some(scalar), true) => (PostLayerScalar::ScaleResidualSum(scalar), PostLayerScalar::ScaleOutput(scalar)),
_ => (PostLayerScalar::None, PostLayerScalar::None),
};
@@ -209,13 +279,13 @@ impl<B: Backend> TransformerLayer<B> {
model_dim,
data_type,
layer_config.rope_config.as_ref(),
- &layer_config.mixer_config,
- &parameter_tree.subtree("mixer"),
+ &layer_config.mixer.body_config,
+ &parameter_tree.subtree("mixer").subtree("body"),
context,
)
.await?;
- let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer_conv_config {
+ let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer.common.conv_config {
Some(config) => (
Some(
TransformerLayerConv::new(
@@ -223,7 +293,7 @@ impl<B: Backend> TransformerLayer<B> {
data_type,
mixer_hadamard_factors,
config,
- &parameter_tree.subtree("mixer_conv"),
+ &parameter_tree.subtree("mixer").subtree("conv"),
context,
)
.await?,
@@ -233,7 +303,7 @@ impl<B: Backend> TransformerLayer<B> {
None => (None, mixer_hadamard_factors),
};
- let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {
+ let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.mixer.common.pre_norm_config {
Some(
Normalization::new(
model_dim,
@@ -247,7 +317,7 @@ impl<B: Backend> TransformerLayer<B> {
},
PostLayerScalar::None,
pre_mixer_norm_config,
- &parameter_tree.subtree("pre_mixer_norm"),
+ &parameter_tree.subtree("mixer").subtree("pre_norm"),
context,
)
.await?,
@@ -259,7 +329,7 @@ impl<B: Backend> TransformerLayer<B> {
None
};
- let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {
+ let post_mixer_norm = if let Some(norm_config) = &layer_config.mixer.common.post_norm_config {
Some(
Normalization::new(
model_dim,
@@ -269,7 +339,7 @@ impl<B: Backend> TransformerLayer<B> {
ShortcutMode::None,
PostLayerScalar::None,
norm_config,
- &parameter_tree.subtree("post_mixer_norm"),
+ &parameter_tree.subtree("mixer").subtree("post_norm"),
context,
)
.await?,
@@ -282,13 +352,13 @@ impl<B: Backend> TransformerLayer<B> {
model_dim,
layer_config.hidden_dim.unwrap_or(hidden_dim),
data_type,
- &layer_config.mlp_config,
- &parameter_tree.subtree("mlp"),
+ &layer_config.mlp.body_config,
+ &parameter_tree.subtree("mlp").subtree("body"),
context,
)
.await?;
- let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp_conv_config {
+ let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp.common.conv_config {
Some(config) => (
Some(
TransformerLayerConv::new(
@@ -296,7 +366,7 @@ impl<B: Backend> TransformerLayer<B> {
data_type,
mlp_input_hadamard_factors,
config,
- &parameter_tree.subtree("mlp_conv"),
+ &parameter_tree.subtree("mlp").subtree("conv"),
context,
)
.await?,
@@ -306,6 +376,11 @@ impl<B: Backend> TransformerLayer<B> {
None => (None, mlp_input_hadamard_factors),
};
+ let Some(pre_mlp_norm_config) = &layer_config.mlp.common.pre_norm_config else {
+ return Err(TransformerLayerError::MissingPreMlpNormConfig {
+ layer_index,
+ });
+ };
let pre_mlp_norm = Normalization::new(
model_dim,
data_type,
@@ -313,13 +388,13 @@ impl<B: Backend> TransformerLayer<B> {
mlp_input_hadamard_factors.map(Arc::new),
ShortcutMode::Add,
residual_sum_scalar,
- &layer_config.pre_mlp_norm_config,
- &parameter_tree.subtree("pre_mlp_norm"),
+ pre_mlp_norm_config,
+ &parameter_tree.subtree("mlp").subtree("pre_norm"),
context,
)
.await?;
- let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
+ let post_mlp_norm = if let Some(norm_config) = &layer_config.mlp.common.post_norm_config {
Some(
Normalization::new(
model_dim,
@@ -329,7 +404,7 @@ impl<B: Backend> TransformerLayer<B> {
ShortcutMode::None,
output_scalar,
norm_config,
- &parameter_tree.subtree("post_mlp_norm"),
+ &parameter_tree.subtree("mlp").subtree("post_norm"),
context,
)
.await?,
@@ -338,8 +413,13 @@ impl<B: Backend> TransformerLayer<B> {
None
};
- let ple_projection = if let Some(ple_config) = &layer_config.ple_config {
- let ple_loader = parameter_tree.subtree("ple");
+ let mixer_connection =
+ ResidualConnection::new(residual_config, data_type, &parameter_tree.subtree("mixer.connection"), context)?;
+ let mlp_connection =
+ ResidualConnection::new(residual_config, data_type, &parameter_tree.subtree("mlp.connection"), context)?;
+
+ let ple_projection = if let Some((op_index, ple_config)) = ple_op {
+ let ple_loader = parameter_tree.subtree("epilogue").subtree(&op_index.to_string());
Some(
PerLayerEmbeddingProjection::new(
model_dim,
@@ -369,6 +449,8 @@ impl<B: Backend> TransformerLayer<B> {
mlp,
post_mlp_norm,
ple_projection,
+ mixer_connection,
+ mlp_connection,
})
}
@@ -419,6 +501,8 @@ impl<B: Backend> TransformerLayer<B> {
.map_err(TransformerLayerError::Backend)?;
}
+ // TODO: the mixer update is currently folded into the stream by the ShortcutMode::Add
+ // fused into this norm kernel; replace with an explicit mixer_connection.encode_update.
hidden = self
.pre_mlp_norm
.encode(&hidden, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)
@@ -448,6 +532,9 @@ impl<B: Backend> TransformerLayer<B> {
.map_err(TransformerLayerError::Backend)?;
}
+ // TODO: the MLP update is folded into the stream by the next layer's pre-norm (or the
+ // output norm); the layer should return `mlp_connection.encode_update(shortcut, hidden)`
+ // so the stream is passed by value, like lalamo's Sublayer does.
if let Some(ple_projection) = &self.ple_projection {
let per_layer_inputs = per_layer_inputs.expect("per-layer inputs required for PLE layer");
ple_projection.encode(
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index 420919bf4..e4ea16421 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -99,7 +99,9 @@ impl<B: Backend> Engine<B> {
let generation_config = config.generation_config;
let end_of_thinking_tag = match &config.token_codec_config {
- AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) => token_codec_config.end_of_thinking_tag.clone(),
+ AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) => {
+ token_codec_config.end_of_thinking_tag().map(str::to_owned)
+ },
AnyTokenCodecConfig::RawTextCodecConfig(_) => None,
};
diff --git a/crates/uzu-engine/src/engine/mod.rs b/crates/uzu-engine/src/engine/mod.rs
index e9593328e..40cc8c6e7 100644
--- a/crates/uzu-engine/src/engine/mod.rs
+++ b/crates/uzu-engine/src/engine/mod.rs
@@ -85,7 +85,12 @@ pub async fn resolve_model_context_length(model_path: &Path) -> Result<Option<u3
Ok(transformer
.layer_configs
.iter()
- .filter(|layer| matches!(layer.mixer_config, AnyTokenMixerConfig::AttentionConfig(_)))
+ .filter(|layer| {
+ matches!(
+ layer.mixer.body_config,
+ AnyTokenMixerConfig::AttentionConfig(_) | AnyTokenMixerConfig::SparseAttentionConfig(_)
+ )
+ })
.filter_map(|layer| layer.rope_config.as_ref().map(|rope| *rope.max_sequence_length()))
.min())
}
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index 812ebe301..fff097ae1 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -14,7 +14,7 @@ use crate::{
backends::common::{
Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,
},
- config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},
+ config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig},
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
@@ -115,11 +115,11 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let data_type = DataType::BF16;
- let config: SpeculatorModelConfig = serde_json::from_slice(&file::read(model_path.join("config.json")).await?)?;
- let AnySpeculatorConfig::DFlashSpeculatorConfig(config) = config.speculator_config;
+ let AnySpeculatorConfig::DFlashSpeculatorConfig(config) =
+ serde_json::from_slice(&file::read(model_path.join("config.json")).await?)?;
let weight_loader = ParameterLoader::new(model_path.join("model.safetensors"), &*context).await?;
- let speculator_tree = weight_loader.tree().subtree("speculator");
+ let speculator_tree = weight_loader.tree();
let dflash = DFlash::new(
data_type,
diff --git a/crates/uzu-engine/src/utils/strict_serde.rs b/crates/uzu-engine/src/utils/strict_serde.rs
index 695f4e44c..969bf246d 100644
--- a/crates/uzu-engine/src/utils/strict_serde.rs
+++ b/crates/uzu-engine/src/utils/strict_serde.rs
@@ -14,7 +14,17 @@ macro_rules! impl_strict {
};
}
-impl_strict!(Unsupported, String, f32, u32, u64, usize, bool, serde_json::Map<String, serde_json::Value>);
+impl_strict!(
+ Unsupported,
+ String,
+ f32,
+ u32,
+ u64,
+ usize,
+ bool,
+ serde_json::Value,
+ serde_json::Map<String, serde_json::Value>
+);
impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Box<T> {}
impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Box<[T]> {}
diff --git a/crates/uzu-engine/src/utils/version.rs b/crates/uzu-engine/src/utils/version.rs
index 03d9fafa0..15aa365d2 100644
--- a/crates/uzu-engine/src/utils/version.rs
+++ b/crates/uzu-engine/src/utils/version.rs
@@ -1,2 +1,2 @@
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-pub const TOOLCHAIN_VERSION: &str = "0.17.2";
+pub const TOOLCHAIN_VERSION: &str = "0.19.0";