A high-performance inference engine for AI models
Support lalamo 0.19 config (#1092)
odilf7568c26bec67
diff --git a/crates/legacy/uzu/src/engine/catalog_tests.rs b/crates/legacy/uzu/src/engine/catalog_tests.rsindex 505632bf5..72cfa4d43 100644--- a/crates/legacy/uzu/src/engine/catalog_tests.rs+++ b/crates/legacy/uzu/src/engine/catalog_tests.rs@@ -131,51 +131,13 @@ async fn refresh_models_waits_for_in_flight_catalog_publication() {}#[tokio::test]-async fn chat_model_info_reads_capabilities_and_context_without_weights() {+async fn chat_model_info_derives_capabilities_from_the_catalog_encoding() {use shoji::types::{basic::Value,model::{ModelSource, ModelSpecialization},};- let (engine, directory) = engine().await;- let norm = serde_json::json!({- "epsilon": 0.00001, "scale_offset": null, "upcast_mode": "only_normalization",- "subtract_mean": false, "has_scale": true, "has_biases": false,- });- let layer = |length| {- serde_json::json!({- "pre_mixer_norm_config": null, "post_mixer_norm_config": null,- "mixer_conv_config": null, "mlp_conv_config": null, "post_mlp_norm_config": null,- "hidden_dim": null, "ple_config": null, "has_post_layer_scalar": false,- "kv_source_layer_index": null, "pre_mlp_norm_config": norm,- "rope_config": { "type": "UnscaledRoPEConfig", "base": 10000.0, "max_sequence_length": length, "head_dim": 4 },- "mixer_config": {- "type": "AttentionConfig", "qkvg_projection_config": {}, "out_projection_config": {},- "query_norm_config": null, "key_norm_config": null, "num_heads": 1, "num_groups": 1,- "head_dim": 4, "is_causal": true, "scale": null, "sliding_window_size": null,- "logit_soft_cap": null, "has_sinks": false, "has_qkvg_biases": false,- "has_out_biases": false, "has_gate": false, "normalize_values": false, "is_kv_sharing": false,- },- "mlp_config": {- "type": "DenseMLPConfig", "linear_config": {}, "activation": { "type": "SiLU", "alpha": 1.0 },- "has_up_biases": false, "has_down_biases": false, "gate_clipping": null, "up_clipping": null,- },- })- };- let config = serde_json::json!({- "type": "LanguageModelConfig", "token_codec_config": { "type": "RawTextCodecConfig" },- "decoder_config": {- "embedding_config": { "type": "TiedEmbeddingConfig", "input_scale": null, "logit_soft_cap": null, "logit_scale": null },- "transformer_config": { "layer_configs": [layer(32768), layer(16384)], "model_dim": 4, "hidden_dim": 4, "output_norm_config": norm },- "vocab_size": 4, "ple_model_config": null, "embedding_norm_config": null,- },- "generation_config": {- "stop_token_ids": [], "temperature": null, "top_k": null, "top_p": null, "min_p": null,- "banned_tokens": null, "repetition_penalty": null, "presence_penalty": null,- "frequency_penalty": null, "suffix_repetition_length": null,- },- });- std::fs::write(directory.path().join("config.json"), config.to_string()).unwrap();+ let (engine, _directory) = engine().await;let model = Model::external("metadata-test".into(),"local".into(),@@ -185,45 +147,64 @@ async fn chat_model_info_reads_capabilities_and_context_without_weights() {"1".into(),vec![ModelSpecialization::Chat {}],ModelAccessibility::OnDevice {- source: ModelSource::Filesystem {- path: directory.path().to_string_lossy().into_owned(),+ source: ModelSource::Registry {+ toolchain_version: "1".into(),+ repository: None,+ source_repository: None,+ files: vec![],},},Some(Value::from(serde_json::json!({ "type": "hanashi", "name": "qwen3.5" }))),);let info = engine.chat_model_info(&model).await.unwrap().unwrap();- assert_eq!(info.context_length, Some(16384));+ assert_eq!(info.context_length, None);assert!(info.capabilities.supports_tools);assert!(info.capabilities.supports_reasoning);assert!(info.capabilities.supports_disable_reasoning);- let mut unlimited = config;- unlimited["decoder_config"]["transformer_config"]["layer_configs"] = serde_json::json!([]);- std::fs::write(directory.path().join("config.json"), unlimited.to_string()).unwrap();- assert_eq!(engine.chat_model_info(&model).await.unwrap().unwrap().context_length, None);+}- let downloadable = Model {- identifier: "not-downloaded".into(),- accessibility: ModelAccessibility::OnDevice {- source: ModelSource::Registry {- toolchain_version: "1".into(),- repository: None,- source_repository: None,- files: vec![],+#[tokio::test]+async fn chat_model_info_returns_none_for_models_without_a_local_uzu_chat_config() {+ use shoji::types::{+ basic::Value,+ model::{ModelSource, ModelSpecialization},+ };++ let (engine, _directory) = engine().await;+ let local = |specializations: Vec<ModelSpecialization>, backend: &str| {+ Model::external(+ "metadata-test".into(),+ "local".into(),+ "Local".into(),+ backend.into(),+ "Uzu".into(),+ "1".into(),+ specializations,+ ModelAccessibility::OnDevice {+ source: ModelSource::Registry {+ toolchain_version: "1".into(),+ repository: None,+ source_repository: None,+ files: vec![],+ },},- },- ..model.clone()+ Some(Value::from(serde_json::json!({ "type": "hanashi", "name": "qwen3.5" }))),+ )};- let pending = engine.chat_model_info(&downloadable).await.unwrap().unwrap();- assert_eq!(pending.context_length, None);- assert_eq!(pending.capabilities, info.capabilities);- assert_eq!(pending.default_reasoning_effort, info.default_reasoning_effort);let remote = Model {accessibility: ModelAccessibility::Remote {repository: None,},- ..model+ ..local(vec![ModelSpecialization::Chat {}], "uzu")};assert!(engine.chat_model_info(&remote).await.unwrap().is_none());+ assert!(+ engine.chat_model_info(&local(vec![ModelSpecialization::Classification {}], "uzu")).await.unwrap().is_none()+ );+ assert!(engine.chat_model_info(&local(vec![ModelSpecialization::Chat {}], "other")).await.unwrap().is_none());+ let mut unencoded = local(vec![ModelSpecialization::Chat {}], "uzu");+ unencoded.encoding = None;+ assert!(engine.chat_model_info(&unencoded).await.is_err());}#[tokio::test]diff --git a/crates/legacy/uzu/src/engine/sampling.rs b/crates/legacy/uzu/src/engine/sampling.rsindex 765a4b7d6..9782685ec 100644--- a/crates/legacy/uzu/src/engine/sampling.rs+++ b/crates/legacy/uzu/src/engine/sampling.rs@@ -58,43 +58,43 @@ mod tests {use super::*;#[test]- fn reads_sampling_defaults_without_weights_and_preserves_disabled_filters() {- let directory = tempfile::tempdir().unwrap();- let config = serde_json::json!({- "type": "LanguageModelConfig",- "token_codec_config": { "type": "RawTextCodecConfig" },- "decoder_config": {- "embedding_config": {- "type": "TiedEmbeddingConfig", "input_scale": null,- "logit_soft_cap": null, "logit_scale": null,- },- "transformer_config": {- "layer_configs": [], "model_dim": 4, "hidden_dim": 4,- "output_norm_config": {- "epsilon": 0.00001, "scale_offset": null,- "upcast_mode": "only_normalization", "subtract_mean": false,- "has_scale": true, "has_biases": false,- },- },- "vocab_size": 4, "ple_model_config": null, "embedding_norm_config": null,- },- "generation_config": {- "stop_token_ids": [], "temperature": null, "top_k": 20, "top_p": null, "min_p": null,- "banned_tokens": null, "repetition_penalty": null, "presence_penalty": null,- "frequency_penalty": null, "suffix_repetition_length": null,- },- });- std::fs::write(directory.path().join("config.json"), serde_json::to_vec(&config).unwrap()).unwrap();+ fn sampling_method_maps_all_fields() {+ assert_eq!(sampling_method(UzuSamplingMethod::Greedy), SamplingMethod::Greedy {});assert_eq!(- futures::executor::block_on(read_sampling_defaults(directory.path())).unwrap(),- Some(SamplingMethod::Stochastic {- temperature: None,+ sampling_method(UzuSamplingMethod::Stochastic {+ temperature: Some(0.6),+ top_k: Some(20),+ top_p: Some(0.95),+ min_p: Some(0.05),+ repetition_penalty: Some(1.1),+ suffix_repetition_length: Some(64),+ }),+ SamplingMethod::Stochastic {+ temperature: Some(0.6),top_k: Some(20),+ top_p: Some(0.95),+ min_p: Some(0.05),+ repetition_penalty: Some(1.1),+ suffix_repetition_length: Some(64),+ }+ );+ assert_eq!(+ sampling_method(UzuSamplingMethod::Stochastic {+ temperature: None,+ top_k: None,+ top_p: None,+ min_p: None,+ repetition_penalty: None,+ suffix_repetition_length: None,+ }),+ SamplingMethod::Stochastic {+ temperature: None,+ top_k: None,top_p: None,min_p: None,repetition_penalty: None,suffix_repetition_length: None,- })+ });}diff --git a/crates/uzu-engine/src/config/embedding/mod.rs b/crates/uzu-engine/src/config/embedding/mod.rsindex 893c790ff..2d49a398f 100644--- a/crates/uzu-engine/src/config/embedding/mod.rs+++ b/crates/uzu-engine/src/config/embedding/mod.rs@@ -8,4 +8,5 @@ pub struct EmbeddingConfig {pub input_scale: Option<f32>,pub logit_soft_cap: Option<f32>,pub logit_scale: Option<f32>,+ pub permute_indices: bool,}diff --git a/crates/uzu-engine/src/config/mod.rs b/crates/uzu-engine/src/config/mod.rsindex 7d60d3c95..cbafb9a77 100644--- a/crates/uzu-engine/src/config/mod.rs+++ b/crates/uzu-engine/src/config/mod.rs@@ -11,8 +11,10 @@ pub mod mlp;pub mod model;pub mod normalization;pub mod per_layer_embedding;+pub mod residual;pub mod rope;pub mod speculator;+pub mod stream_op;pub mod token_codec;pub mod token_mixer;pub mod transformer;diff --git a/crates/uzu-engine/src/config/per_layer_embedding.rs b/crates/uzu-engine/src/config/per_layer_embedding.rsindex f97f5866b..b4469ad96 100644--- a/crates/uzu-engine/src/config/per_layer_embedding.rs+++ b/crates/uzu-engine/src/config/per_layer_embedding.rs@@ -1,6 +1,6 @@use uzu_engine_macros::uzu_config;-use super::{activation::AnyActivation, linear::LinearConfig, normalization::NormalizationConfig};+use super::{linear::LinearConfig, normalization::NormalizationConfig};#[uzu_config]pub struct PLEModelConfig {@@ -13,11 +13,3 @@ pub struct PLEModelConfig {pub linear_config: LinearConfig,pub norm_config: NormalizationConfig,}--#[uzu_config]-pub struct PLELayerConfig {- pub linear_config: LinearConfig,- pub norm_config: NormalizationConfig,- pub ple_dim: u32,- pub activation: AnyActivation,-}diff --git a/crates/uzu-engine/src/config/residual/additive.rs b/crates/uzu-engine/src/config/residual/additive.rsnew file mode 100644index 000000000..8a5731aca--- /dev/null+++ b/crates/uzu-engine/src/config/residual/additive.rs@@ -0,0 +1,5 @@+use uzu_engine_macros::uzu_config;++#[uzu_config(super::ResidualConfig)]+#[derive(Default)]+pub struct AdditiveResidualConfig {}diff --git a/crates/uzu-engine/src/config/residual/gated.rs b/crates/uzu-engine/src/config/residual/gated.rsnew file mode 100644index 000000000..1a8e8e745--- /dev/null+++ b/crates/uzu-engine/src/config/residual/gated.rs@@ -0,0 +1,11 @@+use uzu_engine_macros::uzu_config;++use crate::config::{linear::LinearConfig, normalization::NormalizationConfig};++#[uzu_config(super::ResidualConfig)]+pub struct GatedResidualConfig {+ pub stream_count: u32,+ pub low_rank_dim: u32,+ pub norm_config: NormalizationConfig,+ pub linear_config: LinearConfig,+}diff --git a/crates/uzu-engine/src/config/residual/mod.rs b/crates/uzu-engine/src/config/residual/mod.rsnew file mode 100644index 000000000..fea8efdff--- /dev/null+++ b/crates/uzu-engine/src/config/residual/mod.rs@@ -0,0 +1,7 @@+use uzu_engine_macros::uzu_config_abstract;++pub mod additive;+pub mod gated;++#[uzu_config_abstract(additive::AdditiveResidualConfig, gated::GatedResidualConfig)]+pub struct ResidualConfig;diff --git a/crates/uzu-engine/src/config/speculator/mod.rs b/crates/uzu-engine/src/config/speculator/mod.rsindex e723adc14..51a6914c2 100644--- a/crates/uzu-engine/src/config/speculator/mod.rs+++ b/crates/uzu-engine/src/config/speculator/mod.rs@@ -1,7 +1,6 @@use uzu_engine_macros::uzu_config_abstract;pub mod dflash;-pub mod model;#[uzu_config_abstract(dflash::DFlashSpeculatorConfig)]pub struct SpeculatorConfig;diff --git a/crates/uzu-engine/src/config/speculator/model.rs b/crates/uzu-engine/src/config/speculator/model.rsdeleted file mode 100644index 4aeddedb9..000000000--- a/crates/uzu-engine/src/config/speculator/model.rs+++ /dev/null@@ -1,8 +0,0 @@-use uzu_engine_macros::uzu_config;--use crate::config::speculator::AnySpeculatorConfig;--#[uzu_config(super::SpeculatorConfig)]-pub struct SpeculatorModelConfig {- pub speculator_config: AnySpeculatorConfig,-}diff --git a/crates/uzu-engine/src/config/stream_op/mod.rs b/crates/uzu-engine/src/config/stream_op/mod.rsnew file mode 100644index 000000000..fb51790be--- /dev/null+++ b/crates/uzu-engine/src/config/stream_op/mod.rs@@ -0,0 +1,12 @@+use uzu_engine_macros::uzu_config_abstract;++pub mod ngram_injection;+pub mod per_layer_embedding_injection;+pub mod residual_scale;++#[uzu_config_abstract(+ ngram_injection::NGramInjectionConfig,+ per_layer_embedding_injection::PerLayerEmbeddingInjectionConfig,+ residual_scale::ResidualScaleConfig+)]+pub struct StreamOpConfig;diff --git a/crates/uzu-engine/src/config/stream_op/ngram_injection.rs b/crates/uzu-engine/src/config/stream_op/ngram_injection.rsnew file mode 100644index 000000000..4f5f88ac7--- /dev/null+++ b/crates/uzu-engine/src/config/stream_op/ngram_injection.rs@@ -0,0 +1,25 @@+use uzu_engine_macros::uzu_config;++use crate::config::{+ linear::LinearConfig, normalization::NormalizationConfig, token_mixer::convolutions::SeparableCausalConvConfig,+};++#[uzu_config]+pub struct NGramHashConfig {+ pub token_vocab_size: u32,+ pub ngram_size: u32,+ pub heads_per_ngram: u32,+ pub eos_token_id: u32,+ pub multipliers: Box<[u64]>,+ pub head_vocab_sizes: Box<[u32]>,+}++#[uzu_config(super::StreamOpConfig)]+pub struct NGramInjectionConfig {+ pub hash_config: NGramHashConfig,+ pub linear_config: LinearConfig,+ pub norm_config: NormalizationConfig,+ pub conv_config: SeparableCausalConvConfig,+ pub embedding_dim: u32,+ pub conv_kernel_size: u32,+}diff --git a/crates/uzu-engine/src/config/stream_op/per_layer_embedding_injection.rs b/crates/uzu-engine/src/config/stream_op/per_layer_embedding_injection.rsnew file mode 100644index 000000000..c4fae76ee--- /dev/null+++ b/crates/uzu-engine/src/config/stream_op/per_layer_embedding_injection.rs@@ -0,0 +1,11 @@+use uzu_engine_macros::uzu_config;++use crate::config::{activation::AnyActivation, linear::LinearConfig, normalization::NormalizationConfig};++#[uzu_config(super::StreamOpConfig)]+pub struct PerLayerEmbeddingInjectionConfig {+ pub linear_config: LinearConfig,+ pub norm_config: NormalizationConfig,+ pub ple_dim: u32,+ pub activation: AnyActivation,+}diff --git a/crates/uzu-engine/src/config/stream_op/residual_scale.rs b/crates/uzu-engine/src/config/stream_op/residual_scale.rsnew file mode 100644index 000000000..6c505e793--- /dev/null+++ b/crates/uzu-engine/src/config/stream_op/residual_scale.rs@@ -0,0 +1,4 @@+use uzu_engine_macros::uzu_config;++#[uzu_config(super::StreamOpConfig)]+pub struct ResidualScaleConfig {}diff --git a/crates/uzu-engine/src/config/token_codec/chat_codec.rs b/crates/uzu-engine/src/config/token_codec/chat_codec.rsindex 8a90b1bb9..7f66df493 100644--- a/crates/uzu-engine/src/config/token_codec/chat_codec.rs+++ b/crates/uzu-engine/src/config/token_codec/chat_codec.rs@@ -7,17 +7,33 @@ pub struct ReasoningConfig {pub reasoning_effort_to_field_value: serde_json::Map<String, serde_json::Value>,}+#[uzu_config]+pub struct ResponseFormatConfig {+ pub reasoning_pattern: Option<String>,+ pub prefixes: Box<[String]>,+ turn_pattern: Option<String>,+ // TODO: This is not respected, we use our own configs (see+ // `hanashi/configs`). It's just here to parse the format. This should be+ // unified eventually.+ tool_calls: serde_json::Value,+}+#[uzu_config(super::TokenCodecConfig)]pub struct ChatCodecConfig {pub prompt_template: String,- pub output_parser_regex: Option<String>,+ pub response_format: Option<ResponseFormatConfig>,pub system_role_name: String,pub user_role_name: String,pub assistant_role_name: String,pub eos_token: Option<String>,pub bos_token: Option<String>,- pub end_of_thinking_tag: Option<String>,pub default_system_prompt: Option<String>,#[serde(default)] // TODO: ??? inconsistency in converted models in registry workaround, remove when resolvedpub reasoning_config: Option<ReasoningConfig>,}++impl ChatCodecConfig {+ pub fn end_of_thinking_tag(&self) -> Option<&str> {+ self.response_format.as_ref()?.prefixes.last().map(String::as_str)+ }+}diff --git a/crates/uzu-engine/src/config/token_codec/mod.rs b/crates/uzu-engine/src/config/token_codec/mod.rsindex a1d48f6b7..2a0ed693f 100644--- a/crates/uzu-engine/src/config/token_codec/mod.rs+++ b/crates/uzu-engine/src/config/token_codec/mod.rs@@ -12,13 +12,16 @@ impl From<&AnyTokenCodecConfig> for ShojiTokenCodecConfig {match config {AnyTokenCodecConfig::ChatCodecConfig(config) => ShojiTokenCodecConfig::Chat(ChatTokenCodecConfig {prompt_template: config.prompt_template.clone(),- output_parser_regex: config.output_parser_regex.clone(),+ output_parser_regex: config+ .response_format+ .as_ref()+ .and_then(|format| format.reasoning_pattern.clone()),system_role_name: config.system_role_name.clone(),user_role_name: config.user_role_name.clone(),assistant_role_name: config.assistant_role_name.clone(),eos_token: config.eos_token.clone(),bos_token: config.bos_token.clone(),- end_of_thinking_tag: config.end_of_thinking_tag.clone(),+ end_of_thinking_tag: config.end_of_thinking_tag().map(str::to_owned),default_system_prompt: config.default_system_prompt.clone(),}),AnyTokenCodecConfig::RawTextCodecConfig(_) => ShojiTokenCodecConfig::RawText,diff --git a/crates/uzu-engine/src/config/token_mixer/delta_net.rs b/crates/uzu-engine/src/config/token_mixer/delta_net.rsindex cddf1314f..55ac744e1 100644--- a/crates/uzu-engine/src/config/token_mixer/delta_net.rs+++ b/crates/uzu-engine/src/config/token_mixer/delta_net.rs@@ -1,7 +1,8 @@use uzu_engine_macros::uzu_config;use crate::config::{- linear::LinearConfig, normalization::NormalizationConfig, token_mixer::convolutions::SeparableCausalConvConfig,+ activation::AnyActivation, linear::LinearConfig, normalization::NormalizationConfig,+ token_mixer::convolutions::SeparableCausalConvConfig,};#[uzu_config(super::TokenMixerConfig)]@@ -16,4 +17,5 @@ pub struct DeltaNetConfig {pub head_dim: u32,pub value_head_dim: u32,pub kernel_size: u32,+ pub output_gate: AnyActivation,}diff --git a/crates/uzu-engine/src/config/token_mixer/mod.rs b/crates/uzu-engine/src/config/token_mixer/mod.rsindex fff58ebd6..71c5fe360 100644--- a/crates/uzu-engine/src/config/token_mixer/mod.rs+++ b/crates/uzu-engine/src/config/token_mixer/mod.rs@@ -4,6 +4,12 @@ pub mod attention;pub mod convolutions;pub mod delta_net;pub mod short_conv;+pub mod sparse_attention;-#[uzu_config_abstract(attention::AttentionConfig, delta_net::DeltaNetConfig, short_conv::ShortConvConfig)]+#[uzu_config_abstract(+ attention::AttentionConfig,+ delta_net::DeltaNetConfig,+ short_conv::ShortConvConfig,+ sparse_attention::SparseAttentionConfig+)]pub struct TokenMixerConfig;diff --git a/crates/uzu-engine/src/config/token_mixer/sparse_attention.rs b/crates/uzu-engine/src/config/token_mixer/sparse_attention.rsnew file mode 100644index 000000000..81313de67--- /dev/null+++ b/crates/uzu-engine/src/config/token_mixer/sparse_attention.rs@@ -0,0 +1,22 @@+use uzu_engine_macros::uzu_config;++use crate::config::{+ linear::LinearConfig, normalization::NormalizationConfig, token_mixer::attention::AttentionConfig,+};++#[uzu_config]+pub struct SparseAttentionIndexerConfig {+ pub projection_config: LinearConfig,+ pub query_norm_config: NormalizationConfig,+ pub key_norm_config: NormalizationConfig,+ pub num_heads: u32,+ pub head_dim: u32,+ pub compression_ratio: u32,+ pub token_budget: u32,+}++#[uzu_config(super::TokenMixerConfig)]+pub struct SparseAttentionConfig {+ pub attention_config: AttentionConfig,+ pub indexer_config: SparseAttentionIndexerConfig,+}diff --git a/crates/uzu-engine/src/config/transformer.rs b/crates/uzu-engine/src/config/transformer.rsindex 205df98f9..90b176c8d 100644--- a/crates/uzu-engine/src/config/transformer.rs+++ b/crates/uzu-engine/src/config/transformer.rs@@ -1,11 +1,14 @@use uzu_engine_macros::uzu_config;-use crate::config::{normalization::NormalizationConfig, transformer_layer::TransformerLayerConfig};+use crate::config::{+ normalization::NormalizationConfig, residual::AnyResidualConfig, transformer_layer::TransformerLayerConfig,+};#[uzu_config]pub struct TransformerConfig {+ pub residual_config: AnyResidualConfig,pub layer_configs: Box<[TransformerLayerConfig]>,- pub output_norm_config: NormalizationConfig,+ pub output_norm_config: Option<NormalizationConfig>,pub model_dim: u32,pub hidden_dim: u32,}diff --git a/crates/uzu-engine/src/config/transformer_layer.rs b/crates/uzu-engine/src/config/transformer_layer.rsindex 3944b8b95..7ab571697 100644--- a/crates/uzu-engine/src/config/transformer_layer.rs+++ b/crates/uzu-engine/src/config/transformer_layer.rs@@ -4,8 +4,8 @@ use crate::config::{linear::LinearConfig,mlp::AnyMLPConfig,normalization::NormalizationConfig,- per_layer_embedding::PLELayerConfig,rope::AnyRoPEConfig,+ stream_op::AnyStreamOpConfig,token_mixer::{AnyTokenMixerConfig, convolutions::SeparableCausalConvConfig},};@@ -17,19 +17,34 @@ pub struct TransformerLayerConvConfig {pub conv_group_size: u32,}+#[uzu_config]+pub struct SublayerConfigCommon {+ pub pre_norm_config: Option<NormalizationConfig>,+ pub conv_config: Option<TransformerLayerConvConfig>,+ pub post_norm_config: Option<NormalizationConfig>,+}++#[uzu_config]+pub struct MixerSublayerConfig {+ #[serde(flatten)]+ pub common: SublayerConfigCommon,+ pub body_config: AnyTokenMixerConfig,+}++#[uzu_config]+pub struct MlpSublayerConfig {+ #[serde(flatten)]+ pub common: SublayerConfigCommon,+ pub body_config: AnyMLPConfig,+}+#[uzu_config]pub struct TransformerLayerConfig {- pub pre_mixer_norm_config: Option<NormalizationConfig>,- pub mixer_config: AnyTokenMixerConfig,- pub mixer_conv_config: Option<TransformerLayerConvConfig>,- pub post_mixer_norm_config: Option<NormalizationConfig>,- pub pre_mlp_norm_config: NormalizationConfig,- pub mlp_config: AnyMLPConfig,- pub mlp_conv_config: Option<TransformerLayerConvConfig>,- pub post_mlp_norm_config: Option<NormalizationConfig>,+ pub prologue_configs: Box<[AnyStreamOpConfig]>,+ pub mixer: MixerSublayerConfig,+ pub mlp: MlpSublayerConfig,+ pub epilogue_configs: Box<[AnyStreamOpConfig]>,pub hidden_dim: Option<u32>,- pub ple_config: Option<PLELayerConfig>,- pub has_post_layer_scalar: bool,pub kv_source_layer_index: Option<u32>,pub rope_config: Option<AnyRoPEConfig>,}diff --git a/crates/uzu-engine/src/config/weight_matrix/hybrid_spec.rs b/crates/uzu-engine/src/config/weight_matrix/hybrid_spec.rsindex 332eda59c..631c9056b 100644--- a/crates/uzu-engine/src/config/weight_matrix/hybrid_spec.rs+++ b/crates/uzu-engine/src/config/weight_matrix/hybrid_spec.rs@@ -10,10 +10,17 @@ pub enum IncoherenceProcessingMode {InputOutput,}+#[uzu_config]+pub struct HadamardSpec {+ #[serde(rename = "type")]+ ty: monostate::MustBe!("HadamardSpec"),+ pub block_size: u32,+ pub processing_mode: IncoherenceProcessingMode,+}+#[uzu_config(super::WeightMatrixSpec)]pub struct HybridSpec {pub quantization_spec: Box<AnyWeightMatrixSpec>,pub adapter_spec: Option<Unsupported>,- pub incoherence_block_size: Option<u32>,- pub incoherence_processing_mode: IncoherenceProcessingMode,+ pub incoherence_spec: Option<HadamardSpec>,}diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rsindex ff4b42bd7..09de58301 100644--- a/crates/uzu-engine/src/encodable_block/dflash.rs+++ b/crates/uzu-engine/src/encodable_block/dflash.rs@@ -5,7 +5,12 @@ use thiserror::Error;use crate::{backends::common::{Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding},- config::{dflash::DFlashDraftConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig},+ config::{+ dflash::DFlashDraftConfig,+ residual::{AnyResidualConfig, additive::AdditiveResidualConfig},+ rope::AnyRoPEConfig,+ token_mixer::AnyTokenMixerConfig,+ },data_type::DataType,encodable_block::{batch_topology::BatchTopology,@@ -121,7 +126,7 @@ impl<B: Backend> DFlash<B> {.await?;let layers_tree = parameter_tree.subtree("layers");let first_layer_config = config.layer_configs.first().expect("DFlash draft model must have at least one layer");- let AnyTokenMixerConfig::AttentionConfig(attention_config) = &first_layer_config.mixer_config else {+ let AnyTokenMixerConfig::AttentionConfig(attention_config) = &first_layer_config.mixer.body_config else {return Err(DFlashNewError::InvalidAttentionConfig("DFlash layers must use attention mixers"));};let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;@@ -135,6 +140,8 @@ impl<B: Backend> DFlash<B> {context,).await?;+ // DFlash draft models are hard-coded with an additive residual.+ let residual_config = AnyResidualConfig::AdditiveResidualConfig(AdditiveResidualConfig::default());let mut layers = Vec::with_capacity(num_layers as usize);for (layer_config, index) in config.layer_configs.iter().zip(0u32..) {layers.push(@@ -145,6 +152,7 @@ impl<B: Backend> DFlash<B> {index,data_type,layer_config,+ &residual_config,&layers_tree.subtree(&index.to_string()),context,)diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex 473dfee45..412c1fe32 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -87,6 +87,12 @@ impl<B: Backend> Embedding<B> {parameter_tree: &ParameterTree<'_, B>,context: &B::Context,) -> Result<(Self, Option<Arc<B::GlobalGpuBuffer>>), EmbeddingError<B>> {+ if *config.permute_indices() {+ return Err(EmbeddingError::UnsupportedConfiguration(+ "embeddings with permuted vocabulary indices are not supported".into(),+ ));+ }+let (tying, readout_input_hadamard_factors) = match config {AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {let embedding_tree = parameter_tree.subtree("embedding");diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rsindex 2ccbba2d7..ef662893c 100644--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs@@ -10,7 +10,7 @@ use crate::{config::weight_matrix::{AnyWeightMatrixSpec, Layout,d4s4_spec::D4S4Spec,- hybrid_spec::{HybridSpec, IncoherenceProcessingMode},+ hybrid_spec::{HadamardSpec, HybridSpec, IncoherenceProcessingMode},},data_type::DataType,encodable_block::weight_matrix::{WeightMatrix, WeightMatrixError},@@ -112,8 +112,12 @@ impl<B: Backend> EmbeddingTable<B> {},AnyWeightMatrixSpec::HybridSpec(HybridSpec {quantization_spec,- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),- incoherence_processing_mode: IncoherenceProcessingMode::Output,+ incoherence_spec:+ Some(HadamardSpec {+ block_size: HADAMARD_TRANSFORM_BLOCK_SIZE,+ processing_mode: IncoherenceProcessingMode::Output,+ ..+ }),..}) if embedding_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE) => {let matrix = load_matrix(&tree.subtree("quantized"), *quantization_spec).await?;diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rsindex ad96f26b5..c5afbe7e2 100644--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs@@ -18,7 +18,7 @@ use crate::{},config::weight_matrix::{AnyWeightMatrixSpec,- hybrid_spec::{HybridSpec, IncoherenceProcessingMode},+ hybrid_spec::{HadamardSpec, HybridSpec, IncoherenceProcessingMode},},data_type::DataType,safetensors::{ParameterLoaderError, ParameterTree},@@ -115,10 +115,14 @@ impl<B: Backend> dyn Linear<B> {Ok(Box::new(block))},AnyWeightMatrixSpec::HybridSpec(HybridSpec {- incoherence_block_size: Some(block_size),- incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,+ incoherence_spec:+ Some(HadamardSpec {+ block_size: HADAMARD_TRANSFORM_BLOCK_SIZE,+ processing_mode: IncoherenceProcessingMode::InputOutput,+ ..+ }),..- }) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => Ok(Box::new(+ }) => Ok(Box::new(RHTLinearWrapper::new(input_dimension,output_dimension,diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex fc13ef196..c17059acf 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -9,7 +9,7 @@ use crate::{},config::weight_matrix::{AnyWeightMatrixSpec,- hybrid_spec::{HybridSpec, IncoherenceProcessingMode},+ hybrid_spec::{HadamardSpec, HybridSpec, IncoherenceProcessingMode},},data_type::DataType,encodable_block::linear::{@@ -40,10 +40,13 @@ fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {matches!(spec,AnyWeightMatrixSpec::HybridSpec(HybridSpec {- incoherence_block_size: Some(block_size),- incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,+ incoherence_spec: Some(HadamardSpec {+ block_size: HADAMARD_TRANSFORM_BLOCK_SIZE,+ processing_mode: IncoherenceProcessingMode::InputOutput,+ ..+ }),..- }) if *block_size == HADAMARD_TRANSFORM_BLOCK_SIZE+ }))}diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rsindex e474d2c10..28e2e4ac7 100644--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs@@ -6,7 +6,7 @@ use crate::{},config::weight_matrix::{AnyWeightMatrixSpec,- hybrid_spec::{HybridSpec, IncoherenceProcessingMode},+ hybrid_spec::{HadamardSpec, HybridSpec, IncoherenceProcessingMode},},data_type::DataType,encodable_block::linear::{@@ -31,8 +31,12 @@ impl<B: Backend> UntiedReadout<B> {) -> Result<Self, LinearMatmulError<B>> {let AnyWeightMatrixSpec::HybridSpec(HybridSpec {quantization_spec,- incoherence_block_size: Some(HADAMARD_TRANSFORM_BLOCK_SIZE),- incoherence_processing_mode: IncoherenceProcessingMode::Input,+ incoherence_spec:+ Some(HadamardSpec {+ block_size: HADAMARD_TRANSFORM_BLOCK_SIZE,+ processing_mode: IncoherenceProcessingMode::Input,+ ..+ }),..}) = specelse {diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rsindex cd4e1d02a..b618c97e9 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs@@ -12,7 +12,10 @@ use crate::{delta_net_tree_verify::DeltaNetTreeVerify as DeltaNetTreeVerifyTrait,},},- config::token_mixer::delta_net::DeltaNetConfig,+ config::{+ activation::{AnyActivation, silu::SiLU},+ token_mixer::delta_net::DeltaNetConfig,+ },data_type::DataType,encodable_block::{batch_topology::BatchTopology,@@ -186,6 +189,14 @@ impl<B: Backend> DeltaNet<B> {)));}+ // TODO: Don't hardcode SiLU+ if config.output_gate != AnyActivation::SiLU(SiLU::default()) {+ return Err(DeltaNetNewError::UnsupportedConfiguration(format!(+ "output_gate must be SiLU, got {:?}",+ config.output_gate.act_type()+ )));+ }+let key_dim = config.num_groups * config.head_dim;let value_dim = config.num_heads * config.value_head_dim;let conv_dim = key_dim * 2 + value_dim;diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rsindex 7f4deeb7f..2f79474e8 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs@@ -86,6 +86,8 @@ pub enum MixerNewError<B: Backend> {DeltaNet(#[from] DeltaNetNewError<B>),#[error("Short conv mixer error: {0}")]ShortConv(#[from] ShortConvNewError<B>),+ #[error("Sparse attention mixers are not supported")]+ SparseAttentionUnsupported,}impl<B: Backend> dyn Mixer<B> {@@ -120,6 +122,7 @@ impl<B: Backend> dyn Mixer<B> {Ok((Box::new(short_conv), in_projection_input_hadamard_factors))},+ AnyTokenMixerConfig::SparseAttentionConfig(_) => Err(MixerNewError::SparseAttentionUnsupported),}}}diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rsindex baa5ab656..1d7e1efb5 100644--- a/crates/uzu-engine/src/encodable_block/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mod.rs@@ -13,6 +13,7 @@ pub mod mlp;pub mod normalization;pub mod per_layer_embedding;pub mod prediction_head;+pub mod residual;pub mod sampling;pub mod transformer;pub mod transformer_layer;diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rsindex b8f8afd99..563cccc37 100644--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs@@ -7,8 +7,8 @@ use crate::{kernel::{GatedActMul, GatedActMulSettings, Kernels, TensorAddBiasKernel, TensorAddScaleKernel},},config::{- activation::AnyActivation,- per_layer_embedding::{PLELayerConfig, PLEModelConfig},+ activation::AnyActivation, per_layer_embedding::PLEModelConfig,+ stream_op::per_layer_embedding_injection::PerLayerEmbeddingInjectionConfig,},data_type::DataType,encodable_block::{@@ -182,7 +182,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {num_layers: u32,data_type: DataType,post_layer_scalar: f32,- config: &PLELayerConfig,+ config: &PerLayerEmbeddingInjectionConfig,parameter_tree: &ParameterTree<'_, B>,context: &B::Context,) -> Result<Self, PerLayerEmbeddingError<B>> {diff --git a/crates/uzu-engine/src/encodable_block/residual.rs b/crates/uzu-engine/src/encodable_block/residual.rsnew file mode 100644index 000000000..0215ab27d--- /dev/null+++ b/crates/uzu-engine/src/encodable_block/residual.rs@@ -0,0 +1,117 @@+use thiserror::Error;++use crate::{+ backends::common::{Backend, BufferRef, CommandBuffer, Kernels, kernel::TensorAddScaleKernel},+ config::residual::AnyResidualConfig,+ data_type::DataType,+ safetensors::ParameterTree,+};++#[derive(Debug, Error)]+pub enum ResidualNewError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Gated residual connections are not supported yet")]+ GatedUnsupported,+}++pub enum ResidualConnection<B: Backend> {+ Additive(AdditiveResidualConnection<B>),+ // TODO: Gated(GatedResidualConnection<B>)+}++pub struct AdditiveResidualConnection<B: Backend> {+ residual_add: <B::Kernels as Kernels>::TensorAddScaleKernel,+}++impl<B: Backend> AdditiveResidualConnection<B> {+ fn new(+ context: &B::Context,+ data_type: DataType,+ ) -> Result<Self, B::Error> {+ Ok(Self {+ residual_add: <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, data_type, false)?,+ })+ }++ fn encode_update(+ &self,+ stream: impl BufferRef<Backend = B>,+ update: impl BufferRef<Backend = B>,+ output: &mut B::ScratchBuffer,+ elements: u32,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {+ self.residual_add.encode(Some(stream), update, output, elements, elements, 1.0, command_buffer);+ Ok(())+ }+}++impl<B: Backend> ResidualConnection<B> {+ pub fn new(+ config: &AnyResidualConfig,+ data_type: DataType,+ parameter_tree: &ParameterTree<'_, B>,+ context: &B::Context,+ ) -> Result<Self, ResidualNewError<B>> {+ match config {+ AnyResidualConfig::AdditiveResidualConfig(_) => {+ // TODO: Load weights for gated residuals.+ let _ = parameter_tree;+ Ok(Self::Additive(+ AdditiveResidualConnection::new(context, data_type).map_err(ResidualNewError::Backend)?,+ ))+ },+ AnyResidualConfig::GatedResidualConfig(_) => Err(ResidualNewError::GatedUnsupported),+ }+ }++ pub fn encode_update(+ &self,+ stream: impl BufferRef<Backend = B>,+ update: impl BufferRef<Backend = B>,+ output: &mut B::ScratchBuffer,+ elements: u32,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {+ match self {+ Self::Additive(connection) => connection.encode_update(stream, update, output, elements, command_buffer),+ }+ }+}++pub struct ResidualHead<B: Backend> {+ connection: ResidualConnection<B>,+}++impl<B: Backend> ResidualHead<B> {+ pub fn new(+ config: &AnyResidualConfig,+ data_type: DataType,+ parameter_tree: &ParameterTree<'_, B>,+ context: &B::Context,+ ) -> Result<Self, ResidualNewError<B>> {+ Ok(Self {+ connection: ResidualConnection::new(config, data_type, parameter_tree, context)?,+ })+ }++ // TODO: gated repeats the embeddings `stream_count` times+ pub fn expand(+ &self,+ embeddings: B::ScratchBuffer,+ ) -> B::ScratchBuffer {+ embeddings+ }++ pub fn encode_update(+ &self,+ stream: impl BufferRef<Backend = B>,+ update: impl BufferRef<Backend = B>,+ output: &mut B::ScratchBuffer,+ elements: u32,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {+ self.connection.encode_update(stream, update, output, elements, command_buffer)+ }+}diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rsindex 0966c0a05..dc04f29c5 100644--- a/crates/uzu-engine/src/encodable_block/transformer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer.rs@@ -4,15 +4,14 @@ use derive_more::Debug;use thiserror::Error;use crate::{- backends::common::{- Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, kernel::TensorAddScaleKernel,- },+ backends::common::{Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding},config::{rope::AnyRoPEConfig, transformer::TransformerConfig},data_type::DataType,encodable_block::{batch_topology::BatchTopology,mixer::{MixerState, attention::rope::PrecalculatedRoPE},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},+ residual::{ResidualHead, ResidualNewError},transformer_layer::{TransformerLayer, TransformerLayerError},},safetensors::ParameterTree,@@ -88,6 +87,10 @@ pub enum TransformerNewError<B: Backend> {Layer(#[from] TransformerLayerError<B>),#[error("Normalization error: {0}")]Normalization(#[from] NormalizationNewError<B>),+ #[error("Residual error: {0}")]+ Residual(#[from] ResidualNewError<B>),+ #[error("Readout expects RHT-rotated input but there is no output norm to apply it")]+ UnappliedReadoutRht,}#[derive(Debug, Error)]@@ -101,9 +104,9 @@ pub enum TransformerEncodeError<B: Backend> {pub struct Transformer<B: Backend> {ropes: Box<[AnyRoPEConfig]>,layers: Box<[(TransformerLayer<B>, Option<usize>)]>,- output_norm: Normalization<B>,+ output_norm: Option<Normalization<B>>,model_dim: u32,- residual_add: <B::Kernels as Kernels>::TensorAddScaleKernel,+ residual_head: ResidualHead<B>,}impl<B: Backend> Transformer<B> {@@ -117,6 +120,10 @@ impl<B: Backend> Transformer<B> {) -> Result<Self, TransformerNewError<B>> {let mut ropes: Vec<AnyRoPEConfig> = Vec::new();+ let residual_config = transformer_config.residual_config.clone();+ let residual_head =+ ResidualHead::new(&residual_config, input_data_type, ¶meter_tree.subtree("residual_head"), context)?;+let num_layers = transformer_config.layer_configs.len() as u32;let mut layers = Vec::with_capacity(num_layers as usize);for (layer_config, layer_index) in transformer_config.layer_configs.iter().zip(0u32..) {@@ -136,6 +143,7 @@ impl<B: Backend> Transformer<B> {layer_index,input_data_type,layer_config,+ &residual_config,&layer_loader,context,)@@ -144,28 +152,31 @@ impl<B: Backend> Transformer<B> {layers.push((layer, rope));}- let output_norm = Normalization::new(- transformer_config.model_dim,- input_data_type,- output_data_type,- output_norm_hadamard_factors,- ShortcutMode::Add,- PostLayerScalar::None,- &transformer_config.output_norm_config,- ¶meter_tree.subtree("output_norm"),- context,- )- .await?;-- let residual_add = <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, input_data_type, false)- .map_err(TransformerNewError::Backend)?;+ let output_norm = match (&transformer_config.output_norm_config, output_norm_hadamard_factors) {+ (Some(output_norm_config), hadamard_factors) => Some(+ Normalization::new(+ transformer_config.model_dim,+ input_data_type,+ output_data_type,+ hadamard_factors,+ ShortcutMode::Add,+ PostLayerScalar::None,+ output_norm_config,+ ¶meter_tree.subtree("output_norm"),+ context,+ )+ .await?,+ ),+ (None, Some(_)) => return Err(TransformerNewError::UnappliedReadoutRht),+ (None, None) => None,+ };Ok(Self {ropes: ropes.into_boxed_slice(),layers: layers.into_boxed_slice(),output_norm,model_dim: transformer_config.model_dim,- residual_add,+ residual_head,})}@@ -178,7 +189,7 @@ impl<B: Backend> Transformer<B> {) -> Result<B::ScratchBuffer, B::Error> {let mut output = command_buffer.allocate_scratch(hidden.size())?;let elements = batch_size * self.model_dim;- self.residual_add.encode(Some(shortcut), hidden, &mut output, elements, elements, 1.0, command_buffer);+ self.residual_head.encode_update(shortcut, hidden, &mut output, elements, command_buffer)?;Ok(output)}@@ -249,7 +260,8 @@ impl<B: Backend> Transformer<B> {mut state: Option<&mut TransformerState<B>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<TransformerEncodeOutput<B>, TransformerEncodeError<B>> {- let mut hidden = input;+ // TODO: change shortcut for residual+ let mut hidden = self.residual_head.expand(input);let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_none() {self.prefill_cache_layer_count()} else {@@ -333,19 +345,25 @@ impl<B: Backend> Transformer<B> {});};- let output_normalized = self- .output_norm- .encode(- &hidden,- output_range.start,- output_range.end - output_range.start,- Some(&mut shortcut),- command_buffer,- )- .map_err(TransformerEncodeError::Backend)?;+ // TODO: route the final combine through `residual_head.readout` (stream + last update)+ // instead of the ShortcutMode::Add fused into the output norm.+ let output = match &self.output_norm {+ Some(output_norm) => output_norm+ .encode(+ &hidden,+ output_range.start,+ output_range.end - output_range.start,+ Some(&mut shortcut),+ command_buffer,+ )+ .map_err(TransformerEncodeError::Backend)?,+ None => self+ .capture_residual(&shortcut, &hidden, batch_dim.size(), command_buffer)+ .map_err(TransformerEncodeError::Backend)?,+ };Ok(TransformerEncodeOutput {- output: Some(output_normalized),+ output: Some(output),hidden_features,})}diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rsindex 6016db49c..1fa7cc133 100644--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs@@ -8,7 +8,11 @@ use crate::{backends::common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, kernel::ActivationTransform,},- config::transformer_layer::{TransformerLayerConfig, TransformerLayerConvConfig},+ config::{+ residual::AnyResidualConfig,+ stream_op::AnyStreamOpConfig,+ transformer_layer::{TransformerLayerConfig, TransformerLayerConvConfig},+ },data_type::DataType,encodable_block::{batch_topology::BatchTopology,@@ -18,6 +22,7 @@ use crate::{mlp::{Mlp, MlpBlockNewError, MlpEncodeError},normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},per_layer_embedding::{PerLayerEmbeddingError, PerLayerEmbeddingProjection},+ residual::{ResidualConnection, ResidualNewError},},safetensors::{ParameterLoaderError, ParameterTree},utils::{dispatch_dtype::dispatch_dtype, maybe_mut::MaybeMut},@@ -47,12 +52,34 @@ pub enum TransformerLayerError<B: Backend> {LinearEncode(#[from] LinearEncodeError<B>),#[error("Per-layer embedding error: {0}")]PerLayerEmbedding(#[from] PerLayerEmbeddingError<B>),- #[error("Layer {layer_index} sets post_layer_scalar but has no post_mlp_norm")]- PostLayerScalarWithoutPostMlpNorm {+ #[error("Residual error: {0}")]+ Residual(#[from] ResidualNewError<B>),+ #[error("Layer {layer_index} sets a residual scale op but has no mlp post_norm")]+ ResidualScaleWithoutPostMlpNorm {+ layer_index: u32,+ },+ #[error("Layer {layer_index} uses prologue stream ops, which are not supported")]+ UnsupportedPrologueOps {+ layer_index: u32,+ },+ #[error("Layer {layer_index} uses an n-gram injection op, which is not supported")]+ UnsupportedNGramInjection {layer_index: u32,},- #[error("Transformer layers except the first one if it doesn't have rht in mixer require pre_mixer_norm_config")]+ #[error("Layer {layer_index} has multiple per-layer embedding injection ops, but only one is applied")]+ MultiplePerLayerEmbeddingInjectionOps {+ layer_index: u32,+ },+ #[error("Layer {layer_index} has multiple residual scale ops, but only one is applied")]+ MultipleResidualScaleOps {+ layer_index: u32,+ },+ #[error("Transformer layers except the first one if it doesn't have rht in mixer require a mixer pre_norm")]MissingPreMixerNormConfig,+ #[error("Layer {layer_index} requires an mlp pre_norm")]+ MissingPreMlpNormConfig {+ layer_index: u32,+ },}struct TransformerLayerConv<B: Backend> {@@ -64,8 +91,6 @@ struct TransformerLayerConv<B: Backend> {coefficient_count: u32,}-// TODO: saner shortcut-pub struct TransformerLayer<B: Backend> {pub layer_index: u32,pub kv_source_layer_index: Option<u32>,@@ -78,6 +103,12 @@ pub struct TransformerLayer<B: Backend> {pub mlp: Box<dyn Mlp<B>>,pub post_mlp_norm: Option<Normalization<B>>,pub ple_projection: Option<PerLayerEmbeddingProjection<B>>,+ // TODO: read these in `encode` once norm kernels stop fusing the residual add (ShortcutMode):+ // the stream update becomes `self.mixer_connection.encode_update(...)` / `mlp_connection`.+ #[allow(dead_code)]+ mixer_connection: ResidualConnection<B>,+ #[allow(dead_code)]+ mlp_connection: ResidualConnection<B>,}impl<B: Backend> TransformerLayerConv<B> {@@ -183,16 +214,55 @@ impl<B: Backend> TransformerLayer<B> {layer_index: u32,data_type: DataType,layer_config: &TransformerLayerConfig,+ residual_config: &AnyResidualConfig,parameter_tree: &ParameterTree<'_, B>,context: &B::Context,) -> Result<Self, TransformerLayerError<B>> {- let post_layer_scalar = if layer_config.has_post_layer_scalar {- if layer_config.post_mlp_norm_config.is_none() {- return Err(TransformerLayerError::PostLayerScalarWithoutPostMlpNorm {+ if !layer_config.prologue_configs.is_empty() {+ return Err(TransformerLayerError::UnsupportedPrologueOps {+ layer_index,+ });+ }++ // TODO: Apply epilogue ops as a sequence properly.+ let mut ple_op = None;+ let mut residual_scale_op_index = None;+ for (index, op) in layer_config.epilogue_configs.iter().enumerate() {+ match op {+ AnyStreamOpConfig::NGramInjectionConfig(_) => {+ return Err(TransformerLayerError::UnsupportedNGramInjection {+ layer_index,+ });+ },+ AnyStreamOpConfig::PerLayerEmbeddingInjectionConfig(config) => {+ if ple_op.replace((index, config)).is_some() {+ return Err(TransformerLayerError::MultiplePerLayerEmbeddingInjectionOps {+ layer_index,+ });+ }+ },+ AnyStreamOpConfig::ResidualScaleConfig(_) => {+ if residual_scale_op_index.replace(index).is_some() {+ return Err(TransformerLayerError::MultipleResidualScaleOps {+ layer_index,+ });+ }+ },+ }+ }++ let post_layer_scalar = if let Some(op_index) = residual_scale_op_index {+ if layer_config.mlp.common.post_norm_config.is_none() {+ return Err(TransformerLayerError::ResidualScaleWithoutPostMlpNorm {layer_index,});}- let scalar = dispatch_dtype!(|(T: data_type)| parameter_tree.read_slice::<T>("post_layer_scalar", &[1]).await?[0].as_());+ let scalar = dispatch_dtype!(|(T: data_type)| parameter_tree+ .subtree("epilogue")+ .subtree(&op_index.to_string())+ .read_slice::<T>("scale", &[1])+ .await?[0]+ .as_());Some(scalar)} else {None@@ -200,7 +270,7 @@ impl<B: Backend> TransformerLayer<B> {// When a PLE projection is present it owns the post-layer scalar (applied// to the combined residual), so the norms must not also apply it.- let (residual_sum_scalar, output_scalar) = match (post_layer_scalar, layer_config.ple_config.is_none()) {+ let (residual_sum_scalar, output_scalar) = match (post_layer_scalar, ple_op.is_none()) {(Some(scalar), true) => (PostLayerScalar::ScaleResidualSum(scalar), PostLayerScalar::ScaleOutput(scalar)),_ => (PostLayerScalar::None, PostLayerScalar::None),};@@ -209,13 +279,13 @@ impl<B: Backend> TransformerLayer<B> {model_dim,data_type,layer_config.rope_config.as_ref(),- &layer_config.mixer_config,- ¶meter_tree.subtree("mixer"),+ &layer_config.mixer.body_config,+ ¶meter_tree.subtree("mixer").subtree("body"),context,).await?;- let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer_conv_config {+ let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer.common.conv_config {Some(config) => (Some(TransformerLayerConv::new(@@ -223,7 +293,7 @@ impl<B: Backend> TransformerLayer<B> {data_type,mixer_hadamard_factors,config,- ¶meter_tree.subtree("mixer_conv"),+ ¶meter_tree.subtree("mixer").subtree("conv"),context,).await?,@@ -233,7 +303,7 @@ impl<B: Backend> TransformerLayer<B> {None => (None, mixer_hadamard_factors),};- let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {+ let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.mixer.common.pre_norm_config {Some(Normalization::new(model_dim,@@ -247,7 +317,7 @@ impl<B: Backend> TransformerLayer<B> {},PostLayerScalar::None,pre_mixer_norm_config,- ¶meter_tree.subtree("pre_mixer_norm"),+ ¶meter_tree.subtree("mixer").subtree("pre_norm"),context,).await?,@@ -259,7 +329,7 @@ impl<B: Backend> TransformerLayer<B> {None};- let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {+ let post_mixer_norm = if let Some(norm_config) = &layer_config.mixer.common.post_norm_config {Some(Normalization::new(model_dim,@@ -269,7 +339,7 @@ impl<B: Backend> TransformerLayer<B> {ShortcutMode::None,PostLayerScalar::None,norm_config,- ¶meter_tree.subtree("post_mixer_norm"),+ ¶meter_tree.subtree("mixer").subtree("post_norm"),context,).await?,@@ -282,13 +352,13 @@ impl<B: Backend> TransformerLayer<B> {model_dim,layer_config.hidden_dim.unwrap_or(hidden_dim),data_type,- &layer_config.mlp_config,- ¶meter_tree.subtree("mlp"),+ &layer_config.mlp.body_config,+ ¶meter_tree.subtree("mlp").subtree("body"),context,).await?;- let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp_conv_config {+ let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp.common.conv_config {Some(config) => (Some(TransformerLayerConv::new(@@ -296,7 +366,7 @@ impl<B: Backend> TransformerLayer<B> {data_type,mlp_input_hadamard_factors,config,- ¶meter_tree.subtree("mlp_conv"),+ ¶meter_tree.subtree("mlp").subtree("conv"),context,).await?,@@ -306,6 +376,11 @@ impl<B: Backend> TransformerLayer<B> {None => (None, mlp_input_hadamard_factors),};+ let Some(pre_mlp_norm_config) = &layer_config.mlp.common.pre_norm_config else {+ return Err(TransformerLayerError::MissingPreMlpNormConfig {+ layer_index,+ });+ };let pre_mlp_norm = Normalization::new(model_dim,data_type,@@ -313,13 +388,13 @@ impl<B: Backend> TransformerLayer<B> {mlp_input_hadamard_factors.map(Arc::new),ShortcutMode::Add,residual_sum_scalar,- &layer_config.pre_mlp_norm_config,- ¶meter_tree.subtree("pre_mlp_norm"),+ pre_mlp_norm_config,+ ¶meter_tree.subtree("mlp").subtree("pre_norm"),context,).await?;- let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {+ let post_mlp_norm = if let Some(norm_config) = &layer_config.mlp.common.post_norm_config {Some(Normalization::new(model_dim,@@ -329,7 +404,7 @@ impl<B: Backend> TransformerLayer<B> {ShortcutMode::None,output_scalar,norm_config,- ¶meter_tree.subtree("post_mlp_norm"),+ ¶meter_tree.subtree("mlp").subtree("post_norm"),context,).await?,@@ -338,8 +413,13 @@ impl<B: Backend> TransformerLayer<B> {None};- let ple_projection = if let Some(ple_config) = &layer_config.ple_config {- let ple_loader = parameter_tree.subtree("ple");+ let mixer_connection =+ ResidualConnection::new(residual_config, data_type, ¶meter_tree.subtree("mixer.connection"), context)?;+ let mlp_connection =+ ResidualConnection::new(residual_config, data_type, ¶meter_tree.subtree("mlp.connection"), context)?;++ let ple_projection = if let Some((op_index, ple_config)) = ple_op {+ let ple_loader = parameter_tree.subtree("epilogue").subtree(&op_index.to_string());Some(PerLayerEmbeddingProjection::new(model_dim,@@ -369,6 +449,8 @@ impl<B: Backend> TransformerLayer<B> {mlp,post_mlp_norm,ple_projection,+ mixer_connection,+ mlp_connection,})}@@ -419,6 +501,8 @@ impl<B: Backend> TransformerLayer<B> {.map_err(TransformerLayerError::Backend)?;}+ // TODO: the mixer update is currently folded into the stream by the ShortcutMode::Add+ // fused into this norm kernel; replace with an explicit mixer_connection.encode_update.hidden = self.pre_mlp_norm.encode(&hidden, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)@@ -448,6 +532,9 @@ impl<B: Backend> TransformerLayer<B> {.map_err(TransformerLayerError::Backend)?;}+ // TODO: the MLP update is folded into the stream by the next layer's pre-norm (or the+ // output norm); the layer should return `mlp_connection.encode_update(shortcut, hidden)`+ // so the stream is passed by value, like lalamo's Sublayer does.if let Some(ple_projection) = &self.ple_projection {let per_layer_inputs = per_layer_inputs.expect("per-layer inputs required for PLE layer");ple_projection.encode(diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rsindex 420919bf4..e4ea16421 100644--- a/crates/uzu-engine/src/engine/language_model/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/mod.rs@@ -99,7 +99,9 @@ impl<B: Backend> Engine<B> {let generation_config = config.generation_config;let end_of_thinking_tag = match &config.token_codec_config {- AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) => token_codec_config.end_of_thinking_tag.clone(),+ AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) => {+ token_codec_config.end_of_thinking_tag().map(str::to_owned)+ },AnyTokenCodecConfig::RawTextCodecConfig(_) => None,};diff --git a/crates/uzu-engine/src/engine/mod.rs b/crates/uzu-engine/src/engine/mod.rsindex e9593328e..40cc8c6e7 100644--- a/crates/uzu-engine/src/engine/mod.rs+++ b/crates/uzu-engine/src/engine/mod.rs@@ -85,7 +85,12 @@ pub async fn resolve_model_context_length(model_path: &Path) -> Result<Option<u3Ok(transformer.layer_configs.iter()- .filter(|layer| matches!(layer.mixer_config, AnyTokenMixerConfig::AttentionConfig(_)))+ .filter(|layer| {+ matches!(+ layer.mixer.body_config,+ AnyTokenMixerConfig::AttentionConfig(_) | AnyTokenMixerConfig::SparseAttentionConfig(_)+ )+ }).filter_map(|layer| layer.rope_config.as_ref().map(|rope| *rope.max_sequence_length())).min())}diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex 812ebe301..fff097ae1 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -14,7 +14,7 @@ use crate::{backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,},- config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},+ config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig},data_type::DataType,encodable_block::{batch_topology::BatchTopology,@@ -115,11 +115,11 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let data_type = DataType::BF16;- let config: SpeculatorModelConfig = serde_json::from_slice(&file::read(model_path.join("config.json")).await?)?;- let AnySpeculatorConfig::DFlashSpeculatorConfig(config) = config.speculator_config;+ let AnySpeculatorConfig::DFlashSpeculatorConfig(config) =+ serde_json::from_slice(&file::read(model_path.join("config.json")).await?)?;let weight_loader = ParameterLoader::new(model_path.join("model.safetensors"), &*context).await?;- let speculator_tree = weight_loader.tree().subtree("speculator");+ let speculator_tree = weight_loader.tree();let dflash = DFlash::new(data_type,diff --git a/crates/uzu-engine/src/utils/strict_serde.rs b/crates/uzu-engine/src/utils/strict_serde.rsindex 695f4e44c..969bf246d 100644--- a/crates/uzu-engine/src/utils/strict_serde.rs+++ b/crates/uzu-engine/src/utils/strict_serde.rs@@ -14,7 +14,17 @@ macro_rules! impl_strict {};}-impl_strict!(Unsupported, String, f32, u32, u64, usize, bool, serde_json::Map<String, serde_json::Value>);+impl_strict!(+ Unsupported,+ String,+ f32,+ u32,+ u64,+ usize,+ bool,+ serde_json::Value,+ serde_json::Map<String, serde_json::Value>+);impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Box<T> {}impl<'de, T: DeserializeStrict<'de>> DeserializeStrict<'de> for Box<[T]> {}diff --git a/crates/uzu-engine/src/utils/version.rs b/crates/uzu-engine/src/utils/version.rsindex 03d9fafa0..15aa365d2 100644--- a/crates/uzu-engine/src/utils/version.rs+++ b/crates/uzu-engine/src/utils/version.rs@@ -1,2 +1,2 @@pub const VERSION: &str = env!("CARGO_PKG_VERSION");-pub const TOOLCHAIN_VERSION: &str = "0.17.2";+pub const TOOLCHAIN_VERSION: &str = "0.19.0";