Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add dflash configs (#530)

CCYehff01478edaa6
diff --git a/crates/backend-uzu/src/config/dflash.rs b/crates/backend-uzu/src/config/dflash.rs
new file mode 100644
index 000000000..37f4f7aea
--- /dev/null
+++ b/crates/backend-uzu/src/config/dflash.rs
@@ -0,0 +1,47 @@
+use proc_macros::uzu_config;
+
+use crate::config::{
+ linear::LinearConfig, mlp::dense_mlp::DenseMLPConfig, normalization::NormalizationConfig, rope::AnyRoPEConfig,
+};
+
+#[uzu_config]
+pub struct DFlashAttentionConfig {
+ pub linear_config: LinearConfig,
+ pub query_norm_config: NormalizationConfig,
+ pub key_norm_config: NormalizationConfig,
+ pub rope_config: AnyRoPEConfig,
+ pub num_heads: usize,
+ pub num_key_value_heads: usize,
+ pub head_dim: usize,
+ pub has_attention_biases: bool,
+ pub has_output_biases: bool,
+ /// Symmetric window: keys within +/- `sliding_window_size / 2` of the query
+ /// position, not the causal sliding window used by uzu attention.
+ pub sliding_window_size: Option<usize>,
+ pub scale: f32,
+}
+
+#[uzu_config]
+pub struct DFlashDraftLayerConfig {
+ pub attention_config: DFlashAttentionConfig,
+ pub input_norm_config: NormalizationConfig,
+ /// Sits in the pre-MLP-norm position: normalizes hidden + attention output
+ /// before the MLP, unlike uzu's post-mixer norm.
+ pub post_attention_norm_config: NormalizationConfig,
+ pub mlp_config: DenseMLPConfig,
+}
+
+#[uzu_config]
+pub struct DFlashDraftConfig {
+ pub model_dim: usize,
+ pub hidden_dim: usize,
+ pub block_size: usize,
+ pub mask_token_id: u64,
+ pub target_layer_ids: Box<[usize]>,
+ pub num_target_layers: usize,
+ pub vocab_size: usize,
+ pub context_projection_config: LinearConfig,
+ pub context_norm_config: NormalizationConfig,
+ pub layer_configs: Box<[DFlashDraftLayerConfig]>,
+ pub output_norm_config: NormalizationConfig,
+}
diff --git a/crates/backend-uzu/src/config/mod.rs b/crates/backend-uzu/src/config/mod.rs
index 8d8df401f..585b0e8bd 100644
--- a/crates/backend-uzu/src/config/mod.rs
+++ b/crates/backend-uzu/src/config/mod.rs
@@ -1,6 +1,7 @@
pub mod activation;
pub mod classifier;
pub mod decoder;
+pub mod dflash;
pub mod embedding;
pub mod linear;
pub mod mlp;
diff --git a/crates/backend-uzu/src/config/model/dflash_speculator.rs b/crates/backend-uzu/src/config/model/dflash_speculator.rs
new file mode 100644
index 000000000..9a59ac8d1
--- /dev/null
+++ b/crates/backend-uzu/src/config/model/dflash_speculator.rs
@@ -0,0 +1,8 @@
+use proc_macros::uzu_config;
+
+use crate::config::dflash::DFlashDraftConfig;
+
+#[uzu_config(super::ModelConfig)]
+pub struct DFlashSpeculatorConfig {
+ pub draft_config: DFlashDraftConfig,
+}
diff --git a/crates/backend-uzu/src/config/model/mod.rs b/crates/backend-uzu/src/config/model/mod.rs
index 0156e345e..68c5201d7 100644
--- a/crates/backend-uzu/src/config/model/mod.rs
+++ b/crates/backend-uzu/src/config/model/mod.rs
@@ -3,6 +3,7 @@ use proc_macros::uzu_config_abstract;
use crate::config::token_codec::AnyTokenCodecConfig;
pub mod classifier_model;
+pub mod dflash_speculator;
pub mod generation;
pub mod language_model;
pub mod tts_model;
@@ -10,7 +11,8 @@ pub mod tts_model;
#[uzu_config_abstract(
language_model::LanguageModelConfig,
classifier_model::ClassifierModelConfig,
- tts_model::TTSModelConfig
+ tts_model::TTSModelConfig,
+ dflash_speculator::DFlashSpeculatorConfig
)]
pub struct ModelConfig {
pub token_codec_config: AnyTokenCodecConfig,
diff --git a/crates/backend-uzu/src/config/token_codec/mod.rs b/crates/backend-uzu/src/config/token_codec/mod.rs
index 8999d3886..c65bca1ef 100644
--- a/crates/backend-uzu/src/config/token_codec/mod.rs
+++ b/crates/backend-uzu/src/config/token_codec/mod.rs
@@ -1,7 +1,8 @@
use proc_macros::uzu_config_abstract;
pub mod chat_codec;
+pub mod raw_text_codec;
pub mod tts_codec;
-#[uzu_config_abstract(chat_codec::ChatCodecConfig, tts_codec::TTSCodecConfig)]
+#[uzu_config_abstract(chat_codec::ChatCodecConfig, tts_codec::TTSCodecConfig, raw_text_codec::RawTextCodecConfig)]
pub struct TokenCodecConfig;
diff --git a/crates/backend-uzu/src/config/token_codec/raw_text_codec.rs b/crates/backend-uzu/src/config/token_codec/raw_text_codec.rs
new file mode 100644
index 000000000..dbd744f83
--- /dev/null
+++ b/crates/backend-uzu/src/config/token_codec/raw_text_codec.rs
@@ -0,0 +1,4 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::TokenCodecConfig)]
+pub struct RawTextCodecConfig;
diff --git a/crates/backend-uzu/src/inference/model_metadata.rs b/crates/backend-uzu/src/inference/model_metadata.rs
index adf2f5030..2dda242cd 100644
--- a/crates/backend-uzu/src/inference/model_metadata.rs
+++ b/crates/backend-uzu/src/inference/model_metadata.rs
@@ -20,5 +20,6 @@ pub fn resolve_model_specialization(model_path: &Path) -> Result<ModelSpecializa
AnyModelConfig::LanguageModelConfig(_) => ModelSpecialization::Chat {},
AnyModelConfig::ClassifierModelConfig(_) => ModelSpecialization::Classification {},
AnyModelConfig::TTSModelConfig(_) => ModelSpecialization::TextToSpeech {},
+ AnyModelConfig::DFlashSpeculatorConfig(_) => ModelSpecialization::Speculation {},
})
}
diff --git a/crates/backend-uzu/unit/forward_pass/traces/trace_validator.rs b/crates/backend-uzu/unit/forward_pass/traces/trace_validator.rs
index 27353fa1b..349c404c5 100644
--- a/crates/backend-uzu/unit/forward_pass/traces/trace_validator.rs
+++ b/crates/backend-uzu/unit/forward_pass/traces/trace_validator.rs
@@ -263,6 +263,11 @@ impl<B: Backend> TraceValidator<B> {
AnyModelConfig::TTSModelConfig(_) => {
return Err(Error::InvalidModelConfig("TTS trace validation is not supported".to_string()));
},
+ AnyModelConfig::DFlashSpeculatorConfig(_) => {
+ return Err(Error::InvalidModelConfig(
+ "DFlash speculator trace validation is not supported".to_string(),
+ ));
+ },
};
Ok(Self {