Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Cleanup dflash (#690)

uuuvnf01f19b27602
diff --git a/crates/backend-uzu/src/config/dflash.rs b/crates/backend-uzu/src/config/dflash.rs
index 84de9a796..71f06dbd8 100644
--- a/crates/backend-uzu/src/config/dflash.rs
+++ b/crates/backend-uzu/src/config/dflash.rs
@@ -1,20 +1,10 @@
use proc_macros::uzu_config;
use crate::config::{
- linear::LinearConfig, mlp::dense_mlp::DenseMLPConfig, normalization::NormalizationConfig, rope::AnyRoPEConfig,
- token_mixer::attention::AttentionConfig,
+ linear::LinearConfig, normalization::NormalizationConfig, rope::AnyRoPEConfig,
+ transformer_layer::TransformerLayerConfig,
};
-#[uzu_config]
-pub struct DFlashDraftLayerConfig {
- pub attention_config: AttentionConfig,
- pub input_norm_config: NormalizationConfig,
- /// Sits in the pre-MLP-norm position: normalizes hidden + attention output
- /// before the MLP, unlike uzu's post-mixer norm.
- pub post_attention_norm_config: NormalizationConfig,
- pub mlp_config: DenseMLPConfig,
-}
-
#[uzu_config]
pub struct DFlashDraftConfig {
pub model_dim: usize,
@@ -27,6 +17,6 @@ pub struct DFlashDraftConfig {
pub context_projection_config: LinearConfig,
pub context_norm_config: NormalizationConfig,
pub rope_config: AnyRoPEConfig,
- pub layer_configs: Box<[DFlashDraftLayerConfig]>,
+ pub layer_configs: Box<[TransformerLayerConfig]>,
pub output_norm_config: NormalizationConfig,
}
diff --git a/crates/backend-uzu/src/config/speculator/dflash.rs b/crates/backend-uzu/src/config/speculator/dflash.rs
index 01d108c0c..626769382 100644
--- a/crates/backend-uzu/src/config/speculator/dflash.rs
+++ b/crates/backend-uzu/src/config/speculator/dflash.rs
@@ -5,5 +5,5 @@ use crate::config::{dflash::DFlashDraftConfig, weaver::WeaverConfig};
#[uzu_config(super::SpeculatorConfig)]
pub struct DFlashSpeculatorConfig {
pub draft_config: DFlashDraftConfig,
- pub weaver_config: Option<WeaverConfig>,
+ pub weaver_config: WeaverConfig,
}
diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rs
index 63cbd6853..6b50c06c5 100644
--- a/crates/backend-uzu/src/encodable_block/dflash.rs
+++ b/crates/backend-uzu/src/encodable_block/dflash.rs
@@ -1,37 +1,36 @@
+use std::any::Any;
+
use thiserror::Error;
use crate::{
array::size_for_shape,
backends::common::{
- Allocation, Backend, Encoder, Kernels, gpu_types::trie::TrieNode, kernel::radix_top_k_small::RadixTopKSmall,
- },
- config::{
- dflash::{DFlashDraftConfig, DFlashDraftLayerConfig},
- mlp::AnyMLPConfig,
- rope::AnyRoPEConfig,
+ Allocation, Backend, Encoder, Kernels,
+ gpu_types::trie::TrieNode,
+ kernel::radix_top_k_small::{MAX_K, RadixTopKSmall},
},
+ config::{dflash::DFlashDraftConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig},
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
embedding::{Embedding, EmbeddingError},
linear::{Linear, LinearBlockError},
mixer::{
- Mixer, MixerState,
+ MixerState,
attention::{
ATTENTION_SUFFIX_CAPACITY, Attention, AttentionNewError, AttentionState, rope::PrecalculatedRoPE,
},
},
- mlp::{Mlp, MlpBlockError},
+ mlp::MlpBlockError,
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
+ transformer_layer::{TransformerLayer, TransformerLayerError},
},
parameters::{ParameterLoaderError, ParameterTree},
utils::maybe_mut::MaybeMut,
};
-const RADIX_TOP_K_MAX: usize = 512;
-
pub struct DFlashState<B: Backend> {
- layer_states: Box<[AttentionState<B>]>,
+ layer_states: Box<[Box<dyn MixerState<B>>]>,
context_length: usize,
context_capacity: usize,
}
@@ -42,12 +41,12 @@ impl<B: Backend> DFlashState<B> {
}
}
-pub struct DFlashDraft<B: Backend> {
+pub struct DFlash<B: Backend> {
target_feature_projection: Box<dyn Linear<B>>,
projected_feature_norm: Normalization<B>,
state_kv_projection: Box<dyn Linear<B>>,
layer_kv_dim: usize,
- layers: Box<[DFlashDraftLayer<B>]>,
+ layers: Box<[TransformerLayer<B>]>,
output_norm: Normalization<B>,
top_k: <B::Kernels as Kernels>::RadixTopKSmall,
rope_config: AnyRoPEConfig,
@@ -59,27 +58,20 @@ pub struct DFlashDraft<B: Backend> {
data_type: DataType,
}
-pub struct DFlashDraftOutput<B: Backend> {
- pub candidates: TopKCandidates<B>,
+pub struct DFlashOutput<B: Backend> {
pub draft_hidden: Allocation<B>,
+ pub candidates: TopKCandidates<B>,
}
pub struct TopKCandidates<B: Backend> {
pub ids: Allocation<B>,
- pub scores: Allocation<B>,
+ pub logits: Allocation<B>,
pub rows: usize,
pub candidates_per_row: usize,
}
-struct DFlashDraftLayer<B: Backend> {
- attention: Attention<B>,
- pre_attention_norm: Normalization<B>,
- pre_mlp_norm: Normalization<B>,
- mlp: Box<dyn Mlp<B>>,
-}
-
#[derive(Debug, Error)]
-pub enum DFlashDraftNewError<B: Backend> {
+pub enum DFlashNewError<B: Backend> {
#[error("Parameter loader error: {0}")]
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("Linear error: {0}")]
@@ -90,6 +82,8 @@ pub enum DFlashDraftNewError<B: Backend> {
Mlp(#[from] MlpBlockError<B>),
#[error("Attention error: {0}")]
Attention(#[from] AttentionNewError<B>),
+ #[error("Transformer layer error: {0}")]
+ TransformerLayer(#[from] TransformerLayerError<B>),
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
#[error("invalid DFlash attention config: {0}")]
@@ -97,30 +91,23 @@ pub enum DFlashDraftNewError<B: Backend> {
}
#[derive(Debug, Error)]
-pub enum DFlashDraftEncodeError<B: Backend> {
+pub enum DFlashEncodeError<B: Backend> {
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
#[error("Embedding error: {0}")]
Embedding(#[from] EmbeddingError<B>),
}
-impl<B: Backend> DFlashDraft<B> {
+impl<B: Backend> DFlash<B> {
pub fn new(
context: &B::Context,
config: &DFlashDraftConfig,
parameter_tree: &ParameterTree<B>,
data_type: DataType,
- ) -> Result<Self, DFlashDraftNewError<B>> {
+ ) -> Result<Self, DFlashNewError<B>> {
let vocab_size = config.vocab_size as u32;
let mask_token_id = config.mask_token_id as u32;
- if config.layer_configs.is_empty() {
- return Err(DFlashDraftNewError::InvalidAttentionConfig("at least one DFlash layer is required"));
- }
- if config.block_size == 0 || config.block_size > ATTENTION_SUFFIX_CAPACITY {
- return Err(DFlashDraftNewError::InvalidAttentionConfig(
- "DFlash block_size must be in 1..=attention suffix capacity",
- ));
- }
+ assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");
let target_feature_projection = <dyn Linear<B>>::new(
config.model_dim * config.target_layer_ids.len(),
[config.model_dim],
@@ -140,8 +127,11 @@ impl<B: Backend> DFlashDraft<B> {
context,
)?;
let layers_tree = parameter_tree.subtree("layers")?;
- let layer_kv_dim =
- 2 * config.layer_configs[0].attention_config.num_groups * config.layer_configs[0].attention_config.head_dim;
+ let first_layer_config = config.layer_configs.first().expect("DFlash draft model must have at least one layer");
+ let AnyTokenMixerConfig::AttentionConfig(attention_config) = &first_layer_config.mixer_config else {
+ return Err(DFlashNewError::InvalidAttentionConfig("DFlash layers must use attention mixers"));
+ };
+ let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;
let state_kv_projection = <dyn Linear<B>>::new(
config.model_dim,
[config.layer_configs.len() * layer_kv_dim],
@@ -155,18 +145,18 @@ impl<B: Backend> DFlashDraft<B> {
.iter()
.enumerate()
.map(|(index, layer_config)| {
- DFlashDraftLayer::new(
+ TransformerLayer::new(
context,
- index,
config.model_dim,
config.hidden_dim,
+ config.layer_configs.len(),
layer_config,
- &config.rope_config,
+ index,
&layers_tree.subtree(&index.to_string())?,
data_type,
)
})
- .collect::<Result<Box<[_]>, DFlashDraftNewError<B>>>()?;
+ .collect::<Result<Box<[_]>, TransformerLayerError<B>>>()?;
let output_norm = Normalization::new(
config.model_dim,
None,
@@ -178,7 +168,7 @@ impl<B: Backend> DFlashDraft<B> {
context,
)?;
let top_k =
- <B::Kernels as Kernels>::RadixTopKSmall::new(context, vocab_size).map_err(DFlashDraftNewError::Backend)?;
+ <B::Kernels as Kernels>::RadixTopKSmall::new(context, vocab_size).map_err(DFlashNewError::Backend)?;
Ok(Self {
target_feature_projection,
@@ -207,7 +197,7 @@ impl<B: Backend> DFlashDraft<B> {
let layer_states = self
.layers
.iter()
- .map(|layer| AttentionState::create_empty(&layer.attention, Some(context_capacity), context))
+ .map(|layer| layer.mixer.create_empty_state(Some(context_capacity), context))
.collect::<Result<Box<[_]>, B::Error>>()?;
Ok(DFlashState {
layer_states,
@@ -216,7 +206,7 @@ impl<B: Backend> DFlashDraft<B> {
})
}
- pub fn append_state(
+ pub fn encode_accept(
&self,
state: &mut DFlashState<B>,
target_features: &[Allocation<B>],
@@ -227,7 +217,7 @@ impl<B: Backend> DFlashDraft<B> {
return Ok(());
}
- encoder.push_debug_group("dflash append state");
+ encoder.push_debug_group("dflash accept");
let num_tokens = accepted_indices.len();
let captured_layer_count = self.target_feature_input_dim / self.model_dim;
@@ -275,11 +265,17 @@ impl<B: Backend> DFlashDraft<B> {
);
}
}
- for ((layer, attention_state), key_value) in
+ for ((layer, mixer_state), key_value) in
self.layers.iter().zip(state.layer_states.iter_mut()).zip(layer_key_values)
{
- attention_state.prepare(state.context_length, num_tokens, encoder.context())?;
- layer.attention.append_projected_kv(key_value, &rope, num_tokens, attention_state, encoder)?;
+ mixer_state.prepare(state.context_length, num_tokens, encoder.context())?;
+ let attention = (layer.mixer.as_ref() as &dyn Any)
+ .downcast_ref::<Attention<B>>()
+ .expect("DFlash draft layers must use attention mixers");
+ let attention_state = (mixer_state.as_mut() as &mut dyn Any)
+ .downcast_mut::<AttentionState<B>>()
+ .expect("DFlash draft layer states must be attention states");
+ attention.append_projected_kv(key_value, &rope, num_tokens, attention_state, encoder)?;
}
state.context_length += num_tokens;
@@ -289,17 +285,28 @@ impl<B: Backend> DFlashDraft<B> {
Ok(())
}
- fn encode_hidden(
+ pub fn encode_draft(
&self,
state: &mut DFlashState<B>,
- token_embeddings: Allocation<B>,
+ target_output_token: u32,
+ target_embedding: &Embedding<B>,
+ candidate_count: usize,
encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
+ ) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {
+ encoder.push_debug_group("dflash draft");
+
let batch_dim = self.block_size;
assert!(
state.context_length + batch_dim <= self.max_context_length,
"DFlash block positions exceed configured RoPE capacity"
);
+
+ let mut tokens = vec![self.mask_token_id; batch_dim];
+ tokens[0] = target_output_token;
+ let token_ids = encoder.allocate_constant_from_slice(&tokens).map_err(DFlashEncodeError::Backend)?;
+
+ let token_embeddings = target_embedding.encode_lookup(&token_ids, batch_dim, encoder)?;
+
let nodes = (0..batch_dim)
.map(|index| TrieNode {
trie_start: index as u32,
@@ -309,142 +316,62 @@ impl<B: Backend> DFlashDraft<B> {
.collect::<Box<[_]>>();
let batch_topology = BatchTopology::new(&nodes, true);
let token_positions = (state.context_length..state.context_length + batch_dim).collect::<Box<[_]>>();
- let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, encoder)?;
+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, encoder)
+ .map_err(DFlashEncodeError::Backend)?;
let mut hidden = token_embeddings;
- let mut residual = encoder.allocate_scratch(hidden.size())?;
- for (layer, attention_state) in self.layers.iter().zip(state.layer_states.iter_mut()) {
- attention_state.prepare(state.context_length, batch_dim, encoder.context())?;
- let normalized = layer.pre_attention_norm.encode(&hidden, 0, batch_dim, Some(&mut residual), encoder)?;
- let attention_output = layer.attention.encode(
- normalized,
- Some(&rope),
- &batch_topology,
- Some(MaybeMut::Mut(attention_state)),
- encoder,
- )?;
- let normalized =
- layer.pre_mlp_norm.encode(&attention_output, 0, batch_dim, Some(&mut residual), encoder)?;
- hidden = layer.mlp.encode(normalized, batch_dim, encoder)?;
+ let mut residual = encoder.allocate_scratch(hidden.size()).map_err(DFlashEncodeError::Backend)?;
+ for (layer, mixer_state) in self.layers.iter().zip(state.layer_states.iter_mut()) {
+ mixer_state
+ .prepare(state.context_length, batch_dim, encoder.context())
+ .map_err(DFlashEncodeError::Backend)?;
+ hidden = layer
+ .encode(
+ hidden,
+ &mut residual,
+ None,
+ Some(&rope),
+ &batch_topology,
+ Some(MaybeMut::Mut(mixer_state.as_mut())),
+ encoder,
+ )
+ .map_err(DFlashEncodeError::Backend)?;
}
-
- self.output_norm.encode(&hidden, 0, batch_dim, Some(&mut residual), encoder)
- }
-
- pub fn encode_draft(
- &self,
- state: &mut DFlashState<B>,
- target_output_token: u32,
- target_embedding: &Embedding<B>,
- candidate_count: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<DFlashDraftOutput<B>, DFlashDraftEncodeError<B>> {
- encoder.push_debug_group("dflash draft");
-
- let mut token_ids = encoder
- .allocate_constant(self.block_size * DataType::U32.size_in_bytes())
- .map_err(DFlashDraftEncodeError::Backend)?;
- let mut tokens = vec![self.mask_token_id; self.block_size];
- tokens[0] = target_output_token;
- token_ids.copyin(&tokens);
-
- let token_embeddings = target_embedding.encode_lookup(&token_ids, self.block_size, encoder)?;
- let draft_hidden =
- self.encode_hidden(state, token_embeddings, encoder).map_err(DFlashDraftEncodeError::Backend)?;
+ let draft_hidden = self
+ .output_norm
+ .encode(&hidden, 0, batch_dim, Some(&mut residual), encoder)
+ .map_err(DFlashEncodeError::Backend)?;
// The first block row is the target's output token; only the lookahead rows are ranked.
- let lookahead_rows = self.block_size - 1;
+ let lookahead_rows = batch_dim - 1;
let row_bytes = target_embedding.model_dim() * DataType::BF16.size_in_bytes();
let mut lookahead_hidden =
- encoder.allocate_scratch(lookahead_rows * row_bytes).map_err(DFlashDraftEncodeError::Backend)?;
- encoder.encode_copy(&draft_hidden, row_bytes..self.block_size * row_bytes, &mut lookahead_hidden, ..);
+ encoder.allocate_scratch(lookahead_rows * row_bytes).map_err(DFlashEncodeError::Backend)?;
+ encoder.encode_copy(&draft_hidden, row_bytes..batch_dim * row_bytes, &mut lookahead_hidden, ..);
let logits = target_embedding.encode_readout(lookahead_rows, &lookahead_hidden, DataType::F32, encoder)?;
- let candidates = self
- .encode_candidates(&logits, lookahead_rows, candidate_count, encoder)
- .map_err(DFlashDraftEncodeError::Backend)?;
- drop(logits);
- drop(token_ids);
- drop(lookahead_hidden);
+
+ assert!(candidate_count > 0 && candidate_count <= MAX_K as usize);
+ let mut ids = encoder
+ .allocate_scratch(size_for_shape(&[lookahead_rows, candidate_count], DataType::U32))
+ .map_err(DFlashEncodeError::Backend)?;
+ let mut candidate_logits = encoder
+ .allocate_scratch(size_for_shape(&[lookahead_rows, candidate_count], DataType::F32))
+ .map_err(DFlashEncodeError::Backend)?;
+ self.top_k
+ .encode(&logits, &mut ids, &mut candidate_logits, lookahead_rows as u32, candidate_count as u32, encoder)
+ .map_err(DFlashEncodeError::Backend)?;
+ let candidates = TopKCandidates {
+ ids,
+ logits: candidate_logits,
+ rows: lookahead_rows,
+ candidates_per_row: candidate_count,
+ };
encoder.pop_debug_group();
- Ok(DFlashDraftOutput {
- candidates,
+ Ok(DFlashOutput {
draft_hidden,
- })
- }
-
- fn encode_candidates(
- &self,
- logits: &Allocation<B>,
- rows: usize,
- k: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<TopKCandidates<B>, B::Error> {
- assert!(k > 0 && k <= RADIX_TOP_K_MAX);
- let mut ids = encoder.allocate_scratch(size_for_shape(&[rows, k], DataType::U32))?;
- let mut scores = encoder.allocate_scratch(size_for_shape(&[rows, k], DataType::F32))?;
- self.top_k.encode(logits, &mut ids, &mut scores, rows as u32, k as u32, encoder)?;
- Ok(TopKCandidates {
- ids,
- scores,
- rows,
- candidates_per_row: k,
- })
- }
-}
-
-impl<B: Backend> DFlashDraftLayer<B> {
- fn new(
- context: &B::Context,
- layer_index: usize,
- model_dim: usize,
- hidden_dim: usize,
- config: &DFlashDraftLayerConfig,
- rope_config: &AnyRoPEConfig,
- parameter_tree: &ParameterTree<B>,
- data_type: DataType,
- ) -> Result<Self, DFlashDraftNewError<B>> {
- let (attention, _) = Attention::new(
- model_dim,
- data_type,
- Some(rope_config),
- &config.attention_config,
- &parameter_tree.subtree("attention")?,
- context,
- )?;
- let pre_attention_norm = Normalization::new(
- model_dim,
- None,
- if layer_index == 0 {
- ShortcutMode::Copy
- } else {
- ShortcutMode::Add
- },
- PostLayerScalar::None,
- data_type,
- &config.input_norm_config,
- &parameter_tree.subtree("input_norm")?,
- context,
- )?;
- let pre_mlp_norm = Normalization::new(
- model_dim,
- None,
- ShortcutMode::Add,
- PostLayerScalar::None,
- data_type,
- &config.post_attention_norm_config,
- &parameter_tree.subtree("post_attention_norm")?,
- context,
- )?;
- let mlp_config = AnyMLPConfig::DenseMLPConfig(config.mlp_config.clone());
- let (mlp, _) =
- <dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, &parameter_tree.subtree("mlp")?, data_type)?;
- Ok(Self {
- attention,
- pre_attention_norm,
- pre_mlp_norm,
- mlp,
+ candidates,
})
}
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/mod.rs
index ba2e3b555..c3ae9178c 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/mod.rs
@@ -48,7 +48,7 @@ impl<'a, B: Backend> MaybeMut<'a, dyn MixerState<B>> {
}
}
-pub trait Mixer<B: Backend>: Send + Sync {
+pub trait Mixer<B: Backend>: Any + Send + Sync {
fn speculation_supported(&self) -> bool;
fn max_context_length(&self) -> Option<usize>;
diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rs
index e6426d7dd..6f9df8dd6 100644
--- a/crates/backend-uzu/src/encodable_block/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mod.rs
@@ -13,3 +13,4 @@ pub mod sampling;
pub mod transformer;
pub mod transformer_layer;
pub mod weaver;
+pub mod weaver_layer;
diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rs
index 21ff38da8..d9982e793 100644
--- a/crates/backend-uzu/src/encodable_block/transformer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer.rs
@@ -122,7 +122,9 @@ impl<B: Backend> Transformer<B> {
let layer = TransformerLayer::new(
context,
- transformer_config,
+ transformer_config.model_dim,
+ transformer_config.hidden_dim,
+ transformer_config.layer_configs.len(),
layer_config,
layer_index,
&layer_loader,
diff --git a/crates/backend-uzu/src/encodable_block/transformer_layer.rs b/crates/backend-uzu/src/encodable_block/transformer_layer.rs
index 32bb92d25..c5047a3ca 100644
--- a/crates/backend-uzu/src/encodable_block/transformer_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer_layer.rs
@@ -2,7 +2,7 @@ use thiserror::Error;
use crate::{
backends::common::{Allocation, Backend, Encoder},
- config::{transformer::TransformerConfig, transformer_layer::TransformerLayerConfig},
+ config::transformer_layer::TransformerLayerConfig,
data_type::DataType,
encodable_block::{
batch_topology::BatchTopology,
@@ -50,7 +50,9 @@ pub struct TransformerLayer<B: Backend> {
impl<B: Backend> TransformerLayer<B> {
pub fn new(
context: &B::Context,
- transformer_config: &TransformerConfig,
+ model_dim: usize,
+ hidden_dim: usize,
+ num_layers: usize,
layer_config: &TransformerLayerConfig,
layer_index: usize,
parameter_tree: &ParameterTree<B>,
@@ -82,7 +84,7 @@ impl<B: Backend> TransformerLayer<B> {
};
let (mixer, mixer_hadamard_factors) = <dyn Mixer<B>>::new(
- transformer_config.model_dim,
+ model_dim,
data_type,
layer_config.rope_config.as_ref(),
&layer_config.mixer_config,
@@ -92,7 +94,7 @@ impl<B: Backend> TransformerLayer<B> {
let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {
Some(Normalization::new(
- transformer_config.model_dim,
+ model_dim,
mixer_hadamard_factors,
if layer_index > 0 {
ShortcutMode::Add
@@ -114,7 +116,7 @@ impl<B: Backend> TransformerLayer<B> {
let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {
Some(Normalization::new(
- transformer_config.model_dim,
+ model_dim,
None,
ShortcutMode::None,
PostLayerScalar::None,
@@ -129,15 +131,15 @@ impl<B: Backend> TransformerLayer<B> {
let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(
&layer_config.mlp_config,
- transformer_config.model_dim,
- layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),
+ model_dim,
+ layer_config.hidden_dim.unwrap_or(hidden_dim),
context,
&parameter_tree.subtree("mlp")?,
data_type,
)?;
let pre_mlp_norm = Normalization::new(
- transformer_config.model_dim,
+ model_dim,
mlp_input_hadamard_factors,
ShortcutMode::Add,
residual_sum_scalar,
@@ -149,7 +151,7 @@ impl<B: Backend> TransformerLayer<B> {
let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(Normalization::new(
- transformer_config.model_dim,
+ model_dim,
None,
ShortcutMode::None,
output_scalar,
@@ -167,8 +169,8 @@ impl<B: Backend> TransformerLayer<B> {
PerLayerEmbeddingProjection::new(
context,
ple_config,
- transformer_config.model_dim,
- transformer_config.layer_configs.len(),
+ model_dim,
+ num_layers,
post_layer_scalar.unwrap_or(1.0),
data_type,
&ple_loader,
diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rs
index 082d67f2a..9d31e5e53 100644
--- a/crates/backend-uzu/src/encodable_block/weaver.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver.rs
@@ -3,104 +3,31 @@ use thiserror::Error;
use crate::{
array::size_for_shape,
backends::common::{
- Allocation, AsBufferRangeMut, Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
gpu_types::weaver::{
CANDIDATES_MAX, FRONTIER_MAX_SLOTS, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx,
},
kernel::{
- AncestorAttentionKernel, AttentionPrepareKernel, WeaverFrontierInsertChildrenKernel,
- WeaverFrontierSelectKernel, WeaverTopChildrenKernel,
+ AncestorAttentionKernel, WeaverFrontierInsertChildrenKernel, WeaverFrontierSelectKernel,
+ WeaverTopChildrenKernel,
},
},
- config::{
- activation::{AnyActivation, silu::SiLU},
- linear::LinearConfig,
- mlp::{AnyMLPConfig, dense_mlp::DenseMLPConfig},
- rope::AnyRoPEConfig,
- weaver::WeaverConfig,
- },
+ config::{rope::AnyRoPEConfig, weaver::WeaverConfig},
data_type::DataType,
encodable_block::{
+ dflash::TopKCandidates,
embedding::{Embedding, EmbeddingError},
linear::{Linear, LinearBlockError},
- mixer::attention::{
- AttentionStateType,
- core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},
- rope::PrecalculatedRoPE,
- },
- mlp::{Mlp, MlpBlockError},
+ mixer::attention::{AttentionStateType, core::AttentionCoreEncodeArguments, rope::PrecalculatedRoPE},
+ mlp::MlpBlockError,
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
+ weaver_layer::{PreparedPrefixAttention, WeaverLayer},
},
parameters::{ParameterLoaderError, ParameterTree},
};
-const DATA_TYPE: DataType = DataType::BF16;
-const ROPE_DATA_TYPE: DataType = DataType::F32;
-
-fn weaver_mlp_config(linear_config: LinearConfig) -> AnyMLPConfig {
- AnyMLPConfig::DenseMLPConfig(DenseMLPConfig::unclipped(
- linear_config,
- AnyActivation::SiLU(SiLU::default()),
- true,
- true,
- ))
-}
-
-struct TopKChildren<B: Backend> {
- token_ids: Allocation<B>,
- logprobs: Allocation<B>,
-}
-
-struct PrefixLayerOutput<B: Backend> {
- mlp_delta: Allocation<B>,
- kv_cache: Allocation<B>,
-}
-
-struct PreparedPrefixAttention<B: Backend> {
- queries: Allocation<B>,
- kv_cache: Allocation<B>,
-}
-
-struct PrefixKvCache<B: Backend> {
- layers: Box<[Allocation<B>]>,
- length: usize,
-}
-
-struct NodeExpansionKvCache<B: Backend> {
- layers: Box<[Allocation<B>]>,
- capacity: u32,
-}
-
-struct NodeBatch<'a, B: Backend> {
- count: usize,
- token_ids: &'a Allocation<B>,
- metadata: &'a Allocation<B>,
- ancestor_indices: &'a Allocation<B>,
- ancestor_stride: usize,
-}
-
-struct CandidateBatch<'a, B: Backend> {
- ids: &'a Allocation<B>,
- logits: &'a Allocation<B>,
- candidates_per_node: usize,
- depth_seeds: &'a Allocation<B>,
-}
-
-pub struct CandidatePool<'a, B: Backend> {
- pub ids: &'a Allocation<B>,
- pub logits: &'a Allocation<B>,
- pub depth_count: usize,
- pub candidates_per_depth: usize,
-}
-
-pub struct WeaverInputs<'a, B: Backend> {
- pub target_hidden: &'a Allocation<B>,
- pub draft_hidden: &'a Allocation<B>,
- pub target_embedding: &'a Embedding<B>,
- pub candidates: CandidatePool<'a, B>,
- pub depth_seeds: &'a [u64],
- pub root_token_id: u32,
-}
+pub(crate) const DATA_TYPE: DataType = DataType::BF16;
+pub(crate) const ROPE_DATA_TYPE: DataType = DataType::F32;
pub struct TreeShape {
pub budget: usize,
@@ -120,7 +47,34 @@ pub struct ProposalNode {
impl<B: Backend> EncodedWeaverTree<B> {
pub fn read_nodes(self) -> Vec<ProposalNode> {
- nodes_from_packed_tree(&self.packed_tree.copyout::<u32>())
+ let packed_tree = &self.packed_tree.copyout::<u32>();
+
+ let tree_slot_count = packed_tree.len() / TreeIdx::COUNT;
+ let field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];
+ let mut slot_to_node_index = vec![usize::MAX; tree_slot_count];
+ let mut nodes: Vec<ProposalNode> = Vec::with_capacity(tree_slot_count);
+ for slot in 0..tree_slot_count {
+ if field(TreeIdx::Valid, slot) == 0 {
+ continue;
+ }
+ let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;
+ let parent = (parent_slot >= 0).then(|| {
+ let parent = slot_to_node_index[parent_slot as usize];
+ assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");
+ parent
+ });
+ let index = nodes.len();
+ slot_to_node_index[slot] = index;
+ if let Some(parent) = parent {
+ nodes[parent].child_indices.push(index);
+ }
+ nodes.push(ProposalNode {
+ token_id: field(TreeIdx::TokenId, slot),
+ depth: field(TreeIdx::Depth, slot) as usize,
+ child_indices: Vec::new(),
+ });
+ }
+ nodes
}
}
@@ -141,27 +95,6 @@ pub struct Weaver<B: Backend> {
max_depth: usize,
}
-struct WeaverLayer<B: Backend> {
- // Attention
- pre_attention_norm: Normalization<B>,
- qkv_projection: Box<dyn Linear<B>>,
- attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
- prefix_attention: AttentionCores<B>,
- ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,
- out_projection: Box<dyn Linear<B>>,
-
- // MLP
- pre_mlp_norm: Normalization<B>,
- mlp: Box<dyn Mlp<B>>,
-
- // Geometry
- attention_scale: f32,
- model_dim: usize,
- num_heads: usize,
- head_dim: usize,
- max_depth: usize,
-}
-
#[derive(Debug, Error)]
pub enum WeaverNewError<B: Backend> {
#[error("parameter loader error: {0}")]
@@ -314,39 +247,111 @@ impl<B: Backend> Weaver<B> {
pub fn encode_tree(
&self,
- inputs: WeaverInputs<'_, B>,
+ target_hidden: &Allocation<B>,
+ draft_hidden: &Allocation<B>,
+ target_embedding: &Embedding<B>,
+ candidates: &TopKCandidates<B>,
+ depth_seeds: &[u64],
+ root_token_id: u32,
shape: TreeShape,
encoder: &mut Encoder<B>,
) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
encoder.push_debug_group("weaver tree");
+
let tree_slot_count = shape.budget + 1;
let frontier_capacity = tree_slot_count * shape.children_per_node;
- let lookahead_count = self.max_depth.min(inputs.candidates.depth_count);
+ let lookahead_count = self.max_depth.min(candidates.rows);
let ancestor_stride = self.max_depth;
if shape.budget == 0
|| shape.frontier_width == 0
|| shape.children_per_node == 0
- || shape.children_per_node > inputs.candidates.candidates_per_depth
+ || shape.children_per_node > candidates.candidates_per_row
|| frontier_capacity > FRONTIER_MAX_SLOTS
|| lookahead_count == 0
- || inputs.depth_seeds.len() != self.max_depth
+ || depth_seeds.len() != self.max_depth
{
return Err(WeaverEncodeError::InvalidTreeInput);
}
- let rope = self.precalculate_rope(encoder).map_err(WeaverEncodeError::Backend)?;
+ let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();
+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+
+ // Prefix pass: run the target token and the draft lookahead rows through
+ // every layer, collecting per-layer KV caches.
+ let prefix_length = lookahead_count + 1;
+ let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();
+ let mut prefix_hidden = encoder
+ .allocate_scratch(size_for_shape(&[prefix_length, self.target_model_dim], DATA_TYPE))
+ .map_err(WeaverEncodeError::Backend)?;
+ encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);
+ encoder.encode_copy(
+ draft_hidden,
+ hidden_row_bytes..prefix_length * hidden_row_bytes,
+ &mut prefix_hidden,
+ hidden_row_bytes..prefix_length * hidden_row_bytes,
+ );
+ let normalized_prefix = self
+ .hidden_state_norm
+ .encode(&prefix_hidden, 0, prefix_length, None, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut residual_input = self
+ .hidden_state_projection
+ .encode(normalized_prefix, prefix_length, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");
+ let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
+ let mut prefix_kv_layers = Vec::with_capacity(self.layers.len());
+ for layer in preceding_layers {
+ let PreparedPrefixAttention {
+ queries,
+ kv_cache,
+ } = layer
+ .encode_prefix_attention(&residual_input, &mut residual_state, &rope, prefix_length, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let state_type = AttentionStateType::Full {
+ length: 0,
+ };
+ let kv_plane_bytes = size_for_shape(&[prefix_length, self.model_dim], DATA_TYPE);
+ let attention_output = layer
+ .prefix_attention
+ .encode(
+ AttentionCoreEncodeArguments {
+ queries: &queries,
+ keys: &kv_cache,
+ values: (&kv_cache, kv_plane_bytes),
+ suffix_length: prefix_length,
+ trie: None,
+ sinks: None,
+ state_type: &state_type,
+ },
+ encoder,
+ )
+ .map_err(WeaverEncodeError::Backend)?;
+ residual_input = layer
+ .encode_post_attention(attention_output, &mut residual_state, prefix_length, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ prefix_kv_layers.push(kv_cache);
+ }
+ prefix_kv_layers.push(
+ last_layer
+ .encode_prefix_attention(&residual_input, &mut residual_state, &rope, prefix_length, encoder)
+ .map_err(WeaverEncodeError::Backend)?
+ .kv_cache,
+ );
- let prefix_cache = self
- .encode_prefix(inputs.target_hidden, inputs.draft_hidden, &rope, lookahead_count, encoder)
+ // Per-layer KV cache for tree nodes, one slot per packed-tree slot.
+ let node_kv_size = size_for_shape(&[2, tree_slot_count, self.model_dim], DATA_TYPE);
+ let mut node_kv_layers = (0..self.layers.len())
+ .map(|_| encoder.allocate_scratch(node_kv_size))
+ .collect::<Result<Vec<_>, _>>()
.map_err(WeaverEncodeError::Backend)?;
- let mut node_cache =
- self.create_node_expansion_kv_cache(tree_slot_count, encoder).map_err(WeaverEncodeError::Backend)?;
let mut tree_init = vec![0u32; TreeIdx::COUNT * tree_slot_count];
for slot in 0..tree_slot_count {
tree_init[TreeIdx::ParentSlot as usize * tree_slot_count + slot] = FRONTIER_NO_WINNER;
}
- tree_init[TreeIdx::TokenId as usize * tree_slot_count] = inputs.root_token_id;
+ tree_init[TreeIdx::TokenId as usize * tree_slot_count] = root_token_id;
tree_init[TreeIdx::Valid as usize * tree_slot_count] = 1;
let mut packed_tree = encoder.allocate_constant_from_slice(&tree_init).map_err(WeaverEncodeError::Backend)?;
@@ -358,7 +363,7 @@ impl<B: Backend> Weaver<B> {
.map_err(WeaverEncodeError::Backend)?;
let mut initial_node_token_ids = vec![0u32; shape.frontier_width];
- initial_node_token_ids[0] = inputs.root_token_id;
+ initial_node_token_ids[0] = root_token_id;
let mut initial_node_valid = vec![0u32; shape.frontier_width];
initial_node_valid[0] = 1;
let mut node_token_ids =
@@ -372,13 +377,13 @@ impl<B: Backend> Weaver<B> {
let mut node_valid =
encoder.allocate_constant_from_slice(&initial_node_valid).map_err(WeaverEncodeError::Backend)?;
let mut node_candidate_ids = encoder
- .allocate_constant_from_slice(&vec![0u32; shape.frontier_width * inputs.candidates.candidates_per_depth])
+ .allocate_constant_from_slice(&vec![0u32; shape.frontier_width * candidates.candidates_per_row])
.map_err(WeaverEncodeError::Backend)?;
let mut node_candidate_logits = encoder
- .allocate_constant_from_slice(&vec![0.0f32; shape.frontier_width * inputs.candidates.candidates_per_depth])
+ .allocate_constant_from_slice(&vec![0.0f32; shape.frontier_width * candidates.candidates_per_row])
.map_err(WeaverEncodeError::Backend)?;
- let depth_seeds =
- encoder.allocate_constant_from_slice(inputs.depth_seeds).map_err(WeaverEncodeError::Backend)?;
+ let depth_seeds_buffer =
+ encoder.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;
let mut batch_start_slot = 0;
while batch_start_slot < tree_slot_count {
@@ -396,8 +401,8 @@ impl<B: Backend> Weaver<B> {
&mut node_metadata,
&mut node_ancestor_indices,
&mut node_valid,
- inputs.candidates.ids,
- inputs.candidates.logits,
+ &candidates.ids,
+ &candidates.logits,
&mut node_candidate_ids,
&mut node_candidate_logits,
frontier_capacity as u32,
@@ -407,8 +412,8 @@ impl<B: Backend> Weaver<B> {
ancestor_stride as u32,
self.max_depth as u32,
lookahead_count as u32,
- inputs.candidates.depth_count as u32,
- inputs.candidates.candidates_per_depth as u32,
+ candidates.rows as u32,
+ candidates.candidates_per_row as u32,
encoder,
);
}
@@ -416,39 +421,103 @@ impl<B: Backend> Weaver<B> {
break; // packed-tree slots filled
}
let (batch_candidate_ids, batch_candidate_logits) = if batch_start_slot == 0 {
- (inputs.candidates.ids, inputs.candidates.logits)
+ (&candidates.ids, &candidates.logits)
} else {
(&node_candidate_ids, &node_candidate_logits)
};
- let nodes = NodeBatch {
- count: batch_node_count,
- token_ids: &node_token_ids,
- metadata: &node_metadata,
- ancestor_indices: &node_ancestor_indices,
- ancestor_stride,
- };
- let candidates = CandidateBatch {
- ids: batch_candidate_ids,
- logits: batch_candidate_logits,
- candidates_per_node: inputs.candidates.candidates_per_depth,
- depth_seeds: &depth_seeds,
- };
- let top_children = self.encode_node_expansion(
- &prefix_cache,
- &nodes,
- &candidates,
- &mut node_cache,
- &rope,
- shape.children_per_node,
- inputs.target_embedding,
+
+ // Node expansion: embed the batch's tokens, run every layer against
+ // the prefix KV and each node's ancestors, then pick its children.
+ let token_embedding = target_embedding.encode_lookup(&node_token_ids, batch_node_count, encoder)?;
+ let normalized_embedding = self
+ .token_embedding_norm
+ .encode(&token_embedding, 0, batch_node_count, None, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut residual_input = self
+ .token_embedding_projection
+ .encode(normalized_embedding, batch_node_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut residual_state =
+ encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
+ let metadata_field_bytes = batch_node_count * DataType::U32.size_in_bytes();
+ for (layer_index, layer) in self.layers.iter().enumerate() {
+ let attention_input = layer
+ .pre_attention_norm
+ .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let current_qkv = layer
+ .qkv_projection
+ .encode(attention_input, batch_node_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut attention_output = encoder
+ .allocate_scratch(size_for_shape(&[batch_node_count, self.model_dim], DATA_TYPE))
+ .map_err(WeaverEncodeError::Backend)?;
+ layer.ancestor_attention.encode(
+ &prefix_kv_layers[layer_index],
+ &mut node_kv_layers[layer_index],
+ &current_qkv,
+ &rope.cosines,
+ &rope.sines,
+ &node_metadata,
+ &node_ancestor_indices,
+ (&node_metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes),
+ (&node_metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes),
+ &mut attention_output,
+ batch_node_count as u32,
+ prefix_length as u32,
+ ancestor_stride as u32,
+ tree_slot_count as u32,
+ layer.max_depth as u32,
+ layer.attention_scale,
+ encoder,
+ );
+ residual_input = layer
+ .encode_post_attention(attention_output, &mut residual_state, batch_node_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ }
+
+ let normalized_output = self
+ .readout_norm
+ .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let query = self
+ .readout_query_projection
+ .encode(normalized_output, batch_node_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let logit_residuals = target_embedding.encode_readout_sparse(
+ &query,
+ batch_candidate_ids,
+ batch_node_count,
+ candidates.candidates_per_row,
encoder,
)?;
+ let mut child_token_ids = encoder
+ .allocate_scratch(size_for_shape(&[batch_node_count, shape.children_per_node], DataType::U32))
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut child_logprobs = encoder
+ .allocate_scratch(size_for_shape(&[batch_node_count, shape.children_per_node], DataType::F32))
+ .map_err(WeaverEncodeError::Backend)?;
+ self.top_children.encode(
+ &logit_residuals,
+ batch_candidate_logits,
+ batch_candidate_ids,
+ &depth_seeds_buffer,
+ &node_metadata,
+ &mut child_token_ids,
+ &mut child_logprobs,
+ batch_node_count as u32,
+ candidates.candidates_per_row as u32,
+ shape.children_per_node as u32,
+ target_embedding.vocab_size() as u32,
+ encoder,
+ );
+
self.frontier_insert_children.encode(
&packed_tree,
&node_metadata,
&node_valid,
- &top_children.token_ids,
- &top_children.logprobs,
+ &child_token_ids,
+ &child_logprobs,
&mut frontier,
frontier_capacity as u32,
tree_slot_count as u32,
@@ -458,441 +527,11 @@ impl<B: Backend> Weaver<B> {
);
batch_start_slot += batch_node_count;
}
+
encoder.pop_debug_group();
+
Ok(EncodedWeaverTree {
packed_tree,
})
}
-
- fn precalculate_rope(
- &self,
- encoder: &mut Encoder<B>,
- ) -> Result<PrecalculatedRoPE<B>, B::Error> {
- let positions = (0..=self.max_depth).collect::<Box<[_]>>();
- PrecalculatedRoPE::precalculate(&self.rope_config, &positions, encoder)
- }
-
- fn encode_prefix(
- &self,
- target_hidden: &Allocation<B>,
- draft_hidden: &Allocation<B>,
- rope: &PrecalculatedRoPE<B>,
- lookahead_count: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<PrefixKvCache<B>, B::Error> {
- assert!((1..=self.max_depth).contains(&lookahead_count));
- let token_count = lookahead_count + 1;
- let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();
-
- let mut prefix_hidden =
- encoder.allocate_scratch(size_for_shape(&[token_count, self.target_model_dim], DATA_TYPE))?;
- encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);
- encoder.encode_copy(
- draft_hidden,
- hidden_row_bytes..token_count * hidden_row_bytes,
- &mut prefix_hidden,
- hidden_row_bytes..token_count * hidden_row_bytes,
- );
- let normalized_prefix = self.hidden_state_norm.encode(&prefix_hidden, 0, token_count, None, encoder)?;
- let mut residual_input = self.hidden_state_projection.encode(normalized_prefix, token_count, encoder)?;
- let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");
- let mut residual_state = encoder.allocate_scratch(residual_input.size())?;
- let mut prefix_layers = Vec::with_capacity(self.layers.len());
- for layer in preceding_layers {
- let PrefixLayerOutput {
- mlp_delta,
- kv_cache,
- } = layer.encode_prefix_tokens(residual_input, &mut residual_state, rope, token_count, encoder)?;
- prefix_layers.push(kv_cache);
- residual_input = mlp_delta;
- }
- prefix_layers.push(
- last_layer
- .encode_prefix_attention(&residual_input, &mut residual_state, rope, token_count, encoder)?
- .kv_cache,
- );
- Ok(PrefixKvCache {
- layers: prefix_layers.into_boxed_slice(),
- length: token_count,
- })
- }
-
- fn create_node_expansion_kv_cache(
- &self,
- capacity: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<NodeExpansionKvCache<B>, B::Error> {
- assert!(capacity > 0, "Weaver node capacity must be positive");
- let kv_size = size_for_shape(&[2, capacity, self.model_dim], DATA_TYPE);
- let layers =
- (0..self.layers.len()).map(|_| encoder.allocate_scratch(kv_size)).collect::<Result<Box<[_]>, _>>()?;
- Ok(NodeExpansionKvCache {
- layers,
- capacity: capacity as u32,
- })
- }
-
- fn encode_node_expansion(
- &self,
- prefix_cache: &PrefixKvCache<B>,
- nodes: &NodeBatch<'_, B>,
- candidates: &CandidateBatch<'_, B>,
- node_cache: &mut NodeExpansionKvCache<B>,
- rope: &PrecalculatedRoPE<B>,
- children_per_node: usize,
- target_embedding: &Embedding<B>,
- encoder: &mut Encoder<B>,
- ) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {
- let token_embedding = target_embedding.encode_lookup(nodes.token_ids, nodes.count, encoder)?;
- let normalized_embedding = self
- .token_embedding_norm
- .encode(&token_embedding, 0, nodes.count, None, encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let mut residual_input = self
- .token_embedding_projection
- .encode(normalized_embedding, nodes.count, encoder)
- .map_err(WeaverEncodeError::Backend)?;
-
- let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
- let node_capacity = node_cache.capacity;
- for (layer_index, layer) in self.layers.iter().enumerate() {
- residual_input = layer
- .encode_node_batch(
- residual_input,
- &mut residual_state,
- &prefix_cache.layers[layer_index],
- &mut node_cache.layers[layer_index],
- node_capacity,
- nodes,
- rope,
- prefix_cache.length,
- encoder,
- )
- .map_err(WeaverEncodeError::Backend)?;
- }
-
- self.encode_child_selection(
- &residual_input,
- &mut residual_state,
- nodes,
- candidates,
- children_per_node,
- target_embedding,
- encoder,
- )
- }
-
- fn encode_child_selection(
- &self,
- residual_output: &Allocation<B>,
- residual_state: &mut Allocation<B>,
- nodes: &NodeBatch<'_, B>,
- candidates: &CandidateBatch<'_, B>,
- children_per_node: usize,
- target_embedding: &Embedding<B>,
- encoder: &mut Encoder<B>,
- ) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {
- let normalized_output = self
- .readout_norm
- .encode(residual_output, 0, nodes.count, Some(residual_state), encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let query = self
- .readout_query_projection
- .encode(normalized_output, nodes.count, encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let logit_residuals = target_embedding.encode_readout_sparse(
- &query,
- candidates.ids,
- nodes.count,
- candidates.candidates_per_node,
- encoder,
- )?;
- let mut token_ids = encoder
- .allocate_scratch(size_for_shape(&[nodes.count, children_per_node], DataType::U32))
- .map_err(WeaverEncodeError::Backend)?;
- let mut logprobs = encoder
- .allocate_scratch(size_for_shape(&[nodes.count, children_per_node], DataType::F32))
- .map_err(WeaverEncodeError::Backend)?;
- self.top_children.encode(
- &logit_residuals,
- candidates.logits,
- candidates.ids,
- candidates.depth_seeds,
- nodes.metadata,
- &mut token_ids,
- &mut logprobs,
- nodes.count as u32,
- candidates.candidates_per_node as u32,
- children_per_node as u32,
- target_embedding.vocab_size() as u32,
- encoder,
- );
- Ok(TopKChildren {
- token_ids,
- logprobs,
- })
- }
-}
-
-impl<B: Backend> WeaverLayer<B> {
- fn new(
- context: &B::Context,
- config: &WeaverConfig,
- add_to_residual: bool,
- parameter_tree: &ParameterTree<B>,
- ) -> Result<Self, WeaverNewError<B>> {
- let WeaverConfig {
- model_dim,
- hidden_dim,
- num_heads,
- max_depth,
- ref norm_config,
- ref linear_config,
- ..
- } = *config;
- let head_dim = model_dim / num_heads;
- let attention_scale = 1.0 / (head_dim as f32).sqrt();
- let qkv_projection = <dyn Linear<B>>::new(
- model_dim,
- [3 * model_dim],
- false,
- context,
- DATA_TYPE,
- &parameter_tree.subtree("qkv_projection")?,
- )?;
- let out_projection = <dyn Linear<B>>::new(
- model_dim,
- [model_dim],
- false,
- context,
- DATA_TYPE,
- &parameter_tree.subtree("out_projection")?,
- )?;
- let prefix_attention = AttentionCores::new(
- AttentionCoreNewArguments {
- head_dim,
- num_groups: num_heads,
- num_q_heads: num_heads,
- has_sinks: false,
- is_kv_cache_ring: false,
- is_causal: true,
- is_trie: false,
- sliding_window_size: None,
- scale: Some(attention_scale),
- data_type: DATA_TYPE,
- },
- context,
- )
- .map_err(WeaverNewError::Backend)?;
- let attention_prepare =
- <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true)
- .map_err(WeaverNewError::Backend)?;
- let pre_attention_norm = Normalization::new(
- model_dim,
- None,
- if add_to_residual {
- ShortcutMode::Add
- } else {
- ShortcutMode::Copy
- },
- PostLayerScalar::None,
- DATA_TYPE,
- norm_config,
- &parameter_tree.subtree("pre_attention_norm")?,
- context,
- )?;
- let pre_mlp_norm = Normalization::new(
- model_dim,
- None,
- ShortcutMode::Add,
- PostLayerScalar::None,
- DATA_TYPE,
- norm_config,
- &parameter_tree.subtree("pre_mlp_norm")?,
- context,
- )?;
- let (mlp, up_input_hadamard_factors) = <dyn Mlp<B>>::new(
- &weaver_mlp_config(linear_config.clone()),
- model_dim,
- hidden_dim,
- context,
- &parameter_tree.subtree("mlp")?,
- DATA_TYPE,
- )?;
- assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");
- let ancestor_attention =
- <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim as u32, num_heads as u32)
- .map_err(WeaverNewError::Backend)?;
- Ok(Self {
- qkv_projection,
- out_projection,
- prefix_attention,
- attention_prepare,
- pre_attention_norm,
- pre_mlp_norm,
- mlp,
- ancestor_attention,
- attention_scale,
- model_dim,
- num_heads,
- head_dim,
- max_depth,
- })
- }
-
- fn encode_prefix_tokens(
- &self,
- residual_input: Allocation<B>,
- residual_state: &mut Allocation<B>,
- rope: &PrecalculatedRoPE<B>,
- token_count: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<PrefixLayerOutput<B>, B::Error> {
- let PreparedPrefixAttention {
- queries,
- kv_cache,
- } = self.encode_prefix_attention(&residual_input, residual_state, rope, token_count, encoder)?;
- let state_type = AttentionStateType::Full {
- length: 0,
- };
- let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
- let attention_output = self.prefix_attention.encode(
- AttentionCoreEncodeArguments {
- queries: &queries,
- keys: &kv_cache,
- values: (&kv_cache, kv_plane_bytes),
- suffix_length: token_count,
- trie: None,
- sinks: None,
- state_type: &state_type,
- },
- encoder,
- )?;
- let mlp_delta = self.encode_post_attention(attention_output, residual_state, token_count, encoder)?;
- Ok(PrefixLayerOutput {
- mlp_delta,
- kv_cache,
- })
- }
-
- fn encode_prefix_attention(
- &self,
- residual_input: &Allocation<B>,
- residual_state: &mut Allocation<B>,
- rope: &PrecalculatedRoPE<B>,
- token_count: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<PreparedPrefixAttention<B>, B::Error> {
- let attention_input =
- self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), encoder)?;
- let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;
- let mut queries =
- encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;
- let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
- let mut kv_cache = encoder.allocate_scratch(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE))?;
- let (keys, values) = kv_cache.as_buffer_range_mut().split_at(kv_plane_bytes);
- self.attention_prepare.encode(
- &qkv,
- &mut queries,
- Some(keys),
- Some(values),
- Some(&rope.cosines),
- Some(&rope.sines),
- self.num_heads as u32,
- Some(self.num_heads as u32),
- self.head_dim as u32,
- Some(rope.dim as u32),
- Some(0),
- token_count as u32,
- encoder,
- );
- Ok(PreparedPrefixAttention {
- queries,
- kv_cache,
- })
- }
-
- fn encode_node_batch(
- &self,
- residual_input: Allocation<B>,
- residual_state: &mut Allocation<B>,
- prefix_kv: &Allocation<B>,
- node_kv_cache: &mut Allocation<B>,
- node_capacity: u32,
- nodes: &NodeBatch<'_, B>,
- rope: &PrecalculatedRoPE<B>,
- prefix_length: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
- let attention_input =
- self.pre_attention_norm.encode(&residual_input, 0, nodes.count, Some(residual_state), encoder)?;
- let current_qkv = self.qkv_projection.encode(attention_input, nodes.count, encoder)?;
- let metadata_field_bytes = nodes.count * DataType::U32.size_in_bytes();
- let ancestor_counts = (nodes.metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes);
- let tree_slot_indices = (nodes.metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes);
- let mut attention_output =
- encoder.allocate_scratch(size_for_shape(&[nodes.count, self.model_dim], DATA_TYPE))?;
- self.ancestor_attention.encode(
- prefix_kv,
- node_kv_cache,
- &current_qkv,
- &rope.cosines,
- &rope.sines,
- nodes.metadata,
- nodes.ancestor_indices,
- ancestor_counts,
- tree_slot_indices,
- &mut attention_output,
- nodes.count as u32,
- prefix_length as u32,
- nodes.ancestor_stride as u32,
- node_capacity,
- self.max_depth as u32,
- self.attention_scale,
- encoder,
- );
- self.encode_post_attention(attention_output, residual_state, nodes.count, encoder)
- }
-
- fn encode_post_attention(
- &self,
- attention_output: Allocation<B>,
- residual_state: &mut Allocation<B>,
- token_count: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
- let projected_attention = self.out_projection.encode(attention_output, token_count, encoder)?;
- let mlp_input =
- self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), encoder)?;
- self.mlp.encode(mlp_input, token_count, encoder)
- }
-}
-
-fn nodes_from_packed_tree(packed_tree: &[u32]) -> Vec<ProposalNode> {
- let tree_slot_count = packed_tree.len() / TreeIdx::COUNT;
- let field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];
- let mut slot_to_node_index = vec![usize::MAX; tree_slot_count];
- let mut nodes: Vec<ProposalNode> = Vec::with_capacity(tree_slot_count);
- for slot in 0..tree_slot_count {
- if field(TreeIdx::Valid, slot) == 0 {
- continue;
- }
- let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;
- let parent = (parent_slot >= 0).then(|| {
- let parent = slot_to_node_index[parent_slot as usize];
- assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");
- parent
- });
- let index = nodes.len();
- slot_to_node_index[slot] = index;
- if let Some(parent) = parent {
- nodes[parent].child_indices.push(index);
- }
- nodes.push(ProposalNode {
- token_id: field(TreeIdx::TokenId, slot),
- depth: field(TreeIdx::Depth, slot) as usize,
- child_indices: Vec::new(),
- });
- }
- nodes
}
diff --git a/crates/backend-uzu/src/encodable_block/weaver_layer.rs b/crates/backend-uzu/src/encodable_block/weaver_layer.rs
new file mode 100644
index 000000000..e309ef9fa
--- /dev/null
+++ b/crates/backend-uzu/src/encodable_block/weaver_layer.rs
@@ -0,0 +1,206 @@
+use crate::{
+ array::size_for_shape,
+ backends::common::{
+ Allocation, AsBufferRangeMut, Backend, Encoder, Kernels,
+ kernel::{AncestorAttentionKernel, AttentionPrepareKernel},
+ },
+ config::{
+ activation::{AnyActivation, silu::SiLU},
+ mlp::{AnyMLPConfig, dense_mlp::DenseMLPConfig},
+ weaver::WeaverConfig,
+ },
+ encodable_block::{
+ linear::Linear,
+ mixer::attention::{
+ core::{AttentionCoreNewArguments, AttentionCores},
+ rope::PrecalculatedRoPE,
+ },
+ mlp::Mlp,
+ normalization::{Normalization, PostLayerScalar, ShortcutMode},
+ weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverNewError},
+ },
+ parameters::ParameterTree,
+};
+
+pub struct PreparedPrefixAttention<B: Backend> {
+ pub queries: Allocation<B>,
+ pub kv_cache: Allocation<B>,
+}
+
+pub struct WeaverLayer<B: Backend> {
+ // Attention
+ pub pre_attention_norm: Normalization<B>,
+ pub qkv_projection: Box<dyn Linear<B>>,
+ pub attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
+ pub prefix_attention: AttentionCores<B>,
+ pub ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,
+ pub out_projection: Box<dyn Linear<B>>,
+
+ // MLP
+ pub pre_mlp_norm: Normalization<B>,
+ pub mlp: Box<dyn Mlp<B>>,
+
+ // Geometry
+ pub attention_scale: f32,
+ pub model_dim: usize,
+ pub num_heads: usize,
+ pub head_dim: usize,
+ pub max_depth: usize,
+}
+
+impl<B: Backend> WeaverLayer<B> {
+ pub fn new(
+ context: &B::Context,
+ config: &WeaverConfig,
+ add_to_residual: bool,
+ parameter_tree: &ParameterTree<B>,
+ ) -> Result<Self, WeaverNewError<B>> {
+ let WeaverConfig {
+ model_dim,
+ hidden_dim,
+ num_heads,
+ max_depth,
+ ref norm_config,
+ ref linear_config,
+ ..
+ } = *config;
+ let head_dim = model_dim / num_heads;
+ let attention_scale = 1.0 / (head_dim as f32).sqrt();
+ let qkv_projection = <dyn Linear<B>>::new(
+ model_dim,
+ [3 * model_dim],
+ false,
+ context,
+ DATA_TYPE,
+ &parameter_tree.subtree("qkv_projection")?,
+ )?;
+ let out_projection = <dyn Linear<B>>::new(
+ model_dim,
+ [model_dim],
+ false,
+ context,
+ DATA_TYPE,
+ &parameter_tree.subtree("out_projection")?,
+ )?;
+ let prefix_attention = AttentionCores::new(
+ AttentionCoreNewArguments {
+ head_dim,
+ num_groups: num_heads,
+ num_q_heads: num_heads,
+ has_sinks: false,
+ is_kv_cache_ring: false,
+ is_causal: true,
+ is_trie: false,
+ sliding_window_size: None,
+ scale: Some(attention_scale),
+ data_type: DATA_TYPE,
+ },
+ context,
+ )
+ .map_err(WeaverNewError::Backend)?;
+ let attention_prepare =
+ <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true)
+ .map_err(WeaverNewError::Backend)?;
+ let pre_attention_norm = Normalization::new(
+ model_dim,
+ None,
+ if add_to_residual {
+ ShortcutMode::Add
+ } else {
+ ShortcutMode::Copy
+ },
+ PostLayerScalar::None,
+ DATA_TYPE,
+ norm_config,
+ &parameter_tree.subtree("pre_attention_norm")?,
+ context,
+ )?;
+ let pre_mlp_norm = Normalization::new(
+ model_dim,
+ None,
+ ShortcutMode::Add,
+ PostLayerScalar::None,
+ DATA_TYPE,
+ norm_config,
+ &parameter_tree.subtree("pre_mlp_norm")?,
+ context,
+ )?;
+ let mlp_config = AnyMLPConfig::DenseMLPConfig(DenseMLPConfig::unclipped(
+ linear_config.clone(),
+ AnyActivation::SiLU(SiLU::default()),
+ true,
+ true,
+ ));
+ let (mlp, up_input_hadamard_factors) =
+ <dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, &parameter_tree.subtree("mlp")?, DATA_TYPE)?;
+ assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");
+ let ancestor_attention =
+ <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim as u32, num_heads as u32)
+ .map_err(WeaverNewError::Backend)?;
+ Ok(Self {
+ qkv_projection,
+ out_projection,
+ prefix_attention,
+ attention_prepare,
+ pre_attention_norm,
+ pre_mlp_norm,
+ mlp,
+ ancestor_attention,
+ attention_scale,
+ model_dim,
+ num_heads,
+ head_dim,
+ max_depth,
+ })
+ }
+
+ pub fn encode_prefix_attention(
+ &self,
+ residual_input: &Allocation<B>,
+ residual_state: &mut Allocation<B>,
+ rope: &PrecalculatedRoPE<B>,
+ token_count: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<PreparedPrefixAttention<B>, B::Error> {
+ let attention_input =
+ self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), encoder)?;
+ let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;
+ let mut queries =
+ encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;
+ let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
+ let mut kv_cache = encoder.allocate_scratch(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE))?;
+ let (keys, values) = kv_cache.as_buffer_range_mut().split_at(kv_plane_bytes);
+ self.attention_prepare.encode(
+ &qkv,
+ &mut queries,
+ Some(keys),
+ Some(values),
+ Some(&rope.cosines),
+ Some(&rope.sines),
+ self.num_heads as u32,
+ Some(self.num_heads as u32),
+ self.head_dim as u32,
+ Some(rope.dim as u32),
+ Some(0),
+ token_count as u32,
+ encoder,
+ );
+ Ok(PreparedPrefixAttention {
+ queries,
+ kv_cache,
+ })
+ }
+
+ pub fn encode_post_attention(
+ &self,
+ attention_output: Allocation<B>,
+ residual_state: &mut Allocation<B>,
+ token_count: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let projected_attention = self.out_projection.encode(attention_output, token_count, encoder)?;
+ let mlp_input =
+ self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), encoder)?;
+ self.mlp.encode(mlp_input, token_count, encoder)
+ }
+}
diff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rs
index bac953aee..c56445d66 100644
--- a/crates/backend-uzu/src/engine/language_model/mod.rs
+++ b/crates/backend-uzu/src/engine/language_model/mod.rs
@@ -12,7 +12,7 @@ use crate::{
},
engine::Engine,
parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
- speculators::dflash_speculator::{DFlashSpeculator, DFlashSpeculatorLoadError},
+ speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},
};
pub mod state;
@@ -24,7 +24,7 @@ pub mod grammar;
pub struct LanguageModel<B: Backend> {
engine: Arc<Engine<B>>,
decoder: Decoder<B>,
- speculator: Option<DFlashSpeculator<B>>,
+ speculator: Option<DFlashTfmSpeculator<B>>,
sampling: Sampling<B>,
context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
generation_config: GenerationConfig,
@@ -104,7 +104,7 @@ impl<B: Backend> Engine<B> {
);
let speculator = speculator_path
- .map(|speculator_path| DFlashSpeculator::load(speculator_path, self.context.clone()))
+ .map(|speculator_path| DFlashTfmSpeculator::new(speculator_path, self.context.clone()))
.transpose()?;
let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);
diff --git a/crates/backend-uzu/src/engine/language_model/stream/mod.rs b/crates/backend-uzu/src/engine/language_model/stream/mod.rs
index a6bc22a76..bb9f12dc8 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/mod.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/mod.rs
@@ -6,7 +6,7 @@ use crate::{
backends::common::Backend,
encodable_block::decoder::DecoderError,
engine::language_model::{LanguageModel, state::LanguageModelState},
- speculators::dflash_speculator::DFlashTreeError,
+ speculators::dflash_tfm::DFlashTreeError,
};
pub use crate::{
encodable_block::sampling::SamplingMethod, engine::language_model::stream::stream::LanguageModelStream,
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index ff4849653..652c131fb 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -14,7 +14,7 @@ use crate::{
gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,
},
data_type::DataType,
- encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},
+ encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod, weaver::TreeShape},
engine::{
capture::CaptureSpan,
language_model::{
@@ -23,7 +23,6 @@ use crate::{
stream::{LanguageModelStreamError, LanguageModelStreamOptions},
},
},
- speculators::dflash_speculator::DFlashTreeOptions,
trie::TrieNode,
};
@@ -302,7 +301,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if let Some(speculator) = model.speculator.as_ref() {
let speculator_state = model_state.speculator_state.as_mut().unwrap();
speculator
- .append_state(
+ .encode_accept(
speculator_state,
decoder_output.hidden_features.as_ref().unwrap(),
&(0..input_chunk.len()).collect::<Box<[usize]>>(),
@@ -359,7 +358,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
}
fn generate(&mut self) -> Result<Option<u64>, LanguageModelStreamError<B>> {
- let (mut prev_output, encoder): (ForwardPassChaining<B>, Option<Encoder<B>>) =
+ let (mut prev_output, mut encoder): (ForwardPassChaining<B>, Option<Encoder<B>>) =
match replace(&mut self.decoding_state, DecodingState::Invalid) {
DecodingState::Seeded {
seed_token,
@@ -449,7 +448,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.map_err(LanguageModelStreamError::Backend)?;
if let Some(speculator) = self.model.speculator.as_ref() {
speculator
- .append_state(
+ .encode_accept(
self.model_state.speculator_state.as_mut().unwrap(),
hidden_features.as_deref().unwrap(),
&accepted_token_indicies,
@@ -550,6 +549,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.max_context_length
.map_or(32, |max_context_length| 32.min(max_context_length - context_length));
+ let mut pending = Vec::new();
let (input_trie, chain_copy, full_accept) = if speculation_batch > 1
&& let Some(speculator) = &self.model.speculator
&& let (root_token, Some(output_norm)) = prev_output.resolve(
@@ -557,19 +557,23 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
#[cfg(grammar)]
self.options.grammar.as_mut(),
)? {
+ if let Some(accept_encoder) = encoder.take() {
+ pending.push(accept_encoder.end_encoding().submit());
+ }
let trie = speculator.propose_tree(
self.model_state.speculator_state.as_mut().unwrap(),
output_norm,
root_token as u32,
self.model.decoder.embedding(),
- &self.model_state.prng,
- #[cfg(grammar)]
- self.options.grammar.as_mut(),
- DFlashTreeOptions {
+ TreeShape {
budget: speculation_batch - 1,
frontier_width: 4,
children_per_node: 4,
},
+ #[cfg(grammar)]
+ self.options.grammar.as_mut(),
+ &self.model_state.prng,
+ self.allocation_pool.clone(),
)?;
(trie, None, false)
} else {
@@ -584,7 +588,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
};
let input_flat_trie = input_trie.linearize();
- let mut pending = Vec::new();
let mut encoder = if let Some(encoder) = encoder {
encoder
} else {
@@ -710,7 +713,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if let Some(speculator) = self.model.speculator.as_ref() {
let speculator_state = self.model_state.speculator_state.as_mut().unwrap();
speculator
- .append_state(
+ .encode_accept(
speculator_state,
decoder_output.hidden_features.as_ref().unwrap(),
&(0..batch_dim.size()).collect::<Box<[usize]>>(),
@@ -799,7 +802,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
self.model_state.transformer_state.encode_accept(&[0], &mut encoder).unwrap();
if let Some(speculator) = self.model.speculator.as_ref() {
speculator
- .append_state(
+ .encode_accept(
self.model_state.speculator_state.as_mut().unwrap(),
in_flight.hidden_features.as_deref().unwrap(),
&[0],
@@ -832,7 +835,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut encoder).unwrap();
if let Some(speculator) = self.model.speculator.as_ref() {
speculator
- .append_state(
+ .encode_accept(
self.model_state.speculator_state.as_mut().unwrap(),
hidden_features.as_deref().unwrap(),
&accepted_token_indicies,
diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rs
deleted file mode 100644
index 3dea1b19c..000000000
--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs
+++ /dev/null
@@ -1,286 +0,0 @@
-use std::{
- fs::File,
- io::{self, BufReader},
- path::Path,
- sync::Arc,
-};
-
-use thiserror::Error;
-
-pub use crate::encodable_block::dflash::DFlashState;
-#[cfg(grammar)]
-use crate::engine::language_model::grammar::Grammar;
-use crate::{
- backends::common::{Allocation, Backend, Encoder},
- config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},
- data_type::DataType,
- encodable_block::{
- dflash::{DFlashDraft, DFlashDraftEncodeError, DFlashDraftNewError, DFlashDraftOutput},
- embedding::Embedding,
- sampling::PRng,
- weaver::{CandidatePool, ProposalNode, TreeShape, Weaver, WeaverEncodeError, WeaverInputs, WeaverNewError},
- },
- parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
- trie::TrieNode,
-};
-
-#[derive(Clone, Copy, Debug)]
-pub struct DFlashTreeOptions {
- pub budget: usize,
- pub frontier_width: usize,
- pub children_per_node: usize,
-}
-
-const MAX_TREE_BUDGET: usize = 4096;
-const MAX_TREE_FRONTIER_WIDTH: usize = 8;
-const MAX_CHILDREN_PER_NODE: usize = 8;
-
-#[derive(Debug, Error)]
-pub enum DFlashTreeError<B: Backend> {
- #[error("backend error: {0}")]
- Backend(#[source] B::Error),
- #[error("DFlash draft error: {0}")]
- Draft(#[from] DFlashDraftEncodeError<B>),
- #[error("Weaver error: {0}")]
- Weaver(#[from] WeaverEncodeError<B>),
- #[error("invalid tree options")]
- InvalidOptions,
-}
-
-#[derive(Debug, Error)]
-pub enum DFlashSpeculatorLoadError<B: Backend> {
- #[error("I/O error: {0}")]
- IO(#[from] io::Error),
- #[error("Serde error: {0}")]
- Serde(#[from] serde_json::Error),
- #[error("HeaderLoading error: {0}")]
- HeaderLoading(#[from] HeaderLoadingError),
- #[error("ParameterLoader error: {0}")]
- ParameterLoader(#[from] ParameterLoaderError<B>),
- #[error("DFlash draft error: {0}")]
- Draft(#[from] DFlashDraftNewError<B>),
- #[error("Weaver error: {0}")]
- Weaver(#[from] WeaverNewError<B>),
-}
-
-pub struct DFlashSpeculator<B: Backend> {
- context: Arc<B::Context>,
- model: DFlashDraft<B>,
- weaver: Option<Weaver<B>>,
- config: DFlashSpeculatorConfig,
-}
-
-impl<B: Backend> DFlashSpeculator<B> {
- pub fn load(
- model_path: &Path,
- context: Arc<B::Context>,
- ) -> Result<Self, DFlashSpeculatorLoadError<B>> {
- let config: SpeculatorModelConfig =
- serde_json::from_reader(BufReader::new(File::open(model_path.join("config.json"))?))?;
- let AnySpeculatorConfig::DFlashSpeculatorConfig(config) = config.speculator_config;
- let data_type = DataType::BF16;
-
- let weights_file = File::open(model_path.join("model.safetensors"))?;
- let weight_loader = ParameterLoader::new(&weights_file, &*context)?;
- let speculator_tree = weight_loader.tree().subtree("speculator")?;
- let model =
- DFlashDraft::new(&*context, &config.draft_config, &speculator_tree.subtree("draft_model")?, data_type)?;
- let weaver = config
- .weaver_config
- .as_ref()
- .map(|weaver_config| Weaver::new(&*context, weaver_config, &speculator_tree.subtree("weaver")?))
- .transpose()?;
-
- weight_loader.tree().assert_all_tensors_validated()?;
-
- Ok(Self {
- context,
- model,
- weaver,
- config,
- })
- }
-
- pub fn hidden_feature_layer_indices(&self) -> &[usize] {
- &self.config.draft_config.target_layer_ids
- }
-
- pub fn empty_state(
- &self,
- context_capacity: usize,
- ) -> Result<DFlashState<B>, B::Error> {
- self.model.empty_state(context_capacity, &self.context)
- }
-
- pub fn append_state(
- &self,
- state: &mut DFlashState<B>,
- target_features: &[Allocation<B>],
- accepted_indices: &[usize],
- encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- self.model.append_state(state, target_features, accepted_indices, encoder)
- }
-
- pub fn propose_tree(
- &self,
- state: &mut DFlashState<B>,
- target_output_norm: &Allocation<B>,
- target_output_token: u32,
- target_embedding: &Embedding<B>,
- prng: &PRng,
- #[cfg(grammar)] grammar: Option<&mut Grammar>,
- options: DFlashTreeOptions,
- ) -> Result<TrieNode, DFlashTreeError<B>> {
- if options.budget == 0
- || options.budget > MAX_TREE_BUDGET
- || options.frontier_width == 0
- || options.frontier_width > MAX_TREE_FRONTIER_WIDTH
- || options.children_per_node == 0
- || options.children_per_node > MAX_CHILDREN_PER_NODE
- {
- return Err(DFlashTreeError::InvalidOptions);
- }
- let vocab_size = self.config.draft_config.vocab_size;
- let target_model_dim = self.config.draft_config.model_dim;
- let candidates_per_row =
- self.config.weaver_config.as_ref().map_or(1, |config| config.candidate_pool_size.min(vocab_size));
- if target_output_token as usize >= vocab_size
- || candidates_per_row == 0
- || options.children_per_node > candidates_per_row
- || target_embedding.vocab_size() != vocab_size
- || target_embedding.model_dim() != target_model_dim
- || target_output_norm.size() != target_model_dim * DataType::BF16.size_in_bytes()
- || self.config.weaver_config.as_ref().is_some_and(|config| config.target_model_dim != target_model_dim)
- {
- return Err(DFlashTreeError::InvalidOptions);
- }
- let root_position = state.context_length();
-
- let mut encoder =
- Encoder::new_with_name(&*self.context, Some("speculator propose")).map_err(DFlashTreeError::Backend)?;
- let DFlashDraftOutput {
- candidates,
- draft_hidden,
- } = self.model.encode_draft(state, target_output_token, target_embedding, candidates_per_row, &mut encoder)?;
-
- if let Some(weaver) = self.weaver.as_ref() {
- let max_depth = self.config.weaver_config.as_ref().expect("a Weaver implies a Weaver config").max_depth;
- let depth_seeds =
- (0..max_depth).map(|depth| prng.derive((root_position + depth) as u64)).collect::<Box<[u64]>>();
- let tree = weaver.encode_tree(
- WeaverInputs {
- target_hidden: target_output_norm,
- draft_hidden: &draft_hidden,
- target_embedding,
- candidates: CandidatePool {
- ids: &candidates.ids,
- logits: &candidates.scores,
- depth_count: candidates.rows,
- candidates_per_depth: candidates.candidates_per_row,
- },
- depth_seeds: &depth_seeds,
- root_token_id: target_output_token,
- },
- TreeShape {
- budget: options.budget,
- frontier_width: options.frontier_width,
- children_per_node: options.children_per_node,
- },
- &mut encoder,
- )?;
- let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
- let nodes = tree.read_nodes();
- drop(candidates);
- drop(draft_hidden);
- drop(completed);
- return Ok(Self::finish_tree(
- nodes,
- prng,
- root_position,
- #[cfg(grammar)]
- grammar,
- ));
- }
-
- let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
- let candidate_rows = candidates.rows;
- let candidates_per_row = candidates.candidates_per_row;
- let candidate_id_values = candidates.ids.copyout::<u32>();
- drop(candidates);
- drop(draft_hidden);
- drop(completed);
-
- let mut nodes = vec![ProposalNode {
- token_id: target_output_token,
- depth: 0,
- child_indices: Vec::new(),
- }];
- for depth in 0..options.budget.min(candidate_rows) {
- let token = candidate_id_values[depth * candidates_per_row];
- let parent = nodes.len() - 1;
- let index = nodes.len();
- nodes.push(ProposalNode {
- token_id: token,
- depth: depth + 1,
- child_indices: Vec::new(),
- });
- nodes[parent].child_indices.push(index);
- }
- Ok(Self::finish_tree(
- nodes,
- prng,
- root_position,
- #[cfg(grammar)]
- grammar,
- ))
- }
-
- fn finish_tree(
- nodes: Vec<ProposalNode>,
- prng: &PRng,
- root_position: usize,
- #[cfg(grammar)] mut grammar: Option<&mut Grammar>,
- ) -> TrieNode {
- fn build(
- nodes: &[ProposalNode],
- index: usize,
- prng: &PRng,
- root_position: usize,
- #[cfg(grammar)] grammar: &mut Option<&mut Grammar>,
- ) -> TrieNode {
- let node = &nodes[index];
- let mut trie_node = TrieNode::new(node.token_id as u64, prng.derive((root_position + node.depth) as u64));
- for &child_index in &node.child_indices {
- #[cfg(grammar)]
- if let Some(grammar) = grammar.as_mut()
- && grammar.accept_token(nodes[child_index].token_id as u64).is_err()
- {
- continue;
- }
- let child = build(
- nodes,
- child_index,
- prng,
- root_position,
- #[cfg(grammar)]
- grammar,
- );
- #[cfg(grammar)]
- if let Some(grammar) = grammar.as_mut() {
- grammar.rollback(1);
- }
- trie_node.add(child).expect("tree children are selected without replacement");
- }
- trie_node
- }
- build(
- &nodes,
- 0,
- prng,
- root_position,
- #[cfg(grammar)]
- &mut grammar,
- )
- }
-}
diff --git a/crates/backend-uzu/src/speculators/dflash_tfm.rs b/crates/backend-uzu/src/speculators/dflash_tfm.rs
new file mode 100644
index 000000000..031fc8ecf
--- /dev/null
+++ b/crates/backend-uzu/src/speculators/dflash_tfm.rs
@@ -0,0 +1,198 @@
+use std::{
+ fs::File,
+ io::{self, BufReader},
+ path::Path,
+ sync::Arc,
+};
+
+use thiserror::Error;
+
+pub use crate::encodable_block::dflash::DFlashState;
+#[cfg(grammar)]
+use crate::engine::language_model::grammar::Grammar;
+use crate::{
+ backends::common::{Allocation, AllocationPool, Backend, Encoder},
+ config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},
+ data_type::DataType,
+ encodable_block::{
+ dflash::{DFlash, DFlashEncodeError, DFlashNewError},
+ embedding::Embedding,
+ sampling::PRng,
+ weaver::{ProposalNode, TreeShape, Weaver, WeaverEncodeError, WeaverNewError},
+ },
+ parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
+ trie::TrieNode,
+};
+
+#[derive(Debug, Error)]
+pub enum DFlashTreeError<B: Backend> {
+ #[error("backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("DFlash draft error: {0}")]
+ DFlash(#[from] DFlashEncodeError<B>),
+ #[error("Weaver error: {0}")]
+ Weaver(#[from] WeaverEncodeError<B>),
+}
+
+#[derive(Debug, Error)]
+pub enum DFlashSpeculatorLoadError<B: Backend> {
+ #[error("I/O error: {0}")]
+ IO(#[from] io::Error),
+ #[error("Serde error: {0}")]
+ Serde(#[from] serde_json::Error),
+ #[error("HeaderLoading error: {0}")]
+ HeaderLoading(#[from] HeaderLoadingError),
+ #[error("ParameterLoader error: {0}")]
+ ParameterLoader(#[from] ParameterLoaderError<B>),
+ #[error("DFlash error: {0}")]
+ DFlash(#[from] DFlashNewError<B>),
+ #[error("Weaver error: {0}")]
+ Weaver(#[from] WeaverNewError<B>),
+}
+
+pub struct DFlashTfmSpeculator<B: Backend> {
+ context: Arc<B::Context>,
+ dflash: DFlash<B>,
+ weaver: Weaver<B>,
+ config: DFlashSpeculatorConfig,
+}
+
+impl<B: Backend> DFlashTfmSpeculator<B> {
+ pub fn new(
+ model_path: &Path,
+ context: Arc<B::Context>,
+ ) -> Result<Self, DFlashSpeculatorLoadError<B>> {
+ let data_type = DataType::BF16;
+
+ let config: SpeculatorModelConfig =
+ serde_json::from_reader(BufReader::new(File::open(model_path.join("config.json"))?))?;
+ let AnySpeculatorConfig::DFlashSpeculatorConfig(config) = config.speculator_config;
+
+ let weights_file = File::open(model_path.join("model.safetensors"))?;
+ let weight_loader = ParameterLoader::new(&weights_file, &*context)?;
+ let speculator_tree = weight_loader.tree().subtree("speculator")?;
+
+ let dflash = DFlash::new(&*context, &config.draft_config, &speculator_tree.subtree("draft_model")?, data_type)?;
+ let weaver = Weaver::new(&*context, &config.weaver_config, &speculator_tree.subtree("weaver")?)?;
+
+ weight_loader.tree().assert_all_tensors_validated()?;
+
+ Ok(Self {
+ context,
+ dflash,
+ weaver,
+ config,
+ })
+ }
+
+ pub fn hidden_feature_layer_indices(&self) -> &[usize] {
+ &self.config.draft_config.target_layer_ids
+ }
+
+ pub fn empty_state(
+ &self,
+ context_capacity: usize,
+ ) -> Result<DFlashState<B>, B::Error> {
+ self.dflash.empty_state(context_capacity, &self.context)
+ }
+
+ pub fn encode_accept(
+ &self,
+ state: &mut DFlashState<B>,
+ target_features: &[Allocation<B>],
+ accepted_indices: &[usize],
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), B::Error> {
+ self.dflash.encode_accept(state, target_features, accepted_indices, encoder)
+ }
+
+ pub fn propose_tree(
+ &self,
+ state: &mut DFlashState<B>,
+ target_output_norm: &Allocation<B>,
+ target_output_token: u32,
+ target_embedding: &Embedding<B>,
+ shape: TreeShape,
+ #[cfg(grammar)] grammar: Option<&mut Grammar>,
+ prng: &PRng,
+ allocation_pool: Arc<AllocationPool<B>>,
+ ) -> Result<TrieNode, DFlashTreeError<B>> {
+ let candidate_pool_size = self.config.weaver_config.candidate_pool_size;
+
+ let root_position = state.context_length();
+
+ let mut encoder = Encoder::new_with_pool_name(&*self.context, allocation_pool, Some("speculator propose"))
+ .map_err(DFlashTreeError::Backend)?;
+
+ let dflash_output = self.dflash.encode_draft(
+ state,
+ target_output_token,
+ target_embedding,
+ candidate_pool_size,
+ &mut encoder,
+ )?;
+
+ let max_depth = self.config.weaver_config.max_depth;
+ let depth_seeds =
+ (0..max_depth).map(|depth| prng.derive((root_position + depth) as u64)).collect::<Box<[u64]>>();
+ let tree = self.weaver.encode_tree(
+ target_output_norm,
+ &dflash_output.draft_hidden,
+ target_embedding,
+ &dflash_output.candidates,
+ &depth_seeds,
+ target_output_token,
+ shape,
+ &mut encoder,
+ )?;
+ let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
+ let nodes = tree.read_nodes();
+ drop(dflash_output);
+ drop(completed);
+
+ fn recursive_build(
+ nodes: &[ProposalNode],
+ index: usize,
+ root_position: usize,
+ #[cfg(grammar)] mut grammar: Option<&mut Grammar>,
+ prng: &PRng,
+ ) -> TrieNode {
+ let node = &nodes[index];
+ let mut trie_node = TrieNode::new(node.token_id as u64, prng.derive((root_position + node.depth) as u64));
+ for &child_index in &node.child_indices {
+ #[cfg(grammar)]
+ if let Some(grammar) = grammar.as_mut()
+ && grammar.accept_token(nodes[child_index].token_id as u64).is_err()
+ {
+ continue;
+ }
+
+ let child = recursive_build(
+ nodes,
+ child_index,
+ root_position,
+ #[cfg(grammar)]
+ grammar.as_deref_mut(),
+ prng,
+ );
+
+ #[cfg(grammar)]
+ if let Some(grammar) = grammar.as_mut() {
+ grammar.rollback(1);
+ }
+
+ trie_node.add(child).expect("tree children are selected without replacement");
+ }
+ trie_node
+ }
+
+ Ok(recursive_build(
+ &nodes,
+ 0,
+ root_position,
+ #[cfg(grammar)]
+ grammar,
+ prng,
+ ))
+ }
+}
diff --git a/crates/backend-uzu/src/speculators/mod.rs b/crates/backend-uzu/src/speculators/mod.rs
index 963522197..8afa45e70 100644
--- a/crates/backend-uzu/src/speculators/mod.rs
+++ b/crates/backend-uzu/src/speculators/mod.rs
@@ -1 +1 @@
-pub mod dflash_speculator;
+pub mod dflash_tfm;