A high-performance inference engine for AI models
Cleanup dflash (#690)
uuuvnf01f19b27602
diff --git a/crates/backend-uzu/src/config/dflash.rs b/crates/backend-uzu/src/config/dflash.rsindex 84de9a796..71f06dbd8 100644--- a/crates/backend-uzu/src/config/dflash.rs+++ b/crates/backend-uzu/src/config/dflash.rs@@ -1,20 +1,10 @@use proc_macros::uzu_config;use crate::config::{- linear::LinearConfig, mlp::dense_mlp::DenseMLPConfig, normalization::NormalizationConfig, rope::AnyRoPEConfig,- token_mixer::attention::AttentionConfig,+ linear::LinearConfig, normalization::NormalizationConfig, rope::AnyRoPEConfig,+ transformer_layer::TransformerLayerConfig,};-#[uzu_config]-pub struct DFlashDraftLayerConfig {- pub attention_config: AttentionConfig,- pub input_norm_config: NormalizationConfig,- /// Sits in the pre-MLP-norm position: normalizes hidden + attention output- /// before the MLP, unlike uzu's post-mixer norm.- pub post_attention_norm_config: NormalizationConfig,- pub mlp_config: DenseMLPConfig,-}-#[uzu_config]pub struct DFlashDraftConfig {pub model_dim: usize,@@ -27,6 +17,6 @@ pub struct DFlashDraftConfig {pub context_projection_config: LinearConfig,pub context_norm_config: NormalizationConfig,pub rope_config: AnyRoPEConfig,- pub layer_configs: Box<[DFlashDraftLayerConfig]>,+ pub layer_configs: Box<[TransformerLayerConfig]>,pub output_norm_config: NormalizationConfig,}diff --git a/crates/backend-uzu/src/config/speculator/dflash.rs b/crates/backend-uzu/src/config/speculator/dflash.rsindex 01d108c0c..626769382 100644--- a/crates/backend-uzu/src/config/speculator/dflash.rs+++ b/crates/backend-uzu/src/config/speculator/dflash.rs@@ -5,5 +5,5 @@ use crate::config::{dflash::DFlashDraftConfig, weaver::WeaverConfig};#[uzu_config(super::SpeculatorConfig)]pub struct DFlashSpeculatorConfig {pub draft_config: DFlashDraftConfig,- pub weaver_config: Option<WeaverConfig>,+ pub weaver_config: WeaverConfig,}diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rsindex 63cbd6853..6b50c06c5 100644--- a/crates/backend-uzu/src/encodable_block/dflash.rs+++ b/crates/backend-uzu/src/encodable_block/dflash.rs@@ -1,37 +1,36 @@+use std::any::Any;+use thiserror::Error;use crate::{array::size_for_shape,backends::common::{- Allocation, Backend, Encoder, Kernels, gpu_types::trie::TrieNode, kernel::radix_top_k_small::RadixTopKSmall,- },- config::{- dflash::{DFlashDraftConfig, DFlashDraftLayerConfig},- mlp::AnyMLPConfig,- rope::AnyRoPEConfig,+ Allocation, Backend, Encoder, Kernels,+ gpu_types::trie::TrieNode,+ kernel::radix_top_k_small::{MAX_K, RadixTopKSmall},},+ config::{dflash::DFlashDraftConfig, rope::AnyRoPEConfig, token_mixer::AnyTokenMixerConfig},data_type::DataType,encodable_block::{batch_topology::BatchTopology,embedding::{Embedding, EmbeddingError},linear::{Linear, LinearBlockError},mixer::{- Mixer, MixerState,+ MixerState,attention::{ATTENTION_SUFFIX_CAPACITY, Attention, AttentionNewError, AttentionState, rope::PrecalculatedRoPE,},},- mlp::{Mlp, MlpBlockError},+ mlp::MlpBlockError,normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},+ transformer_layer::{TransformerLayer, TransformerLayerError},},parameters::{ParameterLoaderError, ParameterTree},utils::maybe_mut::MaybeMut,};-const RADIX_TOP_K_MAX: usize = 512;-pub struct DFlashState<B: Backend> {- layer_states: Box<[AttentionState<B>]>,+ layer_states: Box<[Box<dyn MixerState<B>>]>,context_length: usize,context_capacity: usize,}@@ -42,12 +41,12 @@ impl<B: Backend> DFlashState<B> {}}-pub struct DFlashDraft<B: Backend> {+pub struct DFlash<B: Backend> {target_feature_projection: Box<dyn Linear<B>>,projected_feature_norm: Normalization<B>,state_kv_projection: Box<dyn Linear<B>>,layer_kv_dim: usize,- layers: Box<[DFlashDraftLayer<B>]>,+ layers: Box<[TransformerLayer<B>]>,output_norm: Normalization<B>,top_k: <B::Kernels as Kernels>::RadixTopKSmall,rope_config: AnyRoPEConfig,@@ -59,27 +58,20 @@ pub struct DFlashDraft<B: Backend> {data_type: DataType,}-pub struct DFlashDraftOutput<B: Backend> {- pub candidates: TopKCandidates<B>,+pub struct DFlashOutput<B: Backend> {pub draft_hidden: Allocation<B>,+ pub candidates: TopKCandidates<B>,}pub struct TopKCandidates<B: Backend> {pub ids: Allocation<B>,- pub scores: Allocation<B>,+ pub logits: Allocation<B>,pub rows: usize,pub candidates_per_row: usize,}-struct DFlashDraftLayer<B: Backend> {- attention: Attention<B>,- pre_attention_norm: Normalization<B>,- pre_mlp_norm: Normalization<B>,- mlp: Box<dyn Mlp<B>>,-}-#[derive(Debug, Error)]-pub enum DFlashDraftNewError<B: Backend> {+pub enum DFlashNewError<B: Backend> {#[error("Parameter loader error: {0}")]ParameterLoader(#[from] ParameterLoaderError<B>),#[error("Linear error: {0}")]@@ -90,6 +82,8 @@ pub enum DFlashDraftNewError<B: Backend> {Mlp(#[from] MlpBlockError<B>),#[error("Attention error: {0}")]Attention(#[from] AttentionNewError<B>),+ #[error("Transformer layer error: {0}")]+ TransformerLayer(#[from] TransformerLayerError<B>),#[error("Backend error: {0}")]Backend(#[source] B::Error),#[error("invalid DFlash attention config: {0}")]@@ -97,30 +91,23 @@ pub enum DFlashDraftNewError<B: Backend> {}#[derive(Debug, Error)]-pub enum DFlashDraftEncodeError<B: Backend> {+pub enum DFlashEncodeError<B: Backend> {#[error("Backend error: {0}")]Backend(#[source] B::Error),#[error("Embedding error: {0}")]Embedding(#[from] EmbeddingError<B>),}-impl<B: Backend> DFlashDraft<B> {+impl<B: Backend> DFlash<B> {pub fn new(context: &B::Context,config: &DFlashDraftConfig,parameter_tree: &ParameterTree<B>,data_type: DataType,- ) -> Result<Self, DFlashDraftNewError<B>> {+ ) -> Result<Self, DFlashNewError<B>> {let vocab_size = config.vocab_size as u32;let mask_token_id = config.mask_token_id as u32;- if config.layer_configs.is_empty() {- return Err(DFlashDraftNewError::InvalidAttentionConfig("at least one DFlash layer is required"));- }- if config.block_size == 0 || config.block_size > ATTENTION_SUFFIX_CAPACITY {- return Err(DFlashDraftNewError::InvalidAttentionConfig(- "DFlash block_size must be in 1..=attention suffix capacity",- ));- }+ assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");let target_feature_projection = <dyn Linear<B>>::new(config.model_dim * config.target_layer_ids.len(),[config.model_dim],@@ -140,8 +127,11 @@ impl<B: Backend> DFlashDraft<B> {context,)?;let layers_tree = parameter_tree.subtree("layers")?;- let layer_kv_dim =- 2 * config.layer_configs[0].attention_config.num_groups * config.layer_configs[0].attention_config.head_dim;+ let first_layer_config = config.layer_configs.first().expect("DFlash draft model must have at least one layer");+ let AnyTokenMixerConfig::AttentionConfig(attention_config) = &first_layer_config.mixer_config else {+ return Err(DFlashNewError::InvalidAttentionConfig("DFlash layers must use attention mixers"));+ };+ let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;let state_kv_projection = <dyn Linear<B>>::new(config.model_dim,[config.layer_configs.len() * layer_kv_dim],@@ -155,18 +145,18 @@ impl<B: Backend> DFlashDraft<B> {.iter().enumerate().map(|(index, layer_config)| {- DFlashDraftLayer::new(+ TransformerLayer::new(context,- index,config.model_dim,config.hidden_dim,+ config.layer_configs.len(),layer_config,- &config.rope_config,+ index,&layers_tree.subtree(&index.to_string())?,data_type,)})- .collect::<Result<Box<[_]>, DFlashDraftNewError<B>>>()?;+ .collect::<Result<Box<[_]>, TransformerLayerError<B>>>()?;let output_norm = Normalization::new(config.model_dim,None,@@ -178,7 +168,7 @@ impl<B: Backend> DFlashDraft<B> {context,)?;let top_k =- <B::Kernels as Kernels>::RadixTopKSmall::new(context, vocab_size).map_err(DFlashDraftNewError::Backend)?;+ <B::Kernels as Kernels>::RadixTopKSmall::new(context, vocab_size).map_err(DFlashNewError::Backend)?;Ok(Self {target_feature_projection,@@ -207,7 +197,7 @@ impl<B: Backend> DFlashDraft<B> {let layer_states = self.layers.iter()- .map(|layer| AttentionState::create_empty(&layer.attention, Some(context_capacity), context))+ .map(|layer| layer.mixer.create_empty_state(Some(context_capacity), context)).collect::<Result<Box<[_]>, B::Error>>()?;Ok(DFlashState {layer_states,@@ -216,7 +206,7 @@ impl<B: Backend> DFlashDraft<B> {})}- pub fn append_state(+ pub fn encode_accept(&self,state: &mut DFlashState<B>,target_features: &[Allocation<B>],@@ -227,7 +217,7 @@ impl<B: Backend> DFlashDraft<B> {return Ok(());}- encoder.push_debug_group("dflash append state");+ encoder.push_debug_group("dflash accept");let num_tokens = accepted_indices.len();let captured_layer_count = self.target_feature_input_dim / self.model_dim;@@ -275,11 +265,17 @@ impl<B: Backend> DFlashDraft<B> {);}}- for ((layer, attention_state), key_value) in+ for ((layer, mixer_state), key_value) inself.layers.iter().zip(state.layer_states.iter_mut()).zip(layer_key_values){- attention_state.prepare(state.context_length, num_tokens, encoder.context())?;- layer.attention.append_projected_kv(key_value, &rope, num_tokens, attention_state, encoder)?;+ mixer_state.prepare(state.context_length, num_tokens, encoder.context())?;+ let attention = (layer.mixer.as_ref() as &dyn Any)+ .downcast_ref::<Attention<B>>()+ .expect("DFlash draft layers must use attention mixers");+ let attention_state = (mixer_state.as_mut() as &mut dyn Any)+ .downcast_mut::<AttentionState<B>>()+ .expect("DFlash draft layer states must be attention states");+ attention.append_projected_kv(key_value, &rope, num_tokens, attention_state, encoder)?;}state.context_length += num_tokens;@@ -289,17 +285,28 @@ impl<B: Backend> DFlashDraft<B> {Ok(())}- fn encode_hidden(+ pub fn encode_draft(&self,state: &mut DFlashState<B>,- token_embeddings: Allocation<B>,+ target_output_token: u32,+ target_embedding: &Embedding<B>,+ candidate_count: usize,encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {+ ) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {+ encoder.push_debug_group("dflash draft");+let batch_dim = self.block_size;assert!(state.context_length + batch_dim <= self.max_context_length,"DFlash block positions exceed configured RoPE capacity");++ let mut tokens = vec![self.mask_token_id; batch_dim];+ tokens[0] = target_output_token;+ let token_ids = encoder.allocate_constant_from_slice(&tokens).map_err(DFlashEncodeError::Backend)?;++ let token_embeddings = target_embedding.encode_lookup(&token_ids, batch_dim, encoder)?;+let nodes = (0..batch_dim).map(|index| TrieNode {trie_start: index as u32,@@ -309,142 +316,62 @@ impl<B: Backend> DFlashDraft<B> {.collect::<Box<[_]>>();let batch_topology = BatchTopology::new(&nodes, true);let token_positions = (state.context_length..state.context_length + batch_dim).collect::<Box<[_]>>();- let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, encoder)?;+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, encoder)+ .map_err(DFlashEncodeError::Backend)?;let mut hidden = token_embeddings;- let mut residual = encoder.allocate_scratch(hidden.size())?;- for (layer, attention_state) in self.layers.iter().zip(state.layer_states.iter_mut()) {- attention_state.prepare(state.context_length, batch_dim, encoder.context())?;- let normalized = layer.pre_attention_norm.encode(&hidden, 0, batch_dim, Some(&mut residual), encoder)?;- let attention_output = layer.attention.encode(- normalized,- Some(&rope),- &batch_topology,- Some(MaybeMut::Mut(attention_state)),- encoder,- )?;- let normalized =- layer.pre_mlp_norm.encode(&attention_output, 0, batch_dim, Some(&mut residual), encoder)?;- hidden = layer.mlp.encode(normalized, batch_dim, encoder)?;+ let mut residual = encoder.allocate_scratch(hidden.size()).map_err(DFlashEncodeError::Backend)?;+ for (layer, mixer_state) in self.layers.iter().zip(state.layer_states.iter_mut()) {+ mixer_state+ .prepare(state.context_length, batch_dim, encoder.context())+ .map_err(DFlashEncodeError::Backend)?;+ hidden = layer+ .encode(+ hidden,+ &mut residual,+ None,+ Some(&rope),+ &batch_topology,+ Some(MaybeMut::Mut(mixer_state.as_mut())),+ encoder,+ )+ .map_err(DFlashEncodeError::Backend)?;}-- self.output_norm.encode(&hidden, 0, batch_dim, Some(&mut residual), encoder)- }-- pub fn encode_draft(- &self,- state: &mut DFlashState<B>,- target_output_token: u32,- target_embedding: &Embedding<B>,- candidate_count: usize,- encoder: &mut Encoder<B>,- ) -> Result<DFlashDraftOutput<B>, DFlashDraftEncodeError<B>> {- encoder.push_debug_group("dflash draft");-- let mut token_ids = encoder- .allocate_constant(self.block_size * DataType::U32.size_in_bytes())- .map_err(DFlashDraftEncodeError::Backend)?;- let mut tokens = vec![self.mask_token_id; self.block_size];- tokens[0] = target_output_token;- token_ids.copyin(&tokens);-- let token_embeddings = target_embedding.encode_lookup(&token_ids, self.block_size, encoder)?;- let draft_hidden =- self.encode_hidden(state, token_embeddings, encoder).map_err(DFlashDraftEncodeError::Backend)?;+ let draft_hidden = self+ .output_norm+ .encode(&hidden, 0, batch_dim, Some(&mut residual), encoder)+ .map_err(DFlashEncodeError::Backend)?;// The first block row is the target's output token; only the lookahead rows are ranked.- let lookahead_rows = self.block_size - 1;+ let lookahead_rows = batch_dim - 1;let row_bytes = target_embedding.model_dim() * DataType::BF16.size_in_bytes();let mut lookahead_hidden =- encoder.allocate_scratch(lookahead_rows * row_bytes).map_err(DFlashDraftEncodeError::Backend)?;- encoder.encode_copy(&draft_hidden, row_bytes..self.block_size * row_bytes, &mut lookahead_hidden, ..);+ encoder.allocate_scratch(lookahead_rows * row_bytes).map_err(DFlashEncodeError::Backend)?;+ encoder.encode_copy(&draft_hidden, row_bytes..batch_dim * row_bytes, &mut lookahead_hidden, ..);let logits = target_embedding.encode_readout(lookahead_rows, &lookahead_hidden, DataType::F32, encoder)?;- let candidates = self- .encode_candidates(&logits, lookahead_rows, candidate_count, encoder)- .map_err(DFlashDraftEncodeError::Backend)?;- drop(logits);- drop(token_ids);- drop(lookahead_hidden);++ assert!(candidate_count > 0 && candidate_count <= MAX_K as usize);+ let mut ids = encoder+ .allocate_scratch(size_for_shape(&[lookahead_rows, candidate_count], DataType::U32))+ .map_err(DFlashEncodeError::Backend)?;+ let mut candidate_logits = encoder+ .allocate_scratch(size_for_shape(&[lookahead_rows, candidate_count], DataType::F32))+ .map_err(DFlashEncodeError::Backend)?;+ self.top_k+ .encode(&logits, &mut ids, &mut candidate_logits, lookahead_rows as u32, candidate_count as u32, encoder)+ .map_err(DFlashEncodeError::Backend)?;+ let candidates = TopKCandidates {+ ids,+ logits: candidate_logits,+ rows: lookahead_rows,+ candidates_per_row: candidate_count,+ };encoder.pop_debug_group();- Ok(DFlashDraftOutput {- candidates,+ Ok(DFlashOutput {draft_hidden,- })- }-- fn encode_candidates(- &self,- logits: &Allocation<B>,- rows: usize,- k: usize,- encoder: &mut Encoder<B>,- ) -> Result<TopKCandidates<B>, B::Error> {- assert!(k > 0 && k <= RADIX_TOP_K_MAX);- let mut ids = encoder.allocate_scratch(size_for_shape(&[rows, k], DataType::U32))?;- let mut scores = encoder.allocate_scratch(size_for_shape(&[rows, k], DataType::F32))?;- self.top_k.encode(logits, &mut ids, &mut scores, rows as u32, k as u32, encoder)?;- Ok(TopKCandidates {- ids,- scores,- rows,- candidates_per_row: k,- })- }-}--impl<B: Backend> DFlashDraftLayer<B> {- fn new(- context: &B::Context,- layer_index: usize,- model_dim: usize,- hidden_dim: usize,- config: &DFlashDraftLayerConfig,- rope_config: &AnyRoPEConfig,- parameter_tree: &ParameterTree<B>,- data_type: DataType,- ) -> Result<Self, DFlashDraftNewError<B>> {- let (attention, _) = Attention::new(- model_dim,- data_type,- Some(rope_config),- &config.attention_config,- ¶meter_tree.subtree("attention")?,- context,- )?;- let pre_attention_norm = Normalization::new(- model_dim,- None,- if layer_index == 0 {- ShortcutMode::Copy- } else {- ShortcutMode::Add- },- PostLayerScalar::None,- data_type,- &config.input_norm_config,- ¶meter_tree.subtree("input_norm")?,- context,- )?;- let pre_mlp_norm = Normalization::new(- model_dim,- None,- ShortcutMode::Add,- PostLayerScalar::None,- data_type,- &config.post_attention_norm_config,- ¶meter_tree.subtree("post_attention_norm")?,- context,- )?;- let mlp_config = AnyMLPConfig::DenseMLPConfig(config.mlp_config.clone());- let (mlp, _) =- <dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, ¶meter_tree.subtree("mlp")?, data_type)?;- Ok(Self {- attention,- pre_attention_norm,- pre_mlp_norm,- mlp,+ candidates,})}}diff --git a/crates/backend-uzu/src/encodable_block/mixer/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/mod.rsindex ba2e3b555..c3ae9178c 100644--- a/crates/backend-uzu/src/encodable_block/mixer/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/mod.rs@@ -48,7 +48,7 @@ impl<'a, B: Backend> MaybeMut<'a, dyn MixerState<B>> {}}-pub trait Mixer<B: Backend>: Send + Sync {+pub trait Mixer<B: Backend>: Any + Send + Sync {fn speculation_supported(&self) -> bool;fn max_context_length(&self) -> Option<usize>;diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rsindex e6426d7dd..6f9df8dd6 100644--- a/crates/backend-uzu/src/encodable_block/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mod.rs@@ -13,3 +13,4 @@ pub mod sampling;pub mod transformer;pub mod transformer_layer;pub mod weaver;+pub mod weaver_layer;diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rsindex 21ff38da8..d9982e793 100644--- a/crates/backend-uzu/src/encodable_block/transformer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer.rs@@ -122,7 +122,9 @@ impl<B: Backend> Transformer<B> {let layer = TransformerLayer::new(context,- transformer_config,+ transformer_config.model_dim,+ transformer_config.hidden_dim,+ transformer_config.layer_configs.len(),layer_config,layer_index,&layer_loader,diff --git a/crates/backend-uzu/src/encodable_block/transformer_layer.rs b/crates/backend-uzu/src/encodable_block/transformer_layer.rsindex 32bb92d25..c5047a3ca 100644--- a/crates/backend-uzu/src/encodable_block/transformer_layer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer_layer.rs@@ -2,7 +2,7 @@ use thiserror::Error;use crate::{backends::common::{Allocation, Backend, Encoder},- config::{transformer::TransformerConfig, transformer_layer::TransformerLayerConfig},+ config::transformer_layer::TransformerLayerConfig,data_type::DataType,encodable_block::{batch_topology::BatchTopology,@@ -50,7 +50,9 @@ pub struct TransformerLayer<B: Backend> {impl<B: Backend> TransformerLayer<B> {pub fn new(context: &B::Context,- transformer_config: &TransformerConfig,+ model_dim: usize,+ hidden_dim: usize,+ num_layers: usize,layer_config: &TransformerLayerConfig,layer_index: usize,parameter_tree: &ParameterTree<B>,@@ -82,7 +84,7 @@ impl<B: Backend> TransformerLayer<B> {};let (mixer, mixer_hadamard_factors) = <dyn Mixer<B>>::new(- transformer_config.model_dim,+ model_dim,data_type,layer_config.rope_config.as_ref(),&layer_config.mixer_config,@@ -92,7 +94,7 @@ impl<B: Backend> TransformerLayer<B> {let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {Some(Normalization::new(- transformer_config.model_dim,+ model_dim,mixer_hadamard_factors,if layer_index > 0 {ShortcutMode::Add@@ -114,7 +116,7 @@ impl<B: Backend> TransformerLayer<B> {let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {Some(Normalization::new(- transformer_config.model_dim,+ model_dim,None,ShortcutMode::None,PostLayerScalar::None,@@ -129,15 +131,15 @@ impl<B: Backend> TransformerLayer<B> {let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(&layer_config.mlp_config,- transformer_config.model_dim,- layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),+ model_dim,+ layer_config.hidden_dim.unwrap_or(hidden_dim),context,¶meter_tree.subtree("mlp")?,data_type,)?;let pre_mlp_norm = Normalization::new(- transformer_config.model_dim,+ model_dim,mlp_input_hadamard_factors,ShortcutMode::Add,residual_sum_scalar,@@ -149,7 +151,7 @@ impl<B: Backend> TransformerLayer<B> {let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(Normalization::new(- transformer_config.model_dim,+ model_dim,None,ShortcutMode::None,output_scalar,@@ -167,8 +169,8 @@ impl<B: Backend> TransformerLayer<B> {PerLayerEmbeddingProjection::new(context,ple_config,- transformer_config.model_dim,- transformer_config.layer_configs.len(),+ model_dim,+ num_layers,post_layer_scalar.unwrap_or(1.0),data_type,&ple_loader,diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rsindex 082d67f2a..9d31e5e53 100644--- a/crates/backend-uzu/src/encodable_block/weaver.rs+++ b/crates/backend-uzu/src/encodable_block/weaver.rs@@ -3,104 +3,31 @@ use thiserror::Error;use crate::{array::size_for_shape,backends::common::{- Allocation, AsBufferRangeMut, Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,gpu_types::weaver::{CANDIDATES_MAX, FRONTIER_MAX_SLOTS, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx,},kernel::{- AncestorAttentionKernel, AttentionPrepareKernel, WeaverFrontierInsertChildrenKernel,- WeaverFrontierSelectKernel, WeaverTopChildrenKernel,+ AncestorAttentionKernel, WeaverFrontierInsertChildrenKernel, WeaverFrontierSelectKernel,+ WeaverTopChildrenKernel,},},- config::{- activation::{AnyActivation, silu::SiLU},- linear::LinearConfig,- mlp::{AnyMLPConfig, dense_mlp::DenseMLPConfig},- rope::AnyRoPEConfig,- weaver::WeaverConfig,- },+ config::{rope::AnyRoPEConfig, weaver::WeaverConfig},data_type::DataType,encodable_block::{+ dflash::TopKCandidates,embedding::{Embedding, EmbeddingError},linear::{Linear, LinearBlockError},- mixer::attention::{- AttentionStateType,- core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},- rope::PrecalculatedRoPE,- },- mlp::{Mlp, MlpBlockError},+ mixer::attention::{AttentionStateType, core::AttentionCoreEncodeArguments, rope::PrecalculatedRoPE},+ mlp::MlpBlockError,normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},+ weaver_layer::{PreparedPrefixAttention, WeaverLayer},},parameters::{ParameterLoaderError, ParameterTree},};-const DATA_TYPE: DataType = DataType::BF16;-const ROPE_DATA_TYPE: DataType = DataType::F32;--fn weaver_mlp_config(linear_config: LinearConfig) -> AnyMLPConfig {- AnyMLPConfig::DenseMLPConfig(DenseMLPConfig::unclipped(- linear_config,- AnyActivation::SiLU(SiLU::default()),- true,- true,- ))-}--struct TopKChildren<B: Backend> {- token_ids: Allocation<B>,- logprobs: Allocation<B>,-}--struct PrefixLayerOutput<B: Backend> {- mlp_delta: Allocation<B>,- kv_cache: Allocation<B>,-}--struct PreparedPrefixAttention<B: Backend> {- queries: Allocation<B>,- kv_cache: Allocation<B>,-}--struct PrefixKvCache<B: Backend> {- layers: Box<[Allocation<B>]>,- length: usize,-}--struct NodeExpansionKvCache<B: Backend> {- layers: Box<[Allocation<B>]>,- capacity: u32,-}--struct NodeBatch<'a, B: Backend> {- count: usize,- token_ids: &'a Allocation<B>,- metadata: &'a Allocation<B>,- ancestor_indices: &'a Allocation<B>,- ancestor_stride: usize,-}--struct CandidateBatch<'a, B: Backend> {- ids: &'a Allocation<B>,- logits: &'a Allocation<B>,- candidates_per_node: usize,- depth_seeds: &'a Allocation<B>,-}--pub struct CandidatePool<'a, B: Backend> {- pub ids: &'a Allocation<B>,- pub logits: &'a Allocation<B>,- pub depth_count: usize,- pub candidates_per_depth: usize,-}--pub struct WeaverInputs<'a, B: Backend> {- pub target_hidden: &'a Allocation<B>,- pub draft_hidden: &'a Allocation<B>,- pub target_embedding: &'a Embedding<B>,- pub candidates: CandidatePool<'a, B>,- pub depth_seeds: &'a [u64],- pub root_token_id: u32,-}+pub(crate) const DATA_TYPE: DataType = DataType::BF16;+pub(crate) const ROPE_DATA_TYPE: DataType = DataType::F32;pub struct TreeShape {pub budget: usize,@@ -120,7 +47,34 @@ pub struct ProposalNode {impl<B: Backend> EncodedWeaverTree<B> {pub fn read_nodes(self) -> Vec<ProposalNode> {- nodes_from_packed_tree(&self.packed_tree.copyout::<u32>())+ let packed_tree = &self.packed_tree.copyout::<u32>();++ let tree_slot_count = packed_tree.len() / TreeIdx::COUNT;+ let field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];+ let mut slot_to_node_index = vec![usize::MAX; tree_slot_count];+ let mut nodes: Vec<ProposalNode> = Vec::with_capacity(tree_slot_count);+ for slot in 0..tree_slot_count {+ if field(TreeIdx::Valid, slot) == 0 {+ continue;+ }+ let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;+ let parent = (parent_slot >= 0).then(|| {+ let parent = slot_to_node_index[parent_slot as usize];+ assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");+ parent+ });+ let index = nodes.len();+ slot_to_node_index[slot] = index;+ if let Some(parent) = parent {+ nodes[parent].child_indices.push(index);+ }+ nodes.push(ProposalNode {+ token_id: field(TreeIdx::TokenId, slot),+ depth: field(TreeIdx::Depth, slot) as usize,+ child_indices: Vec::new(),+ });+ }+ nodes}}@@ -141,27 +95,6 @@ pub struct Weaver<B: Backend> {max_depth: usize,}-struct WeaverLayer<B: Backend> {- // Attention- pre_attention_norm: Normalization<B>,- qkv_projection: Box<dyn Linear<B>>,- attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,- prefix_attention: AttentionCores<B>,- ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,- out_projection: Box<dyn Linear<B>>,-- // MLP- pre_mlp_norm: Normalization<B>,- mlp: Box<dyn Mlp<B>>,-- // Geometry- attention_scale: f32,- model_dim: usize,- num_heads: usize,- head_dim: usize,- max_depth: usize,-}-#[derive(Debug, Error)]pub enum WeaverNewError<B: Backend> {#[error("parameter loader error: {0}")]@@ -314,39 +247,111 @@ impl<B: Backend> Weaver<B> {pub fn encode_tree(&self,- inputs: WeaverInputs<'_, B>,+ target_hidden: &Allocation<B>,+ draft_hidden: &Allocation<B>,+ target_embedding: &Embedding<B>,+ candidates: &TopKCandidates<B>,+ depth_seeds: &[u64],+ root_token_id: u32,shape: TreeShape,encoder: &mut Encoder<B>,) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {encoder.push_debug_group("weaver tree");+let tree_slot_count = shape.budget + 1;let frontier_capacity = tree_slot_count * shape.children_per_node;- let lookahead_count = self.max_depth.min(inputs.candidates.depth_count);+ let lookahead_count = self.max_depth.min(candidates.rows);let ancestor_stride = self.max_depth;if shape.budget == 0|| shape.frontier_width == 0|| shape.children_per_node == 0- || shape.children_per_node > inputs.candidates.candidates_per_depth+ || shape.children_per_node > candidates.candidates_per_row|| frontier_capacity > FRONTIER_MAX_SLOTS|| lookahead_count == 0- || inputs.depth_seeds.len() != self.max_depth+ || depth_seeds.len() != self.max_depth{return Err(WeaverEncodeError::InvalidTreeInput);}- let rope = self.precalculate_rope(encoder).map_err(WeaverEncodeError::Backend)?;+ let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, encoder)+ .map_err(WeaverEncodeError::Backend)?;++ // Prefix pass: run the target token and the draft lookahead rows through+ // every layer, collecting per-layer KV caches.+ let prefix_length = lookahead_count + 1;+ let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();+ let mut prefix_hidden = encoder+ .allocate_scratch(size_for_shape(&[prefix_length, self.target_model_dim], DATA_TYPE))+ .map_err(WeaverEncodeError::Backend)?;+ encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);+ encoder.encode_copy(+ draft_hidden,+ hidden_row_bytes..prefix_length * hidden_row_bytes,+ &mut prefix_hidden,+ hidden_row_bytes..prefix_length * hidden_row_bytes,+ );+ let normalized_prefix = self+ .hidden_state_norm+ .encode(&prefix_hidden, 0, prefix_length, None, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut residual_input = self+ .hidden_state_projection+ .encode(normalized_prefix, prefix_length, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");+ let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;+ let mut prefix_kv_layers = Vec::with_capacity(self.layers.len());+ for layer in preceding_layers {+ let PreparedPrefixAttention {+ queries,+ kv_cache,+ } = layer+ .encode_prefix_attention(&residual_input, &mut residual_state, &rope, prefix_length, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let state_type = AttentionStateType::Full {+ length: 0,+ };+ let kv_plane_bytes = size_for_shape(&[prefix_length, self.model_dim], DATA_TYPE);+ let attention_output = layer+ .prefix_attention+ .encode(+ AttentionCoreEncodeArguments {+ queries: &queries,+ keys: &kv_cache,+ values: (&kv_cache, kv_plane_bytes),+ suffix_length: prefix_length,+ trie: None,+ sinks: None,+ state_type: &state_type,+ },+ encoder,+ )+ .map_err(WeaverEncodeError::Backend)?;+ residual_input = layer+ .encode_post_attention(attention_output, &mut residual_state, prefix_length, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ prefix_kv_layers.push(kv_cache);+ }+ prefix_kv_layers.push(+ last_layer+ .encode_prefix_attention(&residual_input, &mut residual_state, &rope, prefix_length, encoder)+ .map_err(WeaverEncodeError::Backend)?+ .kv_cache,+ );- let prefix_cache = self- .encode_prefix(inputs.target_hidden, inputs.draft_hidden, &rope, lookahead_count, encoder)+ // Per-layer KV cache for tree nodes, one slot per packed-tree slot.+ let node_kv_size = size_for_shape(&[2, tree_slot_count, self.model_dim], DATA_TYPE);+ let mut node_kv_layers = (0..self.layers.len())+ .map(|_| encoder.allocate_scratch(node_kv_size))+ .collect::<Result<Vec<_>, _>>().map_err(WeaverEncodeError::Backend)?;- let mut node_cache =- self.create_node_expansion_kv_cache(tree_slot_count, encoder).map_err(WeaverEncodeError::Backend)?;let mut tree_init = vec![0u32; TreeIdx::COUNT * tree_slot_count];for slot in 0..tree_slot_count {tree_init[TreeIdx::ParentSlot as usize * tree_slot_count + slot] = FRONTIER_NO_WINNER;}- tree_init[TreeIdx::TokenId as usize * tree_slot_count] = inputs.root_token_id;+ tree_init[TreeIdx::TokenId as usize * tree_slot_count] = root_token_id;tree_init[TreeIdx::Valid as usize * tree_slot_count] = 1;let mut packed_tree = encoder.allocate_constant_from_slice(&tree_init).map_err(WeaverEncodeError::Backend)?;@@ -358,7 +363,7 @@ impl<B: Backend> Weaver<B> {.map_err(WeaverEncodeError::Backend)?;let mut initial_node_token_ids = vec![0u32; shape.frontier_width];- initial_node_token_ids[0] = inputs.root_token_id;+ initial_node_token_ids[0] = root_token_id;let mut initial_node_valid = vec![0u32; shape.frontier_width];initial_node_valid[0] = 1;let mut node_token_ids =@@ -372,13 +377,13 @@ impl<B: Backend> Weaver<B> {let mut node_valid =encoder.allocate_constant_from_slice(&initial_node_valid).map_err(WeaverEncodeError::Backend)?;let mut node_candidate_ids = encoder- .allocate_constant_from_slice(&vec![0u32; shape.frontier_width * inputs.candidates.candidates_per_depth])+ .allocate_constant_from_slice(&vec![0u32; shape.frontier_width * candidates.candidates_per_row]).map_err(WeaverEncodeError::Backend)?;let mut node_candidate_logits = encoder- .allocate_constant_from_slice(&vec![0.0f32; shape.frontier_width * inputs.candidates.candidates_per_depth])+ .allocate_constant_from_slice(&vec![0.0f32; shape.frontier_width * candidates.candidates_per_row]).map_err(WeaverEncodeError::Backend)?;- let depth_seeds =- encoder.allocate_constant_from_slice(inputs.depth_seeds).map_err(WeaverEncodeError::Backend)?;+ let depth_seeds_buffer =+ encoder.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;let mut batch_start_slot = 0;while batch_start_slot < tree_slot_count {@@ -396,8 +401,8 @@ impl<B: Backend> Weaver<B> {&mut node_metadata,&mut node_ancestor_indices,&mut node_valid,- inputs.candidates.ids,- inputs.candidates.logits,+ &candidates.ids,+ &candidates.logits,&mut node_candidate_ids,&mut node_candidate_logits,frontier_capacity as u32,@@ -407,8 +412,8 @@ impl<B: Backend> Weaver<B> {ancestor_stride as u32,self.max_depth as u32,lookahead_count as u32,- inputs.candidates.depth_count as u32,- inputs.candidates.candidates_per_depth as u32,+ candidates.rows as u32,+ candidates.candidates_per_row as u32,encoder,);}@@ -416,39 +421,103 @@ impl<B: Backend> Weaver<B> {break; // packed-tree slots filled}let (batch_candidate_ids, batch_candidate_logits) = if batch_start_slot == 0 {- (inputs.candidates.ids, inputs.candidates.logits)+ (&candidates.ids, &candidates.logits)} else {(&node_candidate_ids, &node_candidate_logits)};- let nodes = NodeBatch {- count: batch_node_count,- token_ids: &node_token_ids,- metadata: &node_metadata,- ancestor_indices: &node_ancestor_indices,- ancestor_stride,- };- let candidates = CandidateBatch {- ids: batch_candidate_ids,- logits: batch_candidate_logits,- candidates_per_node: inputs.candidates.candidates_per_depth,- depth_seeds: &depth_seeds,- };- let top_children = self.encode_node_expansion(- &prefix_cache,- &nodes,- &candidates,- &mut node_cache,- &rope,- shape.children_per_node,- inputs.target_embedding,++ // Node expansion: embed the batch's tokens, run every layer against+ // the prefix KV and each node's ancestors, then pick its children.+ let token_embedding = target_embedding.encode_lookup(&node_token_ids, batch_node_count, encoder)?;+ let normalized_embedding = self+ .token_embedding_norm+ .encode(&token_embedding, 0, batch_node_count, None, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut residual_input = self+ .token_embedding_projection+ .encode(normalized_embedding, batch_node_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut residual_state =+ encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;+ let metadata_field_bytes = batch_node_count * DataType::U32.size_in_bytes();+ for (layer_index, layer) in self.layers.iter().enumerate() {+ let attention_input = layer+ .pre_attention_norm+ .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let current_qkv = layer+ .qkv_projection+ .encode(attention_input, batch_node_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut attention_output = encoder+ .allocate_scratch(size_for_shape(&[batch_node_count, self.model_dim], DATA_TYPE))+ .map_err(WeaverEncodeError::Backend)?;+ layer.ancestor_attention.encode(+ &prefix_kv_layers[layer_index],+ &mut node_kv_layers[layer_index],+ ¤t_qkv,+ &rope.cosines,+ &rope.sines,+ &node_metadata,+ &node_ancestor_indices,+ (&node_metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes),+ (&node_metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes),+ &mut attention_output,+ batch_node_count as u32,+ prefix_length as u32,+ ancestor_stride as u32,+ tree_slot_count as u32,+ layer.max_depth as u32,+ layer.attention_scale,+ encoder,+ );+ residual_input = layer+ .encode_post_attention(attention_output, &mut residual_state, batch_node_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ }++ let normalized_output = self+ .readout_norm+ .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let query = self+ .readout_query_projection+ .encode(normalized_output, batch_node_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let logit_residuals = target_embedding.encode_readout_sparse(+ &query,+ batch_candidate_ids,+ batch_node_count,+ candidates.candidates_per_row,encoder,)?;+ let mut child_token_ids = encoder+ .allocate_scratch(size_for_shape(&[batch_node_count, shape.children_per_node], DataType::U32))+ .map_err(WeaverEncodeError::Backend)?;+ let mut child_logprobs = encoder+ .allocate_scratch(size_for_shape(&[batch_node_count, shape.children_per_node], DataType::F32))+ .map_err(WeaverEncodeError::Backend)?;+ self.top_children.encode(+ &logit_residuals,+ batch_candidate_logits,+ batch_candidate_ids,+ &depth_seeds_buffer,+ &node_metadata,+ &mut child_token_ids,+ &mut child_logprobs,+ batch_node_count as u32,+ candidates.candidates_per_row as u32,+ shape.children_per_node as u32,+ target_embedding.vocab_size() as u32,+ encoder,+ );+self.frontier_insert_children.encode(&packed_tree,&node_metadata,&node_valid,- &top_children.token_ids,- &top_children.logprobs,+ &child_token_ids,+ &child_logprobs,&mut frontier,frontier_capacity as u32,tree_slot_count as u32,@@ -458,441 +527,11 @@ impl<B: Backend> Weaver<B> {);batch_start_slot += batch_node_count;}+encoder.pop_debug_group();+Ok(EncodedWeaverTree {packed_tree,})}-- fn precalculate_rope(- &self,- encoder: &mut Encoder<B>,- ) -> Result<PrecalculatedRoPE<B>, B::Error> {- let positions = (0..=self.max_depth).collect::<Box<[_]>>();- PrecalculatedRoPE::precalculate(&self.rope_config, &positions, encoder)- }-- fn encode_prefix(- &self,- target_hidden: &Allocation<B>,- draft_hidden: &Allocation<B>,- rope: &PrecalculatedRoPE<B>,- lookahead_count: usize,- encoder: &mut Encoder<B>,- ) -> Result<PrefixKvCache<B>, B::Error> {- assert!((1..=self.max_depth).contains(&lookahead_count));- let token_count = lookahead_count + 1;- let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();-- let mut prefix_hidden =- encoder.allocate_scratch(size_for_shape(&[token_count, self.target_model_dim], DATA_TYPE))?;- encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);- encoder.encode_copy(- draft_hidden,- hidden_row_bytes..token_count * hidden_row_bytes,- &mut prefix_hidden,- hidden_row_bytes..token_count * hidden_row_bytes,- );- let normalized_prefix = self.hidden_state_norm.encode(&prefix_hidden, 0, token_count, None, encoder)?;- let mut residual_input = self.hidden_state_projection.encode(normalized_prefix, token_count, encoder)?;- let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");- let mut residual_state = encoder.allocate_scratch(residual_input.size())?;- let mut prefix_layers = Vec::with_capacity(self.layers.len());- for layer in preceding_layers {- let PrefixLayerOutput {- mlp_delta,- kv_cache,- } = layer.encode_prefix_tokens(residual_input, &mut residual_state, rope, token_count, encoder)?;- prefix_layers.push(kv_cache);- residual_input = mlp_delta;- }- prefix_layers.push(- last_layer- .encode_prefix_attention(&residual_input, &mut residual_state, rope, token_count, encoder)?- .kv_cache,- );- Ok(PrefixKvCache {- layers: prefix_layers.into_boxed_slice(),- length: token_count,- })- }-- fn create_node_expansion_kv_cache(- &self,- capacity: usize,- encoder: &mut Encoder<B>,- ) -> Result<NodeExpansionKvCache<B>, B::Error> {- assert!(capacity > 0, "Weaver node capacity must be positive");- let kv_size = size_for_shape(&[2, capacity, self.model_dim], DATA_TYPE);- let layers =- (0..self.layers.len()).map(|_| encoder.allocate_scratch(kv_size)).collect::<Result<Box<[_]>, _>>()?;- Ok(NodeExpansionKvCache {- layers,- capacity: capacity as u32,- })- }-- fn encode_node_expansion(- &self,- prefix_cache: &PrefixKvCache<B>,- nodes: &NodeBatch<'_, B>,- candidates: &CandidateBatch<'_, B>,- node_cache: &mut NodeExpansionKvCache<B>,- rope: &PrecalculatedRoPE<B>,- children_per_node: usize,- target_embedding: &Embedding<B>,- encoder: &mut Encoder<B>,- ) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {- let token_embedding = target_embedding.encode_lookup(nodes.token_ids, nodes.count, encoder)?;- let normalized_embedding = self- .token_embedding_norm- .encode(&token_embedding, 0, nodes.count, None, encoder)- .map_err(WeaverEncodeError::Backend)?;- let mut residual_input = self- .token_embedding_projection- .encode(normalized_embedding, nodes.count, encoder)- .map_err(WeaverEncodeError::Backend)?;-- let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;- let node_capacity = node_cache.capacity;- for (layer_index, layer) in self.layers.iter().enumerate() {- residual_input = layer- .encode_node_batch(- residual_input,- &mut residual_state,- &prefix_cache.layers[layer_index],- &mut node_cache.layers[layer_index],- node_capacity,- nodes,- rope,- prefix_cache.length,- encoder,- )- .map_err(WeaverEncodeError::Backend)?;- }-- self.encode_child_selection(- &residual_input,- &mut residual_state,- nodes,- candidates,- children_per_node,- target_embedding,- encoder,- )- }-- fn encode_child_selection(- &self,- residual_output: &Allocation<B>,- residual_state: &mut Allocation<B>,- nodes: &NodeBatch<'_, B>,- candidates: &CandidateBatch<'_, B>,- children_per_node: usize,- target_embedding: &Embedding<B>,- encoder: &mut Encoder<B>,- ) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {- let normalized_output = self- .readout_norm- .encode(residual_output, 0, nodes.count, Some(residual_state), encoder)- .map_err(WeaverEncodeError::Backend)?;- let query = self- .readout_query_projection- .encode(normalized_output, nodes.count, encoder)- .map_err(WeaverEncodeError::Backend)?;- let logit_residuals = target_embedding.encode_readout_sparse(- &query,- candidates.ids,- nodes.count,- candidates.candidates_per_node,- encoder,- )?;- let mut token_ids = encoder- .allocate_scratch(size_for_shape(&[nodes.count, children_per_node], DataType::U32))- .map_err(WeaverEncodeError::Backend)?;- let mut logprobs = encoder- .allocate_scratch(size_for_shape(&[nodes.count, children_per_node], DataType::F32))- .map_err(WeaverEncodeError::Backend)?;- self.top_children.encode(- &logit_residuals,- candidates.logits,- candidates.ids,- candidates.depth_seeds,- nodes.metadata,- &mut token_ids,- &mut logprobs,- nodes.count as u32,- candidates.candidates_per_node as u32,- children_per_node as u32,- target_embedding.vocab_size() as u32,- encoder,- );- Ok(TopKChildren {- token_ids,- logprobs,- })- }-}--impl<B: Backend> WeaverLayer<B> {- fn new(- context: &B::Context,- config: &WeaverConfig,- add_to_residual: bool,- parameter_tree: &ParameterTree<B>,- ) -> Result<Self, WeaverNewError<B>> {- let WeaverConfig {- model_dim,- hidden_dim,- num_heads,- max_depth,- ref norm_config,- ref linear_config,- ..- } = *config;- let head_dim = model_dim / num_heads;- let attention_scale = 1.0 / (head_dim as f32).sqrt();- let qkv_projection = <dyn Linear<B>>::new(- model_dim,- [3 * model_dim],- false,- context,- DATA_TYPE,- ¶meter_tree.subtree("qkv_projection")?,- )?;- let out_projection = <dyn Linear<B>>::new(- model_dim,- [model_dim],- false,- context,- DATA_TYPE,- ¶meter_tree.subtree("out_projection")?,- )?;- let prefix_attention = AttentionCores::new(- AttentionCoreNewArguments {- head_dim,- num_groups: num_heads,- num_q_heads: num_heads,- has_sinks: false,- is_kv_cache_ring: false,- is_causal: true,- is_trie: false,- sliding_window_size: None,- scale: Some(attention_scale),- data_type: DATA_TYPE,- },- context,- )- .map_err(WeaverNewError::Backend)?;- let attention_prepare =- <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true)- .map_err(WeaverNewError::Backend)?;- let pre_attention_norm = Normalization::new(- model_dim,- None,- if add_to_residual {- ShortcutMode::Add- } else {- ShortcutMode::Copy- },- PostLayerScalar::None,- DATA_TYPE,- norm_config,- ¶meter_tree.subtree("pre_attention_norm")?,- context,- )?;- let pre_mlp_norm = Normalization::new(- model_dim,- None,- ShortcutMode::Add,- PostLayerScalar::None,- DATA_TYPE,- norm_config,- ¶meter_tree.subtree("pre_mlp_norm")?,- context,- )?;- let (mlp, up_input_hadamard_factors) = <dyn Mlp<B>>::new(- &weaver_mlp_config(linear_config.clone()),- model_dim,- hidden_dim,- context,- ¶meter_tree.subtree("mlp")?,- DATA_TYPE,- )?;- assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");- let ancestor_attention =- <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim as u32, num_heads as u32)- .map_err(WeaverNewError::Backend)?;- Ok(Self {- qkv_projection,- out_projection,- prefix_attention,- attention_prepare,- pre_attention_norm,- pre_mlp_norm,- mlp,- ancestor_attention,- attention_scale,- model_dim,- num_heads,- head_dim,- max_depth,- })- }-- fn encode_prefix_tokens(- &self,- residual_input: Allocation<B>,- residual_state: &mut Allocation<B>,- rope: &PrecalculatedRoPE<B>,- token_count: usize,- encoder: &mut Encoder<B>,- ) -> Result<PrefixLayerOutput<B>, B::Error> {- let PreparedPrefixAttention {- queries,- kv_cache,- } = self.encode_prefix_attention(&residual_input, residual_state, rope, token_count, encoder)?;- let state_type = AttentionStateType::Full {- length: 0,- };- let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);- let attention_output = self.prefix_attention.encode(- AttentionCoreEncodeArguments {- queries: &queries,- keys: &kv_cache,- values: (&kv_cache, kv_plane_bytes),- suffix_length: token_count,- trie: None,- sinks: None,- state_type: &state_type,- },- encoder,- )?;- let mlp_delta = self.encode_post_attention(attention_output, residual_state, token_count, encoder)?;- Ok(PrefixLayerOutput {- mlp_delta,- kv_cache,- })- }-- fn encode_prefix_attention(- &self,- residual_input: &Allocation<B>,- residual_state: &mut Allocation<B>,- rope: &PrecalculatedRoPE<B>,- token_count: usize,- encoder: &mut Encoder<B>,- ) -> Result<PreparedPrefixAttention<B>, B::Error> {- let attention_input =- self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), encoder)?;- let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;- let mut queries =- encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;- let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);- let mut kv_cache = encoder.allocate_scratch(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE))?;- let (keys, values) = kv_cache.as_buffer_range_mut().split_at(kv_plane_bytes);- self.attention_prepare.encode(- &qkv,- &mut queries,- Some(keys),- Some(values),- Some(&rope.cosines),- Some(&rope.sines),- self.num_heads as u32,- Some(self.num_heads as u32),- self.head_dim as u32,- Some(rope.dim as u32),- Some(0),- token_count as u32,- encoder,- );- Ok(PreparedPrefixAttention {- queries,- kv_cache,- })- }-- fn encode_node_batch(- &self,- residual_input: Allocation<B>,- residual_state: &mut Allocation<B>,- prefix_kv: &Allocation<B>,- node_kv_cache: &mut Allocation<B>,- node_capacity: u32,- nodes: &NodeBatch<'_, B>,- rope: &PrecalculatedRoPE<B>,- prefix_length: usize,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {- let attention_input =- self.pre_attention_norm.encode(&residual_input, 0, nodes.count, Some(residual_state), encoder)?;- let current_qkv = self.qkv_projection.encode(attention_input, nodes.count, encoder)?;- let metadata_field_bytes = nodes.count * DataType::U32.size_in_bytes();- let ancestor_counts = (nodes.metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes);- let tree_slot_indices = (nodes.metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes);- let mut attention_output =- encoder.allocate_scratch(size_for_shape(&[nodes.count, self.model_dim], DATA_TYPE))?;- self.ancestor_attention.encode(- prefix_kv,- node_kv_cache,- ¤t_qkv,- &rope.cosines,- &rope.sines,- nodes.metadata,- nodes.ancestor_indices,- ancestor_counts,- tree_slot_indices,- &mut attention_output,- nodes.count as u32,- prefix_length as u32,- nodes.ancestor_stride as u32,- node_capacity,- self.max_depth as u32,- self.attention_scale,- encoder,- );- self.encode_post_attention(attention_output, residual_state, nodes.count, encoder)- }-- fn encode_post_attention(- &self,- attention_output: Allocation<B>,- residual_state: &mut Allocation<B>,- token_count: usize,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {- let projected_attention = self.out_projection.encode(attention_output, token_count, encoder)?;- let mlp_input =- self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), encoder)?;- self.mlp.encode(mlp_input, token_count, encoder)- }-}--fn nodes_from_packed_tree(packed_tree: &[u32]) -> Vec<ProposalNode> {- let tree_slot_count = packed_tree.len() / TreeIdx::COUNT;- let field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];- let mut slot_to_node_index = vec![usize::MAX; tree_slot_count];- let mut nodes: Vec<ProposalNode> = Vec::with_capacity(tree_slot_count);- for slot in 0..tree_slot_count {- if field(TreeIdx::Valid, slot) == 0 {- continue;- }- let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;- let parent = (parent_slot >= 0).then(|| {- let parent = slot_to_node_index[parent_slot as usize];- assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");- parent- });- let index = nodes.len();- slot_to_node_index[slot] = index;- if let Some(parent) = parent {- nodes[parent].child_indices.push(index);- }- nodes.push(ProposalNode {- token_id: field(TreeIdx::TokenId, slot),- depth: field(TreeIdx::Depth, slot) as usize,- child_indices: Vec::new(),- });- }- nodes}diff --git a/crates/backend-uzu/src/encodable_block/weaver_layer.rs b/crates/backend-uzu/src/encodable_block/weaver_layer.rsnew file mode 100644index 000000000..e309ef9fa--- /dev/null+++ b/crates/backend-uzu/src/encodable_block/weaver_layer.rs@@ -0,0 +1,206 @@+use crate::{+ array::size_for_shape,+ backends::common::{+ Allocation, AsBufferRangeMut, Backend, Encoder, Kernels,+ kernel::{AncestorAttentionKernel, AttentionPrepareKernel},+ },+ config::{+ activation::{AnyActivation, silu::SiLU},+ mlp::{AnyMLPConfig, dense_mlp::DenseMLPConfig},+ weaver::WeaverConfig,+ },+ encodable_block::{+ linear::Linear,+ mixer::attention::{+ core::{AttentionCoreNewArguments, AttentionCores},+ rope::PrecalculatedRoPE,+ },+ mlp::Mlp,+ normalization::{Normalization, PostLayerScalar, ShortcutMode},+ weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverNewError},+ },+ parameters::ParameterTree,+};++pub struct PreparedPrefixAttention<B: Backend> {+ pub queries: Allocation<B>,+ pub kv_cache: Allocation<B>,+}++pub struct WeaverLayer<B: Backend> {+ // Attention+ pub pre_attention_norm: Normalization<B>,+ pub qkv_projection: Box<dyn Linear<B>>,+ pub attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,+ pub prefix_attention: AttentionCores<B>,+ pub ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,+ pub out_projection: Box<dyn Linear<B>>,++ // MLP+ pub pre_mlp_norm: Normalization<B>,+ pub mlp: Box<dyn Mlp<B>>,++ // Geometry+ pub attention_scale: f32,+ pub model_dim: usize,+ pub num_heads: usize,+ pub head_dim: usize,+ pub max_depth: usize,+}++impl<B: Backend> WeaverLayer<B> {+ pub fn new(+ context: &B::Context,+ config: &WeaverConfig,+ add_to_residual: bool,+ parameter_tree: &ParameterTree<B>,+ ) -> Result<Self, WeaverNewError<B>> {+ let WeaverConfig {+ model_dim,+ hidden_dim,+ num_heads,+ max_depth,+ ref norm_config,+ ref linear_config,+ ..+ } = *config;+ let head_dim = model_dim / num_heads;+ let attention_scale = 1.0 / (head_dim as f32).sqrt();+ let qkv_projection = <dyn Linear<B>>::new(+ model_dim,+ [3 * model_dim],+ false,+ context,+ DATA_TYPE,+ ¶meter_tree.subtree("qkv_projection")?,+ )?;+ let out_projection = <dyn Linear<B>>::new(+ model_dim,+ [model_dim],+ false,+ context,+ DATA_TYPE,+ ¶meter_tree.subtree("out_projection")?,+ )?;+ let prefix_attention = AttentionCores::new(+ AttentionCoreNewArguments {+ head_dim,+ num_groups: num_heads,+ num_q_heads: num_heads,+ has_sinks: false,+ is_kv_cache_ring: false,+ is_causal: true,+ is_trie: false,+ sliding_window_size: None,+ scale: Some(attention_scale),+ data_type: DATA_TYPE,+ },+ context,+ )+ .map_err(WeaverNewError::Backend)?;+ let attention_prepare =+ <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true)+ .map_err(WeaverNewError::Backend)?;+ let pre_attention_norm = Normalization::new(+ model_dim,+ None,+ if add_to_residual {+ ShortcutMode::Add+ } else {+ ShortcutMode::Copy+ },+ PostLayerScalar::None,+ DATA_TYPE,+ norm_config,+ ¶meter_tree.subtree("pre_attention_norm")?,+ context,+ )?;+ let pre_mlp_norm = Normalization::new(+ model_dim,+ None,+ ShortcutMode::Add,+ PostLayerScalar::None,+ DATA_TYPE,+ norm_config,+ ¶meter_tree.subtree("pre_mlp_norm")?,+ context,+ )?;+ let mlp_config = AnyMLPConfig::DenseMLPConfig(DenseMLPConfig::unclipped(+ linear_config.clone(),+ AnyActivation::SiLU(SiLU::default()),+ true,+ true,+ ));+ let (mlp, up_input_hadamard_factors) =+ <dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, ¶meter_tree.subtree("mlp")?, DATA_TYPE)?;+ assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");+ let ancestor_attention =+ <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim as u32, num_heads as u32)+ .map_err(WeaverNewError::Backend)?;+ Ok(Self {+ qkv_projection,+ out_projection,+ prefix_attention,+ attention_prepare,+ pre_attention_norm,+ pre_mlp_norm,+ mlp,+ ancestor_attention,+ attention_scale,+ model_dim,+ num_heads,+ head_dim,+ max_depth,+ })+ }++ pub fn encode_prefix_attention(+ &self,+ residual_input: &Allocation<B>,+ residual_state: &mut Allocation<B>,+ rope: &PrecalculatedRoPE<B>,+ token_count: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<PreparedPrefixAttention<B>, B::Error> {+ let attention_input =+ self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), encoder)?;+ let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;+ let mut queries =+ encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;+ let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);+ let mut kv_cache = encoder.allocate_scratch(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE))?;+ let (keys, values) = kv_cache.as_buffer_range_mut().split_at(kv_plane_bytes);+ self.attention_prepare.encode(+ &qkv,+ &mut queries,+ Some(keys),+ Some(values),+ Some(&rope.cosines),+ Some(&rope.sines),+ self.num_heads as u32,+ Some(self.num_heads as u32),+ self.head_dim as u32,+ Some(rope.dim as u32),+ Some(0),+ token_count as u32,+ encoder,+ );+ Ok(PreparedPrefixAttention {+ queries,+ kv_cache,+ })+ }++ pub fn encode_post_attention(+ &self,+ attention_output: Allocation<B>,+ residual_state: &mut Allocation<B>,+ token_count: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let projected_attention = self.out_projection.encode(attention_output, token_count, encoder)?;+ let mlp_input =+ self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), encoder)?;+ self.mlp.encode(mlp_input, token_count, encoder)+ }+}diff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rsindex bac953aee..c56445d66 100644--- a/crates/backend-uzu/src/engine/language_model/mod.rs+++ b/crates/backend-uzu/src/engine/language_model/mod.rs@@ -12,7 +12,7 @@ use crate::{},engine::Engine,parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},- speculators::dflash_speculator::{DFlashSpeculator, DFlashSpeculatorLoadError},+ speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},};pub mod state;@@ -24,7 +24,7 @@ pub mod grammar;pub struct LanguageModel<B: Backend> {engine: Arc<Engine<B>>,decoder: Decoder<B>,- speculator: Option<DFlashSpeculator<B>>,+ speculator: Option<DFlashTfmSpeculator<B>>,sampling: Sampling<B>,context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,generation_config: GenerationConfig,@@ -104,7 +104,7 @@ impl<B: Backend> Engine<B> {);let speculator = speculator_path- .map(|speculator_path| DFlashSpeculator::load(speculator_path, self.context.clone()))+ .map(|speculator_path| DFlashTfmSpeculator::new(speculator_path, self.context.clone())).transpose()?;let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);diff --git a/crates/backend-uzu/src/engine/language_model/stream/mod.rs b/crates/backend-uzu/src/engine/language_model/stream/mod.rsindex a6bc22a76..bb9f12dc8 100644--- a/crates/backend-uzu/src/engine/language_model/stream/mod.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/mod.rs@@ -6,7 +6,7 @@ use crate::{backends::common::Backend,encodable_block::decoder::DecoderError,engine::language_model::{LanguageModel, state::LanguageModelState},- speculators::dflash_speculator::DFlashTreeError,+ speculators::dflash_tfm::DFlashTreeError,};pub use crate::{encodable_block::sampling::SamplingMethod, engine::language_model::stream::stream::LanguageModelStream,diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex ff4849653..652c131fb 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -14,7 +14,7 @@ use crate::{gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,},data_type::DataType,- encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},+ encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod, weaver::TreeShape},engine::{capture::CaptureSpan,language_model::{@@ -23,7 +23,6 @@ use crate::{stream::{LanguageModelStreamError, LanguageModelStreamOptions},},},- speculators::dflash_speculator::DFlashTreeOptions,trie::TrieNode,};@@ -302,7 +301,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if let Some(speculator) = model.speculator.as_ref() {let speculator_state = model_state.speculator_state.as_mut().unwrap();speculator- .append_state(+ .encode_accept(speculator_state,decoder_output.hidden_features.as_ref().unwrap(),&(0..input_chunk.len()).collect::<Box<[usize]>>(),@@ -359,7 +358,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {}fn generate(&mut self) -> Result<Option<u64>, LanguageModelStreamError<B>> {- let (mut prev_output, encoder): (ForwardPassChaining<B>, Option<Encoder<B>>) =+ let (mut prev_output, mut encoder): (ForwardPassChaining<B>, Option<Encoder<B>>) =match replace(&mut self.decoding_state, DecodingState::Invalid) {DecodingState::Seeded {seed_token,@@ -449,7 +448,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.map_err(LanguageModelStreamError::Backend)?;if let Some(speculator) = self.model.speculator.as_ref() {speculator- .append_state(+ .encode_accept(self.model_state.speculator_state.as_mut().unwrap(),hidden_features.as_deref().unwrap(),&accepted_token_indicies,@@ -550,6 +549,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.max_context_length.map_or(32, |max_context_length| 32.min(max_context_length - context_length));+ let mut pending = Vec::new();let (input_trie, chain_copy, full_accept) = if speculation_batch > 1&& let Some(speculator) = &self.model.speculator&& let (root_token, Some(output_norm)) = prev_output.resolve(@@ -557,19 +557,23 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {#[cfg(grammar)]self.options.grammar.as_mut(),)? {+ if let Some(accept_encoder) = encoder.take() {+ pending.push(accept_encoder.end_encoding().submit());+ }let trie = speculator.propose_tree(self.model_state.speculator_state.as_mut().unwrap(),output_norm,root_token as u32,self.model.decoder.embedding(),- &self.model_state.prng,- #[cfg(grammar)]- self.options.grammar.as_mut(),- DFlashTreeOptions {+ TreeShape {budget: speculation_batch - 1,frontier_width: 4,children_per_node: 4,},+ #[cfg(grammar)]+ self.options.grammar.as_mut(),+ &self.model_state.prng,+ self.allocation_pool.clone(),)?;(trie, None, false)} else {@@ -584,7 +588,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {};let input_flat_trie = input_trie.linearize();- let mut pending = Vec::new();let mut encoder = if let Some(encoder) = encoder {encoder} else {@@ -710,7 +713,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if let Some(speculator) = self.model.speculator.as_ref() {let speculator_state = self.model_state.speculator_state.as_mut().unwrap();speculator- .append_state(+ .encode_accept(speculator_state,decoder_output.hidden_features.as_ref().unwrap(),&(0..batch_dim.size()).collect::<Box<[usize]>>(),@@ -799,7 +802,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {self.model_state.transformer_state.encode_accept(&[0], &mut encoder).unwrap();if let Some(speculator) = self.model.speculator.as_ref() {speculator- .append_state(+ .encode_accept(self.model_state.speculator_state.as_mut().unwrap(),in_flight.hidden_features.as_deref().unwrap(),&[0],@@ -832,7 +835,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut encoder).unwrap();if let Some(speculator) = self.model.speculator.as_ref() {speculator- .append_state(+ .encode_accept(self.model_state.speculator_state.as_mut().unwrap(),hidden_features.as_deref().unwrap(),&accepted_token_indicies,diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rsdeleted file mode 100644index 3dea1b19c..000000000--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs+++ /dev/null@@ -1,286 +0,0 @@-use std::{- fs::File,- io::{self, BufReader},- path::Path,- sync::Arc,-};--use thiserror::Error;--pub use crate::encodable_block::dflash::DFlashState;-#[cfg(grammar)]-use crate::engine::language_model::grammar::Grammar;-use crate::{- backends::common::{Allocation, Backend, Encoder},- config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},- data_type::DataType,- encodable_block::{- dflash::{DFlashDraft, DFlashDraftEncodeError, DFlashDraftNewError, DFlashDraftOutput},- embedding::Embedding,- sampling::PRng,- weaver::{CandidatePool, ProposalNode, TreeShape, Weaver, WeaverEncodeError, WeaverInputs, WeaverNewError},- },- parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},- trie::TrieNode,-};--#[derive(Clone, Copy, Debug)]-pub struct DFlashTreeOptions {- pub budget: usize,- pub frontier_width: usize,- pub children_per_node: usize,-}--const MAX_TREE_BUDGET: usize = 4096;-const MAX_TREE_FRONTIER_WIDTH: usize = 8;-const MAX_CHILDREN_PER_NODE: usize = 8;--#[derive(Debug, Error)]-pub enum DFlashTreeError<B: Backend> {- #[error("backend error: {0}")]- Backend(#[source] B::Error),- #[error("DFlash draft error: {0}")]- Draft(#[from] DFlashDraftEncodeError<B>),- #[error("Weaver error: {0}")]- Weaver(#[from] WeaverEncodeError<B>),- #[error("invalid tree options")]- InvalidOptions,-}--#[derive(Debug, Error)]-pub enum DFlashSpeculatorLoadError<B: Backend> {- #[error("I/O error: {0}")]- IO(#[from] io::Error),- #[error("Serde error: {0}")]- Serde(#[from] serde_json::Error),- #[error("HeaderLoading error: {0}")]- HeaderLoading(#[from] HeaderLoadingError),- #[error("ParameterLoader error: {0}")]- ParameterLoader(#[from] ParameterLoaderError<B>),- #[error("DFlash draft error: {0}")]- Draft(#[from] DFlashDraftNewError<B>),- #[error("Weaver error: {0}")]- Weaver(#[from] WeaverNewError<B>),-}--pub struct DFlashSpeculator<B: Backend> {- context: Arc<B::Context>,- model: DFlashDraft<B>,- weaver: Option<Weaver<B>>,- config: DFlashSpeculatorConfig,-}--impl<B: Backend> DFlashSpeculator<B> {- pub fn load(- model_path: &Path,- context: Arc<B::Context>,- ) -> Result<Self, DFlashSpeculatorLoadError<B>> {- let config: SpeculatorModelConfig =- serde_json::from_reader(BufReader::new(File::open(model_path.join("config.json"))?))?;- let AnySpeculatorConfig::DFlashSpeculatorConfig(config) = config.speculator_config;- let data_type = DataType::BF16;-- let weights_file = File::open(model_path.join("model.safetensors"))?;- let weight_loader = ParameterLoader::new(&weights_file, &*context)?;- let speculator_tree = weight_loader.tree().subtree("speculator")?;- let model =- DFlashDraft::new(&*context, &config.draft_config, &speculator_tree.subtree("draft_model")?, data_type)?;- let weaver = config- .weaver_config- .as_ref()- .map(|weaver_config| Weaver::new(&*context, weaver_config, &speculator_tree.subtree("weaver")?))- .transpose()?;-- weight_loader.tree().assert_all_tensors_validated()?;-- Ok(Self {- context,- model,- weaver,- config,- })- }-- pub fn hidden_feature_layer_indices(&self) -> &[usize] {- &self.config.draft_config.target_layer_ids- }-- pub fn empty_state(- &self,- context_capacity: usize,- ) -> Result<DFlashState<B>, B::Error> {- self.model.empty_state(context_capacity, &self.context)- }-- pub fn append_state(- &self,- state: &mut DFlashState<B>,- target_features: &[Allocation<B>],- accepted_indices: &[usize],- encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- self.model.append_state(state, target_features, accepted_indices, encoder)- }-- pub fn propose_tree(- &self,- state: &mut DFlashState<B>,- target_output_norm: &Allocation<B>,- target_output_token: u32,- target_embedding: &Embedding<B>,- prng: &PRng,- #[cfg(grammar)] grammar: Option<&mut Grammar>,- options: DFlashTreeOptions,- ) -> Result<TrieNode, DFlashTreeError<B>> {- if options.budget == 0- || options.budget > MAX_TREE_BUDGET- || options.frontier_width == 0- || options.frontier_width > MAX_TREE_FRONTIER_WIDTH- || options.children_per_node == 0- || options.children_per_node > MAX_CHILDREN_PER_NODE- {- return Err(DFlashTreeError::InvalidOptions);- }- let vocab_size = self.config.draft_config.vocab_size;- let target_model_dim = self.config.draft_config.model_dim;- let candidates_per_row =- self.config.weaver_config.as_ref().map_or(1, |config| config.candidate_pool_size.min(vocab_size));- if target_output_token as usize >= vocab_size- || candidates_per_row == 0- || options.children_per_node > candidates_per_row- || target_embedding.vocab_size() != vocab_size- || target_embedding.model_dim() != target_model_dim- || target_output_norm.size() != target_model_dim * DataType::BF16.size_in_bytes()- || self.config.weaver_config.as_ref().is_some_and(|config| config.target_model_dim != target_model_dim)- {- return Err(DFlashTreeError::InvalidOptions);- }- let root_position = state.context_length();-- let mut encoder =- Encoder::new_with_name(&*self.context, Some("speculator propose")).map_err(DFlashTreeError::Backend)?;- let DFlashDraftOutput {- candidates,- draft_hidden,- } = self.model.encode_draft(state, target_output_token, target_embedding, candidates_per_row, &mut encoder)?;-- if let Some(weaver) = self.weaver.as_ref() {- let max_depth = self.config.weaver_config.as_ref().expect("a Weaver implies a Weaver config").max_depth;- let depth_seeds =- (0..max_depth).map(|depth| prng.derive((root_position + depth) as u64)).collect::<Box<[u64]>>();- let tree = weaver.encode_tree(- WeaverInputs {- target_hidden: target_output_norm,- draft_hidden: &draft_hidden,- target_embedding,- candidates: CandidatePool {- ids: &candidates.ids,- logits: &candidates.scores,- depth_count: candidates.rows,- candidates_per_depth: candidates.candidates_per_row,- },- depth_seeds: &depth_seeds,- root_token_id: target_output_token,- },- TreeShape {- budget: options.budget,- frontier_width: options.frontier_width,- children_per_node: options.children_per_node,- },- &mut encoder,- )?;- let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;- let nodes = tree.read_nodes();- drop(candidates);- drop(draft_hidden);- drop(completed);- return Ok(Self::finish_tree(- nodes,- prng,- root_position,- #[cfg(grammar)]- grammar,- ));- }-- let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;- let candidate_rows = candidates.rows;- let candidates_per_row = candidates.candidates_per_row;- let candidate_id_values = candidates.ids.copyout::<u32>();- drop(candidates);- drop(draft_hidden);- drop(completed);-- let mut nodes = vec![ProposalNode {- token_id: target_output_token,- depth: 0,- child_indices: Vec::new(),- }];- for depth in 0..options.budget.min(candidate_rows) {- let token = candidate_id_values[depth * candidates_per_row];- let parent = nodes.len() - 1;- let index = nodes.len();- nodes.push(ProposalNode {- token_id: token,- depth: depth + 1,- child_indices: Vec::new(),- });- nodes[parent].child_indices.push(index);- }- Ok(Self::finish_tree(- nodes,- prng,- root_position,- #[cfg(grammar)]- grammar,- ))- }-- fn finish_tree(- nodes: Vec<ProposalNode>,- prng: &PRng,- root_position: usize,- #[cfg(grammar)] mut grammar: Option<&mut Grammar>,- ) -> TrieNode {- fn build(- nodes: &[ProposalNode],- index: usize,- prng: &PRng,- root_position: usize,- #[cfg(grammar)] grammar: &mut Option<&mut Grammar>,- ) -> TrieNode {- let node = &nodes[index];- let mut trie_node = TrieNode::new(node.token_id as u64, prng.derive((root_position + node.depth) as u64));- for &child_index in &node.child_indices {- #[cfg(grammar)]- if let Some(grammar) = grammar.as_mut()- && grammar.accept_token(nodes[child_index].token_id as u64).is_err()- {- continue;- }- let child = build(- nodes,- child_index,- prng,- root_position,- #[cfg(grammar)]- grammar,- );- #[cfg(grammar)]- if let Some(grammar) = grammar.as_mut() {- grammar.rollback(1);- }- trie_node.add(child).expect("tree children are selected without replacement");- }- trie_node- }- build(- &nodes,- 0,- prng,- root_position,- #[cfg(grammar)]- &mut grammar,- )- }-}diff --git a/crates/backend-uzu/src/speculators/dflash_tfm.rs b/crates/backend-uzu/src/speculators/dflash_tfm.rsnew file mode 100644index 000000000..031fc8ecf--- /dev/null+++ b/crates/backend-uzu/src/speculators/dflash_tfm.rs@@ -0,0 +1,198 @@+use std::{+ fs::File,+ io::{self, BufReader},+ path::Path,+ sync::Arc,+};++use thiserror::Error;++pub use crate::encodable_block::dflash::DFlashState;+#[cfg(grammar)]+use crate::engine::language_model::grammar::Grammar;+use crate::{+ backends::common::{Allocation, AllocationPool, Backend, Encoder},+ config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},+ data_type::DataType,+ encodable_block::{+ dflash::{DFlash, DFlashEncodeError, DFlashNewError},+ embedding::Embedding,+ sampling::PRng,+ weaver::{ProposalNode, TreeShape, Weaver, WeaverEncodeError, WeaverNewError},+ },+ parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},+ trie::TrieNode,+};++#[derive(Debug, Error)]+pub enum DFlashTreeError<B: Backend> {+ #[error("backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("DFlash draft error: {0}")]+ DFlash(#[from] DFlashEncodeError<B>),+ #[error("Weaver error: {0}")]+ Weaver(#[from] WeaverEncodeError<B>),+}++#[derive(Debug, Error)]+pub enum DFlashSpeculatorLoadError<B: Backend> {+ #[error("I/O error: {0}")]+ IO(#[from] io::Error),+ #[error("Serde error: {0}")]+ Serde(#[from] serde_json::Error),+ #[error("HeaderLoading error: {0}")]+ HeaderLoading(#[from] HeaderLoadingError),+ #[error("ParameterLoader error: {0}")]+ ParameterLoader(#[from] ParameterLoaderError<B>),+ #[error("DFlash error: {0}")]+ DFlash(#[from] DFlashNewError<B>),+ #[error("Weaver error: {0}")]+ Weaver(#[from] WeaverNewError<B>),+}++pub struct DFlashTfmSpeculator<B: Backend> {+ context: Arc<B::Context>,+ dflash: DFlash<B>,+ weaver: Weaver<B>,+ config: DFlashSpeculatorConfig,+}++impl<B: Backend> DFlashTfmSpeculator<B> {+ pub fn new(+ model_path: &Path,+ context: Arc<B::Context>,+ ) -> Result<Self, DFlashSpeculatorLoadError<B>> {+ let data_type = DataType::BF16;++ let config: SpeculatorModelConfig =+ serde_json::from_reader(BufReader::new(File::open(model_path.join("config.json"))?))?;+ let AnySpeculatorConfig::DFlashSpeculatorConfig(config) = config.speculator_config;++ let weights_file = File::open(model_path.join("model.safetensors"))?;+ let weight_loader = ParameterLoader::new(&weights_file, &*context)?;+ let speculator_tree = weight_loader.tree().subtree("speculator")?;++ let dflash = DFlash::new(&*context, &config.draft_config, &speculator_tree.subtree("draft_model")?, data_type)?;+ let weaver = Weaver::new(&*context, &config.weaver_config, &speculator_tree.subtree("weaver")?)?;++ weight_loader.tree().assert_all_tensors_validated()?;++ Ok(Self {+ context,+ dflash,+ weaver,+ config,+ })+ }++ pub fn hidden_feature_layer_indices(&self) -> &[usize] {+ &self.config.draft_config.target_layer_ids+ }++ pub fn empty_state(+ &self,+ context_capacity: usize,+ ) -> Result<DFlashState<B>, B::Error> {+ self.dflash.empty_state(context_capacity, &self.context)+ }++ pub fn encode_accept(+ &self,+ state: &mut DFlashState<B>,+ target_features: &[Allocation<B>],+ accepted_indices: &[usize],+ encoder: &mut Encoder<B>,+ ) -> Result<(), B::Error> {+ self.dflash.encode_accept(state, target_features, accepted_indices, encoder)+ }++ pub fn propose_tree(+ &self,+ state: &mut DFlashState<B>,+ target_output_norm: &Allocation<B>,+ target_output_token: u32,+ target_embedding: &Embedding<B>,+ shape: TreeShape,+ #[cfg(grammar)] grammar: Option<&mut Grammar>,+ prng: &PRng,+ allocation_pool: Arc<AllocationPool<B>>,+ ) -> Result<TrieNode, DFlashTreeError<B>> {+ let candidate_pool_size = self.config.weaver_config.candidate_pool_size;++ let root_position = state.context_length();++ let mut encoder = Encoder::new_with_pool_name(&*self.context, allocation_pool, Some("speculator propose"))+ .map_err(DFlashTreeError::Backend)?;++ let dflash_output = self.dflash.encode_draft(+ state,+ target_output_token,+ target_embedding,+ candidate_pool_size,+ &mut encoder,+ )?;++ let max_depth = self.config.weaver_config.max_depth;+ let depth_seeds =+ (0..max_depth).map(|depth| prng.derive((root_position + depth) as u64)).collect::<Box<[u64]>>();+ let tree = self.weaver.encode_tree(+ target_output_norm,+ &dflash_output.draft_hidden,+ target_embedding,+ &dflash_output.candidates,+ &depth_seeds,+ target_output_token,+ shape,+ &mut encoder,+ )?;+ let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;+ let nodes = tree.read_nodes();+ drop(dflash_output);+ drop(completed);++ fn recursive_build(+ nodes: &[ProposalNode],+ index: usize,+ root_position: usize,+ #[cfg(grammar)] mut grammar: Option<&mut Grammar>,+ prng: &PRng,+ ) -> TrieNode {+ let node = &nodes[index];+ let mut trie_node = TrieNode::new(node.token_id as u64, prng.derive((root_position + node.depth) as u64));+ for &child_index in &node.child_indices {+ #[cfg(grammar)]+ if let Some(grammar) = grammar.as_mut()+ && grammar.accept_token(nodes[child_index].token_id as u64).is_err()+ {+ continue;+ }++ let child = recursive_build(+ nodes,+ child_index,+ root_position,+ #[cfg(grammar)]+ grammar.as_deref_mut(),+ prng,+ );++ #[cfg(grammar)]+ if let Some(grammar) = grammar.as_mut() {+ grammar.rollback(1);+ }++ trie_node.add(child).expect("tree children are selected without replacement");+ }+ trie_node+ }++ Ok(recursive_build(+ &nodes,+ 0,+ root_position,+ #[cfg(grammar)]+ grammar,+ prng,+ ))+ }+}diff --git a/crates/backend-uzu/src/speculators/mod.rs b/crates/backend-uzu/src/speculators/mod.rsindex 963522197..8afa45e70 100644--- a/crates/backend-uzu/src/speculators/mod.rs+++ b/crates/backend-uzu/src/speculators/mod.rs@@ -1 +1 @@-pub mod dflash_speculator;+pub mod dflash_tfm;