A high-performance inference engine for AI models
DFlash cleanup (#654)
CCYehc0ec32d12ca2
diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rsindex 55abe8edc..d125f0f4a 100644--- a/crates/backend-uzu/src/encodable_block/dflash.rs+++ b/crates/backend-uzu/src/encodable_block/dflash.rs@@ -3,19 +3,17 @@ use thiserror::Error;use crate::{array::size_for_shape,backends::common::{- Allocation, Backend, Encoder, Kernels,- gpu_types::trie::TrieNode,- kernel::{TensorAddSwapKernel, radix_top_k_small::RadixTopKSmall},+ Allocation, Backend, Encoder, Kernels, gpu_types::trie::TrieNode, kernel::radix_top_k_small::RadixTopKSmall,},config::{dflash::{DFlashDraftConfig, DFlashDraftLayerConfig},mlp::AnyMLPConfig,- normalization::NormalizationConfig,rope::AnyRoPEConfig,},data_type::DataType,encodable_block::{batch_topology::BatchTopology,+ embedding::{Embedding, EmbeddingError},linear::{Linear, LinearBlockError},mixer::{Mixer, MixerState,@@ -39,14 +37,14 @@ pub struct DFlashState<B: Backend> {}impl<B: Backend> DFlashState<B> {- pub(crate) fn context_length(&self) -> usize {+ pub fn context_length(&self) -> usize {self.context_length}}-pub(crate) struct DFlashDraft<B: Backend> {- context_projection: Box<dyn Linear<B>>,- context_norm: Normalization<B>,+pub struct DFlashDraft<B: Backend> {+ target_feature_projection: Box<dyn Linear<B>>,+ projected_feature_norm: Normalization<B>,layers: Box<[DFlashDraftLayer<B>]>,output_norm: Normalization<B>,top_k: <B::Kernels as Kernels>::RadixTopKSmall,@@ -54,16 +52,28 @@ pub(crate) struct DFlashDraft<B: Backend> {model_dim: usize,max_context_length: usize,block_size: usize,- target_feature_width: usize,+ mask_token_id: u32,+ target_feature_input_dim: usize,data_type: DataType,}+pub struct DFlashDraftOutput<B: Backend> {+ pub candidates: TopKCandidates<B>,+ pub draft_hidden: Allocation<B>,+}++pub struct TopKCandidates<B: Backend> {+ pub ids: Allocation<B>,+ pub scores: Allocation<B>,+ pub rows: usize,+ pub candidates_per_row: usize,+}+struct DFlashDraftLayer<B: Backend> {attention: Attention<B>,- input_norm: Normalization<B>,- post_attention_norm: Normalization<B>,+ pre_attention_norm: Normalization<B>,+ pre_mlp_norm: Normalization<B>,mlp: Box<dyn Mlp<B>>,- residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,}#[derive(Debug, Error)]@@ -84,32 +94,23 @@ pub enum DFlashDraftNewError<B: Backend> {InvalidAttentionConfig(&'static str),}-fn plain_norm<B: Backend>(- context: &B::Context,- model_dim: usize,- config: &NormalizationConfig,- parameter_tree: &ParameterTree<B>,- data_type: DataType,-) -> Result<Normalization<B>, NormalizationNewError<B>> {- Normalization::new(- model_dim,- None,- ShortcutMode::None,- PostLayerScalar::None,- data_type,- config,- parameter_tree,- context,- )+#[derive(Debug, Error)]+pub enum DFlashDraftEncodeError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Embedding error: {0}")]+ Embedding(#[from] EmbeddingError<B>),}impl<B: Backend> DFlashDraft<B> {- pub(crate) fn new(+ pub fn new(context: &B::Context,config: &DFlashDraftConfig,parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Self, DFlashDraftNewError<B>> {+ let vocab_size = config.vocab_size as u32;+ let mask_token_id = config.mask_token_id as u32;if config.layer_configs.is_empty() {return Err(DFlashDraftNewError::InvalidAttentionConfig("at least one DFlash layer is required"));}@@ -119,7 +120,7 @@ impl<B: Backend> DFlashDraft<B> {));}- let context_projection = <dyn Linear<B>>::new(+ let target_feature_projection = <dyn Linear<B>>::new(config.model_dim * config.target_layer_ids.len(),[config.model_dim],false,@@ -127,12 +128,15 @@ impl<B: Backend> DFlashDraft<B> {data_type,¶meter_tree.subtree("context_projection")?,)?;- let context_norm = plain_norm(- context,+ let projected_feature_norm = Normalization::new(config.model_dim,+ None,+ ShortcutMode::None,+ PostLayerScalar::None,+ data_type,&config.context_norm_config,¶meter_tree.subtree("context_norm")?,- data_type,+ context,)?;let layers_tree = parameter_tree.subtree("layers")?;let layers = config@@ -142,6 +146,7 @@ impl<B: Backend> DFlashDraft<B> {.map(|(index, layer_config)| {DFlashDraftLayer::new(context,+ index,config.model_dim,config.hidden_dim,layer_config,@@ -151,19 +156,22 @@ impl<B: Backend> DFlashDraft<B> {)}).collect::<Result<Box<[_]>, DFlashDraftNewError<B>>>()?;- let output_norm = plain_norm(- context,+ let output_norm = Normalization::new(config.model_dim,+ None,+ ShortcutMode::Add,+ PostLayerScalar::None,+ data_type,&config.output_norm_config,¶meter_tree.subtree("output_norm")?,- data_type,+ context,)?;- let top_k = <B::Kernels as Kernels>::RadixTopKSmall::new(context, config.vocab_size as u32)- .map_err(DFlashDraftNewError::Backend)?;+ let top_k =+ <B::Kernels as Kernels>::RadixTopKSmall::new(context, vocab_size).map_err(DFlashDraftNewError::Backend)?;Ok(Self {- context_projection,- context_norm,+ target_feature_projection,+ projected_feature_norm,layers,output_norm,top_k,@@ -171,12 +179,13 @@ impl<B: Backend> DFlashDraft<B> {model_dim: config.model_dim,max_context_length: *config.rope_config.max_sequence_length(),block_size: config.block_size,- target_feature_width: config.model_dim * config.target_layer_ids.len(),+ mask_token_id,+ target_feature_input_dim: config.model_dim * config.target_layer_ids.len(),data_type,})}- pub(crate) fn empty_state(+ pub fn empty_state(&self,context_capacity: usize,context: &B::Context,@@ -194,7 +203,7 @@ impl<B: Backend> DFlashDraft<B> {})}- pub(crate) fn append_state(+ pub fn append_state(&self,state: &mut DFlashState<B>,target_features: &[Allocation<B>],@@ -206,16 +215,16 @@ impl<B: Backend> DFlashDraft<B> {}let num_tokens = accepted_indices.len();- let expected_feature_count = self.target_feature_width / self.model_dim;- assert_eq!(target_features.len(), expected_feature_count);+ let captured_layer_count = self.target_feature_input_dim / self.model_dim;+ assert_eq!(target_features.len(), captured_layer_count);let layer_feature_bytes = self.model_dim * self.data_type.size_in_bytes();assert!(target_features.iter().all(|features| features.size() % layer_feature_bytes == 0));let context_length = state.context_length + num_tokens;assert!(context_length <= state.context_capacity, "DFlash state capacity exceeded");assert!(context_length <= self.max_context_length, "DFlash context exceeds configured RoPE capacity");- let mut projection_input =- encoder.allocate_scratch(num_tokens * self.target_feature_width * self.data_type.size_in_bytes())?;+ let mut packed_target_features =+ encoder.allocate_scratch(num_tokens * self.target_feature_input_dim * self.data_type.size_in_bytes())?;for (layer_index, features) in target_features.iter().enumerate() {for (token_index, &accepted_index) in accepted_indices.iter().enumerate() {let source_start = accepted_index * layer_feature_bytes;@@ -224,36 +233,38 @@ impl<B: Backend> DFlashDraft<B> {encoder.encode_copy(features,source_start..source_start + layer_feature_bytes,- &mut projection_input,+ &mut packed_target_features,destination_start..destination_start + layer_feature_bytes,);}}- let projected = self.context_projection.encode(projection_input, num_tokens, encoder)?;- let projected = self.context_norm.encode(&projected, 0, num_tokens, None, encoder)?;+ let projected_features = self.target_feature_projection.encode(packed_target_features, num_tokens, encoder)?;+ let normalized_features =+ self.projected_feature_norm.encode(&projected_features, 0, num_tokens, None, encoder)?;let token_positions = (state.context_length..state.context_length + num_tokens).collect::<Box<[_]>>();let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, encoder)?;- let layer_count = self.layers.len();- let mut projected = Some(projected);- for (index, (layer, state_layer)) in self.layers.iter().zip(state.layer_states.iter_mut()).enumerate() {- state_layer.prepare(state.context_length, num_tokens, encoder.context())?;- let layer_input = if index + 1 == layer_count {- projected.take().expect("projected available for last layer")+ let dflash_layer_count = self.layers.len();+ let mut normalized_features = Some(normalized_features);+ for (layer_index, (layer, attention_state)) in self.layers.iter().zip(state.layer_states.iter_mut()).enumerate()+ {+ attention_state.prepare(state.context_length, num_tokens, encoder.context())?;+ let kv_input = if layer_index + 1 == dflash_layer_count {+ normalized_features.take().expect("normalized features available for last layer")} else {- let source = projected.as_ref().expect("projected available");- let mut layer_input = encoder.allocate_scratch(source.size())?;- encoder.encode_copy(source, .., &mut layer_input, ..);- layer_input+ let source = normalized_features.as_ref().expect("normalized features available");+ let mut kv_input = encoder.allocate_scratch(source.size())?;+ encoder.encode_copy(source, .., &mut kv_input, ..);+ kv_input};- layer.attention.append_kv(layer_input, Some(&rope), num_tokens, state_layer, encoder)?;+ layer.attention.append_kv(kv_input, Some(&rope), num_tokens, attention_state, encoder)?;}state.context_length += num_tokens;Ok(())}- pub(crate) fn encode_block(+ fn encode_hidden(&self,state: &mut DFlashState<B>,token_embeddings: Allocation<B>,@@ -276,58 +287,88 @@ impl<B: Backend> DFlashDraft<B> {let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &token_positions, encoder)?;let mut hidden = token_embeddings;- for (layer, state_layer) in self.layers.iter().zip(state.layer_states.iter_mut()) {- state_layer.prepare(state.context_length, batch_dim, encoder.context())?;- let normalized = layer.input_norm.encode(&hidden, 0, batch_dim, None, encoder)?;- let mut attention_output = layer.attention.encode(+ let mut residual = encoder.allocate_scratch(hidden.size())?;+ for (layer, attention_state) in self.layers.iter().zip(state.layer_states.iter_mut()) {+ attention_state.prepare(state.context_length, batch_dim, encoder.context())?;+ let normalized = layer.pre_attention_norm.encode(&hidden, 0, batch_dim, Some(&mut residual), encoder)?;+ let attention_output = layer.attention.encode(normalized,Some(&rope),&batch_topology,- Some(MaybeMut::Mut(state_layer)),+ Some(MaybeMut::Mut(attention_state)),encoder,)?;- let mut attention_residual = encoder.allocate_scratch(hidden.size())?;- encoder.encode_copy(&hidden, .., &mut attention_residual, ..);- layer.residual_add.encode(- &mut attention_residual,- &mut attention_output,- (batch_dim * self.model_dim) as u32,- encoder,- );- let normalized = layer.post_attention_norm.encode(&attention_residual, 0, batch_dim, None, encoder)?;- let mut mlp_output = layer.mlp.encode(normalized, batch_dim, encoder)?;-- let mut output = encoder.allocate_scratch(hidden.size())?;- encoder.encode_copy(&attention_residual, .., &mut output, ..);- layer.residual_add.encode(&mut output, &mut mlp_output, (batch_dim * self.model_dim) as u32, encoder);- hidden = output;+ let normalized =+ layer.pre_mlp_norm.encode(&attention_output, 0, batch_dim, Some(&mut residual), encoder)?;+ hidden = layer.mlp.encode(normalized, batch_dim, encoder)?;}- self.output_norm.encode(&hidden, 0, batch_dim, None, encoder)+ self.output_norm.encode(&hidden, 0, batch_dim, Some(&mut residual), encoder)+ }++ pub fn encode_draft(+ &self,+ state: &mut DFlashState<B>,+ target_output_token: u32,+ target_embedding: &Embedding<B>,+ candidate_count: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<DFlashDraftOutput<B>, DFlashDraftEncodeError<B>> {+ let mut token_ids = encoder+ .allocate_constant(self.block_size * DataType::U32.size_in_bytes())+ .map_err(DFlashDraftEncodeError::Backend)?;+ let mut tokens = vec![self.mask_token_id; self.block_size];+ tokens[0] = target_output_token;+ token_ids.copyin(&tokens);++ let token_embeddings = target_embedding.encode_lookup(&token_ids, self.block_size, encoder)?;+ let draft_hidden =+ self.encode_hidden(state, token_embeddings, encoder).map_err(DFlashDraftEncodeError::Backend)?;++ // The first block row is the target's output token; only the lookahead rows are ranked.+ let lookahead_rows = self.block_size - 1;+ let row_bytes = target_embedding.model_dim() * DataType::BF16.size_in_bytes();+ let mut lookahead_hidden =+ encoder.allocate_scratch(lookahead_rows * row_bytes).map_err(DFlashDraftEncodeError::Backend)?;+ encoder.encode_copy(&draft_hidden, row_bytes..self.block_size * row_bytes, &mut lookahead_hidden, ..);+ let logits = target_embedding.encode_readout(lookahead_rows, &lookahead_hidden, DataType::F32, encoder)?;+ let candidates = self+ .encode_candidates(&logits, lookahead_rows, candidate_count, encoder)+ .map_err(DFlashDraftEncodeError::Backend)?;+ drop(logits);+ drop(token_ids);+ drop(lookahead_hidden);++ Ok(DFlashDraftOutput {+ candidates,+ draft_hidden,+ })}- pub(crate) fn encode_top_k(+ fn encode_candidates(&self,logits: &Allocation<B>,rows: usize,k: usize,encoder: &mut Encoder<B>,- ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {+ ) -> Result<TopKCandidates<B>, B::Error> {assert!(k > 0 && k <= RADIX_TOP_K_MAX);let mut ids = encoder.allocate_scratch(size_for_shape(&[rows, k], DataType::U32))?;let mut scores = encoder.allocate_scratch(size_for_shape(&[rows, k], DataType::F32))?;self.top_k.encode(logits, &mut ids, &mut scores, rows as u32, k as u32, encoder)?;- Ok((ids, scores))- }-- pub(crate) fn block_size(&self) -> usize {- self.block_size+ Ok(TopKCandidates {+ ids,+ scores,+ rows,+ candidates_per_row: k,+ })}}impl<B: Backend> DFlashDraftLayer<B> {fn new(context: &B::Context,+ layer_index: usize,model_dim: usize,hidden_dim: usize,config: &DFlashDraftLayerConfig,@@ -349,32 +390,38 @@ impl<B: Backend> DFlashDraftLayer<B> {¶meter_tree.subtree("attention")?,context,)?;- let input_norm = plain_norm(- context,+ let pre_attention_norm = Normalization::new(model_dim,+ None,+ if layer_index == 0 {+ ShortcutMode::Copy+ } else {+ ShortcutMode::Add+ },+ PostLayerScalar::None,+ data_type,&config.input_norm_config,¶meter_tree.subtree("input_norm")?,- data_type,- )?;- let post_attention_norm = plain_norm(context,+ )?;+ let pre_mlp_norm = Normalization::new(model_dim,+ None,+ ShortcutMode::Add,+ PostLayerScalar::None,+ data_type,&config.post_attention_norm_config,¶meter_tree.subtree("post_attention_norm")?,- data_type,+ context,)?;let mlp_config = AnyMLPConfig::DenseMLPConfig(config.mlp_config.clone());let (mlp, _) =<dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, ¶meter_tree.subtree("mlp")?, data_type)?;- let residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type)- .map_err(DFlashDraftNewError::Backend)?;-Ok(Self {attention,- input_norm,- post_attention_norm,+ pre_attention_norm,+ pre_mlp_norm,mlp,- residual_add,})}}diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rsindex b187148c8..bbee7a1f1 100644--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs@@ -15,8 +15,8 @@ use crate::{config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},data_type::DataType,encodable_block::{- dflash::{DFlashDraft, DFlashDraftNewError},- embedding::{Embedding, EmbeddingError},+ dflash::{DFlashDraft, DFlashDraftEncodeError, DFlashDraftNewError, DFlashDraftOutput},+ embedding::Embedding,sampling::PRng,weaver::{ProposalNode, Weaver, WeaverEncodeError, WeaverNewError, WeaverTreeInput},},@@ -39,21 +39,14 @@ const MAX_CHILDREN_PER_NODE: usize = 8;pub enum DFlashTreeError<B: Backend> {#[error("backend error: {0}")]Backend(#[source] B::Error),- #[error("embedding error: {0}")]- Embedding(#[from] EmbeddingError<B>),+ #[error("DFlash draft error: {0}")]+ Draft(#[from] DFlashDraftEncodeError<B>),#[error("Weaver error: {0}")]Weaver(#[from] WeaverEncodeError<B>),#[error("invalid tree options")]InvalidOptions,}-struct DFlashChainOutput<B: Backend> {- pool_ids: Allocation<B>,- pool_scores: Allocation<B>,- draft_logits: Allocation<B>,- draft_hidden: Allocation<B>,-}-#[derive(Debug, Error)]pub enum DFlashSpeculatorLoadError<B: Backend> {#[error("I/O error: {0}")]@@ -68,11 +61,6 @@ pub enum DFlashSpeculatorLoadError<B: Backend> {Draft(#[from] DFlashDraftNewError<B>),#[error("Weaver error: {0}")]Weaver(#[from] WeaverNewError<B>),- #[error("DFlash mask_token_id {mask_token_id} is outside vocabulary size {vocab_size}")]- InvalidMaskTokenId {- mask_token_id: u64,- vocab_size: usize,- },}pub struct DFlashSpeculator<B: Backend> {@@ -90,14 +78,6 @@ impl<B: Backend> DFlashSpeculator<B> {let config: SpeculatorModelConfig =serde_json::from_reader(BufReader::new(File::open(model_path.join("config.json"))?))?;let AnySpeculatorConfig::DFlashSpeculatorConfig(config) = config.speculator_config;- let draft_config = &config.draft_config;- if draft_config.mask_token_id >= draft_config.vocab_size as u64 {- return Err(DFlashSpeculatorLoadError::InvalidMaskTokenId {- mask_token_id: draft_config.mask_token_id,- vocab_size: draft_config.vocab_size,- });- }-let data_type = DataType::BF16;let weights_file = File::open(model_path.join("model.safetensors"))?;@@ -161,16 +141,16 @@ impl<B: Backend> DFlashSpeculator<B> {{return Err(DFlashTreeError::InvalidOptions);}- let block_size = self.model.block_size();- let target_model_dim = self.config.draft_config.model_dim;let vocab_size = self.config.draft_config.vocab_size;- let pool_size =+ let target_model_dim = self.config.draft_config.model_dim;+ let candidates_per_row =self.config.weaver_config.as_ref().map_or(1, |config| config.candidate_pool_size.min(vocab_size));if target_output_token as usize >= vocab_size- || pool_size == 0- || options.children_per_node > pool_size- || target_embedding.vocab_size() != self.config.draft_config.vocab_size+ || candidates_per_row == 0+ || options.children_per_node > candidates_per_row+ || target_embedding.vocab_size() != vocab_size|| target_embedding.model_dim() != target_model_dim+ || target_output_norm.size() != target_model_dim * DataType::BF16.size_in_bytes()|| self.config.weaver_config.as_ref().is_some_and(|config| config.target_model_dim != target_model_dim){return Err(DFlashTreeError::InvalidOptions);@@ -178,21 +158,10 @@ impl<B: Backend> DFlashSpeculator<B> {let root_position = state.context_length();let mut encoder = Encoder::new(&*self.context).map_err(DFlashTreeError::Backend)?;- let DFlashChainOutput {- pool_ids,- pool_scores,- draft_logits,+ let DFlashDraftOutput {+ candidates,draft_hidden,- } = self.encode_dflash_chain(- &mut encoder,- state,- target_output_norm,- target_output_token,- target_embedding,- block_size,- target_model_dim,- pool_size,- )?;+ } = self.model.encode_draft(state, target_output_token, target_embedding, candidates_per_row, &mut encoder)?;if let Some(weaver) = self.weaver.as_ref() {let max_depth = self.config.weaver_config.as_ref().expect("a Weaver implies a Weaver config").max_depth;@@ -203,10 +172,10 @@ impl<B: Backend> DFlashSpeculator<B> {target_hidden: target_output_norm,draft_hidden: &draft_hidden,target_embedding,- candidate_ids: &pool_ids,- candidate_scores: &pool_scores,- candidate_rows: block_size - 1,- candidates_per_row: pool_size,+ candidate_ids: &candidates.ids,+ candidate_scores: &candidates.scores,+ candidate_rows: candidates.rows,+ candidates_per_row: candidates.candidates_per_row,depth_seeds: &depth_seeds,root_token_id: target_output_token,tree_budget: options.budget,@@ -218,9 +187,7 @@ impl<B: Backend> DFlashSpeculator<B> {)?;let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;let nodes = tree.decode();- drop(pool_ids);- drop(pool_scores);- drop(draft_logits);+ drop(candidates);drop(draft_hidden);drop(completed);return Ok(Self::finish_tree(@@ -233,10 +200,10 @@ impl<B: Backend> DFlashSpeculator<B> {}let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;- let pool_id_values = pool_ids.copyout::<u32>();- drop(pool_ids);- drop(pool_scores);- drop(draft_logits);+ let candidate_rows = candidates.rows;+ let candidates_per_row = candidates.candidates_per_row;+ let candidate_id_values = candidates.ids.copyout::<u32>();+ drop(candidates);drop(draft_hidden);drop(completed);@@ -245,8 +212,8 @@ impl<B: Backend> DFlashSpeculator<B> {depth: 0,children: Vec::new(),}];- for depth in 0..options.budget.min(block_size.saturating_sub(1)) {- let token = pool_id_values[depth * pool_size];+ for depth in 0..options.budget.min(candidate_rows) {+ let token = candidate_id_values[depth * candidates_per_row];let parent = nodes.len() - 1;let index = nodes.len();nodes.push(ProposalNode {@@ -265,50 +232,6 @@ impl<B: Backend> DFlashSpeculator<B> {))}- fn encode_dflash_chain(- &self,- encoder: &mut Encoder<B>,- state: &mut DFlashState<B>,- target_output_norm: &Allocation<B>,- target_output_token: u32,- target_embedding: &Embedding<B>,- block_size: usize,- target_model_dim: usize,- pool_size: usize,- ) -> Result<DFlashChainOutput<B>, DFlashTreeError<B>> {- if target_output_norm.size() != target_model_dim * DataType::BF16.size_in_bytes() {- return Err(DFlashTreeError::InvalidOptions);- }-- let mut noise_ids =- encoder.allocate_constant(block_size * DataType::U32.size_in_bytes()).map_err(DFlashTreeError::Backend)?;- let mut noise = vec![self.config.draft_config.mask_token_id as u32; block_size];- noise[0] = target_output_token;- noise_ids.copyin(&noise);- let token_embeddings = target_embedding.encode_lookup(&noise_ids, block_size, encoder)?;- let draft_hidden =- self.model.encode_block(state, token_embeddings, encoder).map_err(DFlashTreeError::Backend)?;- // The first block row is the target's output token; only the lookahead rows are ranked.- let row_bytes = target_embedding.model_dim() * DataType::BF16.size_in_bytes();- let mut lookahead_hidden =- encoder.allocate_scratch((block_size - 1) * row_bytes).map_err(DFlashTreeError::Backend)?;- encoder.encode_copy(&draft_hidden, row_bytes..block_size * row_bytes, &mut lookahead_hidden, ..);- let draft_logits =- target_embedding.encode_readout(block_size - 1, &lookahead_hidden, DataType::F32, encoder)?;- let (pool_ids, pool_scores) = self- .model- .encode_top_k(&draft_logits, block_size - 1, pool_size, encoder)- .map_err(DFlashTreeError::Backend)?;- drop(noise_ids);- drop(lookahead_hidden);- Ok(DFlashChainOutput {- pool_ids,- pool_scores,- draft_logits,- draft_hidden,- })- }-fn finish_tree(nodes: Vec<ProposalNode>,prng: &PRng,