A high-performance inference engine for AI models
Add shared-Gumbel noise to the Weaver final pruning score (#870)
Maksim Nakhodnovea3ca0b42738
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs b/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rsindex d9f53fc58..7c6b384e7 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs@@ -20,6 +20,7 @@ pub fn weaver_frontier_insert_children(node_valid: *const u32,child_ids: *const u32,child_logprobs: *const f32,+ child_prune_logprobs: *const f32,frontier: *mut u32,frontier_capacity: u32,tree_slot_count: u32,@@ -38,6 +39,7 @@ pub fn weaver_frontier_insert_children(let node_valid = unsafe { std::slice::from_raw_parts(node_valid, node_count) };let child_ids = unsafe { std::slice::from_raw_parts(child_ids, node_count * expand_width) };let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, node_count * expand_width) };+ let child_prune_logprobs = unsafe { std::slice::from_raw_parts(child_prune_logprobs, node_count * expand_width) };let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * frontier_capacity) };for index in 0..node_count * expand_width {@@ -58,7 +60,8 @@ pub fn weaver_frontier_insert_children(frontier[FrontierIdx::Depth as usize * frontier_capacity + slot] =packed_tree[TreeIdx::Depth as usize * tree_slot_count + parent] + 1;frontier[FrontierIdx::PathLogprobBits as usize * frontier_capacity + slot] = cumulative_logprob.to_bits();- frontier[FrontierIdx::EdgeLogprobBits as usize * frontier_capacity + slot] = logprob.to_bits();+ frontier[FrontierIdx::EdgeLogprobBits as usize * frontier_capacity + slot] =+ child_prune_logprobs[index].to_bits();frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot] = top_k_score_key(cumulative_logprob);frontier[FrontierIdx::Active as usize * frontier_capacity + slot] = 1;}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_top_children.rs b/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_top_children.rsindex bfee25425..310e5bad9 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_top_children.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_top_children.rs@@ -15,10 +15,13 @@ pub fn weaver_top_children(node_metadata: *const u32,output_token_ids: *mut u32,output_model_logprobs: *mut f32,+ #[optional(has_prune_noise)] output_prune_logprobs: Option<*mut f32>,rows: u32,candidates: u32,expand_width: u32,vocab_size: u32,+ #[optional(has_prune_noise)] prune_noise_scale: Option<f32>,+ #[specialize] has_prune_noise: bool,) {let rows = rows as usize;let candidates = candidates as usize;@@ -39,6 +42,17 @@ pub fn weaver_top_children(let perturbed = (0..candidates).map(|index| logits[index] + gumbel_float(seed, revidx(token(index), vocab_size))).collect::<Vec<_>>();+ // Final pruning weighs each edge by the pool softmax of logits plus the target's noise scaled by 1 / sigma.+ let prune = has_prune_noise.then(|| {+ let scale = prune_noise_scale.unwrap();+ let prune_logits = (0..candidates)+ .map(|index| logits[index] + scale * gumbel_float(seed, revidx(token(index), vocab_size)))+ .collect::<Vec<_>>();+ let prune_max = prune_logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);+ let prune_log_sum =+ prune_logits.iter().map(|value| (value - prune_max).exp()).sum::<f32>().ln() + prune_max;+ (prune_logits, prune_log_sum)+ });let mut indices = (0..candidates).collect::<Vec<_>>();indices.sort_by(|&left, &right| {perturbed[right].total_cmp(&perturbed[left]).then_with(|| token(left).cmp(&token(right)))@@ -47,6 +61,10 @@ pub fn weaver_top_children(unsafe {*output_token_ids.add(row * expand_width + rank) = token(index);*output_model_logprobs.add(row * expand_width + rank) = logits[index] - log_sum;+ if let Some((prune_logits, prune_log_sum)) = &prune {+ *output_prune_logprobs.unwrap().add(row * expand_width + rank) =+ prune_logits[index] - prune_log_sum;+ }}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal b/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metalindex 1159407db..dcaed28f0 100644--- a/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal@@ -11,6 +11,7 @@ PUBLIC KERNEL(WeaverFrontierInsertChildren)(const device uint* node_valid,const device uint* child_ids,const device float* child_logprobs,+ const device float* child_prune_logprobs,device uint* frontier,constant uint& frontier_capacity,constant uint& tree_slot_count,@@ -45,7 +46,8 @@ PUBLIC KERNEL(WeaverFrontierInsertChildren)(frontier[uint(FrontierIdx::Depth) * frontier_capacity + slot] =packed_tree[uint(TreeIdx::Depth) * tree_slot_count + parent] + 1u;frontier[uint(FrontierIdx::PathLogprobBits) * frontier_capacity + slot] = as_type<uint>(cumulative_logprob);- frontier[uint(FrontierIdx::EdgeLogprobBits) * frontier_capacity + slot] = as_type<uint>(logprob);+ frontier[uint(FrontierIdx::EdgeLogprobBits) * frontier_capacity + slot] =+ as_type<uint>(child_prune_logprobs[row * expand_width + child]);frontier[uint(FrontierIdx::PathScoreKey) * frontier_capacity + slot] = top_k_score_key(cumulative_logprob);frontier[uint(FrontierIdx::Active) * frontier_capacity + slot] = 1u;}diff --git a/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_top_children.metal b/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_top_children.metalindex c4757d6ac..156462a5c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_top_children.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_top_children.metal@@ -2,6 +2,7 @@#include "../common/defines.h"#include "../common/dsl.h"#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"#include "../common/top_k.h"#include "../rng.h"#include "weaver_frontier.h"@@ -26,6 +27,15 @@ METAL_FUNC float gumbel_noise(uint64_t seed, uint logit_idx, uint vocab_size) {return -log(-log(uniform));}+// The noise the target sampler adds for this token: the 24-bit uniform of `uniform_float` in rng.h keeps it finite.+METAL_FUNC float target_gumbel_noise(uint64_t seed, uint logit_idx, uint vocab_size) {+ const uint2 offset_word = gumbel_revidx(logit_idx, vocab_size);+ PhiloxState rng;+ philox_init(&rng, seed, offset_word.x);+ const float uniform = float(max(rng.output[offset_word.y] >> 8, 1u)) * (1.0f / 16777216.0f);+ return -log(-log(uniform));+}+METAL_FUNC bool weaver_better(uint score, uint token, uint index, uint best_score, uint best_token, uint best_index) {return score > best_score ||(score == best_score && (token < best_token || (token == best_token && index < best_index)));@@ -39,10 +49,13 @@ PUBLIC KERNEL(WeaverTopChildren)(const device uint* node_metadata,device uint* output_token_ids,device float* output_model_logprobs,+ device float* output_prune_logprobs OPTIONAL(has_prune_noise),constant uint& rows,constant uint& candidates,constant uint& expand_width,constant uint& vocab_size,+ constant float& prune_noise_scale OPTIONAL(has_prune_noise),+ const bool has_prune_noise SPECIALIZE,threadgroup float reduce_float[TOP_CHILDREN_SIMDGROUPS],threadgroup uint reduce_score[TOP_CHILDREN_SIMDGROUPS],threadgroup uint reduce_token[TOP_CHILDREN_SIMDGROUPS],@@ -112,6 +125,31 @@ PUBLIC KERNEL(WeaverTopChildren)(}threadgroup_barrier(mem_flags::mem_threadgroup);+ // Final pruning weighs each edge by the pool softmax of logits plus the target's noise scaled by 1 / sigma.+ float first_prune_logit = -INFINITY;+ float second_prune_logit = -INFINITY;+ float prune_log_sum = 0.0f;+ if (has_prune_noise) {+ if (first_valid) {+ first_prune_logit = first_logit + prune_noise_scale * target_gumbel_noise(seed, first_token, vocab_size);+ }+ if (second_valid) {+ second_prune_logit = second_logit + prune_noise_scale * target_gumbel_noise(seed, second_token, vocab_size);+ }+ const float prune_max = threadgroup_cooperative_reduce<SimdReduceMax<float>, TOP_CHILDREN_THREADS>(+ fmax(first_prune_logit, second_prune_logit),+ reduce_float,+ thread_context+ );+ const float prune_total = threadgroup_cooperative_reduce<SimdReduceSum<float>, TOP_CHILDREN_THREADS>(+ (first_valid ? exp(first_prune_logit - prune_max) : 0.0f) ++ (second_valid ? exp(second_prune_logit - prune_max) : 0.0f),+ reduce_float,+ thread_context+ );+ prune_log_sum = log(prune_total) + prune_max;+ }+for (uint child = 0; child < expand_width; ++child) {uint local_score = 0u;uint local_token = 0xffffffffu;@@ -162,6 +200,10 @@ PUBLIC KERNEL(WeaverTopChildren)(output_token_ids[row * expand_width + child] = winner_token;output_model_logprobs[row * expand_width + child] = winner_logit - log_sum;}+ if (has_prune_noise && (first_index == winner_index || second_index == winner_index)) {+ output_prune_logprobs[row * expand_width + child] =+ (first_index == winner_index ? first_prune_logit : second_prune_logit) - prune_log_sum;+ }first_active = first_active && first_index != winner_index;second_active = second_active && second_index != winner_index;}diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex 309fc4697..479c704d9 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -36,6 +36,7 @@ pub struct WeaverTreeShape {pub rounds: u32,pub expand_per_round: u32,pub expand_width: u32,+ pub prune_noise_scale: Option<f32>,}impl WeaverTreeShape {@@ -121,6 +122,7 @@ pub struct Weaver<B: Backend> {rope_config: AnyRoPEConfig,top_k: <B::Kernels as Kernels>::RadixTopKSmall,top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,+ top_children_with_prune_noise: <B::Kernels as Kernels>::WeaverTopChildrenKernel,frontier_select: <B::Kernels as Kernels>::WeaverFrontierSelectKernel,frontier_insert_children: <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel,model_dim: u32,@@ -257,7 +259,9 @@ impl<B: Backend> Weaver<B> {let top_k =<B::Kernels as Kernels>::RadixTopKSmall::new(context, vocab_size).map_err(WeaverNewError::Backend)?;let top_children =- <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;+ <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context, false).map_err(WeaverNewError::Backend)?;+ let top_children_with_prune_noise =+ <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context, true).map_err(WeaverNewError::Backend)?;let frontier_select =<B::Kernels as Kernels>::WeaverFrontierSelectKernel::new(context).map_err(WeaverNewError::Backend)?;let frontier_insert_children = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(context)@@ -273,6 +277,7 @@ impl<B: Backend> Weaver<B> {rope_config: config.rope_config.clone(),top_k,top_children,+ top_children_with_prune_noise,frontier_select,frontier_insert_children,model_dim: config.model_dim,@@ -488,7 +493,17 @@ impl<B: Backend> Weaver<B> {let mut child_logprobs = encoder.allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::F32)).map_err(WeaverEncodeError::Backend)?;- self.top_children.encode(+ let mut child_prune_logprobs = shape+ .prune_noise_scale+ .map(|_| encoder.allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::F32)))+ .transpose()+ .map_err(WeaverEncodeError::Backend)?;+ let top_children = if shape.prune_noise_scale.is_some() {+ &self.top_children_with_prune_noise+ } else {+ &self.top_children+ };+ top_children.encode(&logit_residuals,batch_candidate_logits,batch_candidate_ids,@@ -496,19 +511,23 @@ impl<B: Backend> Weaver<B> {&*node_metadata,&mut child_token_ids,&mut child_logprobs,+ child_prune_logprobs.as_mut(),batch_node_count,self.candidate_pool_size,shape.expand_width,target_embedding.vocab_size(),+ shape.prune_noise_scale,encoder,);+ // The edge lane feeds only final pruning; without prune noise it keeps the model logprobs.self.frontier_insert_children.encode(&*packed_tree,&*node_metadata,&*node_valid,&child_token_ids,&child_logprobs,+ child_prune_logprobs.as_ref().unwrap_or(&child_logprobs),frontier,frontier_capacity,tree_slot_count,@@ -547,6 +566,7 @@ impl<B: Backend> Weaver<B> {|| shape.expand_width > self.candidate_pool_size|| tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width|| depth_seeds.len() as u32 != self.max_depth+ || shape.prune_noise_scale.is_some_and(|scale| !(scale > 0.0 && scale.is_finite())){return Err(WeaverEncodeError::InvalidTreeInput);}diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 955342c29..18216ab7e 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -562,6 +562,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let (input_trie, chain_copy, full_accept) = if let Some(speculator) = &self.model.speculator&& let Some(shape) = speculator.make_shape(self.model_state.max_context_length.map(|max_context_length| max_context_length - context_length),+ &self.options.sampling_method,)&& let (root_token, Some(output_norm)) = prev_output.resolve(&mut self.model_state.tokens,diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex 2ce2a3fee..ecb16e409 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -65,9 +65,19 @@ pub enum DFlashTfmTreeConstructionMethod {rounds: u32,expand_per_round: u32,expand_width: u32,+ /// A shape without it prunes with `DEFAULT_PRUNE_SIGMA`; `null` prunes on the model logprobs.+ #[serde(default = "default_prune_sigma")]+ prune_sigma: Option<f32>,},}+/// The final pruning noise scale calibrated for DFlash + Weaver trees.+pub const DEFAULT_PRUNE_SIGMA: f32 = 1.5;++fn default_prune_sigma() -> Option<f32> {+ Some(DEFAULT_PRUNE_SIGMA)+}+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]pub struct DFlashTfmTreeShape {pub tree_budget: u32,@@ -164,6 +174,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {pub fn make_shape(&self,max_depth: Option<u32>,+ sampling_method: &SamplingMethod,) -> Option<DFlashTfmTreeShape> {if max_depth.is_some_and(|max_depth| max_depth < 2) {return None;@@ -183,6 +194,16 @@ impl<B: Backend> DFlashTfmSpeculator<B> {}}+ // Greedy verification adds no Gumbel noise to the target logits, so pruning has none to anticipate.+ if matches!(sampling_method, SamplingMethod::Greedy)+ && let DFlashTfmTreeConstructionMethod::Weaver {+ prune_sigma,+ ..+ } = &mut shape.construction_method+ {+ *prune_sigma = None;+ }+Some(shape)}@@ -279,6 +300,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {rounds,expand_per_round,expand_width,+ prune_sigma,} => {let weaver =self.weaver.as_ref().expect("weaver tree construction requires a speculator with weaver weights");@@ -298,6 +320,13 @@ impl<B: Backend> DFlashTfmSpeculator<B> {dflash_depth)));}+ if let Some(prune_sigma) = prune_sigma+ && !(prune_sigma > 0.0 && prune_sigma.is_finite() && prune_sigma.recip().is_finite())+ {+ return Err(DFlashTreeError::InvalidTreeShape(format!(+ "prune sigma {prune_sigma} is not positive and finite"+ )));+ }let dflash_output = self.dflash.encode_draft(state,target_output_token,@@ -322,6 +351,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {rounds,expand_per_round,expand_width,+ prune_noise_scale: prune_sigma.map(f32::recip),},&mut encoder,)?;@@ -387,3 +417,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {Ok(trie)}}++#[cfg(test)]+#[path = "../../unit/speculators/dflash_tfm_test.rs"]+mod tests;diff --git a/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_frontier_test.rs b/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_frontier_test.rsindex 4136ebb2c..c5a2d67de 100644--- a/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_frontier_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_frontier_test.rs@@ -77,11 +77,15 @@ fn select<B: Backend>() -> Vec<u32> {.collect()}+const TREE_PATH_LOGPROBS: [f32; 4] = [0.5, -1.0, 2.0, 4.0];+const MODEL_LOGPROBS: [f32; 9] = [-0.1, -0.2, -0.3, 8.0, 8.0, 8.0, 0.1, 0.2, 0.3];+const PRUNE_LOGPROBS: [f32; 9] = [-1.1, -1.2, -1.3, -9.0, -9.0, -9.0, -0.6, -0.7, -0.8];+fn insert_children<B: Backend>() -> Vec<u32> {let context = create_context::<B>();let mut tree = vec![0; TreeIdx::COUNT * 4];tree[TreeIdx::PathLogprobBits as usize * 4..(TreeIdx::PathLogprobBits as usize + 1) * 4]- .copy_from_slice(&[0.5, -1.0, 2.0, 4.0].map(f32::to_bits));+ .copy_from_slice(&TREE_PATH_LOGPROBS.map(f32::to_bits));tree[TreeIdx::Depth as usize * 4..(TreeIdx::Depth as usize + 1) * 4].copy_from_slice(&[0, 2, 4, 6]);let tree = alloc_allocation_with_data::<B, u32>(&context, &tree);let mut metadata = vec![0; MetadataIdx::COUNT * 3];@@ -89,15 +93,40 @@ fn insert_children<B: Backend>() -> Vec<u32> {let metadata = alloc_allocation_with_data::<B, u32>(&context, &metadata);let valid = alloc_allocation_with_data::<B, u32>(&context, &[1, 0, 1]);let ids = alloc_allocation_with_data::<B, u32>(&context, &(10..19).collect::<Vec<_>>());- let scores = alloc_allocation_with_data::<B, f32>(&context, &[-0.1, -0.2, -0.3, 8.0, 8.0, 8.0, 0.1, 0.2, 0.3]);+ let scores = alloc_allocation_with_data::<B, f32>(&context, &MODEL_LOGPROBS);+ let prune_scores = alloc_allocation_with_data::<B, f32>(&context, &PRUNE_LOGPROBS);let mut frontier = alloc_allocation_with_data::<B, u32>(&context, &[42; FrontierIdx::COUNT * 16]);let kernel = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(&context).unwrap();let mut encoder = Encoder::new(context.as_ref()).unwrap();- kernel.encode(&tree, &metadata, &valid, &ids, &scores, &mut frontier, 16, 4, 3, 3, &mut encoder);+ kernel.encode(&tree, &metadata, &valid, &ids, &scores, &prune_scores, &mut frontier, 16, 4, 3, 3, &mut encoder);encoder.end_encoding().submit().wait_until_completed().unwrap();allocation_to_vec(&frontier)}+/// Expansion follows the model logprobs (path and its order key), final pruning reads the edge lane, which takes the+/// prune channel. Rows 0 and 2 expand tree slots 1 and 0 into frontier slots 3..6 and 0..3; row 1 is invalid, so no+/// other slot is written.+#[uzu_test]+fn weaver_frontier_insert_children_splits_path_and_edge() {+ let frontier = insert_children::<Cpu>();+ let lane = |field: FrontierIdx, slot: usize| frontier[field as usize * 16 + slot];+ for (row, parent) in [(0, 1), (2, 0)] {+ for child in 0..3 {+ let slot = parent * 3 + child;+ let index = row * 3 + child;+ assert_eq!(lane(FrontierIdx::EdgeLogprobBits, slot), PRUNE_LOGPROBS[index].to_bits());+ assert_eq!(+ lane(FrontierIdx::PathLogprobBits, slot),+ (TREE_PATH_LOGPROBS[parent] + MODEL_LOGPROBS[index]).to_bits()+ );+ assert_eq!(lane(FrontierIdx::Active, slot), 1);+ }+ }+ for slot in 6..16 {+ assert!((0..FrontierIdx::COUNT).all(|field| frontier[field * 16 + slot] == 42));+ }+}+#[uzu_test]fn weaver_frontier_kernels_match_cpu() {for_each_non_cpu_backend!(|B| {diff --git a/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_top_children_test.rs b/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_top_children_test.rsindex 4ce4a33cd..2e513eeb6 100644--- a/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_top_children_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_top_children_test.rs@@ -6,6 +6,7 @@ use crate::{common::{Backend, Encoder, Kernels, gpu_types::weaver::MetadataIdx, kernel::WeaverTopChildrenKernel},cpu::Cpu,},+ encodable_block::sampling::{gumbel_float, revidx},tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context, for_each_non_cpu_backend,},@@ -16,12 +17,27 @@ const CHILDREN: usize = 8;const VOCAB_SIZE: u32 = 131_072;const DEPTHS: [u32; 3] = [0, 1, 2];const DEPTH_SEEDS: [u64; 3] = [0x9E3779B97F4A7C15, 0xD1B54A32D192ED03, 0x2545F4914F6CDD1D];+const ROWS: usize = 3;+const PRUNE_NOISE_SCALE: f32 = 1.0 / 1.5;++fn inputs() -> (Vec<bf16>, Vec<f32>, Vec<u32>) {+ let residual = (0..ROWS * CANDIDATES)+ .map(|index| bf16::from_f32(((index as f32 * 0.017).cos() * 4.0).round() * 0.125))+ .collect::<Vec<_>>();+ let candidate_logits =+ (0..ROWS * CANDIDATES).map(|index| ((index as f32 * 0.011).sin() * 3.0).round() * 0.125).collect::<Vec<_>>();+ let ids = (0..ROWS)+ .flat_map(|row| (0..CANDIDATES).rev().map(move |index| 70_000 + (row * CANDIDATES + index) as u32))+ .collect::<Vec<_>>();+ (residual, candidate_logits, ids)+}fn top_children<B: Backend>(residual: &[bf16],candidate_logits: &[f32],ids: &[u32],-) -> (Vec<u32>, Vec<f32>) {+ prune_noise_scale: Option<f32>,+) -> (Vec<u32>, Vec<f32>, Option<Vec<f32>>) {let rows = residual.len() / CANDIDATES;assert_eq!(residual.len(), rows * CANDIDATES);assert_eq!(candidate_logits.len(), rows * CANDIDATES);@@ -39,7 +55,8 @@ fn top_children<B: Backend>(let node_metadata = alloc_allocation_with_data::<B, u32>(&context, &metadata_values);let mut output_token_ids = alloc_allocation::<B, u32>(&context, rows * CHILDREN);let mut output_model_logprobs = alloc_allocation::<B, f32>(&context, rows * CHILDREN);- let kernel = <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(&context).unwrap();+ let mut output_prune_logprobs = prune_noise_scale.map(|_| alloc_allocation::<B, f32>(&context, rows * CHILDREN));+ let kernel = <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(&context, prune_noise_scale.is_some()).unwrap();let mut encoder = Encoder::new(context.as_ref()).unwrap();kernel.encode(&residual,@@ -49,34 +66,81 @@ fn top_children<B: Backend>(&node_metadata,&mut output_token_ids,&mut output_model_logprobs,+ output_prune_logprobs.as_mut(),rows as u32,CANDIDATES as u32,CHILDREN as u32,VOCAB_SIZE,+ prune_noise_scale,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- (allocation_to_vec(&output_token_ids), allocation_to_vec(&output_model_logprobs))+ (+ allocation_to_vec(&output_token_ids),+ allocation_to_vec(&output_model_logprobs),+ output_prune_logprobs.as_ref().map(allocation_to_vec),+ )}#[uzu_test]fn weaver_top_children_matches_cpu() {- const ROWS: usize = 3;- let residual = (0..ROWS * CANDIDATES)- .map(|index| bf16::from_f32(((index as f32 * 0.017).cos() * 4.0).round() * 0.125))- .collect::<Vec<_>>();- let candidate_logits =- (0..ROWS * CANDIDATES).map(|index| ((index as f32 * 0.011).sin() * 3.0).round() * 0.125).collect::<Vec<_>>();- let ids = (0..ROWS)- .flat_map(|row| (0..CANDIDATES).rev().map(move |index| 70_000 + (row * CANDIDATES + index) as u32))- .collect::<Vec<_>>();+ let (residual, candidate_logits, ids) = inputs();++ for prune_noise_scale in [None, Some(PRUNE_NOISE_SCALE)] {+ for_each_non_cpu_backend!(|B| {+ let expected = top_children::<Cpu>(&residual, &candidate_logits, &ids, prune_noise_scale);+ let actual = top_children::<B>(&residual, &candidate_logits, &ids, prune_noise_scale);+ assert_eq!(actual.0, expected.0);+ for (actual, expected) in actual.1.iter().zip(expected.1) {+ assert!((actual - expected).abs() < 1e-5);+ }+ assert_eq!(actual.2.is_some(), expected.2.is_some());+ // The prune channel also carries the Gumbel transform, which Metal evaluates with fast-math logs.+ for (actual, expected) in actual.2.iter().flatten().zip(expected.2.iter().flatten()) {+ assert!((actual - expected).abs() < 1e-3);+ }+ });+ }+}++/// The prune channel is the log-softmax over the whole pool of logits plus the target sampler's noise at the+/// parent depth's seed, scaled by `prune_noise_scale`; selection and model logprobs stay as without it. The reference+/// takes that noise from `gumbel_float` on purpose, since matching the target sampler is the contract; the seed, the+/// token indexing and the log-sum-exp are computed here independently, in f64.+#[uzu_test]+fn weaver_top_children_prune_logprobs_match_reference() {+ let (residual, candidate_logits, ids) = inputs();+ let (tokens, model, none) = top_children::<Cpu>(&residual, &candidate_logits, &ids, None);+ let (prune_tokens, prune_model, prune) =+ top_children::<Cpu>(&residual, &candidate_logits, &ids, Some(PRUNE_NOISE_SCALE));+ assert!(none.is_none());+ let prune = prune.unwrap();+ assert_eq!(prune_tokens, tokens);+ assert_eq!(+ prune_model.iter().map(|value| value.to_bits()).collect::<Vec<_>>(),+ model.iter().map(|value| value.to_bits()).collect::<Vec<_>>()+ );- for_each_non_cpu_backend!(|B| {- let expected = top_children::<Cpu>(&residual, &candidate_logits, &ids);- let actual = top_children::<B>(&residual, &candidate_logits, &ids);- assert_eq!(actual.0, expected.0);- for (actual, expected) in actual.1.iter().zip(expected.1) {- assert!((actual - expected).abs() < 1e-5);+ for row in 0..ROWS {+ let seed = DEPTH_SEEDS[DEPTHS[row] as usize];+ let values = (0..CANDIDATES)+ .map(|index| {+ let flat = row * CANDIDATES + index;+ candidate_logits[flat] as f64+ + residual[flat].to_f64()+ + PRUNE_NOISE_SCALE as f64 * gumbel_float(seed, revidx(ids[flat], VOCAB_SIZE)) as f64+ })+ .collect::<Vec<_>>();+ let max = values.iter().copied().fold(f64::NEG_INFINITY, f64::max);+ let log_sum = values.iter().map(|value| (value - max).exp()).sum::<f64>().ln() + max;+ for child in 0..CHILDREN {+ let token = tokens[row * CHILDREN + child];+ let index = (0..CANDIDATES).find(|&index| ids[row * CANDIDATES + index] == token).unwrap();+ let actual = prune[row * CHILDREN + child];+ // An f32 log-sum-exp over 512 values of magnitude below 20 keeps about 1e-4 of absolute accuracy.+ assert!((actual as f64 - (values[index] - log_sum)).abs() < 1e-4);+ assert!(actual <= 0.0);}- });+ }+ assert!(prune.iter().zip(&model).any(|(prune, model)| (prune - model).abs() > 0.1));}diff --git a/crates/uzu-engine/unit/speculators/dflash_tfm_test.rs b/crates/uzu-engine/unit/speculators/dflash_tfm_test.rsnew file mode 100644index 000000000..835e130e3--- /dev/null+++ b/crates/uzu-engine/unit/speculators/dflash_tfm_test.rs@@ -0,0 +1,54 @@+use uzu_engine_macros::uzu_test;++use super::{DFlashTfmTreeConstructionMethod, DFlashTfmTreeShape};++fn weaver_shape(construction_method: &str) -> DFlashTfmTreeShape {+ serde_json::from_str(&format!(+ r#"{{"tree_budget":16,"max_tree_depth":16,"dflash_depth_override":null,"construction_method":{construction_method}}}"#+ ))+ .unwrap()+}++/// A shape without `prune_sigma` prunes with noise scale 1.5, `null` turns the noise off, and a number is the scale+/// itself, 0 included: parsing does not reinterpret it, the shape check rejects it.+#[uzu_test]+fn weaver_shape_prune_sigma_is_optional() {+ let weaver = |prune_sigma| DFlashTfmTreeConstructionMethod::Weaver {+ rounds: 16,+ expand_per_round: 4,+ expand_width: 4,+ prune_sigma,+ };+ let method = |prune_sigma: &str| {+ format!(r#"{{"type":"Weaver","rounds":16,"expand_per_round":4,"expand_width":4{prune_sigma}}}"#)+ };+ for (field, expected) in [+ ("", Some(1.5)),+ (r#","prune_sigma":null"#, None),+ (r#","prune_sigma":1.5"#, Some(1.5)),+ (r#","prune_sigma":2"#, Some(2.0)),+ (r#","prune_sigma":0"#, Some(0.0)),+ ] {+ assert_eq!(weaver_shape(&method(field)).construction_method, weaver(expected), "{field:?}");+ }+}++/// A shape written back to JSON reads as the same shape, with the noise on or off.+#[uzu_test]+fn weaver_shape_prune_sigma_round_trips() {+ for prune_sigma in [Some(1.5), Some(2.0), None] {+ let shape = DFlashTfmTreeShape {+ tree_budget: 16,+ max_tree_depth: 16,+ dflash_depth_override: None,+ construction_method: DFlashTfmTreeConstructionMethod::Weaver {+ rounds: 16,+ expand_per_round: 4,+ expand_width: 4,+ prune_sigma,+ },+ };+ let json = serde_json::to_string(&shape).unwrap();+ assert_eq!(serde_json::from_str::<DFlashTfmTreeShape>(&json).unwrap(), shape, "{json}");+ }+}