Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add shared-Gumbel noise to the Weaver final pruning score (#870)

Maksim Nakhodnovea3ca0b42738
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs b/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs
index d9f53fc58..7c6b384e7 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs
@@ -20,6 +20,7 @@ pub fn weaver_frontier_insert_children(
node_valid: *const u32,
child_ids: *const u32,
child_logprobs: *const f32,
+ child_prune_logprobs: *const f32,
frontier: *mut u32,
frontier_capacity: u32,
tree_slot_count: u32,
@@ -38,6 +39,7 @@ pub fn weaver_frontier_insert_children(
let node_valid = unsafe { std::slice::from_raw_parts(node_valid, node_count) };
let child_ids = unsafe { std::slice::from_raw_parts(child_ids, node_count * expand_width) };
let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, node_count * expand_width) };
+ let child_prune_logprobs = unsafe { std::slice::from_raw_parts(child_prune_logprobs, node_count * expand_width) };
let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * frontier_capacity) };
for index in 0..node_count * expand_width {
@@ -58,7 +60,8 @@ pub fn weaver_frontier_insert_children(
frontier[FrontierIdx::Depth as usize * frontier_capacity + slot] =
packed_tree[TreeIdx::Depth as usize * tree_slot_count + parent] + 1;
frontier[FrontierIdx::PathLogprobBits as usize * frontier_capacity + slot] = cumulative_logprob.to_bits();
- frontier[FrontierIdx::EdgeLogprobBits as usize * frontier_capacity + slot] = logprob.to_bits();
+ frontier[FrontierIdx::EdgeLogprobBits as usize * frontier_capacity + slot] =
+ child_prune_logprobs[index].to_bits();
frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot] = top_k_score_key(cumulative_logprob);
frontier[FrontierIdx::Active as usize * frontier_capacity + slot] = 1;
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_top_children.rs b/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_top_children.rs
index bfee25425..310e5bad9 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_top_children.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/weaver/weaver_top_children.rs
@@ -15,10 +15,13 @@ pub fn weaver_top_children(
node_metadata: *const u32,
output_token_ids: *mut u32,
output_model_logprobs: *mut f32,
+ #[optional(has_prune_noise)] output_prune_logprobs: Option<*mut f32>,
rows: u32,
candidates: u32,
expand_width: u32,
vocab_size: u32,
+ #[optional(has_prune_noise)] prune_noise_scale: Option<f32>,
+ #[specialize] has_prune_noise: bool,
) {
let rows = rows as usize;
let candidates = candidates as usize;
@@ -39,6 +42,17 @@ pub fn weaver_top_children(
let perturbed = (0..candidates)
.map(|index| logits[index] + gumbel_float(seed, revidx(token(index), vocab_size)))
.collect::<Vec<_>>();
+ // Final pruning weighs each edge by the pool softmax of logits plus the target's noise scaled by 1 / sigma.
+ let prune = has_prune_noise.then(|| {
+ let scale = prune_noise_scale.unwrap();
+ let prune_logits = (0..candidates)
+ .map(|index| logits[index] + scale * gumbel_float(seed, revidx(token(index), vocab_size)))
+ .collect::<Vec<_>>();
+ let prune_max = prune_logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
+ let prune_log_sum =
+ prune_logits.iter().map(|value| (value - prune_max).exp()).sum::<f32>().ln() + prune_max;
+ (prune_logits, prune_log_sum)
+ });
let mut indices = (0..candidates).collect::<Vec<_>>();
indices.sort_by(|&left, &right| {
perturbed[right].total_cmp(&perturbed[left]).then_with(|| token(left).cmp(&token(right)))
@@ -47,6 +61,10 @@ pub fn weaver_top_children(
unsafe {
*output_token_ids.add(row * expand_width + rank) = token(index);
*output_model_logprobs.add(row * expand_width + rank) = logits[index] - log_sum;
+ if let Some((prune_logits, prune_log_sum)) = &prune {
+ *output_prune_logprobs.unwrap().add(row * expand_width + rank) =
+ prune_logits[index] - prune_log_sum;
+ }
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal b/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal
index 1159407db..dcaed28f0 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal
@@ -11,6 +11,7 @@ PUBLIC KERNEL(WeaverFrontierInsertChildren)(
const device uint* node_valid,
const device uint* child_ids,
const device float* child_logprobs,
+ const device float* child_prune_logprobs,
device uint* frontier,
constant uint& frontier_capacity,
constant uint& tree_slot_count,
@@ -45,7 +46,8 @@ PUBLIC KERNEL(WeaverFrontierInsertChildren)(
frontier[uint(FrontierIdx::Depth) * frontier_capacity + slot] =
packed_tree[uint(TreeIdx::Depth) * tree_slot_count + parent] + 1u;
frontier[uint(FrontierIdx::PathLogprobBits) * frontier_capacity + slot] = as_type<uint>(cumulative_logprob);
- frontier[uint(FrontierIdx::EdgeLogprobBits) * frontier_capacity + slot] = as_type<uint>(logprob);
+ frontier[uint(FrontierIdx::EdgeLogprobBits) * frontier_capacity + slot] =
+ as_type<uint>(child_prune_logprobs[row * expand_width + child]);
frontier[uint(FrontierIdx::PathScoreKey) * frontier_capacity + slot] = top_k_score_key(cumulative_logprob);
frontier[uint(FrontierIdx::Active) * frontier_capacity + slot] = 1u;
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_top_children.metal b/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_top_children.metal
index c4757d6ac..156462a5c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_top_children.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/weaver/weaver_top_children.metal
@@ -2,6 +2,7 @@
#include "../common/defines.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
#include "../common/top_k.h"
#include "../rng.h"
#include "weaver_frontier.h"
@@ -26,6 +27,15 @@ METAL_FUNC float gumbel_noise(uint64_t seed, uint logit_idx, uint vocab_size) {
return -log(-log(uniform));
}
+// The noise the target sampler adds for this token: the 24-bit uniform of `uniform_float` in rng.h keeps it finite.
+METAL_FUNC float target_gumbel_noise(uint64_t seed, uint logit_idx, uint vocab_size) {
+ const uint2 offset_word = gumbel_revidx(logit_idx, vocab_size);
+ PhiloxState rng;
+ philox_init(&rng, seed, offset_word.x);
+ const float uniform = float(max(rng.output[offset_word.y] >> 8, 1u)) * (1.0f / 16777216.0f);
+ return -log(-log(uniform));
+}
+
METAL_FUNC bool weaver_better(uint score, uint token, uint index, uint best_score, uint best_token, uint best_index) {
return score > best_score ||
(score == best_score && (token < best_token || (token == best_token && index < best_index)));
@@ -39,10 +49,13 @@ PUBLIC KERNEL(WeaverTopChildren)(
const device uint* node_metadata,
device uint* output_token_ids,
device float* output_model_logprobs,
+ device float* output_prune_logprobs OPTIONAL(has_prune_noise),
constant uint& rows,
constant uint& candidates,
constant uint& expand_width,
constant uint& vocab_size,
+ constant float& prune_noise_scale OPTIONAL(has_prune_noise),
+ const bool has_prune_noise SPECIALIZE,
threadgroup float reduce_float[TOP_CHILDREN_SIMDGROUPS],
threadgroup uint reduce_score[TOP_CHILDREN_SIMDGROUPS],
threadgroup uint reduce_token[TOP_CHILDREN_SIMDGROUPS],
@@ -112,6 +125,31 @@ PUBLIC KERNEL(WeaverTopChildren)(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
+ // Final pruning weighs each edge by the pool softmax of logits plus the target's noise scaled by 1 / sigma.
+ float first_prune_logit = -INFINITY;
+ float second_prune_logit = -INFINITY;
+ float prune_log_sum = 0.0f;
+ if (has_prune_noise) {
+ if (first_valid) {
+ first_prune_logit = first_logit + prune_noise_scale * target_gumbel_noise(seed, first_token, vocab_size);
+ }
+ if (second_valid) {
+ second_prune_logit = second_logit + prune_noise_scale * target_gumbel_noise(seed, second_token, vocab_size);
+ }
+ const float prune_max = threadgroup_cooperative_reduce<SimdReduceMax<float>, TOP_CHILDREN_THREADS>(
+ fmax(first_prune_logit, second_prune_logit),
+ reduce_float,
+ thread_context
+ );
+ const float prune_total = threadgroup_cooperative_reduce<SimdReduceSum<float>, TOP_CHILDREN_THREADS>(
+ (first_valid ? exp(first_prune_logit - prune_max) : 0.0f) +
+ (second_valid ? exp(second_prune_logit - prune_max) : 0.0f),
+ reduce_float,
+ thread_context
+ );
+ prune_log_sum = log(prune_total) + prune_max;
+ }
+
for (uint child = 0; child < expand_width; ++child) {
uint local_score = 0u;
uint local_token = 0xffffffffu;
@@ -162,6 +200,10 @@ PUBLIC KERNEL(WeaverTopChildren)(
output_token_ids[row * expand_width + child] = winner_token;
output_model_logprobs[row * expand_width + child] = winner_logit - log_sum;
}
+ if (has_prune_noise && (first_index == winner_index || second_index == winner_index)) {
+ output_prune_logprobs[row * expand_width + child] =
+ (first_index == winner_index ? first_prune_logit : second_prune_logit) - prune_log_sum;
+ }
first_active = first_active && first_index != winner_index;
second_active = second_active && second_index != winner_index;
}
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index 309fc4697..479c704d9 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -36,6 +36,7 @@ pub struct WeaverTreeShape {
pub rounds: u32,
pub expand_per_round: u32,
pub expand_width: u32,
+ pub prune_noise_scale: Option<f32>,
}
impl WeaverTreeShape {
@@ -121,6 +122,7 @@ pub struct Weaver<B: Backend> {
rope_config: AnyRoPEConfig,
top_k: <B::Kernels as Kernels>::RadixTopKSmall,
top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,
+ top_children_with_prune_noise: <B::Kernels as Kernels>::WeaverTopChildrenKernel,
frontier_select: <B::Kernels as Kernels>::WeaverFrontierSelectKernel,
frontier_insert_children: <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel,
model_dim: u32,
@@ -257,7 +259,9 @@ impl<B: Backend> Weaver<B> {
let top_k =
<B::Kernels as Kernels>::RadixTopKSmall::new(context, vocab_size).map_err(WeaverNewError::Backend)?;
let top_children =
- <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;
+ <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context, false).map_err(WeaverNewError::Backend)?;
+ let top_children_with_prune_noise =
+ <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context, true).map_err(WeaverNewError::Backend)?;
let frontier_select =
<B::Kernels as Kernels>::WeaverFrontierSelectKernel::new(context).map_err(WeaverNewError::Backend)?;
let frontier_insert_children = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(context)
@@ -273,6 +277,7 @@ impl<B: Backend> Weaver<B> {
rope_config: config.rope_config.clone(),
top_k,
top_children,
+ top_children_with_prune_noise,
frontier_select,
frontier_insert_children,
model_dim: config.model_dim,
@@ -488,7 +493,17 @@ impl<B: Backend> Weaver<B> {
let mut child_logprobs = encoder
.allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::F32))
.map_err(WeaverEncodeError::Backend)?;
- self.top_children.encode(
+ let mut child_prune_logprobs = shape
+ .prune_noise_scale
+ .map(|_| encoder.allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::F32)))
+ .transpose()
+ .map_err(WeaverEncodeError::Backend)?;
+ let top_children = if shape.prune_noise_scale.is_some() {
+ &self.top_children_with_prune_noise
+ } else {
+ &self.top_children
+ };
+ top_children.encode(
&logit_residuals,
batch_candidate_logits,
batch_candidate_ids,
@@ -496,19 +511,23 @@ impl<B: Backend> Weaver<B> {
&*node_metadata,
&mut child_token_ids,
&mut child_logprobs,
+ child_prune_logprobs.as_mut(),
batch_node_count,
self.candidate_pool_size,
shape.expand_width,
target_embedding.vocab_size(),
+ shape.prune_noise_scale,
encoder,
);
+ // The edge lane feeds only final pruning; without prune noise it keeps the model logprobs.
self.frontier_insert_children.encode(
&*packed_tree,
&*node_metadata,
&*node_valid,
&child_token_ids,
&child_logprobs,
+ child_prune_logprobs.as_ref().unwrap_or(&child_logprobs),
frontier,
frontier_capacity,
tree_slot_count,
@@ -547,6 +566,7 @@ impl<B: Backend> Weaver<B> {
|| shape.expand_width > self.candidate_pool_size
|| tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width
|| depth_seeds.len() as u32 != self.max_depth
+ || shape.prune_noise_scale.is_some_and(|scale| !(scale > 0.0 && scale.is_finite()))
{
return Err(WeaverEncodeError::InvalidTreeInput);
}
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 955342c29..18216ab7e 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -562,6 +562,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let (input_trie, chain_copy, full_accept) = if let Some(speculator) = &self.model.speculator
&& let Some(shape) = speculator.make_shape(
self.model_state.max_context_length.map(|max_context_length| max_context_length - context_length),
+ &self.options.sampling_method,
)
&& let (root_token, Some(output_norm)) = prev_output.resolve(
&mut self.model_state.tokens,
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index 2ce2a3fee..ecb16e409 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -65,9 +65,19 @@ pub enum DFlashTfmTreeConstructionMethod {
rounds: u32,
expand_per_round: u32,
expand_width: u32,
+ /// A shape without it prunes with `DEFAULT_PRUNE_SIGMA`; `null` prunes on the model logprobs.
+ #[serde(default = "default_prune_sigma")]
+ prune_sigma: Option<f32>,
},
}
+/// The final pruning noise scale calibrated for DFlash + Weaver trees.
+pub const DEFAULT_PRUNE_SIGMA: f32 = 1.5;
+
+fn default_prune_sigma() -> Option<f32> {
+ Some(DEFAULT_PRUNE_SIGMA)
+}
+
#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]
pub struct DFlashTfmTreeShape {
pub tree_budget: u32,
@@ -164,6 +174,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
pub fn make_shape(
&self,
max_depth: Option<u32>,
+ sampling_method: &SamplingMethod,
) -> Option<DFlashTfmTreeShape> {
if max_depth.is_some_and(|max_depth| max_depth < 2) {
return None;
@@ -183,6 +194,16 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
}
}
+ // Greedy verification adds no Gumbel noise to the target logits, so pruning has none to anticipate.
+ if matches!(sampling_method, SamplingMethod::Greedy)
+ && let DFlashTfmTreeConstructionMethod::Weaver {
+ prune_sigma,
+ ..
+ } = &mut shape.construction_method
+ {
+ *prune_sigma = None;
+ }
+
Some(shape)
}
@@ -279,6 +300,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
rounds,
expand_per_round,
expand_width,
+ prune_sigma,
} => {
let weaver =
self.weaver.as_ref().expect("weaver tree construction requires a speculator with weaver weights");
@@ -298,6 +320,13 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
dflash_depth
)));
}
+ if let Some(prune_sigma) = prune_sigma
+ && !(prune_sigma > 0.0 && prune_sigma.is_finite() && prune_sigma.recip().is_finite())
+ {
+ return Err(DFlashTreeError::InvalidTreeShape(format!(
+ "prune sigma {prune_sigma} is not positive and finite"
+ )));
+ }
let dflash_output = self.dflash.encode_draft(
state,
target_output_token,
@@ -322,6 +351,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
rounds,
expand_per_round,
expand_width,
+ prune_noise_scale: prune_sigma.map(f32::recip),
},
&mut encoder,
)?;
@@ -387,3 +417,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
Ok(trie)
}
}
+
+#[cfg(test)]
+#[path = "../../unit/speculators/dflash_tfm_test.rs"]
+mod tests;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_frontier_test.rs b/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
index 4136ebb2c..c5a2d67de 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
@@ -77,11 +77,15 @@ fn select<B: Backend>() -> Vec<u32> {
.collect()
}
+const TREE_PATH_LOGPROBS: [f32; 4] = [0.5, -1.0, 2.0, 4.0];
+const MODEL_LOGPROBS: [f32; 9] = [-0.1, -0.2, -0.3, 8.0, 8.0, 8.0, 0.1, 0.2, 0.3];
+const PRUNE_LOGPROBS: [f32; 9] = [-1.1, -1.2, -1.3, -9.0, -9.0, -9.0, -0.6, -0.7, -0.8];
+
fn insert_children<B: Backend>() -> Vec<u32> {
let context = create_context::<B>();
let mut tree = vec![0; TreeIdx::COUNT * 4];
tree[TreeIdx::PathLogprobBits as usize * 4..(TreeIdx::PathLogprobBits as usize + 1) * 4]
- .copy_from_slice(&[0.5, -1.0, 2.0, 4.0].map(f32::to_bits));
+ .copy_from_slice(&TREE_PATH_LOGPROBS.map(f32::to_bits));
tree[TreeIdx::Depth as usize * 4..(TreeIdx::Depth as usize + 1) * 4].copy_from_slice(&[0, 2, 4, 6]);
let tree = alloc_allocation_with_data::<B, u32>(&context, &tree);
let mut metadata = vec![0; MetadataIdx::COUNT * 3];
@@ -89,15 +93,40 @@ fn insert_children<B: Backend>() -> Vec<u32> {
let metadata = alloc_allocation_with_data::<B, u32>(&context, &metadata);
let valid = alloc_allocation_with_data::<B, u32>(&context, &[1, 0, 1]);
let ids = alloc_allocation_with_data::<B, u32>(&context, &(10..19).collect::<Vec<_>>());
- let scores = alloc_allocation_with_data::<B, f32>(&context, &[-0.1, -0.2, -0.3, 8.0, 8.0, 8.0, 0.1, 0.2, 0.3]);
+ let scores = alloc_allocation_with_data::<B, f32>(&context, &MODEL_LOGPROBS);
+ let prune_scores = alloc_allocation_with_data::<B, f32>(&context, &PRUNE_LOGPROBS);
let mut frontier = alloc_allocation_with_data::<B, u32>(&context, &[42; FrontierIdx::COUNT * 16]);
let kernel = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(&context).unwrap();
let mut encoder = Encoder::new(context.as_ref()).unwrap();
- kernel.encode(&tree, &metadata, &valid, &ids, &scores, &mut frontier, 16, 4, 3, 3, &mut encoder);
+ kernel.encode(&tree, &metadata, &valid, &ids, &scores, &prune_scores, &mut frontier, 16, 4, 3, 3, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
allocation_to_vec(&frontier)
}
+/// Expansion follows the model logprobs (path and its order key), final pruning reads the edge lane, which takes the
+/// prune channel. Rows 0 and 2 expand tree slots 1 and 0 into frontier slots 3..6 and 0..3; row 1 is invalid, so no
+/// other slot is written.
+#[uzu_test]
+fn weaver_frontier_insert_children_splits_path_and_edge() {
+ let frontier = insert_children::<Cpu>();
+ let lane = |field: FrontierIdx, slot: usize| frontier[field as usize * 16 + slot];
+ for (row, parent) in [(0, 1), (2, 0)] {
+ for child in 0..3 {
+ let slot = parent * 3 + child;
+ let index = row * 3 + child;
+ assert_eq!(lane(FrontierIdx::EdgeLogprobBits, slot), PRUNE_LOGPROBS[index].to_bits());
+ assert_eq!(
+ lane(FrontierIdx::PathLogprobBits, slot),
+ (TREE_PATH_LOGPROBS[parent] + MODEL_LOGPROBS[index]).to_bits()
+ );
+ assert_eq!(lane(FrontierIdx::Active, slot), 1);
+ }
+ }
+ for slot in 6..16 {
+ assert!((0..FrontierIdx::COUNT).all(|field| frontier[field * 16 + slot] == 42));
+ }
+}
+
#[uzu_test]
fn weaver_frontier_kernels_match_cpu() {
for_each_non_cpu_backend!(|B| {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_top_children_test.rs b/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
index 4ce4a33cd..2e513eeb6 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
@@ -6,6 +6,7 @@ use crate::{
common::{Backend, Encoder, Kernels, gpu_types::weaver::MetadataIdx, kernel::WeaverTopChildrenKernel},
cpu::Cpu,
},
+ encodable_block::sampling::{gumbel_float, revidx},
tests::helpers::{
alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context, for_each_non_cpu_backend,
},
@@ -16,12 +17,27 @@ const CHILDREN: usize = 8;
const VOCAB_SIZE: u32 = 131_072;
const DEPTHS: [u32; 3] = [0, 1, 2];
const DEPTH_SEEDS: [u64; 3] = [0x9E3779B97F4A7C15, 0xD1B54A32D192ED03, 0x2545F4914F6CDD1D];
+const ROWS: usize = 3;
+const PRUNE_NOISE_SCALE: f32 = 1.0 / 1.5;
+
+fn inputs() -> (Vec<bf16>, Vec<f32>, Vec<u32>) {
+ let residual = (0..ROWS * CANDIDATES)
+ .map(|index| bf16::from_f32(((index as f32 * 0.017).cos() * 4.0).round() * 0.125))
+ .collect::<Vec<_>>();
+ let candidate_logits =
+ (0..ROWS * CANDIDATES).map(|index| ((index as f32 * 0.011).sin() * 3.0).round() * 0.125).collect::<Vec<_>>();
+ let ids = (0..ROWS)
+ .flat_map(|row| (0..CANDIDATES).rev().map(move |index| 70_000 + (row * CANDIDATES + index) as u32))
+ .collect::<Vec<_>>();
+ (residual, candidate_logits, ids)
+}
fn top_children<B: Backend>(
residual: &[bf16],
candidate_logits: &[f32],
ids: &[u32],
-) -> (Vec<u32>, Vec<f32>) {
+ prune_noise_scale: Option<f32>,
+) -> (Vec<u32>, Vec<f32>, Option<Vec<f32>>) {
let rows = residual.len() / CANDIDATES;
assert_eq!(residual.len(), rows * CANDIDATES);
assert_eq!(candidate_logits.len(), rows * CANDIDATES);
@@ -39,7 +55,8 @@ fn top_children<B: Backend>(
let node_metadata = alloc_allocation_with_data::<B, u32>(&context, &metadata_values);
let mut output_token_ids = alloc_allocation::<B, u32>(&context, rows * CHILDREN);
let mut output_model_logprobs = alloc_allocation::<B, f32>(&context, rows * CHILDREN);
- let kernel = <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(&context).unwrap();
+ let mut output_prune_logprobs = prune_noise_scale.map(|_| alloc_allocation::<B, f32>(&context, rows * CHILDREN));
+ let kernel = <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(&context, prune_noise_scale.is_some()).unwrap();
let mut encoder = Encoder::new(context.as_ref()).unwrap();
kernel.encode(
&residual,
@@ -49,34 +66,81 @@ fn top_children<B: Backend>(
&node_metadata,
&mut output_token_ids,
&mut output_model_logprobs,
+ output_prune_logprobs.as_mut(),
rows as u32,
CANDIDATES as u32,
CHILDREN as u32,
VOCAB_SIZE,
+ prune_noise_scale,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (allocation_to_vec(&output_token_ids), allocation_to_vec(&output_model_logprobs))
+ (
+ allocation_to_vec(&output_token_ids),
+ allocation_to_vec(&output_model_logprobs),
+ output_prune_logprobs.as_ref().map(allocation_to_vec),
+ )
}
#[uzu_test]
fn weaver_top_children_matches_cpu() {
- const ROWS: usize = 3;
- let residual = (0..ROWS * CANDIDATES)
- .map(|index| bf16::from_f32(((index as f32 * 0.017).cos() * 4.0).round() * 0.125))
- .collect::<Vec<_>>();
- let candidate_logits =
- (0..ROWS * CANDIDATES).map(|index| ((index as f32 * 0.011).sin() * 3.0).round() * 0.125).collect::<Vec<_>>();
- let ids = (0..ROWS)
- .flat_map(|row| (0..CANDIDATES).rev().map(move |index| 70_000 + (row * CANDIDATES + index) as u32))
- .collect::<Vec<_>>();
+ let (residual, candidate_logits, ids) = inputs();
+
+ for prune_noise_scale in [None, Some(PRUNE_NOISE_SCALE)] {
+ for_each_non_cpu_backend!(|B| {
+ let expected = top_children::<Cpu>(&residual, &candidate_logits, &ids, prune_noise_scale);
+ let actual = top_children::<B>(&residual, &candidate_logits, &ids, prune_noise_scale);
+ assert_eq!(actual.0, expected.0);
+ for (actual, expected) in actual.1.iter().zip(expected.1) {
+ assert!((actual - expected).abs() < 1e-5);
+ }
+ assert_eq!(actual.2.is_some(), expected.2.is_some());
+ // The prune channel also carries the Gumbel transform, which Metal evaluates with fast-math logs.
+ for (actual, expected) in actual.2.iter().flatten().zip(expected.2.iter().flatten()) {
+ assert!((actual - expected).abs() < 1e-3);
+ }
+ });
+ }
+}
+
+/// The prune channel is the log-softmax over the whole pool of logits plus the target sampler's noise at the
+/// parent depth's seed, scaled by `prune_noise_scale`; selection and model logprobs stay as without it. The reference
+/// takes that noise from `gumbel_float` on purpose, since matching the target sampler is the contract; the seed, the
+/// token indexing and the log-sum-exp are computed here independently, in f64.
+#[uzu_test]
+fn weaver_top_children_prune_logprobs_match_reference() {
+ let (residual, candidate_logits, ids) = inputs();
+ let (tokens, model, none) = top_children::<Cpu>(&residual, &candidate_logits, &ids, None);
+ let (prune_tokens, prune_model, prune) =
+ top_children::<Cpu>(&residual, &candidate_logits, &ids, Some(PRUNE_NOISE_SCALE));
+ assert!(none.is_none());
+ let prune = prune.unwrap();
+ assert_eq!(prune_tokens, tokens);
+ assert_eq!(
+ prune_model.iter().map(|value| value.to_bits()).collect::<Vec<_>>(),
+ model.iter().map(|value| value.to_bits()).collect::<Vec<_>>()
+ );
- for_each_non_cpu_backend!(|B| {
- let expected = top_children::<Cpu>(&residual, &candidate_logits, &ids);
- let actual = top_children::<B>(&residual, &candidate_logits, &ids);
- assert_eq!(actual.0, expected.0);
- for (actual, expected) in actual.1.iter().zip(expected.1) {
- assert!((actual - expected).abs() < 1e-5);
+ for row in 0..ROWS {
+ let seed = DEPTH_SEEDS[DEPTHS[row] as usize];
+ let values = (0..CANDIDATES)
+ .map(|index| {
+ let flat = row * CANDIDATES + index;
+ candidate_logits[flat] as f64
+ + residual[flat].to_f64()
+ + PRUNE_NOISE_SCALE as f64 * gumbel_float(seed, revidx(ids[flat], VOCAB_SIZE)) as f64
+ })
+ .collect::<Vec<_>>();
+ let max = values.iter().copied().fold(f64::NEG_INFINITY, f64::max);
+ let log_sum = values.iter().map(|value| (value - max).exp()).sum::<f64>().ln() + max;
+ for child in 0..CHILDREN {
+ let token = tokens[row * CHILDREN + child];
+ let index = (0..CANDIDATES).find(|&index| ids[row * CANDIDATES + index] == token).unwrap();
+ let actual = prune[row * CHILDREN + child];
+ // An f32 log-sum-exp over 512 values of magnitude below 20 keeps about 1e-4 of absolute accuracy.
+ assert!((actual as f64 - (values[index] - log_sum)).abs() < 1e-4);
+ assert!(actual <= 0.0);
}
- });
+ }
+ assert!(prune.iter().zip(&model).any(|(prune, model)| (prune - model).abs() > 0.1));
}
diff --git a/crates/uzu-engine/unit/speculators/dflash_tfm_test.rs b/crates/uzu-engine/unit/speculators/dflash_tfm_test.rs
new file mode 100644
index 000000000..835e130e3
--- /dev/null
+++ b/crates/uzu-engine/unit/speculators/dflash_tfm_test.rs
@@ -0,0 +1,54 @@
+use uzu_engine_macros::uzu_test;
+
+use super::{DFlashTfmTreeConstructionMethod, DFlashTfmTreeShape};
+
+fn weaver_shape(construction_method: &str) -> DFlashTfmTreeShape {
+ serde_json::from_str(&format!(
+ r#"{{"tree_budget":16,"max_tree_depth":16,"dflash_depth_override":null,"construction_method":{construction_method}}}"#
+ ))
+ .unwrap()
+}
+
+/// A shape without `prune_sigma` prunes with noise scale 1.5, `null` turns the noise off, and a number is the scale
+/// itself, 0 included: parsing does not reinterpret it, the shape check rejects it.
+#[uzu_test]
+fn weaver_shape_prune_sigma_is_optional() {
+ let weaver = |prune_sigma| DFlashTfmTreeConstructionMethod::Weaver {
+ rounds: 16,
+ expand_per_round: 4,
+ expand_width: 4,
+ prune_sigma,
+ };
+ let method = |prune_sigma: &str| {
+ format!(r#"{{"type":"Weaver","rounds":16,"expand_per_round":4,"expand_width":4{prune_sigma}}}"#)
+ };
+ for (field, expected) in [
+ ("", Some(1.5)),
+ (r#","prune_sigma":null"#, None),
+ (r#","prune_sigma":1.5"#, Some(1.5)),
+ (r#","prune_sigma":2"#, Some(2.0)),
+ (r#","prune_sigma":0"#, Some(0.0)),
+ ] {
+ assert_eq!(weaver_shape(&method(field)).construction_method, weaver(expected), "{field:?}");
+ }
+}
+
+/// A shape written back to JSON reads as the same shape, with the noise on or off.
+#[uzu_test]
+fn weaver_shape_prune_sigma_round_trips() {
+ for prune_sigma in [Some(1.5), Some(2.0), None] {
+ let shape = DFlashTfmTreeShape {
+ tree_budget: 16,
+ max_tree_depth: 16,
+ dflash_depth_override: None,
+ construction_method: DFlashTfmTreeConstructionMethod::Weaver {
+ rounds: 16,
+ expand_per_round: 4,
+ expand_width: 4,
+ prune_sigma,
+ },
+ };
+ let json = serde_json::to_string(&shape).unwrap();
+ assert_eq!(serde_json::from_str::<DFlashTfmTreeShape>(&json).unwrap(), shape, "{json}");
+ }
+}