A high-performance inference engine for AI models
RAII debug groups (idea stolen from #906) (#935)
uuuvn6d021ae30e09
diff --git a/crates/uzu-engine/build/metal/bindgen/mod.rs b/crates/uzu-engine/build/metal/bindgen/mod.rsindex df497d193..588a61f5d 100644--- a/crates/uzu-engine/build/metal/bindgen/mod.rs+++ b/crates/uzu-engine/build/metal/bindgen/mod.rs@@ -122,13 +122,12 @@ pub fn bindgen(#(#encode_argument_definitions),*) {#empty_dispatch_guards- command_buffer.push_debug_group(#kernel_name);+ let command_buffer = &mut *command_buffer.debug_group(#kernel_name);#(#encode_deconstructs)*#encode_accesses_callcommand_buffer.compute_encoder.set_compute_pipeline_state(&self.pipeline);#(#encode_set_calls)*#dispatch_code- command_buffer.pop_debug_group();}}};diff --git a/crates/uzu-engine/src/backends/common/backend.rs b/crates/uzu-engine/src/backends/common/backend.rsindex 6acdcbc50..3aa01329a 100644--- a/crates/uzu-engine/src/backends/common/backend.rs+++ b/crates/uzu-engine/src/backends/common/backend.rs@@ -1,10 +1,11 @@-use std::error::Error;+use std::{error::Error, ops::DerefMut};use super::{CommandBuffer, ConstantBuffer, Context, GlobalBuffer, Kernels, ScratchBuffer, SparseBuffer};pub trait Backend: Clone + Send + Sync + 'static {type Context: Context<Backend = Self>;type CommandBuffer: CommandBuffer<Backend = Self>;+ type DebugGroupSpan<'a>: DerefMut<Target = <Self::CommandBuffer as CommandBuffer>::Encoding> + 'a;type GlobalBuffer: GlobalBuffer<Backend = Self>;type ConstantBuffer: ConstantBuffer<Backend = Self>;type ScratchBuffer: ScratchBuffer<Backend = Self>;diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rsindex 0ed3d0034..c98712b8e 100644--- a/crates/uzu-engine/src/backends/common/command_buffer.rs+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs@@ -73,14 +73,13 @@ pub trait CommandBufferEncoding {value: u8,);- fn push_debug_group(+ #[must_use]+ fn debug_group(&mut self,name: &str,- );-- fn pop_debug_group(&mut self);+ ) -> <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DebugGroupSpan<'_>;- fn add_debug_point(+ fn debug_point(&mut self,name: &str,);diff --git a/crates/uzu-engine/src/backends/cpu/backend.rs b/crates/uzu-engine/src/backends/cpu/backend.rsindex 335858916..0555fcdaf 100644--- a/crates/uzu-engine/src/backends/cpu/backend.rs+++ b/crates/uzu-engine/src/backends/cpu/backend.rs@@ -10,7 +10,7 @@ use crate::backends::{},cpu::{buffer::{dense::CpuBuffer, sparse::CpuSparseBuffer},- command_buffer::CpuCommandBuffer,+ command_buffer::{CpuCommandBuffer, CpuCommandBufferEncoding},context::CpuContext,error::CpuError,kernel::CpuKernels,@@ -23,6 +23,7 @@ pub struct Cpu;impl Backend for Cpu {type Context = CpuContext;type CommandBuffer = CpuCommandBuffer;+ type DebugGroupSpan<'a> = &'a mut CpuCommandBufferEncoding;type GlobalBuffer = CpuBuffer;type ConstantBuffer = BumpAllocation<Self::GlobalBuffer>;type ScratchBuffer = PoolAllocation<Self::GlobalBuffer>;diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rsindex 04ca0e2bb..4441b9977 100644--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs@@ -110,15 +110,14 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {});}- fn push_debug_group(+ fn debug_group(&mut self,_name: &str,- ) {+ ) -> <Cpu as Backend>::DebugGroupSpan<'_> {+ self}- fn pop_debug_group(&mut self) {}-- fn add_debug_point(+ fn debug_point(&mut self,_name: &str,) {diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rsindex 21b07f2d1..0c0f28688 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs@@ -64,28 +64,29 @@ impl AttentionKernel for AttentionCpuKernel {arguments.trie.is_some(),config.sliding_window_size.is_some(),)?;- command_buffer.push_debug_group("attention core");- single_pass.encode(- arguments.queries,- arguments.keys,- arguments.values,- &mut output,- config.num_q_heads / config.num_groups,- arguments.cache.prefix_len() + arguments.suffix_length,- config.head_dim,- config.num_groups * config.head_dim,- config.head_dim,- config.num_groups * config.head_dim,- arguments.cache.ring_params(),- config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),- arguments.trie,- config.sliding_window_size,- arguments.sinks,- config.num_q_heads,- arguments.suffix_length,- command_buffer,- );- command_buffer.pop_debug_group();+ {+ let command_buffer = &mut *command_buffer.debug_group("attention core");+ single_pass.encode(+ arguments.queries,+ arguments.keys,+ arguments.values,+ &mut output,+ config.num_q_heads / config.num_groups,+ arguments.cache.prefix_len() + arguments.suffix_length,+ config.head_dim,+ config.num_groups * config.head_dim,+ config.head_dim,+ config.num_groups * config.head_dim,+ arguments.cache.ring_params(),+ config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),+ arguments.trie,+ config.sliding_window_size,+ arguments.sinks,+ config.num_q_heads,+ arguments.suffix_length,+ command_buffer,+ );+ }if let (Some(gate), Some(gate_row_stride)) = (arguments.gate, config.gate_row_stride) {<sigmoid_gate::SigmoidGateCpuKernel as SigmoidGateKernel>::new(command_buffer.context(), config.data_type)?.encode(diff --git a/crates/uzu-engine/src/backends/metal/backend.rs b/crates/uzu-engine/src/backends/metal/backend.rsindex 3986a0f47..bd8a2d71f 100644--- a/crates/uzu-engine/src/backends/metal/backend.rs+++ b/crates/uzu-engine/src/backends/metal/backend.rs@@ -10,7 +10,7 @@ use crate::backends::{metal::{buffer::{dense::MetalDenseBuffer, sparse::MetalSparseBuffer},capture::MetalCaptureSpan,- command_buffer::MetalCommandBuffer,+ command_buffer::{MetalCommandBuffer, MetalDebugGroupSpan},context::MetalContext,error::MetalError,kernel::MetalKernels,@@ -23,6 +23,7 @@ pub struct Metal;impl Backend for Metal {type Context = MetalContext;type CommandBuffer = MetalCommandBuffer;+ type DebugGroupSpan<'a> = MetalDebugGroupSpan<'a>;type GlobalBuffer = BlockAllocation<MetalDenseBuffer>;type ConstantBuffer = BumpAllocation<Self::GlobalBuffer>;type ScratchBuffer = PoolAllocation<Self::GlobalBuffer>;diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rsindex 02565f5e8..4c8af0b49 100644--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs@@ -1,4 +1,5 @@use std::{+ ops::{Deref, DerefMut},range::Range,sync::{Arc, mpsc},time::Duration,@@ -195,19 +196,17 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {kernel.encode(dst, (range.end - range.start) as u64, u32::from(value) * 0x0101_0101, self);}- // TODO: maybe port previous debug command_buffer labels- fn push_debug_group(+ fn debug_group(&mut self,name: &str,- ) {+ ) -> MetalDebugGroupSpan<'_> {ProtocolObject::<dyn MTL4CommandEncoder>::push_debug_group(self.compute_encoder.as_ref(), name);+ MetalDebugGroupSpan {+ command_buffer: self,+ }}- fn pop_debug_group(&mut self) {- self.compute_encoder.pop_debug_group();- }-- fn add_debug_point(+ fn debug_point(&mut self,name: &str,) {@@ -240,6 +239,30 @@ impl Drop for MetalCommandBufferEncoding {}}+pub struct MetalDebugGroupSpan<'a> {+ command_buffer: &'a mut MetalCommandBufferEncoding,+}++impl Deref for MetalDebugGroupSpan<'_> {+ type Target = MetalCommandBufferEncoding;++ fn deref(&self) -> &Self::Target {+ self.command_buffer+ }+}++impl DerefMut for MetalDebugGroupSpan<'_> {+ fn deref_mut(&mut self) -> &mut Self::Target {+ self.command_buffer+ }+}++impl Drop for MetalDebugGroupSpan<'_> {+ fn drop(&mut self) {+ self.command_buffer.compute_encoder.pop_debug_group();+ }+}+pub struct MetalCommandBufferExecutable {command_allocator: Retained<ProtocolObject<dyn MTL4CommandAllocator>>,command_buffer: Retained<ProtocolObject<dyn MTL4CommandBuffer>>,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rsindex d16465c6f..4452fe8a1 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs@@ -101,9 +101,9 @@ impl AttentionKernel for AttentionMetalKernel {>,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {- command_buffer.push_debug_group("attention core");+ let command_buffer = &mut *command_buffer.debug_group("attention core");let (gate, suffix_length) = (arguments.gate, arguments.suffix_length);- let result = self.encode_impl(arguments, command_buffer).map(|(mut output, gate_applied)| {+ self.encode_impl(arguments, command_buffer).map(|(mut output, gate_applied)| {if let (Some(gate), Some((kernel, stride)), false) = (gate, &self.gate, gate_applied) {kernel.encode(gate,@@ -115,9 +115,7 @@ impl AttentionKernel for AttentionMetalKernel {);}output- });- command_buffer.pop_debug_group();- result+ })}}diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rsindex f44077a95..fc24277d2 100644--- a/crates/uzu-engine/src/encodable_block/classifier.rs+++ b/crates/uzu-engine/src/encodable_block/classifier.rs@@ -118,7 +118,7 @@ impl<B: Backend> Classifier<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, ClassifierError<B>> {- command_buffer.push_debug_group("classifier");+ let command_buffer = &mut *command_buffer.debug_group("classifier");let embedded = self.embedding.encode_lookup(token_ids, batch_dim, command_buffer)?;@@ -156,8 +156,6 @@ impl<B: Backend> Classifier<B> {let logits = self.prediction_head.encode(pooled, 1, command_buffer).map_err(ClassifierError::Backend)?;- command_buffer.pop_debug_group();-Ok(logits)}}diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rsindex 7b1c8bbfd..5c7dfcfc2 100644--- a/crates/uzu-engine/src/encodable_block/decoder.rs+++ b/crates/uzu-engine/src/encodable_block/decoder.rs@@ -143,7 +143,7 @@ impl<B: Backend> Decoder<B> {state: &mut TransformerState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {- command_buffer.push_debug_group("decoder");+ let command_buffer = &mut *command_buffer.debug_group("decoder");let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), command_buffer)?;let embedded = if let Some(embedding_norm) = &self.embedding_norm {@@ -196,8 +196,6 @@ impl<B: Backend> Decoder<B> {transformer_output.output};- command_buffer.pop_debug_group();-Ok(DecoderEncodeOutput {logits,hidden_features: transformer_output.hidden_features,diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rsindex b35383899..67d38fd36 100644--- a/crates/uzu-engine/src/encodable_block/dflash.rs+++ b/crates/uzu-engine/src/encodable_block/dflash.rs@@ -207,7 +207,7 @@ impl<B: Backend> DFlash<B> {return Ok(());}- command_buffer.push_debug_group("dflash accept");+ let command_buffer = &mut *command_buffer.debug_group("dflash accept");let num_tokens = accepted_indices.len() as u32;let captured_layer_count = self.target_feature_input_dim / self.model_dim;@@ -274,8 +274,6 @@ impl<B: Backend> DFlash<B> {state.context_length += num_tokens;- command_buffer.pop_debug_group();-Ok(())}@@ -287,7 +285,7 @@ impl<B: Backend> DFlash<B> {batch_size: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {- command_buffer.push_debug_group("dflash draft");+ let command_buffer = &mut *command_buffer.debug_group("dflash draft");assert!(batch_size >= 2 && batch_size <= self.block_size, "batch size exceeds DFlash block size");assert!(@@ -348,8 +346,6 @@ impl<B: Backend> DFlash<B> {command_buffer,)?;- command_buffer.pop_debug_group();-Ok(DFlashOutput {draft_hidden,logits,diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex 0cd58c0f7..d64fa10a2 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -185,7 +185,7 @@ impl<B: Backend> Embedding<B> {apply_logit_transform: bool,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, EmbeddingError<B>> {- command_buffer.push_debug_group("embedding readout");+ let command_buffer = &mut *command_buffer.debug_group("embedding readout");assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");let mut output_buffer = match &self.tying {@@ -236,8 +236,6 @@ impl<B: Backend> Embedding<B> {);}- command_buffer.pop_debug_group();-Ok(output_buffer)}}diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rsindex d4faa24b9..a98d5599f 100644--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs@@ -175,7 +175,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("linear (qlora)");+ let command_buffer = &mut *command_buffer.debug_group("linear (qlora)");let mut intermediate =command_buffer.allocate_scratch_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type)?;@@ -261,8 +261,6 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {);}- command_buffer.pop_debug_group();-Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex d38dc43b8..4135fe01a 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -193,7 +193,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {LinearInput::FullPrecision(input) => input,input => return self.inner_linear.encode_input(input, batch_dim, command_buffer),};- command_buffer.push_debug_group("linear (rht)");+ let command_buffer = &mut *command_buffer.debug_group("linear (rht)");let format = self.inner_linear.select_activation_format(batch_dim, command_buffer.context());let input = self.input_rht.prepare_in_place(input, batch_dim, format, command_buffer)?;@@ -204,7 +204,6 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {command_buffer,)?;- command_buffer.pop_debug_group();Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rsindex d9b72f349..f3a63c1c6 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs@@ -216,7 +216,7 @@ impl<B: Backend> Mixer<B> for Attention<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("attention");+ let command_buffer = &mut *command_buffer.debug_group("attention");assert_eq!(precalculated_rope.is_some(), self.max_rope_length.is_some(), "precalculated rope mismatch");@@ -224,8 +224,6 @@ impl<B: Backend> Mixer<B> for Attention<B> {state.map(|state| state.downcast::<AttentionState<B>>().expect("incorrect type of attention state"));let output = self.attend(hidden, precalculated_rope, batch_dim, state, command_buffer)?;- command_buffer.pop_debug_group();-Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rsindex 33e793265..8ac89be55 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs@@ -470,7 +470,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("delta net");+ let command_buffer = &mut *command_buffer.debug_group("delta net");assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");@@ -487,8 +487,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {if !batch_dim.full_accept() {let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;- command_buffer.pop_debug_group();-return Ok(output);}@@ -631,8 +629,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {let output = self.out_projection.encode(delta_output, batch_dim.size(), command_buffer)?;- command_buffer.pop_debug_group();-Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rsindex 878668519..753113be2 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs@@ -243,7 +243,7 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("mamba2");+ let command_buffer = &mut *command_buffer.debug_group("mamba2");assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");@@ -421,8 +421,6 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {let output = self.out_projection.encode(ssd_output, batch_dim.size(), command_buffer)?;- command_buffer.pop_debug_group();-Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rsindex 93f9cbbb2..040b8cc4a 100644--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs@@ -308,7 +308,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("short conv");+ let command_buffer = &mut *command_buffer.debug_group("short conv");assert!(precalculated_rope.is_none(), "unexpected rope for short conv mixer");@@ -344,8 +344,6 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {let output = self.out_projection.encode(conv_output, batch_dim.size(), command_buffer)?;- command_buffer.pop_debug_group();-Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rsindex a0881dee7..a6a021470 100644--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs@@ -35,15 +35,13 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("mlp (dense)");+ let command_buffer = &mut *command_buffer.debug_group("mlp (dense)");let fused_up = self.up.encode(input, batch_dim, command_buffer)?;let act_format = self.down.select_activation_format(batch_dim, command_buffer.context());let down_input = self.gate.encode_for_linear(command_buffer, &fused_up, batch_dim, act_format)?;let output = self.down.encode_input(down_input, batch_dim, command_buffer)?;- command_buffer.pop_debug_group();-Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rsindex a2920a9ea..6f5ab40d1 100644--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs@@ -110,7 +110,7 @@ impl<B: Backend> PerLayerEmbedding<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("per layer embedding");+ let command_buffer = &mut *command_buffer.debug_group("per layer embedding");let total_ple_dim = self.num_layers * self.ple_dim;let total_rows = batch_dim * self.num_layers;@@ -150,8 +150,6 @@ impl<B: Backend> PerLayerEmbedding<B> {command_buffer,);- command_buffer.pop_debug_group();-Ok(per_layer_inputs)}}@@ -245,7 +243,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.push_debug_group("per layer embedding projection");+ let command_buffer = &mut *command_buffer.debug_group("per layer embedding projection");let length = batch_dim * self.model_dim;@@ -289,8 +287,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {command_buffer,);- command_buffer.pop_debug_group();-Ok(())}}diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rsindex 351661ee1..0ebb99779 100644--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs@@ -93,7 +93,7 @@ impl<B: Backend> PredictionHead<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("prediction head");+ let command_buffer = &mut *command_buffer.debug_group("prediction head");let mut hidden = self.dense_projection.encode(input, batch_dim, command_buffer)?;self.activation_kernel.encode(@@ -107,8 +107,6 @@ impl<B: Backend> PredictionHead<B> {self.normalization.encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;let logits = self.readout.encode(normalized, batch_dim, command_buffer)?;- command_buffer.pop_debug_group();-Ok(logits)}}diff --git a/crates/uzu-engine/src/encodable_block/sampling/mod.rs b/crates/uzu-engine/src/encodable_block/sampling/mod.rsindex 1142bd587..1d2a1e29a 100644--- a/crates/uzu-engine/src/encodable_block/sampling/mod.rs+++ b/crates/uzu-engine/src/encodable_block/sampling/mod.rs@@ -92,7 +92,7 @@ impl<B: Backend> Sampling<B> {sampling_range: Range<u32>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::GlobalBuffer, B::Error> {- command_buffer.push_debug_group("sampling");+ let command_buffer = &mut *command_buffer.debug_group("sampling");let sampling_length = sampling_range.end - sampling_range.start;@@ -194,8 +194,6 @@ impl<B: Backend> Sampling<B> {command_buffer,);- command_buffer.pop_debug_group();-Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rsindex 2bb7b8bea..8cdcdb62b 100644--- a/crates/uzu-engine/src/encodable_block/transformer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer.rs@@ -60,21 +60,21 @@ impl<B: Backend> TransformerState<B> {accepted_indices: &[u32],command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.push_debug_group("transformer accept");+ {+ let command_buffer = &mut *command_buffer.debug_group("transformer accept");- for layer_state in &mut self.layer_states {- let TransformerLayerStateType::Owned(layer_state) = layer_state else {- continue;- };+ for layer_state in &mut self.layer_states {+ let TransformerLayerStateType::Owned(layer_state) = layer_state else {+ continue;+ };- layer_state.encode_accept(accepted_indices, command_buffer)?;- }-- self.context_length += accepted_indices.len() as u32;+ layer_state.encode_accept(accepted_indices, command_buffer)?;+ }- command_buffer.pop_debug_group();+ self.context_length += accepted_indices.len() as u32;+ }- command_buffer.add_debug_point(&format!("accepted {} tokens", accepted_indices.len()));+ command_buffer.debug_point(&format!("accepted {} tokens", accepted_indices.len()));Ok(())}diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rsindex 567a92c2a..654d9be9b 100644--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs@@ -350,7 +350,7 @@ impl<B: Backend> TransformerLayer<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&format!("transformer layer {}", self.layer_index));+ let command_buffer = &mut *command_buffer.debug_group(&format!("transformer layer {}", self.layer_index));let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?@@ -418,8 +418,6 @@ impl<B: Backend> TransformerLayer<B> {command_buffer.encode_fill(&mut hidden, 0);}- command_buffer.pop_debug_group();-Ok(hidden)}}diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex ac02f6e78..9df512a47 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -295,7 +295,7 @@ impl<B: Backend> Weaver<B> {depth: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<Vec<B::ScratchBuffer>, WeaverEncodeError<B>> {- command_buffer.push_debug_group("weaver prefix");+ let command_buffer = &mut *command_buffer.debug_group("weaver prefix");let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);let mut prefix_hidden = command_buffer@@ -356,8 +356,6 @@ impl<B: Backend> Weaver<B> {.kv_cache,);- command_buffer.pop_debug_group();-Ok(prefix_kv_layers)}@@ -525,158 +523,161 @@ impl<B: Backend> Weaver<B> {shape: WeaverTreeShape,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {- command_buffer.push_debug_group("weaver tree");-- let tree_slot_count = shape.slot_count();- let ancestor_stride = self.max_depth;- if shape.tree_budget == 0- || shape.rounds == 0- || shape.max_depth < 2- || shape.max_depth > self.max_depth + 1- || shape.dflash_depth < shape.max_depth- || shape.dflash_depth > self.max_depth + 1- || shape.expand_per_round == 0- || shape.expand_per_round > FRONTIER_MAX_WIDTH- || shape.expand_width == 0- || shape.expand_width > self.candidate_pool_size- || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width- || depth_seeds.len() as u32 != self.max_depth- || shape.prune_noise_scale.is_some_and(|scale| !(scale > 0.0 && scale.is_finite()))- {- return Err(WeaverEncodeError::InvalidTreeInput);- }- let frontier_capacity = tree_slot_count * shape.expand_width;- let pool_depth_count = shape.dflash_depth - 1;-- // Rank the draft logits: the top `candidate_pool_size` tokens per- // lookahead row form the candidate pool node expansions draw from.- let vocab_size = target_embedding.vocab_size();- assert!(- logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DATA_TYPE),- "draft logits do not cover the lookahead rows"- );- let mut candidate_ids = command_buffer- .allocate_scratch_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::U32)- .map_err(WeaverEncodeError::Backend)?;- let mut candidate_logits = command_buffer- .allocate_scratch_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::F32)- .map_err(WeaverEncodeError::Backend)?;- self.top_k- .encode(- logits,- &mut candidate_ids,- &mut candidate_logits,- pool_depth_count,- self.candidate_pool_size,- command_buffer,- )- .map_err(WeaverEncodeError::Backend)?;+ let (packed_tree, frontier) = {+ let command_buffer = &mut *command_buffer.debug_group("weaver tree");++ let tree_slot_count = shape.slot_count();+ let ancestor_stride = self.max_depth;+ if shape.tree_budget == 0+ || shape.rounds == 0+ || shape.max_depth < 2+ || shape.max_depth > self.max_depth + 1+ || shape.dflash_depth < shape.max_depth+ || shape.dflash_depth > self.max_depth + 1+ || shape.expand_per_round == 0+ || shape.expand_per_round > FRONTIER_MAX_WIDTH+ || shape.expand_width == 0+ || shape.expand_width > self.candidate_pool_size+ || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width+ || depth_seeds.len() as u32 != self.max_depth+ || shape.prune_noise_scale.is_some_and(|scale| !(scale > 0.0 && scale.is_finite()))+ {+ return Err(WeaverEncodeError::InvalidTreeInput);+ }+ let frontier_capacity = tree_slot_count * shape.expand_width;+ let pool_depth_count = shape.dflash_depth - 1;++ // Rank the draft logits: the top `candidate_pool_size` tokens per+ // lookahead row form the candidate pool node expansions draw from.+ let vocab_size = target_embedding.vocab_size();+ assert!(+ logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DATA_TYPE),+ "draft logits do not cover the lookahead rows"+ );+ let mut candidate_ids = command_buffer+ .allocate_scratch_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::U32)+ .map_err(WeaverEncodeError::Backend)?;+ let mut candidate_logits = command_buffer+ .allocate_scratch_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::F32)+ .map_err(WeaverEncodeError::Backend)?;+ self.top_k+ .encode(+ logits,+ &mut candidate_ids,+ &mut candidate_logits,+ pool_depth_count,+ self.candidate_pool_size,+ command_buffer,+ )+ .map_err(WeaverEncodeError::Backend)?;- let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();- let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, command_buffer)- .map_err(WeaverEncodeError::Backend)?;+ let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, command_buffer)+ .map_err(WeaverEncodeError::Backend)?;- let prefix_kv_layers =- self.encode_prefix(target_hidden, draft_hidden, &rope, shape.dflash_depth, command_buffer)?;+ let prefix_kv_layers =+ self.encode_prefix(target_hidden, draft_hidden, &rope, shape.dflash_depth, command_buffer)?;- // Per-layer KV cache for tree nodes, one slot per packed-tree slot.- let node_kv_size = size_for_shape(&[2, tree_slot_count, self.model_dim], DATA_TYPE);- let mut node_kv_layers = (0..self.layers.len())- .map(|_| command_buffer.allocate_scratch(node_kv_size))- .collect::<Result<Vec<_>, _>>()- .map_err(WeaverEncodeError::Backend)?;+ // Per-layer KV cache for tree nodes, one slot per packed-tree slot.+ let node_kv_size = size_for_shape(&[2, tree_slot_count, self.model_dim], DATA_TYPE);+ let mut node_kv_layers = (0..self.layers.len())+ .map(|_| command_buffer.allocate_scratch(node_kv_size))+ .collect::<Result<Vec<_>, _>>()+ .map_err(WeaverEncodeError::Backend)?;- let root =- command_buffer.allocate_constant_from_slice(&[root_token_id, 1u32]).map_err(WeaverEncodeError::Backend)?;- let word_bytes = DataType::U32.size_in_bytes();- let tree_field_bytes = size_for_shape(&[tree_slot_count], DataType::U32);- let mut packed_tree =- command_buffer.allocate_scratch(TreeIdx::COUNT * tree_field_bytes).map_err(WeaverEncodeError::Backend)?;- command_buffer.encode_fill(&mut packed_tree, 0);- let parent_offset = TreeIdx::ParentSlot as usize * tree_field_bytes;- command_buffer.encode_fill(- packed_tree.subrange_mut(parent_offset..parent_offset + tree_field_bytes),- 0xff, // FRONTIER_NO_WINNER- );- command_buffer.encode_copy(root.subrange(..word_bytes), packed_tree.subrange_mut(..word_bytes));- let valid_offset = TreeIdx::Valid as usize * tree_field_bytes;- command_buffer.encode_copy(- root.subrange(word_bytes..),- packed_tree.subrange_mut(valid_offset..valid_offset + word_bytes),- );+ let root = command_buffer+ .allocate_constant_from_slice(&[root_token_id, 1u32])+ .map_err(WeaverEncodeError::Backend)?;+ let word_bytes = DataType::U32.size_in_bytes();+ let tree_field_bytes = size_for_shape(&[tree_slot_count], DataType::U32);+ let mut packed_tree = command_buffer+ .allocate_scratch(TreeIdx::COUNT * tree_field_bytes)+ .map_err(WeaverEncodeError::Backend)?;+ command_buffer.encode_fill(&mut packed_tree, 0);+ let parent_offset = TreeIdx::ParentSlot as usize * tree_field_bytes;+ command_buffer.encode_fill(+ packed_tree.subrange_mut(parent_offset..parent_offset + tree_field_bytes),+ 0xff, // FRONTIER_NO_WINNER+ );+ command_buffer.encode_copy(root.subrange(..word_bytes), packed_tree.subrange_mut(..word_bytes));+ let valid_offset = TreeIdx::Valid as usize * tree_field_bytes;+ command_buffer.encode_copy(+ root.subrange(word_bytes..),+ packed_tree.subrange_mut(valid_offset..valid_offset + word_bytes),+ );- let mut frontier = command_buffer- .allocate_scratch_for_shape(&[FrontierIdx::COUNT as u32, frontier_capacity], DataType::U32)- .map_err(WeaverEncodeError::Backend)?;- command_buffer.encode_fill(&mut frontier, 0);- let mut slot_ancestors = command_buffer- .allocate_scratch_for_shape(&[tree_slot_count, ancestor_stride], DataType::U32)- .map_err(WeaverEncodeError::Backend)?;+ let mut frontier = command_buffer+ .allocate_scratch_for_shape(&[FrontierIdx::COUNT as u32, frontier_capacity], DataType::U32)+ .map_err(WeaverEncodeError::Backend)?;+ command_buffer.encode_fill(&mut frontier, 0);+ let mut slot_ancestors = command_buffer+ .allocate_scratch_for_shape(&[tree_slot_count, ancestor_stride], DataType::U32)+ .map_err(WeaverEncodeError::Backend)?;- let mut node_token_ids = command_buffer- .allocate_scratch_for_shape(&[shape.expand_per_round], DataType::U32)- .map_err(WeaverEncodeError::Backend)?;- command_buffer.encode_fill(&mut node_token_ids, 0);- command_buffer.encode_copy(root.subrange(..word_bytes), node_token_ids.subrange_mut(..word_bytes));+ let mut node_token_ids = command_buffer+ .allocate_scratch_for_shape(&[shape.expand_per_round], DataType::U32)+ .map_err(WeaverEncodeError::Backend)?;+ command_buffer.encode_fill(&mut node_token_ids, 0);+ command_buffer.encode_copy(root.subrange(..word_bytes), node_token_ids.subrange_mut(..word_bytes));- let mut node_metadata = command_buffer- .allocate_scratch_for_shape(&[MetadataIdx::COUNT as u32, shape.expand_per_round], DataType::U32)- .map_err(WeaverEncodeError::Backend)?;- command_buffer.encode_fill(&mut node_metadata, 0);- let mut node_ancestor_indices = command_buffer- .allocate_scratch_for_shape(&[shape.expand_per_round, ancestor_stride], DataType::U32)- .map_err(WeaverEncodeError::Backend)?;- command_buffer.encode_fill(&mut node_ancestor_indices, 0);- let mut node_valid = command_buffer- .allocate_scratch_for_shape(&[shape.expand_per_round], DataType::U32)- .map_err(WeaverEncodeError::Backend)?;- command_buffer.encode_fill(&mut node_valid, 0);- command_buffer.encode_copy(root.subrange(word_bytes..), node_valid.subrange_mut(..word_bytes));+ let mut node_metadata = command_buffer+ .allocate_scratch_for_shape(&[MetadataIdx::COUNT as u32, shape.expand_per_round], DataType::U32)+ .map_err(WeaverEncodeError::Backend)?;+ command_buffer.encode_fill(&mut node_metadata, 0);+ let mut node_ancestor_indices = command_buffer+ .allocate_scratch_for_shape(&[shape.expand_per_round, ancestor_stride], DataType::U32)+ .map_err(WeaverEncodeError::Backend)?;+ command_buffer.encode_fill(&mut node_ancestor_indices, 0);+ let mut node_valid = command_buffer+ .allocate_scratch_for_shape(&[shape.expand_per_round], DataType::U32)+ .map_err(WeaverEncodeError::Backend)?;+ command_buffer.encode_fill(&mut node_valid, 0);+ command_buffer.encode_copy(root.subrange(word_bytes..), node_valid.subrange_mut(..word_bytes));- let mut node_candidate_ids = command_buffer- .allocate_scratch_for_shape(&[shape.expand_per_round, self.candidate_pool_size], DataType::U32)- .map_err(WeaverEncodeError::Backend)?;- let mut node_candidate_logits = command_buffer- .allocate_scratch_for_shape(&[shape.expand_per_round, self.candidate_pool_size], DataType::F32)- .map_err(WeaverEncodeError::Backend)?;- let depth_seeds_buffer =- command_buffer.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;-- let mut batch_start_slot = 0;- for round in 0..shape.rounds {- let batch_node_count = if round == 0 {- 1- } else {- shape.expand_per_round- };- command_buffer.push_debug_group("weaver step");- self.encode_step(- target_embedding,- prefix_kv_layers.iter(),- &rope,- node_kv_layers.iter_mut(),- &mut packed_tree,- &mut frontier,- &mut slot_ancestors,- &mut node_token_ids,- &mut node_metadata,- &mut node_ancestor_indices,- &mut node_valid,- &mut node_candidate_ids,- &mut node_candidate_logits,- &depth_seeds_buffer,- &candidate_ids,- &candidate_logits,- &shape,- batch_node_count,- batch_start_slot,- command_buffer,- )?;- command_buffer.pop_debug_group();- batch_start_slot += batch_node_count;- }+ let mut node_candidate_ids = command_buffer+ .allocate_scratch_for_shape(&[shape.expand_per_round, self.candidate_pool_size], DataType::U32)+ .map_err(WeaverEncodeError::Backend)?;+ let mut node_candidate_logits = command_buffer+ .allocate_scratch_for_shape(&[shape.expand_per_round, self.candidate_pool_size], DataType::F32)+ .map_err(WeaverEncodeError::Backend)?;+ let depth_seeds_buffer =+ command_buffer.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;++ let mut batch_start_slot = 0;+ for round in 0..shape.rounds {+ let batch_node_count = if round == 0 {+ 1+ } else {+ shape.expand_per_round+ };+ let command_buffer = &mut *command_buffer.debug_group("weaver step");+ self.encode_step(+ target_embedding,+ prefix_kv_layers.iter(),+ &rope,+ node_kv_layers.iter_mut(),+ &mut packed_tree,+ &mut frontier,+ &mut slot_ancestors,+ &mut node_token_ids,+ &mut node_metadata,+ &mut node_ancestor_indices,+ &mut node_valid,+ &mut node_candidate_ids,+ &mut node_candidate_logits,+ &depth_seeds_buffer,+ &candidate_ids,+ &candidate_logits,+ &shape,+ batch_node_count,+ batch_start_slot,+ command_buffer,+ )?;+ batch_start_slot += batch_node_count;+ }- command_buffer.pop_debug_group();+ (packed_tree, frontier)+ };let mut packed_tree_readback =command_buffer.context().create_buffer(packed_tree.size()).map_err(WeaverEncodeError::Backend)?;diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex cdd1be257..f5a74c487 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -211,7 +211,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.create_command_buffer(Some("prefill"), Some(allocation_pool.clone())).map_err(LanguageModelStreamError::Backend)?;- command_buffer.add_debug_point(&format!(+ command_buffer.debug_point(&format!("prefill at context {}, suffix {} ({} batches)",model_state.transformer_state.context_length(),input.len(),@@ -590,7 +590,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {).map_err(LanguageModelStreamError::Backend)?;- command_buffer.add_debug_point(&format!(+ command_buffer.debug_point(&format!("decode round {} at context {} suffix {}",self.metrics.num_decode_forward_passes,context_length,