A high-performance inference engine for AI models
More debug groups everywhere (#676)
uuuvnac28cb128f03
diff --git a/crates/backend-uzu/build/metal/bindgen/mod.rs b/crates/backend-uzu/build/metal/bindgen/mod.rsindex 2e64a608f..dda895ef4 100644--- a/crates/backend-uzu/build/metal/bindgen/mod.rs+++ b/crates/backend-uzu/build/metal/bindgen/mod.rs@@ -111,12 +111,14 @@ pub fn bindgen(#(#encode_argument_definitions),*) {#empty_dispatch_guards+ encoder.push_debug_group(#kernel_name);#(#encode_deconstructs)*#encode_accesses_calllet compute_encoder = encoder.as_command_buffer_mut().ensure_compute();compute_encoder.set_compute_pipeline_state(&self.pipeline);#(#encode_set_calls)*#dispatch_code+ encoder.pop_debug_group();}}};diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rsindex 64115e083..6f393e899 100644--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs@@ -1,5 +1,10 @@-use std::{sync::Arc, time::Duration};+use std::{+ iter::{chain, once},+ sync::{Arc, LazyLock},+ time::Duration,+};+use itertools::Itertools;use metal::{MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferStatus,MTLCommandEncoder, MTLCommandEncoderExt, MTLCommandQueue, MTLComputeCommandEncoder,@@ -14,6 +19,8 @@ use crate::backends::{metal::{Metal, MetalContext, error::MetalError},};+static DEBUG_ENCODER_LABELS: LazyLock<bool> = LazyLock::new(|| std::env::var("UZU_METAL_DEBUG_ENCODER_LABELS").is_ok());+pub struct MetalCommandBuffer;impl CommandBuffer for MetalCommandBuffer {@@ -50,6 +57,7 @@ impl CommandBufferInitial for MetalCommandBufferInitial {MetalCommandBufferEncoding {command_buffer: self.command_buffer,encoding_state: MetalCommandBufferEncodingEncodingState::None,+ debug_group_stack: vec![],context: self.context,}}@@ -64,17 +72,24 @@ enum MetalCommandBufferEncodingEncodingState {pub struct MetalCommandBufferEncoding {command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,encoding_state: MetalCommandBufferEncodingEncodingState,+ debug_group_stack: Vec<String>,context: Arc<MetalContext>,}impl MetalCommandBufferEncoding {fn ensure_none(&mut self) {- match &mut self.encoding_state {+ let encoder: &ProtocolObject<dyn MTLCommandEncoder> = match &self.encoding_state {MetalCommandBufferEncodingEncodingState::None => return,- MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) => compute_encoder.end_encoding(),- MetalCommandBufferEncodingEncodingState::Blit(blit_encoder) => blit_encoder.end_encoding(),+ MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) => compute_encoder.as_ref(),+ MetalCommandBufferEncodingEncodingState::Blit(blit_encoder) => blit_encoder.as_ref(),};+ for _ in &self.debug_group_stack {+ encoder.pop_debug_group();+ }++ encoder.end_encoding();+self.encoding_state = MetalCommandBufferEncodingEncodingState::None;}@@ -83,7 +98,7 @@ impl MetalCommandBufferEncoding {self.ensure_none();let compute_encoder =self.command_buffer.compute_command_encoder().expect("Failed to create compute command encoder");- compute_encoder.set_label(self.command_buffer.label().as_deref());+ self.ensure_common(compute_encoder.as_ref());self.encoding_state = MetalCommandBufferEncodingEncodingState::Compute(compute_encoder);}@@ -96,9 +111,10 @@ impl MetalCommandBufferEncoding {fn ensure_blit(&mut self) -> &mut Retained<ProtocolObject<dyn MTLBlitCommandEncoder>> {if !matches!(self.encoding_state, MetalCommandBufferEncodingEncodingState::Blit(_)) {self.ensure_none();- self.encoding_state = MetalCommandBufferEncodingEncodingState::Blit(- self.command_buffer.blit_command_encoder().expect("Failed to create blit command encoder"),- );+ let blit_encoder =+ self.command_buffer.blit_command_encoder().expect("Failed to create blit command encoder");+ self.ensure_common(blit_encoder.as_ref());+ self.encoding_state = MetalCommandBufferEncodingEncodingState::Blit(blit_encoder);}let MetalCommandBufferEncodingEncodingState::Blit(blit_encoder) = &mut self.encoding_state else {@@ -106,6 +122,31 @@ impl MetalCommandBufferEncoding {};blit_encoder}++ fn ensure_common(+ &self,+ encoder: &ProtocolObject<dyn MTLCommandEncoder>,+ ) {+ let command_buffer_label = self.command_buffer.label();+ let label = if *DEBUG_ENCODER_LABELS && (command_buffer_label.is_some() || !self.debug_group_stack.is_empty()) {+ Some(+ chain(+ once(command_buffer_label.as_deref()),+ self.debug_group_stack.iter().map(|label| Some(label.as_str())),+ )+ .flatten()+ .join("."),+ )+ } else {+ command_buffer_label+ };+ if label.is_some() {+ encoder.set_label(label.as_deref());+ }+ for debug_group in &self.debug_group_stack {+ encoder.push_debug_group(debug_group);+ }+ }}impl Drop for MetalCommandBufferEncoding {@@ -162,11 +203,37 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {&mut self,name: &str,) {- self.command_buffer.push_debug_group(name);+ if *DEBUG_ENCODER_LABELS {+ self.ensure_none();+ }++ self.debug_group_stack.push(name.to_string());++ match &self.encoding_state {+ MetalCommandBufferEncodingEncodingState::None => (),+ MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) => compute_encoder.push_debug_group(name),+ MetalCommandBufferEncodingEncodingState::Blit(blit_encoder) => {+ let encoder: &ProtocolObject<dyn MTLCommandEncoder> = blit_encoder.as_ref();+ encoder.push_debug_group(name);+ },+ }}fn pop_debug_group(&mut self) {- self.command_buffer.pop_debug_group();+ if *DEBUG_ENCODER_LABELS {+ self.ensure_none();+ }++ self.debug_group_stack.pop().expect("debug group stack underflow");++ match &self.encoding_state {+ MetalCommandBufferEncodingEncodingState::None => (),+ MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) => compute_encoder.pop_debug_group(),+ MetalCommandBufferEncodingEncodingState::Blit(blit_encoder) => {+ let encoder: &ProtocolObject<dyn MTLCommandEncoder> = blit_encoder.as_ref();+ encoder.pop_debug_group();+ },+ }}fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {@@ -193,6 +260,7 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {{let cmd_buffer = cmd_queue.command_buffer().expect("Failed to create command buffer");+ cmd_buffer.set_label(Some("sync (wait)"));cmd_buffer.encode_wait_for_event_value(self.context.timeline_event(), wait_value);cmd_buffer.commit();}@@ -201,6 +269,7 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {{let cmd_buffer = cmd_queue.command_buffer().expect("Failed to create command buffer");+ cmd_buffer.set_label(Some("sync (signal)"));cmd_buffer.encode_signal_event_value(self.context.timeline_event(), wait_value + 1);cmd_buffer.commit();}diff --git a/crates/backend-uzu/src/encodable_block/classifier.rs b/crates/backend-uzu/src/encodable_block/classifier.rsindex 9c3991a3a..1d2ef92ae 100644--- a/crates/backend-uzu/src/encodable_block/classifier.rs+++ b/crates/backend-uzu/src/encodable_block/classifier.rs@@ -117,6 +117,8 @@ impl<B: Backend> Classifier<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, ClassifierError<B>> {+ encoder.push_debug_group("classifier");+let embedded = self.embedding.encode_lookup(token_ids, batch_dim, encoder)?;let hidden =@@ -143,6 +145,8 @@ impl<B: Backend> Classifier<B> {let logits = self.prediction_head.encode(pooled, 1, encoder).map_err(ClassifierError::Backend)?;+ encoder.pop_debug_group();+Ok(logits)}}diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 2541d4742..ef35118e1 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -540,6 +540,8 @@ impl<B: Backend> Embedding<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, EmbeddingError<B>> {+ encoder.push_debug_group("embedding lookup");+let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.model_dim as usize], self.data_type)).map_err(EmbeddingError::BackendError)?;@@ -618,6 +620,8 @@ impl<B: Backend> Embedding<B> {},};+ encoder.pop_debug_group();+Ok(output)}@@ -628,6 +632,8 @@ impl<B: Backend> Embedding<B> {output_data_type: DataType,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, EmbeddingError<B>> {+ encoder.push_debug_group("embedding readout");+assert!(batch_dim > 0, "Embedding readout requires at least one row");let native_output = output_data_type == self.data_type;let input_hadamard = match &self.tying {@@ -776,6 +782,8 @@ impl<B: Backend> Embedding<B> {}}+ encoder.pop_debug_group();+Ok(output_allocation)}@@ -789,6 +797,8 @@ impl<B: Backend> Embedding<B> {ids_per_row: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, EmbeddingError<B>> {+ encoder.push_debug_group("embedding readout (sparse)");+assert!(rows > 0 && ids_per_row > 0);let (b, readout, input_hadamard) = match &self.tying {EmbeddingTying::Tied {@@ -895,6 +905,8 @@ impl<B: Backend> Embedding<B> {).map_err(EmbeddingError::BackendError)?;+ encoder.pop_debug_group();+Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex 38e54e978..1d440dd54 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -349,13 +349,19 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- self.encode_with_a(+ encoder.push_debug_group("matmul");++ let output = self.encode_with_a(MatmulA::FullPrecision {values: &input,offset: 0,},batch_dim,encoder,- )+ )?;++ encoder.pop_debug_group();++ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex c27e02963..6dbb4ccb9 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -160,6 +160,8 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("linear (qlora)");+let mut intermediate =encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type))?;@@ -242,6 +244,8 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {);}+ encoder.pop_debug_group();+Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rsindex 9058603aa..2120aa45d 100644--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs@@ -178,6 +178,8 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("linear (rht)");+if let Some(quantize_transform) = &self.quantize_transform&& self.inner_linear.select_path(batch_dim, encoder.context()) == MatmulPath::Gemm{@@ -200,7 +202,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {self.input_dimension as u32,encoder,);- return self.inner_linear.encode_with_a(+ let output = self.inner_linear.encode_with_a(MatmulA::Int8Symmetric {values: &values,scales: &scales,@@ -208,7 +210,11 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {},batch_dim,encoder,- );+ )?;++ encoder.pop_debug_group();++ return Ok(output);}let mut input = input;@@ -219,6 +225,10 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {self.input_dimension as u32,encoder,);- self.inner_linear.encode(input, batch_dim, encoder)+ let output = self.inner_linear.encode(input, batch_dim, encoder)?;++ encoder.pop_debug_group();++ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rsindex abe2f03c6..456d02d30 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/core/mod.rs@@ -76,7 +76,9 @@ impl<B: Backend> AttentionCores<B> {arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, <B as Backend>::Error> {- if arguments.suffix_length > 8+ encoder.push_debug_group("attention core");++ let output = if arguments.suffix_length > 8&& let Some(gemm) = &self.gemm{gemm.encode(arguments, encoder)@@ -88,7 +90,11 @@ impl<B: Backend> AttentionCores<B> {self.two_pass.encode(arguments, encoder)} else {self.single_pass.encode(arguments, encoder)- }+ }?;++ encoder.pop_debug_group();++ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rsindex 700c870ac..2976e9838 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs@@ -265,10 +265,16 @@ impl<B: Backend> Mixer<B> for Attention<B> {state: Option<MaybeMut<dyn MixerState<B>>>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("attention");+assert_eq!(precalculated_rope.is_some(), self.max_rope_length.is_some(), "precalculated rope mismatch");let state =state.map(|state| state.downcast::<AttentionState<B>>().expect("incorrect type of attention state"));- self.attend(hidden, precalculated_rope, batch_dim, state, encoder)+ let output = self.attend(hidden, precalculated_rope, batch_dim, state, encoder)?;++ encoder.pop_debug_group();++ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rsindex 73fa3748a..9949bdb94 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/qkv_norm.rs@@ -138,6 +138,8 @@ impl<B: Backend> QKVNorm<B> {q_heads: usize,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {+ encoder.push_debug_group("qkv norm");+let kv = self.num_kv_heads;let total_heads = q_heads + 2 * kv;let heads = [(&self.query, 0, q_heads), (&self.key, q_heads, kv), (&self.value, q_heads + kv, kv)];@@ -163,6 +165,9 @@ impl<B: Backend> QKVNorm<B> {encoder,);}++ encoder.pop_debug_group();+Ok(())}}diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rsindex 0a15d89c9..c0e05171e 100644--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs@@ -486,6 +486,8 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {state: Option<MaybeMut<dyn MixerState<B>>>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("delta net");+assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");let state = state.expect("delta net requires state");@@ -499,7 +501,11 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {let mut in_projected = self.in_projection.encode(hidden, batch_dim.size(), encoder)?;if !batch_dim.full_accept() {- return self.encode_tree_verify(in_projected, batch_dim, state, encoder);+ let output = self.encode_tree_verify(in_projected, batch_dim, state, encoder)?;++ encoder.pop_debug_group();++ return Ok(output);}let mut delta_output =@@ -639,6 +645,10 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {suffix_length: batch_dim.size(),});- self.out_projection.encode(delta_output, batch_dim.size(), encoder)+ let output = self.out_projection.encode(delta_output, batch_dim.size(), encoder)?;++ encoder.pop_debug_group();++ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rsindex e3cc82932..3affc5cc9 100644--- a/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/mamba2.rs@@ -251,6 +251,8 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {state: Option<MaybeMut<dyn MixerState<B>>>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("mamba2");+assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");if !batch_dim.full_accept() {@@ -439,7 +441,11 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {state.suffix_length = Some(batch_dim.size());- self.out_projection.encode(ssd_output, batch_dim.size(), encoder)+ let output = self.out_projection.encode(ssd_output, batch_dim.size(), encoder)?;++ encoder.pop_debug_group();++ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rsindex b17c704f6..dc2ee69d2 100644--- a/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/short_conv.rs@@ -317,6 +317,8 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {state: Option<MaybeMut<dyn MixerState<B>>>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("short conv");+assert!(precalculated_rope.is_none(), "unexpected rope for short conv mixer");let state = state.expect("short conv requires state");@@ -350,6 +352,10 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {conv_output};- self.out_projection.encode(conv_output, batch_dim.size(), encoder)+ let output = self.out_projection.encode(conv_output, batch_dim.size(), encoder)?;++ encoder.pop_debug_group();++ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/mlp/dense.rs b/crates/backend-uzu/src/encodable_block/mlp/dense.rsindex f45cadd8b..eafd4ac4b 100644--- a/crates/backend-uzu/src/encodable_block/mlp/dense.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/dense.rs@@ -35,8 +35,14 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("mlp (dense)");+let fused_up = self.up.encode(input, batch_dim, encoder)?;let hidden = self.gate.encode(encoder, &fused_up, batch_dim)?;- self.down.encode(hidden, batch_dim, encoder)+ let output = self.down.encode(hidden, batch_dim, encoder)?;++ encoder.pop_debug_group();++ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rsindex f602d8c7c..231dcb145 100644--- a/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/gate_act_mul.rs@@ -38,6 +38,8 @@ impl<B: Backend> MlpGateActMulEncodable<B> {fused_up: &Allocation<B>,batch_dim: usize,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("gate act mul");+if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")}@@ -54,6 +56,9 @@ impl<B: Backend> MlpGateActMulEncodable<B> {self.activation.act_type(),encoder,);++ encoder.pop_debug_group();+Ok(hidden)}}diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rsindex 1b48e55a1..9002bf9ba 100644--- a/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/moe/mod.rs@@ -216,6 +216,8 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("mlp (moe)");+let total_rows = batch_dim * self.num_active_experts;let num_blocks = batch_dim.div_ceil(256);let num_tiles = self.num_routed_experts.div_ceil(512);@@ -341,6 +343,8 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {encoder,);+ encoder.pop_debug_group();+Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rsindex 4674084a7..38df87f75 100644--- a/crates/backend-uzu/src/encodable_block/normalization.rs+++ b/crates/backend-uzu/src/encodable_block/normalization.rs@@ -115,6 +115,8 @@ impl<B: Backend> Normalization<B> {shortcut: Option<&mut Allocation<B>>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("normalization");+let row_size = self.element_count * self.data_type.size_in_bytes();let row_offset_bytes = row_offset * row_size;let shortcut = shortcut.map(|shortcut| (shortcut, row_offset_bytes));@@ -133,6 +135,9 @@ impl<B: Backend> Normalization<B> {self.post_layer_scalar_value,encoder,);++ encoder.pop_debug_group();+Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rsindex 180982a7c..2916f6399 100644--- a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs+++ b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs@@ -116,6 +116,8 @@ impl<B: Backend> PerLayerEmbedding<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("per layer embedding");+let total_ple_dim = self.num_layers * self.ple_dim;let total_rows = batch_dim * self.num_layers;let total_elements = batch_dim * total_ple_dim;@@ -151,6 +153,8 @@ impl<B: Backend> PerLayerEmbedding<B> {encoder,);+ encoder.pop_debug_group();+Ok(per_layer_inputs)}}@@ -240,6 +244,8 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {+ encoder.push_debug_group("per layer embedding projection");+let length = batch_dim * self.model_dim;self.residual_finalize.encode(@@ -283,6 +289,8 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {encoder,);+ encoder.pop_debug_group();+Ok(())}}diff --git a/crates/backend-uzu/src/encodable_block/prediction_head.rs b/crates/backend-uzu/src/encodable_block/prediction_head.rsindex b08c0e5f8..c2e5f7ee9 100644--- a/crates/backend-uzu/src/encodable_block/prediction_head.rs+++ b/crates/backend-uzu/src/encodable_block/prediction_head.rs@@ -94,6 +94,8 @@ impl<B: Backend> PredictionHead<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("prediction head");+let mut hidden = self.dense_projection.encode(input, batch_dim, encoder)?;self.activation_kernel.encode(None::<&Allocation<B>>,@@ -103,6 +105,10 @@ impl<B: Backend> PredictionHead<B> {encoder,);let normalized = self.normalization.encode(&hidden, 0, batch_dim, None, encoder)?;- self.readout.encode(normalized, batch_dim, encoder)+ let logits = self.readout.encode(normalized, batch_dim, encoder)?;++ encoder.pop_debug_group();++ Ok(logits)}}diff --git a/crates/backend-uzu/src/encodable_block/transformer_layer.rs b/crates/backend-uzu/src/encodable_block/transformer_layer.rsindex 5527ddcdf..32bb92d25 100644--- a/crates/backend-uzu/src/encodable_block/transformer_layer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer_layer.rs@@ -199,6 +199,8 @@ impl<B: Backend> TransformerLayer<B> {state: Option<MaybeMut<dyn MixerState<B>>>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group(&format!("transformer layer {}", self.layer_index));+let hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut), encoder)?} else {@@ -228,6 +230,8 @@ impl<B: Backend> TransformerLayer<B> {encoder.encode_fill(&mut hidden, 0);}+ encoder.pop_debug_group();+Ok(hidden)}}diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rsindex 4913cdde2..57f927a70 100644--- a/crates/backend-uzu/src/encodable_block/weaver.rs+++ b/crates/backend-uzu/src/encodable_block/weaver.rs@@ -301,7 +301,7 @@ impl<B: Backend> Weaver<B> {shape: TreeShape,encoder: &mut Encoder<B>,) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {- encoder.push_debug_group("dflash weaver");+ encoder.push_debug_group("weaver tree");let tree_slot_count = shape.budget + 1;let frontier_capacity = tree_slot_count * shape.children_per_node;let lookahead_count = self.max_depth.min(inputs.candidates.depth_count);diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex bafecc337..ff4849653 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -81,7 +81,7 @@ struct DecodingStatePending<B: Backend> {input_trie: TrieNode,full_accept: bool,pending: Box<[Pending<B>]>,- _capture_span: Option<CaptureSpan<B>>,+ capture_span: Option<CaptureSpan<B>>,hidden_features: Option<Box<[Allocation<B>]>>,output_norm: Option<Allocation<B>>,output_tokens: Allocation<B>,@@ -97,6 +97,7 @@ enum DecodingState<B: Backend> {num_accepted: usize,hidden_features: Option<Box<[Allocation<B>]>>,output_norm: Option<(Allocation<B>, usize)>,+ capture_span: Option<CaptureSpan<B>>,},Halted,Invalid,@@ -334,7 +335,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {input_trie: TrieNode::new(0, 0),full_accept: true,pending,- _capture_span: capture_span,+ capture_span,hidden_features: None,output_norm,output_tokens: output_tokens.unwrap(),@@ -407,6 +408,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {num_accepted: 0,hidden_features: forward_pass_pending.hidden_features,output_norm,+ capture_span: forward_pass_pending.capture_span,};return self.generate();}@@ -416,6 +418,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {num_accepted,hidden_features,output_norm,+ capture_span,} => {let output_token_id = full[num_accepted].2;@@ -427,6 +430,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {num_accepted: num_accepted + 1,hidden_features,output_norm,+ capture_span,};return Ok(Some(output_token_id));} else {@@ -483,6 +487,22 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {);encoder.pop_debug_group();}+ if let Some(capture_span) = capture_span {+ encoder+ .end_encoding()+ .submit()+ .wait_until_completed()+ .map_err(LanguageModelStreamError::Backend)?;++ drop(capture_span);++ encoder = Encoder::<B>::new_with_pool_name(+ &self.model.engine.context,+ self.allocation_pool.clone(),+ Some("decode"),+ )+ .map_err(LanguageModelStreamError::Backend)?;+ }self.model_state.tokens.extend(accepted_output_token_ids);(ForwardPassChaining::Constant {@@ -525,14 +545,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {None};- let mut pending = Vec::new();- let mut encoder = if let Some(encoder) = encoder {- encoder- } else {- Encoder::<B>::new_with_pool_name(&self.model.engine.context, self.allocation_pool.clone(), Some("decode"))- .map_err(LanguageModelStreamError::Backend)?- };-let speculation_batch = self.model_state.max_context_length@@ -545,13 +557,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {#[cfg(grammar)]self.options.grammar.as_mut(),)? {- pending.push(encoder.end_encoding().submit());- encoder = Encoder::<B>::new_with_pool_name(- &self.model.engine.context,- self.allocation_pool.clone(),- Some("decode"),- )- .map_err(LanguageModelStreamError::Backend)?;let trie = speculator.propose_tree(self.model_state.speculator_state.as_mut().unwrap(),output_norm,@@ -579,6 +584,14 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {};let input_flat_trie = input_trie.linearize();+ let mut pending = Vec::new();+ let mut encoder = if let Some(encoder) = encoder {+ encoder+ } else {+ Encoder::<B>::new_with_pool_name(&self.model.engine.context, self.allocation_pool.clone(), Some("decode"))+ .map_err(LanguageModelStreamError::Backend)?+ };+let token_ids = if let Some(chain_copy) = chain_copy {let mut token_ids =encoder.allocate_scratch(DataType::U32.size_in_bytes()).map_err(LanguageModelStreamError::Backend)?;@@ -731,7 +744,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {input_trie,full_accept,pending: pending.into_boxed_slice(),- _capture_span: capture_span,+ capture_span,hidden_features: if full_accept {None} else {@@ -804,6 +817,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {num_accepted,hidden_features,output_norm: _,+ capture_span,} => {assert!(num_accepted > 0 && num_accepted < full.len());@@ -828,6 +842,8 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {}encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(capture_span);+self.model_state.tokens.extend(full.iter().take(num_accepted).map(|(_, _, t)| *t));Some(full[num_accepted].2)