Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

RAII debug groups (idea stolen from #906) (#935)

uuuvn6d021ae30e09
diff --git a/crates/uzu-engine/build/metal/bindgen/mod.rs b/crates/uzu-engine/build/metal/bindgen/mod.rs
index df497d193..588a61f5d 100644
--- a/crates/uzu-engine/build/metal/bindgen/mod.rs
+++ b/crates/uzu-engine/build/metal/bindgen/mod.rs
@@ -122,13 +122,12 @@ pub fn bindgen(
#(#encode_argument_definitions),*
) {
#empty_dispatch_guards
- command_buffer.push_debug_group(#kernel_name);
+ let command_buffer = &mut *command_buffer.debug_group(#kernel_name);
#(#encode_deconstructs)*
#encode_accesses_call
command_buffer.compute_encoder.set_compute_pipeline_state(&self.pipeline);
#(#encode_set_calls)*
#dispatch_code
- command_buffer.pop_debug_group();
}
}
};
diff --git a/crates/uzu-engine/src/backends/common/backend.rs b/crates/uzu-engine/src/backends/common/backend.rs
index 6acdcbc50..3aa01329a 100644
--- a/crates/uzu-engine/src/backends/common/backend.rs
+++ b/crates/uzu-engine/src/backends/common/backend.rs
@@ -1,10 +1,11 @@
-use std::error::Error;
+use std::{error::Error, ops::DerefMut};
use super::{CommandBuffer, ConstantBuffer, Context, GlobalBuffer, Kernels, ScratchBuffer, SparseBuffer};
pub trait Backend: Clone + Send + Sync + 'static {
type Context: Context<Backend = Self>;
type CommandBuffer: CommandBuffer<Backend = Self>;
+ type DebugGroupSpan<'a>: DerefMut<Target = <Self::CommandBuffer as CommandBuffer>::Encoding> + 'a;
type GlobalBuffer: GlobalBuffer<Backend = Self>;
type ConstantBuffer: ConstantBuffer<Backend = Self>;
type ScratchBuffer: ScratchBuffer<Backend = Self>;
diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rs
index 0ed3d0034..c98712b8e 100644
--- a/crates/uzu-engine/src/backends/common/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs
@@ -73,14 +73,13 @@ pub trait CommandBufferEncoding {
value: u8,
);
- fn push_debug_group(
+ #[must_use]
+ fn debug_group(
&mut self,
name: &str,
- );
-
- fn pop_debug_group(&mut self);
+ ) -> <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DebugGroupSpan<'_>;
- fn add_debug_point(
+ fn debug_point(
&mut self,
name: &str,
);
diff --git a/crates/uzu-engine/src/backends/cpu/backend.rs b/crates/uzu-engine/src/backends/cpu/backend.rs
index 335858916..0555fcdaf 100644
--- a/crates/uzu-engine/src/backends/cpu/backend.rs
+++ b/crates/uzu-engine/src/backends/cpu/backend.rs
@@ -10,7 +10,7 @@ use crate::backends::{
},
cpu::{
buffer::{dense::CpuBuffer, sparse::CpuSparseBuffer},
- command_buffer::CpuCommandBuffer,
+ command_buffer::{CpuCommandBuffer, CpuCommandBufferEncoding},
context::CpuContext,
error::CpuError,
kernel::CpuKernels,
@@ -23,6 +23,7 @@ pub struct Cpu;
impl Backend for Cpu {
type Context = CpuContext;
type CommandBuffer = CpuCommandBuffer;
+ type DebugGroupSpan<'a> = &'a mut CpuCommandBufferEncoding;
type GlobalBuffer = CpuBuffer;
type ConstantBuffer = BumpAllocation<Self::GlobalBuffer>;
type ScratchBuffer = PoolAllocation<Self::GlobalBuffer>;
diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
index 04ca0e2bb..4441b9977 100644
--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
@@ -110,15 +110,14 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
});
}
- fn push_debug_group(
+ fn debug_group(
&mut self,
_name: &str,
- ) {
+ ) -> <Cpu as Backend>::DebugGroupSpan<'_> {
+ self
}
- fn pop_debug_group(&mut self) {}
-
- fn add_debug_point(
+ fn debug_point(
&mut self,
_name: &str,
) {
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
index 21b07f2d1..0c0f28688 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
@@ -64,28 +64,29 @@ impl AttentionKernel for AttentionCpuKernel {
arguments.trie.is_some(),
config.sliding_window_size.is_some(),
)?;
- command_buffer.push_debug_group("attention core");
- single_pass.encode(
- arguments.queries,
- arguments.keys,
- arguments.values,
- &mut output,
- config.num_q_heads / config.num_groups,
- arguments.cache.prefix_len() + arguments.suffix_length,
- config.head_dim,
- config.num_groups * config.head_dim,
- config.head_dim,
- config.num_groups * config.head_dim,
- arguments.cache.ring_params(),
- config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),
- arguments.trie,
- config.sliding_window_size,
- arguments.sinks,
- config.num_q_heads,
- arguments.suffix_length,
- command_buffer,
- );
- command_buffer.pop_debug_group();
+ {
+ let command_buffer = &mut *command_buffer.debug_group("attention core");
+ single_pass.encode(
+ arguments.queries,
+ arguments.keys,
+ arguments.values,
+ &mut output,
+ config.num_q_heads / config.num_groups,
+ arguments.cache.prefix_len() + arguments.suffix_length,
+ config.head_dim,
+ config.num_groups * config.head_dim,
+ config.head_dim,
+ config.num_groups * config.head_dim,
+ arguments.cache.ring_params(),
+ config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),
+ arguments.trie,
+ config.sliding_window_size,
+ arguments.sinks,
+ config.num_q_heads,
+ arguments.suffix_length,
+ command_buffer,
+ );
+ }
if let (Some(gate), Some(gate_row_stride)) = (arguments.gate, config.gate_row_stride) {
<sigmoid_gate::SigmoidGateCpuKernel as SigmoidGateKernel>::new(command_buffer.context(), config.data_type)?
.encode(
diff --git a/crates/uzu-engine/src/backends/metal/backend.rs b/crates/uzu-engine/src/backends/metal/backend.rs
index 3986a0f47..bd8a2d71f 100644
--- a/crates/uzu-engine/src/backends/metal/backend.rs
+++ b/crates/uzu-engine/src/backends/metal/backend.rs
@@ -10,7 +10,7 @@ use crate::backends::{
metal::{
buffer::{dense::MetalDenseBuffer, sparse::MetalSparseBuffer},
capture::MetalCaptureSpan,
- command_buffer::MetalCommandBuffer,
+ command_buffer::{MetalCommandBuffer, MetalDebugGroupSpan},
context::MetalContext,
error::MetalError,
kernel::MetalKernels,
@@ -23,6 +23,7 @@ pub struct Metal;
impl Backend for Metal {
type Context = MetalContext;
type CommandBuffer = MetalCommandBuffer;
+ type DebugGroupSpan<'a> = MetalDebugGroupSpan<'a>;
type GlobalBuffer = BlockAllocation<MetalDenseBuffer>;
type ConstantBuffer = BumpAllocation<Self::GlobalBuffer>;
type ScratchBuffer = PoolAllocation<Self::GlobalBuffer>;
diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rs
index 02565f5e8..4c8af0b49 100644
--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs
@@ -1,4 +1,5 @@
use std::{
+ ops::{Deref, DerefMut},
range::Range,
sync::{Arc, mpsc},
time::Duration,
@@ -195,19 +196,17 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
kernel.encode(dst, (range.end - range.start) as u64, u32::from(value) * 0x0101_0101, self);
}
- // TODO: maybe port previous debug command_buffer labels
- fn push_debug_group(
+ fn debug_group(
&mut self,
name: &str,
- ) {
+ ) -> MetalDebugGroupSpan<'_> {
ProtocolObject::<dyn MTL4CommandEncoder>::push_debug_group(self.compute_encoder.as_ref(), name);
+ MetalDebugGroupSpan {
+ command_buffer: self,
+ }
}
- fn pop_debug_group(&mut self) {
- self.compute_encoder.pop_debug_group();
- }
-
- fn add_debug_point(
+ fn debug_point(
&mut self,
name: &str,
) {
@@ -240,6 +239,30 @@ impl Drop for MetalCommandBufferEncoding {
}
}
+pub struct MetalDebugGroupSpan<'a> {
+ command_buffer: &'a mut MetalCommandBufferEncoding,
+}
+
+impl Deref for MetalDebugGroupSpan<'_> {
+ type Target = MetalCommandBufferEncoding;
+
+ fn deref(&self) -> &Self::Target {
+ self.command_buffer
+ }
+}
+
+impl DerefMut for MetalDebugGroupSpan<'_> {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ self.command_buffer
+ }
+}
+
+impl Drop for MetalDebugGroupSpan<'_> {
+ fn drop(&mut self) {
+ self.command_buffer.compute_encoder.pop_debug_group();
+ }
+}
+
pub struct MetalCommandBufferExecutable {
command_allocator: Retained<ProtocolObject<dyn MTL4CommandAllocator>>,
command_buffer: Retained<ProtocolObject<dyn MTL4CommandBuffer>>,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
index d16465c6f..4452fe8a1 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
@@ -101,9 +101,9 @@ impl AttentionKernel for AttentionMetalKernel {
>,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
- command_buffer.push_debug_group("attention core");
+ let command_buffer = &mut *command_buffer.debug_group("attention core");
let (gate, suffix_length) = (arguments.gate, arguments.suffix_length);
- let result = self.encode_impl(arguments, command_buffer).map(|(mut output, gate_applied)| {
+ self.encode_impl(arguments, command_buffer).map(|(mut output, gate_applied)| {
if let (Some(gate), Some((kernel, stride)), false) = (gate, &self.gate, gate_applied) {
kernel.encode(
gate,
@@ -115,9 +115,7 @@ impl AttentionKernel for AttentionMetalKernel {
);
}
output
- });
- command_buffer.pop_debug_group();
- result
+ })
}
}
diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rs
index f44077a95..fc24277d2 100644
--- a/crates/uzu-engine/src/encodable_block/classifier.rs
+++ b/crates/uzu-engine/src/encodable_block/classifier.rs
@@ -118,7 +118,7 @@ impl<B: Backend> Classifier<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, ClassifierError<B>> {
- command_buffer.push_debug_group("classifier");
+ let command_buffer = &mut *command_buffer.debug_group("classifier");
let embedded = self.embedding.encode_lookup(token_ids, batch_dim, command_buffer)?;
@@ -156,8 +156,6 @@ impl<B: Backend> Classifier<B> {
let logits = self.prediction_head.encode(pooled, 1, command_buffer).map_err(ClassifierError::Backend)?;
- command_buffer.pop_debug_group();
-
Ok(logits)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rs
index 7b1c8bbfd..5c7dfcfc2 100644
--- a/crates/uzu-engine/src/encodable_block/decoder.rs
+++ b/crates/uzu-engine/src/encodable_block/decoder.rs
@@ -143,7 +143,7 @@ impl<B: Backend> Decoder<B> {
state: &mut TransformerState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {
- command_buffer.push_debug_group("decoder");
+ let command_buffer = &mut *command_buffer.debug_group("decoder");
let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), command_buffer)?;
let embedded = if let Some(embedding_norm) = &self.embedding_norm {
@@ -196,8 +196,6 @@ impl<B: Backend> Decoder<B> {
transformer_output.output
};
- command_buffer.pop_debug_group();
-
Ok(DecoderEncodeOutput {
logits,
hidden_features: transformer_output.hidden_features,
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index b35383899..67d38fd36 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -207,7 +207,7 @@ impl<B: Backend> DFlash<B> {
return Ok(());
}
- command_buffer.push_debug_group("dflash accept");
+ let command_buffer = &mut *command_buffer.debug_group("dflash accept");
let num_tokens = accepted_indices.len() as u32;
let captured_layer_count = self.target_feature_input_dim / self.model_dim;
@@ -274,8 +274,6 @@ impl<B: Backend> DFlash<B> {
state.context_length += num_tokens;
- command_buffer.pop_debug_group();
-
Ok(())
}
@@ -287,7 +285,7 @@ impl<B: Backend> DFlash<B> {
batch_size: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {
- command_buffer.push_debug_group("dflash draft");
+ let command_buffer = &mut *command_buffer.debug_group("dflash draft");
assert!(batch_size >= 2 && batch_size <= self.block_size, "batch size exceeds DFlash block size");
assert!(
@@ -348,8 +346,6 @@ impl<B: Backend> DFlash<B> {
command_buffer,
)?;
- command_buffer.pop_debug_group();
-
Ok(DFlashOutput {
draft_hidden,
logits,
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index 0cd58c0f7..d64fa10a2 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -185,7 +185,7 @@ impl<B: Backend> Embedding<B> {
apply_logit_transform: bool,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, EmbeddingError<B>> {
- command_buffer.push_debug_group("embedding readout");
+ let command_buffer = &mut *command_buffer.debug_group("embedding readout");
assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");
let mut output_buffer = match &self.tying {
@@ -236,8 +236,6 @@ impl<B: Backend> Embedding<B> {
);
}
- command_buffer.pop_debug_group();
-
Ok(output_buffer)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
index d4faa24b9..a98d5599f 100644
--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
@@ -175,7 +175,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("linear (qlora)");
+ let command_buffer = &mut *command_buffer.debug_group("linear (qlora)");
let mut intermediate =
command_buffer.allocate_scratch_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type)?;
@@ -261,8 +261,6 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
);
}
- command_buffer.pop_debug_group();
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index d38dc43b8..4135fe01a 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -193,7 +193,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
LinearInput::FullPrecision(input) => input,
input => return self.inner_linear.encode_input(input, batch_dim, command_buffer),
};
- command_buffer.push_debug_group("linear (rht)");
+ let command_buffer = &mut *command_buffer.debug_group("linear (rht)");
let format = self.inner_linear.select_activation_format(batch_dim, command_buffer.context());
let input = self.input_rht.prepare_in_place(input, batch_dim, format, command_buffer)?;
@@ -204,7 +204,6 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
command_buffer,
)?;
- command_buffer.pop_debug_group();
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
index d9b72f349..f3a63c1c6 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
@@ -216,7 +216,7 @@ impl<B: Backend> Mixer<B> for Attention<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("attention");
+ let command_buffer = &mut *command_buffer.debug_group("attention");
assert_eq!(precalculated_rope.is_some(), self.max_rope_length.is_some(), "precalculated rope mismatch");
@@ -224,8 +224,6 @@ impl<B: Backend> Mixer<B> for Attention<B> {
state.map(|state| state.downcast::<AttentionState<B>>().expect("incorrect type of attention state"));
let output = self.attend(hidden, precalculated_rope, batch_dim, state, command_buffer)?;
- command_buffer.pop_debug_group();
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
index 33e793265..8ac89be55 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
@@ -470,7 +470,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("delta net");
+ let command_buffer = &mut *command_buffer.debug_group("delta net");
assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");
@@ -487,8 +487,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
if !batch_dim.full_accept() {
let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;
- command_buffer.pop_debug_group();
-
return Ok(output);
}
@@ -631,8 +629,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
let output = self.out_projection.encode(delta_output, batch_dim.size(), command_buffer)?;
- command_buffer.pop_debug_group();
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
index 878668519..753113be2 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
@@ -243,7 +243,7 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("mamba2");
+ let command_buffer = &mut *command_buffer.debug_group("mamba2");
assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");
@@ -421,8 +421,6 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
let output = self.out_projection.encode(ssd_output, batch_dim.size(), command_buffer)?;
- command_buffer.pop_debug_group();
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
index 93f9cbbb2..040b8cc4a 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
@@ -308,7 +308,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("short conv");
+ let command_buffer = &mut *command_buffer.debug_group("short conv");
assert!(precalculated_rope.is_none(), "unexpected rope for short conv mixer");
@@ -344,8 +344,6 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
let output = self.out_projection.encode(conv_output, batch_dim.size(), command_buffer)?;
- command_buffer.pop_debug_group();
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
index a0881dee7..a6a021470 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
@@ -35,15 +35,13 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("mlp (dense)");
+ let command_buffer = &mut *command_buffer.debug_group("mlp (dense)");
let fused_up = self.up.encode(input, batch_dim, command_buffer)?;
let act_format = self.down.select_activation_format(batch_dim, command_buffer.context());
let down_input = self.gate.encode_for_linear(command_buffer, &fused_up, batch_dim, act_format)?;
let output = self.down.encode_input(down_input, batch_dim, command_buffer)?;
- command_buffer.pop_debug_group();
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
index a2920a9ea..6f5ab40d1 100644
--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
@@ -110,7 +110,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("per layer embedding");
+ let command_buffer = &mut *command_buffer.debug_group("per layer embedding");
let total_ple_dim = self.num_layers * self.ple_dim;
let total_rows = batch_dim * self.num_layers;
@@ -150,8 +150,6 @@ impl<B: Backend> PerLayerEmbedding<B> {
command_buffer,
);
- command_buffer.pop_debug_group();
-
Ok(per_layer_inputs)
}
}
@@ -245,7 +243,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.push_debug_group("per layer embedding projection");
+ let command_buffer = &mut *command_buffer.debug_group("per layer embedding projection");
let length = batch_dim * self.model_dim;
@@ -289,8 +287,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
command_buffer,
);
- command_buffer.pop_debug_group();
-
Ok(())
}
}
diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rs
index 351661ee1..0ebb99779 100644
--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs
+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs
@@ -93,7 +93,7 @@ impl<B: Backend> PredictionHead<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("prediction head");
+ let command_buffer = &mut *command_buffer.debug_group("prediction head");
let mut hidden = self.dense_projection.encode(input, batch_dim, command_buffer)?;
self.activation_kernel.encode(
@@ -107,8 +107,6 @@ impl<B: Backend> PredictionHead<B> {
self.normalization.encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;
let logits = self.readout.encode(normalized, batch_dim, command_buffer)?;
- command_buffer.pop_debug_group();
-
Ok(logits)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/sampling/mod.rs b/crates/uzu-engine/src/encodable_block/sampling/mod.rs
index 1142bd587..1d2a1e29a 100644
--- a/crates/uzu-engine/src/encodable_block/sampling/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/sampling/mod.rs
@@ -92,7 +92,7 @@ impl<B: Backend> Sampling<B> {
sampling_range: Range<u32>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::GlobalBuffer, B::Error> {
- command_buffer.push_debug_group("sampling");
+ let command_buffer = &mut *command_buffer.debug_group("sampling");
let sampling_length = sampling_range.end - sampling_range.start;
@@ -194,8 +194,6 @@ impl<B: Backend> Sampling<B> {
command_buffer,
);
- command_buffer.pop_debug_group();
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rs
index 2bb7b8bea..8cdcdb62b 100644
--- a/crates/uzu-engine/src/encodable_block/transformer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer.rs
@@ -60,21 +60,21 @@ impl<B: Backend> TransformerState<B> {
accepted_indices: &[u32],
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.push_debug_group("transformer accept");
+ {
+ let command_buffer = &mut *command_buffer.debug_group("transformer accept");
- for layer_state in &mut self.layer_states {
- let TransformerLayerStateType::Owned(layer_state) = layer_state else {
- continue;
- };
+ for layer_state in &mut self.layer_states {
+ let TransformerLayerStateType::Owned(layer_state) = layer_state else {
+ continue;
+ };
- layer_state.encode_accept(accepted_indices, command_buffer)?;
- }
-
- self.context_length += accepted_indices.len() as u32;
+ layer_state.encode_accept(accepted_indices, command_buffer)?;
+ }
- command_buffer.pop_debug_group();
+ self.context_length += accepted_indices.len() as u32;
+ }
- command_buffer.add_debug_point(&format!("accepted {} tokens", accepted_indices.len()));
+ command_buffer.debug_point(&format!("accepted {} tokens", accepted_indices.len()));
Ok(())
}
diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
index 567a92c2a..654d9be9b 100644
--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
@@ -350,7 +350,7 @@ impl<B: Backend> TransformerLayer<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&format!("transformer layer {}", self.layer_index));
+ let command_buffer = &mut *command_buffer.debug_group(&format!("transformer layer {}", self.layer_index));
let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {
pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?
@@ -418,8 +418,6 @@ impl<B: Backend> TransformerLayer<B> {
command_buffer.encode_fill(&mut hidden, 0);
}
- command_buffer.pop_debug_group();
-
Ok(hidden)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index ac02f6e78..9df512a47 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -295,7 +295,7 @@ impl<B: Backend> Weaver<B> {
depth: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<Vec<B::ScratchBuffer>, WeaverEncodeError<B>> {
- command_buffer.push_debug_group("weaver prefix");
+ let command_buffer = &mut *command_buffer.debug_group("weaver prefix");
let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);
let mut prefix_hidden = command_buffer
@@ -356,8 +356,6 @@ impl<B: Backend> Weaver<B> {
.kv_cache,
);
- command_buffer.pop_debug_group();
-
Ok(prefix_kv_layers)
}
@@ -525,158 +523,161 @@ impl<B: Backend> Weaver<B> {
shape: WeaverTreeShape,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
- command_buffer.push_debug_group("weaver tree");
-
- let tree_slot_count = shape.slot_count();
- let ancestor_stride = self.max_depth;
- if shape.tree_budget == 0
- || shape.rounds == 0
- || shape.max_depth < 2
- || shape.max_depth > self.max_depth + 1
- || shape.dflash_depth < shape.max_depth
- || shape.dflash_depth > self.max_depth + 1
- || shape.expand_per_round == 0
- || shape.expand_per_round > FRONTIER_MAX_WIDTH
- || shape.expand_width == 0
- || shape.expand_width > self.candidate_pool_size
- || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width
- || depth_seeds.len() as u32 != self.max_depth
- || shape.prune_noise_scale.is_some_and(|scale| !(scale > 0.0 && scale.is_finite()))
- {
- return Err(WeaverEncodeError::InvalidTreeInput);
- }
- let frontier_capacity = tree_slot_count * shape.expand_width;
- let pool_depth_count = shape.dflash_depth - 1;
-
- // Rank the draft logits: the top `candidate_pool_size` tokens per
- // lookahead row form the candidate pool node expansions draw from.
- let vocab_size = target_embedding.vocab_size();
- assert!(
- logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DATA_TYPE),
- "draft logits do not cover the lookahead rows"
- );
- let mut candidate_ids = command_buffer
- .allocate_scratch_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::U32)
- .map_err(WeaverEncodeError::Backend)?;
- let mut candidate_logits = command_buffer
- .allocate_scratch_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::F32)
- .map_err(WeaverEncodeError::Backend)?;
- self.top_k
- .encode(
- logits,
- &mut candidate_ids,
- &mut candidate_logits,
- pool_depth_count,
- self.candidate_pool_size,
- command_buffer,
- )
- .map_err(WeaverEncodeError::Backend)?;
+ let (packed_tree, frontier) = {
+ let command_buffer = &mut *command_buffer.debug_group("weaver tree");
+
+ let tree_slot_count = shape.slot_count();
+ let ancestor_stride = self.max_depth;
+ if shape.tree_budget == 0
+ || shape.rounds == 0
+ || shape.max_depth < 2
+ || shape.max_depth > self.max_depth + 1
+ || shape.dflash_depth < shape.max_depth
+ || shape.dflash_depth > self.max_depth + 1
+ || shape.expand_per_round == 0
+ || shape.expand_per_round > FRONTIER_MAX_WIDTH
+ || shape.expand_width == 0
+ || shape.expand_width > self.candidate_pool_size
+ || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width
+ || depth_seeds.len() as u32 != self.max_depth
+ || shape.prune_noise_scale.is_some_and(|scale| !(scale > 0.0 && scale.is_finite()))
+ {
+ return Err(WeaverEncodeError::InvalidTreeInput);
+ }
+ let frontier_capacity = tree_slot_count * shape.expand_width;
+ let pool_depth_count = shape.dflash_depth - 1;
+
+ // Rank the draft logits: the top `candidate_pool_size` tokens per
+ // lookahead row form the candidate pool node expansions draw from.
+ let vocab_size = target_embedding.vocab_size();
+ assert!(
+ logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DATA_TYPE),
+ "draft logits do not cover the lookahead rows"
+ );
+ let mut candidate_ids = command_buffer
+ .allocate_scratch_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::U32)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut candidate_logits = command_buffer
+ .allocate_scratch_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::F32)
+ .map_err(WeaverEncodeError::Backend)?;
+ self.top_k
+ .encode(
+ logits,
+ &mut candidate_ids,
+ &mut candidate_logits,
+ pool_depth_count,
+ self.candidate_pool_size,
+ command_buffer,
+ )
+ .map_err(WeaverEncodeError::Backend)?;
- let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();
- let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, command_buffer)
- .map_err(WeaverEncodeError::Backend)?;
+ let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();
+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, command_buffer)
+ .map_err(WeaverEncodeError::Backend)?;
- let prefix_kv_layers =
- self.encode_prefix(target_hidden, draft_hidden, &rope, shape.dflash_depth, command_buffer)?;
+ let prefix_kv_layers =
+ self.encode_prefix(target_hidden, draft_hidden, &rope, shape.dflash_depth, command_buffer)?;
- // Per-layer KV cache for tree nodes, one slot per packed-tree slot.
- let node_kv_size = size_for_shape(&[2, tree_slot_count, self.model_dim], DATA_TYPE);
- let mut node_kv_layers = (0..self.layers.len())
- .map(|_| command_buffer.allocate_scratch(node_kv_size))
- .collect::<Result<Vec<_>, _>>()
- .map_err(WeaverEncodeError::Backend)?;
+ // Per-layer KV cache for tree nodes, one slot per packed-tree slot.
+ let node_kv_size = size_for_shape(&[2, tree_slot_count, self.model_dim], DATA_TYPE);
+ let mut node_kv_layers = (0..self.layers.len())
+ .map(|_| command_buffer.allocate_scratch(node_kv_size))
+ .collect::<Result<Vec<_>, _>>()
+ .map_err(WeaverEncodeError::Backend)?;
- let root =
- command_buffer.allocate_constant_from_slice(&[root_token_id, 1u32]).map_err(WeaverEncodeError::Backend)?;
- let word_bytes = DataType::U32.size_in_bytes();
- let tree_field_bytes = size_for_shape(&[tree_slot_count], DataType::U32);
- let mut packed_tree =
- command_buffer.allocate_scratch(TreeIdx::COUNT * tree_field_bytes).map_err(WeaverEncodeError::Backend)?;
- command_buffer.encode_fill(&mut packed_tree, 0);
- let parent_offset = TreeIdx::ParentSlot as usize * tree_field_bytes;
- command_buffer.encode_fill(
- packed_tree.subrange_mut(parent_offset..parent_offset + tree_field_bytes),
- 0xff, // FRONTIER_NO_WINNER
- );
- command_buffer.encode_copy(root.subrange(..word_bytes), packed_tree.subrange_mut(..word_bytes));
- let valid_offset = TreeIdx::Valid as usize * tree_field_bytes;
- command_buffer.encode_copy(
- root.subrange(word_bytes..),
- packed_tree.subrange_mut(valid_offset..valid_offset + word_bytes),
- );
+ let root = command_buffer
+ .allocate_constant_from_slice(&[root_token_id, 1u32])
+ .map_err(WeaverEncodeError::Backend)?;
+ let word_bytes = DataType::U32.size_in_bytes();
+ let tree_field_bytes = size_for_shape(&[tree_slot_count], DataType::U32);
+ let mut packed_tree = command_buffer
+ .allocate_scratch(TreeIdx::COUNT * tree_field_bytes)
+ .map_err(WeaverEncodeError::Backend)?;
+ command_buffer.encode_fill(&mut packed_tree, 0);
+ let parent_offset = TreeIdx::ParentSlot as usize * tree_field_bytes;
+ command_buffer.encode_fill(
+ packed_tree.subrange_mut(parent_offset..parent_offset + tree_field_bytes),
+ 0xff, // FRONTIER_NO_WINNER
+ );
+ command_buffer.encode_copy(root.subrange(..word_bytes), packed_tree.subrange_mut(..word_bytes));
+ let valid_offset = TreeIdx::Valid as usize * tree_field_bytes;
+ command_buffer.encode_copy(
+ root.subrange(word_bytes..),
+ packed_tree.subrange_mut(valid_offset..valid_offset + word_bytes),
+ );
- let mut frontier = command_buffer
- .allocate_scratch_for_shape(&[FrontierIdx::COUNT as u32, frontier_capacity], DataType::U32)
- .map_err(WeaverEncodeError::Backend)?;
- command_buffer.encode_fill(&mut frontier, 0);
- let mut slot_ancestors = command_buffer
- .allocate_scratch_for_shape(&[tree_slot_count, ancestor_stride], DataType::U32)
- .map_err(WeaverEncodeError::Backend)?;
+ let mut frontier = command_buffer
+ .allocate_scratch_for_shape(&[FrontierIdx::COUNT as u32, frontier_capacity], DataType::U32)
+ .map_err(WeaverEncodeError::Backend)?;
+ command_buffer.encode_fill(&mut frontier, 0);
+ let mut slot_ancestors = command_buffer
+ .allocate_scratch_for_shape(&[tree_slot_count, ancestor_stride], DataType::U32)
+ .map_err(WeaverEncodeError::Backend)?;
- let mut node_token_ids = command_buffer
- .allocate_scratch_for_shape(&[shape.expand_per_round], DataType::U32)
- .map_err(WeaverEncodeError::Backend)?;
- command_buffer.encode_fill(&mut node_token_ids, 0);
- command_buffer.encode_copy(root.subrange(..word_bytes), node_token_ids.subrange_mut(..word_bytes));
+ let mut node_token_ids = command_buffer
+ .allocate_scratch_for_shape(&[shape.expand_per_round], DataType::U32)
+ .map_err(WeaverEncodeError::Backend)?;
+ command_buffer.encode_fill(&mut node_token_ids, 0);
+ command_buffer.encode_copy(root.subrange(..word_bytes), node_token_ids.subrange_mut(..word_bytes));
- let mut node_metadata = command_buffer
- .allocate_scratch_for_shape(&[MetadataIdx::COUNT as u32, shape.expand_per_round], DataType::U32)
- .map_err(WeaverEncodeError::Backend)?;
- command_buffer.encode_fill(&mut node_metadata, 0);
- let mut node_ancestor_indices = command_buffer
- .allocate_scratch_for_shape(&[shape.expand_per_round, ancestor_stride], DataType::U32)
- .map_err(WeaverEncodeError::Backend)?;
- command_buffer.encode_fill(&mut node_ancestor_indices, 0);
- let mut node_valid = command_buffer
- .allocate_scratch_for_shape(&[shape.expand_per_round], DataType::U32)
- .map_err(WeaverEncodeError::Backend)?;
- command_buffer.encode_fill(&mut node_valid, 0);
- command_buffer.encode_copy(root.subrange(word_bytes..), node_valid.subrange_mut(..word_bytes));
+ let mut node_metadata = command_buffer
+ .allocate_scratch_for_shape(&[MetadataIdx::COUNT as u32, shape.expand_per_round], DataType::U32)
+ .map_err(WeaverEncodeError::Backend)?;
+ command_buffer.encode_fill(&mut node_metadata, 0);
+ let mut node_ancestor_indices = command_buffer
+ .allocate_scratch_for_shape(&[shape.expand_per_round, ancestor_stride], DataType::U32)
+ .map_err(WeaverEncodeError::Backend)?;
+ command_buffer.encode_fill(&mut node_ancestor_indices, 0);
+ let mut node_valid = command_buffer
+ .allocate_scratch_for_shape(&[shape.expand_per_round], DataType::U32)
+ .map_err(WeaverEncodeError::Backend)?;
+ command_buffer.encode_fill(&mut node_valid, 0);
+ command_buffer.encode_copy(root.subrange(word_bytes..), node_valid.subrange_mut(..word_bytes));
- let mut node_candidate_ids = command_buffer
- .allocate_scratch_for_shape(&[shape.expand_per_round, self.candidate_pool_size], DataType::U32)
- .map_err(WeaverEncodeError::Backend)?;
- let mut node_candidate_logits = command_buffer
- .allocate_scratch_for_shape(&[shape.expand_per_round, self.candidate_pool_size], DataType::F32)
- .map_err(WeaverEncodeError::Backend)?;
- let depth_seeds_buffer =
- command_buffer.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;
-
- let mut batch_start_slot = 0;
- for round in 0..shape.rounds {
- let batch_node_count = if round == 0 {
- 1
- } else {
- shape.expand_per_round
- };
- command_buffer.push_debug_group("weaver step");
- self.encode_step(
- target_embedding,
- prefix_kv_layers.iter(),
- &rope,
- node_kv_layers.iter_mut(),
- &mut packed_tree,
- &mut frontier,
- &mut slot_ancestors,
- &mut node_token_ids,
- &mut node_metadata,
- &mut node_ancestor_indices,
- &mut node_valid,
- &mut node_candidate_ids,
- &mut node_candidate_logits,
- &depth_seeds_buffer,
- &candidate_ids,
- &candidate_logits,
- &shape,
- batch_node_count,
- batch_start_slot,
- command_buffer,
- )?;
- command_buffer.pop_debug_group();
- batch_start_slot += batch_node_count;
- }
+ let mut node_candidate_ids = command_buffer
+ .allocate_scratch_for_shape(&[shape.expand_per_round, self.candidate_pool_size], DataType::U32)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut node_candidate_logits = command_buffer
+ .allocate_scratch_for_shape(&[shape.expand_per_round, self.candidate_pool_size], DataType::F32)
+ .map_err(WeaverEncodeError::Backend)?;
+ let depth_seeds_buffer =
+ command_buffer.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;
+
+ let mut batch_start_slot = 0;
+ for round in 0..shape.rounds {
+ let batch_node_count = if round == 0 {
+ 1
+ } else {
+ shape.expand_per_round
+ };
+ let command_buffer = &mut *command_buffer.debug_group("weaver step");
+ self.encode_step(
+ target_embedding,
+ prefix_kv_layers.iter(),
+ &rope,
+ node_kv_layers.iter_mut(),
+ &mut packed_tree,
+ &mut frontier,
+ &mut slot_ancestors,
+ &mut node_token_ids,
+ &mut node_metadata,
+ &mut node_ancestor_indices,
+ &mut node_valid,
+ &mut node_candidate_ids,
+ &mut node_candidate_logits,
+ &depth_seeds_buffer,
+ &candidate_ids,
+ &candidate_logits,
+ &shape,
+ batch_node_count,
+ batch_start_slot,
+ command_buffer,
+ )?;
+ batch_start_slot += batch_node_count;
+ }
- command_buffer.pop_debug_group();
+ (packed_tree, frontier)
+ };
let mut packed_tree_readback =
command_buffer.context().create_buffer(packed_tree.size()).map_err(WeaverEncodeError::Backend)?;
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index cdd1be257..f5a74c487 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -211,7 +211,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.create_command_buffer(Some("prefill"), Some(allocation_pool.clone()))
.map_err(LanguageModelStreamError::Backend)?;
- command_buffer.add_debug_point(&format!(
+ command_buffer.debug_point(&format!(
"prefill at context {}, suffix {} ({} batches)",
model_state.transformer_state.context_length(),
input.len(),
@@ -590,7 +590,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
.map_err(LanguageModelStreamError::Backend)?;
- command_buffer.add_debug_point(&format!(
+ command_buffer.debug_point(&format!(
"decode round {} at context {} suffix {}",
self.metrics.num_decode_forward_passes,
context_length,