A high-performance inference engine for AI models
gpu timestamps recording
eugene8ad7fa4c3b80
diff --git a/Cargo.lock b/Cargo.lockindex eee205fe4..acf8f6327 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3689,9 +3689,9 @@ dependencies = [[[package]]name = "mtl-rs"-version = "0.2.2"+version = "0.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d9f1de2bca596db925db758d497cbf9cb394a2981933319b3777510097e1d360"+checksum = "4aa9291872296e9e734b0caf6e7a933fe6861d25ed86240461d6114355f7dbbf"dependencies = ["bitflags 2.13.0","block2",diff --git a/Cargo.toml b/Cargo.tomlindex 759972957..420503b9d 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -76,7 +76,7 @@ objc2-foundation = "0.3.2"objc2-core-foundation = "0.3.2"objc2-io-kit = { version = "0.3.2", default-features = false, features = ["libc", "ps"] }obfstr = "0.4.4"-metal = { package = "mtl-rs", version = "0.2.2" }+metal = { package = "mtl-rs", version = "0.3.0" }block2 = "0.6"bytemuck = "1.23"bytesize = "2"diff --git a/apps/mirai-gui/desktop/Cargo.lock b/apps/mirai-gui/desktop/Cargo.lockindex a2481a32f..a6849f8a7 100644--- a/apps/mirai-gui/desktop/Cargo.lock+++ b/apps/mirai-gui/desktop/Cargo.lock@@ -3565,9 +3565,9 @@ dependencies = [[[package]]name = "mtl-rs"-version = "0.2.2"+version = "0.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d9f1de2bca596db925db758d497cbf9cb394a2981933319b3777510097e1d360"+checksum = "4aa9291872296e9e734b0caf6e7a933fe6861d25ed86240461d6114355f7dbbf"dependencies = ["bitflags 2.13.0","block2",diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rsindex 0377b6a70..4d8168aa6 100644--- a/crates/uzu-engine/src/backends/common/command_buffer.rs+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs@@ -4,7 +4,7 @@ use bytemuck::{AnyBitPattern, NoUninit};use crate::{array::size_for_shape,- backends::common::{Backend, BufferMut, BufferRef},+ backends::common::{Backend, BufferMut, BufferRef, TimestampSpan},data_type::DataType,};@@ -80,6 +80,15 @@ pub trait CommandBufferEncoding {fn pop_debug_group(&mut self);+ fn enable_timestamps(&mut self);++ fn sample_start_timestamp(+ &mut self,+ name: &str,+ );++ fn sample_end_timestamp(&mut self);+fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;}@@ -104,4 +113,6 @@ pub trait CommandBufferCompleted: Send {type CommandBuffer: CommandBuffer<Completed = Self>;fn gpu_execution_time(&self) -> Duration;++ fn timestamps(&self) -> &[TimestampSpan];}diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rsindex a07c947d6..47663a079 100644--- a/crates/uzu-engine/src/backends/common/mod.rs+++ b/crates/uzu-engine/src/backends/common/mod.rs@@ -6,6 +6,7 @@ mod context;mod device_capabilities;pub mod gpu_types;pub mod kernel;+mod timestamps;pub use backend::Backend;pub use buffer::{@@ -23,3 +24,4 @@ pub use command_buffer::{pub use context::Context;pub use device_capabilities::DeviceCapabilities;pub use kernel::Kernels;+pub use timestamps::{TimestampSpan, TimestampSpanRecorder};diff --git a/crates/uzu-engine/src/backends/common/timestamps/mod.rs b/crates/uzu-engine/src/backends/common/timestamps/mod.rsnew file mode 100644index 000000000..4d2eca4b4--- /dev/null+++ b/crates/uzu-engine/src/backends/common/timestamps/mod.rs@@ -0,0 +1,6 @@+mod span_boundary;+mod timestamp_span;+mod timestamp_span_recorder;++pub use timestamp_span::TimestampSpan;+pub use timestamp_span_recorder::TimestampSpanRecorder;diff --git a/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs b/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rsnew file mode 100644index 000000000..a26a37153--- /dev/null+++ b/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs@@ -0,0 +1,4 @@+pub enum SpanBoundary {+ Start(String),+ End,+}diff --git a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rsnew file mode 100644index 000000000..f4315085f--- /dev/null+++ b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs@@ -0,0 +1,8 @@+use std::time::Instant;++#[derive(Debug, Clone)]+pub struct TimestampSpan {+ pub name: String,+ pub start: Instant,+ pub end: Instant,+}diff --git a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rsnew file mode 100644index 000000000..b1c6ce04f--- /dev/null+++ b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs@@ -0,0 +1,59 @@+use std::time::Instant;++use super::{TimestampSpan, span_boundary::SpanBoundary};++#[derive(Default)]+pub struct TimestampSpanRecorder {+ boundaries: Vec<SpanBoundary>,+}++impl TimestampSpanRecorder {+ pub fn start(+ &mut self,+ name: String,+ ) -> usize {+ self.push(SpanBoundary::Start(name))+ }++ pub fn end(&mut self) -> usize {+ self.push(SpanBoundary::End)+ }++ fn push(+ &mut self,+ boundary: SpanBoundary,+ ) -> usize {+ self.boundaries.push(boundary);+ self.boundaries.len() - 1+ }++ pub fn slot_count(&self) -> usize {+ self.boundaries.len()+ }++ pub fn spans(+ &self,+ instants: &[Instant],+ ) -> Box<[TimestampSpan]> {+ let mut spans = Vec::new();+ let mut open_span_indices = Vec::new();+ for (boundary, &instant) in self.boundaries.iter().zip(instants) {+ match boundary {+ SpanBoundary::Start(name) => {+ open_span_indices.push(spans.len());+ spans.push(TimestampSpan {+ name: name.clone(),+ start: instant,+ end: instant,+ });+ },+ SpanBoundary::End => {+ let index = open_span_indices.pop().expect("timestamp ended without a start");+ spans[index].end = instant;+ },+ }+ }+ assert!(open_span_indices.is_empty(), "timestamps were started but never ended");+ spans.into_boxed_slice()+ }+}diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rsindex 62ba458aa..78722176e 100644--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs@@ -7,12 +7,14 @@ use crate::{backends::{common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,- CommandBufferExecutable, CommandBufferPending, allocator::bump::BumpAllocator,+ CommandBufferExecutable, CommandBufferPending, TimestampSpan, TimestampSpanRecorder,+ allocator::bump::BumpAllocator,},cpu::{Cpu,buffer::{CpuBufferExt, dense::CpuBuffer},context::CpuContext,+ cpu_command::CpuCommand,error::CpuError,},},@@ -31,10 +33,11 @@ impl CommandBuffer for CpuCommandBuffer {}pub struct CpuCommandBufferEncoding {- commands: Vec<Box<dyn FnOnce() + Send>>,+ commands: Vec<CpuCommand>,constant_allocator: BumpAllocator<<Cpu as Backend>::GlobalBuffer>,allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,context: Arc<CpuContext>,+ timestamp_spans: Option<TimestampSpanRecorder>,}impl CpuCommandBufferEncoding {@@ -48,6 +51,7 @@ impl CpuCommandBufferEncoding {constant_allocator,allocation_pool,context,+ timestamp_spans: None,}}@@ -55,7 +59,11 @@ impl CpuCommandBufferEncoding {&mut self,command: impl FnOnce() + Send + 'static,) {- self.commands.push(Box::new(command))+ self.commands.push(CpuCommand::Run(Box::new(command)))+ }++ fn write_timestamp(&mut self) {+ self.commands.push(CpuCommand::Timestamp);}}@@ -118,6 +126,28 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {fn pop_debug_group(&mut self) {}+ fn enable_timestamps(&mut self) {+ assert!(self.timestamp_spans.is_none(), "timestamps already enabled");+ self.timestamp_spans = Some(TimestampSpanRecorder::default());+ }++ fn sample_start_timestamp(+ &mut self,+ name: &str,+ ) {+ if let Some(spans) = &mut self.timestamp_spans {+ spans.start(name.to_owned());+ self.write_timestamp();+ }+ }++ fn sample_end_timestamp(&mut self) {+ if let Some(spans) = &mut self.timestamp_spans {+ spans.end();+ self.write_timestamp();+ }+ }+fn end_encoding(self) -> CpuCommandBufferExecutable {assert!(self.constant_allocator.is_done(), "attempted to end encoding while constants are still alive");CpuCommandBufferExecutable {@@ -125,15 +155,17 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {constant_allocator: self.constant_allocator,allocation_pool: self.allocation_pool,context: self.context,+ timestamp_spans: self.timestamp_spans,}}}pub struct CpuCommandBufferExecutable {- commands: Vec<Box<dyn FnOnce() + Send>>,+ commands: Vec<CpuCommand>,constant_allocator: BumpAllocator<<Cpu as Backend>::GlobalBuffer>,allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,context: Arc<CpuContext>,+ timestamp_spans: Option<TimestampSpanRecorder>,}impl CommandBufferExecutable for CpuCommandBufferExecutable {@@ -148,14 +180,23 @@ impl CommandBufferExecutable for CpuCommandBufferExecutable {.send(Box::new(move || {let start = Instant::now();- for command in self.commands {- command()- }+ let instants = self+ .commands+ .into_iter()+ .filter_map(|command| match command {+ CpuCommand::Run(run) => {+ run();+ None+ },+ CpuCommand::Timestamp => Some(Instant::now()),+ })+ .collect::<Box<[Instant]>>();let gpu_execution_time = start.elapsed();let completed = CpuCommandBufferCompleted {gpu_execution_time,+ timestamps: self.timestamp_spans.map_or_else(Box::default, |spans| spans.spans(&instants)),_allocation_pool: self.allocation_pool,};@@ -186,6 +227,7 @@ impl CommandBufferPending for CpuCommandBufferPending {pub struct CpuCommandBufferCompleted {gpu_execution_time: Duration,+ timestamps: Box<[TimestampSpan]>,_allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,}@@ -195,4 +237,8 @@ impl CommandBufferCompleted for CpuCommandBufferCompleted {fn gpu_execution_time(&self) -> Duration {self.gpu_execution_time}++ fn timestamps(&self) -> &[TimestampSpan] {+ &self.timestamps+ }}diff --git a/crates/uzu-engine/src/backends/cpu/cpu_command.rs b/crates/uzu-engine/src/backends/cpu/cpu_command.rsnew file mode 100644index 000000000..6dd851e4c--- /dev/null+++ b/crates/uzu-engine/src/backends/cpu/cpu_command.rs@@ -0,0 +1,4 @@+pub enum CpuCommand {+ Run(Box<dyn FnOnce() + Send>),+ Timestamp,+}diff --git a/crates/uzu-engine/src/backends/cpu/mod.rs b/crates/uzu-engine/src/backends/cpu/mod.rsindex f569a2da4..7d6afffc5 100644--- a/crates/uzu-engine/src/backends/cpu/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/mod.rs@@ -2,6 +2,7 @@ mod backend;mod buffer;mod command_buffer;mod context;+mod cpu_command;mod error;pub(crate) mod kernel; // TODO: This should not be pub!!!diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rsindex 985a6d0e5..a584b4cb0 100644--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs@@ -16,9 +16,11 @@ use rangemap::RangeSet;use crate::backends::{common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,- CommandBufferExecutable, CommandBufferPending, Context, allocator::bump::BumpAllocator,+ CommandBufferExecutable, CommandBufferPending, Context, TimestampSpan, allocator::bump::BumpAllocator,+ },+ metal::{+ Metal, MetalContext, buffer::MetalBufferExt, error::MetalError, timestamp_recorder::MetalTimestampRecorder,},- metal::{Metal, MetalContext, buffer::MetalBufferExt, error::MetalError},};pub struct MetalCommandBuffer;@@ -47,6 +49,7 @@ pub struct MetalCommandBufferEncoding {constant_allocator: Option<BumpAllocator<<Metal as Backend>::GlobalBuffer>>,allocation_pool: Arc<<Metal as Backend>::AllocationPool>,pub(super) context: Arc<MetalContext>,+ timestamps: Option<MetalTimestampRecorder>,}impl MetalCommandBufferEncoding {@@ -101,6 +104,7 @@ impl MetalCommandBufferEncoding {constant_allocator: Some(constant_allocator),allocation_pool,context,+ timestamps: None,})}@@ -213,6 +217,26 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {self.compute_encoder.pop_debug_group();}+ fn enable_timestamps(&mut self) {+ assert!(self.timestamps.is_none(), "timestamps already enabled");+ self.timestamps = Some(MetalTimestampRecorder::new(self.context.device.clone()));+ }++ fn sample_start_timestamp(+ &mut self,+ name: &str,+ ) {+ if let Some(timestamps) = &mut self.timestamps {+ timestamps.start(name.to_owned(), &self.compute_encoder);+ }+ }++ fn sample_end_timestamp(&mut self) {+ if let Some(timestamps) = &mut self.timestamps {+ timestamps.end(&self.compute_encoder);+ }+ }+fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {let constant_allocator = self.constant_allocator.take().unwrap();assert!(constant_allocator.is_done(), "attempted to end encoding while constants are still alive");@@ -223,6 +247,7 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {constant_allocator,allocation_pool: self.allocation_pool.clone(),context: self.context.clone(),+ timestamps: self.timestamps.take(),}}}@@ -245,6 +270,7 @@ pub struct MetalCommandBufferExecutable {constant_allocator: BumpAllocator<<Metal as Backend>::GlobalBuffer>,allocation_pool: Arc<<Metal as Backend>::AllocationPool>,context: Arc<MetalContext>,+ timestamps: Option<MetalTimestampRecorder>,}impl CommandBufferExecutable for MetalCommandBufferExecutable {@@ -266,14 +292,18 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {let command_allocator = self.command_allocator.clone();let command_buffer = self.command_buffer.clone();let context_clone = self.context.clone();+ let timestamps = self.timestamps;let constant_allocator = self.constant_allocator;let allocation_pool = self.allocation_pool.clone();let feedback_handler = move |feedback: &ProtocolObject<dyn MTL4CommitFeedback>| {let message = if let Some(error) = feedback.error() {- Err(error.to_string())+ Err(MetalError::CommandBufferExecution(error.to_string()))} else {- Ok(Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()))+ timestamps+ .as_ref()+ .map_or_else(|| Ok(Box::default()), MetalTimestampRecorder::resolve)+ .map(|spans| (Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()), spans))};let _ = sender.send(message);context_clone.command_buffer_cache.lock().push(MetalCommandBufferCache {@@ -296,19 +326,17 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {pub struct MetalCommandBufferPending {allocation_pool: Arc<<Metal as Backend>::AllocationPool>,- receiver: mpsc::Receiver<Result<Duration, String>>,+ receiver: mpsc::Receiver<Result<(Duration, Box<[TimestampSpan]>), MetalError>>,}impl CommandBufferPending for MetalCommandBufferPending {type CommandBuffer = MetalCommandBuffer;fn wait_until_completed(self) -> Result<MetalCommandBufferCompleted, MetalError> {+ let (gpu_execution_time, timestamps) = self.receiver.recv().map_err(MetalError::CommandBufferWait)??;Ok(MetalCommandBufferCompleted {- gpu_execution_time: self- .receiver- .recv()- .map_err(MetalError::CommandBufferWait)?- .map_err(MetalError::CommandBufferExecution)?,+ gpu_execution_time,+ timestamps,_allocation_pool: self.allocation_pool,})}@@ -316,6 +344,7 @@ impl CommandBufferPending for MetalCommandBufferPending {pub struct MetalCommandBufferCompleted {gpu_execution_time: Duration,+ timestamps: Box<[TimestampSpan]>,_allocation_pool: Arc<<Metal as Backend>::AllocationPool>,}@@ -325,6 +354,10 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {fn gpu_execution_time(&self) -> Duration {self.gpu_execution_time}++ fn timestamps(&self) -> &[TimestampSpan] {+ &self.timestamps+ }}pub struct MetalCommandBufferCache {diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rsindex e51351964..83b5033e6 100644--- a/crates/uzu-engine/src/backends/metal/error.rs+++ b/crates/uzu-engine/src/backends/metal/error.rs@@ -27,6 +27,10 @@ pub enum MetalError {CannotCreateBuffer,#[error("Cannot create heap")]CannotCreateHeap,+ #[error("Cannot resolve counter heap")]+ CannotResolveCounterHeap,+ #[error("Timestamp {0} was not written by the GPU")]+ UnwrittenTimestamp(usize),#[error("Cannot create command buffer")]CannotCreateCommandBuffer,#[error("Cannot create argument table: {0}")]diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rsindex 01324c58f..5fc8da6ee 100644--- a/crates/uzu-engine/src/backends/metal/mod.rs+++ b/crates/uzu-engine/src/backends/metal/mod.rs@@ -7,6 +7,7 @@ mod error;mod heaps;mod kernel;mod metal_extensions;+mod timestamp_recorder;pub use backend::Metal;pub use context::MetalContext; // TODO: This should be removeddiff --git a/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rsnew file mode 100644index 000000000..919245b51--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs@@ -0,0 +1,102 @@+use std::time::{Duration, Instant};++use metal::{+ MTL4ComputeCommandEncoder, MTL4CounterHeap, MTL4CounterHeapDescriptor, MTL4CounterHeapExt, MTL4CounterHeapType,+ MTL4TimestampGranularity, MTLDevice, MTLDeviceExt,+};+use objc2::{rc::Retained, runtime::ProtocolObject};++use crate::backends::{+ common::{TimestampSpan, TimestampSpanRecorder},+ metal::error::MetalError,+};++// "Maximum counter sample buffer length" is 32 KB, i.e. 4096 timestamp entries of 8 bytes (`MTL4TimestampHeapEntry`):+// https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf+const COUNTER_HEAP_CAPACITY: usize = 4096;++pub struct MetalTimestampRecorder {+ device: Retained<ProtocolObject<dyn MTLDevice>>,+ spans: TimestampSpanRecorder,+ counter_heaps: Vec<Retained<ProtocolObject<dyn MTL4CounterHeap>>>,+}++impl MetalTimestampRecorder {+ pub fn new(device: Retained<ProtocolObject<dyn MTLDevice>>) -> Self {+ Self {+ device,+ spans: TimestampSpanRecorder::default(),+ counter_heaps: Vec::new(),+ }+ }++ pub fn start(+ &mut self,+ name: String,+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,+ ) {+ let slot = self.spans.start(name);+ self.write(slot, encoder);+ }++ pub fn end(+ &mut self,+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,+ ) {+ let slot = self.spans.end();+ self.write(slot, encoder);+ }++ fn write(+ &mut self,+ slot: usize,+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,+ ) {+ if slot / COUNTER_HEAP_CAPACITY == self.counter_heaps.len() {+ let descriptor = MTL4CounterHeapDescriptor::new();+ descriptor.set_type(MTL4CounterHeapType::TIMESTAMP);+ descriptor.set_count(COUNTER_HEAP_CAPACITY);+ let counter_heap =+ self.device.new_counter_heap_with_descriptor(&descriptor).expect("Failed to create a counter heap");+ counter_heap.invalidate_counter_range(0..COUNTER_HEAP_CAPACITY);+ self.counter_heaps.push(counter_heap);+ }+ encoder.write_timestamp_with_granularity_into_heap_at_index(+ MTL4TimestampGranularity::PRECISE,+ &self.counter_heaps[slot / COUNTER_HEAP_CAPACITY],+ slot % COUNTER_HEAP_CAPACITY,+ );+ }++ pub fn resolve(&self) -> Result<Box<[TimestampSpan]>, MetalError> {+ let count = self.spans.slot_count();+ let mut ticks = Vec::with_capacity(count);+ for (index, counter_heap) in self.counter_heaps.iter().enumerate() {+ let slots = 0..(count - index * COUNTER_HEAP_CAPACITY).min(COUNTER_HEAP_CAPACITY);+ let bytes = counter_heap.resolve_counter_range(slots).ok_or(MetalError::CannotResolveCounterHeap)?;+ let (entries, _) = bytes.as_chunks::<{ size_of::<u64>() }>();+ ticks.extend(entries.iter().map(|entry| u64::from_ne_bytes(*entry)));+ }+ let frequency = self.device.query_timestamp_frequency() as u128;+ let (mut cpu_now, mut gpu_now) = (0, 0);+ let now = Instant::now();+ self.device.sample_timestamps_gpu_timestamp(&mut cpu_now, &mut gpu_now);+ let to_instant = |ticks: u64| {+ let nanoseconds = (ticks as u128 * 1_000_000_000 / frequency) as u64;+ now - Duration::from_nanos(gpu_now - nanoseconds)+ };+ let instants = ticks+ .iter()+ .enumerate()+ .scan(None, |previous_timestamp, (slot, &slot_ticks)| {+ let timestamp = match slot_ticks {+ 0 => previous_timestamp.ok_or(MetalError::UnwrittenTimestamp(slot)),+ slot_ticks => Ok(to_instant(slot_ticks)),+ };+ *previous_timestamp = timestamp.as_ref().ok().copied();+ Some(timestamp)+ })+ .collect::<Result<Box<[Instant]>, MetalError>>()?;+ Ok(self.spans.spans(&instants))+ }+}diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rsindex f44077a95..d343d583e 100644--- a/crates/uzu-engine/src/encodable_block/classifier.rs+++ b/crates/uzu-engine/src/encodable_block/classifier.rs@@ -35,6 +35,7 @@ pub enum ClassifierError<B: Backend> {}pub struct Classifier<B: Backend> {+ name: String,hidden_dim: u32,data_type: DataType,embedding: Embedding<B>,@@ -46,12 +47,14 @@ pub struct Classifier<B: Backend> {impl<B: Backend> Classifier<B> {pub fn new(+ name: String,context: &B::Context,config: &ClassifierConfig,parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Self, ClassifierError<B>> {let (embedding, _) = Embedding::new(+ format!("{name}/embedding"),context,config.vocab_size,config.transformer_config.model_dim,@@ -61,6 +64,7 @@ impl<B: Backend> Classifier<B> {)?;let embedding_norm = Normalization::new(+ format!("{name}/embedding norm"),config.transformer_config.model_dim,None,ShortcutMode::None,@@ -72,6 +76,7 @@ impl<B: Backend> Classifier<B> {)?;let transformer = Transformer::new(+ format!("{name}/transformer"),context,None,data_type,@@ -89,6 +94,7 @@ impl<B: Backend> Classifier<B> {<B::Kernels as Kernels>::PoolingMeanKernel::new(context, data_type).map_err(ClassifierError::Backend)?;let prediction_head = PredictionHead::new(+ format!("{name}/prediction head"),config.hidden_dim,config.num_labels,data_type,@@ -98,6 +104,7 @@ impl<B: Backend> Classifier<B> {)?;Ok(Self {+ name,hidden_dim: config.hidden_dim,data_type,embedding,@@ -118,7 +125,8 @@ impl<B: Backend> Classifier<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, ClassifierError<B>> {- command_buffer.push_debug_group("classifier");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let embedded = self.embedding.encode_lookup(token_ids, batch_dim, command_buffer)?;@@ -156,6 +164,7 @@ impl<B: Backend> Classifier<B> {let logits = self.prediction_head.encode(pooled, 1, command_buffer).map_err(ClassifierError::Backend)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(logits)diff --git a/crates/uzu-engine/src/encodable_block/context_ring_update.rs b/crates/uzu-engine/src/encodable_block/context_ring_update.rsnew file mode 100644index 000000000..1573b04b7--- /dev/null+++ b/crates/uzu-engine/src/encodable_block/context_ring_update.rs@@ -0,0 +1,35 @@+use crate::backends::common::{+ Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, kernel::ContextRingUpdateKernel,+};++pub struct ContextRingUpdate<B: Backend> {+ name: String,+ kernel: <B::Kernels as Kernels>::ContextRingUpdateKernel,+}++impl<B: Backend> ContextRingUpdate<B> {+ pub fn new(+ name: String,+ context: &B::Context,+ ) -> Result<Self, B::Error> {+ Ok(Self {+ name,+ kernel: <B::Kernels as Kernels>::ContextRingUpdateKernel::new(context)?,+ })+ }++ pub fn encode(+ &self,+ input: impl BufferRef<Backend = B>,+ context_ring: impl BufferMut<Backend = B>,+ suffix_repetition_length: u32,+ input_length: u32,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) {+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);+ self.kernel.encode(input, context_ring, suffix_repetition_length, input_length, command_buffer);+ command_buffer.sample_end_timestamp();+ command_buffer.pop_debug_group();+ }+}diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rsindex c6c852b6a..1f9afb10e 100644--- a/crates/uzu-engine/src/encodable_block/convolution.rs+++ b/crates/uzu-engine/src/encodable_block/convolution.rs@@ -21,6 +21,7 @@ pub enum ConvolutionNewError<B: Backend> {}pub struct SeparableCausalConv<B: Backend> {+ name: String,model_dim: u32,data_type: DataType,weights: B::GlobalBuffer,@@ -30,6 +31,7 @@ pub struct SeparableCausalConv<B: Backend> {impl<B: Backend> SeparableCausalConv<B> {pub fn new(+ name: String,model_dim: u32,kernel_size: u32,group_size: u32,@@ -62,6 +64,7 @@ impl<B: Backend> SeparableCausalConv<B> {.map_err(ConvolutionNewError::Backend)?;Ok(Self {+ name,model_dim,data_type,weights,@@ -79,7 +82,8 @@ impl<B: Backend> SeparableCausalConv<B> {sequence_length: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("SeparableCausalConv");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let mut output =command_buffer.allocate_scratch_for_shape(&[sequence_length, self.model_dim], self.data_type)?;@@ -95,6 +99,7 @@ impl<B: Backend> SeparableCausalConv<B> {command_buffer,);+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)}diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rsindex 7b1c8bbfd..5476b386a 100644--- a/crates/uzu-engine/src/encodable_block/decoder.rs+++ b/crates/uzu-engine/src/encodable_block/decoder.rs@@ -32,6 +32,7 @@ pub enum DecoderError<B: Backend> {}pub struct Decoder<B: Backend> {+ name: String,embedding: Embedding<B>,embedding_norm: Option<Normalization<B>>,per_layer_embedding: Option<PerLayerEmbedding<B>>,@@ -50,12 +51,14 @@ impl<B: Backend> Decoder<B> {}pub fn new(+ name: String,context: &B::Context,config: &DecoderConfig,parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Self, DecoderError<B>> {let (embedding, readout_input_hadamard_factors) = Embedding::new(+ format!("{name}/embedding"),context,config.vocab_size,config.transformer_config.model_dim,@@ -69,6 +72,7 @@ impl<B: Backend> Decoder<B> {.as_ref().map(|norm_config| {Normalization::new(+ format!("{name}/embedding norm"),config.transformer_config.model_dim,None,ShortcutMode::None,@@ -88,6 +92,7 @@ impl<B: Backend> Decoder<B> {"per-layer embedding num_layers must match transformer layer count");Some(PerLayerEmbedding::new(+ format!("{name}/per layer embedding"),context,ple_config,config.transformer_config.model_dim,@@ -99,6 +104,7 @@ impl<B: Backend> Decoder<B> {};let transformer = Transformer::new(+ format!("{name}/transformer"),context,readout_input_hadamard_factors,data_type,@@ -107,6 +113,7 @@ impl<B: Backend> Decoder<B> {)?;Ok(Self {+ name,embedding,embedding_norm,per_layer_embedding,@@ -143,7 +150,8 @@ impl<B: Backend> Decoder<B> {state: &mut TransformerState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {- command_buffer.push_debug_group("decoder");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), command_buffer)?;let embedded = if let Some(embedding_norm) = &self.embedding_norm {@@ -196,6 +204,7 @@ impl<B: Backend> Decoder<B> {transformer_output.output};+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(DecoderEncodeOutput {diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rsindex b35383899..4c90073fa 100644--- a/crates/uzu-engine/src/encodable_block/dflash.rs+++ b/crates/uzu-engine/src/encodable_block/dflash.rs@@ -41,6 +41,8 @@ impl<B: Backend> DFlashState<B> {}pub struct DFlash<B: Backend> {+ accept_name: String,+ draft_name: String,target_feature_projection: Box<dyn Linear<B>>,projected_feature_norm: Normalization<B>,state_kv_projection: Box<dyn Linear<B>>,@@ -89,6 +91,7 @@ pub enum DFlashEncodeError<B: Backend> {impl<B: Backend> DFlash<B> {pub fn new(+ name: String,context: &B::Context,config: &DFlashDraftConfig,parameter_tree: &ParameterTree<B>,@@ -97,6 +100,7 @@ impl<B: Backend> DFlash<B> {let mask_token_id = config.mask_token_id as u32;assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");let target_feature_projection = <dyn Linear<B>>::new(+ format!("{name}/target feature projection"),config.model_dim * config.target_layer_ids.len() as u32,[config.model_dim],false,@@ -105,6 +109,7 @@ impl<B: Backend> DFlash<B> {¶meter_tree.subtree("context_projection"),)?;let projected_feature_norm = Normalization::new(+ format!("{name}/projected feature norm"),config.model_dim,None,ShortcutMode::None,@@ -122,6 +127,7 @@ impl<B: Backend> DFlash<B> {let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;let num_layers = config.layer_configs.len() as u32;let state_kv_projection = <dyn Linear<B>>::new(+ format!("{name}/state kv projection"),config.model_dim,[num_layers * layer_kv_dim],false,@@ -135,6 +141,7 @@ impl<B: Backend> DFlash<B> {.zip(0u32..).map(|(layer_config, index)| {TransformerLayer::new(+ format!("{name}/layer {index}"),context,config.model_dim,config.hidden_dim,@@ -147,6 +154,7 @@ impl<B: Backend> DFlash<B> {}).collect::<Result<Box<[_]>, TransformerLayerError<B>>>()?;let output_norm = Normalization::new(+ format!("{name}/output norm"),config.model_dim,None,ShortcutMode::Add,@@ -158,6 +166,8 @@ impl<B: Backend> DFlash<B> {)?;Ok(Self {+ accept_name: format!("{name}/accept"),+ draft_name: format!("{name}/draft"),target_feature_projection,projected_feature_norm,state_kv_projection,@@ -207,7 +217,8 @@ impl<B: Backend> DFlash<B> {return Ok(());}- command_buffer.push_debug_group("dflash accept");+ command_buffer.push_debug_group(&self.accept_name);+ command_buffer.sample_start_timestamp(&self.accept_name);let num_tokens = accepted_indices.len() as u32;let captured_layer_count = self.target_feature_input_dim / self.model_dim;@@ -274,6 +285,7 @@ impl<B: Backend> DFlash<B> {state.context_length += num_tokens;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(())@@ -287,7 +299,8 @@ impl<B: Backend> DFlash<B> {batch_size: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {- command_buffer.push_debug_group("dflash draft");+ command_buffer.push_debug_group(&self.draft_name);+ command_buffer.sample_start_timestamp(&self.draft_name);assert!(batch_size >= 2 && batch_size <= self.block_size, "batch size exceeds DFlash block size");assert!(@@ -348,6 +361,7 @@ impl<B: Backend> DFlash<B> {command_buffer,)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(DFlashOutput {diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex fd218c54a..8f386db84 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -45,6 +45,7 @@ enum EmbeddingTying<B: Backend> {}pub struct Embedding<B: Backend> {+ readout_name: String,tying: EmbeddingTying<B>,input_scale: f32,data_type: DataType,@@ -69,6 +70,7 @@ impl<B: Backend> Embedding<B> {}pub fn new(+ name: String,context: &B::Context,vocab_size: u32,model_dim: u32,@@ -79,7 +81,14 @@ impl<B: Backend> Embedding<B> {let (tying, readout_input_hadamard_factors) = match config {AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {let embedding_tree = parameter_tree.subtree("embedding");- let table = EmbeddingTable::load(context, &embedding_tree, vocab_size, model_dim, data_type)?;+ let table = EmbeddingTable::load(+ format!("{name}/lookup"),+ context,+ &embedding_tree,+ vocab_size,+ model_dim,+ data_type,+ )?;if table.as_matrix().is_none() {return Err(EmbeddingError::UnsupportedConfiguration("tied embeddings need a matrix table".into()));}@@ -100,11 +109,18 @@ impl<B: Backend> Embedding<B> {},AnyEmbeddingConfig::UntiedEmbeddingConfig(_) => {let input_embedding_tree = parameter_tree.subtree("input_embedding");- let input_table =- EmbeddingTable::load(context, &input_embedding_tree, vocab_size, model_dim, data_type)?;+ let input_table = EmbeddingTable::load(+ format!("{name}/lookup"),+ context,+ &input_embedding_tree,+ vocab_size,+ model_dim,+ data_type,+ )?;let output_embedding_tree = parameter_tree.subtree("output_embedding");let output_embedding_spec = output_embedding_tree.metadata("spec")?;let output = UntiedReadout::load(+ format!("{name}/readout/untied"),context,&output_embedding_tree,output_embedding_spec,@@ -140,6 +156,7 @@ impl<B: Backend> Embedding<B> {Ok((Self {+ readout_name: format!("{name}/readout"),tying,input_scale,data_type,@@ -157,8 +174,6 @@ impl<B: Backend> Embedding<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, EmbeddingError<B>> {- command_buffer.push_debug_group("embedding lookup");-let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type).map_err(EmbeddingError::BackendError)?;@@ -175,8 +190,6 @@ impl<B: Backend> Embedding<B> {};table.encode_lookup(token_ids, &mut output, batch_dim, self.input_scale, command_buffer);- command_buffer.pop_debug_group();-Ok(output)}@@ -189,7 +202,8 @@ impl<B: Backend> Embedding<B> {apply_logit_transform: bool,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, EmbeddingError<B>> {- command_buffer.push_debug_group("embedding readout");+ command_buffer.push_debug_group(&self.readout_name);+ command_buffer.sample_start_timestamp(&self.readout_name);assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");let mut output_buffer = match &self.tying {@@ -240,6 +254,7 @@ impl<B: Backend> Embedding<B> {);}+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output_buffer)diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rsindex 7dae6806a..dee8a1974 100644--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs@@ -3,7 +3,7 @@ use thiserror::Error;use crate::{backends::common::{- Backend, BufferMut, BufferRef, CommandBuffer, Kernels,+ Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,gpu_types::{EmbeddingTableKind, HADAMARD_TRANSFORM_BLOCK_SIZE, d4s4},kernel::InputEmbeddingLookupKernel,},@@ -81,6 +81,7 @@ impl<B: Backend> Storage<B> {}pub struct EmbeddingTable<B: Backend> {+ name: String,storage: Storage<B>,output_hadamard_factors: Option<B::GlobalBuffer>,lookup: LookupKernel<B>,@@ -90,6 +91,7 @@ pub struct EmbeddingTable<B: Backend> {impl<B: Backend> EmbeddingTable<B> {pub fn load(+ name: String,context: &B::Context,tree: &ParameterTree<B>,vocab_size: u32,@@ -150,6 +152,7 @@ impl<B: Backend> EmbeddingTable<B> {.map_err(EmbeddingTableError::BackendError)?;Ok(Self {+ name,storage,output_hadamard_factors,lookup,@@ -175,6 +178,8 @@ impl<B: Backend> EmbeddingTable<B> {scale: f32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) {+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let bindings = self.storage.lookup_bindings();self.lookup.encode(token_ids,@@ -193,6 +198,8 @@ impl<B: Backend> EmbeddingTable<B> {scale,command_buffer,);+ command_buffer.sample_end_timestamp();+ command_buffer.pop_debug_group();}}diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rsindex 2c82b4d93..4db7675fe 100644--- a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs@@ -10,6 +10,7 @@ use crate::{};pub(super) struct InputRht<B: Backend> {+ name: String,rht_signs: B::GlobalBuffer,rht: ActivationTransform<B>,quantizer: Option<ActivationTransform<B>>,@@ -17,6 +18,7 @@ pub(super) struct InputRht<B: Backend> {impl<B: Backend> InputRht<B> {pub(super) fn new(+ name: String,context: &B::Context,data_type: DataType,preparation: LinearInputPreparation<B>,@@ -32,6 +34,7 @@ impl<B: Backend> InputRht<B> {.transpose()?;Ok(Self {+ name,rht_signs,rht,quantizer,@@ -45,6 +48,7 @@ impl<B: Backend> InputRht<B> {format: ActivationFormat,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<LinearInput<B>, B::Error> {+ command_buffer.sample_start_timestamp(&self.name);if format == ActivationFormat::Int8&& let Some(quantizer) = &self.quantizer{@@ -70,6 +74,7 @@ impl<B: Backend> InputRht<B> {command_buffer,);+ command_buffer.sample_end_timestamp();return Ok(LinearInput::Int8Symmetric {values,scales,@@ -82,6 +87,7 @@ impl<B: Backend> InputRht<B> {let input_dim = self.input_dim();let mut transformed = command_buffer.allocate_scratch(input.size())?;self.rht.encode_fp(input, &mut transformed, &self.rht_signs, batch_dim, input_dim, command_buffer);+ command_buffer.sample_end_timestamp();Ok(LinearInput::FullPrecision(transformed))}@@ -96,8 +102,10 @@ impl<B: Backend> InputRht<B> {return self.prepare(&input, batch_dim, format, command_buffer);}+ command_buffer.sample_start_timestamp(&self.name);let input_dim = self.input_dim();self.rht.encode_fp_in_place(&mut input, &self.rht_signs, None, batch_dim, input_dim, command_buffer);+ command_buffer.sample_end_timestamp();Ok(LinearInput::FullPrecision(input))}diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rsindex 4f7f6a9ad..da0ff8c77 100644--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs@@ -37,6 +37,7 @@ pub enum LinearMatmulError<B: Backend> {}pub struct LinearMatmul<B: Backend> {+ name: String,kernel: Mutex<<B::Kernels as Kernels>::MatmulKernel>,matrix: WeightMatrix<B>,biases: Option<B::GlobalBuffer>,@@ -66,6 +67,7 @@ impl<B: Backend> LinearMatmul<B> {/// Loads a linear over any parsed spec — full precision, MLX or Int. Hybrid/// compositions live in the wrappers, not here.pub fn load(+ name: String,context: &B::Context,spec: AnyWeightMatrixSpec,input_dim: u32,@@ -98,6 +100,7 @@ impl<B: Backend> LinearMatmul<B> {.map_err(LinearMatmulError::BackendError)?;Ok(Self {+ name,kernel: Mutex::new(kernel),matrix,biases,@@ -125,6 +128,7 @@ impl<B: Backend> LinearMatmul<B> {gather: Option<Gather<impl BufferRef<Backend = B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.name);let (output_dim, gather_indices) =gather.map_or((self.output_dim, None), |gather| (gather.output_dim, Some(gather.indices)));let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, output_dim], self.output_data_type)?;@@ -143,6 +147,7 @@ impl<B: Backend> LinearMatmul<B> {command_buffer,)?;+ command_buffer.sample_end_timestamp();Ok(output)}@@ -190,7 +195,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("matmul");+ command_buffer.push_debug_group(&self.name);let output = self.encode_with_a(MatmulA::FullPrecision {diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rsindex 78dfdfb93..9c49aae44 100644--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs@@ -123,6 +123,7 @@ pub enum LinearBlockError<B: Backend> {impl<B: Backend> dyn Linear<B> {pub fn new_mixed_precision(+ name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -140,6 +141,7 @@ impl<B: Backend> dyn Linear<B> {| AnyWeightMatrixSpec::MLXSpec(_)| AnyWeightMatrixSpec::IntSpec(_)) => {let block = LinearMatmul::load(+ name,context,spec,input_dimension,@@ -159,6 +161,7 @@ impl<B: Backend> dyn Linear<B> {incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,..}) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => Ok(Box::new(RHTLinearWrapper::new(+ name,context,input_dimension,output_dimension_sum,@@ -181,6 +184,7 @@ impl<B: Backend> dyn Linear<B> {return Err(LinearBlockError::UnsupportedConfiguration(format!("{adapter_spec:?}")));};Ok(Box::new(QLoRALinearWrapper::new(+ name,context,*quantization_spec,adapter_spec,@@ -199,6 +203,7 @@ impl<B: Backend> dyn Linear<B> {}pub fn new(+ name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -207,6 +212,7 @@ impl<B: Backend> dyn Linear<B> {parameter_tree: &ParameterTree<B>,) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {Self::new_mixed_precision(+ name,input_dimension,output_dimensions,has_biases,@@ -219,6 +225,7 @@ impl<B: Backend> dyn Linear<B> {}pub fn new_with_input_rht_mixed_precision(+ name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -230,6 +237,7 @@ impl<B: Backend> dyn Linear<B> {) -> Result<(Box<dyn Linear<B>>, Option<B::GlobalBuffer>), LinearBlockError<B>> {let output_dimension_sum: u32 = output_dimensions.as_ref().iter().sum();if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(+ name.clone(),context,input_dimension,output_dimension_sum,@@ -244,6 +252,7 @@ impl<B: Backend> dyn Linear<B> {}let linear = Self::new_mixed_precision(+ name,input_dimension,output_dimensions,has_biases,@@ -257,6 +266,7 @@ impl<B: Backend> dyn Linear<B> {}pub fn new_for_fused_input(+ name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -266,6 +276,7 @@ impl<B: Backend> dyn Linear<B> {) -> Result<(Box<dyn Linear<B>>, Option<LinearInputPreparation<B>>), LinearBlockError<B>> {let output_dimension_sum: u32 = output_dimensions.as_ref().iter().sum();if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(+ name.clone(),context,input_dimension,output_dimension_sum,@@ -280,6 +291,7 @@ impl<B: Backend> dyn Linear<B> {}let linear = Self::new_mixed_precision(+ name,input_dimension,output_dimensions,has_biases,@@ -293,6 +305,7 @@ impl<B: Backend> dyn Linear<B> {}pub fn new_with_input_rht(+ name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -301,6 +314,7 @@ impl<B: Backend> dyn Linear<B> {parameter_tree: &ParameterTree<B>,) -> Result<(Box<dyn Linear<B>>, Option<B::GlobalBuffer>), LinearBlockError<B>> {Self::new_with_input_rht_mixed_precision(+ name,input_dimension,output_dimensions,has_biases,diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rsindex d4faa24b9..dfdf74851 100644--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs@@ -35,6 +35,7 @@ pub enum QLoRALinearWrapperError<B: Backend> {}pub struct QLoRALinearWrapper<B: Backend> {+ name: String,base_linear: LinearMatmul<B>,input_hadamard: Option<(ActivationTransform<B>, B::GlobalBuffer)>,output_hadamard: Option<(ActivationTransform<B>, B::GlobalBuffer)>,@@ -51,6 +52,7 @@ pub struct QLoRALinearWrapper<B: Backend> {impl<B: Backend> QLoRALinearWrapper<B> {pub fn new(+ name: String,context: &B::Context,quantization_spec: AnyWeightMatrixSpec,adapter_spec: LowRankSpec,@@ -85,6 +87,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {));};let base_linear = LinearMatmul::load(+ format!("{name}/base"),context,quantization_spec,input_dim,@@ -152,6 +155,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {.read_buffer()?;Ok(Self {+ name,base_linear,input_hadamard,output_hadamard,@@ -175,7 +179,8 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("linear (qlora)");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let mut intermediate =command_buffer.allocate_scratch_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type)?;@@ -261,6 +266,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {);}+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex c4dbbe8d8..bcfaebc11 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -27,6 +27,7 @@ pub enum RHTLinearWrapperError<B: Backend> {}pub struct RHTLinearWrapper<B: Backend> {+ name: String,input_rht: InputRht<B>,inner_linear: LinearMatmul<B>,}@@ -45,6 +46,7 @@ fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {impl<B: Backend> RHTLinearWrapper<B> {pub(super) fn new(+ name: String,context: &B::Context,input_dimension: u32,output_dimension: u32,@@ -61,6 +63,7 @@ impl<B: Backend> RHTLinearWrapper<B> {}let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(+ format!("{name}/matmul"),context,input_dimension,output_dimension,@@ -72,6 +75,7 @@ impl<B: Backend> RHTLinearWrapper<B> {)?;let activation_quantization = inner_linear.prepare_a8(context);Self::build_self_contained(+ name,context,input_data_type,LinearInputPreparation {@@ -83,6 +87,7 @@ impl<B: Backend> RHTLinearWrapper<B> {}pub(super) fn try_new_with_input_preparation(+ name: String,context: &B::Context,input_dimension: u32,output_dimension: u32,@@ -100,6 +105,7 @@ impl<B: Backend> RHTLinearWrapper<B> {}let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(+ format!("{name}/matmul"),context,input_dimension,output_dimension,@@ -114,7 +120,7 @@ impl<B: Backend> RHTLinearWrapper<B> {activation_quantization: inner_linear.prepare_a8(context),};if input_preparation.activation_quantization.is_some() && !allow_prequantized_activation {- let wrapper = Self::build_self_contained(context, input_data_type, input_preparation, inner_linear)?;+ let wrapper = Self::build_self_contained(name, context, input_data_type, input_preparation, inner_linear)?;Ok(Some((Box::new(wrapper), None)))} else {Ok(Some((Box::new(inner_linear), Some(input_preparation))))@@ -122,6 +128,7 @@ impl<B: Backend> RHTLinearWrapper<B> {}fn load_inner_with_output_rht(+ name: String,context: &B::Context,input_dimension: u32,output_dimension: u32,@@ -143,6 +150,7 @@ impl<B: Backend> RHTLinearWrapper<B> {.validate(&[output_dimension], DataType::I32)?.read_buffer()?;let inner_linear = LinearMatmul::load(+ name,context,quantization_spec,input_dimension,@@ -158,15 +166,17 @@ impl<B: Backend> RHTLinearWrapper<B> {}fn build_self_contained(+ name: String,context: &B::Context,input_data_type: DataType,input_preparation: LinearInputPreparation<B>,inner_linear: LinearMatmul<B>,) -> Result<Self, RHTLinearWrapperError<B>> {- let input_rht = InputRht::new(context, input_data_type, input_preparation, true)+ let input_rht = InputRht::new(format!("{name}/prepare"), context, input_data_type, input_preparation, true).map_err(RHTLinearWrapperError::BackendError)?;Ok(Self {+ name,input_rht,inner_linear,})@@ -189,12 +199,14 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("linear (rht)");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let input = match input {LinearInput::FullPrecision(input) => input,input => {let output = self.inner_linear.encode_input(input, batch_dim, command_buffer);+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();return output;},@@ -208,6 +220,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {command_buffer,)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)}diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rsindex a993ff14e..65fb42175 100644--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs@@ -16,12 +16,14 @@ use crate::{};pub struct UntiedReadout<B: Backend> {+ name: String,linear: LinearMatmul<B>,input_rht: Option<InputRht<B>>,}impl<B: Backend> UntiedReadout<B> {pub fn load(+ name: String,context: &B::Context,tree: &ParameterTree<B>,spec: AnyWeightMatrixSpec,@@ -39,13 +41,25 @@ impl<B: Backend> UntiedReadout<B> {else {return Ok(Self {linear: LinearMatmul::load(- context, spec, model_dim, vocab_size, data_type, data_type, data_type, tree, None, None,+ format!("{name}/matmul"),+ context,+ spec,+ model_dim,+ vocab_size,+ data_type,+ data_type,+ data_type,+ tree,+ None,+ None,)?,input_rht: None,+ name,});};let mut linear = LinearMatmul::load(+ format!("{name}/matmul"),context,*quantization_spec,model_dim,@@ -66,10 +80,12 @@ impl<B: Backend> UntiedReadout<B> {rht_signs,activation_quantization: linear.prepare_a8(context),};- let input_rht = InputRht::new(context, data_type, preparation, /* in_place */ false)- .map_err(LinearMatmulError::BackendError)?;+ let input_rht =+ InputRht::new(format!("{name}/prepare"), context, data_type, preparation, /* in_place */ false)+ .map_err(LinearMatmulError::BackendError)?;Ok(Self {+ name,linear,input_rht: Some(input_rht),})@@ -82,12 +98,15 @@ impl<B: Backend> UntiedReadout<B> {gather: Option<Gather<impl BufferRef<Backend = B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.name);let Some(input_rht) = &self.input_rht else {let a = MatmulA::FullPrecision {values: input,offset: 0,};- return self.linear.encode_with_a(a, batch_dim, gather, command_buffer);+ let output = self.linear.encode_with_a(a, batch_dim, gather, command_buffer)?;+ command_buffer.sample_end_timestamp();+ return Ok(output);};let format = if gather.is_some() {@@ -96,6 +115,8 @@ impl<B: Backend> UntiedReadout<B> {self.linear.select_activation_format(batch_dim, command_buffer.context())};let input = input_rht.prepare(input, batch_dim, format, command_buffer)?;- self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)+ let output = self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)?;+ command_buffer.sample_end_timestamp();+ Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rsindex 9c2b21056..9b540abef 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs@@ -33,6 +33,11 @@ pub use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, KVCacheView};pub mod rope;pub struct Attention<B: Backend> {+ name: String,+ core_name: String,+ prepare_kv_and_queries_name: String,+ prepare_queries_name: String,+ append_projected_kv_name: String,head_dim: u32,num_q_heads: u32,num_kv_heads: Option<u32>,@@ -62,6 +67,7 @@ pub enum AttentionNewError<B: Backend> {impl<B: Backend> Attention<B> {pub fn new(+ name: String,hidden_dim: u32,data_type: DataType,rope_config: Option<&AnyRoPEConfig>,@@ -95,6 +101,7 @@ impl<B: Backend> Attention<B> {qkv_dim};let (projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(+ format!("{name}/qkv projection"),hidden_dim,[projection_dim],config.has_qkvg_biases,@@ -110,6 +117,7 @@ impl<B: Backend> Attention<B> {let qkv_norm = (query_norm_config.is_some() || key_norm_config.is_some() || value_norm_config.is_some()).then(|| {QKVNorm::new(+ format!("{name}/qkv norm"),context,data_type,query_norm_config,@@ -164,6 +172,7 @@ impl<B: Backend> Attention<B> {.map_err(AttentionNewError::Backend)?;let out_projection = <dyn Linear<B>>::new(+ format!("{name}/out projection"),q_dim,[hidden_dim],config.has_out_biases,@@ -174,6 +183,11 @@ impl<B: Backend> Attention<B> {Ok((Self {+ core_name: format!("{name}/core"),+ prepare_kv_and_queries_name: format!("{name}/prepare kv and queries"),+ prepare_queries_name: format!("{name}/prepare queries"),+ append_projected_kv_name: format!("{name}/append projected kv"),+ name,head_dim,num_q_heads,num_kv_heads,@@ -221,7 +235,8 @@ impl<B: Backend> Mixer<B> for Attention<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("attention");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);assert_eq!(precalculated_rope.is_some(), self.max_rope_length.is_some(), "precalculated rope mismatch");@@ -229,6 +244,7 @@ impl<B: Backend> Mixer<B> for Attention<B> {state.map(|state| state.downcast::<AttentionState<B>>().expect("incorrect type of attention state"));let output = self.attend(hidden, precalculated_rope, batch_dim, state, command_buffer)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rsindex 75b337ac1..f1d6b7a9d 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs@@ -127,6 +127,7 @@ impl<B: Backend> Attention<B> {state: &mut AttentionState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {+ command_buffer.sample_start_timestamp(&self.append_projected_kv_name);if let Some(norm) = &self.projection.norm {norm.encode_key_value(key_value.reborrow(), batch_dim, command_buffer)?;}@@ -142,6 +143,7 @@ impl<B: Backend> Attention<B> {command_buffer,)?;state.encode_accept(&(0..batch_dim).collect::<Box<[u32]>>(), command_buffer)?;+ command_buffer.sample_end_timestamp();Ok(())}@@ -152,13 +154,14 @@ impl<B: Backend> Attention<B> {state: &AttentionState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.core_name);let trie = if batch_dim.is_flat() {None} else {Some(command_buffer.allocate_constant_from_slice(batch_dim.nodes())?)};- self.kernel.encode(+ let output = self.kernel.encode(AttentionArguments {queries,keys: state.keys.as_ref(),@@ -169,7 +172,9 @@ impl<B: Backend> Attention<B> {cache: state.view(),},command_buffer,- )+ )?;+ command_buffer.sample_end_timestamp();+ Ok(output)}fn prepare_kv_and_queries(@@ -183,6 +188,7 @@ impl<B: Backend> Attention<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.prepare_kv_and_queries_name);let num_kv_heads = self.num_kv_heads.expect("KV prepare requires KV heads");// Appended KV is tightly packed; attention projections may have a trailing gate segment.let input_row_stride = if num_q_heads == 0 {@@ -211,6 +217,7 @@ impl<B: Backend> Attention<B> {batch_dim,command_buffer,);+ command_buffer.sample_end_timestamp();Ok(queries)}@@ -221,6 +228,7 @@ impl<B: Backend> Attention<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.prepare_queries_name);let mut queries =command_buffer.allocate_scratch_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type)?;self.prepare.encode(@@ -239,6 +247,7 @@ impl<B: Backend> Attention<B> {batch_dim,command_buffer,);+ command_buffer.sample_end_timestamp();Ok(queries)}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rsindex 979998d7a..23c1054cb 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs@@ -26,6 +26,7 @@ struct Head<B: Backend> {}pub struct QKVNorm<B: Backend> {+ name: String,query: Option<Head<B>>,key: Option<Head<B>>,value: Option<Head<B>>,@@ -37,6 +38,7 @@ pub struct QKVNorm<B: Backend> {impl<B: Backend> QKVNorm<B> {pub fn new(+ name: String,context: &B::Context,intermediate_data_type: DataType,query_config: Option<NormalizationConfig>,@@ -75,6 +77,7 @@ impl<B: Backend> QKVNorm<B> {.transpose()?;Ok(Self {+ name,query,key,value,@@ -152,7 +155,8 @@ impl<B: Backend> QKVNorm<B> {"QKV norm input row stride ({input_row_stride}) is smaller than its packed row width ({packed_row_width})");- command_buffer.push_debug_group("qkv norm");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let kv = self.num_kv_heads;let heads = [(&self.query, 0, q_heads), (&self.key, q_heads, kv), (&self.value, q_heads + kv, kv)];@@ -179,6 +183,7 @@ impl<B: Backend> QKVNorm<B> {);}+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(())diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rsindex 967fc08cd..6430332e5 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs@@ -3,7 +3,7 @@ use std::any::Any;use crate::{array::size_for_shape,backends::common::{- Backend, Buffer, CommandBuffer, Context, DeviceCapabilities, Kernels, SparseBuffer,+ Backend, Buffer, CommandBuffer, CommandBufferEncoding, Context, DeviceCapabilities, Kernels, SparseBuffer,gpu_types::{Copy, ring::RingParams},kernel::KVCacheUpdateKernel,},@@ -161,6 +161,7 @@ impl KVCacheState {}pub struct AttentionState<B: Backend> {+ accept_name: String,pub element_dim: u32,pub data_type: DataType,cache: KVCacheState,@@ -215,6 +216,7 @@ impl<B: Backend> AttentionState<B> {let kv_cache_update = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(context, data_type)?;Ok(Self {+ accept_name: format!("{}/accept", attention.name),element_dim,data_type,cache,@@ -253,6 +255,7 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {accepted_indices: &[u32],command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {+ command_buffer.sample_start_timestamp(&self.accept_name);let copies = self.cache.accept(accepted_indices);if !copies.is_empty() {@@ -266,6 +269,7 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {);}+ command_buffer.sample_end_timestamp();Ok(())}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rsindex 33e793265..986db8154 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs@@ -48,6 +48,7 @@ enum DeltaNetSuffixStatus<B: Backend> {}pub struct DeltaNetState<B: Backend> {+ accept_name: String,conv_state: B::GlobalBuffer,ssm_state: B::GlobalBuffer,suffix_status: Option<DeltaNetSuffixStatus<B>>,@@ -68,6 +69,7 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {accepted_indices: &[u32],command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {+ command_buffer.sample_start_timestamp(&self.accept_name);let suffix_status = self.suffix_status.take().expect("delta net state has no suffix to accept");let accepted_index = *accepted_indices.last().expect("delta net state attempted to accept zero indices");@@ -109,11 +111,14 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {);},}+ command_buffer.sample_end_timestamp();Ok(())}}pub struct DeltaNet<B: Backend> {+ name: String,+ tree_verify_name: String,num_heads: u32,head_dim: u32,num_groups: u32,@@ -159,6 +164,7 @@ pub enum DeltaNetNewError<B: Backend> {impl<B: Backend> DeltaNet<B> {pub fn new(+ name: String,hidden_dim: u32,outer_data_type: DataType,config: &DeltaNetConfig,@@ -191,6 +197,7 @@ impl<B: Backend> DeltaNet<B> {let (in_projection, in_projection_input_hadamard_factors) =<dyn Linear<B>>::new_with_input_rht_mixed_precision(+ format!("{name}/in projection"),hidden_dim,[total_proj_dim],false,@@ -281,6 +288,7 @@ impl<B: Backend> DeltaNet<B> {.map_err(DeltaNetNewError::Backend)?;let out_projection = <dyn Linear<B>>::new_mixed_precision(+ format!("{name}/out projection"),value_dim,[hidden_dim],false,@@ -293,6 +301,8 @@ impl<B: Backend> DeltaNet<B> {Ok((Self {+ tree_verify_name: format!("{name}/tree verify"),+ name,num_heads: config.num_heads,head_dim: config.head_dim,num_groups: config.num_groups,@@ -334,6 +344,7 @@ impl<B: Backend> DeltaNet<B> {state: &mut DeltaNetState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.tree_verify_name);let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");let tree_size = batch_dim.size();let parents = command_buffer.allocate_constant_from_slice(batch_dim.parents())?;@@ -417,6 +428,7 @@ impl<B: Backend> DeltaNet<B> {beta,parents: batch_dim.parents().into(),});+ command_buffer.sample_end_timestamp();Ok(output)}}@@ -455,6 +467,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {)?;Ok(Box::new(DeltaNetState {+ accept_name: format!("{}/accept", self.name),conv_state,ssm_state,suffix_status: None,@@ -470,7 +483,8 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("delta net");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");@@ -487,6 +501,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {if !batch_dim.full_accept() {let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();return Ok(output);@@ -631,6 +646,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {let output = self.out_projection.encode(delta_output, batch_dim.size(), command_buffer)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rsindex 878668519..7027e63cf 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs@@ -55,6 +55,7 @@ enum Mamba2SSDPrefillVariant<B: Backend> {}pub struct Mamba2<B: Backend> {+ name: String,kernel_size: u32,num_heads: u32,num_groups: u32,@@ -91,6 +92,7 @@ pub enum Mamba2NewError<B: Backend> {impl<B: Backend> Mamba2<B> {pub fn new(+ name: String,hidden_dim: u32,outer_data_type: DataType,config: &Mamba2Config,@@ -114,6 +116,7 @@ impl<B: Backend> Mamba2<B> {let (in_projection, in_projection_input_hadamard_factors) =<dyn Linear<B>>::new_with_input_rht_mixed_precision(+ format!("{name}/in projection"),hidden_dim,[conv_dim, inner_dim, num_heads],config.has_in_biases,@@ -165,6 +168,7 @@ impl<B: Backend> Mamba2<B> {};let out_projection = <dyn Linear<B>>::new_mixed_precision(+ format!("{name}/out projection"),inner_dim,[hidden_dim],config.has_out_biases,@@ -177,6 +181,7 @@ impl<B: Backend> Mamba2<B> {Ok((Self {+ name,kernel_size,num_heads,num_groups,@@ -243,7 +248,8 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("mamba2");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");@@ -421,6 +427,7 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {let output = self.out_projection.encode(ssd_output, batch_dim.size(), command_buffer)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rsindex 5e0c30d66..e9fe51377 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs@@ -83,6 +83,7 @@ pub enum MixerNewError<B: Backend> {impl<B: Backend> dyn Mixer<B> {pub fn new(+ layer_name: &str,hidden_dim: u32,data_type: DataType,rope_config: Option<&AnyRoPEConfig>,@@ -92,32 +93,57 @@ impl<B: Backend> dyn Mixer<B> {) -> Result<(Box<dyn Mixer<B>>, Option<B::GlobalBuffer>), MixerNewError<B>> {match config {AnyTokenMixerConfig::AttentionConfig(config) => {- let (attention, in_projection_input_hadamard_factors) =- Attention::new(hidden_dim, data_type, rope_config, config, parameter_tree, context)?;+ let (attention, in_projection_input_hadamard_factors) = Attention::new(+ format!("{layer_name}/attention"),+ hidden_dim,+ data_type,+ rope_config,+ config,+ parameter_tree,+ context,+ )?;Ok((Box::new(attention), in_projection_input_hadamard_factors))},AnyTokenMixerConfig::DeltaNetConfig(config) => {assert!(rope_config.is_none(), "unexpected rope for delta net mixer");- let (delta_net, in_projection_input_hadamard_factors) =- DeltaNet::new(hidden_dim, data_type, config, parameter_tree, context)?;+ let (delta_net, in_projection_input_hadamard_factors) = DeltaNet::new(+ format!("{layer_name}/delta net"),+ hidden_dim,+ data_type,+ config,+ parameter_tree,+ context,+ )?;Ok((Box::new(delta_net), in_projection_input_hadamard_factors))},AnyTokenMixerConfig::Mamba2Config(config) => {assert!(rope_config.is_none(), "unexpected rope for mamba2 mixer");- let (mamba2, in_projection_input_hadamard_factors) =- Mamba2::new(hidden_dim, data_type, config, parameter_tree, context)?;+ let (mamba2, in_projection_input_hadamard_factors) = Mamba2::new(+ format!("{layer_name}/mamba2"),+ hidden_dim,+ data_type,+ config,+ parameter_tree,+ context,+ )?;Ok((Box::new(mamba2), in_projection_input_hadamard_factors))},AnyTokenMixerConfig::ShortConvConfig(config) => {assert!(rope_config.is_none(), "unexpected rope for short conv mixer");- let (short_conv, in_projection_input_hadamard_factors) =- ShortConv::new(hidden_dim, data_type, config, parameter_tree, context)?;+ let (short_conv, in_projection_input_hadamard_factors) = ShortConv::new(+ format!("{layer_name}/short conv"),+ hidden_dim,+ data_type,+ config,+ parameter_tree,+ context,+ )?;Ok((Box::new(short_conv), in_projection_input_hadamard_factors))},diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rsindex 93f9cbbb2..4c878daf1 100644--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs@@ -75,6 +75,10 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {}pub struct ShortConv<B: Backend> {+ name: String,+ decode_conv_name: String,+ prefill_conv_name: String,+ trie_conv_name: String,hidden_dim: u32,data_type: DataType,kernel_size: u32,@@ -102,6 +106,7 @@ pub enum ShortConvNewError<B: Backend> {impl<B: Backend> ShortConv<B> {pub fn new(+ name: String,hidden_dim: u32,data_type: DataType,config: &ShortConvConfig,@@ -117,6 +122,7 @@ impl<B: Backend> ShortConv<B> {}let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(+ format!("{name}/in projection"),hidden_dim,[hidden_dim * 3],false,@@ -126,6 +132,7 @@ impl<B: Backend> ShortConv<B> {)?;let out_projection = <dyn Linear<B>>::new(+ format!("{name}/out projection"),hidden_dim,[hidden_dim],false,@@ -161,6 +168,10 @@ impl<B: Backend> ShortConv<B> {Ok((Self {+ decode_conv_name: format!("{name}/decode conv"),+ prefill_conv_name: format!("{name}/prefill conv"),+ trie_conv_name: format!("{name}/trie conv"),+ name,hidden_dim,data_type,kernel_size,@@ -183,6 +194,7 @@ impl<B: Backend> ShortConv<B> {state: &mut ShortConvState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.decode_conv_name);let mut conv_output = command_buffer.allocate_scratch_for_shape(&[self.hidden_dim], self.data_type)?;self.short_conv_decode.encode(in_projected,@@ -198,6 +210,7 @@ impl<B: Backend> ShortConv<B> {self.hidden_dim,command_buffer,);+ command_buffer.sample_end_timestamp();Ok(conv_output)}@@ -208,6 +221,7 @@ impl<B: Backend> ShortConv<B> {state: &mut ShortConvState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.prefill_conv_name);let state_stride = self.kernel_size - 1;let padded_rows = state_stride + batch_dim;@@ -239,6 +253,7 @@ impl<B: Backend> ShortConv<B> {self.hidden_dim,command_buffer,);+ command_buffer.sample_end_timestamp();Ok(conv_output)}@@ -250,6 +265,7 @@ impl<B: Backend> ShortConv<B> {state: &mut ShortConvState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {+ command_buffer.sample_start_timestamp(&self.trie_conv_name);let mut conv_output =command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;let mut conv_states = command_buffer@@ -269,6 +285,7 @@ impl<B: Backend> ShortConv<B> {self.hidden_dim,command_buffer,);+ command_buffer.sample_end_timestamp();Ok((conv_output, conv_states))}}@@ -308,7 +325,8 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("short conv");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);assert!(precalculated_rope.is_none(), "unexpected rope for short conv mixer");@@ -344,6 +362,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {let output = self.out_projection.encode(conv_output, batch_dim.size(), command_buffer)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rsindex a0881dee7..d48d52ddc 100644--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs@@ -9,6 +9,7 @@ use crate::{};pub struct DenseMlp<B: Backend> {+ name: String,up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,down: Box<dyn Linear<B>>,@@ -16,11 +17,13 @@ pub struct DenseMlp<B: Backend> {impl<B: Backend> DenseMlp<B> {pub fn new(+ name: String,up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,down: Box<dyn Linear<B>>,) -> Self {Self {+ name,up,gate,down,@@ -35,13 +38,15 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("mlp (dense)");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let fused_up = self.up.encode(input, batch_dim, command_buffer)?;let act_format = self.down.select_activation_format(batch_dim, command_buffer.context());let down_input = self.gate.encode_for_linear(command_buffer, &fused_up, batch_dim, act_format)?;let output = self.down.encode_input(down_input, batch_dim, command_buffer)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rsindex 4eab50d2f..45911b711 100644--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs@@ -10,6 +10,7 @@ use crate::{};pub struct MlpGateActMulEncodable<B: Backend> {+ name: String,fp_kernel: GatedActMul<B>,activation: AnyActivation,hidden_dim: u32,@@ -21,6 +22,7 @@ pub struct MlpGateActMulEncodable<B: Backend> {impl<B: Backend> MlpGateActMulEncodable<B> {pub fn new(+ name: String,context: &B::Context,data_type: DataType,activation: AnyActivation,@@ -41,6 +43,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {.map(|quantization| GatedActMul::quantized(context, data_type, quantization, settings)).transpose()?;Ok(Self {+ name,fp_kernel,activation,hidden_dim,@@ -58,7 +61,8 @@ impl<B: Backend> MlpGateActMulEncodable<B> {batch_dim: u32,act_format: ActivationFormat,) -> Result<LinearInput<B>, B::Error> {- command_buffer.push_debug_group("gate act mul");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")@@ -115,6 +119,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {LinearInput::FullPrecision(hidden)};+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(input)diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rsindex 5ae64a74e..30407b40e 100644--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs@@ -35,6 +35,7 @@ pub enum MlpBlockError<B: Backend> {impl<B: Backend> dyn Mlp<B> {pub fn new(+ name: String,config: &AnyMLPConfig,model_dimension: u32,hidden_dimension: u32,@@ -45,6 +46,7 @@ impl<B: Backend> dyn Mlp<B> {match config {AnyMLPConfig::DenseMLPConfig(dense_config) => {let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(+ format!("{name}/up projection"),model_dimension,[2 * hidden_dimension],dense_config.has_up_biases,@@ -54,6 +56,7 @@ impl<B: Backend> dyn Mlp<B> {)?;let (down_projection, down_input_preparation) = <dyn Linear<B>>::new_for_fused_input(+ format!("{name}/down projection"),hidden_dimension,[model_dimension],dense_config.has_down_biases,@@ -63,6 +66,7 @@ impl<B: Backend> dyn Mlp<B> {)?;let gate = MlpGateActMulEncodable::new(+ format!("{name}/gate act mul"),context,data_type,dense_config.activation.clone(),@@ -73,7 +77,7 @@ impl<B: Backend> dyn Mlp<B> {).map_err(MlpBlockError::BackendError)?;- Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))+ Ok((Box::new(DenseMlp::new(name, up_projection, gate, down_projection)), up_input_hadamard_factors))},AnyMLPConfig::MixtureOfExpertsConfig(_) => Err(MlpBlockError::UnsupportedMixtureOfExperts),}diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rsindex 9f6c79366..b3f893482 100644--- a/crates/uzu-engine/src/encodable_block/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mod.rs@@ -1,5 +1,6 @@pub mod batch_topology;pub mod classifier;+pub mod context_ring_update;pub mod convolution;pub mod decoder;pub mod dflash;diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rsindex 6fe74ae30..6ce97da6c 100644--- a/crates/uzu-engine/src/encodable_block/normalization.rs+++ b/crates/uzu-engine/src/encodable_block/normalization.rs@@ -37,6 +37,7 @@ pub enum NormalizationNewError<B: Backend> {}pub struct Normalization<B: Backend> {+ name: String,epsilon: f32,scale_offset: Option<f32>,scales: Option<B::GlobalBuffer>,@@ -50,6 +51,7 @@ pub struct Normalization<B: Backend> {impl<B: Backend> Normalization<B> {pub fn new(+ name: String,element_count: u32,hadamard_factors: Option<B::GlobalBuffer>,shortcut_mode: ShortcutMode,@@ -100,6 +102,7 @@ impl<B: Backend> Normalization<B> {.map_err(NormalizationNewError::Backend)?;Ok(Self {+ name,epsilon: config.epsilon,scale_offset: config.scale_offset,scales,@@ -120,7 +123,8 @@ impl<B: Backend> Normalization<B> {shortcut: Option<impl BufferMut<Backend = B>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("normalization");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let row_size = size_for_shape(&[self.element_count], self.data_type);let row_offset_bytes = row_offset as usize * row_size;@@ -141,6 +145,7 @@ impl<B: Backend> Normalization<B> {command_buffer,);+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rsindex a2920a9ea..6c9c5a9ee 100644--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs@@ -32,6 +32,7 @@ pub enum PerLayerEmbeddingError<B: Backend> {}pub struct PerLayerEmbedding<B: Backend> {+ name: String,token_embedding: EmbeddingTable<B>,model_projection: Box<dyn Linear<B>>,projection_norm: Normalization<B>,@@ -45,6 +46,7 @@ pub struct PerLayerEmbedding<B: Backend> {impl<B: Backend> PerLayerEmbedding<B> {pub fn new(+ name: String,context: &B::Context,config: &PLEModelConfig,model_dim: u32,@@ -54,6 +56,7 @@ impl<B: Backend> PerLayerEmbedding<B> {let total_ple_dim = config.num_layers * config.ple_dim;let token_embedding = EmbeddingTable::load(+ format!("{name}/token lookup"),context,¶meter_tree.subtree("token_embedding"),config.ple_vocab_size,@@ -62,6 +65,7 @@ impl<B: Backend> PerLayerEmbedding<B> {)?;let model_projection = <dyn Linear<B>>::new(+ format!("{name}/model projection"),model_dim,[total_ple_dim],false,@@ -77,6 +81,7 @@ impl<B: Backend> PerLayerEmbedding<B> {adjusted};let projection_norm = Normalization::new(+ format!("{name}/projection norm"),config.ple_dim,None,ShortcutMode::None,@@ -91,6 +96,7 @@ impl<B: Backend> PerLayerEmbedding<B> {.map_err(PerLayerEmbeddingError::BackendError)?;Ok(Self {+ name,token_embedding,model_projection,projection_norm,@@ -110,7 +116,8 @@ impl<B: Backend> PerLayerEmbedding<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("per layer embedding");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let total_ple_dim = self.num_layers * self.ple_dim;let total_rows = batch_dim * self.num_layers;@@ -150,6 +157,7 @@ impl<B: Backend> PerLayerEmbedding<B> {command_buffer,);+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(per_layer_inputs)@@ -157,6 +165,7 @@ impl<B: Backend> PerLayerEmbedding<B> {}pub struct PerLayerEmbeddingProjection<B: Backend> {+ name: String,gate: Box<dyn Linear<B>>,projection: Box<dyn Linear<B>>,norm: Normalization<B>,@@ -173,6 +182,7 @@ pub struct PerLayerEmbeddingProjection<B: Backend> {impl<B: Backend> PerLayerEmbeddingProjection<B> {pub fn new(+ name: String,context: &B::Context,config: &PLELayerConfig,model_dim: u32,@@ -182,6 +192,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {parameter_tree: &ParameterTree<B>,) -> Result<Self, PerLayerEmbeddingError<B>> {let gate = <dyn Linear<B>>::new(+ format!("{name}/gate"),model_dim,[config.ple_dim],false,@@ -190,6 +201,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {¶meter_tree.subtree("gate"),)?;let projection = <dyn Linear<B>>::new(+ format!("{name}/projection"),config.ple_dim,[model_dim],false,@@ -198,6 +210,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {¶meter_tree.subtree("projection"),)?;let norm = Normalization::new(+ format!("{name}/norm"),model_dim,None,ShortcutMode::None,@@ -221,6 +234,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {.map_err(PerLayerEmbeddingError::BackendError)?;Ok(Self {+ name,gate,projection,norm,@@ -245,7 +259,8 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.push_debug_group("per layer embedding projection");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let length = batch_dim * self.model_dim;@@ -289,6 +304,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {command_buffer,);+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(())diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rsindex 351661ee1..010b45944 100644--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs@@ -27,6 +27,7 @@ pub enum PredictionHeadError<B: Backend> {}pub struct PredictionHead<B: Backend> {+ name: String,hidden_dim: u32,activation: ActivationType,dense_projection: Box<dyn Linear<B>>,@@ -37,6 +38,7 @@ pub struct PredictionHead<B: Backend> {impl<B: Backend> PredictionHead<B> {pub fn new(+ name: String,hidden_dim: u32,num_labels: u32,data_type: DataType,@@ -45,6 +47,7 @@ impl<B: Backend> PredictionHead<B> {context: &B::Context,) -> Result<Self, PredictionHeadError<B>> {let dense_projection = <dyn Linear<B>>::new(+ format!("{name}/dense projection"),hidden_dim,[hidden_dim],config.use_dense_bias,@@ -58,6 +61,7 @@ impl<B: Backend> PredictionHead<B> {.map_err(PredictionHeadError::Backend)?;let normalization = Normalization::new(+ format!("{name}/norm"),hidden_dim,None,ShortcutMode::None,@@ -69,6 +73,7 @@ impl<B: Backend> PredictionHead<B> {)?;let readout = <dyn Linear<B>>::new(+ format!("{name}/readout"),hidden_dim,[num_labels],true,@@ -78,6 +83,7 @@ impl<B: Backend> PredictionHead<B> {)?;Ok(Self {+ name,hidden_dim,activation,dense_projection,@@ -93,7 +99,8 @@ impl<B: Backend> PredictionHead<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("prediction head");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let mut hidden = self.dense_projection.encode(input, batch_dim, command_buffer)?;self.activation_kernel.encode(@@ -107,6 +114,7 @@ impl<B: Backend> PredictionHead<B> {self.normalization.encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;let logits = self.readout.encode(normalized, batch_dim, command_buffer)?;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(logits)diff --git a/crates/uzu-engine/src/encodable_block/sampling/mod.rs b/crates/uzu-engine/src/encodable_block/sampling/mod.rsindex 1142bd587..16721b71b 100644--- a/crates/uzu-engine/src/encodable_block/sampling/mod.rs+++ b/crates/uzu-engine/src/encodable_block/sampling/mod.rs@@ -24,6 +24,7 @@ pub use gumbel::{gumbel_float, revidx};pub use prng::PRng;pub struct Sampling<B: Backend> {+ name: String,vocab_size: u32,data_type: DataType,unified_kernels: Mutex<HashMap<UnifiedSamplingKey, <B::Kernels as Kernels>::UnifiedSamplingKernel>>,@@ -31,10 +32,12 @@ pub struct Sampling<B: Backend> {impl<B: Backend> Sampling<B> {pub fn new(+ name: String,data_type: DataType,vocab_size: u32,) -> Self {Self {+ name,vocab_size,data_type,unified_kernels: Mutex::new(HashMap::new()),@@ -92,7 +95,8 @@ impl<B: Backend> Sampling<B> {sampling_range: Range<u32>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::GlobalBuffer, B::Error> {- command_buffer.push_debug_group("sampling");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let sampling_length = sampling_range.end - sampling_range.start;@@ -194,6 +198,7 @@ impl<B: Backend> Sampling<B> {command_buffer,);+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rsindex c73b991b6..a38d15288 100644--- a/crates/uzu-engine/src/encodable_block/transformer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer.rs@@ -25,6 +25,7 @@ enum TransformerLayerStateType<B: Backend> {}pub struct TransformerState<B: Backend> {+ accept_name: String,layer_states: Box<[TransformerLayerStateType<B>]>,context_length: u32,}@@ -60,7 +61,8 @@ impl<B: Backend> TransformerState<B> {accepted_indices: &[u32],command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.push_debug_group("transformer accept");+ command_buffer.push_debug_group(&self.accept_name);+ command_buffer.sample_start_timestamp(&self.accept_name);for layer_state in &mut self.layer_states {let TransformerLayerStateType::Owned(layer_state) = layer_state else {@@ -72,6 +74,7 @@ impl<B: Backend> TransformerState<B> {self.context_length += accepted_indices.len() as u32;+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(())@@ -89,6 +92,8 @@ pub enum TransformerNewError<B: Backend> {}pub struct Transformer<B: Backend> {+ name: String,+ capture_residual_name: String,ropes: Box<[AnyRoPEConfig]>,layers: Box<[(TransformerLayer<B>, Option<usize>)]>,output_norm: Normalization<B>,@@ -98,6 +103,7 @@ pub struct Transformer<B: Backend> {impl<B: Backend> Transformer<B> {pub fn new(+ name: String,context: &B::Context,output_norm_hadamard_factors: Option<B::GlobalBuffer>,data_type: DataType,@@ -122,6 +128,7 @@ impl<B: Backend> Transformer<B> {});let layer = TransformerLayer::new(+ format!("{name}/layer {layer_index}"),context,transformer_config.model_dim,transformer_config.hidden_dim,@@ -137,6 +144,7 @@ impl<B: Backend> Transformer<B> {.collect::<Result<Box<[_]>, TransformerNewError<B>>>()?;let output_norm = Normalization::new(+ format!("{name}/output norm"),transformer_config.model_dim,output_norm_hadamard_factors,ShortcutMode::Add,@@ -151,6 +159,8 @@ impl<B: Backend> Transformer<B> {.map_err(TransformerNewError::Backend)?;Ok(Self {+ capture_residual_name: format!("{name}/capture residual"),+ name,ropes: ropes.into_boxed_slice(),layers,output_norm,@@ -166,9 +176,11 @@ impl<B: Backend> Transformer<B> {batch_size: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.capture_residual_name);let mut output = command_buffer.allocate_scratch(hidden.size())?;let elements = batch_size * self.model_dim;self.residual_add.encode(Some(shortcut), hidden, &mut output, elements, elements, 1.0, command_buffer);+ command_buffer.sample_end_timestamp();Ok(output)}@@ -220,6 +232,7 @@ impl<B: Backend> Transformer<B> {let context_length = 0;Ok(TransformerState {+ accept_name: format!("{}/accept", self.name),layer_states,context_length,})@@ -235,6 +248,7 @@ impl<B: Backend> Transformer<B> {mut state: Option<&mut TransformerState<B>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<TransformerEncodeOutput<B>, B::Error> {+ command_buffer.sample_start_timestamp(&self.name);let mut hidden = input;let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_none() {self.prefill_cache_layer_count()@@ -310,6 +324,7 @@ impl<B: Backend> Transformer<B> {});let Some(output_range) = output_range else {+ command_buffer.sample_end_timestamp();return Ok(TransformerEncodeOutput {output: None,hidden_features,@@ -324,6 +339,7 @@ impl<B: Backend> Transformer<B> {command_buffer,)?;+ command_buffer.sample_end_timestamp();Ok(TransformerEncodeOutput {output: Some(output_normalized),hidden_features,diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rsindex 567a92c2a..40f0e10ff 100644--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs@@ -45,6 +45,7 @@ pub enum TransformerLayerError<B: Backend> {}struct TransformerLayerConv<B: Backend> {+ pre_convolution_name: String,model_dim: u32,pre_conv: SeparableCausalConv<B>,kernel_projection: Box<dyn Linear<B>>,@@ -56,6 +57,7 @@ struct TransformerLayerConv<B: Backend> {// TODO: saner shortcutpub struct TransformerLayer<B: Backend> {+ name: String,pub layer_index: u32,pub kv_source_layer_index: Option<u32>,pub pre_mixer_norm: Option<Normalization<B>>,@@ -71,6 +73,7 @@ pub struct TransformerLayer<B: Backend> {impl<B: Backend> TransformerLayerConv<B> {fn new(+ name: String,context: &B::Context,model_dim: u32,config: &TransformerLayerConvConfig,@@ -79,6 +82,7 @@ impl<B: Backend> TransformerLayerConv<B> {input_hadamard_factors: Option<B::GlobalBuffer>,) -> Result<Self, TransformerLayerError<B>> {let pre_conv = SeparableCausalConv::new(+ format!("{name}/pre conv"),model_dim,config.conv_kernel_size,config.conv_group_size,@@ -88,6 +92,7 @@ impl<B: Backend> TransformerLayerConv<B> {context,)?;let post_conv = SeparableCausalConv::new(+ format!("{name}/post conv"),model_dim,config.conv_kernel_size,config.conv_group_size,@@ -100,6 +105,7 @@ impl<B: Backend> TransformerLayerConv<B> {let coefficient_count = config.conv_kernel_size * (model_dim / config.conv_group_size);let projection_dim = coefficient_count * 2;let kernel_projection = <dyn Linear<B>>::new(+ format!("{name}/kernel projection"),model_dim,[projection_dim],false,@@ -114,6 +120,7 @@ impl<B: Backend> TransformerLayerConv<B> {.map_err(TransformerLayerError::Backend)?;Ok(Self {+ pre_convolution_name: format!("{name}/pre convolution"),model_dim,pre_conv,kernel_projection,@@ -129,6 +136,7 @@ impl<B: Backend> TransformerLayerConv<B> {sequence_length: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {+ command_buffer.sample_start_timestamp(&self.pre_convolution_name);let mut projection_input = command_buffer.allocate_scratch(input.size())?;command_buffer.encode_copy(input, &mut projection_input);let coefficients = self.kernel_projection.encode(projection_input, sequence_length, command_buffer)?;@@ -143,6 +151,7 @@ impl<B: Backend> TransformerLayerConv<B> {if let Some((transform, factors)) = &self.input_rht {transform.encode_fp_in_place(&mut output, factors, None, sequence_length, self.model_dim, command_buffer);}+ command_buffer.sample_end_timestamp();Ok((output, coefficients))}@@ -166,6 +175,7 @@ impl<B: Backend> TransformerLayerConv<B> {impl<B: Backend> TransformerLayer<B> {pub fn new(+ name: String,context: &B::Context,model_dim: u32,hidden_dim: u32,@@ -201,6 +211,7 @@ impl<B: Backend> TransformerLayer<B> {};let (mixer, mixer_hadamard_factors) = <dyn Mixer<B>>::new(+ &name,model_dim,data_type,layer_config.rope_config.as_ref(),@@ -212,6 +223,7 @@ impl<B: Backend> TransformerLayer<B> {let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer_conv_config {Some(config) => (Some(TransformerLayerConv::new(+ format!("{name}/mixer conv"),context,model_dim,config,@@ -226,6 +238,7 @@ impl<B: Backend> TransformerLayer<B> {let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {Some(Normalization::new(+ format!("{name}/pre mixer norm"),model_dim,mixer_hadamard_factors,if layer_index > 0 {@@ -248,6 +261,7 @@ impl<B: Backend> TransformerLayer<B> {let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {Some(Normalization::new(+ format!("{name}/post mixer norm"),model_dim,None,ShortcutMode::None,@@ -262,6 +276,7 @@ impl<B: Backend> TransformerLayer<B> {};let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(+ format!("{name}/mlp"),&layer_config.mlp_config,model_dim,layer_config.hidden_dim.unwrap_or(hidden_dim),@@ -273,6 +288,7 @@ impl<B: Backend> TransformerLayer<B> {let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp_conv_config {Some(config) => (Some(TransformerLayerConv::new(+ format!("{name}/mlp conv"),context,model_dim,config,@@ -286,6 +302,7 @@ impl<B: Backend> TransformerLayer<B> {};let pre_mlp_norm = Normalization::new(+ format!("{name}/pre mlp norm"),model_dim,mlp_input_hadamard_factors,ShortcutMode::Add,@@ -298,6 +315,7 @@ impl<B: Backend> TransformerLayer<B> {let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(Normalization::new(+ format!("{name}/post mlp norm"),model_dim,None,ShortcutMode::None,@@ -314,6 +332,7 @@ impl<B: Backend> TransformerLayer<B> {let ple_projection = layer_config.ple_config.as_ref().map(|ple_config| {let ple_loader = parameter_tree.subtree("ple");PerLayerEmbeddingProjection::new(+ format!("{name}/per layer embedding projection"),context,ple_config,model_dim,@@ -326,6 +345,7 @@ impl<B: Backend> TransformerLayer<B> {});Ok(Self {+ name,layer_index,kv_source_layer_index: layer_config.kv_source_layer_index,pre_mixer_norm,@@ -350,7 +370,8 @@ impl<B: Backend> TransformerLayer<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&format!("transformer layer {}", self.layer_index));+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?@@ -418,6 +439,7 @@ impl<B: Backend> TransformerLayer<B> {command_buffer.encode_fill(&mut hidden, 0);}+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(hidden)diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex d446df08c..a3a15c378 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -112,6 +112,9 @@ impl<B: Backend> EncodedWeaverTree<B> {}pub struct Weaver<B: Backend> {+ name: String,+ prefix_name: String,+ step_name: String,token_embedding_norm: Normalization<B>,token_embedding_projection: Box<dyn Linear<B>>,hidden_state_norm: Normalization<B>,@@ -171,6 +174,7 @@ pub enum WeaverEncodeError<B: Backend> {impl<B: Backend> Weaver<B> {pub fn new(+ name: String,context: &B::Context,config: &WeaverConfig,vocab_size: u32,@@ -199,6 +203,7 @@ impl<B: Backend> Weaver<B> {});}let token_embedding_norm = Normalization::new(+ format!("{name}/token embedding norm"),config.target_embedding_dim,None,ShortcutMode::None,@@ -209,6 +214,7 @@ impl<B: Backend> Weaver<B> {context,)?;let hidden_state_norm = Normalization::new(+ format!("{name}/hidden state norm"),config.target_model_dim,None,ShortcutMode::None,@@ -219,6 +225,7 @@ impl<B: Backend> Weaver<B> {context,)?;let token_embedding_projection = <dyn Linear<B>>::new(+ format!("{name}/token embedding projection"),config.target_embedding_dim,[config.model_dim],true,@@ -228,9 +235,18 @@ impl<B: Backend> Weaver<B> {)?;let layer_parameters = parameter_tree.subtree("blocks");let layers = (0..config.num_layers)- .map(|index| WeaverLayer::new(context, config, index > 0, &layer_parameters.subtree(&index.to_string())))+ .map(|index| {+ WeaverLayer::new(+ format!("{name}/layer {index}"),+ context,+ config,+ index > 0,+ &layer_parameters.subtree(&index.to_string()),+ )+ }).collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;let readout_norm = Normalization::new(+ format!("{name}/readout norm"),config.model_dim,None,ShortcutMode::Add,@@ -241,6 +257,7 @@ impl<B: Backend> Weaver<B> {context,)?;let hidden_state_projection = <dyn Linear<B>>::new(+ format!("{name}/hidden state projection"),config.target_model_dim,[config.model_dim],true,@@ -249,6 +266,7 @@ impl<B: Backend> Weaver<B> {¶meter_tree.subtree("hidden_state_projection"),)?;let readout_query_projection = <dyn Linear<B>>::new(+ format!("{name}/readout query projection"),config.model_dim,[config.target_model_dim],false,@@ -267,6 +285,9 @@ impl<B: Backend> Weaver<B> {let frontier_insert_children = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;Ok(Self {+ prefix_name: format!("{name}/prefix"),+ step_name: format!("{name}/step"),+ name,token_embedding_norm,token_embedding_projection,hidden_state_norm,@@ -299,7 +320,8 @@ impl<B: Backend> Weaver<B> {depth: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<Vec<B::ScratchBuffer>, WeaverEncodeError<B>> {- command_buffer.push_debug_group("weaver prefix");+ command_buffer.push_debug_group(&self.prefix_name);+ command_buffer.sample_start_timestamp(&self.prefix_name);let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);let mut prefix_hidden = command_buffer@@ -359,6 +381,7 @@ impl<B: Backend> Weaver<B> {.kv_cache,);+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(prefix_kv_layers)@@ -387,6 +410,8 @@ impl<B: Backend> Weaver<B> {batch_start_slot: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), WeaverEncodeError<B>> {+ command_buffer.push_debug_group(&self.step_name);+ command_buffer.sample_start_timestamp(&self.step_name);let tree_slot_count = shape.slot_count();let ancestor_stride = self.max_depth;let frontier_capacity = tree_slot_count * shape.expand_width;@@ -539,6 +564,8 @@ impl<B: Backend> Weaver<B> {command_buffer,);+ command_buffer.sample_end_timestamp();+ command_buffer.pop_debug_group();Ok(())}@@ -553,7 +580,8 @@ impl<B: Backend> Weaver<B> {shape: WeaverTreeShape,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {- command_buffer.push_debug_group("weaver tree");+ command_buffer.push_debug_group(&self.name);+ command_buffer.sample_start_timestamp(&self.name);let tree_slot_count = shape.slot_count();let ancestor_stride = self.max_depth;@@ -677,7 +705,6 @@ impl<B: Backend> Weaver<B> {} else {shape.expand_per_round};- command_buffer.push_debug_group("weaver step");self.encode_step(target_embedding,prefix_kv_layers.iter(),@@ -700,10 +727,10 @@ impl<B: Backend> Weaver<B> {batch_start_slot,command_buffer,)?;- command_buffer.pop_debug_group();batch_start_slot += batch_node_count;}+ command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();let mut packed_tree_readback =diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rsindex 2b9080e6b..8621bedce 100644--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs@@ -25,6 +25,8 @@ pub struct PreparedPrefixAttention<B: Backend> {}pub struct WeaverLayer<B: Backend> {+ prefix_attention_name: String,+ post_attention_name: String,pub pre_attention_norm: Normalization<B>,pub qkv_projection: Box<dyn Linear<B>>,attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,@@ -42,6 +44,7 @@ pub struct WeaverLayer<B: Backend> {impl<B: Backend> WeaverLayer<B> {pub fn new(+ name: String,context: &B::Context,config: &WeaverConfig,add_to_residual: bool,@@ -59,6 +62,7 @@ impl<B: Backend> WeaverLayer<B> {let head_dim = model_dim / num_heads;let attention_scale = 1.0 / (head_dim as f32).sqrt();let qkv_projection = <dyn Linear<B>>::new(+ format!("{name}/qkv projection"),model_dim,[3 * model_dim],false,@@ -67,6 +71,7 @@ impl<B: Backend> WeaverLayer<B> {¶meter_tree.subtree("qkv_projection"),)?;let out_projection = <dyn Linear<B>>::new(+ format!("{name}/out projection"),model_dim,[model_dim],false,@@ -93,6 +98,7 @@ impl<B: Backend> WeaverLayer<B> {<B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true).map_err(WeaverNewError::Backend)?;let pre_attention_norm = Normalization::new(+ format!("{name}/pre attention norm"),model_dim,None,if add_to_residual {@@ -107,6 +113,7 @@ impl<B: Backend> WeaverLayer<B> {context,)?;let pre_mlp_norm = Normalization::new(+ format!("{name}/pre mlp norm"),model_dim,None,ShortcutMode::Add,@@ -122,12 +129,21 @@ impl<B: Backend> WeaverLayer<B> {true,true,));- let (mlp, up_input_hadamard_factors) =- <dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, ¶meter_tree.subtree("mlp"), DATA_TYPE)?;+ let (mlp, up_input_hadamard_factors) = <dyn Mlp<B>>::new(+ format!("{name}/mlp"),+ &mlp_config,+ model_dim,+ hidden_dim,+ context,+ ¶meter_tree.subtree("mlp"),+ DATA_TYPE,+ )?;assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");let ancestor_attention = <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim, num_heads).map_err(WeaverNewError::Backend)?;Ok(Self {+ prefix_attention_name: format!("{name}/prefix attention"),+ post_attention_name: format!("{name}/post attention"),qkv_projection,out_projection,prefix_attention,@@ -152,6 +168,7 @@ impl<B: Backend> WeaverLayer<B> {token_count: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<PreparedPrefixAttention<B>, B::Error> {+ command_buffer.sample_start_timestamp(&self.prefix_attention_name);let attention_input =self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), command_buffer)?;let qkv = self.qkv_projection.encode(attention_input, token_count, command_buffer)?;@@ -176,6 +193,7 @@ impl<B: Backend> WeaverLayer<B> {token_count,command_buffer,);+ command_buffer.sample_end_timestamp();Ok(PreparedPrefixAttention {queries,kv_cache,@@ -189,9 +207,12 @@ impl<B: Backend> WeaverLayer<B> {token_count: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {+ command_buffer.sample_start_timestamp(&self.post_attention_name);let projected_attention = self.out_projection.encode(attention_output, token_count, command_buffer)?;let mlp_input =self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), command_buffer)?;- self.mlp.encode(mlp_input, token_count, command_buffer)+ let output = self.mlp.encode(mlp_input, token_count, command_buffer)?;+ command_buffer.sample_end_timestamp();+ Ok(output)}}diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rsindex d92e350e7..0d63e996a 100644--- a/crates/uzu-engine/src/engine/classifier_model.rs+++ b/crates/uzu-engine/src/engine/classifier_model.rs@@ -66,6 +66,7 @@ impl<B: Backend> Engine<B> {let weight_loader = ParameterLoader::new(&weights_file, context.as_ref())?;let classifier = ClassifierEncodable::new(+ String::from("classifier"),context.as_ref(),&config.classifier_config,&weight_loader.tree().subtree("classifier"),diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rsindex 1255404b9..54a2ff84a 100644--- a/crates/uzu-engine/src/engine/language_model/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/mod.rs@@ -5,13 +5,14 @@ use thiserror::Error;use tokenizers::Tokenizer;use crate::{- backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},+ backends::common::{Backend, Context, DeviceCapabilities},config::{model::{generation::GenerationConfig, language_model::LanguageModelConfig},token_codec::AnyTokenCodecConfig,},data_type::DataType,encodable_block::{+ context_ring_update::ContextRingUpdate,decoder::{Decoder, DecoderError},sampling::{Sampling, SamplingMethod},},@@ -31,7 +32,7 @@ pub struct LanguageModel<B: Backend> {decoder: Decoder<B>,speculator: Option<DFlashTfmSpeculator<B>>,sampling: Sampling<B>,- context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,+ context_ring_update: ContextRingUpdate<B>,generation_config: GenerationConfig,end_of_thinking_tag: Option<String>,tokenizer: Arc<Tokenizer>,@@ -77,6 +78,7 @@ impl<B: Backend> Engine<B> {let data_type = DataType::BF16;let decoder = Decoder::new(+ String::from("decoder"),self.context.as_ref(),&config.decoder_config,&weight_loader.tree().subtree("decoder"),@@ -94,10 +96,11 @@ impl<B: Backend> Engine<B> {.transpose()?.flatten();- let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);+ let sampling = Sampling::new(String::from("sampling"), data_type, config.decoder_config.vocab_size);- let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&self.context)- .map_err(EngineLoadLanguageModelError::Backend)?;+ let context_ring_update =+ ContextRingUpdate::new(String::from("update repetition penalty ring"), self.context.as_ref())+ .map_err(EngineLoadLanguageModelError::Backend)?;weight_loader.tree().assert_all_tensors_validated()?;diff --git a/crates/uzu-engine/src/engine/language_model/stream/mod.rs b/crates/uzu-engine/src/engine/language_model/stream/mod.rsindex fc456ba8d..8ca87d672 100644--- a/crates/uzu-engine/src/engine/language_model/stream/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/mod.rs@@ -1,10 +1,12 @@+use std::sync::mpsc::Sender;+use derive_more::Debug;use thiserror::Error;#[cfg(grammar)]use crate::engine::language_model::grammar::{Grammar, GrammarError};use crate::{- backends::common::Backend,+ backends::common::{Backend, TimestampSpan},encodable_block::decoder::DecoderError,engine::language_model::{LanguageModel, state::LanguageModelState},speculators::dflash_tfm::DFlashTreeError,@@ -17,6 +19,7 @@ mod stream;pub struct LanguageModelStreamOptions {pub sampling_method: SamplingMethod,+ pub timestamps: Option<Sender<Box<[TimestampSpan]>>>,#[cfg(grammar)]pub grammar: Option<Grammar>,}@@ -44,6 +47,7 @@ impl<B: Backend> LanguageModel<B> {pub fn default_stream_options(&self) -> LanguageModelStreamOptions {LanguageModelStreamOptions {sampling_method: self.default_sampling_method(),+ timestamps: None,#[cfg(grammar)]grammar: None,}diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 7663cf362..1e826f915 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -2,7 +2,7 @@ use std::{iter::{once, repeat_n},mem::replace,range::Range,- sync::Arc,+ sync::{Arc, mpsc::Sender},};use shoji::traits::backend::chat_token::TokenStreamMetrics;@@ -12,8 +12,8 @@ use crate::engine::language_model::grammar::Grammar;use crate::{array::size_for_shape,backends::common::{- Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,- CommandBufferPending, Context, gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,+ Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context,+ TimestampSpan, gpu_types::trie::TrieNode as GpuTrieNode,},data_type::DataType,encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},@@ -26,6 +26,7 @@ use crate::{},},trie::TrieNode,+ utils::timestamps::{create_command_buffer, wait},};enum ForwardPassChaining<B: Backend> {@@ -40,6 +41,7 @@ impl<B: Backend> ForwardPassChaining<B> {fn resolve<'a>(&'a mut self,tokens: &mut Vec<u64>,+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,#[cfg(grammar)] grammar: Option<&mut Grammar>,) -> Result<(u64, Option<&'a B::ScratchBuffer>), LanguageModelStreamError<B>> {match self {@@ -50,7 +52,7 @@ impl<B: Backend> ForwardPassChaining<B> {Self::InFlight(in_flight) => {assert!(in_flight.full_accept);for pending in replace(&mut in_flight.pending, Box::new([])) {- pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;+ wait(pending, timestamps).map_err(LanguageModelStreamError::Backend)?;}let output_tokens = in_flight.output_tokens.as_slice::<u32>();assert_eq!(output_tokens.len(), 1);@@ -212,11 +214,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {).map_err(LanguageModelStreamError::Backend)?;- let mut command_buffer = model- .engine- .context- .create_command_buffer(Some("prefill"), Some(allocation_pool.clone()))- .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = create_command_buffer::<B>(+ &model.engine.context,+ "prefill",+ &allocation_pool,+ options.timestamps.as_ref(),+ )+ .map_err(LanguageModelStreamError::Backend)?;let mut output_tokens = None;let mut output_norm = None;@@ -401,7 +405,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {(ForwardPassChaining::InFlight(forward_pass_pending), None)} else {for pending in forward_pass_pending.pending {- pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;+ wait(pending, self.options.timestamps.as_ref()).map_err(LanguageModelStreamError::Backend)?;}let sampled_tokens = forward_pass_pending.output_tokens@@ -454,12 +458,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();- let mut command_buffer = self- .model- .engine- .context- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))- .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = create_command_buffer::<B>(+ &self.model.engine.context,+ "decode",+ &self.allocation_pool,+ self.options.timestamps.as_ref(),+ )+ .map_err(LanguageModelStreamError::Backend)?;self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut command_buffer)@@ -485,7 +490,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {None};if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {- command_buffer.push_debug_group("update repetition penalty ring");let accepted_input_token_ids_const = command_buffer.allocate_constant_from_slice(&accepted_input_token_ids@@ -501,23 +505,20 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {full.len() as u32,&mut command_buffer,);- command_buffer.pop_debug_group();}if let Some(capture_span) = capture_span {- command_buffer- .end_encoding()- .submit()- .wait_until_completed()+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).map_err(LanguageModelStreamError::Backend)?;drop(capture_span);- command_buffer = self- .model- .engine- .context- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))- .map_err(LanguageModelStreamError::Backend)?;+ command_buffer = create_command_buffer::<B>(+ &self.model.engine.context,+ "decode",+ &self.allocation_pool,+ self.options.timestamps.as_ref(),+ )+ .map_err(LanguageModelStreamError::Backend)?;}self.model_state.tokens.extend(accepted_output_token_ids);(@@ -541,6 +542,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {prev_output.resolve(&mut self.model_state.tokens,+ self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)?@@ -553,6 +555,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {{prev_output.resolve(&mut self.model_state.tokens,+ self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)?;@@ -569,6 +572,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {)&& let (root_token, Some(output_norm)) = prev_output.resolve(&mut self.model_state.tokens,+ self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)? {@@ -585,6 +589,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.options.grammar.as_mut(),&self.model_state.prng,self.allocation_pool.clone(),+ self.options.timestamps.as_ref(),)?;prev_output.drop_output_norm();(trie, None, false)@@ -607,12 +612,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.allocation_pool = self.model.engine.context.create_allocation_pool();- let mut command_buffer = self- .model- .engine- .context- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))- .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = create_command_buffer::<B>(+ &self.model.engine.context,+ "decode",+ &self.allocation_pool,+ self.options.timestamps.as_ref(),+ )+ .map_err(LanguageModelStreamError::Backend)?;let scratch_token_ids;let mut constant_token_ids = None;@@ -659,12 +665,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if chain_copy.is_some() {pending.push(command_buffer.end_encoding().submit());- let mut command_buffer = self- .model- .engine- .context- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))- .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = create_command_buffer::<B>(+ &self.model.engine.context,+ "decode",+ &self.allocation_pool,+ self.options.timestamps.as_ref(),+ )+ .map_err(LanguageModelStreamError::Backend)?;let mut bitmask = command_buffer.allocate_constant(@@ -672,7 +679,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {).map_err(LanguageModelStreamError::Backend)?;- prev_output.resolve(&mut self.model_state.tokens, Some(grammar))?;+ prev_output.resolve(&mut self.model_state.tokens, self.options.timestamps.as_ref(), Some(grammar))?;if grammar.next_bitmask(bitmask.as_slice_mut()) {(Some(bitmask), command_buffer)} else {@@ -786,6 +793,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {prev_output.resolve(&mut self.model_state.tokens,+ self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)?@@ -814,16 +822,17 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {} => Some(seed_token),DecodingState::ForwardPassPending(in_flight) => {for pending in in_flight.pending {- pending.wait_until_completed().unwrap();+ wait(pending, self.options.timestamps.as_ref()).unwrap();}if !in_flight.full_accept {- let mut command_buffer = self- .model- .engine- .context- .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))- .unwrap();+ let mut command_buffer = create_command_buffer::<B>(+ &self.model.engine.context,+ "drop accept",+ &self.allocation_pool,+ self.options.timestamps.as_ref(),+ )+ .unwrap();self.model_state.transformer_state.encode_accept(&[0], &mut command_buffer).unwrap();if let Some(speculator) = self.model.speculator.as_ref() {speculator@@ -835,7 +844,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {).unwrap();}- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).unwrap();}Some(in_flight.output_tokens.as_slice::<u32>()[0] as u64)@@ -849,12 +858,13 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {} => {assert!(num_accepted > 0 && num_accepted < full.len());- let mut command_buffer = self- .model- .engine- .context- .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))- .unwrap();+ let mut command_buffer = create_command_buffer::<B>(+ &self.model.engine.context,+ "drop accept",+ &self.allocation_pool,+ self.options.timestamps.as_ref(),+ )+ .unwrap();let accepted_token_indicies =full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();self.model_state@@ -871,7 +881,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {).unwrap();}- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).unwrap();drop(capture_span);diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex 6c41724b5..2d1e02a60 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -3,7 +3,7 @@ use std::{fs::File,io::{self, BufReader},path::Path,- sync::Arc,+ sync::{Arc, mpsc::Sender},};use derive_more::Debug;@@ -15,8 +15,8 @@ pub use crate::encodable_block::dflash::DFlashState;use crate::engine::language_model::grammar::Grammar;use crate::{backends::common::{- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,- Context, gpu_types::trie::TrieNode as GpuTrieNode,+ Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,+ gpu_types::trie::TrieNode as GpuTrieNode,},config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},data_type::DataType,@@ -29,6 +29,7 @@ use crate::{},parameters::{ParameterLoader, ParameterLoaderError},trie::TrieNode,+ utils::timestamps::{create_command_buffer, wait},};#[derive(Debug, Error)]@@ -122,12 +123,19 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let weight_loader = ParameterLoader::new(&weights_file, &*context)?;let speculator_tree = weight_loader.tree().subtree("speculator");- let dflash = DFlash::new(&*context, &config.draft_config, &speculator_tree.subtree("draft_model"), data_type)?;+ let dflash = DFlash::new(+ String::from("dflash"),+ &*context,+ &config.draft_config,+ &speculator_tree.subtree("draft_model"),+ data_type,+ )?;let weaver = config.weaver_config.as_ref().map(|weaver_config| {Weaver::new(+ String::from("weaver"),&*context,weaver_config,config.draft_config.vocab_size,@@ -138,7 +146,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {weight_loader.tree().assert_all_tensors_validated()?;- let sampling = Sampling::new(data_type, config.draft_config.vocab_size);+ let sampling = Sampling::new(String::from("sampling"), data_type, config.draft_config.vocab_size);Ok(Some(Self {context,@@ -217,6 +225,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {#[cfg(grammar)] grammar: Option<&mut Grammar>,prng: &PRng,allocation_pool: Arc<B::AllocationPool>,+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,) -> Result<TrieNode, DFlashTreeError<B>> {assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");@@ -230,10 +239,9 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let root_position = state.context_length();- let mut command_buffer = self- .context- .create_command_buffer(Some("speculator propose"), Some(allocation_pool))- .map_err(DFlashTreeError::Backend)?;+ let mut command_buffer =+ create_command_buffer::<B>(&self.context, "speculator propose", &allocation_pool, timestamps)+ .map_err(DFlashTreeError::Backend)?;let nodes = match shape.construction_method {DFlashTfmTreeConstructionMethod::Argmax => {@@ -283,7 +291,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {).map_err(DFlashTreeError::Backend)?;let completed =- command_buffer.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;+ wait(command_buffer.end_encoding().submit(), timestamps).map_err(DFlashTreeError::Backend)?;let tokens = sampled.copyout::<u32>();drop(completed);nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {@@ -358,7 +366,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {&mut command_buffer,)?;let completed =- command_buffer.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;+ wait(command_buffer.end_encoding().submit(), timestamps).map_err(DFlashTreeError::Backend)?;let nodes = tree.read_nodes();drop(completed);nodesdiff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rsindex 11e88e2c5..17b686018 100644--- a/crates/uzu-engine/src/utils/mod.rs+++ b/crates/uzu-engine/src/utils/mod.rs@@ -4,4 +4,5 @@ pub mod maybe_mut;#[cfg(backend = "cpu")]pub mod pointers;pub mod strict_serde;+pub mod timestamps;pub mod version;diff --git a/crates/uzu-engine/src/utils/timestamps.rs b/crates/uzu-engine/src/utils/timestamps.rsnew file mode 100644index 000000000..fcf6d60f6--- /dev/null+++ b/crates/uzu-engine/src/utils/timestamps.rs@@ -0,0 +1,32 @@+use std::sync::{Arc, mpsc::Sender};++use crate::backends::common::{+ Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferPending, Context, TimestampSpan,+};++pub fn create_command_buffer<B: Backend>(+ context: &B::Context,+ name: &str,+ allocation_pool: &Arc<B::AllocationPool>,+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,+) -> Result<<B::CommandBuffer as CommandBuffer>::Encoding, B::Error> {+ let mut command_buffer = context.create_command_buffer(Some(name), Some(allocation_pool.clone()))?;+ if timestamps.is_some() {+ command_buffer.enable_timestamps();+ }+ Ok(command_buffer)+}++pub fn wait<Pending: CommandBufferPending>(+ pending: Pending,+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,+) -> Result<+ <Pending::CommandBuffer as CommandBuffer>::Completed,+ <<Pending::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,+> {+ let completed = pending.wait_until_completed()?;+ if let Some(sender) = timestamps {+ let _ = sender.send(completed.timestamps().into());+ }+ Ok(completed)+}diff --git a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rsindex 15a85cc3b..c9639d5d8 100644--- a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs+++ b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs@@ -30,6 +30,7 @@ fn run_key_value_row_stride_test<B: Backend>() {let key = QKVNorm::<B>::build_head(&context, DataType::F32, config, None, HEAD_DIM).expect("failed to construct key norm");let norm = QKVNorm {+ name: String::from("test/qkv norm"),query: None,key: Some(key),value: None,diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rsindex f00b7bfff..50932575b 100644--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs@@ -44,7 +44,7 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(method: &SamplingMethod,batch_size: u32,) -> Result<SamplingTestResults, TestCaseError> {- let sampling = Sampling::<B>::new(T::data_type(), vocab_size as u32);+ let sampling = Sampling::<B>::new(String::from("test/sampling"), T::data_type(), vocab_size as u32);let mut command_buffer = context.create_command_buffer(None, None).unwrap();let logits_buffer = command_buffer.allocate_constant_from_slice(logits).unwrap();