A high-performance inference engine for AI models
gpu timestamps recording
eugenec812cf9bae50
diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rsindex 38cdf576f..ddb17038b 100644--- a/crates/uzu-engine/src/backends/common/command_buffer.rs+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs@@ -3,7 +3,7 @@ use std::{mem::size_of_val, time::Duration};use bytemuck::{AnyBitPattern, NoUninit};use crate::{- backends::common::{Backend, BufferMut, BufferRef},+ backends::common::{Backend, BufferMut, BufferRef, TimestampSpan},data_type::DataType,utils::size_for_shape::size_for_shape,};@@ -84,6 +84,8 @@ pub trait CommandBufferEncoding: Send {name: &str,);+ fn with_timestamps(self) -> Self;+fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;}@@ -110,4 +112,6 @@ pub trait CommandBufferCompleted: Send {type CommandBuffer: CommandBuffer<Completed = Self>;fn gpu_execution_time(&self) -> Duration;++ fn timestamps(&self) -> &[TimestampSpan];}diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rsindex 0b3094e9d..9bf74ac18 100644--- a/crates/uzu-engine/src/backends/common/mod.rs+++ b/crates/uzu-engine/src/backends/common/mod.rs@@ -7,6 +7,7 @@ mod context;mod device_capabilities;pub mod gpu_types;pub mod kernel;+mod timestamp_span;pub use backend::Backend;pub use buffer::{@@ -24,3 +25,4 @@ pub use command_buffer::{pub use context::Context;pub use device_capabilities::DeviceCapabilities;pub use kernel::Kernels;+pub use timestamp_span::TimestampSpan;diff --git a/crates/uzu-engine/src/backends/common/timestamp_span.rs b/crates/uzu-engine/src/backends/common/timestamp_span.rsnew file mode 100644index 000000000..f4315085f--- /dev/null+++ b/crates/uzu-engine/src/backends/common/timestamp_span.rs@@ -0,0 +1,8 @@+use std::time::Instant;++#[derive(Debug, Clone)]+pub struct TimestampSpan {+ pub name: String,+ pub start: Instant,+ pub end: Instant,+}diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rsindex b0fb837d6..e76af9676 100644--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs@@ -9,7 +9,7 @@ use crate::{backends::{common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,- CommandBufferExecutable, CommandBufferPending, allocator::bump::BumpAllocator,+ CommandBufferExecutable, CommandBufferPending, TimestampSpan, allocator::bump::BumpAllocator,},cpu::{Cpu,@@ -125,6 +125,10 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {) {}+ fn with_timestamps(self) -> Self {+ panic!("the CPU backend does not record timestamps")+ }+fn end_encoding(self) -> CpuCommandBufferExecutable {assert!(self.constant_allocator.is_done(), "attempted to end encoding while constants are still alive");CpuCommandBufferExecutable {@@ -202,4 +206,8 @@ impl CommandBufferCompleted for CpuCommandBufferCompleted {fn gpu_execution_time(&self) -> Duration {self.gpu_execution_time}++ fn timestamps(&self) -> &[TimestampSpan] {+ &[]+ }}diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rsindex 70492e705..b4e433d22 100644--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs@@ -18,7 +18,7 @@ use rangemap::RangeSet;use crate::backends::{common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,- CommandBufferExecutable, CommandBufferPending, Context, allocator::bump::BumpAllocator,+ CommandBufferExecutable, CommandBufferPending, Context, TimestampSpan, allocator::bump::BumpAllocator,},metal::{Metal,@@ -26,6 +26,7 @@ use crate::backends::{context::MetalContext,error::MetalError,kernel::{BlitCopyMetalKernel, BlitFillMetalKernel},+ timestamp_recorder::MetalTimestampRecorder,},};@@ -55,6 +56,7 @@ pub struct MetalCommandBufferEncoding {constant_allocator: Option<BumpAllocator<<Metal as Backend>::GlobalBuffer>>,allocation_pool: Arc<<Metal as Backend>::AllocationPool>,pub(super) context: Arc<MetalContext>,+ timestamps: Option<MetalTimestampRecorder>,}unsafe impl Send for MetalCommandBufferEncoding {}@@ -108,6 +110,7 @@ impl MetalCommandBufferEncoding {constant_allocator: Some(constant_allocator),allocation_pool,context,+ timestamps: None,})}@@ -205,6 +208,9 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {name: &str,) -> MetalDebugGroupSpan<'_> {ProtocolObject::<dyn MTL4CommandEncoder>::push_debug_group(self.compute_encoder.as_ref(), name);+ if let Some(timestamps) = &mut self.timestamps {+ timestamps.start(name, &self.context.device, &self.compute_encoder);+ }MetalDebugGroupSpan {command_buffer: self,}@@ -217,6 +223,11 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {ProtocolObject::<dyn MTL4CommandEncoder>::insert_debug_signpost(self.compute_encoder.as_ref(), name);}+ fn with_timestamps(mut self) -> Self {+ self.timestamps = Some(MetalTimestampRecorder::default());+ self+ }+fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {let constant_allocator = self.constant_allocator.take().unwrap();assert!(constant_allocator.is_done(), "attempted to end encoding while constants are still alive");@@ -227,6 +238,7 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {constant_allocator,allocation_pool: self.allocation_pool.clone(),context: self.context.clone(),+ timestamps: self.timestamps.take(),}}}@@ -263,7 +275,11 @@ impl DerefMut for MetalDebugGroupSpan<'_> {impl Drop for MetalDebugGroupSpan<'_> {fn drop(&mut self) {- self.command_buffer.compute_encoder.pop_debug_group();+ let command_buffer = &mut *self.command_buffer;+ if let Some(timestamps) = &mut command_buffer.timestamps {+ timestamps.end(&command_buffer.context.device, &command_buffer.compute_encoder);+ }+ command_buffer.compute_encoder.pop_debug_group();}}@@ -273,6 +289,7 @@ pub struct MetalCommandBufferExecutable {constant_allocator: BumpAllocator<<Metal as Backend>::GlobalBuffer>,allocation_pool: Arc<<Metal as Backend>::AllocationPool>,context: Arc<MetalContext>,+ timestamps: Option<MetalTimestampRecorder>,}impl CommandBufferExecutable for MetalCommandBufferExecutable {@@ -294,14 +311,18 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {let command_allocator = self.command_allocator.clone();let command_buffer = self.command_buffer.clone();let context_clone = self.context.clone();+ let timestamps = self.timestamps;let constant_allocator = self.constant_allocator;let allocation_pool = self.allocation_pool.clone();let feedback_handler = move |feedback: &ProtocolObject<dyn MTL4CommitFeedback>| {let message = if let Some(error) = feedback.error() {- Err(error.to_string())+ Err(MetalError::CommandBufferExecution(error.to_string()))} else {- Ok(Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()))+ timestamps+ .as_ref()+ .map_or_else(|| Ok(Box::default()), |timestamps| timestamps.resolve(&context_clone.device))+ .map(|spans| (Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()), spans))};let _ = sender.unbounded_send(message);context_clone.command_allocator_cache.lock().push(command_allocator.clone());@@ -319,26 +340,25 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {}pub struct MetalCommandBufferPending {- receiver: mpsc::UnboundedReceiver<Result<Duration, String>>,+ receiver: mpsc::UnboundedReceiver<Result<(Duration, Box<[TimestampSpan]>), MetalError>>,}impl CommandBufferPending for MetalCommandBufferPending {type CommandBuffer = MetalCommandBuffer;async fn wait_until_completed(mut self) -> Result<MetalCommandBufferCompleted, MetalError> {+ let (gpu_execution_time, timestamps) =+ self.receiver.recv().await.map_err(|_| MetalError::CommandBufferWait)??;Ok(MetalCommandBufferCompleted {- gpu_execution_time: self- .receiver- .recv()- .await- .map_err(|_| MetalError::CommandBufferWait)?- .map_err(MetalError::CommandBufferExecution)?,+ gpu_execution_time,+ timestamps,})}}pub struct MetalCommandBufferCompleted {gpu_execution_time: Duration,+ timestamps: Box<[TimestampSpan]>,}impl CommandBufferCompleted for MetalCommandBufferCompleted {@@ -347,4 +367,8 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {fn gpu_execution_time(&self) -> Duration {self.gpu_execution_time}++ fn timestamps(&self) -> &[TimestampSpan] {+ &self.timestamps+ }}diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rsindex 92eb54b7a..515b43361 100644--- a/crates/uzu-engine/src/backends/metal/error.rs+++ b/crates/uzu-engine/src/backends/metal/error.rs@@ -18,6 +18,10 @@ pub enum MetalError {CannotCreateBuffer,#[error("Cannot create heap")]CannotCreateHeap,+ #[error("Cannot resolve counter heap")]+ CannotResolveCounterHeap,+ #[error("Timestamp {0} was not written by the GPU")]+ UnwrittenTimestamp(usize),#[error("Cannot create command buffer")]CannotCreateCommandBuffer,#[error("Cannot create argument table: {0}")]diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rsindex 2057b7b17..cd14f3e84 100644--- a/crates/uzu-engine/src/backends/metal/mod.rs+++ b/crates/uzu-engine/src/backends/metal/mod.rs@@ -9,5 +9,6 @@ mod gpu_types;mod heaps;mod kernel;mod metal_extensions;+mod timestamp_recorder;pub use backend::Metal;diff --git a/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rsnew file mode 100644index 000000000..b49efce5a--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs@@ -0,0 +1,121 @@+use std::{+ ops::Range,+ time::{Duration, Instant},+};++use metal::{+ MTL4ComputeCommandEncoder, MTL4CounterHeap, MTL4CounterHeapDescriptor, MTL4CounterHeapExt, MTL4CounterHeapType,+ MTL4TimestampGranularity, MTLDevice, MTLDeviceExt,+};+use objc2::{rc::Retained, runtime::ProtocolObject};++use crate::backends::{common::TimestampSpan, metal::error::MetalError};++// "Maximum counter sample buffer length" is 32 KB, i.e. 4096 timestamp entries of 8 bytes (`MTL4TimestampHeapEntry`):+// https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf+const COUNTER_HEAP_CAPACITY: usize = 4096;++#[derive(Default)]+pub struct MetalTimestampRecorder {+ spans: Vec<(String, Range<usize>)>,+ open_spans: Vec<usize>,+ slot_count: usize,+ counter_heaps: Vec<Retained<ProtocolObject<dyn MTL4CounterHeap>>>,+}++impl MetalTimestampRecorder {+ pub fn start(+ &mut self,+ label: &str,+ device: &ProtocolObject<dyn MTLDevice>,+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,+ ) {+ let name = match self.open_spans.last() {+ Some(&parent) => format!("{}/{label}", self.spans[parent].0),+ None => label.to_owned(),+ };+ let slot = self.write(device, encoder);+ self.open_spans.push(self.spans.len());+ self.spans.push((name, slot..slot));+ }++ pub fn end(+ &mut self,+ device: &ProtocolObject<dyn MTLDevice>,+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,+ ) {+ let span = self.open_spans.pop().expect("timestamp ended without a start");+ self.spans[span].1.end = self.write(device, encoder);+ }++ fn write(+ &mut self,+ device: &ProtocolObject<dyn MTLDevice>,+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,+ ) -> usize {+ let slot = self.slot_count;+ if slot / COUNTER_HEAP_CAPACITY == self.counter_heaps.len() {+ let descriptor = MTL4CounterHeapDescriptor::new();+ descriptor.set_type(MTL4CounterHeapType::TIMESTAMP);+ descriptor.set_count(COUNTER_HEAP_CAPACITY);+ let counter_heap =+ device.new_counter_heap_with_descriptor(&descriptor).expect("Failed to create a counter heap");+ counter_heap.invalidate_counter_range(0..COUNTER_HEAP_CAPACITY);+ self.counter_heaps.push(counter_heap);+ }+ encoder.write_timestamp_with_granularity_into_heap_at_index(+ MTL4TimestampGranularity::PRECISE,+ &self.counter_heaps[slot / COUNTER_HEAP_CAPACITY],+ slot % COUNTER_HEAP_CAPACITY,+ );+ self.slot_count += 1;+ slot+ }++ pub fn resolve(+ &self,+ device: &ProtocolObject<dyn MTLDevice>,+ ) -> Result<Box<[TimestampSpan]>, MetalError> {+ let count = self.slot_count;+ let mut ticks = Vec::with_capacity(count);+ for (index, counter_heap) in self.counter_heaps.iter().enumerate() {+ let slots = 0..(count - index * COUNTER_HEAP_CAPACITY).min(COUNTER_HEAP_CAPACITY);+ let bytes = counter_heap.resolve_counter_range(slots).ok_or(MetalError::CannotResolveCounterHeap)?;+ let (entries, _) = bytes.as_chunks::<{ size_of::<u64>() }>();+ ticks.extend(entries.iter().map(|entry| u64::from_ne_bytes(*entry)));+ }+ let frequency = device.query_timestamp_frequency() as u128;+ let (mut cpu_now, mut gpu_now) = (0, 0);+ let now = Instant::now();+ device.sample_timestamps_gpu_timestamp(&mut cpu_now, &mut gpu_now);+ let to_instant = |ticks: u64| {+ let nanoseconds = (ticks as u128 * 1_000_000_000 / frequency) as u64;+ now - Duration::from_nanos(gpu_now - nanoseconds)+ };+ let instants = ticks+ .iter()+ .enumerate()+ .scan(None, |previous_timestamp, (slot, &slot_ticks)| {+ let timestamp = match slot_ticks {+ 0 => previous_timestamp.ok_or(MetalError::UnwrittenTimestamp(slot)),+ slot_ticks => Ok(to_instant(slot_ticks)),+ };+ *previous_timestamp = timestamp.as_ref().ok().copied();+ Some(timestamp)+ })+ .collect::<Result<Box<[Instant]>, MetalError>>()?;+ Ok(self+ .spans+ .iter()+ .map(|(name, slots)| TimestampSpan {+ name: name.clone(),+ start: instants[slots.start],+ end: instants[slots.end],+ })+ .collect())+ }+}++#[cfg(test)]+#[path = "../../../unit/backends/metal/timestamp_recorder_test.rs"]+mod tests;diff --git a/crates/uzu-engine/src/engine/language_model/stream/mod.rs b/crates/uzu-engine/src/engine/language_model/stream/mod.rsindex cd35827ba..43de853aa 100644--- a/crates/uzu-engine/src/engine/language_model/stream/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/mod.rs@@ -1,10 +1,12 @@+use std::sync::mpsc::Sender;+use derive_more::Debug;use thiserror::Error;#[cfg(grammar)]use crate::engine::language_model::grammar::{Grammar, GrammarError};use crate::{- backends::common::{Backend, capture_matcher::CaptureError},+ backends::common::{Backend, TimestampSpan, capture_matcher::CaptureError},encodable_block::decoder::DecoderError,engine::language_model::{LanguageModel, state::LanguageModelState},speculators::dflash_tfm::DFlashTreeError,@@ -17,6 +19,7 @@ mod stream;pub struct LanguageModelStreamOptions {pub sampling_method: SamplingMethod,+ pub timestamps: Option<Sender<Box<[TimestampSpan]>>>,#[cfg(grammar)]pub grammar: Option<Grammar>,}@@ -46,6 +49,7 @@ impl<B: Backend> LanguageModel<B> {pub fn default_stream_options(&self) -> LanguageModelStreamOptions {LanguageModelStreamOptions {sampling_method: self.default_sampling_method(),+ timestamps: None,#[cfg(grammar)]grammar: None,}diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 911d3188e..365e82c22 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -2,7 +2,7 @@ use std::{iter::{once, repeat_n},mem::replace,range::Range,- sync::Arc,+ sync::{Arc, mpsc::Sender},};use futures::executor::block_on;@@ -12,8 +12,8 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;use crate::engine::language_model::grammar::Grammar;use crate::{backends::common::{- Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,- CommandBufferPending, Context,+ Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context,+ TimestampSpan,capture_matcher::{CAPTURE_MATCHER, CaptureMatcherSpan},gpu_types::trie::TrieNode as GpuTrieNode,kernel::ContextRingUpdateKernel,@@ -26,7 +26,10 @@ use crate::{stream::{LanguageModelStreamError, LanguageModelStreamOptions},},trie::TrieNode,- utils::size_for_shape::size_for_shape,+ utils::{+ size_for_shape::size_for_shape,+ timestamps::{record, wait},+ },};enum ForwardPassChaining<B: Backend> {@@ -41,6 +44,7 @@ impl<B: Backend> ForwardPassChaining<B> {async fn resolve<'a>(&'a mut self,tokens: &mut Vec<u64>,+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,#[cfg(grammar)] grammar: Option<&mut Grammar>,) -> Result<(u64, Option<&'a B::ScratchBuffer>), LanguageModelStreamError<B>> {match self {@@ -51,7 +55,7 @@ impl<B: Backend> ForwardPassChaining<B> {Self::InFlight(in_flight) => {assert!(in_flight.full_accept);for pending in replace(&mut in_flight.pending, Box::new([])) {- pending.wait_until_completed().await.map_err(LanguageModelStreamError::Backend)?;+ wait(pending, timestamps).await.map_err(LanguageModelStreamError::Backend)?;}let output_tokens = in_flight.output_tokens.as_slice::<u32>();assert_eq!(output_tokens.len(), 1);@@ -207,11 +211,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {).map_err(LanguageModelStreamError::Backend)?;- let mut command_buffer = model+ let command_buffer = model.engine.context.create_command_buffer(Some("prefill"), Some(allocation_pool.clone())).map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = record(command_buffer, options.timestamps.as_ref());command_buffer.debug_point(&format!("prefill at context {}, suffix {} ({} batches)",@@ -401,7 +406,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {(ForwardPassChaining::InFlight(forward_pass_pending), None)} else {for pending in forward_pass_pending.pending {- pending.wait_until_completed().await.map_err(LanguageModelStreamError::Backend)?;+ wait(pending, self.options.timestamps.as_ref())+ .await+ .map_err(LanguageModelStreamError::Backend)?;}let sampled_tokens = forward_pass_pending.output_tokens@@ -454,7 +461,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();- let mut command_buffer = self+ let command_buffer = self.model.engine.context@@ -463,6 +470,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {Some(self.allocation_pool.clone()),).map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut command_buffer)@@ -525,6 +533,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {prev_output.resolve(&mut self.model_state.tokens,+ self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)@@ -550,6 +559,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let resolved = prev_output.resolve(&mut self.model_state.tokens,+ self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)@@ -569,6 +579,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {&self.model_state.prng,self.allocation_pool.clone(),Some(&format!("decode {} (speculate)", self.metrics.num_decode_forward_passes)),+ self.options.timestamps.as_ref(),).await?;prev_output.drop_output_norm();@@ -588,7 +599,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.allocation_pool = self.model.engine.context.create_allocation_pool();- let mut command_buffer = self+ let command_buffer = self.model.engine.context@@ -597,6 +608,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {Some(self.allocation_pool.clone()),).map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());command_buffer.debug_point(&format!("decode round {} at context {} suffix {}",@@ -650,7 +662,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if chain_copy.is_some() {pending.push(command_buffer.end_encoding().submit());- let mut command_buffer = self+ let command_buffer = self.model.engine.context@@ -659,6 +671,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {Some(self.allocation_pool.clone()),).map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());let mut bitmask = command_buffer.allocate_constant(@@ -666,7 +679,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {).map_err(LanguageModelStreamError::Backend)?;- prev_output.resolve(&mut self.model_state.tokens, Some(grammar)).await?;+ prev_output+ .resolve(&mut self.model_state.tokens, self.options.timestamps.as_ref(), Some(grammar))+ .await?;if grammar.next_bitmask(bitmask.as_slice_mut()) {(Some(bitmask), command_buffer)} else {@@ -782,6 +797,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {prev_output.resolve(&mut self.model_state.tokens,+ self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)@@ -801,11 +817,11 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {} => Some(seed_token),DecodingState::ForwardPassPending(in_flight) => {for pending in in_flight.pending {- pending.wait_until_completed().await.map_err(LanguageModelStreamError::Backend)?;+ wait(pending, self.options.timestamps.as_ref()).await.map_err(LanguageModelStreamError::Backend)?;}if !in_flight.full_accept {- let mut command_buffer = self+ let command_buffer = self.model.engine.context@@ -814,6 +830,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {Some(self.allocation_pool.clone()),).map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());self.model_state.transformer_state.encode_accept(&[0], &mut command_buffer)@@ -826,10 +843,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {&mut command_buffer,)?;}- command_buffer- .end_encoding()- .submit()- .wait_until_completed()+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).await.map_err(LanguageModelStreamError::Backend)?;}@@ -845,7 +859,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {} => {assert!(num_accepted > 0 && num_accepted < full.len());- let mut command_buffer = self+ let command_buffer = self.model.engine.context@@ -854,6 +868,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {Some(self.allocation_pool.clone()),).map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());let accepted_token_indicies =full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();self.model_state@@ -868,10 +883,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {&mut command_buffer,)?;}- command_buffer- .end_encoding()- .submit()- .wait_until_completed()+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).await.map_err(LanguageModelStreamError::Backend)?;diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex 0806f00b2..83c108e8c 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -3,7 +3,7 @@ use std::{fs::File,io::{self, BufReader},path::Path,- sync::Arc,+ sync::{Arc, mpsc::Sender},};use derive_more::Debug;@@ -15,8 +15,8 @@ pub use crate::encodable_block::dflash::DFlashState;use crate::engine::language_model::grammar::Grammar;use crate::{backends::common::{- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,- Context, gpu_types::trie::TrieNode as GpuTrieNode,+ Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,+ gpu_types::trie::TrieNode as GpuTrieNode,},config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},data_type::DataType,@@ -29,6 +29,7 @@ use crate::{},safetensors::{ParameterLoader, ParameterLoaderError},trie::TrieNode,+ utils::timestamps::{record, wait},};#[derive(Debug, Error)]@@ -206,6 +207,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {prng: &PRng,allocation_pool: Arc<B::AllocationPool>,command_buffer_name: Option<&str>,+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,) -> Result<TrieNode, DFlashTreeError<B>> {assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");@@ -219,10 +221,11 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let root_position = state.context_length();- let mut command_buffer = self+ let command_buffer = self.context.create_command_buffer(command_buffer_name, Some(allocation_pool)).map_err(DFlashTreeError::Backend)?;+ let mut command_buffer = record(command_buffer, timestamps);let nodes = match shape.construction_method {DFlashTfmTreeConstructionMethod::Argmax => {@@ -271,12 +274,8 @@ impl<B: Backend> DFlashTfmSpeculator<B> {&mut command_buffer,).map_err(DFlashTreeError::Backend)?;- let completed = command_buffer- .end_encoding()- .submit()- .wait_until_completed()- .await- .map_err(DFlashTreeError::Backend)?;+ let completed =+ wait(command_buffer.end_encoding().submit(), timestamps).await.map_err(DFlashTreeError::Backend)?;let tokens = sampled.copyout::<u32>();drop(completed);nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {@@ -349,12 +348,8 @@ impl<B: Backend> DFlashTfmSpeculator<B> {},&mut command_buffer,)?;- let completed = command_buffer- .end_encoding()- .submit()- .wait_until_completed()- .await- .map_err(DFlashTreeError::Backend)?;+ let completed =+ wait(command_buffer.end_encoding().submit(), timestamps).await.map_err(DFlashTreeError::Backend)?;let nodes = tree.read_nodes();drop(completed);nodesdiff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rsindex 0137b4219..bf1465e61 100644--- a/crates/uzu-engine/src/utils/mod.rs+++ b/crates/uzu-engine/src/utils/mod.rs@@ -5,4 +5,5 @@ pub mod maybe_mut;pub mod pointers;pub mod size_for_shape;pub mod strict_serde;+pub mod timestamps;pub mod version;diff --git a/crates/uzu-engine/src/utils/timestamps.rs b/crates/uzu-engine/src/utils/timestamps.rsnew file mode 100644index 000000000..5a22567ac--- /dev/null+++ b/crates/uzu-engine/src/utils/timestamps.rs@@ -0,0 +1,29 @@+use std::sync::mpsc::Sender;++use crate::backends::common::{+ Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferPending, TimestampSpan,+};++pub fn record<Encoding: CommandBufferEncoding>(+ command_buffer: Encoding,+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,+) -> Encoding {+ match timestamps {+ Some(_) => command_buffer.with_timestamps(),+ None => command_buffer,+ }+}++pub async fn wait<Pending: CommandBufferPending>(+ pending: Pending,+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,+) -> Result<+ <Pending::CommandBuffer as CommandBuffer>::Completed,+ <<Pending::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,+> {+ let completed = pending.wait_until_completed().await?;+ if let Some(sender) = timestamps {+ let _ = sender.send(completed.timestamps().into());+ }+ Ok(completed)+}diff --git a/crates/uzu-engine/unit/backends/metal/timestamp_recorder_test.rs b/crates/uzu-engine/unit/backends/metal/timestamp_recorder_test.rsnew file mode 100644index 000000000..13c6dfdf4--- /dev/null+++ b/crates/uzu-engine/unit/backends/metal/timestamp_recorder_test.rs@@ -0,0 +1,64 @@+use futures::executor::block_on;+use uzu_engine_macros::uzu_test;++use crate::{+ backends::{+ common::{+ Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,+ CommandBufferPending, Context, TimestampSpan,+ },+ metal::Metal,+ },+ tests::helpers::create_context,+};++type Encoding = <<Metal as Backend>::CommandBuffer as CommandBuffer>::Encoding;++fn fill(command_buffer: &mut Encoding) {+ let mut buffer = command_buffer.allocate_scratch(256).unwrap();+ command_buffer.encode_fill(&mut buffer, 7);+}++fn record(mut command_buffer: Encoding) -> Box<[TimestampSpan]> {+ {+ let command_buffer = &mut *command_buffer.debug_group("decoder");+ fill(command_buffer);+ for layer in ["layer 0", "layer 1"] {+ let command_buffer = &mut *command_buffer.debug_group(layer);+ fill(command_buffer);+ }+ }+ let completed = block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();+ completed.timestamps().into()+}++#[uzu_test]+fn debug_groups_time_their_full_path_down_to_kernels() {+ let context = create_context::<Metal>();+ let spans = record(context.create_command_buffer(None, None).unwrap().with_timestamps());+ assert_eq!(+ spans.iter().map(|span| span.name.as_str()).collect::<Vec<_>>(),+ [+ "decoder",+ "decoder/BlitFill",+ "decoder/layer 0",+ "decoder/layer 0/BlitFill",+ "decoder/layer 1",+ "decoder/layer 1/BlitFill"+ ]+ );+ for span in &spans {+ assert!(span.start <= span.end, "{} ends before it starts", span.name);+ if let Some((parent_name, _)) = span.name.rsplit_once('/') {+ let parent = spans.iter().find(|parent| parent.name == parent_name).unwrap();+ assert!(parent.start <= span.start && span.end <= parent.end, "{} is outside {parent_name}", span.name);+ }+ }+}++#[uzu_test]+fn only_command_buffers_asked_for_timestamps_record_them() {+ let context = create_context::<Metal>();+ assert_eq!(record(context.create_command_buffer(None, None).unwrap().with_timestamps()).len(), 6);+ assert!(record(context.create_command_buffer(None, None).unwrap()).is_empty());+}