Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

gpu timestamps recording

eugenec812cf9bae50
diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rs
index 38cdf576f..ddb17038b 100644
--- a/crates/uzu-engine/src/backends/common/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs
@@ -3,7 +3,7 @@ use std::{mem::size_of_val, time::Duration};
use bytemuck::{AnyBitPattern, NoUninit};
use crate::{
- backends::common::{Backend, BufferMut, BufferRef},
+ backends::common::{Backend, BufferMut, BufferRef, TimestampSpan},
data_type::DataType,
utils::size_for_shape::size_for_shape,
};
@@ -84,6 +84,8 @@ pub trait CommandBufferEncoding: Send {
name: &str,
);
+ fn with_timestamps(self) -> Self;
+
fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;
}
@@ -110,4 +112,6 @@ pub trait CommandBufferCompleted: Send {
type CommandBuffer: CommandBuffer<Completed = Self>;
fn gpu_execution_time(&self) -> Duration;
+
+ fn timestamps(&self) -> &[TimestampSpan];
}
diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rs
index 0b3094e9d..9bf74ac18 100644
--- a/crates/uzu-engine/src/backends/common/mod.rs
+++ b/crates/uzu-engine/src/backends/common/mod.rs
@@ -7,6 +7,7 @@ mod context;
mod device_capabilities;
pub mod gpu_types;
pub mod kernel;
+mod timestamp_span;
pub use backend::Backend;
pub use buffer::{
@@ -24,3 +25,4 @@ pub use command_buffer::{
pub use context::Context;
pub use device_capabilities::DeviceCapabilities;
pub use kernel::Kernels;
+pub use timestamp_span::TimestampSpan;
diff --git a/crates/uzu-engine/src/backends/common/timestamp_span.rs b/crates/uzu-engine/src/backends/common/timestamp_span.rs
new file mode 100644
index 000000000..f4315085f
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/timestamp_span.rs
@@ -0,0 +1,8 @@
+use std::time::Instant;
+
+#[derive(Debug, Clone)]
+pub struct TimestampSpan {
+ pub name: String,
+ pub start: Instant,
+ pub end: Instant,
+}
diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
index b0fb837d6..e76af9676 100644
--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
@@ -9,7 +9,7 @@ use crate::{
backends::{
common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,
- CommandBufferExecutable, CommandBufferPending, allocator::bump::BumpAllocator,
+ CommandBufferExecutable, CommandBufferPending, TimestampSpan, allocator::bump::BumpAllocator,
},
cpu::{
Cpu,
@@ -125,6 +125,10 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
) {
}
+ fn with_timestamps(self) -> Self {
+ panic!("the CPU backend does not record timestamps")
+ }
+
fn end_encoding(self) -> CpuCommandBufferExecutable {
assert!(self.constant_allocator.is_done(), "attempted to end encoding while constants are still alive");
CpuCommandBufferExecutable {
@@ -202,4 +206,8 @@ impl CommandBufferCompleted for CpuCommandBufferCompleted {
fn gpu_execution_time(&self) -> Duration {
self.gpu_execution_time
}
+
+ fn timestamps(&self) -> &[TimestampSpan] {
+ &[]
+ }
}
diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rs
index 70492e705..b4e433d22 100644
--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs
@@ -18,7 +18,7 @@ use rangemap::RangeSet;
use crate::backends::{
common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,
- CommandBufferExecutable, CommandBufferPending, Context, allocator::bump::BumpAllocator,
+ CommandBufferExecutable, CommandBufferPending, Context, TimestampSpan, allocator::bump::BumpAllocator,
},
metal::{
Metal,
@@ -26,6 +26,7 @@ use crate::backends::{
context::MetalContext,
error::MetalError,
kernel::{BlitCopyMetalKernel, BlitFillMetalKernel},
+ timestamp_recorder::MetalTimestampRecorder,
},
};
@@ -55,6 +56,7 @@ pub struct MetalCommandBufferEncoding {
constant_allocator: Option<BumpAllocator<<Metal as Backend>::GlobalBuffer>>,
allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
pub(super) context: Arc<MetalContext>,
+ timestamps: Option<MetalTimestampRecorder>,
}
unsafe impl Send for MetalCommandBufferEncoding {}
@@ -108,6 +110,7 @@ impl MetalCommandBufferEncoding {
constant_allocator: Some(constant_allocator),
allocation_pool,
context,
+ timestamps: None,
})
}
@@ -205,6 +208,9 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
name: &str,
) -> MetalDebugGroupSpan<'_> {
ProtocolObject::<dyn MTL4CommandEncoder>::push_debug_group(self.compute_encoder.as_ref(), name);
+ if let Some(timestamps) = &mut self.timestamps {
+ timestamps.start(name, &self.context.device, &self.compute_encoder);
+ }
MetalDebugGroupSpan {
command_buffer: self,
}
@@ -217,6 +223,11 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
ProtocolObject::<dyn MTL4CommandEncoder>::insert_debug_signpost(self.compute_encoder.as_ref(), name);
}
+ fn with_timestamps(mut self) -> Self {
+ self.timestamps = Some(MetalTimestampRecorder::default());
+ self
+ }
+
fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {
let constant_allocator = self.constant_allocator.take().unwrap();
assert!(constant_allocator.is_done(), "attempted to end encoding while constants are still alive");
@@ -227,6 +238,7 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
constant_allocator,
allocation_pool: self.allocation_pool.clone(),
context: self.context.clone(),
+ timestamps: self.timestamps.take(),
}
}
}
@@ -263,7 +275,11 @@ impl DerefMut for MetalDebugGroupSpan<'_> {
impl Drop for MetalDebugGroupSpan<'_> {
fn drop(&mut self) {
- self.command_buffer.compute_encoder.pop_debug_group();
+ let command_buffer = &mut *self.command_buffer;
+ if let Some(timestamps) = &mut command_buffer.timestamps {
+ timestamps.end(&command_buffer.context.device, &command_buffer.compute_encoder);
+ }
+ command_buffer.compute_encoder.pop_debug_group();
}
}
@@ -273,6 +289,7 @@ pub struct MetalCommandBufferExecutable {
constant_allocator: BumpAllocator<<Metal as Backend>::GlobalBuffer>,
allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
context: Arc<MetalContext>,
+ timestamps: Option<MetalTimestampRecorder>,
}
impl CommandBufferExecutable for MetalCommandBufferExecutable {
@@ -294,14 +311,18 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {
let command_allocator = self.command_allocator.clone();
let command_buffer = self.command_buffer.clone();
let context_clone = self.context.clone();
+ let timestamps = self.timestamps;
let constant_allocator = self.constant_allocator;
let allocation_pool = self.allocation_pool.clone();
let feedback_handler = move |feedback: &ProtocolObject<dyn MTL4CommitFeedback>| {
let message = if let Some(error) = feedback.error() {
- Err(error.to_string())
+ Err(MetalError::CommandBufferExecution(error.to_string()))
} else {
- Ok(Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()))
+ timestamps
+ .as_ref()
+ .map_or_else(|| Ok(Box::default()), |timestamps| timestamps.resolve(&context_clone.device))
+ .map(|spans| (Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()), spans))
};
let _ = sender.unbounded_send(message);
context_clone.command_allocator_cache.lock().push(command_allocator.clone());
@@ -319,26 +340,25 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {
}
pub struct MetalCommandBufferPending {
- receiver: mpsc::UnboundedReceiver<Result<Duration, String>>,
+ receiver: mpsc::UnboundedReceiver<Result<(Duration, Box<[TimestampSpan]>), MetalError>>,
}
impl CommandBufferPending for MetalCommandBufferPending {
type CommandBuffer = MetalCommandBuffer;
async fn wait_until_completed(mut self) -> Result<MetalCommandBufferCompleted, MetalError> {
+ let (gpu_execution_time, timestamps) =
+ self.receiver.recv().await.map_err(|_| MetalError::CommandBufferWait)??;
Ok(MetalCommandBufferCompleted {
- gpu_execution_time: self
- .receiver
- .recv()
- .await
- .map_err(|_| MetalError::CommandBufferWait)?
- .map_err(MetalError::CommandBufferExecution)?,
+ gpu_execution_time,
+ timestamps,
})
}
}
pub struct MetalCommandBufferCompleted {
gpu_execution_time: Duration,
+ timestamps: Box<[TimestampSpan]>,
}
impl CommandBufferCompleted for MetalCommandBufferCompleted {
@@ -347,4 +367,8 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {
fn gpu_execution_time(&self) -> Duration {
self.gpu_execution_time
}
+
+ fn timestamps(&self) -> &[TimestampSpan] {
+ &self.timestamps
+ }
}
diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rs
index 92eb54b7a..515b43361 100644
--- a/crates/uzu-engine/src/backends/metal/error.rs
+++ b/crates/uzu-engine/src/backends/metal/error.rs
@@ -18,6 +18,10 @@ pub enum MetalError {
CannotCreateBuffer,
#[error("Cannot create heap")]
CannotCreateHeap,
+ #[error("Cannot resolve counter heap")]
+ CannotResolveCounterHeap,
+ #[error("Timestamp {0} was not written by the GPU")]
+ UnwrittenTimestamp(usize),
#[error("Cannot create command buffer")]
CannotCreateCommandBuffer,
#[error("Cannot create argument table: {0}")]
diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rs
index 2057b7b17..cd14f3e84 100644
--- a/crates/uzu-engine/src/backends/metal/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/mod.rs
@@ -9,5 +9,6 @@ mod gpu_types;
mod heaps;
mod kernel;
mod metal_extensions;
+mod timestamp_recorder;
pub use backend::Metal;
diff --git a/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs
new file mode 100644
index 000000000..b49efce5a
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs
@@ -0,0 +1,121 @@
+use std::{
+ ops::Range,
+ time::{Duration, Instant},
+};
+
+use metal::{
+ MTL4ComputeCommandEncoder, MTL4CounterHeap, MTL4CounterHeapDescriptor, MTL4CounterHeapExt, MTL4CounterHeapType,
+ MTL4TimestampGranularity, MTLDevice, MTLDeviceExt,
+};
+use objc2::{rc::Retained, runtime::ProtocolObject};
+
+use crate::backends::{common::TimestampSpan, metal::error::MetalError};
+
+// "Maximum counter sample buffer length" is 32 KB, i.e. 4096 timestamp entries of 8 bytes (`MTL4TimestampHeapEntry`):
+// https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
+const COUNTER_HEAP_CAPACITY: usize = 4096;
+
+#[derive(Default)]
+pub struct MetalTimestampRecorder {
+ spans: Vec<(String, Range<usize>)>,
+ open_spans: Vec<usize>,
+ slot_count: usize,
+ counter_heaps: Vec<Retained<ProtocolObject<dyn MTL4CounterHeap>>>,
+}
+
+impl MetalTimestampRecorder {
+ pub fn start(
+ &mut self,
+ label: &str,
+ device: &ProtocolObject<dyn MTLDevice>,
+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,
+ ) {
+ let name = match self.open_spans.last() {
+ Some(&parent) => format!("{}/{label}", self.spans[parent].0),
+ None => label.to_owned(),
+ };
+ let slot = self.write(device, encoder);
+ self.open_spans.push(self.spans.len());
+ self.spans.push((name, slot..slot));
+ }
+
+ pub fn end(
+ &mut self,
+ device: &ProtocolObject<dyn MTLDevice>,
+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,
+ ) {
+ let span = self.open_spans.pop().expect("timestamp ended without a start");
+ self.spans[span].1.end = self.write(device, encoder);
+ }
+
+ fn write(
+ &mut self,
+ device: &ProtocolObject<dyn MTLDevice>,
+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,
+ ) -> usize {
+ let slot = self.slot_count;
+ if slot / COUNTER_HEAP_CAPACITY == self.counter_heaps.len() {
+ let descriptor = MTL4CounterHeapDescriptor::new();
+ descriptor.set_type(MTL4CounterHeapType::TIMESTAMP);
+ descriptor.set_count(COUNTER_HEAP_CAPACITY);
+ let counter_heap =
+ device.new_counter_heap_with_descriptor(&descriptor).expect("Failed to create a counter heap");
+ counter_heap.invalidate_counter_range(0..COUNTER_HEAP_CAPACITY);
+ self.counter_heaps.push(counter_heap);
+ }
+ encoder.write_timestamp_with_granularity_into_heap_at_index(
+ MTL4TimestampGranularity::PRECISE,
+ &self.counter_heaps[slot / COUNTER_HEAP_CAPACITY],
+ slot % COUNTER_HEAP_CAPACITY,
+ );
+ self.slot_count += 1;
+ slot
+ }
+
+ pub fn resolve(
+ &self,
+ device: &ProtocolObject<dyn MTLDevice>,
+ ) -> Result<Box<[TimestampSpan]>, MetalError> {
+ let count = self.slot_count;
+ let mut ticks = Vec::with_capacity(count);
+ for (index, counter_heap) in self.counter_heaps.iter().enumerate() {
+ let slots = 0..(count - index * COUNTER_HEAP_CAPACITY).min(COUNTER_HEAP_CAPACITY);
+ let bytes = counter_heap.resolve_counter_range(slots).ok_or(MetalError::CannotResolveCounterHeap)?;
+ let (entries, _) = bytes.as_chunks::<{ size_of::<u64>() }>();
+ ticks.extend(entries.iter().map(|entry| u64::from_ne_bytes(*entry)));
+ }
+ let frequency = device.query_timestamp_frequency() as u128;
+ let (mut cpu_now, mut gpu_now) = (0, 0);
+ let now = Instant::now();
+ device.sample_timestamps_gpu_timestamp(&mut cpu_now, &mut gpu_now);
+ let to_instant = |ticks: u64| {
+ let nanoseconds = (ticks as u128 * 1_000_000_000 / frequency) as u64;
+ now - Duration::from_nanos(gpu_now - nanoseconds)
+ };
+ let instants = ticks
+ .iter()
+ .enumerate()
+ .scan(None, |previous_timestamp, (slot, &slot_ticks)| {
+ let timestamp = match slot_ticks {
+ 0 => previous_timestamp.ok_or(MetalError::UnwrittenTimestamp(slot)),
+ slot_ticks => Ok(to_instant(slot_ticks)),
+ };
+ *previous_timestamp = timestamp.as_ref().ok().copied();
+ Some(timestamp)
+ })
+ .collect::<Result<Box<[Instant]>, MetalError>>()?;
+ Ok(self
+ .spans
+ .iter()
+ .map(|(name, slots)| TimestampSpan {
+ name: name.clone(),
+ start: instants[slots.start],
+ end: instants[slots.end],
+ })
+ .collect())
+ }
+}
+
+#[cfg(test)]
+#[path = "../../../unit/backends/metal/timestamp_recorder_test.rs"]
+mod tests;
diff --git a/crates/uzu-engine/src/engine/language_model/stream/mod.rs b/crates/uzu-engine/src/engine/language_model/stream/mod.rs
index cd35827ba..43de853aa 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/mod.rs
@@ -1,10 +1,12 @@
+use std::sync::mpsc::Sender;
+
use derive_more::Debug;
use thiserror::Error;
#[cfg(grammar)]
use crate::engine::language_model::grammar::{Grammar, GrammarError};
use crate::{
- backends::common::{Backend, capture_matcher::CaptureError},
+ backends::common::{Backend, TimestampSpan, capture_matcher::CaptureError},
encodable_block::decoder::DecoderError,
engine::language_model::{LanguageModel, state::LanguageModelState},
speculators::dflash_tfm::DFlashTreeError,
@@ -17,6 +19,7 @@ mod stream;
pub struct LanguageModelStreamOptions {
pub sampling_method: SamplingMethod,
+ pub timestamps: Option<Sender<Box<[TimestampSpan]>>>,
#[cfg(grammar)]
pub grammar: Option<Grammar>,
}
@@ -46,6 +49,7 @@ impl<B: Backend> LanguageModel<B> {
pub fn default_stream_options(&self) -> LanguageModelStreamOptions {
LanguageModelStreamOptions {
sampling_method: self.default_sampling_method(),
+ timestamps: None,
#[cfg(grammar)]
grammar: None,
}
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 911d3188e..365e82c22 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -2,7 +2,7 @@ use std::{
iter::{once, repeat_n},
mem::replace,
range::Range,
- sync::Arc,
+ sync::{Arc, mpsc::Sender},
};
use futures::executor::block_on;
@@ -12,8 +12,8 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;
use crate::engine::language_model::grammar::Grammar;
use crate::{
backends::common::{
- Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,
- CommandBufferPending, Context,
+ Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context,
+ TimestampSpan,
capture_matcher::{CAPTURE_MATCHER, CaptureMatcherSpan},
gpu_types::trie::TrieNode as GpuTrieNode,
kernel::ContextRingUpdateKernel,
@@ -26,7 +26,10 @@ use crate::{
stream::{LanguageModelStreamError, LanguageModelStreamOptions},
},
trie::TrieNode,
- utils::size_for_shape::size_for_shape,
+ utils::{
+ size_for_shape::size_for_shape,
+ timestamps::{record, wait},
+ },
};
enum ForwardPassChaining<B: Backend> {
@@ -41,6 +44,7 @@ impl<B: Backend> ForwardPassChaining<B> {
async fn resolve<'a>(
&'a mut self,
tokens: &mut Vec<u64>,
+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
#[cfg(grammar)] grammar: Option<&mut Grammar>,
) -> Result<(u64, Option<&'a B::ScratchBuffer>), LanguageModelStreamError<B>> {
match self {
@@ -51,7 +55,7 @@ impl<B: Backend> ForwardPassChaining<B> {
Self::InFlight(in_flight) => {
assert!(in_flight.full_accept);
for pending in replace(&mut in_flight.pending, Box::new([])) {
- pending.wait_until_completed().await.map_err(LanguageModelStreamError::Backend)?;
+ wait(pending, timestamps).await.map_err(LanguageModelStreamError::Backend)?;
}
let output_tokens = in_flight.output_tokens.as_slice::<u32>();
assert_eq!(output_tokens.len(), 1);
@@ -207,11 +211,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
.map_err(LanguageModelStreamError::Backend)?;
- let mut command_buffer = model
+ let command_buffer = model
.engine
.context
.create_command_buffer(Some("prefill"), Some(allocation_pool.clone()))
.map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = record(command_buffer, options.timestamps.as_ref());
command_buffer.debug_point(&format!(
"prefill at context {}, suffix {} ({} batches)",
@@ -401,7 +406,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
(ForwardPassChaining::InFlight(forward_pass_pending), None)
} else {
for pending in forward_pass_pending.pending {
- pending.wait_until_completed().await.map_err(LanguageModelStreamError::Backend)?;
+ wait(pending, self.options.timestamps.as_ref())
+ .await
+ .map_err(LanguageModelStreamError::Backend)?;
}
let sampled_tokens = forward_pass_pending
.output_tokens
@@ -454,7 +461,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();
let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();
- let mut command_buffer = self
+ let command_buffer = self
.model
.engine
.context
@@ -463,6 +470,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
Some(self.allocation_pool.clone()),
)
.map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
self.model_state
.transformer_state
.encode_accept(&accepted_token_indicies, &mut command_buffer)
@@ -525,6 +533,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
prev_output
.resolve(
&mut self.model_state.tokens,
+ self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)
@@ -550,6 +559,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let resolved = prev_output
.resolve(
&mut self.model_state.tokens,
+ self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)
@@ -569,6 +579,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
&self.model_state.prng,
self.allocation_pool.clone(),
Some(&format!("decode {} (speculate)", self.metrics.num_decode_forward_passes)),
+ self.options.timestamps.as_ref(),
)
.await?;
prev_output.drop_output_norm();
@@ -588,7 +599,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.allocation_pool = self.model.engine.context.create_allocation_pool();
- let mut command_buffer = self
+ let command_buffer = self
.model
.engine
.context
@@ -597,6 +608,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
Some(self.allocation_pool.clone()),
)
.map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
command_buffer.debug_point(&format!(
"decode round {} at context {} suffix {}",
@@ -650,7 +662,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if chain_copy.is_some() {
pending.push(command_buffer.end_encoding().submit());
- let mut command_buffer = self
+ let command_buffer = self
.model
.engine
.context
@@ -659,6 +671,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
Some(self.allocation_pool.clone()),
)
.map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
let mut bitmask = command_buffer
.allocate_constant(
@@ -666,7 +679,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
.map_err(LanguageModelStreamError::Backend)?;
- prev_output.resolve(&mut self.model_state.tokens, Some(grammar)).await?;
+ prev_output
+ .resolve(&mut self.model_state.tokens, self.options.timestamps.as_ref(), Some(grammar))
+ .await?;
if grammar.next_bitmask(bitmask.as_slice_mut()) {
(Some(bitmask), command_buffer)
} else {
@@ -782,6 +797,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
prev_output
.resolve(
&mut self.model_state.tokens,
+ self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)
@@ -801,11 +817,11 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
} => Some(seed_token),
DecodingState::ForwardPassPending(in_flight) => {
for pending in in_flight.pending {
- pending.wait_until_completed().await.map_err(LanguageModelStreamError::Backend)?;
+ wait(pending, self.options.timestamps.as_ref()).await.map_err(LanguageModelStreamError::Backend)?;
}
if !in_flight.full_accept {
- let mut command_buffer = self
+ let command_buffer = self
.model
.engine
.context
@@ -814,6 +830,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
Some(self.allocation_pool.clone()),
)
.map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
self.model_state
.transformer_state
.encode_accept(&[0], &mut command_buffer)
@@ -826,10 +843,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
&mut command_buffer,
)?;
}
- command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref())
.await
.map_err(LanguageModelStreamError::Backend)?;
}
@@ -845,7 +859,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
} => {
assert!(num_accepted > 0 && num_accepted < full.len());
- let mut command_buffer = self
+ let command_buffer = self
.model
.engine
.context
@@ -854,6 +868,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
Some(self.allocation_pool.clone()),
)
.map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = record(command_buffer, self.options.timestamps.as_ref());
let accepted_token_indicies =
full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
self.model_state
@@ -868,10 +883,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
&mut command_buffer,
)?;
}
- command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref())
.await
.map_err(LanguageModelStreamError::Backend)?;
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index 0806f00b2..83c108e8c 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -3,7 +3,7 @@ use std::{
fs::File,
io::{self, BufReader},
path::Path,
- sync::Arc,
+ sync::{Arc, mpsc::Sender},
};
use derive_more::Debug;
@@ -15,8 +15,8 @@ pub use crate::encodable_block::dflash::DFlashState;
use crate::engine::language_model::grammar::Grammar;
use crate::{
backends::common::{
- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,
- Context, gpu_types::trie::TrieNode as GpuTrieNode,
+ Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,
+ gpu_types::trie::TrieNode as GpuTrieNode,
},
config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},
data_type::DataType,
@@ -29,6 +29,7 @@ use crate::{
},
safetensors::{ParameterLoader, ParameterLoaderError},
trie::TrieNode,
+ utils::timestamps::{record, wait},
};
#[derive(Debug, Error)]
@@ -206,6 +207,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
prng: &PRng,
allocation_pool: Arc<B::AllocationPool>,
command_buffer_name: Option<&str>,
+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
) -> Result<TrieNode, DFlashTreeError<B>> {
assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");
@@ -219,10 +221,11 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let root_position = state.context_length();
- let mut command_buffer = self
+ let command_buffer = self
.context
.create_command_buffer(command_buffer_name, Some(allocation_pool))
.map_err(DFlashTreeError::Backend)?;
+ let mut command_buffer = record(command_buffer, timestamps);
let nodes = match shape.construction_method {
DFlashTfmTreeConstructionMethod::Argmax => {
@@ -271,12 +274,8 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
&mut command_buffer,
)
.map_err(DFlashTreeError::Backend)?;
- let completed = command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
- .await
- .map_err(DFlashTreeError::Backend)?;
+ let completed =
+ wait(command_buffer.end_encoding().submit(), timestamps).await.map_err(DFlashTreeError::Backend)?;
let tokens = sampled.copyout::<u32>();
drop(completed);
nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {
@@ -349,12 +348,8 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
},
&mut command_buffer,
)?;
- let completed = command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
- .await
- .map_err(DFlashTreeError::Backend)?;
+ let completed =
+ wait(command_buffer.end_encoding().submit(), timestamps).await.map_err(DFlashTreeError::Backend)?;
let nodes = tree.read_nodes();
drop(completed);
nodes
diff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rs
index 0137b4219..bf1465e61 100644
--- a/crates/uzu-engine/src/utils/mod.rs
+++ b/crates/uzu-engine/src/utils/mod.rs
@@ -5,4 +5,5 @@ pub mod maybe_mut;
pub mod pointers;
pub mod size_for_shape;
pub mod strict_serde;
+pub mod timestamps;
pub mod version;
diff --git a/crates/uzu-engine/src/utils/timestamps.rs b/crates/uzu-engine/src/utils/timestamps.rs
new file mode 100644
index 000000000..5a22567ac
--- /dev/null
+++ b/crates/uzu-engine/src/utils/timestamps.rs
@@ -0,0 +1,29 @@
+use std::sync::mpsc::Sender;
+
+use crate::backends::common::{
+ Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferPending, TimestampSpan,
+};
+
+pub fn record<Encoding: CommandBufferEncoding>(
+ command_buffer: Encoding,
+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
+) -> Encoding {
+ match timestamps {
+ Some(_) => command_buffer.with_timestamps(),
+ None => command_buffer,
+ }
+}
+
+pub async fn wait<Pending: CommandBufferPending>(
+ pending: Pending,
+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
+) -> Result<
+ <Pending::CommandBuffer as CommandBuffer>::Completed,
+ <<Pending::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
+> {
+ let completed = pending.wait_until_completed().await?;
+ if let Some(sender) = timestamps {
+ let _ = sender.send(completed.timestamps().into());
+ }
+ Ok(completed)
+}
diff --git a/crates/uzu-engine/unit/backends/metal/timestamp_recorder_test.rs b/crates/uzu-engine/unit/backends/metal/timestamp_recorder_test.rs
new file mode 100644
index 000000000..13c6dfdf4
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/metal/timestamp_recorder_test.rs
@@ -0,0 +1,64 @@
+use futures::executor::block_on;
+use uzu_engine_macros::uzu_test;
+
+use crate::{
+ backends::{
+ common::{
+ Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
+ CommandBufferPending, Context, TimestampSpan,
+ },
+ metal::Metal,
+ },
+ tests::helpers::create_context,
+};
+
+type Encoding = <<Metal as Backend>::CommandBuffer as CommandBuffer>::Encoding;
+
+fn fill(command_buffer: &mut Encoding) {
+ let mut buffer = command_buffer.allocate_scratch(256).unwrap();
+ command_buffer.encode_fill(&mut buffer, 7);
+}
+
+fn record(mut command_buffer: Encoding) -> Box<[TimestampSpan]> {
+ {
+ let command_buffer = &mut *command_buffer.debug_group("decoder");
+ fill(command_buffer);
+ for layer in ["layer 0", "layer 1"] {
+ let command_buffer = &mut *command_buffer.debug_group(layer);
+ fill(command_buffer);
+ }
+ }
+ let completed = block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
+ completed.timestamps().into()
+}
+
+#[uzu_test]
+fn debug_groups_time_their_full_path_down_to_kernels() {
+ let context = create_context::<Metal>();
+ let spans = record(context.create_command_buffer(None, None).unwrap().with_timestamps());
+ assert_eq!(
+ spans.iter().map(|span| span.name.as_str()).collect::<Vec<_>>(),
+ [
+ "decoder",
+ "decoder/BlitFill",
+ "decoder/layer 0",
+ "decoder/layer 0/BlitFill",
+ "decoder/layer 1",
+ "decoder/layer 1/BlitFill"
+ ]
+ );
+ for span in &spans {
+ assert!(span.start <= span.end, "{} ends before it starts", span.name);
+ if let Some((parent_name, _)) = span.name.rsplit_once('/') {
+ let parent = spans.iter().find(|parent| parent.name == parent_name).unwrap();
+ assert!(parent.start <= span.start && span.end <= parent.end, "{} is outside {parent_name}", span.name);
+ }
+ }
+}
+
+#[uzu_test]
+fn only_command_buffers_asked_for_timestamps_record_them() {
+ let context = create_context::<Metal>();
+ assert_eq!(record(context.create_command_buffer(None, None).unwrap().with_timestamps()).len(), 6);
+ assert!(record(context.create_command_buffer(None, None).unwrap()).is_empty());
+}