Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

gpu timestamps recording

eugene8ad7fa4c3b80
diff --git a/Cargo.lock b/Cargo.lock
index eee205fe4..acf8f6327 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3689,9 +3689,9 @@ dependencies = [
[[package]]
name = "mtl-rs"
-version = "0.2.2"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9f1de2bca596db925db758d497cbf9cb394a2981933319b3777510097e1d360"
+checksum = "4aa9291872296e9e734b0caf6e7a933fe6861d25ed86240461d6114355f7dbbf"
dependencies = [
"bitflags 2.13.0",
"block2",
diff --git a/Cargo.toml b/Cargo.toml
index 759972957..420503b9d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -76,7 +76,7 @@ objc2-foundation = "0.3.2"
objc2-core-foundation = "0.3.2"
objc2-io-kit = { version = "0.3.2", default-features = false, features = ["libc", "ps"] }
obfstr = "0.4.4"
-metal = { package = "mtl-rs", version = "0.2.2" }
+metal = { package = "mtl-rs", version = "0.3.0" }
block2 = "0.6"
bytemuck = "1.23"
bytesize = "2"
diff --git a/apps/mirai-gui/desktop/Cargo.lock b/apps/mirai-gui/desktop/Cargo.lock
index a2481a32f..a6849f8a7 100644
--- a/apps/mirai-gui/desktop/Cargo.lock
+++ b/apps/mirai-gui/desktop/Cargo.lock
@@ -3565,9 +3565,9 @@ dependencies = [
[[package]]
name = "mtl-rs"
-version = "0.2.2"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9f1de2bca596db925db758d497cbf9cb394a2981933319b3777510097e1d360"
+checksum = "4aa9291872296e9e734b0caf6e7a933fe6861d25ed86240461d6114355f7dbbf"
dependencies = [
"bitflags 2.13.0",
"block2",
diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rs
index 0377b6a70..4d8168aa6 100644
--- a/crates/uzu-engine/src/backends/common/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs
@@ -4,7 +4,7 @@ use bytemuck::{AnyBitPattern, NoUninit};
use crate::{
array::size_for_shape,
- backends::common::{Backend, BufferMut, BufferRef},
+ backends::common::{Backend, BufferMut, BufferRef, TimestampSpan},
data_type::DataType,
};
@@ -80,6 +80,15 @@ pub trait CommandBufferEncoding {
fn pop_debug_group(&mut self);
+ fn enable_timestamps(&mut self);
+
+ fn sample_start_timestamp(
+ &mut self,
+ name: &str,
+ );
+
+ fn sample_end_timestamp(&mut self);
+
fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;
}
@@ -104,4 +113,6 @@ pub trait CommandBufferCompleted: Send {
type CommandBuffer: CommandBuffer<Completed = Self>;
fn gpu_execution_time(&self) -> Duration;
+
+ fn timestamps(&self) -> &[TimestampSpan];
}
diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rs
index a07c947d6..47663a079 100644
--- a/crates/uzu-engine/src/backends/common/mod.rs
+++ b/crates/uzu-engine/src/backends/common/mod.rs
@@ -6,6 +6,7 @@ mod context;
mod device_capabilities;
pub mod gpu_types;
pub mod kernel;
+mod timestamps;
pub use backend::Backend;
pub use buffer::{
@@ -23,3 +24,4 @@ pub use command_buffer::{
pub use context::Context;
pub use device_capabilities::DeviceCapabilities;
pub use kernel::Kernels;
+pub use timestamps::{TimestampSpan, TimestampSpanRecorder};
diff --git a/crates/uzu-engine/src/backends/common/timestamps/mod.rs b/crates/uzu-engine/src/backends/common/timestamps/mod.rs
new file mode 100644
index 000000000..4d2eca4b4
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/timestamps/mod.rs
@@ -0,0 +1,6 @@
+mod span_boundary;
+mod timestamp_span;
+mod timestamp_span_recorder;
+
+pub use timestamp_span::TimestampSpan;
+pub use timestamp_span_recorder::TimestampSpanRecorder;
diff --git a/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs b/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs
new file mode 100644
index 000000000..a26a37153
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs
@@ -0,0 +1,4 @@
+pub enum SpanBoundary {
+ Start(String),
+ End,
+}
diff --git a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs
new file mode 100644
index 000000000..f4315085f
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs
@@ -0,0 +1,8 @@
+use std::time::Instant;
+
+#[derive(Debug, Clone)]
+pub struct TimestampSpan {
+ pub name: String,
+ pub start: Instant,
+ pub end: Instant,
+}
diff --git a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs
new file mode 100644
index 000000000..b1c6ce04f
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs
@@ -0,0 +1,59 @@
+use std::time::Instant;
+
+use super::{TimestampSpan, span_boundary::SpanBoundary};
+
+#[derive(Default)]
+pub struct TimestampSpanRecorder {
+ boundaries: Vec<SpanBoundary>,
+}
+
+impl TimestampSpanRecorder {
+ pub fn start(
+ &mut self,
+ name: String,
+ ) -> usize {
+ self.push(SpanBoundary::Start(name))
+ }
+
+ pub fn end(&mut self) -> usize {
+ self.push(SpanBoundary::End)
+ }
+
+ fn push(
+ &mut self,
+ boundary: SpanBoundary,
+ ) -> usize {
+ self.boundaries.push(boundary);
+ self.boundaries.len() - 1
+ }
+
+ pub fn slot_count(&self) -> usize {
+ self.boundaries.len()
+ }
+
+ pub fn spans(
+ &self,
+ instants: &[Instant],
+ ) -> Box<[TimestampSpan]> {
+ let mut spans = Vec::new();
+ let mut open_span_indices = Vec::new();
+ for (boundary, &instant) in self.boundaries.iter().zip(instants) {
+ match boundary {
+ SpanBoundary::Start(name) => {
+ open_span_indices.push(spans.len());
+ spans.push(TimestampSpan {
+ name: name.clone(),
+ start: instant,
+ end: instant,
+ });
+ },
+ SpanBoundary::End => {
+ let index = open_span_indices.pop().expect("timestamp ended without a start");
+ spans[index].end = instant;
+ },
+ }
+ }
+ assert!(open_span_indices.is_empty(), "timestamps were started but never ended");
+ spans.into_boxed_slice()
+ }
+}
diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
index 62ba458aa..78722176e 100644
--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
@@ -7,12 +7,14 @@ use crate::{
backends::{
common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,
- CommandBufferExecutable, CommandBufferPending, allocator::bump::BumpAllocator,
+ CommandBufferExecutable, CommandBufferPending, TimestampSpan, TimestampSpanRecorder,
+ allocator::bump::BumpAllocator,
},
cpu::{
Cpu,
buffer::{CpuBufferExt, dense::CpuBuffer},
context::CpuContext,
+ cpu_command::CpuCommand,
error::CpuError,
},
},
@@ -31,10 +33,11 @@ impl CommandBuffer for CpuCommandBuffer {
}
pub struct CpuCommandBufferEncoding {
- commands: Vec<Box<dyn FnOnce() + Send>>,
+ commands: Vec<CpuCommand>,
constant_allocator: BumpAllocator<<Cpu as Backend>::GlobalBuffer>,
allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,
context: Arc<CpuContext>,
+ timestamp_spans: Option<TimestampSpanRecorder>,
}
impl CpuCommandBufferEncoding {
@@ -48,6 +51,7 @@ impl CpuCommandBufferEncoding {
constant_allocator,
allocation_pool,
context,
+ timestamp_spans: None,
}
}
@@ -55,7 +59,11 @@ impl CpuCommandBufferEncoding {
&mut self,
command: impl FnOnce() + Send + 'static,
) {
- self.commands.push(Box::new(command))
+ self.commands.push(CpuCommand::Run(Box::new(command)))
+ }
+
+ fn write_timestamp(&mut self) {
+ self.commands.push(CpuCommand::Timestamp);
}
}
@@ -118,6 +126,28 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
fn pop_debug_group(&mut self) {}
+ fn enable_timestamps(&mut self) {
+ assert!(self.timestamp_spans.is_none(), "timestamps already enabled");
+ self.timestamp_spans = Some(TimestampSpanRecorder::default());
+ }
+
+ fn sample_start_timestamp(
+ &mut self,
+ name: &str,
+ ) {
+ if let Some(spans) = &mut self.timestamp_spans {
+ spans.start(name.to_owned());
+ self.write_timestamp();
+ }
+ }
+
+ fn sample_end_timestamp(&mut self) {
+ if let Some(spans) = &mut self.timestamp_spans {
+ spans.end();
+ self.write_timestamp();
+ }
+ }
+
fn end_encoding(self) -> CpuCommandBufferExecutable {
assert!(self.constant_allocator.is_done(), "attempted to end encoding while constants are still alive");
CpuCommandBufferExecutable {
@@ -125,15 +155,17 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
constant_allocator: self.constant_allocator,
allocation_pool: self.allocation_pool,
context: self.context,
+ timestamp_spans: self.timestamp_spans,
}
}
}
pub struct CpuCommandBufferExecutable {
- commands: Vec<Box<dyn FnOnce() + Send>>,
+ commands: Vec<CpuCommand>,
constant_allocator: BumpAllocator<<Cpu as Backend>::GlobalBuffer>,
allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,
context: Arc<CpuContext>,
+ timestamp_spans: Option<TimestampSpanRecorder>,
}
impl CommandBufferExecutable for CpuCommandBufferExecutable {
@@ -148,14 +180,23 @@ impl CommandBufferExecutable for CpuCommandBufferExecutable {
.send(Box::new(move || {
let start = Instant::now();
- for command in self.commands {
- command()
- }
+ let instants = self
+ .commands
+ .into_iter()
+ .filter_map(|command| match command {
+ CpuCommand::Run(run) => {
+ run();
+ None
+ },
+ CpuCommand::Timestamp => Some(Instant::now()),
+ })
+ .collect::<Box<[Instant]>>();
let gpu_execution_time = start.elapsed();
let completed = CpuCommandBufferCompleted {
gpu_execution_time,
+ timestamps: self.timestamp_spans.map_or_else(Box::default, |spans| spans.spans(&instants)),
_allocation_pool: self.allocation_pool,
};
@@ -186,6 +227,7 @@ impl CommandBufferPending for CpuCommandBufferPending {
pub struct CpuCommandBufferCompleted {
gpu_execution_time: Duration,
+ timestamps: Box<[TimestampSpan]>,
_allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,
}
@@ -195,4 +237,8 @@ impl CommandBufferCompleted for CpuCommandBufferCompleted {
fn gpu_execution_time(&self) -> Duration {
self.gpu_execution_time
}
+
+ fn timestamps(&self) -> &[TimestampSpan] {
+ &self.timestamps
+ }
}
diff --git a/crates/uzu-engine/src/backends/cpu/cpu_command.rs b/crates/uzu-engine/src/backends/cpu/cpu_command.rs
new file mode 100644
index 000000000..6dd851e4c
--- /dev/null
+++ b/crates/uzu-engine/src/backends/cpu/cpu_command.rs
@@ -0,0 +1,4 @@
+pub enum CpuCommand {
+ Run(Box<dyn FnOnce() + Send>),
+ Timestamp,
+}
diff --git a/crates/uzu-engine/src/backends/cpu/mod.rs b/crates/uzu-engine/src/backends/cpu/mod.rs
index f569a2da4..7d6afffc5 100644
--- a/crates/uzu-engine/src/backends/cpu/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/mod.rs
@@ -2,6 +2,7 @@ mod backend;
mod buffer;
mod command_buffer;
mod context;
+mod cpu_command;
mod error;
pub(crate) mod kernel; // TODO: This should not be pub!!!
diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rs
index 985a6d0e5..a584b4cb0 100644
--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs
@@ -16,9 +16,11 @@ use rangemap::RangeSet;
use crate::backends::{
common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,
- CommandBufferExecutable, CommandBufferPending, Context, allocator::bump::BumpAllocator,
+ CommandBufferExecutable, CommandBufferPending, Context, TimestampSpan, allocator::bump::BumpAllocator,
+ },
+ metal::{
+ Metal, MetalContext, buffer::MetalBufferExt, error::MetalError, timestamp_recorder::MetalTimestampRecorder,
},
- metal::{Metal, MetalContext, buffer::MetalBufferExt, error::MetalError},
};
pub struct MetalCommandBuffer;
@@ -47,6 +49,7 @@ pub struct MetalCommandBufferEncoding {
constant_allocator: Option<BumpAllocator<<Metal as Backend>::GlobalBuffer>>,
allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
pub(super) context: Arc<MetalContext>,
+ timestamps: Option<MetalTimestampRecorder>,
}
impl MetalCommandBufferEncoding {
@@ -101,6 +104,7 @@ impl MetalCommandBufferEncoding {
constant_allocator: Some(constant_allocator),
allocation_pool,
context,
+ timestamps: None,
})
}
@@ -213,6 +217,26 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
self.compute_encoder.pop_debug_group();
}
+ fn enable_timestamps(&mut self) {
+ assert!(self.timestamps.is_none(), "timestamps already enabled");
+ self.timestamps = Some(MetalTimestampRecorder::new(self.context.device.clone()));
+ }
+
+ fn sample_start_timestamp(
+ &mut self,
+ name: &str,
+ ) {
+ if let Some(timestamps) = &mut self.timestamps {
+ timestamps.start(name.to_owned(), &self.compute_encoder);
+ }
+ }
+
+ fn sample_end_timestamp(&mut self) {
+ if let Some(timestamps) = &mut self.timestamps {
+ timestamps.end(&self.compute_encoder);
+ }
+ }
+
fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {
let constant_allocator = self.constant_allocator.take().unwrap();
assert!(constant_allocator.is_done(), "attempted to end encoding while constants are still alive");
@@ -223,6 +247,7 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
constant_allocator,
allocation_pool: self.allocation_pool.clone(),
context: self.context.clone(),
+ timestamps: self.timestamps.take(),
}
}
}
@@ -245,6 +270,7 @@ pub struct MetalCommandBufferExecutable {
constant_allocator: BumpAllocator<<Metal as Backend>::GlobalBuffer>,
allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
context: Arc<MetalContext>,
+ timestamps: Option<MetalTimestampRecorder>,
}
impl CommandBufferExecutable for MetalCommandBufferExecutable {
@@ -266,14 +292,18 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {
let command_allocator = self.command_allocator.clone();
let command_buffer = self.command_buffer.clone();
let context_clone = self.context.clone();
+ let timestamps = self.timestamps;
let constant_allocator = self.constant_allocator;
let allocation_pool = self.allocation_pool.clone();
let feedback_handler = move |feedback: &ProtocolObject<dyn MTL4CommitFeedback>| {
let message = if let Some(error) = feedback.error() {
- Err(error.to_string())
+ Err(MetalError::CommandBufferExecution(error.to_string()))
} else {
- Ok(Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()))
+ timestamps
+ .as_ref()
+ .map_or_else(|| Ok(Box::default()), MetalTimestampRecorder::resolve)
+ .map(|spans| (Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()), spans))
};
let _ = sender.send(message);
context_clone.command_buffer_cache.lock().push(MetalCommandBufferCache {
@@ -296,19 +326,17 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {
pub struct MetalCommandBufferPending {
allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
- receiver: mpsc::Receiver<Result<Duration, String>>,
+ receiver: mpsc::Receiver<Result<(Duration, Box<[TimestampSpan]>), MetalError>>,
}
impl CommandBufferPending for MetalCommandBufferPending {
type CommandBuffer = MetalCommandBuffer;
fn wait_until_completed(self) -> Result<MetalCommandBufferCompleted, MetalError> {
+ let (gpu_execution_time, timestamps) = self.receiver.recv().map_err(MetalError::CommandBufferWait)??;
Ok(MetalCommandBufferCompleted {
- gpu_execution_time: self
- .receiver
- .recv()
- .map_err(MetalError::CommandBufferWait)?
- .map_err(MetalError::CommandBufferExecution)?,
+ gpu_execution_time,
+ timestamps,
_allocation_pool: self.allocation_pool,
})
}
@@ -316,6 +344,7 @@ impl CommandBufferPending for MetalCommandBufferPending {
pub struct MetalCommandBufferCompleted {
gpu_execution_time: Duration,
+ timestamps: Box<[TimestampSpan]>,
_allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
}
@@ -325,6 +354,10 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {
fn gpu_execution_time(&self) -> Duration {
self.gpu_execution_time
}
+
+ fn timestamps(&self) -> &[TimestampSpan] {
+ &self.timestamps
+ }
}
pub struct MetalCommandBufferCache {
diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rs
index e51351964..83b5033e6 100644
--- a/crates/uzu-engine/src/backends/metal/error.rs
+++ b/crates/uzu-engine/src/backends/metal/error.rs
@@ -27,6 +27,10 @@ pub enum MetalError {
CannotCreateBuffer,
#[error("Cannot create heap")]
CannotCreateHeap,
+ #[error("Cannot resolve counter heap")]
+ CannotResolveCounterHeap,
+ #[error("Timestamp {0} was not written by the GPU")]
+ UnwrittenTimestamp(usize),
#[error("Cannot create command buffer")]
CannotCreateCommandBuffer,
#[error("Cannot create argument table: {0}")]
diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rs
index 01324c58f..5fc8da6ee 100644
--- a/crates/uzu-engine/src/backends/metal/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/mod.rs
@@ -7,6 +7,7 @@ mod error;
mod heaps;
mod kernel;
mod metal_extensions;
+mod timestamp_recorder;
pub use backend::Metal;
pub use context::MetalContext; // TODO: This should be removed
diff --git a/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs
new file mode 100644
index 000000000..919245b51
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs
@@ -0,0 +1,102 @@
+use std::time::{Duration, Instant};
+
+use metal::{
+ MTL4ComputeCommandEncoder, MTL4CounterHeap, MTL4CounterHeapDescriptor, MTL4CounterHeapExt, MTL4CounterHeapType,
+ MTL4TimestampGranularity, MTLDevice, MTLDeviceExt,
+};
+use objc2::{rc::Retained, runtime::ProtocolObject};
+
+use crate::backends::{
+ common::{TimestampSpan, TimestampSpanRecorder},
+ metal::error::MetalError,
+};
+
+// "Maximum counter sample buffer length" is 32 KB, i.e. 4096 timestamp entries of 8 bytes (`MTL4TimestampHeapEntry`):
+// https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
+const COUNTER_HEAP_CAPACITY: usize = 4096;
+
+pub struct MetalTimestampRecorder {
+ device: Retained<ProtocolObject<dyn MTLDevice>>,
+ spans: TimestampSpanRecorder,
+ counter_heaps: Vec<Retained<ProtocolObject<dyn MTL4CounterHeap>>>,
+}
+
+impl MetalTimestampRecorder {
+ pub fn new(device: Retained<ProtocolObject<dyn MTLDevice>>) -> Self {
+ Self {
+ device,
+ spans: TimestampSpanRecorder::default(),
+ counter_heaps: Vec::new(),
+ }
+ }
+
+ pub fn start(
+ &mut self,
+ name: String,
+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,
+ ) {
+ let slot = self.spans.start(name);
+ self.write(slot, encoder);
+ }
+
+ pub fn end(
+ &mut self,
+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,
+ ) {
+ let slot = self.spans.end();
+ self.write(slot, encoder);
+ }
+
+ fn write(
+ &mut self,
+ slot: usize,
+ encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,
+ ) {
+ if slot / COUNTER_HEAP_CAPACITY == self.counter_heaps.len() {
+ let descriptor = MTL4CounterHeapDescriptor::new();
+ descriptor.set_type(MTL4CounterHeapType::TIMESTAMP);
+ descriptor.set_count(COUNTER_HEAP_CAPACITY);
+ let counter_heap =
+ self.device.new_counter_heap_with_descriptor(&descriptor).expect("Failed to create a counter heap");
+ counter_heap.invalidate_counter_range(0..COUNTER_HEAP_CAPACITY);
+ self.counter_heaps.push(counter_heap);
+ }
+ encoder.write_timestamp_with_granularity_into_heap_at_index(
+ MTL4TimestampGranularity::PRECISE,
+ &self.counter_heaps[slot / COUNTER_HEAP_CAPACITY],
+ slot % COUNTER_HEAP_CAPACITY,
+ );
+ }
+
+ pub fn resolve(&self) -> Result<Box<[TimestampSpan]>, MetalError> {
+ let count = self.spans.slot_count();
+ let mut ticks = Vec::with_capacity(count);
+ for (index, counter_heap) in self.counter_heaps.iter().enumerate() {
+ let slots = 0..(count - index * COUNTER_HEAP_CAPACITY).min(COUNTER_HEAP_CAPACITY);
+ let bytes = counter_heap.resolve_counter_range(slots).ok_or(MetalError::CannotResolveCounterHeap)?;
+ let (entries, _) = bytes.as_chunks::<{ size_of::<u64>() }>();
+ ticks.extend(entries.iter().map(|entry| u64::from_ne_bytes(*entry)));
+ }
+ let frequency = self.device.query_timestamp_frequency() as u128;
+ let (mut cpu_now, mut gpu_now) = (0, 0);
+ let now = Instant::now();
+ self.device.sample_timestamps_gpu_timestamp(&mut cpu_now, &mut gpu_now);
+ let to_instant = |ticks: u64| {
+ let nanoseconds = (ticks as u128 * 1_000_000_000 / frequency) as u64;
+ now - Duration::from_nanos(gpu_now - nanoseconds)
+ };
+ let instants = ticks
+ .iter()
+ .enumerate()
+ .scan(None, |previous_timestamp, (slot, &slot_ticks)| {
+ let timestamp = match slot_ticks {
+ 0 => previous_timestamp.ok_or(MetalError::UnwrittenTimestamp(slot)),
+ slot_ticks => Ok(to_instant(slot_ticks)),
+ };
+ *previous_timestamp = timestamp.as_ref().ok().copied();
+ Some(timestamp)
+ })
+ .collect::<Result<Box<[Instant]>, MetalError>>()?;
+ Ok(self.spans.spans(&instants))
+ }
+}
diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rs
index f44077a95..d343d583e 100644
--- a/crates/uzu-engine/src/encodable_block/classifier.rs
+++ b/crates/uzu-engine/src/encodable_block/classifier.rs
@@ -35,6 +35,7 @@ pub enum ClassifierError<B: Backend> {
}
pub struct Classifier<B: Backend> {
+ name: String,
hidden_dim: u32,
data_type: DataType,
embedding: Embedding<B>,
@@ -46,12 +47,14 @@ pub struct Classifier<B: Backend> {
impl<B: Backend> Classifier<B> {
pub fn new(
+ name: String,
context: &B::Context,
config: &ClassifierConfig,
parameter_tree: &ParameterTree<B>,
data_type: DataType,
) -> Result<Self, ClassifierError<B>> {
let (embedding, _) = Embedding::new(
+ format!("{name}/embedding"),
context,
config.vocab_size,
config.transformer_config.model_dim,
@@ -61,6 +64,7 @@ impl<B: Backend> Classifier<B> {
)?;
let embedding_norm = Normalization::new(
+ format!("{name}/embedding norm"),
config.transformer_config.model_dim,
None,
ShortcutMode::None,
@@ -72,6 +76,7 @@ impl<B: Backend> Classifier<B> {
)?;
let transformer = Transformer::new(
+ format!("{name}/transformer"),
context,
None,
data_type,
@@ -89,6 +94,7 @@ impl<B: Backend> Classifier<B> {
<B::Kernels as Kernels>::PoolingMeanKernel::new(context, data_type).map_err(ClassifierError::Backend)?;
let prediction_head = PredictionHead::new(
+ format!("{name}/prediction head"),
config.hidden_dim,
config.num_labels,
data_type,
@@ -98,6 +104,7 @@ impl<B: Backend> Classifier<B> {
)?;
Ok(Self {
+ name,
hidden_dim: config.hidden_dim,
data_type,
embedding,
@@ -118,7 +125,8 @@ impl<B: Backend> Classifier<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, ClassifierError<B>> {
- command_buffer.push_debug_group("classifier");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let embedded = self.embedding.encode_lookup(token_ids, batch_dim, command_buffer)?;
@@ -156,6 +164,7 @@ impl<B: Backend> Classifier<B> {
let logits = self.prediction_head.encode(pooled, 1, command_buffer).map_err(ClassifierError::Backend)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(logits)
diff --git a/crates/uzu-engine/src/encodable_block/context_ring_update.rs b/crates/uzu-engine/src/encodable_block/context_ring_update.rs
new file mode 100644
index 000000000..1573b04b7
--- /dev/null
+++ b/crates/uzu-engine/src/encodable_block/context_ring_update.rs
@@ -0,0 +1,35 @@
+use crate::backends::common::{
+ Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, kernel::ContextRingUpdateKernel,
+};
+
+pub struct ContextRingUpdate<B: Backend> {
+ name: String,
+ kernel: <B::Kernels as Kernels>::ContextRingUpdateKernel,
+}
+
+impl<B: Backend> ContextRingUpdate<B> {
+ pub fn new(
+ name: String,
+ context: &B::Context,
+ ) -> Result<Self, B::Error> {
+ Ok(Self {
+ name,
+ kernel: <B::Kernels as Kernels>::ContextRingUpdateKernel::new(context)?,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ input: impl BufferRef<Backend = B>,
+ context_ring: impl BufferMut<Backend = B>,
+ suffix_repetition_length: u32,
+ input_length: u32,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) {
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
+ self.kernel.encode(input, context_ring, suffix_repetition_length, input_length, command_buffer);
+ command_buffer.sample_end_timestamp();
+ command_buffer.pop_debug_group();
+ }
+}
diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rs
index c6c852b6a..1f9afb10e 100644
--- a/crates/uzu-engine/src/encodable_block/convolution.rs
+++ b/crates/uzu-engine/src/encodable_block/convolution.rs
@@ -21,6 +21,7 @@ pub enum ConvolutionNewError<B: Backend> {
}
pub struct SeparableCausalConv<B: Backend> {
+ name: String,
model_dim: u32,
data_type: DataType,
weights: B::GlobalBuffer,
@@ -30,6 +31,7 @@ pub struct SeparableCausalConv<B: Backend> {
impl<B: Backend> SeparableCausalConv<B> {
pub fn new(
+ name: String,
model_dim: u32,
kernel_size: u32,
group_size: u32,
@@ -62,6 +64,7 @@ impl<B: Backend> SeparableCausalConv<B> {
.map_err(ConvolutionNewError::Backend)?;
Ok(Self {
+ name,
model_dim,
data_type,
weights,
@@ -79,7 +82,8 @@ impl<B: Backend> SeparableCausalConv<B> {
sequence_length: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("SeparableCausalConv");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let mut output =
command_buffer.allocate_scratch_for_shape(&[sequence_length, self.model_dim], self.data_type)?;
@@ -95,6 +99,7 @@ impl<B: Backend> SeparableCausalConv<B> {
command_buffer,
);
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
}
diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rs
index 7b1c8bbfd..5476b386a 100644
--- a/crates/uzu-engine/src/encodable_block/decoder.rs
+++ b/crates/uzu-engine/src/encodable_block/decoder.rs
@@ -32,6 +32,7 @@ pub enum DecoderError<B: Backend> {
}
pub struct Decoder<B: Backend> {
+ name: String,
embedding: Embedding<B>,
embedding_norm: Option<Normalization<B>>,
per_layer_embedding: Option<PerLayerEmbedding<B>>,
@@ -50,12 +51,14 @@ impl<B: Backend> Decoder<B> {
}
pub fn new(
+ name: String,
context: &B::Context,
config: &DecoderConfig,
parameter_tree: &ParameterTree<B>,
data_type: DataType,
) -> Result<Self, DecoderError<B>> {
let (embedding, readout_input_hadamard_factors) = Embedding::new(
+ format!("{name}/embedding"),
context,
config.vocab_size,
config.transformer_config.model_dim,
@@ -69,6 +72,7 @@ impl<B: Backend> Decoder<B> {
.as_ref()
.map(|norm_config| {
Normalization::new(
+ format!("{name}/embedding norm"),
config.transformer_config.model_dim,
None,
ShortcutMode::None,
@@ -88,6 +92,7 @@ impl<B: Backend> Decoder<B> {
"per-layer embedding num_layers must match transformer layer count"
);
Some(PerLayerEmbedding::new(
+ format!("{name}/per layer embedding"),
context,
ple_config,
config.transformer_config.model_dim,
@@ -99,6 +104,7 @@ impl<B: Backend> Decoder<B> {
};
let transformer = Transformer::new(
+ format!("{name}/transformer"),
context,
readout_input_hadamard_factors,
data_type,
@@ -107,6 +113,7 @@ impl<B: Backend> Decoder<B> {
)?;
Ok(Self {
+ name,
embedding,
embedding_norm,
per_layer_embedding,
@@ -143,7 +150,8 @@ impl<B: Backend> Decoder<B> {
state: &mut TransformerState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {
- command_buffer.push_debug_group("decoder");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), command_buffer)?;
let embedded = if let Some(embedding_norm) = &self.embedding_norm {
@@ -196,6 +204,7 @@ impl<B: Backend> Decoder<B> {
transformer_output.output
};
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(DecoderEncodeOutput {
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index b35383899..4c90073fa 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -41,6 +41,8 @@ impl<B: Backend> DFlashState<B> {
}
pub struct DFlash<B: Backend> {
+ accept_name: String,
+ draft_name: String,
target_feature_projection: Box<dyn Linear<B>>,
projected_feature_norm: Normalization<B>,
state_kv_projection: Box<dyn Linear<B>>,
@@ -89,6 +91,7 @@ pub enum DFlashEncodeError<B: Backend> {
impl<B: Backend> DFlash<B> {
pub fn new(
+ name: String,
context: &B::Context,
config: &DFlashDraftConfig,
parameter_tree: &ParameterTree<B>,
@@ -97,6 +100,7 @@ impl<B: Backend> DFlash<B> {
let mask_token_id = config.mask_token_id as u32;
assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");
let target_feature_projection = <dyn Linear<B>>::new(
+ format!("{name}/target feature projection"),
config.model_dim * config.target_layer_ids.len() as u32,
[config.model_dim],
false,
@@ -105,6 +109,7 @@ impl<B: Backend> DFlash<B> {
&parameter_tree.subtree("context_projection"),
)?;
let projected_feature_norm = Normalization::new(
+ format!("{name}/projected feature norm"),
config.model_dim,
None,
ShortcutMode::None,
@@ -122,6 +127,7 @@ impl<B: Backend> DFlash<B> {
let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;
let num_layers = config.layer_configs.len() as u32;
let state_kv_projection = <dyn Linear<B>>::new(
+ format!("{name}/state kv projection"),
config.model_dim,
[num_layers * layer_kv_dim],
false,
@@ -135,6 +141,7 @@ impl<B: Backend> DFlash<B> {
.zip(0u32..)
.map(|(layer_config, index)| {
TransformerLayer::new(
+ format!("{name}/layer {index}"),
context,
config.model_dim,
config.hidden_dim,
@@ -147,6 +154,7 @@ impl<B: Backend> DFlash<B> {
})
.collect::<Result<Box<[_]>, TransformerLayerError<B>>>()?;
let output_norm = Normalization::new(
+ format!("{name}/output norm"),
config.model_dim,
None,
ShortcutMode::Add,
@@ -158,6 +166,8 @@ impl<B: Backend> DFlash<B> {
)?;
Ok(Self {
+ accept_name: format!("{name}/accept"),
+ draft_name: format!("{name}/draft"),
target_feature_projection,
projected_feature_norm,
state_kv_projection,
@@ -207,7 +217,8 @@ impl<B: Backend> DFlash<B> {
return Ok(());
}
- command_buffer.push_debug_group("dflash accept");
+ command_buffer.push_debug_group(&self.accept_name);
+ command_buffer.sample_start_timestamp(&self.accept_name);
let num_tokens = accepted_indices.len() as u32;
let captured_layer_count = self.target_feature_input_dim / self.model_dim;
@@ -274,6 +285,7 @@ impl<B: Backend> DFlash<B> {
state.context_length += num_tokens;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(())
@@ -287,7 +299,8 @@ impl<B: Backend> DFlash<B> {
batch_size: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {
- command_buffer.push_debug_group("dflash draft");
+ command_buffer.push_debug_group(&self.draft_name);
+ command_buffer.sample_start_timestamp(&self.draft_name);
assert!(batch_size >= 2 && batch_size <= self.block_size, "batch size exceeds DFlash block size");
assert!(
@@ -348,6 +361,7 @@ impl<B: Backend> DFlash<B> {
command_buffer,
)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(DFlashOutput {
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index fd218c54a..8f386db84 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -45,6 +45,7 @@ enum EmbeddingTying<B: Backend> {
}
pub struct Embedding<B: Backend> {
+ readout_name: String,
tying: EmbeddingTying<B>,
input_scale: f32,
data_type: DataType,
@@ -69,6 +70,7 @@ impl<B: Backend> Embedding<B> {
}
pub fn new(
+ name: String,
context: &B::Context,
vocab_size: u32,
model_dim: u32,
@@ -79,7 +81,14 @@ impl<B: Backend> Embedding<B> {
let (tying, readout_input_hadamard_factors) = match config {
AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {
let embedding_tree = parameter_tree.subtree("embedding");
- let table = EmbeddingTable::load(context, &embedding_tree, vocab_size, model_dim, data_type)?;
+ let table = EmbeddingTable::load(
+ format!("{name}/lookup"),
+ context,
+ &embedding_tree,
+ vocab_size,
+ model_dim,
+ data_type,
+ )?;
if table.as_matrix().is_none() {
return Err(EmbeddingError::UnsupportedConfiguration("tied embeddings need a matrix table".into()));
}
@@ -100,11 +109,18 @@ impl<B: Backend> Embedding<B> {
},
AnyEmbeddingConfig::UntiedEmbeddingConfig(_) => {
let input_embedding_tree = parameter_tree.subtree("input_embedding");
- let input_table =
- EmbeddingTable::load(context, &input_embedding_tree, vocab_size, model_dim, data_type)?;
+ let input_table = EmbeddingTable::load(
+ format!("{name}/lookup"),
+ context,
+ &input_embedding_tree,
+ vocab_size,
+ model_dim,
+ data_type,
+ )?;
let output_embedding_tree = parameter_tree.subtree("output_embedding");
let output_embedding_spec = output_embedding_tree.metadata("spec")?;
let output = UntiedReadout::load(
+ format!("{name}/readout/untied"),
context,
&output_embedding_tree,
output_embedding_spec,
@@ -140,6 +156,7 @@ impl<B: Backend> Embedding<B> {
Ok((
Self {
+ readout_name: format!("{name}/readout"),
tying,
input_scale,
data_type,
@@ -157,8 +174,6 @@ impl<B: Backend> Embedding<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, EmbeddingError<B>> {
- command_buffer.push_debug_group("embedding lookup");
-
let mut output = command_buffer
.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)
.map_err(EmbeddingError::BackendError)?;
@@ -175,8 +190,6 @@ impl<B: Backend> Embedding<B> {
};
table.encode_lookup(token_ids, &mut output, batch_dim, self.input_scale, command_buffer);
- command_buffer.pop_debug_group();
-
Ok(output)
}
@@ -189,7 +202,8 @@ impl<B: Backend> Embedding<B> {
apply_logit_transform: bool,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, EmbeddingError<B>> {
- command_buffer.push_debug_group("embedding readout");
+ command_buffer.push_debug_group(&self.readout_name);
+ command_buffer.sample_start_timestamp(&self.readout_name);
assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");
let mut output_buffer = match &self.tying {
@@ -240,6 +254,7 @@ impl<B: Backend> Embedding<B> {
);
}
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output_buffer)
diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rs
index 7dae6806a..dee8a1974 100644
--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs
@@ -3,7 +3,7 @@ use thiserror::Error;
use crate::{
backends::common::{
- Backend, BufferMut, BufferRef, CommandBuffer, Kernels,
+ Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,
gpu_types::{EmbeddingTableKind, HADAMARD_TRANSFORM_BLOCK_SIZE, d4s4},
kernel::InputEmbeddingLookupKernel,
},
@@ -81,6 +81,7 @@ impl<B: Backend> Storage<B> {
}
pub struct EmbeddingTable<B: Backend> {
+ name: String,
storage: Storage<B>,
output_hadamard_factors: Option<B::GlobalBuffer>,
lookup: LookupKernel<B>,
@@ -90,6 +91,7 @@ pub struct EmbeddingTable<B: Backend> {
impl<B: Backend> EmbeddingTable<B> {
pub fn load(
+ name: String,
context: &B::Context,
tree: &ParameterTree<B>,
vocab_size: u32,
@@ -150,6 +152,7 @@ impl<B: Backend> EmbeddingTable<B> {
.map_err(EmbeddingTableError::BackendError)?;
Ok(Self {
+ name,
storage,
output_hadamard_factors,
lookup,
@@ -175,6 +178,8 @@ impl<B: Backend> EmbeddingTable<B> {
scale: f32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let bindings = self.storage.lookup_bindings();
self.lookup.encode(
token_ids,
@@ -193,6 +198,8 @@ impl<B: Backend> EmbeddingTable<B> {
scale,
command_buffer,
);
+ command_buffer.sample_end_timestamp();
+ command_buffer.pop_debug_group();
}
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
index 2c82b4d93..4db7675fe 100644
--- a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
@@ -10,6 +10,7 @@ use crate::{
};
pub(super) struct InputRht<B: Backend> {
+ name: String,
rht_signs: B::GlobalBuffer,
rht: ActivationTransform<B>,
quantizer: Option<ActivationTransform<B>>,
@@ -17,6 +18,7 @@ pub(super) struct InputRht<B: Backend> {
impl<B: Backend> InputRht<B> {
pub(super) fn new(
+ name: String,
context: &B::Context,
data_type: DataType,
preparation: LinearInputPreparation<B>,
@@ -32,6 +34,7 @@ impl<B: Backend> InputRht<B> {
.transpose()?;
Ok(Self {
+ name,
rht_signs,
rht,
quantizer,
@@ -45,6 +48,7 @@ impl<B: Backend> InputRht<B> {
format: ActivationFormat,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<LinearInput<B>, B::Error> {
+ command_buffer.sample_start_timestamp(&self.name);
if format == ActivationFormat::Int8
&& let Some(quantizer) = &self.quantizer
{
@@ -70,6 +74,7 @@ impl<B: Backend> InputRht<B> {
command_buffer,
);
+ command_buffer.sample_end_timestamp();
return Ok(LinearInput::Int8Symmetric {
values,
scales,
@@ -82,6 +87,7 @@ impl<B: Backend> InputRht<B> {
let input_dim = self.input_dim();
let mut transformed = command_buffer.allocate_scratch(input.size())?;
self.rht.encode_fp(input, &mut transformed, &self.rht_signs, batch_dim, input_dim, command_buffer);
+ command_buffer.sample_end_timestamp();
Ok(LinearInput::FullPrecision(transformed))
}
@@ -96,8 +102,10 @@ impl<B: Backend> InputRht<B> {
return self.prepare(&input, batch_dim, format, command_buffer);
}
+ command_buffer.sample_start_timestamp(&self.name);
let input_dim = self.input_dim();
self.rht.encode_fp_in_place(&mut input, &self.rht_signs, None, batch_dim, input_dim, command_buffer);
+ command_buffer.sample_end_timestamp();
Ok(LinearInput::FullPrecision(input))
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
index 4f7f6a9ad..da0ff8c77 100644
--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
@@ -37,6 +37,7 @@ pub enum LinearMatmulError<B: Backend> {
}
pub struct LinearMatmul<B: Backend> {
+ name: String,
kernel: Mutex<<B::Kernels as Kernels>::MatmulKernel>,
matrix: WeightMatrix<B>,
biases: Option<B::GlobalBuffer>,
@@ -66,6 +67,7 @@ impl<B: Backend> LinearMatmul<B> {
/// Loads a linear over any parsed spec — full precision, MLX or Int. Hybrid
/// compositions live in the wrappers, not here.
pub fn load(
+ name: String,
context: &B::Context,
spec: AnyWeightMatrixSpec,
input_dim: u32,
@@ -98,6 +100,7 @@ impl<B: Backend> LinearMatmul<B> {
.map_err(LinearMatmulError::BackendError)?;
Ok(Self {
+ name,
kernel: Mutex::new(kernel),
matrix,
biases,
@@ -125,6 +128,7 @@ impl<B: Backend> LinearMatmul<B> {
gather: Option<Gather<impl BufferRef<Backend = B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.name);
let (output_dim, gather_indices) =
gather.map_or((self.output_dim, None), |gather| (gather.output_dim, Some(gather.indices)));
let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, output_dim], self.output_data_type)?;
@@ -143,6 +147,7 @@ impl<B: Backend> LinearMatmul<B> {
command_buffer,
)?;
+ command_buffer.sample_end_timestamp();
Ok(output)
}
@@ -190,7 +195,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("matmul");
+ command_buffer.push_debug_group(&self.name);
let output = self.encode_with_a(
MatmulA::FullPrecision {
diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rs
index 78dfdfb93..9c49aae44 100644
--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs
@@ -123,6 +123,7 @@ pub enum LinearBlockError<B: Backend> {
impl<B: Backend> dyn Linear<B> {
pub fn new_mixed_precision(
+ name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -140,6 +141,7 @@ impl<B: Backend> dyn Linear<B> {
| AnyWeightMatrixSpec::MLXSpec(_)
| AnyWeightMatrixSpec::IntSpec(_)) => {
let block = LinearMatmul::load(
+ name,
context,
spec,
input_dimension,
@@ -159,6 +161,7 @@ impl<B: Backend> dyn Linear<B> {
incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
..
}) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => Ok(Box::new(RHTLinearWrapper::new(
+ name,
context,
input_dimension,
output_dimension_sum,
@@ -181,6 +184,7 @@ impl<B: Backend> dyn Linear<B> {
return Err(LinearBlockError::UnsupportedConfiguration(format!("{adapter_spec:?}")));
};
Ok(Box::new(QLoRALinearWrapper::new(
+ name,
context,
*quantization_spec,
adapter_spec,
@@ -199,6 +203,7 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new(
+ name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -207,6 +212,7 @@ impl<B: Backend> dyn Linear<B> {
parameter_tree: &ParameterTree<B>,
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
Self::new_mixed_precision(
+ name,
input_dimension,
output_dimensions,
has_biases,
@@ -219,6 +225,7 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new_with_input_rht_mixed_precision(
+ name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -230,6 +237,7 @@ impl<B: Backend> dyn Linear<B> {
) -> Result<(Box<dyn Linear<B>>, Option<B::GlobalBuffer>), LinearBlockError<B>> {
let output_dimension_sum: u32 = output_dimensions.as_ref().iter().sum();
if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(
+ name.clone(),
context,
input_dimension,
output_dimension_sum,
@@ -244,6 +252,7 @@ impl<B: Backend> dyn Linear<B> {
}
let linear = Self::new_mixed_precision(
+ name,
input_dimension,
output_dimensions,
has_biases,
@@ -257,6 +266,7 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new_for_fused_input(
+ name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -266,6 +276,7 @@ impl<B: Backend> dyn Linear<B> {
) -> Result<(Box<dyn Linear<B>>, Option<LinearInputPreparation<B>>), LinearBlockError<B>> {
let output_dimension_sum: u32 = output_dimensions.as_ref().iter().sum();
if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(
+ name.clone(),
context,
input_dimension,
output_dimension_sum,
@@ -280,6 +291,7 @@ impl<B: Backend> dyn Linear<B> {
}
let linear = Self::new_mixed_precision(
+ name,
input_dimension,
output_dimensions,
has_biases,
@@ -293,6 +305,7 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new_with_input_rht(
+ name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -301,6 +314,7 @@ impl<B: Backend> dyn Linear<B> {
parameter_tree: &ParameterTree<B>,
) -> Result<(Box<dyn Linear<B>>, Option<B::GlobalBuffer>), LinearBlockError<B>> {
Self::new_with_input_rht_mixed_precision(
+ name,
input_dimension,
output_dimensions,
has_biases,
diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
index d4faa24b9..dfdf74851 100644
--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
@@ -35,6 +35,7 @@ pub enum QLoRALinearWrapperError<B: Backend> {
}
pub struct QLoRALinearWrapper<B: Backend> {
+ name: String,
base_linear: LinearMatmul<B>,
input_hadamard: Option<(ActivationTransform<B>, B::GlobalBuffer)>,
output_hadamard: Option<(ActivationTransform<B>, B::GlobalBuffer)>,
@@ -51,6 +52,7 @@ pub struct QLoRALinearWrapper<B: Backend> {
impl<B: Backend> QLoRALinearWrapper<B> {
pub fn new(
+ name: String,
context: &B::Context,
quantization_spec: AnyWeightMatrixSpec,
adapter_spec: LowRankSpec,
@@ -85,6 +87,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
));
};
let base_linear = LinearMatmul::load(
+ format!("{name}/base"),
context,
quantization_spec,
input_dim,
@@ -152,6 +155,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
.read_buffer()?;
Ok(Self {
+ name,
base_linear,
input_hadamard,
output_hadamard,
@@ -175,7 +179,8 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("linear (qlora)");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let mut intermediate =
command_buffer.allocate_scratch_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type)?;
@@ -261,6 +266,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
);
}
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index c4dbbe8d8..bcfaebc11 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -27,6 +27,7 @@ pub enum RHTLinearWrapperError<B: Backend> {
}
pub struct RHTLinearWrapper<B: Backend> {
+ name: String,
input_rht: InputRht<B>,
inner_linear: LinearMatmul<B>,
}
@@ -45,6 +46,7 @@ fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {
impl<B: Backend> RHTLinearWrapper<B> {
pub(super) fn new(
+ name: String,
context: &B::Context,
input_dimension: u32,
output_dimension: u32,
@@ -61,6 +63,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(
+ format!("{name}/matmul"),
context,
input_dimension,
output_dimension,
@@ -72,6 +75,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
)?;
let activation_quantization = inner_linear.prepare_a8(context);
Self::build_self_contained(
+ name,
context,
input_data_type,
LinearInputPreparation {
@@ -83,6 +87,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
pub(super) fn try_new_with_input_preparation(
+ name: String,
context: &B::Context,
input_dimension: u32,
output_dimension: u32,
@@ -100,6 +105,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(
+ format!("{name}/matmul"),
context,
input_dimension,
output_dimension,
@@ -114,7 +120,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
activation_quantization: inner_linear.prepare_a8(context),
};
if input_preparation.activation_quantization.is_some() && !allow_prequantized_activation {
- let wrapper = Self::build_self_contained(context, input_data_type, input_preparation, inner_linear)?;
+ let wrapper = Self::build_self_contained(name, context, input_data_type, input_preparation, inner_linear)?;
Ok(Some((Box::new(wrapper), None)))
} else {
Ok(Some((Box::new(inner_linear), Some(input_preparation))))
@@ -122,6 +128,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
fn load_inner_with_output_rht(
+ name: String,
context: &B::Context,
input_dimension: u32,
output_dimension: u32,
@@ -143,6 +150,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
.validate(&[output_dimension], DataType::I32)?
.read_buffer()?;
let inner_linear = LinearMatmul::load(
+ name,
context,
quantization_spec,
input_dimension,
@@ -158,15 +166,17 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
fn build_self_contained(
+ name: String,
context: &B::Context,
input_data_type: DataType,
input_preparation: LinearInputPreparation<B>,
inner_linear: LinearMatmul<B>,
) -> Result<Self, RHTLinearWrapperError<B>> {
- let input_rht = InputRht::new(context, input_data_type, input_preparation, true)
+ let input_rht = InputRht::new(format!("{name}/prepare"), context, input_data_type, input_preparation, true)
.map_err(RHTLinearWrapperError::BackendError)?;
Ok(Self {
+ name,
input_rht,
inner_linear,
})
@@ -189,12 +199,14 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("linear (rht)");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let input = match input {
LinearInput::FullPrecision(input) => input,
input => {
let output = self.inner_linear.encode_input(input, batch_dim, command_buffer);
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
return output;
},
@@ -208,6 +220,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
command_buffer,
)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
index a993ff14e..65fb42175 100644
--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
@@ -16,12 +16,14 @@ use crate::{
};
pub struct UntiedReadout<B: Backend> {
+ name: String,
linear: LinearMatmul<B>,
input_rht: Option<InputRht<B>>,
}
impl<B: Backend> UntiedReadout<B> {
pub fn load(
+ name: String,
context: &B::Context,
tree: &ParameterTree<B>,
spec: AnyWeightMatrixSpec,
@@ -39,13 +41,25 @@ impl<B: Backend> UntiedReadout<B> {
else {
return Ok(Self {
linear: LinearMatmul::load(
- context, spec, model_dim, vocab_size, data_type, data_type, data_type, tree, None, None,
+ format!("{name}/matmul"),
+ context,
+ spec,
+ model_dim,
+ vocab_size,
+ data_type,
+ data_type,
+ data_type,
+ tree,
+ None,
+ None,
)?,
input_rht: None,
+ name,
});
};
let mut linear = LinearMatmul::load(
+ format!("{name}/matmul"),
context,
*quantization_spec,
model_dim,
@@ -66,10 +80,12 @@ impl<B: Backend> UntiedReadout<B> {
rht_signs,
activation_quantization: linear.prepare_a8(context),
};
- let input_rht = InputRht::new(context, data_type, preparation, /* in_place */ false)
- .map_err(LinearMatmulError::BackendError)?;
+ let input_rht =
+ InputRht::new(format!("{name}/prepare"), context, data_type, preparation, /* in_place */ false)
+ .map_err(LinearMatmulError::BackendError)?;
Ok(Self {
+ name,
linear,
input_rht: Some(input_rht),
})
@@ -82,12 +98,15 @@ impl<B: Backend> UntiedReadout<B> {
gather: Option<Gather<impl BufferRef<Backend = B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.name);
let Some(input_rht) = &self.input_rht else {
let a = MatmulA::FullPrecision {
values: input,
offset: 0,
};
- return self.linear.encode_with_a(a, batch_dim, gather, command_buffer);
+ let output = self.linear.encode_with_a(a, batch_dim, gather, command_buffer)?;
+ command_buffer.sample_end_timestamp();
+ return Ok(output);
};
let format = if gather.is_some() {
@@ -96,6 +115,8 @@ impl<B: Backend> UntiedReadout<B> {
self.linear.select_activation_format(batch_dim, command_buffer.context())
};
let input = input_rht.prepare(input, batch_dim, format, command_buffer)?;
- self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)
+ let output = self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)?;
+ command_buffer.sample_end_timestamp();
+ Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
index 9c2b21056..9b540abef 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
@@ -33,6 +33,11 @@ pub use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, KVCacheView};
pub mod rope;
pub struct Attention<B: Backend> {
+ name: String,
+ core_name: String,
+ prepare_kv_and_queries_name: String,
+ prepare_queries_name: String,
+ append_projected_kv_name: String,
head_dim: u32,
num_q_heads: u32,
num_kv_heads: Option<u32>,
@@ -62,6 +67,7 @@ pub enum AttentionNewError<B: Backend> {
impl<B: Backend> Attention<B> {
pub fn new(
+ name: String,
hidden_dim: u32,
data_type: DataType,
rope_config: Option<&AnyRoPEConfig>,
@@ -95,6 +101,7 @@ impl<B: Backend> Attention<B> {
qkv_dim
};
let (projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(
+ format!("{name}/qkv projection"),
hidden_dim,
[projection_dim],
config.has_qkvg_biases,
@@ -110,6 +117,7 @@ impl<B: Backend> Attention<B> {
let qkv_norm = (query_norm_config.is_some() || key_norm_config.is_some() || value_norm_config.is_some())
.then(|| {
QKVNorm::new(
+ format!("{name}/qkv norm"),
context,
data_type,
query_norm_config,
@@ -164,6 +172,7 @@ impl<B: Backend> Attention<B> {
.map_err(AttentionNewError::Backend)?;
let out_projection = <dyn Linear<B>>::new(
+ format!("{name}/out projection"),
q_dim,
[hidden_dim],
config.has_out_biases,
@@ -174,6 +183,11 @@ impl<B: Backend> Attention<B> {
Ok((
Self {
+ core_name: format!("{name}/core"),
+ prepare_kv_and_queries_name: format!("{name}/prepare kv and queries"),
+ prepare_queries_name: format!("{name}/prepare queries"),
+ append_projected_kv_name: format!("{name}/append projected kv"),
+ name,
head_dim,
num_q_heads,
num_kv_heads,
@@ -221,7 +235,8 @@ impl<B: Backend> Mixer<B> for Attention<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("attention");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
assert_eq!(precalculated_rope.is_some(), self.max_rope_length.is_some(), "precalculated rope mismatch");
@@ -229,6 +244,7 @@ impl<B: Backend> Mixer<B> for Attention<B> {
state.map(|state| state.downcast::<AttentionState<B>>().expect("incorrect type of attention state"));
let output = self.attend(hidden, precalculated_rope, batch_dim, state, command_buffer)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
index 75b337ac1..f1d6b7a9d 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
@@ -127,6 +127,7 @@ impl<B: Backend> Attention<B> {
state: &mut AttentionState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
+ command_buffer.sample_start_timestamp(&self.append_projected_kv_name);
if let Some(norm) = &self.projection.norm {
norm.encode_key_value(key_value.reborrow(), batch_dim, command_buffer)?;
}
@@ -142,6 +143,7 @@ impl<B: Backend> Attention<B> {
command_buffer,
)?;
state.encode_accept(&(0..batch_dim).collect::<Box<[u32]>>(), command_buffer)?;
+ command_buffer.sample_end_timestamp();
Ok(())
}
@@ -152,13 +154,14 @@ impl<B: Backend> Attention<B> {
state: &AttentionState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.core_name);
let trie = if batch_dim.is_flat() {
None
} else {
Some(command_buffer.allocate_constant_from_slice(batch_dim.nodes())?)
};
- self.kernel.encode(
+ let output = self.kernel.encode(
AttentionArguments {
queries,
keys: state.keys.as_ref(),
@@ -169,7 +172,9 @@ impl<B: Backend> Attention<B> {
cache: state.view(),
},
command_buffer,
- )
+ )?;
+ command_buffer.sample_end_timestamp();
+ Ok(output)
}
fn prepare_kv_and_queries(
@@ -183,6 +188,7 @@ impl<B: Backend> Attention<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.prepare_kv_and_queries_name);
let num_kv_heads = self.num_kv_heads.expect("KV prepare requires KV heads");
// Appended KV is tightly packed; attention projections may have a trailing gate segment.
let input_row_stride = if num_q_heads == 0 {
@@ -211,6 +217,7 @@ impl<B: Backend> Attention<B> {
batch_dim,
command_buffer,
);
+ command_buffer.sample_end_timestamp();
Ok(queries)
}
@@ -221,6 +228,7 @@ impl<B: Backend> Attention<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.prepare_queries_name);
let mut queries =
command_buffer.allocate_scratch_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type)?;
self.prepare.encode(
@@ -239,6 +247,7 @@ impl<B: Backend> Attention<B> {
batch_dim,
command_buffer,
);
+ command_buffer.sample_end_timestamp();
Ok(queries)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
index 979998d7a..23c1054cb 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
@@ -26,6 +26,7 @@ struct Head<B: Backend> {
}
pub struct QKVNorm<B: Backend> {
+ name: String,
query: Option<Head<B>>,
key: Option<Head<B>>,
value: Option<Head<B>>,
@@ -37,6 +38,7 @@ pub struct QKVNorm<B: Backend> {
impl<B: Backend> QKVNorm<B> {
pub fn new(
+ name: String,
context: &B::Context,
intermediate_data_type: DataType,
query_config: Option<NormalizationConfig>,
@@ -75,6 +77,7 @@ impl<B: Backend> QKVNorm<B> {
.transpose()?;
Ok(Self {
+ name,
query,
key,
value,
@@ -152,7 +155,8 @@ impl<B: Backend> QKVNorm<B> {
"QKV norm input row stride ({input_row_stride}) is smaller than its packed row width ({packed_row_width})"
);
- command_buffer.push_debug_group("qkv norm");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let kv = self.num_kv_heads;
let heads = [(&self.query, 0, q_heads), (&self.key, q_heads, kv), (&self.value, q_heads + kv, kv)];
@@ -179,6 +183,7 @@ impl<B: Backend> QKVNorm<B> {
);
}
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(())
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
index 967fc08cd..6430332e5 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
@@ -3,7 +3,7 @@ use std::any::Any;
use crate::{
array::size_for_shape,
backends::common::{
- Backend, Buffer, CommandBuffer, Context, DeviceCapabilities, Kernels, SparseBuffer,
+ Backend, Buffer, CommandBuffer, CommandBufferEncoding, Context, DeviceCapabilities, Kernels, SparseBuffer,
gpu_types::{Copy, ring::RingParams},
kernel::KVCacheUpdateKernel,
},
@@ -161,6 +161,7 @@ impl KVCacheState {
}
pub struct AttentionState<B: Backend> {
+ accept_name: String,
pub element_dim: u32,
pub data_type: DataType,
cache: KVCacheState,
@@ -215,6 +216,7 @@ impl<B: Backend> AttentionState<B> {
let kv_cache_update = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(context, data_type)?;
Ok(Self {
+ accept_name: format!("{}/accept", attention.name),
element_dim,
data_type,
cache,
@@ -253,6 +255,7 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
accepted_indices: &[u32],
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
+ command_buffer.sample_start_timestamp(&self.accept_name);
let copies = self.cache.accept(accepted_indices);
if !copies.is_empty() {
@@ -266,6 +269,7 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
);
}
+ command_buffer.sample_end_timestamp();
Ok(())
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
index 33e793265..986db8154 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
@@ -48,6 +48,7 @@ enum DeltaNetSuffixStatus<B: Backend> {
}
pub struct DeltaNetState<B: Backend> {
+ accept_name: String,
conv_state: B::GlobalBuffer,
ssm_state: B::GlobalBuffer,
suffix_status: Option<DeltaNetSuffixStatus<B>>,
@@ -68,6 +69,7 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {
accepted_indices: &[u32],
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
+ command_buffer.sample_start_timestamp(&self.accept_name);
let suffix_status = self.suffix_status.take().expect("delta net state has no suffix to accept");
let accepted_index = *accepted_indices.last().expect("delta net state attempted to accept zero indices");
@@ -109,11 +111,14 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {
);
},
}
+ command_buffer.sample_end_timestamp();
Ok(())
}
}
pub struct DeltaNet<B: Backend> {
+ name: String,
+ tree_verify_name: String,
num_heads: u32,
head_dim: u32,
num_groups: u32,
@@ -159,6 +164,7 @@ pub enum DeltaNetNewError<B: Backend> {
impl<B: Backend> DeltaNet<B> {
pub fn new(
+ name: String,
hidden_dim: u32,
outer_data_type: DataType,
config: &DeltaNetConfig,
@@ -191,6 +197,7 @@ impl<B: Backend> DeltaNet<B> {
let (in_projection, in_projection_input_hadamard_factors) =
<dyn Linear<B>>::new_with_input_rht_mixed_precision(
+ format!("{name}/in projection"),
hidden_dim,
[total_proj_dim],
false,
@@ -281,6 +288,7 @@ impl<B: Backend> DeltaNet<B> {
.map_err(DeltaNetNewError::Backend)?;
let out_projection = <dyn Linear<B>>::new_mixed_precision(
+ format!("{name}/out projection"),
value_dim,
[hidden_dim],
false,
@@ -293,6 +301,8 @@ impl<B: Backend> DeltaNet<B> {
Ok((
Self {
+ tree_verify_name: format!("{name}/tree verify"),
+ name,
num_heads: config.num_heads,
head_dim: config.head_dim,
num_groups: config.num_groups,
@@ -334,6 +344,7 @@ impl<B: Backend> DeltaNet<B> {
state: &mut DeltaNetState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.tree_verify_name);
let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");
let tree_size = batch_dim.size();
let parents = command_buffer.allocate_constant_from_slice(batch_dim.parents())?;
@@ -417,6 +428,7 @@ impl<B: Backend> DeltaNet<B> {
beta,
parents: batch_dim.parents().into(),
});
+ command_buffer.sample_end_timestamp();
Ok(output)
}
}
@@ -455,6 +467,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
)?;
Ok(Box::new(DeltaNetState {
+ accept_name: format!("{}/accept", self.name),
conv_state,
ssm_state,
suffix_status: None,
@@ -470,7 +483,8 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("delta net");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");
@@ -487,6 +501,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
if !batch_dim.full_accept() {
let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
return Ok(output);
@@ -631,6 +646,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
let output = self.out_projection.encode(delta_output, batch_dim.size(), command_buffer)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
index 878668519..7027e63cf 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
@@ -55,6 +55,7 @@ enum Mamba2SSDPrefillVariant<B: Backend> {
}
pub struct Mamba2<B: Backend> {
+ name: String,
kernel_size: u32,
num_heads: u32,
num_groups: u32,
@@ -91,6 +92,7 @@ pub enum Mamba2NewError<B: Backend> {
impl<B: Backend> Mamba2<B> {
pub fn new(
+ name: String,
hidden_dim: u32,
outer_data_type: DataType,
config: &Mamba2Config,
@@ -114,6 +116,7 @@ impl<B: Backend> Mamba2<B> {
let (in_projection, in_projection_input_hadamard_factors) =
<dyn Linear<B>>::new_with_input_rht_mixed_precision(
+ format!("{name}/in projection"),
hidden_dim,
[conv_dim, inner_dim, num_heads],
config.has_in_biases,
@@ -165,6 +168,7 @@ impl<B: Backend> Mamba2<B> {
};
let out_projection = <dyn Linear<B>>::new_mixed_precision(
+ format!("{name}/out projection"),
inner_dim,
[hidden_dim],
config.has_out_biases,
@@ -177,6 +181,7 @@ impl<B: Backend> Mamba2<B> {
Ok((
Self {
+ name,
kernel_size,
num_heads,
num_groups,
@@ -243,7 +248,8 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("mamba2");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");
@@ -421,6 +427,7 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
let output = self.out_projection.encode(ssd_output, batch_dim.size(), command_buffer)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
index 5e0c30d66..e9fe51377 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
@@ -83,6 +83,7 @@ pub enum MixerNewError<B: Backend> {
impl<B: Backend> dyn Mixer<B> {
pub fn new(
+ layer_name: &str,
hidden_dim: u32,
data_type: DataType,
rope_config: Option<&AnyRoPEConfig>,
@@ -92,32 +93,57 @@ impl<B: Backend> dyn Mixer<B> {
) -> Result<(Box<dyn Mixer<B>>, Option<B::GlobalBuffer>), MixerNewError<B>> {
match config {
AnyTokenMixerConfig::AttentionConfig(config) => {
- let (attention, in_projection_input_hadamard_factors) =
- Attention::new(hidden_dim, data_type, rope_config, config, parameter_tree, context)?;
+ let (attention, in_projection_input_hadamard_factors) = Attention::new(
+ format!("{layer_name}/attention"),
+ hidden_dim,
+ data_type,
+ rope_config,
+ config,
+ parameter_tree,
+ context,
+ )?;
Ok((Box::new(attention), in_projection_input_hadamard_factors))
},
AnyTokenMixerConfig::DeltaNetConfig(config) => {
assert!(rope_config.is_none(), "unexpected rope for delta net mixer");
- let (delta_net, in_projection_input_hadamard_factors) =
- DeltaNet::new(hidden_dim, data_type, config, parameter_tree, context)?;
+ let (delta_net, in_projection_input_hadamard_factors) = DeltaNet::new(
+ format!("{layer_name}/delta net"),
+ hidden_dim,
+ data_type,
+ config,
+ parameter_tree,
+ context,
+ )?;
Ok((Box::new(delta_net), in_projection_input_hadamard_factors))
},
AnyTokenMixerConfig::Mamba2Config(config) => {
assert!(rope_config.is_none(), "unexpected rope for mamba2 mixer");
- let (mamba2, in_projection_input_hadamard_factors) =
- Mamba2::new(hidden_dim, data_type, config, parameter_tree, context)?;
+ let (mamba2, in_projection_input_hadamard_factors) = Mamba2::new(
+ format!("{layer_name}/mamba2"),
+ hidden_dim,
+ data_type,
+ config,
+ parameter_tree,
+ context,
+ )?;
Ok((Box::new(mamba2), in_projection_input_hadamard_factors))
},
AnyTokenMixerConfig::ShortConvConfig(config) => {
assert!(rope_config.is_none(), "unexpected rope for short conv mixer");
- let (short_conv, in_projection_input_hadamard_factors) =
- ShortConv::new(hidden_dim, data_type, config, parameter_tree, context)?;
+ let (short_conv, in_projection_input_hadamard_factors) = ShortConv::new(
+ format!("{layer_name}/short conv"),
+ hidden_dim,
+ data_type,
+ config,
+ parameter_tree,
+ context,
+ )?;
Ok((Box::new(short_conv), in_projection_input_hadamard_factors))
},
diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
index 93f9cbbb2..4c878daf1 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
@@ -75,6 +75,10 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {
}
pub struct ShortConv<B: Backend> {
+ name: String,
+ decode_conv_name: String,
+ prefill_conv_name: String,
+ trie_conv_name: String,
hidden_dim: u32,
data_type: DataType,
kernel_size: u32,
@@ -102,6 +106,7 @@ pub enum ShortConvNewError<B: Backend> {
impl<B: Backend> ShortConv<B> {
pub fn new(
+ name: String,
hidden_dim: u32,
data_type: DataType,
config: &ShortConvConfig,
@@ -117,6 +122,7 @@ impl<B: Backend> ShortConv<B> {
}
let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(
+ format!("{name}/in projection"),
hidden_dim,
[hidden_dim * 3],
false,
@@ -126,6 +132,7 @@ impl<B: Backend> ShortConv<B> {
)?;
let out_projection = <dyn Linear<B>>::new(
+ format!("{name}/out projection"),
hidden_dim,
[hidden_dim],
false,
@@ -161,6 +168,10 @@ impl<B: Backend> ShortConv<B> {
Ok((
Self {
+ decode_conv_name: format!("{name}/decode conv"),
+ prefill_conv_name: format!("{name}/prefill conv"),
+ trie_conv_name: format!("{name}/trie conv"),
+ name,
hidden_dim,
data_type,
kernel_size,
@@ -183,6 +194,7 @@ impl<B: Backend> ShortConv<B> {
state: &mut ShortConvState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.decode_conv_name);
let mut conv_output = command_buffer.allocate_scratch_for_shape(&[self.hidden_dim], self.data_type)?;
self.short_conv_decode.encode(
in_projected,
@@ -198,6 +210,7 @@ impl<B: Backend> ShortConv<B> {
self.hidden_dim,
command_buffer,
);
+ command_buffer.sample_end_timestamp();
Ok(conv_output)
}
@@ -208,6 +221,7 @@ impl<B: Backend> ShortConv<B> {
state: &mut ShortConvState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.prefill_conv_name);
let state_stride = self.kernel_size - 1;
let padded_rows = state_stride + batch_dim;
@@ -239,6 +253,7 @@ impl<B: Backend> ShortConv<B> {
self.hidden_dim,
command_buffer,
);
+ command_buffer.sample_end_timestamp();
Ok(conv_output)
}
@@ -250,6 +265,7 @@ impl<B: Backend> ShortConv<B> {
state: &mut ShortConvState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {
+ command_buffer.sample_start_timestamp(&self.trie_conv_name);
let mut conv_output =
command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;
let mut conv_states = command_buffer
@@ -269,6 +285,7 @@ impl<B: Backend> ShortConv<B> {
self.hidden_dim,
command_buffer,
);
+ command_buffer.sample_end_timestamp();
Ok((conv_output, conv_states))
}
}
@@ -308,7 +325,8 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("short conv");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
assert!(precalculated_rope.is_none(), "unexpected rope for short conv mixer");
@@ -344,6 +362,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
let output = self.out_projection.encode(conv_output, batch_dim.size(), command_buffer)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
index a0881dee7..d48d52ddc 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
@@ -9,6 +9,7 @@ use crate::{
};
pub struct DenseMlp<B: Backend> {
+ name: String,
up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
down: Box<dyn Linear<B>>,
@@ -16,11 +17,13 @@ pub struct DenseMlp<B: Backend> {
impl<B: Backend> DenseMlp<B> {
pub fn new(
+ name: String,
up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
down: Box<dyn Linear<B>>,
) -> Self {
Self {
+ name,
up,
gate,
down,
@@ -35,13 +38,15 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("mlp (dense)");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let fused_up = self.up.encode(input, batch_dim, command_buffer)?;
let act_format = self.down.select_activation_format(batch_dim, command_buffer.context());
let down_input = self.gate.encode_for_linear(command_buffer, &fused_up, batch_dim, act_format)?;
let output = self.down.encode_input(down_input, batch_dim, command_buffer)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
index 4eab50d2f..45911b711 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
@@ -10,6 +10,7 @@ use crate::{
};
pub struct MlpGateActMulEncodable<B: Backend> {
+ name: String,
fp_kernel: GatedActMul<B>,
activation: AnyActivation,
hidden_dim: u32,
@@ -21,6 +22,7 @@ pub struct MlpGateActMulEncodable<B: Backend> {
impl<B: Backend> MlpGateActMulEncodable<B> {
pub fn new(
+ name: String,
context: &B::Context,
data_type: DataType,
activation: AnyActivation,
@@ -41,6 +43,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
.map(|quantization| GatedActMul::quantized(context, data_type, quantization, settings))
.transpose()?;
Ok(Self {
+ name,
fp_kernel,
activation,
hidden_dim,
@@ -58,7 +61,8 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
batch_dim: u32,
act_format: ActivationFormat,
) -> Result<LinearInput<B>, B::Error> {
- command_buffer.push_debug_group("gate act mul");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
if self.activation.act_type() == ActivationType::IDENTITY {
panic!("Identity activation is not supported for kernel")
@@ -115,6 +119,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
LinearInput::FullPrecision(hidden)
};
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(input)
diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
index 5ae64a74e..30407b40e 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
@@ -35,6 +35,7 @@ pub enum MlpBlockError<B: Backend> {
impl<B: Backend> dyn Mlp<B> {
pub fn new(
+ name: String,
config: &AnyMLPConfig,
model_dimension: u32,
hidden_dimension: u32,
@@ -45,6 +46,7 @@ impl<B: Backend> dyn Mlp<B> {
match config {
AnyMLPConfig::DenseMLPConfig(dense_config) => {
let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(
+ format!("{name}/up projection"),
model_dimension,
[2 * hidden_dimension],
dense_config.has_up_biases,
@@ -54,6 +56,7 @@ impl<B: Backend> dyn Mlp<B> {
)?;
let (down_projection, down_input_preparation) = <dyn Linear<B>>::new_for_fused_input(
+ format!("{name}/down projection"),
hidden_dimension,
[model_dimension],
dense_config.has_down_biases,
@@ -63,6 +66,7 @@ impl<B: Backend> dyn Mlp<B> {
)?;
let gate = MlpGateActMulEncodable::new(
+ format!("{name}/gate act mul"),
context,
data_type,
dense_config.activation.clone(),
@@ -73,7 +77,7 @@ impl<B: Backend> dyn Mlp<B> {
)
.map_err(MlpBlockError::BackendError)?;
- Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))
+ Ok((Box::new(DenseMlp::new(name, up_projection, gate, down_projection)), up_input_hadamard_factors))
},
AnyMLPConfig::MixtureOfExpertsConfig(_) => Err(MlpBlockError::UnsupportedMixtureOfExperts),
}
diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rs
index 9f6c79366..b3f893482 100644
--- a/crates/uzu-engine/src/encodable_block/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mod.rs
@@ -1,5 +1,6 @@
pub mod batch_topology;
pub mod classifier;
+pub mod context_ring_update;
pub mod convolution;
pub mod decoder;
pub mod dflash;
diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rs
index 6fe74ae30..6ce97da6c 100644
--- a/crates/uzu-engine/src/encodable_block/normalization.rs
+++ b/crates/uzu-engine/src/encodable_block/normalization.rs
@@ -37,6 +37,7 @@ pub enum NormalizationNewError<B: Backend> {
}
pub struct Normalization<B: Backend> {
+ name: String,
epsilon: f32,
scale_offset: Option<f32>,
scales: Option<B::GlobalBuffer>,
@@ -50,6 +51,7 @@ pub struct Normalization<B: Backend> {
impl<B: Backend> Normalization<B> {
pub fn new(
+ name: String,
element_count: u32,
hadamard_factors: Option<B::GlobalBuffer>,
shortcut_mode: ShortcutMode,
@@ -100,6 +102,7 @@ impl<B: Backend> Normalization<B> {
.map_err(NormalizationNewError::Backend)?;
Ok(Self {
+ name,
epsilon: config.epsilon,
scale_offset: config.scale_offset,
scales,
@@ -120,7 +123,8 @@ impl<B: Backend> Normalization<B> {
shortcut: Option<impl BufferMut<Backend = B>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("normalization");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let row_size = size_for_shape(&[self.element_count], self.data_type);
let row_offset_bytes = row_offset as usize * row_size;
@@ -141,6 +145,7 @@ impl<B: Backend> Normalization<B> {
command_buffer,
);
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
index a2920a9ea..6c9c5a9ee 100644
--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
@@ -32,6 +32,7 @@ pub enum PerLayerEmbeddingError<B: Backend> {
}
pub struct PerLayerEmbedding<B: Backend> {
+ name: String,
token_embedding: EmbeddingTable<B>,
model_projection: Box<dyn Linear<B>>,
projection_norm: Normalization<B>,
@@ -45,6 +46,7 @@ pub struct PerLayerEmbedding<B: Backend> {
impl<B: Backend> PerLayerEmbedding<B> {
pub fn new(
+ name: String,
context: &B::Context,
config: &PLEModelConfig,
model_dim: u32,
@@ -54,6 +56,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
let total_ple_dim = config.num_layers * config.ple_dim;
let token_embedding = EmbeddingTable::load(
+ format!("{name}/token lookup"),
context,
&parameter_tree.subtree("token_embedding"),
config.ple_vocab_size,
@@ -62,6 +65,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
)?;
let model_projection = <dyn Linear<B>>::new(
+ format!("{name}/model projection"),
model_dim,
[total_ple_dim],
false,
@@ -77,6 +81,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
adjusted
};
let projection_norm = Normalization::new(
+ format!("{name}/projection norm"),
config.ple_dim,
None,
ShortcutMode::None,
@@ -91,6 +96,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
.map_err(PerLayerEmbeddingError::BackendError)?;
Ok(Self {
+ name,
token_embedding,
model_projection,
projection_norm,
@@ -110,7 +116,8 @@ impl<B: Backend> PerLayerEmbedding<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("per layer embedding");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let total_ple_dim = self.num_layers * self.ple_dim;
let total_rows = batch_dim * self.num_layers;
@@ -150,6 +157,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
command_buffer,
);
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(per_layer_inputs)
@@ -157,6 +165,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
}
pub struct PerLayerEmbeddingProjection<B: Backend> {
+ name: String,
gate: Box<dyn Linear<B>>,
projection: Box<dyn Linear<B>>,
norm: Normalization<B>,
@@ -173,6 +182,7 @@ pub struct PerLayerEmbeddingProjection<B: Backend> {
impl<B: Backend> PerLayerEmbeddingProjection<B> {
pub fn new(
+ name: String,
context: &B::Context,
config: &PLELayerConfig,
model_dim: u32,
@@ -182,6 +192,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
parameter_tree: &ParameterTree<B>,
) -> Result<Self, PerLayerEmbeddingError<B>> {
let gate = <dyn Linear<B>>::new(
+ format!("{name}/gate"),
model_dim,
[config.ple_dim],
false,
@@ -190,6 +201,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
&parameter_tree.subtree("gate"),
)?;
let projection = <dyn Linear<B>>::new(
+ format!("{name}/projection"),
config.ple_dim,
[model_dim],
false,
@@ -198,6 +210,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
&parameter_tree.subtree("projection"),
)?;
let norm = Normalization::new(
+ format!("{name}/norm"),
model_dim,
None,
ShortcutMode::None,
@@ -221,6 +234,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
.map_err(PerLayerEmbeddingError::BackendError)?;
Ok(Self {
+ name,
gate,
projection,
norm,
@@ -245,7 +259,8 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.push_debug_group("per layer embedding projection");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let length = batch_dim * self.model_dim;
@@ -289,6 +304,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
command_buffer,
);
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(())
diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rs
index 351661ee1..010b45944 100644
--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs
+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs
@@ -27,6 +27,7 @@ pub enum PredictionHeadError<B: Backend> {
}
pub struct PredictionHead<B: Backend> {
+ name: String,
hidden_dim: u32,
activation: ActivationType,
dense_projection: Box<dyn Linear<B>>,
@@ -37,6 +38,7 @@ pub struct PredictionHead<B: Backend> {
impl<B: Backend> PredictionHead<B> {
pub fn new(
+ name: String,
hidden_dim: u32,
num_labels: u32,
data_type: DataType,
@@ -45,6 +47,7 @@ impl<B: Backend> PredictionHead<B> {
context: &B::Context,
) -> Result<Self, PredictionHeadError<B>> {
let dense_projection = <dyn Linear<B>>::new(
+ format!("{name}/dense projection"),
hidden_dim,
[hidden_dim],
config.use_dense_bias,
@@ -58,6 +61,7 @@ impl<B: Backend> PredictionHead<B> {
.map_err(PredictionHeadError::Backend)?;
let normalization = Normalization::new(
+ format!("{name}/norm"),
hidden_dim,
None,
ShortcutMode::None,
@@ -69,6 +73,7 @@ impl<B: Backend> PredictionHead<B> {
)?;
let readout = <dyn Linear<B>>::new(
+ format!("{name}/readout"),
hidden_dim,
[num_labels],
true,
@@ -78,6 +83,7 @@ impl<B: Backend> PredictionHead<B> {
)?;
Ok(Self {
+ name,
hidden_dim,
activation,
dense_projection,
@@ -93,7 +99,8 @@ impl<B: Backend> PredictionHead<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("prediction head");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let mut hidden = self.dense_projection.encode(input, batch_dim, command_buffer)?;
self.activation_kernel.encode(
@@ -107,6 +114,7 @@ impl<B: Backend> PredictionHead<B> {
self.normalization.encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;
let logits = self.readout.encode(normalized, batch_dim, command_buffer)?;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(logits)
diff --git a/crates/uzu-engine/src/encodable_block/sampling/mod.rs b/crates/uzu-engine/src/encodable_block/sampling/mod.rs
index 1142bd587..16721b71b 100644
--- a/crates/uzu-engine/src/encodable_block/sampling/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/sampling/mod.rs
@@ -24,6 +24,7 @@ pub use gumbel::{gumbel_float, revidx};
pub use prng::PRng;
pub struct Sampling<B: Backend> {
+ name: String,
vocab_size: u32,
data_type: DataType,
unified_kernels: Mutex<HashMap<UnifiedSamplingKey, <B::Kernels as Kernels>::UnifiedSamplingKernel>>,
@@ -31,10 +32,12 @@ pub struct Sampling<B: Backend> {
impl<B: Backend> Sampling<B> {
pub fn new(
+ name: String,
data_type: DataType,
vocab_size: u32,
) -> Self {
Self {
+ name,
vocab_size,
data_type,
unified_kernels: Mutex::new(HashMap::new()),
@@ -92,7 +95,8 @@ impl<B: Backend> Sampling<B> {
sampling_range: Range<u32>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::GlobalBuffer, B::Error> {
- command_buffer.push_debug_group("sampling");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let sampling_length = sampling_range.end - sampling_range.start;
@@ -194,6 +198,7 @@ impl<B: Backend> Sampling<B> {
command_buffer,
);
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rs
index c73b991b6..a38d15288 100644
--- a/crates/uzu-engine/src/encodable_block/transformer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer.rs
@@ -25,6 +25,7 @@ enum TransformerLayerStateType<B: Backend> {
}
pub struct TransformerState<B: Backend> {
+ accept_name: String,
layer_states: Box<[TransformerLayerStateType<B>]>,
context_length: u32,
}
@@ -60,7 +61,8 @@ impl<B: Backend> TransformerState<B> {
accepted_indices: &[u32],
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.push_debug_group("transformer accept");
+ command_buffer.push_debug_group(&self.accept_name);
+ command_buffer.sample_start_timestamp(&self.accept_name);
for layer_state in &mut self.layer_states {
let TransformerLayerStateType::Owned(layer_state) = layer_state else {
@@ -72,6 +74,7 @@ impl<B: Backend> TransformerState<B> {
self.context_length += accepted_indices.len() as u32;
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(())
@@ -89,6 +92,8 @@ pub enum TransformerNewError<B: Backend> {
}
pub struct Transformer<B: Backend> {
+ name: String,
+ capture_residual_name: String,
ropes: Box<[AnyRoPEConfig]>,
layers: Box<[(TransformerLayer<B>, Option<usize>)]>,
output_norm: Normalization<B>,
@@ -98,6 +103,7 @@ pub struct Transformer<B: Backend> {
impl<B: Backend> Transformer<B> {
pub fn new(
+ name: String,
context: &B::Context,
output_norm_hadamard_factors: Option<B::GlobalBuffer>,
data_type: DataType,
@@ -122,6 +128,7 @@ impl<B: Backend> Transformer<B> {
});
let layer = TransformerLayer::new(
+ format!("{name}/layer {layer_index}"),
context,
transformer_config.model_dim,
transformer_config.hidden_dim,
@@ -137,6 +144,7 @@ impl<B: Backend> Transformer<B> {
.collect::<Result<Box<[_]>, TransformerNewError<B>>>()?;
let output_norm = Normalization::new(
+ format!("{name}/output norm"),
transformer_config.model_dim,
output_norm_hadamard_factors,
ShortcutMode::Add,
@@ -151,6 +159,8 @@ impl<B: Backend> Transformer<B> {
.map_err(TransformerNewError::Backend)?;
Ok(Self {
+ capture_residual_name: format!("{name}/capture residual"),
+ name,
ropes: ropes.into_boxed_slice(),
layers,
output_norm,
@@ -166,9 +176,11 @@ impl<B: Backend> Transformer<B> {
batch_size: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.capture_residual_name);
let mut output = command_buffer.allocate_scratch(hidden.size())?;
let elements = batch_size * self.model_dim;
self.residual_add.encode(Some(shortcut), hidden, &mut output, elements, elements, 1.0, command_buffer);
+ command_buffer.sample_end_timestamp();
Ok(output)
}
@@ -220,6 +232,7 @@ impl<B: Backend> Transformer<B> {
let context_length = 0;
Ok(TransformerState {
+ accept_name: format!("{}/accept", self.name),
layer_states,
context_length,
})
@@ -235,6 +248,7 @@ impl<B: Backend> Transformer<B> {
mut state: Option<&mut TransformerState<B>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<TransformerEncodeOutput<B>, B::Error> {
+ command_buffer.sample_start_timestamp(&self.name);
let mut hidden = input;
let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_none() {
self.prefill_cache_layer_count()
@@ -310,6 +324,7 @@ impl<B: Backend> Transformer<B> {
});
let Some(output_range) = output_range else {
+ command_buffer.sample_end_timestamp();
return Ok(TransformerEncodeOutput {
output: None,
hidden_features,
@@ -324,6 +339,7 @@ impl<B: Backend> Transformer<B> {
command_buffer,
)?;
+ command_buffer.sample_end_timestamp();
Ok(TransformerEncodeOutput {
output: Some(output_normalized),
hidden_features,
diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
index 567a92c2a..40f0e10ff 100644
--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
@@ -45,6 +45,7 @@ pub enum TransformerLayerError<B: Backend> {
}
struct TransformerLayerConv<B: Backend> {
+ pre_convolution_name: String,
model_dim: u32,
pre_conv: SeparableCausalConv<B>,
kernel_projection: Box<dyn Linear<B>>,
@@ -56,6 +57,7 @@ struct TransformerLayerConv<B: Backend> {
// TODO: saner shortcut
pub struct TransformerLayer<B: Backend> {
+ name: String,
pub layer_index: u32,
pub kv_source_layer_index: Option<u32>,
pub pre_mixer_norm: Option<Normalization<B>>,
@@ -71,6 +73,7 @@ pub struct TransformerLayer<B: Backend> {
impl<B: Backend> TransformerLayerConv<B> {
fn new(
+ name: String,
context: &B::Context,
model_dim: u32,
config: &TransformerLayerConvConfig,
@@ -79,6 +82,7 @@ impl<B: Backend> TransformerLayerConv<B> {
input_hadamard_factors: Option<B::GlobalBuffer>,
) -> Result<Self, TransformerLayerError<B>> {
let pre_conv = SeparableCausalConv::new(
+ format!("{name}/pre conv"),
model_dim,
config.conv_kernel_size,
config.conv_group_size,
@@ -88,6 +92,7 @@ impl<B: Backend> TransformerLayerConv<B> {
context,
)?;
let post_conv = SeparableCausalConv::new(
+ format!("{name}/post conv"),
model_dim,
config.conv_kernel_size,
config.conv_group_size,
@@ -100,6 +105,7 @@ impl<B: Backend> TransformerLayerConv<B> {
let coefficient_count = config.conv_kernel_size * (model_dim / config.conv_group_size);
let projection_dim = coefficient_count * 2;
let kernel_projection = <dyn Linear<B>>::new(
+ format!("{name}/kernel projection"),
model_dim,
[projection_dim],
false,
@@ -114,6 +120,7 @@ impl<B: Backend> TransformerLayerConv<B> {
.map_err(TransformerLayerError::Backend)?;
Ok(Self {
+ pre_convolution_name: format!("{name}/pre convolution"),
model_dim,
pre_conv,
kernel_projection,
@@ -129,6 +136,7 @@ impl<B: Backend> TransformerLayerConv<B> {
sequence_length: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {
+ command_buffer.sample_start_timestamp(&self.pre_convolution_name);
let mut projection_input = command_buffer.allocate_scratch(input.size())?;
command_buffer.encode_copy(input, &mut projection_input);
let coefficients = self.kernel_projection.encode(projection_input, sequence_length, command_buffer)?;
@@ -143,6 +151,7 @@ impl<B: Backend> TransformerLayerConv<B> {
if let Some((transform, factors)) = &self.input_rht {
transform.encode_fp_in_place(&mut output, factors, None, sequence_length, self.model_dim, command_buffer);
}
+ command_buffer.sample_end_timestamp();
Ok((output, coefficients))
}
@@ -166,6 +175,7 @@ impl<B: Backend> TransformerLayerConv<B> {
impl<B: Backend> TransformerLayer<B> {
pub fn new(
+ name: String,
context: &B::Context,
model_dim: u32,
hidden_dim: u32,
@@ -201,6 +211,7 @@ impl<B: Backend> TransformerLayer<B> {
};
let (mixer, mixer_hadamard_factors) = <dyn Mixer<B>>::new(
+ &name,
model_dim,
data_type,
layer_config.rope_config.as_ref(),
@@ -212,6 +223,7 @@ impl<B: Backend> TransformerLayer<B> {
let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer_conv_config {
Some(config) => (
Some(TransformerLayerConv::new(
+ format!("{name}/mixer conv"),
context,
model_dim,
config,
@@ -226,6 +238,7 @@ impl<B: Backend> TransformerLayer<B> {
let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {
Some(Normalization::new(
+ format!("{name}/pre mixer norm"),
model_dim,
mixer_hadamard_factors,
if layer_index > 0 {
@@ -248,6 +261,7 @@ impl<B: Backend> TransformerLayer<B> {
let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {
Some(Normalization::new(
+ format!("{name}/post mixer norm"),
model_dim,
None,
ShortcutMode::None,
@@ -262,6 +276,7 @@ impl<B: Backend> TransformerLayer<B> {
};
let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(
+ format!("{name}/mlp"),
&layer_config.mlp_config,
model_dim,
layer_config.hidden_dim.unwrap_or(hidden_dim),
@@ -273,6 +288,7 @@ impl<B: Backend> TransformerLayer<B> {
let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp_conv_config {
Some(config) => (
Some(TransformerLayerConv::new(
+ format!("{name}/mlp conv"),
context,
model_dim,
config,
@@ -286,6 +302,7 @@ impl<B: Backend> TransformerLayer<B> {
};
let pre_mlp_norm = Normalization::new(
+ format!("{name}/pre mlp norm"),
model_dim,
mlp_input_hadamard_factors,
ShortcutMode::Add,
@@ -298,6 +315,7 @@ impl<B: Backend> TransformerLayer<B> {
let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(Normalization::new(
+ format!("{name}/post mlp norm"),
model_dim,
None,
ShortcutMode::None,
@@ -314,6 +332,7 @@ impl<B: Backend> TransformerLayer<B> {
let ple_projection = layer_config.ple_config.as_ref().map(|ple_config| {
let ple_loader = parameter_tree.subtree("ple");
PerLayerEmbeddingProjection::new(
+ format!("{name}/per layer embedding projection"),
context,
ple_config,
model_dim,
@@ -326,6 +345,7 @@ impl<B: Backend> TransformerLayer<B> {
});
Ok(Self {
+ name,
layer_index,
kv_source_layer_index: layer_config.kv_source_layer_index,
pre_mixer_norm,
@@ -350,7 +370,8 @@ impl<B: Backend> TransformerLayer<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&format!("transformer layer {}", self.layer_index));
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {
pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?
@@ -418,6 +439,7 @@ impl<B: Backend> TransformerLayer<B> {
command_buffer.encode_fill(&mut hidden, 0);
}
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(hidden)
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index d446df08c..a3a15c378 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -112,6 +112,9 @@ impl<B: Backend> EncodedWeaverTree<B> {
}
pub struct Weaver<B: Backend> {
+ name: String,
+ prefix_name: String,
+ step_name: String,
token_embedding_norm: Normalization<B>,
token_embedding_projection: Box<dyn Linear<B>>,
hidden_state_norm: Normalization<B>,
@@ -171,6 +174,7 @@ pub enum WeaverEncodeError<B: Backend> {
impl<B: Backend> Weaver<B> {
pub fn new(
+ name: String,
context: &B::Context,
config: &WeaverConfig,
vocab_size: u32,
@@ -199,6 +203,7 @@ impl<B: Backend> Weaver<B> {
});
}
let token_embedding_norm = Normalization::new(
+ format!("{name}/token embedding norm"),
config.target_embedding_dim,
None,
ShortcutMode::None,
@@ -209,6 +214,7 @@ impl<B: Backend> Weaver<B> {
context,
)?;
let hidden_state_norm = Normalization::new(
+ format!("{name}/hidden state norm"),
config.target_model_dim,
None,
ShortcutMode::None,
@@ -219,6 +225,7 @@ impl<B: Backend> Weaver<B> {
context,
)?;
let token_embedding_projection = <dyn Linear<B>>::new(
+ format!("{name}/token embedding projection"),
config.target_embedding_dim,
[config.model_dim],
true,
@@ -228,9 +235,18 @@ impl<B: Backend> Weaver<B> {
)?;
let layer_parameters = parameter_tree.subtree("blocks");
let layers = (0..config.num_layers)
- .map(|index| WeaverLayer::new(context, config, index > 0, &layer_parameters.subtree(&index.to_string())))
+ .map(|index| {
+ WeaverLayer::new(
+ format!("{name}/layer {index}"),
+ context,
+ config,
+ index > 0,
+ &layer_parameters.subtree(&index.to_string()),
+ )
+ })
.collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;
let readout_norm = Normalization::new(
+ format!("{name}/readout norm"),
config.model_dim,
None,
ShortcutMode::Add,
@@ -241,6 +257,7 @@ impl<B: Backend> Weaver<B> {
context,
)?;
let hidden_state_projection = <dyn Linear<B>>::new(
+ format!("{name}/hidden state projection"),
config.target_model_dim,
[config.model_dim],
true,
@@ -249,6 +266,7 @@ impl<B: Backend> Weaver<B> {
&parameter_tree.subtree("hidden_state_projection"),
)?;
let readout_query_projection = <dyn Linear<B>>::new(
+ format!("{name}/readout query projection"),
config.model_dim,
[config.target_model_dim],
false,
@@ -267,6 +285,9 @@ impl<B: Backend> Weaver<B> {
let frontier_insert_children = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(context)
.map_err(WeaverNewError::Backend)?;
Ok(Self {
+ prefix_name: format!("{name}/prefix"),
+ step_name: format!("{name}/step"),
+ name,
token_embedding_norm,
token_embedding_projection,
hidden_state_norm,
@@ -299,7 +320,8 @@ impl<B: Backend> Weaver<B> {
depth: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<Vec<B::ScratchBuffer>, WeaverEncodeError<B>> {
- command_buffer.push_debug_group("weaver prefix");
+ command_buffer.push_debug_group(&self.prefix_name);
+ command_buffer.sample_start_timestamp(&self.prefix_name);
let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);
let mut prefix_hidden = command_buffer
@@ -359,6 +381,7 @@ impl<B: Backend> Weaver<B> {
.kv_cache,
);
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(prefix_kv_layers)
@@ -387,6 +410,8 @@ impl<B: Backend> Weaver<B> {
batch_start_slot: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), WeaverEncodeError<B>> {
+ command_buffer.push_debug_group(&self.step_name);
+ command_buffer.sample_start_timestamp(&self.step_name);
let tree_slot_count = shape.slot_count();
let ancestor_stride = self.max_depth;
let frontier_capacity = tree_slot_count * shape.expand_width;
@@ -539,6 +564,8 @@ impl<B: Backend> Weaver<B> {
command_buffer,
);
+ command_buffer.sample_end_timestamp();
+ command_buffer.pop_debug_group();
Ok(())
}
@@ -553,7 +580,8 @@ impl<B: Backend> Weaver<B> {
shape: WeaverTreeShape,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
- command_buffer.push_debug_group("weaver tree");
+ command_buffer.push_debug_group(&self.name);
+ command_buffer.sample_start_timestamp(&self.name);
let tree_slot_count = shape.slot_count();
let ancestor_stride = self.max_depth;
@@ -677,7 +705,6 @@ impl<B: Backend> Weaver<B> {
} else {
shape.expand_per_round
};
- command_buffer.push_debug_group("weaver step");
self.encode_step(
target_embedding,
prefix_kv_layers.iter(),
@@ -700,10 +727,10 @@ impl<B: Backend> Weaver<B> {
batch_start_slot,
command_buffer,
)?;
- command_buffer.pop_debug_group();
batch_start_slot += batch_node_count;
}
+ command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
let mut packed_tree_readback =
diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
index 2b9080e6b..8621bedce 100644
--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
@@ -25,6 +25,8 @@ pub struct PreparedPrefixAttention<B: Backend> {
}
pub struct WeaverLayer<B: Backend> {
+ prefix_attention_name: String,
+ post_attention_name: String,
pub pre_attention_norm: Normalization<B>,
pub qkv_projection: Box<dyn Linear<B>>,
attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
@@ -42,6 +44,7 @@ pub struct WeaverLayer<B: Backend> {
impl<B: Backend> WeaverLayer<B> {
pub fn new(
+ name: String,
context: &B::Context,
config: &WeaverConfig,
add_to_residual: bool,
@@ -59,6 +62,7 @@ impl<B: Backend> WeaverLayer<B> {
let head_dim = model_dim / num_heads;
let attention_scale = 1.0 / (head_dim as f32).sqrt();
let qkv_projection = <dyn Linear<B>>::new(
+ format!("{name}/qkv projection"),
model_dim,
[3 * model_dim],
false,
@@ -67,6 +71,7 @@ impl<B: Backend> WeaverLayer<B> {
&parameter_tree.subtree("qkv_projection"),
)?;
let out_projection = <dyn Linear<B>>::new(
+ format!("{name}/out projection"),
model_dim,
[model_dim],
false,
@@ -93,6 +98,7 @@ impl<B: Backend> WeaverLayer<B> {
<B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true)
.map_err(WeaverNewError::Backend)?;
let pre_attention_norm = Normalization::new(
+ format!("{name}/pre attention norm"),
model_dim,
None,
if add_to_residual {
@@ -107,6 +113,7 @@ impl<B: Backend> WeaverLayer<B> {
context,
)?;
let pre_mlp_norm = Normalization::new(
+ format!("{name}/pre mlp norm"),
model_dim,
None,
ShortcutMode::Add,
@@ -122,12 +129,21 @@ impl<B: Backend> WeaverLayer<B> {
true,
true,
));
- let (mlp, up_input_hadamard_factors) =
- <dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, &parameter_tree.subtree("mlp"), DATA_TYPE)?;
+ let (mlp, up_input_hadamard_factors) = <dyn Mlp<B>>::new(
+ format!("{name}/mlp"),
+ &mlp_config,
+ model_dim,
+ hidden_dim,
+ context,
+ &parameter_tree.subtree("mlp"),
+ DATA_TYPE,
+ )?;
assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");
let ancestor_attention = <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim, num_heads)
.map_err(WeaverNewError::Backend)?;
Ok(Self {
+ prefix_attention_name: format!("{name}/prefix attention"),
+ post_attention_name: format!("{name}/post attention"),
qkv_projection,
out_projection,
prefix_attention,
@@ -152,6 +168,7 @@ impl<B: Backend> WeaverLayer<B> {
token_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<PreparedPrefixAttention<B>, B::Error> {
+ command_buffer.sample_start_timestamp(&self.prefix_attention_name);
let attention_input =
self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), command_buffer)?;
let qkv = self.qkv_projection.encode(attention_input, token_count, command_buffer)?;
@@ -176,6 +193,7 @@ impl<B: Backend> WeaverLayer<B> {
token_count,
command_buffer,
);
+ command_buffer.sample_end_timestamp();
Ok(PreparedPrefixAttention {
queries,
kv_cache,
@@ -189,9 +207,12 @@ impl<B: Backend> WeaverLayer<B> {
token_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
+ command_buffer.sample_start_timestamp(&self.post_attention_name);
let projected_attention = self.out_projection.encode(attention_output, token_count, command_buffer)?;
let mlp_input =
self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), command_buffer)?;
- self.mlp.encode(mlp_input, token_count, command_buffer)
+ let output = self.mlp.encode(mlp_input, token_count, command_buffer)?;
+ command_buffer.sample_end_timestamp();
+ Ok(output)
}
}
diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rs
index d92e350e7..0d63e996a 100644
--- a/crates/uzu-engine/src/engine/classifier_model.rs
+++ b/crates/uzu-engine/src/engine/classifier_model.rs
@@ -66,6 +66,7 @@ impl<B: Backend> Engine<B> {
let weight_loader = ParameterLoader::new(&weights_file, context.as_ref())?;
let classifier = ClassifierEncodable::new(
+ String::from("classifier"),
context.as_ref(),
&config.classifier_config,
&weight_loader.tree().subtree("classifier"),
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index 1255404b9..54a2ff84a 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -5,13 +5,14 @@ use thiserror::Error;
use tokenizers::Tokenizer;
use crate::{
- backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},
+ backends::common::{Backend, Context, DeviceCapabilities},
config::{
model::{generation::GenerationConfig, language_model::LanguageModelConfig},
token_codec::AnyTokenCodecConfig,
},
data_type::DataType,
encodable_block::{
+ context_ring_update::ContextRingUpdate,
decoder::{Decoder, DecoderError},
sampling::{Sampling, SamplingMethod},
},
@@ -31,7 +32,7 @@ pub struct LanguageModel<B: Backend> {
decoder: Decoder<B>,
speculator: Option<DFlashTfmSpeculator<B>>,
sampling: Sampling<B>,
- context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
+ context_ring_update: ContextRingUpdate<B>,
generation_config: GenerationConfig,
end_of_thinking_tag: Option<String>,
tokenizer: Arc<Tokenizer>,
@@ -77,6 +78,7 @@ impl<B: Backend> Engine<B> {
let data_type = DataType::BF16;
let decoder = Decoder::new(
+ String::from("decoder"),
self.context.as_ref(),
&config.decoder_config,
&weight_loader.tree().subtree("decoder"),
@@ -94,10 +96,11 @@ impl<B: Backend> Engine<B> {
.transpose()?
.flatten();
- let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);
+ let sampling = Sampling::new(String::from("sampling"), data_type, config.decoder_config.vocab_size);
- let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&self.context)
- .map_err(EngineLoadLanguageModelError::Backend)?;
+ let context_ring_update =
+ ContextRingUpdate::new(String::from("update repetition penalty ring"), self.context.as_ref())
+ .map_err(EngineLoadLanguageModelError::Backend)?;
weight_loader.tree().assert_all_tensors_validated()?;
diff --git a/crates/uzu-engine/src/engine/language_model/stream/mod.rs b/crates/uzu-engine/src/engine/language_model/stream/mod.rs
index fc456ba8d..8ca87d672 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/mod.rs
@@ -1,10 +1,12 @@
+use std::sync::mpsc::Sender;
+
use derive_more::Debug;
use thiserror::Error;
#[cfg(grammar)]
use crate::engine::language_model::grammar::{Grammar, GrammarError};
use crate::{
- backends::common::Backend,
+ backends::common::{Backend, TimestampSpan},
encodable_block::decoder::DecoderError,
engine::language_model::{LanguageModel, state::LanguageModelState},
speculators::dflash_tfm::DFlashTreeError,
@@ -17,6 +19,7 @@ mod stream;
pub struct LanguageModelStreamOptions {
pub sampling_method: SamplingMethod,
+ pub timestamps: Option<Sender<Box<[TimestampSpan]>>>,
#[cfg(grammar)]
pub grammar: Option<Grammar>,
}
@@ -44,6 +47,7 @@ impl<B: Backend> LanguageModel<B> {
pub fn default_stream_options(&self) -> LanguageModelStreamOptions {
LanguageModelStreamOptions {
sampling_method: self.default_sampling_method(),
+ timestamps: None,
#[cfg(grammar)]
grammar: None,
}
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 7663cf362..1e826f915 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -2,7 +2,7 @@ use std::{
iter::{once, repeat_n},
mem::replace,
range::Range,
- sync::Arc,
+ sync::{Arc, mpsc::Sender},
};
use shoji::traits::backend::chat_token::TokenStreamMetrics;
@@ -12,8 +12,8 @@ use crate::engine::language_model::grammar::Grammar;
use crate::{
array::size_for_shape,
backends::common::{
- Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,
- CommandBufferPending, Context, gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,
+ Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context,
+ TimestampSpan, gpu_types::trie::TrieNode as GpuTrieNode,
},
data_type::DataType,
encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},
@@ -26,6 +26,7 @@ use crate::{
},
},
trie::TrieNode,
+ utils::timestamps::{create_command_buffer, wait},
};
enum ForwardPassChaining<B: Backend> {
@@ -40,6 +41,7 @@ impl<B: Backend> ForwardPassChaining<B> {
fn resolve<'a>(
&'a mut self,
tokens: &mut Vec<u64>,
+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
#[cfg(grammar)] grammar: Option<&mut Grammar>,
) -> Result<(u64, Option<&'a B::ScratchBuffer>), LanguageModelStreamError<B>> {
match self {
@@ -50,7 +52,7 @@ impl<B: Backend> ForwardPassChaining<B> {
Self::InFlight(in_flight) => {
assert!(in_flight.full_accept);
for pending in replace(&mut in_flight.pending, Box::new([])) {
- pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;
+ wait(pending, timestamps).map_err(LanguageModelStreamError::Backend)?;
}
let output_tokens = in_flight.output_tokens.as_slice::<u32>();
assert_eq!(output_tokens.len(), 1);
@@ -212,11 +214,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
.map_err(LanguageModelStreamError::Backend)?;
- let mut command_buffer = model
- .engine
- .context
- .create_command_buffer(Some("prefill"), Some(allocation_pool.clone()))
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = create_command_buffer::<B>(
+ &model.engine.context,
+ "prefill",
+ &allocation_pool,
+ options.timestamps.as_ref(),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
let mut output_tokens = None;
let mut output_norm = None;
@@ -401,7 +405,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
(ForwardPassChaining::InFlight(forward_pass_pending), None)
} else {
for pending in forward_pass_pending.pending {
- pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;
+ wait(pending, self.options.timestamps.as_ref()).map_err(LanguageModelStreamError::Backend)?;
}
let sampled_tokens = forward_pass_pending
.output_tokens
@@ -454,12 +458,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();
let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();
- let mut command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = create_command_buffer::<B>(
+ &self.model.engine.context,
+ "decode",
+ &self.allocation_pool,
+ self.options.timestamps.as_ref(),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
self.model_state
.transformer_state
.encode_accept(&accepted_token_indicies, &mut command_buffer)
@@ -485,7 +490,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
None
};
if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {
- command_buffer.push_debug_group("update repetition penalty ring");
let accepted_input_token_ids_const = command_buffer
.allocate_constant_from_slice(
&accepted_input_token_ids
@@ -501,23 +505,20 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
full.len() as u32,
&mut command_buffer,
);
- command_buffer.pop_debug_group();
}
if let Some(capture_span) = capture_span {
- command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref())
.map_err(LanguageModelStreamError::Backend)?;
drop(capture_span);
- command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
- .map_err(LanguageModelStreamError::Backend)?;
+ command_buffer = create_command_buffer::<B>(
+ &self.model.engine.context,
+ "decode",
+ &self.allocation_pool,
+ self.options.timestamps.as_ref(),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
}
self.model_state.tokens.extend(accepted_output_token_ids);
(
@@ -541,6 +542,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
prev_output
.resolve(
&mut self.model_state.tokens,
+ self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)?
@@ -553,6 +555,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
{
prev_output.resolve(
&mut self.model_state.tokens,
+ self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)?;
@@ -569,6 +572,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
&& let (root_token, Some(output_norm)) = prev_output.resolve(
&mut self.model_state.tokens,
+ self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)? {
@@ -585,6 +589,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.options.grammar.as_mut(),
&self.model_state.prng,
self.allocation_pool.clone(),
+ self.options.timestamps.as_ref(),
)?;
prev_output.drop_output_norm();
(trie, None, false)
@@ -607,12 +612,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.allocation_pool = self.model.engine.context.create_allocation_pool();
- let mut command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = create_command_buffer::<B>(
+ &self.model.engine.context,
+ "decode",
+ &self.allocation_pool,
+ self.options.timestamps.as_ref(),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
let scratch_token_ids;
let mut constant_token_ids = None;
@@ -659,12 +665,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if chain_copy.is_some() {
pending.push(command_buffer.end_encoding().submit());
- let mut command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = create_command_buffer::<B>(
+ &self.model.engine.context,
+ "decode",
+ &self.allocation_pool,
+ self.options.timestamps.as_ref(),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
let mut bitmask = command_buffer
.allocate_constant(
@@ -672,7 +679,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
.map_err(LanguageModelStreamError::Backend)?;
- prev_output.resolve(&mut self.model_state.tokens, Some(grammar))?;
+ prev_output.resolve(&mut self.model_state.tokens, self.options.timestamps.as_ref(), Some(grammar))?;
if grammar.next_bitmask(bitmask.as_slice_mut()) {
(Some(bitmask), command_buffer)
} else {
@@ -786,6 +793,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
prev_output
.resolve(
&mut self.model_state.tokens,
+ self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)?
@@ -814,16 +822,17 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
} => Some(seed_token),
DecodingState::ForwardPassPending(in_flight) => {
for pending in in_flight.pending {
- pending.wait_until_completed().unwrap();
+ wait(pending, self.options.timestamps.as_ref()).unwrap();
}
if !in_flight.full_accept {
- let mut command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))
- .unwrap();
+ let mut command_buffer = create_command_buffer::<B>(
+ &self.model.engine.context,
+ "drop accept",
+ &self.allocation_pool,
+ self.options.timestamps.as_ref(),
+ )
+ .unwrap();
self.model_state.transformer_state.encode_accept(&[0], &mut command_buffer).unwrap();
if let Some(speculator) = self.model.speculator.as_ref() {
speculator
@@ -835,7 +844,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
)
.unwrap();
}
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).unwrap();
}
Some(in_flight.output_tokens.as_slice::<u32>()[0] as u64)
@@ -849,12 +858,13 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
} => {
assert!(num_accepted > 0 && num_accepted < full.len());
- let mut command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))
- .unwrap();
+ let mut command_buffer = create_command_buffer::<B>(
+ &self.model.engine.context,
+ "drop accept",
+ &self.allocation_pool,
+ self.options.timestamps.as_ref(),
+ )
+ .unwrap();
let accepted_token_indicies =
full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
self.model_state
@@ -871,7 +881,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
)
.unwrap();
}
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).unwrap();
drop(capture_span);
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index 6c41724b5..2d1e02a60 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -3,7 +3,7 @@ use std::{
fs::File,
io::{self, BufReader},
path::Path,
- sync::Arc,
+ sync::{Arc, mpsc::Sender},
};
use derive_more::Debug;
@@ -15,8 +15,8 @@ pub use crate::encodable_block::dflash::DFlashState;
use crate::engine::language_model::grammar::Grammar;
use crate::{
backends::common::{
- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,
- Context, gpu_types::trie::TrieNode as GpuTrieNode,
+ Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,
+ gpu_types::trie::TrieNode as GpuTrieNode,
},
config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},
data_type::DataType,
@@ -29,6 +29,7 @@ use crate::{
},
parameters::{ParameterLoader, ParameterLoaderError},
trie::TrieNode,
+ utils::timestamps::{create_command_buffer, wait},
};
#[derive(Debug, Error)]
@@ -122,12 +123,19 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let weight_loader = ParameterLoader::new(&weights_file, &*context)?;
let speculator_tree = weight_loader.tree().subtree("speculator");
- let dflash = DFlash::new(&*context, &config.draft_config, &speculator_tree.subtree("draft_model"), data_type)?;
+ let dflash = DFlash::new(
+ String::from("dflash"),
+ &*context,
+ &config.draft_config,
+ &speculator_tree.subtree("draft_model"),
+ data_type,
+ )?;
let weaver = config
.weaver_config
.as_ref()
.map(|weaver_config| {
Weaver::new(
+ String::from("weaver"),
&*context,
weaver_config,
config.draft_config.vocab_size,
@@ -138,7 +146,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
weight_loader.tree().assert_all_tensors_validated()?;
- let sampling = Sampling::new(data_type, config.draft_config.vocab_size);
+ let sampling = Sampling::new(String::from("sampling"), data_type, config.draft_config.vocab_size);
Ok(Some(Self {
context,
@@ -217,6 +225,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
#[cfg(grammar)] grammar: Option<&mut Grammar>,
prng: &PRng,
allocation_pool: Arc<B::AllocationPool>,
+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
) -> Result<TrieNode, DFlashTreeError<B>> {
assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");
@@ -230,10 +239,9 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let root_position = state.context_length();
- let mut command_buffer = self
- .context
- .create_command_buffer(Some("speculator propose"), Some(allocation_pool))
- .map_err(DFlashTreeError::Backend)?;
+ let mut command_buffer =
+ create_command_buffer::<B>(&self.context, "speculator propose", &allocation_pool, timestamps)
+ .map_err(DFlashTreeError::Backend)?;
let nodes = match shape.construction_method {
DFlashTfmTreeConstructionMethod::Argmax => {
@@ -283,7 +291,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
)
.map_err(DFlashTreeError::Backend)?;
let completed =
- command_buffer.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
+ wait(command_buffer.end_encoding().submit(), timestamps).map_err(DFlashTreeError::Backend)?;
let tokens = sampled.copyout::<u32>();
drop(completed);
nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {
@@ -358,7 +366,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
&mut command_buffer,
)?;
let completed =
- command_buffer.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
+ wait(command_buffer.end_encoding().submit(), timestamps).map_err(DFlashTreeError::Backend)?;
let nodes = tree.read_nodes();
drop(completed);
nodes
diff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rs
index 11e88e2c5..17b686018 100644
--- a/crates/uzu-engine/src/utils/mod.rs
+++ b/crates/uzu-engine/src/utils/mod.rs
@@ -4,4 +4,5 @@ pub mod maybe_mut;
#[cfg(backend = "cpu")]
pub mod pointers;
pub mod strict_serde;
+pub mod timestamps;
pub mod version;
diff --git a/crates/uzu-engine/src/utils/timestamps.rs b/crates/uzu-engine/src/utils/timestamps.rs
new file mode 100644
index 000000000..fcf6d60f6
--- /dev/null
+++ b/crates/uzu-engine/src/utils/timestamps.rs
@@ -0,0 +1,32 @@
+use std::sync::{Arc, mpsc::Sender};
+
+use crate::backends::common::{
+ Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferPending, Context, TimestampSpan,
+};
+
+pub fn create_command_buffer<B: Backend>(
+ context: &B::Context,
+ name: &str,
+ allocation_pool: &Arc<B::AllocationPool>,
+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
+) -> Result<<B::CommandBuffer as CommandBuffer>::Encoding, B::Error> {
+ let mut command_buffer = context.create_command_buffer(Some(name), Some(allocation_pool.clone()))?;
+ if timestamps.is_some() {
+ command_buffer.enable_timestamps();
+ }
+ Ok(command_buffer)
+}
+
+pub fn wait<Pending: CommandBufferPending>(
+ pending: Pending,
+ timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
+) -> Result<
+ <Pending::CommandBuffer as CommandBuffer>::Completed,
+ <<Pending::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
+> {
+ let completed = pending.wait_until_completed()?;
+ if let Some(sender) = timestamps {
+ let _ = sender.send(completed.timestamps().into());
+ }
+ Ok(completed)
+}
diff --git a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs
index 15a85cc3b..c9639d5d8 100644
--- a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs
@@ -30,6 +30,7 @@ fn run_key_value_row_stride_test<B: Backend>() {
let key = QKVNorm::<B>::build_head(&context, DataType::F32, config, None, HEAD_DIM)
.expect("failed to construct key norm");
let norm = QKVNorm {
+ name: String::from("test/qkv norm"),
query: None,
key: Some(key),
value: None,
diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
index f00b7bfff..50932575b 100644
--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
@@ -44,7 +44,7 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(
method: &SamplingMethod,
batch_size: u32,
) -> Result<SamplingTestResults, TestCaseError> {
- let sampling = Sampling::<B>::new(T::data_type(), vocab_size as u32);
+ let sampling = Sampling::<B>::new(String::from("test/sampling"), T::data_type(), vocab_size as u32);
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
let logits_buffer = command_buffer.allocate_constant_from_slice(logits).unwrap();