Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

New capture (#929)

uuuvn55dc41d5cbf7
diff --git a/crates/uzu-engine/src/backends/common/backend.rs b/crates/uzu-engine/src/backends/common/backend.rs
index 6ce7985d3..6acdcbc50 100644
--- a/crates/uzu-engine/src/backends/common/backend.rs
+++ b/crates/uzu-engine/src/backends/common/backend.rs
@@ -10,6 +10,7 @@ pub trait Backend: Clone + Send + Sync + 'static {
type ScratchBuffer: ScratchBuffer<Backend = Self>;
type SparseBuffer: SparseBuffer<Backend = Self>;
type AllocationPool: Send + Sync;
+ type CaptureSpan: Send + Sync;
type Kernels: Kernels<Backend = Self>;
type Error: Error;
diff --git a/crates/uzu-engine/src/backends/common/capture_matcher.rs b/crates/uzu-engine/src/backends/common/capture_matcher.rs
new file mode 100644
index 000000000..2c3a91f74
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/capture_matcher.rs
@@ -0,0 +1,110 @@
+use std::{
+ collections::HashMap,
+ env,
+ path::PathBuf,
+ sync::{
+ LazyLock,
+ mpsc::{self, Receiver, Sender},
+ },
+};
+
+use parking_lot::Mutex;
+use thiserror::Error;
+
+use crate::backends::common::{Backend, Context};
+
+pub static CAPTURE_MATCHER: LazyLock<CaptureMatcher> = LazyLock::new(CaptureMatcher::new);
+
+struct CaptureMatcherInner {
+ starts: HashMap<String, (Sender<Box<dyn Send + Sync>>, PathBuf)>,
+ ends: HashMap<String, Receiver<Box<dyn Send + Sync>>>,
+}
+
+pub struct CaptureMatcher {
+ enabled: bool,
+ inner: Mutex<CaptureMatcherInner>,
+}
+
+#[derive(Debug, Error)]
+pub enum CaptureError<B: Backend> {
+ #[error("Cannot start backend capture: {0}")]
+ BackendCapture(#[source] B::Error),
+ #[error("Attempted to start capture whose endpoint already happened")]
+ StartAfterEnd,
+}
+
+impl CaptureMatcher {
+ fn new() -> Self {
+ let mut enabled = false;
+ let mut starts = HashMap::new();
+ let mut ends = HashMap::new();
+
+ if let Some(capture) = env::var_os("UZU_CAPTURE").map(|s| s.into_string().unwrap()) {
+ let spans = capture.split(',').filter(|s| !s.is_empty());
+ let auto_paths = spans.clone().filter(|s| !s.contains('@')).count();
+ let mut auto_index = 0;
+
+ for span in spans {
+ enabled = true;
+
+ let (span, output_path) = match span.split_once('@') {
+ Some((span, path)) => (span, PathBuf::from(path)),
+ None if auto_paths == 1 => (span, PathBuf::from("/tmp/uzu.gputrace")),
+ None => {
+ let output_path = PathBuf::from(format!("/tmp/uzu-{auto_index}.gputrace"));
+ auto_index += 1;
+ (span, output_path)
+ },
+ };
+ let (start, end) = span.split_once("..").unwrap_or((span, span));
+
+ let (sender, receiver) = mpsc::channel();
+
+ assert!(!starts.contains_key(start), "duplicate capture start: {start:?}");
+ starts.insert(start.to_owned(), (sender, output_path));
+
+ assert!(!ends.contains_key(end), "duplicate capture end: {end:?}");
+ ends.insert(end.to_owned(), receiver);
+ }
+ }
+
+ Self {
+ enabled,
+ inner: Mutex::new(CaptureMatcherInner {
+ starts,
+ ends,
+ }),
+ }
+ }
+
+ pub fn enabled(&self) -> bool {
+ self.enabled
+ }
+
+ pub fn capture<B: Backend>(
+ &self,
+ context: &B::Context,
+ span: &str,
+ ) -> Result<Option<CaptureMatcherSpan>, CaptureError<B>> {
+ if self.enabled {
+ let mut inner = self.inner.lock();
+ if let Some((start, output_path)) = inner.starts.remove(span) {
+ start
+ .send(Box::new(context.start_capture(&output_path).map_err(CaptureError::BackendCapture)?))
+ .map_err(|_| CaptureError::StartAfterEnd)?;
+ }
+ if let Some(end) = inner.ends.remove(span) {
+ return Ok(Some(CaptureMatcherSpan {
+ _end: end,
+ }));
+ }
+ }
+
+ Ok(None)
+ }
+}
+
+pub struct CaptureMatcherSpan {
+ _end: Receiver<Box<dyn Send + Sync>>,
+}
+unsafe impl Sync for CaptureMatcherSpan {}
diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rs
index 0377b6a70..0ed3d0034 100644
--- a/crates/uzu-engine/src/backends/common/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs
@@ -80,6 +80,11 @@ pub trait CommandBufferEncoding {
fn pop_debug_group(&mut self);
+ fn add_debug_point(
+ &mut self,
+ name: &str,
+ );
+
fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;
}
diff --git a/crates/uzu-engine/src/backends/common/context.rs b/crates/uzu-engine/src/backends/common/context.rs
index 0b07354ba..66196980d 100644
--- a/crates/uzu-engine/src/backends/common/context.rs
+++ b/crates/uzu-engine/src/backends/common/context.rs
@@ -29,14 +29,10 @@ pub trait Context: Sized + Send + Sync {
fn peak_memory_usage(&self) -> Option<usize>;
- fn enable_capture();
-
fn start_capture(
&self,
- trace_path: &Path,
- ) -> Result<(), <Self::Backend as Backend>::Error>;
-
- fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;
+ output_path: &Path,
+ ) -> Result<<Self::Backend as Backend>::CaptureSpan, <Self::Backend as Backend>::Error>;
fn device_capabilities(&self) -> DeviceCapabilities;
}
diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rs
index a07c947d6..59a6e5e67 100644
--- a/crates/uzu-engine/src/backends/common/mod.rs
+++ b/crates/uzu-engine/src/backends/common/mod.rs
@@ -1,6 +1,7 @@
pub(super) mod allocator;
mod backend;
mod buffer;
+pub mod capture_matcher;
mod command_buffer;
mod context;
mod device_capabilities;
diff --git a/crates/uzu-engine/src/backends/cpu/backend.rs b/crates/uzu-engine/src/backends/cpu/backend.rs
index 22ed5b72e..335858916 100644
--- a/crates/uzu-engine/src/backends/cpu/backend.rs
+++ b/crates/uzu-engine/src/backends/cpu/backend.rs
@@ -1,3 +1,5 @@
+use std::convert::Infallible;
+
use crate::backends::{
common::{
Backend,
@@ -26,6 +28,7 @@ impl Backend for Cpu {
type ScratchBuffer = PoolAllocation<Self::GlobalBuffer>;
type SparseBuffer = CpuSparseBuffer;
type AllocationPool = PoolAllocator<Self::GlobalBuffer>;
+ type CaptureSpan = Infallible;
type Kernels = CpuKernels;
type Error = CpuError;
diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
index 62ba458aa..04ca0e2bb 100644
--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
@@ -118,6 +118,12 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
fn pop_debug_group(&mut self) {}
+ fn add_debug_point(
+ &mut self,
+ _name: &str,
+ ) {
+ }
+
fn end_encoding(self) -> CpuCommandBufferExecutable {
assert!(self.constant_allocator.is_done(), "attempted to end encoding while constants are still alive");
CpuCommandBufferExecutable {
diff --git a/crates/uzu-engine/src/backends/cpu/context.rs b/crates/uzu-engine/src/backends/cpu/context.rs
index 9f923ece7..fa6000162 100644
--- a/crates/uzu-engine/src/backends/cpu/context.rs
+++ b/crates/uzu-engine/src/backends/cpu/context.rs
@@ -1,4 +1,5 @@
use std::{
+ convert::Infallible,
path::Path,
sync::{Arc, Weak, mpsc},
thread,
@@ -71,16 +72,10 @@ impl Context for CpuContext {
None
}
- fn enable_capture() {}
-
fn start_capture(
&self,
- _trace_path: &Path,
- ) -> Result<(), CpuError> {
- Err(CpuError::NotSupported)
- }
-
- fn stop_capture(&self) -> Result<(), CpuError> {
+ _output_path: &Path,
+ ) -> Result<Infallible, CpuError> {
Err(CpuError::NotSupported)
}
diff --git a/crates/uzu-engine/src/backends/metal/backend.rs b/crates/uzu-engine/src/backends/metal/backend.rs
index fa2936be7..3986a0f47 100644
--- a/crates/uzu-engine/src/backends/metal/backend.rs
+++ b/crates/uzu-engine/src/backends/metal/backend.rs
@@ -9,6 +9,7 @@ use crate::backends::{
},
metal::{
buffer::{dense::MetalDenseBuffer, sparse::MetalSparseBuffer},
+ capture::MetalCaptureSpan,
command_buffer::MetalCommandBuffer,
context::MetalContext,
error::MetalError,
@@ -27,6 +28,7 @@ impl Backend for Metal {
type ScratchBuffer = PoolAllocation<Self::GlobalBuffer>;
type SparseBuffer = MetalSparseBuffer;
type AllocationPool = PoolAllocator<Self::GlobalBuffer>;
+ type CaptureSpan = MetalCaptureSpan;
type Kernels = MetalKernels;
type Error = MetalError;
diff --git a/crates/uzu-engine/src/backends/metal/buffer/sparse.rs b/crates/uzu-engine/src/backends/metal/buffer/sparse.rs
index 9e41a6cad..a60f8b9e9 100644
--- a/crates/uzu-engine/src/backends/metal/buffer/sparse.rs
+++ b/crates/uzu-engine/src/backends/metal/buffer/sparse.rs
@@ -2,8 +2,8 @@ use std::sync::Arc;
use metal::{
MTL4CommandQueue, MTL4CommandQueueExt, MTL4UpdateSparseBufferMappingOperation, MTLBuffer, MTLDeviceExt,
- MTLResidencySet, MTLResourceOptions, MTLSharedEventExt, MTLSharedEventListener, MTLSharedEventNotificationBlock,
- MTLSparseTextureMappingMode,
+ MTLEventExt, MTLResidencySet, MTLResourceOptions, MTLSharedEventExt, MTLSharedEventListener,
+ MTLSharedEventNotificationBlock, MTLSparseTextureMappingMode,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
@@ -118,6 +118,7 @@ impl Drop for MetalSparseBuffer {
);
sparse_state.did_ops = true;
let event = self.context.sparse_queue.device().new_shared_event().unwrap();
+ event.set_label(Some("sparse unmap event"));
let retain = (self.buffer.clone(), std::mem::take(&mut self.pages), event.clone());
event.notify_listener_at_value(
&MTLSharedEventListener::shared_listener(),
diff --git a/crates/uzu-engine/src/backends/metal/capture.rs b/crates/uzu-engine/src/backends/metal/capture.rs
new file mode 100644
index 000000000..2286b38f9
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/capture.rs
@@ -0,0 +1,31 @@
+use std::path::Path;
+
+use metal::{MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCaptureTarget};
+
+use crate::backends::metal::error::MetalError;
+
+pub struct MetalCaptureSpan;
+
+impl MetalCaptureSpan {
+ pub fn new(
+ target: &MTLCaptureTarget,
+ output_path: &Path,
+ ) -> Result<Self, MetalError> {
+ let capture_descriptor = MTLCaptureDescriptor::new();
+ capture_descriptor.set_capture_object(Some(target));
+ capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);
+ capture_descriptor.set_output_path(Some(output_path));
+
+ MTLCaptureManager::shared_capture_manager()
+ .start_capture_with_descriptor(&capture_descriptor)
+ .map_err(|error| MetalError::CannotStartGpuCapture(error.to_string()))?;
+
+ Ok(Self)
+ }
+}
+
+impl Drop for MetalCaptureSpan {
+ fn drop(&mut self) {
+ MTLCaptureManager::shared_capture_manager().stop_capture();
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rs
index 985a6d0e5..727de6a8d 100644
--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs
@@ -55,23 +55,19 @@ impl MetalCommandBufferEncoding {
name: Option<&str>,
allocation_pool: Option<Arc<<Metal as Backend>::AllocationPool>>,
) -> Result<Self, MetalError> {
- let (command_allocator, command_buffer) = if let mut command_buffer_cache = context.command_buffer_cache.lock()
- && let Some(command_buffer_cached) = command_buffer_cache.pop()
- {
- command_buffer_cached.command_allocator.reset();
- (command_buffer_cached.command_allocator, command_buffer_cached.command_buffer)
+ let command_allocator = if let Some(command_allocator) = context.command_allocator_cache.lock().pop() {
+ command_allocator.reset();
+ command_allocator
} else {
- (
- context.device.new_command_allocator().ok_or(MetalError::CannotCreateCommandBuffer)?,
- context.device.new_mtl4_command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?,
- )
+ context.device.new_command_allocator().ok_or(MetalError::CannotCreateCommandBuffer)?
};
- command_buffer.set_label(name);
-
+ let command_buffer = context.device.new_mtl4_command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?;
command_buffer.begin_command_buffer_with_allocator(&command_allocator);
+ command_buffer.set_label(name);
let compute_encoder = command_buffer.compute_command_encoder().unwrap();
+ ProtocolObject::<dyn MTL4CommandEncoder>::set_label(compute_encoder.as_ref(), name);
compute_encoder.barrier_after_queue_stages_before_stages_visibility_options(
MTLStages::Dispatch | MTLStages::Blit | MTLStages::ResourceState,
@@ -80,6 +76,7 @@ impl MetalCommandBufferEncoding {
);
let argument_table_descriptor = MTL4ArgumentTableDescriptor::new();
+ argument_table_descriptor.set_label(name.map(|name| format!("{name} argument table")).as_deref());
argument_table_descriptor.set_max_buffer_bind_count(31);
let argument_table = context
.device
@@ -213,6 +210,13 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
self.compute_encoder.pop_debug_group();
}
+ fn add_debug_point(
+ &mut self,
+ name: &str,
+ ) {
+ ProtocolObject::<dyn MTL4CommandEncoder>::insert_debug_signpost(self.compute_encoder.as_ref(), name);
+ }
+
fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {
let constant_allocator = self.constant_allocator.take().unwrap();
assert!(constant_allocator.is_done(), "attempted to end encoding while constants are still alive");
@@ -276,11 +280,8 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {
Ok(Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()))
};
let _ = sender.send(message);
- context_clone.command_buffer_cache.lock().push(MetalCommandBufferCache {
- command_allocator: command_allocator.clone(),
- command_buffer: command_buffer.clone(),
- });
- let _keep_alive = (&constant_allocator, &allocation_pool);
+ context_clone.command_allocator_cache.lock().push(command_allocator.clone());
+ let _keep_alive = (&command_buffer, &constant_allocator, &allocation_pool);
};
let options = MTL4CommitOptions::new();
@@ -326,8 +327,3 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {
self.gpu_execution_time
}
}
-
-pub struct MetalCommandBufferCache {
- command_allocator: Retained<ProtocolObject<dyn MTL4CommandAllocator>>,
- command_buffer: Retained<ProtocolObject<dyn MTL4CommandBuffer>>,
-}
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rs
index 90ee23b35..01318dfab 100644
--- a/crates/uzu-engine/src/backends/metal/context.rs
+++ b/crates/uzu-engine/src/backends/metal/context.rs
@@ -8,8 +8,8 @@ use std::{
};
use metal::{
- MTL4CommandQueue, MTL4Compiler, MTL4CompilerDescriptor, MTLCaptureDescriptor, MTLCaptureDestination,
- MTLCaptureManager, MTLCaptureTarget, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,
+ MTL4CommandAllocator, MTL4CommandQueue, MTL4CommandQueueDescriptor, MTL4Compiler, MTL4CompilerDescriptor,
+ MTLCaptureTarget, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLEventExt,
MTLFunctionConstantValues, MTLGPUFamily, MTLLibrary, MTLResidencySet, MTLResidencySetDescriptor, MTLSparsePageSize,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
@@ -19,11 +19,13 @@ use crate::backends::{
common::{
Backend, Context, DeviceCapabilities,
allocator::{block::BlockAllocator, pool::PoolAllocator},
+ capture_matcher::CAPTURE_MATCHER,
},
metal::{
Metal,
buffer::{dense::MetalDenseBuffer, sparse::MetalSparseBuffer},
- command_buffer::{MetalCommandBufferCache, MetalCommandBufferEncoding},
+ capture::MetalCaptureSpan,
+ command_buffer::MetalCommandBufferEncoding,
decompression,
error::MetalError,
heaps::MetalHeaps,
@@ -53,7 +55,7 @@ pub struct MetalContext {
pub(super) block_allocator: Arc<BlockAllocator<MetalDenseBuffer>>,
pub(super) heaps: Arc<MetalHeaps>,
pub(super) peak_memory_usage: Arc<AtomicUsize>,
- pub(super) command_buffer_cache: Mutex<Vec<MetalCommandBufferCache>>,
+ pub(super) command_allocator_cache: Mutex<Vec<Retained<ProtocolObject<dyn MTL4CommandAllocator>>>>,
library_cache: Mutex<HashMap<usize, Retained<ProtocolObject<dyn MTLLibrary>>>>,
pipeline_cache: Mutex<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,
weak_self: Weak<MetalContext>,
@@ -101,7 +103,7 @@ impl MetalContext {
}
let library = self.library(library_data, library_compressed)?;
- let pipeline = self.compiler.compute_pipeline_state(&library, function_name, constants)?;
+ let pipeline = self.compiler.compute_pipeline_state(&library, function_name, constants, Some(cache_key))?;
self.pipeline_cache.lock().insert(cache_key.to_string(), pipeline.clone());
Ok(pipeline)
@@ -112,7 +114,14 @@ impl Context for MetalContext {
type Backend = Metal;
fn new() -> Result<Arc<Self>, MetalError> {
+ if CAPTURE_MATCHER.enabled() {
+ unsafe {
+ std::env::set_var("METAL_CAPTURE_ENABLED", "1");
+ }
+ }
+
let device = <dyn MTLDevice>::system_default().ok_or(MetalError::CannotOpenDevice)?;
+
let device_name = device.name();
let gpu_core_count = device.gpu_core_count();
let apple_gpu_family = device.newest_supported_apple_gpu_family();
@@ -126,13 +135,21 @@ impl Context for MetalContext {
.new_residency_set_with_descriptor(&residency_set_descriptor)
.map_err(|nserror| MetalError::CannotCreateResidencySet(nserror.to_string()))?;
- let command_queue = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueue)?;
+ let queue_descriptor = MTL4CommandQueueDescriptor::new();
+ queue_descriptor.set_label(Some("compute queue"));
+ let command_queue = device
+ .new_mtl4_command_queue_with_descriptor(&queue_descriptor)
+ .map_err(|_| MetalError::CannotCreateCommandQueue)?;
command_queue.add_residency_set(&residency_set);
- let sparse_queue = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueue)?;
+ queue_descriptor.set_label(Some("sparse queue"));
+ let sparse_queue = device
+ .new_mtl4_command_queue_with_descriptor(&queue_descriptor)
+ .map_err(|_| MetalError::CannotCreateCommandQueue)?;
sparse_queue.add_residency_set(&residency_set);
let sparse_event = device.new_event().ok_or(MetalError::CannotCreateEvent)?;
+ sparse_event.set_label(Some("sparse event"));
let sparse_state = SparseState {
did_ops: false,
@@ -162,7 +179,7 @@ impl Context for MetalContext {
block_allocator,
heaps,
peak_memory_usage,
- command_buffer_cache: Mutex::new(Vec::with_capacity(32)),
+ command_allocator_cache: Mutex::new(Vec::with_capacity(32)),
library_cache: Mutex::new(HashMap::new()),
pipeline_cache: Mutex::new(HashMap::new()),
weak_self: weak_self.clone(),
@@ -203,32 +220,11 @@ impl Context for MetalContext {
Some(self.peak_memory_usage.load(Ordering::Relaxed))
}
- fn enable_capture() {
- unsafe {
- std::env::set_var("METAL_CAPTURE_ENABLED", "1");
- }
- }
-
fn start_capture(
&self,
- trace_path: &Path,
- ) -> Result<(), <Self::Backend as Backend>::Error> {
- let capture_descriptor = MTLCaptureDescriptor::new();
- capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);
- capture_descriptor.set_output_path(Some(&trace_path.with_added_extension("gputrace")));
- capture_descriptor.set_capture_object(Some(&MTLCaptureTarget::Device(self.device.clone())));
-
- MTLCaptureManager::shared_capture_manager()
- .start_capture_with_descriptor(&capture_descriptor)
- .map_err(|error| MetalError::CannotStartGpuCapture(error.to_string()))?;
-
- Ok(())
- }
-
- fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error> {
- MTLCaptureManager::shared_capture_manager().stop_capture();
-
- Ok(())
+ output_path: &Path,
+ ) -> Result<MetalCaptureSpan, MetalError> {
+ MetalCaptureSpan::new(&MTLCaptureTarget::Device(self.device.clone()), output_path)
}
fn device_capabilities(&self) -> DeviceCapabilities {
diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rs
index e51351964..ce3c505a8 100644
--- a/crates/uzu-engine/src/backends/metal/error.rs
+++ b/crates/uzu-engine/src/backends/metal/error.rs
@@ -13,7 +13,7 @@ pub enum MetalError {
CannotOpenDevice,
#[error("Cannot create residency set: {0}")]
CannotCreateResidencySet(String),
- #[error("Cannot start gpu capture {0}")]
+ #[error("Cannot start GPU capture: {0}")]
CannotStartGpuCapture(String),
#[error("Cannot create library: {0}")]
CannotCreateLibrary(String),
diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rs
index 653c31f41..15de231b5 100644
--- a/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rs
+++ b/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rs
@@ -15,6 +15,7 @@ pub trait CompilerPipelineExtensions {
library: &ProtocolObject<dyn MTLLibrary>,
function_name: &str,
constants: Option<&MTLFunctionConstantValues>,
+ label: Option<&str>,
) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError>;
}
@@ -24,12 +25,14 @@ impl CompilerPipelineExtensions for ProtocolObject<dyn MTL4Compiler> {
library: &ProtocolObject<dyn MTLLibrary>,
function_name: &str,
constants: Option<&MTLFunctionConstantValues>,
+ label: Option<&str>,
) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {
let library_function = MTL4LibraryFunctionDescriptor::new();
library_function.set_library(Some(library));
library_function.set_name(Some(function_name));
let pipeline_descriptor = MTL4ComputePipelineDescriptor::new();
+ pipeline_descriptor.set_label(label);
match constants {
Some(constants) => {
let specialized_function = MTL4SpecializedFunctionDescriptor::new();
diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rs
index 01324c58f..3b0320734 100644
--- a/crates/uzu-engine/src/backends/metal/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/mod.rs
@@ -1,5 +1,6 @@
mod backend;
mod buffer;
+mod capture;
mod command_buffer;
mod context;
mod decompression;
diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rs
index c6c852b6a..7fc633724 100644
--- a/crates/uzu-engine/src/encodable_block/convolution.rs
+++ b/crates/uzu-engine/src/encodable_block/convolution.rs
@@ -79,8 +79,6 @@ impl<B: Backend> SeparableCausalConv<B> {
sequence_length: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("SeparableCausalConv");
-
let mut output =
command_buffer.allocate_scratch_for_shape(&[sequence_length, self.model_dim], self.data_type)?;
let coefficients_offset_bytes = size_for_shape(&[coefficient_column_offset], self.data_type);
@@ -95,7 +93,6 @@ impl<B: Backend> SeparableCausalConv<B> {
command_buffer,
);
- command_buffer.pop_debug_group();
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index fd218c54a..0cd58c0f7 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -157,8 +157,6 @@ impl<B: Backend> Embedding<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, EmbeddingError<B>> {
- command_buffer.push_debug_group("embedding lookup");
-
let mut output = command_buffer
.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)
.map_err(EmbeddingError::BackendError)?;
@@ -175,8 +173,6 @@ impl<B: Backend> Embedding<B> {
};
table.encode_lookup(token_ids, &mut output, batch_dim, self.input_scale, command_buffer);
- command_buffer.pop_debug_group();
-
Ok(output)
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
index 4f7f6a9ad..b1fcce5dd 100644
--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
@@ -190,9 +190,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("matmul");
-
- let output = self.encode_with_a(
+ self.encode_with_a(
MatmulA::FullPrecision {
values: &input,
offset: 0,
@@ -200,11 +198,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
batch_dim,
None::<Gather<&B::ScratchBuffer>>,
command_buffer,
- )?;
-
- command_buffer.pop_debug_group();
-
- Ok(output)
+ )
}
fn encode_input(
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index c4dbbe8d8..d38dc43b8 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -189,16 +189,12 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("linear (rht)");
-
let input = match input {
LinearInput::FullPrecision(input) => input,
- input => {
- let output = self.inner_linear.encode_input(input, batch_dim, command_buffer);
- command_buffer.pop_debug_group();
- return output;
- },
+ input => return self.inner_linear.encode_input(input, batch_dim, command_buffer),
};
+ command_buffer.push_debug_group("linear (rht)");
+
let format = self.inner_linear.select_activation_format(batch_dim, command_buffer.context());
let input = self.input_rht.prepare_in_place(input, batch_dim, format, command_buffer)?;
let output = self.inner_linear.encode_with_a(
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
index 979998d7a..02beb5503 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
@@ -3,7 +3,7 @@ use thiserror::Error;
use crate::{
backends::common::{
- Backend, BufferMut, CommandBuffer, CommandBufferEncoding,
+ Backend, BufferMut, CommandBuffer,
kernel::{Kernels, QKVNormKernel},
},
config::normalization::{NormalizationConfig, UpcastMode},
@@ -152,8 +152,6 @@ impl<B: Backend> QKVNorm<B> {
"QKV norm input row stride ({input_row_stride}) is smaller than its packed row width ({packed_row_width})"
);
- command_buffer.push_debug_group("qkv norm");
-
let kv = self.num_kv_heads;
let heads = [(&self.query, 0, q_heads), (&self.key, q_heads, kv), (&self.value, q_heads + kv, kv)];
for (head, head_offset, head_count) in heads {
@@ -179,8 +177,6 @@ impl<B: Backend> QKVNorm<B> {
);
}
- command_buffer.pop_debug_group();
-
Ok(())
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
index 4eab50d2f..9ffeb4f3d 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
@@ -58,8 +58,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
batch_dim: u32,
act_format: ActivationFormat,
) -> Result<LinearInput<B>, B::Error> {
- command_buffer.push_debug_group("gate act mul");
-
if self.activation.act_type() == ActivationType::IDENTITY {
panic!("Identity activation is not supported for kernel")
}
@@ -115,8 +113,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
LinearInput::FullPrecision(hidden)
};
- command_buffer.pop_debug_group();
-
Ok(input)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rs
index 6fe74ae30..1a1ec09d7 100644
--- a/crates/uzu-engine/src/encodable_block/normalization.rs
+++ b/crates/uzu-engine/src/encodable_block/normalization.rs
@@ -120,8 +120,6 @@ impl<B: Backend> Normalization<B> {
shortcut: Option<impl BufferMut<Backend = B>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("normalization");
-
let row_size = size_for_shape(&[self.element_count], self.data_type);
let row_offset_bytes = row_offset as usize * row_size;
let shortcut = shortcut.map(|shortcut| shortcut.subrange_mut(row_offset_bytes..));
@@ -141,8 +139,6 @@ impl<B: Backend> Normalization<B> {
command_buffer,
);
- command_buffer.pop_debug_group();
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rs
index c73b991b6..2bb7b8bea 100644
--- a/crates/uzu-engine/src/encodable_block/transformer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer.rs
@@ -74,6 +74,8 @@ impl<B: Backend> TransformerState<B> {
command_buffer.pop_debug_group();
+ command_buffer.add_debug_point(&format!("accepted {} tokens", accepted_indices.len()));
+
Ok(())
}
}
diff --git a/crates/uzu-engine/src/engine/capture.rs b/crates/uzu-engine/src/engine/capture.rs
deleted file mode 100644
index 93713fea3..000000000
--- a/crates/uzu-engine/src/engine/capture.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-use std::{
- path::PathBuf,
- sync::{
- Arc, LazyLock,
- atomic::{AtomicBool, Ordering},
- },
-};
-
-use crate::backends::common::{Backend, Context};
-
-static CAPTURE_FIRST_PREFILL: LazyLock<AtomicBool> =
- LazyLock::new(|| AtomicBool::new(std::env::var("UZU_CAPTURE_FIRST_PREFILL").is_ok()));
-static CAPTURE_FIRST_DECODE: LazyLock<AtomicBool> =
- LazyLock::new(|| AtomicBool::new(std::env::var("UZU_CAPTURE_FIRST_DECODE").is_ok()));
-
-pub struct CaptureRequest<B: Backend> {
- context: Arc<B::Context>,
- path: PathBuf,
-}
-
-impl<B: Backend> CaptureRequest<B> {
- fn new(
- context: Arc<B::Context>,
- path: PathBuf,
- ) -> Self {
- Self {
- context,
- path,
- }
- }
-
- pub fn start(self) -> Result<CaptureSpan<B>, B::Error> {
- self.context.start_capture(&self.path)?;
- eprintln!("Started capturing to {:?}", self.path);
-
- Ok(CaptureSpan {
- context: self.context,
- path: self.path,
- })
- }
-}
-
-pub struct CaptureSpan<B: Backend> {
- context: Arc<B::Context>,
- path: PathBuf,
-}
-
-impl<B: Backend> Drop for CaptureSpan<B> {
- fn drop(&mut self) {
- match self.context.stop_capture() {
- Ok(()) => {
- eprintln!("Successfully saved capture to {:?}", self.path);
- },
- Err(err) => {
- eprintln!("Error when saving capture to {:?}: {:?}", self.path, err);
- },
- }
- }
-}
-
-pub struct CaptureManager<B: Backend> {
- context: Arc<B::Context>,
-}
-
-impl<B: Backend> CaptureManager<B> {
- pub fn pre_load_enable() -> bool {
- let capture_enabled =
- CAPTURE_FIRST_PREFILL.load(Ordering::Relaxed) || CAPTURE_FIRST_DECODE.load(Ordering::Relaxed);
-
- if capture_enabled {
- <B::Context as Context>::enable_capture();
- }
-
- capture_enabled
- }
-
- pub fn new(context: Arc<B::Context>) -> Self {
- Self {
- context,
- }
- }
-
- fn capture_path(suffix: &str) -> PathBuf {
- PathBuf::from("/tmp").join(format!("uzu-capture-{}-{}", B::NAME, suffix))
- }
-
- pub fn maybe_capture_prefill_step(&self) -> Option<CaptureRequest<B>> {
- CAPTURE_FIRST_PREFILL
- .swap(false, Ordering::Relaxed)
- .then(|| CaptureRequest::new(self.context.clone(), Self::capture_path("prefill")))
- }
-
- pub fn maybe_capture_decode_step(&self) -> Option<CaptureRequest<B>> {
- CAPTURE_FIRST_DECODE
- .swap(false, Ordering::Relaxed)
- .then(|| CaptureRequest::new(self.context.clone(), Self::capture_path("decode")))
- }
-}
diff --git a/crates/uzu-engine/src/engine/language_model/stream/mod.rs b/crates/uzu-engine/src/engine/language_model/stream/mod.rs
index fc456ba8d..cd35827ba 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/mod.rs
@@ -4,7 +4,7 @@ use thiserror::Error;
#[cfg(grammar)]
use crate::engine::language_model::grammar::{Grammar, GrammarError};
use crate::{
- backends::common::Backend,
+ backends::common::{Backend, capture_matcher::CaptureError},
encodable_block::decoder::DecoderError,
engine::language_model::{LanguageModel, state::LanguageModelState},
speculators::dflash_tfm::DFlashTreeError,
@@ -25,6 +25,8 @@ pub struct LanguageModelStreamOptions {
pub enum LanguageModelStreamError<B: Backend> {
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
+ #[error("Capture error: {0}")]
+ Capture(#[from] CaptureError<B>),
#[error("Decoder error: {0}")]
Decoder(#[from] DecoderError<B>),
#[cfg(grammar)]
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 7663cf362..cdd1be257 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -13,17 +13,17 @@ use crate::{
array::size_for_shape,
backends::common::{
Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,
- CommandBufferPending, Context, gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,
+ CommandBufferPending, Context,
+ capture_matcher::{CAPTURE_MATCHER, CaptureMatcherSpan},
+ gpu_types::trie::TrieNode as GpuTrieNode,
+ kernel::ContextRingUpdateKernel,
},
data_type::DataType,
encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},
- engine::{
- capture::CaptureSpan,
- language_model::{
- LanguageModel,
- state::LanguageModelState,
- stream::{LanguageModelStreamError, LanguageModelStreamOptions},
- },
+ engine::language_model::{
+ LanguageModel,
+ state::LanguageModelState,
+ stream::{LanguageModelStreamError, LanguageModelStreamOptions},
},
trie::TrieNode,
};
@@ -92,7 +92,7 @@ struct DecodingStatePending<B: Backend> {
input_trie: TrieNode,
full_accept: bool,
pending: Box<[<B::CommandBuffer as CommandBuffer>::Pending]>,
- capture_span: Option<CaptureSpan<B>>,
+ capture_span: Option<CaptureMatcherSpan>,
hidden_features: Option<Box<[B::ScratchBuffer]>>,
output_norm: Option<B::ScratchBuffer>,
output_tokens: Arc<B::GlobalBuffer>,
@@ -109,7 +109,7 @@ enum DecodingState<B: Backend> {
num_accepted: usize,
hidden_features: Option<Box<[B::ScratchBuffer]>>,
output_norm: Option<(B::ScratchBuffer, usize)>,
- capture_span: Option<CaptureSpan<B>>,
+ capture_span: Option<CaptureMatcherSpan>,
},
Halted,
Invalid,
@@ -158,14 +158,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
return Err(LanguageModelStreamError::ContextOverflow);
}
- let capture_span = if let Some(capture_manager) = &model.engine.capture_manager
- && let Some(capture_request) = capture_manager.maybe_capture_prefill_step()
- {
- Some(capture_request.start().map_err(LanguageModelStreamError::Backend)?)
- } else {
- None
- };
-
let allocation_pool = model.engine.context.create_allocation_pool();
let mut context_ring = if let Some(suffix_repetition_length) =
@@ -198,6 +190,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let mut metrics = TokenStreamMetrics::default();
let decoding_state = if !input.is_empty() {
+ let _capture_span = CAPTURE_MATCHER.capture::<B>(&model.engine.context, "prefill")?;
model_state.last_output_token.take();
// NOTE: this is required for attention correctness (hardcoded suffix 1024). This is really bad design, attention should be rewritten to allow on-demand suffix length
@@ -218,6 +211,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.create_command_buffer(Some("prefill"), Some(allocation_pool.clone()))
.map_err(LanguageModelStreamError::Backend)?;
+ command_buffer.add_debug_point(&format!(
+ "prefill at context {}, suffix {} ({} batches)",
+ model_state.transformer_state.context_length(),
+ input.len(),
+ number_of_batches
+ ));
+
let mut output_tokens = None;
let mut output_norm = None;
let split_logits_row = model.decoder.prefill_cache_skips_trailing_layers();
@@ -349,7 +349,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
input_trie: TrieNode::new(0, 0, 0.0),
full_accept: true,
pending,
- capture_span,
+ capture_span: None,
hidden_features: None,
output_norm,
output_tokens: Arc::new(output_tokens.unwrap()),
@@ -374,9 +374,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
}
fn generate(&mut self) -> Result<Option<u64>, LanguageModelStreamError<B>> {
- let (mut prev_output, mut command_buffer): (
+ let (mut prev_output, acceptance): (
ForwardPassChaining<B>,
- Option<<B::CommandBuffer as CommandBuffer>::Encoding>,
+ Option<(<B::CommandBuffer as CommandBuffer>::Encoding, Option<CaptureMatcherSpan>)>,
) = match replace(&mut self.decoding_state, DecodingState::Invalid) {
DecodingState::Seeded {
seed_token,
@@ -458,7 +458,10 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.model
.engine
.context
- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
+ .create_command_buffer(
+ Some(&format!("decode {} (accept)", self.metrics.num_decode_forward_passes - 1)),
+ Some(self.allocation_pool.clone()),
+ )
.map_err(LanguageModelStreamError::Backend)?;
self.model_state
.transformer_state
@@ -485,7 +488,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
None
};
if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {
- command_buffer.push_debug_group("update repetition penalty ring");
let accepted_input_token_ids_const = command_buffer
.allocate_constant_from_slice(
&accepted_input_token_ids
@@ -501,23 +503,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
full.len() as u32,
&mut command_buffer,
);
- command_buffer.pop_debug_group();
- }
- if let Some(capture_span) = capture_span {
- command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
- .map_err(LanguageModelStreamError::Backend)?;
-
- drop(capture_span);
-
- command_buffer = self
- .model
- .engine
- .context
- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
- .map_err(LanguageModelStreamError::Backend)?;
}
self.model_state.tokens.extend(accepted_output_token_ids);
(
@@ -525,7 +510,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
token: output_token_id,
output_norm,
},
- Some(command_buffer),
+ Some((command_buffer, capture_span)),
)
}
},
@@ -548,20 +533,14 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
));
}
- let capture_span = if let Some(capture_manager) = &self.model.engine.capture_manager
- && let Some(capture_request) = capture_manager.maybe_capture_decode_step()
- {
- prev_output.resolve(
- &mut self.model_state.tokens,
- #[cfg(grammar)]
- self.options.grammar.as_mut(),
- )?;
- Some(capture_request.start().map_err(LanguageModelStreamError::Backend)?)
- } else {
- None
- };
-
let mut pending = Vec::new();
+ if let Some((accept_command_buffer, _capture_span)) = acceptance {
+ pending.push(accept_command_buffer.end_encoding().submit());
+ }
+
+ let mut capture_span = CAPTURE_MATCHER
+ .capture::<B>(&self.model.engine.context, &format!("decode:{}", self.metrics.num_decode_forward_passes))?;
+
let (input_trie, chain_copy, full_accept) = if let Some(speculator) = &self.model.speculator
&& let Some(shape) = speculator.make_shape(
self.model_state.max_context_length.map(|max_context_length| max_context_length - context_length),
@@ -572,9 +551,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
#[cfg(grammar)]
self.options.grammar.as_mut(),
)? {
- if let Some(accept_command_buffer) = command_buffer.take() {
- pending.push(accept_command_buffer.end_encoding().submit());
- }
let trie = speculator.propose_tree(
self.model_state.speculator_state.as_mut().unwrap(),
output_norm,
@@ -585,6 +561,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.options.grammar.as_mut(),
&self.model_state.prng,
self.allocation_pool.clone(),
+ Some(&format!("decode {} (speculate)", self.metrics.num_decode_forward_passes)),
)?;
prev_output.drop_output_norm();
(trie, None, false)
@@ -601,19 +578,25 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
};
let input_flat_trie = input_trie.linearize();
- if let Some(accept_command_buffer) = command_buffer.take() {
- pending.push(accept_command_buffer.end_encoding().submit());
- }
-
self.allocation_pool = self.model.engine.context.create_allocation_pool();
let mut command_buffer = self
.model
.engine
.context
- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
+ .create_command_buffer(
+ Some(&format!("decode {} (forward)", self.metrics.num_decode_forward_passes)),
+ Some(self.allocation_pool.clone()),
+ )
.map_err(LanguageModelStreamError::Backend)?;
+ command_buffer.add_debug_point(&format!(
+ "decode round {} at context {} suffix {}",
+ self.metrics.num_decode_forward_passes,
+ context_length,
+ input_flat_trie.len()
+ ));
+
let scratch_token_ids;
let mut constant_token_ids = None;
let token_ids: &dyn Buffer<Backend = B> = if let Some(chain_copy) = chain_copy.as_deref() {
@@ -663,7 +646,10 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.model
.engine
.context
- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
+ .create_command_buffer(
+ Some(&format!("decode {} (sample and accept)", self.metrics.num_decode_forward_passes)),
+ Some(self.allocation_pool.clone()),
+ )
.map_err(LanguageModelStreamError::Backend)?;
let mut bitmask = command_buffer
@@ -761,6 +747,10 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
drop(constant_token_ids);
pending.push(command_buffer.end_encoding().submit());
+ if full_accept {
+ capture_span.take();
+ }
+
self.metrics.num_decode_forward_passes += 1;
self.metrics.num_tokens_proposed += input_flat_trie.len();
if full_accept {
@@ -822,7 +812,10 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
.model
.engine
.context
- .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))
+ .create_command_buffer(
+ Some(&format!("decode {} (accept on drop)", self.metrics.num_decode_forward_passes - 1)),
+ Some(self.allocation_pool.clone()),
+ )
.unwrap();
self.model_state.transformer_state.encode_accept(&[0], &mut command_buffer).unwrap();
if let Some(speculator) = self.model.speculator.as_ref() {
@@ -853,7 +846,10 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
.model
.engine
.context
- .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))
+ .create_command_buffer(
+ Some(&format!("decode {} (accept on drop)", self.metrics.num_decode_forward_passes - 1)),
+ Some(self.allocation_pool.clone()),
+ )
.unwrap();
let accepted_token_indicies =
full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
diff --git a/crates/uzu-engine/src/engine/mod.rs b/crates/uzu-engine/src/engine/mod.rs
index 8f1f941b6..c62f5bca7 100644
--- a/crates/uzu-engine/src/engine/mod.rs
+++ b/crates/uzu-engine/src/engine/mod.rs
@@ -4,39 +4,38 @@ use derive_more::Debug;
use thiserror::Error;
use crate::{
- backends::common::{Backend, Context},
+ backends::common::{
+ Backend, Context,
+ capture_matcher::{CAPTURE_MATCHER, CaptureError, CaptureMatcherSpan},
+ },
config::model::AnyModelConfig,
encodable_block::sampling::SamplingMethod,
- engine::capture::CaptureManager,
};
pub mod classifier_model;
pub mod language_model;
-mod capture;
-
pub struct Engine<B: Backend> {
+ _capture_span: Option<CaptureMatcherSpan>,
context: Arc<B::Context>,
- capture_manager: Option<CaptureManager<B>>,
}
#[derive(Debug, Error)]
pub enum EngineNewError<B: Backend> {
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
+ #[error("Capture error: {0}")]
+ Capture(#[from] CaptureError<B>),
}
impl<B: Backend> Engine<B> {
pub fn new() -> Result<Arc<Self>, EngineNewError<B>> {
- let capture_enabled = CaptureManager::<B>::pre_load_enable();
-
let context = <B::Context as Context>::new().map_err(EngineNewError::Backend)?;
-
- let capture_manager = capture_enabled.then(|| CaptureManager::new(context.clone()));
+ let capture_span = CAPTURE_MATCHER.capture::<B>(&context, "")?;
Ok(Arc::new(Self {
+ _capture_span: capture_span,
context,
- capture_manager,
}))
}
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index 6c41724b5..7a37d05e3 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -217,6 +217,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
#[cfg(grammar)] grammar: Option<&mut Grammar>,
prng: &PRng,
allocation_pool: Arc<B::AllocationPool>,
+ command_buffer_name: Option<&str>,
) -> Result<TrieNode, DFlashTreeError<B>> {
assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");
@@ -232,7 +233,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let mut command_buffer = self
.context
- .create_command_buffer(Some("speculator propose"), Some(allocation_pool))
+ .create_command_buffer(command_buffer_name, Some(allocation_pool))
.map_err(DFlashTreeError::Backend)?;
let nodes = match shape.construction_method {
diff --git a/crates/uzu-engine/unit/BENCHMARKS.md b/crates/uzu-engine/unit/BENCHMARKS.md
index 06662faca..f539bfbe3 100644
--- a/crates/uzu-engine/unit/BENCHMARKS.md
+++ b/crates/uzu-engine/unit/BENCHMARKS.md
@@ -58,9 +58,9 @@ CRITERION_HOME="$PWD/target/criterion/m2_max" cargo bench \
--save-baseline matmul_baseline_m2_max
```
-Set `UZU_CAPTURE_BENCH=1` to capture the first matching benchmark command buffer as a Metal `.gputrace`.
-`UZU_CAPTURE_BENCH_FILTER` is an optional benchmark path substring; `UZU_CAPTURE_BENCH_DIR` defaults to the current
-directory.
+Set `UZU_CAPTURE='<benchmark path>'` to capture the first matching batch; named helpers use exact Criterion paths.
+Append `@path.gputrace` to choose the output (default `/tmp/uzu.gputrace` for one capture).
+Streams also expose `prefill` and zero-based `decode:N` per stream; ranges use `start..end`, e.g. `prefill..decode:2`.
## Running on iPhone (via `cargo-dinghy`)
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
index 3a542dbed..e21376da3 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
@@ -99,12 +99,11 @@ fn bench_build_tree_out_type<T: ArrayElement + Float>(
for &batch_size in BATCH_SIZES {
for &tree_size in TREE_SIZES {
let mut buffers = make_buffers::<T>(context, batch_size, tree_size);
- let benchmark_path = format!(
- "Metal/Kernel/GDNTreeVerify/BuildTreeOut/{data_type_name}/{path}/{tile_name}/B{batch_size}_T{tree_size}"
- );
- let benchmark_id = BenchmarkId::from_parameter(format!(
- "B{batch_size}_T{tree_size}_QK{QK_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}_V{HEAD_V_DIM}"
- ));
+ let parameter =
+ format!("B{batch_size}_T{tree_size}_QK{QK_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}_V{HEAD_V_DIM}");
+ let benchmark_path =
+ format!("Metal/Kernel/GDNTreeVerify/BuildTreeOut/{data_type_name}/{path}/{tile_name}/{parameter}");
+ let benchmark_id = BenchmarkId::from_parameter(parameter);
let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(
context,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
index 99fc7e66c..577d8d1f7 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
@@ -168,11 +168,9 @@ fn bench_build_tree_gram(c: &mut Criterion) {
)
.expect("BuildTreeGramKernel");
let scale = (HEAD_K_DIM as f32).sqrt().recip();
- let benchmark_path =
- format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}/B{batch_size}_T{tree_size}");
- let benchmark_id = BenchmarkId::from_parameter(format!(
- "B{batch_size}_T{tree_size}_Hg{K_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}"
- ));
+ let parameter = format!("B{batch_size}_T{tree_size}_Hg{K_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}");
+ let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}/{parameter}");
+ let benchmark_id = BenchmarkId::from_parameter(parameter);
let mut buffers = ColdPool::new(buffers_bytes(batch_size, tree_size), || {
make_buffers(&context, batch_size, tree_size).0
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
index ac165afe3..9b3c14c8f 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
@@ -187,10 +187,9 @@ fn bench_tree_update_solve(c: &mut Criterion) {
group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));
for &(batch_size, tree_size) in BENCH_SHAPES {
- let benchmark_path =
- format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}/B{batch_size}_T{tree_size}");
- let benchmark_id =
- BenchmarkId::from_parameter(format!("B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}"));
+ let parameter = format!("B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}");
+ let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}/{parameter}");
+ let benchmark_id = BenchmarkId::from_parameter(parameter);
let mut buffers =
ColdPool::new(buffers_bytes(batch_size, tree_size), || make_buffers(&context, batch_size, tree_size));
diff --git a/crates/uzu-engine/unit/common/env_vars.rs b/crates/uzu-engine/unit/common/env_vars.rs
deleted file mode 100644
index 15d5912d8..000000000
--- a/crates/uzu-engine/unit/common/env_vars.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-pub const UZU_CAPTURE_BENCH: &str = "UZU_CAPTURE_BENCH";
-pub const UZU_CAPTURE_BENCH_DIR: &str = "UZU_CAPTURE_BENCH_DIR";
-pub const UZU_CAPTURE_BENCH_FILTER: &str = "UZU_CAPTURE_BENCH_FILTER";
-
-pub fn enabled(name: &str) -> bool {
- std::env::var(name).is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("yes") || v.eq_ignore_ascii_case("true"))
-}
diff --git a/crates/uzu-engine/unit/common/harness.rs b/crates/uzu-engine/unit/common/harness.rs
index 1aa1de161..30a9dcc1a 100644
--- a/crates/uzu-engine/unit/common/harness.rs
+++ b/crates/uzu-engine/unit/common/harness.rs
@@ -1,8 +1,5 @@
use libtest_mimic::{Arguments, Trial};
-const METAL_CAPTURE_ENABLED: &str = "METAL_CAPTURE_ENABLED";
-const UZU_CAPTURE_BENCH: &str = "UZU_CAPTURE_BENCH";
-
pub enum UzuTest {
Bench(&'static dyn Fn()),
Test(&'static UzuTestCase),
@@ -30,7 +27,6 @@ pub fn uzu_harness(tests: &[&UzuTest]) {
if benchmarks {
#[cfg(target_os = "ios")]
ios_set_current_dir();
- enable_benchmark_gpu_capture_if_requested();
let bench_tests: Vec<&dyn Fn()> = tests
.iter()
.filter_map(|test| match test {
@@ -56,15 +52,3 @@ pub fn uzu_harness(tests: &[&UzuTest]) {
libtest_mimic::run(&Arguments::from_args(), trials).exit();
}
}
-
-fn enable_benchmark_gpu_capture_if_requested() {
- if enabled(UZU_CAPTURE_BENCH) {
- unsafe {
- std::env::set_var(METAL_CAPTURE_ENABLED, "1");
- }
- }
-}
-
-fn enabled(name: &str) -> bool {
- std::env::var(name).is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("yes") || v.eq_ignore_ascii_case("true"))
-}
diff --git a/crates/uzu-engine/unit/common/matmul/bench.rs b/crates/uzu-engine/unit/common/matmul/bench.rs
index daca0ee2a..5e538e948 100644
--- a/crates/uzu-engine/unit/common/matmul/bench.rs
+++ b/crates/uzu-engine/unit/common/matmul/bench.rs
@@ -1,57 +1,18 @@
-use std::{
- env,
- path::PathBuf,
- sync::atomic::{AtomicBool, Ordering},
- time::{SystemTime, UNIX_EPOCH},
-};
-
use criterion::Bencher;
-use super::super::env_vars;
use crate::backends::common::{
Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
- CommandBufferPending, Context,
+ CommandBufferPending, Context, capture_matcher::CAPTURE_MATCHER,
};
-static CAPTURE_TAKEN: AtomicBool = AtomicBool::new(false);
-
-fn should_capture_benchmark(benchmark_path: &str) -> bool {
- env_vars::enabled(env_vars::UZU_CAPTURE_BENCH)
- && benchmark_path.starts_with("Metal/")
- && env::var(env_vars::UZU_CAPTURE_BENCH_FILTER).map_or(true, |filter| benchmark_path.contains(&filter))
- && !CAPTURE_TAKEN.swap(true, Ordering::AcqRel)
-}
-
-fn benchmark_capture_path() -> PathBuf {
- let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).expect("system clock before Unix epoch").as_secs();
- env::var(env_vars::UZU_CAPTURE_BENCH_DIR)
- .map(PathBuf::from)
- .unwrap_or(env::current_dir().unwrap())
- .join(format!("uzu_bench-{timestamp}.gputrace"))
-}
-
-fn start_benchmark_capture<B: Backend>(
- context: &B::Context,
- benchmark_path: &str,
-) -> bool {
- if !should_capture_benchmark(benchmark_path) {
- return false;
- }
-
- let path = benchmark_capture_path();
- context.start_capture(&path).expect("failed to start benchmark GPU capture");
- println!("GPU benchmark capture started for {benchmark_path}: {path:?}");
- true
-}
-
pub fn iter_encode_loop<B: Backend, F>(
context: &B::Context,
bencher: &mut Bencher,
- mut encode: F,
+ encode: F,
) where
F: FnMut(&mut <B::CommandBuffer as CommandBuffer>::Encoding),
{
- iter_encode_loop_named::<B, _>(context, bencher, "unnamed_benchmark", |command_buffer| encode(command_buffer));
+ iter_encode_loop_named::<B, _>(context, bencher, "unnamed_benchmark", encode);
}
pub fn iter_encode_loop_named<B: Backend, F>(
@@ -63,15 +24,14 @@ pub fn iter_encode_loop_named<B: Backend, F>(
F: FnMut(&mut <B::CommandBuffer as CommandBuffer>::Encoding),
{
bencher.iter_custom(|n_iters| {
- let capture = start_benchmark_capture::<B>(context, benchmark_path);
- let mut command_buffer = context.create_command_buffer(None, None).unwrap();
+ let _capture_span = CAPTURE_MATCHER
+ .capture::<B>(context, benchmark_path)
+ .unwrap_or_else(|error| panic!("failed to capture benchmark {benchmark_path}: {error}"));
+ let mut command_buffer = context.create_command_buffer(Some(benchmark_path), None).unwrap();
for _ in 0..n_iters {
encode(&mut command_buffer);
}
let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- if capture {
- context.stop_capture().expect("failed to stop benchmark GPU capture");
- }
completed.gpu_execution_time()
});
}
diff --git a/crates/uzu-engine/unit/common/mod.rs b/crates/uzu-engine/unit/common/mod.rs
index dfa417228..2448a1500 100644
--- a/crates/uzu-engine/unit/common/mod.rs
+++ b/crates/uzu-engine/unit/common/mod.rs
@@ -3,7 +3,6 @@ compile_error!("uzu tests require cpu backend");
pub mod assert;
pub mod cold_pool;
-pub mod env_vars;
pub mod harness;
pub mod helpers;
pub mod matmul;