A high-performance inference engine for AI models
New capture (#929)
uuuvn55dc41d5cbf7
diff --git a/crates/uzu-engine/src/backends/common/backend.rs b/crates/uzu-engine/src/backends/common/backend.rsindex 6ce7985d3..6acdcbc50 100644--- a/crates/uzu-engine/src/backends/common/backend.rs+++ b/crates/uzu-engine/src/backends/common/backend.rs@@ -10,6 +10,7 @@ pub trait Backend: Clone + Send + Sync + 'static {type ScratchBuffer: ScratchBuffer<Backend = Self>;type SparseBuffer: SparseBuffer<Backend = Self>;type AllocationPool: Send + Sync;+ type CaptureSpan: Send + Sync;type Kernels: Kernels<Backend = Self>;type Error: Error;diff --git a/crates/uzu-engine/src/backends/common/capture_matcher.rs b/crates/uzu-engine/src/backends/common/capture_matcher.rsnew file mode 100644index 000000000..2c3a91f74--- /dev/null+++ b/crates/uzu-engine/src/backends/common/capture_matcher.rs@@ -0,0 +1,110 @@+use std::{+ collections::HashMap,+ env,+ path::PathBuf,+ sync::{+ LazyLock,+ mpsc::{self, Receiver, Sender},+ },+};++use parking_lot::Mutex;+use thiserror::Error;++use crate::backends::common::{Backend, Context};++pub static CAPTURE_MATCHER: LazyLock<CaptureMatcher> = LazyLock::new(CaptureMatcher::new);++struct CaptureMatcherInner {+ starts: HashMap<String, (Sender<Box<dyn Send + Sync>>, PathBuf)>,+ ends: HashMap<String, Receiver<Box<dyn Send + Sync>>>,+}++pub struct CaptureMatcher {+ enabled: bool,+ inner: Mutex<CaptureMatcherInner>,+}++#[derive(Debug, Error)]+pub enum CaptureError<B: Backend> {+ #[error("Cannot start backend capture: {0}")]+ BackendCapture(#[source] B::Error),+ #[error("Attempted to start capture whose endpoint already happened")]+ StartAfterEnd,+}++impl CaptureMatcher {+ fn new() -> Self {+ let mut enabled = false;+ let mut starts = HashMap::new();+ let mut ends = HashMap::new();++ if let Some(capture) = env::var_os("UZU_CAPTURE").map(|s| s.into_string().unwrap()) {+ let spans = capture.split(',').filter(|s| !s.is_empty());+ let auto_paths = spans.clone().filter(|s| !s.contains('@')).count();+ let mut auto_index = 0;++ for span in spans {+ enabled = true;++ let (span, output_path) = match span.split_once('@') {+ Some((span, path)) => (span, PathBuf::from(path)),+ None if auto_paths == 1 => (span, PathBuf::from("/tmp/uzu.gputrace")),+ None => {+ let output_path = PathBuf::from(format!("/tmp/uzu-{auto_index}.gputrace"));+ auto_index += 1;+ (span, output_path)+ },+ };+ let (start, end) = span.split_once("..").unwrap_or((span, span));++ let (sender, receiver) = mpsc::channel();++ assert!(!starts.contains_key(start), "duplicate capture start: {start:?}");+ starts.insert(start.to_owned(), (sender, output_path));++ assert!(!ends.contains_key(end), "duplicate capture end: {end:?}");+ ends.insert(end.to_owned(), receiver);+ }+ }++ Self {+ enabled,+ inner: Mutex::new(CaptureMatcherInner {+ starts,+ ends,+ }),+ }+ }++ pub fn enabled(&self) -> bool {+ self.enabled+ }++ pub fn capture<B: Backend>(+ &self,+ context: &B::Context,+ span: &str,+ ) -> Result<Option<CaptureMatcherSpan>, CaptureError<B>> {+ if self.enabled {+ let mut inner = self.inner.lock();+ if let Some((start, output_path)) = inner.starts.remove(span) {+ start+ .send(Box::new(context.start_capture(&output_path).map_err(CaptureError::BackendCapture)?))+ .map_err(|_| CaptureError::StartAfterEnd)?;+ }+ if let Some(end) = inner.ends.remove(span) {+ return Ok(Some(CaptureMatcherSpan {+ _end: end,+ }));+ }+ }++ Ok(None)+ }+}++pub struct CaptureMatcherSpan {+ _end: Receiver<Box<dyn Send + Sync>>,+}+unsafe impl Sync for CaptureMatcherSpan {}diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rsindex 0377b6a70..0ed3d0034 100644--- a/crates/uzu-engine/src/backends/common/command_buffer.rs+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs@@ -80,6 +80,11 @@ pub trait CommandBufferEncoding {fn pop_debug_group(&mut self);+ fn add_debug_point(+ &mut self,+ name: &str,+ );+fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;}diff --git a/crates/uzu-engine/src/backends/common/context.rs b/crates/uzu-engine/src/backends/common/context.rsindex 0b07354ba..66196980d 100644--- a/crates/uzu-engine/src/backends/common/context.rs+++ b/crates/uzu-engine/src/backends/common/context.rs@@ -29,14 +29,10 @@ pub trait Context: Sized + Send + Sync {fn peak_memory_usage(&self) -> Option<usize>;- fn enable_capture();-fn start_capture(&self,- trace_path: &Path,- ) -> Result<(), <Self::Backend as Backend>::Error>;-- fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;+ output_path: &Path,+ ) -> Result<<Self::Backend as Backend>::CaptureSpan, <Self::Backend as Backend>::Error>;fn device_capabilities(&self) -> DeviceCapabilities;}diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rsindex a07c947d6..59a6e5e67 100644--- a/crates/uzu-engine/src/backends/common/mod.rs+++ b/crates/uzu-engine/src/backends/common/mod.rs@@ -1,6 +1,7 @@pub(super) mod allocator;mod backend;mod buffer;+pub mod capture_matcher;mod command_buffer;mod context;mod device_capabilities;diff --git a/crates/uzu-engine/src/backends/cpu/backend.rs b/crates/uzu-engine/src/backends/cpu/backend.rsindex 22ed5b72e..335858916 100644--- a/crates/uzu-engine/src/backends/cpu/backend.rs+++ b/crates/uzu-engine/src/backends/cpu/backend.rs@@ -1,3 +1,5 @@+use std::convert::Infallible;+use crate::backends::{common::{Backend,@@ -26,6 +28,7 @@ impl Backend for Cpu {type ScratchBuffer = PoolAllocation<Self::GlobalBuffer>;type SparseBuffer = CpuSparseBuffer;type AllocationPool = PoolAllocator<Self::GlobalBuffer>;+ type CaptureSpan = Infallible;type Kernels = CpuKernels;type Error = CpuError;diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rsindex 62ba458aa..04ca0e2bb 100644--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs@@ -118,6 +118,12 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {fn pop_debug_group(&mut self) {}+ fn add_debug_point(+ &mut self,+ _name: &str,+ ) {+ }+fn end_encoding(self) -> CpuCommandBufferExecutable {assert!(self.constant_allocator.is_done(), "attempted to end encoding while constants are still alive");CpuCommandBufferExecutable {diff --git a/crates/uzu-engine/src/backends/cpu/context.rs b/crates/uzu-engine/src/backends/cpu/context.rsindex 9f923ece7..fa6000162 100644--- a/crates/uzu-engine/src/backends/cpu/context.rs+++ b/crates/uzu-engine/src/backends/cpu/context.rs@@ -1,4 +1,5 @@use std::{+ convert::Infallible,path::Path,sync::{Arc, Weak, mpsc},thread,@@ -71,16 +72,10 @@ impl Context for CpuContext {None}- fn enable_capture() {}-fn start_capture(&self,- _trace_path: &Path,- ) -> Result<(), CpuError> {- Err(CpuError::NotSupported)- }-- fn stop_capture(&self) -> Result<(), CpuError> {+ _output_path: &Path,+ ) -> Result<Infallible, CpuError> {Err(CpuError::NotSupported)}diff --git a/crates/uzu-engine/src/backends/metal/backend.rs b/crates/uzu-engine/src/backends/metal/backend.rsindex fa2936be7..3986a0f47 100644--- a/crates/uzu-engine/src/backends/metal/backend.rs+++ b/crates/uzu-engine/src/backends/metal/backend.rs@@ -9,6 +9,7 @@ use crate::backends::{},metal::{buffer::{dense::MetalDenseBuffer, sparse::MetalSparseBuffer},+ capture::MetalCaptureSpan,command_buffer::MetalCommandBuffer,context::MetalContext,error::MetalError,@@ -27,6 +28,7 @@ impl Backend for Metal {type ScratchBuffer = PoolAllocation<Self::GlobalBuffer>;type SparseBuffer = MetalSparseBuffer;type AllocationPool = PoolAllocator<Self::GlobalBuffer>;+ type CaptureSpan = MetalCaptureSpan;type Kernels = MetalKernels;type Error = MetalError;diff --git a/crates/uzu-engine/src/backends/metal/buffer/sparse.rs b/crates/uzu-engine/src/backends/metal/buffer/sparse.rsindex 9e41a6cad..a60f8b9e9 100644--- a/crates/uzu-engine/src/backends/metal/buffer/sparse.rs+++ b/crates/uzu-engine/src/backends/metal/buffer/sparse.rs@@ -2,8 +2,8 @@ use std::sync::Arc;use metal::{MTL4CommandQueue, MTL4CommandQueueExt, MTL4UpdateSparseBufferMappingOperation, MTLBuffer, MTLDeviceExt,- MTLResidencySet, MTLResourceOptions, MTLSharedEventExt, MTLSharedEventListener, MTLSharedEventNotificationBlock,- MTLSparseTextureMappingMode,+ MTLEventExt, MTLResidencySet, MTLResourceOptions, MTLSharedEventExt, MTLSharedEventListener,+ MTLSharedEventNotificationBlock, MTLSparseTextureMappingMode,};use objc2::{rc::Retained, runtime::ProtocolObject};@@ -118,6 +118,7 @@ impl Drop for MetalSparseBuffer {);sparse_state.did_ops = true;let event = self.context.sparse_queue.device().new_shared_event().unwrap();+ event.set_label(Some("sparse unmap event"));let retain = (self.buffer.clone(), std::mem::take(&mut self.pages), event.clone());event.notify_listener_at_value(&MTLSharedEventListener::shared_listener(),diff --git a/crates/uzu-engine/src/backends/metal/capture.rs b/crates/uzu-engine/src/backends/metal/capture.rsnew file mode 100644index 000000000..2286b38f9--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/capture.rs@@ -0,0 +1,31 @@+use std::path::Path;++use metal::{MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCaptureTarget};++use crate::backends::metal::error::MetalError;++pub struct MetalCaptureSpan;++impl MetalCaptureSpan {+ pub fn new(+ target: &MTLCaptureTarget,+ output_path: &Path,+ ) -> Result<Self, MetalError> {+ let capture_descriptor = MTLCaptureDescriptor::new();+ capture_descriptor.set_capture_object(Some(target));+ capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);+ capture_descriptor.set_output_path(Some(output_path));++ MTLCaptureManager::shared_capture_manager()+ .start_capture_with_descriptor(&capture_descriptor)+ .map_err(|error| MetalError::CannotStartGpuCapture(error.to_string()))?;++ Ok(Self)+ }+}++impl Drop for MetalCaptureSpan {+ fn drop(&mut self) {+ MTLCaptureManager::shared_capture_manager().stop_capture();+ }+}diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rsindex 985a6d0e5..727de6a8d 100644--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs@@ -55,23 +55,19 @@ impl MetalCommandBufferEncoding {name: Option<&str>,allocation_pool: Option<Arc<<Metal as Backend>::AllocationPool>>,) -> Result<Self, MetalError> {- let (command_allocator, command_buffer) = if let mut command_buffer_cache = context.command_buffer_cache.lock()- && let Some(command_buffer_cached) = command_buffer_cache.pop()- {- command_buffer_cached.command_allocator.reset();- (command_buffer_cached.command_allocator, command_buffer_cached.command_buffer)+ let command_allocator = if let Some(command_allocator) = context.command_allocator_cache.lock().pop() {+ command_allocator.reset();+ command_allocator} else {- (- context.device.new_command_allocator().ok_or(MetalError::CannotCreateCommandBuffer)?,- context.device.new_mtl4_command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?,- )+ context.device.new_command_allocator().ok_or(MetalError::CannotCreateCommandBuffer)?};- command_buffer.set_label(name);-+ let command_buffer = context.device.new_mtl4_command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?;command_buffer.begin_command_buffer_with_allocator(&command_allocator);+ command_buffer.set_label(name);let compute_encoder = command_buffer.compute_command_encoder().unwrap();+ ProtocolObject::<dyn MTL4CommandEncoder>::set_label(compute_encoder.as_ref(), name);compute_encoder.barrier_after_queue_stages_before_stages_visibility_options(MTLStages::Dispatch | MTLStages::Blit | MTLStages::ResourceState,@@ -80,6 +76,7 @@ impl MetalCommandBufferEncoding {);let argument_table_descriptor = MTL4ArgumentTableDescriptor::new();+ argument_table_descriptor.set_label(name.map(|name| format!("{name} argument table")).as_deref());argument_table_descriptor.set_max_buffer_bind_count(31);let argument_table = context.device@@ -213,6 +210,13 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {self.compute_encoder.pop_debug_group();}+ fn add_debug_point(+ &mut self,+ name: &str,+ ) {+ ProtocolObject::<dyn MTL4CommandEncoder>::insert_debug_signpost(self.compute_encoder.as_ref(), name);+ }+fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {let constant_allocator = self.constant_allocator.take().unwrap();assert!(constant_allocator.is_done(), "attempted to end encoding while constants are still alive");@@ -276,11 +280,8 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {Ok(Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()))};let _ = sender.send(message);- context_clone.command_buffer_cache.lock().push(MetalCommandBufferCache {- command_allocator: command_allocator.clone(),- command_buffer: command_buffer.clone(),- });- let _keep_alive = (&constant_allocator, &allocation_pool);+ context_clone.command_allocator_cache.lock().push(command_allocator.clone());+ let _keep_alive = (&command_buffer, &constant_allocator, &allocation_pool);};let options = MTL4CommitOptions::new();@@ -326,8 +327,3 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {self.gpu_execution_time}}--pub struct MetalCommandBufferCache {- command_allocator: Retained<ProtocolObject<dyn MTL4CommandAllocator>>,- command_buffer: Retained<ProtocolObject<dyn MTL4CommandBuffer>>,-}diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rsindex 90ee23b35..01318dfab 100644--- a/crates/uzu-engine/src/backends/metal/context.rs+++ b/crates/uzu-engine/src/backends/metal/context.rs@@ -8,8 +8,8 @@ use std::{};use metal::{- MTL4CommandQueue, MTL4Compiler, MTL4CompilerDescriptor, MTLCaptureDescriptor, MTLCaptureDestination,- MTLCaptureManager, MTLCaptureTarget, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,+ MTL4CommandAllocator, MTL4CommandQueue, MTL4CommandQueueDescriptor, MTL4Compiler, MTL4CompilerDescriptor,+ MTLCaptureTarget, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLEventExt,MTLFunctionConstantValues, MTLGPUFamily, MTLLibrary, MTLResidencySet, MTLResidencySetDescriptor, MTLSparsePageSize,};use objc2::{rc::Retained, runtime::ProtocolObject};@@ -19,11 +19,13 @@ use crate::backends::{common::{Backend, Context, DeviceCapabilities,allocator::{block::BlockAllocator, pool::PoolAllocator},+ capture_matcher::CAPTURE_MATCHER,},metal::{Metal,buffer::{dense::MetalDenseBuffer, sparse::MetalSparseBuffer},- command_buffer::{MetalCommandBufferCache, MetalCommandBufferEncoding},+ capture::MetalCaptureSpan,+ command_buffer::MetalCommandBufferEncoding,decompression,error::MetalError,heaps::MetalHeaps,@@ -53,7 +55,7 @@ pub struct MetalContext {pub(super) block_allocator: Arc<BlockAllocator<MetalDenseBuffer>>,pub(super) heaps: Arc<MetalHeaps>,pub(super) peak_memory_usage: Arc<AtomicUsize>,- pub(super) command_buffer_cache: Mutex<Vec<MetalCommandBufferCache>>,+ pub(super) command_allocator_cache: Mutex<Vec<Retained<ProtocolObject<dyn MTL4CommandAllocator>>>>,library_cache: Mutex<HashMap<usize, Retained<ProtocolObject<dyn MTLLibrary>>>>,pipeline_cache: Mutex<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,weak_self: Weak<MetalContext>,@@ -101,7 +103,7 @@ impl MetalContext {}let library = self.library(library_data, library_compressed)?;- let pipeline = self.compiler.compute_pipeline_state(&library, function_name, constants)?;+ let pipeline = self.compiler.compute_pipeline_state(&library, function_name, constants, Some(cache_key))?;self.pipeline_cache.lock().insert(cache_key.to_string(), pipeline.clone());Ok(pipeline)@@ -112,7 +114,14 @@ impl Context for MetalContext {type Backend = Metal;fn new() -> Result<Arc<Self>, MetalError> {+ if CAPTURE_MATCHER.enabled() {+ unsafe {+ std::env::set_var("METAL_CAPTURE_ENABLED", "1");+ }+ }+let device = <dyn MTLDevice>::system_default().ok_or(MetalError::CannotOpenDevice)?;+let device_name = device.name();let gpu_core_count = device.gpu_core_count();let apple_gpu_family = device.newest_supported_apple_gpu_family();@@ -126,13 +135,21 @@ impl Context for MetalContext {.new_residency_set_with_descriptor(&residency_set_descriptor).map_err(|nserror| MetalError::CannotCreateResidencySet(nserror.to_string()))?;- let command_queue = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueue)?;+ let queue_descriptor = MTL4CommandQueueDescriptor::new();+ queue_descriptor.set_label(Some("compute queue"));+ let command_queue = device+ .new_mtl4_command_queue_with_descriptor(&queue_descriptor)+ .map_err(|_| MetalError::CannotCreateCommandQueue)?;command_queue.add_residency_set(&residency_set);- let sparse_queue = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueue)?;+ queue_descriptor.set_label(Some("sparse queue"));+ let sparse_queue = device+ .new_mtl4_command_queue_with_descriptor(&queue_descriptor)+ .map_err(|_| MetalError::CannotCreateCommandQueue)?;sparse_queue.add_residency_set(&residency_set);let sparse_event = device.new_event().ok_or(MetalError::CannotCreateEvent)?;+ sparse_event.set_label(Some("sparse event"));let sparse_state = SparseState {did_ops: false,@@ -162,7 +179,7 @@ impl Context for MetalContext {block_allocator,heaps,peak_memory_usage,- command_buffer_cache: Mutex::new(Vec::with_capacity(32)),+ command_allocator_cache: Mutex::new(Vec::with_capacity(32)),library_cache: Mutex::new(HashMap::new()),pipeline_cache: Mutex::new(HashMap::new()),weak_self: weak_self.clone(),@@ -203,32 +220,11 @@ impl Context for MetalContext {Some(self.peak_memory_usage.load(Ordering::Relaxed))}- fn enable_capture() {- unsafe {- std::env::set_var("METAL_CAPTURE_ENABLED", "1");- }- }-fn start_capture(&self,- trace_path: &Path,- ) -> Result<(), <Self::Backend as Backend>::Error> {- let capture_descriptor = MTLCaptureDescriptor::new();- capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);- capture_descriptor.set_output_path(Some(&trace_path.with_added_extension("gputrace")));- capture_descriptor.set_capture_object(Some(&MTLCaptureTarget::Device(self.device.clone())));-- MTLCaptureManager::shared_capture_manager()- .start_capture_with_descriptor(&capture_descriptor)- .map_err(|error| MetalError::CannotStartGpuCapture(error.to_string()))?;-- Ok(())- }-- fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error> {- MTLCaptureManager::shared_capture_manager().stop_capture();-- Ok(())+ output_path: &Path,+ ) -> Result<MetalCaptureSpan, MetalError> {+ MetalCaptureSpan::new(&MTLCaptureTarget::Device(self.device.clone()), output_path)}fn device_capabilities(&self) -> DeviceCapabilities {diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rsindex e51351964..ce3c505a8 100644--- a/crates/uzu-engine/src/backends/metal/error.rs+++ b/crates/uzu-engine/src/backends/metal/error.rs@@ -13,7 +13,7 @@ pub enum MetalError {CannotOpenDevice,#[error("Cannot create residency set: {0}")]CannotCreateResidencySet(String),- #[error("Cannot start gpu capture {0}")]+ #[error("Cannot start GPU capture: {0}")]CannotStartGpuCapture(String),#[error("Cannot create library: {0}")]CannotCreateLibrary(String),diff --git a/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rs b/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rsindex 653c31f41..15de231b5 100644--- a/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rs+++ b/crates/uzu-engine/src/backends/metal/metal_extensions/compiler_extensions_pipeline.rs@@ -15,6 +15,7 @@ pub trait CompilerPipelineExtensions {library: &ProtocolObject<dyn MTLLibrary>,function_name: &str,constants: Option<&MTLFunctionConstantValues>,+ label: Option<&str>,) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError>;}@@ -24,12 +25,14 @@ impl CompilerPipelineExtensions for ProtocolObject<dyn MTL4Compiler> {library: &ProtocolObject<dyn MTLLibrary>,function_name: &str,constants: Option<&MTLFunctionConstantValues>,+ label: Option<&str>,) -> Result<Retained<ProtocolObject<dyn MTLComputePipelineState>>, MetalError> {let library_function = MTL4LibraryFunctionDescriptor::new();library_function.set_library(Some(library));library_function.set_name(Some(function_name));let pipeline_descriptor = MTL4ComputePipelineDescriptor::new();+ pipeline_descriptor.set_label(label);match constants {Some(constants) => {let specialized_function = MTL4SpecializedFunctionDescriptor::new();diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rsindex 01324c58f..3b0320734 100644--- a/crates/uzu-engine/src/backends/metal/mod.rs+++ b/crates/uzu-engine/src/backends/metal/mod.rs@@ -1,5 +1,6 @@mod backend;mod buffer;+mod capture;mod command_buffer;mod context;mod decompression;diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rsindex c6c852b6a..7fc633724 100644--- a/crates/uzu-engine/src/encodable_block/convolution.rs+++ b/crates/uzu-engine/src/encodable_block/convolution.rs@@ -79,8 +79,6 @@ impl<B: Backend> SeparableCausalConv<B> {sequence_length: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("SeparableCausalConv");-let mut output =command_buffer.allocate_scratch_for_shape(&[sequence_length, self.model_dim], self.data_type)?;let coefficients_offset_bytes = size_for_shape(&[coefficient_column_offset], self.data_type);@@ -95,7 +93,6 @@ impl<B: Backend> SeparableCausalConv<B> {command_buffer,);- command_buffer.pop_debug_group();Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex fd218c54a..0cd58c0f7 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -157,8 +157,6 @@ impl<B: Backend> Embedding<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, EmbeddingError<B>> {- command_buffer.push_debug_group("embedding lookup");-let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type).map_err(EmbeddingError::BackendError)?;@@ -175,8 +173,6 @@ impl<B: Backend> Embedding<B> {};table.encode_lookup(token_ids, &mut output, batch_dim, self.input_scale, command_buffer);- command_buffer.pop_debug_group();-Ok(output)}diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rsindex 4f7f6a9ad..b1fcce5dd 100644--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs@@ -190,9 +190,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("matmul");-- let output = self.encode_with_a(+ self.encode_with_a(MatmulA::FullPrecision {values: &input,offset: 0,@@ -200,11 +198,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim,None::<Gather<&B::ScratchBuffer>>,command_buffer,- )?;-- command_buffer.pop_debug_group();-- Ok(output)+ )}fn encode_input(diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex c4dbbe8d8..d38dc43b8 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -189,16 +189,12 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("linear (rht)");-let input = match input {LinearInput::FullPrecision(input) => input,- input => {- let output = self.inner_linear.encode_input(input, batch_dim, command_buffer);- command_buffer.pop_debug_group();- return output;- },+ input => return self.inner_linear.encode_input(input, batch_dim, command_buffer),};+ command_buffer.push_debug_group("linear (rht)");+let format = self.inner_linear.select_activation_format(batch_dim, command_buffer.context());let input = self.input_rht.prepare_in_place(input, batch_dim, format, command_buffer)?;let output = self.inner_linear.encode_with_a(diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rsindex 979998d7a..02beb5503 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs@@ -3,7 +3,7 @@ use thiserror::Error;use crate::{backends::common::{- Backend, BufferMut, CommandBuffer, CommandBufferEncoding,+ Backend, BufferMut, CommandBuffer,kernel::{Kernels, QKVNormKernel},},config::normalization::{NormalizationConfig, UpcastMode},@@ -152,8 +152,6 @@ impl<B: Backend> QKVNorm<B> {"QKV norm input row stride ({input_row_stride}) is smaller than its packed row width ({packed_row_width})");- command_buffer.push_debug_group("qkv norm");-let kv = self.num_kv_heads;let heads = [(&self.query, 0, q_heads), (&self.key, q_heads, kv), (&self.value, q_heads + kv, kv)];for (head, head_offset, head_count) in heads {@@ -179,8 +177,6 @@ impl<B: Backend> QKVNorm<B> {);}- command_buffer.pop_debug_group();-Ok(())}}diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rsindex 4eab50d2f..9ffeb4f3d 100644--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs@@ -58,8 +58,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {batch_dim: u32,act_format: ActivationFormat,) -> Result<LinearInput<B>, B::Error> {- command_buffer.push_debug_group("gate act mul");-if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")}@@ -115,8 +113,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {LinearInput::FullPrecision(hidden)};- command_buffer.pop_debug_group();-Ok(input)}}diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rsindex 6fe74ae30..1a1ec09d7 100644--- a/crates/uzu-engine/src/encodable_block/normalization.rs+++ b/crates/uzu-engine/src/encodable_block/normalization.rs@@ -120,8 +120,6 @@ impl<B: Backend> Normalization<B> {shortcut: Option<impl BufferMut<Backend = B>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("normalization");-let row_size = size_for_shape(&[self.element_count], self.data_type);let row_offset_bytes = row_offset as usize * row_size;let shortcut = shortcut.map(|shortcut| shortcut.subrange_mut(row_offset_bytes..));@@ -141,8 +139,6 @@ impl<B: Backend> Normalization<B> {command_buffer,);- command_buffer.pop_debug_group();-Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rsindex c73b991b6..2bb7b8bea 100644--- a/crates/uzu-engine/src/encodable_block/transformer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer.rs@@ -74,6 +74,8 @@ impl<B: Backend> TransformerState<B> {command_buffer.pop_debug_group();+ command_buffer.add_debug_point(&format!("accepted {} tokens", accepted_indices.len()));+Ok(())}}diff --git a/crates/uzu-engine/src/engine/capture.rs b/crates/uzu-engine/src/engine/capture.rsdeleted file mode 100644index 93713fea3..000000000--- a/crates/uzu-engine/src/engine/capture.rs+++ /dev/null@@ -1,98 +0,0 @@-use std::{- path::PathBuf,- sync::{- Arc, LazyLock,- atomic::{AtomicBool, Ordering},- },-};--use crate::backends::common::{Backend, Context};--static CAPTURE_FIRST_PREFILL: LazyLock<AtomicBool> =- LazyLock::new(|| AtomicBool::new(std::env::var("UZU_CAPTURE_FIRST_PREFILL").is_ok()));-static CAPTURE_FIRST_DECODE: LazyLock<AtomicBool> =- LazyLock::new(|| AtomicBool::new(std::env::var("UZU_CAPTURE_FIRST_DECODE").is_ok()));--pub struct CaptureRequest<B: Backend> {- context: Arc<B::Context>,- path: PathBuf,-}--impl<B: Backend> CaptureRequest<B> {- fn new(- context: Arc<B::Context>,- path: PathBuf,- ) -> Self {- Self {- context,- path,- }- }-- pub fn start(self) -> Result<CaptureSpan<B>, B::Error> {- self.context.start_capture(&self.path)?;- eprintln!("Started capturing to {:?}", self.path);-- Ok(CaptureSpan {- context: self.context,- path: self.path,- })- }-}--pub struct CaptureSpan<B: Backend> {- context: Arc<B::Context>,- path: PathBuf,-}--impl<B: Backend> Drop for CaptureSpan<B> {- fn drop(&mut self) {- match self.context.stop_capture() {- Ok(()) => {- eprintln!("Successfully saved capture to {:?}", self.path);- },- Err(err) => {- eprintln!("Error when saving capture to {:?}: {:?}", self.path, err);- },- }- }-}--pub struct CaptureManager<B: Backend> {- context: Arc<B::Context>,-}--impl<B: Backend> CaptureManager<B> {- pub fn pre_load_enable() -> bool {- let capture_enabled =- CAPTURE_FIRST_PREFILL.load(Ordering::Relaxed) || CAPTURE_FIRST_DECODE.load(Ordering::Relaxed);-- if capture_enabled {- <B::Context as Context>::enable_capture();- }-- capture_enabled- }-- pub fn new(context: Arc<B::Context>) -> Self {- Self {- context,- }- }-- fn capture_path(suffix: &str) -> PathBuf {- PathBuf::from("/tmp").join(format!("uzu-capture-{}-{}", B::NAME, suffix))- }-- pub fn maybe_capture_prefill_step(&self) -> Option<CaptureRequest<B>> {- CAPTURE_FIRST_PREFILL- .swap(false, Ordering::Relaxed)- .then(|| CaptureRequest::new(self.context.clone(), Self::capture_path("prefill")))- }-- pub fn maybe_capture_decode_step(&self) -> Option<CaptureRequest<B>> {- CAPTURE_FIRST_DECODE- .swap(false, Ordering::Relaxed)- .then(|| CaptureRequest::new(self.context.clone(), Self::capture_path("decode")))- }-}diff --git a/crates/uzu-engine/src/engine/language_model/stream/mod.rs b/crates/uzu-engine/src/engine/language_model/stream/mod.rsindex fc456ba8d..cd35827ba 100644--- a/crates/uzu-engine/src/engine/language_model/stream/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/mod.rs@@ -4,7 +4,7 @@ use thiserror::Error;#[cfg(grammar)]use crate::engine::language_model::grammar::{Grammar, GrammarError};use crate::{- backends::common::Backend,+ backends::common::{Backend, capture_matcher::CaptureError},encodable_block::decoder::DecoderError,engine::language_model::{LanguageModel, state::LanguageModelState},speculators::dflash_tfm::DFlashTreeError,@@ -25,6 +25,8 @@ pub struct LanguageModelStreamOptions {pub enum LanguageModelStreamError<B: Backend> {#[error("Backend error: {0}")]Backend(#[source] B::Error),+ #[error("Capture error: {0}")]+ Capture(#[from] CaptureError<B>),#[error("Decoder error: {0}")]Decoder(#[from] DecoderError<B>),#[cfg(grammar)]diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 7663cf362..cdd1be257 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -13,17 +13,17 @@ use crate::{array::size_for_shape,backends::common::{Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,- CommandBufferPending, Context, gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,+ CommandBufferPending, Context,+ capture_matcher::{CAPTURE_MATCHER, CaptureMatcherSpan},+ gpu_types::trie::TrieNode as GpuTrieNode,+ kernel::ContextRingUpdateKernel,},data_type::DataType,encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},- engine::{- capture::CaptureSpan,- language_model::{- LanguageModel,- state::LanguageModelState,- stream::{LanguageModelStreamError, LanguageModelStreamOptions},- },+ engine::language_model::{+ LanguageModel,+ state::LanguageModelState,+ stream::{LanguageModelStreamError, LanguageModelStreamOptions},},trie::TrieNode,};@@ -92,7 +92,7 @@ struct DecodingStatePending<B: Backend> {input_trie: TrieNode,full_accept: bool,pending: Box<[<B::CommandBuffer as CommandBuffer>::Pending]>,- capture_span: Option<CaptureSpan<B>>,+ capture_span: Option<CaptureMatcherSpan>,hidden_features: Option<Box<[B::ScratchBuffer]>>,output_norm: Option<B::ScratchBuffer>,output_tokens: Arc<B::GlobalBuffer>,@@ -109,7 +109,7 @@ enum DecodingState<B: Backend> {num_accepted: usize,hidden_features: Option<Box<[B::ScratchBuffer]>>,output_norm: Option<(B::ScratchBuffer, usize)>,- capture_span: Option<CaptureSpan<B>>,+ capture_span: Option<CaptureMatcherSpan>,},Halted,Invalid,@@ -158,14 +158,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {return Err(LanguageModelStreamError::ContextOverflow);}- let capture_span = if let Some(capture_manager) = &model.engine.capture_manager- && let Some(capture_request) = capture_manager.maybe_capture_prefill_step()- {- Some(capture_request.start().map_err(LanguageModelStreamError::Backend)?)- } else {- None- };-let allocation_pool = model.engine.context.create_allocation_pool();let mut context_ring = if let Some(suffix_repetition_length) =@@ -198,6 +190,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let mut metrics = TokenStreamMetrics::default();let decoding_state = if !input.is_empty() {+ let _capture_span = CAPTURE_MATCHER.capture::<B>(&model.engine.context, "prefill")?;model_state.last_output_token.take();// NOTE: this is required for attention correctness (hardcoded suffix 1024). This is really bad design, attention should be rewritten to allow on-demand suffix length@@ -218,6 +211,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.create_command_buffer(Some("prefill"), Some(allocation_pool.clone())).map_err(LanguageModelStreamError::Backend)?;+ command_buffer.add_debug_point(&format!(+ "prefill at context {}, suffix {} ({} batches)",+ model_state.transformer_state.context_length(),+ input.len(),+ number_of_batches+ ));+let mut output_tokens = None;let mut output_norm = None;let split_logits_row = model.decoder.prefill_cache_skips_trailing_layers();@@ -349,7 +349,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {input_trie: TrieNode::new(0, 0, 0.0),full_accept: true,pending,- capture_span,+ capture_span: None,hidden_features: None,output_norm,output_tokens: Arc::new(output_tokens.unwrap()),@@ -374,9 +374,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {}fn generate(&mut self) -> Result<Option<u64>, LanguageModelStreamError<B>> {- let (mut prev_output, mut command_buffer): (+ let (mut prev_output, acceptance): (ForwardPassChaining<B>,- Option<<B::CommandBuffer as CommandBuffer>::Encoding>,+ Option<(<B::CommandBuffer as CommandBuffer>::Encoding, Option<CaptureMatcherSpan>)>,) = match replace(&mut self.decoding_state, DecodingState::Invalid) {DecodingState::Seeded {seed_token,@@ -458,7 +458,10 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.model.engine.context- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))+ .create_command_buffer(+ Some(&format!("decode {} (accept)", self.metrics.num_decode_forward_passes - 1)),+ Some(self.allocation_pool.clone()),+ ).map_err(LanguageModelStreamError::Backend)?;self.model_state.transformer_state@@ -485,7 +488,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {None};if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {- command_buffer.push_debug_group("update repetition penalty ring");let accepted_input_token_ids_const = command_buffer.allocate_constant_from_slice(&accepted_input_token_ids@@ -501,23 +503,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {full.len() as u32,&mut command_buffer,);- command_buffer.pop_debug_group();- }- if let Some(capture_span) = capture_span {- command_buffer- .end_encoding()- .submit()- .wait_until_completed()- .map_err(LanguageModelStreamError::Backend)?;-- drop(capture_span);-- command_buffer = self- .model- .engine- .context- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))- .map_err(LanguageModelStreamError::Backend)?;}self.model_state.tokens.extend(accepted_output_token_ids);(@@ -525,7 +510,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {token: output_token_id,output_norm,},- Some(command_buffer),+ Some((command_buffer, capture_span)),)}},@@ -548,20 +533,14 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {));}- let capture_span = if let Some(capture_manager) = &self.model.engine.capture_manager- && let Some(capture_request) = capture_manager.maybe_capture_decode_step()- {- prev_output.resolve(- &mut self.model_state.tokens,- #[cfg(grammar)]- self.options.grammar.as_mut(),- )?;- Some(capture_request.start().map_err(LanguageModelStreamError::Backend)?)- } else {- None- };-let mut pending = Vec::new();+ if let Some((accept_command_buffer, _capture_span)) = acceptance {+ pending.push(accept_command_buffer.end_encoding().submit());+ }++ let mut capture_span = CAPTURE_MATCHER+ .capture::<B>(&self.model.engine.context, &format!("decode:{}", self.metrics.num_decode_forward_passes))?;+let (input_trie, chain_copy, full_accept) = if let Some(speculator) = &self.model.speculator&& let Some(shape) = speculator.make_shape(self.model_state.max_context_length.map(|max_context_length| max_context_length - context_length),@@ -572,9 +551,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {#[cfg(grammar)]self.options.grammar.as_mut(),)? {- if let Some(accept_command_buffer) = command_buffer.take() {- pending.push(accept_command_buffer.end_encoding().submit());- }let trie = speculator.propose_tree(self.model_state.speculator_state.as_mut().unwrap(),output_norm,@@ -585,6 +561,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.options.grammar.as_mut(),&self.model_state.prng,self.allocation_pool.clone(),+ Some(&format!("decode {} (speculate)", self.metrics.num_decode_forward_passes)),)?;prev_output.drop_output_norm();(trie, None, false)@@ -601,19 +578,25 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {};let input_flat_trie = input_trie.linearize();- if let Some(accept_command_buffer) = command_buffer.take() {- pending.push(accept_command_buffer.end_encoding().submit());- }-self.allocation_pool = self.model.engine.context.create_allocation_pool();let mut command_buffer = self.model.engine.context- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))+ .create_command_buffer(+ Some(&format!("decode {} (forward)", self.metrics.num_decode_forward_passes)),+ Some(self.allocation_pool.clone()),+ ).map_err(LanguageModelStreamError::Backend)?;+ command_buffer.add_debug_point(&format!(+ "decode round {} at context {} suffix {}",+ self.metrics.num_decode_forward_passes,+ context_length,+ input_flat_trie.len()+ ));+let scratch_token_ids;let mut constant_token_ids = None;let token_ids: &dyn Buffer<Backend = B> = if let Some(chain_copy) = chain_copy.as_deref() {@@ -663,7 +646,10 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {.model.engine.context- .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))+ .create_command_buffer(+ Some(&format!("decode {} (sample and accept)", self.metrics.num_decode_forward_passes)),+ Some(self.allocation_pool.clone()),+ ).map_err(LanguageModelStreamError::Backend)?;let mut bitmask = command_buffer@@ -761,6 +747,10 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {drop(constant_token_ids);pending.push(command_buffer.end_encoding().submit());+ if full_accept {+ capture_span.take();+ }+self.metrics.num_decode_forward_passes += 1;self.metrics.num_tokens_proposed += input_flat_trie.len();if full_accept {@@ -822,7 +812,10 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {.model.engine.context- .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))+ .create_command_buffer(+ Some(&format!("decode {} (accept on drop)", self.metrics.num_decode_forward_passes - 1)),+ Some(self.allocation_pool.clone()),+ ).unwrap();self.model_state.transformer_state.encode_accept(&[0], &mut command_buffer).unwrap();if let Some(speculator) = self.model.speculator.as_ref() {@@ -853,7 +846,10 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {.model.engine.context- .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))+ .create_command_buffer(+ Some(&format!("decode {} (accept on drop)", self.metrics.num_decode_forward_passes - 1)),+ Some(self.allocation_pool.clone()),+ ).unwrap();let accepted_token_indicies =full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();diff --git a/crates/uzu-engine/src/engine/mod.rs b/crates/uzu-engine/src/engine/mod.rsindex 8f1f941b6..c62f5bca7 100644--- a/crates/uzu-engine/src/engine/mod.rs+++ b/crates/uzu-engine/src/engine/mod.rs@@ -4,39 +4,38 @@ use derive_more::Debug;use thiserror::Error;use crate::{- backends::common::{Backend, Context},+ backends::common::{+ Backend, Context,+ capture_matcher::{CAPTURE_MATCHER, CaptureError, CaptureMatcherSpan},+ },config::model::AnyModelConfig,encodable_block::sampling::SamplingMethod,- engine::capture::CaptureManager,};pub mod classifier_model;pub mod language_model;-mod capture;-pub struct Engine<B: Backend> {+ _capture_span: Option<CaptureMatcherSpan>,context: Arc<B::Context>,- capture_manager: Option<CaptureManager<B>>,}#[derive(Debug, Error)]pub enum EngineNewError<B: Backend> {#[error("Backend error: {0}")]Backend(#[source] B::Error),+ #[error("Capture error: {0}")]+ Capture(#[from] CaptureError<B>),}impl<B: Backend> Engine<B> {pub fn new() -> Result<Arc<Self>, EngineNewError<B>> {- let capture_enabled = CaptureManager::<B>::pre_load_enable();-let context = <B::Context as Context>::new().map_err(EngineNewError::Backend)?;-- let capture_manager = capture_enabled.then(|| CaptureManager::new(context.clone()));+ let capture_span = CAPTURE_MATCHER.capture::<B>(&context, "")?;Ok(Arc::new(Self {+ _capture_span: capture_span,context,- capture_manager,}))}diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex 6c41724b5..7a37d05e3 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -217,6 +217,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {#[cfg(grammar)] grammar: Option<&mut Grammar>,prng: &PRng,allocation_pool: Arc<B::AllocationPool>,+ command_buffer_name: Option<&str>,) -> Result<TrieNode, DFlashTreeError<B>> {assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");@@ -232,7 +233,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let mut command_buffer = self.context- .create_command_buffer(Some("speculator propose"), Some(allocation_pool))+ .create_command_buffer(command_buffer_name, Some(allocation_pool)).map_err(DFlashTreeError::Backend)?;let nodes = match shape.construction_method {diff --git a/crates/uzu-engine/unit/BENCHMARKS.md b/crates/uzu-engine/unit/BENCHMARKS.mdindex 06662faca..f539bfbe3 100644--- a/crates/uzu-engine/unit/BENCHMARKS.md+++ b/crates/uzu-engine/unit/BENCHMARKS.md@@ -58,9 +58,9 @@ CRITERION_HOME="$PWD/target/criterion/m2_max" cargo bench \--save-baseline matmul_baseline_m2_max```-Set `UZU_CAPTURE_BENCH=1` to capture the first matching benchmark command buffer as a Metal `.gputrace`.-`UZU_CAPTURE_BENCH_FILTER` is an optional benchmark path substring; `UZU_CAPTURE_BENCH_DIR` defaults to the current-directory.+Set `UZU_CAPTURE='<benchmark path>'` to capture the first matching batch; named helpers use exact Criterion paths.+Append `@path.gputrace` to choose the output (default `/tmp/uzu.gputrace` for one capture).+Streams also expose `prefill` and zero-based `decode:N` per stream; ranges use `start..end`, e.g. `prefill..decode:2`.## Running on iPhone (via `cargo-dinghy`)diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rsindex 3a542dbed..e21376da3 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs@@ -99,12 +99,11 @@ fn bench_build_tree_out_type<T: ArrayElement + Float>(for &batch_size in BATCH_SIZES {for &tree_size in TREE_SIZES {let mut buffers = make_buffers::<T>(context, batch_size, tree_size);- let benchmark_path = format!(- "Metal/Kernel/GDNTreeVerify/BuildTreeOut/{data_type_name}/{path}/{tile_name}/B{batch_size}_T{tree_size}"- );- let benchmark_id = BenchmarkId::from_parameter(format!(- "B{batch_size}_T{tree_size}_QK{QK_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}_V{HEAD_V_DIM}"- ));+ let parameter =+ format!("B{batch_size}_T{tree_size}_QK{QK_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}_V{HEAD_V_DIM}");+ let benchmark_path =+ format!("Metal/Kernel/GDNTreeVerify/BuildTreeOut/{data_type_name}/{path}/{tile_name}/{parameter}");+ let benchmark_id = BenchmarkId::from_parameter(parameter);let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(context,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rsindex 99fc7e66c..577d8d1f7 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs@@ -168,11 +168,9 @@ fn bench_build_tree_gram(c: &mut Criterion) {).expect("BuildTreeGramKernel");let scale = (HEAD_K_DIM as f32).sqrt().recip();- let benchmark_path =- format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}/B{batch_size}_T{tree_size}");- let benchmark_id = BenchmarkId::from_parameter(format!(- "B{batch_size}_T{tree_size}_Hg{K_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}"- ));+ let parameter = format!("B{batch_size}_T{tree_size}_Hg{K_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}");+ let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}/{parameter}");+ let benchmark_id = BenchmarkId::from_parameter(parameter);let mut buffers = ColdPool::new(buffers_bytes(batch_size, tree_size), || {make_buffers(&context, batch_size, tree_size).0diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rsindex ac165afe3..9b3c14c8f 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs@@ -187,10 +187,9 @@ fn bench_tree_update_solve(c: &mut Criterion) {group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));for &(batch_size, tree_size) in BENCH_SHAPES {- let benchmark_path =- format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}/B{batch_size}_T{tree_size}");- let benchmark_id =- BenchmarkId::from_parameter(format!("B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}"));+ let parameter = format!("B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}");+ let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}/{parameter}");+ let benchmark_id = BenchmarkId::from_parameter(parameter);let mut buffers =ColdPool::new(buffers_bytes(batch_size, tree_size), || make_buffers(&context, batch_size, tree_size));diff --git a/crates/uzu-engine/unit/common/env_vars.rs b/crates/uzu-engine/unit/common/env_vars.rsdeleted file mode 100644index 15d5912d8..000000000--- a/crates/uzu-engine/unit/common/env_vars.rs+++ /dev/null@@ -1,7 +0,0 @@-pub const UZU_CAPTURE_BENCH: &str = "UZU_CAPTURE_BENCH";-pub const UZU_CAPTURE_BENCH_DIR: &str = "UZU_CAPTURE_BENCH_DIR";-pub const UZU_CAPTURE_BENCH_FILTER: &str = "UZU_CAPTURE_BENCH_FILTER";--pub fn enabled(name: &str) -> bool {- std::env::var(name).is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("yes") || v.eq_ignore_ascii_case("true"))-}diff --git a/crates/uzu-engine/unit/common/harness.rs b/crates/uzu-engine/unit/common/harness.rsindex 1aa1de161..30a9dcc1a 100644--- a/crates/uzu-engine/unit/common/harness.rs+++ b/crates/uzu-engine/unit/common/harness.rs@@ -1,8 +1,5 @@use libtest_mimic::{Arguments, Trial};-const METAL_CAPTURE_ENABLED: &str = "METAL_CAPTURE_ENABLED";-const UZU_CAPTURE_BENCH: &str = "UZU_CAPTURE_BENCH";-pub enum UzuTest {Bench(&'static dyn Fn()),Test(&'static UzuTestCase),@@ -30,7 +27,6 @@ pub fn uzu_harness(tests: &[&UzuTest]) {if benchmarks {#[cfg(target_os = "ios")]ios_set_current_dir();- enable_benchmark_gpu_capture_if_requested();let bench_tests: Vec<&dyn Fn()> = tests.iter().filter_map(|test| match test {@@ -56,15 +52,3 @@ pub fn uzu_harness(tests: &[&UzuTest]) {libtest_mimic::run(&Arguments::from_args(), trials).exit();}}--fn enable_benchmark_gpu_capture_if_requested() {- if enabled(UZU_CAPTURE_BENCH) {- unsafe {- std::env::set_var(METAL_CAPTURE_ENABLED, "1");- }- }-}--fn enabled(name: &str) -> bool {- std::env::var(name).is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("yes") || v.eq_ignore_ascii_case("true"))-}diff --git a/crates/uzu-engine/unit/common/matmul/bench.rs b/crates/uzu-engine/unit/common/matmul/bench.rsindex daca0ee2a..5e538e948 100644--- a/crates/uzu-engine/unit/common/matmul/bench.rs+++ b/crates/uzu-engine/unit/common/matmul/bench.rs@@ -1,57 +1,18 @@-use std::{- env,- path::PathBuf,- sync::atomic::{AtomicBool, Ordering},- time::{SystemTime, UNIX_EPOCH},-};-use criterion::Bencher;-use super::super::env_vars;use crate::backends::common::{Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,- CommandBufferPending, Context,+ CommandBufferPending, Context, capture_matcher::CAPTURE_MATCHER,};-static CAPTURE_TAKEN: AtomicBool = AtomicBool::new(false);--fn should_capture_benchmark(benchmark_path: &str) -> bool {- env_vars::enabled(env_vars::UZU_CAPTURE_BENCH)- && benchmark_path.starts_with("Metal/")- && env::var(env_vars::UZU_CAPTURE_BENCH_FILTER).map_or(true, |filter| benchmark_path.contains(&filter))- && !CAPTURE_TAKEN.swap(true, Ordering::AcqRel)-}--fn benchmark_capture_path() -> PathBuf {- let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).expect("system clock before Unix epoch").as_secs();- env::var(env_vars::UZU_CAPTURE_BENCH_DIR)- .map(PathBuf::from)- .unwrap_or(env::current_dir().unwrap())- .join(format!("uzu_bench-{timestamp}.gputrace"))-}--fn start_benchmark_capture<B: Backend>(- context: &B::Context,- benchmark_path: &str,-) -> bool {- if !should_capture_benchmark(benchmark_path) {- return false;- }-- let path = benchmark_capture_path();- context.start_capture(&path).expect("failed to start benchmark GPU capture");- println!("GPU benchmark capture started for {benchmark_path}: {path:?}");- true-}-pub fn iter_encode_loop<B: Backend, F>(context: &B::Context,bencher: &mut Bencher,- mut encode: F,+ encode: F,) whereF: FnMut(&mut <B::CommandBuffer as CommandBuffer>::Encoding),{- iter_encode_loop_named::<B, _>(context, bencher, "unnamed_benchmark", |command_buffer| encode(command_buffer));+ iter_encode_loop_named::<B, _>(context, bencher, "unnamed_benchmark", encode);}pub fn iter_encode_loop_named<B: Backend, F>(@@ -63,15 +24,14 @@ pub fn iter_encode_loop_named<B: Backend, F>(F: FnMut(&mut <B::CommandBuffer as CommandBuffer>::Encoding),{bencher.iter_custom(|n_iters| {- let capture = start_benchmark_capture::<B>(context, benchmark_path);- let mut command_buffer = context.create_command_buffer(None, None).unwrap();+ let _capture_span = CAPTURE_MATCHER+ .capture::<B>(context, benchmark_path)+ .unwrap_or_else(|error| panic!("failed to capture benchmark {benchmark_path}: {error}"));+ let mut command_buffer = context.create_command_buffer(Some(benchmark_path), None).unwrap();for _ in 0..n_iters {encode(&mut command_buffer);}let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- if capture {- context.stop_capture().expect("failed to stop benchmark GPU capture");- }completed.gpu_execution_time()});}diff --git a/crates/uzu-engine/unit/common/mod.rs b/crates/uzu-engine/unit/common/mod.rsindex dfa417228..2448a1500 100644--- a/crates/uzu-engine/unit/common/mod.rs+++ b/crates/uzu-engine/unit/common/mod.rs@@ -3,7 +3,6 @@ compile_error!("uzu tests require cpu backend");pub mod assert;pub mod cold_pool;-pub mod env_vars;pub mod harness;pub mod helpers;pub mod matmul;