A high-performance inference engine for AI models
Revert "gpu timestamps recording" (#912)
uuuvn4e7004b6ebdd
diff --git a/Cargo.lock b/Cargo.lockindex acf8f6327..eee205fe4 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3689,9 +3689,9 @@ dependencies = [[[package]]name = "mtl-rs"-version = "0.3.0"+version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4aa9291872296e9e734b0caf6e7a933fe6861d25ed86240461d6114355f7dbbf"+checksum = "d9f1de2bca596db925db758d497cbf9cb394a2981933319b3777510097e1d360"dependencies = ["bitflags 2.13.0","block2",diff --git a/Cargo.toml b/Cargo.tomlindex 420503b9d..759972957 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -76,7 +76,7 @@ objc2-foundation = "0.3.2"objc2-core-foundation = "0.3.2"objc2-io-kit = { version = "0.3.2", default-features = false, features = ["libc", "ps"] }obfstr = "0.4.4"-metal = { package = "mtl-rs", version = "0.3.0" }+metal = { package = "mtl-rs", version = "0.2.2" }block2 = "0.6"bytemuck = "1.23"bytesize = "2"diff --git a/apps/mirai-gui/desktop/Cargo.lock b/apps/mirai-gui/desktop/Cargo.lockindex a6849f8a7..a2481a32f 100644--- a/apps/mirai-gui/desktop/Cargo.lock+++ b/apps/mirai-gui/desktop/Cargo.lock@@ -3565,9 +3565,9 @@ dependencies = [[[package]]name = "mtl-rs"-version = "0.3.0"+version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4aa9291872296e9e734b0caf6e7a933fe6861d25ed86240461d6114355f7dbbf"+checksum = "d9f1de2bca596db925db758d497cbf9cb394a2981933319b3777510097e1d360"dependencies = ["bitflags 2.13.0","block2",diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rsindex 4d8168aa6..0377b6a70 100644--- a/crates/uzu-engine/src/backends/common/command_buffer.rs+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs@@ -4,7 +4,7 @@ use bytemuck::{AnyBitPattern, NoUninit};use crate::{array::size_for_shape,- backends::common::{Backend, BufferMut, BufferRef, TimestampSpan},+ backends::common::{Backend, BufferMut, BufferRef},data_type::DataType,};@@ -80,15 +80,6 @@ pub trait CommandBufferEncoding {fn pop_debug_group(&mut self);- fn enable_timestamps(&mut self);-- fn sample_start_timestamp(- &mut self,- name: &str,- );-- fn sample_end_timestamp(&mut self);-fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;}@@ -113,6 +104,4 @@ pub trait CommandBufferCompleted: Send {type CommandBuffer: CommandBuffer<Completed = Self>;fn gpu_execution_time(&self) -> Duration;-- fn timestamps(&self) -> &[TimestampSpan];}diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rsindex 47663a079..a07c947d6 100644--- a/crates/uzu-engine/src/backends/common/mod.rs+++ b/crates/uzu-engine/src/backends/common/mod.rs@@ -6,7 +6,6 @@ mod context;mod device_capabilities;pub mod gpu_types;pub mod kernel;-mod timestamps;pub use backend::Backend;pub use buffer::{@@ -24,4 +23,3 @@ pub use command_buffer::{pub use context::Context;pub use device_capabilities::DeviceCapabilities;pub use kernel::Kernels;-pub use timestamps::{TimestampSpan, TimestampSpanRecorder};diff --git a/crates/uzu-engine/src/backends/common/timestamps/mod.rs b/crates/uzu-engine/src/backends/common/timestamps/mod.rsdeleted file mode 100644index 4d2eca4b4..000000000--- a/crates/uzu-engine/src/backends/common/timestamps/mod.rs+++ /dev/null@@ -1,6 +0,0 @@-mod span_boundary;-mod timestamp_span;-mod timestamp_span_recorder;--pub use timestamp_span::TimestampSpan;-pub use timestamp_span_recorder::TimestampSpanRecorder;diff --git a/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs b/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rsdeleted file mode 100644index a26a37153..000000000--- a/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs+++ /dev/null@@ -1,4 +0,0 @@-pub enum SpanBoundary {- Start(String),- End,-}diff --git a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rsdeleted file mode 100644index f4315085f..000000000--- a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs+++ /dev/null@@ -1,8 +0,0 @@-use std::time::Instant;--#[derive(Debug, Clone)]-pub struct TimestampSpan {- pub name: String,- pub start: Instant,- pub end: Instant,-}diff --git a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rsdeleted file mode 100644index b1c6ce04f..000000000--- a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs+++ /dev/null@@ -1,59 +0,0 @@-use std::time::Instant;--use super::{TimestampSpan, span_boundary::SpanBoundary};--#[derive(Default)]-pub struct TimestampSpanRecorder {- boundaries: Vec<SpanBoundary>,-}--impl TimestampSpanRecorder {- pub fn start(- &mut self,- name: String,- ) -> usize {- self.push(SpanBoundary::Start(name))- }-- pub fn end(&mut self) -> usize {- self.push(SpanBoundary::End)- }-- fn push(- &mut self,- boundary: SpanBoundary,- ) -> usize {- self.boundaries.push(boundary);- self.boundaries.len() - 1- }-- pub fn slot_count(&self) -> usize {- self.boundaries.len()- }-- pub fn spans(- &self,- instants: &[Instant],- ) -> Box<[TimestampSpan]> {- let mut spans = Vec::new();- let mut open_span_indices = Vec::new();- for (boundary, &instant) in self.boundaries.iter().zip(instants) {- match boundary {- SpanBoundary::Start(name) => {- open_span_indices.push(spans.len());- spans.push(TimestampSpan {- name: name.clone(),- start: instant,- end: instant,- });- },- SpanBoundary::End => {- let index = open_span_indices.pop().expect("timestamp ended without a start");- spans[index].end = instant;- },- }- }- assert!(open_span_indices.is_empty(), "timestamps were started but never ended");- spans.into_boxed_slice()- }-}diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rsindex 78722176e..62ba458aa 100644--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs@@ -7,14 +7,12 @@ use crate::{backends::{common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,- CommandBufferExecutable, CommandBufferPending, TimestampSpan, TimestampSpanRecorder,- allocator::bump::BumpAllocator,+ CommandBufferExecutable, CommandBufferPending, allocator::bump::BumpAllocator,},cpu::{Cpu,buffer::{CpuBufferExt, dense::CpuBuffer},context::CpuContext,- cpu_command::CpuCommand,error::CpuError,},},@@ -33,11 +31,10 @@ impl CommandBuffer for CpuCommandBuffer {}pub struct CpuCommandBufferEncoding {- commands: Vec<CpuCommand>,+ commands: Vec<Box<dyn FnOnce() + Send>>,constant_allocator: BumpAllocator<<Cpu as Backend>::GlobalBuffer>,allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,context: Arc<CpuContext>,- timestamp_spans: Option<TimestampSpanRecorder>,}impl CpuCommandBufferEncoding {@@ -51,7 +48,6 @@ impl CpuCommandBufferEncoding {constant_allocator,allocation_pool,context,- timestamp_spans: None,}}@@ -59,11 +55,7 @@ impl CpuCommandBufferEncoding {&mut self,command: impl FnOnce() + Send + 'static,) {- self.commands.push(CpuCommand::Run(Box::new(command)))- }-- fn write_timestamp(&mut self) {- self.commands.push(CpuCommand::Timestamp);+ self.commands.push(Box::new(command))}}@@ -126,28 +118,6 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {fn pop_debug_group(&mut self) {}- fn enable_timestamps(&mut self) {- assert!(self.timestamp_spans.is_none(), "timestamps already enabled");- self.timestamp_spans = Some(TimestampSpanRecorder::default());- }-- fn sample_start_timestamp(- &mut self,- name: &str,- ) {- if let Some(spans) = &mut self.timestamp_spans {- spans.start(name.to_owned());- self.write_timestamp();- }- }-- fn sample_end_timestamp(&mut self) {- if let Some(spans) = &mut self.timestamp_spans {- spans.end();- self.write_timestamp();- }- }-fn end_encoding(self) -> CpuCommandBufferExecutable {assert!(self.constant_allocator.is_done(), "attempted to end encoding while constants are still alive");CpuCommandBufferExecutable {@@ -155,17 +125,15 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {constant_allocator: self.constant_allocator,allocation_pool: self.allocation_pool,context: self.context,- timestamp_spans: self.timestamp_spans,}}}pub struct CpuCommandBufferExecutable {- commands: Vec<CpuCommand>,+ commands: Vec<Box<dyn FnOnce() + Send>>,constant_allocator: BumpAllocator<<Cpu as Backend>::GlobalBuffer>,allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,context: Arc<CpuContext>,- timestamp_spans: Option<TimestampSpanRecorder>,}impl CommandBufferExecutable for CpuCommandBufferExecutable {@@ -180,23 +148,14 @@ impl CommandBufferExecutable for CpuCommandBufferExecutable {.send(Box::new(move || {let start = Instant::now();- let instants = self- .commands- .into_iter()- .filter_map(|command| match command {- CpuCommand::Run(run) => {- run();- None- },- CpuCommand::Timestamp => Some(Instant::now()),- })- .collect::<Box<[Instant]>>();+ for command in self.commands {+ command()+ }let gpu_execution_time = start.elapsed();let completed = CpuCommandBufferCompleted {gpu_execution_time,- timestamps: self.timestamp_spans.map_or_else(Box::default, |spans| spans.spans(&instants)),_allocation_pool: self.allocation_pool,};@@ -227,7 +186,6 @@ impl CommandBufferPending for CpuCommandBufferPending {pub struct CpuCommandBufferCompleted {gpu_execution_time: Duration,- timestamps: Box<[TimestampSpan]>,_allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,}@@ -237,8 +195,4 @@ impl CommandBufferCompleted for CpuCommandBufferCompleted {fn gpu_execution_time(&self) -> Duration {self.gpu_execution_time}-- fn timestamps(&self) -> &[TimestampSpan] {- &self.timestamps- }}diff --git a/crates/uzu-engine/src/backends/cpu/cpu_command.rs b/crates/uzu-engine/src/backends/cpu/cpu_command.rsdeleted file mode 100644index 6dd851e4c..000000000--- a/crates/uzu-engine/src/backends/cpu/cpu_command.rs+++ /dev/null@@ -1,4 +0,0 @@-pub enum CpuCommand {- Run(Box<dyn FnOnce() + Send>),- Timestamp,-}diff --git a/crates/uzu-engine/src/backends/cpu/mod.rs b/crates/uzu-engine/src/backends/cpu/mod.rsindex 7d6afffc5..f569a2da4 100644--- a/crates/uzu-engine/src/backends/cpu/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/mod.rs@@ -2,7 +2,6 @@ mod backend;mod buffer;mod command_buffer;mod context;-mod cpu_command;mod error;pub(crate) mod kernel; // TODO: This should not be pub!!!diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rsindex a584b4cb0..985a6d0e5 100644--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs@@ -16,11 +16,9 @@ use rangemap::RangeSet;use crate::backends::{common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,- CommandBufferExecutable, CommandBufferPending, Context, TimestampSpan, allocator::bump::BumpAllocator,- },- metal::{- Metal, MetalContext, buffer::MetalBufferExt, error::MetalError, timestamp_recorder::MetalTimestampRecorder,+ CommandBufferExecutable, CommandBufferPending, Context, allocator::bump::BumpAllocator,},+ metal::{Metal, MetalContext, buffer::MetalBufferExt, error::MetalError},};pub struct MetalCommandBuffer;@@ -49,7 +47,6 @@ pub struct MetalCommandBufferEncoding {constant_allocator: Option<BumpAllocator<<Metal as Backend>::GlobalBuffer>>,allocation_pool: Arc<<Metal as Backend>::AllocationPool>,pub(super) context: Arc<MetalContext>,- timestamps: Option<MetalTimestampRecorder>,}impl MetalCommandBufferEncoding {@@ -104,7 +101,6 @@ impl MetalCommandBufferEncoding {constant_allocator: Some(constant_allocator),allocation_pool,context,- timestamps: None,})}@@ -217,26 +213,6 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {self.compute_encoder.pop_debug_group();}- fn enable_timestamps(&mut self) {- assert!(self.timestamps.is_none(), "timestamps already enabled");- self.timestamps = Some(MetalTimestampRecorder::new(self.context.device.clone()));- }-- fn sample_start_timestamp(- &mut self,- name: &str,- ) {- if let Some(timestamps) = &mut self.timestamps {- timestamps.start(name.to_owned(), &self.compute_encoder);- }- }-- fn sample_end_timestamp(&mut self) {- if let Some(timestamps) = &mut self.timestamps {- timestamps.end(&self.compute_encoder);- }- }-fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {let constant_allocator = self.constant_allocator.take().unwrap();assert!(constant_allocator.is_done(), "attempted to end encoding while constants are still alive");@@ -247,7 +223,6 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {constant_allocator,allocation_pool: self.allocation_pool.clone(),context: self.context.clone(),- timestamps: self.timestamps.take(),}}}@@ -270,7 +245,6 @@ pub struct MetalCommandBufferExecutable {constant_allocator: BumpAllocator<<Metal as Backend>::GlobalBuffer>,allocation_pool: Arc<<Metal as Backend>::AllocationPool>,context: Arc<MetalContext>,- timestamps: Option<MetalTimestampRecorder>,}impl CommandBufferExecutable for MetalCommandBufferExecutable {@@ -292,18 +266,14 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {let command_allocator = self.command_allocator.clone();let command_buffer = self.command_buffer.clone();let context_clone = self.context.clone();- let timestamps = self.timestamps;let constant_allocator = self.constant_allocator;let allocation_pool = self.allocation_pool.clone();let feedback_handler = move |feedback: &ProtocolObject<dyn MTL4CommitFeedback>| {let message = if let Some(error) = feedback.error() {- Err(MetalError::CommandBufferExecution(error.to_string()))+ Err(error.to_string())} else {- timestamps- .as_ref()- .map_or_else(|| Ok(Box::default()), MetalTimestampRecorder::resolve)- .map(|spans| (Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()), spans))+ Ok(Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()))};let _ = sender.send(message);context_clone.command_buffer_cache.lock().push(MetalCommandBufferCache {@@ -326,17 +296,19 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {pub struct MetalCommandBufferPending {allocation_pool: Arc<<Metal as Backend>::AllocationPool>,- receiver: mpsc::Receiver<Result<(Duration, Box<[TimestampSpan]>), MetalError>>,+ receiver: mpsc::Receiver<Result<Duration, String>>,}impl CommandBufferPending for MetalCommandBufferPending {type CommandBuffer = MetalCommandBuffer;fn wait_until_completed(self) -> Result<MetalCommandBufferCompleted, MetalError> {- let (gpu_execution_time, timestamps) = self.receiver.recv().map_err(MetalError::CommandBufferWait)??;Ok(MetalCommandBufferCompleted {- gpu_execution_time,- timestamps,+ gpu_execution_time: self+ .receiver+ .recv()+ .map_err(MetalError::CommandBufferWait)?+ .map_err(MetalError::CommandBufferExecution)?,_allocation_pool: self.allocation_pool,})}@@ -344,7 +316,6 @@ impl CommandBufferPending for MetalCommandBufferPending {pub struct MetalCommandBufferCompleted {gpu_execution_time: Duration,- timestamps: Box<[TimestampSpan]>,_allocation_pool: Arc<<Metal as Backend>::AllocationPool>,}@@ -354,10 +325,6 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {fn gpu_execution_time(&self) -> Duration {self.gpu_execution_time}-- fn timestamps(&self) -> &[TimestampSpan] {- &self.timestamps- }}pub struct MetalCommandBufferCache {diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rsindex 83b5033e6..e51351964 100644--- a/crates/uzu-engine/src/backends/metal/error.rs+++ b/crates/uzu-engine/src/backends/metal/error.rs@@ -27,10 +27,6 @@ pub enum MetalError {CannotCreateBuffer,#[error("Cannot create heap")]CannotCreateHeap,- #[error("Cannot resolve counter heap")]- CannotResolveCounterHeap,- #[error("Timestamp {0} was not written by the GPU")]- UnwrittenTimestamp(usize),#[error("Cannot create command buffer")]CannotCreateCommandBuffer,#[error("Cannot create argument table: {0}")]diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rsindex 5fc8da6ee..01324c58f 100644--- a/crates/uzu-engine/src/backends/metal/mod.rs+++ b/crates/uzu-engine/src/backends/metal/mod.rs@@ -7,7 +7,6 @@ mod error;mod heaps;mod kernel;mod metal_extensions;-mod timestamp_recorder;pub use backend::Metal;pub use context::MetalContext; // TODO: This should be removeddiff --git a/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rsdeleted file mode 100644index 919245b51..000000000--- a/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs+++ /dev/null@@ -1,102 +0,0 @@-use std::time::{Duration, Instant};--use metal::{- MTL4ComputeCommandEncoder, MTL4CounterHeap, MTL4CounterHeapDescriptor, MTL4CounterHeapExt, MTL4CounterHeapType,- MTL4TimestampGranularity, MTLDevice, MTLDeviceExt,-};-use objc2::{rc::Retained, runtime::ProtocolObject};--use crate::backends::{- common::{TimestampSpan, TimestampSpanRecorder},- metal::error::MetalError,-};--// "Maximum counter sample buffer length" is 32 KB, i.e. 4096 timestamp entries of 8 bytes (`MTL4TimestampHeapEntry`):-// https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf-const COUNTER_HEAP_CAPACITY: usize = 4096;--pub struct MetalTimestampRecorder {- device: Retained<ProtocolObject<dyn MTLDevice>>,- spans: TimestampSpanRecorder,- counter_heaps: Vec<Retained<ProtocolObject<dyn MTL4CounterHeap>>>,-}--impl MetalTimestampRecorder {- pub fn new(device: Retained<ProtocolObject<dyn MTLDevice>>) -> Self {- Self {- device,- spans: TimestampSpanRecorder::default(),- counter_heaps: Vec::new(),- }- }-- pub fn start(- &mut self,- name: String,- encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,- ) {- let slot = self.spans.start(name);- self.write(slot, encoder);- }-- pub fn end(- &mut self,- encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,- ) {- let slot = self.spans.end();- self.write(slot, encoder);- }-- fn write(- &mut self,- slot: usize,- encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,- ) {- if slot / COUNTER_HEAP_CAPACITY == self.counter_heaps.len() {- let descriptor = MTL4CounterHeapDescriptor::new();- descriptor.set_type(MTL4CounterHeapType::TIMESTAMP);- descriptor.set_count(COUNTER_HEAP_CAPACITY);- let counter_heap =- self.device.new_counter_heap_with_descriptor(&descriptor).expect("Failed to create a counter heap");- counter_heap.invalidate_counter_range(0..COUNTER_HEAP_CAPACITY);- self.counter_heaps.push(counter_heap);- }- encoder.write_timestamp_with_granularity_into_heap_at_index(- MTL4TimestampGranularity::PRECISE,- &self.counter_heaps[slot / COUNTER_HEAP_CAPACITY],- slot % COUNTER_HEAP_CAPACITY,- );- }-- pub fn resolve(&self) -> Result<Box<[TimestampSpan]>, MetalError> {- let count = self.spans.slot_count();- let mut ticks = Vec::with_capacity(count);- for (index, counter_heap) in self.counter_heaps.iter().enumerate() {- let slots = 0..(count - index * COUNTER_HEAP_CAPACITY).min(COUNTER_HEAP_CAPACITY);- let bytes = counter_heap.resolve_counter_range(slots).ok_or(MetalError::CannotResolveCounterHeap)?;- let (entries, _) = bytes.as_chunks::<{ size_of::<u64>() }>();- ticks.extend(entries.iter().map(|entry| u64::from_ne_bytes(*entry)));- }- let frequency = self.device.query_timestamp_frequency() as u128;- let (mut cpu_now, mut gpu_now) = (0, 0);- let now = Instant::now();- self.device.sample_timestamps_gpu_timestamp(&mut cpu_now, &mut gpu_now);- let to_instant = |ticks: u64| {- let nanoseconds = (ticks as u128 * 1_000_000_000 / frequency) as u64;- now - Duration::from_nanos(gpu_now - nanoseconds)- };- let instants = ticks- .iter()- .enumerate()- .scan(None, |previous_timestamp, (slot, &slot_ticks)| {- let timestamp = match slot_ticks {- 0 => previous_timestamp.ok_or(MetalError::UnwrittenTimestamp(slot)),- slot_ticks => Ok(to_instant(slot_ticks)),- };- *previous_timestamp = timestamp.as_ref().ok().copied();- Some(timestamp)- })- .collect::<Result<Box<[Instant]>, MetalError>>()?;- Ok(self.spans.spans(&instants))- }-}diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rsindex d343d583e..f44077a95 100644--- a/crates/uzu-engine/src/encodable_block/classifier.rs+++ b/crates/uzu-engine/src/encodable_block/classifier.rs@@ -35,7 +35,6 @@ pub enum ClassifierError<B: Backend> {}pub struct Classifier<B: Backend> {- name: String,hidden_dim: u32,data_type: DataType,embedding: Embedding<B>,@@ -47,14 +46,12 @@ pub struct Classifier<B: Backend> {impl<B: Backend> Classifier<B> {pub fn new(- name: String,context: &B::Context,config: &ClassifierConfig,parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Self, ClassifierError<B>> {let (embedding, _) = Embedding::new(- format!("{name}/embedding"),context,config.vocab_size,config.transformer_config.model_dim,@@ -64,7 +61,6 @@ impl<B: Backend> Classifier<B> {)?;let embedding_norm = Normalization::new(- format!("{name}/embedding norm"),config.transformer_config.model_dim,None,ShortcutMode::None,@@ -76,7 +72,6 @@ impl<B: Backend> Classifier<B> {)?;let transformer = Transformer::new(- format!("{name}/transformer"),context,None,data_type,@@ -94,7 +89,6 @@ impl<B: Backend> Classifier<B> {<B::Kernels as Kernels>::PoolingMeanKernel::new(context, data_type).map_err(ClassifierError::Backend)?;let prediction_head = PredictionHead::new(- format!("{name}/prediction head"),config.hidden_dim,config.num_labels,data_type,@@ -104,7 +98,6 @@ impl<B: Backend> Classifier<B> {)?;Ok(Self {- name,hidden_dim: config.hidden_dim,data_type,embedding,@@ -125,8 +118,7 @@ impl<B: Backend> Classifier<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, ClassifierError<B>> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("classifier");let embedded = self.embedding.encode_lookup(token_ids, batch_dim, command_buffer)?;@@ -164,7 +156,6 @@ impl<B: Backend> Classifier<B> {let logits = self.prediction_head.encode(pooled, 1, command_buffer).map_err(ClassifierError::Backend)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(logits)diff --git a/crates/uzu-engine/src/encodable_block/context_ring_update.rs b/crates/uzu-engine/src/encodable_block/context_ring_update.rsdeleted file mode 100644index 1573b04b7..000000000--- a/crates/uzu-engine/src/encodable_block/context_ring_update.rs+++ /dev/null@@ -1,35 +0,0 @@-use crate::backends::common::{- Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, kernel::ContextRingUpdateKernel,-};--pub struct ContextRingUpdate<B: Backend> {- name: String,- kernel: <B::Kernels as Kernels>::ContextRingUpdateKernel,-}--impl<B: Backend> ContextRingUpdate<B> {- pub fn new(- name: String,- context: &B::Context,- ) -> Result<Self, B::Error> {- Ok(Self {- name,- kernel: <B::Kernels as Kernels>::ContextRingUpdateKernel::new(context)?,- })- }-- pub fn encode(- &self,- input: impl BufferRef<Backend = B>,- context_ring: impl BufferMut<Backend = B>,- suffix_repetition_length: u32,- input_length: u32,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);- self.kernel.encode(input, context_ring, suffix_repetition_length, input_length, command_buffer);- command_buffer.sample_end_timestamp();- command_buffer.pop_debug_group();- }-}diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rsindex 1f9afb10e..c6c852b6a 100644--- a/crates/uzu-engine/src/encodable_block/convolution.rs+++ b/crates/uzu-engine/src/encodable_block/convolution.rs@@ -21,7 +21,6 @@ pub enum ConvolutionNewError<B: Backend> {}pub struct SeparableCausalConv<B: Backend> {- name: String,model_dim: u32,data_type: DataType,weights: B::GlobalBuffer,@@ -31,7 +30,6 @@ pub struct SeparableCausalConv<B: Backend> {impl<B: Backend> SeparableCausalConv<B> {pub fn new(- name: String,model_dim: u32,kernel_size: u32,group_size: u32,@@ -64,7 +62,6 @@ impl<B: Backend> SeparableCausalConv<B> {.map_err(ConvolutionNewError::Backend)?;Ok(Self {- name,model_dim,data_type,weights,@@ -82,8 +79,7 @@ impl<B: Backend> SeparableCausalConv<B> {sequence_length: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("SeparableCausalConv");let mut output =command_buffer.allocate_scratch_for_shape(&[sequence_length, self.model_dim], self.data_type)?;@@ -99,7 +95,6 @@ impl<B: Backend> SeparableCausalConv<B> {command_buffer,);- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)}diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rsindex 5476b386a..7b1c8bbfd 100644--- a/crates/uzu-engine/src/encodable_block/decoder.rs+++ b/crates/uzu-engine/src/encodable_block/decoder.rs@@ -32,7 +32,6 @@ pub enum DecoderError<B: Backend> {}pub struct Decoder<B: Backend> {- name: String,embedding: Embedding<B>,embedding_norm: Option<Normalization<B>>,per_layer_embedding: Option<PerLayerEmbedding<B>>,@@ -51,14 +50,12 @@ impl<B: Backend> Decoder<B> {}pub fn new(- name: String,context: &B::Context,config: &DecoderConfig,parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Self, DecoderError<B>> {let (embedding, readout_input_hadamard_factors) = Embedding::new(- format!("{name}/embedding"),context,config.vocab_size,config.transformer_config.model_dim,@@ -72,7 +69,6 @@ impl<B: Backend> Decoder<B> {.as_ref().map(|norm_config| {Normalization::new(- format!("{name}/embedding norm"),config.transformer_config.model_dim,None,ShortcutMode::None,@@ -92,7 +88,6 @@ impl<B: Backend> Decoder<B> {"per-layer embedding num_layers must match transformer layer count");Some(PerLayerEmbedding::new(- format!("{name}/per layer embedding"),context,ple_config,config.transformer_config.model_dim,@@ -104,7 +99,6 @@ impl<B: Backend> Decoder<B> {};let transformer = Transformer::new(- format!("{name}/transformer"),context,readout_input_hadamard_factors,data_type,@@ -113,7 +107,6 @@ impl<B: Backend> Decoder<B> {)?;Ok(Self {- name,embedding,embedding_norm,per_layer_embedding,@@ -150,8 +143,7 @@ impl<B: Backend> Decoder<B> {state: &mut TransformerState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("decoder");let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), command_buffer)?;let embedded = if let Some(embedding_norm) = &self.embedding_norm {@@ -204,7 +196,6 @@ impl<B: Backend> Decoder<B> {transformer_output.output};- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(DecoderEncodeOutput {diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rsindex 4c90073fa..b35383899 100644--- a/crates/uzu-engine/src/encodable_block/dflash.rs+++ b/crates/uzu-engine/src/encodable_block/dflash.rs@@ -41,8 +41,6 @@ impl<B: Backend> DFlashState<B> {}pub struct DFlash<B: Backend> {- accept_name: String,- draft_name: String,target_feature_projection: Box<dyn Linear<B>>,projected_feature_norm: Normalization<B>,state_kv_projection: Box<dyn Linear<B>>,@@ -91,7 +89,6 @@ pub enum DFlashEncodeError<B: Backend> {impl<B: Backend> DFlash<B> {pub fn new(- name: String,context: &B::Context,config: &DFlashDraftConfig,parameter_tree: &ParameterTree<B>,@@ -100,7 +97,6 @@ impl<B: Backend> DFlash<B> {let mask_token_id = config.mask_token_id as u32;assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");let target_feature_projection = <dyn Linear<B>>::new(- format!("{name}/target feature projection"),config.model_dim * config.target_layer_ids.len() as u32,[config.model_dim],false,@@ -109,7 +105,6 @@ impl<B: Backend> DFlash<B> {¶meter_tree.subtree("context_projection"),)?;let projected_feature_norm = Normalization::new(- format!("{name}/projected feature norm"),config.model_dim,None,ShortcutMode::None,@@ -127,7 +122,6 @@ impl<B: Backend> DFlash<B> {let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;let num_layers = config.layer_configs.len() as u32;let state_kv_projection = <dyn Linear<B>>::new(- format!("{name}/state kv projection"),config.model_dim,[num_layers * layer_kv_dim],false,@@ -141,7 +135,6 @@ impl<B: Backend> DFlash<B> {.zip(0u32..).map(|(layer_config, index)| {TransformerLayer::new(- format!("{name}/layer {index}"),context,config.model_dim,config.hidden_dim,@@ -154,7 +147,6 @@ impl<B: Backend> DFlash<B> {}).collect::<Result<Box<[_]>, TransformerLayerError<B>>>()?;let output_norm = Normalization::new(- format!("{name}/output norm"),config.model_dim,None,ShortcutMode::Add,@@ -166,8 +158,6 @@ impl<B: Backend> DFlash<B> {)?;Ok(Self {- accept_name: format!("{name}/accept"),- draft_name: format!("{name}/draft"),target_feature_projection,projected_feature_norm,state_kv_projection,@@ -217,8 +207,7 @@ impl<B: Backend> DFlash<B> {return Ok(());}- command_buffer.push_debug_group(&self.accept_name);- command_buffer.sample_start_timestamp(&self.accept_name);+ command_buffer.push_debug_group("dflash accept");let num_tokens = accepted_indices.len() as u32;let captured_layer_count = self.target_feature_input_dim / self.model_dim;@@ -285,7 +274,6 @@ impl<B: Backend> DFlash<B> {state.context_length += num_tokens;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(())@@ -299,8 +287,7 @@ impl<B: Backend> DFlash<B> {batch_size: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {- command_buffer.push_debug_group(&self.draft_name);- command_buffer.sample_start_timestamp(&self.draft_name);+ command_buffer.push_debug_group("dflash draft");assert!(batch_size >= 2 && batch_size <= self.block_size, "batch size exceeds DFlash block size");assert!(@@ -361,7 +348,6 @@ impl<B: Backend> DFlash<B> {command_buffer,)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(DFlashOutput {diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex 8f386db84..fd218c54a 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -45,7 +45,6 @@ enum EmbeddingTying<B: Backend> {}pub struct Embedding<B: Backend> {- readout_name: String,tying: EmbeddingTying<B>,input_scale: f32,data_type: DataType,@@ -70,7 +69,6 @@ impl<B: Backend> Embedding<B> {}pub fn new(- name: String,context: &B::Context,vocab_size: u32,model_dim: u32,@@ -81,14 +79,7 @@ impl<B: Backend> Embedding<B> {let (tying, readout_input_hadamard_factors) = match config {AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {let embedding_tree = parameter_tree.subtree("embedding");- let table = EmbeddingTable::load(- format!("{name}/lookup"),- context,- &embedding_tree,- vocab_size,- model_dim,- data_type,- )?;+ let table = EmbeddingTable::load(context, &embedding_tree, vocab_size, model_dim, data_type)?;if table.as_matrix().is_none() {return Err(EmbeddingError::UnsupportedConfiguration("tied embeddings need a matrix table".into()));}@@ -109,18 +100,11 @@ impl<B: Backend> Embedding<B> {},AnyEmbeddingConfig::UntiedEmbeddingConfig(_) => {let input_embedding_tree = parameter_tree.subtree("input_embedding");- let input_table = EmbeddingTable::load(- format!("{name}/lookup"),- context,- &input_embedding_tree,- vocab_size,- model_dim,- data_type,- )?;+ let input_table =+ EmbeddingTable::load(context, &input_embedding_tree, vocab_size, model_dim, data_type)?;let output_embedding_tree = parameter_tree.subtree("output_embedding");let output_embedding_spec = output_embedding_tree.metadata("spec")?;let output = UntiedReadout::load(- format!("{name}/readout/untied"),context,&output_embedding_tree,output_embedding_spec,@@ -156,7 +140,6 @@ impl<B: Backend> Embedding<B> {Ok((Self {- readout_name: format!("{name}/readout"),tying,input_scale,data_type,@@ -174,6 +157,8 @@ impl<B: Backend> Embedding<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, EmbeddingError<B>> {+ command_buffer.push_debug_group("embedding lookup");+let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type).map_err(EmbeddingError::BackendError)?;@@ -190,6 +175,8 @@ impl<B: Backend> Embedding<B> {};table.encode_lookup(token_ids, &mut output, batch_dim, self.input_scale, command_buffer);+ command_buffer.pop_debug_group();+Ok(output)}@@ -202,8 +189,7 @@ impl<B: Backend> Embedding<B> {apply_logit_transform: bool,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, EmbeddingError<B>> {- command_buffer.push_debug_group(&self.readout_name);- command_buffer.sample_start_timestamp(&self.readout_name);+ command_buffer.push_debug_group("embedding readout");assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");let mut output_buffer = match &self.tying {@@ -254,7 +240,6 @@ impl<B: Backend> Embedding<B> {);}- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output_buffer)diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rsindex dee8a1974..7dae6806a 100644--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs@@ -3,7 +3,7 @@ use thiserror::Error;use crate::{backends::common::{- Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,+ Backend, BufferMut, BufferRef, CommandBuffer, Kernels,gpu_types::{EmbeddingTableKind, HADAMARD_TRANSFORM_BLOCK_SIZE, d4s4},kernel::InputEmbeddingLookupKernel,},@@ -81,7 +81,6 @@ impl<B: Backend> Storage<B> {}pub struct EmbeddingTable<B: Backend> {- name: String,storage: Storage<B>,output_hadamard_factors: Option<B::GlobalBuffer>,lookup: LookupKernel<B>,@@ -91,7 +90,6 @@ pub struct EmbeddingTable<B: Backend> {impl<B: Backend> EmbeddingTable<B> {pub fn load(- name: String,context: &B::Context,tree: &ParameterTree<B>,vocab_size: u32,@@ -152,7 +150,6 @@ impl<B: Backend> EmbeddingTable<B> {.map_err(EmbeddingTableError::BackendError)?;Ok(Self {- name,storage,output_hadamard_factors,lookup,@@ -178,8 +175,6 @@ impl<B: Backend> EmbeddingTable<B> {scale: f32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);let bindings = self.storage.lookup_bindings();self.lookup.encode(token_ids,@@ -198,8 +193,6 @@ impl<B: Backend> EmbeddingTable<B> {scale,command_buffer,);- command_buffer.sample_end_timestamp();- command_buffer.pop_debug_group();}}diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rsindex 4db7675fe..2c82b4d93 100644--- a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs@@ -10,7 +10,6 @@ use crate::{};pub(super) struct InputRht<B: Backend> {- name: String,rht_signs: B::GlobalBuffer,rht: ActivationTransform<B>,quantizer: Option<ActivationTransform<B>>,@@ -18,7 +17,6 @@ pub(super) struct InputRht<B: Backend> {impl<B: Backend> InputRht<B> {pub(super) fn new(- name: String,context: &B::Context,data_type: DataType,preparation: LinearInputPreparation<B>,@@ -34,7 +32,6 @@ impl<B: Backend> InputRht<B> {.transpose()?;Ok(Self {- name,rht_signs,rht,quantizer,@@ -48,7 +45,6 @@ impl<B: Backend> InputRht<B> {format: ActivationFormat,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<LinearInput<B>, B::Error> {- command_buffer.sample_start_timestamp(&self.name);if format == ActivationFormat::Int8&& let Some(quantizer) = &self.quantizer{@@ -74,7 +70,6 @@ impl<B: Backend> InputRht<B> {command_buffer,);- command_buffer.sample_end_timestamp();return Ok(LinearInput::Int8Symmetric {values,scales,@@ -87,7 +82,6 @@ impl<B: Backend> InputRht<B> {let input_dim = self.input_dim();let mut transformed = command_buffer.allocate_scratch(input.size())?;self.rht.encode_fp(input, &mut transformed, &self.rht_signs, batch_dim, input_dim, command_buffer);- command_buffer.sample_end_timestamp();Ok(LinearInput::FullPrecision(transformed))}@@ -102,10 +96,8 @@ impl<B: Backend> InputRht<B> {return self.prepare(&input, batch_dim, format, command_buffer);}- command_buffer.sample_start_timestamp(&self.name);let input_dim = self.input_dim();self.rht.encode_fp_in_place(&mut input, &self.rht_signs, None, batch_dim, input_dim, command_buffer);- command_buffer.sample_end_timestamp();Ok(LinearInput::FullPrecision(input))}diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rsindex da0ff8c77..4f7f6a9ad 100644--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs@@ -37,7 +37,6 @@ pub enum LinearMatmulError<B: Backend> {}pub struct LinearMatmul<B: Backend> {- name: String,kernel: Mutex<<B::Kernels as Kernels>::MatmulKernel>,matrix: WeightMatrix<B>,biases: Option<B::GlobalBuffer>,@@ -67,7 +66,6 @@ impl<B: Backend> LinearMatmul<B> {/// Loads a linear over any parsed spec — full precision, MLX or Int. Hybrid/// compositions live in the wrappers, not here.pub fn load(- name: String,context: &B::Context,spec: AnyWeightMatrixSpec,input_dim: u32,@@ -100,7 +98,6 @@ impl<B: Backend> LinearMatmul<B> {.map_err(LinearMatmulError::BackendError)?;Ok(Self {- name,kernel: Mutex::new(kernel),matrix,biases,@@ -128,7 +125,6 @@ impl<B: Backend> LinearMatmul<B> {gather: Option<Gather<impl BufferRef<Backend = B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.name);let (output_dim, gather_indices) =gather.map_or((self.output_dim, None), |gather| (gather.output_dim, Some(gather.indices)));let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, output_dim], self.output_data_type)?;@@ -147,7 +143,6 @@ impl<B: Backend> LinearMatmul<B> {command_buffer,)?;- command_buffer.sample_end_timestamp();Ok(output)}@@ -195,7 +190,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);+ command_buffer.push_debug_group("matmul");let output = self.encode_with_a(MatmulA::FullPrecision {diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rsindex 9c49aae44..78dfdfb93 100644--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs@@ -123,7 +123,6 @@ pub enum LinearBlockError<B: Backend> {impl<B: Backend> dyn Linear<B> {pub fn new_mixed_precision(- name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -141,7 +140,6 @@ impl<B: Backend> dyn Linear<B> {| AnyWeightMatrixSpec::MLXSpec(_)| AnyWeightMatrixSpec::IntSpec(_)) => {let block = LinearMatmul::load(- name,context,spec,input_dimension,@@ -161,7 +159,6 @@ impl<B: Backend> dyn Linear<B> {incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,..}) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => Ok(Box::new(RHTLinearWrapper::new(- name,context,input_dimension,output_dimension_sum,@@ -184,7 +181,6 @@ impl<B: Backend> dyn Linear<B> {return Err(LinearBlockError::UnsupportedConfiguration(format!("{adapter_spec:?}")));};Ok(Box::new(QLoRALinearWrapper::new(- name,context,*quantization_spec,adapter_spec,@@ -203,7 +199,6 @@ impl<B: Backend> dyn Linear<B> {}pub fn new(- name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -212,7 +207,6 @@ impl<B: Backend> dyn Linear<B> {parameter_tree: &ParameterTree<B>,) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {Self::new_mixed_precision(- name,input_dimension,output_dimensions,has_biases,@@ -225,7 +219,6 @@ impl<B: Backend> dyn Linear<B> {}pub fn new_with_input_rht_mixed_precision(- name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -237,7 +230,6 @@ impl<B: Backend> dyn Linear<B> {) -> Result<(Box<dyn Linear<B>>, Option<B::GlobalBuffer>), LinearBlockError<B>> {let output_dimension_sum: u32 = output_dimensions.as_ref().iter().sum();if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(- name.clone(),context,input_dimension,output_dimension_sum,@@ -252,7 +244,6 @@ impl<B: Backend> dyn Linear<B> {}let linear = Self::new_mixed_precision(- name,input_dimension,output_dimensions,has_biases,@@ -266,7 +257,6 @@ impl<B: Backend> dyn Linear<B> {}pub fn new_for_fused_input(- name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -276,7 +266,6 @@ impl<B: Backend> dyn Linear<B> {) -> Result<(Box<dyn Linear<B>>, Option<LinearInputPreparation<B>>), LinearBlockError<B>> {let output_dimension_sum: u32 = output_dimensions.as_ref().iter().sum();if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(- name.clone(),context,input_dimension,output_dimension_sum,@@ -291,7 +280,6 @@ impl<B: Backend> dyn Linear<B> {}let linear = Self::new_mixed_precision(- name,input_dimension,output_dimensions,has_biases,@@ -305,7 +293,6 @@ impl<B: Backend> dyn Linear<B> {}pub fn new_with_input_rht(- name: String,input_dimension: u32,output_dimensions: impl AsRef<[u32]>,has_biases: bool,@@ -314,7 +301,6 @@ impl<B: Backend> dyn Linear<B> {parameter_tree: &ParameterTree<B>,) -> Result<(Box<dyn Linear<B>>, Option<B::GlobalBuffer>), LinearBlockError<B>> {Self::new_with_input_rht_mixed_precision(- name,input_dimension,output_dimensions,has_biases,diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rsindex dfdf74851..d4faa24b9 100644--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs@@ -35,7 +35,6 @@ pub enum QLoRALinearWrapperError<B: Backend> {}pub struct QLoRALinearWrapper<B: Backend> {- name: String,base_linear: LinearMatmul<B>,input_hadamard: Option<(ActivationTransform<B>, B::GlobalBuffer)>,output_hadamard: Option<(ActivationTransform<B>, B::GlobalBuffer)>,@@ -52,7 +51,6 @@ pub struct QLoRALinearWrapper<B: Backend> {impl<B: Backend> QLoRALinearWrapper<B> {pub fn new(- name: String,context: &B::Context,quantization_spec: AnyWeightMatrixSpec,adapter_spec: LowRankSpec,@@ -87,7 +85,6 @@ impl<B: Backend> QLoRALinearWrapper<B> {));};let base_linear = LinearMatmul::load(- format!("{name}/base"),context,quantization_spec,input_dim,@@ -155,7 +152,6 @@ impl<B: Backend> QLoRALinearWrapper<B> {.read_buffer()?;Ok(Self {- name,base_linear,input_hadamard,output_hadamard,@@ -179,8 +175,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("linear (qlora)");let mut intermediate =command_buffer.allocate_scratch_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type)?;@@ -266,7 +261,6 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {);}- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex bcfaebc11..c4dbbe8d8 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -27,7 +27,6 @@ pub enum RHTLinearWrapperError<B: Backend> {}pub struct RHTLinearWrapper<B: Backend> {- name: String,input_rht: InputRht<B>,inner_linear: LinearMatmul<B>,}@@ -46,7 +45,6 @@ fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {impl<B: Backend> RHTLinearWrapper<B> {pub(super) fn new(- name: String,context: &B::Context,input_dimension: u32,output_dimension: u32,@@ -63,7 +61,6 @@ impl<B: Backend> RHTLinearWrapper<B> {}let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(- format!("{name}/matmul"),context,input_dimension,output_dimension,@@ -75,7 +72,6 @@ impl<B: Backend> RHTLinearWrapper<B> {)?;let activation_quantization = inner_linear.prepare_a8(context);Self::build_self_contained(- name,context,input_data_type,LinearInputPreparation {@@ -87,7 +83,6 @@ impl<B: Backend> RHTLinearWrapper<B> {}pub(super) fn try_new_with_input_preparation(- name: String,context: &B::Context,input_dimension: u32,output_dimension: u32,@@ -105,7 +100,6 @@ impl<B: Backend> RHTLinearWrapper<B> {}let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(- format!("{name}/matmul"),context,input_dimension,output_dimension,@@ -120,7 +114,7 @@ impl<B: Backend> RHTLinearWrapper<B> {activation_quantization: inner_linear.prepare_a8(context),};if input_preparation.activation_quantization.is_some() && !allow_prequantized_activation {- let wrapper = Self::build_self_contained(name, context, input_data_type, input_preparation, inner_linear)?;+ let wrapper = Self::build_self_contained(context, input_data_type, input_preparation, inner_linear)?;Ok(Some((Box::new(wrapper), None)))} else {Ok(Some((Box::new(inner_linear), Some(input_preparation))))@@ -128,7 +122,6 @@ impl<B: Backend> RHTLinearWrapper<B> {}fn load_inner_with_output_rht(- name: String,context: &B::Context,input_dimension: u32,output_dimension: u32,@@ -150,7 +143,6 @@ impl<B: Backend> RHTLinearWrapper<B> {.validate(&[output_dimension], DataType::I32)?.read_buffer()?;let inner_linear = LinearMatmul::load(- name,context,quantization_spec,input_dimension,@@ -166,17 +158,15 @@ impl<B: Backend> RHTLinearWrapper<B> {}fn build_self_contained(- name: String,context: &B::Context,input_data_type: DataType,input_preparation: LinearInputPreparation<B>,inner_linear: LinearMatmul<B>,) -> Result<Self, RHTLinearWrapperError<B>> {- let input_rht = InputRht::new(format!("{name}/prepare"), context, input_data_type, input_preparation, true)+ let input_rht = InputRht::new(context, input_data_type, input_preparation, true).map_err(RHTLinearWrapperError::BackendError)?;Ok(Self {- name,input_rht,inner_linear,})@@ -199,14 +189,12 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("linear (rht)");let input = match input {LinearInput::FullPrecision(input) => input,input => {let output = self.inner_linear.encode_input(input, batch_dim, command_buffer);- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();return output;},@@ -220,7 +208,6 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {command_buffer,)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)}diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rsindex 65fb42175..a993ff14e 100644--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs@@ -16,14 +16,12 @@ use crate::{};pub struct UntiedReadout<B: Backend> {- name: String,linear: LinearMatmul<B>,input_rht: Option<InputRht<B>>,}impl<B: Backend> UntiedReadout<B> {pub fn load(- name: String,context: &B::Context,tree: &ParameterTree<B>,spec: AnyWeightMatrixSpec,@@ -41,25 +39,13 @@ impl<B: Backend> UntiedReadout<B> {else {return Ok(Self {linear: LinearMatmul::load(- format!("{name}/matmul"),- context,- spec,- model_dim,- vocab_size,- data_type,- data_type,- data_type,- tree,- None,- None,+ context, spec, model_dim, vocab_size, data_type, data_type, data_type, tree, None, None,)?,input_rht: None,- name,});};let mut linear = LinearMatmul::load(- format!("{name}/matmul"),context,*quantization_spec,model_dim,@@ -80,12 +66,10 @@ impl<B: Backend> UntiedReadout<B> {rht_signs,activation_quantization: linear.prepare_a8(context),};- let input_rht =- InputRht::new(format!("{name}/prepare"), context, data_type, preparation, /* in_place */ false)- .map_err(LinearMatmulError::BackendError)?;+ let input_rht = InputRht::new(context, data_type, preparation, /* in_place */ false)+ .map_err(LinearMatmulError::BackendError)?;Ok(Self {- name,linear,input_rht: Some(input_rht),})@@ -98,15 +82,12 @@ impl<B: Backend> UntiedReadout<B> {gather: Option<Gather<impl BufferRef<Backend = B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.name);let Some(input_rht) = &self.input_rht else {let a = MatmulA::FullPrecision {values: input,offset: 0,};- let output = self.linear.encode_with_a(a, batch_dim, gather, command_buffer)?;- command_buffer.sample_end_timestamp();- return Ok(output);+ return self.linear.encode_with_a(a, batch_dim, gather, command_buffer);};let format = if gather.is_some() {@@ -115,8 +96,6 @@ impl<B: Backend> UntiedReadout<B> {self.linear.select_activation_format(batch_dim, command_buffer.context())};let input = input_rht.prepare(input, batch_dim, format, command_buffer)?;- let output = self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)?;- command_buffer.sample_end_timestamp();- Ok(output)+ self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rsindex 9b540abef..9c2b21056 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs@@ -33,11 +33,6 @@ pub use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, KVCacheView};pub mod rope;pub struct Attention<B: Backend> {- name: String,- core_name: String,- prepare_kv_and_queries_name: String,- prepare_queries_name: String,- append_projected_kv_name: String,head_dim: u32,num_q_heads: u32,num_kv_heads: Option<u32>,@@ -67,7 +62,6 @@ pub enum AttentionNewError<B: Backend> {impl<B: Backend> Attention<B> {pub fn new(- name: String,hidden_dim: u32,data_type: DataType,rope_config: Option<&AnyRoPEConfig>,@@ -101,7 +95,6 @@ impl<B: Backend> Attention<B> {qkv_dim};let (projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(- format!("{name}/qkv projection"),hidden_dim,[projection_dim],config.has_qkvg_biases,@@ -117,7 +110,6 @@ impl<B: Backend> Attention<B> {let qkv_norm = (query_norm_config.is_some() || key_norm_config.is_some() || value_norm_config.is_some()).then(|| {QKVNorm::new(- format!("{name}/qkv norm"),context,data_type,query_norm_config,@@ -172,7 +164,6 @@ impl<B: Backend> Attention<B> {.map_err(AttentionNewError::Backend)?;let out_projection = <dyn Linear<B>>::new(- format!("{name}/out projection"),q_dim,[hidden_dim],config.has_out_biases,@@ -183,11 +174,6 @@ impl<B: Backend> Attention<B> {Ok((Self {- core_name: format!("{name}/core"),- prepare_kv_and_queries_name: format!("{name}/prepare kv and queries"),- prepare_queries_name: format!("{name}/prepare queries"),- append_projected_kv_name: format!("{name}/append projected kv"),- name,head_dim,num_q_heads,num_kv_heads,@@ -235,8 +221,7 @@ impl<B: Backend> Mixer<B> for Attention<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("attention");assert_eq!(precalculated_rope.is_some(), self.max_rope_length.is_some(), "precalculated rope mismatch");@@ -244,7 +229,6 @@ impl<B: Backend> Mixer<B> for Attention<B> {state.map(|state| state.downcast::<AttentionState<B>>().expect("incorrect type of attention state"));let output = self.attend(hidden, precalculated_rope, batch_dim, state, command_buffer)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rsindex f1d6b7a9d..75b337ac1 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs@@ -127,7 +127,6 @@ impl<B: Backend> Attention<B> {state: &mut AttentionState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.sample_start_timestamp(&self.append_projected_kv_name);if let Some(norm) = &self.projection.norm {norm.encode_key_value(key_value.reborrow(), batch_dim, command_buffer)?;}@@ -143,7 +142,6 @@ impl<B: Backend> Attention<B> {command_buffer,)?;state.encode_accept(&(0..batch_dim).collect::<Box<[u32]>>(), command_buffer)?;- command_buffer.sample_end_timestamp();Ok(())}@@ -154,14 +152,13 @@ impl<B: Backend> Attention<B> {state: &AttentionState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.core_name);let trie = if batch_dim.is_flat() {None} else {Some(command_buffer.allocate_constant_from_slice(batch_dim.nodes())?)};- let output = self.kernel.encode(+ self.kernel.encode(AttentionArguments {queries,keys: state.keys.as_ref(),@@ -172,9 +169,7 @@ impl<B: Backend> Attention<B> {cache: state.view(),},command_buffer,- )?;- command_buffer.sample_end_timestamp();- Ok(output)+ )}fn prepare_kv_and_queries(@@ -188,7 +183,6 @@ impl<B: Backend> Attention<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.prepare_kv_and_queries_name);let num_kv_heads = self.num_kv_heads.expect("KV prepare requires KV heads");// Appended KV is tightly packed; attention projections may have a trailing gate segment.let input_row_stride = if num_q_heads == 0 {@@ -217,7 +211,6 @@ impl<B: Backend> Attention<B> {batch_dim,command_buffer,);- command_buffer.sample_end_timestamp();Ok(queries)}@@ -228,7 +221,6 @@ impl<B: Backend> Attention<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.prepare_queries_name);let mut queries =command_buffer.allocate_scratch_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type)?;self.prepare.encode(@@ -247,7 +239,6 @@ impl<B: Backend> Attention<B> {batch_dim,command_buffer,);- command_buffer.sample_end_timestamp();Ok(queries)}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rsindex 23c1054cb..979998d7a 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs@@ -26,7 +26,6 @@ struct Head<B: Backend> {}pub struct QKVNorm<B: Backend> {- name: String,query: Option<Head<B>>,key: Option<Head<B>>,value: Option<Head<B>>,@@ -38,7 +37,6 @@ pub struct QKVNorm<B: Backend> {impl<B: Backend> QKVNorm<B> {pub fn new(- name: String,context: &B::Context,intermediate_data_type: DataType,query_config: Option<NormalizationConfig>,@@ -77,7 +75,6 @@ impl<B: Backend> QKVNorm<B> {.transpose()?;Ok(Self {- name,query,key,value,@@ -155,8 +152,7 @@ impl<B: Backend> QKVNorm<B> {"QKV norm input row stride ({input_row_stride}) is smaller than its packed row width ({packed_row_width})");- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("qkv norm");let kv = self.num_kv_heads;let heads = [(&self.query, 0, q_heads), (&self.key, q_heads, kv), (&self.value, q_heads + kv, kv)];@@ -183,7 +179,6 @@ impl<B: Backend> QKVNorm<B> {);}- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(())diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rsindex 6430332e5..967fc08cd 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs@@ -3,7 +3,7 @@ use std::any::Any;use crate::{array::size_for_shape,backends::common::{- Backend, Buffer, CommandBuffer, CommandBufferEncoding, Context, DeviceCapabilities, Kernels, SparseBuffer,+ Backend, Buffer, CommandBuffer, Context, DeviceCapabilities, Kernels, SparseBuffer,gpu_types::{Copy, ring::RingParams},kernel::KVCacheUpdateKernel,},@@ -161,7 +161,6 @@ impl KVCacheState {}pub struct AttentionState<B: Backend> {- accept_name: String,pub element_dim: u32,pub data_type: DataType,cache: KVCacheState,@@ -216,7 +215,6 @@ impl<B: Backend> AttentionState<B> {let kv_cache_update = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(context, data_type)?;Ok(Self {- accept_name: format!("{}/accept", attention.name),element_dim,data_type,cache,@@ -255,7 +253,6 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {accepted_indices: &[u32],command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.sample_start_timestamp(&self.accept_name);let copies = self.cache.accept(accepted_indices);if !copies.is_empty() {@@ -269,7 +266,6 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {);}- command_buffer.sample_end_timestamp();Ok(())}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rsindex 986db8154..33e793265 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs@@ -48,7 +48,6 @@ enum DeltaNetSuffixStatus<B: Backend> {}pub struct DeltaNetState<B: Backend> {- accept_name: String,conv_state: B::GlobalBuffer,ssm_state: B::GlobalBuffer,suffix_status: Option<DeltaNetSuffixStatus<B>>,@@ -69,7 +68,6 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {accepted_indices: &[u32],command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.sample_start_timestamp(&self.accept_name);let suffix_status = self.suffix_status.take().expect("delta net state has no suffix to accept");let accepted_index = *accepted_indices.last().expect("delta net state attempted to accept zero indices");@@ -111,14 +109,11 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {);},}- command_buffer.sample_end_timestamp();Ok(())}}pub struct DeltaNet<B: Backend> {- name: String,- tree_verify_name: String,num_heads: u32,head_dim: u32,num_groups: u32,@@ -164,7 +159,6 @@ pub enum DeltaNetNewError<B: Backend> {impl<B: Backend> DeltaNet<B> {pub fn new(- name: String,hidden_dim: u32,outer_data_type: DataType,config: &DeltaNetConfig,@@ -197,7 +191,6 @@ impl<B: Backend> DeltaNet<B> {let (in_projection, in_projection_input_hadamard_factors) =<dyn Linear<B>>::new_with_input_rht_mixed_precision(- format!("{name}/in projection"),hidden_dim,[total_proj_dim],false,@@ -288,7 +281,6 @@ impl<B: Backend> DeltaNet<B> {.map_err(DeltaNetNewError::Backend)?;let out_projection = <dyn Linear<B>>::new_mixed_precision(- format!("{name}/out projection"),value_dim,[hidden_dim],false,@@ -301,8 +293,6 @@ impl<B: Backend> DeltaNet<B> {Ok((Self {- tree_verify_name: format!("{name}/tree verify"),- name,num_heads: config.num_heads,head_dim: config.head_dim,num_groups: config.num_groups,@@ -344,7 +334,6 @@ impl<B: Backend> DeltaNet<B> {state: &mut DeltaNetState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.tree_verify_name);let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");let tree_size = batch_dim.size();let parents = command_buffer.allocate_constant_from_slice(batch_dim.parents())?;@@ -428,7 +417,6 @@ impl<B: Backend> DeltaNet<B> {beta,parents: batch_dim.parents().into(),});- command_buffer.sample_end_timestamp();Ok(output)}}@@ -467,7 +455,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {)?;Ok(Box::new(DeltaNetState {- accept_name: format!("{}/accept", self.name),conv_state,ssm_state,suffix_status: None,@@ -483,8 +470,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("delta net");assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");@@ -501,7 +487,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {if !batch_dim.full_accept() {let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();return Ok(output);@@ -646,7 +631,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {let output = self.out_projection.encode(delta_output, batch_dim.size(), command_buffer)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rsindex 7027e63cf..878668519 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs@@ -55,7 +55,6 @@ enum Mamba2SSDPrefillVariant<B: Backend> {}pub struct Mamba2<B: Backend> {- name: String,kernel_size: u32,num_heads: u32,num_groups: u32,@@ -92,7 +91,6 @@ pub enum Mamba2NewError<B: Backend> {impl<B: Backend> Mamba2<B> {pub fn new(- name: String,hidden_dim: u32,outer_data_type: DataType,config: &Mamba2Config,@@ -116,7 +114,6 @@ impl<B: Backend> Mamba2<B> {let (in_projection, in_projection_input_hadamard_factors) =<dyn Linear<B>>::new_with_input_rht_mixed_precision(- format!("{name}/in projection"),hidden_dim,[conv_dim, inner_dim, num_heads],config.has_in_biases,@@ -168,7 +165,6 @@ impl<B: Backend> Mamba2<B> {};let out_projection = <dyn Linear<B>>::new_mixed_precision(- format!("{name}/out projection"),inner_dim,[hidden_dim],config.has_out_biases,@@ -181,7 +177,6 @@ impl<B: Backend> Mamba2<B> {Ok((Self {- name,kernel_size,num_heads,num_groups,@@ -248,8 +243,7 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("mamba2");assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");@@ -427,7 +421,6 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {let output = self.out_projection.encode(ssd_output, batch_dim.size(), command_buffer)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rsindex e9fe51377..5e0c30d66 100644--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs@@ -83,7 +83,6 @@ pub enum MixerNewError<B: Backend> {impl<B: Backend> dyn Mixer<B> {pub fn new(- layer_name: &str,hidden_dim: u32,data_type: DataType,rope_config: Option<&AnyRoPEConfig>,@@ -93,57 +92,32 @@ impl<B: Backend> dyn Mixer<B> {) -> Result<(Box<dyn Mixer<B>>, Option<B::GlobalBuffer>), MixerNewError<B>> {match config {AnyTokenMixerConfig::AttentionConfig(config) => {- let (attention, in_projection_input_hadamard_factors) = Attention::new(- format!("{layer_name}/attention"),- hidden_dim,- data_type,- rope_config,- config,- parameter_tree,- context,- )?;+ let (attention, in_projection_input_hadamard_factors) =+ Attention::new(hidden_dim, data_type, rope_config, config, parameter_tree, context)?;Ok((Box::new(attention), in_projection_input_hadamard_factors))},AnyTokenMixerConfig::DeltaNetConfig(config) => {assert!(rope_config.is_none(), "unexpected rope for delta net mixer");- let (delta_net, in_projection_input_hadamard_factors) = DeltaNet::new(- format!("{layer_name}/delta net"),- hidden_dim,- data_type,- config,- parameter_tree,- context,- )?;+ let (delta_net, in_projection_input_hadamard_factors) =+ DeltaNet::new(hidden_dim, data_type, config, parameter_tree, context)?;Ok((Box::new(delta_net), in_projection_input_hadamard_factors))},AnyTokenMixerConfig::Mamba2Config(config) => {assert!(rope_config.is_none(), "unexpected rope for mamba2 mixer");- let (mamba2, in_projection_input_hadamard_factors) = Mamba2::new(- format!("{layer_name}/mamba2"),- hidden_dim,- data_type,- config,- parameter_tree,- context,- )?;+ let (mamba2, in_projection_input_hadamard_factors) =+ Mamba2::new(hidden_dim, data_type, config, parameter_tree, context)?;Ok((Box::new(mamba2), in_projection_input_hadamard_factors))},AnyTokenMixerConfig::ShortConvConfig(config) => {assert!(rope_config.is_none(), "unexpected rope for short conv mixer");- let (short_conv, in_projection_input_hadamard_factors) = ShortConv::new(- format!("{layer_name}/short conv"),- hidden_dim,- data_type,- config,- parameter_tree,- context,- )?;+ let (short_conv, in_projection_input_hadamard_factors) =+ ShortConv::new(hidden_dim, data_type, config, parameter_tree, context)?;Ok((Box::new(short_conv), in_projection_input_hadamard_factors))},diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rsindex 4c878daf1..93f9cbbb2 100644--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs@@ -75,10 +75,6 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {}pub struct ShortConv<B: Backend> {- name: String,- decode_conv_name: String,- prefill_conv_name: String,- trie_conv_name: String,hidden_dim: u32,data_type: DataType,kernel_size: u32,@@ -106,7 +102,6 @@ pub enum ShortConvNewError<B: Backend> {impl<B: Backend> ShortConv<B> {pub fn new(- name: String,hidden_dim: u32,data_type: DataType,config: &ShortConvConfig,@@ -122,7 +117,6 @@ impl<B: Backend> ShortConv<B> {}let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(- format!("{name}/in projection"),hidden_dim,[hidden_dim * 3],false,@@ -132,7 +126,6 @@ impl<B: Backend> ShortConv<B> {)?;let out_projection = <dyn Linear<B>>::new(- format!("{name}/out projection"),hidden_dim,[hidden_dim],false,@@ -168,10 +161,6 @@ impl<B: Backend> ShortConv<B> {Ok((Self {- decode_conv_name: format!("{name}/decode conv"),- prefill_conv_name: format!("{name}/prefill conv"),- trie_conv_name: format!("{name}/trie conv"),- name,hidden_dim,data_type,kernel_size,@@ -194,7 +183,6 @@ impl<B: Backend> ShortConv<B> {state: &mut ShortConvState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.decode_conv_name);let mut conv_output = command_buffer.allocate_scratch_for_shape(&[self.hidden_dim], self.data_type)?;self.short_conv_decode.encode(in_projected,@@ -210,7 +198,6 @@ impl<B: Backend> ShortConv<B> {self.hidden_dim,command_buffer,);- command_buffer.sample_end_timestamp();Ok(conv_output)}@@ -221,7 +208,6 @@ impl<B: Backend> ShortConv<B> {state: &mut ShortConvState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.prefill_conv_name);let state_stride = self.kernel_size - 1;let padded_rows = state_stride + batch_dim;@@ -253,7 +239,6 @@ impl<B: Backend> ShortConv<B> {self.hidden_dim,command_buffer,);- command_buffer.sample_end_timestamp();Ok(conv_output)}@@ -265,7 +250,6 @@ impl<B: Backend> ShortConv<B> {state: &mut ShortConvState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {- command_buffer.sample_start_timestamp(&self.trie_conv_name);let mut conv_output =command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;let mut conv_states = command_buffer@@ -285,7 +269,6 @@ impl<B: Backend> ShortConv<B> {self.hidden_dim,command_buffer,);- command_buffer.sample_end_timestamp();Ok((conv_output, conv_states))}}@@ -325,8 +308,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("short conv");assert!(precalculated_rope.is_none(), "unexpected rope for short conv mixer");@@ -362,7 +344,6 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {let output = self.out_projection.encode(conv_output, batch_dim.size(), command_buffer)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rsindex d48d52ddc..a0881dee7 100644--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs@@ -9,7 +9,6 @@ use crate::{};pub struct DenseMlp<B: Backend> {- name: String,up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,down: Box<dyn Linear<B>>,@@ -17,13 +16,11 @@ pub struct DenseMlp<B: Backend> {impl<B: Backend> DenseMlp<B> {pub fn new(- name: String,up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,down: Box<dyn Linear<B>>,) -> Self {Self {- name,up,gate,down,@@ -38,15 +35,13 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("mlp (dense)");let fused_up = self.up.encode(input, batch_dim, command_buffer)?;let act_format = self.down.select_activation_format(batch_dim, command_buffer.context());let down_input = self.gate.encode_for_linear(command_buffer, &fused_up, batch_dim, act_format)?;let output = self.down.encode_input(down_input, batch_dim, command_buffer)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rsindex 45911b711..4eab50d2f 100644--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs@@ -10,7 +10,6 @@ use crate::{};pub struct MlpGateActMulEncodable<B: Backend> {- name: String,fp_kernel: GatedActMul<B>,activation: AnyActivation,hidden_dim: u32,@@ -22,7 +21,6 @@ pub struct MlpGateActMulEncodable<B: Backend> {impl<B: Backend> MlpGateActMulEncodable<B> {pub fn new(- name: String,context: &B::Context,data_type: DataType,activation: AnyActivation,@@ -43,7 +41,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {.map(|quantization| GatedActMul::quantized(context, data_type, quantization, settings)).transpose()?;Ok(Self {- name,fp_kernel,activation,hidden_dim,@@ -61,8 +58,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {batch_dim: u32,act_format: ActivationFormat,) -> Result<LinearInput<B>, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("gate act mul");if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")@@ -119,7 +115,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {LinearInput::FullPrecision(hidden)};- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(input)diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rsindex 30407b40e..5ae64a74e 100644--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs@@ -35,7 +35,6 @@ pub enum MlpBlockError<B: Backend> {impl<B: Backend> dyn Mlp<B> {pub fn new(- name: String,config: &AnyMLPConfig,model_dimension: u32,hidden_dimension: u32,@@ -46,7 +45,6 @@ impl<B: Backend> dyn Mlp<B> {match config {AnyMLPConfig::DenseMLPConfig(dense_config) => {let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(- format!("{name}/up projection"),model_dimension,[2 * hidden_dimension],dense_config.has_up_biases,@@ -56,7 +54,6 @@ impl<B: Backend> dyn Mlp<B> {)?;let (down_projection, down_input_preparation) = <dyn Linear<B>>::new_for_fused_input(- format!("{name}/down projection"),hidden_dimension,[model_dimension],dense_config.has_down_biases,@@ -66,7 +63,6 @@ impl<B: Backend> dyn Mlp<B> {)?;let gate = MlpGateActMulEncodable::new(- format!("{name}/gate act mul"),context,data_type,dense_config.activation.clone(),@@ -77,7 +73,7 @@ impl<B: Backend> dyn Mlp<B> {).map_err(MlpBlockError::BackendError)?;- Ok((Box::new(DenseMlp::new(name, up_projection, gate, down_projection)), up_input_hadamard_factors))+ Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))},AnyMLPConfig::MixtureOfExpertsConfig(_) => Err(MlpBlockError::UnsupportedMixtureOfExperts),}diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rsindex b3f893482..9f6c79366 100644--- a/crates/uzu-engine/src/encodable_block/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mod.rs@@ -1,6 +1,5 @@pub mod batch_topology;pub mod classifier;-pub mod context_ring_update;pub mod convolution;pub mod decoder;pub mod dflash;diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rsindex 6ce97da6c..6fe74ae30 100644--- a/crates/uzu-engine/src/encodable_block/normalization.rs+++ b/crates/uzu-engine/src/encodable_block/normalization.rs@@ -37,7 +37,6 @@ pub enum NormalizationNewError<B: Backend> {}pub struct Normalization<B: Backend> {- name: String,epsilon: f32,scale_offset: Option<f32>,scales: Option<B::GlobalBuffer>,@@ -51,7 +50,6 @@ pub struct Normalization<B: Backend> {impl<B: Backend> Normalization<B> {pub fn new(- name: String,element_count: u32,hadamard_factors: Option<B::GlobalBuffer>,shortcut_mode: ShortcutMode,@@ -102,7 +100,6 @@ impl<B: Backend> Normalization<B> {.map_err(NormalizationNewError::Backend)?;Ok(Self {- name,epsilon: config.epsilon,scale_offset: config.scale_offset,scales,@@ -123,8 +120,7 @@ impl<B: Backend> Normalization<B> {shortcut: Option<impl BufferMut<Backend = B>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("normalization");let row_size = size_for_shape(&[self.element_count], self.data_type);let row_offset_bytes = row_offset as usize * row_size;@@ -145,7 +141,6 @@ impl<B: Backend> Normalization<B> {command_buffer,);- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rsindex 6c9c5a9ee..a2920a9ea 100644--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs@@ -32,7 +32,6 @@ pub enum PerLayerEmbeddingError<B: Backend> {}pub struct PerLayerEmbedding<B: Backend> {- name: String,token_embedding: EmbeddingTable<B>,model_projection: Box<dyn Linear<B>>,projection_norm: Normalization<B>,@@ -46,7 +45,6 @@ pub struct PerLayerEmbedding<B: Backend> {impl<B: Backend> PerLayerEmbedding<B> {pub fn new(- name: String,context: &B::Context,config: &PLEModelConfig,model_dim: u32,@@ -56,7 +54,6 @@ impl<B: Backend> PerLayerEmbedding<B> {let total_ple_dim = config.num_layers * config.ple_dim;let token_embedding = EmbeddingTable::load(- format!("{name}/token lookup"),context,¶meter_tree.subtree("token_embedding"),config.ple_vocab_size,@@ -65,7 +62,6 @@ impl<B: Backend> PerLayerEmbedding<B> {)?;let model_projection = <dyn Linear<B>>::new(- format!("{name}/model projection"),model_dim,[total_ple_dim],false,@@ -81,7 +77,6 @@ impl<B: Backend> PerLayerEmbedding<B> {adjusted};let projection_norm = Normalization::new(- format!("{name}/projection norm"),config.ple_dim,None,ShortcutMode::None,@@ -96,7 +91,6 @@ impl<B: Backend> PerLayerEmbedding<B> {.map_err(PerLayerEmbeddingError::BackendError)?;Ok(Self {- name,token_embedding,model_projection,projection_norm,@@ -116,8 +110,7 @@ impl<B: Backend> PerLayerEmbedding<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("per layer embedding");let total_ple_dim = self.num_layers * self.ple_dim;let total_rows = batch_dim * self.num_layers;@@ -157,7 +150,6 @@ impl<B: Backend> PerLayerEmbedding<B> {command_buffer,);- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(per_layer_inputs)@@ -165,7 +157,6 @@ impl<B: Backend> PerLayerEmbedding<B> {}pub struct PerLayerEmbeddingProjection<B: Backend> {- name: String,gate: Box<dyn Linear<B>>,projection: Box<dyn Linear<B>>,norm: Normalization<B>,@@ -182,7 +173,6 @@ pub struct PerLayerEmbeddingProjection<B: Backend> {impl<B: Backend> PerLayerEmbeddingProjection<B> {pub fn new(- name: String,context: &B::Context,config: &PLELayerConfig,model_dim: u32,@@ -192,7 +182,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {parameter_tree: &ParameterTree<B>,) -> Result<Self, PerLayerEmbeddingError<B>> {let gate = <dyn Linear<B>>::new(- format!("{name}/gate"),model_dim,[config.ple_dim],false,@@ -201,7 +190,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {¶meter_tree.subtree("gate"),)?;let projection = <dyn Linear<B>>::new(- format!("{name}/projection"),config.ple_dim,[model_dim],false,@@ -210,7 +198,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {¶meter_tree.subtree("projection"),)?;let norm = Normalization::new(- format!("{name}/norm"),model_dim,None,ShortcutMode::None,@@ -234,7 +221,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {.map_err(PerLayerEmbeddingError::BackendError)?;Ok(Self {- name,gate,projection,norm,@@ -259,8 +245,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("per layer embedding projection");let length = batch_dim * self.model_dim;@@ -304,7 +289,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {command_buffer,);- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(())diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rsindex 010b45944..351661ee1 100644--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs@@ -27,7 +27,6 @@ pub enum PredictionHeadError<B: Backend> {}pub struct PredictionHead<B: Backend> {- name: String,hidden_dim: u32,activation: ActivationType,dense_projection: Box<dyn Linear<B>>,@@ -38,7 +37,6 @@ pub struct PredictionHead<B: Backend> {impl<B: Backend> PredictionHead<B> {pub fn new(- name: String,hidden_dim: u32,num_labels: u32,data_type: DataType,@@ -47,7 +45,6 @@ impl<B: Backend> PredictionHead<B> {context: &B::Context,) -> Result<Self, PredictionHeadError<B>> {let dense_projection = <dyn Linear<B>>::new(- format!("{name}/dense projection"),hidden_dim,[hidden_dim],config.use_dense_bias,@@ -61,7 +58,6 @@ impl<B: Backend> PredictionHead<B> {.map_err(PredictionHeadError::Backend)?;let normalization = Normalization::new(- format!("{name}/norm"),hidden_dim,None,ShortcutMode::None,@@ -73,7 +69,6 @@ impl<B: Backend> PredictionHead<B> {)?;let readout = <dyn Linear<B>>::new(- format!("{name}/readout"),hidden_dim,[num_labels],true,@@ -83,7 +78,6 @@ impl<B: Backend> PredictionHead<B> {)?;Ok(Self {- name,hidden_dim,activation,dense_projection,@@ -99,8 +93,7 @@ impl<B: Backend> PredictionHead<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("prediction head");let mut hidden = self.dense_projection.encode(input, batch_dim, command_buffer)?;self.activation_kernel.encode(@@ -114,7 +107,6 @@ impl<B: Backend> PredictionHead<B> {self.normalization.encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;let logits = self.readout.encode(normalized, batch_dim, command_buffer)?;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(logits)diff --git a/crates/uzu-engine/src/encodable_block/sampling/mod.rs b/crates/uzu-engine/src/encodable_block/sampling/mod.rsindex 16721b71b..1142bd587 100644--- a/crates/uzu-engine/src/encodable_block/sampling/mod.rs+++ b/crates/uzu-engine/src/encodable_block/sampling/mod.rs@@ -24,7 +24,6 @@ pub use gumbel::{gumbel_float, revidx};pub use prng::PRng;pub struct Sampling<B: Backend> {- name: String,vocab_size: u32,data_type: DataType,unified_kernels: Mutex<HashMap<UnifiedSamplingKey, <B::Kernels as Kernels>::UnifiedSamplingKernel>>,@@ -32,12 +31,10 @@ pub struct Sampling<B: Backend> {impl<B: Backend> Sampling<B> {pub fn new(- name: String,data_type: DataType,vocab_size: u32,) -> Self {Self {- name,vocab_size,data_type,unified_kernels: Mutex::new(HashMap::new()),@@ -95,8 +92,7 @@ impl<B: Backend> Sampling<B> {sampling_range: Range<u32>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::GlobalBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("sampling");let sampling_length = sampling_range.end - sampling_range.start;@@ -198,7 +194,6 @@ impl<B: Backend> Sampling<B> {command_buffer,);- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rsindex a38d15288..c73b991b6 100644--- a/crates/uzu-engine/src/encodable_block/transformer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer.rs@@ -25,7 +25,6 @@ enum TransformerLayerStateType<B: Backend> {}pub struct TransformerState<B: Backend> {- accept_name: String,layer_states: Box<[TransformerLayerStateType<B>]>,context_length: u32,}@@ -61,8 +60,7 @@ impl<B: Backend> TransformerState<B> {accepted_indices: &[u32],command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.push_debug_group(&self.accept_name);- command_buffer.sample_start_timestamp(&self.accept_name);+ command_buffer.push_debug_group("transformer accept");for layer_state in &mut self.layer_states {let TransformerLayerStateType::Owned(layer_state) = layer_state else {@@ -74,7 +72,6 @@ impl<B: Backend> TransformerState<B> {self.context_length += accepted_indices.len() as u32;- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(())@@ -92,8 +89,6 @@ pub enum TransformerNewError<B: Backend> {}pub struct Transformer<B: Backend> {- name: String,- capture_residual_name: String,ropes: Box<[AnyRoPEConfig]>,layers: Box<[(TransformerLayer<B>, Option<usize>)]>,output_norm: Normalization<B>,@@ -103,7 +98,6 @@ pub struct Transformer<B: Backend> {impl<B: Backend> Transformer<B> {pub fn new(- name: String,context: &B::Context,output_norm_hadamard_factors: Option<B::GlobalBuffer>,data_type: DataType,@@ -128,7 +122,6 @@ impl<B: Backend> Transformer<B> {});let layer = TransformerLayer::new(- format!("{name}/layer {layer_index}"),context,transformer_config.model_dim,transformer_config.hidden_dim,@@ -144,7 +137,6 @@ impl<B: Backend> Transformer<B> {.collect::<Result<Box<[_]>, TransformerNewError<B>>>()?;let output_norm = Normalization::new(- format!("{name}/output norm"),transformer_config.model_dim,output_norm_hadamard_factors,ShortcutMode::Add,@@ -159,8 +151,6 @@ impl<B: Backend> Transformer<B> {.map_err(TransformerNewError::Backend)?;Ok(Self {- capture_residual_name: format!("{name}/capture residual"),- name,ropes: ropes.into_boxed_slice(),layers,output_norm,@@ -176,11 +166,9 @@ impl<B: Backend> Transformer<B> {batch_size: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.capture_residual_name);let mut output = command_buffer.allocate_scratch(hidden.size())?;let elements = batch_size * self.model_dim;self.residual_add.encode(Some(shortcut), hidden, &mut output, elements, elements, 1.0, command_buffer);- command_buffer.sample_end_timestamp();Ok(output)}@@ -232,7 +220,6 @@ impl<B: Backend> Transformer<B> {let context_length = 0;Ok(TransformerState {- accept_name: format!("{}/accept", self.name),layer_states,context_length,})@@ -248,7 +235,6 @@ impl<B: Backend> Transformer<B> {mut state: Option<&mut TransformerState<B>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<TransformerEncodeOutput<B>, B::Error> {- command_buffer.sample_start_timestamp(&self.name);let mut hidden = input;let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_none() {self.prefill_cache_layer_count()@@ -324,7 +310,6 @@ impl<B: Backend> Transformer<B> {});let Some(output_range) = output_range else {- command_buffer.sample_end_timestamp();return Ok(TransformerEncodeOutput {output: None,hidden_features,@@ -339,7 +324,6 @@ impl<B: Backend> Transformer<B> {command_buffer,)?;- command_buffer.sample_end_timestamp();Ok(TransformerEncodeOutput {output: Some(output_normalized),hidden_features,diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rsindex 40f0e10ff..567a92c2a 100644--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs@@ -45,7 +45,6 @@ pub enum TransformerLayerError<B: Backend> {}struct TransformerLayerConv<B: Backend> {- pre_convolution_name: String,model_dim: u32,pre_conv: SeparableCausalConv<B>,kernel_projection: Box<dyn Linear<B>>,@@ -57,7 +56,6 @@ struct TransformerLayerConv<B: Backend> {// TODO: saner shortcutpub struct TransformerLayer<B: Backend> {- name: String,pub layer_index: u32,pub kv_source_layer_index: Option<u32>,pub pre_mixer_norm: Option<Normalization<B>>,@@ -73,7 +71,6 @@ pub struct TransformerLayer<B: Backend> {impl<B: Backend> TransformerLayerConv<B> {fn new(- name: String,context: &B::Context,model_dim: u32,config: &TransformerLayerConvConfig,@@ -82,7 +79,6 @@ impl<B: Backend> TransformerLayerConv<B> {input_hadamard_factors: Option<B::GlobalBuffer>,) -> Result<Self, TransformerLayerError<B>> {let pre_conv = SeparableCausalConv::new(- format!("{name}/pre conv"),model_dim,config.conv_kernel_size,config.conv_group_size,@@ -92,7 +88,6 @@ impl<B: Backend> TransformerLayerConv<B> {context,)?;let post_conv = SeparableCausalConv::new(- format!("{name}/post conv"),model_dim,config.conv_kernel_size,config.conv_group_size,@@ -105,7 +100,6 @@ impl<B: Backend> TransformerLayerConv<B> {let coefficient_count = config.conv_kernel_size * (model_dim / config.conv_group_size);let projection_dim = coefficient_count * 2;let kernel_projection = <dyn Linear<B>>::new(- format!("{name}/kernel projection"),model_dim,[projection_dim],false,@@ -120,7 +114,6 @@ impl<B: Backend> TransformerLayerConv<B> {.map_err(TransformerLayerError::Backend)?;Ok(Self {- pre_convolution_name: format!("{name}/pre convolution"),model_dim,pre_conv,kernel_projection,@@ -136,7 +129,6 @@ impl<B: Backend> TransformerLayerConv<B> {sequence_length: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {- command_buffer.sample_start_timestamp(&self.pre_convolution_name);let mut projection_input = command_buffer.allocate_scratch(input.size())?;command_buffer.encode_copy(input, &mut projection_input);let coefficients = self.kernel_projection.encode(projection_input, sequence_length, command_buffer)?;@@ -151,7 +143,6 @@ impl<B: Backend> TransformerLayerConv<B> {if let Some((transform, factors)) = &self.input_rht {transform.encode_fp_in_place(&mut output, factors, None, sequence_length, self.model_dim, command_buffer);}- command_buffer.sample_end_timestamp();Ok((output, coefficients))}@@ -175,7 +166,6 @@ impl<B: Backend> TransformerLayerConv<B> {impl<B: Backend> TransformerLayer<B> {pub fn new(- name: String,context: &B::Context,model_dim: u32,hidden_dim: u32,@@ -211,7 +201,6 @@ impl<B: Backend> TransformerLayer<B> {};let (mixer, mixer_hadamard_factors) = <dyn Mixer<B>>::new(- &name,model_dim,data_type,layer_config.rope_config.as_ref(),@@ -223,7 +212,6 @@ impl<B: Backend> TransformerLayer<B> {let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer_conv_config {Some(config) => (Some(TransformerLayerConv::new(- format!("{name}/mixer conv"),context,model_dim,config,@@ -238,7 +226,6 @@ impl<B: Backend> TransformerLayer<B> {let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {Some(Normalization::new(- format!("{name}/pre mixer norm"),model_dim,mixer_hadamard_factors,if layer_index > 0 {@@ -261,7 +248,6 @@ impl<B: Backend> TransformerLayer<B> {let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {Some(Normalization::new(- format!("{name}/post mixer norm"),model_dim,None,ShortcutMode::None,@@ -276,7 +262,6 @@ impl<B: Backend> TransformerLayer<B> {};let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(- format!("{name}/mlp"),&layer_config.mlp_config,model_dim,layer_config.hidden_dim.unwrap_or(hidden_dim),@@ -288,7 +273,6 @@ impl<B: Backend> TransformerLayer<B> {let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp_conv_config {Some(config) => (Some(TransformerLayerConv::new(- format!("{name}/mlp conv"),context,model_dim,config,@@ -302,7 +286,6 @@ impl<B: Backend> TransformerLayer<B> {};let pre_mlp_norm = Normalization::new(- format!("{name}/pre mlp norm"),model_dim,mlp_input_hadamard_factors,ShortcutMode::Add,@@ -315,7 +298,6 @@ impl<B: Backend> TransformerLayer<B> {let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(Normalization::new(- format!("{name}/post mlp norm"),model_dim,None,ShortcutMode::None,@@ -332,7 +314,6 @@ impl<B: Backend> TransformerLayer<B> {let ple_projection = layer_config.ple_config.as_ref().map(|ple_config| {let ple_loader = parameter_tree.subtree("ple");PerLayerEmbeddingProjection::new(- format!("{name}/per layer embedding projection"),context,ple_config,model_dim,@@ -345,7 +326,6 @@ impl<B: Backend> TransformerLayer<B> {});Ok(Self {- name,layer_index,kv_source_layer_index: layer_config.kv_source_layer_index,pre_mixer_norm,@@ -370,8 +350,7 @@ impl<B: Backend> TransformerLayer<B> {state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group(&format!("transformer layer {}", self.layer_index));let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?@@ -439,7 +418,6 @@ impl<B: Backend> TransformerLayer<B> {command_buffer.encode_fill(&mut hidden, 0);}- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(hidden)diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex a3a15c378..d446df08c 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -112,9 +112,6 @@ impl<B: Backend> EncodedWeaverTree<B> {}pub struct Weaver<B: Backend> {- name: String,- prefix_name: String,- step_name: String,token_embedding_norm: Normalization<B>,token_embedding_projection: Box<dyn Linear<B>>,hidden_state_norm: Normalization<B>,@@ -174,7 +171,6 @@ pub enum WeaverEncodeError<B: Backend> {impl<B: Backend> Weaver<B> {pub fn new(- name: String,context: &B::Context,config: &WeaverConfig,vocab_size: u32,@@ -203,7 +199,6 @@ impl<B: Backend> Weaver<B> {});}let token_embedding_norm = Normalization::new(- format!("{name}/token embedding norm"),config.target_embedding_dim,None,ShortcutMode::None,@@ -214,7 +209,6 @@ impl<B: Backend> Weaver<B> {context,)?;let hidden_state_norm = Normalization::new(- format!("{name}/hidden state norm"),config.target_model_dim,None,ShortcutMode::None,@@ -225,7 +219,6 @@ impl<B: Backend> Weaver<B> {context,)?;let token_embedding_projection = <dyn Linear<B>>::new(- format!("{name}/token embedding projection"),config.target_embedding_dim,[config.model_dim],true,@@ -235,18 +228,9 @@ impl<B: Backend> Weaver<B> {)?;let layer_parameters = parameter_tree.subtree("blocks");let layers = (0..config.num_layers)- .map(|index| {- WeaverLayer::new(- format!("{name}/layer {index}"),- context,- config,- index > 0,- &layer_parameters.subtree(&index.to_string()),- )- })+ .map(|index| WeaverLayer::new(context, config, index > 0, &layer_parameters.subtree(&index.to_string()))).collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;let readout_norm = Normalization::new(- format!("{name}/readout norm"),config.model_dim,None,ShortcutMode::Add,@@ -257,7 +241,6 @@ impl<B: Backend> Weaver<B> {context,)?;let hidden_state_projection = <dyn Linear<B>>::new(- format!("{name}/hidden state projection"),config.target_model_dim,[config.model_dim],true,@@ -266,7 +249,6 @@ impl<B: Backend> Weaver<B> {¶meter_tree.subtree("hidden_state_projection"),)?;let readout_query_projection = <dyn Linear<B>>::new(- format!("{name}/readout query projection"),config.model_dim,[config.target_model_dim],false,@@ -285,9 +267,6 @@ impl<B: Backend> Weaver<B> {let frontier_insert_children = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;Ok(Self {- prefix_name: format!("{name}/prefix"),- step_name: format!("{name}/step"),- name,token_embedding_norm,token_embedding_projection,hidden_state_norm,@@ -320,8 +299,7 @@ impl<B: Backend> Weaver<B> {depth: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<Vec<B::ScratchBuffer>, WeaverEncodeError<B>> {- command_buffer.push_debug_group(&self.prefix_name);- command_buffer.sample_start_timestamp(&self.prefix_name);+ command_buffer.push_debug_group("weaver prefix");let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);let mut prefix_hidden = command_buffer@@ -381,7 +359,6 @@ impl<B: Backend> Weaver<B> {.kv_cache,);- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();Ok(prefix_kv_layers)@@ -410,8 +387,6 @@ impl<B: Backend> Weaver<B> {batch_start_slot: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), WeaverEncodeError<B>> {- command_buffer.push_debug_group(&self.step_name);- command_buffer.sample_start_timestamp(&self.step_name);let tree_slot_count = shape.slot_count();let ancestor_stride = self.max_depth;let frontier_capacity = tree_slot_count * shape.expand_width;@@ -564,8 +539,6 @@ impl<B: Backend> Weaver<B> {command_buffer,);- command_buffer.sample_end_timestamp();- command_buffer.pop_debug_group();Ok(())}@@ -580,8 +553,7 @@ impl<B: Backend> Weaver<B> {shape: WeaverTreeShape,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {- command_buffer.push_debug_group(&self.name);- command_buffer.sample_start_timestamp(&self.name);+ command_buffer.push_debug_group("weaver tree");let tree_slot_count = shape.slot_count();let ancestor_stride = self.max_depth;@@ -705,6 +677,7 @@ impl<B: Backend> Weaver<B> {} else {shape.expand_per_round};+ command_buffer.push_debug_group("weaver step");self.encode_step(target_embedding,prefix_kv_layers.iter(),@@ -727,10 +700,10 @@ impl<B: Backend> Weaver<B> {batch_start_slot,command_buffer,)?;+ command_buffer.pop_debug_group();batch_start_slot += batch_node_count;}- command_buffer.sample_end_timestamp();command_buffer.pop_debug_group();let mut packed_tree_readback =diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rsindex 8621bedce..2b9080e6b 100644--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs@@ -25,8 +25,6 @@ pub struct PreparedPrefixAttention<B: Backend> {}pub struct WeaverLayer<B: Backend> {- prefix_attention_name: String,- post_attention_name: String,pub pre_attention_norm: Normalization<B>,pub qkv_projection: Box<dyn Linear<B>>,attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,@@ -44,7 +42,6 @@ pub struct WeaverLayer<B: Backend> {impl<B: Backend> WeaverLayer<B> {pub fn new(- name: String,context: &B::Context,config: &WeaverConfig,add_to_residual: bool,@@ -62,7 +59,6 @@ impl<B: Backend> WeaverLayer<B> {let head_dim = model_dim / num_heads;let attention_scale = 1.0 / (head_dim as f32).sqrt();let qkv_projection = <dyn Linear<B>>::new(- format!("{name}/qkv projection"),model_dim,[3 * model_dim],false,@@ -71,7 +67,6 @@ impl<B: Backend> WeaverLayer<B> {¶meter_tree.subtree("qkv_projection"),)?;let out_projection = <dyn Linear<B>>::new(- format!("{name}/out projection"),model_dim,[model_dim],false,@@ -98,7 +93,6 @@ impl<B: Backend> WeaverLayer<B> {<B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true).map_err(WeaverNewError::Backend)?;let pre_attention_norm = Normalization::new(- format!("{name}/pre attention norm"),model_dim,None,if add_to_residual {@@ -113,7 +107,6 @@ impl<B: Backend> WeaverLayer<B> {context,)?;let pre_mlp_norm = Normalization::new(- format!("{name}/pre mlp norm"),model_dim,None,ShortcutMode::Add,@@ -129,21 +122,12 @@ impl<B: Backend> WeaverLayer<B> {true,true,));- let (mlp, up_input_hadamard_factors) = <dyn Mlp<B>>::new(- format!("{name}/mlp"),- &mlp_config,- model_dim,- hidden_dim,- context,- ¶meter_tree.subtree("mlp"),- DATA_TYPE,- )?;+ let (mlp, up_input_hadamard_factors) =+ <dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, ¶meter_tree.subtree("mlp"), DATA_TYPE)?;assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");let ancestor_attention = <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim, num_heads).map_err(WeaverNewError::Backend)?;Ok(Self {- prefix_attention_name: format!("{name}/prefix attention"),- post_attention_name: format!("{name}/post attention"),qkv_projection,out_projection,prefix_attention,@@ -168,7 +152,6 @@ impl<B: Backend> WeaverLayer<B> {token_count: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<PreparedPrefixAttention<B>, B::Error> {- command_buffer.sample_start_timestamp(&self.prefix_attention_name);let attention_input =self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), command_buffer)?;let qkv = self.qkv_projection.encode(attention_input, token_count, command_buffer)?;@@ -193,7 +176,6 @@ impl<B: Backend> WeaverLayer<B> {token_count,command_buffer,);- command_buffer.sample_end_timestamp();Ok(PreparedPrefixAttention {queries,kv_cache,@@ -207,12 +189,9 @@ impl<B: Backend> WeaverLayer<B> {token_count: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.sample_start_timestamp(&self.post_attention_name);let projected_attention = self.out_projection.encode(attention_output, token_count, command_buffer)?;let mlp_input =self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), command_buffer)?;- let output = self.mlp.encode(mlp_input, token_count, command_buffer)?;- command_buffer.sample_end_timestamp();- Ok(output)+ self.mlp.encode(mlp_input, token_count, command_buffer)}}diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rsindex 0d63e996a..d92e350e7 100644--- a/crates/uzu-engine/src/engine/classifier_model.rs+++ b/crates/uzu-engine/src/engine/classifier_model.rs@@ -66,7 +66,6 @@ impl<B: Backend> Engine<B> {let weight_loader = ParameterLoader::new(&weights_file, context.as_ref())?;let classifier = ClassifierEncodable::new(- String::from("classifier"),context.as_ref(),&config.classifier_config,&weight_loader.tree().subtree("classifier"),diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rsindex 54a2ff84a..1255404b9 100644--- a/crates/uzu-engine/src/engine/language_model/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/mod.rs@@ -5,14 +5,13 @@ use thiserror::Error;use tokenizers::Tokenizer;use crate::{- backends::common::{Backend, Context, DeviceCapabilities},+ backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},config::{model::{generation::GenerationConfig, language_model::LanguageModelConfig},token_codec::AnyTokenCodecConfig,},data_type::DataType,encodable_block::{- context_ring_update::ContextRingUpdate,decoder::{Decoder, DecoderError},sampling::{Sampling, SamplingMethod},},@@ -32,7 +31,7 @@ pub struct LanguageModel<B: Backend> {decoder: Decoder<B>,speculator: Option<DFlashTfmSpeculator<B>>,sampling: Sampling<B>,- context_ring_update: ContextRingUpdate<B>,+ context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,generation_config: GenerationConfig,end_of_thinking_tag: Option<String>,tokenizer: Arc<Tokenizer>,@@ -78,7 +77,6 @@ impl<B: Backend> Engine<B> {let data_type = DataType::BF16;let decoder = Decoder::new(- String::from("decoder"),self.context.as_ref(),&config.decoder_config,&weight_loader.tree().subtree("decoder"),@@ -96,11 +94,10 @@ impl<B: Backend> Engine<B> {.transpose()?.flatten();- let sampling = Sampling::new(String::from("sampling"), data_type, config.decoder_config.vocab_size);+ let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);- let context_ring_update =- ContextRingUpdate::new(String::from("update repetition penalty ring"), self.context.as_ref())- .map_err(EngineLoadLanguageModelError::Backend)?;+ let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&self.context)+ .map_err(EngineLoadLanguageModelError::Backend)?;weight_loader.tree().assert_all_tensors_validated()?;diff --git a/crates/uzu-engine/src/engine/language_model/stream/mod.rs b/crates/uzu-engine/src/engine/language_model/stream/mod.rsindex 8ca87d672..fc456ba8d 100644--- a/crates/uzu-engine/src/engine/language_model/stream/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/mod.rs@@ -1,12 +1,10 @@-use std::sync::mpsc::Sender;-use derive_more::Debug;use thiserror::Error;#[cfg(grammar)]use crate::engine::language_model::grammar::{Grammar, GrammarError};use crate::{- backends::common::{Backend, TimestampSpan},+ backends::common::Backend,encodable_block::decoder::DecoderError,engine::language_model::{LanguageModel, state::LanguageModelState},speculators::dflash_tfm::DFlashTreeError,@@ -19,7 +17,6 @@ mod stream;pub struct LanguageModelStreamOptions {pub sampling_method: SamplingMethod,- pub timestamps: Option<Sender<Box<[TimestampSpan]>>>,#[cfg(grammar)]pub grammar: Option<Grammar>,}@@ -47,7 +44,6 @@ impl<B: Backend> LanguageModel<B> {pub fn default_stream_options(&self) -> LanguageModelStreamOptions {LanguageModelStreamOptions {sampling_method: self.default_sampling_method(),- timestamps: None,#[cfg(grammar)]grammar: None,}diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 1e826f915..7663cf362 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -2,7 +2,7 @@ use std::{iter::{once, repeat_n},mem::replace,range::Range,- sync::{Arc, mpsc::Sender},+ sync::Arc,};use shoji::traits::backend::chat_token::TokenStreamMetrics;@@ -12,8 +12,8 @@ use crate::engine::language_model::grammar::Grammar;use crate::{array::size_for_shape,backends::common::{- Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context,- TimestampSpan, gpu_types::trie::TrieNode as GpuTrieNode,+ Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,+ CommandBufferPending, Context, gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,},data_type::DataType,encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},@@ -26,7 +26,6 @@ use crate::{},},trie::TrieNode,- utils::timestamps::{create_command_buffer, wait},};enum ForwardPassChaining<B: Backend> {@@ -41,7 +40,6 @@ impl<B: Backend> ForwardPassChaining<B> {fn resolve<'a>(&'a mut self,tokens: &mut Vec<u64>,- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,#[cfg(grammar)] grammar: Option<&mut Grammar>,) -> Result<(u64, Option<&'a B::ScratchBuffer>), LanguageModelStreamError<B>> {match self {@@ -52,7 +50,7 @@ impl<B: Backend> ForwardPassChaining<B> {Self::InFlight(in_flight) => {assert!(in_flight.full_accept);for pending in replace(&mut in_flight.pending, Box::new([])) {- wait(pending, timestamps).map_err(LanguageModelStreamError::Backend)?;+ pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;}let output_tokens = in_flight.output_tokens.as_slice::<u32>();assert_eq!(output_tokens.len(), 1);@@ -214,13 +212,11 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {).map_err(LanguageModelStreamError::Backend)?;- let mut command_buffer = create_command_buffer::<B>(- &model.engine.context,- "prefill",- &allocation_pool,- options.timestamps.as_ref(),- )- .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = model+ .engine+ .context+ .create_command_buffer(Some("prefill"), Some(allocation_pool.clone()))+ .map_err(LanguageModelStreamError::Backend)?;let mut output_tokens = None;let mut output_norm = None;@@ -405,7 +401,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {(ForwardPassChaining::InFlight(forward_pass_pending), None)} else {for pending in forward_pass_pending.pending {- wait(pending, self.options.timestamps.as_ref()).map_err(LanguageModelStreamError::Backend)?;+ pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;}let sampled_tokens = forward_pass_pending.output_tokens@@ -458,13 +454,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();- let mut command_buffer = create_command_buffer::<B>(- &self.model.engine.context,- "decode",- &self.allocation_pool,- self.options.timestamps.as_ref(),- )- .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = self+ .model+ .engine+ .context+ .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))+ .map_err(LanguageModelStreamError::Backend)?;self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut command_buffer)@@ -490,6 +485,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {None};if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {+ command_buffer.push_debug_group("update repetition penalty ring");let accepted_input_token_ids_const = command_buffer.allocate_constant_from_slice(&accepted_input_token_ids@@ -505,20 +501,23 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {full.len() as u32,&mut command_buffer,);+ command_buffer.pop_debug_group();}if let Some(capture_span) = capture_span {- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref())+ command_buffer+ .end_encoding()+ .submit()+ .wait_until_completed().map_err(LanguageModelStreamError::Backend)?;drop(capture_span);- command_buffer = create_command_buffer::<B>(- &self.model.engine.context,- "decode",- &self.allocation_pool,- self.options.timestamps.as_ref(),- )- .map_err(LanguageModelStreamError::Backend)?;+ command_buffer = self+ .model+ .engine+ .context+ .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))+ .map_err(LanguageModelStreamError::Backend)?;}self.model_state.tokens.extend(accepted_output_token_ids);(@@ -542,7 +541,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {prev_output.resolve(&mut self.model_state.tokens,- self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)?@@ -555,7 +553,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {{prev_output.resolve(&mut self.model_state.tokens,- self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)?;@@ -572,7 +569,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {)&& let (root_token, Some(output_norm)) = prev_output.resolve(&mut self.model_state.tokens,- self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)? {@@ -589,7 +585,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.options.grammar.as_mut(),&self.model_state.prng,self.allocation_pool.clone(),- self.options.timestamps.as_ref(),)?;prev_output.drop_output_norm();(trie, None, false)@@ -612,13 +607,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.allocation_pool = self.model.engine.context.create_allocation_pool();- let mut command_buffer = create_command_buffer::<B>(- &self.model.engine.context,- "decode",- &self.allocation_pool,- self.options.timestamps.as_ref(),- )- .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = self+ .model+ .engine+ .context+ .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))+ .map_err(LanguageModelStreamError::Backend)?;let scratch_token_ids;let mut constant_token_ids = None;@@ -665,13 +659,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if chain_copy.is_some() {pending.push(command_buffer.end_encoding().submit());- let mut command_buffer = create_command_buffer::<B>(- &self.model.engine.context,- "decode",- &self.allocation_pool,- self.options.timestamps.as_ref(),- )- .map_err(LanguageModelStreamError::Backend)?;+ let mut command_buffer = self+ .model+ .engine+ .context+ .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))+ .map_err(LanguageModelStreamError::Backend)?;let mut bitmask = command_buffer.allocate_constant(@@ -679,7 +672,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {).map_err(LanguageModelStreamError::Backend)?;- prev_output.resolve(&mut self.model_state.tokens, self.options.timestamps.as_ref(), Some(grammar))?;+ prev_output.resolve(&mut self.model_state.tokens, Some(grammar))?;if grammar.next_bitmask(bitmask.as_slice_mut()) {(Some(bitmask), command_buffer)} else {@@ -793,7 +786,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {prev_output.resolve(&mut self.model_state.tokens,- self.options.timestamps.as_ref(),#[cfg(grammar)]self.options.grammar.as_mut(),)?@@ -822,17 +814,16 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {} => Some(seed_token),DecodingState::ForwardPassPending(in_flight) => {for pending in in_flight.pending {- wait(pending, self.options.timestamps.as_ref()).unwrap();+ pending.wait_until_completed().unwrap();}if !in_flight.full_accept {- let mut command_buffer = create_command_buffer::<B>(- &self.model.engine.context,- "drop accept",- &self.allocation_pool,- self.options.timestamps.as_ref(),- )- .unwrap();+ let mut command_buffer = self+ .model+ .engine+ .context+ .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))+ .unwrap();self.model_state.transformer_state.encode_accept(&[0], &mut command_buffer).unwrap();if let Some(speculator) = self.model.speculator.as_ref() {speculator@@ -844,7 +835,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {).unwrap();}- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();}Some(in_flight.output_tokens.as_slice::<u32>()[0] as u64)@@ -858,13 +849,12 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {} => {assert!(num_accepted > 0 && num_accepted < full.len());- let mut command_buffer = create_command_buffer::<B>(- &self.model.engine.context,- "drop accept",- &self.allocation_pool,- self.options.timestamps.as_ref(),- )- .unwrap();+ let mut command_buffer = self+ .model+ .engine+ .context+ .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))+ .unwrap();let accepted_token_indicies =full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();self.model_state@@ -881,7 +871,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {).unwrap();}- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();drop(capture_span);diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex 2d1e02a60..6c41724b5 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -3,7 +3,7 @@ use std::{fs::File,io::{self, BufReader},path::Path,- sync::{Arc, mpsc::Sender},+ sync::Arc,};use derive_more::Debug;@@ -15,8 +15,8 @@ pub use crate::encodable_block::dflash::DFlashState;use crate::engine::language_model::grammar::Grammar;use crate::{backends::common::{- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,- gpu_types::trie::TrieNode as GpuTrieNode,+ Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,+ Context, gpu_types::trie::TrieNode as GpuTrieNode,},config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},data_type::DataType,@@ -29,7 +29,6 @@ use crate::{},parameters::{ParameterLoader, ParameterLoaderError},trie::TrieNode,- utils::timestamps::{create_command_buffer, wait},};#[derive(Debug, Error)]@@ -123,19 +122,12 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let weight_loader = ParameterLoader::new(&weights_file, &*context)?;let speculator_tree = weight_loader.tree().subtree("speculator");- let dflash = DFlash::new(- String::from("dflash"),- &*context,- &config.draft_config,- &speculator_tree.subtree("draft_model"),- data_type,- )?;+ let dflash = DFlash::new(&*context, &config.draft_config, &speculator_tree.subtree("draft_model"), data_type)?;let weaver = config.weaver_config.as_ref().map(|weaver_config| {Weaver::new(- String::from("weaver"),&*context,weaver_config,config.draft_config.vocab_size,@@ -146,7 +138,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {weight_loader.tree().assert_all_tensors_validated()?;- let sampling = Sampling::new(String::from("sampling"), data_type, config.draft_config.vocab_size);+ let sampling = Sampling::new(data_type, config.draft_config.vocab_size);Ok(Some(Self {context,@@ -225,7 +217,6 @@ impl<B: Backend> DFlashTfmSpeculator<B> {#[cfg(grammar)] grammar: Option<&mut Grammar>,prng: &PRng,allocation_pool: Arc<B::AllocationPool>,- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,) -> Result<TrieNode, DFlashTreeError<B>> {assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");@@ -239,9 +230,10 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let root_position = state.context_length();- let mut command_buffer =- create_command_buffer::<B>(&self.context, "speculator propose", &allocation_pool, timestamps)- .map_err(DFlashTreeError::Backend)?;+ let mut command_buffer = self+ .context+ .create_command_buffer(Some("speculator propose"), Some(allocation_pool))+ .map_err(DFlashTreeError::Backend)?;let nodes = match shape.construction_method {DFlashTfmTreeConstructionMethod::Argmax => {@@ -291,7 +283,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {).map_err(DFlashTreeError::Backend)?;let completed =- wait(command_buffer.end_encoding().submit(), timestamps).map_err(DFlashTreeError::Backend)?;+ command_buffer.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;let tokens = sampled.copyout::<u32>();drop(completed);nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {@@ -366,7 +358,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {&mut command_buffer,)?;let completed =- wait(command_buffer.end_encoding().submit(), timestamps).map_err(DFlashTreeError::Backend)?;+ command_buffer.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;let nodes = tree.read_nodes();drop(completed);nodesdiff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rsindex 17b686018..11e88e2c5 100644--- a/crates/uzu-engine/src/utils/mod.rs+++ b/crates/uzu-engine/src/utils/mod.rs@@ -4,5 +4,4 @@ pub mod maybe_mut;#[cfg(backend = "cpu")]pub mod pointers;pub mod strict_serde;-pub mod timestamps;pub mod version;diff --git a/crates/uzu-engine/src/utils/timestamps.rs b/crates/uzu-engine/src/utils/timestamps.rsdeleted file mode 100644index fcf6d60f6..000000000--- a/crates/uzu-engine/src/utils/timestamps.rs+++ /dev/null@@ -1,32 +0,0 @@-use std::sync::{Arc, mpsc::Sender};--use crate::backends::common::{- Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferPending, Context, TimestampSpan,-};--pub fn create_command_buffer<B: Backend>(- context: &B::Context,- name: &str,- allocation_pool: &Arc<B::AllocationPool>,- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,-) -> Result<<B::CommandBuffer as CommandBuffer>::Encoding, B::Error> {- let mut command_buffer = context.create_command_buffer(Some(name), Some(allocation_pool.clone()))?;- if timestamps.is_some() {- command_buffer.enable_timestamps();- }- Ok(command_buffer)-}--pub fn wait<Pending: CommandBufferPending>(- pending: Pending,- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,-) -> Result<- <Pending::CommandBuffer as CommandBuffer>::Completed,- <<Pending::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,-> {- let completed = pending.wait_until_completed()?;- if let Some(sender) = timestamps {- let _ = sender.send(completed.timestamps().into());- }- Ok(completed)-}diff --git a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rsindex c9639d5d8..15a85cc3b 100644--- a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs+++ b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs@@ -30,7 +30,6 @@ fn run_key_value_row_stride_test<B: Backend>() {let key = QKVNorm::<B>::build_head(&context, DataType::F32, config, None, HEAD_DIM).expect("failed to construct key norm");let norm = QKVNorm {- name: String::from("test/qkv norm"),query: None,key: Some(key),value: None,diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rsindex 50932575b..f00b7bfff 100644--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs@@ -44,7 +44,7 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(method: &SamplingMethod,batch_size: u32,) -> Result<SamplingTestResults, TestCaseError> {- let sampling = Sampling::<B>::new(String::from("test/sampling"), T::data_type(), vocab_size as u32);+ let sampling = Sampling::<B>::new(T::data_type(), vocab_size as u32);let mut command_buffer = context.create_command_buffer(None, None).unwrap();let logits_buffer = command_buffer.allocate_constant_from_slice(logits).unwrap();