Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Revert "gpu timestamps recording" (#912)

uuuvn4e7004b6ebdd
diff --git a/Cargo.lock b/Cargo.lock
index acf8f6327..eee205fe4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3689,9 +3689,9 @@ dependencies = [
[[package]]
name = "mtl-rs"
-version = "0.3.0"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4aa9291872296e9e734b0caf6e7a933fe6861d25ed86240461d6114355f7dbbf"
+checksum = "d9f1de2bca596db925db758d497cbf9cb394a2981933319b3777510097e1d360"
dependencies = [
"bitflags 2.13.0",
"block2",
diff --git a/Cargo.toml b/Cargo.toml
index 420503b9d..759972957 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -76,7 +76,7 @@ objc2-foundation = "0.3.2"
objc2-core-foundation = "0.3.2"
objc2-io-kit = { version = "0.3.2", default-features = false, features = ["libc", "ps"] }
obfstr = "0.4.4"
-metal = { package = "mtl-rs", version = "0.3.0" }
+metal = { package = "mtl-rs", version = "0.2.2" }
block2 = "0.6"
bytemuck = "1.23"
bytesize = "2"
diff --git a/apps/mirai-gui/desktop/Cargo.lock b/apps/mirai-gui/desktop/Cargo.lock
index a6849f8a7..a2481a32f 100644
--- a/apps/mirai-gui/desktop/Cargo.lock
+++ b/apps/mirai-gui/desktop/Cargo.lock
@@ -3565,9 +3565,9 @@ dependencies = [
[[package]]
name = "mtl-rs"
-version = "0.3.0"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4aa9291872296e9e734b0caf6e7a933fe6861d25ed86240461d6114355f7dbbf"
+checksum = "d9f1de2bca596db925db758d497cbf9cb394a2981933319b3777510097e1d360"
dependencies = [
"bitflags 2.13.0",
"block2",
diff --git a/crates/uzu-engine/src/backends/common/command_buffer.rs b/crates/uzu-engine/src/backends/common/command_buffer.rs
index 4d8168aa6..0377b6a70 100644
--- a/crates/uzu-engine/src/backends/common/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/common/command_buffer.rs
@@ -4,7 +4,7 @@ use bytemuck::{AnyBitPattern, NoUninit};
use crate::{
array::size_for_shape,
- backends::common::{Backend, BufferMut, BufferRef, TimestampSpan},
+ backends::common::{Backend, BufferMut, BufferRef},
data_type::DataType,
};
@@ -80,15 +80,6 @@ pub trait CommandBufferEncoding {
fn pop_debug_group(&mut self);
- fn enable_timestamps(&mut self);
-
- fn sample_start_timestamp(
- &mut self,
- name: &str,
- );
-
- fn sample_end_timestamp(&mut self);
-
fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;
}
@@ -113,6 +104,4 @@ pub trait CommandBufferCompleted: Send {
type CommandBuffer: CommandBuffer<Completed = Self>;
fn gpu_execution_time(&self) -> Duration;
-
- fn timestamps(&self) -> &[TimestampSpan];
}
diff --git a/crates/uzu-engine/src/backends/common/mod.rs b/crates/uzu-engine/src/backends/common/mod.rs
index 47663a079..a07c947d6 100644
--- a/crates/uzu-engine/src/backends/common/mod.rs
+++ b/crates/uzu-engine/src/backends/common/mod.rs
@@ -6,7 +6,6 @@ mod context;
mod device_capabilities;
pub mod gpu_types;
pub mod kernel;
-mod timestamps;
pub use backend::Backend;
pub use buffer::{
@@ -24,4 +23,3 @@ pub use command_buffer::{
pub use context::Context;
pub use device_capabilities::DeviceCapabilities;
pub use kernel::Kernels;
-pub use timestamps::{TimestampSpan, TimestampSpanRecorder};
diff --git a/crates/uzu-engine/src/backends/common/timestamps/mod.rs b/crates/uzu-engine/src/backends/common/timestamps/mod.rs
deleted file mode 100644
index 4d2eca4b4..000000000
--- a/crates/uzu-engine/src/backends/common/timestamps/mod.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-mod span_boundary;
-mod timestamp_span;
-mod timestamp_span_recorder;
-
-pub use timestamp_span::TimestampSpan;
-pub use timestamp_span_recorder::TimestampSpanRecorder;
diff --git a/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs b/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs
deleted file mode 100644
index a26a37153..000000000
--- a/crates/uzu-engine/src/backends/common/timestamps/span_boundary.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-pub enum SpanBoundary {
- Start(String),
- End,
-}
diff --git a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs
deleted file mode 100644
index f4315085f..000000000
--- a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-use std::time::Instant;
-
-#[derive(Debug, Clone)]
-pub struct TimestampSpan {
- pub name: String,
- pub start: Instant,
- pub end: Instant,
-}
diff --git a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs b/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs
deleted file mode 100644
index b1c6ce04f..000000000
--- a/crates/uzu-engine/src/backends/common/timestamps/timestamp_span_recorder.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-use std::time::Instant;
-
-use super::{TimestampSpan, span_boundary::SpanBoundary};
-
-#[derive(Default)]
-pub struct TimestampSpanRecorder {
- boundaries: Vec<SpanBoundary>,
-}
-
-impl TimestampSpanRecorder {
- pub fn start(
- &mut self,
- name: String,
- ) -> usize {
- self.push(SpanBoundary::Start(name))
- }
-
- pub fn end(&mut self) -> usize {
- self.push(SpanBoundary::End)
- }
-
- fn push(
- &mut self,
- boundary: SpanBoundary,
- ) -> usize {
- self.boundaries.push(boundary);
- self.boundaries.len() - 1
- }
-
- pub fn slot_count(&self) -> usize {
- self.boundaries.len()
- }
-
- pub fn spans(
- &self,
- instants: &[Instant],
- ) -> Box<[TimestampSpan]> {
- let mut spans = Vec::new();
- let mut open_span_indices = Vec::new();
- for (boundary, &instant) in self.boundaries.iter().zip(instants) {
- match boundary {
- SpanBoundary::Start(name) => {
- open_span_indices.push(spans.len());
- spans.push(TimestampSpan {
- name: name.clone(),
- start: instant,
- end: instant,
- });
- },
- SpanBoundary::End => {
- let index = open_span_indices.pop().expect("timestamp ended without a start");
- spans[index].end = instant;
- },
- }
- }
- assert!(open_span_indices.is_empty(), "timestamps were started but never ended");
- spans.into_boxed_slice()
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/command_buffer.rs b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
index 78722176e..62ba458aa 100644
--- a/crates/uzu-engine/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/cpu/command_buffer.rs
@@ -7,14 +7,12 @@ use crate::{
backends::{
common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,
- CommandBufferExecutable, CommandBufferPending, TimestampSpan, TimestampSpanRecorder,
- allocator::bump::BumpAllocator,
+ CommandBufferExecutable, CommandBufferPending, allocator::bump::BumpAllocator,
},
cpu::{
Cpu,
buffer::{CpuBufferExt, dense::CpuBuffer},
context::CpuContext,
- cpu_command::CpuCommand,
error::CpuError,
},
},
@@ -33,11 +31,10 @@ impl CommandBuffer for CpuCommandBuffer {
}
pub struct CpuCommandBufferEncoding {
- commands: Vec<CpuCommand>,
+ commands: Vec<Box<dyn FnOnce() + Send>>,
constant_allocator: BumpAllocator<<Cpu as Backend>::GlobalBuffer>,
allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,
context: Arc<CpuContext>,
- timestamp_spans: Option<TimestampSpanRecorder>,
}
impl CpuCommandBufferEncoding {
@@ -51,7 +48,6 @@ impl CpuCommandBufferEncoding {
constant_allocator,
allocation_pool,
context,
- timestamp_spans: None,
}
}
@@ -59,11 +55,7 @@ impl CpuCommandBufferEncoding {
&mut self,
command: impl FnOnce() + Send + 'static,
) {
- self.commands.push(CpuCommand::Run(Box::new(command)))
- }
-
- fn write_timestamp(&mut self) {
- self.commands.push(CpuCommand::Timestamp);
+ self.commands.push(Box::new(command))
}
}
@@ -126,28 +118,6 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
fn pop_debug_group(&mut self) {}
- fn enable_timestamps(&mut self) {
- assert!(self.timestamp_spans.is_none(), "timestamps already enabled");
- self.timestamp_spans = Some(TimestampSpanRecorder::default());
- }
-
- fn sample_start_timestamp(
- &mut self,
- name: &str,
- ) {
- if let Some(spans) = &mut self.timestamp_spans {
- spans.start(name.to_owned());
- self.write_timestamp();
- }
- }
-
- fn sample_end_timestamp(&mut self) {
- if let Some(spans) = &mut self.timestamp_spans {
- spans.end();
- self.write_timestamp();
- }
- }
-
fn end_encoding(self) -> CpuCommandBufferExecutable {
assert!(self.constant_allocator.is_done(), "attempted to end encoding while constants are still alive");
CpuCommandBufferExecutable {
@@ -155,17 +125,15 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
constant_allocator: self.constant_allocator,
allocation_pool: self.allocation_pool,
context: self.context,
- timestamp_spans: self.timestamp_spans,
}
}
}
pub struct CpuCommandBufferExecutable {
- commands: Vec<CpuCommand>,
+ commands: Vec<Box<dyn FnOnce() + Send>>,
constant_allocator: BumpAllocator<<Cpu as Backend>::GlobalBuffer>,
allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,
context: Arc<CpuContext>,
- timestamp_spans: Option<TimestampSpanRecorder>,
}
impl CommandBufferExecutable for CpuCommandBufferExecutable {
@@ -180,23 +148,14 @@ impl CommandBufferExecutable for CpuCommandBufferExecutable {
.send(Box::new(move || {
let start = Instant::now();
- let instants = self
- .commands
- .into_iter()
- .filter_map(|command| match command {
- CpuCommand::Run(run) => {
- run();
- None
- },
- CpuCommand::Timestamp => Some(Instant::now()),
- })
- .collect::<Box<[Instant]>>();
+ for command in self.commands {
+ command()
+ }
let gpu_execution_time = start.elapsed();
let completed = CpuCommandBufferCompleted {
gpu_execution_time,
- timestamps: self.timestamp_spans.map_or_else(Box::default, |spans| spans.spans(&instants)),
_allocation_pool: self.allocation_pool,
};
@@ -227,7 +186,6 @@ impl CommandBufferPending for CpuCommandBufferPending {
pub struct CpuCommandBufferCompleted {
gpu_execution_time: Duration,
- timestamps: Box<[TimestampSpan]>,
_allocation_pool: Arc<<Cpu as Backend>::AllocationPool>,
}
@@ -237,8 +195,4 @@ impl CommandBufferCompleted for CpuCommandBufferCompleted {
fn gpu_execution_time(&self) -> Duration {
self.gpu_execution_time
}
-
- fn timestamps(&self) -> &[TimestampSpan] {
- &self.timestamps
- }
}
diff --git a/crates/uzu-engine/src/backends/cpu/cpu_command.rs b/crates/uzu-engine/src/backends/cpu/cpu_command.rs
deleted file mode 100644
index 6dd851e4c..000000000
--- a/crates/uzu-engine/src/backends/cpu/cpu_command.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-pub enum CpuCommand {
- Run(Box<dyn FnOnce() + Send>),
- Timestamp,
-}
diff --git a/crates/uzu-engine/src/backends/cpu/mod.rs b/crates/uzu-engine/src/backends/cpu/mod.rs
index 7d6afffc5..f569a2da4 100644
--- a/crates/uzu-engine/src/backends/cpu/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/mod.rs
@@ -2,7 +2,6 @@ mod backend;
mod buffer;
mod command_buffer;
mod context;
-mod cpu_command;
mod error;
pub(crate) mod kernel; // TODO: This should not be pub!!!
diff --git a/crates/uzu-engine/src/backends/metal/command_buffer.rs b/crates/uzu-engine/src/backends/metal/command_buffer.rs
index a584b4cb0..985a6d0e5 100644
--- a/crates/uzu-engine/src/backends/metal/command_buffer.rs
+++ b/crates/uzu-engine/src/backends/metal/command_buffer.rs
@@ -16,11 +16,9 @@ use rangemap::RangeSet;
use crate::backends::{
common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding,
- CommandBufferExecutable, CommandBufferPending, Context, TimestampSpan, allocator::bump::BumpAllocator,
- },
- metal::{
- Metal, MetalContext, buffer::MetalBufferExt, error::MetalError, timestamp_recorder::MetalTimestampRecorder,
+ CommandBufferExecutable, CommandBufferPending, Context, allocator::bump::BumpAllocator,
},
+ metal::{Metal, MetalContext, buffer::MetalBufferExt, error::MetalError},
};
pub struct MetalCommandBuffer;
@@ -49,7 +47,6 @@ pub struct MetalCommandBufferEncoding {
constant_allocator: Option<BumpAllocator<<Metal as Backend>::GlobalBuffer>>,
allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
pub(super) context: Arc<MetalContext>,
- timestamps: Option<MetalTimestampRecorder>,
}
impl MetalCommandBufferEncoding {
@@ -104,7 +101,6 @@ impl MetalCommandBufferEncoding {
constant_allocator: Some(constant_allocator),
allocation_pool,
context,
- timestamps: None,
})
}
@@ -217,26 +213,6 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
self.compute_encoder.pop_debug_group();
}
- fn enable_timestamps(&mut self) {
- assert!(self.timestamps.is_none(), "timestamps already enabled");
- self.timestamps = Some(MetalTimestampRecorder::new(self.context.device.clone()));
- }
-
- fn sample_start_timestamp(
- &mut self,
- name: &str,
- ) {
- if let Some(timestamps) = &mut self.timestamps {
- timestamps.start(name.to_owned(), &self.compute_encoder);
- }
- }
-
- fn sample_end_timestamp(&mut self) {
- if let Some(timestamps) = &mut self.timestamps {
- timestamps.end(&self.compute_encoder);
- }
- }
-
fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {
let constant_allocator = self.constant_allocator.take().unwrap();
assert!(constant_allocator.is_done(), "attempted to end encoding while constants are still alive");
@@ -247,7 +223,6 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
constant_allocator,
allocation_pool: self.allocation_pool.clone(),
context: self.context.clone(),
- timestamps: self.timestamps.take(),
}
}
}
@@ -270,7 +245,6 @@ pub struct MetalCommandBufferExecutable {
constant_allocator: BumpAllocator<<Metal as Backend>::GlobalBuffer>,
allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
context: Arc<MetalContext>,
- timestamps: Option<MetalTimestampRecorder>,
}
impl CommandBufferExecutable for MetalCommandBufferExecutable {
@@ -292,18 +266,14 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {
let command_allocator = self.command_allocator.clone();
let command_buffer = self.command_buffer.clone();
let context_clone = self.context.clone();
- let timestamps = self.timestamps;
let constant_allocator = self.constant_allocator;
let allocation_pool = self.allocation_pool.clone();
let feedback_handler = move |feedback: &ProtocolObject<dyn MTL4CommitFeedback>| {
let message = if let Some(error) = feedback.error() {
- Err(MetalError::CommandBufferExecution(error.to_string()))
+ Err(error.to_string())
} else {
- timestamps
- .as_ref()
- .map_or_else(|| Ok(Box::default()), MetalTimestampRecorder::resolve)
- .map(|spans| (Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()), spans))
+ Ok(Duration::from_secs_f64(feedback.gpu_end_time() - feedback.gpu_start_time()))
};
let _ = sender.send(message);
context_clone.command_buffer_cache.lock().push(MetalCommandBufferCache {
@@ -326,17 +296,19 @@ impl CommandBufferExecutable for MetalCommandBufferExecutable {
pub struct MetalCommandBufferPending {
allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
- receiver: mpsc::Receiver<Result<(Duration, Box<[TimestampSpan]>), MetalError>>,
+ receiver: mpsc::Receiver<Result<Duration, String>>,
}
impl CommandBufferPending for MetalCommandBufferPending {
type CommandBuffer = MetalCommandBuffer;
fn wait_until_completed(self) -> Result<MetalCommandBufferCompleted, MetalError> {
- let (gpu_execution_time, timestamps) = self.receiver.recv().map_err(MetalError::CommandBufferWait)??;
Ok(MetalCommandBufferCompleted {
- gpu_execution_time,
- timestamps,
+ gpu_execution_time: self
+ .receiver
+ .recv()
+ .map_err(MetalError::CommandBufferWait)?
+ .map_err(MetalError::CommandBufferExecution)?,
_allocation_pool: self.allocation_pool,
})
}
@@ -344,7 +316,6 @@ impl CommandBufferPending for MetalCommandBufferPending {
pub struct MetalCommandBufferCompleted {
gpu_execution_time: Duration,
- timestamps: Box<[TimestampSpan]>,
_allocation_pool: Arc<<Metal as Backend>::AllocationPool>,
}
@@ -354,10 +325,6 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {
fn gpu_execution_time(&self) -> Duration {
self.gpu_execution_time
}
-
- fn timestamps(&self) -> &[TimestampSpan] {
- &self.timestamps
- }
}
pub struct MetalCommandBufferCache {
diff --git a/crates/uzu-engine/src/backends/metal/error.rs b/crates/uzu-engine/src/backends/metal/error.rs
index 83b5033e6..e51351964 100644
--- a/crates/uzu-engine/src/backends/metal/error.rs
+++ b/crates/uzu-engine/src/backends/metal/error.rs
@@ -27,10 +27,6 @@ pub enum MetalError {
CannotCreateBuffer,
#[error("Cannot create heap")]
CannotCreateHeap,
- #[error("Cannot resolve counter heap")]
- CannotResolveCounterHeap,
- #[error("Timestamp {0} was not written by the GPU")]
- UnwrittenTimestamp(usize),
#[error("Cannot create command buffer")]
CannotCreateCommandBuffer,
#[error("Cannot create argument table: {0}")]
diff --git a/crates/uzu-engine/src/backends/metal/mod.rs b/crates/uzu-engine/src/backends/metal/mod.rs
index 5fc8da6ee..01324c58f 100644
--- a/crates/uzu-engine/src/backends/metal/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/mod.rs
@@ -7,7 +7,6 @@ mod error;
mod heaps;
mod kernel;
mod metal_extensions;
-mod timestamp_recorder;
pub use backend::Metal;
pub use context::MetalContext; // TODO: This should be removed
diff --git a/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs b/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs
deleted file mode 100644
index 919245b51..000000000
--- a/crates/uzu-engine/src/backends/metal/timestamp_recorder.rs
+++ /dev/null
@@ -1,102 +0,0 @@
-use std::time::{Duration, Instant};
-
-use metal::{
- MTL4ComputeCommandEncoder, MTL4CounterHeap, MTL4CounterHeapDescriptor, MTL4CounterHeapExt, MTL4CounterHeapType,
- MTL4TimestampGranularity, MTLDevice, MTLDeviceExt,
-};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use crate::backends::{
- common::{TimestampSpan, TimestampSpanRecorder},
- metal::error::MetalError,
-};
-
-// "Maximum counter sample buffer length" is 32 KB, i.e. 4096 timestamp entries of 8 bytes (`MTL4TimestampHeapEntry`):
-// https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
-const COUNTER_HEAP_CAPACITY: usize = 4096;
-
-pub struct MetalTimestampRecorder {
- device: Retained<ProtocolObject<dyn MTLDevice>>,
- spans: TimestampSpanRecorder,
- counter_heaps: Vec<Retained<ProtocolObject<dyn MTL4CounterHeap>>>,
-}
-
-impl MetalTimestampRecorder {
- pub fn new(device: Retained<ProtocolObject<dyn MTLDevice>>) -> Self {
- Self {
- device,
- spans: TimestampSpanRecorder::default(),
- counter_heaps: Vec::new(),
- }
- }
-
- pub fn start(
- &mut self,
- name: String,
- encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,
- ) {
- let slot = self.spans.start(name);
- self.write(slot, encoder);
- }
-
- pub fn end(
- &mut self,
- encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,
- ) {
- let slot = self.spans.end();
- self.write(slot, encoder);
- }
-
- fn write(
- &mut self,
- slot: usize,
- encoder: &ProtocolObject<dyn MTL4ComputeCommandEncoder>,
- ) {
- if slot / COUNTER_HEAP_CAPACITY == self.counter_heaps.len() {
- let descriptor = MTL4CounterHeapDescriptor::new();
- descriptor.set_type(MTL4CounterHeapType::TIMESTAMP);
- descriptor.set_count(COUNTER_HEAP_CAPACITY);
- let counter_heap =
- self.device.new_counter_heap_with_descriptor(&descriptor).expect("Failed to create a counter heap");
- counter_heap.invalidate_counter_range(0..COUNTER_HEAP_CAPACITY);
- self.counter_heaps.push(counter_heap);
- }
- encoder.write_timestamp_with_granularity_into_heap_at_index(
- MTL4TimestampGranularity::PRECISE,
- &self.counter_heaps[slot / COUNTER_HEAP_CAPACITY],
- slot % COUNTER_HEAP_CAPACITY,
- );
- }
-
- pub fn resolve(&self) -> Result<Box<[TimestampSpan]>, MetalError> {
- let count = self.spans.slot_count();
- let mut ticks = Vec::with_capacity(count);
- for (index, counter_heap) in self.counter_heaps.iter().enumerate() {
- let slots = 0..(count - index * COUNTER_HEAP_CAPACITY).min(COUNTER_HEAP_CAPACITY);
- let bytes = counter_heap.resolve_counter_range(slots).ok_or(MetalError::CannotResolveCounterHeap)?;
- let (entries, _) = bytes.as_chunks::<{ size_of::<u64>() }>();
- ticks.extend(entries.iter().map(|entry| u64::from_ne_bytes(*entry)));
- }
- let frequency = self.device.query_timestamp_frequency() as u128;
- let (mut cpu_now, mut gpu_now) = (0, 0);
- let now = Instant::now();
- self.device.sample_timestamps_gpu_timestamp(&mut cpu_now, &mut gpu_now);
- let to_instant = |ticks: u64| {
- let nanoseconds = (ticks as u128 * 1_000_000_000 / frequency) as u64;
- now - Duration::from_nanos(gpu_now - nanoseconds)
- };
- let instants = ticks
- .iter()
- .enumerate()
- .scan(None, |previous_timestamp, (slot, &slot_ticks)| {
- let timestamp = match slot_ticks {
- 0 => previous_timestamp.ok_or(MetalError::UnwrittenTimestamp(slot)),
- slot_ticks => Ok(to_instant(slot_ticks)),
- };
- *previous_timestamp = timestamp.as_ref().ok().copied();
- Some(timestamp)
- })
- .collect::<Result<Box<[Instant]>, MetalError>>()?;
- Ok(self.spans.spans(&instants))
- }
-}
diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rs
index d343d583e..f44077a95 100644
--- a/crates/uzu-engine/src/encodable_block/classifier.rs
+++ b/crates/uzu-engine/src/encodable_block/classifier.rs
@@ -35,7 +35,6 @@ pub enum ClassifierError<B: Backend> {
}
pub struct Classifier<B: Backend> {
- name: String,
hidden_dim: u32,
data_type: DataType,
embedding: Embedding<B>,
@@ -47,14 +46,12 @@ pub struct Classifier<B: Backend> {
impl<B: Backend> Classifier<B> {
pub fn new(
- name: String,
context: &B::Context,
config: &ClassifierConfig,
parameter_tree: &ParameterTree<B>,
data_type: DataType,
) -> Result<Self, ClassifierError<B>> {
let (embedding, _) = Embedding::new(
- format!("{name}/embedding"),
context,
config.vocab_size,
config.transformer_config.model_dim,
@@ -64,7 +61,6 @@ impl<B: Backend> Classifier<B> {
)?;
let embedding_norm = Normalization::new(
- format!("{name}/embedding norm"),
config.transformer_config.model_dim,
None,
ShortcutMode::None,
@@ -76,7 +72,6 @@ impl<B: Backend> Classifier<B> {
)?;
let transformer = Transformer::new(
- format!("{name}/transformer"),
context,
None,
data_type,
@@ -94,7 +89,6 @@ impl<B: Backend> Classifier<B> {
<B::Kernels as Kernels>::PoolingMeanKernel::new(context, data_type).map_err(ClassifierError::Backend)?;
let prediction_head = PredictionHead::new(
- format!("{name}/prediction head"),
config.hidden_dim,
config.num_labels,
data_type,
@@ -104,7 +98,6 @@ impl<B: Backend> Classifier<B> {
)?;
Ok(Self {
- name,
hidden_dim: config.hidden_dim,
data_type,
embedding,
@@ -125,8 +118,7 @@ impl<B: Backend> Classifier<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, ClassifierError<B>> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("classifier");
let embedded = self.embedding.encode_lookup(token_ids, batch_dim, command_buffer)?;
@@ -164,7 +156,6 @@ impl<B: Backend> Classifier<B> {
let logits = self.prediction_head.encode(pooled, 1, command_buffer).map_err(ClassifierError::Backend)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(logits)
diff --git a/crates/uzu-engine/src/encodable_block/context_ring_update.rs b/crates/uzu-engine/src/encodable_block/context_ring_update.rs
deleted file mode 100644
index 1573b04b7..000000000
--- a/crates/uzu-engine/src/encodable_block/context_ring_update.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-use crate::backends::common::{
- Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels, kernel::ContextRingUpdateKernel,
-};
-
-pub struct ContextRingUpdate<B: Backend> {
- name: String,
- kernel: <B::Kernels as Kernels>::ContextRingUpdateKernel,
-}
-
-impl<B: Backend> ContextRingUpdate<B> {
- pub fn new(
- name: String,
- context: &B::Context,
- ) -> Result<Self, B::Error> {
- Ok(Self {
- name,
- kernel: <B::Kernels as Kernels>::ContextRingUpdateKernel::new(context)?,
- })
- }
-
- pub fn encode(
- &self,
- input: impl BufferRef<Backend = B>,
- context_ring: impl BufferMut<Backend = B>,
- suffix_repetition_length: u32,
- input_length: u32,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
- self.kernel.encode(input, context_ring, suffix_repetition_length, input_length, command_buffer);
- command_buffer.sample_end_timestamp();
- command_buffer.pop_debug_group();
- }
-}
diff --git a/crates/uzu-engine/src/encodable_block/convolution.rs b/crates/uzu-engine/src/encodable_block/convolution.rs
index 1f9afb10e..c6c852b6a 100644
--- a/crates/uzu-engine/src/encodable_block/convolution.rs
+++ b/crates/uzu-engine/src/encodable_block/convolution.rs
@@ -21,7 +21,6 @@ pub enum ConvolutionNewError<B: Backend> {
}
pub struct SeparableCausalConv<B: Backend> {
- name: String,
model_dim: u32,
data_type: DataType,
weights: B::GlobalBuffer,
@@ -31,7 +30,6 @@ pub struct SeparableCausalConv<B: Backend> {
impl<B: Backend> SeparableCausalConv<B> {
pub fn new(
- name: String,
model_dim: u32,
kernel_size: u32,
group_size: u32,
@@ -64,7 +62,6 @@ impl<B: Backend> SeparableCausalConv<B> {
.map_err(ConvolutionNewError::Backend)?;
Ok(Self {
- name,
model_dim,
data_type,
weights,
@@ -82,8 +79,7 @@ impl<B: Backend> SeparableCausalConv<B> {
sequence_length: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("SeparableCausalConv");
let mut output =
command_buffer.allocate_scratch_for_shape(&[sequence_length, self.model_dim], self.data_type)?;
@@ -99,7 +95,6 @@ impl<B: Backend> SeparableCausalConv<B> {
command_buffer,
);
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
}
diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rs
index 5476b386a..7b1c8bbfd 100644
--- a/crates/uzu-engine/src/encodable_block/decoder.rs
+++ b/crates/uzu-engine/src/encodable_block/decoder.rs
@@ -32,7 +32,6 @@ pub enum DecoderError<B: Backend> {
}
pub struct Decoder<B: Backend> {
- name: String,
embedding: Embedding<B>,
embedding_norm: Option<Normalization<B>>,
per_layer_embedding: Option<PerLayerEmbedding<B>>,
@@ -51,14 +50,12 @@ impl<B: Backend> Decoder<B> {
}
pub fn new(
- name: String,
context: &B::Context,
config: &DecoderConfig,
parameter_tree: &ParameterTree<B>,
data_type: DataType,
) -> Result<Self, DecoderError<B>> {
let (embedding, readout_input_hadamard_factors) = Embedding::new(
- format!("{name}/embedding"),
context,
config.vocab_size,
config.transformer_config.model_dim,
@@ -72,7 +69,6 @@ impl<B: Backend> Decoder<B> {
.as_ref()
.map(|norm_config| {
Normalization::new(
- format!("{name}/embedding norm"),
config.transformer_config.model_dim,
None,
ShortcutMode::None,
@@ -92,7 +88,6 @@ impl<B: Backend> Decoder<B> {
"per-layer embedding num_layers must match transformer layer count"
);
Some(PerLayerEmbedding::new(
- format!("{name}/per layer embedding"),
context,
ple_config,
config.transformer_config.model_dim,
@@ -104,7 +99,6 @@ impl<B: Backend> Decoder<B> {
};
let transformer = Transformer::new(
- format!("{name}/transformer"),
context,
readout_input_hadamard_factors,
data_type,
@@ -113,7 +107,6 @@ impl<B: Backend> Decoder<B> {
)?;
Ok(Self {
- name,
embedding,
embedding_norm,
per_layer_embedding,
@@ -150,8 +143,7 @@ impl<B: Backend> Decoder<B> {
state: &mut TransformerState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("decoder");
let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), command_buffer)?;
let embedded = if let Some(embedding_norm) = &self.embedding_norm {
@@ -204,7 +196,6 @@ impl<B: Backend> Decoder<B> {
transformer_output.output
};
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(DecoderEncodeOutput {
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index 4c90073fa..b35383899 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -41,8 +41,6 @@ impl<B: Backend> DFlashState<B> {
}
pub struct DFlash<B: Backend> {
- accept_name: String,
- draft_name: String,
target_feature_projection: Box<dyn Linear<B>>,
projected_feature_norm: Normalization<B>,
state_kv_projection: Box<dyn Linear<B>>,
@@ -91,7 +89,6 @@ pub enum DFlashEncodeError<B: Backend> {
impl<B: Backend> DFlash<B> {
pub fn new(
- name: String,
context: &B::Context,
config: &DFlashDraftConfig,
parameter_tree: &ParameterTree<B>,
@@ -100,7 +97,6 @@ impl<B: Backend> DFlash<B> {
let mask_token_id = config.mask_token_id as u32;
assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");
let target_feature_projection = <dyn Linear<B>>::new(
- format!("{name}/target feature projection"),
config.model_dim * config.target_layer_ids.len() as u32,
[config.model_dim],
false,
@@ -109,7 +105,6 @@ impl<B: Backend> DFlash<B> {
&parameter_tree.subtree("context_projection"),
)?;
let projected_feature_norm = Normalization::new(
- format!("{name}/projected feature norm"),
config.model_dim,
None,
ShortcutMode::None,
@@ -127,7 +122,6 @@ impl<B: Backend> DFlash<B> {
let layer_kv_dim = 2 * attention_config.num_groups * attention_config.head_dim;
let num_layers = config.layer_configs.len() as u32;
let state_kv_projection = <dyn Linear<B>>::new(
- format!("{name}/state kv projection"),
config.model_dim,
[num_layers * layer_kv_dim],
false,
@@ -141,7 +135,6 @@ impl<B: Backend> DFlash<B> {
.zip(0u32..)
.map(|(layer_config, index)| {
TransformerLayer::new(
- format!("{name}/layer {index}"),
context,
config.model_dim,
config.hidden_dim,
@@ -154,7 +147,6 @@ impl<B: Backend> DFlash<B> {
})
.collect::<Result<Box<[_]>, TransformerLayerError<B>>>()?;
let output_norm = Normalization::new(
- format!("{name}/output norm"),
config.model_dim,
None,
ShortcutMode::Add,
@@ -166,8 +158,6 @@ impl<B: Backend> DFlash<B> {
)?;
Ok(Self {
- accept_name: format!("{name}/accept"),
- draft_name: format!("{name}/draft"),
target_feature_projection,
projected_feature_norm,
state_kv_projection,
@@ -217,8 +207,7 @@ impl<B: Backend> DFlash<B> {
return Ok(());
}
- command_buffer.push_debug_group(&self.accept_name);
- command_buffer.sample_start_timestamp(&self.accept_name);
+ command_buffer.push_debug_group("dflash accept");
let num_tokens = accepted_indices.len() as u32;
let captured_layer_count = self.target_feature_input_dim / self.model_dim;
@@ -285,7 +274,6 @@ impl<B: Backend> DFlash<B> {
state.context_length += num_tokens;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(())
@@ -299,8 +287,7 @@ impl<B: Backend> DFlash<B> {
batch_size: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<DFlashOutput<B>, DFlashEncodeError<B>> {
- command_buffer.push_debug_group(&self.draft_name);
- command_buffer.sample_start_timestamp(&self.draft_name);
+ command_buffer.push_debug_group("dflash draft");
assert!(batch_size >= 2 && batch_size <= self.block_size, "batch size exceeds DFlash block size");
assert!(
@@ -361,7 +348,6 @@ impl<B: Backend> DFlash<B> {
command_buffer,
)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(DFlashOutput {
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index 8f386db84..fd218c54a 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -45,7 +45,6 @@ enum EmbeddingTying<B: Backend> {
}
pub struct Embedding<B: Backend> {
- readout_name: String,
tying: EmbeddingTying<B>,
input_scale: f32,
data_type: DataType,
@@ -70,7 +69,6 @@ impl<B: Backend> Embedding<B> {
}
pub fn new(
- name: String,
context: &B::Context,
vocab_size: u32,
model_dim: u32,
@@ -81,14 +79,7 @@ impl<B: Backend> Embedding<B> {
let (tying, readout_input_hadamard_factors) = match config {
AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {
let embedding_tree = parameter_tree.subtree("embedding");
- let table = EmbeddingTable::load(
- format!("{name}/lookup"),
- context,
- &embedding_tree,
- vocab_size,
- model_dim,
- data_type,
- )?;
+ let table = EmbeddingTable::load(context, &embedding_tree, vocab_size, model_dim, data_type)?;
if table.as_matrix().is_none() {
return Err(EmbeddingError::UnsupportedConfiguration("tied embeddings need a matrix table".into()));
}
@@ -109,18 +100,11 @@ impl<B: Backend> Embedding<B> {
},
AnyEmbeddingConfig::UntiedEmbeddingConfig(_) => {
let input_embedding_tree = parameter_tree.subtree("input_embedding");
- let input_table = EmbeddingTable::load(
- format!("{name}/lookup"),
- context,
- &input_embedding_tree,
- vocab_size,
- model_dim,
- data_type,
- )?;
+ let input_table =
+ EmbeddingTable::load(context, &input_embedding_tree, vocab_size, model_dim, data_type)?;
let output_embedding_tree = parameter_tree.subtree("output_embedding");
let output_embedding_spec = output_embedding_tree.metadata("spec")?;
let output = UntiedReadout::load(
- format!("{name}/readout/untied"),
context,
&output_embedding_tree,
output_embedding_spec,
@@ -156,7 +140,6 @@ impl<B: Backend> Embedding<B> {
Ok((
Self {
- readout_name: format!("{name}/readout"),
tying,
input_scale,
data_type,
@@ -174,6 +157,8 @@ impl<B: Backend> Embedding<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, EmbeddingError<B>> {
+ command_buffer.push_debug_group("embedding lookup");
+
let mut output = command_buffer
.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)
.map_err(EmbeddingError::BackendError)?;
@@ -190,6 +175,8 @@ impl<B: Backend> Embedding<B> {
};
table.encode_lookup(token_ids, &mut output, batch_dim, self.input_scale, command_buffer);
+ command_buffer.pop_debug_group();
+
Ok(output)
}
@@ -202,8 +189,7 @@ impl<B: Backend> Embedding<B> {
apply_logit_transform: bool,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, EmbeddingError<B>> {
- command_buffer.push_debug_group(&self.readout_name);
- command_buffer.sample_start_timestamp(&self.readout_name);
+ command_buffer.push_debug_group("embedding readout");
assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");
let mut output_buffer = match &self.tying {
@@ -254,7 +240,6 @@ impl<B: Backend> Embedding<B> {
);
}
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output_buffer)
diff --git a/crates/uzu-engine/src/encodable_block/embedding_table.rs b/crates/uzu-engine/src/encodable_block/embedding_table.rs
index dee8a1974..7dae6806a 100644
--- a/crates/uzu-engine/src/encodable_block/embedding_table.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding_table.rs
@@ -3,7 +3,7 @@ use thiserror::Error;
use crate::{
backends::common::{
- Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,
+ Backend, BufferMut, BufferRef, CommandBuffer, Kernels,
gpu_types::{EmbeddingTableKind, HADAMARD_TRANSFORM_BLOCK_SIZE, d4s4},
kernel::InputEmbeddingLookupKernel,
},
@@ -81,7 +81,6 @@ impl<B: Backend> Storage<B> {
}
pub struct EmbeddingTable<B: Backend> {
- name: String,
storage: Storage<B>,
output_hadamard_factors: Option<B::GlobalBuffer>,
lookup: LookupKernel<B>,
@@ -91,7 +90,6 @@ pub struct EmbeddingTable<B: Backend> {
impl<B: Backend> EmbeddingTable<B> {
pub fn load(
- name: String,
context: &B::Context,
tree: &ParameterTree<B>,
vocab_size: u32,
@@ -152,7 +150,6 @@ impl<B: Backend> EmbeddingTable<B> {
.map_err(EmbeddingTableError::BackendError)?;
Ok(Self {
- name,
storage,
output_hadamard_factors,
lookup,
@@ -178,8 +175,6 @@ impl<B: Backend> EmbeddingTable<B> {
scale: f32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
let bindings = self.storage.lookup_bindings();
self.lookup.encode(
token_ids,
@@ -198,8 +193,6 @@ impl<B: Backend> EmbeddingTable<B> {
scale,
command_buffer,
);
- command_buffer.sample_end_timestamp();
- command_buffer.pop_debug_group();
}
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
index 4db7675fe..2c82b4d93 100644
--- a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
@@ -10,7 +10,6 @@ use crate::{
};
pub(super) struct InputRht<B: Backend> {
- name: String,
rht_signs: B::GlobalBuffer,
rht: ActivationTransform<B>,
quantizer: Option<ActivationTransform<B>>,
@@ -18,7 +17,6 @@ pub(super) struct InputRht<B: Backend> {
impl<B: Backend> InputRht<B> {
pub(super) fn new(
- name: String,
context: &B::Context,
data_type: DataType,
preparation: LinearInputPreparation<B>,
@@ -34,7 +32,6 @@ impl<B: Backend> InputRht<B> {
.transpose()?;
Ok(Self {
- name,
rht_signs,
rht,
quantizer,
@@ -48,7 +45,6 @@ impl<B: Backend> InputRht<B> {
format: ActivationFormat,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<LinearInput<B>, B::Error> {
- command_buffer.sample_start_timestamp(&self.name);
if format == ActivationFormat::Int8
&& let Some(quantizer) = &self.quantizer
{
@@ -74,7 +70,6 @@ impl<B: Backend> InputRht<B> {
command_buffer,
);
- command_buffer.sample_end_timestamp();
return Ok(LinearInput::Int8Symmetric {
values,
scales,
@@ -87,7 +82,6 @@ impl<B: Backend> InputRht<B> {
let input_dim = self.input_dim();
let mut transformed = command_buffer.allocate_scratch(input.size())?;
self.rht.encode_fp(input, &mut transformed, &self.rht_signs, batch_dim, input_dim, command_buffer);
- command_buffer.sample_end_timestamp();
Ok(LinearInput::FullPrecision(transformed))
}
@@ -102,10 +96,8 @@ impl<B: Backend> InputRht<B> {
return self.prepare(&input, batch_dim, format, command_buffer);
}
- command_buffer.sample_start_timestamp(&self.name);
let input_dim = self.input_dim();
self.rht.encode_fp_in_place(&mut input, &self.rht_signs, None, batch_dim, input_dim, command_buffer);
- command_buffer.sample_end_timestamp();
Ok(LinearInput::FullPrecision(input))
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
index da0ff8c77..4f7f6a9ad 100644
--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
@@ -37,7 +37,6 @@ pub enum LinearMatmulError<B: Backend> {
}
pub struct LinearMatmul<B: Backend> {
- name: String,
kernel: Mutex<<B::Kernels as Kernels>::MatmulKernel>,
matrix: WeightMatrix<B>,
biases: Option<B::GlobalBuffer>,
@@ -67,7 +66,6 @@ impl<B: Backend> LinearMatmul<B> {
/// Loads a linear over any parsed spec — full precision, MLX or Int. Hybrid
/// compositions live in the wrappers, not here.
pub fn load(
- name: String,
context: &B::Context,
spec: AnyWeightMatrixSpec,
input_dim: u32,
@@ -100,7 +98,6 @@ impl<B: Backend> LinearMatmul<B> {
.map_err(LinearMatmulError::BackendError)?;
Ok(Self {
- name,
kernel: Mutex::new(kernel),
matrix,
biases,
@@ -128,7 +125,6 @@ impl<B: Backend> LinearMatmul<B> {
gather: Option<Gather<impl BufferRef<Backend = B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.name);
let (output_dim, gather_indices) =
gather.map_or((self.output_dim, None), |gather| (gather.output_dim, Some(gather.indices)));
let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, output_dim], self.output_data_type)?;
@@ -147,7 +143,6 @@ impl<B: Backend> LinearMatmul<B> {
command_buffer,
)?;
- command_buffer.sample_end_timestamp();
Ok(output)
}
@@ -195,7 +190,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
+ command_buffer.push_debug_group("matmul");
let output = self.encode_with_a(
MatmulA::FullPrecision {
diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rs
index 9c49aae44..78dfdfb93 100644
--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs
@@ -123,7 +123,6 @@ pub enum LinearBlockError<B: Backend> {
impl<B: Backend> dyn Linear<B> {
pub fn new_mixed_precision(
- name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -141,7 +140,6 @@ impl<B: Backend> dyn Linear<B> {
| AnyWeightMatrixSpec::MLXSpec(_)
| AnyWeightMatrixSpec::IntSpec(_)) => {
let block = LinearMatmul::load(
- name,
context,
spec,
input_dimension,
@@ -161,7 +159,6 @@ impl<B: Backend> dyn Linear<B> {
incoherence_processing_mode: IncoherenceProcessingMode::InputOutput,
..
}) if block_size == HADAMARD_TRANSFORM_BLOCK_SIZE => Ok(Box::new(RHTLinearWrapper::new(
- name,
context,
input_dimension,
output_dimension_sum,
@@ -184,7 +181,6 @@ impl<B: Backend> dyn Linear<B> {
return Err(LinearBlockError::UnsupportedConfiguration(format!("{adapter_spec:?}")));
};
Ok(Box::new(QLoRALinearWrapper::new(
- name,
context,
*quantization_spec,
adapter_spec,
@@ -203,7 +199,6 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new(
- name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -212,7 +207,6 @@ impl<B: Backend> dyn Linear<B> {
parameter_tree: &ParameterTree<B>,
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
Self::new_mixed_precision(
- name,
input_dimension,
output_dimensions,
has_biases,
@@ -225,7 +219,6 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new_with_input_rht_mixed_precision(
- name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -237,7 +230,6 @@ impl<B: Backend> dyn Linear<B> {
) -> Result<(Box<dyn Linear<B>>, Option<B::GlobalBuffer>), LinearBlockError<B>> {
let output_dimension_sum: u32 = output_dimensions.as_ref().iter().sum();
if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(
- name.clone(),
context,
input_dimension,
output_dimension_sum,
@@ -252,7 +244,6 @@ impl<B: Backend> dyn Linear<B> {
}
let linear = Self::new_mixed_precision(
- name,
input_dimension,
output_dimensions,
has_biases,
@@ -266,7 +257,6 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new_for_fused_input(
- name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -276,7 +266,6 @@ impl<B: Backend> dyn Linear<B> {
) -> Result<(Box<dyn Linear<B>>, Option<LinearInputPreparation<B>>), LinearBlockError<B>> {
let output_dimension_sum: u32 = output_dimensions.as_ref().iter().sum();
if let Some(linear) = RHTLinearWrapper::try_new_with_input_preparation(
- name.clone(),
context,
input_dimension,
output_dimension_sum,
@@ -291,7 +280,6 @@ impl<B: Backend> dyn Linear<B> {
}
let linear = Self::new_mixed_precision(
- name,
input_dimension,
output_dimensions,
has_biases,
@@ -305,7 +293,6 @@ impl<B: Backend> dyn Linear<B> {
}
pub fn new_with_input_rht(
- name: String,
input_dimension: u32,
output_dimensions: impl AsRef<[u32]>,
has_biases: bool,
@@ -314,7 +301,6 @@ impl<B: Backend> dyn Linear<B> {
parameter_tree: &ParameterTree<B>,
) -> Result<(Box<dyn Linear<B>>, Option<B::GlobalBuffer>), LinearBlockError<B>> {
Self::new_with_input_rht_mixed_precision(
- name,
input_dimension,
output_dimensions,
has_biases,
diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
index dfdf74851..d4faa24b9 100644
--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
@@ -35,7 +35,6 @@ pub enum QLoRALinearWrapperError<B: Backend> {
}
pub struct QLoRALinearWrapper<B: Backend> {
- name: String,
base_linear: LinearMatmul<B>,
input_hadamard: Option<(ActivationTransform<B>, B::GlobalBuffer)>,
output_hadamard: Option<(ActivationTransform<B>, B::GlobalBuffer)>,
@@ -52,7 +51,6 @@ pub struct QLoRALinearWrapper<B: Backend> {
impl<B: Backend> QLoRALinearWrapper<B> {
pub fn new(
- name: String,
context: &B::Context,
quantization_spec: AnyWeightMatrixSpec,
adapter_spec: LowRankSpec,
@@ -87,7 +85,6 @@ impl<B: Backend> QLoRALinearWrapper<B> {
));
};
let base_linear = LinearMatmul::load(
- format!("{name}/base"),
context,
quantization_spec,
input_dim,
@@ -155,7 +152,6 @@ impl<B: Backend> QLoRALinearWrapper<B> {
.read_buffer()?;
Ok(Self {
- name,
base_linear,
input_hadamard,
output_hadamard,
@@ -179,8 +175,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("linear (qlora)");
let mut intermediate =
command_buffer.allocate_scratch_for_shape(&[batch_dim, self.lora_rank], self.weights_data_type)?;
@@ -266,7 +261,6 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
);
}
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index bcfaebc11..c4dbbe8d8 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -27,7 +27,6 @@ pub enum RHTLinearWrapperError<B: Backend> {
}
pub struct RHTLinearWrapper<B: Backend> {
- name: String,
input_rht: InputRht<B>,
inner_linear: LinearMatmul<B>,
}
@@ -46,7 +45,6 @@ fn has_input_output_rht(spec: &AnyWeightMatrixSpec) -> bool {
impl<B: Backend> RHTLinearWrapper<B> {
pub(super) fn new(
- name: String,
context: &B::Context,
input_dimension: u32,
output_dimension: u32,
@@ -63,7 +61,6 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(
- format!("{name}/matmul"),
context,
input_dimension,
output_dimension,
@@ -75,7 +72,6 @@ impl<B: Backend> RHTLinearWrapper<B> {
)?;
let activation_quantization = inner_linear.prepare_a8(context);
Self::build_self_contained(
- name,
context,
input_data_type,
LinearInputPreparation {
@@ -87,7 +83,6 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
pub(super) fn try_new_with_input_preparation(
- name: String,
context: &B::Context,
input_dimension: u32,
output_dimension: u32,
@@ -105,7 +100,6 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(
- format!("{name}/matmul"),
context,
input_dimension,
output_dimension,
@@ -120,7 +114,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
activation_quantization: inner_linear.prepare_a8(context),
};
if input_preparation.activation_quantization.is_some() && !allow_prequantized_activation {
- let wrapper = Self::build_self_contained(name, context, input_data_type, input_preparation, inner_linear)?;
+ let wrapper = Self::build_self_contained(context, input_data_type, input_preparation, inner_linear)?;
Ok(Some((Box::new(wrapper), None)))
} else {
Ok(Some((Box::new(inner_linear), Some(input_preparation))))
@@ -128,7 +122,6 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
fn load_inner_with_output_rht(
- name: String,
context: &B::Context,
input_dimension: u32,
output_dimension: u32,
@@ -150,7 +143,6 @@ impl<B: Backend> RHTLinearWrapper<B> {
.validate(&[output_dimension], DataType::I32)?
.read_buffer()?;
let inner_linear = LinearMatmul::load(
- name,
context,
quantization_spec,
input_dimension,
@@ -166,17 +158,15 @@ impl<B: Backend> RHTLinearWrapper<B> {
}
fn build_self_contained(
- name: String,
context: &B::Context,
input_data_type: DataType,
input_preparation: LinearInputPreparation<B>,
inner_linear: LinearMatmul<B>,
) -> Result<Self, RHTLinearWrapperError<B>> {
- let input_rht = InputRht::new(format!("{name}/prepare"), context, input_data_type, input_preparation, true)
+ let input_rht = InputRht::new(context, input_data_type, input_preparation, true)
.map_err(RHTLinearWrapperError::BackendError)?;
Ok(Self {
- name,
input_rht,
inner_linear,
})
@@ -199,14 +189,12 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("linear (rht)");
let input = match input {
LinearInput::FullPrecision(input) => input,
input => {
let output = self.inner_linear.encode_input(input, batch_dim, command_buffer);
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
return output;
},
@@ -220,7 +208,6 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
command_buffer,
)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
index 65fb42175..a993ff14e 100644
--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
@@ -16,14 +16,12 @@ use crate::{
};
pub struct UntiedReadout<B: Backend> {
- name: String,
linear: LinearMatmul<B>,
input_rht: Option<InputRht<B>>,
}
impl<B: Backend> UntiedReadout<B> {
pub fn load(
- name: String,
context: &B::Context,
tree: &ParameterTree<B>,
spec: AnyWeightMatrixSpec,
@@ -41,25 +39,13 @@ impl<B: Backend> UntiedReadout<B> {
else {
return Ok(Self {
linear: LinearMatmul::load(
- format!("{name}/matmul"),
- context,
- spec,
- model_dim,
- vocab_size,
- data_type,
- data_type,
- data_type,
- tree,
- None,
- None,
+ context, spec, model_dim, vocab_size, data_type, data_type, data_type, tree, None, None,
)?,
input_rht: None,
- name,
});
};
let mut linear = LinearMatmul::load(
- format!("{name}/matmul"),
context,
*quantization_spec,
model_dim,
@@ -80,12 +66,10 @@ impl<B: Backend> UntiedReadout<B> {
rht_signs,
activation_quantization: linear.prepare_a8(context),
};
- let input_rht =
- InputRht::new(format!("{name}/prepare"), context, data_type, preparation, /* in_place */ false)
- .map_err(LinearMatmulError::BackendError)?;
+ let input_rht = InputRht::new(context, data_type, preparation, /* in_place */ false)
+ .map_err(LinearMatmulError::BackendError)?;
Ok(Self {
- name,
linear,
input_rht: Some(input_rht),
})
@@ -98,15 +82,12 @@ impl<B: Backend> UntiedReadout<B> {
gather: Option<Gather<impl BufferRef<Backend = B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.name);
let Some(input_rht) = &self.input_rht else {
let a = MatmulA::FullPrecision {
values: input,
offset: 0,
};
- let output = self.linear.encode_with_a(a, batch_dim, gather, command_buffer)?;
- command_buffer.sample_end_timestamp();
- return Ok(output);
+ return self.linear.encode_with_a(a, batch_dim, gather, command_buffer);
};
let format = if gather.is_some() {
@@ -115,8 +96,6 @@ impl<B: Backend> UntiedReadout<B> {
self.linear.select_activation_format(batch_dim, command_buffer.context())
};
let input = input_rht.prepare(input, batch_dim, format, command_buffer)?;
- let output = self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)?;
- command_buffer.sample_end_timestamp();
- Ok(output)
+ self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
index 9b540abef..9c2b21056 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
@@ -33,11 +33,6 @@ pub use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, KVCacheView};
pub mod rope;
pub struct Attention<B: Backend> {
- name: String,
- core_name: String,
- prepare_kv_and_queries_name: String,
- prepare_queries_name: String,
- append_projected_kv_name: String,
head_dim: u32,
num_q_heads: u32,
num_kv_heads: Option<u32>,
@@ -67,7 +62,6 @@ pub enum AttentionNewError<B: Backend> {
impl<B: Backend> Attention<B> {
pub fn new(
- name: String,
hidden_dim: u32,
data_type: DataType,
rope_config: Option<&AnyRoPEConfig>,
@@ -101,7 +95,6 @@ impl<B: Backend> Attention<B> {
qkv_dim
};
let (projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(
- format!("{name}/qkv projection"),
hidden_dim,
[projection_dim],
config.has_qkvg_biases,
@@ -117,7 +110,6 @@ impl<B: Backend> Attention<B> {
let qkv_norm = (query_norm_config.is_some() || key_norm_config.is_some() || value_norm_config.is_some())
.then(|| {
QKVNorm::new(
- format!("{name}/qkv norm"),
context,
data_type,
query_norm_config,
@@ -172,7 +164,6 @@ impl<B: Backend> Attention<B> {
.map_err(AttentionNewError::Backend)?;
let out_projection = <dyn Linear<B>>::new(
- format!("{name}/out projection"),
q_dim,
[hidden_dim],
config.has_out_biases,
@@ -183,11 +174,6 @@ impl<B: Backend> Attention<B> {
Ok((
Self {
- core_name: format!("{name}/core"),
- prepare_kv_and_queries_name: format!("{name}/prepare kv and queries"),
- prepare_queries_name: format!("{name}/prepare queries"),
- append_projected_kv_name: format!("{name}/append projected kv"),
- name,
head_dim,
num_q_heads,
num_kv_heads,
@@ -235,8 +221,7 @@ impl<B: Backend> Mixer<B> for Attention<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("attention");
assert_eq!(precalculated_rope.is_some(), self.max_rope_length.is_some(), "precalculated rope mismatch");
@@ -244,7 +229,6 @@ impl<B: Backend> Mixer<B> for Attention<B> {
state.map(|state| state.downcast::<AttentionState<B>>().expect("incorrect type of attention state"));
let output = self.attend(hidden, precalculated_rope, batch_dim, state, command_buffer)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
index f1d6b7a9d..75b337ac1 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
@@ -127,7 +127,6 @@ impl<B: Backend> Attention<B> {
state: &mut AttentionState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.sample_start_timestamp(&self.append_projected_kv_name);
if let Some(norm) = &self.projection.norm {
norm.encode_key_value(key_value.reborrow(), batch_dim, command_buffer)?;
}
@@ -143,7 +142,6 @@ impl<B: Backend> Attention<B> {
command_buffer,
)?;
state.encode_accept(&(0..batch_dim).collect::<Box<[u32]>>(), command_buffer)?;
- command_buffer.sample_end_timestamp();
Ok(())
}
@@ -154,14 +152,13 @@ impl<B: Backend> Attention<B> {
state: &AttentionState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.core_name);
let trie = if batch_dim.is_flat() {
None
} else {
Some(command_buffer.allocate_constant_from_slice(batch_dim.nodes())?)
};
- let output = self.kernel.encode(
+ self.kernel.encode(
AttentionArguments {
queries,
keys: state.keys.as_ref(),
@@ -172,9 +169,7 @@ impl<B: Backend> Attention<B> {
cache: state.view(),
},
command_buffer,
- )?;
- command_buffer.sample_end_timestamp();
- Ok(output)
+ )
}
fn prepare_kv_and_queries(
@@ -188,7 +183,6 @@ impl<B: Backend> Attention<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.prepare_kv_and_queries_name);
let num_kv_heads = self.num_kv_heads.expect("KV prepare requires KV heads");
// Appended KV is tightly packed; attention projections may have a trailing gate segment.
let input_row_stride = if num_q_heads == 0 {
@@ -217,7 +211,6 @@ impl<B: Backend> Attention<B> {
batch_dim,
command_buffer,
);
- command_buffer.sample_end_timestamp();
Ok(queries)
}
@@ -228,7 +221,6 @@ impl<B: Backend> Attention<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.prepare_queries_name);
let mut queries =
command_buffer.allocate_scratch_for_shape(&[self.num_q_heads, batch_dim, self.head_dim], self.data_type)?;
self.prepare.encode(
@@ -247,7 +239,6 @@ impl<B: Backend> Attention<B> {
batch_dim,
command_buffer,
);
- command_buffer.sample_end_timestamp();
Ok(queries)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
index 23c1054cb..979998d7a 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
@@ -26,7 +26,6 @@ struct Head<B: Backend> {
}
pub struct QKVNorm<B: Backend> {
- name: String,
query: Option<Head<B>>,
key: Option<Head<B>>,
value: Option<Head<B>>,
@@ -38,7 +37,6 @@ pub struct QKVNorm<B: Backend> {
impl<B: Backend> QKVNorm<B> {
pub fn new(
- name: String,
context: &B::Context,
intermediate_data_type: DataType,
query_config: Option<NormalizationConfig>,
@@ -77,7 +75,6 @@ impl<B: Backend> QKVNorm<B> {
.transpose()?;
Ok(Self {
- name,
query,
key,
value,
@@ -155,8 +152,7 @@ impl<B: Backend> QKVNorm<B> {
"QKV norm input row stride ({input_row_stride}) is smaller than its packed row width ({packed_row_width})"
);
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("qkv norm");
let kv = self.num_kv_heads;
let heads = [(&self.query, 0, q_heads), (&self.key, q_heads, kv), (&self.value, q_heads + kv, kv)];
@@ -183,7 +179,6 @@ impl<B: Backend> QKVNorm<B> {
);
}
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(())
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
index 6430332e5..967fc08cd 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
@@ -3,7 +3,7 @@ use std::any::Any;
use crate::{
array::size_for_shape,
backends::common::{
- Backend, Buffer, CommandBuffer, CommandBufferEncoding, Context, DeviceCapabilities, Kernels, SparseBuffer,
+ Backend, Buffer, CommandBuffer, Context, DeviceCapabilities, Kernels, SparseBuffer,
gpu_types::{Copy, ring::RingParams},
kernel::KVCacheUpdateKernel,
},
@@ -161,7 +161,6 @@ impl KVCacheState {
}
pub struct AttentionState<B: Backend> {
- accept_name: String,
pub element_dim: u32,
pub data_type: DataType,
cache: KVCacheState,
@@ -216,7 +215,6 @@ impl<B: Backend> AttentionState<B> {
let kv_cache_update = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(context, data_type)?;
Ok(Self {
- accept_name: format!("{}/accept", attention.name),
element_dim,
data_type,
cache,
@@ -255,7 +253,6 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
accepted_indices: &[u32],
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.sample_start_timestamp(&self.accept_name);
let copies = self.cache.accept(accepted_indices);
if !copies.is_empty() {
@@ -269,7 +266,6 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
);
}
- command_buffer.sample_end_timestamp();
Ok(())
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
index 986db8154..33e793265 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
@@ -48,7 +48,6 @@ enum DeltaNetSuffixStatus<B: Backend> {
}
pub struct DeltaNetState<B: Backend> {
- accept_name: String,
conv_state: B::GlobalBuffer,
ssm_state: B::GlobalBuffer,
suffix_status: Option<DeltaNetSuffixStatus<B>>,
@@ -69,7 +68,6 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {
accepted_indices: &[u32],
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.sample_start_timestamp(&self.accept_name);
let suffix_status = self.suffix_status.take().expect("delta net state has no suffix to accept");
let accepted_index = *accepted_indices.last().expect("delta net state attempted to accept zero indices");
@@ -111,14 +109,11 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {
);
},
}
- command_buffer.sample_end_timestamp();
Ok(())
}
}
pub struct DeltaNet<B: Backend> {
- name: String,
- tree_verify_name: String,
num_heads: u32,
head_dim: u32,
num_groups: u32,
@@ -164,7 +159,6 @@ pub enum DeltaNetNewError<B: Backend> {
impl<B: Backend> DeltaNet<B> {
pub fn new(
- name: String,
hidden_dim: u32,
outer_data_type: DataType,
config: &DeltaNetConfig,
@@ -197,7 +191,6 @@ impl<B: Backend> DeltaNet<B> {
let (in_projection, in_projection_input_hadamard_factors) =
<dyn Linear<B>>::new_with_input_rht_mixed_precision(
- format!("{name}/in projection"),
hidden_dim,
[total_proj_dim],
false,
@@ -288,7 +281,6 @@ impl<B: Backend> DeltaNet<B> {
.map_err(DeltaNetNewError::Backend)?;
let out_projection = <dyn Linear<B>>::new_mixed_precision(
- format!("{name}/out projection"),
value_dim,
[hidden_dim],
false,
@@ -301,8 +293,6 @@ impl<B: Backend> DeltaNet<B> {
Ok((
Self {
- tree_verify_name: format!("{name}/tree verify"),
- name,
num_heads: config.num_heads,
head_dim: config.head_dim,
num_groups: config.num_groups,
@@ -344,7 +334,6 @@ impl<B: Backend> DeltaNet<B> {
state: &mut DeltaNetState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.tree_verify_name);
let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");
let tree_size = batch_dim.size();
let parents = command_buffer.allocate_constant_from_slice(batch_dim.parents())?;
@@ -428,7 +417,6 @@ impl<B: Backend> DeltaNet<B> {
beta,
parents: batch_dim.parents().into(),
});
- command_buffer.sample_end_timestamp();
Ok(output)
}
}
@@ -467,7 +455,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
)?;
Ok(Box::new(DeltaNetState {
- accept_name: format!("{}/accept", self.name),
conv_state,
ssm_state,
suffix_status: None,
@@ -483,8 +470,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("delta net");
assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");
@@ -501,7 +487,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
if !batch_dim.full_accept() {
let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
return Ok(output);
@@ -646,7 +631,6 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
let output = self.out_projection.encode(delta_output, batch_dim.size(), command_buffer)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
index 7027e63cf..878668519 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mamba2.rs
@@ -55,7 +55,6 @@ enum Mamba2SSDPrefillVariant<B: Backend> {
}
pub struct Mamba2<B: Backend> {
- name: String,
kernel_size: u32,
num_heads: u32,
num_groups: u32,
@@ -92,7 +91,6 @@ pub enum Mamba2NewError<B: Backend> {
impl<B: Backend> Mamba2<B> {
pub fn new(
- name: String,
hidden_dim: u32,
outer_data_type: DataType,
config: &Mamba2Config,
@@ -116,7 +114,6 @@ impl<B: Backend> Mamba2<B> {
let (in_projection, in_projection_input_hadamard_factors) =
<dyn Linear<B>>::new_with_input_rht_mixed_precision(
- format!("{name}/in projection"),
hidden_dim,
[conv_dim, inner_dim, num_heads],
config.has_in_biases,
@@ -168,7 +165,6 @@ impl<B: Backend> Mamba2<B> {
};
let out_projection = <dyn Linear<B>>::new_mixed_precision(
- format!("{name}/out projection"),
inner_dim,
[hidden_dim],
config.has_out_biases,
@@ -181,7 +177,6 @@ impl<B: Backend> Mamba2<B> {
Ok((
Self {
- name,
kernel_size,
num_heads,
num_groups,
@@ -248,8 +243,7 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("mamba2");
assert!(precalculated_rope.is_none(), "unexpected rope for mamba2 mixer");
@@ -427,7 +421,6 @@ impl<B: Backend> Mixer<B> for Mamba2<B> {
let output = self.out_projection.encode(ssd_output, batch_dim.size(), command_buffer)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
index e9fe51377..5e0c30d66 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/mod.rs
@@ -83,7 +83,6 @@ pub enum MixerNewError<B: Backend> {
impl<B: Backend> dyn Mixer<B> {
pub fn new(
- layer_name: &str,
hidden_dim: u32,
data_type: DataType,
rope_config: Option<&AnyRoPEConfig>,
@@ -93,57 +92,32 @@ impl<B: Backend> dyn Mixer<B> {
) -> Result<(Box<dyn Mixer<B>>, Option<B::GlobalBuffer>), MixerNewError<B>> {
match config {
AnyTokenMixerConfig::AttentionConfig(config) => {
- let (attention, in_projection_input_hadamard_factors) = Attention::new(
- format!("{layer_name}/attention"),
- hidden_dim,
- data_type,
- rope_config,
- config,
- parameter_tree,
- context,
- )?;
+ let (attention, in_projection_input_hadamard_factors) =
+ Attention::new(hidden_dim, data_type, rope_config, config, parameter_tree, context)?;
Ok((Box::new(attention), in_projection_input_hadamard_factors))
},
AnyTokenMixerConfig::DeltaNetConfig(config) => {
assert!(rope_config.is_none(), "unexpected rope for delta net mixer");
- let (delta_net, in_projection_input_hadamard_factors) = DeltaNet::new(
- format!("{layer_name}/delta net"),
- hidden_dim,
- data_type,
- config,
- parameter_tree,
- context,
- )?;
+ let (delta_net, in_projection_input_hadamard_factors) =
+ DeltaNet::new(hidden_dim, data_type, config, parameter_tree, context)?;
Ok((Box::new(delta_net), in_projection_input_hadamard_factors))
},
AnyTokenMixerConfig::Mamba2Config(config) => {
assert!(rope_config.is_none(), "unexpected rope for mamba2 mixer");
- let (mamba2, in_projection_input_hadamard_factors) = Mamba2::new(
- format!("{layer_name}/mamba2"),
- hidden_dim,
- data_type,
- config,
- parameter_tree,
- context,
- )?;
+ let (mamba2, in_projection_input_hadamard_factors) =
+ Mamba2::new(hidden_dim, data_type, config, parameter_tree, context)?;
Ok((Box::new(mamba2), in_projection_input_hadamard_factors))
},
AnyTokenMixerConfig::ShortConvConfig(config) => {
assert!(rope_config.is_none(), "unexpected rope for short conv mixer");
- let (short_conv, in_projection_input_hadamard_factors) = ShortConv::new(
- format!("{layer_name}/short conv"),
- hidden_dim,
- data_type,
- config,
- parameter_tree,
- context,
- )?;
+ let (short_conv, in_projection_input_hadamard_factors) =
+ ShortConv::new(hidden_dim, data_type, config, parameter_tree, context)?;
Ok((Box::new(short_conv), in_projection_input_hadamard_factors))
},
diff --git a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
index 4c878daf1..93f9cbbb2 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/short_conv.rs
@@ -75,10 +75,6 @@ impl<B: Backend> MixerState<B> for ShortConvState<B> {
}
pub struct ShortConv<B: Backend> {
- name: String,
- decode_conv_name: String,
- prefill_conv_name: String,
- trie_conv_name: String,
hidden_dim: u32,
data_type: DataType,
kernel_size: u32,
@@ -106,7 +102,6 @@ pub enum ShortConvNewError<B: Backend> {
impl<B: Backend> ShortConv<B> {
pub fn new(
- name: String,
hidden_dim: u32,
data_type: DataType,
config: &ShortConvConfig,
@@ -122,7 +117,6 @@ impl<B: Backend> ShortConv<B> {
}
let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(
- format!("{name}/in projection"),
hidden_dim,
[hidden_dim * 3],
false,
@@ -132,7 +126,6 @@ impl<B: Backend> ShortConv<B> {
)?;
let out_projection = <dyn Linear<B>>::new(
- format!("{name}/out projection"),
hidden_dim,
[hidden_dim],
false,
@@ -168,10 +161,6 @@ impl<B: Backend> ShortConv<B> {
Ok((
Self {
- decode_conv_name: format!("{name}/decode conv"),
- prefill_conv_name: format!("{name}/prefill conv"),
- trie_conv_name: format!("{name}/trie conv"),
- name,
hidden_dim,
data_type,
kernel_size,
@@ -194,7 +183,6 @@ impl<B: Backend> ShortConv<B> {
state: &mut ShortConvState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.decode_conv_name);
let mut conv_output = command_buffer.allocate_scratch_for_shape(&[self.hidden_dim], self.data_type)?;
self.short_conv_decode.encode(
in_projected,
@@ -210,7 +198,6 @@ impl<B: Backend> ShortConv<B> {
self.hidden_dim,
command_buffer,
);
- command_buffer.sample_end_timestamp();
Ok(conv_output)
}
@@ -221,7 +208,6 @@ impl<B: Backend> ShortConv<B> {
state: &mut ShortConvState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.prefill_conv_name);
let state_stride = self.kernel_size - 1;
let padded_rows = state_stride + batch_dim;
@@ -253,7 +239,6 @@ impl<B: Backend> ShortConv<B> {
self.hidden_dim,
command_buffer,
);
- command_buffer.sample_end_timestamp();
Ok(conv_output)
}
@@ -265,7 +250,6 @@ impl<B: Backend> ShortConv<B> {
state: &mut ShortConvState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {
- command_buffer.sample_start_timestamp(&self.trie_conv_name);
let mut conv_output =
command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;
let mut conv_states = command_buffer
@@ -285,7 +269,6 @@ impl<B: Backend> ShortConv<B> {
self.hidden_dim,
command_buffer,
);
- command_buffer.sample_end_timestamp();
Ok((conv_output, conv_states))
}
}
@@ -325,8 +308,7 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("short conv");
assert!(precalculated_rope.is_none(), "unexpected rope for short conv mixer");
@@ -362,7 +344,6 @@ impl<B: Backend> Mixer<B> for ShortConv<B> {
let output = self.out_projection.encode(conv_output, batch_dim.size(), command_buffer)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mlp/dense.rs b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
index d48d52ddc..a0881dee7 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/dense.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/dense.rs
@@ -9,7 +9,6 @@ use crate::{
};
pub struct DenseMlp<B: Backend> {
- name: String,
up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
down: Box<dyn Linear<B>>,
@@ -17,13 +16,11 @@ pub struct DenseMlp<B: Backend> {
impl<B: Backend> DenseMlp<B> {
pub fn new(
- name: String,
up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
down: Box<dyn Linear<B>>,
) -> Self {
Self {
- name,
up,
gate,
down,
@@ -38,15 +35,13 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("mlp (dense)");
let fused_up = self.up.encode(input, batch_dim, command_buffer)?;
let act_format = self.down.select_activation_format(batch_dim, command_buffer.context());
let down_input = self.gate.encode_for_linear(command_buffer, &fused_up, batch_dim, act_format)?;
let output = self.down.encode_input(down_input, batch_dim, command_buffer)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
index 45911b711..4eab50d2f 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
@@ -10,7 +10,6 @@ use crate::{
};
pub struct MlpGateActMulEncodable<B: Backend> {
- name: String,
fp_kernel: GatedActMul<B>,
activation: AnyActivation,
hidden_dim: u32,
@@ -22,7 +21,6 @@ pub struct MlpGateActMulEncodable<B: Backend> {
impl<B: Backend> MlpGateActMulEncodable<B> {
pub fn new(
- name: String,
context: &B::Context,
data_type: DataType,
activation: AnyActivation,
@@ -43,7 +41,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
.map(|quantization| GatedActMul::quantized(context, data_type, quantization, settings))
.transpose()?;
Ok(Self {
- name,
fp_kernel,
activation,
hidden_dim,
@@ -61,8 +58,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
batch_dim: u32,
act_format: ActivationFormat,
) -> Result<LinearInput<B>, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("gate act mul");
if self.activation.act_type() == ActivationType::IDENTITY {
panic!("Identity activation is not supported for kernel")
@@ -119,7 +115,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
LinearInput::FullPrecision(hidden)
};
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(input)
diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
index 30407b40e..5ae64a74e 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
@@ -35,7 +35,6 @@ pub enum MlpBlockError<B: Backend> {
impl<B: Backend> dyn Mlp<B> {
pub fn new(
- name: String,
config: &AnyMLPConfig,
model_dimension: u32,
hidden_dimension: u32,
@@ -46,7 +45,6 @@ impl<B: Backend> dyn Mlp<B> {
match config {
AnyMLPConfig::DenseMLPConfig(dense_config) => {
let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_with_input_rht(
- format!("{name}/up projection"),
model_dimension,
[2 * hidden_dimension],
dense_config.has_up_biases,
@@ -56,7 +54,6 @@ impl<B: Backend> dyn Mlp<B> {
)?;
let (down_projection, down_input_preparation) = <dyn Linear<B>>::new_for_fused_input(
- format!("{name}/down projection"),
hidden_dimension,
[model_dimension],
dense_config.has_down_biases,
@@ -66,7 +63,6 @@ impl<B: Backend> dyn Mlp<B> {
)?;
let gate = MlpGateActMulEncodable::new(
- format!("{name}/gate act mul"),
context,
data_type,
dense_config.activation.clone(),
@@ -77,7 +73,7 @@ impl<B: Backend> dyn Mlp<B> {
)
.map_err(MlpBlockError::BackendError)?;
- Ok((Box::new(DenseMlp::new(name, up_projection, gate, down_projection)), up_input_hadamard_factors))
+ Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))
},
AnyMLPConfig::MixtureOfExpertsConfig(_) => Err(MlpBlockError::UnsupportedMixtureOfExperts),
}
diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rs
index b3f893482..9f6c79366 100644
--- a/crates/uzu-engine/src/encodable_block/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mod.rs
@@ -1,6 +1,5 @@
pub mod batch_topology;
pub mod classifier;
-pub mod context_ring_update;
pub mod convolution;
pub mod decoder;
pub mod dflash;
diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rs
index 6ce97da6c..6fe74ae30 100644
--- a/crates/uzu-engine/src/encodable_block/normalization.rs
+++ b/crates/uzu-engine/src/encodable_block/normalization.rs
@@ -37,7 +37,6 @@ pub enum NormalizationNewError<B: Backend> {
}
pub struct Normalization<B: Backend> {
- name: String,
epsilon: f32,
scale_offset: Option<f32>,
scales: Option<B::GlobalBuffer>,
@@ -51,7 +50,6 @@ pub struct Normalization<B: Backend> {
impl<B: Backend> Normalization<B> {
pub fn new(
- name: String,
element_count: u32,
hadamard_factors: Option<B::GlobalBuffer>,
shortcut_mode: ShortcutMode,
@@ -102,7 +100,6 @@ impl<B: Backend> Normalization<B> {
.map_err(NormalizationNewError::Backend)?;
Ok(Self {
- name,
epsilon: config.epsilon,
scale_offset: config.scale_offset,
scales,
@@ -123,8 +120,7 @@ impl<B: Backend> Normalization<B> {
shortcut: Option<impl BufferMut<Backend = B>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("normalization");
let row_size = size_for_shape(&[self.element_count], self.data_type);
let row_offset_bytes = row_offset as usize * row_size;
@@ -145,7 +141,6 @@ impl<B: Backend> Normalization<B> {
command_buffer,
);
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
index 6c9c5a9ee..a2920a9ea 100644
--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
@@ -32,7 +32,6 @@ pub enum PerLayerEmbeddingError<B: Backend> {
}
pub struct PerLayerEmbedding<B: Backend> {
- name: String,
token_embedding: EmbeddingTable<B>,
model_projection: Box<dyn Linear<B>>,
projection_norm: Normalization<B>,
@@ -46,7 +45,6 @@ pub struct PerLayerEmbedding<B: Backend> {
impl<B: Backend> PerLayerEmbedding<B> {
pub fn new(
- name: String,
context: &B::Context,
config: &PLEModelConfig,
model_dim: u32,
@@ -56,7 +54,6 @@ impl<B: Backend> PerLayerEmbedding<B> {
let total_ple_dim = config.num_layers * config.ple_dim;
let token_embedding = EmbeddingTable::load(
- format!("{name}/token lookup"),
context,
&parameter_tree.subtree("token_embedding"),
config.ple_vocab_size,
@@ -65,7 +62,6 @@ impl<B: Backend> PerLayerEmbedding<B> {
)?;
let model_projection = <dyn Linear<B>>::new(
- format!("{name}/model projection"),
model_dim,
[total_ple_dim],
false,
@@ -81,7 +77,6 @@ impl<B: Backend> PerLayerEmbedding<B> {
adjusted
};
let projection_norm = Normalization::new(
- format!("{name}/projection norm"),
config.ple_dim,
None,
ShortcutMode::None,
@@ -96,7 +91,6 @@ impl<B: Backend> PerLayerEmbedding<B> {
.map_err(PerLayerEmbeddingError::BackendError)?;
Ok(Self {
- name,
token_embedding,
model_projection,
projection_norm,
@@ -116,8 +110,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("per layer embedding");
let total_ple_dim = self.num_layers * self.ple_dim;
let total_rows = batch_dim * self.num_layers;
@@ -157,7 +150,6 @@ impl<B: Backend> PerLayerEmbedding<B> {
command_buffer,
);
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(per_layer_inputs)
@@ -165,7 +157,6 @@ impl<B: Backend> PerLayerEmbedding<B> {
}
pub struct PerLayerEmbeddingProjection<B: Backend> {
- name: String,
gate: Box<dyn Linear<B>>,
projection: Box<dyn Linear<B>>,
norm: Normalization<B>,
@@ -182,7 +173,6 @@ pub struct PerLayerEmbeddingProjection<B: Backend> {
impl<B: Backend> PerLayerEmbeddingProjection<B> {
pub fn new(
- name: String,
context: &B::Context,
config: &PLELayerConfig,
model_dim: u32,
@@ -192,7 +182,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
parameter_tree: &ParameterTree<B>,
) -> Result<Self, PerLayerEmbeddingError<B>> {
let gate = <dyn Linear<B>>::new(
- format!("{name}/gate"),
model_dim,
[config.ple_dim],
false,
@@ -201,7 +190,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
&parameter_tree.subtree("gate"),
)?;
let projection = <dyn Linear<B>>::new(
- format!("{name}/projection"),
config.ple_dim,
[model_dim],
false,
@@ -210,7 +198,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
&parameter_tree.subtree("projection"),
)?;
let norm = Normalization::new(
- format!("{name}/norm"),
model_dim,
None,
ShortcutMode::None,
@@ -234,7 +221,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
.map_err(PerLayerEmbeddingError::BackendError)?;
Ok(Self {
- name,
gate,
projection,
norm,
@@ -259,8 +245,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("per layer embedding projection");
let length = batch_dim * self.model_dim;
@@ -304,7 +289,6 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
command_buffer,
);
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(())
diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rs
index 010b45944..351661ee1 100644
--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs
+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs
@@ -27,7 +27,6 @@ pub enum PredictionHeadError<B: Backend> {
}
pub struct PredictionHead<B: Backend> {
- name: String,
hidden_dim: u32,
activation: ActivationType,
dense_projection: Box<dyn Linear<B>>,
@@ -38,7 +37,6 @@ pub struct PredictionHead<B: Backend> {
impl<B: Backend> PredictionHead<B> {
pub fn new(
- name: String,
hidden_dim: u32,
num_labels: u32,
data_type: DataType,
@@ -47,7 +45,6 @@ impl<B: Backend> PredictionHead<B> {
context: &B::Context,
) -> Result<Self, PredictionHeadError<B>> {
let dense_projection = <dyn Linear<B>>::new(
- format!("{name}/dense projection"),
hidden_dim,
[hidden_dim],
config.use_dense_bias,
@@ -61,7 +58,6 @@ impl<B: Backend> PredictionHead<B> {
.map_err(PredictionHeadError::Backend)?;
let normalization = Normalization::new(
- format!("{name}/norm"),
hidden_dim,
None,
ShortcutMode::None,
@@ -73,7 +69,6 @@ impl<B: Backend> PredictionHead<B> {
)?;
let readout = <dyn Linear<B>>::new(
- format!("{name}/readout"),
hidden_dim,
[num_labels],
true,
@@ -83,7 +78,6 @@ impl<B: Backend> PredictionHead<B> {
)?;
Ok(Self {
- name,
hidden_dim,
activation,
dense_projection,
@@ -99,8 +93,7 @@ impl<B: Backend> PredictionHead<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("prediction head");
let mut hidden = self.dense_projection.encode(input, batch_dim, command_buffer)?;
self.activation_kernel.encode(
@@ -114,7 +107,6 @@ impl<B: Backend> PredictionHead<B> {
self.normalization.encode(&hidden, 0, batch_dim, None::<&mut B::ScratchBuffer>, command_buffer)?;
let logits = self.readout.encode(normalized, batch_dim, command_buffer)?;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(logits)
diff --git a/crates/uzu-engine/src/encodable_block/sampling/mod.rs b/crates/uzu-engine/src/encodable_block/sampling/mod.rs
index 16721b71b..1142bd587 100644
--- a/crates/uzu-engine/src/encodable_block/sampling/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/sampling/mod.rs
@@ -24,7 +24,6 @@ pub use gumbel::{gumbel_float, revidx};
pub use prng::PRng;
pub struct Sampling<B: Backend> {
- name: String,
vocab_size: u32,
data_type: DataType,
unified_kernels: Mutex<HashMap<UnifiedSamplingKey, <B::Kernels as Kernels>::UnifiedSamplingKernel>>,
@@ -32,12 +31,10 @@ pub struct Sampling<B: Backend> {
impl<B: Backend> Sampling<B> {
pub fn new(
- name: String,
data_type: DataType,
vocab_size: u32,
) -> Self {
Self {
- name,
vocab_size,
data_type,
unified_kernels: Mutex::new(HashMap::new()),
@@ -95,8 +92,7 @@ impl<B: Backend> Sampling<B> {
sampling_range: Range<u32>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::GlobalBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("sampling");
let sampling_length = sampling_range.end - sampling_range.start;
@@ -198,7 +194,6 @@ impl<B: Backend> Sampling<B> {
command_buffer,
);
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rs
index a38d15288..c73b991b6 100644
--- a/crates/uzu-engine/src/encodable_block/transformer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer.rs
@@ -25,7 +25,6 @@ enum TransformerLayerStateType<B: Backend> {
}
pub struct TransformerState<B: Backend> {
- accept_name: String,
layer_states: Box<[TransformerLayerStateType<B>]>,
context_length: u32,
}
@@ -61,8 +60,7 @@ impl<B: Backend> TransformerState<B> {
accepted_indices: &[u32],
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.push_debug_group(&self.accept_name);
- command_buffer.sample_start_timestamp(&self.accept_name);
+ command_buffer.push_debug_group("transformer accept");
for layer_state in &mut self.layer_states {
let TransformerLayerStateType::Owned(layer_state) = layer_state else {
@@ -74,7 +72,6 @@ impl<B: Backend> TransformerState<B> {
self.context_length += accepted_indices.len() as u32;
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(())
@@ -92,8 +89,6 @@ pub enum TransformerNewError<B: Backend> {
}
pub struct Transformer<B: Backend> {
- name: String,
- capture_residual_name: String,
ropes: Box<[AnyRoPEConfig]>,
layers: Box<[(TransformerLayer<B>, Option<usize>)]>,
output_norm: Normalization<B>,
@@ -103,7 +98,6 @@ pub struct Transformer<B: Backend> {
impl<B: Backend> Transformer<B> {
pub fn new(
- name: String,
context: &B::Context,
output_norm_hadamard_factors: Option<B::GlobalBuffer>,
data_type: DataType,
@@ -128,7 +122,6 @@ impl<B: Backend> Transformer<B> {
});
let layer = TransformerLayer::new(
- format!("{name}/layer {layer_index}"),
context,
transformer_config.model_dim,
transformer_config.hidden_dim,
@@ -144,7 +137,6 @@ impl<B: Backend> Transformer<B> {
.collect::<Result<Box<[_]>, TransformerNewError<B>>>()?;
let output_norm = Normalization::new(
- format!("{name}/output norm"),
transformer_config.model_dim,
output_norm_hadamard_factors,
ShortcutMode::Add,
@@ -159,8 +151,6 @@ impl<B: Backend> Transformer<B> {
.map_err(TransformerNewError::Backend)?;
Ok(Self {
- capture_residual_name: format!("{name}/capture residual"),
- name,
ropes: ropes.into_boxed_slice(),
layers,
output_norm,
@@ -176,11 +166,9 @@ impl<B: Backend> Transformer<B> {
batch_size: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.capture_residual_name);
let mut output = command_buffer.allocate_scratch(hidden.size())?;
let elements = batch_size * self.model_dim;
self.residual_add.encode(Some(shortcut), hidden, &mut output, elements, elements, 1.0, command_buffer);
- command_buffer.sample_end_timestamp();
Ok(output)
}
@@ -232,7 +220,6 @@ impl<B: Backend> Transformer<B> {
let context_length = 0;
Ok(TransformerState {
- accept_name: format!("{}/accept", self.name),
layer_states,
context_length,
})
@@ -248,7 +235,6 @@ impl<B: Backend> Transformer<B> {
mut state: Option<&mut TransformerState<B>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<TransformerEncodeOutput<B>, B::Error> {
- command_buffer.sample_start_timestamp(&self.name);
let mut hidden = input;
let layer_count = if output_range.is_none() && hidden_feature_layer_indices.is_none() {
self.prefill_cache_layer_count()
@@ -324,7 +310,6 @@ impl<B: Backend> Transformer<B> {
});
let Some(output_range) = output_range else {
- command_buffer.sample_end_timestamp();
return Ok(TransformerEncodeOutput {
output: None,
hidden_features,
@@ -339,7 +324,6 @@ impl<B: Backend> Transformer<B> {
command_buffer,
)?;
- command_buffer.sample_end_timestamp();
Ok(TransformerEncodeOutput {
output: Some(output_normalized),
hidden_features,
diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
index 40f0e10ff..567a92c2a 100644
--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
@@ -45,7 +45,6 @@ pub enum TransformerLayerError<B: Backend> {
}
struct TransformerLayerConv<B: Backend> {
- pre_convolution_name: String,
model_dim: u32,
pre_conv: SeparableCausalConv<B>,
kernel_projection: Box<dyn Linear<B>>,
@@ -57,7 +56,6 @@ struct TransformerLayerConv<B: Backend> {
// TODO: saner shortcut
pub struct TransformerLayer<B: Backend> {
- name: String,
pub layer_index: u32,
pub kv_source_layer_index: Option<u32>,
pub pre_mixer_norm: Option<Normalization<B>>,
@@ -73,7 +71,6 @@ pub struct TransformerLayer<B: Backend> {
impl<B: Backend> TransformerLayerConv<B> {
fn new(
- name: String,
context: &B::Context,
model_dim: u32,
config: &TransformerLayerConvConfig,
@@ -82,7 +79,6 @@ impl<B: Backend> TransformerLayerConv<B> {
input_hadamard_factors: Option<B::GlobalBuffer>,
) -> Result<Self, TransformerLayerError<B>> {
let pre_conv = SeparableCausalConv::new(
- format!("{name}/pre conv"),
model_dim,
config.conv_kernel_size,
config.conv_group_size,
@@ -92,7 +88,6 @@ impl<B: Backend> TransformerLayerConv<B> {
context,
)?;
let post_conv = SeparableCausalConv::new(
- format!("{name}/post conv"),
model_dim,
config.conv_kernel_size,
config.conv_group_size,
@@ -105,7 +100,6 @@ impl<B: Backend> TransformerLayerConv<B> {
let coefficient_count = config.conv_kernel_size * (model_dim / config.conv_group_size);
let projection_dim = coefficient_count * 2;
let kernel_projection = <dyn Linear<B>>::new(
- format!("{name}/kernel projection"),
model_dim,
[projection_dim],
false,
@@ -120,7 +114,6 @@ impl<B: Backend> TransformerLayerConv<B> {
.map_err(TransformerLayerError::Backend)?;
Ok(Self {
- pre_convolution_name: format!("{name}/pre convolution"),
model_dim,
pre_conv,
kernel_projection,
@@ -136,7 +129,6 @@ impl<B: Backend> TransformerLayerConv<B> {
sequence_length: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(B::ScratchBuffer, B::ScratchBuffer), B::Error> {
- command_buffer.sample_start_timestamp(&self.pre_convolution_name);
let mut projection_input = command_buffer.allocate_scratch(input.size())?;
command_buffer.encode_copy(input, &mut projection_input);
let coefficients = self.kernel_projection.encode(projection_input, sequence_length, command_buffer)?;
@@ -151,7 +143,6 @@ impl<B: Backend> TransformerLayerConv<B> {
if let Some((transform, factors)) = &self.input_rht {
transform.encode_fp_in_place(&mut output, factors, None, sequence_length, self.model_dim, command_buffer);
}
- command_buffer.sample_end_timestamp();
Ok((output, coefficients))
}
@@ -175,7 +166,6 @@ impl<B: Backend> TransformerLayerConv<B> {
impl<B: Backend> TransformerLayer<B> {
pub fn new(
- name: String,
context: &B::Context,
model_dim: u32,
hidden_dim: u32,
@@ -211,7 +201,6 @@ impl<B: Backend> TransformerLayer<B> {
};
let (mixer, mixer_hadamard_factors) = <dyn Mixer<B>>::new(
- &name,
model_dim,
data_type,
layer_config.rope_config.as_ref(),
@@ -223,7 +212,6 @@ impl<B: Backend> TransformerLayer<B> {
let (mixer_conv, mixer_hadamard_factors) = match &layer_config.mixer_conv_config {
Some(config) => (
Some(TransformerLayerConv::new(
- format!("{name}/mixer conv"),
context,
model_dim,
config,
@@ -238,7 +226,6 @@ impl<B: Backend> TransformerLayer<B> {
let pre_mixer_norm = if let Some(pre_mixer_norm_config) = &layer_config.pre_mixer_norm_config {
Some(Normalization::new(
- format!("{name}/pre mixer norm"),
model_dim,
mixer_hadamard_factors,
if layer_index > 0 {
@@ -261,7 +248,6 @@ impl<B: Backend> TransformerLayer<B> {
let post_mixer_norm = if let Some(norm_config) = &layer_config.post_mixer_norm_config {
Some(Normalization::new(
- format!("{name}/post mixer norm"),
model_dim,
None,
ShortcutMode::None,
@@ -276,7 +262,6 @@ impl<B: Backend> TransformerLayer<B> {
};
let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(
- format!("{name}/mlp"),
&layer_config.mlp_config,
model_dim,
layer_config.hidden_dim.unwrap_or(hidden_dim),
@@ -288,7 +273,6 @@ impl<B: Backend> TransformerLayer<B> {
let (mlp_conv, mlp_input_hadamard_factors) = match &layer_config.mlp_conv_config {
Some(config) => (
Some(TransformerLayerConv::new(
- format!("{name}/mlp conv"),
context,
model_dim,
config,
@@ -302,7 +286,6 @@ impl<B: Backend> TransformerLayer<B> {
};
let pre_mlp_norm = Normalization::new(
- format!("{name}/pre mlp norm"),
model_dim,
mlp_input_hadamard_factors,
ShortcutMode::Add,
@@ -315,7 +298,6 @@ impl<B: Backend> TransformerLayer<B> {
let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(Normalization::new(
- format!("{name}/post mlp norm"),
model_dim,
None,
ShortcutMode::None,
@@ -332,7 +314,6 @@ impl<B: Backend> TransformerLayer<B> {
let ple_projection = layer_config.ple_config.as_ref().map(|ple_config| {
let ple_loader = parameter_tree.subtree("ple");
PerLayerEmbeddingProjection::new(
- format!("{name}/per layer embedding projection"),
context,
ple_config,
model_dim,
@@ -345,7 +326,6 @@ impl<B: Backend> TransformerLayer<B> {
});
Ok(Self {
- name,
layer_index,
kv_source_layer_index: layer_config.kv_source_layer_index,
pre_mixer_norm,
@@ -370,8 +350,7 @@ impl<B: Backend> TransformerLayer<B> {
state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group(&format!("transformer layer {}", self.layer_index));
let mut hidden = if let Some(pre_mixer_norm) = &self.pre_mixer_norm {
pre_mixer_norm.encode(&input, 0, batch_dim.size(), Some(shortcut.reborrow()), command_buffer)?
@@ -439,7 +418,6 @@ impl<B: Backend> TransformerLayer<B> {
command_buffer.encode_fill(&mut hidden, 0);
}
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(hidden)
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index a3a15c378..d446df08c 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -112,9 +112,6 @@ impl<B: Backend> EncodedWeaverTree<B> {
}
pub struct Weaver<B: Backend> {
- name: String,
- prefix_name: String,
- step_name: String,
token_embedding_norm: Normalization<B>,
token_embedding_projection: Box<dyn Linear<B>>,
hidden_state_norm: Normalization<B>,
@@ -174,7 +171,6 @@ pub enum WeaverEncodeError<B: Backend> {
impl<B: Backend> Weaver<B> {
pub fn new(
- name: String,
context: &B::Context,
config: &WeaverConfig,
vocab_size: u32,
@@ -203,7 +199,6 @@ impl<B: Backend> Weaver<B> {
});
}
let token_embedding_norm = Normalization::new(
- format!("{name}/token embedding norm"),
config.target_embedding_dim,
None,
ShortcutMode::None,
@@ -214,7 +209,6 @@ impl<B: Backend> Weaver<B> {
context,
)?;
let hidden_state_norm = Normalization::new(
- format!("{name}/hidden state norm"),
config.target_model_dim,
None,
ShortcutMode::None,
@@ -225,7 +219,6 @@ impl<B: Backend> Weaver<B> {
context,
)?;
let token_embedding_projection = <dyn Linear<B>>::new(
- format!("{name}/token embedding projection"),
config.target_embedding_dim,
[config.model_dim],
true,
@@ -235,18 +228,9 @@ impl<B: Backend> Weaver<B> {
)?;
let layer_parameters = parameter_tree.subtree("blocks");
let layers = (0..config.num_layers)
- .map(|index| {
- WeaverLayer::new(
- format!("{name}/layer {index}"),
- context,
- config,
- index > 0,
- &layer_parameters.subtree(&index.to_string()),
- )
- })
+ .map(|index| WeaverLayer::new(context, config, index > 0, &layer_parameters.subtree(&index.to_string())))
.collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;
let readout_norm = Normalization::new(
- format!("{name}/readout norm"),
config.model_dim,
None,
ShortcutMode::Add,
@@ -257,7 +241,6 @@ impl<B: Backend> Weaver<B> {
context,
)?;
let hidden_state_projection = <dyn Linear<B>>::new(
- format!("{name}/hidden state projection"),
config.target_model_dim,
[config.model_dim],
true,
@@ -266,7 +249,6 @@ impl<B: Backend> Weaver<B> {
&parameter_tree.subtree("hidden_state_projection"),
)?;
let readout_query_projection = <dyn Linear<B>>::new(
- format!("{name}/readout query projection"),
config.model_dim,
[config.target_model_dim],
false,
@@ -285,9 +267,6 @@ impl<B: Backend> Weaver<B> {
let frontier_insert_children = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(context)
.map_err(WeaverNewError::Backend)?;
Ok(Self {
- prefix_name: format!("{name}/prefix"),
- step_name: format!("{name}/step"),
- name,
token_embedding_norm,
token_embedding_projection,
hidden_state_norm,
@@ -320,8 +299,7 @@ impl<B: Backend> Weaver<B> {
depth: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<Vec<B::ScratchBuffer>, WeaverEncodeError<B>> {
- command_buffer.push_debug_group(&self.prefix_name);
- command_buffer.sample_start_timestamp(&self.prefix_name);
+ command_buffer.push_debug_group("weaver prefix");
let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);
let mut prefix_hidden = command_buffer
@@ -381,7 +359,6 @@ impl<B: Backend> Weaver<B> {
.kv_cache,
);
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
Ok(prefix_kv_layers)
@@ -410,8 +387,6 @@ impl<B: Backend> Weaver<B> {
batch_start_slot: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), WeaverEncodeError<B>> {
- command_buffer.push_debug_group(&self.step_name);
- command_buffer.sample_start_timestamp(&self.step_name);
let tree_slot_count = shape.slot_count();
let ancestor_stride = self.max_depth;
let frontier_capacity = tree_slot_count * shape.expand_width;
@@ -564,8 +539,6 @@ impl<B: Backend> Weaver<B> {
command_buffer,
);
- command_buffer.sample_end_timestamp();
- command_buffer.pop_debug_group();
Ok(())
}
@@ -580,8 +553,7 @@ impl<B: Backend> Weaver<B> {
shape: WeaverTreeShape,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
- command_buffer.push_debug_group(&self.name);
- command_buffer.sample_start_timestamp(&self.name);
+ command_buffer.push_debug_group("weaver tree");
let tree_slot_count = shape.slot_count();
let ancestor_stride = self.max_depth;
@@ -705,6 +677,7 @@ impl<B: Backend> Weaver<B> {
} else {
shape.expand_per_round
};
+ command_buffer.push_debug_group("weaver step");
self.encode_step(
target_embedding,
prefix_kv_layers.iter(),
@@ -727,10 +700,10 @@ impl<B: Backend> Weaver<B> {
batch_start_slot,
command_buffer,
)?;
+ command_buffer.pop_debug_group();
batch_start_slot += batch_node_count;
}
- command_buffer.sample_end_timestamp();
command_buffer.pop_debug_group();
let mut packed_tree_readback =
diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
index 8621bedce..2b9080e6b 100644
--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
@@ -25,8 +25,6 @@ pub struct PreparedPrefixAttention<B: Backend> {
}
pub struct WeaverLayer<B: Backend> {
- prefix_attention_name: String,
- post_attention_name: String,
pub pre_attention_norm: Normalization<B>,
pub qkv_projection: Box<dyn Linear<B>>,
attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
@@ -44,7 +42,6 @@ pub struct WeaverLayer<B: Backend> {
impl<B: Backend> WeaverLayer<B> {
pub fn new(
- name: String,
context: &B::Context,
config: &WeaverConfig,
add_to_residual: bool,
@@ -62,7 +59,6 @@ impl<B: Backend> WeaverLayer<B> {
let head_dim = model_dim / num_heads;
let attention_scale = 1.0 / (head_dim as f32).sqrt();
let qkv_projection = <dyn Linear<B>>::new(
- format!("{name}/qkv projection"),
model_dim,
[3 * model_dim],
false,
@@ -71,7 +67,6 @@ impl<B: Backend> WeaverLayer<B> {
&parameter_tree.subtree("qkv_projection"),
)?;
let out_projection = <dyn Linear<B>>::new(
- format!("{name}/out projection"),
model_dim,
[model_dim],
false,
@@ -98,7 +93,6 @@ impl<B: Backend> WeaverLayer<B> {
<B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, ROPE_DATA_TYPE, true, true)
.map_err(WeaverNewError::Backend)?;
let pre_attention_norm = Normalization::new(
- format!("{name}/pre attention norm"),
model_dim,
None,
if add_to_residual {
@@ -113,7 +107,6 @@ impl<B: Backend> WeaverLayer<B> {
context,
)?;
let pre_mlp_norm = Normalization::new(
- format!("{name}/pre mlp norm"),
model_dim,
None,
ShortcutMode::Add,
@@ -129,21 +122,12 @@ impl<B: Backend> WeaverLayer<B> {
true,
true,
));
- let (mlp, up_input_hadamard_factors) = <dyn Mlp<B>>::new(
- format!("{name}/mlp"),
- &mlp_config,
- model_dim,
- hidden_dim,
- context,
- &parameter_tree.subtree("mlp"),
- DATA_TYPE,
- )?;
+ let (mlp, up_input_hadamard_factors) =
+ <dyn Mlp<B>>::new(&mlp_config, model_dim, hidden_dim, context, &parameter_tree.subtree("mlp"), DATA_TYPE)?;
assert!(up_input_hadamard_factors.is_none(), "Weaver MLP does not support input Hadamard factors");
let ancestor_attention = <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim, num_heads)
.map_err(WeaverNewError::Backend)?;
Ok(Self {
- prefix_attention_name: format!("{name}/prefix attention"),
- post_attention_name: format!("{name}/post attention"),
qkv_projection,
out_projection,
prefix_attention,
@@ -168,7 +152,6 @@ impl<B: Backend> WeaverLayer<B> {
token_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<PreparedPrefixAttention<B>, B::Error> {
- command_buffer.sample_start_timestamp(&self.prefix_attention_name);
let attention_input =
self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), command_buffer)?;
let qkv = self.qkv_projection.encode(attention_input, token_count, command_buffer)?;
@@ -193,7 +176,6 @@ impl<B: Backend> WeaverLayer<B> {
token_count,
command_buffer,
);
- command_buffer.sample_end_timestamp();
Ok(PreparedPrefixAttention {
queries,
kv_cache,
@@ -207,12 +189,9 @@ impl<B: Backend> WeaverLayer<B> {
token_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.sample_start_timestamp(&self.post_attention_name);
let projected_attention = self.out_projection.encode(attention_output, token_count, command_buffer)?;
let mlp_input =
self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), command_buffer)?;
- let output = self.mlp.encode(mlp_input, token_count, command_buffer)?;
- command_buffer.sample_end_timestamp();
- Ok(output)
+ self.mlp.encode(mlp_input, token_count, command_buffer)
}
}
diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rs
index 0d63e996a..d92e350e7 100644
--- a/crates/uzu-engine/src/engine/classifier_model.rs
+++ b/crates/uzu-engine/src/engine/classifier_model.rs
@@ -66,7 +66,6 @@ impl<B: Backend> Engine<B> {
let weight_loader = ParameterLoader::new(&weights_file, context.as_ref())?;
let classifier = ClassifierEncodable::new(
- String::from("classifier"),
context.as_ref(),
&config.classifier_config,
&weight_loader.tree().subtree("classifier"),
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index 54a2ff84a..1255404b9 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -5,14 +5,13 @@ use thiserror::Error;
use tokenizers::Tokenizer;
use crate::{
- backends::common::{Backend, Context, DeviceCapabilities},
+ backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},
config::{
model::{generation::GenerationConfig, language_model::LanguageModelConfig},
token_codec::AnyTokenCodecConfig,
},
data_type::DataType,
encodable_block::{
- context_ring_update::ContextRingUpdate,
decoder::{Decoder, DecoderError},
sampling::{Sampling, SamplingMethod},
},
@@ -32,7 +31,7 @@ pub struct LanguageModel<B: Backend> {
decoder: Decoder<B>,
speculator: Option<DFlashTfmSpeculator<B>>,
sampling: Sampling<B>,
- context_ring_update: ContextRingUpdate<B>,
+ context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
generation_config: GenerationConfig,
end_of_thinking_tag: Option<String>,
tokenizer: Arc<Tokenizer>,
@@ -78,7 +77,6 @@ impl<B: Backend> Engine<B> {
let data_type = DataType::BF16;
let decoder = Decoder::new(
- String::from("decoder"),
self.context.as_ref(),
&config.decoder_config,
&weight_loader.tree().subtree("decoder"),
@@ -96,11 +94,10 @@ impl<B: Backend> Engine<B> {
.transpose()?
.flatten();
- let sampling = Sampling::new(String::from("sampling"), data_type, config.decoder_config.vocab_size);
+ let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);
- let context_ring_update =
- ContextRingUpdate::new(String::from("update repetition penalty ring"), self.context.as_ref())
- .map_err(EngineLoadLanguageModelError::Backend)?;
+ let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&self.context)
+ .map_err(EngineLoadLanguageModelError::Backend)?;
weight_loader.tree().assert_all_tensors_validated()?;
diff --git a/crates/uzu-engine/src/engine/language_model/stream/mod.rs b/crates/uzu-engine/src/engine/language_model/stream/mod.rs
index 8ca87d672..fc456ba8d 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/mod.rs
@@ -1,12 +1,10 @@
-use std::sync::mpsc::Sender;
-
use derive_more::Debug;
use thiserror::Error;
#[cfg(grammar)]
use crate::engine::language_model::grammar::{Grammar, GrammarError};
use crate::{
- backends::common::{Backend, TimestampSpan},
+ backends::common::Backend,
encodable_block::decoder::DecoderError,
engine::language_model::{LanguageModel, state::LanguageModelState},
speculators::dflash_tfm::DFlashTreeError,
@@ -19,7 +17,6 @@ mod stream;
pub struct LanguageModelStreamOptions {
pub sampling_method: SamplingMethod,
- pub timestamps: Option<Sender<Box<[TimestampSpan]>>>,
#[cfg(grammar)]
pub grammar: Option<Grammar>,
}
@@ -47,7 +44,6 @@ impl<B: Backend> LanguageModel<B> {
pub fn default_stream_options(&self) -> LanguageModelStreamOptions {
LanguageModelStreamOptions {
sampling_method: self.default_sampling_method(),
- timestamps: None,
#[cfg(grammar)]
grammar: None,
}
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 1e826f915..7663cf362 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -2,7 +2,7 @@ use std::{
iter::{once, repeat_n},
mem::replace,
range::Range,
- sync::{Arc, mpsc::Sender},
+ sync::Arc,
};
use shoji::traits::backend::chat_token::TokenStreamMetrics;
@@ -12,8 +12,8 @@ use crate::engine::language_model::grammar::Grammar;
use crate::{
array::size_for_shape,
backends::common::{
- Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context,
- TimestampSpan, gpu_types::trie::TrieNode as GpuTrieNode,
+ Backend, Buffer, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable,
+ CommandBufferPending, Context, gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,
},
data_type::DataType,
encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},
@@ -26,7 +26,6 @@ use crate::{
},
},
trie::TrieNode,
- utils::timestamps::{create_command_buffer, wait},
};
enum ForwardPassChaining<B: Backend> {
@@ -41,7 +40,6 @@ impl<B: Backend> ForwardPassChaining<B> {
fn resolve<'a>(
&'a mut self,
tokens: &mut Vec<u64>,
- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
#[cfg(grammar)] grammar: Option<&mut Grammar>,
) -> Result<(u64, Option<&'a B::ScratchBuffer>), LanguageModelStreamError<B>> {
match self {
@@ -52,7 +50,7 @@ impl<B: Backend> ForwardPassChaining<B> {
Self::InFlight(in_flight) => {
assert!(in_flight.full_accept);
for pending in replace(&mut in_flight.pending, Box::new([])) {
- wait(pending, timestamps).map_err(LanguageModelStreamError::Backend)?;
+ pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;
}
let output_tokens = in_flight.output_tokens.as_slice::<u32>();
assert_eq!(output_tokens.len(), 1);
@@ -214,13 +212,11 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
.map_err(LanguageModelStreamError::Backend)?;
- let mut command_buffer = create_command_buffer::<B>(
- &model.engine.context,
- "prefill",
- &allocation_pool,
- options.timestamps.as_ref(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = model
+ .engine
+ .context
+ .create_command_buffer(Some("prefill"), Some(allocation_pool.clone()))
+ .map_err(LanguageModelStreamError::Backend)?;
let mut output_tokens = None;
let mut output_norm = None;
@@ -405,7 +401,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
(ForwardPassChaining::InFlight(forward_pass_pending), None)
} else {
for pending in forward_pass_pending.pending {
- wait(pending, self.options.timestamps.as_ref()).map_err(LanguageModelStreamError::Backend)?;
+ pending.wait_until_completed().map_err(LanguageModelStreamError::Backend)?;
}
let sampled_tokens = forward_pass_pending
.output_tokens
@@ -458,13 +454,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let accepted_token_indicies = full.iter().map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();
let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();
- let mut command_buffer = create_command_buffer::<B>(
- &self.model.engine.context,
- "decode",
- &self.allocation_pool,
- self.options.timestamps.as_ref(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = self
+ .model
+ .engine
+ .context
+ .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
+ .map_err(LanguageModelStreamError::Backend)?;
self.model_state
.transformer_state
.encode_accept(&accepted_token_indicies, &mut command_buffer)
@@ -490,6 +485,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
None
};
if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length() {
+ command_buffer.push_debug_group("update repetition penalty ring");
let accepted_input_token_ids_const = command_buffer
.allocate_constant_from_slice(
&accepted_input_token_ids
@@ -505,20 +501,23 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
full.len() as u32,
&mut command_buffer,
);
+ command_buffer.pop_debug_group();
}
if let Some(capture_span) = capture_span {
- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref())
+ command_buffer
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
.map_err(LanguageModelStreamError::Backend)?;
drop(capture_span);
- command_buffer = create_command_buffer::<B>(
- &self.model.engine.context,
- "decode",
- &self.allocation_pool,
- self.options.timestamps.as_ref(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ command_buffer = self
+ .model
+ .engine
+ .context
+ .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
+ .map_err(LanguageModelStreamError::Backend)?;
}
self.model_state.tokens.extend(accepted_output_token_ids);
(
@@ -542,7 +541,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
prev_output
.resolve(
&mut self.model_state.tokens,
- self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)?
@@ -555,7 +553,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
{
prev_output.resolve(
&mut self.model_state.tokens,
- self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)?;
@@ -572,7 +569,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
&& let (root_token, Some(output_norm)) = prev_output.resolve(
&mut self.model_state.tokens,
- self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)? {
@@ -589,7 +585,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.options.grammar.as_mut(),
&self.model_state.prng,
self.allocation_pool.clone(),
- self.options.timestamps.as_ref(),
)?;
prev_output.drop_output_norm();
(trie, None, false)
@@ -612,13 +607,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.allocation_pool = self.model.engine.context.create_allocation_pool();
- let mut command_buffer = create_command_buffer::<B>(
- &self.model.engine.context,
- "decode",
- &self.allocation_pool,
- self.options.timestamps.as_ref(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = self
+ .model
+ .engine
+ .context
+ .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
+ .map_err(LanguageModelStreamError::Backend)?;
let scratch_token_ids;
let mut constant_token_ids = None;
@@ -665,13 +659,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if chain_copy.is_some() {
pending.push(command_buffer.end_encoding().submit());
- let mut command_buffer = create_command_buffer::<B>(
- &self.model.engine.context,
- "decode",
- &self.allocation_pool,
- self.options.timestamps.as_ref(),
- )
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut command_buffer = self
+ .model
+ .engine
+ .context
+ .create_command_buffer(Some("decode"), Some(self.allocation_pool.clone()))
+ .map_err(LanguageModelStreamError::Backend)?;
let mut bitmask = command_buffer
.allocate_constant(
@@ -679,7 +672,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
.map_err(LanguageModelStreamError::Backend)?;
- prev_output.resolve(&mut self.model_state.tokens, self.options.timestamps.as_ref(), Some(grammar))?;
+ prev_output.resolve(&mut self.model_state.tokens, Some(grammar))?;
if grammar.next_bitmask(bitmask.as_slice_mut()) {
(Some(bitmask), command_buffer)
} else {
@@ -793,7 +786,6 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
prev_output
.resolve(
&mut self.model_state.tokens,
- self.options.timestamps.as_ref(),
#[cfg(grammar)]
self.options.grammar.as_mut(),
)?
@@ -822,17 +814,16 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
} => Some(seed_token),
DecodingState::ForwardPassPending(in_flight) => {
for pending in in_flight.pending {
- wait(pending, self.options.timestamps.as_ref()).unwrap();
+ pending.wait_until_completed().unwrap();
}
if !in_flight.full_accept {
- let mut command_buffer = create_command_buffer::<B>(
- &self.model.engine.context,
- "drop accept",
- &self.allocation_pool,
- self.options.timestamps.as_ref(),
- )
- .unwrap();
+ let mut command_buffer = self
+ .model
+ .engine
+ .context
+ .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))
+ .unwrap();
self.model_state.transformer_state.encode_accept(&[0], &mut command_buffer).unwrap();
if let Some(speculator) = self.model.speculator.as_ref() {
speculator
@@ -844,7 +835,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
)
.unwrap();
}
- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
}
Some(in_flight.output_tokens.as_slice::<u32>()[0] as u64)
@@ -858,13 +849,12 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
} => {
assert!(num_accepted > 0 && num_accepted < full.len());
- let mut command_buffer = create_command_buffer::<B>(
- &self.model.engine.context,
- "drop accept",
- &self.allocation_pool,
- self.options.timestamps.as_ref(),
- )
- .unwrap();
+ let mut command_buffer = self
+ .model
+ .engine
+ .context
+ .create_command_buffer(Some("drop accept"), Some(self.allocation_pool.clone()))
+ .unwrap();
let accepted_token_indicies =
full.iter().take(num_accepted + 1).map(|(i, _, _)| *i as u32).collect::<Box<[u32]>>();
self.model_state
@@ -881,7 +871,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
)
.unwrap();
}
- wait(command_buffer.end_encoding().submit(), self.options.timestamps.as_ref()).unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
drop(capture_span);
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index 2d1e02a60..6c41724b5 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -3,7 +3,7 @@ use std::{
fs::File,
io::{self, BufReader},
path::Path,
- sync::{Arc, mpsc::Sender},
+ sync::Arc,
};
use derive_more::Debug;
@@ -15,8 +15,8 @@ pub use crate::encodable_block::dflash::DFlashState;
use crate::engine::language_model::grammar::Grammar;
use crate::{
backends::common::{
- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, Context, TimestampSpan,
- gpu_types::trie::TrieNode as GpuTrieNode,
+ Backend, BufferRef, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,
+ Context, gpu_types::trie::TrieNode as GpuTrieNode,
},
config::speculator::{AnySpeculatorConfig, dflash::DFlashSpeculatorConfig, model::SpeculatorModelConfig},
data_type::DataType,
@@ -29,7 +29,6 @@ use crate::{
},
parameters::{ParameterLoader, ParameterLoaderError},
trie::TrieNode,
- utils::timestamps::{create_command_buffer, wait},
};
#[derive(Debug, Error)]
@@ -123,19 +122,12 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let weight_loader = ParameterLoader::new(&weights_file, &*context)?;
let speculator_tree = weight_loader.tree().subtree("speculator");
- let dflash = DFlash::new(
- String::from("dflash"),
- &*context,
- &config.draft_config,
- &speculator_tree.subtree("draft_model"),
- data_type,
- )?;
+ let dflash = DFlash::new(&*context, &config.draft_config, &speculator_tree.subtree("draft_model"), data_type)?;
let weaver = config
.weaver_config
.as_ref()
.map(|weaver_config| {
Weaver::new(
- String::from("weaver"),
&*context,
weaver_config,
config.draft_config.vocab_size,
@@ -146,7 +138,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
weight_loader.tree().assert_all_tensors_validated()?;
- let sampling = Sampling::new(String::from("sampling"), data_type, config.draft_config.vocab_size);
+ let sampling = Sampling::new(data_type, config.draft_config.vocab_size);
Ok(Some(Self {
context,
@@ -225,7 +217,6 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
#[cfg(grammar)] grammar: Option<&mut Grammar>,
prng: &PRng,
allocation_pool: Arc<B::AllocationPool>,
- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
) -> Result<TrieNode, DFlashTreeError<B>> {
assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");
@@ -239,9 +230,10 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let root_position = state.context_length();
- let mut command_buffer =
- create_command_buffer::<B>(&self.context, "speculator propose", &allocation_pool, timestamps)
- .map_err(DFlashTreeError::Backend)?;
+ let mut command_buffer = self
+ .context
+ .create_command_buffer(Some("speculator propose"), Some(allocation_pool))
+ .map_err(DFlashTreeError::Backend)?;
let nodes = match shape.construction_method {
DFlashTfmTreeConstructionMethod::Argmax => {
@@ -291,7 +283,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
)
.map_err(DFlashTreeError::Backend)?;
let completed =
- wait(command_buffer.end_encoding().submit(), timestamps).map_err(DFlashTreeError::Backend)?;
+ command_buffer.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
let tokens = sampled.copyout::<u32>();
drop(completed);
nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {
@@ -366,7 +358,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
&mut command_buffer,
)?;
let completed =
- wait(command_buffer.end_encoding().submit(), timestamps).map_err(DFlashTreeError::Backend)?;
+ command_buffer.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
let nodes = tree.read_nodes();
drop(completed);
nodes
diff --git a/crates/uzu-engine/src/utils/mod.rs b/crates/uzu-engine/src/utils/mod.rs
index 17b686018..11e88e2c5 100644
--- a/crates/uzu-engine/src/utils/mod.rs
+++ b/crates/uzu-engine/src/utils/mod.rs
@@ -4,5 +4,4 @@ pub mod maybe_mut;
#[cfg(backend = "cpu")]
pub mod pointers;
pub mod strict_serde;
-pub mod timestamps;
pub mod version;
diff --git a/crates/uzu-engine/src/utils/timestamps.rs b/crates/uzu-engine/src/utils/timestamps.rs
deleted file mode 100644
index fcf6d60f6..000000000
--- a/crates/uzu-engine/src/utils/timestamps.rs
+++ /dev/null
@@ -1,32 +0,0 @@
-use std::sync::{Arc, mpsc::Sender};
-
-use crate::backends::common::{
- Backend, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferPending, Context, TimestampSpan,
-};
-
-pub fn create_command_buffer<B: Backend>(
- context: &B::Context,
- name: &str,
- allocation_pool: &Arc<B::AllocationPool>,
- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
-) -> Result<<B::CommandBuffer as CommandBuffer>::Encoding, B::Error> {
- let mut command_buffer = context.create_command_buffer(Some(name), Some(allocation_pool.clone()))?;
- if timestamps.is_some() {
- command_buffer.enable_timestamps();
- }
- Ok(command_buffer)
-}
-
-pub fn wait<Pending: CommandBufferPending>(
- pending: Pending,
- timestamps: Option<&Sender<Box<[TimestampSpan]>>>,
-) -> Result<
- <Pending::CommandBuffer as CommandBuffer>::Completed,
- <<Pending::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
-> {
- let completed = pending.wait_until_completed()?;
- if let Some(sender) = timestamps {
- let _ = sender.send(completed.timestamps().into());
- }
- Ok(completed)
-}
diff --git a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs
index c9639d5d8..15a85cc3b 100644
--- a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs
@@ -30,7 +30,6 @@ fn run_key_value_row_stride_test<B: Backend>() {
let key = QKVNorm::<B>::build_head(&context, DataType::F32, config, None, HEAD_DIM)
.expect("failed to construct key norm");
let norm = QKVNorm {
- name: String::from("test/qkv norm"),
query: None,
key: Some(key),
value: None,
diff --git a/crates/uzu-engine/unit/encodable_block/sampling_test.rs b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
index 50932575b..f00b7bfff 100644
--- a/crates/uzu-engine/unit/encodable_block/sampling_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/sampling_test.rs
@@ -44,7 +44,7 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(
method: &SamplingMethod,
batch_size: u32,
) -> Result<SamplingTestResults, TestCaseError> {
- let sampling = Sampling::<B>::new(String::from("test/sampling"), T::data_type(), vocab_size as u32);
+ let sampling = Sampling::<B>::new(T::data_type(), vocab_size as u32);
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
let logits_buffer = command_buffer.allocate_constant_from_slice(logits).unwrap();