Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Better labels (#672)

uuuvnf68d506ef44e
diff --git a/crates/backend-uzu/src/backends/common/command_buffer.rs b/crates/backend-uzu/src/backends/common/command_buffer.rs
index d0f7b72d5..e9ba90c48 100644
--- a/crates/backend-uzu/src/backends/common/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/common/command_buffer.rs
@@ -91,6 +91,13 @@ pub trait CommandBufferEncoding {
before: AccessFlags,
);
+ fn push_debug_group(
+ &mut self,
+ name: &str,
+ );
+
+ fn pop_debug_group(&mut self);
+
fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;
}
diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rs
index 640ab6b99..df10f9b74 100644
--- a/crates/backend-uzu/src/backends/common/context.rs
+++ b/crates/backend-uzu/src/backends/common/context.rs
@@ -8,7 +8,8 @@ pub trait Context: Sized + Send + Sync {
fn new() -> Result<Arc<Self>, <Self::Backend as Backend>::Error>;
fn create_command_buffer(
- &self
+ &self,
+ name: Option<&str>,
) -> Result<<<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Initial, <Self::Backend as Backend>::Error>;
fn create_buffer(
diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rs
index 1710f3fc3..995608391 100644
--- a/crates/backend-uzu/src/backends/common/encoder.rs
+++ b/crates/backend-uzu/src/backends/common/encoder.rs
@@ -43,14 +43,22 @@ pub struct Encoder<'encoding, B: Backend> {
impl<'encoding, B: Backend> Encoder<'encoding, B> {
pub fn new(context: &'encoding B::Context) -> Result<Self, B::Error> {
- Self::new_with_pool(context, Arc::new(context.create_allocation_pool(false)))
+ Self::new_with_name(context, None)
}
- pub fn new_with_pool(
+ pub fn new_with_name(
+ context: &'encoding B::Context,
+ name: Option<&str>,
+ ) -> Result<Self, B::Error> {
+ Self::new_with_pool_name(context, Arc::new(context.create_allocation_pool(false)), name)
+ }
+
+ pub fn new_with_pool_name(
context: &'encoding B::Context,
allocation_pool: Arc<AllocationPool<B>>,
+ name: Option<&str>,
) -> Result<Self, B::Error> {
- let command_buffer = context.create_command_buffer()?.start_encoding();
+ let command_buffer = context.create_command_buffer(name)?.start_encoding();
let hazard_tracker = HazardTracker::new();
Ok(Self {
@@ -144,6 +152,17 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
self.command_buffer.encode_fill(dst_buffer_range, value);
}
+ pub fn push_debug_group(
+ &mut self,
+ name: &str,
+ ) {
+ self.command_buffer.push_debug_group(name);
+ }
+
+ pub fn pop_debug_group(&mut self) {
+ self.command_buffer.pop_debug_group();
+ }
+
pub fn access(
&mut self,
accesses: &[Access],
diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
index fbcf55c3b..f04581dd9 100644
--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
@@ -110,6 +110,14 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
) {
}
+ fn push_debug_group(
+ &mut self,
+ _name: &str,
+ ) {
+ }
+
+ fn pop_debug_group(&mut self) {}
+
fn end_encoding(self) -> CpuCommandBufferExecutable {
CpuCommandBufferExecutable {
command_queue: self.command_queue,
diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rs
index cbb9cf074..9055f98d2 100644
--- a/crates/backend-uzu/src/backends/cpu/context.rs
+++ b/crates/backend-uzu/src/backends/cpu/context.rs
@@ -54,7 +54,10 @@ impl Context for CpuContext {
self.allocator.create_pool(reusable)
}
- fn create_command_buffer(&self) -> Result<CpuCommandBufferInitial, CpuError> {
+ fn create_command_buffer(
+ &self,
+ _name: Option<&str>,
+ ) -> Result<CpuCommandBufferInitial, CpuError> {
Ok(CpuCommandBufferInitial::new(self.command_queue.clone()))
}
diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rs
index a46c71430..64115e083 100644
--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs
@@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration};
use metal::{
MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferStatus,
- MTLCommandEncoder, MTLCommandQueue, MTLComputeCommandEncoder,
+ MTLCommandEncoder, MTLCommandEncoderExt, MTLCommandQueue, MTLComputeCommandEncoder,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
@@ -81,9 +81,10 @@ impl MetalCommandBufferEncoding {
pub(super) fn ensure_compute(&mut self) -> &mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>> {
if !matches!(self.encoding_state, MetalCommandBufferEncodingEncodingState::Compute(_)) {
self.ensure_none();
- self.encoding_state = MetalCommandBufferEncodingEncodingState::Compute(
- self.command_buffer.compute_command_encoder().expect("Failed to create compute command encoder"),
- );
+ let compute_encoder =
+ self.command_buffer.compute_command_encoder().expect("Failed to create compute command encoder");
+ compute_encoder.set_label(self.command_buffer.label().as_deref());
+ self.encoding_state = MetalCommandBufferEncodingEncodingState::Compute(compute_encoder);
}
let MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) = &mut self.encoding_state else {
@@ -157,6 +158,17 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
) {
}
+ fn push_debug_group(
+ &mut self,
+ name: &str,
+ ) {
+ self.command_buffer.push_debug_group(name);
+ }
+
+ fn pop_debug_group(&mut self) {
+ self.command_buffer.pop_debug_group();
+ }
+
fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {
self.ensure_none();
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index 75d35ab4d..f31b27958 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -11,8 +11,8 @@ use std::{
use metal::MTLSharedEvent;
use metal::{
MTL4CommandQueue, MTL4CommandQueueExt, MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager,
- MTLCommandQueue, MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,
- MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions, MTLSparsePageSize,
+ MTLCommandBufferExt, MTLCommandQueue, MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt,
+ MTLEvent, MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions, MTLSparsePageSize,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use parking_lot::{Mutex, MutexGuard};
@@ -210,11 +210,14 @@ impl Context for MetalContext {
self.allocator.create_pool(reusable)
}
- fn create_command_buffer(&self) -> Result<MetalCommandBufferInitial, MetalError> {
- Ok(MetalCommandBufferInitial::new(
- self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?,
- self.weak_self.upgrade().unwrap(), // never fails
- ))
+ fn create_command_buffer(
+ &self,
+ name: Option<&str>,
+ ) -> Result<MetalCommandBufferInitial, MetalError> {
+ let command_buffer = self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?;
+ command_buffer.set_label(name);
+ let context = self.weak_self.upgrade().unwrap(); // never fails
+ Ok(MetalCommandBufferInitial::new(command_buffer, context))
}
fn create_sparse_buffer(
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index a33d7cc84..1b2e16bc0 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -124,6 +124,8 @@ impl<B: Backend> Decoder<B> {
state: &mut TransformerState<B>,
encoder: &mut Encoder<B>,
) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {
+ encoder.push_debug_group("decoder");
+
let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), encoder)?;
let per_layer_inputs = if let Some(per_layer_embedding) = &self.per_layer_embedding {
@@ -161,6 +163,8 @@ impl<B: Backend> Decoder<B> {
transformer_output.output
};
+ encoder.pop_debug_group();
+
Ok(DecoderEncodeOutput {
logits,
hidden_features: transformer_output.hidden_features,
diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rs
index 0461c7328..63cbd6853 100644
--- a/crates/backend-uzu/src/encodable_block/dflash.rs
+++ b/crates/backend-uzu/src/encodable_block/dflash.rs
@@ -227,6 +227,8 @@ impl<B: Backend> DFlashDraft<B> {
return Ok(());
}
+ encoder.push_debug_group("dflash append state");
+
let num_tokens = accepted_indices.len();
let captured_layer_count = self.target_feature_input_dim / self.model_dim;
assert_eq!(target_features.len(), captured_layer_count);
@@ -281,6 +283,9 @@ impl<B: Backend> DFlashDraft<B> {
}
state.context_length += num_tokens;
+
+ encoder.pop_debug_group();
+
Ok(())
}
@@ -334,6 +339,8 @@ impl<B: Backend> DFlashDraft<B> {
candidate_count: usize,
encoder: &mut Encoder<B>,
) -> Result<DFlashDraftOutput<B>, DFlashDraftEncodeError<B>> {
+ encoder.push_debug_group("dflash draft");
+
let mut token_ids = encoder
.allocate_constant(self.block_size * DataType::U32.size_in_bytes())
.map_err(DFlashDraftEncodeError::Backend)?;
@@ -359,6 +366,8 @@ impl<B: Backend> DFlashDraft<B> {
drop(token_ids);
drop(lookahead_hidden);
+ encoder.pop_debug_group();
+
Ok(DFlashDraftOutput {
candidates,
draft_hidden,
diff --git a/crates/backend-uzu/src/encodable_block/sampling/mod.rs b/crates/backend-uzu/src/encodable_block/sampling/mod.rs
index 2f322b98f..135f513c0 100644
--- a/crates/backend-uzu/src/encodable_block/sampling/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/sampling/mod.rs
@@ -92,6 +92,8 @@ impl<B: Backend> Sampling<B> {
sampling_range: Range<usize>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
+ encoder.push_debug_group("sampling");
+
let (is_stochastic, temperature, top_k, top_p, min_p, repetition_penalty, suffix_repetition_length) =
match sampling_method {
SamplingMethod::Greedy => (false, None, None, None, None, None, None),
@@ -185,6 +187,8 @@ impl<B: Backend> Sampling<B> {
encoder,
);
+ encoder.pop_debug_group();
+
Ok(output)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rs
index 37459df24..21ff38da8 100644
--- a/crates/backend-uzu/src/encodable_block/transformer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer.rs
@@ -58,6 +58,8 @@ impl<B: Backend> TransformerState<B> {
accepted_indices: &[usize],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
+ encoder.push_debug_group("transformer accept");
+
for layer_state in &mut self.layer_states {
let TransformerLayerStateType::Owned(layer_state) = layer_state else {
continue;
@@ -68,6 +70,8 @@ impl<B: Backend> TransformerState<B> {
self.context_length += accepted_indices.len();
+ encoder.pop_debug_group();
+
Ok(())
}
}
diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rs
index b0f928c55..4913cdde2 100644
--- a/crates/backend-uzu/src/encodable_block/weaver.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver.rs
@@ -301,6 +301,7 @@ impl<B: Backend> Weaver<B> {
shape: TreeShape,
encoder: &mut Encoder<B>,
) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
+ encoder.push_debug_group("dflash weaver");
let tree_slot_count = shape.budget + 1;
let frontier_capacity = tree_slot_count * shape.children_per_node;
let lookahead_count = self.max_depth.min(inputs.candidates.depth_count);
@@ -437,6 +438,7 @@ impl<B: Backend> Weaver<B> {
);
batch_start_slot += batch_node_count;
}
+ encoder.pop_debug_group();
Ok(EncodedWeaverTree {
packed_tree,
})
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index b7d01ad76..bafecc337 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -201,8 +201,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
)
.map_err(LanguageModelStreamError::Backend)?;
- let mut encoder = Encoder::<B>::new_with_pool(&model.engine.context, allocation_pool.clone())
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut encoder =
+ Encoder::<B>::new_with_pool_name(&model.engine.context, allocation_pool.clone(), Some("prefill"))
+ .map_err(LanguageModelStreamError::Backend)?;
let mut output_tokens = None;
let mut output_norm = None;
@@ -432,9 +433,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let accepted_token_indicies = full.iter().map(|(i, _, _)| *i).collect::<Box<[usize]>>();
let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();
let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();
- let mut encoder =
- Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut encoder = Encoder::<B>::new_with_pool_name(
+ &self.model.engine.context,
+ self.allocation_pool.clone(),
+ Some("decode"),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
self.model_state
.transformer_state
.encode_accept(&accepted_token_indicies, &mut encoder)
@@ -460,6 +464,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
};
if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length()
{
+ encoder.push_debug_group("update repetition penalty ring");
let mut accepted_input_token_ids_const = encoder
.allocate_constant(full.len() * DataType::U32.size_in_bytes())
.map_err(LanguageModelStreamError::Backend)?;
@@ -476,6 +481,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
full.len() as u32,
&mut encoder,
);
+ encoder.pop_debug_group();
}
self.model_state.tokens.extend(accepted_output_token_ids);
(
@@ -523,7 +529,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let mut encoder = if let Some(encoder) = encoder {
encoder
} else {
- Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())
+ Encoder::<B>::new_with_pool_name(&self.model.engine.context, self.allocation_pool.clone(), Some("decode"))
.map_err(LanguageModelStreamError::Backend)?
};
@@ -540,8 +546,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.options.grammar.as_mut(),
)? {
pending.push(encoder.end_encoding().submit());
- encoder = Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())
- .map_err(LanguageModelStreamError::Backend)?;
+ encoder = Encoder::<B>::new_with_pool_name(
+ &self.model.engine.context,
+ self.allocation_pool.clone(),
+ Some("decode"),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
let trie = speculator.propose_tree(
self.model_state.speculator_state.as_mut().unwrap(),
output_norm,
@@ -608,8 +618,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if chain_copy.is_some() {
pending.push(encoder.end_encoding().submit());
- let mut encoder = Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())
- .map_err(LanguageModelStreamError::Backend)?;
+ let mut encoder = Encoder::<B>::new_with_pool_name(
+ &self.model.engine.context,
+ self.allocation_pool.clone(),
+ Some("decode"),
+ )
+ .map_err(LanguageModelStreamError::Backend)?;
let mut bitmask = encoder
.allocate_constant(
@@ -763,8 +777,12 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
}
if !in_flight.full_accept {
- let mut encoder =
- Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone()).unwrap();
+ let mut encoder = Encoder::<B>::new_with_pool_name(
+ &self.model.engine.context,
+ self.allocation_pool.clone(),
+ Some("drop accept"),
+ )
+ .unwrap();
self.model_state.transformer_state.encode_accept(&[0], &mut encoder).unwrap();
if let Some(speculator) = self.model.speculator.as_ref() {
speculator
@@ -789,8 +807,12 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
} => {
assert!(num_accepted > 0 && num_accepted < full.len());
- let mut encoder =
- Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone()).unwrap();
+ let mut encoder = Encoder::<B>::new_with_pool_name(
+ &self.model.engine.context,
+ self.allocation_pool.clone(),
+ Some("drop accept"),
+ )
+ .unwrap();
let accepted_token_indicies =
full.iter().take(num_accepted + 1).map(|(i, _, _)| *i).collect::<Box<[usize]>>();
self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut encoder).unwrap();
diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rs
index 6ec23c0ad..3dea1b19c 100644
--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs
+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs
@@ -157,7 +157,8 @@ impl<B: Backend> DFlashSpeculator<B> {
}
let root_position = state.context_length();
- let mut encoder = Encoder::new(&*self.context).map_err(DFlashTreeError::Backend)?;
+ let mut encoder =
+ Encoder::new_with_name(&*self.context, Some("speculator propose")).map_err(DFlashTreeError::Backend)?;
let DFlashDraftOutput {
candidates,
draft_hidden,