A high-performance inference engine for AI models
Better labels (#672)
uuuvnf68d506ef44e
diff --git a/crates/backend-uzu/src/backends/common/command_buffer.rs b/crates/backend-uzu/src/backends/common/command_buffer.rsindex d0f7b72d5..e9ba90c48 100644--- a/crates/backend-uzu/src/backends/common/command_buffer.rs+++ b/crates/backend-uzu/src/backends/common/command_buffer.rs@@ -91,6 +91,13 @@ pub trait CommandBufferEncoding {before: AccessFlags,);+ fn push_debug_group(+ &mut self,+ name: &str,+ );++ fn pop_debug_group(&mut self);+fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;}diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rsindex 640ab6b99..df10f9b74 100644--- a/crates/backend-uzu/src/backends/common/context.rs+++ b/crates/backend-uzu/src/backends/common/context.rs@@ -8,7 +8,8 @@ pub trait Context: Sized + Send + Sync {fn new() -> Result<Arc<Self>, <Self::Backend as Backend>::Error>;fn create_command_buffer(- &self+ &self,+ name: Option<&str>,) -> Result<<<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Initial, <Self::Backend as Backend>::Error>;fn create_buffer(diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rsindex 1710f3fc3..995608391 100644--- a/crates/backend-uzu/src/backends/common/encoder.rs+++ b/crates/backend-uzu/src/backends/common/encoder.rs@@ -43,14 +43,22 @@ pub struct Encoder<'encoding, B: Backend> {impl<'encoding, B: Backend> Encoder<'encoding, B> {pub fn new(context: &'encoding B::Context) -> Result<Self, B::Error> {- Self::new_with_pool(context, Arc::new(context.create_allocation_pool(false)))+ Self::new_with_name(context, None)}- pub fn new_with_pool(+ pub fn new_with_name(+ context: &'encoding B::Context,+ name: Option<&str>,+ ) -> Result<Self, B::Error> {+ Self::new_with_pool_name(context, Arc::new(context.create_allocation_pool(false)), name)+ }++ pub fn new_with_pool_name(context: &'encoding B::Context,allocation_pool: Arc<AllocationPool<B>>,+ name: Option<&str>,) -> Result<Self, B::Error> {- let command_buffer = context.create_command_buffer()?.start_encoding();+ let command_buffer = context.create_command_buffer(name)?.start_encoding();let hazard_tracker = HazardTracker::new();Ok(Self {@@ -144,6 +152,17 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {self.command_buffer.encode_fill(dst_buffer_range, value);}+ pub fn push_debug_group(+ &mut self,+ name: &str,+ ) {+ self.command_buffer.push_debug_group(name);+ }++ pub fn pop_debug_group(&mut self) {+ self.command_buffer.pop_debug_group();+ }+pub fn access(&mut self,accesses: &[Access],diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rsindex fbcf55c3b..f04581dd9 100644--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs@@ -110,6 +110,14 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {) {}+ fn push_debug_group(+ &mut self,+ _name: &str,+ ) {+ }++ fn pop_debug_group(&mut self) {}+fn end_encoding(self) -> CpuCommandBufferExecutable {CpuCommandBufferExecutable {command_queue: self.command_queue,diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rsindex cbb9cf074..9055f98d2 100644--- a/crates/backend-uzu/src/backends/cpu/context.rs+++ b/crates/backend-uzu/src/backends/cpu/context.rs@@ -54,7 +54,10 @@ impl Context for CpuContext {self.allocator.create_pool(reusable)}- fn create_command_buffer(&self) -> Result<CpuCommandBufferInitial, CpuError> {+ fn create_command_buffer(+ &self,+ _name: Option<&str>,+ ) -> Result<CpuCommandBufferInitial, CpuError> {Ok(CpuCommandBufferInitial::new(self.command_queue.clone()))}diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rsindex a46c71430..64115e083 100644--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs@@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration};use metal::{MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferStatus,- MTLCommandEncoder, MTLCommandQueue, MTLComputeCommandEncoder,+ MTLCommandEncoder, MTLCommandEncoderExt, MTLCommandQueue, MTLComputeCommandEncoder,};use objc2::{rc::Retained, runtime::ProtocolObject};@@ -81,9 +81,10 @@ impl MetalCommandBufferEncoding {pub(super) fn ensure_compute(&mut self) -> &mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>> {if !matches!(self.encoding_state, MetalCommandBufferEncodingEncodingState::Compute(_)) {self.ensure_none();- self.encoding_state = MetalCommandBufferEncodingEncodingState::Compute(- self.command_buffer.compute_command_encoder().expect("Failed to create compute command encoder"),- );+ let compute_encoder =+ self.command_buffer.compute_command_encoder().expect("Failed to create compute command encoder");+ compute_encoder.set_label(self.command_buffer.label().as_deref());+ self.encoding_state = MetalCommandBufferEncodingEncodingState::Compute(compute_encoder);}let MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) = &mut self.encoding_state else {@@ -157,6 +158,17 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {) {}+ fn push_debug_group(+ &mut self,+ name: &str,+ ) {+ self.command_buffer.push_debug_group(name);+ }++ fn pop_debug_group(&mut self) {+ self.command_buffer.pop_debug_group();+ }+fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {self.ensure_none();diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex 75d35ab4d..f31b27958 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -11,8 +11,8 @@ use std::{use metal::MTLSharedEvent;use metal::{MTL4CommandQueue, MTL4CommandQueueExt, MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager,- MTLCommandQueue, MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,- MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions, MTLSparsePageSize,+ MTLCommandBufferExt, MTLCommandQueue, MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt,+ MTLEvent, MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions, MTLSparsePageSize,};use objc2::{rc::Retained, runtime::ProtocolObject};use parking_lot::{Mutex, MutexGuard};@@ -210,11 +210,14 @@ impl Context for MetalContext {self.allocator.create_pool(reusable)}- fn create_command_buffer(&self) -> Result<MetalCommandBufferInitial, MetalError> {- Ok(MetalCommandBufferInitial::new(- self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?,- self.weak_self.upgrade().unwrap(), // never fails- ))+ fn create_command_buffer(+ &self,+ name: Option<&str>,+ ) -> Result<MetalCommandBufferInitial, MetalError> {+ let command_buffer = self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?;+ command_buffer.set_label(name);+ let context = self.weak_self.upgrade().unwrap(); // never fails+ Ok(MetalCommandBufferInitial::new(command_buffer, context))}fn create_sparse_buffer(diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex a33d7cc84..1b2e16bc0 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -124,6 +124,8 @@ impl<B: Backend> Decoder<B> {state: &mut TransformerState<B>,encoder: &mut Encoder<B>,) -> Result<DecoderEncodeOutput<B>, DecoderError<B>> {+ encoder.push_debug_group("decoder");+let embedded = self.embedding.encode_lookup(token_ids, batch_dim.size(), encoder)?;let per_layer_inputs = if let Some(per_layer_embedding) = &self.per_layer_embedding {@@ -161,6 +163,8 @@ impl<B: Backend> Decoder<B> {transformer_output.output};+ encoder.pop_debug_group();+Ok(DecoderEncodeOutput {logits,hidden_features: transformer_output.hidden_features,diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rsindex 0461c7328..63cbd6853 100644--- a/crates/backend-uzu/src/encodable_block/dflash.rs+++ b/crates/backend-uzu/src/encodable_block/dflash.rs@@ -227,6 +227,8 @@ impl<B: Backend> DFlashDraft<B> {return Ok(());}+ encoder.push_debug_group("dflash append state");+let num_tokens = accepted_indices.len();let captured_layer_count = self.target_feature_input_dim / self.model_dim;assert_eq!(target_features.len(), captured_layer_count);@@ -281,6 +283,9 @@ impl<B: Backend> DFlashDraft<B> {}state.context_length += num_tokens;++ encoder.pop_debug_group();+Ok(())}@@ -334,6 +339,8 @@ impl<B: Backend> DFlashDraft<B> {candidate_count: usize,encoder: &mut Encoder<B>,) -> Result<DFlashDraftOutput<B>, DFlashDraftEncodeError<B>> {+ encoder.push_debug_group("dflash draft");+let mut token_ids = encoder.allocate_constant(self.block_size * DataType::U32.size_in_bytes()).map_err(DFlashDraftEncodeError::Backend)?;@@ -359,6 +366,8 @@ impl<B: Backend> DFlashDraft<B> {drop(token_ids);drop(lookahead_hidden);+ encoder.pop_debug_group();+Ok(DFlashDraftOutput {candidates,draft_hidden,diff --git a/crates/backend-uzu/src/encodable_block/sampling/mod.rs b/crates/backend-uzu/src/encodable_block/sampling/mod.rsindex 2f322b98f..135f513c0 100644--- a/crates/backend-uzu/src/encodable_block/sampling/mod.rs+++ b/crates/backend-uzu/src/encodable_block/sampling/mod.rs@@ -92,6 +92,8 @@ impl<B: Backend> Sampling<B> {sampling_range: Range<usize>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {+ encoder.push_debug_group("sampling");+let (is_stochastic, temperature, top_k, top_p, min_p, repetition_penalty, suffix_repetition_length) =match sampling_method {SamplingMethod::Greedy => (false, None, None, None, None, None, None),@@ -185,6 +187,8 @@ impl<B: Backend> Sampling<B> {encoder,);+ encoder.pop_debug_group();+Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rsindex 37459df24..21ff38da8 100644--- a/crates/backend-uzu/src/encodable_block/transformer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer.rs@@ -58,6 +58,8 @@ impl<B: Backend> TransformerState<B> {accepted_indices: &[usize],encoder: &mut Encoder<B>,) -> Result<(), B::Error> {+ encoder.push_debug_group("transformer accept");+for layer_state in &mut self.layer_states {let TransformerLayerStateType::Owned(layer_state) = layer_state else {continue;@@ -68,6 +70,8 @@ impl<B: Backend> TransformerState<B> {self.context_length += accepted_indices.len();+ encoder.pop_debug_group();+Ok(())}}diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rsindex b0f928c55..4913cdde2 100644--- a/crates/backend-uzu/src/encodable_block/weaver.rs+++ b/crates/backend-uzu/src/encodable_block/weaver.rs@@ -301,6 +301,7 @@ impl<B: Backend> Weaver<B> {shape: TreeShape,encoder: &mut Encoder<B>,) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {+ encoder.push_debug_group("dflash weaver");let tree_slot_count = shape.budget + 1;let frontier_capacity = tree_slot_count * shape.children_per_node;let lookahead_count = self.max_depth.min(inputs.candidates.depth_count);@@ -437,6 +438,7 @@ impl<B: Backend> Weaver<B> {);batch_start_slot += batch_node_count;}+ encoder.pop_debug_group();Ok(EncodedWeaverTree {packed_tree,})diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex b7d01ad76..bafecc337 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -201,8 +201,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {).map_err(LanguageModelStreamError::Backend)?;- let mut encoder = Encoder::<B>::new_with_pool(&model.engine.context, allocation_pool.clone())- .map_err(LanguageModelStreamError::Backend)?;+ let mut encoder =+ Encoder::<B>::new_with_pool_name(&model.engine.context, allocation_pool.clone(), Some("prefill"))+ .map_err(LanguageModelStreamError::Backend)?;let mut output_tokens = None;let mut output_norm = None;@@ -432,9 +433,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let accepted_token_indicies = full.iter().map(|(i, _, _)| *i).collect::<Box<[usize]>>();let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();- let mut encoder =- Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())- .map_err(LanguageModelStreamError::Backend)?;+ let mut encoder = Encoder::<B>::new_with_pool_name(+ &self.model.engine.context,+ self.allocation_pool.clone(),+ Some("decode"),+ )+ .map_err(LanguageModelStreamError::Backend)?;self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut encoder)@@ -460,6 +464,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {};if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length(){+ encoder.push_debug_group("update repetition penalty ring");let mut accepted_input_token_ids_const = encoder.allocate_constant(full.len() * DataType::U32.size_in_bytes()).map_err(LanguageModelStreamError::Backend)?;@@ -476,6 +481,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {full.len() as u32,&mut encoder,);+ encoder.pop_debug_group();}self.model_state.tokens.extend(accepted_output_token_ids);(@@ -523,7 +529,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let mut encoder = if let Some(encoder) = encoder {encoder} else {- Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())+ Encoder::<B>::new_with_pool_name(&self.model.engine.context, self.allocation_pool.clone(), Some("decode")).map_err(LanguageModelStreamError::Backend)?};@@ -540,8 +546,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {self.options.grammar.as_mut(),)? {pending.push(encoder.end_encoding().submit());- encoder = Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())- .map_err(LanguageModelStreamError::Backend)?;+ encoder = Encoder::<B>::new_with_pool_name(+ &self.model.engine.context,+ self.allocation_pool.clone(),+ Some("decode"),+ )+ .map_err(LanguageModelStreamError::Backend)?;let trie = speculator.propose_tree(self.model_state.speculator_state.as_mut().unwrap(),output_norm,@@ -608,8 +618,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if chain_copy.is_some() {pending.push(encoder.end_encoding().submit());- let mut encoder = Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())- .map_err(LanguageModelStreamError::Backend)?;+ let mut encoder = Encoder::<B>::new_with_pool_name(+ &self.model.engine.context,+ self.allocation_pool.clone(),+ Some("decode"),+ )+ .map_err(LanguageModelStreamError::Backend)?;let mut bitmask = encoder.allocate_constant(@@ -763,8 +777,12 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {}if !in_flight.full_accept {- let mut encoder =- Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone()).unwrap();+ let mut encoder = Encoder::<B>::new_with_pool_name(+ &self.model.engine.context,+ self.allocation_pool.clone(),+ Some("drop accept"),+ )+ .unwrap();self.model_state.transformer_state.encode_accept(&[0], &mut encoder).unwrap();if let Some(speculator) = self.model.speculator.as_ref() {speculator@@ -789,8 +807,12 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {} => {assert!(num_accepted > 0 && num_accepted < full.len());- let mut encoder =- Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone()).unwrap();+ let mut encoder = Encoder::<B>::new_with_pool_name(+ &self.model.engine.context,+ self.allocation_pool.clone(),+ Some("drop accept"),+ )+ .unwrap();let accepted_token_indicies =full.iter().take(num_accepted + 1).map(|(i, _, _)| *i).collect::<Box<[usize]>>();self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut encoder).unwrap();diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rsindex 6ec23c0ad..3dea1b19c 100644--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs@@ -157,7 +157,8 @@ impl<B: Backend> DFlashSpeculator<B> {}let root_position = state.context_length();- let mut encoder = Encoder::new(&*self.context).map_err(DFlashTreeError::Backend)?;+ let mut encoder =+ Encoder::new_with_name(&*self.context, Some("speculator propose")).map_err(DFlashTreeError::Backend)?;let DFlashDraftOutput {candidates,draft_hidden,