A high-performance inference engine for AI models
Async decoding (#52)
Aleksei Savin0930977fdbd5
diff --git a/Cargo.lock b/Cargo.lockindex d0097c700..b12c69953 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2643,6 +2643,7 @@ name = "uzu"version = "0.1.6"dependencies = ["bitflags 2.9.4",+ "block","bytemuck","clap","half",diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 800a969d4..5becfbbac 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -15,6 +15,7 @@ mpsgraph = "0.2.0"objc2 = "0.6.2"objc2-foundation = "0.3.1"metal = "0.32.0"+block = "0.1.6"bytemuck = "1.23.2"num-traits = "0.2.19"regex = "1.11.2"diff --git a/crates/uzu/src/backends/metal/array.rs b/crates/uzu/src/backends/metal/array.rsindex bd8b1f40f..6b854a0b0 100644--- a/crates/uzu/src/backends/metal/array.rs+++ b/crates/uzu/src/backends/metal/array.rs@@ -16,6 +16,8 @@ pub struct MetalArray {shape: Box<[usize]>,/// Data type of the elementsdata_type: DataType,+ /// Byte offset into the buffer (for indexed async buffers)+ offset: usize,}impl Array for MetalArray {@@ -30,7 +32,7 @@ impl Array for MetalArray {fn buffer(&self) -> &[u8] {unsafe {std::slice::from_raw_parts(- self.buffer.contents() as *const u8,+ (self.buffer.contents() as *const u8).add(self.offset),self.size_in_bytes(),)}@@ -39,7 +41,7 @@ impl Array for MetalArray {fn buffer_mut(&mut self) -> &mut [u8] {unsafe {std::slice::from_raw_parts_mut(- self.buffer.contents() as *mut u8,+ (self.buffer.contents() as *mut u8).add(self.offset),self.size_in_bytes(),)}@@ -47,28 +49,46 @@ impl Array for MetalArray {}impl MetalArray {- /// Create a new Array+ /// Create a new Array at offset 0pub unsafe fn new(buffer: MTLBuffer,shape: &[usize],data_type: DataType,+ ) -> Self {+ unsafe { Self::new_with_offset(buffer, shape, data_type, 0) }+ }++ /// Create a new Array at a byte offset into the buffer.+ /// Used for indexed async buffers where each pass uses a different offset.+ pub unsafe fn new_with_offset(+ buffer: MTLBuffer,+ shape: &[usize],+ data_type: DataType,+ offset: usize,) -> Self {let required_bytes = array_size_in_bytes(shape, data_type);assert!(- required_bytes <= buffer.length() as usize,- "Shape {:?} with data type {:?} requires {} bytes, but buffer length is {} bytes",+ offset + required_bytes <= buffer.length() as usize,+ "Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",shape,data_type,- required_bytes,+ offset,+ offset + required_bytes,buffer.length());Self {buffer,shape: shape.into(),data_type,+ offset,}}+ /// Returns the byte offset into the underlying buffer+ pub fn buffer_offset(&self) -> usize {+ self.offset+ }+/// Wraps the underlying MTLBuffer into MPSTensorData for use with MPSGraph.pub unsafe fn to_mps_tensor_data(&mut self) -> Retained<TensorData> {TensorData::new_with_mtl_buffer(@@ -80,6 +100,8 @@ impl MetalArray {}/// Returns the underlying MTLBuffer.+ /// NOTE: For arrays created with `new_with_offset`, callers must use `buffer_offset()`+ /// and pass it to `encoder.set_buffer(..., offset)` - Metal has no sub-buffer concept.pub unsafe fn mtl_buffer(&mut self) -> &MTLBuffer {&self.buffer}diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex b3c533e18..5ef3e1712 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -1,7 +1,7 @@#![allow(dead_code)]include!(concat!(env!("OUT_DIR"), "/metal_lib.rs"));-use std::rc::Rc;+use std::{cell::RefCell, collections::HashMap, rc::Rc};use metal::{CommandQueue as MTLCommandQueue,@@ -18,6 +18,7 @@ pub struct MTLContext {pub device: MTLDevice,pub command_queue: MTLCommandQueue,library: MTLLibrary,+ pipeline_cache: RefCell<HashMap<String, MTLComputePipelineState>>,}impl MTLContext {@@ -39,6 +40,7 @@ impl MTLContext {device,command_queue,library,+ pipeline_cache: RefCell::new(HashMap::new()),})}@@ -47,7 +49,33 @@ impl MTLContext {function_name: &str,constants: Option<&FunctionConstantValues>,) -> Result<MTLComputePipelineState, MTLError> {- self.library.compute_pipeline_state(function_name, constants)+ // Only cache pipelines without constants+ if constants.is_some() {+ return self+ .library+ .compute_pipeline_state(function_name, constants);+ }+ self.compute_pipeline_state_cached(function_name, function_name, None)+ }++ pub fn compute_pipeline_state_cached(+ &self,+ cache_key: &str,+ function_name: &str,+ constants: Option<&FunctionConstantValues>,+ ) -> Result<MTLComputePipelineState, MTLError> {+ if let Some(pipeline) = self.pipeline_cache.borrow().get(cache_key) {+ return Ok(pipeline.clone());+ }++ let pipeline =+ self.library.compute_pipeline_state(function_name, constants)?;++ self.pipeline_cache+ .borrow_mut()+ .insert(cache_key.to_string(), pipeline.clone());++ Ok(pipeline)}pub fn compute_pipeline_state_with_reflection(@@ -55,8 +83,39 @@ impl MTLContext {function_name: &str,constants: Option<&FunctionConstantValues>,) -> Result<(MTLComputePipelineState, Vec<String>), MTLError> {- self.library- .compute_pipeline_state_with_reflection(function_name, constants)+ // Only cache pipelines without constants+ if constants.is_some() {+ return self.library.compute_pipeline_state_with_reflection(+ function_name,+ constants,+ );+ }+ self.compute_pipeline_state_with_reflection_cached(+ function_name,+ function_name,+ None,+ )+ }++ pub fn compute_pipeline_state_with_reflection_cached(+ &self,+ cache_key: &str,+ function_name: &str,+ constants: Option<&FunctionConstantValues>,+ ) -> Result<(MTLComputePipelineState, Vec<String>), MTLError> {+ if let Some(pipeline) = self.pipeline_cache.borrow().get(cache_key) {+ return Ok((pipeline.clone(), Vec::new()));+ }++ let (pipeline, arg_names) = self+ .library+ .compute_pipeline_state_with_reflection(function_name, constants)?;++ self.pipeline_cache+ .borrow_mut()+ .insert(cache_key.to_string(), pipeline.clone());++ Ok((pipeline, arg_names))}}diff --git a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rsindex 40a4b83a2..9113bfb03 100644--- a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs+++ b/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs@@ -221,4 +221,8 @@ impl EncodableWithState for DecoderExecutables {state.copy_array(ArrayId::Logits, traces.borrow().logits.clone());}}++ fn supports_shared_encoder(&self) -> bool {+ false+ }}diff --git a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rsindex 98e26526f..a4dcb4b25 100644--- a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs+++ b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs@@ -1,5 +1,6 @@use std::rc::Rc;+use metal::ComputeCommandEncoderRef;use mpsgraph::CommandBuffer as MPSCommandBuffer;use objc2::rc::autoreleasepool;@@ -9,7 +10,7 @@ use crate::{MTLContext,compilation_parameters::CompilationConfig,forward_pass::{- ArrayId, ForwardPassState,+ ArrayId, EncoderResolver, ForwardPassState,encodable_with_state::{EncodableWithState, EncodingParameters},transformer_layer,},@@ -293,8 +294,170 @@ impl LayerExecutables {}}-impl EncodableWithState for LayerExecutables {- fn encode(+impl LayerExecutables {+ fn encode_with_shared_encoder_impl(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,+ ) {+ let layer_traces = if let Some(traces) = state.traces.clone() {+ traces.borrow().layer_results.get(self.layer_index).cloned()+ } else {+ None+ };++ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().inputs.clone(),+ );+ }++ self.copy_main_to_shortcut+ .encode_with_shared_encoder(state, encoder, parameters);++ self.pre_attention_norm+ .encode_with_shared_encoder(state, encoder, parameters);+ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().pre_attention_norm.clone(),+ );+ }++ match &self.mixer {+ MixerExecutables::Attention {+ qkv_projection,+ qk_norm,+ rope,+ attention,+ out_projection,+ } => {+ qkv_projection+ .encode_with_shared_encoder(state, encoder, parameters);+ if let Some(norm) = qk_norm {+ norm.encode_with_shared_encoder(state, encoder, parameters);+ }+ rope.encode_with_shared_encoder(state, encoder, parameters);+ attention+ .encode_with_shared_encoder(state, encoder, parameters);+ out_projection+ .encode_with_shared_encoder(state, encoder, parameters);+ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().attention.clone(),+ );+ }+ },+ MixerExecutables::StateSpace {+ mixer,+ } => {+ mixer.encode_with_shared_encoder(state, encoder, parameters);+ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().attention.clone(),+ );+ }+ },+ }++ if let Some(post_attention_norm) = &self.post_attention_norm {+ post_attention_norm+ .encode_with_shared_encoder(state, encoder, parameters);+ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().post_attention_norm.clone(),+ );+ }+ }++ self.main_shortcut_add_swap+ .encode_with_shared_encoder(state, encoder, parameters);+ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().mlp_inputs.clone(),+ );+ }++ self.pre_mlp_norm+ .encode_with_shared_encoder(state, encoder, parameters);+ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().pre_mlp_norm.clone(),+ );+ }++ self.mlp.encode_with_shared_encoder(state, encoder, parameters);+ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(ArrayId::Main, layer_traces.borrow().mlp.clone());+ }++ if let Some(post_mlp_norm) = &self.post_mlp_norm {+ post_mlp_norm+ .encode_with_shared_encoder(state, encoder, parameters);+ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().post_mlp_norm.clone(),+ );+ }+ }++ self.main_shortcut_add_swap+ .encode_with_shared_encoder(state, encoder, parameters);+ if let Some(layer_traces) = layer_traces.clone() {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().outputs.clone(),+ );+ }+ }++ fn all_blocks_support_shared_encoder(&self) -> bool {+ let mixer_supports = match &self.mixer {+ MixerExecutables::Attention {+ qkv_projection,+ qk_norm,+ rope,+ attention,+ out_projection,+ } => {+ qkv_projection.supports_shared_encoder()+ && qk_norm+ .as_ref()+ .map_or(true, |n| n.supports_shared_encoder())+ && rope.supports_shared_encoder()+ && attention.supports_shared_encoder()+ && out_projection.supports_shared_encoder()+ },+ MixerExecutables::StateSpace {+ mixer,+ } => mixer.supports_shared_encoder(),+ };++ self.copy_main_to_shortcut.supports_shared_encoder()+ && self.pre_attention_norm.supports_shared_encoder()+ && mixer_supports+ && self+ .post_attention_norm+ .as_ref()+ .map_or(true, |n| n.supports_shared_encoder())+ && self.main_shortcut_add_swap.supports_shared_encoder()+ && self.pre_mlp_norm.supports_shared_encoder()+ && self.mlp.supports_shared_encoder()+ && self+ .post_mlp_norm+ .as_ref()+ .map_or(true, |n| n.supports_shared_encoder())+ }++ fn encode_with_resolver(&self,state: &mut ForwardPassState,command_buffer: &MPSCommandBuffer,@@ -306,6 +469,8 @@ impl EncodableWithState for LayerExecutables {None};+ let mut resolver = EncoderResolver::new(command_buffer);+if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main,@@ -313,10 +478,9 @@ impl EncodableWithState for LayerExecutables {);}- self.copy_main_to_shortcut.encode(state, command_buffer, parameters);- // shortcut = input+ resolver.encode(self.copy_main_to_shortcut.as_ref(), state, parameters);- self.pre_attention_norm.encode(state, command_buffer, parameters);+ resolver.encode(self.pre_attention_norm.as_ref(), state, parameters);if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main,@@ -332,13 +496,13 @@ impl EncodableWithState for LayerExecutables {attention,out_projection,} => {- qkv_projection.encode(state, command_buffer, parameters);+ resolver.encode(qkv_projection.as_ref(), state, parameters);if let Some(norm) = qk_norm {- norm.encode(state, command_buffer, parameters);+ resolver.encode(norm.as_ref(), state, parameters);}- rope.encode(state, command_buffer, parameters);- attention.encode(state, command_buffer, parameters);- out_projection.encode(state, command_buffer, parameters);+ resolver.encode(rope.as_ref().as_ref(), state, parameters);+ resolver.encode(attention.as_ref(), state, parameters);+ resolver.encode(out_projection.as_ref(), state, parameters);if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main,@@ -349,7 +513,7 @@ impl EncodableWithState for LayerExecutables {MixerExecutables::StateSpace {mixer,} => {- mixer.encode(state, command_buffer, parameters);+ resolver.encode(mixer.as_ref(), state, parameters);if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main,@@ -360,7 +524,7 @@ impl EncodableWithState for LayerExecutables {}if let Some(post_attention_norm) = &self.post_attention_norm {- post_attention_norm.encode(state, command_buffer, parameters);+ resolver.encode(post_attention_norm.as_ref(), state, parameters);if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main,@@ -368,11 +532,12 @@ impl EncodableWithState for LayerExecutables {);}}- //main = attention_result- self.main_shortcut_add_swap.encode(state, command_buffer, parameters);- // shortcut = input + attention_result- // main = input + attention_result+ resolver.encode(+ self.main_shortcut_add_swap.as_ref(),+ state,+ parameters,+ );if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main,@@ -380,7 +545,7 @@ impl EncodableWithState for LayerExecutables {);}- self.pre_mlp_norm.encode(state, command_buffer, parameters);+ resolver.encode(self.pre_mlp_norm.as_ref(), state, parameters);if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main,@@ -388,13 +553,13 @@ impl EncodableWithState for LayerExecutables {);}- self.mlp.encode(state, command_buffer, parameters);+ resolver.encode(self.mlp.as_ref(), state, parameters);if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main, layer_traces.borrow().mlp.clone());}if let Some(post_mlp_norm) = &self.post_mlp_norm {- post_mlp_norm.encode(state, command_buffer, parameters);+ resolver.encode(post_mlp_norm.as_ref(), state, parameters);if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main,@@ -402,16 +567,50 @@ impl EncodableWithState for LayerExecutables {);}}- // main = mlp_result- self.main_shortcut_add_swap.encode(state, command_buffer, parameters);- // shortcut = input + attention_result + mlp_result- // main = input + attention_result + mlp_result+ resolver.encode(+ self.main_shortcut_add_swap.as_ref(),+ state,+ parameters,+ );if let Some(layer_traces) = layer_traces.clone() {state.copy_array(ArrayId::Main,layer_traces.borrow().outputs.clone(),);}++ resolver.finish();+ }+}++impl EncodableWithState for LayerExecutables {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ self.encode_with_resolver(state, command_buffer, parameters);++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ self.all_blocks_support_shared_encoder()+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,+ ) {+ self.encode_with_shared_encoder_impl(state, encoder, parameters);}}diff --git a/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs b/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rsindex 9a602adb2..1c486dd47 100644--- a/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs+++ b/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs@@ -1,15 +1,18 @@+use metal::{Buffer, BufferRef, ComputeCommandEncoderRef};use mpsgraph::CommandBuffer as MPSCommandBuffer;use super::ForwardPassState;-pub struct EncodingParameters {+#[derive(Clone)]+pub struct EncodingParameters<'a> {pub warmup: bool,pub enable_commit: bool,pub wait_until_completed: bool,pub projection_step: Option<usize>,+ pub predicate: Option<&'a Buffer>,}-impl EncodingParameters {+impl<'a> EncodingParameters<'a> {pub fn new(warmup: bool,enable_commit: bool,@@ -20,6 +23,7 @@ impl EncodingParameters {enable_commit,wait_until_completed,projection_step: None,+ predicate: None,}}@@ -30,6 +34,18 @@ impl EncodingParameters {self.projection_step = Some(projection_step);self}++ pub fn with_predicate(+ mut self,+ predicate: &'a Buffer,+ ) -> Self {+ self.predicate = Some(predicate);+ self+ }++ pub fn predicate_ref(&self) -> Option<&BufferRef> {+ self.predicate.map(|buffer| buffer.as_ref())+ }}pub trait EncodableWithState {@@ -39,4 +55,15 @@ pub trait EncodableWithState {command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,);++ fn supports_shared_encoder(&self) -> bool;++ fn encode_with_shared_encoder(+ &self,+ _state: &mut ForwardPassState,+ _encoder: &ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ panic!("encode_with_shared_encoder called on unsupported type");+ }}diff --git a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rsnew file mode 100644index 000000000..2fbe157cc--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs@@ -0,0 +1,57 @@+use metal::ComputeCommandEncoder;+use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{+ ForwardPassState,+ encodable_with_state::{EncodableWithState, EncodingParameters},+};++pub struct EncoderResolver<'a> {+ command_buffer: &'a MPSCommandBuffer,+ encoder: Option<ComputeCommandEncoder>,+}++impl<'a> EncoderResolver<'a> {+ pub fn new(command_buffer: &'a MPSCommandBuffer) -> Self {+ Self {+ command_buffer,+ encoder: None,+ }+ }++ pub fn encode(+ &mut self,+ block: &dyn EncodableWithState,+ state: &mut ForwardPassState,+ parameters: &EncodingParameters,+ ) {+ if block.supports_shared_encoder() {+ if self.encoder.is_none() {+ self.encoder = Some(+ self.command_buffer+ .root_command_buffer()+ .new_compute_command_encoder()+ .to_owned(),+ );+ }+ block.encode_with_shared_encoder(+ state,+ self.encoder.as_ref().unwrap(),+ parameters,+ );+ } else {+ self.end_current_encoder();+ block.encode(state, self.command_buffer, parameters);+ }+ }++ pub fn end_current_encoder(&mut self) {+ if let Some(encoder) = self.encoder.take() {+ encoder.end_encoding();+ }+ }++ pub fn finish(mut self) {+ self.end_current_encoder();+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rsindex 92ce766f3..26c569cc0 100644--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs@@ -1,6 +1,7 @@mod buffers;pub mod cache_layers;pub mod encodable_with_state;+mod encoder_resolver;mod io_arrays;pub mod kv_cache_layer;mod model_shape;@@ -12,6 +13,7 @@ pub mod transformer_layer;pub use buffers::ForwardPassBuffers;pub use cache_layers::{CacheLayer, CacheLayers};+pub use encoder_resolver::EncoderResolver;pub use io_arrays::IOArrays;pub use kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState};pub use model_shape::ModelShape;diff --git a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rsindex de9819e7a..52fcfef7d 100644--- a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs+++ b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs@@ -103,4 +103,8 @@ impl EncodableWithState for MPSGraphBlock {}}}++ fn supports_shared_encoder(&self) -> bool {+ false // MPSGraph uses its own encoding mechanism, not compute encoders+ }}diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rsindex 454dea87f..8ddb134ee 100644--- a/crates/uzu/src/backends/metal/forward_pass/state.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs@@ -942,6 +942,9 @@ impl ForwardPassState {token_seeds: &[u64],trace: bool,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,+ skip_token_ids_copy: bool,+ async_positions: Option<(&metal::Buffer, usize)>,+ async_seeds: Option<(&metal::Buffer, usize)>,) -> Self {let suffix_length = token_ids.len();assert_eq!(@@ -971,46 +974,79 @@ impl ForwardPassState {// --------------------// Token IDs// --------------------- let mut token_ids_array = unsafe {- MetalArray::new(- scratch.token_ids.clone(),- &[suffix_length],- DataType::U64,- )+ let token_ids_refcell = {+ let mut token_ids_array = unsafe {+ MetalArray::new(+ scratch.token_ids.clone(),+ &[suffix_length],+ DataType::U64,+ )+ };+ if !skip_token_ids_copy {+ context.copy_from_view(&mut token_ids_array, token_ids.into());+ }+ RefCell::new(token_ids_array)};- context.copy_from_view(&mut token_ids_array, token_ids.into());- let token_ids_refcell = RefCell::new(token_ids_array);// --------------------// Token Positions (i32)// --------------------- let mut token_positions_array = unsafe {- MetalArray::new(- scratch.token_positions.clone(),- &[suffix_length],- DataType::I32,- )- };- let token_positions_i32: Box<[i32]> =- token_positions.iter().map(|p| *p as i32).collect();- context.copy_from_view(- &mut token_positions_array,- token_positions_i32.as_ref().into(),- );- let token_positions_refcell = RefCell::new(token_positions_array);+ let token_positions_refcell =+ if let Some((buffer, offset)) = async_positions {+ // Use pre-allocated async positions buffer at offset+ let token_positions_array = unsafe {+ MetalArray::new_with_offset(+ buffer.clone(),+ &[suffix_length],+ DataType::I32,+ offset * std::mem::size_of::<i32>(),+ )+ };+ RefCell::new(token_positions_array)+ } else {+ // Copy positions from CPU+ let mut token_positions_array = unsafe {+ MetalArray::new(+ scratch.token_positions.clone(),+ &[suffix_length],+ DataType::I32,+ )+ };+ let token_positions_i32: Box<[i32]> =+ token_positions.iter().map(|p| *p as i32).collect();+ context.copy_from_view(+ &mut token_positions_array,+ token_positions_i32.as_ref().into(),+ );+ RefCell::new(token_positions_array)+ };// --------------------// Token Seeds// --------------------- let mut token_seeds_array = unsafe {- MetalArray::new(- scratch.token_seeds.clone(),- &[suffix_length],- DataType::U64,- )+ let token_seeds_refcell = if let Some((buffer, offset)) = async_seeds {+ // Use pre-allocated async seeds buffer at offset+ let token_seeds_array = unsafe {+ MetalArray::new_with_offset(+ buffer.clone(),+ &[suffix_length],+ DataType::U64,+ offset * std::mem::size_of::<u64>(),+ )+ };+ RefCell::new(token_seeds_array)+ } else {+ // Copy seeds from CPU+ let mut token_seeds_array = unsafe {+ MetalArray::new(+ scratch.token_seeds.clone(),+ &[suffix_length],+ DataType::U64,+ )+ };+ context.copy_from_view(&mut token_seeds_array, token_seeds.into());+ RefCell::new(token_seeds_array)};- context.copy_from_view(&mut token_seeds_array, token_seeds.into());- let token_seeds_refcell = RefCell::new(token_seeds_array);// --------------------// Attention Biasdiff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rsindex 43faaf496..1a6a1f65e 100644--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs@@ -16,6 +16,7 @@ use crate::{ArrayId, ForwardPassState, HashMapId,encodable_with_state::{EncodableWithState, EncodingParameters},},+ metal_extensions::ComputeEncoderConditional,},};@@ -168,24 +169,30 @@ impl AttentionKernel {let function_constants = make_function_constants(has_sinks_value);for &head_dim in &supported_head_dims {+ let single_pass_name = format!(+ "attention_single_pass_{}_{}",+ data_suffix, head_dim+ );+ let single_pass_key =+ format!("{}_sinks_{}", single_pass_name, has_sinks_value);if let Ok((pipeline, _)) = context- .compute_pipeline_state_with_reflection(- &format!(- "attention_single_pass_{}_{}",- data_suffix, head_dim- ),+ .compute_pipeline_state_with_reflection_cached(+ &single_pass_key,+ &single_pass_name,Some(&function_constants),){single_pass.insert((head_dim, has_sinks_value), pipeline);}+ let two_pass_1_name =+ format!("attention_2pass_1_{}_{}", data_suffix, head_dim);+ let two_pass_1_key =+ format!("{}_sinks_{}", two_pass_1_name, has_sinks_value);if let Ok((pipeline, _)) = context- .compute_pipeline_state_with_reflection(- &format!(- "attention_2pass_1_{}_{}",- data_suffix, head_dim- ),+ .compute_pipeline_state_with_reflection_cached(+ &two_pass_1_key,+ &two_pass_1_name,Some(&function_constants),){@@ -193,12 +200,18 @@ impl AttentionKernel {}if !two_pass_2.contains_key(&head_dim) {+ let two_pass_2_name = format!(+ "attention_2pass_2_{}_{}",+ data_suffix, head_dim+ );+ let two_pass_2_key = format!(+ "{}_sinks_{}",+ two_pass_2_name, has_sinks_value+ );if let Ok((pipeline, _)) = context- .compute_pipeline_state_with_reflection(- &format!(- "attention_2pass_2_{}_{}",- data_suffix, head_dim- ),+ .compute_pipeline_state_with_reflection_cached(+ &two_pass_2_key,+ &two_pass_2_name,Some(&function_constants),){@@ -580,6 +593,51 @@ impl EncodableWithState for AttentionKernelEncodable {state: &mut ForwardPassState,command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,+ ) {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();++ compute_encoder.condition(+ parameters.predicate_ref(),+ || {+ self.encode_with_encoder_impl(+ state,+ compute_encoder,+ parameters,+ );+ },+ None::<fn()>,+ );++ compute_encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,+ ) {+ self.encode_with_encoder_impl(state, encoder, parameters);+ }+}++impl AttentionKernelEncodable {+ fn encode_with_encoder_impl(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,) {let (suffix_length,@@ -708,13 +766,8 @@ impl EncodableWithState for AttentionKernelEncodable {binding[0].borrow_mut().mtl_buffer().clone()});- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();-- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();-if let Err(e) = self.kernel.encode_kv_cache_update(- &compute_encoder,+ encoder,KVCacheUpdateArguments {rotated_keys_buffer: &rotated_keys_buffer,qkv_buffer: &qkv_buffer,@@ -729,7 +782,6 @@ impl EncodableWithState for AttentionKernelEncodable {},) {eprintln!("Failed to encode KV cache update: {:?}", e);- compute_encoder.end_encoding();return;}@@ -741,7 +793,7 @@ impl EncodableWithState for AttentionKernelEncodable {match variant {AttentionKernelVariant::SinglePass => {if let Err(e) = self.kernel.encode_single_pass(- &compute_encoder,+ encoder,AttentionSinglePassArguments {queries_buffer: &queries_buffer,keys_buffer: &key_cache_buffer,@@ -772,7 +824,7 @@ impl EncodableWithState for AttentionKernelEncodable {},AttentionKernelVariant::TwoPass => {if let Err(e) = self.kernel.encode_two_pass(- &compute_encoder,+ encoder,AttentionTwoPassArguments {queries_buffer: &queries_buffer,keys_buffer: &key_cache_buffer,@@ -802,12 +854,5 @@ impl EncodableWithState for AttentionKernelEncodable {}},}-- compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }}}diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rsindex 0c8d5b693..14ecbf88e 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs@@ -12,6 +12,7 @@ use super::super::{ArrayId, ForwardPassState,encodable_with_state::{EncodableWithState, EncodingParameters},},+ metal_extensions::ComputeEncoderConditional,};use crate::{Array, DataType, backends::metal::MTLError, config::QuantizationMode,@@ -82,7 +83,7 @@ impl QuantizedEmbeddingLookupKernel {&self,encoder: &ComputeCommandEncoderRef,args: QuantizedEmbeddingLookupArguments,- ) -> Result<(), QuantizedEmbeddingError> {+ ) {encoder.set_compute_pipeline_state(&self.pipeline);// Set buffers@@ -124,8 +125,6 @@ impl QuantizedEmbeddingLookupKernel {MTLSize::new(threadgroups, 1, 1),MTLSize::new(threads_per_threadgroup, 1, 1),);-- Ok(())}}@@ -284,6 +283,41 @@ impl EncodableWithState for QuantizedEmbeddingLookupKernelBlock {state: &mut ForwardPassState,command_buffer: &CommandBuffer,parameters: &EncodingParameters,+ ) {+ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ self.encode_impl(state, encoder);++ encoder.end_encoding();++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &metal::ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ self.encode_impl(state, encoder);+ }+}++impl QuantizedEmbeddingLookupKernelBlock {+ fn encode_impl(+ &self,+ state: &mut ForwardPassState,+ encoder: &metal::ComputeCommandEncoderRef,) {let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);let batch_size = state.active_suffix_length();@@ -293,9 +327,6 @@ impl EncodableWithState for QuantizedEmbeddingLookupKernelBlock {let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };let output_buffer = unsafe { output_array_mut.mtl_buffer() };- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();-let args = QuantizedEmbeddingLookupArguments {token_ids_buffer,weights_buffer: &self.weights_buffer,@@ -308,18 +339,7 @@ impl EncodableWithState for QuantizedEmbeddingLookupKernelBlock {group_size: self.group_size,};- self.kernel- .encode(encoder, args)- .expect("Failed to encode quantized embedding lookup kernel");-- encoder.end_encoding();-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }+ self.kernel.encode(encoder, args);}}@@ -489,6 +509,47 @@ impl EncodableWithState for QuantizedEmbeddingReadoutKernelBlock {state: &mut ForwardPassState,command_buffer: &CommandBuffer,parameters: &EncodingParameters,+ ) {+ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ encoder.condition(+ parameters.predicate_ref(),+ || {+ self.encode_impl(state, encoder);+ },+ None::<fn()>,+ );++ encoder.end_encoding();++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &metal::ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ self.encode_impl(state, encoder);+ }+}++impl QuantizedEmbeddingReadoutKernelBlock {+ fn encode_impl(+ &self,+ state: &mut ForwardPassState,+ encoder: &metal::ComputeCommandEncoderRef,) {let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);let batch_size = state.active_suffix_length();@@ -498,9 +559,6 @@ impl EncodableWithState for QuantizedEmbeddingReadoutKernelBlock {let input_buffer = unsafe { input_array_mut.mtl_buffer() };let output_buffer = unsafe { output_array_mut.mtl_buffer() };- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();-// For transposed matmul: input @ weights.T// where weights is [vocab_size, model_dim]use super::super::kernel::quant_matmul::{@@ -522,14 +580,5 @@ impl EncodableWithState for QuantizedEmbeddingReadoutKernelBlock {self.kernel.encode(encoder, args).expect("Failed to encode quantized embedding readout kernel");-- encoder.end_encoding();-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }}}diff --git a/crates/uzu/src/backends/metal/kernel/linear/mod.rs b/crates/uzu/src/backends/metal/kernel/linear/mod.rsindex b28dc52bf..676d9b998 100644--- a/crates/uzu/src/backends/metal/kernel/linear/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/linear/mod.rs@@ -1,6 +1,6 @@use std::rc::Rc;-use metal::Buffer as MTLBuffer;+use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};use mpsgraph::CommandBuffer as MPSCommandBuffer;use super::{@@ -271,6 +271,7 @@ impl EncodableWithState for QuantizedLinearKernelBlock {self.output_dim,total_len,&retained_cb,+ parameters.predicate,);}@@ -281,4 +282,55 @@ impl EncodableWithState for QuantizedLinearKernelBlock {mtl_command_buffer.wait_until_completed();}}++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let batch_size = state.active_suffix_length();++ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let args = QuantizedMatmulArguments {+ a_buffer: input_buffer,+ b_buffer: &self.weights_buffer,+ scales_buffer: &self.scales_buffer,+ zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,+ output_buffer: output_buffer,+ batch: batch_size as i32,+ input_dim: self.input_dim as i32,+ output_dim: self.output_dim as i32,+ quantization_type: self.quantization_type,+ };++ self.kernel+ .encode(encoder, args)+ .expect("Failed to encode quantized matmul kernel");++ if let (Some(bias_add), Some(bias_buf)) =+ (&self.bias_add_kernel, &self.biases_buffer)+ {+ let total_len = batch_size * self.output_dim;+ bias_add.encode_with_encoder(+ &output_buffer,+ bias_buf,+ &output_buffer,+ self.output_dim,+ total_len,+ encoder,+ parameters.predicate,+ );+ }+ }}diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rsindex 5f61ada8e..769fd723f 100644--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs@@ -94,6 +94,10 @@ impl EncodableWithState for ScalePadNormalizeImage {) {let _ = command_buffer;}++ fn supports_shared_encoder(&self) -> bool {+ false // Image processing uses its own encoding path+ }}#[derive(Debug)]diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rsindex f0f038d5a..4f58d1d46 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs@@ -1,6 +1,5 @@use metal::{- Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,- MTLSize,+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState, MTLSize,};use mpsgraph::CommandBuffer as MPSCommandBuffer;@@ -14,6 +13,7 @@ use crate::{encodable_with_state::{EncodableWithState, EncodingParameters},},kernel::QuantizedLinearKernelBlock,+ metal_extensions::ComputeEncoderConditional,},config::Activation,};@@ -49,7 +49,7 @@ impl MlpGateActMulEncodable {fused_up: &MTLBuffer,hidden: &MTLBuffer,m: i32,- ) -> Result<(), MTLError> {+ ) {self.kernel.encode(encoder,&self.activation,@@ -57,7 +57,7 @@ impl MlpGateActMulEncodable {hidden,m,self.hidden_dim as i32,- )+ );}}@@ -107,7 +107,7 @@ impl MlpGateActMulKernel {hidden_buffer: &MTLBuffer,m: i32,h: i32,- ) -> Result<(), MTLError> {+ ) {let act_code = Self::act_code(activation);let threads_per_tg = MTLSize::new(64, 1, 1);encoder.set_compute_pipeline_state(&self.pipeline);@@ -131,7 +131,6 @@ impl MlpGateActMulKernel {let grid = MTLSize::new(h as u64, m as u64, 1);encoder.dispatch_threads(grid, threads_per_tg);- Ok(())}}@@ -169,13 +168,21 @@ impl EncodableWithState for MlpBlockEncodable {let mut fused = arrays[0].borrow_mut();let mut hidden = arrays[1].borrow_mut();let m = fused.shape()[0] as i32;- let root = command_buffer.root_command_buffer();- let encoder = root.new_compute_command_encoder();+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let encoder = mtl_command_buffer.new_compute_command_encoder();+let fused_buf = unsafe { fused.mtl_buffer() };let hidden_buf = unsafe { hidden.mtl_buffer() };- self.gate- .encode(encoder, fused_buf, hidden_buf, m)- .expect("Failed to encode MLP activation/mul kernel");++ encoder.condition(+ params.predicate_ref(),+ || {+ self.gate.encode(encoder, fused_buf, hidden_buf, m);+ },+ None::<fn()>,+ );+encoder.end_encoding();drop(fused);drop(hidden);@@ -189,4 +196,41 @@ impl EncodableWithState for MlpBlockEncodable {mtl_command_buffer.wait_until_completed();}}++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,+ ) {+ self.encode_with_encoder_impl(state, encoder, parameters);+ }+}++impl MlpBlockEncodable {+ fn encode_with_encoder_impl(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ params: &EncodingParameters,+ ) {+ // Up+ self.up.encode_with_shared_encoder(state, encoder, params);+ // Gate act+mul (fused_up -> hidden)+ let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);+ let mut fused = arrays[0].borrow_mut();+ let mut hidden = arrays[1].borrow_mut();+ let m = fused.shape()[0] as i32;+ let fused_buf = unsafe { fused.mtl_buffer() };+ let hidden_buf = unsafe { hidden.mtl_buffer() };+ self.gate.encode(encoder, fused_buf, hidden_buf, m);+ drop(fused);+ drop(hidden);+ // Down+ self.down.encode_with_shared_encoder(state, encoder, params);+ }}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 70fcca455..5911a90b4 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -14,6 +14,7 @@ pub mod ssm;mod tensor_add_bias;mod tensor_add_swap;mod tensor_copy;+pub mod token_copy;pub use attention::{AttentionKernel, AttentionKernelEncodable};pub use data_type::KernelDataType;@@ -37,12 +38,12 @@ pub(crate) use ssm::MambaMixerEncodable;pub use ssm::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,- SSDUpdateKernel, SSMKernelError,- SplitInProjArguments, SplitInProjKernel,+ SSDUpdateKernel, SSMKernelError, SplitInProjArguments, SplitInProjKernel,};pub use tensor_add_bias::TensorAddBias;pub use tensor_add_swap::TensorAddSwap;pub use tensor_copy::TensorCopy;+pub use token_copy::TokenCopyKernel;use super::{MTLContext, metal_extensions};pub use crate::backends::metal::kernel::linear::QuantizedLinearKernelBlock;diff --git a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs b/crates/uzu/src/backends/metal/kernel/moe/encodable.rsindex 4124a597c..f58a406da 100644--- a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/encodable.rs@@ -637,4 +637,8 @@ impl EncodableWithState for MoeBlockEncodable {mtl_command_buffer.wait_until_completed();}}++ fn supports_shared_encoder(&self) -> bool {+ false // MoE uses blit encoders + complex multi-kernel pattern+ }}diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rsindex a13a7eaed..8d8ffc35d 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/experts.rs@@ -136,8 +136,14 @@ impl MoeExpertsTwoPassDecodeKernel {"moe_experts_decode_pass_a_indirect_{}",dtype_suffix);+ let cache_key =+ format!("{}_gate_{}_tile_{}", kernel_name, gate, tile_h);pass_a_indirect[gate as usize].push(- ctx.compute_pipeline_state(&kernel_name, Some(&fcv))?,+ ctx.compute_pipeline_state_cached(+ &cache_key,+ &kernel_name,+ Some(&fcv),+ )?,);}}@@ -334,8 +340,13 @@ impl MoeExpertsTwoPassPrefillKernel {"moe_two_pass_prefill_pass_a_indirect_{}",dtype_suffix);+ let cache_key = format!("{}_gate_{}", kernel_name, gate);pass_a_indirect[gate as usize].push(- ctx.compute_pipeline_state(&kernel_name, Some(&fcv))?,+ ctx.compute_pipeline_state_cached(+ &cache_key,+ &kernel_name,+ Some(&fcv),+ )?,);}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rsindex 122401609..824227fd8 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs@@ -124,8 +124,10 @@ impl QuantizedMatmulKernel {mode,)?;+ let cache_key_mv = format!("{}_mlx_{}", kernel_name_mv, use_mlx_quant);let (pipeline_mv, _) = mtl_context- .compute_pipeline_state_with_reflection(+ .compute_pipeline_state_with_reflection_cached(+ &cache_key_mv,&kernel_name_mv,Some(&function_constants),)@@ -141,8 +143,10 @@ impl QuantizedMatmulKernel {mode,)?;+ let cache_key_mm = format!("{}_mlx_{}", kernel_name_mm, use_mlx_quant);let (pipeline_mm, _) = mtl_context- .compute_pipeline_state_with_reflection(+ .compute_pipeline_state_with_reflection_cached(+ &cache_key_mm,&kernel_name_mm,Some(&function_constants),)diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rsindex 825e41d12..3753f8d3a 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs@@ -14,6 +14,7 @@ use crate::{ArrayId, ForwardPassState,encodable_with_state::{EncodableWithState, EncodingParameters},},+ metal_extensions::ComputeEncoderConditional,},config::{RMSNormConfig, UpcastMode},parameters::ParameterTree,@@ -208,10 +209,12 @@ impl RMSNormKernel {&self,compute_encoder: &ComputeCommandEncoderRef,args: RMSNormArguments,- ) -> Result<(), RMSNormError> {- if self.kernel_type != RMSNormKernelType::Standard {- return Err(RMSNormError::InvalidKernelType);- }+ ) {+ assert_eq!(+ self.kernel_type,+ RMSNormKernelType::Standard,+ "Invalid kernel type for standard RMS norm operation"+ );compute_encoder.set_compute_pipeline_state(&self.pipeline);@@ -258,18 +261,18 @@ impl RMSNormKernel {threadgroups_per_grid,threads_per_threadgroup,);-- Ok(())}pub fn encode_qk_norm(&self,compute_encoder: &ComputeCommandEncoderRef,args: QKNormArguments,- ) -> Result<(), RMSNormError> {- if self.kernel_type != RMSNormKernelType::QueryKey {- return Err(RMSNormError::InvalidKernelType);- }+ ) {+ assert_eq!(+ self.kernel_type,+ RMSNormKernelType::QueryKey,+ "Invalid kernel type for QK norm operation"+ );compute_encoder.set_compute_pipeline_state(&self.pipeline);@@ -350,8 +353,6 @@ impl RMSNormKernel {threadgroups_per_grid,threads_per_threadgroup,);-- Ok(())}}@@ -453,8 +454,62 @@ impl EncodableWithState for RMSNormKernelEncodable {command_buffer.root_command_buffer().to_owned();let compute_encoder = mtl_command_buffer.new_compute_command_encoder();- if let Err(e) = self.kernel.encode(- &compute_encoder,+ compute_encoder.condition(+ parameters.predicate_ref(),+ || {+ self.kernel.encode(+ &compute_encoder,+ RMSNormArguments {+ input_buffer: &input_buffer,+ scales_buffer: &self.scales_buffer,+ output_buffer: &output_buffer,+ batch_size,+ model_dim,+ epsilon: self.config.epsilon,+ scale_offset: self.config.scale_offset.unwrap_or(0.0),+ },+ );+ },+ None::<fn()>,+ );++ compute_encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ let input_binding = state.arrays(&[self.input_array_id]);+ let output_binding = state.arrays(&[self.output_array_id]);++ let input_shape = {+ let input_array = input_binding[0].borrow();+ input_array.shape().to_vec()+ };++ let mut input_array = input_binding[0].borrow_mut();+ let mut output_array = output_binding[0].borrow_mut();++ let input_buffer = unsafe { input_array.mtl_buffer() };+ let output_buffer = unsafe { output_array.mtl_buffer() };++ let batch_size = input_shape[0] as i32;+ let model_dim = input_shape[1] as i32;++ self.kernel.encode(+ encoder,RMSNormArguments {input_buffer: &input_buffer,scales_buffer: &self.scales_buffer,@@ -464,16 +519,7 @@ impl EncodableWithState for RMSNormKernelEncodable {epsilon: self.config.epsilon,scale_offset: self.config.scale_offset.unwrap_or(0.0),},- ) {- eprintln!("Failed to encode RMS norm kernel: {:?}", e);- }-- compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }+ );}}@@ -622,6 +668,46 @@ impl EncodableWithState for QKNormKernelEncodable {state: &mut ForwardPassState,command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,+ ) {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();++ compute_encoder.condition(+ parameters.predicate_ref(),+ || {+ self.encode_with_encoder_impl(state, compute_encoder);+ },+ None::<fn()>,+ );++ compute_encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ self.encode_with_encoder_impl(state, encoder);+ }+}++impl QKNormKernelEncodable {+ fn encode_with_encoder_impl(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,) {let qkv_binding = state.arrays(&[self.qkv_array_id]);let qkv_shape = {@@ -635,10 +721,6 @@ impl EncodableWithState for QKNormKernelEncodable {let batch_size = qkv_shape[0] as i32;let head_dim = self.head_dim as i32;- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();-// Process query normalization if configuredif let (Some(query_kernel),@@ -647,14 +729,13 @@ impl EncodableWithState for QKNormKernelEncodable {) =(&self.query_kernel, &self.query_scales_buffer, &self.query_config){- if let Err(e) = query_kernel.encode_qk_norm(- &compute_encoder,+ query_kernel.encode_qk_norm(+ encoder,QKNormArguments {qkv_input_buffer: &qkv_buffer,scales_buffer: query_scales_buffer,qkv_output_buffer: &qkv_buffer,batch_size,- // Always pass actual head counts (needed for correct buffer addressing)num_q_heads: self.num_q_heads as i32,num_kv_heads: self.num_kv_heads as i32,head_dim,@@ -662,26 +743,19 @@ impl EncodableWithState for QKNormKernelEncodable {scale_offset: query_config.scale_offset.unwrap_or(0.0),target: QKNormTarget::QueryHeads,},- ) {- eprintln!(- "Failed to encode query normalization kernel: {:?}",- e- );- }+ );}- // Process key normalization if configuredif let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =(&self.key_kernel, &self.key_scales_buffer, &self.key_config){- if let Err(e) = key_kernel.encode_qk_norm(- &compute_encoder,+ key_kernel.encode_qk_norm(+ encoder,QKNormArguments {qkv_input_buffer: &qkv_buffer,scales_buffer: key_scales_buffer,qkv_output_buffer: &qkv_buffer,batch_size,- // Always pass actual head counts (needed for correct buffer addressing)num_q_heads: self.num_q_heads as i32,num_kv_heads: self.num_kv_heads as i32,head_dim,@@ -689,16 +763,7 @@ impl EncodableWithState for QKNormKernelEncodable {scale_offset: key_config.scale_offset.unwrap_or(0.0),target: QKNormTarget::KeyHeads,},- ) {- eprintln!("Failed to encode key normalization kernel: {:?}", e);- }- }-- compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ );}}}diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rsindex be166cea4..ec4964985 100644--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/rope/mod.rs@@ -15,6 +15,7 @@ use crate::{ArrayId, ForwardPassState, RopeType,encodable_with_state::{EncodableWithState, EncodingParameters},},+ metal_extensions::ComputeEncoderConditional,},};@@ -156,6 +157,46 @@ impl EncodableWithState for RopeKernelEncodable {state: &mut ForwardPassState,command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,+ ) {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();++ compute_encoder.condition(+ parameters.predicate_ref(),+ || {+ self.encode_with_encoder_impl(state, compute_encoder);+ },+ None::<fn()>,+ );++ compute_encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ self.encode_with_encoder_impl(state, encoder);+ }+}++impl RopeKernelEncodable {+ fn encode_with_encoder_impl(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,) {let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);@@ -200,12 +241,8 @@ impl EncodableWithState for RopeKernelEncodable {state.arrays(&[ArrayId::RopeSines(self.rope_type)]);let mut rope_sines = sin_buffer_binding[0].borrow_mut();- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();-self.kernel.encode(- &compute_encoder,+ encoder,RopeKernelArguments {qkv_buffer: unsafe { &qkv.mtl_buffer() },cosines_buffer: unsafe { &rope_cosines.mtl_buffer() },@@ -224,12 +261,5 @@ impl EncodableWithState for RopeKernelEncodable {max_sequence_length: rope_max_seq_len,},);-- compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }}}diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rsindex 2ef7ff152..f1c5a3c59 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs@@ -247,6 +247,7 @@ impl SamplingKernel {self.encode_with_encoder(logits_buffer,seeds_buffer,+ 0, // No offset for non-async pathsampled_tokens_buffer,sampling_method,batch_size,@@ -261,6 +262,7 @@ impl SamplingKernel {&self,logits_buffer: &MTLBuffer,seeds_buffer: Option<&MTLBuffer>,+ seeds_offset: usize,sampled_tokens_buffer: &MTLBuffer,sampling_method: SamplingMethod,batch_size: usize,@@ -327,6 +329,7 @@ impl SamplingKernel {self.encode_gumbel(last_logits_buffer,seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,+ seeds_offset,&self.partial_gumbel_buffer,batch_size as u32,vocab_size as u32,@@ -467,6 +470,7 @@ impl SamplingKernel {&self,logits_buffer: &MTLBuffer,seeds_buffer: &MTLBuffer,+ seeds_offset: usize,processed_logits_buffer: &MTLBuffer,batch_size: u32,vocab_size: u32,@@ -475,7 +479,7 @@ impl SamplingKernel {compute_encoder.set_compute_pipeline_state(&self.gumbel_pipeline);compute_encoder.set_buffer(0, Some(logits_buffer), 0);- compute_encoder.set_buffer(1, Some(seeds_buffer), 0);+ compute_encoder.set_buffer(1, Some(seeds_buffer), seeds_offset as u64);compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);compute_encoder.set_bytes(3,@@ -639,6 +643,40 @@ impl EncodableWithState for SamplingKernelEncodable {state: &mut ForwardPassState,command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,+ ) {+ let root_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let encoder = root_command_buffer.new_compute_command_encoder();++ self.encode_impl(state, encoder);++ encoder.end_encoding();++ if parameters.wait_until_completed {+ command_buffer.commit_and_continue();+ root_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &metal::ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ self.encode_impl(state, encoder);+ }+}++impl SamplingKernelEncodable {+ fn encode_impl(+ &self,+ state: &mut ForwardPassState,+ encoder: &metal::ComputeCommandEncoderRef,) {assert!(state.sampling_output.is_some(),@@ -665,24 +703,19 @@ impl EncodableWithState for SamplingKernelEncodable {state.sampling_output.as_ref().unwrap().borrow_mut();let sampling_method = state.sampling_method.unwrap();+ let seeds_offset = seeds.buffer_offset();- let root_command_buffer =- command_buffer.root_command_buffer().to_owned();- if let Err(e) = self.kernel.encode(+ if let Err(e) = self.kernel.encode_with_encoder(unsafe { &logits.mtl_buffer() },unsafe { Some(&seeds.mtl_buffer()) },+ seeds_offset,unsafe { &output_buffer_ref.mtl_buffer() },sampling_method,batch_size,vocab_size,- &root_command_buffer,+ encoder,) {panic!("Sampling encoding failed: {:?}", e);}-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- root_command_buffer.wait_until_completed();- }}}diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rsindex 45cdd7e1d..031af0d15 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs@@ -104,14 +104,25 @@ impl Conv1dScanKernel {let decode_name =format!("conv1d_decode_kernel_{}", fn_suffix(data_type));let function_constants = make_function_constants(activation);+ let activation_id = activation_to_int(activation);+ let cache_key = format!("{}_act_{}", fn_name, activation_id);let pipeline = context- .compute_pipeline_state(&fn_name, Some(&function_constants))+ .compute_pipeline_state_cached(+ &cache_key,+ &fn_name,+ Some(&function_constants),+ ).map_err(SSMKernelError::MetalError)?;let pack_pipeline = context.compute_pipeline_state(&pack_name, None).map_err(SSMKernelError::MetalError)?;+ let decode_cache_key = format!("{}_act_{}", decode_name, activation_id);let decode_pipeline = context- .compute_pipeline_state(&decode_name, Some(&function_constants))+ .compute_pipeline_state_cached(+ &decode_cache_key,+ &decode_name,+ Some(&function_constants),+ ).map_err(SSMKernelError::MetalError)?;Ok(Self {pipeline,diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs b/crates/uzu/src/backends/metal/kernel/ssm/mamba.rsindex 5fc14fa75..a4d82b506 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs+++ b/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs@@ -1,12 +1,13 @@use std::{env, rc::Rc};+use metal::ComputeCommandEncoderRef;use mpsgraph::CommandBuffer as MPSCommandBuffer;use super::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,- SSDPrefillArguments, SSDPrefillKernel,- SSDPrefillMode, SSDUpdateArguments, SSDUpdateKernel, SplitInProjArguments,- SplitInProjKernel, conv1d_scan::Conv1dDecodeArguments,+ SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,+ SSDUpdateKernel, SplitInProjArguments, SplitInProjKernel,+ conv1d_scan::Conv1dDecodeArguments,};use crate::{DataType,@@ -19,7 +20,7 @@ use crate::{transformer_layer,},},- config::{DecoderLayerType, mamba::Mamba2Config},+ config::{DecoderLayerType, LinearConfig, mamba::Mamba2Config},parameters::ParameterTree,};@@ -28,6 +29,7 @@ pub(crate) struct MambaMixerEncodable {config: Mamba2Config,in_projection: Box<dyn EncodableWithState>,out_projection: Box<dyn EncodableWithState>,+ projections_support_shared_encoder: bool,split_inproj: SplitInProjKernel,conv_scan: Conv1dScanKernel,ssm_prefill: SSDPrefillKernel,@@ -67,15 +69,23 @@ impl MambaMixerEncodable {mamba_config.in_projection_config.activation_precision().into();let kernel_data_type: KernelDataType = data_type.into();+ let projections_are_quantized = matches!(+ &mamba_config.in_projection_config,+ LinearConfig::Quantized(_) | LinearConfig::MLXQuantized(_)+ ) && matches!(+ &mamba_config.out_projection_config,+ LinearConfig::Quantized(_) | LinearConfig::MLXQuantized(_)+ );++ let in_projection_output_dim = mamba_config.conv_dim()+ + mamba_config.inner_dim()+ + mamba_config.num_heads;+let in_projection = transformer_layer::linear_block(&mamba_config.in_projection_config,mamba_config.has_in_biases,model_dim,- [- mamba_config.conv_dim(),- mamba_config.inner_dim(),- mamba_config.num_heads,- ],+ [in_projection_output_dim],mtl_context,&resolve_subtree(decoder_layer_loader,@@ -131,6 +141,7 @@ impl MambaMixerEncodable {config: mamba_config,in_projection,out_projection,+ projections_support_shared_encoder: projections_are_quantized,split_inproj,conv_scan,ssm_prefill,@@ -143,7 +154,42 @@ impl MambaMixerEncodable {}}- fn encode_pipeline(+ fn encode_pipeline_with_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,+ ) {+ let suffix_length = state.aux_buffers_suffix_length();+ let active_suffix_length = state.active_suffix_length();+ if suffix_length == 0 || active_suffix_length == 0 {+ return;+ }++ self.in_projection+ .encode_with_shared_encoder(state, encoder, parameters);+ self.split_inproj_with_encoder(state, encoder, active_suffix_length);+ self.run_conv_scan_with_encoder(state, encoder, active_suffix_length);++ if suffix_length == 1 {+ self.run_decode_ssm_with_encoder(+ state,+ encoder,+ active_suffix_length,+ );+ } else {+ self.run_prefill_ssm_with_encoder(+ state,+ encoder,+ active_suffix_length,+ );+ }++ self.out_projection+ .encode_with_shared_encoder(state, encoder, parameters);+ }++ fn encode_pipeline_with_command_buffer(&self,state: &mut ForwardPassState,command_buffer: &MPSCommandBuffer,@@ -156,29 +202,34 @@ impl MambaMixerEncodable {}self.in_projection.encode(state, command_buffer, parameters);- self.split_inproj(state, command_buffer, active_suffix_length);- self.run_conv_scan(state, command_buffer, active_suffix_length);++ let cmd = command_buffer.root_command_buffer().to_owned();+ let encoder = cmd.new_compute_command_encoder();+ self.split_inproj_with_encoder(state, encoder, active_suffix_length);+ self.run_conv_scan_with_encoder(state, encoder, active_suffix_length);if suffix_length == 1 {- self.run_decode_ssm(state, command_buffer, active_suffix_length);+ self.run_decode_ssm_with_encoder(+ state,+ encoder,+ active_suffix_length,+ );} else {- self.run_prefill_ssm(state, command_buffer, active_suffix_length);+ self.run_prefill_ssm_with_encoder(+ state,+ encoder,+ active_suffix_length,+ );}+ encoder.end_encoding();self.out_projection.encode(state, command_buffer, parameters);-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }}- fn split_inproj(+ fn split_inproj_with_encoder(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ encoder: &ComputeCommandEncoderRef,suffix_length: usize,) {let arrays = state.arrays(&[@@ -204,12 +255,9 @@ impl MambaMixerEncodable {let num_heads = self.config.num_heads;let total_dim = conv_dim + inner_dim + num_heads;- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute = mtl_command_buffer.new_compute_command_encoder();self.split_inproj.encode(- &compute,+ encoder,SplitInProjArguments {input: &input_buf,conv_out: &conv_buf,@@ -224,13 +272,12 @@ impl MambaMixerEncodable {},).expect("Failed to encode split in-projection kernel");- compute.end_encoding();}- fn run_conv_scan(+ fn run_conv_scan_with_encoder(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ encoder: &ComputeCommandEncoderRef,suffix_length: usize,) {let arrays = state.arrays(&[@@ -262,15 +309,13 @@ impl MambaMixerEncodable {let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();let proj_dim = self.config.num_groups * self.config.state_dim;- let cmd = command_buffer.root_command_buffer().to_owned();let state_stride = self.config.kernel_size.saturating_sub(1);drop(conv_state);if suffix_length == 1 {- let compute = cmd.new_compute_command_encoder();self.conv_scan.encode_decode(- &compute,+ encoder,Conv1dDecodeArguments {x: &input_buf,w: &weight_buf,@@ -290,7 +335,6 @@ impl MambaMixerEncodable {},).expect("Failed to encode conv decode kernel");- compute.end_encoding();} else {let padded_buf = if state_stride > 0 {let array = state@@ -300,24 +344,20 @@ impl MambaMixerEncodable {let buf = unsafe { borrow.mtl_buffer().to_owned() };drop(borrow);- {- let pack_encoder = cmd.new_compute_command_encoder();- self.conv_scan- .encode_pack(- &pack_encoder,- Conv1dPackArguments {- state_in: &state_buf,- x: &input_buf,- padded: &buf,- state_stride,- row_stride: conv_dim,- suffix_len: suffix_length,- channels: conv_dim,- },- )- .expect("Failed to encode conv pack kernel");- pack_encoder.end_encoding();- }+ self.conv_scan+ .encode_pack(+ encoder,+ Conv1dPackArguments {+ state_in: &state_buf,+ x: &input_buf,+ padded: &buf,+ state_stride,+ row_stride: conv_dim,+ suffix_len: suffix_length,+ channels: conv_dim,+ },+ )+ .expect("Failed to encode conv pack kernel");Some(buf)} else {@@ -326,10 +366,9 @@ impl MambaMixerEncodable {let conv_source = padded_buf.as_ref().unwrap_or(&input_buf);- let compute = cmd.new_compute_command_encoder();self.conv_scan.encode(- &compute,+ encoder,Conv1dScanArguments {padded: conv_source,w: &weight_buf,@@ -348,14 +387,13 @@ impl MambaMixerEncodable {},).expect("Failed to encode conv scan kernel");- compute.end_encoding();}}- fn run_prefill_ssm(+ fn run_prefill_ssm_with_encoder(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ encoder: &ComputeCommandEncoderRef,suffix_length: usize,) {let base_arrays = state.arrays(&[@@ -392,11 +430,9 @@ impl MambaMixerEncodable {let mut skip_weights = self.skip_connection_weight.clone();let skip = unsafe { skip_weights.mtl_buffer().to_owned() };- let cmd = command_buffer.root_command_buffer().to_owned();- let compute = cmd.new_compute_command_encoder();self.ssm_prefill.encode(- &compute,+ encoder,SSDPrefillArguments {x: &x_buf,dt: &dt_buf,@@ -432,13 +468,12 @@ impl MambaMixerEncodable {self.prefill_mode,).expect("Failed to encode SSD prefill kernel");- compute.end_encoding();}- fn run_decode_ssm(+ fn run_decode_ssm_with_encoder(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ encoder: &ComputeCommandEncoderRef,suffix_length: usize,) {let arrays = state.arrays(&[@@ -485,11 +520,9 @@ impl MambaMixerEncodable {let group_size = (h / g) as i32;let state_size = n as i32;- let cmd = command_buffer.root_command_buffer().to_owned();- let compute = cmd.new_compute_command_encoder();self.ssd_update.encode(- &compute,+ encoder,SSDUpdateArguments {x: &x_buf,dt: &dt_buf,@@ -512,7 +545,6 @@ impl MambaMixerEncodable {},).expect("Failed to encode SSD decode kernel");- compute.end_encoding();}}@@ -536,7 +568,37 @@ impl EncodableWithState for MambaMixerEncodable {command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,) {- self.encode_pipeline(state, command_buffer, parameters);+ if self.projections_support_shared_encoder {+ let cmd = command_buffer.root_command_buffer().to_owned();+ let encoder = cmd.new_compute_command_encoder();+ self.encode_pipeline_with_encoder(state, encoder, parameters);+ encoder.end_encoding();+ } else {+ self.encode_pipeline_with_command_buffer(+ state,+ command_buffer,+ parameters,+ );+ }++ if parameters.wait_until_completed {+ let cmd = command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ cmd.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ self.projections_support_shared_encoder+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,+ ) {+ self.encode_pipeline_with_encoder(state, encoder, parameters);}}diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rsindex 61838092b..fd6b0afd4 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs@@ -2,11 +2,12 @@ use std::mem::size_of;use metal::{Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,- ComputePipelineState as MTLComputePipelineState,+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,};use super::{- KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,+ KernelDataType, MTLContext,+ metal_extensions::{ComputeEncoderConditional, ComputeEncoderDispatch},};use crate::backends::metal::error::MTLError;@@ -37,24 +38,50 @@ impl TensorAddBias {num_cols: usize,total_len: usize,command_buffer: &MTLCommandBuffer,+ predicate: Option<&MTLBuffer>,) {let encoder = command_buffer.new_compute_command_encoder();- encoder.set_label("Tensor Add Bias");- encoder.set_compute_pipeline_state(&self.pipeline_state);- encoder.set_buffer(0, Some(input), 0);- encoder.set_buffer(1, Some(bias), 0);- encoder.set_buffer(2, Some(output), 0);- encoder.set_bytes(- 3,- size_of::<i32>() as u64,- &(num_cols as i32) as *const _ as *const _,+ self.encode_with_encoder(+ input, bias, output, num_cols, total_len, encoder, predicate,);- encoder.set_bytes(- 4,- size_of::<i32>() as u64,- &(total_len as i32) as *const _ as *const _,- );- encoder.dispatch_1d_exactly(&self.pipeline_state, total_len, None);encoder.end_encoding();}++ pub fn encode_with_encoder(+ &self,+ input: &MTLBuffer,+ bias: &MTLBuffer,+ output: &MTLBuffer,+ num_cols: usize,+ total_len: usize,+ encoder: &ComputeCommandEncoderRef,+ predicate: Option<&MTLBuffer>,+ ) {+ encoder.condition(+ predicate.map(|b| b.as_ref()),+ || {+ encoder.set_label("Tensor Add Bias");+ encoder.set_compute_pipeline_state(&self.pipeline_state);+ encoder.set_buffer(0, Some(input), 0);+ encoder.set_buffer(1, Some(bias), 0);+ encoder.set_buffer(2, Some(output), 0);+ encoder.set_bytes(+ 3,+ size_of::<i32>() as u64,+ &(num_cols as i32) as *const _ as *const _,+ );+ encoder.set_bytes(+ 4,+ size_of::<i32>() as u64,+ &(total_len as i32) as *const _ as *const _,+ );+ encoder.dispatch_1d_exactly(+ &self.pipeline_state,+ total_len,+ None,+ );+ },+ None::<fn()>,+ );+ }}diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rsindex 6a71a2dd3..19336e899 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs@@ -2,7 +2,7 @@ use std::mem::size_of;use metal::{Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,- ComputePipelineState as MTLComputePipelineState,+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,};use mpsgraph::CommandBuffer as MPSCommandBuffer;@@ -51,22 +51,27 @@ impl TensorAddSwap {length: usize,command_buffer: &MTLCommandBuffer,) {- let compute_encoder = command_buffer.new_compute_command_encoder();-- compute_encoder.set_label("Tensor Add-Swap");-- compute_encoder.set_buffer(0, Some(skip_buffer), 0);- compute_encoder.set_buffer(1, Some(main_buffer), 0);+ let encoder = command_buffer.new_compute_command_encoder();+ self.encode_with_encoder_raw(skip_buffer, main_buffer, length, encoder);+ encoder.end_encoding();+ }- compute_encoder.set_bytes(+ fn encode_with_encoder_raw(+ &self,+ skip_buffer: &MTLBuffer,+ main_buffer: &MTLBuffer,+ length: usize,+ encoder: &ComputeCommandEncoderRef,+ ) {+ encoder.set_label("Tensor Add-Swap");+ encoder.set_buffer(0, Some(skip_buffer), 0);+ encoder.set_buffer(1, Some(main_buffer), 0);+ encoder.set_bytes(2,size_of::<i32>() as u64,&(length as i32) as *const _ as *const std::ffi::c_void,);-- compute_encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);-- compute_encoder.end_encoding();+ encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);}}@@ -102,4 +107,32 @@ impl EncodableWithState for TensorAddSwap {retained_mtl_command_buffer.wait_until_completed();}}++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&self.argument_arrays);+ assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");++ let length = arrays[0].borrow().num_elements();++ let mut skip_array = arrays[0].borrow_mut();+ let mut main_array = arrays[1].borrow_mut();+ let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };+ let main_mtl_buffer = unsafe { main_array.mtl_buffer() };++ self.encode_with_encoder_raw(+ &skip_mtl_buffer,+ &main_mtl_buffer,+ length,+ encoder,+ );+ }}diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rsindex 5db33a53e..d278a8ba6 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs@@ -2,7 +2,7 @@ use std::mem::size_of;use metal::{Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,- ComputePipelineState as MTLComputePipelineState,+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,};use mpsgraph::CommandBuffer as MPSCommandBuffer;@@ -46,20 +46,34 @@ impl TensorCopy {destination_buffer: &MTLBuffer,length: usize,command_buffer: &MTLCommandBuffer,+ _predicate: Option<&MTLBuffer>,) {- let compute_encoder = command_buffer.new_compute_command_encoder();- compute_encoder.set_label("Tensor Copy");+ let encoder = command_buffer.new_compute_command_encoder();+ self.encode_with_encoder_raw(+ source_buffer,+ destination_buffer,+ length,+ encoder,+ );+ encoder.end_encoding();+ }- compute_encoder.set_buffer(0, Some(source_buffer), 0);- compute_encoder.set_buffer(1, Some(destination_buffer), 0);- compute_encoder.set_bytes(+ fn encode_with_encoder_raw(+ &self,+ source_buffer: &MTLBuffer,+ destination_buffer: &MTLBuffer,+ length: usize,+ encoder: &ComputeCommandEncoderRef,+ ) {+ encoder.set_label("Tensor Copy");+ encoder.set_buffer(0, Some(source_buffer), 0);+ encoder.set_buffer(1, Some(destination_buffer), 0);+ encoder.set_bytes(2,size_of::<i32>() as u64,&(length as i32) as *const _ as *const std::ffi::c_void,);-- compute_encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);- compute_encoder.end_encoding();+ encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);}}@@ -88,6 +102,7 @@ impl EncodableWithState for TensorCopy {&destination_mtl_buffer,length,&retained_mtl_command_buffer,+ parameters.predicate,);if parameters.wait_until_completed {@@ -95,4 +110,32 @@ impl EncodableWithState for TensorCopy {retained_mtl_command_buffer.wait_until_completed();}}++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&self.argument_arrays);+ assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");++ let length = arrays[0].borrow().num_elements();++ let mut source_array = arrays[0].borrow_mut();+ let mut destination_array = arrays[1].borrow_mut();+ let source_mtl_buffer = unsafe { source_array.mtl_buffer() };+ let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };++ self.encode_with_encoder_raw(+ &source_mtl_buffer,+ &destination_mtl_buffer,+ length,+ encoder,+ );+ }}diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/token_copy/mod.rsnew file mode 100644index 000000000..15e449c41--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs@@ -0,0 +1,67 @@+use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState, MTLSize,+};++use crate::backends::metal::MTLContext;++/// Kernel for copying sampled tokens in async pipeline.+/// Two operations:+/// 1. copy_to_token_ids: u32 → u64 for next forward pass embedding+/// 2. copy_to_results: u32 → u32[offset] for callback to read+pub struct TokenCopyKernel {+ copy_to_token_ids: MTLComputePipelineState,+ copy_to_results: MTLComputePipelineState,+}++impl TokenCopyKernel {+ pub fn new(+ context: &MTLContext+ ) -> Result<Self, crate::backends::metal::error::MTLError> {+ let (copy_to_token_ids, _) = context+ .compute_pipeline_state_with_reflection(+ "copy_sampled_token",+ None,+ )?;+ let (copy_to_results, _) = context+ .compute_pipeline_state_with_reflection(+ "copy_token_to_results",+ None,+ )?;++ Ok(Self {+ copy_to_token_ids,+ copy_to_results,+ })+ }++ /// Copies sampling_output[0] (u32) → token_ids[0] (u64)+ /// For next forward pass to read the token.+ pub fn encode_to_token_ids(+ &self,+ sampling_output: &MTLBuffer,+ token_ids: &MTLBuffer,+ encoder: &ComputeCommandEncoderRef,+ ) {+ encoder.set_compute_pipeline_state(&self.copy_to_token_ids);+ encoder.set_buffer(0, Some(sampling_output), 0);+ encoder.set_buffer(1, Some(token_ids), 0);+ encoder.dispatch_threads(MTLSize::new(1, 1, 1), MTLSize::new(1, 1, 1));+ }++ /// Copies sampling_output[0] (u32) → results[offset] (u32)+ /// For callback to read without race condition.+ pub fn encode_to_results(+ &self,+ sampling_output: &MTLBuffer,+ results: &MTLBuffer,+ pass_idx: usize,+ encoder: &ComputeCommandEncoderRef,+ ) {+ let offset = (pass_idx * std::mem::size_of::<u32>()) as u64;+ encoder.set_compute_pipeline_state(&self.copy_to_results);+ encoder.set_buffer(0, Some(sampling_output), 0);+ encoder.set_buffer(1, Some(results), offset);+ encoder.dispatch_threads(MTLSize::new(1, 1, 1), MTLSize::new(1, 1, 1));+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metalnew file mode 100644index 000000000..b9f458c16--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal@@ -0,0 +1,26 @@+#include <metal_stdlib>+using namespace metal;++/// Copies a single sampled token (u32) to the token_ids buffer (u64).+/// Used in async pipeline to pass sampled token to next forward pass.+kernel void copy_sampled_token(+ device const uint32_t* src [[buffer(0)]],+ device uint64_t* dst [[buffer(1)]],+ uint idx [[thread_position_in_grid]]+) {+ if (idx == 0) {+ dst[0] = static_cast<uint64_t>(src[0]);+ }+}++/// Copies a single sampled token (u32) to an indexed results buffer (u32).+/// Used in async pipeline to store result for callback access.+kernel void copy_token_to_results(+ device const uint32_t* src [[buffer(0)]],+ device uint32_t* dst [[buffer(1)]],+ uint idx [[thread_position_in_grid]]+) {+ if (idx == 0) {+ dst[0] = src[0];+ }+}diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rsnew file mode 100644index 000000000..cae917be8--- /dev/null+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs@@ -0,0 +1,131 @@+#![allow(unexpected_cfgs)]++use metal::{+ BufferRef, ComputeCommandEncoder, ComputeCommandEncoderRef,+ MTLCompareFunction,+ foreign_types::{ForeignType, ForeignTypeRef},+ objc::{msg_send, runtime::Object, sel, sel_impl},+};++/// Low-level, unsafe conditional control of Metal encoders.+/// This is internal; users should prefer the safe `ComputeEncoderConditional::condition`.+trait ComputeEncoderRawConditional {+ unsafe fn encode_start_if(+ &self,+ predicate: &BufferRef,+ offset: usize,+ comparison: MTLCompareFunction,+ reference_value: u32,+ );++ unsafe fn encode_start_else(&self);++ unsafe fn encode_end_if(&self) -> bool;+}++/// Safe conditional wrapper that uses the raw unsafe methods internally.+pub trait ComputeEncoderConditional {+ fn condition<IfBlock, ElseBlock>(+ &self,+ predicate: Option<&BufferRef>,+ if_block: IfBlock,+ else_block: Option<ElseBlock>,+ ) where+ IfBlock: FnOnce(),+ ElseBlock: FnOnce();+}++impl ComputeEncoderRawConditional for ComputeCommandEncoder {+ unsafe fn encode_start_if(+ &self,+ predicate: &BufferRef,+ offset: usize,+ comparison: MTLCompareFunction,+ reference_value: u32,+ ) {+ let obj = self.as_ptr() as *mut Object;+ let predicate_ptr = predicate.as_ptr() as *mut Object;+ let _: () = msg_send![+ obj,+ encodeStartIf: predicate_ptr+ offset: offset+ comparison: comparison+ referenceValue: reference_value+ ];+ }++ unsafe fn encode_start_else(&self) {+ let obj = self.as_ptr() as *mut Object;+ let _: () = msg_send![obj, encodeStartElse];+ }++ unsafe fn encode_end_if(&self) -> bool {+ let obj = self.as_ptr() as *mut Object;+ let result: bool = msg_send![obj, encodeEndIf];+ result+ }+}++impl ComputeEncoderRawConditional for ComputeCommandEncoderRef {+ unsafe fn encode_start_if(+ &self,+ predicate: &BufferRef,+ offset: usize,+ comparison: MTLCompareFunction,+ reference_value: u32,+ ) {+ let obj = self as *const _ as *mut Object;+ let predicate_ptr = predicate.as_ptr() as *mut Object;+ let _: () = msg_send![+ obj,+ encodeStartIf:predicate_ptr+ offset:offset+ comparison:comparison+ referenceValue:reference_value+ ];+ }++ unsafe fn encode_start_else(&self) {+ let obj = self as *const _ as *mut Object;+ let _: () = msg_send![obj, encodeStartElse];+ }++ unsafe fn encode_end_if(&self) -> bool {+ let obj = self as *const _ as *mut Object;+ let result: bool = msg_send![obj, encodeEndIf];+ result+ }+}++impl<T> ComputeEncoderConditional for T+where+ T: ComputeEncoderRawConditional,+{+ fn condition<IfBlock, ElseBlock>(+ &self,+ predicate: Option<&BufferRef>,+ if_block: IfBlock,+ else_block: Option<ElseBlock>,+ ) where+ IfBlock: FnOnce(),+ ElseBlock: FnOnce(),+ {+ match (predicate, else_block) {+ (Some(p), Some(else_fn)) => unsafe {+ self.encode_start_if(p, 0, MTLCompareFunction::Equal, 0);+ if_block();+ self.encode_start_else();+ else_fn();+ self.encode_end_if();+ },+ (Some(p), None) => unsafe {+ self.encode_start_if(p, 0, MTLCompareFunction::Equal, 0);+ if_block();+ self.encode_end_if();+ },+ (None, _) => {+ if_block();+ },+ }+ }+}diff --git a/crates/uzu/src/backends/metal/metal_extensions/mod.rs b/crates/uzu/src/backends/metal/metal_extensions/mod.rsindex 6d7545a16..59efb625c 100644--- a/crates/uzu/src/backends/metal/metal_extensions/mod.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/mod.rs@@ -1,6 +1,7 @@#![allow(dead_code, unused_variables, unused_imports)]mod command_encoder_extensions_device;+mod compute_command_encoder_extensions_conditional;mod compute_command_encoder_extensions_dispatch;mod compute_pipeline_state_extensions_device;mod compute_pipeline_state_extensions_threads;@@ -8,5 +9,6 @@ mod device_extensions_features;mod library_extensions_pipeline;pub mod command_buffer_extensions;+pub use compute_command_encoder_extensions_conditional::ComputeEncoderConditional;pub use compute_command_encoder_extensions_dispatch::ComputeEncoderDispatch;pub use library_extensions_pipeline::LibraryPipelineExtensions;diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rsindex 8bf09a4f8..72e0a84b6 100644--- a/crates/uzu/src/config/decoder.rs+++ b/crates/uzu/src/config/decoder.rs@@ -229,6 +229,16 @@ impl DecoderConfig {pub fn group_size(&self) -> usize {self.num_heads * self.num_groups}++ pub fn has_attention_layers(&self) -> bool {+ if let Some(layer_types) = &self.layer_types {+ layer_types+ .iter()+ .any(|lt| matches!(lt, DecoderLayerType::Transformer))+ } else {+ matches!(self.layer_config.mixer_config, MixerConfig::Attention(_))+ }+ }}#[cfg(test)]diff --git a/crates/uzu/src/generator/context.rs b/crates/uzu/src/generator/context.rsindex b8d6e3fcc..edbbcb95f 100644--- a/crates/uzu/src/generator/context.rs+++ b/crates/uzu/src/generator/context.rs@@ -1,5 +1,12 @@-use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};+use std::{+ cell::{Cell, RefCell},+ fs::File,+ io::BufReader,+ path::Path,+ rc::Rc,+};+use metal::Event as MTLEvent;use mpsgraph::CommandBuffer as MPSCommandBuffer;use objc2::rc::Retained;@@ -10,7 +17,7 @@ use crate::{MTLContext, ModelShape,compilation_parameters::CompilationConfig,forward_pass::{ForwardPassBuffers, SharedBuffers},- kernel::SamplingKernelEncodable,+ kernel::{SamplingKernelEncodable, TokenCopyKernel},},config::{LanguageModelConfig, ModelMetadata},generator::rng::DerivableSeed,@@ -22,6 +29,105 @@ use crate::{},};+/// Pre-allocated buffers for async generation pipeline.+/// Indexed by pass_idx to avoid race conditions between GPU passes.+pub struct AsyncBuffers {+ /// Positions buffer: [max_tokens] i32+ /// Pre-populated with [prefill_count, prefill_count+1, ...]+ pub positions: metal::Buffer,+ /// Seeds buffer: [max_tokens] u64+ /// Pre-populated with deterministic seed sequence+ pub seeds: metal::Buffer,+ /// Results buffer: [batch_size] u32+ /// Each pass writes its sampled token to results[pass_idx % batch_size]+ pub results: metal::Buffer,+ /// Metal event for GPU-side synchronization between passes+ pub event: MTLEvent,+ /// Current event counter (pass N waits on N, signals N+1)+ pub counter: Cell<u64>,+ /// Number of tokens after prefill (base for position calculation)+ pub prefill_count: Cell<usize>,+ /// Batch size (number of passes to keep in flight)+ pub batch_size: usize,+}++impl AsyncBuffers {+ pub fn new(+ device: &metal::DeviceRef,+ max_tokens: usize,+ batch_size: usize,+ ) -> Self {+ let positions = device.new_buffer(+ (max_tokens * std::mem::size_of::<i32>()) as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );+ let seeds = device.new_buffer(+ (max_tokens * std::mem::size_of::<u64>()) as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );+ let results = device.new_buffer(+ (batch_size * std::mem::size_of::<u32>()) as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );+ let event = device.new_event();++ Self {+ positions,+ seeds,+ results,+ event,+ counter: Cell::new(0),+ prefill_count: Cell::new(0),+ batch_size,+ }+ }++ /// Prepare positions buffer: [prefill_count-1, prefill_count, ...]+ /// Uses prefill_count-1 to match sync path which uses self.tokens.len()-1+ pub fn prepare_positions(+ &self,+ prefill_count: usize,+ tokens_to_generate: usize,+ ) {+ self.prefill_count.set(prefill_count);+ let base_position = prefill_count.saturating_sub(1);+ let ptr = self.positions.contents() as *mut i32;+ for i in 0..tokens_to_generate {+ unsafe {+ *ptr.add(i) = (base_position + i) as i32;+ }+ }+ }++ /// Prepare seeds buffer with deterministic sequence+ pub fn prepare_seeds(+ &self,+ seed_source: &mut DerivableSeed,+ tokens_to_generate: usize,+ ) {+ let ptr = self.seeds.contents() as *mut u64;+ for i in 0..tokens_to_generate {+ unsafe {+ *ptr.add(i) = seed_source.next();+ }+ }+ }++ /// Reset event counter before async generation+ pub fn reset_counter(&self) {+ self.counter.set(0);+ }++ /// Read sampled token from results buffer at given pass index+ pub fn read_result(+ &self,+ pass_idx: usize,+ ) -> u32 {+ let ptr = self.results.contents() as *const u32;+ unsafe { *ptr.add(pass_idx % self.batch_size) }+ }+}+pub struct GeneratorContext {pub mtl_context: Rc<MTLContext>,pub command_buffer: Retained<MPSCommandBuffer>,@@ -36,6 +142,11 @@ pub struct GeneratorContext {pub kv_cache_update: Box<KVCacheUpdate>,pub gpu_sampler: SamplingKernelEncodable,pub next_seed: DerivableSeed,++ /// Kernel for copying sampled tokens in async pipeline+ pub token_copy: TokenCopyKernel,+ /// Pre-allocated buffers for async generation+ pub async_buffers: AsyncBuffers,}impl GeneratorContext {@@ -139,6 +250,17 @@ impl GeneratorContext {).map_err(|_| Error::UnableToCreateMetalContext)?;+ let token_copy = TokenCopyKernel::new(&mtl_context)+ .map_err(|_| Error::UnableToCreateMetalContext)?;++ let async_batch_size =+ decoding_config.async_batch_size.resolve(model_path);+ let async_buffers = AsyncBuffers::new(+ &mtl_context.device,+ max_prefix_length,+ async_batch_size,+ );+let base_seed = decoding_config.sampling_seed.resolve();let next_seed = DerivableSeed::new(base_seed);@@ -154,6 +276,8 @@ impl GeneratorContext {kv_cache_update,gpu_sampler,next_seed,+ token_copy,+ async_buffers,};return Ok(context);diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rsindex 3bea1cf9d..7b47ff0f6 100644--- a/crates/uzu/src/generator/generator.rs+++ b/crates/uzu/src/generator/generator.rs@@ -1,5 +1,6 @@-use std::{collections::HashMap, path::Path, time::Instant};+use std::{collections::HashMap, path::Path, sync::Arc, time::Instant};+use block::ConcreteBlock;use mpsgraph::CommandBuffer;use super::{@@ -565,4 +566,164 @@ impl Generator {self.registered_prefix_len = desired_prefix_len;}}++ /// Prepares async buffers for generation.+ /// Must be called after prefill, before async_generate loop.+ pub fn prepare_async(+ &mut self,+ tokens_to_generate: usize,+ ) {+ let prefill_count = self.tokens.len();+ self.context+ .async_buffers+ .prepare_positions(prefill_count, tokens_to_generate);+ self.context+ .async_buffers+ .prepare_seeds(&mut self.context.next_seed, tokens_to_generate);+ self.context.async_buffers.reset_counter();+ }++ /// Submits a single async forward pass.+ /// Does NOT block. Callback fires when GPU completes.+ ///+ /// - `pass_idx`: Index of this pass (0, 1, 2, ...)+ /// - `sampling_method`: Sampling configuration+ /// - `on_complete`: Callback receiving sampled token as u64+ pub fn async_generate<F>(+ &mut self,+ pass_idx: usize,+ sampling_method: SamplingMethod,+ on_complete: F,+ ) -> Result<(), Error>+ where+ F: FnOnce(u64) + Send + 'static,+ {+ assert_eq!(+ self.decoding_config.generate_suffix_length(),+ 1,+ "async_generate only supports suffix_length=1"+ );++ // Extract values from async_buffers before mutable borrow+ let current_counter = self.context.async_buffers.counter.get();+ let is_continuation = current_counter > 0;+ let batch_size = self.context.async_buffers.batch_size;+ let slot = pass_idx % batch_size;+ let async_event = self.context.async_buffers.event.clone();+ let results_buffer = self.context.async_buffers.results.clone();+ let async_positions_buffer =+ self.context.async_buffers.positions.clone();+ let async_seeds_buffer = self.context.async_buffers.seeds.clone();++ let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;++ let task = GeneratorRunTask {+ token_ids: vec![last_token],+ token_positions: vec![0], // Ignored, using async buffer+ token_seeds: vec![0], // Ignored, using async buffer+ expected_number_of_new_tokens: 1,+ active_suffix_length: 1,+ is_prefilling: false,+ };++ let async_positions = Some((&async_positions_buffer, pass_idx));+ let async_seeds = Some((&async_seeds_buffer, pass_idx));++ let mut state = ForwardPassState::new(+ self.context.mtl_context.clone(),+ &self.context.model_config.decoder_config,+ &self.context.model_shape,+ &self.context.scratch_buffers,+ self.context.cache_layers.clone(),+ self.context.shared_buffers.clone(),+ &task.token_ids,+ &task.token_positions,+ task.active_suffix_length,+ task.is_prefilling,+ &task.token_seeds,+ false,+ None,+ is_continuation,+ async_positions,+ async_seeds,+ );+ state.sampling_method = Some(sampling_method);++ self.context.reset_command_buffer();+ let root_command_buffer =+ self.context.command_buffer.root_command_buffer();++ // Wait on previous pass if this is a continuation+ if is_continuation {+ root_command_buffer+ .encode_wait_for_event(&async_event, current_counter);+ }++ // Encode forward pass+ self.context.executables.encode(+ &mut state,+ &self.context.command_buffer,+ &EncodingParameters::new(false, false, false),+ );++ // Encode sampling+ self.context.gpu_sampler.encode(+ &mut state,+ &self.context.command_buffer,+ &EncodingParameters::new(false, false, false),+ );++ // Copy sampled token: sampling_output → token_ids (for next pass)+ // and sampling_output → results[slot] (for callback)+ let sampling_output = state+ .sampling_output+ .as_ref()+ .expect("sampling_output must exist after sampling encode");+ let sampling_output_buffer =+ unsafe { sampling_output.borrow_mut().mtl_buffer().clone() };+ let token_ids_buffer = self.context.scratch_buffers.token_ids.clone();++ let encoder = root_command_buffer.new_compute_command_encoder();+ self.context.token_copy.encode_to_token_ids(+ &sampling_output_buffer,+ &token_ids_buffer,+ encoder,+ );+ self.context.token_copy.encode_to_results(+ &sampling_output_buffer,+ &results_buffer,+ slot,+ encoder,+ );+ encoder.end_encoding();++ // Signal event for next pass+ let next_counter = current_counter + 1;+ root_command_buffer.encode_signal_event(&async_event, next_counter);+ self.context.async_buffers.counter.set(next_counter);++ // Add completion handler+ let results_buffer_clone = results_buffer.clone();+ let callback = Arc::new(std::sync::Mutex::new(Some(on_complete)));++ let block = ConcreteBlock::new(move |_: &metal::CommandBufferRef| {+ let token = {+ let ptr = results_buffer_clone.contents() as *const u32;+ unsafe { *ptr.add(slot) as u64 }+ };+ if let Some(cb) = callback.lock().unwrap().take() {+ cb(token);+ }+ });+ let block = block.copy();++ root_command_buffer.add_completed_handler(&block);+ self.context.command_buffer.commit_and_continue();++ Ok(())+ }++ pub fn has_attention_layers(&self) -> bool {+ self.context.model_config.decoder_config.has_attention_layers()+ }}diff --git a/crates/uzu/src/generator/tasks.rs b/crates/uzu/src/generator/tasks.rsindex a04cdbda0..9d97f175d 100644--- a/crates/uzu/src/generator/tasks.rs+++ b/crates/uzu/src/generator/tasks.rs@@ -1,3 +1,7 @@+use std::mem::size_of;++use metal::{Buffer, MTLResourceOptions};+use super::context::GeneratorContext;use crate::backends::metal::{ForwardPassState,@@ -8,6 +12,24 @@ use crate::backends::metal::{pub struct GeneratorEncodedTask {pub key: String,+ predicate_buffer: Buffer,+}++impl GeneratorEncodedTask {+ pub fn predicate_buffer(&self) -> &Buffer {+ &self.predicate_buffer+ }++ pub fn disable_execution(&self) {+ unsafe {+ let ptr = self.predicate_buffer.contents() as *mut u32;+ *ptr = 1;+ self.predicate_buffer.did_modify_range(metal::NSRange::new(+ 0,+ size_of::<u32>() as u64,+ ));+ }+ }}#[derive(Debug, Clone)]@@ -61,6 +83,9 @@ impl GeneratorRunTask {&self.token_seeds,false,external_bias_fn,+ false,+ None,+ None,);return state;@@ -74,8 +99,15 @@ impl GeneratorRunTask {key: String,) -> GeneratorEncodedTask {context.executables.encode(state, &context.command_buffer, parameters);- GeneratorEncodedTask {++ let encoded_task = GeneratorEncodedTask {key,- }+ predicate_buffer: context.mtl_context.device.new_buffer(+ size_of::<u32>() as u64,+ MTLResourceOptions::StorageModeShared,+ ),+ };++ encoded_task}}diff --git a/crates/uzu/src/runners/decoder_runner.rs b/crates/uzu/src/runners/decoder_runner.rsindex 34b77814b..e740cdabb 100644--- a/crates/uzu/src/runners/decoder_runner.rs+++ b/crates/uzu/src/runners/decoder_runner.rs@@ -132,6 +132,9 @@ impl DecoderTestContext {&token_seeds,false,None,+ false,+ None,+ None,);let executables = DecoderExecutables::new(diff --git a/crates/uzu/src/session/config/decoding_config.rs b/crates/uzu/src/session/config/decoding_config.rsindex 350feff98..05a752456 100644--- a/crates/uzu/src/session/config/decoding_config.rs+++ b/crates/uzu/src/session/config/decoding_config.rs@@ -1,6 +1,9 @@use crate::session::{config::SpeculatorConfig,- parameter::{ContextLength, ContextMode, PrefillStepSize, SamplingSeed},+ parameter::{+ AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize,+ SamplingSeed,+ },};#[derive(Clone)]@@ -11,6 +14,7 @@ pub struct DecodingConfig {pub speculator_config: SpeculatorConfig,pub sampling_seed: SamplingSeed,pub allow_pre_encode: bool,+ pub async_batch_size: AsyncBatchSize,}impl DecodingConfig {@@ -29,6 +33,7 @@ impl DecodingConfig {speculator_config,sampling_seed,allow_pre_encode,+ async_batch_size: AsyncBatchSize::default(),}}@@ -46,6 +51,7 @@ impl Default for DecodingConfig {speculator_config: SpeculatorConfig::default(),sampling_seed: SamplingSeed::default(),allow_pre_encode: true,+ async_batch_size: AsyncBatchSize::default(),}}}@@ -110,4 +116,14 @@ impl DecodingConfig {..self.clone()}}++ pub fn with_async_batch_size(+ &self,+ async_batch_size: AsyncBatchSize,+ ) -> Self {+ Self {+ async_batch_size,+ ..self.clone()+ }+ }}diff --git a/crates/uzu/src/session/parameter/async_batch_size.rs b/crates/uzu/src/session/parameter/async_batch_size.rsnew file mode 100644index 000000000..4c0093c70--- /dev/null+++ b/crates/uzu/src/session/parameter/async_batch_size.rs@@ -0,0 +1,45 @@+use std::path::Path;++use crate::utils::{DeviceClass, ModelSize};++#[derive(Debug, Clone, Copy)]+pub enum AsyncBatchSize {+ Default,+ Custom(usize),+}++impl Default for AsyncBatchSize {+ fn default() -> Self {+ AsyncBatchSize::Default+ }+}++impl AsyncBatchSize {+ pub fn resolve(+ &self,+ model_path: &Path,+ ) -> usize {+ match self {+ AsyncBatchSize::Default => {+ let device = DeviceClass::detect();+ let model_size = ModelSize::from_path(model_path);+ default_batch_size(device, model_size)+ },+ AsyncBatchSize::Custom(value) => *value,+ }+ }+}++fn default_batch_size(+ device: DeviceClass,+ model_size: ModelSize,+) -> usize {+ match (model_size, device.is_high_end(), device) {+ (ModelSize::Large, true, _) => 32,+ (ModelSize::Large, _, DeviceClass::IPhone) => 4,+ (ModelSize::Large, _, _) => 8,+ (ModelSize::Small, true, _) => 256,+ (ModelSize::Small, _, DeviceClass::IPhone) => 8,+ (ModelSize::Small, _, _) => 64,+ }+}diff --git a/crates/uzu/src/session/parameter/mod.rs b/crates/uzu/src/session/parameter/mod.rsindex 7a0299bcf..6470f6549 100644--- a/crates/uzu/src/session/parameter/mod.rs+++ b/crates/uzu/src/session/parameter/mod.rs@@ -1,9 +1,11 @@+mod async_batch_size;mod context_length;mod context_mode;mod prefill_step_size;mod resolvable_value;mod sampling_policy;mod sampling_seed;+pub use async_batch_size::AsyncBatchSize;pub use context_length::ContextLength;pub use context_mode::ContextMode;pub use prefill_step_size::PrefillStepSize;diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rsindex 69f3ffc1d..9d9388781 100644--- a/crates/uzu/src/session/session.rs+++ b/crates/uzu/src/session/session.rs@@ -1,4 +1,18 @@-use std::{fs::File, io::BufReader, path::PathBuf, time::Instant};+use std::{+ fs::File,+ io::BufReader,+ path::PathBuf,+ sync::{+ Arc,+ atomic::{AtomicU64, Ordering},+ mpsc,+ },+ time::Instant,+};++fn nanos_to_secs(nanos: u64) -> f64 {+ nanos as f64 / 1_000_000_000.0+}use objc2::rc::autoreleasepool;use tokenizers::Tokenizer;@@ -16,7 +30,7 @@ use crate::{Context, InputProcessor, InputProcessorDefault, OutputParser,is_directory_fits_ram,},- parameter::{ConfigResolvableValue, ContextMode},+ parameter::{ConfigResolvableValue, ContextMode, SamplingMethod},types::{Error, FinishReason, Input, Output, RunStats, Stats, StepStats,TotalStats,@@ -24,6 +38,18 @@ use crate::{},};+struct RunContext {+ eos_tokens: Vec<u64>,+ context_length: usize,+ prefix_len_before: usize,+ input_tokens_len: usize,+ tokens_limit: usize,+ prefill_result: PrefillResult,+ prefill_duration: f64,+ prefill_suffix_length: usize,+ run_start: Instant,+}+pub struct Session {pub model_path: PathBuf,pub model_metadata: ModelMetadata,@@ -151,13 +177,13 @@ impl Session {if self.static_context.is_none() {let mut prefill_config = config.clone();prefill_config.tokens_limit = 0;- let (_out, ctx) =+ let (_out, run_context) =self.extend(input.clone(), None, prefill_config)?;- self.static_context = Some(ctx);+ self.static_context = Some(run_context);}let tmp = self.static_context.take();- if let Some(ref ctx) = tmp {- self.reconfigure_generator(Some(ctx))?;+ if let Some(ref run_context) = tmp {+ self.reconfigure_generator(Some(run_context))?;}self.static_context = tmp;},@@ -237,8 +263,6 @@ impl Session {return Err(Error::ContextLengthExceeded);}- let prefix_len_before = generator.tokens.len().saturating_sub(1);-let eos_tokens: Vec<u64> = self.model_metadata.model_config@@ -248,47 +272,12 @@ impl Session {.map(|&x| x as u64).collect();- let finish_reason = |generator: &Generator,- tokens_new: Vec<u64>|- -> Option<FinishReason> {- let start_idx = prefix_len_before + tokens.len();- let total_tokens = generator.tokens.len();- let total_new_tokens = generator.tokens[start_idx..].len();- let has_eos_token =- tokens_new.iter().any(|token| eos_tokens.contains(token));- let context_limit_reached = total_tokens >= context_length;-- if has_eos_token {- Some(FinishReason::Stop)- } else if total_new_tokens >= config.tokens_limit as usize {- Some(FinishReason::Length)- } else if context_limit_reached {- Some(FinishReason::ContextLimitReached)- } else {- None- }- };-- let build_generated_text = |generator: &Generator,- tokenizer: &Tokenizer|- -> Result<String, Error> {- let start_idx = prefix_len_before + tokens.len();- let generated_tokens: Vec<u32> = generator.tokens[start_idx..]- .to_vec()- .iter()- .map(|value| *value as u32)- .collect();- let generated_text = tokenizer- .decode(&generated_tokens, true)- .map_err(|_| Error::UnableToDecodeText)?;- Ok(generated_text)- };-let sampling_method =config.sampling_policy.resolve(&self.model_metadata.model_config);let prefill_start = Instant::now();let prefix_offset = generator.tokens.len();+ let prefix_len_before = prefix_offset.saturating_sub(1);let sample_suffix = config.tokens_limit > 0;let prefill_result = generator.prefill(@@ -301,36 +290,38 @@ impl Session {let prefill_duration = prefill_start.elapsed().as_secs_f64();generator.clear_cache();- let prefill_finish_reason =- finish_reason(generator, prefill_tokens.clone());- let prefill_generated_text =- build_generated_text(generator, &self.tokenizer)?;- let prefill_parsed_text =- self.output_parser.parse(prefill_generated_text);-let prefill_suffix_length = generator.decoding_config.prefill_step_size.resolve(&self.model_metadata.model_config);- let prefill_output = Output {- text: prefill_parsed_text,- stats: Self::build_stats(- &self.model_metadata,- prefill_result.clone(),- prefill_duration,- prefill_suffix_length,- Vec::new(),- Vec::new(),- generator.decoding_config.generate_suffix_length(),- run_start.elapsed().as_secs_f64(),- tokens.len(),- generator.tokens[prefix_len_before + tokens.len()..].len(),- ),- finish_reason: prefill_finish_reason.clone(),++ let run_context = RunContext {+ eos_tokens,+ context_length,+ prefix_len_before,+ input_tokens_len: tokens.len(),+ tokens_limit: config.tokens_limit as usize,+ prefill_result: prefill_result.clone(),+ prefill_duration,+ prefill_suffix_length,+ run_start,};- let prefill_should_continue = if let Some(progress) = &progress {- progress(prefill_output.clone())+ let prefill_finish_reason =+ Self::check_finish_reason(&run_context, generator, &prefill_tokens);+ let prefill_output = Self::build_output(+ &self.model_metadata,+ &self.tokenizer,+ &self.output_parser,+ &run_context,+ generator,+ &[],+ &[],+ prefill_finish_reason.clone(),+ )?;++ let prefill_should_continue = if let Some(ref p) = progress {+ p(prefill_output.clone())} else {true};@@ -344,55 +335,45 @@ impl Session {}}- let mut generate_results: Vec<GenerateResult> = Vec::new();- let mut generate_durations: Vec<f64> = Vec::new();- let generate_output = loop {- let generate_start = Instant::now();- let generate_result = generator.generate(sampling_method)?;- let generate_tokens = generate_result.tokens.clone();- let generate_duration = generate_start.elapsed().as_secs_f64();- generate_results.push(generate_result);- generate_durations.push(generate_duration);-- let generate_finish_reason =- finish_reason(generator, generate_tokens);- let generate_generated_text =- build_generated_text(generator, &self.tokenizer)?;- let generate_parsed_text =- self.output_parser.parse(generate_generated_text);-- let generate_output = Output {- text: generate_parsed_text,- stats: Self::build_stats(- &self.model_metadata,- prefill_result.clone(),- prefill_duration,- prefill_suffix_length,- generate_results.clone(),- generate_durations.clone(),- generator.decoding_config.generate_suffix_length(),- run_start.elapsed().as_secs_f64(),- tokens.len(),- generator.tokens[prefix_len_before + tokens.len()..].len(),- ),- finish_reason: generate_finish_reason.clone(),- };-- let generate_should_continue = if let Some(progress) = &progress {- progress(generate_output.clone())- } else {- true- };+ let can_use_async = generator.decoding_config.generate_suffix_length()+ == 1+ && !generator.has_attention_layers();- if !generate_should_continue || generate_finish_reason.is_some() {- if generate_should_continue {- break generate_output;- } else {- break generate_output.clone_with_finish_reason(Some(- FinishReason::Cancelled,- ));- }- }+ let generate_output = if can_use_async {+ let batch_size = generator+ .decoding_config+ .async_batch_size+ .resolve(&self.model_path);+ let (results, durations, finish_reason) = Self::run_async_batch(+ &self.model_metadata,+ &self.tokenizer,+ &self.output_parser,+ &run_context,+ generator,+ sampling_method,+ &progress,+ batch_size,+ )?;+ Self::build_output(+ &self.model_metadata,+ &self.tokenizer,+ &self.output_parser,+ &run_context,+ generator,+ &results,+ &durations,+ Some(finish_reason),+ )?+ } else {+ Self::run_sync_generate(+ &self.model_metadata,+ &self.tokenizer,+ &self.output_parser,+ &run_context,+ generator,+ sampling_method,+ &progress,+ )?};generator.clear_cache();@@ -407,16 +388,16 @@ impl Session {let generator =self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;generator.reset_state();- if let Some(ctx) = context {+ if let Some(run_context) = context {let mut generator_state =generator.context.cache_layers.borrow_mut();- for (ctx_layer, gen_layer) in ctx+ for (run_context_layer, gen_layer) in run_context.cache_layers.data.iter().zip(generator_state.data.iter_mut()){- match (ctx_layer, gen_layer) {+ match (run_context_layer, gen_layer) {(CacheLayer::Transformer(src),CacheLayer::Transformer(dst),@@ -470,7 +451,7 @@ impl Session {}drop(generator_state);- generator.tokens = ctx.tokens.clone();+ generator.tokens = run_context.tokens.clone();}Ok(())}@@ -492,6 +473,249 @@ impl Session {}}+impl Session {+ fn decode_generated_tokens(+ tokenizer: &Tokenizer,+ generator: &Generator,+ run_context: &RunContext,+ ) -> Result<String, Error> {+ let start_idx =+ run_context.prefix_len_before + run_context.input_tokens_len;+ let generated_tokens: Vec<u32> =+ generator.tokens[start_idx..].iter().map(|&v| v as u32).collect();+ tokenizer+ .decode(&generated_tokens, true)+ .map_err(|_| Error::UnableToDecodeText)+ }++ fn check_finish_reason(+ run_context: &RunContext,+ generator: &Generator,+ new_tokens: &[u64],+ ) -> Option<FinishReason> {+ let start_idx =+ run_context.prefix_len_before + run_context.input_tokens_len;+ let total_new_tokens = generator.tokens[start_idx..].len();+ let has_eos =+ new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));+ let context_limit =+ generator.tokens.len() >= run_context.context_length;++ if has_eos {+ Some(FinishReason::Stop)+ } else if total_new_tokens >= run_context.tokens_limit {+ Some(FinishReason::Length)+ } else if context_limit {+ Some(FinishReason::ContextLimitReached)+ } else {+ None+ }+ }++ fn build_output(+ model_metadata: &ModelMetadata,+ tokenizer: &Tokenizer,+ output_parser: &OutputParser,+ run_context: &RunContext,+ generator: &Generator,+ generate_results: &[GenerateResult],+ generate_durations: &[f64],+ finish_reason: Option<FinishReason>,+ ) -> Result<Output, Error> {+ let text =+ Self::decode_generated_tokens(tokenizer, generator, run_context)?;+ let parsed = output_parser.parse(text);+ let start_idx =+ run_context.prefix_len_before + run_context.input_tokens_len;+ let output_tokens = generator.tokens[start_idx..].len();++ Ok(Output {+ text: parsed,+ stats: Self::build_stats(+ model_metadata,+ run_context.prefill_result.clone(),+ run_context.prefill_duration,+ run_context.prefill_suffix_length,+ generate_results.to_vec(),+ generate_durations.to_vec(),+ generator.decoding_config.generate_suffix_length(),+ run_context.run_start.elapsed().as_secs_f64(),+ run_context.input_tokens_len,+ output_tokens,+ ),+ finish_reason,+ })+ }++ fn run_async_batch<F>(+ model_metadata: &ModelMetadata,+ tokenizer: &Tokenizer,+ output_parser: &OutputParser,+ run_context: &RunContext,+ generator: &mut Generator,+ sampling_method: SamplingMethod,+ progress: &Option<F>,+ batch_size: usize,+ ) -> Result<(Vec<GenerateResult>, Vec<f64>, FinishReason), Error>+ where+ F: Fn(Output) -> bool,+ {+ let remaining_by_limit = run_context+ .tokens_limit+ .saturating_sub(run_context.prefill_result.tokens.len());+ let remaining_by_context =+ run_context.context_length.saturating_sub(generator.tokens.len());+ let tokens_to_generate = remaining_by_limit.min(remaining_by_context);++ generator.prepare_async(tokens_to_generate);++ let (sender, receiver) = mpsc::channel::<(usize, u64, u64)>();+ let start_time = Instant::now();+ let last_nanos = Arc::new(AtomicU64::new(0));++ let mut results: Vec<GenerateResult> =+ Vec::with_capacity(tokens_to_generate);+ let mut durations: Vec<f64> = Vec::with_capacity(tokens_to_generate);+ let mut finish_reason = FinishReason::Length;+ let mut next_to_submit = 0;+ let mut in_flight = 0;++ while in_flight > 0 || next_to_submit < tokens_to_generate {+ let batch_end =+ std::cmp::min(next_to_submit + batch_size, tokens_to_generate);+ for idx in next_to_submit..batch_end {+ let batch_sender = sender.clone();+ let last_callback_nanos = last_nanos.clone();+ let batch_start_time = start_time;+ generator.async_generate(+ idx,+ sampling_method,+ move |token| {+ let now_nanos =+ batch_start_time.elapsed().as_nanos() as u64;+ let prev_nanos = last_callback_nanos+ .swap(now_nanos, Ordering::SeqCst);+ let _ = batch_sender.send((+ idx,+ token,+ now_nanos.saturating_sub(prev_nanos),+ ));+ },+ )?;+ in_flight += 1;+ }+ let batch_submitted = batch_end - next_to_submit;+ next_to_submit = batch_end;++ for _ in 0..batch_submitted {+ let (_, token, duration_nanos) =+ receiver.recv().map_err(|_| Error::SamplingFailed)?;+ in_flight -= 1;++ let duration = nanos_to_secs(duration_nanos);+ generator.tokens.push(token);+ results.push(GenerateResult {+ tokens: vec![token],+ forwardpass_duration: duration,+ });+ durations.push(duration);++ let should_stop = if run_context.eos_tokens.contains(&token) {+ finish_reason = FinishReason::Stop;+ true+ } else if generator.tokens.len() >= run_context.context_length {+ finish_reason = FinishReason::ContextLimitReached;+ true+ } else if let Some(progress_fn) = progress {+ let output = Self::build_output(+ model_metadata,+ tokenizer,+ output_parser,+ run_context,+ generator,+ &results,+ &durations,+ None,+ )?;+ if !progress_fn(output) {+ finish_reason = FinishReason::Cancelled;+ true+ } else {+ false+ }+ } else {+ false+ };++ if should_stop {+ while in_flight > 0 {+ let _ = receiver.recv();+ in_flight -= 1;+ }+ return Ok((results, durations, finish_reason));+ }+ }+ }++ Ok((results, durations, finish_reason))+ }++ fn run_sync_generate<F>(+ model_metadata: &ModelMetadata,+ tokenizer: &Tokenizer,+ output_parser: &OutputParser,+ run_context: &RunContext,+ generator: &mut Generator,+ sampling_method: SamplingMethod,+ progress: &Option<F>,+ ) -> Result<Output, Error>+ where+ F: Fn(Output) -> bool,+ {+ let mut results: Vec<GenerateResult> = Vec::new();+ let mut durations: Vec<f64> = Vec::new();++ loop {+ let start = Instant::now();+ let result = generator.generate(sampling_method)?;+ let new_tokens = result.tokens.clone();+ let duration = start.elapsed().as_secs_f64();++ results.push(result);+ durations.push(duration);++ let finish_reason =+ Self::check_finish_reason(run_context, generator, &new_tokens);+ let output = Self::build_output(+ model_metadata,+ tokenizer,+ output_parser,+ run_context,+ generator,+ &results,+ &durations,+ finish_reason.clone(),+ )?;++ let should_continue = if let Some(progress_fn) = progress {+ progress_fn(output.clone())+ } else {+ true+ };++ if !should_continue || finish_reason.is_some() {+ if should_continue {+ return Ok(output);+ } else {+ return Ok(output.clone_with_finish_reason(Some(+ FinishReason::Cancelled,+ )));+ }+ }+ }+ }+}+impl Session {fn build_stats(model_metadata: &ModelMetadata,diff --git a/crates/uzu/src/tracer/tracer.rs b/crates/uzu/src/tracer/tracer.rsindex ec284ceb7..1b1f01ec8 100644--- a/crates/uzu/src/tracer/tracer.rs+++ b/crates/uzu/src/tracer/tracer.rs@@ -318,6 +318,9 @@ impl Tracer {&token_seeds,true,None,+ false,+ None,+ None,);let root_command_buffer = selfdiff --git a/crates/uzu/src/utils/device.rs b/crates/uzu/src/utils/device.rsnew file mode 100644index 000000000..de3c76ae4--- /dev/null+++ b/crates/uzu/src/utils/device.rs@@ -0,0 +1,59 @@+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DeviceClass {+ Ultra,+ Max,+ Pro,+ Base,+ IPhone,+ Unknown,+}++impl DeviceClass {+ pub fn detect() -> Self {+ let device = match metal::Device::system_default() {+ Some(d) => d,+ None => return DeviceClass::Unknown,+ };++ let name = device.name().to_lowercase();++ if name.contains("ultra") {+ DeviceClass::Ultra+ } else if name.contains("max") {+ DeviceClass::Max+ } else if name.contains("pro") {+ DeviceClass::Pro+ } else if name.contains("iphone") || name.contains("a1") {+ DeviceClass::IPhone+ } else {+ DeviceClass::Base+ }+ }++ pub fn is_high_end(&self) -> bool {+ matches!(self, DeviceClass::Ultra | DeviceClass::Max)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_device_detection() {+ let device = DeviceClass::detect();+ println!("Detected device: {:?}", device);+ println!("Is high-end: {}", device.is_high_end());+ assert_ne!(device, DeviceClass::Unknown);+ }++ #[test]+ fn test_is_high_end() {+ assert!(DeviceClass::Ultra.is_high_end());+ assert!(DeviceClass::Max.is_high_end());+ assert!(!DeviceClass::Pro.is_high_end());+ assert!(!DeviceClass::Base.is_high_end());+ assert!(!DeviceClass::IPhone.is_high_end());+ assert!(!DeviceClass::Unknown.is_high_end());+ }+}diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rsindex 1c4996f25..13264fa5a 100644--- a/crates/uzu/src/utils/mod.rs+++ b/crates/uzu/src/utils/mod.rs@@ -1,6 +1,11 @@+pub mod device;+pub mod env_utils;+pub mod model_size;pub mod storage;-pub use storage::*;pub mod version;-pub use version::VERSION;-pub mod env_utils;++pub use device::DeviceClass;pub use env_utils::*;+pub use model_size::ModelSize;+pub use storage::*;+pub use version::VERSION;diff --git a/crates/uzu/src/utils/model_size.rs b/crates/uzu/src/utils/model_size.rsnew file mode 100644index 000000000..5e7a0eb89--- /dev/null+++ b/crates/uzu/src/utils/model_size.rs@@ -0,0 +1,63 @@+use std::path::Path;++const FOUR_GB: u64 = 4 * 1024 * 1024 * 1024;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ModelSize {+ Large,+ Small,+}++impl ModelSize {+ pub fn from_path(model_path: &Path) -> Self {+ let weights_path = model_path.join("model.safetensors");+ let size_bytes =+ std::fs::metadata(&weights_path).map(|m| m.len()).unwrap_or(0);++ Self::from_bytes(size_bytes)+ }++ pub fn from_bytes(size_bytes: u64) -> Self {+ if size_bytes > FOUR_GB {+ ModelSize::Large+ } else {+ ModelSize::Small+ }+ }+}++#[cfg(test)]+mod tests {+ use std::path::PathBuf;++ use super::*;+ use crate::VERSION;+ pub const MODEL_DIR_NAME: &str = "Llama-3.2-1B-Instruct";++ pub fn get_test_model_path() -> PathBuf {+ let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .join("..")+ .join("..")+ .join("models")+ .join(VERSION)+ .join(MODEL_DIR_NAME);+ if !path.exists() {+ panic!(+ "Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.",+ path+ );+ }+ path+ }++ #[test]+ fn test_model_size_from_path() {+ let model_path = get_test_model_path();++ if model_path.exists() {+ let size = ModelSize::from_path(&model_path);+ println!("Model size: {:?}", size);+ assert_eq!(size, ModelSize::Small);+ }+ }+}diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rsindex 611f73eac..f00e091a4 100644--- a/crates/uzu/tests/ssm_update_test.rs+++ b/crates/uzu/tests/ssm_update_test.rs@@ -4,9 +4,7 @@ use half::bf16;use metal::MTLResourceOptions;use uzu::backends::metal::{KernelDataType, MTLContext,- kernel::ssm::{- SSDUpdateArguments, SSDUpdateKernel,- },+ kernel::ssm::{SSDUpdateArguments, SSDUpdateKernel},};fn ssm_update_ref_bf16(x: &[bf16],