Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Async decoding (#52)

Aleksei Savin0930977fdbd5
diff --git a/Cargo.lock b/Cargo.lock
index d0097c700..b12c69953 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2643,6 +2643,7 @@ name = "uzu"
version = "0.1.6"
dependencies = [
"bitflags 2.9.4",
+ "block",
"bytemuck",
"clap",
"half",
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 800a969d4..5becfbbac 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -15,6 +15,7 @@ mpsgraph = "0.2.0"
objc2 = "0.6.2"
objc2-foundation = "0.3.1"
metal = "0.32.0"
+block = "0.1.6"
bytemuck = "1.23.2"
num-traits = "0.2.19"
regex = "1.11.2"
diff --git a/crates/uzu/src/backends/metal/array.rs b/crates/uzu/src/backends/metal/array.rs
index bd8b1f40f..6b854a0b0 100644
--- a/crates/uzu/src/backends/metal/array.rs
+++ b/crates/uzu/src/backends/metal/array.rs
@@ -16,6 +16,8 @@ pub struct MetalArray {
shape: Box<[usize]>,
/// Data type of the elements
data_type: DataType,
+ /// Byte offset into the buffer (for indexed async buffers)
+ offset: usize,
}
impl Array for MetalArray {
@@ -30,7 +32,7 @@ impl Array for MetalArray {
fn buffer(&self) -> &[u8] {
unsafe {
std::slice::from_raw_parts(
- self.buffer.contents() as *const u8,
+ (self.buffer.contents() as *const u8).add(self.offset),
self.size_in_bytes(),
)
}
@@ -39,7 +41,7 @@ impl Array for MetalArray {
fn buffer_mut(&mut self) -> &mut [u8] {
unsafe {
std::slice::from_raw_parts_mut(
- self.buffer.contents() as *mut u8,
+ (self.buffer.contents() as *mut u8).add(self.offset),
self.size_in_bytes(),
)
}
@@ -47,28 +49,46 @@ impl Array for MetalArray {
}
impl MetalArray {
- /// Create a new Array
+ /// Create a new Array at offset 0
pub unsafe fn new(
buffer: MTLBuffer,
shape: &[usize],
data_type: DataType,
+ ) -> Self {
+ unsafe { Self::new_with_offset(buffer, shape, data_type, 0) }
+ }
+
+ /// Create a new Array at a byte offset into the buffer.
+ /// Used for indexed async buffers where each pass uses a different offset.
+ pub unsafe fn new_with_offset(
+ buffer: MTLBuffer,
+ shape: &[usize],
+ data_type: DataType,
+ offset: usize,
) -> Self {
let required_bytes = array_size_in_bytes(shape, data_type);
assert!(
- required_bytes <= buffer.length() as usize,
- "Shape {:?} with data type {:?} requires {} bytes, but buffer length is {} bytes",
+ offset + required_bytes <= buffer.length() as usize,
+ "Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",
shape,
data_type,
- required_bytes,
+ offset,
+ offset + required_bytes,
buffer.length()
);
Self {
buffer,
shape: shape.into(),
data_type,
+ offset,
}
}
+ /// Returns the byte offset into the underlying buffer
+ pub fn buffer_offset(&self) -> usize {
+ self.offset
+ }
+
/// Wraps the underlying MTLBuffer into MPSTensorData for use with MPSGraph.
pub unsafe fn to_mps_tensor_data(&mut self) -> Retained<TensorData> {
TensorData::new_with_mtl_buffer(
@@ -80,6 +100,8 @@ impl MetalArray {
}
/// Returns the underlying MTLBuffer.
+ /// NOTE: For arrays created with `new_with_offset`, callers must use `buffer_offset()`
+ /// and pass it to `encoder.set_buffer(..., offset)` - Metal has no sub-buffer concept.
pub unsafe fn mtl_buffer(&mut self) -> &MTLBuffer {
&self.buffer
}
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index b3c533e18..5ef3e1712 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -1,7 +1,7 @@
#![allow(dead_code)]
include!(concat!(env!("OUT_DIR"), "/metal_lib.rs"));
-use std::rc::Rc;
+use std::{cell::RefCell, collections::HashMap, rc::Rc};
use metal::{
CommandQueue as MTLCommandQueue,
@@ -18,6 +18,7 @@ pub struct MTLContext {
pub device: MTLDevice,
pub command_queue: MTLCommandQueue,
library: MTLLibrary,
+ pipeline_cache: RefCell<HashMap<String, MTLComputePipelineState>>,
}
impl MTLContext {
@@ -39,6 +40,7 @@ impl MTLContext {
device,
command_queue,
library,
+ pipeline_cache: RefCell::new(HashMap::new()),
})
}
@@ -47,7 +49,33 @@ impl MTLContext {
function_name: &str,
constants: Option<&FunctionConstantValues>,
) -> Result<MTLComputePipelineState, MTLError> {
- self.library.compute_pipeline_state(function_name, constants)
+ // Only cache pipelines without constants
+ if constants.is_some() {
+ return self
+ .library
+ .compute_pipeline_state(function_name, constants);
+ }
+ self.compute_pipeline_state_cached(function_name, function_name, None)
+ }
+
+ pub fn compute_pipeline_state_cached(
+ &self,
+ cache_key: &str,
+ function_name: &str,
+ constants: Option<&FunctionConstantValues>,
+ ) -> Result<MTLComputePipelineState, MTLError> {
+ if let Some(pipeline) = self.pipeline_cache.borrow().get(cache_key) {
+ return Ok(pipeline.clone());
+ }
+
+ let pipeline =
+ self.library.compute_pipeline_state(function_name, constants)?;
+
+ self.pipeline_cache
+ .borrow_mut()
+ .insert(cache_key.to_string(), pipeline.clone());
+
+ Ok(pipeline)
}
pub fn compute_pipeline_state_with_reflection(
@@ -55,8 +83,39 @@ impl MTLContext {
function_name: &str,
constants: Option<&FunctionConstantValues>,
) -> Result<(MTLComputePipelineState, Vec<String>), MTLError> {
- self.library
- .compute_pipeline_state_with_reflection(function_name, constants)
+ // Only cache pipelines without constants
+ if constants.is_some() {
+ return self.library.compute_pipeline_state_with_reflection(
+ function_name,
+ constants,
+ );
+ }
+ self.compute_pipeline_state_with_reflection_cached(
+ function_name,
+ function_name,
+ None,
+ )
+ }
+
+ pub fn compute_pipeline_state_with_reflection_cached(
+ &self,
+ cache_key: &str,
+ function_name: &str,
+ constants: Option<&FunctionConstantValues>,
+ ) -> Result<(MTLComputePipelineState, Vec<String>), MTLError> {
+ if let Some(pipeline) = self.pipeline_cache.borrow().get(cache_key) {
+ return Ok((pipeline.clone(), Vec::new()));
+ }
+
+ let (pipeline, arg_names) = self
+ .library
+ .compute_pipeline_state_with_reflection(function_name, constants)?;
+
+ self.pipeline_cache
+ .borrow_mut()
+ .insert(cache_key.to_string(), pipeline.clone());
+
+ Ok((pipeline, arg_names))
}
}
diff --git a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs
index 40a4b83a2..9113bfb03 100644
--- a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs
+++ b/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs
@@ -221,4 +221,8 @@ impl EncodableWithState for DecoderExecutables {
state.copy_array(ArrayId::Logits, traces.borrow().logits.clone());
}
}
+
+ fn supports_shared_encoder(&self) -> bool {
+ false
+ }
}
diff --git a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs
index 98e26526f..a4dcb4b25 100644
--- a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs
+++ b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs
@@ -1,5 +1,6 @@
use std::rc::Rc;
+use metal::ComputeCommandEncoderRef;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use objc2::rc::autoreleasepool;
@@ -9,7 +10,7 @@ use crate::{
MTLContext,
compilation_parameters::CompilationConfig,
forward_pass::{
- ArrayId, ForwardPassState,
+ ArrayId, EncoderResolver, ForwardPassState,
encodable_with_state::{EncodableWithState, EncodingParameters},
transformer_layer,
},
@@ -293,8 +294,170 @@ impl LayerExecutables {
}
}
-impl EncodableWithState for LayerExecutables {
- fn encode(
+impl LayerExecutables {
+ fn encode_with_shared_encoder_impl(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
+ ) {
+ let layer_traces = if let Some(traces) = state.traces.clone() {
+ traces.borrow().layer_results.get(self.layer_index).cloned()
+ } else {
+ None
+ };
+
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().inputs.clone(),
+ );
+ }
+
+ self.copy_main_to_shortcut
+ .encode_with_shared_encoder(state, encoder, parameters);
+
+ self.pre_attention_norm
+ .encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().pre_attention_norm.clone(),
+ );
+ }
+
+ match &self.mixer {
+ MixerExecutables::Attention {
+ qkv_projection,
+ qk_norm,
+ rope,
+ attention,
+ out_projection,
+ } => {
+ qkv_projection
+ .encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(norm) = qk_norm {
+ norm.encode_with_shared_encoder(state, encoder, parameters);
+ }
+ rope.encode_with_shared_encoder(state, encoder, parameters);
+ attention
+ .encode_with_shared_encoder(state, encoder, parameters);
+ out_projection
+ .encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().attention.clone(),
+ );
+ }
+ },
+ MixerExecutables::StateSpace {
+ mixer,
+ } => {
+ mixer.encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().attention.clone(),
+ );
+ }
+ },
+ }
+
+ if let Some(post_attention_norm) = &self.post_attention_norm {
+ post_attention_norm
+ .encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().post_attention_norm.clone(),
+ );
+ }
+ }
+
+ self.main_shortcut_add_swap
+ .encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().mlp_inputs.clone(),
+ );
+ }
+
+ self.pre_mlp_norm
+ .encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().pre_mlp_norm.clone(),
+ );
+ }
+
+ self.mlp.encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(ArrayId::Main, layer_traces.borrow().mlp.clone());
+ }
+
+ if let Some(post_mlp_norm) = &self.post_mlp_norm {
+ post_mlp_norm
+ .encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().post_mlp_norm.clone(),
+ );
+ }
+ }
+
+ self.main_shortcut_add_swap
+ .encode_with_shared_encoder(state, encoder, parameters);
+ if let Some(layer_traces) = layer_traces.clone() {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().outputs.clone(),
+ );
+ }
+ }
+
+ fn all_blocks_support_shared_encoder(&self) -> bool {
+ let mixer_supports = match &self.mixer {
+ MixerExecutables::Attention {
+ qkv_projection,
+ qk_norm,
+ rope,
+ attention,
+ out_projection,
+ } => {
+ qkv_projection.supports_shared_encoder()
+ && qk_norm
+ .as_ref()
+ .map_or(true, |n| n.supports_shared_encoder())
+ && rope.supports_shared_encoder()
+ && attention.supports_shared_encoder()
+ && out_projection.supports_shared_encoder()
+ },
+ MixerExecutables::StateSpace {
+ mixer,
+ } => mixer.supports_shared_encoder(),
+ };
+
+ self.copy_main_to_shortcut.supports_shared_encoder()
+ && self.pre_attention_norm.supports_shared_encoder()
+ && mixer_supports
+ && self
+ .post_attention_norm
+ .as_ref()
+ .map_or(true, |n| n.supports_shared_encoder())
+ && self.main_shortcut_add_swap.supports_shared_encoder()
+ && self.pre_mlp_norm.supports_shared_encoder()
+ && self.mlp.supports_shared_encoder()
+ && self
+ .post_mlp_norm
+ .as_ref()
+ .map_or(true, |n| n.supports_shared_encoder())
+ }
+
+ fn encode_with_resolver(
&self,
state: &mut ForwardPassState,
command_buffer: &MPSCommandBuffer,
@@ -306,6 +469,8 @@ impl EncodableWithState for LayerExecutables {
None
};
+ let mut resolver = EncoderResolver::new(command_buffer);
+
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
ArrayId::Main,
@@ -313,10 +478,9 @@ impl EncodableWithState for LayerExecutables {
);
}
- self.copy_main_to_shortcut.encode(state, command_buffer, parameters);
- // shortcut = input
+ resolver.encode(self.copy_main_to_shortcut.as_ref(), state, parameters);
- self.pre_attention_norm.encode(state, command_buffer, parameters);
+ resolver.encode(self.pre_attention_norm.as_ref(), state, parameters);
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
ArrayId::Main,
@@ -332,13 +496,13 @@ impl EncodableWithState for LayerExecutables {
attention,
out_projection,
} => {
- qkv_projection.encode(state, command_buffer, parameters);
+ resolver.encode(qkv_projection.as_ref(), state, parameters);
if let Some(norm) = qk_norm {
- norm.encode(state, command_buffer, parameters);
+ resolver.encode(norm.as_ref(), state, parameters);
}
- rope.encode(state, command_buffer, parameters);
- attention.encode(state, command_buffer, parameters);
- out_projection.encode(state, command_buffer, parameters);
+ resolver.encode(rope.as_ref().as_ref(), state, parameters);
+ resolver.encode(attention.as_ref(), state, parameters);
+ resolver.encode(out_projection.as_ref(), state, parameters);
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
ArrayId::Main,
@@ -349,7 +513,7 @@ impl EncodableWithState for LayerExecutables {
MixerExecutables::StateSpace {
mixer,
} => {
- mixer.encode(state, command_buffer, parameters);
+ resolver.encode(mixer.as_ref(), state, parameters);
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
ArrayId::Main,
@@ -360,7 +524,7 @@ impl EncodableWithState for LayerExecutables {
}
if let Some(post_attention_norm) = &self.post_attention_norm {
- post_attention_norm.encode(state, command_buffer, parameters);
+ resolver.encode(post_attention_norm.as_ref(), state, parameters);
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
ArrayId::Main,
@@ -368,11 +532,12 @@ impl EncodableWithState for LayerExecutables {
);
}
}
- //main = attention_result
- self.main_shortcut_add_swap.encode(state, command_buffer, parameters);
- // shortcut = input + attention_result
- // main = input + attention_result
+ resolver.encode(
+ self.main_shortcut_add_swap.as_ref(),
+ state,
+ parameters,
+ );
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
ArrayId::Main,
@@ -380,7 +545,7 @@ impl EncodableWithState for LayerExecutables {
);
}
- self.pre_mlp_norm.encode(state, command_buffer, parameters);
+ resolver.encode(self.pre_mlp_norm.as_ref(), state, parameters);
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
ArrayId::Main,
@@ -388,13 +553,13 @@ impl EncodableWithState for LayerExecutables {
);
}
- self.mlp.encode(state, command_buffer, parameters);
+ resolver.encode(self.mlp.as_ref(), state, parameters);
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(ArrayId::Main, layer_traces.borrow().mlp.clone());
}
if let Some(post_mlp_norm) = &self.post_mlp_norm {
- post_mlp_norm.encode(state, command_buffer, parameters);
+ resolver.encode(post_mlp_norm.as_ref(), state, parameters);
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
ArrayId::Main,
@@ -402,16 +567,50 @@ impl EncodableWithState for LayerExecutables {
);
}
}
- // main = mlp_result
- self.main_shortcut_add_swap.encode(state, command_buffer, parameters);
- // shortcut = input + attention_result + mlp_result
- // main = input + attention_result + mlp_result
+ resolver.encode(
+ self.main_shortcut_add_swap.as_ref(),
+ state,
+ parameters,
+ );
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
ArrayId::Main,
layer_traces.borrow().outputs.clone(),
);
}
+
+ resolver.finish();
+ }
+}
+
+impl EncodableWithState for LayerExecutables {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ self.encode_with_resolver(state, command_buffer, parameters);
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ self.all_blocks_support_shared_encoder()
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
+ ) {
+ self.encode_with_shared_encoder_impl(state, encoder, parameters);
}
}
diff --git a/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs b/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs
index 9a602adb2..1c486dd47 100644
--- a/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/encodable_with_state.rs
@@ -1,15 +1,18 @@
+use metal::{Buffer, BufferRef, ComputeCommandEncoderRef};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use super::ForwardPassState;
-pub struct EncodingParameters {
+#[derive(Clone)]
+pub struct EncodingParameters<'a> {
pub warmup: bool,
pub enable_commit: bool,
pub wait_until_completed: bool,
pub projection_step: Option<usize>,
+ pub predicate: Option<&'a Buffer>,
}
-impl EncodingParameters {
+impl<'a> EncodingParameters<'a> {
pub fn new(
warmup: bool,
enable_commit: bool,
@@ -20,6 +23,7 @@ impl EncodingParameters {
enable_commit,
wait_until_completed,
projection_step: None,
+ predicate: None,
}
}
@@ -30,6 +34,18 @@ impl EncodingParameters {
self.projection_step = Some(projection_step);
self
}
+
+ pub fn with_predicate(
+ mut self,
+ predicate: &'a Buffer,
+ ) -> Self {
+ self.predicate = Some(predicate);
+ self
+ }
+
+ pub fn predicate_ref(&self) -> Option<&BufferRef> {
+ self.predicate.map(|buffer| buffer.as_ref())
+ }
}
pub trait EncodableWithState {
@@ -39,4 +55,15 @@ pub trait EncodableWithState {
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
);
+
+ fn supports_shared_encoder(&self) -> bool;
+
+ fn encode_with_shared_encoder(
+ &self,
+ _state: &mut ForwardPassState,
+ _encoder: &ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ panic!("encode_with_shared_encoder called on unsupported type");
+ }
}
diff --git a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
new file mode 100644
index 000000000..2fbe157cc
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
@@ -0,0 +1,57 @@
+use metal::ComputeCommandEncoder;
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{
+ ForwardPassState,
+ encodable_with_state::{EncodableWithState, EncodingParameters},
+};
+
+pub struct EncoderResolver<'a> {
+ command_buffer: &'a MPSCommandBuffer,
+ encoder: Option<ComputeCommandEncoder>,
+}
+
+impl<'a> EncoderResolver<'a> {
+ pub fn new(command_buffer: &'a MPSCommandBuffer) -> Self {
+ Self {
+ command_buffer,
+ encoder: None,
+ }
+ }
+
+ pub fn encode(
+ &mut self,
+ block: &dyn EncodableWithState,
+ state: &mut ForwardPassState,
+ parameters: &EncodingParameters,
+ ) {
+ if block.supports_shared_encoder() {
+ if self.encoder.is_none() {
+ self.encoder = Some(
+ self.command_buffer
+ .root_command_buffer()
+ .new_compute_command_encoder()
+ .to_owned(),
+ );
+ }
+ block.encode_with_shared_encoder(
+ state,
+ self.encoder.as_ref().unwrap(),
+ parameters,
+ );
+ } else {
+ self.end_current_encoder();
+ block.encode(state, self.command_buffer, parameters);
+ }
+ }
+
+ pub fn end_current_encoder(&mut self) {
+ if let Some(encoder) = self.encoder.take() {
+ encoder.end_encoding();
+ }
+ }
+
+ pub fn finish(mut self) {
+ self.end_current_encoder();
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rs
index 92ce766f3..26c569cc0 100644
--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs
@@ -1,6 +1,7 @@
mod buffers;
pub mod cache_layers;
pub mod encodable_with_state;
+mod encoder_resolver;
mod io_arrays;
pub mod kv_cache_layer;
mod model_shape;
@@ -12,6 +13,7 @@ pub mod transformer_layer;
pub use buffers::ForwardPassBuffers;
pub use cache_layers::{CacheLayer, CacheLayers};
+pub use encoder_resolver::EncoderResolver;
pub use io_arrays::IOArrays;
pub use kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState};
pub use model_shape::ModelShape;
diff --git a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
index de9819e7a..52fcfef7d 100644
--- a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
@@ -103,4 +103,8 @@ impl EncodableWithState for MPSGraphBlock {
}
}
}
+
+ fn supports_shared_encoder(&self) -> bool {
+ false // MPSGraph uses its own encoding mechanism, not compute encoders
+ }
}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rs
index 454dea87f..8ddb134ee 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs
@@ -942,6 +942,9 @@ impl ForwardPassState {
token_seeds: &[u64],
trace: bool,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
+ skip_token_ids_copy: bool,
+ async_positions: Option<(&metal::Buffer, usize)>,
+ async_seeds: Option<(&metal::Buffer, usize)>,
) -> Self {
let suffix_length = token_ids.len();
assert_eq!(
@@ -971,46 +974,79 @@ impl ForwardPassState {
// --------------------
// Token IDs
// --------------------
- let mut token_ids_array = unsafe {
- MetalArray::new(
- scratch.token_ids.clone(),
- &[suffix_length],
- DataType::U64,
- )
+ let token_ids_refcell = {
+ let mut token_ids_array = unsafe {
+ MetalArray::new(
+ scratch.token_ids.clone(),
+ &[suffix_length],
+ DataType::U64,
+ )
+ };
+ if !skip_token_ids_copy {
+ context.copy_from_view(&mut token_ids_array, token_ids.into());
+ }
+ RefCell::new(token_ids_array)
};
- context.copy_from_view(&mut token_ids_array, token_ids.into());
- let token_ids_refcell = RefCell::new(token_ids_array);
// --------------------
// Token Positions (i32)
// --------------------
- let mut token_positions_array = unsafe {
- MetalArray::new(
- scratch.token_positions.clone(),
- &[suffix_length],
- DataType::I32,
- )
- };
- let token_positions_i32: Box<[i32]> =
- token_positions.iter().map(|p| *p as i32).collect();
- context.copy_from_view(
- &mut token_positions_array,
- token_positions_i32.as_ref().into(),
- );
- let token_positions_refcell = RefCell::new(token_positions_array);
+ let token_positions_refcell =
+ if let Some((buffer, offset)) = async_positions {
+ // Use pre-allocated async positions buffer at offset
+ let token_positions_array = unsafe {
+ MetalArray::new_with_offset(
+ buffer.clone(),
+ &[suffix_length],
+ DataType::I32,
+ offset * std::mem::size_of::<i32>(),
+ )
+ };
+ RefCell::new(token_positions_array)
+ } else {
+ // Copy positions from CPU
+ let mut token_positions_array = unsafe {
+ MetalArray::new(
+ scratch.token_positions.clone(),
+ &[suffix_length],
+ DataType::I32,
+ )
+ };
+ let token_positions_i32: Box<[i32]> =
+ token_positions.iter().map(|p| *p as i32).collect();
+ context.copy_from_view(
+ &mut token_positions_array,
+ token_positions_i32.as_ref().into(),
+ );
+ RefCell::new(token_positions_array)
+ };
// --------------------
// Token Seeds
// --------------------
- let mut token_seeds_array = unsafe {
- MetalArray::new(
- scratch.token_seeds.clone(),
- &[suffix_length],
- DataType::U64,
- )
+ let token_seeds_refcell = if let Some((buffer, offset)) = async_seeds {
+ // Use pre-allocated async seeds buffer at offset
+ let token_seeds_array = unsafe {
+ MetalArray::new_with_offset(
+ buffer.clone(),
+ &[suffix_length],
+ DataType::U64,
+ offset * std::mem::size_of::<u64>(),
+ )
+ };
+ RefCell::new(token_seeds_array)
+ } else {
+ // Copy seeds from CPU
+ let mut token_seeds_array = unsafe {
+ MetalArray::new(
+ scratch.token_seeds.clone(),
+ &[suffix_length],
+ DataType::U64,
+ )
+ };
+ context.copy_from_view(&mut token_seeds_array, token_seeds.into());
+ RefCell::new(token_seeds_array)
};
- context.copy_from_view(&mut token_seeds_array, token_seeds.into());
- let token_seeds_refcell = RefCell::new(token_seeds_array);
// --------------------
// Attention Bias
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
index 43faaf496..1a6a1f65e 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
@@ -16,6 +16,7 @@ use crate::{
ArrayId, ForwardPassState, HashMapId,
encodable_with_state::{EncodableWithState, EncodingParameters},
},
+ metal_extensions::ComputeEncoderConditional,
},
};
@@ -168,24 +169,30 @@ impl AttentionKernel {
let function_constants = make_function_constants(has_sinks_value);
for &head_dim in &supported_head_dims {
+ let single_pass_name = format!(
+ "attention_single_pass_{}_{}",
+ data_suffix, head_dim
+ );
+ let single_pass_key =
+ format!("{}_sinks_{}", single_pass_name, has_sinks_value);
if let Ok((pipeline, _)) = context
- .compute_pipeline_state_with_reflection(
- &format!(
- "attention_single_pass_{}_{}",
- data_suffix, head_dim
- ),
+ .compute_pipeline_state_with_reflection_cached(
+ &single_pass_key,
+ &single_pass_name,
Some(&function_constants),
)
{
single_pass.insert((head_dim, has_sinks_value), pipeline);
}
+ let two_pass_1_name =
+ format!("attention_2pass_1_{}_{}", data_suffix, head_dim);
+ let two_pass_1_key =
+ format!("{}_sinks_{}", two_pass_1_name, has_sinks_value);
if let Ok((pipeline, _)) = context
- .compute_pipeline_state_with_reflection(
- &format!(
- "attention_2pass_1_{}_{}",
- data_suffix, head_dim
- ),
+ .compute_pipeline_state_with_reflection_cached(
+ &two_pass_1_key,
+ &two_pass_1_name,
Some(&function_constants),
)
{
@@ -193,12 +200,18 @@ impl AttentionKernel {
}
if !two_pass_2.contains_key(&head_dim) {
+ let two_pass_2_name = format!(
+ "attention_2pass_2_{}_{}",
+ data_suffix, head_dim
+ );
+ let two_pass_2_key = format!(
+ "{}_sinks_{}",
+ two_pass_2_name, has_sinks_value
+ );
if let Ok((pipeline, _)) = context
- .compute_pipeline_state_with_reflection(
- &format!(
- "attention_2pass_2_{}_{}",
- data_suffix, head_dim
- ),
+ .compute_pipeline_state_with_reflection_cached(
+ &two_pass_2_key,
+ &two_pass_2_name,
Some(&function_constants),
)
{
@@ -580,6 +593,51 @@ impl EncodableWithState for AttentionKernelEncodable {
state: &mut ForwardPassState,
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
+ ) {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+
+ compute_encoder.condition(
+ parameters.predicate_ref(),
+ || {
+ self.encode_with_encoder_impl(
+ state,
+ compute_encoder,
+ parameters,
+ );
+ },
+ None::<fn()>,
+ );
+
+ compute_encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
+ ) {
+ self.encode_with_encoder_impl(state, encoder, parameters);
+ }
+}
+
+impl AttentionKernelEncodable {
+ fn encode_with_encoder_impl(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
) {
let (
suffix_length,
@@ -708,13 +766,8 @@ impl EncodableWithState for AttentionKernelEncodable {
binding[0].borrow_mut().mtl_buffer().clone()
});
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
-
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
-
if let Err(e) = self.kernel.encode_kv_cache_update(
- &compute_encoder,
+ encoder,
KVCacheUpdateArguments {
rotated_keys_buffer: &rotated_keys_buffer,
qkv_buffer: &qkv_buffer,
@@ -729,7 +782,6 @@ impl EncodableWithState for AttentionKernelEncodable {
},
) {
eprintln!("Failed to encode KV cache update: {:?}", e);
- compute_encoder.end_encoding();
return;
}
@@ -741,7 +793,7 @@ impl EncodableWithState for AttentionKernelEncodable {
match variant {
AttentionKernelVariant::SinglePass => {
if let Err(e) = self.kernel.encode_single_pass(
- &compute_encoder,
+ encoder,
AttentionSinglePassArguments {
queries_buffer: &queries_buffer,
keys_buffer: &key_cache_buffer,
@@ -772,7 +824,7 @@ impl EncodableWithState for AttentionKernelEncodable {
},
AttentionKernelVariant::TwoPass => {
if let Err(e) = self.kernel.encode_two_pass(
- &compute_encoder,
+ encoder,
AttentionTwoPassArguments {
queries_buffer: &queries_buffer,
keys_buffer: &key_cache_buffer,
@@ -802,12 +854,5 @@ impl EncodableWithState for AttentionKernelEncodable {
}
},
}
-
- compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
index 0c8d5b693..14ecbf88e 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
@@ -12,6 +12,7 @@ use super::super::{
ArrayId, ForwardPassState,
encodable_with_state::{EncodableWithState, EncodingParameters},
},
+ metal_extensions::ComputeEncoderConditional,
};
use crate::{
Array, DataType, backends::metal::MTLError, config::QuantizationMode,
@@ -82,7 +83,7 @@ impl QuantizedEmbeddingLookupKernel {
&self,
encoder: &ComputeCommandEncoderRef,
args: QuantizedEmbeddingLookupArguments,
- ) -> Result<(), QuantizedEmbeddingError> {
+ ) {
encoder.set_compute_pipeline_state(&self.pipeline);
// Set buffers
@@ -124,8 +125,6 @@ impl QuantizedEmbeddingLookupKernel {
MTLSize::new(threadgroups, 1, 1),
MTLSize::new(threads_per_threadgroup, 1, 1),
);
-
- Ok(())
}
}
@@ -284,6 +283,41 @@ impl EncodableWithState for QuantizedEmbeddingLookupKernelBlock {
state: &mut ForwardPassState,
command_buffer: &CommandBuffer,
parameters: &EncodingParameters,
+ ) {
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ self.encode_impl(state, encoder);
+
+ encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &metal::ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ self.encode_impl(state, encoder);
+ }
+}
+
+impl QuantizedEmbeddingLookupKernelBlock {
+ fn encode_impl(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &metal::ComputeCommandEncoderRef,
) {
let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
let batch_size = state.active_suffix_length();
@@ -293,9 +327,6 @@ impl EncodableWithState for QuantizedEmbeddingLookupKernelBlock {
let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
-
let args = QuantizedEmbeddingLookupArguments {
token_ids_buffer,
weights_buffer: &self.weights_buffer,
@@ -308,18 +339,7 @@ impl EncodableWithState for QuantizedEmbeddingLookupKernelBlock {
group_size: self.group_size,
};
- self.kernel
- .encode(encoder, args)
- .expect("Failed to encode quantized embedding lookup kernel");
-
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
+ self.kernel.encode(encoder, args);
}
}
@@ -489,6 +509,47 @@ impl EncodableWithState for QuantizedEmbeddingReadoutKernelBlock {
state: &mut ForwardPassState,
command_buffer: &CommandBuffer,
parameters: &EncodingParameters,
+ ) {
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ encoder.condition(
+ parameters.predicate_ref(),
+ || {
+ self.encode_impl(state, encoder);
+ },
+ None::<fn()>,
+ );
+
+ encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &metal::ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ self.encode_impl(state, encoder);
+ }
+}
+
+impl QuantizedEmbeddingReadoutKernelBlock {
+ fn encode_impl(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &metal::ComputeCommandEncoderRef,
) {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
let batch_size = state.active_suffix_length();
@@ -498,9 +559,6 @@ impl EncodableWithState for QuantizedEmbeddingReadoutKernelBlock {
let input_buffer = unsafe { input_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
-
// For transposed matmul: input @ weights.T
// where weights is [vocab_size, model_dim]
use super::super::kernel::quant_matmul::{
@@ -522,14 +580,5 @@ impl EncodableWithState for QuantizedEmbeddingReadoutKernelBlock {
self.kernel
.encode(encoder, args)
.expect("Failed to encode quantized embedding readout kernel");
-
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/linear/mod.rs b/crates/uzu/src/backends/metal/kernel/linear/mod.rs
index b28dc52bf..676d9b998 100644
--- a/crates/uzu/src/backends/metal/kernel/linear/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/linear/mod.rs
@@ -1,6 +1,6 @@
use std::rc::Rc;
-use metal::Buffer as MTLBuffer;
+use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use super::{
@@ -271,6 +271,7 @@ impl EncodableWithState for QuantizedLinearKernelBlock {
self.output_dim,
total_len,
&retained_cb,
+ parameters.predicate,
);
}
@@ -281,4 +282,55 @@ impl EncodableWithState for QuantizedLinearKernelBlock {
mtl_command_buffer.wait_until_completed();
}
}
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let batch_size = state.active_suffix_length();
+
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let args = QuantizedMatmulArguments {
+ a_buffer: input_buffer,
+ b_buffer: &self.weights_buffer,
+ scales_buffer: &self.scales_buffer,
+ zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,
+ output_buffer: output_buffer,
+ batch: batch_size as i32,
+ input_dim: self.input_dim as i32,
+ output_dim: self.output_dim as i32,
+ quantization_type: self.quantization_type,
+ };
+
+ self.kernel
+ .encode(encoder, args)
+ .expect("Failed to encode quantized matmul kernel");
+
+ if let (Some(bias_add), Some(bias_buf)) =
+ (&self.bias_add_kernel, &self.biases_buffer)
+ {
+ let total_len = batch_size * self.output_dim;
+ bias_add.encode_with_encoder(
+ &output_buffer,
+ bias_buf,
+ &output_buffer,
+ self.output_dim,
+ total_len,
+ encoder,
+ parameters.predicate,
+ );
+ }
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
index 5f61ada8e..769fd723f 100644
--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
@@ -94,6 +94,10 @@ impl EncodableWithState for ScalePadNormalizeImage {
) {
let _ = command_buffer;
}
+
+ fn supports_shared_encoder(&self) -> bool {
+ false // Image processing uses its own encoding path
+ }
}
#[derive(Debug)]
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
index f0f038d5a..4f58d1d46 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
@@ -1,6 +1,5 @@
use metal::{
- Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
- MTLSize,
+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState, MTLSize,
};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
@@ -14,6 +13,7 @@ use crate::{
encodable_with_state::{EncodableWithState, EncodingParameters},
},
kernel::QuantizedLinearKernelBlock,
+ metal_extensions::ComputeEncoderConditional,
},
config::Activation,
};
@@ -49,7 +49,7 @@ impl MlpGateActMulEncodable {
fused_up: &MTLBuffer,
hidden: &MTLBuffer,
m: i32,
- ) -> Result<(), MTLError> {
+ ) {
self.kernel.encode(
encoder,
&self.activation,
@@ -57,7 +57,7 @@ impl MlpGateActMulEncodable {
hidden,
m,
self.hidden_dim as i32,
- )
+ );
}
}
@@ -107,7 +107,7 @@ impl MlpGateActMulKernel {
hidden_buffer: &MTLBuffer,
m: i32,
h: i32,
- ) -> Result<(), MTLError> {
+ ) {
let act_code = Self::act_code(activation);
let threads_per_tg = MTLSize::new(64, 1, 1);
encoder.set_compute_pipeline_state(&self.pipeline);
@@ -131,7 +131,6 @@ impl MlpGateActMulKernel {
let grid = MTLSize::new(h as u64, m as u64, 1);
encoder.dispatch_threads(grid, threads_per_tg);
- Ok(())
}
}
@@ -169,13 +168,21 @@ impl EncodableWithState for MlpBlockEncodable {
let mut fused = arrays[0].borrow_mut();
let mut hidden = arrays[1].borrow_mut();
let m = fused.shape()[0] as i32;
- let root = command_buffer.root_command_buffer();
- let encoder = root.new_compute_command_encoder();
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let encoder = mtl_command_buffer.new_compute_command_encoder();
+
let fused_buf = unsafe { fused.mtl_buffer() };
let hidden_buf = unsafe { hidden.mtl_buffer() };
- self.gate
- .encode(encoder, fused_buf, hidden_buf, m)
- .expect("Failed to encode MLP activation/mul kernel");
+
+ encoder.condition(
+ params.predicate_ref(),
+ || {
+ self.gate.encode(encoder, fused_buf, hidden_buf, m);
+ },
+ None::<fn()>,
+ );
+
encoder.end_encoding();
drop(fused);
drop(hidden);
@@ -189,4 +196,41 @@ impl EncodableWithState for MlpBlockEncodable {
mtl_command_buffer.wait_until_completed();
}
}
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
+ ) {
+ self.encode_with_encoder_impl(state, encoder, parameters);
+ }
+}
+
+impl MlpBlockEncodable {
+ fn encode_with_encoder_impl(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ params: &EncodingParameters,
+ ) {
+ // Up
+ self.up.encode_with_shared_encoder(state, encoder, params);
+ // Gate act+mul (fused_up -> hidden)
+ let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
+ let mut fused = arrays[0].borrow_mut();
+ let mut hidden = arrays[1].borrow_mut();
+ let m = fused.shape()[0] as i32;
+ let fused_buf = unsafe { fused.mtl_buffer() };
+ let hidden_buf = unsafe { hidden.mtl_buffer() };
+ self.gate.encode(encoder, fused_buf, hidden_buf, m);
+ drop(fused);
+ drop(hidden);
+ // Down
+ self.down.encode_with_shared_encoder(state, encoder, params);
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 70fcca455..5911a90b4 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -14,6 +14,7 @@ pub mod ssm;
mod tensor_add_bias;
mod tensor_add_swap;
mod tensor_copy;
+pub mod token_copy;
pub use attention::{AttentionKernel, AttentionKernelEncodable};
pub use data_type::KernelDataType;
@@ -37,12 +38,12 @@ pub(crate) use ssm::MambaMixerEncodable;
pub use ssm::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,
- SSDUpdateKernel, SSMKernelError,
- SplitInProjArguments, SplitInProjKernel,
+ SSDUpdateKernel, SSMKernelError, SplitInProjArguments, SplitInProjKernel,
};
pub use tensor_add_bias::TensorAddBias;
pub use tensor_add_swap::TensorAddSwap;
pub use tensor_copy::TensorCopy;
+pub use token_copy::TokenCopyKernel;
use super::{MTLContext, metal_extensions};
pub use crate::backends::metal::kernel::linear::QuantizedLinearKernelBlock;
diff --git a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs b/crates/uzu/src/backends/metal/kernel/moe/encodable.rs
index 4124a597c..f58a406da 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/encodable.rs
@@ -637,4 +637,8 @@ impl EncodableWithState for MoeBlockEncodable {
mtl_command_buffer.wait_until_completed();
}
}
+
+ fn supports_shared_encoder(&self) -> bool {
+ false // MoE uses blit encoders + complex multi-kernel pattern
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
index a13a7eaed..8d8ffc35d 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
@@ -136,8 +136,14 @@ impl MoeExpertsTwoPassDecodeKernel {
"moe_experts_decode_pass_a_indirect_{}",
dtype_suffix
);
+ let cache_key =
+ format!("{}_gate_{}_tile_{}", kernel_name, gate, tile_h);
pass_a_indirect[gate as usize].push(
- ctx.compute_pipeline_state(&kernel_name, Some(&fcv))?,
+ ctx.compute_pipeline_state_cached(
+ &cache_key,
+ &kernel_name,
+ Some(&fcv),
+ )?,
);
}
}
@@ -334,8 +340,13 @@ impl MoeExpertsTwoPassPrefillKernel {
"moe_two_pass_prefill_pass_a_indirect_{}",
dtype_suffix
);
+ let cache_key = format!("{}_gate_{}", kernel_name, gate);
pass_a_indirect[gate as usize].push(
- ctx.compute_pipeline_state(&kernel_name, Some(&fcv))?,
+ ctx.compute_pipeline_state_cached(
+ &cache_key,
+ &kernel_name,
+ Some(&fcv),
+ )?,
);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
index 122401609..824227fd8 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -124,8 +124,10 @@ impl QuantizedMatmulKernel {
mode,
)?;
+ let cache_key_mv = format!("{}_mlx_{}", kernel_name_mv, use_mlx_quant);
let (pipeline_mv, _) = mtl_context
- .compute_pipeline_state_with_reflection(
+ .compute_pipeline_state_with_reflection_cached(
+ &cache_key_mv,
&kernel_name_mv,
Some(&function_constants),
)
@@ -141,8 +143,10 @@ impl QuantizedMatmulKernel {
mode,
)?;
+ let cache_key_mm = format!("{}_mlx_{}", kernel_name_mm, use_mlx_quant);
let (pipeline_mm, _) = mtl_context
- .compute_pipeline_state_with_reflection(
+ .compute_pipeline_state_with_reflection_cached(
+ &cache_key_mm,
&kernel_name_mm,
Some(&function_constants),
)
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
index 825e41d12..3753f8d3a 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
@@ -14,6 +14,7 @@ use crate::{
ArrayId, ForwardPassState,
encodable_with_state::{EncodableWithState, EncodingParameters},
},
+ metal_extensions::ComputeEncoderConditional,
},
config::{RMSNormConfig, UpcastMode},
parameters::ParameterTree,
@@ -208,10 +209,12 @@ impl RMSNormKernel {
&self,
compute_encoder: &ComputeCommandEncoderRef,
args: RMSNormArguments,
- ) -> Result<(), RMSNormError> {
- if self.kernel_type != RMSNormKernelType::Standard {
- return Err(RMSNormError::InvalidKernelType);
- }
+ ) {
+ assert_eq!(
+ self.kernel_type,
+ RMSNormKernelType::Standard,
+ "Invalid kernel type for standard RMS norm operation"
+ );
compute_encoder.set_compute_pipeline_state(&self.pipeline);
@@ -258,18 +261,18 @@ impl RMSNormKernel {
threadgroups_per_grid,
threads_per_threadgroup,
);
-
- Ok(())
}
pub fn encode_qk_norm(
&self,
compute_encoder: &ComputeCommandEncoderRef,
args: QKNormArguments,
- ) -> Result<(), RMSNormError> {
- if self.kernel_type != RMSNormKernelType::QueryKey {
- return Err(RMSNormError::InvalidKernelType);
- }
+ ) {
+ assert_eq!(
+ self.kernel_type,
+ RMSNormKernelType::QueryKey,
+ "Invalid kernel type for QK norm operation"
+ );
compute_encoder.set_compute_pipeline_state(&self.pipeline);
@@ -350,8 +353,6 @@ impl RMSNormKernel {
threadgroups_per_grid,
threads_per_threadgroup,
);
-
- Ok(())
}
}
@@ -453,8 +454,62 @@ impl EncodableWithState for RMSNormKernelEncodable {
command_buffer.root_command_buffer().to_owned();
let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
- if let Err(e) = self.kernel.encode(
- &compute_encoder,
+ compute_encoder.condition(
+ parameters.predicate_ref(),
+ || {
+ self.kernel.encode(
+ &compute_encoder,
+ RMSNormArguments {
+ input_buffer: &input_buffer,
+ scales_buffer: &self.scales_buffer,
+ output_buffer: &output_buffer,
+ batch_size,
+ model_dim,
+ epsilon: self.config.epsilon,
+ scale_offset: self.config.scale_offset.unwrap_or(0.0),
+ },
+ );
+ },
+ None::<fn()>,
+ );
+
+ compute_encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ let input_binding = state.arrays(&[self.input_array_id]);
+ let output_binding = state.arrays(&[self.output_array_id]);
+
+ let input_shape = {
+ let input_array = input_binding[0].borrow();
+ input_array.shape().to_vec()
+ };
+
+ let mut input_array = input_binding[0].borrow_mut();
+ let mut output_array = output_binding[0].borrow_mut();
+
+ let input_buffer = unsafe { input_array.mtl_buffer() };
+ let output_buffer = unsafe { output_array.mtl_buffer() };
+
+ let batch_size = input_shape[0] as i32;
+ let model_dim = input_shape[1] as i32;
+
+ self.kernel.encode(
+ encoder,
RMSNormArguments {
input_buffer: &input_buffer,
scales_buffer: &self.scales_buffer,
@@ -464,16 +519,7 @@ impl EncodableWithState for RMSNormKernelEncodable {
epsilon: self.config.epsilon,
scale_offset: self.config.scale_offset.unwrap_or(0.0),
},
- ) {
- eprintln!("Failed to encode RMS norm kernel: {:?}", e);
- }
-
- compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
+ );
}
}
@@ -622,6 +668,46 @@ impl EncodableWithState for QKNormKernelEncodable {
state: &mut ForwardPassState,
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
+ ) {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+
+ compute_encoder.condition(
+ parameters.predicate_ref(),
+ || {
+ self.encode_with_encoder_impl(state, compute_encoder);
+ },
+ None::<fn()>,
+ );
+
+ compute_encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ self.encode_with_encoder_impl(state, encoder);
+ }
+}
+
+impl QKNormKernelEncodable {
+ fn encode_with_encoder_impl(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
) {
let qkv_binding = state.arrays(&[self.qkv_array_id]);
let qkv_shape = {
@@ -635,10 +721,6 @@ impl EncodableWithState for QKNormKernelEncodable {
let batch_size = qkv_shape[0] as i32;
let head_dim = self.head_dim as i32;
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
-
// Process query normalization if configured
if let (
Some(query_kernel),
@@ -647,14 +729,13 @@ impl EncodableWithState for QKNormKernelEncodable {
) =
(&self.query_kernel, &self.query_scales_buffer, &self.query_config)
{
- if let Err(e) = query_kernel.encode_qk_norm(
- &compute_encoder,
+ query_kernel.encode_qk_norm(
+ encoder,
QKNormArguments {
qkv_input_buffer: &qkv_buffer,
scales_buffer: query_scales_buffer,
qkv_output_buffer: &qkv_buffer,
batch_size,
- // Always pass actual head counts (needed for correct buffer addressing)
num_q_heads: self.num_q_heads as i32,
num_kv_heads: self.num_kv_heads as i32,
head_dim,
@@ -662,26 +743,19 @@ impl EncodableWithState for QKNormKernelEncodable {
scale_offset: query_config.scale_offset.unwrap_or(0.0),
target: QKNormTarget::QueryHeads,
},
- ) {
- eprintln!(
- "Failed to encode query normalization kernel: {:?}",
- e
- );
- }
+ );
}
- // Process key normalization if configured
if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =
(&self.key_kernel, &self.key_scales_buffer, &self.key_config)
{
- if let Err(e) = key_kernel.encode_qk_norm(
- &compute_encoder,
+ key_kernel.encode_qk_norm(
+ encoder,
QKNormArguments {
qkv_input_buffer: &qkv_buffer,
scales_buffer: key_scales_buffer,
qkv_output_buffer: &qkv_buffer,
batch_size,
- // Always pass actual head counts (needed for correct buffer addressing)
num_q_heads: self.num_q_heads as i32,
num_kv_heads: self.num_kv_heads as i32,
head_dim,
@@ -689,16 +763,7 @@ impl EncodableWithState for QKNormKernelEncodable {
scale_offset: key_config.scale_offset.unwrap_or(0.0),
target: QKNormTarget::KeyHeads,
},
- ) {
- eprintln!("Failed to encode key normalization kernel: {:?}", e);
- }
- }
-
- compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ );
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
index be166cea4..ec4964985 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
@@ -15,6 +15,7 @@ use crate::{
ArrayId, ForwardPassState, RopeType,
encodable_with_state::{EncodableWithState, EncodingParameters},
},
+ metal_extensions::ComputeEncoderConditional,
},
};
@@ -156,6 +157,46 @@ impl EncodableWithState for RopeKernelEncodable {
state: &mut ForwardPassState,
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
+ ) {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+
+ compute_encoder.condition(
+ parameters.predicate_ref(),
+ || {
+ self.encode_with_encoder_impl(state, compute_encoder);
+ },
+ None::<fn()>,
+ );
+
+ compute_encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ self.encode_with_encoder_impl(state, encoder);
+ }
+}
+
+impl RopeKernelEncodable {
+ fn encode_with_encoder_impl(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
) {
let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
@@ -200,12 +241,8 @@ impl EncodableWithState for RopeKernelEncodable {
state.arrays(&[ArrayId::RopeSines(self.rope_type)]);
let mut rope_sines = sin_buffer_binding[0].borrow_mut();
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
-
self.kernel.encode(
- &compute_encoder,
+ encoder,
RopeKernelArguments {
qkv_buffer: unsafe { &qkv.mtl_buffer() },
cosines_buffer: unsafe { &rope_cosines.mtl_buffer() },
@@ -224,12 +261,5 @@ impl EncodableWithState for RopeKernelEncodable {
max_sequence_length: rope_max_seq_len,
},
);
-
- compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
index 2ef7ff152..f1c5a3c59 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
@@ -247,6 +247,7 @@ impl SamplingKernel {
self.encode_with_encoder(
logits_buffer,
seeds_buffer,
+ 0, // No offset for non-async path
sampled_tokens_buffer,
sampling_method,
batch_size,
@@ -261,6 +262,7 @@ impl SamplingKernel {
&self,
logits_buffer: &MTLBuffer,
seeds_buffer: Option<&MTLBuffer>,
+ seeds_offset: usize,
sampled_tokens_buffer: &MTLBuffer,
sampling_method: SamplingMethod,
batch_size: usize,
@@ -327,6 +329,7 @@ impl SamplingKernel {
self.encode_gumbel(
last_logits_buffer,
seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,
+ seeds_offset,
&self.partial_gumbel_buffer,
batch_size as u32,
vocab_size as u32,
@@ -467,6 +470,7 @@ impl SamplingKernel {
&self,
logits_buffer: &MTLBuffer,
seeds_buffer: &MTLBuffer,
+ seeds_offset: usize,
processed_logits_buffer: &MTLBuffer,
batch_size: u32,
vocab_size: u32,
@@ -475,7 +479,7 @@ impl SamplingKernel {
compute_encoder.set_compute_pipeline_state(&self.gumbel_pipeline);
compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(seeds_buffer), 0);
+ compute_encoder.set_buffer(1, Some(seeds_buffer), seeds_offset as u64);
compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);
compute_encoder.set_bytes(
3,
@@ -639,6 +643,40 @@ impl EncodableWithState for SamplingKernelEncodable {
state: &mut ForwardPassState,
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
+ ) {
+ let root_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ self.encode_impl(state, encoder);
+
+ encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ command_buffer.commit_and_continue();
+ root_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &metal::ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ self.encode_impl(state, encoder);
+ }
+}
+
+impl SamplingKernelEncodable {
+ fn encode_impl(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &metal::ComputeCommandEncoderRef,
) {
assert!(
state.sampling_output.is_some(),
@@ -665,24 +703,19 @@ impl EncodableWithState for SamplingKernelEncodable {
state.sampling_output.as_ref().unwrap().borrow_mut();
let sampling_method = state.sampling_method.unwrap();
+ let seeds_offset = seeds.buffer_offset();
- let root_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- if let Err(e) = self.kernel.encode(
+ if let Err(e) = self.kernel.encode_with_encoder(
unsafe { &logits.mtl_buffer() },
unsafe { Some(&seeds.mtl_buffer()) },
+ seeds_offset,
unsafe { &output_buffer_ref.mtl_buffer() },
sampling_method,
batch_size,
vocab_size,
- &root_command_buffer,
+ encoder,
) {
panic!("Sampling encoding failed: {:?}", e);
}
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- root_command_buffer.wait_until_completed();
- }
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
index 45cdd7e1d..031af0d15 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
@@ -104,14 +104,25 @@ impl Conv1dScanKernel {
let decode_name =
format!("conv1d_decode_kernel_{}", fn_suffix(data_type));
let function_constants = make_function_constants(activation);
+ let activation_id = activation_to_int(activation);
+ let cache_key = format!("{}_act_{}", fn_name, activation_id);
let pipeline = context
- .compute_pipeline_state(&fn_name, Some(&function_constants))
+ .compute_pipeline_state_cached(
+ &cache_key,
+ &fn_name,
+ Some(&function_constants),
+ )
.map_err(SSMKernelError::MetalError)?;
let pack_pipeline = context
.compute_pipeline_state(&pack_name, None)
.map_err(SSMKernelError::MetalError)?;
+ let decode_cache_key = format!("{}_act_{}", decode_name, activation_id);
let decode_pipeline = context
- .compute_pipeline_state(&decode_name, Some(&function_constants))
+ .compute_pipeline_state_cached(
+ &decode_cache_key,
+ &decode_name,
+ Some(&function_constants),
+ )
.map_err(SSMKernelError::MetalError)?;
Ok(Self {
pipeline,
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs b/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs
index 5fc14fa75..a4d82b506 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/mamba.rs
@@ -1,12 +1,13 @@
use std::{env, rc::Rc};
+use metal::ComputeCommandEncoderRef;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use super::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
- SSDPrefillArguments, SSDPrefillKernel,
- SSDPrefillMode, SSDUpdateArguments, SSDUpdateKernel, SplitInProjArguments,
- SplitInProjKernel, conv1d_scan::Conv1dDecodeArguments,
+ SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,
+ SSDUpdateKernel, SplitInProjArguments, SplitInProjKernel,
+ conv1d_scan::Conv1dDecodeArguments,
};
use crate::{
DataType,
@@ -19,7 +20,7 @@ use crate::{
transformer_layer,
},
},
- config::{DecoderLayerType, mamba::Mamba2Config},
+ config::{DecoderLayerType, LinearConfig, mamba::Mamba2Config},
parameters::ParameterTree,
};
@@ -28,6 +29,7 @@ pub(crate) struct MambaMixerEncodable {
config: Mamba2Config,
in_projection: Box<dyn EncodableWithState>,
out_projection: Box<dyn EncodableWithState>,
+ projections_support_shared_encoder: bool,
split_inproj: SplitInProjKernel,
conv_scan: Conv1dScanKernel,
ssm_prefill: SSDPrefillKernel,
@@ -67,15 +69,23 @@ impl MambaMixerEncodable {
mamba_config.in_projection_config.activation_precision().into();
let kernel_data_type: KernelDataType = data_type.into();
+ let projections_are_quantized = matches!(
+ &mamba_config.in_projection_config,
+ LinearConfig::Quantized(_) | LinearConfig::MLXQuantized(_)
+ ) && matches!(
+ &mamba_config.out_projection_config,
+ LinearConfig::Quantized(_) | LinearConfig::MLXQuantized(_)
+ );
+
+ let in_projection_output_dim = mamba_config.conv_dim()
+ + mamba_config.inner_dim()
+ + mamba_config.num_heads;
+
let in_projection = transformer_layer::linear_block(
&mamba_config.in_projection_config,
mamba_config.has_in_biases,
model_dim,
- [
- mamba_config.conv_dim(),
- mamba_config.inner_dim(),
- mamba_config.num_heads,
- ],
+ [in_projection_output_dim],
mtl_context,
&resolve_subtree(
decoder_layer_loader,
@@ -131,6 +141,7 @@ impl MambaMixerEncodable {
config: mamba_config,
in_projection,
out_projection,
+ projections_support_shared_encoder: projections_are_quantized,
split_inproj,
conv_scan,
ssm_prefill,
@@ -143,7 +154,42 @@ impl MambaMixerEncodable {
}
}
- fn encode_pipeline(
+ fn encode_pipeline_with_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
+ ) {
+ let suffix_length = state.aux_buffers_suffix_length();
+ let active_suffix_length = state.active_suffix_length();
+ if suffix_length == 0 || active_suffix_length == 0 {
+ return;
+ }
+
+ self.in_projection
+ .encode_with_shared_encoder(state, encoder, parameters);
+ self.split_inproj_with_encoder(state, encoder, active_suffix_length);
+ self.run_conv_scan_with_encoder(state, encoder, active_suffix_length);
+
+ if suffix_length == 1 {
+ self.run_decode_ssm_with_encoder(
+ state,
+ encoder,
+ active_suffix_length,
+ );
+ } else {
+ self.run_prefill_ssm_with_encoder(
+ state,
+ encoder,
+ active_suffix_length,
+ );
+ }
+
+ self.out_projection
+ .encode_with_shared_encoder(state, encoder, parameters);
+ }
+
+ fn encode_pipeline_with_command_buffer(
&self,
state: &mut ForwardPassState,
command_buffer: &MPSCommandBuffer,
@@ -156,29 +202,34 @@ impl MambaMixerEncodable {
}
self.in_projection.encode(state, command_buffer, parameters);
- self.split_inproj(state, command_buffer, active_suffix_length);
- self.run_conv_scan(state, command_buffer, active_suffix_length);
+
+ let cmd = command_buffer.root_command_buffer().to_owned();
+ let encoder = cmd.new_compute_command_encoder();
+ self.split_inproj_with_encoder(state, encoder, active_suffix_length);
+ self.run_conv_scan_with_encoder(state, encoder, active_suffix_length);
if suffix_length == 1 {
- self.run_decode_ssm(state, command_buffer, active_suffix_length);
+ self.run_decode_ssm_with_encoder(
+ state,
+ encoder,
+ active_suffix_length,
+ );
} else {
- self.run_prefill_ssm(state, command_buffer, active_suffix_length);
+ self.run_prefill_ssm_with_encoder(
+ state,
+ encoder,
+ active_suffix_length,
+ );
}
+ encoder.end_encoding();
self.out_projection.encode(state, command_buffer, parameters);
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
}
- fn split_inproj(
+ fn split_inproj_with_encoder(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ encoder: &ComputeCommandEncoderRef,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -204,12 +255,9 @@ impl MambaMixerEncodable {
let num_heads = self.config.num_heads;
let total_dim = conv_dim + inner_dim + num_heads;
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute = mtl_command_buffer.new_compute_command_encoder();
self.split_inproj
.encode(
- &compute,
+ encoder,
SplitInProjArguments {
input: &input_buf,
conv_out: &conv_buf,
@@ -224,13 +272,12 @@ impl MambaMixerEncodable {
},
)
.expect("Failed to encode split in-projection kernel");
- compute.end_encoding();
}
- fn run_conv_scan(
+ fn run_conv_scan_with_encoder(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ encoder: &ComputeCommandEncoderRef,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -262,15 +309,13 @@ impl MambaMixerEncodable {
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
let proj_dim = self.config.num_groups * self.config.state_dim;
- let cmd = command_buffer.root_command_buffer().to_owned();
let state_stride = self.config.kernel_size.saturating_sub(1);
drop(conv_state);
if suffix_length == 1 {
- let compute = cmd.new_compute_command_encoder();
self.conv_scan
.encode_decode(
- &compute,
+ encoder,
Conv1dDecodeArguments {
x: &input_buf,
w: &weight_buf,
@@ -290,7 +335,6 @@ impl MambaMixerEncodable {
},
)
.expect("Failed to encode conv decode kernel");
- compute.end_encoding();
} else {
let padded_buf = if state_stride > 0 {
let array = state
@@ -300,24 +344,20 @@ impl MambaMixerEncodable {
let buf = unsafe { borrow.mtl_buffer().to_owned() };
drop(borrow);
- {
- let pack_encoder = cmd.new_compute_command_encoder();
- self.conv_scan
- .encode_pack(
- &pack_encoder,
- Conv1dPackArguments {
- state_in: &state_buf,
- x: &input_buf,
- padded: &buf,
- state_stride,
- row_stride: conv_dim,
- suffix_len: suffix_length,
- channels: conv_dim,
- },
- )
- .expect("Failed to encode conv pack kernel");
- pack_encoder.end_encoding();
- }
+ self.conv_scan
+ .encode_pack(
+ encoder,
+ Conv1dPackArguments {
+ state_in: &state_buf,
+ x: &input_buf,
+ padded: &buf,
+ state_stride,
+ row_stride: conv_dim,
+ suffix_len: suffix_length,
+ channels: conv_dim,
+ },
+ )
+ .expect("Failed to encode conv pack kernel");
Some(buf)
} else {
@@ -326,10 +366,9 @@ impl MambaMixerEncodable {
let conv_source = padded_buf.as_ref().unwrap_or(&input_buf);
- let compute = cmd.new_compute_command_encoder();
self.conv_scan
.encode(
- &compute,
+ encoder,
Conv1dScanArguments {
padded: conv_source,
w: &weight_buf,
@@ -348,14 +387,13 @@ impl MambaMixerEncodable {
},
)
.expect("Failed to encode conv scan kernel");
- compute.end_encoding();
}
}
- fn run_prefill_ssm(
+ fn run_prefill_ssm_with_encoder(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ encoder: &ComputeCommandEncoderRef,
suffix_length: usize,
) {
let base_arrays = state.arrays(&[
@@ -392,11 +430,9 @@ impl MambaMixerEncodable {
let mut skip_weights = self.skip_connection_weight.clone();
let skip = unsafe { skip_weights.mtl_buffer().to_owned() };
- let cmd = command_buffer.root_command_buffer().to_owned();
- let compute = cmd.new_compute_command_encoder();
self.ssm_prefill
.encode(
- &compute,
+ encoder,
SSDPrefillArguments {
x: &x_buf,
dt: &dt_buf,
@@ -432,13 +468,12 @@ impl MambaMixerEncodable {
self.prefill_mode,
)
.expect("Failed to encode SSD prefill kernel");
- compute.end_encoding();
}
- fn run_decode_ssm(
+ fn run_decode_ssm_with_encoder(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ encoder: &ComputeCommandEncoderRef,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -485,11 +520,9 @@ impl MambaMixerEncodable {
let group_size = (h / g) as i32;
let state_size = n as i32;
- let cmd = command_buffer.root_command_buffer().to_owned();
- let compute = cmd.new_compute_command_encoder();
self.ssd_update
.encode(
- &compute,
+ encoder,
SSDUpdateArguments {
x: &x_buf,
dt: &dt_buf,
@@ -512,7 +545,6 @@ impl MambaMixerEncodable {
},
)
.expect("Failed to encode SSD decode kernel");
- compute.end_encoding();
}
}
@@ -536,7 +568,37 @@ impl EncodableWithState for MambaMixerEncodable {
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
) {
- self.encode_pipeline(state, command_buffer, parameters);
+ if self.projections_support_shared_encoder {
+ let cmd = command_buffer.root_command_buffer().to_owned();
+ let encoder = cmd.new_compute_command_encoder();
+ self.encode_pipeline_with_encoder(state, encoder, parameters);
+ encoder.end_encoding();
+ } else {
+ self.encode_pipeline_with_command_buffer(
+ state,
+ command_buffer,
+ parameters,
+ );
+ }
+
+ if parameters.wait_until_completed {
+ let cmd = command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ cmd.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ self.projections_support_shared_encoder
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
+ ) {
+ self.encode_pipeline_with_encoder(state, encoder, parameters);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs
index 61838092b..fd6b0afd4 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs
@@ -2,11 +2,12 @@ use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
- ComputePipelineState as MTLComputePipelineState,
+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
};
use super::{
- KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,
+ KernelDataType, MTLContext,
+ metal_extensions::{ComputeEncoderConditional, ComputeEncoderDispatch},
};
use crate::backends::metal::error::MTLError;
@@ -37,24 +38,50 @@ impl TensorAddBias {
num_cols: usize,
total_len: usize,
command_buffer: &MTLCommandBuffer,
+ predicate: Option<&MTLBuffer>,
) {
let encoder = command_buffer.new_compute_command_encoder();
- encoder.set_label("Tensor Add Bias");
- encoder.set_compute_pipeline_state(&self.pipeline_state);
- encoder.set_buffer(0, Some(input), 0);
- encoder.set_buffer(1, Some(bias), 0);
- encoder.set_buffer(2, Some(output), 0);
- encoder.set_bytes(
- 3,
- size_of::<i32>() as u64,
- &(num_cols as i32) as *const _ as *const _,
+ self.encode_with_encoder(
+ input, bias, output, num_cols, total_len, encoder, predicate,
);
- encoder.set_bytes(
- 4,
- size_of::<i32>() as u64,
- &(total_len as i32) as *const _ as *const _,
- );
- encoder.dispatch_1d_exactly(&self.pipeline_state, total_len, None);
encoder.end_encoding();
}
+
+ pub fn encode_with_encoder(
+ &self,
+ input: &MTLBuffer,
+ bias: &MTLBuffer,
+ output: &MTLBuffer,
+ num_cols: usize,
+ total_len: usize,
+ encoder: &ComputeCommandEncoderRef,
+ predicate: Option<&MTLBuffer>,
+ ) {
+ encoder.condition(
+ predicate.map(|b| b.as_ref()),
+ || {
+ encoder.set_label("Tensor Add Bias");
+ encoder.set_compute_pipeline_state(&self.pipeline_state);
+ encoder.set_buffer(0, Some(input), 0);
+ encoder.set_buffer(1, Some(bias), 0);
+ encoder.set_buffer(2, Some(output), 0);
+ encoder.set_bytes(
+ 3,
+ size_of::<i32>() as u64,
+ &(num_cols as i32) as *const _ as *const _,
+ );
+ encoder.set_bytes(
+ 4,
+ size_of::<i32>() as u64,
+ &(total_len as i32) as *const _ as *const _,
+ );
+ encoder.dispatch_1d_exactly(
+ &self.pipeline_state,
+ total_len,
+ None,
+ );
+ },
+ None::<fn()>,
+ );
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
index 6a71a2dd3..19336e899 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
@@ -2,7 +2,7 @@ use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
- ComputePipelineState as MTLComputePipelineState,
+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
@@ -51,22 +51,27 @@ impl TensorAddSwap {
length: usize,
command_buffer: &MTLCommandBuffer,
) {
- let compute_encoder = command_buffer.new_compute_command_encoder();
-
- compute_encoder.set_label("Tensor Add-Swap");
-
- compute_encoder.set_buffer(0, Some(skip_buffer), 0);
- compute_encoder.set_buffer(1, Some(main_buffer), 0);
+ let encoder = command_buffer.new_compute_command_encoder();
+ self.encode_with_encoder_raw(skip_buffer, main_buffer, length, encoder);
+ encoder.end_encoding();
+ }
- compute_encoder.set_bytes(
+ fn encode_with_encoder_raw(
+ &self,
+ skip_buffer: &MTLBuffer,
+ main_buffer: &MTLBuffer,
+ length: usize,
+ encoder: &ComputeCommandEncoderRef,
+ ) {
+ encoder.set_label("Tensor Add-Swap");
+ encoder.set_buffer(0, Some(skip_buffer), 0);
+ encoder.set_buffer(1, Some(main_buffer), 0);
+ encoder.set_bytes(
2,
size_of::<i32>() as u64,
&(length as i32) as *const _ as *const std::ffi::c_void,
);
-
- compute_encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);
-
- compute_encoder.end_encoding();
+ encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);
}
}
@@ -102,4 +107,32 @@ impl EncodableWithState for TensorAddSwap {
retained_mtl_command_buffer.wait_until_completed();
}
}
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&self.argument_arrays);
+ assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");
+
+ let length = arrays[0].borrow().num_elements();
+
+ let mut skip_array = arrays[0].borrow_mut();
+ let mut main_array = arrays[1].borrow_mut();
+ let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };
+ let main_mtl_buffer = unsafe { main_array.mtl_buffer() };
+
+ self.encode_with_encoder_raw(
+ &skip_mtl_buffer,
+ &main_mtl_buffer,
+ length,
+ encoder,
+ );
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
index 5db33a53e..d278a8ba6 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
@@ -2,7 +2,7 @@ use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
- ComputePipelineState as MTLComputePipelineState,
+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
@@ -46,20 +46,34 @@ impl TensorCopy {
destination_buffer: &MTLBuffer,
length: usize,
command_buffer: &MTLCommandBuffer,
+ _predicate: Option<&MTLBuffer>,
) {
- let compute_encoder = command_buffer.new_compute_command_encoder();
- compute_encoder.set_label("Tensor Copy");
+ let encoder = command_buffer.new_compute_command_encoder();
+ self.encode_with_encoder_raw(
+ source_buffer,
+ destination_buffer,
+ length,
+ encoder,
+ );
+ encoder.end_encoding();
+ }
- compute_encoder.set_buffer(0, Some(source_buffer), 0);
- compute_encoder.set_buffer(1, Some(destination_buffer), 0);
- compute_encoder.set_bytes(
+ fn encode_with_encoder_raw(
+ &self,
+ source_buffer: &MTLBuffer,
+ destination_buffer: &MTLBuffer,
+ length: usize,
+ encoder: &ComputeCommandEncoderRef,
+ ) {
+ encoder.set_label("Tensor Copy");
+ encoder.set_buffer(0, Some(source_buffer), 0);
+ encoder.set_buffer(1, Some(destination_buffer), 0);
+ encoder.set_bytes(
2,
size_of::<i32>() as u64,
&(length as i32) as *const _ as *const std::ffi::c_void,
);
-
- compute_encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);
- compute_encoder.end_encoding();
+ encoder.dispatch_1d_exactly(&self.pipeline_state, length, None);
}
}
@@ -88,6 +102,7 @@ impl EncodableWithState for TensorCopy {
&destination_mtl_buffer,
length,
&retained_mtl_command_buffer,
+ parameters.predicate,
);
if parameters.wait_until_completed {
@@ -95,4 +110,32 @@ impl EncodableWithState for TensorCopy {
retained_mtl_command_buffer.wait_until_completed();
}
}
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&self.argument_arrays);
+ assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");
+
+ let length = arrays[0].borrow().num_elements();
+
+ let mut source_array = arrays[0].borrow_mut();
+ let mut destination_array = arrays[1].borrow_mut();
+ let source_mtl_buffer = unsafe { source_array.mtl_buffer() };
+ let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };
+
+ self.encode_with_encoder_raw(
+ &source_mtl_buffer,
+ &destination_mtl_buffer,
+ length,
+ encoder,
+ );
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs
new file mode 100644
index 000000000..15e449c41
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/token_copy/mod.rs
@@ -0,0 +1,67 @@
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState, MTLSize,
+};
+
+use crate::backends::metal::MTLContext;
+
+/// Kernel for copying sampled tokens in async pipeline.
+/// Two operations:
+/// 1. copy_to_token_ids: u32 → u64 for next forward pass embedding
+/// 2. copy_to_results: u32 → u32[offset] for callback to read
+pub struct TokenCopyKernel {
+ copy_to_token_ids: MTLComputePipelineState,
+ copy_to_results: MTLComputePipelineState,
+}
+
+impl TokenCopyKernel {
+ pub fn new(
+ context: &MTLContext
+ ) -> Result<Self, crate::backends::metal::error::MTLError> {
+ let (copy_to_token_ids, _) = context
+ .compute_pipeline_state_with_reflection(
+ "copy_sampled_token",
+ None,
+ )?;
+ let (copy_to_results, _) = context
+ .compute_pipeline_state_with_reflection(
+ "copy_token_to_results",
+ None,
+ )?;
+
+ Ok(Self {
+ copy_to_token_ids,
+ copy_to_results,
+ })
+ }
+
+ /// Copies sampling_output[0] (u32) → token_ids[0] (u64)
+ /// For next forward pass to read the token.
+ pub fn encode_to_token_ids(
+ &self,
+ sampling_output: &MTLBuffer,
+ token_ids: &MTLBuffer,
+ encoder: &ComputeCommandEncoderRef,
+ ) {
+ encoder.set_compute_pipeline_state(&self.copy_to_token_ids);
+ encoder.set_buffer(0, Some(sampling_output), 0);
+ encoder.set_buffer(1, Some(token_ids), 0);
+ encoder.dispatch_threads(MTLSize::new(1, 1, 1), MTLSize::new(1, 1, 1));
+ }
+
+ /// Copies sampling_output[0] (u32) → results[offset] (u32)
+ /// For callback to read without race condition.
+ pub fn encode_to_results(
+ &self,
+ sampling_output: &MTLBuffer,
+ results: &MTLBuffer,
+ pass_idx: usize,
+ encoder: &ComputeCommandEncoderRef,
+ ) {
+ let offset = (pass_idx * std::mem::size_of::<u32>()) as u64;
+ encoder.set_compute_pipeline_state(&self.copy_to_results);
+ encoder.set_buffer(0, Some(sampling_output), 0);
+ encoder.set_buffer(1, Some(results), offset);
+ encoder.dispatch_threads(MTLSize::new(1, 1, 1), MTLSize::new(1, 1, 1));
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
new file mode 100644
index 000000000..b9f458c16
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
@@ -0,0 +1,26 @@
+#include <metal_stdlib>
+using namespace metal;
+
+/// Copies a single sampled token (u32) to the token_ids buffer (u64).
+/// Used in async pipeline to pass sampled token to next forward pass.
+kernel void copy_sampled_token(
+ device const uint32_t* src [[buffer(0)]],
+ device uint64_t* dst [[buffer(1)]],
+ uint idx [[thread_position_in_grid]]
+) {
+ if (idx == 0) {
+ dst[0] = static_cast<uint64_t>(src[0]);
+ }
+}
+
+/// Copies a single sampled token (u32) to an indexed results buffer (u32).
+/// Used in async pipeline to store result for callback access.
+kernel void copy_token_to_results(
+ device const uint32_t* src [[buffer(0)]],
+ device uint32_t* dst [[buffer(1)]],
+ uint idx [[thread_position_in_grid]]
+) {
+ if (idx == 0) {
+ dst[0] = src[0];
+ }
+}
diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
new file mode 100644
index 000000000..cae917be8
--- /dev/null
+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
@@ -0,0 +1,131 @@
+#![allow(unexpected_cfgs)]
+
+use metal::{
+ BufferRef, ComputeCommandEncoder, ComputeCommandEncoderRef,
+ MTLCompareFunction,
+ foreign_types::{ForeignType, ForeignTypeRef},
+ objc::{msg_send, runtime::Object, sel, sel_impl},
+};
+
+/// Low-level, unsafe conditional control of Metal encoders.
+/// This is internal; users should prefer the safe `ComputeEncoderConditional::condition`.
+trait ComputeEncoderRawConditional {
+ unsafe fn encode_start_if(
+ &self,
+ predicate: &BufferRef,
+ offset: usize,
+ comparison: MTLCompareFunction,
+ reference_value: u32,
+ );
+
+ unsafe fn encode_start_else(&self);
+
+ unsafe fn encode_end_if(&self) -> bool;
+}
+
+/// Safe conditional wrapper that uses the raw unsafe methods internally.
+pub trait ComputeEncoderConditional {
+ fn condition<IfBlock, ElseBlock>(
+ &self,
+ predicate: Option<&BufferRef>,
+ if_block: IfBlock,
+ else_block: Option<ElseBlock>,
+ ) where
+ IfBlock: FnOnce(),
+ ElseBlock: FnOnce();
+}
+
+impl ComputeEncoderRawConditional for ComputeCommandEncoder {
+ unsafe fn encode_start_if(
+ &self,
+ predicate: &BufferRef,
+ offset: usize,
+ comparison: MTLCompareFunction,
+ reference_value: u32,
+ ) {
+ let obj = self.as_ptr() as *mut Object;
+ let predicate_ptr = predicate.as_ptr() as *mut Object;
+ let _: () = msg_send![
+ obj,
+ encodeStartIf: predicate_ptr
+ offset: offset
+ comparison: comparison
+ referenceValue: reference_value
+ ];
+ }
+
+ unsafe fn encode_start_else(&self) {
+ let obj = self.as_ptr() as *mut Object;
+ let _: () = msg_send![obj, encodeStartElse];
+ }
+
+ unsafe fn encode_end_if(&self) -> bool {
+ let obj = self.as_ptr() as *mut Object;
+ let result: bool = msg_send![obj, encodeEndIf];
+ result
+ }
+}
+
+impl ComputeEncoderRawConditional for ComputeCommandEncoderRef {
+ unsafe fn encode_start_if(
+ &self,
+ predicate: &BufferRef,
+ offset: usize,
+ comparison: MTLCompareFunction,
+ reference_value: u32,
+ ) {
+ let obj = self as *const _ as *mut Object;
+ let predicate_ptr = predicate.as_ptr() as *mut Object;
+ let _: () = msg_send![
+ obj,
+ encodeStartIf:predicate_ptr
+ offset:offset
+ comparison:comparison
+ referenceValue:reference_value
+ ];
+ }
+
+ unsafe fn encode_start_else(&self) {
+ let obj = self as *const _ as *mut Object;
+ let _: () = msg_send![obj, encodeStartElse];
+ }
+
+ unsafe fn encode_end_if(&self) -> bool {
+ let obj = self as *const _ as *mut Object;
+ let result: bool = msg_send![obj, encodeEndIf];
+ result
+ }
+}
+
+impl<T> ComputeEncoderConditional for T
+where
+ T: ComputeEncoderRawConditional,
+{
+ fn condition<IfBlock, ElseBlock>(
+ &self,
+ predicate: Option<&BufferRef>,
+ if_block: IfBlock,
+ else_block: Option<ElseBlock>,
+ ) where
+ IfBlock: FnOnce(),
+ ElseBlock: FnOnce(),
+ {
+ match (predicate, else_block) {
+ (Some(p), Some(else_fn)) => unsafe {
+ self.encode_start_if(p, 0, MTLCompareFunction::Equal, 0);
+ if_block();
+ self.encode_start_else();
+ else_fn();
+ self.encode_end_if();
+ },
+ (Some(p), None) => unsafe {
+ self.encode_start_if(p, 0, MTLCompareFunction::Equal, 0);
+ if_block();
+ self.encode_end_if();
+ },
+ (None, _) => {
+ if_block();
+ },
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/metal_extensions/mod.rs b/crates/uzu/src/backends/metal/metal_extensions/mod.rs
index 6d7545a16..59efb625c 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/mod.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/mod.rs
@@ -1,6 +1,7 @@
#![allow(dead_code, unused_variables, unused_imports)]
mod command_encoder_extensions_device;
+mod compute_command_encoder_extensions_conditional;
mod compute_command_encoder_extensions_dispatch;
mod compute_pipeline_state_extensions_device;
mod compute_pipeline_state_extensions_threads;
@@ -8,5 +9,6 @@ mod device_extensions_features;
mod library_extensions_pipeline;
pub mod command_buffer_extensions;
+pub use compute_command_encoder_extensions_conditional::ComputeEncoderConditional;
pub use compute_command_encoder_extensions_dispatch::ComputeEncoderDispatch;
pub use library_extensions_pipeline::LibraryPipelineExtensions;
diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rs
index 8bf09a4f8..72e0a84b6 100644
--- a/crates/uzu/src/config/decoder.rs
+++ b/crates/uzu/src/config/decoder.rs
@@ -229,6 +229,16 @@ impl DecoderConfig {
pub fn group_size(&self) -> usize {
self.num_heads * self.num_groups
}
+
+ pub fn has_attention_layers(&self) -> bool {
+ if let Some(layer_types) = &self.layer_types {
+ layer_types
+ .iter()
+ .any(|lt| matches!(lt, DecoderLayerType::Transformer))
+ } else {
+ matches!(self.layer_config.mixer_config, MixerConfig::Attention(_))
+ }
+ }
}
#[cfg(test)]
diff --git a/crates/uzu/src/generator/context.rs b/crates/uzu/src/generator/context.rs
index b8d6e3fcc..edbbcb95f 100644
--- a/crates/uzu/src/generator/context.rs
+++ b/crates/uzu/src/generator/context.rs
@@ -1,5 +1,12 @@
-use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
+use std::{
+ cell::{Cell, RefCell},
+ fs::File,
+ io::BufReader,
+ path::Path,
+ rc::Rc,
+};
+use metal::Event as MTLEvent;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use objc2::rc::Retained;
@@ -10,7 +17,7 @@ use crate::{
MTLContext, ModelShape,
compilation_parameters::CompilationConfig,
forward_pass::{ForwardPassBuffers, SharedBuffers},
- kernel::SamplingKernelEncodable,
+ kernel::{SamplingKernelEncodable, TokenCopyKernel},
},
config::{LanguageModelConfig, ModelMetadata},
generator::rng::DerivableSeed,
@@ -22,6 +29,105 @@ use crate::{
},
};
+/// Pre-allocated buffers for async generation pipeline.
+/// Indexed by pass_idx to avoid race conditions between GPU passes.
+pub struct AsyncBuffers {
+ /// Positions buffer: [max_tokens] i32
+ /// Pre-populated with [prefill_count, prefill_count+1, ...]
+ pub positions: metal::Buffer,
+ /// Seeds buffer: [max_tokens] u64
+ /// Pre-populated with deterministic seed sequence
+ pub seeds: metal::Buffer,
+ /// Results buffer: [batch_size] u32
+ /// Each pass writes its sampled token to results[pass_idx % batch_size]
+ pub results: metal::Buffer,
+ /// Metal event for GPU-side synchronization between passes
+ pub event: MTLEvent,
+ /// Current event counter (pass N waits on N, signals N+1)
+ pub counter: Cell<u64>,
+ /// Number of tokens after prefill (base for position calculation)
+ pub prefill_count: Cell<usize>,
+ /// Batch size (number of passes to keep in flight)
+ pub batch_size: usize,
+}
+
+impl AsyncBuffers {
+ pub fn new(
+ device: &metal::DeviceRef,
+ max_tokens: usize,
+ batch_size: usize,
+ ) -> Self {
+ let positions = device.new_buffer(
+ (max_tokens * std::mem::size_of::<i32>()) as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ let seeds = device.new_buffer(
+ (max_tokens * std::mem::size_of::<u64>()) as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ let results = device.new_buffer(
+ (batch_size * std::mem::size_of::<u32>()) as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ let event = device.new_event();
+
+ Self {
+ positions,
+ seeds,
+ results,
+ event,
+ counter: Cell::new(0),
+ prefill_count: Cell::new(0),
+ batch_size,
+ }
+ }
+
+ /// Prepare positions buffer: [prefill_count-1, prefill_count, ...]
+ /// Uses prefill_count-1 to match sync path which uses self.tokens.len()-1
+ pub fn prepare_positions(
+ &self,
+ prefill_count: usize,
+ tokens_to_generate: usize,
+ ) {
+ self.prefill_count.set(prefill_count);
+ let base_position = prefill_count.saturating_sub(1);
+ let ptr = self.positions.contents() as *mut i32;
+ for i in 0..tokens_to_generate {
+ unsafe {
+ *ptr.add(i) = (base_position + i) as i32;
+ }
+ }
+ }
+
+ /// Prepare seeds buffer with deterministic sequence
+ pub fn prepare_seeds(
+ &self,
+ seed_source: &mut DerivableSeed,
+ tokens_to_generate: usize,
+ ) {
+ let ptr = self.seeds.contents() as *mut u64;
+ for i in 0..tokens_to_generate {
+ unsafe {
+ *ptr.add(i) = seed_source.next();
+ }
+ }
+ }
+
+ /// Reset event counter before async generation
+ pub fn reset_counter(&self) {
+ self.counter.set(0);
+ }
+
+ /// Read sampled token from results buffer at given pass index
+ pub fn read_result(
+ &self,
+ pass_idx: usize,
+ ) -> u32 {
+ let ptr = self.results.contents() as *const u32;
+ unsafe { *ptr.add(pass_idx % self.batch_size) }
+ }
+}
+
pub struct GeneratorContext {
pub mtl_context: Rc<MTLContext>,
pub command_buffer: Retained<MPSCommandBuffer>,
@@ -36,6 +142,11 @@ pub struct GeneratorContext {
pub kv_cache_update: Box<KVCacheUpdate>,
pub gpu_sampler: SamplingKernelEncodable,
pub next_seed: DerivableSeed,
+
+ /// Kernel for copying sampled tokens in async pipeline
+ pub token_copy: TokenCopyKernel,
+ /// Pre-allocated buffers for async generation
+ pub async_buffers: AsyncBuffers,
}
impl GeneratorContext {
@@ -139,6 +250,17 @@ impl GeneratorContext {
)
.map_err(|_| Error::UnableToCreateMetalContext)?;
+ let token_copy = TokenCopyKernel::new(&mtl_context)
+ .map_err(|_| Error::UnableToCreateMetalContext)?;
+
+ let async_batch_size =
+ decoding_config.async_batch_size.resolve(model_path);
+ let async_buffers = AsyncBuffers::new(
+ &mtl_context.device,
+ max_prefix_length,
+ async_batch_size,
+ );
+
let base_seed = decoding_config.sampling_seed.resolve();
let next_seed = DerivableSeed::new(base_seed);
@@ -154,6 +276,8 @@ impl GeneratorContext {
kv_cache_update,
gpu_sampler,
next_seed,
+ token_copy,
+ async_buffers,
};
return Ok(context);
diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rs
index 3bea1cf9d..7b47ff0f6 100644
--- a/crates/uzu/src/generator/generator.rs
+++ b/crates/uzu/src/generator/generator.rs
@@ -1,5 +1,6 @@
-use std::{collections::HashMap, path::Path, time::Instant};
+use std::{collections::HashMap, path::Path, sync::Arc, time::Instant};
+use block::ConcreteBlock;
use mpsgraph::CommandBuffer;
use super::{
@@ -565,4 +566,164 @@ impl Generator {
self.registered_prefix_len = desired_prefix_len;
}
}
+
+ /// Prepares async buffers for generation.
+ /// Must be called after prefill, before async_generate loop.
+ pub fn prepare_async(
+ &mut self,
+ tokens_to_generate: usize,
+ ) {
+ let prefill_count = self.tokens.len();
+ self.context
+ .async_buffers
+ .prepare_positions(prefill_count, tokens_to_generate);
+ self.context
+ .async_buffers
+ .prepare_seeds(&mut self.context.next_seed, tokens_to_generate);
+ self.context.async_buffers.reset_counter();
+ }
+
+ /// Submits a single async forward pass.
+ /// Does NOT block. Callback fires when GPU completes.
+ ///
+ /// - `pass_idx`: Index of this pass (0, 1, 2, ...)
+ /// - `sampling_method`: Sampling configuration
+ /// - `on_complete`: Callback receiving sampled token as u64
+ pub fn async_generate<F>(
+ &mut self,
+ pass_idx: usize,
+ sampling_method: SamplingMethod,
+ on_complete: F,
+ ) -> Result<(), Error>
+ where
+ F: FnOnce(u64) + Send + 'static,
+ {
+ assert_eq!(
+ self.decoding_config.generate_suffix_length(),
+ 1,
+ "async_generate only supports suffix_length=1"
+ );
+
+ // Extract values from async_buffers before mutable borrow
+ let current_counter = self.context.async_buffers.counter.get();
+ let is_continuation = current_counter > 0;
+ let batch_size = self.context.async_buffers.batch_size;
+ let slot = pass_idx % batch_size;
+ let async_event = self.context.async_buffers.event.clone();
+ let results_buffer = self.context.async_buffers.results.clone();
+ let async_positions_buffer =
+ self.context.async_buffers.positions.clone();
+ let async_seeds_buffer = self.context.async_buffers.seeds.clone();
+
+ let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;
+
+ let task = GeneratorRunTask {
+ token_ids: vec![last_token],
+ token_positions: vec![0], // Ignored, using async buffer
+ token_seeds: vec![0], // Ignored, using async buffer
+ expected_number_of_new_tokens: 1,
+ active_suffix_length: 1,
+ is_prefilling: false,
+ };
+
+ let async_positions = Some((&async_positions_buffer, pass_idx));
+ let async_seeds = Some((&async_seeds_buffer, pass_idx));
+
+ let mut state = ForwardPassState::new(
+ self.context.mtl_context.clone(),
+ &self.context.model_config.decoder_config,
+ &self.context.model_shape,
+ &self.context.scratch_buffers,
+ self.context.cache_layers.clone(),
+ self.context.shared_buffers.clone(),
+ &task.token_ids,
+ &task.token_positions,
+ task.active_suffix_length,
+ task.is_prefilling,
+ &task.token_seeds,
+ false,
+ None,
+ is_continuation,
+ async_positions,
+ async_seeds,
+ );
+ state.sampling_method = Some(sampling_method);
+
+ self.context.reset_command_buffer();
+ let root_command_buffer =
+ self.context.command_buffer.root_command_buffer();
+
+ // Wait on previous pass if this is a continuation
+ if is_continuation {
+ root_command_buffer
+ .encode_wait_for_event(&async_event, current_counter);
+ }
+
+ // Encode forward pass
+ self.context.executables.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &EncodingParameters::new(false, false, false),
+ );
+
+ // Encode sampling
+ self.context.gpu_sampler.encode(
+ &mut state,
+ &self.context.command_buffer,
+ &EncodingParameters::new(false, false, false),
+ );
+
+ // Copy sampled token: sampling_output → token_ids (for next pass)
+ // and sampling_output → results[slot] (for callback)
+ let sampling_output = state
+ .sampling_output
+ .as_ref()
+ .expect("sampling_output must exist after sampling encode");
+ let sampling_output_buffer =
+ unsafe { sampling_output.borrow_mut().mtl_buffer().clone() };
+ let token_ids_buffer = self.context.scratch_buffers.token_ids.clone();
+
+ let encoder = root_command_buffer.new_compute_command_encoder();
+ self.context.token_copy.encode_to_token_ids(
+ &sampling_output_buffer,
+ &token_ids_buffer,
+ encoder,
+ );
+ self.context.token_copy.encode_to_results(
+ &sampling_output_buffer,
+ &results_buffer,
+ slot,
+ encoder,
+ );
+ encoder.end_encoding();
+
+ // Signal event for next pass
+ let next_counter = current_counter + 1;
+ root_command_buffer.encode_signal_event(&async_event, next_counter);
+ self.context.async_buffers.counter.set(next_counter);
+
+ // Add completion handler
+ let results_buffer_clone = results_buffer.clone();
+ let callback = Arc::new(std::sync::Mutex::new(Some(on_complete)));
+
+ let block = ConcreteBlock::new(move |_: &metal::CommandBufferRef| {
+ let token = {
+ let ptr = results_buffer_clone.contents() as *const u32;
+ unsafe { *ptr.add(slot) as u64 }
+ };
+ if let Some(cb) = callback.lock().unwrap().take() {
+ cb(token);
+ }
+ });
+ let block = block.copy();
+
+ root_command_buffer.add_completed_handler(&block);
+ self.context.command_buffer.commit_and_continue();
+
+ Ok(())
+ }
+
+ pub fn has_attention_layers(&self) -> bool {
+ self.context.model_config.decoder_config.has_attention_layers()
+ }
}
diff --git a/crates/uzu/src/generator/tasks.rs b/crates/uzu/src/generator/tasks.rs
index a04cdbda0..9d97f175d 100644
--- a/crates/uzu/src/generator/tasks.rs
+++ b/crates/uzu/src/generator/tasks.rs
@@ -1,3 +1,7 @@
+use std::mem::size_of;
+
+use metal::{Buffer, MTLResourceOptions};
+
use super::context::GeneratorContext;
use crate::backends::metal::{
ForwardPassState,
@@ -8,6 +12,24 @@ use crate::backends::metal::{
pub struct GeneratorEncodedTask {
pub key: String,
+ predicate_buffer: Buffer,
+}
+
+impl GeneratorEncodedTask {
+ pub fn predicate_buffer(&self) -> &Buffer {
+ &self.predicate_buffer
+ }
+
+ pub fn disable_execution(&self) {
+ unsafe {
+ let ptr = self.predicate_buffer.contents() as *mut u32;
+ *ptr = 1;
+ self.predicate_buffer.did_modify_range(metal::NSRange::new(
+ 0,
+ size_of::<u32>() as u64,
+ ));
+ }
+ }
}
#[derive(Debug, Clone)]
@@ -61,6 +83,9 @@ impl GeneratorRunTask {
&self.token_seeds,
false,
external_bias_fn,
+ false,
+ None,
+ None,
);
return state;
@@ -74,8 +99,15 @@ impl GeneratorRunTask {
key: String,
) -> GeneratorEncodedTask {
context.executables.encode(state, &context.command_buffer, parameters);
- GeneratorEncodedTask {
+
+ let encoded_task = GeneratorEncodedTask {
key,
- }
+ predicate_buffer: context.mtl_context.device.new_buffer(
+ size_of::<u32>() as u64,
+ MTLResourceOptions::StorageModeShared,
+ ),
+ };
+
+ encoded_task
}
}
diff --git a/crates/uzu/src/runners/decoder_runner.rs b/crates/uzu/src/runners/decoder_runner.rs
index 34b77814b..e740cdabb 100644
--- a/crates/uzu/src/runners/decoder_runner.rs
+++ b/crates/uzu/src/runners/decoder_runner.rs
@@ -132,6 +132,9 @@ impl DecoderTestContext {
&token_seeds,
false,
None,
+ false,
+ None,
+ None,
);
let executables = DecoderExecutables::new(
diff --git a/crates/uzu/src/session/config/decoding_config.rs b/crates/uzu/src/session/config/decoding_config.rs
index 350feff98..05a752456 100644
--- a/crates/uzu/src/session/config/decoding_config.rs
+++ b/crates/uzu/src/session/config/decoding_config.rs
@@ -1,6 +1,9 @@
use crate::session::{
config::SpeculatorConfig,
- parameter::{ContextLength, ContextMode, PrefillStepSize, SamplingSeed},
+ parameter::{
+ AsyncBatchSize, ContextLength, ContextMode, PrefillStepSize,
+ SamplingSeed,
+ },
};
#[derive(Clone)]
@@ -11,6 +14,7 @@ pub struct DecodingConfig {
pub speculator_config: SpeculatorConfig,
pub sampling_seed: SamplingSeed,
pub allow_pre_encode: bool,
+ pub async_batch_size: AsyncBatchSize,
}
impl DecodingConfig {
@@ -29,6 +33,7 @@ impl DecodingConfig {
speculator_config,
sampling_seed,
allow_pre_encode,
+ async_batch_size: AsyncBatchSize::default(),
}
}
@@ -46,6 +51,7 @@ impl Default for DecodingConfig {
speculator_config: SpeculatorConfig::default(),
sampling_seed: SamplingSeed::default(),
allow_pre_encode: true,
+ async_batch_size: AsyncBatchSize::default(),
}
}
}
@@ -110,4 +116,14 @@ impl DecodingConfig {
..self.clone()
}
}
+
+ pub fn with_async_batch_size(
+ &self,
+ async_batch_size: AsyncBatchSize,
+ ) -> Self {
+ Self {
+ async_batch_size,
+ ..self.clone()
+ }
+ }
}
diff --git a/crates/uzu/src/session/parameter/async_batch_size.rs b/crates/uzu/src/session/parameter/async_batch_size.rs
new file mode 100644
index 000000000..4c0093c70
--- /dev/null
+++ b/crates/uzu/src/session/parameter/async_batch_size.rs
@@ -0,0 +1,45 @@
+use std::path::Path;
+
+use crate::utils::{DeviceClass, ModelSize};
+
+#[derive(Debug, Clone, Copy)]
+pub enum AsyncBatchSize {
+ Default,
+ Custom(usize),
+}
+
+impl Default for AsyncBatchSize {
+ fn default() -> Self {
+ AsyncBatchSize::Default
+ }
+}
+
+impl AsyncBatchSize {
+ pub fn resolve(
+ &self,
+ model_path: &Path,
+ ) -> usize {
+ match self {
+ AsyncBatchSize::Default => {
+ let device = DeviceClass::detect();
+ let model_size = ModelSize::from_path(model_path);
+ default_batch_size(device, model_size)
+ },
+ AsyncBatchSize::Custom(value) => *value,
+ }
+ }
+}
+
+fn default_batch_size(
+ device: DeviceClass,
+ model_size: ModelSize,
+) -> usize {
+ match (model_size, device.is_high_end(), device) {
+ (ModelSize::Large, true, _) => 32,
+ (ModelSize::Large, _, DeviceClass::IPhone) => 4,
+ (ModelSize::Large, _, _) => 8,
+ (ModelSize::Small, true, _) => 256,
+ (ModelSize::Small, _, DeviceClass::IPhone) => 8,
+ (ModelSize::Small, _, _) => 64,
+ }
+}
diff --git a/crates/uzu/src/session/parameter/mod.rs b/crates/uzu/src/session/parameter/mod.rs
index 7a0299bcf..6470f6549 100644
--- a/crates/uzu/src/session/parameter/mod.rs
+++ b/crates/uzu/src/session/parameter/mod.rs
@@ -1,9 +1,11 @@
+mod async_batch_size;
mod context_length;
mod context_mode;
mod prefill_step_size;
mod resolvable_value;
mod sampling_policy;
mod sampling_seed;
+pub use async_batch_size::AsyncBatchSize;
pub use context_length::ContextLength;
pub use context_mode::ContextMode;
pub use prefill_step_size::PrefillStepSize;
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rs
index 69f3ffc1d..9d9388781 100644
--- a/crates/uzu/src/session/session.rs
+++ b/crates/uzu/src/session/session.rs
@@ -1,4 +1,18 @@
-use std::{fs::File, io::BufReader, path::PathBuf, time::Instant};
+use std::{
+ fs::File,
+ io::BufReader,
+ path::PathBuf,
+ sync::{
+ Arc,
+ atomic::{AtomicU64, Ordering},
+ mpsc,
+ },
+ time::Instant,
+};
+
+fn nanos_to_secs(nanos: u64) -> f64 {
+ nanos as f64 / 1_000_000_000.0
+}
use objc2::rc::autoreleasepool;
use tokenizers::Tokenizer;
@@ -16,7 +30,7 @@ use crate::{
Context, InputProcessor, InputProcessorDefault, OutputParser,
is_directory_fits_ram,
},
- parameter::{ConfigResolvableValue, ContextMode},
+ parameter::{ConfigResolvableValue, ContextMode, SamplingMethod},
types::{
Error, FinishReason, Input, Output, RunStats, Stats, StepStats,
TotalStats,
@@ -24,6 +38,18 @@ use crate::{
},
};
+struct RunContext {
+ eos_tokens: Vec<u64>,
+ context_length: usize,
+ prefix_len_before: usize,
+ input_tokens_len: usize,
+ tokens_limit: usize,
+ prefill_result: PrefillResult,
+ prefill_duration: f64,
+ prefill_suffix_length: usize,
+ run_start: Instant,
+}
+
pub struct Session {
pub model_path: PathBuf,
pub model_metadata: ModelMetadata,
@@ -151,13 +177,13 @@ impl Session {
if self.static_context.is_none() {
let mut prefill_config = config.clone();
prefill_config.tokens_limit = 0;
- let (_out, ctx) =
+ let (_out, run_context) =
self.extend(input.clone(), None, prefill_config)?;
- self.static_context = Some(ctx);
+ self.static_context = Some(run_context);
}
let tmp = self.static_context.take();
- if let Some(ref ctx) = tmp {
- self.reconfigure_generator(Some(ctx))?;
+ if let Some(ref run_context) = tmp {
+ self.reconfigure_generator(Some(run_context))?;
}
self.static_context = tmp;
},
@@ -237,8 +263,6 @@ impl Session {
return Err(Error::ContextLengthExceeded);
}
- let prefix_len_before = generator.tokens.len().saturating_sub(1);
-
let eos_tokens: Vec<u64> = self
.model_metadata
.model_config
@@ -248,47 +272,12 @@ impl Session {
.map(|&x| x as u64)
.collect();
- let finish_reason = |generator: &Generator,
- tokens_new: Vec<u64>|
- -> Option<FinishReason> {
- let start_idx = prefix_len_before + tokens.len();
- let total_tokens = generator.tokens.len();
- let total_new_tokens = generator.tokens[start_idx..].len();
- let has_eos_token =
- tokens_new.iter().any(|token| eos_tokens.contains(token));
- let context_limit_reached = total_tokens >= context_length;
-
- if has_eos_token {
- Some(FinishReason::Stop)
- } else if total_new_tokens >= config.tokens_limit as usize {
- Some(FinishReason::Length)
- } else if context_limit_reached {
- Some(FinishReason::ContextLimitReached)
- } else {
- None
- }
- };
-
- let build_generated_text = |generator: &Generator,
- tokenizer: &Tokenizer|
- -> Result<String, Error> {
- let start_idx = prefix_len_before + tokens.len();
- let generated_tokens: Vec<u32> = generator.tokens[start_idx..]
- .to_vec()
- .iter()
- .map(|value| *value as u32)
- .collect();
- let generated_text = tokenizer
- .decode(&generated_tokens, true)
- .map_err(|_| Error::UnableToDecodeText)?;
- Ok(generated_text)
- };
-
let sampling_method =
config.sampling_policy.resolve(&self.model_metadata.model_config);
let prefill_start = Instant::now();
let prefix_offset = generator.tokens.len();
+ let prefix_len_before = prefix_offset.saturating_sub(1);
let sample_suffix = config.tokens_limit > 0;
let prefill_result = generator.prefill(
@@ -301,36 +290,38 @@ impl Session {
let prefill_duration = prefill_start.elapsed().as_secs_f64();
generator.clear_cache();
- let prefill_finish_reason =
- finish_reason(generator, prefill_tokens.clone());
- let prefill_generated_text =
- build_generated_text(generator, &self.tokenizer)?;
- let prefill_parsed_text =
- self.output_parser.parse(prefill_generated_text);
-
let prefill_suffix_length = generator
.decoding_config
.prefill_step_size
.resolve(&self.model_metadata.model_config);
- let prefill_output = Output {
- text: prefill_parsed_text,
- stats: Self::build_stats(
- &self.model_metadata,
- prefill_result.clone(),
- prefill_duration,
- prefill_suffix_length,
- Vec::new(),
- Vec::new(),
- generator.decoding_config.generate_suffix_length(),
- run_start.elapsed().as_secs_f64(),
- tokens.len(),
- generator.tokens[prefix_len_before + tokens.len()..].len(),
- ),
- finish_reason: prefill_finish_reason.clone(),
+
+ let run_context = RunContext {
+ eos_tokens,
+ context_length,
+ prefix_len_before,
+ input_tokens_len: tokens.len(),
+ tokens_limit: config.tokens_limit as usize,
+ prefill_result: prefill_result.clone(),
+ prefill_duration,
+ prefill_suffix_length,
+ run_start,
};
- let prefill_should_continue = if let Some(progress) = &progress {
- progress(prefill_output.clone())
+ let prefill_finish_reason =
+ Self::check_finish_reason(&run_context, generator, &prefill_tokens);
+ let prefill_output = Self::build_output(
+ &self.model_metadata,
+ &self.tokenizer,
+ &self.output_parser,
+ &run_context,
+ generator,
+ &[],
+ &[],
+ prefill_finish_reason.clone(),
+ )?;
+
+ let prefill_should_continue = if let Some(ref p) = progress {
+ p(prefill_output.clone())
} else {
true
};
@@ -344,55 +335,45 @@ impl Session {
}
}
- let mut generate_results: Vec<GenerateResult> = Vec::new();
- let mut generate_durations: Vec<f64> = Vec::new();
- let generate_output = loop {
- let generate_start = Instant::now();
- let generate_result = generator.generate(sampling_method)?;
- let generate_tokens = generate_result.tokens.clone();
- let generate_duration = generate_start.elapsed().as_secs_f64();
- generate_results.push(generate_result);
- generate_durations.push(generate_duration);
-
- let generate_finish_reason =
- finish_reason(generator, generate_tokens);
- let generate_generated_text =
- build_generated_text(generator, &self.tokenizer)?;
- let generate_parsed_text =
- self.output_parser.parse(generate_generated_text);
-
- let generate_output = Output {
- text: generate_parsed_text,
- stats: Self::build_stats(
- &self.model_metadata,
- prefill_result.clone(),
- prefill_duration,
- prefill_suffix_length,
- generate_results.clone(),
- generate_durations.clone(),
- generator.decoding_config.generate_suffix_length(),
- run_start.elapsed().as_secs_f64(),
- tokens.len(),
- generator.tokens[prefix_len_before + tokens.len()..].len(),
- ),
- finish_reason: generate_finish_reason.clone(),
- };
-
- let generate_should_continue = if let Some(progress) = &progress {
- progress(generate_output.clone())
- } else {
- true
- };
+ let can_use_async = generator.decoding_config.generate_suffix_length()
+ == 1
+ && !generator.has_attention_layers();
- if !generate_should_continue || generate_finish_reason.is_some() {
- if generate_should_continue {
- break generate_output;
- } else {
- break generate_output.clone_with_finish_reason(Some(
- FinishReason::Cancelled,
- ));
- }
- }
+ let generate_output = if can_use_async {
+ let batch_size = generator
+ .decoding_config
+ .async_batch_size
+ .resolve(&self.model_path);
+ let (results, durations, finish_reason) = Self::run_async_batch(
+ &self.model_metadata,
+ &self.tokenizer,
+ &self.output_parser,
+ &run_context,
+ generator,
+ sampling_method,
+ &progress,
+ batch_size,
+ )?;
+ Self::build_output(
+ &self.model_metadata,
+ &self.tokenizer,
+ &self.output_parser,
+ &run_context,
+ generator,
+ &results,
+ &durations,
+ Some(finish_reason),
+ )?
+ } else {
+ Self::run_sync_generate(
+ &self.model_metadata,
+ &self.tokenizer,
+ &self.output_parser,
+ &run_context,
+ generator,
+ sampling_method,
+ &progress,
+ )?
};
generator.clear_cache();
@@ -407,16 +388,16 @@ impl Session {
let generator =
self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
generator.reset_state();
- if let Some(ctx) = context {
+ if let Some(run_context) = context {
let mut generator_state =
generator.context.cache_layers.borrow_mut();
- for (ctx_layer, gen_layer) in ctx
+ for (run_context_layer, gen_layer) in run_context
.cache_layers
.data
.iter()
.zip(generator_state.data.iter_mut())
{
- match (ctx_layer, gen_layer) {
+ match (run_context_layer, gen_layer) {
(
CacheLayer::Transformer(src),
CacheLayer::Transformer(dst),
@@ -470,7 +451,7 @@ impl Session {
}
drop(generator_state);
- generator.tokens = ctx.tokens.clone();
+ generator.tokens = run_context.tokens.clone();
}
Ok(())
}
@@ -492,6 +473,249 @@ impl Session {
}
}
+impl Session {
+ fn decode_generated_tokens(
+ tokenizer: &Tokenizer,
+ generator: &Generator,
+ run_context: &RunContext,
+ ) -> Result<String, Error> {
+ let start_idx =
+ run_context.prefix_len_before + run_context.input_tokens_len;
+ let generated_tokens: Vec<u32> =
+ generator.tokens[start_idx..].iter().map(|&v| v as u32).collect();
+ tokenizer
+ .decode(&generated_tokens, true)
+ .map_err(|_| Error::UnableToDecodeText)
+ }
+
+ fn check_finish_reason(
+ run_context: &RunContext,
+ generator: &Generator,
+ new_tokens: &[u64],
+ ) -> Option<FinishReason> {
+ let start_idx =
+ run_context.prefix_len_before + run_context.input_tokens_len;
+ let total_new_tokens = generator.tokens[start_idx..].len();
+ let has_eos =
+ new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));
+ let context_limit =
+ generator.tokens.len() >= run_context.context_length;
+
+ if has_eos {
+ Some(FinishReason::Stop)
+ } else if total_new_tokens >= run_context.tokens_limit {
+ Some(FinishReason::Length)
+ } else if context_limit {
+ Some(FinishReason::ContextLimitReached)
+ } else {
+ None
+ }
+ }
+
+ fn build_output(
+ model_metadata: &ModelMetadata,
+ tokenizer: &Tokenizer,
+ output_parser: &OutputParser,
+ run_context: &RunContext,
+ generator: &Generator,
+ generate_results: &[GenerateResult],
+ generate_durations: &[f64],
+ finish_reason: Option<FinishReason>,
+ ) -> Result<Output, Error> {
+ let text =
+ Self::decode_generated_tokens(tokenizer, generator, run_context)?;
+ let parsed = output_parser.parse(text);
+ let start_idx =
+ run_context.prefix_len_before + run_context.input_tokens_len;
+ let output_tokens = generator.tokens[start_idx..].len();
+
+ Ok(Output {
+ text: parsed,
+ stats: Self::build_stats(
+ model_metadata,
+ run_context.prefill_result.clone(),
+ run_context.prefill_duration,
+ run_context.prefill_suffix_length,
+ generate_results.to_vec(),
+ generate_durations.to_vec(),
+ generator.decoding_config.generate_suffix_length(),
+ run_context.run_start.elapsed().as_secs_f64(),
+ run_context.input_tokens_len,
+ output_tokens,
+ ),
+ finish_reason,
+ })
+ }
+
+ fn run_async_batch<F>(
+ model_metadata: &ModelMetadata,
+ tokenizer: &Tokenizer,
+ output_parser: &OutputParser,
+ run_context: &RunContext,
+ generator: &mut Generator,
+ sampling_method: SamplingMethod,
+ progress: &Option<F>,
+ batch_size: usize,
+ ) -> Result<(Vec<GenerateResult>, Vec<f64>, FinishReason), Error>
+ where
+ F: Fn(Output) -> bool,
+ {
+ let remaining_by_limit = run_context
+ .tokens_limit
+ .saturating_sub(run_context.prefill_result.tokens.len());
+ let remaining_by_context =
+ run_context.context_length.saturating_sub(generator.tokens.len());
+ let tokens_to_generate = remaining_by_limit.min(remaining_by_context);
+
+ generator.prepare_async(tokens_to_generate);
+
+ let (sender, receiver) = mpsc::channel::<(usize, u64, u64)>();
+ let start_time = Instant::now();
+ let last_nanos = Arc::new(AtomicU64::new(0));
+
+ let mut results: Vec<GenerateResult> =
+ Vec::with_capacity(tokens_to_generate);
+ let mut durations: Vec<f64> = Vec::with_capacity(tokens_to_generate);
+ let mut finish_reason = FinishReason::Length;
+ let mut next_to_submit = 0;
+ let mut in_flight = 0;
+
+ while in_flight > 0 || next_to_submit < tokens_to_generate {
+ let batch_end =
+ std::cmp::min(next_to_submit + batch_size, tokens_to_generate);
+ for idx in next_to_submit..batch_end {
+ let batch_sender = sender.clone();
+ let last_callback_nanos = last_nanos.clone();
+ let batch_start_time = start_time;
+ generator.async_generate(
+ idx,
+ sampling_method,
+ move |token| {
+ let now_nanos =
+ batch_start_time.elapsed().as_nanos() as u64;
+ let prev_nanos = last_callback_nanos
+ .swap(now_nanos, Ordering::SeqCst);
+ let _ = batch_sender.send((
+ idx,
+ token,
+ now_nanos.saturating_sub(prev_nanos),
+ ));
+ },
+ )?;
+ in_flight += 1;
+ }
+ let batch_submitted = batch_end - next_to_submit;
+ next_to_submit = batch_end;
+
+ for _ in 0..batch_submitted {
+ let (_, token, duration_nanos) =
+ receiver.recv().map_err(|_| Error::SamplingFailed)?;
+ in_flight -= 1;
+
+ let duration = nanos_to_secs(duration_nanos);
+ generator.tokens.push(token);
+ results.push(GenerateResult {
+ tokens: vec![token],
+ forwardpass_duration: duration,
+ });
+ durations.push(duration);
+
+ let should_stop = if run_context.eos_tokens.contains(&token) {
+ finish_reason = FinishReason::Stop;
+ true
+ } else if generator.tokens.len() >= run_context.context_length {
+ finish_reason = FinishReason::ContextLimitReached;
+ true
+ } else if let Some(progress_fn) = progress {
+ let output = Self::build_output(
+ model_metadata,
+ tokenizer,
+ output_parser,
+ run_context,
+ generator,
+ &results,
+ &durations,
+ None,
+ )?;
+ if !progress_fn(output) {
+ finish_reason = FinishReason::Cancelled;
+ true
+ } else {
+ false
+ }
+ } else {
+ false
+ };
+
+ if should_stop {
+ while in_flight > 0 {
+ let _ = receiver.recv();
+ in_flight -= 1;
+ }
+ return Ok((results, durations, finish_reason));
+ }
+ }
+ }
+
+ Ok((results, durations, finish_reason))
+ }
+
+ fn run_sync_generate<F>(
+ model_metadata: &ModelMetadata,
+ tokenizer: &Tokenizer,
+ output_parser: &OutputParser,
+ run_context: &RunContext,
+ generator: &mut Generator,
+ sampling_method: SamplingMethod,
+ progress: &Option<F>,
+ ) -> Result<Output, Error>
+ where
+ F: Fn(Output) -> bool,
+ {
+ let mut results: Vec<GenerateResult> = Vec::new();
+ let mut durations: Vec<f64> = Vec::new();
+
+ loop {
+ let start = Instant::now();
+ let result = generator.generate(sampling_method)?;
+ let new_tokens = result.tokens.clone();
+ let duration = start.elapsed().as_secs_f64();
+
+ results.push(result);
+ durations.push(duration);
+
+ let finish_reason =
+ Self::check_finish_reason(run_context, generator, &new_tokens);
+ let output = Self::build_output(
+ model_metadata,
+ tokenizer,
+ output_parser,
+ run_context,
+ generator,
+ &results,
+ &durations,
+ finish_reason.clone(),
+ )?;
+
+ let should_continue = if let Some(progress_fn) = progress {
+ progress_fn(output.clone())
+ } else {
+ true
+ };
+
+ if !should_continue || finish_reason.is_some() {
+ if should_continue {
+ return Ok(output);
+ } else {
+ return Ok(output.clone_with_finish_reason(Some(
+ FinishReason::Cancelled,
+ )));
+ }
+ }
+ }
+ }
+}
+
impl Session {
fn build_stats(
model_metadata: &ModelMetadata,
diff --git a/crates/uzu/src/tracer/tracer.rs b/crates/uzu/src/tracer/tracer.rs
index ec284ceb7..1b1f01ec8 100644
--- a/crates/uzu/src/tracer/tracer.rs
+++ b/crates/uzu/src/tracer/tracer.rs
@@ -318,6 +318,9 @@ impl Tracer {
&token_seeds,
true,
None,
+ false,
+ None,
+ None,
);
let root_command_buffer = self
diff --git a/crates/uzu/src/utils/device.rs b/crates/uzu/src/utils/device.rs
new file mode 100644
index 000000000..de3c76ae4
--- /dev/null
+++ b/crates/uzu/src/utils/device.rs
@@ -0,0 +1,59 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DeviceClass {
+ Ultra,
+ Max,
+ Pro,
+ Base,
+ IPhone,
+ Unknown,
+}
+
+impl DeviceClass {
+ pub fn detect() -> Self {
+ let device = match metal::Device::system_default() {
+ Some(d) => d,
+ None => return DeviceClass::Unknown,
+ };
+
+ let name = device.name().to_lowercase();
+
+ if name.contains("ultra") {
+ DeviceClass::Ultra
+ } else if name.contains("max") {
+ DeviceClass::Max
+ } else if name.contains("pro") {
+ DeviceClass::Pro
+ } else if name.contains("iphone") || name.contains("a1") {
+ DeviceClass::IPhone
+ } else {
+ DeviceClass::Base
+ }
+ }
+
+ pub fn is_high_end(&self) -> bool {
+ matches!(self, DeviceClass::Ultra | DeviceClass::Max)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_device_detection() {
+ let device = DeviceClass::detect();
+ println!("Detected device: {:?}", device);
+ println!("Is high-end: {}", device.is_high_end());
+ assert_ne!(device, DeviceClass::Unknown);
+ }
+
+ #[test]
+ fn test_is_high_end() {
+ assert!(DeviceClass::Ultra.is_high_end());
+ assert!(DeviceClass::Max.is_high_end());
+ assert!(!DeviceClass::Pro.is_high_end());
+ assert!(!DeviceClass::Base.is_high_end());
+ assert!(!DeviceClass::IPhone.is_high_end());
+ assert!(!DeviceClass::Unknown.is_high_end());
+ }
+}
diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rs
index 1c4996f25..13264fa5a 100644
--- a/crates/uzu/src/utils/mod.rs
+++ b/crates/uzu/src/utils/mod.rs
@@ -1,6 +1,11 @@
+pub mod device;
+pub mod env_utils;
+pub mod model_size;
pub mod storage;
-pub use storage::*;
pub mod version;
-pub use version::VERSION;
-pub mod env_utils;
+
+pub use device::DeviceClass;
pub use env_utils::*;
+pub use model_size::ModelSize;
+pub use storage::*;
+pub use version::VERSION;
diff --git a/crates/uzu/src/utils/model_size.rs b/crates/uzu/src/utils/model_size.rs
new file mode 100644
index 000000000..5e7a0eb89
--- /dev/null
+++ b/crates/uzu/src/utils/model_size.rs
@@ -0,0 +1,63 @@
+use std::path::Path;
+
+const FOUR_GB: u64 = 4 * 1024 * 1024 * 1024;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ModelSize {
+ Large,
+ Small,
+}
+
+impl ModelSize {
+ pub fn from_path(model_path: &Path) -> Self {
+ let weights_path = model_path.join("model.safetensors");
+ let size_bytes =
+ std::fs::metadata(&weights_path).map(|m| m.len()).unwrap_or(0);
+
+ Self::from_bytes(size_bytes)
+ }
+
+ pub fn from_bytes(size_bytes: u64) -> Self {
+ if size_bytes > FOUR_GB {
+ ModelSize::Large
+ } else {
+ ModelSize::Small
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::path::PathBuf;
+
+ use super::*;
+ use crate::VERSION;
+ pub const MODEL_DIR_NAME: &str = "Llama-3.2-1B-Instruct";
+
+ pub fn get_test_model_path() -> PathBuf {
+ let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+ .join("..")
+ .join("..")
+ .join("models")
+ .join(VERSION)
+ .join(MODEL_DIR_NAME);
+ if !path.exists() {
+ panic!(
+ "Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.",
+ path
+ );
+ }
+ path
+ }
+
+ #[test]
+ fn test_model_size_from_path() {
+ let model_path = get_test_model_path();
+
+ if model_path.exists() {
+ let size = ModelSize::from_path(&model_path);
+ println!("Model size: {:?}", size);
+ assert_eq!(size, ModelSize::Small);
+ }
+ }
+}
diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rs
index 611f73eac..f00e091a4 100644
--- a/crates/uzu/tests/ssm_update_test.rs
+++ b/crates/uzu/tests/ssm_update_test.rs
@@ -4,9 +4,7 @@ use half::bf16;
use metal::MTLResourceOptions;
use uzu::backends::metal::{
KernelDataType, MTLContext,
- kernel::ssm::{
- SSDUpdateArguments, SSDUpdateKernel,
- },
+ kernel::ssm::{SSDUpdateArguments, SSDUpdateKernel},
};
fn ssm_update_ref_bf16(
x: &[bf16],