Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

more generic kernel wrappers and encodable blocks (#199)

uuuvn7f9ef618f85e
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index db22f269a..13ad56878 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -12,4 +12,7 @@ pub trait CommandBuffer {
&self,
callback: impl FnOnce(&<Self::Backend as Backend>::CopyEncoder) -> T,
) -> T;
+
+ fn submit(&self);
+ fn wait_until_completed(&self);
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/common/kernel/attention.rs
similarity index 67%
rename from crates/uzu/src/backends/metal/kernel/attention/mod.rs
rename to crates/uzu/src/backends/common/kernel/attention.rs
index 2f2f5cc67..76fd12630 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/attention.rs
@@ -3,29 +3,24 @@ use std::{
collections::{HashMap, hash_map::Entry},
};
-use metal::{MTLBuffer, MTLComputeCommandEncoder};
-use objc2::{Message, runtime::ProtocolObject};
-
use crate::{
DataType,
- backends::{
- common::{
- gpu_types::{AttnMaskParams, AttnParams},
- kernel::AttentionGemmKernel,
- },
- metal::{MetalContext, MetalError, kernel::dsl::AttentionGemmMetalKernel},
+ backends::common::{
+ Backend, Kernels,
+ gpu_types::{AttnMaskParams, AttnParams},
+ kernel::AttentionGemmKernel,
},
};
const BQ: usize = 32;
-pub struct AttentionGemmArguments<'a> {
- pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
- pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)
- pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)
- pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(6)
- pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(7)
+pub struct AttentionGemmArguments<'a, B: Backend> {
+ pub queries_buffer: &'a B::NativeBuffer, // buffer(0)
+ pub keys_buffer: &'a B::NativeBuffer, // buffer(1)
+ pub values_buffer: &'a B::NativeBuffer, // buffer(2)
+ pub output_buffer: &'a B::NativeBuffer, // buffer(3)
+ pub mask_buffer: Option<&'a B::NativeBuffer>, // buffer(6)
+ pub sinks_buffer: Option<&'a B::NativeBuffer>, // buffer(7)
pub num_heads: usize,
pub num_groups: usize,
pub suffix_length: usize, // qL
@@ -37,12 +32,12 @@ pub struct AttentionGemmArguments<'a> {
pub scale: f32,
}
-pub struct AttentionGemmBlock {
+pub struct AttentionGemmBlock<B: Backend> {
data_type: DataType,
- cache: RefCell<HashMap<KernelKey, AttentionGemmMetalKernel>>,
+ cache: RefCell<HashMap<KernelKey, <B::Kernels as Kernels>::AttentionGemmKernel>>,
}
-impl AttentionGemmBlock {
+impl<B: Backend> AttentionGemmBlock<B> {
pub fn new(data_type: DataType) -> Self {
let cache = RefCell::new(HashMap::new());
Self {
@@ -53,10 +48,10 @@ impl AttentionGemmBlock {
pub fn encode(
&self,
- context: &MetalContext,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: &AttentionGemmArguments,
- ) -> Result<(), MetalError> {
+ context: &B::Context,
+ compute_encoder: &B::ComputeEncoder,
+ args: &AttentionGemmArguments<B>,
+ ) -> Result<(), B::Error> {
let bk: usize = if args.head_dim < 128 {
32
} else {
@@ -80,7 +75,7 @@ impl AttentionGemmBlock {
let kernel = match map.entry(key) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
- let kernel = AttentionGemmMetalKernel::new(
+ let kernel = <B::Kernels as Kernels>::AttentionGemmKernel::new(
context,
self.data_type,
bk as u32,
@@ -130,22 +125,17 @@ impl AttentionGemmBlock {
// We use a shared bias matrix for all heads/batches.
m_strides: [0, 0, args.sequence_length as i64],
};
- let attn_mask_params_opt = args.mask_buffer.map(|_| std::slice::from_ref(&attn_mask_params));
-
- let mask_buffer_retain = args.mask_buffer.map(|b| b.retain());
- let mask_buffer_opt = mask_buffer_retain.as_ref().map(|b| b);
- let sinks_buffer_retain = args.sinks_buffer.map(|b| b.retain());
- let sinks_buffer_opt = sinks_buffer_retain.as_ref().map(|b| b);
+ let attn_mask_params_opt = args.mask_buffer.map(|_| attn_mask_params);
kernel.encode(
- &args.queries_buffer.retain(),
- &args.keys_buffer.retain(),
- &args.values_buffer.retain(),
- &args.output_buffer.retain(),
- std::slice::from_ref(&params),
+ args.queries_buffer,
+ args.keys_buffer,
+ args.values_buffer,
+ args.output_buffer,
+ params,
attn_mask_params_opt,
- mask_buffer_opt,
- sinks_buffer_opt,
+ args.mask_buffer,
+ args.sinks_buffer,
args.num_heads as u32,
args.suffix_length as u32,
&compute_encoder,
diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
new file mode 100644
index 000000000..7139b277c
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
@@ -0,0 +1,47 @@
+use crate::{
+ DataType,
+ backends::common::{Backend, Kernels, kernel::MlpGateActMulKernel},
+ config::Activation,
+};
+
+pub struct MlpGateActMulEncodable<B: Backend> {
+ kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,
+ activation: Activation,
+ hidden_dim: usize,
+}
+
+impl<B: Backend> MlpGateActMulEncodable<B> {
+ pub fn new(
+ context: &B::Context,
+ data_type: DataType,
+ activation: Activation,
+ hidden_dim: usize,
+ ) -> Result<Self, B::Error> {
+ let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type)?;
+ Ok(Self {
+ kernel,
+ activation,
+ hidden_dim,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &B::ComputeEncoder,
+ fused_up: &B::NativeBuffer,
+ hidden: &B::NativeBuffer,
+ m: i32,
+ ) -> Result<(), B::Error> {
+ let act_type = match self.activation {
+ Activation::SiLU {
+ ..
+ } => 0,
+ Activation::Gelu => 1,
+ Activation::Identity => {
+ panic!("Identity activation is not supported for kernel")
+ },
+ };
+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, encoder);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rs
index ca042ab4e..1ff1f73f7 100644
--- a/crates/uzu/src/backends/common/kernel/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/mod.rs
@@ -2,6 +2,9 @@
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
+pub mod attention;
pub mod kv_cache_update;
pub mod matmul;
+pub mod mlp_gate_act_mul;
pub mod sampling;
+pub mod ssd_prefill;
diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
new file mode 100644
index 000000000..b65adaa73
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
@@ -0,0 +1,134 @@
+use crate::{
+ DataType,
+ backends::common::{
+ Backend, Kernels,
+ kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},
+ },
+};
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum SSDPrefillMode {
+ Sequential,
+ SinglePass,
+}
+
+pub struct SSDPrefillArguments<'a, B: Backend> {
+ pub x: &'a B::NativeBuffer,
+ pub dt: &'a B::NativeBuffer, // raw dt values
+ pub b: &'a B::NativeBuffer,
+ pub c: &'a B::NativeBuffer,
+ pub d: &'a B::NativeBuffer,
+ pub z: &'a B::NativeBuffer,
+ pub state: &'a B::NativeBuffer,
+ pub y: &'a B::NativeBuffer,
+ pub suffix_len: usize,
+ pub group_size: i32,
+ pub state_size: i32,
+ pub x_strides: [usize; 3],
+ pub dt_strides: [usize; 2],
+ pub cb_strides: [usize; 3],
+ pub state_strides: [usize; 3],
+ pub channels: usize,
+ pub head_dim: usize,
+}
+
+pub struct SSDPrefillKernels<B: Backend> {
+ single: <B::Kernels as Kernels>::SSDPrefillKernel,
+ single_64: <B::Kernels as Kernels>::SSDPrefill64Kernel,
+ sequential: <B::Kernels as Kernels>::SSDPrefillSequentialKernel,
+}
+
+impl<B: Backend> SSDPrefillKernels<B> {
+ pub fn new(
+ context: &B::Context,
+ data_type: DataType,
+ ) -> Result<Self, B::Error> {
+ let single = <B::Kernels as Kernels>::SSDPrefillKernel::new(context, data_type)?;
+ let single_64 = <B::Kernels as Kernels>::SSDPrefill64Kernel::new(context, data_type)?;
+ let sequential = <B::Kernels as Kernels>::SSDPrefillSequentialKernel::new(context, data_type)?;
+ Ok(Self {
+ single,
+ single_64,
+ sequential,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ args: SSDPrefillArguments<B>,
+ mode: SSDPrefillMode,
+ ) {
+ let x_strides: Vec<u32> = args.x_strides.iter().map(|x| *x as u32).collect();
+ let dt_strides: Vec<u32> = args.dt_strides.iter().map(|x| *x as u32).collect();
+ let cb_strides: Vec<u32> = args.cb_strides.iter().map(|x| *x as u32).collect();
+ let state_strides: Vec<u32> = args.state_strides.iter().map(|x| *x as u32).collect();
+
+ if mode == SSDPrefillMode::SinglePass {
+ if args.state_size == 64 {
+ self.single_64.encode(
+ args.x,
+ args.dt,
+ args.b,
+ args.c,
+ args.d,
+ args.z,
+ args.state,
+ args.y,
+ args.suffix_len as u32,
+ args.group_size,
+ args.state_size,
+ x_strides.as_slice(),
+ dt_strides.as_slice(),
+ cb_strides.as_slice(),
+ state_strides.as_slice(),
+ args.channels as u32,
+ args.head_dim as u32,
+ compute_encoder,
+ )
+ } else {
+ self.single.encode(
+ args.x,
+ args.dt,
+ args.b,
+ args.c,
+ args.d,
+ args.z,
+ args.state,
+ args.y,
+ args.suffix_len as u32,
+ args.group_size,
+ args.state_size,
+ x_strides.as_slice(),
+ dt_strides.as_slice(),
+ cb_strides.as_slice(),
+ state_strides.as_slice(),
+ args.channels as u32,
+ args.head_dim as u32,
+ compute_encoder,
+ )
+ }
+ } else if mode == SSDPrefillMode::Sequential {
+ self.sequential.encode(
+ args.x,
+ args.dt,
+ args.b,
+ args.c,
+ args.d,
+ args.z,
+ args.state,
+ args.y,
+ args.suffix_len as u32,
+ args.group_size,
+ args.state_size,
+ x_strides.as_slice(),
+ dt_strides.as_slice(),
+ cb_strides.as_slice(),
+ state_strides.as_slice(),
+ args.channels as u32,
+ args.head_dim as u32,
+ compute_encoder,
+ )
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rs
index 296a1ea3f..b300da23d 100644
--- a/crates/uzu/src/backends/metal/classifier_context.rs
+++ b/crates/uzu/src/backends/metal/classifier_context.rs
@@ -6,7 +6,7 @@ use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{
MetalContext,
encodable_block::{
- ClassifierLayer, ClassifierPredictionHead,
+ ClassifierLayer,
transformer_layer::{embed_block, linear_block},
},
kernel::dsl::SigmoidMetalKernel,
@@ -18,7 +18,7 @@ use crate::{
metal::{Metal, error::ClassifierError},
},
config::{ClassifierModelConfig, ModelMetadata},
- encodable_block::{Activation, EncodableBlock, Normalization, Pooling, Rope},
+ encodable_block::{Activation, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope},
forward_pass::{
model_shape::ModelShape,
scratch_buffers::ScratchBuffers,
@@ -40,7 +40,7 @@ pub struct ClassifierContext {
pub embed: Box<dyn EncodableBlock<Metal>>,
pub embedding_norm: Normalization<Metal>,
- pub layers: Box<[ClassifierLayer]>,
+ pub layers: Box<[ClassifierLayer<Metal>]>,
pub output_norm: Normalization<Metal>,
pub global_rope: Rc<Box<dyn EncodableBlock<Metal>>>,
pub local_rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index 904dbdbe4..5523cea83 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -1,3 +1,5 @@
+use std::ops::Deref;
+
use metal::{MTLCommandBuffer, MTLCommandEncoder};
use objc2::{rc::Retained, runtime::ProtocolObject};
@@ -32,4 +34,12 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
ret
}
+
+ fn submit(&self) {
+ self.commit();
+ }
+
+ fn wait_until_completed(&self) {
+ self.deref().wait_until_completed();
+ }
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
index 7684e4728..290f445d8 100644
--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
@@ -1,43 +1,44 @@
use std::rc::Rc;
-#[cfg(not(feature = "tracing"))]
-use metal::MTLCommandEncoder;
-use metal::{MTLCommandBuffer, MTLComputeCommandEncoder};
-use objc2::{
- rc::{Retained, autoreleasepool},
- runtime::ProtocolObject,
-};
+use objc2::rc::autoreleasepool;
-use super::{Attention, transformer_layer};
+use super::transformer_layer;
+#[cfg(not(feature = "tracing"))]
+use crate::backends::common::CommandBuffer;
use crate::{
DataType,
- backends::metal::{Metal, MetalContext},
+ backends::{
+ common::Backend,
+ metal::{Metal, MetalContext},
+ },
config::TransformerLayerConfig,
- encodable_block::{EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy},
+ encodable_block::{
+ Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,
+ },
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
-pub struct ClassifierLayer {
+pub struct ClassifierLayer<B: Backend> {
#[cfg_attr(not(feature = "tracing"), allow(dead_code))]
layer_index: usize,
- copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<Metal>>,
- pre_attention_norm: Option<Box<dyn EncodableBlock<Metal>>>,
- qkv_projection: Box<dyn EncodableBlock<Metal>>,
- qk_norm: Option<Box<dyn EncodableBlock<Metal>>>,
- rope: Rc<Box<dyn EncodableBlock<Metal>>>,
- attention: Box<dyn EncodableBlock<Metal>>,
- out_projection: Box<dyn EncodableBlock<Metal>>,
- post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>>,
- mixer_residual_add: Box<dyn EncodableBlock<Metal>>,
- copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<Metal>>,
- pre_mlp_norm: Box<dyn EncodableBlock<Metal>>,
- mlp: Box<dyn EncodableBlock<Metal>>,
- post_mlp_norm: Option<Box<dyn EncodableBlock<Metal>>>,
- mlp_residual_add: Box<dyn EncodableBlock<Metal>>,
+ copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<B>>,
+ pre_attention_norm: Option<Box<dyn EncodableBlock<B>>>,
+ qkv_projection: Box<dyn EncodableBlock<B>>,
+ qk_norm: Option<Box<dyn EncodableBlock<B>>>,
+ rope: Rc<Box<dyn EncodableBlock<B>>>,
+ attention: Box<dyn EncodableBlock<B>>,
+ out_projection: Box<dyn EncodableBlock<B>>,
+ post_attention_norm: Option<Box<dyn EncodableBlock<B>>>,
+ mixer_residual_add: Box<dyn EncodableBlock<B>>,
+ copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<B>>,
+ pre_mlp_norm: Box<dyn EncodableBlock<B>>,
+ mlp: Box<dyn EncodableBlock<B>>,
+ post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,
+ mlp_residual_add: Box<dyn EncodableBlock<B>>,
}
-impl ClassifierLayer {
+impl ClassifierLayer<Metal> {
pub fn new(
mtl_context: Rc<MetalContext>,
layer_config: &TransformerLayerConfig,
@@ -247,20 +248,18 @@ impl ClassifierLayer {
}
}
-impl EncodableBlock<Metal> for ClassifierLayer {
+impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {
fn encode(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
) {
#[cfg(not(feature = "tracing"))]
{
if self.supports_shared_encoder() {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, parameters, &encoder);
- encoder.end_encoding();
+ command_buffer
+ .with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
return;
}
}
@@ -384,9 +383,9 @@ impl EncodableBlock<Metal> for ClassifierLayer {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
) {
debug_assert!(
self.supports_shared_encoder(),
diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
index 03ddee61b..ff065fb21 100644
--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
@@ -2,15 +2,15 @@
use std::rc::Rc;
-use metal::{MTLCommandBuffer, MTLComputeCommandEncoder};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
use super::LayerExecutables;
use crate::{
DataType, DecoderConfig,
- backends::metal::{
- Metal, MetalContext,
- encodable_block::transformer_layer::{embed_block, readout_block},
+ backends::{
+ common::Backend,
+ metal::{
+ Metal, MetalContext,
+ encodable_block::transformer_layer::{embed_block, readout_block},
+ },
},
config::{DecoderLayerType, MixerConfig},
encodable_block::{EncodableBlock, EncodingParameters, RMSNorm, Rope},
@@ -22,16 +22,16 @@ use crate::{
};
/// Full decoder executable with all layers and components.
-pub struct Decoder {
- pub embed: Box<dyn EncodableBlock<Metal>>,
- pub layers: Box<[LayerExecutables]>,
- pub norm: Box<dyn EncodableBlock<Metal>>,
- pub readout: Box<dyn EncodableBlock<Metal>>,
- pub global_rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,
- pub local_rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,
+pub struct Decoder<B: Backend> {
+ pub embed: Box<dyn EncodableBlock<B>>,
+ pub layers: Box<[LayerExecutables<B>]>,
+ pub norm: Box<dyn EncodableBlock<B>>,
+ pub readout: Box<dyn EncodableBlock<B>>,
+ pub global_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
+ pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
}
-impl Decoder {
+impl Decoder<Metal> {
pub fn new(
mtl_context: Rc<MetalContext>,
decoder_config: Rc<DecoderConfig>,
@@ -167,21 +167,21 @@ impl Decoder {
}
}
-impl EncodableBlock<Metal> for Decoder {
+impl<B: Backend> EncodableBlock<B> for Decoder<B> {
fn encode_with_shared_encoder(
&self,
- _state: &mut ForwardPassState<Metal>,
- _parameters: &EncodingParameters<Metal>,
- _encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ _state: &mut ForwardPassState<B>,
+ _parameters: &EncodingParameters<B>,
+ _encoder: &B::ComputeEncoder,
) {
unimplemented!("Decoder does not support shared encoder")
}
fn encode(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
) {
self.embed.encode(state, parameters, command_buffer);
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
index a847f842c..ff7922f69 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
@@ -2,41 +2,40 @@
use std::rc::Rc;
-#[cfg(not(feature = "tracing"))]
-use metal::MTLCommandEncoder;
-use metal::{MTLCommandBuffer, MTLComputeCommandEncoder};
-use objc2::{
- rc::{Retained, autoreleasepool},
- runtime::ProtocolObject,
-};
+use objc2::rc::autoreleasepool;
-use super::{
- super::{Attention, MambaMixer, ShortConvMixer, transformer_layer},
- MixerExecutables,
-};
+use super::{super::transformer_layer, MixerExecutables};
+#[cfg(not(feature = "tracing"))]
+use crate::backends::common::CommandBuffer;
use crate::{
DataType, DecoderLayerConfig,
- backends::metal::{Metal, MetalContext},
+ backends::{
+ common::Backend,
+ metal::{Metal, MetalContext},
+ },
config::{DecoderLayerType, MixerConfig},
- encodable_block::{EncodableBlock, EncodingParameters, QKNorm, RMSNorm, TensorAddSwap, TensorCopy},
+ encodable_block::{
+ Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,
+ TensorCopy,
+ },
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
/// A single decoder layer with all its components.
-pub struct LayerExecutables {
+pub struct LayerExecutables<B: Backend> {
pub layer_index: usize,
- pub copy_main_to_shortcut: Box<dyn EncodableBlock<Metal>>,
- pub pre_attention_norm: Box<dyn EncodableBlock<Metal>>,
- pub(crate) mixer: MixerExecutables,
- pub post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>>,
- pub main_shortcut_add_swap: Box<dyn EncodableBlock<Metal>>,
- pub pre_mlp_norm: Box<dyn EncodableBlock<Metal>>,
- pub mlp: Box<dyn EncodableBlock<Metal>>,
- pub post_mlp_norm: Option<Box<dyn EncodableBlock<Metal>>>,
+ pub copy_main_to_shortcut: Box<dyn EncodableBlock<B>>,
+ pub pre_attention_norm: Box<dyn EncodableBlock<B>>,
+ pub(crate) mixer: MixerExecutables<B>,
+ pub post_attention_norm: Option<Box<dyn EncodableBlock<B>>>,
+ pub main_shortcut_add_swap: Box<dyn EncodableBlock<B>>,
+ pub pre_mlp_norm: Box<dyn EncodableBlock<B>>,
+ pub mlp: Box<dyn EncodableBlock<B>>,
+ pub post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,
}
-impl LayerExecutables {
+impl LayerExecutables<Metal> {
#[allow(clippy::too_many_arguments)]
pub fn new(
mtl_context: Rc<MetalContext>,
@@ -259,22 +258,20 @@ impl LayerExecutables {
}
}
-impl EncodableBlock<Metal> for LayerExecutables {
+impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
fn encode(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
) {
// In non-tracing builds, if every sub-block supports shared encoding,
// we can run the entire layer in a single compute encoder.
#[cfg(not(feature = "tracing"))]
{
if self.supports_shared_encoder() {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, parameters, &encoder);
- encoder.end_encoding();
+ command_buffer
+ .with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
return;
}
}
@@ -432,9 +429,9 @@ impl EncodableBlock<Metal> for LayerExecutables {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
) {
debug_assert!(
self.supports_shared_encoder(),
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs
index 0ac31a7b4..2ac32fc5d 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs
@@ -2,21 +2,21 @@
use std::rc::Rc;
-use crate::{backends::metal::Metal, encodable_block::EncodableBlock};
+use crate::{backends::common::Backend, encodable_block::EncodableBlock};
/// Mixer component - either attention, state space, or short conv.
-pub(crate) enum MixerExecutables {
+pub(crate) enum MixerExecutables<B: Backend> {
Attention {
- qkv_projection: Box<dyn EncodableBlock<Metal>>,
- qk_norm: Option<Box<dyn EncodableBlock<Metal>>>,
- rope: Rc<Box<dyn EncodableBlock<Metal>>>,
- attention: Box<dyn EncodableBlock<Metal>>,
- out_projection: Box<dyn EncodableBlock<Metal>>,
+ qkv_projection: Box<dyn EncodableBlock<B>>,
+ qk_norm: Option<Box<dyn EncodableBlock<B>>>,
+ rope: Rc<Box<dyn EncodableBlock<B>>>,
+ attention: Box<dyn EncodableBlock<B>>,
+ out_projection: Box<dyn EncodableBlock<B>>,
},
StateSpace {
- mixer: Box<dyn EncodableBlock<Metal>>,
+ mixer: Box<dyn EncodableBlock<B>>,
},
ShortConv {
- mixer: Box<dyn EncodableBlock<Metal>>,
+ mixer: Box<dyn EncodableBlock<B>>,
},
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 7c704bc60..ebcf7ef7d 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -6,20 +6,14 @@
//! Encodables implement `EncodableBlock` and orchestrate one or more
//! kernels to perform operations on `ForwardPassState`.
-mod attention;
mod classifier_layer;
mod decoder;
mod embedding;
mod layer;
mod linear;
-mod mamba_mixer;
-mod mlp;
mod moe_block;
-mod prediction_head;
-mod short_conv_mixer;
pub mod transformer_layer;
-pub use attention::Attention;
pub use classifier_layer::ClassifierLayer;
pub use decoder::Decoder;
pub use embedding::{
@@ -28,8 +22,4 @@ pub use embedding::{
};
pub use layer::LayerExecutables;
pub use linear::{FullPrecisionLinear, QuantizedLinear};
-pub(crate) use mamba_mixer::MambaMixer;
-pub use mlp::MlpBlock;
pub use moe_block::{MoeBlock, SharedMoeWeights};
-pub use prediction_head::ClassifierPredictionHead;
-pub(crate) use short_conv_mixer::ShortConvMixer;
diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
deleted file mode 100644
index 71154dbc1..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
+++ /dev/null
@@ -1,106 +0,0 @@
-//! Prediction head encodable for classification output.
-
-#[cfg(feature = "tracing")]
-use metal::MTLBlitCommandEncoder;
-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-#[cfg(feature = "tracing")]
-use crate::forward_pass::state::ArrayId;
-use crate::{
- backends::metal::Metal,
- encodable_block::{EncodableBlock, EncodingParameters},
- forward_pass::state::ForwardPassState,
-};
-
-pub struct ClassifierPredictionHead {
- dense: Box<dyn EncodableBlock<Metal>>,
- activation: Box<dyn EncodableBlock<Metal>>,
- norm: Box<dyn EncodableBlock<Metal>>,
- readout: Box<dyn EncodableBlock<Metal>>,
- #[cfg_attr(not(feature = "tracing"), allow(dead_code))]
- num_labels: usize,
-}
-
-impl ClassifierPredictionHead {
- pub fn new(
- dense: Box<dyn EncodableBlock<Metal>>,
- activation: Box<dyn EncodableBlock<Metal>>,
- norm: Box<dyn EncodableBlock<Metal>>,
- readout: Box<dyn EncodableBlock<Metal>>,
- num_labels: usize,
- ) -> Self {
- Self {
- dense,
- activation,
- norm,
- readout,
- num_labels,
- }
- }
-}
-
-impl EncodableBlock<Metal> for ClassifierPredictionHead {
- fn encode(
- &self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- ) {
- if self.supports_shared_encoder() {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, parameters, &encoder);
- encoder.end_encoding();
- } else {
- self.dense.encode(state, parameters, command_buffer);
- self.activation.encode(state, parameters, command_buffer);
- self.norm.encode(state, parameters, command_buffer);
- self.readout.encode(state, parameters, command_buffer);
- }
-
- #[cfg(feature = "tracing")]
- {
- let traces_rc = state.traces().clone();
- let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
- let logits_array_ref = logits_arrays[0].borrow();
- let linear_output_buffer = logits_array_ref.buffer();
- let data_type = logits_array_ref.data_type();
- let batch_size = logits_array_ref.shape()[0];
-
- let traces_ref = traces_rc.borrow();
- let trace_logits = traces_ref.logits.borrow();
- let dst_trace_buf = trace_logits.buffer();
-
- let copy_size_bytes = batch_size * self.num_labels * data_type.size_in_bytes();
-
- let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");
- blit.copy_buffer_to_buffer(linear_output_buffer, 0, dst_trace_buf, 0, copy_size_bytes);
- blit.end_encoding();
- }
-
- if parameters.wait_until_completed {
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- self.dense.supports_shared_encoder()
- && self.activation.supports_shared_encoder()
- && self.norm.supports_shared_encoder()
- && self.readout.supports_shared_encoder()
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- ) {
- self.dense.encode_with_shared_encoder(state, parameters, encoder);
- self.activation.encode_with_shared_encoder(state, parameters, encoder);
- self.norm.encode_with_shared_encoder(state, parameters, encoder);
- self.readout.encode_with_shared_encoder(state, parameters, encoder);
- }
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 8bee2f346..dcc4987c2 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -1,12 +1,15 @@
use super::{
- FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock, MoeBlock,
+ FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, MoeBlock,
QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,
};
use crate::{
DataType,
- backends::metal::{Metal, MetalContext, MetalError, kernel::mlp::MlpGateActMulEncodable},
+ backends::{
+ common::kernel::mlp_gate_act_mul::MlpGateActMulEncodable,
+ metal::{Metal, MetalContext, MetalError},
+ },
config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
- encodable_block::EncodableBlock,
+ encodable_block::{EncodableBlock, MlpBlock},
forward_pass::state::ArrayId,
parameters::ParameterTree,
};
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index 7f9dd6c63..06ed9897c 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -45,8 +45,8 @@ KERNEL(AttentionGemm)(
const device T* k,
const device T* v,
device T* o,
- const constant AttnParams* params,
- const constant AttnMaskParams* mask_params OPTIONAL(has_mask),
+ const constant AttnParams& params,
+ const constant AttnMaskParams& mask_params OPTIONAL(has_mask),
const device T* mask OPTIONAL(has_mask),
const device float* sinks OPTIONAL(has_sinks),
const constant uint& num_heads,
@@ -78,21 +78,21 @@ KERNEL(AttentionGemm)(
const uint head_idx = tgid_y;
const uint q_tile_idx = tgid_x;
- q += batch_idx * params->q_strides[0] + head_idx * params->q_strides[1] +
- q_tile_idx * int64_t(BQ) * params->q_strides[2];
+ q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] +
+ q_tile_idx * int64_t(BQ) * params.q_strides[2];
- const int kv_head_idx = int(tgid_y) / params->gqa_factor;
- k += batch_idx * params->k_strides[0] +
- int64_t(kv_head_idx) * params->k_strides[1];
- v += batch_idx * params->v_strides[0] +
- int64_t(kv_head_idx) * params->v_strides[1];
+ const int kv_head_idx = int(tgid_y) / params.gqa_factor;
+ k += batch_idx * params.k_strides[0] +
+ int64_t(kv_head_idx) * params.k_strides[1];
+ v += batch_idx * params.v_strides[0] +
+ int64_t(kv_head_idx) * params.v_strides[1];
- o += batch_idx * params->o_strides[0] + head_idx * params->o_strides[1] +
- q_tile_idx * int64_t(BQ) * params->o_strides[2];
+ o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] +
+ q_tile_idx * int64_t(BQ) * params.o_strides[2];
if (has_mask) {
- mask += batch_idx * mask_params->m_strides[0] +
- head_idx * mask_params->m_strides[1];
+ mask += batch_idx * mask_params.m_strides[0] +
+ head_idx * mask_params.m_strides[1];
}
// -------------------------------------------------------------------------
@@ -105,10 +105,6 @@ KERNEL(AttentionGemm)(
constexpr short LDK_tgp = BD + padK; // K stored as [BK, BD] row-major
constexpr short LDV_tgp = BD + padV;
- constexpr short tgp_mem_k = BK * (BD + padK);
- constexpr short tgp_mem_v = BK * (BD + padV);
- constexpr short tgp_mem_s = tgp_mem_k > tgp_mem_v ? tgp_mem_k : tgp_mem_v;
-
threadgroup T* Qs = q_smem;
threadgroup T* Ks = kv_smem;
threadgroup T* Vs = kv_smem;
@@ -140,15 +136,15 @@ KERNEL(AttentionGemm)(
/*reduction_dim=*/0,
/*tgp_size=*/WM * WN * 32>;
- const int q_src_ld = int(params->q_strides[2]);
- const int k_src_ld = int(params->k_strides[2]);
- const int v_src_ld = int(params->v_strides[2]);
+ const int q_src_ld = int(params.q_strides[2]);
+ const int k_src_ld = int(params.k_strides[2]);
+ const int v_src_ld = int(params.v_strides[2]);
thread QBlockLoader loader_q(q, q_src_ld, Qs, simd_group_id, simd_lane_id);
thread KBlockLoader loader_k(k, k_src_ld, Ks, simd_group_id, simd_lane_id);
thread VBlockLoader loader_v(v, v_src_ld, Vs, simd_group_id, simd_lane_id);
- TransformScale<T> ts(static_cast<T>(params->scale * M_LOG2E_F));
+ TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));
// -------------------------------------------------------------------------
// MMA tiles
@@ -204,8 +200,8 @@ KERNEL(AttentionGemm)(
// Load Q block once (and apply scaling)
threadgroup_barrier(mem_flags::mem_threadgroup);
- if (!align_q && int(tgid_x) == params->nq_aligned) {
- loader_q.load_safe(short2(BD, params->q_rem));
+ if (!align_q && int(tgid_x) == params.nq_aligned) {
+ loader_q.load_safe(short2(BD, params.q_rem));
} else {
loader_q.load_unsafe();
}
@@ -223,22 +219,22 @@ KERNEL(AttentionGemm)(
}
// Determine K block loop limit (causal can early-stop)
- int kb_lim = params->nk;
+ int kb_lim = params.nk;
if (do_causal) {
- const int q_max = (int(tgid_x) + 1) * BQ + params->q_off;
+ const int q_max = (int(tgid_x) + 1) * BQ + params.q_off;
kb_lim = (q_max + BK - 1) / BK;
- kb_lim = min(params->nk, kb_lim);
+ kb_lim = min(params.nk, kb_lim);
}
const int q_rel = int(tgid_x) * BQ + int(tm) + int(sm); // [0, q_len)
- const int q_abs = q_rel + params->q_off; // [0, k_len)
+ const int q_abs = q_rel + params.q_off; // [0, k_len)
// Loop over KV blocks
for (int kb = 0; kb < kb_lim; kb++) {
// Load K block
threadgroup_barrier(mem_flags::mem_threadgroup);
- if (!align_k && kb == params->nk_aligned) {
- loader_k.load_safe(short2(BD, params->k_rem));
+ if (!align_k && kb == params.nk_aligned) {
+ loader_k.load_safe(short2(BD, params.k_rem));
} else {
loader_k.load_unsafe();
}
@@ -264,8 +260,8 @@ KERNEL(AttentionGemm)(
}
// Mask out tail keys for the last (unaligned) K block
- if (!align_k && kb == params->nk_aligned) {
- const int k_rem = params->k_rem;
+ if (!align_k && kb == params.nk_aligned) {
+ const int k_rem = params.k_rem;
UZU_PRAGMA_UNROLL
for (short j = 0; j < TK; j++) {
thread auto& frag = Stile.frag_at(0, j);
@@ -299,8 +295,8 @@ KERNEL(AttentionGemm)(
}
// Add external mask (additive bias in natural-log domain; convert to log2)
- if (has_mask && q_rel < params->q_len) {
- const int64_t row_stride = mask_params->m_strides[2];
+ if (has_mask && q_rel < params.q_len) {
+ const int64_t row_stride = mask_params.m_strides[2];
const int64_t row_base = int64_t(q_rel) * row_stride;
UZU_PRAGMA_UNROLL
@@ -311,12 +307,12 @@ KERNEL(AttentionGemm)(
const int k0 = col_base;
const int k1 = col_base + 1;
- if (k0 < params->k_len) {
+ if (k0 < params.k_len) {
AccumType mv = static_cast<AccumType>(mask[row_base + int64_t(k0)]);
mv = metal::max(mv, static_cast<AccumType>(-1e9f));
frag[0] += M_LOG2E_F * mv;
}
- if (k1 < params->k_len) {
+ if (k1 < params.k_len) {
AccumType mv = static_cast<AccumType>(mask[row_base + int64_t(k1)]);
mv = metal::max(mv, static_cast<AccumType>(-1e9f));
frag[1] += M_LOG2E_F * mv;
@@ -326,8 +322,8 @@ KERNEL(AttentionGemm)(
// Load V block (overwriting K in shared memory)
threadgroup_barrier(mem_flags::mem_threadgroup);
- if (!align_k && kb == params->nk_aligned) {
- loader_v.load_safe(short2(BD, params->k_rem));
+ if (!align_k && kb == params.nk_aligned) {
+ loader_v.load_safe(short2(BD, params.k_rem));
} else {
loader_v.load_unsafe();
}
@@ -415,11 +411,11 @@ KERNEL(AttentionGemm)(
threadgroup_barrier(mem_flags::mem_none);
- // Store results (O is row-major with row-stride params->o_strides[2])
- o += int64_t(tm + sm) * params->o_strides[2] + int64_t(sn);
+ // Store results (O is row-major with row-stride params.o_strides[2])
+ o += int64_t(tm + sm) * params.o_strides[2] + int64_t(sn);
- if (!align_q && int(tgid_x) == params->nq_aligned) {
- const short2 dst_tile_dims = short2(BD - sn, params->q_rem - (tm + sm));
+ if (!align_q && int(tgid_x) == params.nq_aligned) {
+ const short2 dst_tile_dims = short2(BD - sn, params.q_rem - (tm + sm));
if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {
return;
@@ -427,10 +423,10 @@ KERNEL(AttentionGemm)(
Otile.template store_safe<T, 1, 1>(
o,
- int(params->o_strides[2]),
+ int(params.o_strides[2]),
dst_tile_dims
);
} else {
- Otile.template store<T, 1, 1>(o, int(params->o_strides[2]));
+ Otile.template store<T, 1, 1>(o, int(params.o_strides[2]));
}
-}
\ No newline at end of file
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
deleted file mode 100644
index 550d5cca3..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
+++ /dev/null
@@ -1,116 +0,0 @@
-use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLFunctionConstantValues};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use crate::{
- DataType,
- backends::{
- common::kernel::MlpGateActMulKernel,
- metal::{FunctionConstantValuesSetValue, MetalContext, MetalError, kernel::dsl::MlpGateActMulMetalKernel},
- },
- config::Activation,
-};
-
-// MLP Fused Epilogue Function Constant Indices
-// These must match the values in kernel/common/mlp_epilogue.h
-pub const MLP_FUSED_FC_INDEX: u64 = 50;
-pub const MLP_HIDDEN_DIM_FC_INDEX: u64 = 51;
-pub const MLP_ACTIVATION_FC_INDEX: u64 = 52;
-
-/// MLP activation type for fused kernels
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-#[repr(u32)]
-pub enum MlpActivationType {
- SiLU = 0,
- Gelu = 1,
-}
-
-impl From<&Activation> for MlpActivationType {
- fn from(act: &Activation) -> Self {
- match act {
- Activation::SiLU {
- ..
- } => MlpActivationType::SiLU,
- Activation::Gelu => MlpActivationType::Gelu,
- Activation::Identity => {
- panic!("Identity activation not supported for MLP fusion")
- },
- }
- }
-}
-
-/// Configuration for MLP fused matmul epilogue
-#[derive(Debug, Clone, Copy)]
-pub struct MlpFusedConfig {
- pub hidden_dim: u32,
- pub activation: MlpActivationType,
-}
-
-impl MlpFusedConfig {
- pub fn new(
- hidden_dim: usize,
- activation: &Activation,
- ) -> Self {
- Self {
- hidden_dim: hidden_dim as u32,
- activation: MlpActivationType::from(activation),
- }
- }
-
- /// Create function constants for MLP fused matmul
- pub fn make_function_constants(&self) -> Retained<MTLFunctionConstantValues> {
- let fcv = MTLFunctionConstantValues::new();
- fcv.set_value(&true, MLP_FUSED_FC_INDEX as usize);
- fcv.set_value(&self.hidden_dim, MLP_HIDDEN_DIM_FC_INDEX as usize);
- fcv.set_value(&(self.activation as u32), MLP_ACTIVATION_FC_INDEX as usize);
- fcv
- }
-}
-
-/// Create function constants for non-fused (standard) matmul
-pub fn make_non_fused_function_constants() -> Retained<MTLFunctionConstantValues> {
- let fcv = MTLFunctionConstantValues::new();
- fcv.set_value(&false, MLP_FUSED_FC_INDEX as usize);
- fcv
-}
-
-pub struct MlpGateActMulEncodable {
- kernel: MlpGateActMulMetalKernel,
- activation: Activation,
- hidden_dim: usize,
-}
-
-impl MlpGateActMulEncodable {
- pub fn new(
- context: &MetalContext,
- data_type: DataType,
- activation: Activation,
- hidden_dim: usize,
- ) -> Result<Self, MetalError> {
- let kernel = MlpGateActMulMetalKernel::new(context, data_type.into())?;
- Ok(Self {
- kernel,
- activation,
- hidden_dim,
- })
- }
-
- pub fn encode(
- &self,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- fused_up: &Retained<ProtocolObject<dyn MTLBuffer>>,
- hidden: &Retained<ProtocolObject<dyn MTLBuffer>>,
- m: i32,
- ) -> Result<(), MetalError> {
- let act_type = match self.activation {
- Activation::SiLU {
- ..
- } => 0,
- Activation::Gelu => 1,
- Activation::Identity => {
- panic!("Identity activation is not supported for kernel")
- },
- };
- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, encoder);
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index e1400f79a..6f02adf2c 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,4 +1,3 @@
-pub mod attention;
use crate::backends::common::kernel::matmul::MatmulKernels;
pub mod dsl {
include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
@@ -6,14 +5,8 @@ pub mod dsl {
pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
pub mod matmul;
-pub mod mlp;
-pub use mlp::{
- MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX, MlpActivationType, MlpFusedConfig,
- make_non_fused_function_constants,
-};
pub mod moe;
pub mod quant_matmul;
-pub mod ssm;
pub use matmul::{MatmulArguments, MatmulKernel};
pub use moe::{
@@ -22,7 +15,6 @@ pub use moe::{
MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,
MoeScatterWithMapArguments,
};
-pub use ssm::{SSDPrefillArguments, SSDPrefillMode};
impl MatmulKernels for dsl::MetalKernels {
type FullPrecisionMatmulKernel = MatmulKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
deleted file mode 100644
index a87e87362..000000000
--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
+++ /dev/null
@@ -1,2 +0,0 @@
-pub mod ssd_prefill;
-pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode};
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs
deleted file mode 100644
index d5a19dd69..000000000
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs
+++ /dev/null
@@ -1,135 +0,0 @@
-use metal::{MTLBuffer, MTLComputeCommandEncoder};
-use objc2::{Message, runtime::ProtocolObject};
-
-use crate::{
- DataType,
- backends::{
- common::kernel::{SSDPrefill64Kernel as _, SSDPrefillKernel as _, SSDPrefillSequentialKernel as _},
- metal::{
- MetalContext, MetalError,
- kernel::dsl::{SSDPrefill64MetalKernel, SSDPrefillMetalKernel, SSDPrefillSequentialMetalKernel},
- },
- },
-};
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum SSDPrefillMode {
- Sequential,
- SinglePass,
-}
-
-pub struct SSDPrefillArguments<'a> {
- pub x: &'a ProtocolObject<dyn MTLBuffer>,
- pub dt: &'a ProtocolObject<dyn MTLBuffer>, // raw dt values
- pub b: &'a ProtocolObject<dyn MTLBuffer>,
- pub c: &'a ProtocolObject<dyn MTLBuffer>,
- pub d: &'a ProtocolObject<dyn MTLBuffer>,
- pub z: &'a ProtocolObject<dyn MTLBuffer>,
- pub state: &'a ProtocolObject<dyn MTLBuffer>,
- pub y: &'a ProtocolObject<dyn MTLBuffer>,
- pub suffix_len: usize,
- pub group_size: i32,
- pub state_size: i32,
- pub x_strides: [usize; 3],
- pub dt_strides: [usize; 2],
- pub cb_strides: [usize; 3],
- pub state_strides: [usize; 3],
- pub channels: usize,
- pub head_dim: usize,
-}
-
-pub struct SSDPrefillKernels {
- single: SSDPrefillMetalKernel,
- single_64: SSDPrefill64MetalKernel,
- sequential: SSDPrefillSequentialMetalKernel,
-}
-
-impl SSDPrefillKernels {
- pub fn new(
- context: &MetalContext,
- data_type: DataType,
- ) -> Result<Self, MetalError> {
- let single = SSDPrefillMetalKernel::new(context, data_type)?;
- let single_64 = SSDPrefill64MetalKernel::new(context, data_type)?;
- let sequential = SSDPrefillSequentialMetalKernel::new(context, data_type)?;
- Ok(Self {
- single,
- single_64,
- sequential,
- })
- }
-
- pub fn encode(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: SSDPrefillArguments,
- mode: SSDPrefillMode,
- ) {
- if mode == SSDPrefillMode::SinglePass {
- if args.state_size == 64 {
- self.single_64.encode(
- &args.x.retain(),
- &args.dt.retain(),
- &args.b.retain(),
- &args.c.retain(),
- &args.d.retain(),
- &args.z.retain(),
- &args.state.retain(),
- &args.y.retain(),
- args.suffix_len as u32,
- args.group_size,
- args.state_size,
- args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.channels as u32,
- args.head_dim as u32,
- compute_encoder,
- )
- } else {
- self.single.encode(
- &args.x.retain(),
- &args.dt.retain(),
- &args.b.retain(),
- &args.c.retain(),
- &args.d.retain(),
- &args.z.retain(),
- &args.state.retain(),
- &args.y.retain(),
- args.suffix_len as u32,
- args.group_size,
- args.state_size,
- args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.channels as u32,
- args.head_dim as u32,
- compute_encoder,
- )
- }
- } else if mode == SSDPrefillMode::Sequential {
- self.sequential.encode(
- &args.x.retain(),
- &args.dt.retain(),
- &args.b.retain(),
- &args.c.retain(),
- &args.d.retain(),
- &args.z.retain(),
- &args.state.retain(),
- &args.y.retain(),
- args.suffix_len as u32,
- args.group_size,
- args.state_size,
- args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),
- args.channels as u32,
- args.head_dim as u32,
- compute_encoder,
- )
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rs
index 755f071eb..147f52c50 100644
--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs
+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs
@@ -140,7 +140,7 @@ pub struct LanguageModelGeneratorContext {
pub model_config: LanguageModelConfig,
pub decoder_config: Rc<DecoderConfig>,
pub model_shape: ModelShape,
- pub executables: Decoder,
+ pub executables: Decoder<Metal>,
pub kv_cache_update: Box<KVCacheUpdate<Metal>>,
pub gpu_sampler: Sampling<Metal>,
pub seed: PRng,
diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rs
index 43a4e5dff..ac610f7bb 100644
--- a/crates/uzu/src/classifier/mod.rs
+++ b/crates/uzu/src/classifier/mod.rs
@@ -7,7 +7,7 @@ pub use result::{ClassificationOutput, ClassificationStats};
// Re-export context from metal backend
pub use crate::backends::metal::ClassifierContext;
// Re-export encodables from metal backend for convenience
-pub use crate::backends::metal::encodable_block::{ClassifierLayer, ClassifierPredictionHead};
-pub use crate::encodable_block::Pooling;
+pub use crate::backends::metal::encodable_block::ClassifierLayer;
+pub use crate::encodable_block::{ClassifierPredictionHead, Pooling};
#[cfg(feature = "tracing")]
pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
similarity index 86%
rename from crates/uzu/src/backends/metal/encodable_block/attention.rs
rename to crates/uzu/src/encodable_block/attention.rs
index bac0a64ec..dc3cc50b4 100644
--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -2,24 +2,13 @@
use std::collections::HashMap;
-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
use crate::{
DataType,
- backends::{
- common::kernel::{
+ backends::common::{
+ Backend, CommandBuffer, Kernels,
+ kernel::{
AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,
- },
- metal::{
- Metal, MetalContext, MetalError,
- kernel::{
- attention::{AttentionGemmArguments, AttentionGemmBlock},
- dsl::{
- AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel,
- AttentionUpdateKVCacheMetalKernel,
- },
- },
+ attention::{AttentionGemmArguments, AttentionGemmBlock},
},
},
encodable_block::{EncodableBlock, EncodingParameters},
@@ -42,12 +31,12 @@ fn env_gemm_attention_enabled() -> bool {
})
}
-pub struct Attention {
- single_pass_kernels: HashMap<KernelKey, AttentionSinglePassMetalKernel>,
- two_pass_1_kernels: HashMap<KernelKey, AttentionTwoPass1MetalKernel>,
- two_pass_2_kernels: HashMap<u32, AttentionTwoPass2MetalKernel>,
- update_kv_cache_kernel: AttentionUpdateKVCacheMetalKernel,
- gemm_block: AttentionGemmBlock,
+pub struct Attention<B: Backend> {
+ single_pass_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionSinglePassKernel>,
+ two_pass_1_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionTwoPass1Kernel>,
+ two_pass_2_kernels: HashMap<u32, <B::Kernels as Kernels>::AttentionTwoPass2Kernel>,
+ update_kv_cache_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,
+ gemm_block: AttentionGemmBlock<B>,
layer_index: usize,
attention_scale: Option<f32>,
has_sinks: bool,
@@ -55,16 +44,16 @@ pub struct Attention {
sliding_window_size: Option<usize>,
}
-impl Attention {
+impl<B: Backend> Attention<B> {
pub fn new(
- context: &MetalContext,
+ context: &B::Context,
data_type: DataType,
layer_index: usize,
attention_scale: Option<f32>,
has_sinks: bool,
is_causal: bool,
sliding_window_size: Option<usize>,
- ) -> Result<Self, MetalError> {
+ ) -> Result<Self, B::Error> {
let mut single_pass_kernels = HashMap::new();
let mut two_pass_1_kernels = HashMap::new();
let mut two_pass_2_kernels = HashMap::new();
@@ -78,22 +67,22 @@ impl Attention {
};
let float_mask = has_mask;
- let sp_kernel = AttentionSinglePassMetalKernel::new(
+ let sp_kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(
context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,
)?;
single_pass_kernels.insert(key, sp_kernel);
- let tp1_kernel = AttentionTwoPass1MetalKernel::new(
+ let tp1_kernel = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,
)?;
two_pass_1_kernels.insert(key, tp1_kernel);
- let tp2_kernel = AttentionTwoPass2MetalKernel::new(context, data_type, head_dim)?;
+ let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)?;
two_pass_2_kernels.insert(head_dim, tp2_kernel);
}
}
- let update_kv_cache_kernel = AttentionUpdateKVCacheMetalKernel::new(context, data_type)?;
+ let update_kv_cache_kernel = <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type)?;
let gemm_block = AttentionGemmBlock::new(data_type);
Ok(Self {
@@ -138,20 +127,17 @@ impl Attention {
}
}
-impl EncodableBlock<Metal> for Attention {
+impl<B: Backend> EncodableBlock<B> for Attention<B> {
fn encode(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
) {
- let compute_encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, parameters, &compute_encoder);
- compute_encoder.end_encoding();
+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, &encoder));
if parameters.wait_until_completed {
- command_buffer.commit();
+ command_buffer.submit();
command_buffer.wait_until_completed();
}
}
@@ -162,9 +148,9 @@ impl EncodableBlock<Metal> for Attention {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ compute_encoder: &B::ComputeEncoder,
) {
let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
@@ -356,12 +342,12 @@ impl EncodableBlock<Metal> for Attention {
match variant {
KernelVariant::Gemm => {
let args = AttentionGemmArguments {
- queries_buffer: &queries_buffer,
+ queries_buffer,
keys_buffer: &key_cache_buffer,
values_buffer: &value_cache_buffer,
- output_buffer: &attention_output_buffer,
- mask_buffer: attention_bias_buffer.as_deref(),
- sinks_buffer: sinks_buffer.as_deref(),
+ output_buffer: attention_output_buffer,
+ mask_buffer: attention_bias_buffer.as_ref(),
+ sinks_buffer: sinks_buffer.as_ref(),
num_heads,
num_groups,
suffix_length,
@@ -373,13 +359,13 @@ impl EncodableBlock<Metal> for Attention {
scale,
};
self.gemm_block
- .encode(state.mtl_context(), compute_encoder, &args)
+ .encode(state.context(), compute_encoder, &args)
.expect("Failed to encode AttentionGemmBlock");
},
KernelVariant::SinglePass => {
let kernel = match self.single_pass_kernels.get(&kernel_key) {
Some(k) => k,
- None => panic!("Can not find AttentionSinglePassMetalKernel for key {:?}", kernel_key),
+ None => panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key),
};
let mask_buffer_opt = attention_bias_buffer.as_ref();
let sinks_buffer_opt = sinks_buffer.as_ref();
@@ -408,11 +394,11 @@ impl EncodableBlock<Metal> for Attention {
KernelVariant::TwoPass => {
let kernel_pass1 = match self.two_pass_1_kernels.get(&kernel_key) {
Some(k) => k,
- None => panic!("Can not find AttentionTwoPass1MetalKernel for key {:?}", kernel_key),
+ None => panic!("Can not find AttentionTwoPass1Kernel for key {:?}", kernel_key),
};
let kernel_pass2 = match self.two_pass_2_kernels.get(&(head_dim as u32)) {
Some(k) => k,
- None => panic!("Can not find AttentionTwoPass2MetalKernel for key {:?}", kernel_key),
+ None => panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key),
};
let mask_buffer_opt = attention_bias_buffer.as_ref().map(|b| b);
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
index 6e10e9855..5a9444f27 100644
--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
@@ -122,7 +122,7 @@ where
let element_size = input_array.data_type().size_in_bytes();
self.kernel.borrow_mut().encode(
- state.mtl_context(),
+ state.context(),
encoder,
FullPrecisionMatmulArguments {
a: input_array.buffer(),
diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rs
index 17b8e83a6..56d32d294 100644
--- a/crates/uzu/src/encodable_block/full_precision_linear.rs
+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs
@@ -148,7 +148,7 @@ where
let output_array = arrays[1].borrow_mut();
self.kernel.borrow_mut().encode(
- state.mtl_context(),
+ state.context(),
encoder,
FullPrecisionMatmulArguments {
a: input_array.buffer(),
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
similarity index 75%
rename from crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
rename to crates/uzu/src/encodable_block/mamba_mixer.rs
index 182533c19..bcaa39755 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -2,54 +2,47 @@
use std::env;
-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};
-use objc2::{Message, rc::Retained, runtime::ProtocolObject};
-
-use super::transformer_layer;
use crate::{
Activation, DataType,
array::Array,
backends::{
- common::kernel::{Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel},
- metal::{
- Metal, MetalContext,
+ common::{
+ Backend, CommandBuffer, Kernels,
kernel::{
- dsl::{
- Conv1dDecodeMetalKernel, Conv1dPackMetalKernel, Conv1dScanMetalKernel, SSDUpdateMetalKernel,
- SplitInProjMetalKernel,
- },
- ssm::ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
+ Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,
+ ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
},
+ metal::{Metal, encodable_block::transformer_layer},
},
config::{DecoderLayerType, Mamba2Config},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
+ parameters::{ParameterTree, resolve_subtree},
};
-pub(crate) struct MambaMixer {
+pub(crate) struct MambaMixer<B: Backend> {
layer_index: usize,
config: Mamba2Config,
- in_projection: Box<dyn EncodableBlock<Metal>>,
- out_projection: Box<dyn EncodableBlock<Metal>>,
- split_inproj: SplitInProjMetalKernel,
- conv_decode: Conv1dDecodeMetalKernel,
- conv_pack: Conv1dPackMetalKernel,
- conv_scan: Conv1dScanMetalKernel,
- ssd_prefill: SSDPrefillKernels,
- ssd_update: SSDUpdateMetalKernel,
- conv_weight: Array<Metal>,
- conv_bias: Option<Array<Metal>>,
- gate_bias: Array<Metal>,
- skip_connection_weight: Array<Metal>,
+ in_projection: Box<dyn EncodableBlock<B>>,
+ out_projection: Box<dyn EncodableBlock<B>>,
+ split_inproj: <B::Kernels as Kernels>::SplitInProjKernel,
+ conv_decode: <B::Kernels as Kernels>::Conv1dDecodeKernel,
+ conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,
+ conv_scan: <B::Kernels as Kernels>::Conv1dScanKernel,
+ ssd_prefill: SSDPrefillKernels<B>,
+ ssd_update: <B::Kernels as Kernels>::SSDUpdateKernel,
+ conv_weight: Array<B>,
+ conv_bias: Option<Array<B>>,
+ gate_bias: Array<B>,
+ skip_connection_weight: Array<B>,
prefill_mode: SSDPrefillMode,
}
-impl MambaMixer {
+impl MambaMixer<Metal> {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
- mtl_context: &MetalContext,
+ context: &<Metal as Backend>::Context,
layer_type: DecoderLayerType,
mamba_config: Mamba2Config,
layer_index: usize,
@@ -57,7 +50,7 @@ impl MambaMixer {
num_heads: usize,
head_dim: usize,
num_groups: usize,
- decoder_layer_loader: &ParameterTree<MetalContext>,
+ decoder_layer_loader: &ParameterTree<<Metal as Backend>::Context>,
) -> Self {
let _ = (num_heads, head_dim, num_groups);
if !matches!(layer_type, DecoderLayerType::StateSpace { .. }) {
@@ -73,7 +66,7 @@ impl MambaMixer {
mamba_config.has_in_biases,
model_dim,
[mamba_config.conv_dim(), mamba_config.inner_dim(), mamba_config.num_heads],
- mtl_context,
+ context,
&resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),
ArrayId::Main,
ArrayId::SsmInProj,
@@ -85,7 +78,7 @@ impl MambaMixer {
mamba_config.has_out_biases,
mamba_config.inner_dim(),
[model_dim],
- mtl_context,
+ context,
&resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),
ArrayId::AttentionOutput,
ArrayId::Main,
@@ -101,25 +94,27 @@ impl MambaMixer {
let gate_bias = split_tree.leaf("gate_bias").unwrap().clone();
let skip_connection_weight = split_tree.leaf("skip_connection_weight").unwrap().clone();
- let split_inproj =
- SplitInProjMetalKernel::new(mtl_context, data_type).expect("Failed to create split in-projection kernel");
- let conv_scan = Conv1dScanMetalKernel::new(
- mtl_context,
+ let split_inproj = <<Metal as Backend>::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)
+ .expect("Failed to create split in-projection kernel");
+ let conv_scan = <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(
+ context,
data_type,
Self::activation_to_int(&mamba_config.activation),
mamba_config.conv_config.has_biases,
)
.expect("Failed to create conv scan kernel");
- let conv_decode = Conv1dDecodeMetalKernel::new(
- mtl_context,
+ let conv_decode = <<Metal as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(
+ context,
data_type,
Self::activation_to_int(&mamba_config.activation),
mamba_config.conv_config.has_biases,
)
.expect("Failed to create conv decode kernel");
- let conv_pack = Conv1dPackMetalKernel::new(mtl_context, data_type).expect("Failed to create conv pack kernel");
- let ssd_prefill = SSDPrefillKernels::new(mtl_context, data_type).expect("Failed to create SSD prefill kernel");
- let ssd_update = SSDUpdateMetalKernel::new(mtl_context, data_type).expect("Failed to create SSD decode kernel");
+ let conv_pack = <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)
+ .expect("Failed to create conv pack kernel");
+ let ssd_prefill = SSDPrefillKernels::new(context, data_type).expect("Failed to create SSD prefill kernel");
+ let ssd_update = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(context, data_type)
+ .expect("Failed to create SSD decode kernel");
let prefill_mode = resolve_prefill_mode_from_env();
Self {
@@ -140,12 +135,14 @@ impl MambaMixer {
prefill_mode,
}
}
+}
+impl<B: Backend> MambaMixer<B> {
fn encode_pipeline_with_encoder(
&self,
- state: &mut ForwardPassState<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters<Metal>,
+ state: &mut ForwardPassState<B>,
+ encoder: &B::ComputeEncoder,
+ parameters: &EncodingParameters<B>,
) {
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
@@ -167,8 +164,8 @@ impl MambaMixer {
fn run_split_inproj(
&self,
- state: &mut ForwardPassState<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ encoder: &B::ComputeEncoder,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -182,12 +179,12 @@ impl MambaMixer {
let gate = arrays[2].borrow_mut();
let dt = arrays[3].borrow_mut();
- let input_buf = in_proj.buffer().to_owned();
- let conv_buf = conv_inputs.buffer().to_owned();
- let gate_buf = gate.buffer().to_owned();
- let dt_buf = dt.buffer().to_owned();
+ let input_buf = in_proj.buffer().clone();
+ let conv_buf = conv_inputs.buffer().clone();
+ let gate_buf = gate.buffer().clone();
+ let dt_buf = dt.buffer().clone();
let gate_bias = self.gate_bias.clone();
- let bias_buf = gate_bias.buffer().to_owned();
+ let bias_buf = gate_bias.buffer().clone();
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
@@ -211,8 +208,8 @@ impl MambaMixer {
fn run_conv_scan(
&self,
- state: &mut ForwardPassState<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ encoder: &B::ComputeEncoder,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -228,11 +225,11 @@ impl MambaMixer {
let b_arr = arrays[3].borrow_mut();
let c_arr = arrays[4].borrow_mut();
- let input_buf = conv_inputs.buffer().to_owned();
+ let input_buf = conv_inputs.buffer().clone();
let state_buf = conv_state.buffer().clone();
- let x_buf = x_arr.buffer().to_owned();
- let b_buf = b_arr.buffer().to_owned();
- let c_buf = c_arr.buffer().to_owned();
+ let x_buf = x_arr.buffer().clone();
+ let b_buf = b_arr.buffer().clone();
+ let c_buf = c_arr.buffer().clone();
let weight_storage = self.conv_weight.clone();
let weight_buf = weight_storage.buffer().clone();
@@ -289,7 +286,7 @@ impl MambaMixer {
};
if conv_dim > 0 && self.config.kernel_size > 0 {
- let conv_source = padded_buf.as_deref().unwrap_or(&input_buf).retain();
+ let conv_source = padded_buf.as_ref().unwrap_or(&input_buf).clone();
self.conv_scan.encode(
&conv_source,
&weight_buf,
@@ -313,8 +310,8 @@ impl MambaMixer {
fn run_prefill_ssm(
&self,
- state: &mut ForwardPassState<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ encoder: &B::ComputeEncoder,
suffix_length: usize,
) {
let base_arrays = state.arrays(&[
@@ -378,8 +375,8 @@ impl MambaMixer {
fn run_decode_ssm(
&self,
- state: &mut ForwardPassState<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ encoder: &B::ComputeEncoder,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -471,21 +468,19 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {
}
}
-impl EncodableBlock<Metal> for MambaMixer {
+impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {
fn encode(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
) {
if self.supports_shared_encoder() {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_pipeline_with_encoder(state, &encoder, parameters);
- encoder.end_encoding();
+ command_buffer
+ .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, encoder, parameters));
if parameters.wait_until_completed {
- command_buffer.commit();
+ command_buffer.submit();
command_buffer.wait_until_completed();
}
return;
@@ -498,20 +493,20 @@ impl EncodableBlock<Metal> for MambaMixer {
self.in_projection.encode(state, parameters, command_buffer);
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.run_split_inproj(state, &encoder, active_suffix_length);
- self.run_conv_scan(state, &encoder, active_suffix_length);
- if active_suffix_length == 1 {
- self.run_decode_ssm(state, &encoder, active_suffix_length);
- } else {
- self.run_prefill_ssm(state, &encoder, active_suffix_length);
- }
- encoder.end_encoding();
+ command_buffer.with_compute_encoder(|encoder| {
+ self.run_split_inproj(state, encoder, active_suffix_length);
+ self.run_conv_scan(state, encoder, active_suffix_length);
+ if active_suffix_length == 1 {
+ self.run_decode_ssm(state, encoder, active_suffix_length);
+ } else {
+ self.run_prefill_ssm(state, encoder, active_suffix_length);
+ }
+ });
self.out_projection.encode(state, parameters, command_buffer);
if parameters.wait_until_completed {
- command_buffer.commit();
+ command_buffer.submit();
command_buffer.wait_until_completed();
}
}
@@ -522,24 +517,10 @@ impl EncodableBlock<Metal> for MambaMixer {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
) {
self.encode_pipeline_with_encoder(state, encoder, parameters);
}
}
-
-fn resolve_subtree<'tree>(
- loader: &'tree ParameterTree<MetalContext>,
- candidates: &[&str],
-) -> ParameterTree<'tree, MetalContext> {
- for candidate in candidates {
- if let Ok(tree) = loader.subtree(candidate) {
- return tree;
- }
- }
-
- let missing = candidates.first().copied().unwrap_or("<missing subtree name>");
- loader.subtree(missing).unwrap_or_else(|_| panic!("Unable to resolve parameter subtree '{missing}'"))
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp.rs
similarity index 59%
rename from crates/uzu/src/backends/metal/encodable_block/mlp.rs
rename to crates/uzu/src/encodable_block/mlp.rs
index 6cfcd4bc0..9a2af53a4 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs
+++ b/crates/uzu/src/encodable_block/mlp.rs
@@ -1,25 +1,22 @@
//! MLP block encodable.
-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
use crate::{
- backends::metal::{Metal, kernel::mlp::MlpGateActMulEncodable},
+ backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::{ArrayId, ForwardPassState},
};
-pub struct MlpBlock {
- up: Box<dyn EncodableBlock<Metal>>,
- gate: MlpGateActMulEncodable,
- down: Box<dyn EncodableBlock<Metal>>,
+pub struct MlpBlock<B: Backend> {
+ up: Box<dyn EncodableBlock<B>>,
+ gate: MlpGateActMulEncodable<B>,
+ down: Box<dyn EncodableBlock<B>>,
}
-impl MlpBlock {
+impl<B: Backend> MlpBlock<B> {
pub fn new(
- up: Box<dyn EncodableBlock<Metal>>,
- gate: MlpGateActMulEncodable,
- down: Box<dyn EncodableBlock<Metal>>,
+ up: Box<dyn EncodableBlock<B>>,
+ gate: MlpGateActMulEncodable<B>,
+ down: Box<dyn EncodableBlock<B>>,
) -> Self {
Self {
up,
@@ -29,24 +26,21 @@ impl MlpBlock {
}
}
-impl EncodableBlock<Metal> for MlpBlock {
+impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
fn encode(
&self,
- state: &mut ForwardPassState<Metal>,
- params: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ state: &mut ForwardPassState<B>,
+ params: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
) {
if self.supports_shared_encoder() {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, params, &encoder);
- encoder.end_encoding();
+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, params, encoder));
} else {
// Up
self.up.encode(state, params, command_buffer);
// Gate act+mul (fused_up -> hidden)
- {
+ command_buffer.with_compute_encoder(|encoder| {
let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
let fused = arrays[0].borrow_mut();
let hidden = arrays[1].borrow_mut();
@@ -54,20 +48,17 @@ impl EncodableBlock<Metal> for MlpBlock {
let fused_buf = fused.buffer();
let hidden_buf = hidden.buffer();
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
self.gate
- .encode(&encoder, fused_buf, hidden_buf, m)
+ .encode(encoder, fused_buf, hidden_buf, m)
.expect("Failed to encode MLP activation/mul kernel");
- encoder.end_encoding();
- }
+ });
// Down
self.down.encode(state, params, command_buffer);
}
if params.wait_until_completed {
- command_buffer.commit();
+ command_buffer.submit();
command_buffer.wait_until_completed();
}
}
@@ -78,9 +69,9 @@ impl EncodableBlock<Metal> for MlpBlock {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState<Metal>,
- params: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ params: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
) {
// Up
self.up.encode_with_shared_encoder(state, params, encoder);
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 79b75fec2..050224bc2 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -4,34 +4,44 @@ use crate::{
};
mod activation;
+mod attention;
mod full_precision_embedding_lookup;
mod full_precision_embedding_readout;
mod full_precision_linear;
mod layer_norm;
+mod mamba_mixer;
+mod mlp;
mod normalization;
mod pooling;
+mod prediction_head;
mod qk_norm;
mod quantized_embedding_lookup;
mod quantized_linear;
mod rms_norm;
mod rope;
mod sampling;
+mod short_conv_mixer;
mod tensor_add_swap;
mod tensor_copy;
pub use activation::Activation;
+pub use attention::Attention;
pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};
pub use full_precision_embedding_readout::{FullPrecisionEmbeddingReadout, FullPrecisionEmbeddingReadoutError};
pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};
pub use layer_norm::{LayerNorm, LayerNormError};
+pub(crate) use mamba_mixer::MambaMixer;
+pub use mlp::MlpBlock;
pub use normalization::{Normalization, NormalizationError};
pub use pooling::Pooling;
+pub use prediction_head::ClassifierPredictionHead;
pub use qk_norm::{QKNorm, QKNormError};
pub use quantized_embedding_lookup::{QuantizedEmbeddingLookup, QuantizedEmbeddingLookupError};
pub use quantized_linear::{QuantizedLinear, QuantizedLinearError};
pub use rms_norm::{RMSNorm, RMSNormError};
pub use rope::Rope;
pub use sampling::Sampling;
+pub use short_conv_mixer::ShortConvMixer;
pub use tensor_add_swap::TensorAddSwap;
pub use tensor_copy::TensorCopy;
diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rs
new file mode 100644
index 000000000..fb3b12d61
--- /dev/null
+++ b/crates/uzu/src/encodable_block/prediction_head.rs
@@ -0,0 +1,88 @@
+//! Prediction head encodable for classification output.
+
+#[cfg(feature = "tracing")]
+use crate::forward_pass::state::ArrayId;
+use crate::{
+ backends::common::{Backend, CommandBuffer},
+ encodable_block::{EncodableBlock, EncodingParameters},
+ forward_pass::state::ForwardPassState,
+};
+
+pub struct ClassifierPredictionHead<B: Backend> {
+ dense: Box<dyn EncodableBlock<B>>,
+ activation: Box<dyn EncodableBlock<B>>,
+ norm: Box<dyn EncodableBlock<B>>,
+ readout: Box<dyn EncodableBlock<B>>,
+ #[allow(dead_code)]
+ num_labels: usize,
+}
+
+impl<B: Backend> ClassifierPredictionHead<B> {
+ pub fn new(
+ dense: Box<dyn EncodableBlock<B>>,
+ activation: Box<dyn EncodableBlock<B>>,
+ norm: Box<dyn EncodableBlock<B>>,
+ readout: Box<dyn EncodableBlock<B>>,
+ num_labels: usize,
+ ) -> Self {
+ Self {
+ dense,
+ activation,
+ norm,
+ readout,
+ num_labels,
+ }
+ }
+}
+
+impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
+ ) {
+ if self.supports_shared_encoder() {
+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
+ } else {
+ self.dense.encode(state, parameters, command_buffer);
+ self.activation.encode(state, parameters, command_buffer);
+ self.norm.encode(state, parameters, command_buffer);
+ self.readout.encode(state, parameters, command_buffer);
+ }
+
+ #[cfg(feature = "tracing")]
+ {
+ let traces = state.traces().clone();
+ state.encode_copy_array(
+ command_buffer,
+ ArrayId::ClassifierPredictionHeadLogits,
+ traces.borrow().logits.clone(),
+ );
+ }
+
+ if parameters.wait_until_completed {
+ command_buffer.submit();
+ command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ self.dense.supports_shared_encoder()
+ && self.activation.supports_shared_encoder()
+ && self.norm.supports_shared_encoder()
+ && self.readout.supports_shared_encoder()
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
+ ) {
+ self.dense.encode_with_shared_encoder(state, parameters, encoder);
+ self.activation.encode_with_shared_encoder(state, parameters, encoder);
+ self.norm.encode_with_shared_encoder(state, parameters, encoder);
+ self.readout.encode_with_shared_encoder(state, parameters, encoder);
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
similarity index 74%
rename from crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
rename to crates/uzu/src/encodable_block/short_conv_mixer.rs
index 7293db9d7..4582af4e7 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -1,64 +1,41 @@
-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::transformer_layer;
use crate::{
DataType,
array::Array,
backends::{
common::{
- Context,
+ Backend, CommandBuffer, Context, Kernels,
kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},
},
- metal::{
- Metal, MetalContext,
- kernel::dsl::{
- ShortConvDecodeMetalKernel, ShortConvPackMetalKernel, ShortConvPrefillMetalKernel,
- ShortConvTrieMetalKernel,
- },
- },
+ metal::{Metal, encodable_block::transformer_layer},
},
config::{DecoderLayerType, ShortConvConfig},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
+ parameters::{ParameterTree, resolve_subtree},
};
-pub(crate) struct ShortConvMixer {
+pub struct ShortConvMixer<B: Backend> {
layer_index: usize,
config: ShortConvConfig,
model_dim: usize,
- in_projection: Box<dyn EncodableBlock<Metal>>,
- out_projection: Box<dyn EncodableBlock<Metal>>,
- short_conv_pack: ShortConvPackMetalKernel,
- short_conv_prefill: ShortConvPrefillMetalKernel,
- short_conv_decode: ShortConvDecodeMetalKernel,
- short_conv_trie: ShortConvTrieMetalKernel,
- conv_weight: Array<Metal>,
- conv_bias: Option<Array<Metal>>,
+ in_projection: Box<dyn EncodableBlock<B>>,
+ out_projection: Box<dyn EncodableBlock<B>>,
+ short_conv_pack: <B::Kernels as Kernels>::ShortConvPackKernel,
+ short_conv_prefill: <B::Kernels as Kernels>::ShortConvPrefillKernel,
+ short_conv_decode: <B::Kernels as Kernels>::ShortConvDecodeKernel,
+ short_conv_trie: <B::Kernels as Kernels>::ShortConvTrieKernel,
+ conv_weight: Array<B>,
+ conv_bias: Option<Array<B>>,
}
-fn resolve_subtree<'tree>(
- tree: &'tree ParameterTree<MetalContext>,
- candidates: &[&str],
-) -> ParameterTree<'tree, MetalContext> {
- for candidate in candidates {
- if let Ok(subtree) = tree.subtree(candidate) {
- return subtree;
- }
- }
- panic!("Could not find any of {:?} in parameter tree", candidates);
-}
-
-impl ShortConvMixer {
- #[allow(clippy::too_many_arguments)]
- pub(crate) fn new(
- mtl_context: &MetalContext,
+impl ShortConvMixer<Metal> {
+ pub fn new(
+ context: &<Metal as Backend>::Context,
layer_type: DecoderLayerType,
short_conv_config: ShortConvConfig,
layer_index: usize,
model_dim: usize,
- decoder_layer_loader: &ParameterTree<MetalContext>,
+ decoder_layer_loader: &ParameterTree<<Metal as Backend>::Context>,
) -> Self {
if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {
panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);
@@ -74,7 +51,7 @@ impl ShortConvMixer {
false,
model_dim,
[model_dim * 3],
- mtl_context,
+ context,
&resolve_subtree(&mixer_tree, &["in_projection", "in_proj"]),
ArrayId::Main,
ArrayId::SsmInProj,
@@ -86,7 +63,7 @@ impl ShortConvMixer {
false,
model_dim,
[model_dim],
- mtl_context,
+ context,
&resolve_subtree(&mixer_tree, &["out_projection", "out_proj"]),
ArrayId::AttentionOutput,
ArrayId::Main,
@@ -101,14 +78,17 @@ impl ShortConvMixer {
};
let has_bias = short_conv_config.conv_config.has_biases;
- let short_conv_pack =
- ShortConvPackMetalKernel::new(mtl_context, data_type).expect("Failed to create short conv pack kernel");
- let short_conv_prefill = ShortConvPrefillMetalKernel::new(mtl_context, data_type, has_bias)
- .expect("Failed to create short conv prefill kernel");
- let short_conv_decode = ShortConvDecodeMetalKernel::new(mtl_context, data_type, has_bias)
- .expect("Failed to create short conv decode kernel");
- let short_conv_trie = ShortConvTrieMetalKernel::new(mtl_context, data_type, has_bias)
- .expect("Failed to create short conv trie kernel");
+ let short_conv_pack = <<Metal as Backend>::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type)
+ .expect("Failed to create short conv pack kernel");
+ let short_conv_prefill =
+ <<Metal as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)
+ .expect("Failed to create short conv prefill kernel");
+ let short_conv_decode =
+ <<Metal as Backend>::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias)
+ .expect("Failed to create short conv decode kernel");
+ let short_conv_trie =
+ <<Metal as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)
+ .expect("Failed to create short conv trie kernel");
Self {
layer_index,
@@ -124,12 +104,14 @@ impl ShortConvMixer {
conv_bias,
}
}
+}
+impl<B: Backend> ShortConvMixer<B> {
fn encode_pipeline(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
) {
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
@@ -138,23 +120,21 @@ impl ShortConvMixer {
self.in_projection.encode(state, parameters, command_buffer);
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.run_conv(state, &encoder, active_suffix_length);
- encoder.end_encoding();
+ command_buffer.with_compute_encoder(|encoder| self.run_conv(state, &encoder, active_suffix_length));
self.out_projection.encode(state, parameters, command_buffer);
if parameters.wait_until_completed {
- command_buffer.commit();
+ command_buffer.submit();
command_buffer.wait_until_completed();
}
}
fn encode_pipeline_with_encoder(
&self,
- state: &mut ForwardPassState<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters<Metal>,
+ state: &mut ForwardPassState<B>,
+ encoder: &B::ComputeEncoder,
+ parameters: &EncodingParameters<B>,
) {
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
@@ -170,7 +150,7 @@ impl ShortConvMixer {
fn clear_suffix_state_valid_range(
&self,
- state: &ForwardPassState<Metal>,
+ state: &ForwardPassState<B>,
) {
let Some(cache_layers) = state.cache_layers() else {
return;
@@ -182,7 +162,7 @@ impl ShortConvMixer {
fn set_suffix_state_valid_range(
&self,
- state: &ForwardPassState<Metal>,
+ state: &ForwardPassState<B>,
start: usize,
len: usize,
) {
@@ -196,8 +176,8 @@ impl ShortConvMixer {
fn run_conv(
&self,
- state: &mut ForwardPassState<Metal>,
- compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ compute: &B::ComputeEncoder,
active_suffix_length: usize,
) {
self.clear_suffix_state_valid_range(state);
@@ -235,8 +215,8 @@ impl ShortConvMixer {
fn run_prefill_conv(
&self,
- state: &mut ForwardPassState<Metal>,
- compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ compute: &B::ComputeEncoder,
suffix_length: usize,
) {
if self.model_dim == 0 || suffix_length == 0 {
@@ -265,7 +245,7 @@ impl ShortConvMixer {
let element_size = data_type.size_in_bytes();
let padded_rows = state_stride + suffix_length;
let padded_size = padded_rows * self.model_dim * element_size;
- let padded_buf = state.mtl_context().create_buffer(padded_size).expect("Failed to create padded buffer");
+ let padded_buf = state.context().create_buffer(padded_size).expect("Failed to create padded buffer");
self.short_conv_pack.encode(
&state_buf,
&in_proj_buf,
@@ -295,8 +275,8 @@ impl ShortConvMixer {
fn run_trie_conv(
&self,
- state: &mut ForwardPassState<Metal>,
- compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ compute: &B::ComputeEncoder,
sampling_start: usize,
trie_len: usize,
) {
@@ -360,8 +340,8 @@ impl ShortConvMixer {
fn run_decode_conv(
&self,
- state: &mut ForwardPassState<Metal>,
- compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ compute: &B::ComputeEncoder,
suffix_length: usize,
) {
if self.model_dim == 0 || suffix_length == 0 {
@@ -402,21 +382,19 @@ impl ShortConvMixer {
}
}
-impl EncodableBlock<Metal> for ShortConvMixer {
+impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {
fn encode(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
) {
if self.supports_shared_encoder() {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_pipeline_with_encoder(state, &encoder, parameters);
- encoder.end_encoding();
+ command_buffer
+ .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, &encoder, parameters));
if parameters.wait_until_completed {
- command_buffer.commit();
+ command_buffer.submit();
command_buffer.wait_until_completed();
}
} else {
@@ -430,9 +408,9 @@ impl EncodableBlock<Metal> for ShortConvMixer {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
) {
self.encode_pipeline_with_encoder(state, encoder, parameters);
}
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index 02c1a076f..7967ea70f 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -448,7 +448,7 @@ impl<B: Backend> ForwardPassState<B> {
// Accessors
// ========================================================================
- pub fn mtl_context(&self) -> &B::Context {
+ pub fn context(&self) -> &B::Context {
self.context.as_ref()
}
diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rs
index ae0b098ed..be1380602 100644
--- a/crates/uzu/src/parameters/loader.rs
+++ b/crates/uzu/src/parameters/loader.rs
@@ -193,3 +193,15 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {
self.loader.read_extract_at(&self.join_prefix(name), buf, shape, data_type)
}
}
+
+pub fn resolve_subtree<'tree, C: Context>(
+ tree: &'tree ParameterTree<C>,
+ candidates: &[&str],
+) -> ParameterTree<'tree, C> {
+ for candidate in candidates {
+ if let Ok(subtree) = tree.subtree(candidate) {
+ return subtree;
+ }
+ }
+ panic!("Could not find any of {:?} in parameter tree", candidates);
+}
diff --git a/crates/uzu/src/parameters/mod.rs b/crates/uzu/src/parameters/mod.rs
index ff3f5ddc2..903847308 100644
--- a/crates/uzu/src/parameters/mod.rs
+++ b/crates/uzu/src/parameters/mod.rs
@@ -1,5 +1,5 @@
mod loader;
-pub use loader::{ParameterLoader, ParameterLoaderError, ParameterTree};
+pub use loader::{ParameterLoader, ParameterLoaderError, ParameterTree, resolve_subtree};
mod safetensors_metadata;
// Re-export the safetensors header reader so other modules (e.g. decoder
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index 3cec1dbd3..40838afea 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -13,14 +13,14 @@ use uzu::{
backends::{
common::{
Context,
- kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},
- },
- metal::{
- MetalContext,
kernel::{
+ AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,
attention::{AttentionGemmArguments, AttentionGemmBlock},
- dsl::{AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},
},
+ },
+ metal::{
+ Metal, MetalContext,
+ kernel::dsl::{AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},
metal_extensions::CommandBufferTimingExt,
},
},
@@ -439,7 +439,7 @@ fn run_single_pass_attention_with_is_causal(
}
fn run_gemm_attention(
- kernel: &AttentionGemmBlock,
+ kernel: &AttentionGemmBlock<Metal>,
context: &MetalContext,
queries: &Array4<f32>,
keys: &Array4<f32>,
@@ -472,8 +472,8 @@ fn run_gemm_attention(
keys_buffer: &key_cache_buffer,
values_buffer: &value_cache_buffer,
output_buffer: &output_buffer,
- mask_buffer: mask_buffer.as_deref(),
- sinks_buffer: sinks_buffer.as_deref(),
+ mask_buffer: mask_buffer.as_ref(),
+ sinks_buffer: sinks_buffer.as_ref(),
num_heads,
num_groups: num_kv_heads,
suffix_length: seq_len,
diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rs
index ddaa4ff4d..436d552e7 100644
--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs
+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs
@@ -11,14 +11,14 @@ use objc2::runtime::ProtocolObject;
use uzu::{
DataType,
backends::{
- common::{Context, kernel::Conv1dScanKernel},
- metal::{
- MetalContext,
+ common::{
+ Context,
kernel::{
- dsl::Conv1dScanMetalKernel,
- ssm::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
+ Conv1dScanKernel,
+ ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
},
+ metal::{Metal, MetalContext, kernel::dsl::Conv1dScanMetalKernel},
},
};
@@ -199,7 +199,7 @@ impl SSDPrefillFixture {
fn run_prefill_kernel_mode(
ctx: &MetalContext,
- kernel: &SSDPrefillKernels,
+ kernel: &SSDPrefillKernels<Metal>,
fixture: &SSDPrefillFixture,
mode: SSDPrefillMode,
) -> (Vec<f32>, Vec<f32>, Option<(Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>)>) {
@@ -228,7 +228,7 @@ fn run_prefill_kernel_mode(
write_buffer(&state_buf, &fixture.state_init);
zero_buffer(&y_buf);
- let args = SSDPrefillArguments {
+ let args = SSDPrefillArguments::<Metal> {
x: &x_buf,
dt: &dt_buf,
b: &b_buf,
@@ -363,7 +363,7 @@ fn assert_deterministic_for_mode(mode: SSDPrefillMode) {
eprintln!("Skipping SSD prefill determinism test: no Metal device");
return;
};
- let kernel = SSDPrefillKernels::new(&ctx, DataType::F32).unwrap();
+ let kernel = SSDPrefillKernels::<Metal>::new(&ctx, DataType::F32).unwrap();
let fixture = SSDPrefillFixture::new();
let (y_a, state_a, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
@@ -378,7 +378,7 @@ fn assert_matches_cpu_reference(mode: SSDPrefillMode) {
eprintln!("Skipping SSD prefill reference test: no Metal device");
return;
};
- let kernel = SSDPrefillKernels::new(&ctx, DataType::F32).unwrap();
+ let kernel = SSDPrefillKernels::<Metal>::new(&ctx, DataType::F32).unwrap();
let fixture = SSDPrefillFixture::new();
let (y_ref, state_ref) = ssd_prefill_cpu_reference(