Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

generic forward pass state 2 (#161)

uuuvn23ad4f02e45f
diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rs
index 850be99b7..15452405e 100644
--- a/crates/uzu/build/common/traitgen.rs
+++ b/crates/uzu/build/common/traitgen.rs
@@ -4,6 +4,7 @@ use anyhow::{Context, bail};
use itertools::Itertools;
use proc_macro2::TokenStream;
use quote::{format_ident, quote};
+use syn::Type;
use crate::common::{
codegen::write_tokens,
@@ -21,7 +22,7 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
let ty = match &p.ty {
KernelParameterType::DType => quote! { DataType },
KernelParameterType::Specialization(ty) => {
- let ty = format_ident!("{}", ty.as_ref());
+ let ty: Type = syn::parse_str(ty.as_ref()).unwrap();
quote! { #ty }
},
};
@@ -39,11 +40,11 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
quote! { &<Self::Backend as Backend>::NativeBuffer }
},
KernelArgumentType::Constant(ty) => {
- let ty = format_ident!("{}", ty.as_ref());
+ let ty: Type = syn::parse_str(ty.as_ref()).unwrap();
quote! { &[#ty] }
},
KernelArgumentType::Scalar(ty) => {
- let ty = format_ident!("{}", ty.as_ref());
+ let ty: Type = syn::parse_str(ty.as_ref()).unwrap();
quote! { #ty }
},
};
@@ -58,8 +59,8 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
fn new(context: &<Self::Backend as Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as Backend>::Error>;
- fn encode(&self, #(#args ,)* encoder: &<Self::Backend as Backend>::EncoderRef);
- fn encode_if(&self, #(#args ,)* encoder: &<Self::Backend as Backend>::EncoderRef, predicate: Option<&<Self::Backend as Backend>::NativeBuffer>);
+ fn encode(&self, #(#args ,)* encoder: &<Self::Backend as Backend>::ComputeEncoder);
+ fn encode_if(&self, #(#args ,)* encoder: &<Self::Backend as Backend>::ComputeEncoder, predicate: Option<&<Self::Backend as Backend>::NativeBuffer>);
}
};
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rs
index df079d2eb..84d2f09fa 100644
--- a/crates/uzu/build/metal/ast.rs
+++ b/crates/uzu/build/metal/ast.rs
@@ -135,16 +135,19 @@ pub struct MetalArgument {
}
impl MetalArgument {
- fn scalar_type_to_rust(c_type: &str) -> anyhow::Result<&'static str> {
+ fn scalar_type_to_rust(c_type: &str) -> anyhow::Result<Box<str>> {
let mut tokens: Vec<_> = c_type.split_whitespace().collect();
if tokens.first() == Some(&"const") {
tokens.remove(0);
}
match tokens.as_slice() {
- ["bool"] => Ok("bool"),
- ["uint"] | ["uint32_t"] | ["unsigned", "int"] => Ok("u32"),
- ["int"] | ["int32_t"] => Ok("i32"),
- ["float"] => Ok("f32"),
+ ["bool"] => Ok("bool".into()),
+ ["uint"] | ["uint32_t"] | ["unsigned", "int"] => Ok("u32".into()),
+ ["int"] | ["int32_t"] => Ok("i32".into()),
+ ["float"] => Ok("f32".into()),
+ [vpath] if vpath.starts_with("uzu::") => Ok(vpath
+ .replacen("uzu::", "crate::backends::common::gpu_types::", 1)
+ .into()),
_ => bail!("unknown scalar type: {c_type}"),
}
}
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index b272f900d..7b86a977b 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -4,7 +4,7 @@ use anyhow::Context;
use itertools::Itertools;
use proc_macro2::TokenStream;
use quote::{format_ident, quote};
-use syn::LitInt;
+use syn::{LitInt, Type};
use crate::metal::ast::MetalGroupsType;
@@ -135,7 +135,7 @@ pub fn bindgen(
Some((def, set, quote! { #arg_name }))
}
MetalArgumentType::Constant((r_type, constant_type)) => {
- let arg_dtype = format_ident!("{r_type}");
+ let arg_dtype: Type = syn::parse_str(&r_type).unwrap();
let (def, set) = match constant_type {
MetalConstantType::Scalar => (
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rs
index 17a2e9782..e0b7b3aaa 100644
--- a/crates/uzu/src/backends/common/backend.rs
+++ b/crates/uzu/src/backends/common/backend.rs
@@ -1,12 +1,13 @@
-use std::error::Error;
+use std::{error::Error, fmt::Debug};
-use super::{CommandBuffer, Context, Kernels, NativeBuffer};
+use super::{CommandBuffer, Context, CopyEncoder, Kernels, NativeBuffer};
-pub trait Backend: Clone {
+pub trait Backend: Debug + Clone {
type Context: Context<Backend = Self>;
- type CommandBuffer: CommandBuffer<Backend = Self>;
type NativeBuffer: NativeBuffer<Backend = Self>;
- type EncoderRef;
+ type CommandBuffer: CommandBuffer<Backend = Self>;
+ type ComputeEncoder;
+ type CopyEncoder: CopyEncoder<Backend = Self>;
type Kernels: Kernels<Backend = Self>;
- type Error: Error;
+ type Error: Error + Debug;
}
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index 0315e12ad..db22f269a 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -3,8 +3,13 @@ use super::Backend;
pub trait CommandBuffer {
type Backend: Backend;
- fn with_encoder<T>(
+ fn with_compute_encoder<T>(
&self,
- callback: impl FnOnce(&<Self::Backend as Backend>::EncoderRef) -> T,
+ callback: impl FnOnce(&<Self::Backend as Backend>::ComputeEncoder) -> T,
+ ) -> T;
+
+ fn with_copy_encoder<T>(
+ &self,
+ callback: impl FnOnce(&<Self::Backend as Backend>::CopyEncoder) -> T,
) -> T;
}
diff --git a/crates/uzu/src/backends/common/copy_encoder.rs b/crates/uzu/src/backends/common/copy_encoder.rs
new file mode 100644
index 000000000..f68665a2f
--- /dev/null
+++ b/crates/uzu/src/backends/common/copy_encoder.rs
@@ -0,0 +1,12 @@
+use super::Backend;
+
+pub trait CopyEncoder {
+ type Backend: Backend;
+
+ fn encode_copy(
+ &self,
+ src: &<Self::Backend as Backend>::NativeBuffer,
+ dst: &<Self::Backend as Backend>::NativeBuffer,
+ size: usize,
+ );
+}
diff --git a/crates/uzu/src/backends/common/gpu_types/kv_cache_update.rs b/crates/uzu/src/backends/common/gpu_types/kv_cache_update.rs
new file mode 100644
index 000000000..193175e2b
--- /dev/null
+++ b/crates/uzu/src/backends/common/gpu_types/kv_cache_update.rs
@@ -0,0 +1,6 @@
+#[derive(Clone, Copy, Debug)]
+#[repr(C)]
+pub struct Swap {
+ pub source: u32,
+ pub destination: u32,
+}
diff --git a/crates/uzu/src/backends/common/gpu_types/mod.rs b/crates/uzu/src/backends/common/gpu_types/mod.rs
index 795fdc68b..a6ba1de5c 100644
--- a/crates/uzu/src/backends/common/gpu_types/mod.rs
+++ b/crates/uzu/src/backends/common/gpu_types/mod.rs
@@ -3,8 +3,10 @@
//! These `#[repr(C)]` structs are the source of truth. The build system uses
//! cbindgen to generate C headers for Metal shaders.
-mod attention;
-mod matmul;
+pub mod attention;
+pub mod kv_cache_update;
+pub mod matmul;
pub use attention::*;
+pub use kv_cache_update::*;
pub use matmul::*;
diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
new file mode 100644
index 000000000..ab6af972f
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -0,0 +1,163 @@
+use thiserror::Error;
+
+use crate::{
+ DataType,
+ backends::common::{
+ Backend, CommandBuffer, Kernels, gpu_types::Swap,
+ kernel::KVCacheUpdateKernel,
+ },
+};
+
+pub struct KVLayerData<B: Backend> {
+ pub key_buffer: B::NativeBuffer,
+ pub key_shape: [usize; 3],
+ pub value_buffer: B::NativeBuffer,
+ pub value_shape: [usize; 3],
+}
+
+pub struct KVCacheUpdate<B: Backend> {
+ kernel: <B::Kernels as Kernels>::KVCacheUpdateKernel,
+ max_sequence_length: usize,
+}
+
+#[derive(Debug, Error)]
+pub enum KVCacheUpdateError<B: Backend> {
+ #[error("Source and destination indices length mismatch")]
+ IndicesCountMismatch,
+ #[error("Shape mismatch between key and value tensors")]
+ ShapeMismatch,
+ #[error("Unsupported data type")]
+ UnsupportedDataType,
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Swap count exceeds maximum sequence length")]
+ MaxSequenceLengthExceeded,
+ #[error("Buffer creation failed")]
+ BufferCreationFailed,
+ #[error("Function not found: {0}")]
+ FunctionNotFound(String),
+}
+
+impl<B: Backend> KVCacheUpdate<B> {
+ pub fn new(
+ context: &B::Context,
+ data_type: DataType,
+ max_sequence_length: usize,
+ ) -> Result<Self, KVCacheUpdateError<B>> {
+ let kernel = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(
+ context, data_type,
+ )
+ .map_err(KVCacheUpdateError::BackendError)?;
+
+ Ok(Self {
+ kernel,
+ max_sequence_length,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ in_place_data: &[KVLayerData<B>],
+ source_indices: &[usize],
+ destination_indices: &[usize],
+ command_buffer: &B::CommandBuffer,
+ ) -> Result<(), KVCacheUpdateError<B>> {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.encode_with_encoder(
+ in_place_data,
+ source_indices,
+ destination_indices,
+ encoder,
+ )
+ })
+ }
+
+ /// Encode the KV cache update operation using a provided compute encoder
+ pub fn encode_with_encoder(
+ &self,
+ in_place_data: &[KVLayerData<B>],
+ source_indices: &[usize],
+ destination_indices: &[usize],
+ encoder: &B::ComputeEncoder,
+ ) -> Result<(), KVCacheUpdateError<B>> {
+ if source_indices.len() != destination_indices.len() {
+ return Err(KVCacheUpdateError::IndicesCountMismatch);
+ }
+
+ let swaps = create_swaps_direct(source_indices, destination_indices);
+ if swaps.len() > self.max_sequence_length {
+ return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);
+ }
+
+ // For small swap counts, use set_bytes to avoid shared buffer race in async mode.
+ // Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.
+ let use_inline_bytes = swaps.len() <= 32;
+ assert!(
+ use_inline_bytes,
+ "non-inline is not supported yet (and is broken anyways due to a data race)"
+ );
+
+ for layer_data in in_place_data {
+ if layer_data.key_shape != layer_data.value_shape {
+ return Err(KVCacheUpdateError::ShapeMismatch);
+ }
+
+ let [num_heads, max_sequence_length, head_dim] =
+ layer_data.key_shape;
+
+ self.kernel.encode(
+ &layer_data.key_buffer,
+ &layer_data.value_buffer,
+ &swaps,
+ swaps.len() as u32,
+ num_heads as u32,
+ max_sequence_length as u32,
+ head_dim as u32,
+ encoder,
+ );
+ }
+
+ Ok(())
+ }
+}
+
+pub fn create_swaps_direct(
+ source_indices: &[usize],
+ destination_indices: &[usize],
+) -> Vec<Swap> {
+ if source_indices.len() != destination_indices.len() {
+ return Vec::new();
+ }
+
+ let mut swaps = Vec::with_capacity(source_indices.len());
+
+ for (&src, &dst) in source_indices.iter().zip(destination_indices.iter()) {
+ if src != dst {
+ swaps.push(Swap {
+ source: src as u32,
+ destination: dst as u32,
+ });
+ }
+ }
+
+ swaps
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_direct_swaps() {
+ let sources = [0, 2, 4];
+ let destinations = [1, 3, 5];
+ let swaps = create_swaps_direct(&sources, &destinations);
+ assert_eq!(swaps.len(), 3);
+ assert_eq!(swaps[0].source, 0);
+ assert_eq!(swaps[0].destination, 1);
+ assert_eq!(swaps[1].source, 2);
+ assert_eq!(swaps[1].destination, 3);
+ assert_eq!(swaps[2].source, 4);
+ assert_eq!(swaps[2].destination, 5);
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rs
index dc38e9b24..fb7523be0 100644
--- a/crates/uzu/src/backends/common/kernel/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/mod.rs
@@ -2,4 +2,5 @@
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
+pub mod kv_cache_update;
pub mod sampling;
diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rs
index bb8737f48..5fc3cffb5 100644
--- a/crates/uzu/src/backends/common/kernel/sampling.rs
+++ b/crates/uzu/src/backends/common/kernel/sampling.rs
@@ -184,7 +184,7 @@ impl<B: Backend> SamplingKernel<B> {
sampling_method: SamplingMethod,
batch_size: usize,
vocab_size: usize,
- compute_encoder: &B::EncoderRef,
+ compute_encoder: &B::ComputeEncoder,
) -> Result<(), SamplingError<B>> {
if batch_size > self.max_batch_size {
return Err(SamplingError::BatchSizeExceeded(
diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rs
index 392acdf4c..2f561b239 100644
--- a/crates/uzu/src/backends/common/mod.rs
+++ b/crates/uzu/src/backends/common/mod.rs
@@ -4,6 +4,7 @@ mod buffer;
mod buffer_lifetime;
mod command_buffer;
mod context;
+mod copy_encoder;
pub mod gpu_types;
pub mod kernel;
mod native_buffer;
@@ -14,5 +15,6 @@ pub use buffer::Buffer;
pub use buffer_lifetime::BufferLifetime;
pub use command_buffer::CommandBuffer;
pub use context::Context;
+pub use copy_encoder::CopyEncoder;
pub use kernel::Kernels;
pub use native_buffer::NativeBuffer;
diff --git a/crates/uzu/src/backends/common/native_buffer.rs b/crates/uzu/src/backends/common/native_buffer.rs
index 80d09ae5c..9ea6e7ef6 100644
--- a/crates/uzu/src/backends/common/native_buffer.rs
+++ b/crates/uzu/src/backends/common/native_buffer.rs
@@ -1,8 +1,8 @@
-use std::{os::raw::c_void, ptr::NonNull};
+use std::{fmt::Debug, os::raw::c_void, ptr::NonNull};
use super::Backend;
-pub trait NativeBuffer: Send + Sync + Clone {
+pub trait NativeBuffer: Send + Sync + Debug + Clone {
type Backend: Backend;
fn set_label(
diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rs
index 9d98fdc4a..763324f34 100644
--- a/crates/uzu/src/backends/metal/backend.rs
+++ b/crates/uzu/src/backends/metal/backend.rs
@@ -1,4 +1,7 @@
-use metal::{MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder};
+use metal::{
+ MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,
+ MTLComputeCommandEncoder,
+};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{MTLContext, MTLError, MetalKernels};
@@ -8,10 +11,11 @@ use crate::backends::common::Backend;
pub struct Metal;
impl Backend for Metal {
- type NativeBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;
type Context = MTLContext;
+ type NativeBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;
type CommandBuffer = Retained<ProtocolObject<dyn MTLCommandBuffer>>;
- type EncoderRef = ProtocolObject<dyn MTLComputeCommandEncoder>;
+ type ComputeEncoder = ProtocolObject<dyn MTLComputeCommandEncoder>;
+ type CopyEncoder = ProtocolObject<dyn MTLBlitCommandEncoder>;
type Kernels = MetalKernels;
type Error = MTLError;
}
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rs
index 01bcef66b..f1ccb6e94 100644
--- a/crates/uzu/src/backends/metal/classifier_context.rs
+++ b/crates/uzu/src/backends/metal/classifier_context.rs
@@ -10,7 +10,6 @@ use super::{
Pooling, Rope,
transformer_layer::{embed_block, linear_block},
},
- forward_pass::{ArrayId, EncodableBlock, RopeType},
kernel::dsl::SigmoidMetalKernel,
};
use crate::{
@@ -20,9 +19,11 @@ use crate::{
metal::error::ClassifierError,
},
config::{ClassifierModelConfig, ModelMetadata},
+ encodable_block::EncodableBlock,
forward_pass::{
- model_shape::ModelShape, scratch_buffers::ScratchBuffers,
- state::SharedBuffers,
+ model_shape::ModelShape,
+ scratch_buffers::ScratchBuffers,
+ state::{ArrayId, RopeType, SharedBuffers},
},
parameters::ParameterLoader,
session::types::Error,
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index b7cfd391a..5741b68a9 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -8,10 +8,10 @@ use super::Metal;
impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
type Backend = Metal;
- fn with_encoder<T>(
+ fn with_compute_encoder<T>(
&self,
callback: impl FnOnce(
- &<Self::Backend as crate::backends::common::Backend>::EncoderRef,
+ &<Self::Backend as crate::backends::common::Backend>::ComputeEncoder,
) -> T,
) -> T {
let encoder = self
@@ -24,4 +24,21 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
ret
}
+
+ fn with_copy_encoder<T>(
+ &self,
+ callback: impl FnOnce(
+ &<Self::Backend as crate::backends::common::Backend>::CopyEncoder,
+ ) -> T,
+ ) -> T {
+ let encoder = self
+ .new_blit_command_encoder()
+ .expect("Failed to create blit command encoder");
+
+ let ret = callback(&encoder);
+
+ encoder.end_encoding();
+
+ ret
+ }
}
diff --git a/crates/uzu/src/backends/metal/copy_encoder.rs b/crates/uzu/src/backends/metal/copy_encoder.rs
new file mode 100644
index 000000000..80394899b
--- /dev/null
+++ b/crates/uzu/src/backends/metal/copy_encoder.rs
@@ -0,0 +1,21 @@
+use metal::{MTLBlitCommandEncoder, MTLBuffer};
+use objc2::runtime::ProtocolObject;
+
+use crate::backends::common::CopyEncoder;
+
+use super::Metal;
+
+impl CopyEncoder for ProtocolObject<dyn MTLBlitCommandEncoder> {
+ type Backend = Metal;
+
+ fn encode_copy(
+ &self,
+ src: &<Self::Backend as crate::backends::common::Backend>::NativeBuffer,
+ dst: &<Self::Backend as crate::backends::common::Backend>::NativeBuffer,
+ size: usize,
+ ) {
+ assert!(src.length() <= size && dst.length() <= size);
+
+ self.copy_buffer_to_buffer(src, 0, dst, 0, size);
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs
index 78bd80f67..d984b3959 100644
--- a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs
@@ -4,15 +4,15 @@ use crate::backends::metal::{
};
use super::{
- super::{
- MTLContext, MTLError,
- forward_pass::{ArrayId, ForwardPassState},
- kernel::dsl::ActivationMetalKernel,
- },
- EncodableBlock, EncodingParameters,
+ super::{MTLContext, MTLError, kernel::dsl::ActivationMetalKernel},
+ EncodableBlock,
};
-use crate::backends::common::kernel::ActivationKernel;
use crate::{DataType, config::Activation as ActivationConfig};
+use crate::{
+ backends::common::kernel::ActivationKernel,
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
+};
pub struct Activation {
kernel: ActivationMetalKernel,
@@ -42,9 +42,9 @@ impl Activation {
impl EncodableBlock<Metal> for Activation {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let encoder = command_buffer
.new_compute_command_encoder()
@@ -61,9 +61,9 @@ impl EncodableBlock<Metal> for Activation {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let input_array = arrays[0].borrow_mut();
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rs
index 8ecff4b22..8b2479303 100644
--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs
@@ -1,16 +1,17 @@
//! Attention kernel encodable.
-use super::{EncodableBlock, EncodingParameters, Metal};
+use super::{EncodableBlock, Metal};
use crate::backends::metal::{
KernelDataType, MTLCommandBuffer, MTLCommandEncoder,
MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained,
- forward_pass::{ArrayId, ForwardPassState, HashMapId},
kernel::attention::{
AttentionError, AttentionGemmArguments, AttentionKernel,
AttentionKernelVariant, AttentionSinglePassArguments,
AttentionTwoPassArguments, KVCacheUpdateArguments,
},
};
+use crate::encodable_block::EncodingParameters;
+use crate::forward_pass::state::{ArrayId, ForwardPassState, HashMapId};
fn env_gemm_attention_enabled() -> bool {
static VALUE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
@@ -62,9 +63,9 @@ impl Attention {
impl EncodableBlock<Metal> for Attention {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let compute_encoder = command_buffer
.new_compute_command_encoder()
@@ -84,9 +85,9 @@ impl EncodableBlock<Metal> for Attention {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let (
suffix_length,
@@ -317,7 +318,7 @@ impl EncodableBlock<Metal> for Attention {
match variant {
AttentionKernelVariant::SinglePass => {
if use_gemm {
- let mtl = &**state.mtl_context();
+ let mtl = state.mtl_context();
if let Err(e) = self.kernel.encode_gemm(
mtl,
compute_encoder,
diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
index b697968c4..a258bd31a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
@@ -9,16 +9,18 @@ use crate::backends::metal::{
use objc2::rc::autoreleasepool;
use super::{
- Attention, EncodableBlock, EncodingParameters, ForwardPassState,
- Normalization, QKNorm, TensorAddSwap, TensorCopy, transformer_layer,
+ Attention, EncodableBlock, Normalization, QKNorm, TensorAddSwap,
+ TensorCopy, transformer_layer,
};
+use crate::forward_pass::state::{ArrayId, ForwardPassState};
use crate::{
DataType,
backends::metal::{
MTLContext, compilation_parameters::CompilationConfig,
- forward_pass::ArrayId, kernel::KernelDataType,
+ kernel::KernelDataType,
},
config::TransformerLayerConfig,
+ encodable_block::EncodingParameters,
parameters::ParameterTree,
};
@@ -283,9 +285,9 @@ impl ClassifierLayer {
impl EncodableBlock<Metal> for ClassifierLayer {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
#[cfg(not(feature = "tracing"))]
{
@@ -475,9 +477,9 @@ impl EncodableBlock<Metal> for ClassifierLayer {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
debug_assert!(
self.supports_shared_encoder(),
diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
index 4f14bc79b..d56b09207 100644
--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
@@ -3,9 +3,7 @@
use crate::backends::metal::Metal;
use std::rc::Rc;
-use super::{
- EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,
-};
+use super::{EncodableBlock, LayerExecutables, RMSNorm, Rope};
use crate::{
DataType, DecoderConfig,
backends::metal::{
@@ -13,10 +11,13 @@ use crate::{
ProtocolObject, Retained,
compilation_parameters::CompilationConfig,
encodable_block::transformer_layer::{embed_block, readout_block},
- forward_pass::{ArrayId, ForwardPassState, RopeType},
},
config::{DecoderLayerType, MixerConfig},
- forward_pass::model_shape::ModelShape,
+ encodable_block::EncodingParameters,
+ forward_pass::{
+ model_shape::ModelShape,
+ state::{ArrayId, ForwardPassState, RopeType},
+ },
parameters::ParameterTree,
};
@@ -202,18 +203,18 @@ impl Decoder {
impl EncodableBlock<Metal> for Decoder {
fn encode_with_shared_encoder(
&self,
- _state: &mut ForwardPassState,
+ _state: &mut ForwardPassState<Metal>,
_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
unimplemented!("Decoder does not support shared encoder")
}
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
self.embed.encode(state, command_buffer, parameters);
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
index 6728697e8..1085df36a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
@@ -1,5 +1,5 @@
use super::{
- super::{EncodableBlock, EncodingParameters, Metal},
+ super::{EncodableBlock, Metal},
EmbeddingError,
};
use crate::{
@@ -9,11 +9,11 @@ use crate::{
metal::{
MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
- Retained,
- forward_pass::{ArrayId, ForwardPassState},
- kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,
+ Retained, kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,
},
},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -86,9 +86,9 @@ impl FullPrecisionEmbeddingLookup {
impl EncodableBlock<Metal> for FullPrecisionEmbeddingLookup {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let encoder = command_buffer
.new_compute_command_encoder()
@@ -108,9 +108,9 @@ impl EncodableBlock<Metal> for FullPrecisionEmbeddingLookup {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
let batch_size = state.active_suffix_length();
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
index 4fe565361..b951ac91f 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
@@ -6,16 +6,17 @@ use crate::backends::metal::{
};
use super::{
- super::{EncodableBlock, EncodingParameters, Metal},
+ super::{EncodableBlock, Metal},
EmbeddingError,
};
use crate::{
DataType,
backends::metal::{
MTLContext, MTLError,
- forward_pass::{ArrayId, ForwardPassState},
kernel::matmul::{MatmulArguments, MatmulKernel},
},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -88,9 +89,9 @@ impl FullPrecisionEmbeddingReadout {
impl EncodableBlock<Metal> for FullPrecisionEmbeddingReadout {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let encoder = command_buffer
.new_compute_command_encoder()
@@ -110,9 +111,9 @@ impl EncodableBlock<Metal> for FullPrecisionEmbeddingReadout {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
let batch_size = state.sampling_length();
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
index 1fb215cb0..a8bd084f6 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
@@ -1,5 +1,5 @@
use super::{
- super::{EncodableBlock, EncodingParameters, Metal},
+ super::{EncodableBlock, Metal},
EmbeddingError,
};
use crate::{
@@ -9,12 +9,12 @@ use crate::{
metal::{
MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
- Retained,
- forward_pass::{ArrayId, ForwardPassState},
- kernel::dsl::QuantizedEmbeddingLookupMetalKernel,
+ Retained, kernel::dsl::QuantizedEmbeddingLookupMetalKernel,
},
},
config::QuantizationMode,
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -221,9 +221,9 @@ impl QuantizedEmbeddingLookup {
impl EncodableBlock<Metal> for QuantizedEmbeddingLookup {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let encoder = command_buffer
.new_compute_command_encoder()
@@ -243,9 +243,9 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingLookup {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
let batch_size = state.active_suffix_length();
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
index 576018b03..bde63ffbf 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
@@ -1,5 +1,5 @@
use super::{
- super::{EncodableBlock, EncodingParameters, Metal},
+ super::{EncodableBlock, Metal},
EmbeddingError,
};
use crate::{
@@ -10,7 +10,6 @@ use crate::{
MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
Retained,
- forward_pass::{ArrayId, ForwardPassState},
kernel::quant_matmul::{
QuantizationType, QuantizedMatmulArguments,
QuantizedMatmulKernel,
@@ -18,6 +17,8 @@ use crate::{
},
},
config::QuantizationMode,
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -219,9 +220,9 @@ impl QuantizedEmbeddingReadout {
impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let encoder = command_buffer
.new_compute_command_encoder()
@@ -241,9 +242,9 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
let batch_size = state.sampling_length();
diff --git a/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rs b/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rs
deleted file mode 100644
index 1b28f7704..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-//! Encoding parameters for forward pass operations.
-
-use crate::backends::metal::Metal;
-
-pub type EncodingParameters<'a> =
- crate::encodable_block::EncodingParameters<'a, Metal>;
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
index 7bb5a9e15..5f86335d5 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
@@ -12,20 +12,20 @@ use objc2::rc::autoreleasepool;
use super::{
super::{
- Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm,
- RMSNorm, ShortConvMixer, TensorAddSwap, TensorCopy, transformer_layer,
+ Attention, EncodableBlock, MambaMixer, QKNorm, RMSNorm, ShortConvMixer,
+ TensorAddSwap, TensorCopy, transformer_layer,
},
MixerExecutables,
};
use crate::{
DataType, DecoderLayerConfig,
backends::metal::{
- MTLContext,
- compilation_parameters::CompilationConfig,
- forward_pass::{ArrayId, ForwardPassState},
+ MTLContext, compilation_parameters::CompilationConfig,
kernel::KernelDataType,
},
config::{DecoderLayerType, MixerConfig},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -311,9 +311,9 @@ impl LayerExecutables {
impl EncodableBlock<Metal> for LayerExecutables {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
// In non-tracing builds, if every sub-block supports shared encoding,
// we can run the entire layer in a single compute encoder.
@@ -538,9 +538,9 @@ impl EncodableBlock<Metal> for LayerExecutables {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
debug_assert!(
self.supports_shared_encoder(),
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
index 32d7279e4..490544054 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
@@ -7,10 +7,11 @@ use crate::{
MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
Retained,
- encodable_block::{EncodableBlock, EncodingParameters},
- forward_pass::{ArrayId, ForwardPassState},
+ encodable_block::EncodableBlock,
kernel::matmul::{MatmulArguments, MatmulKernel},
},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -106,9 +107,9 @@ impl FullPrecisionLinear {
impl EncodableBlock<Metal> for FullPrecisionLinear {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
@@ -162,9 +163,9 @@ impl EncodableBlock<Metal> for FullPrecisionLinear {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
index 77adc96ff..357700ced 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
@@ -12,8 +12,7 @@ use crate::{
DataType,
backends::metal::{
MTLContext, MTLError,
- encodable_block::{EncodableBlock, EncodingParameters},
- forward_pass::{ArrayId, ForwardPassState},
+ encodable_block::EncodableBlock,
kernel::{
dsl::TensorAddBiasMetalKernel,
quant_matmul::{
@@ -23,6 +22,8 @@ use crate::{
},
},
config::QuantizationConfig,
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -229,9 +230,9 @@ impl QuantizedLinear {
impl EncodableBlock<Metal> for QuantizedLinear {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
@@ -292,9 +293,9 @@ impl EncodableBlock<Metal> for QuantizedLinear {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
index 99e47ffd9..f7c110a2c 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
@@ -1,6 +1,6 @@
//! Mamba2 SSM mixer encodable.
-use super::{EncodableBlock, EncodingParameters, Metal, transformer_layer};
+use super::{EncodableBlock, Metal, transformer_layer};
use crate::backends::metal::kernel::ssm::ssd_prefill::{
SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode,
};
@@ -13,7 +13,6 @@ use crate::{
MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,
Retained,
compilation_parameters::CompilationConfig,
- forward_pass::{ArrayId, ForwardPassState},
kernel::dsl::{SSDUpdateMetalKernel, SplitInProjMetalKernel},
kernel::ssm::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
@@ -22,6 +21,8 @@ use crate::{
},
},
config::{DecoderLayerType, Mamba2Config},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
use std::{env, rc::Rc};
@@ -149,9 +150,9 @@ impl MambaMixer {
fn encode_pipeline_with_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
@@ -175,7 +176,7 @@ impl MambaMixer {
fn run_split_inproj(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
@@ -219,7 +220,7 @@ impl MambaMixer {
fn run_conv_scan(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
@@ -335,7 +336,7 @@ impl MambaMixer {
fn run_prefill_ssm(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
@@ -413,7 +414,7 @@ impl MambaMixer {
fn run_decode_ssm(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
@@ -501,9 +502,9 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {
impl EncodableBlock<Metal> for MambaMixer {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
let encoder = command_buffer
@@ -553,9 +554,9 @@ impl EncodableBlock<Metal> for MambaMixer {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
self.encode_pipeline_with_encoder(state, encoder, parameters);
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
index b32a00eec..6eb7f499f 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
@@ -7,12 +7,11 @@ use crate::backends::metal::{
ProtocolObject, Retained,
};
-use super::{EncodableBlock, EncodingParameters, Metal};
+use super::{EncodableBlock, Metal};
use crate::{
DataType,
backends::metal::{
MTLContext,
- forward_pass::{ArrayId, ForwardPassState},
kernel::{
mlp::{MlpActivationType, MlpGateActMulEncodable},
mlp_fused::{MlpFusedArguments, MlpFusedKernel},
@@ -23,6 +22,8 @@ use crate::{
},
},
config::Activation,
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
};
pub struct MlpBlock {
@@ -48,9 +49,9 @@ impl MlpBlock {
impl EncodableBlock<Metal> for MlpBlock {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- params: &EncodingParameters,
+ params: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
let encoder = command_buffer
@@ -97,9 +98,9 @@ impl EncodableBlock<Metal> for MlpBlock {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- params: &EncodingParameters,
+ params: &EncodingParameters<Metal>,
) {
// Up
self.up.encode_with_shared_encoder(state, encoder, params);
@@ -317,9 +318,9 @@ impl MlpFusedBlock {
impl EncodableBlock<Metal> for MlpFusedBlock {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- params: &EncodingParameters,
+ params: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
let encoder = command_buffer
@@ -361,9 +362,9 @@ impl EncodableBlock<Metal> for MlpFusedBlock {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- params: &EncodingParameters,
+ params: &EncodingParameters<Metal>,
) {
// Fused up + activation
let arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 26e150c67..fc7339204 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -7,7 +7,6 @@
//! kernels to perform operations on `ForwardPassState`.
pub use super::Metal;
-use super::forward_pass::ForwardPassState;
pub use crate::encodable_block::EncodableBlock;
mod activation;
@@ -15,7 +14,6 @@ mod attention;
mod classifier_layer;
mod decoder;
mod embedding;
-mod encoding_parameters;
mod layer;
mod linear;
mod mamba_mixer;
@@ -39,7 +37,6 @@ pub use embedding::{
FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,
QuantizedEmbeddingReadout,
};
-pub use encoding_parameters::EncodingParameters;
pub use layer::LayerExecutables;
pub use linear::{FullPrecisionLinear, QuantizedLinear};
pub(crate) use mamba_mixer::MambaMixer;
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
index e2bc760a1..21053540e 100644
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
@@ -3,7 +3,7 @@
use std::{cell::RefCell, rc::Rc};
use super::{
- super::{EncodableBlock, EncodingParameters, Metal},
+ super::{EncodableBlock, Metal},
SharedMoeWeights,
};
use crate::{
@@ -15,7 +15,6 @@ use crate::{
KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,
MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext,
MetalArray, NSRange, ProtocolObject, Retained,
- forward_pass::{ArrayId, ForwardPassState},
kernel::{
MoeGatherKernels,
dsl::{
@@ -32,6 +31,8 @@ use crate::{
},
},
},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -300,9 +301,9 @@ impl MoeBlock {
impl EncodableBlock<Metal> for MoeBlock {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let suffix_length = state.active_suffix_length();
let arrays = state.arrays(&[
@@ -644,9 +645,9 @@ impl EncodableBlock<Metal> for MoeBlock {
fn encode_with_shared_encoder(
&self,
- _state: &mut ForwardPassState,
+ _state: &mut ForwardPassState<Metal>,
_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
unreachable!("MoeBlock does not support shared compute encoder");
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
index d88dabdee..b4fe040d8 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
@@ -1,6 +1,6 @@
//! LayerNorm encodable.
-use super::super::{EncodableBlock, EncodingParameters, Metal};
+use super::super::{EncodableBlock, Metal};
use crate::{
DataType,
backends::{
@@ -9,11 +9,12 @@ use crate::{
MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
MTLResourceOptions, ProtocolObject, Retained,
- forward_pass::{ArrayId, ForwardPassState},
kernel::dsl::LayerNormMetalKernel,
},
},
config::{NormalizationConfig, UpcastMode},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -77,9 +78,9 @@ impl LayerNorm {
impl EncodableBlock<Metal> for LayerNorm {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let compute_encoder = command_buffer
.new_compute_command_encoder()
@@ -99,9 +100,9 @@ impl EncodableBlock<Metal> for LayerNorm {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let input_binding = state.arrays(&[self.input_array_id]);
let output_binding = state.arrays(&[self.output_array_id]);
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
index e892320d8..e7b65f5ea 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
@@ -8,17 +8,17 @@ pub use layer_norm::LayerNorm;
pub use qk_norm::QKNorm;
pub use rms_norm::RMSNorm;
-use super::{EncodableBlock, EncodingParameters, Metal};
+use super::{EncodableBlock, Metal};
use crate::backends::metal::MTLError;
use crate::{
DataType,
backends::metal::{
MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, ProtocolObject,
- Retained,
- forward_pass::{ArrayId, ForwardPassState},
- kernel::RMSNormError,
+ Retained, kernel::RMSNormError,
},
config::NormalizationConfig,
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -75,9 +75,9 @@ impl Normalization {
impl EncodableBlock<Metal> for Normalization {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
match self {
Normalization::LayerNorm(layer_norm) => {
@@ -102,9 +102,9 @@ impl EncodableBlock<Metal> for Normalization {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
match self {
Normalization::LayerNorm(layer_norm) => layer_norm
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
index a9694ee31..2eedae5de 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
@@ -1,19 +1,20 @@
//! QK Normalization encodable.
-use super::super::{EncodableBlock, EncodingParameters, Metal};
+use super::super::{EncodableBlock, Metal};
use crate::{
DataType,
backends::metal::{
MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
MTLResourceOptions, ProtocolObject, Retained,
- forward_pass::{ArrayId, ForwardPassState},
kernel::rms_norm::{
QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel,
RMSNormKernelType,
},
},
config::{NormalizationConfig, UpcastMode},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -157,9 +158,9 @@ impl QKNorm {
impl EncodableBlock<Metal> for QKNorm {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let compute_encoder = command_buffer
.new_compute_command_encoder()
@@ -179,9 +180,9 @@ impl EncodableBlock<Metal> for QKNorm {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let qkv_binding = state.arrays(&[self.qkv_array_id]);
let qkv_shape = {
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
index 4ec1da37c..4644e1f4a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
@@ -1,18 +1,19 @@
//! RMS Normalization encodable.
-use super::super::{EncodableBlock, EncodingParameters, Metal};
+use super::super::{EncodableBlock, Metal};
use crate::{
DataType,
backends::metal::{
MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
MTLResourceOptions, ProtocolObject, Retained,
- forward_pass::{ArrayId, ForwardPassState},
kernel::rms_norm::{
RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType,
},
},
config::{NormalizationConfig, UpcastMode},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -100,9 +101,9 @@ impl RMSNorm {
impl EncodableBlock<Metal> for RMSNorm {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let compute_encoder = command_buffer
.new_compute_command_encoder()
@@ -122,9 +123,9 @@ impl EncodableBlock<Metal> for RMSNorm {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let input_binding = state.arrays(&[self.input_array_id]);
let output_binding = state.arrays(&[self.output_array_id]);
diff --git a/crates/uzu/src/backends/metal/encodable_block/pooling.rs b/crates/uzu/src/backends/metal/encodable_block/pooling.rs
index 9899cc8d2..0fe17f7b2 100644
--- a/crates/uzu/src/backends/metal/encodable_block/pooling.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/pooling.rs
@@ -1,6 +1,6 @@
//! Pooling encodable for sequence-level aggregation.
-use super::{EncodableBlock, EncodingParameters, Metal};
+use super::{EncodableBlock, Metal};
#[cfg(feature = "tracing")]
use crate::backends::metal::MTLBlitCommandEncoder;
use crate::{
@@ -10,11 +10,12 @@ use crate::{
KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
Retained,
- forward_pass::{ArrayId, ForwardPassState},
kernel::dsl::{PoolingClsMetalKernel, PoolingMeanMetalKernel},
},
},
config::PoolingType,
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
};
enum PoolingKernel {
@@ -74,9 +75,9 @@ impl Pooling {
impl EncodableBlock<Metal> for Pooling {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let encoder = command_buffer
.new_compute_command_encoder()
@@ -124,9 +125,9 @@ impl EncodableBlock<Metal> for Pooling {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let batch_size = 1;
let seq_len = state.aux_buffers_suffix_length();
diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
index b47b01a87..efddba52a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
@@ -1,12 +1,16 @@
//! Prediction head encodable for classification output.
-use super::{EncodableBlock, EncodingParameters, Metal};
+use super::{EncodableBlock, Metal};
#[cfg(feature = "tracing")]
-use crate::backends::metal::{MTLBlitCommandEncoder, forward_pass::ArrayId};
+use crate::backends::metal::MTLBlitCommandEncoder;
use crate::backends::metal::{
MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
- ProtocolObject, Retained, forward_pass::ForwardPassState,
+ ProtocolObject, Retained,
};
+use crate::encodable_block::EncodingParameters;
+#[cfg(feature = "tracing")]
+use crate::forward_pass::state::ArrayId;
+use crate::forward_pass::state::ForwardPassState;
pub struct ClassifierPredictionHead {
dense: Box<dyn EncodableBlock<Metal>>,
@@ -38,9 +42,9 @@ impl ClassifierPredictionHead {
impl EncodableBlock<Metal> for ClassifierPredictionHead {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
let encoder = command_buffer
@@ -100,9 +104,9 @@ impl EncodableBlock<Metal> for ClassifierPredictionHead {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
self.dense.encode_with_shared_encoder(state, encoder, parameters);
self.activation.encode_with_shared_encoder(state, encoder, parameters);
diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rs
index d3b6e847a..d4671fb7a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs
@@ -5,12 +5,13 @@ use crate::backends::metal::{
ProtocolObject, Retained,
};
-use super::{EncodableBlock, EncodingParameters, Metal};
+use super::{EncodableBlock, Metal};
use crate::backends::metal::{
KernelDataType, MTLContext,
- forward_pass::{ArrayId, ForwardPassState, RopeType},
kernel::rope::{RopeError, RopeKernel, RopeKernelArguments},
};
+use crate::encodable_block::EncodingParameters;
+use crate::forward_pass::state::{ArrayId, ForwardPassState, RopeType};
pub struct Rope {
kernel: RopeKernel,
@@ -34,9 +35,9 @@ impl Rope {
impl EncodableBlock<Metal> for Rope {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let compute_encoder = command_buffer
.new_compute_command_encoder()
@@ -56,9 +57,9 @@ impl EncodableBlock<Metal> for Rope {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
index 65c524b26..be4122178 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -1,6 +1,6 @@
use std::rc::Rc;
-use super::{EncodableBlock, EncodingParameters, Metal, transformer_layer};
+use super::{EncodableBlock, Metal, transformer_layer};
use crate::{
DataType,
backends::{
@@ -10,7 +10,6 @@ use crate::{
MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,
Retained,
compilation_parameters::CompilationConfig,
- forward_pass::{ArrayId, ForwardPassState},
kernel::short_conv::{
ShortConvDecodeArguments, ShortConvKernel,
ShortConvPackArguments, ShortConvPrefillArguments,
@@ -18,6 +17,8 @@ use crate::{
},
},
config::{DecoderLayerType, ShortConvConfig},
+ encodable_block::EncodingParameters,
+ forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -120,9 +121,9 @@ impl ShortConvMixer {
fn encode_pipeline(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
@@ -151,9 +152,9 @@ impl ShortConvMixer {
fn encode_pipeline_with_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
@@ -175,7 +176,7 @@ impl ShortConvMixer {
fn run_prefill_conv(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
@@ -249,7 +250,7 @@ impl ShortConvMixer {
fn run_decode_conv(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,
suffix_length: usize,
) {
@@ -300,9 +301,9 @@ impl ShortConvMixer {
impl EncodableBlock<Metal> for ShortConvMixer {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
if self.supports_shared_encoder() {
let encoder = command_buffer
@@ -327,9 +328,9 @@ impl EncodableBlock<Metal> for ShortConvMixer {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
self.encode_pipeline_with_encoder(state, encoder, parameters);
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
index 524876baa..4a8abc476 100644
--- a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
@@ -8,12 +8,12 @@ use crate::backends::{
},
};
-use super::{EncodableBlock, EncodingParameters, Metal};
+use super::{EncodableBlock, Metal};
use crate::backends::metal::{
- MTLContext, MTLError,
- forward_pass::{ArrayId, ForwardPassState},
- kernel::dsl::TensorAddSwapMetalKernel,
+ MTLContext, MTLError, kernel::dsl::TensorAddSwapMetalKernel,
};
+use crate::encodable_block::EncodingParameters;
+use crate::forward_pass::state::{ArrayId, ForwardPassState};
pub struct TensorAddSwap {
kernel: TensorAddSwapMetalKernel,
@@ -37,9 +37,9 @@ impl TensorAddSwap {
impl EncodableBlock<Metal> for TensorAddSwap {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let encoder = command_buffer
.new_compute_command_encoder()
@@ -59,9 +59,9 @@ impl EncodableBlock<Metal> for TensorAddSwap {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&self.argument_arrays);
assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");
diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
index 301dcfec7..916b5a1e6 100644
--- a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
@@ -8,12 +8,12 @@ use crate::backends::{
},
};
-use super::{EncodableBlock, EncodingParameters, Metal};
+use super::{EncodableBlock, Metal};
use crate::backends::metal::{
- MTLContext, MTLError,
- forward_pass::{ArrayId, ForwardPassState},
- kernel::dsl::TensorCopyMetalKernel,
+ MTLContext, MTLError, kernel::dsl::TensorCopyMetalKernel,
};
+use crate::encodable_block::EncodingParameters;
+use crate::forward_pass::state::{ArrayId, ForwardPassState};
pub struct TensorCopy {
kernel: TensorCopyMetalKernel,
@@ -37,9 +37,9 @@ impl TensorCopy {
impl EncodableBlock<Metal> for TensorCopy {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- parameters: &EncodingParameters,
+ parameters: &EncodingParameters<Metal>,
) {
let encoder = command_buffer
.new_compute_command_encoder()
@@ -59,9 +59,9 @@ impl EncodableBlock<Metal> for TensorCopy {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let arrays = state.arrays(&self.argument_arrays);
assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 9c32dda6d..82d938c94 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -11,10 +11,10 @@ use crate::{
DataType,
backends::metal::{
MTLContext, MTLError,
- forward_pass::ArrayId,
kernel::{mlp::MlpGateActMulEncodable, quant_matmul::QuantizationType},
},
config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
+ forward_pass::state::ArrayId,
parameters::ParameterTree,
};
diff --git a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
deleted file mode 100644
index 90854cadb..000000000
--- a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-use super::ForwardPassState;
-use crate::backends::metal::{
- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, Metal,
- ProtocolObject, Retained,
- encodable_block::{EncodableBlock, EncodingParameters},
-};
-
-pub struct EncoderResolver<'a> {
- command_buffer: &'a Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- encoder: Option<Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>>,
-}
-
-impl<'a> EncoderResolver<'a> {
- pub fn new(
- command_buffer: &'a Retained<ProtocolObject<dyn MTLCommandBuffer>>
- ) -> Self {
- Self {
- command_buffer,
- encoder: None,
- }
- }
-
- pub fn encode(
- &mut self,
- block: &dyn EncodableBlock<Metal>,
- state: &mut ForwardPassState,
- parameters: &EncodingParameters,
- ) {
- if block.supports_shared_encoder() {
- if self.encoder.is_none() {
- self.encoder = Some(
- self.command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder"),
- );
- }
- block.encode_with_shared_encoder(
- state,
- self.encoder.as_ref().unwrap(),
- parameters,
- );
- } else {
- self.end_current_encoder();
- block.encode(state, self.command_buffer, parameters);
- }
- }
-
- pub fn end_current_encoder(&mut self) {
- if let Some(encoder) = self.encoder.take() {
- encoder.end_encoding();
- }
- }
-
- pub fn finish(mut self) {
- self.end_current_encoder();
- }
-}
diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rs
deleted file mode 100644
index c4f610d07..000000000
--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-pub mod cache_layers;
-mod encoder_resolver;
-pub mod kv_cache_layer;
-pub mod short_conv_layer;
-pub mod ssm_layer;
-pub mod state;
-#[cfg(feature = "tracing")]
-pub mod traces;
-
-pub use cache_layers::{CacheLayer, CacheLayers};
-pub use encoder_resolver::EncoderResolver;
-pub use kv_cache_layer::{
- AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,
-};
-pub use short_conv_layer::ShortConvLayer;
-pub use ssm_layer::SSMLayer;
-pub use crate::forward_pass::state::{ArrayId, HashMapId, RopeType};
-pub use state::{
- ArrayCell, ClassifierModeState, ForwardPassMode, ForwardPassState,
- LanguageModelGeneratorModeState,
-};
-
-pub use super::encodable_block::{EncodableBlock, EncodingParameters};
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mode.rs b/crates/uzu/src/backends/metal/forward_pass/state/mode.rs
deleted file mode 100644
index 1826f1189..000000000
--- a/crates/uzu/src/backends/metal/forward_pass/state/mode.rs
+++ /dev/null
@@ -1,36 +0,0 @@
-use std::{cell::RefCell, rc::Rc};
-
-use super::super::cache_layers::CacheLayers;
-#[cfg(feature = "tracing")]
-use super::super::traces::ActivationTrace;
-use crate::{backends::metal::MetalArray, session::parameter::SamplingMethod};
-
-type ArrayCell = RefCell<MetalArray>;
-
-pub enum ForwardPassMode {
- LanguageModelGenerator(LanguageModelGeneratorModeState),
- Classifier(ClassifierModeState),
-}
-
-pub struct LanguageModelGeneratorModeState {
- pub cache_layers: Rc<RefCell<CacheLayers>>,
- pub token_seeds: ArrayCell,
- pub logits: ArrayCell,
- pub sampling_output: Option<ArrayCell>,
- pub sampling_method: Option<SamplingMethod>,
- #[cfg(feature = "tracing")]
- pub traces: Rc<RefCell<ActivationTrace>>,
- pub active_suffix_length: usize,
- pub sampling_start: usize,
- pub sampling_length: usize,
- pub is_prefilling: bool,
-}
-
-pub struct ClassifierModeState {
- pub pooling: ArrayCell,
- pub dense: ArrayCell,
- pub norm: ArrayCell,
- pub classifier_logits: ArrayCell,
- #[cfg(feature = "tracing")]
- pub traces: Rc<RefCell<ActivationTrace>>,
-}
diff --git a/crates/uzu/src/backends/metal/generated/kv_cache_update.h b/crates/uzu/src/backends/metal/generated/kv_cache_update.h
new file mode 100644
index 000000000..fe32897f8
--- /dev/null
+++ b/crates/uzu/src/backends/metal/generated/kv_cache_update.h
@@ -0,0 +1,27 @@
+// Auto-generated from gpu_types/kv_cache_update.rs - do not edit manually
+#pragma once
+
+#ifndef UZU_KV_CACHE_UPDATE_H
+#define UZU_KV_CACHE_UPDATE_H
+
+#ifdef __METAL_VERSION__
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu {
+namespace kv_cache_update {
+#else
+#include <stdint.h>
+#endif
+
+typedef struct {
+ uint32_t source;
+ uint32_t destination;
+} Swap;
+
+#ifdef __METAL_VERSION__
+} // namespace kv_cache_update
+} // namespace uzu
+#endif
+
+#endif // UZU_KV_CACHE_UPDATE_H
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
index 855d5e1d1..84564431c 100644
--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
@@ -1,58 +1,36 @@
#include <metal_stdlib>
#include "../definitions.metal"
+#include "kv_cache_update.h"
+
template <typename T>
-void swap(device T* buffer, const int sourceIdx, const int destIdx) {
+void swap(device T* buffer, const uint sourceIdx, const uint destIdx) {
const T temp = buffer[sourceIdx];
buffer[sourceIdx] = buffer[destIdx];
buffer[destIdx] = temp;
}
template <typename T>
-void updateKVCache(
- device T* inPlaceKeys,
- device T* inPlaceValues,
- const constant int2* swaps,
- const constant int& swapCount,
- const constant int& numHeads,
- const constant int& maxSequenceLength,
- const constant int& headDim,
- const uint2 position
+VARIANTS(T, float, bfloat, half)
+KERNEL(KVCacheUpdate) (
+ device T* in_place_keys,
+ device T* in_place_values,
+ const constant uzu::kv_cache_update::Swap* swaps,
+ const constant uint& swap_count,
+ const constant uint& num_heads,
+ const constant uint& max_sequence_length,
+ const constant uint& head_dim,
+ const uint head_idx AXIS(num_heads, 32),
+ const uint channel_idx AXIS(head_dim, 32)
) {
- for (int i = 0; i < swapCount; ++i) {
- // [headIdx: 0..numHeads, tokenIdx: 0..maxSequenceLength, channelIdx:
- // 0..headDim]
- const int headOffset = position.x * maxSequenceLength * headDim;
- const int channelOffset = position.y;
- const int sourceIdx = headOffset + swaps[i].x * headDim + channelOffset;
- const int destIdx = headOffset + swaps[i].y * headDim + channelOffset;
+ for (uint i = 0; i < swap_count; ++i) {
+ // [headIdx: 0..num_heads, tokenIdx: 0..max_sequence_length, channelIdx:
+ // 0..head_dim]
+ const uint head_offset = head_idx * max_sequence_length * head_dim;
+ const uint sourceIdx = head_offset + swaps[i].source * head_dim + channel_idx;
+ const uint destIdx = head_offset + swaps[i].destination * head_dim + channel_idx;
- swap(inPlaceKeys, sourceIdx, destIdx);
- swap(inPlaceValues, sourceIdx, destIdx);
+ swap(in_place_keys, sourceIdx, destIdx);
+ swap(in_place_values, sourceIdx, destIdx);
}
}
-
-#define outerArguments(T) \
- (device T * inPlaceKeys [[buffer(0)]], \
- device T * inPlaceValues [[buffer(1)]], \
- const constant int2* swaps [[buffer(2)]], \
- const constant int& swapCount [[buffer(3)]], \
- const constant int& numHeads [[buffer(4)]], \
- const constant int& maxSequenceLength [[buffer(5)]], \
- const constant int& headDim [[buffer(6)]], \
- const uint2 position [[thread_position_in_grid]])
-
-#define innerArguments \
- (inPlaceKeys, \
- inPlaceValues, \
- swaps, \
- swapCount, \
- numHeads, \
- maxSequenceLength, \
- headDim, \
- position)
-
-generateKernels(1024, updateKVCache)
-
-#undef outerArguments
-#undef innerArguments
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs b/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs
deleted file mode 100644
index a01d915f6..000000000
--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs
+++ /dev/null
@@ -1,262 +0,0 @@
-use std::{mem::size_of, ptr::NonNull};
-
-use thiserror::Error;
-
-use super::{
- super::MTLError, KernelDataType, MTLContext,
- metal_extensions::ComputeEncoderDispatch,
-};
-use crate::backends::{
- common::Context,
- metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandEncoderExt,
- MTLComputeCommandEncoder, MTLComputePipelineState, MTLSize,
- ProtocolObject, Retained,
- },
-};
-
-#[derive(Clone, Copy, Debug)]
-#[repr(C)]
-pub struct Swap {
- pub source: i32,
- pub destination: i32,
-}
-
-pub struct KVLayerData {
- pub key_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- pub key_shape: [usize; 3],
- pub value_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- pub value_shape: [usize; 3],
-}
-
-pub struct KVCacheUpdate {
- pipeline_state: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- indices_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- max_sequence_length: usize,
-}
-
-#[derive(Debug, Error)]
-pub enum KVCacheUpdateError {
- #[error("Source and destination indices length mismatch")]
- IndicesCountMismatch,
- #[error("Shape mismatch between key and value tensors")]
- ShapeMismatch,
- #[error("Unsupported data type")]
- UnsupportedDataType,
- #[error("Metal error: {0}")]
- MetalError(#[from] MTLError),
- #[error("Swap count exceeds maximum sequence length")]
- MaxSequenceLengthExceeded,
- #[error("Buffer creation failed")]
- BufferCreationFailed,
- #[error("Function not found: {0}")]
- FunctionNotFound(String),
-}
-
-impl KVCacheUpdate {
- pub fn new(
- context: &MTLContext,
- data_type: KernelDataType,
- max_sequence_length: usize,
- ) -> Result<Self, KVCacheUpdateError> {
- // Format function name with the appropriate suffix
- let function_name =
- format!("updateKVCache_{}", data_type.function_name_suffix());
-
- let pipeline_state = context
- .compute_pipeline_state(&function_name, None)
- .map_err(|e| KVCacheUpdateError::MetalError(e))?;
-
- let indices_buffer = context
- .create_buffer(max_sequence_length * size_of::<Swap>())
- .map_err(|_| KVCacheUpdateError::BufferCreationFailed)?;
-
- Ok(Self {
- pipeline_state,
- indices_buffer,
- max_sequence_length,
- })
- }
-
- pub fn encode(
- &self,
- in_place_data: &[KVLayerData],
- source_indices: &[usize],
- destination_indices: &[usize],
- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
- ) -> Result<(), KVCacheUpdateError> {
- let compute_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder");
- compute_encoder.set_label(Some("KV Cache Update"));
-
- self.encode_with_encoder(
- in_place_data,
- source_indices,
- destination_indices,
- &*compute_encoder,
- )?;
-
- compute_encoder.end_encoding();
- Ok(())
- }
-
- /// Encode the KV cache update operation using a provided compute encoder
- pub fn encode_with_encoder(
- &self,
- in_place_data: &[KVLayerData],
- source_indices: &[usize],
- destination_indices: &[usize],
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- ) -> Result<(), KVCacheUpdateError> {
- if source_indices.len() != destination_indices.len() {
- return Err(KVCacheUpdateError::IndicesCountMismatch);
- }
-
- let swaps = create_swaps_direct(source_indices, destination_indices);
- if swaps.len() > self.max_sequence_length {
- return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);
- }
-
- // For small swap counts, use set_bytes to avoid shared buffer race in async mode.
- // Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.
- let use_inline_bytes = swaps.len() <= 32;
-
- if !use_inline_bytes {
- let indices_ptr =
- self.indices_buffer.contents().as_ptr() as *mut Swap;
- unsafe {
- for (i, swap) in swaps.iter().enumerate() {
- *indices_ptr.add(i) = *swap;
- }
- }
- }
-
- for layer_data in in_place_data {
- if layer_data.key_shape != layer_data.value_shape {
- return Err(KVCacheUpdateError::ShapeMismatch);
- }
-
- let [num_heads, max_sequence_length, head_dim] =
- layer_data.key_shape;
-
- // Set buffers and parameters
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(&layer_data.key_buffer),
- 0,
- 0,
- );
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(&layer_data.value_buffer),
- 0,
- 1,
- );
- if use_inline_bytes {
- unsafe {
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(swaps.as_ptr() as *mut _),
- swaps.len() * size_of::<Swap>(),
- 2,
- );
- }
- } else {
- MTLComputeCommandEncoder::set_buffer(
- compute_encoder,
- Some(&self.indices_buffer),
- 0,
- 2,
- );
- }
- unsafe {
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(
- &(swaps.len() as i32) as *const _ as *mut _,
- ),
- size_of::<i32>(),
- 3,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(
- &(num_heads as i32) as *const _ as *mut _,
- ),
- size_of::<i32>(),
- 4,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(
- &(max_sequence_length as i32) as *const _ as *mut _,
- ),
- size_of::<i32>(),
- 5,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(
- &(head_dim as i32) as *const _ as *mut _,
- ),
- size_of::<i32>(),
- 6,
- );
- }
-
- compute_encoder.dispatch_2d_exactly(
- &self.pipeline_state,
- MTLSize {
- width: num_heads as usize,
- height: head_dim as usize,
- depth: 1,
- },
- None,
- );
- }
-
- Ok(())
- }
-}
-
-pub fn create_swaps_direct(
- source_indices: &[usize],
- destination_indices: &[usize],
-) -> Vec<Swap> {
- if source_indices.len() != destination_indices.len() {
- return Vec::new();
- }
-
- let mut swaps = Vec::with_capacity(source_indices.len());
-
- for (&src, &dst) in source_indices.iter().zip(destination_indices.iter()) {
- if src != dst {
- swaps.push(Swap {
- source: src as i32,
- destination: dst as i32,
- });
- }
- }
-
- swaps
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_direct_swaps() {
- let sources = [0, 2, 4];
- let destinations = [1, 3, 5];
- let swaps = create_swaps_direct(&sources, &destinations);
- assert_eq!(swaps.len(), 3);
- assert_eq!(swaps[0].source, 0);
- assert_eq!(swaps[0].destination, 1);
- assert_eq!(swaps[1].source, 2);
- assert_eq!(swaps[1].destination, 3);
- assert_eq!(swaps[2].source, 4);
- assert_eq!(swaps[2].destination, 5);
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
index 7a945dd23..ee493532d 100644
--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
@@ -1,15 +1,16 @@
use std::{mem, ptr::NonNull};
-use crate::backends::metal::Metal;
+use crate::{
+ backends::metal::Metal,
+ encodable_block::{EncodableBlock, EncodingParameters},
+ forward_pass::state::ForwardPassState,
+};
use objc2::rc::Retained;
use crate::backends::metal::{
MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandEncoderExt,
MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLSize,
- ProtocolObject,
- error::MTLError,
- forward_pass::{EncodableBlock, EncodingParameters, ForwardPassState},
- image::Image,
+ ProtocolObject, error::MTLError, image::Image,
metal_extensions::ComputeEncoderDispatch,
};
@@ -83,9 +84,9 @@ impl ScalePadNormalizeImage {
impl EncodableBlock<Metal> for ScalePadNormalizeImage {
fn encode(
&self,
- _state: &mut ForwardPassState,
+ _state: &mut ForwardPassState<Metal>,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
let _ = command_buffer;
}
@@ -96,9 +97,9 @@ impl EncodableBlock<Metal> for ScalePadNormalizeImage {
fn encode_with_shared_encoder(
&self,
- _state: &mut ForwardPassState,
+ _state: &mut ForwardPassState<Metal>,
_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters,
+ _parameters: &EncodingParameters<Metal>,
) {
unreachable!(
"ScalePadNormalizeImage does not support shared compute encoder"
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index c3c1f8d38..a24d4084a 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -6,7 +6,6 @@ pub mod dsl {
pub(super) const MTLB: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
-pub mod kv_cache_update;
pub mod matmul;
pub mod media_kernels;
pub mod mlp;
@@ -29,7 +28,6 @@ pub use attention::{
KVCacheUpdateArguments,
};
pub use data_type::KernelDataType;
-pub use kv_cache_update::KVCacheUpdate;
pub use matmul::{MatmulArguments, MatmulKernel};
pub use moe::{
MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError,
@@ -53,8 +51,6 @@ pub use ssm::{
};
pub use token_copy::TokenCopyKernel;
-use super::{MTLContext, metal_extensions};
-
pub mod media {
pub use super::media_kernels::{
ExtractImagePatches, ScalePadNormalizeImage,
diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rs
index 37e15d9ab..1b621d72d 100644
--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs
+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs
@@ -7,7 +7,7 @@ use std::{
};
use super::{
- CacheLayers, Decoder, KVCacheUpdate, KernelDataType, MTLContext, Metal,
+ Decoder, KVCacheUpdate, KernelDataType, MTLContext, Metal,
compilation_parameters::CompilationConfig, kernel::TokenCopyKernel,
};
use crate::{
@@ -23,8 +23,8 @@ use crate::{
config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
encodable_block::Sampling,
forward_pass::{
- model_shape::ModelShape, scratch_buffers::ScratchBuffers,
- state::SharedBuffers,
+ cache_layers::CacheLayers, model_shape::ModelShape,
+ scratch_buffers::ScratchBuffers, state::SharedBuffers,
},
language_model::rng::PRng,
parameters::ParameterLoader,
@@ -136,7 +136,7 @@ pub struct LanguageModelGeneratorContext {
pub mtl_context: Rc<MTLContext>,
pub command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- pub cache_layers: Rc<RefCell<CacheLayers>>,
+ pub cache_layers: Rc<RefCell<CacheLayers<Metal>>>,
pub shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,
pub scratch_buffers: ScratchBuffers<Metal>,
@@ -144,7 +144,7 @@ pub struct LanguageModelGeneratorContext {
pub decoder_config: Rc<DecoderConfig>,
pub model_shape: ModelShape,
pub executables: Decoder,
- pub kv_cache_update: Box<KVCacheUpdate>,
+ pub kv_cache_update: Box<KVCacheUpdate<Metal>>,
pub gpu_sampler: Sampling<Metal>,
pub seed: PRng,
@@ -234,7 +234,7 @@ impl LanguageModelGeneratorContext {
);
let cache_layers = Rc::new(RefCell::new(CacheLayers::new(
- &context,
+ context.as_ref(),
&model_shape,
max_prefix_length,
max_suffix_length,
@@ -244,8 +244,12 @@ impl LanguageModelGeneratorContext {
decoder_config.output_norm_config.scale_precision.into();
let kernel_data_type: KernelDataType = intermediate_data_type.into();
let kv_cache_update = Box::new(
- KVCacheUpdate::new(&context, kernel_data_type, max_prefix_length)
- .map_err(|_| Error::UnableToCreateMetalContext)?,
+ KVCacheUpdate::new(
+ context.as_ref(),
+ intermediate_data_type,
+ max_prefix_length,
+ )
+ .map_err(|_| Error::UnableToCreateMetalContext)?,
);
let gpu_sampler = Sampling::<Metal>::new(
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 019247024..6ca1d398b 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -5,9 +5,9 @@ mod classifier_context;
mod command_buffer;
pub mod compilation_parameters;
mod context;
+mod copy_encoder;
pub mod encodable_block;
pub mod error;
-pub mod forward_pass;
pub mod image;
pub mod kernel;
mod language_model_generator_context;
@@ -15,6 +15,7 @@ pub mod media_utils;
pub mod metal_extensions;
pub mod placement_analysis;
pub mod utils;
+pub use crate::backends::common::kernel::kv_cache_update::KVCacheUpdate;
pub use array::MetalArray;
pub use backend::Metal;
pub use classifier_context::ClassifierContext;
@@ -23,10 +24,7 @@ pub use context::{
};
pub use encodable_block::Decoder;
pub use error::MTLError;
-pub use forward_pass::{CacheLayers, ForwardPassState};
-pub use kernel::{
- KVCacheUpdate, KernelDataType, RopeKernel, dsl::MetalKernels,
-};
+pub use kernel::{KernelDataType, RopeKernel, dsl::MetalKernels};
pub use language_model_generator_context::LanguageModelGeneratorContext;
pub use media_utils::{
ImagePreprocessingParams, ImagePreprocessingRequirements,
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 094325df3..7aad798f7 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -7,14 +7,12 @@ use objc2::rc::autoreleasepool;
#[cfg(feature = "tracing")]
use super::ActivationTrace;
use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
+use crate::backends::metal::Metal;
use crate::{
DataType,
- backends::metal::{
- MTLCommandBuffer,
- forward_pass::{
- ArrayId, EncodableBlock, EncodingParameters, ForwardPassState,
- },
- },
+ backends::metal::MTLCommandBuffer,
+ encodable_block::{EncodableBlock, EncodingParameters},
+ forward_pass::state::{ArrayId, ForwardPassState},
session::types::Error,
};
@@ -83,7 +81,7 @@ impl Classifier {
&mut self,
token_ids: &[u64],
token_positions: &[usize],
- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace>>), Error> {
+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<Metal>>>), Error> {
self.forward_pass(token_ids, token_positions)
}
@@ -92,7 +90,7 @@ impl Classifier {
&mut self,
token_ids: &[u64],
token_positions: &[usize],
- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace>>), Error> {
+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<Metal>>>), Error> {
autoreleasepool(|_| {
let num_labels = self.context.model_config.model_config.num_labels;
let mut state = ForwardPassState::new_classifier(
@@ -241,7 +239,7 @@ impl Classifier {
fn copy_logits_from_state(
&self,
- state: &ForwardPassState,
+ state: &ForwardPassState<Metal>,
) -> Result<Box<[f32]>, Error> {
let logits_arrays =
state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rs
index 680439779..48d0f31b3 100644
--- a/crates/uzu/src/classifier/mod.rs
+++ b/crates/uzu/src/classifier/mod.rs
@@ -11,6 +11,4 @@ pub use crate::backends::metal::encodable_block::{
ClassifierLayer, ClassifierPredictionHead, Pooling,
};
#[cfg(feature = "tracing")]
-pub use crate::backends::metal::forward_pass::traces::{
- ActivationTrace, LayerActivationTrace,
-};
+pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 5a89ecfb4..87e85ed6f 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -1,6 +1,6 @@
-use crate::backends::{
- common::{Backend, CommandBuffer},
- metal::ForwardPassState,
+use crate::{
+ backends::common::{Backend, CommandBuffer},
+ forward_pass::state::ForwardPassState,
};
mod sampling;
@@ -55,8 +55,8 @@ impl<'a, B: Backend> EncodingParameters<'a, B> {
pub trait EncodableBlock<B: Backend> {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
- encoder: &B::EncoderRef,
+ state: &mut ForwardPassState<B>,
+ encoder: &B::ComputeEncoder,
parameters: &EncodingParameters<B>,
);
@@ -66,11 +66,11 @@ pub trait EncodableBlock<B: Backend> {
fn encode(
&self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<B>,
command_buffer: &B::CommandBuffer,
parameters: &EncodingParameters<B>,
) {
- command_buffer.with_encoder(|encoder| {
+ command_buffer.with_compute_encoder(|encoder| {
self.encode_with_shared_encoder(state, encoder, parameters)
});
}
diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rs
index c6cb660c2..51c775ace 100644
--- a/crates/uzu/src/encodable_block/sampling.rs
+++ b/crates/uzu/src/encodable_block/sampling.rs
@@ -2,16 +2,11 @@
use crate::{
DataType,
- backends::{
- common::{
- Backend,
- kernel::sampling::{ArgmaxStrategy, SamplingError, SamplingKernel},
- },
- metal::{
- MTLComputeCommandEncoder, Metal, ProtocolObject,
- forward_pass::{ArrayId, ForwardPassState},
- },
+ backends::common::{
+ Backend,
+ kernel::sampling::{ArgmaxStrategy, SamplingError, SamplingKernel},
},
+ forward_pass::state::{ArrayId, ForwardPassState},
};
use super::{EncodableBlock, EncodingParameters};
@@ -58,12 +53,12 @@ impl<B: Backend> Sampling<B> {
}
}
-impl EncodableBlock<Metal> for Sampling<Metal> {
+impl<B: Backend> EncodableBlock<B> for Sampling<B> {
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- _parameters: &EncodingParameters<Metal>,
+ state: &mut ForwardPassState<B>,
+ encoder: &B::ComputeEncoder,
+ _parameters: &EncodingParameters<B>,
) {
assert!(
state.sampling_output().is_some(),
diff --git a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rs
similarity index 91%
rename from crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
rename to crates/uzu/src/forward_pass/cache_layers.rs
index baaf49075..b5da2272a 100644
--- a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/forward_pass/cache_layers.rs
@@ -1,33 +1,30 @@
use std::{cell::RefCell, collections::HashMap};
-use super::{
- super::{MTLContext, MetalArray},
- kv_cache_layer::{
- AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,
- KVSlice,
- },
- short_conv_layer::ShortConvLayer,
- ssm_layer::SSMLayer,
-};
use crate::{
- array::ArrayContextExt,
- backends::metal::{
- MTLCommandBuffer, ProtocolObject, Retained, kernel::KVCacheUpdate,
- },
+ array::{Array, ArrayContextExt},
+ backends::common::{Backend, kernel::kv_cache_update::KVCacheUpdate},
config::DecoderLayerType,
- forward_pass::model_shape::ModelShape,
+ forward_pass::{
+ kv_cache_layer::{
+ AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer,
+ KVCacheLayerState, KVSlice,
+ },
+ model_shape::ModelShape,
+ short_conv_layer::ShortConvLayer,
+ ssm_layer::SSMLayer,
+ },
};
#[derive(Debug)]
-pub enum CacheLayer {
- Transformer(KVCacheLayer),
- StateSpace(SSMLayer),
- ShortConv(ShortConvLayer),
+pub enum CacheLayer<B: Backend> {
+ Transformer(KVCacheLayer<B>),
+ StateSpace(SSMLayer<B>),
+ ShortConv(ShortConvLayer<B>),
}
#[derive(Clone)]
-pub enum CacheLayerSlice {
- Transformer(KVSlice),
+pub enum CacheLayerSlice<B: Backend> {
+ Transformer(KVSlice<B>),
StateSpace,
ShortConv,
}
@@ -41,43 +38,43 @@ const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str =
const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str =
"cache_layers_short_conv_conv_state";
-impl CacheLayer {
- pub fn as_transformer(&self) -> Option<&KVCacheLayer> {
+impl<B: Backend> CacheLayer<B> {
+ pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {
match self {
CacheLayer::Transformer(layer) => Some(layer),
_ => None,
}
}
- pub fn as_transformer_mut(&mut self) -> Option<&mut KVCacheLayer> {
+ pub fn as_transformer_mut(&mut self) -> Option<&mut KVCacheLayer<B>> {
match self {
CacheLayer::Transformer(layer) => Some(layer),
_ => None,
}
}
- pub fn as_state_space(&self) -> Option<&SSMLayer> {
+ pub fn as_state_space(&self) -> Option<&SSMLayer<B>> {
match self {
CacheLayer::StateSpace(layer) => Some(layer),
_ => None,
}
}
- pub fn as_state_space_mut(&mut self) -> Option<&mut SSMLayer> {
+ pub fn as_state_space_mut(&mut self) -> Option<&mut SSMLayer<B>> {
match self {
CacheLayer::StateSpace(layer) => Some(layer),
_ => None,
}
}
- pub fn as_short_conv(&self) -> Option<&ShortConvLayer> {
+ pub fn as_short_conv(&self) -> Option<&ShortConvLayer<B>> {
match self {
CacheLayer::ShortConv(layer) => Some(layer),
_ => None,
}
}
- pub fn as_short_conv_mut(&mut self) -> Option<&mut ShortConvLayer> {
+ pub fn as_short_conv_mut(&mut self) -> Option<&mut ShortConvLayer<B>> {
match self {
CacheLayer::ShortConv(layer) => Some(layer),
_ => None,
@@ -85,20 +82,20 @@ impl CacheLayer {
}
}
-pub struct CacheLayers {
+pub struct CacheLayers<B: Backend> {
max_suffix_length: usize,
max_prefix_length: usize,
- pub data: Box<[CacheLayer]>,
+ pub data: Box<[CacheLayer<B>]>,
}
#[derive(Clone)]
-pub struct CacheLayersSlice {
- pub layers: Vec<CacheLayerSlice>,
+pub struct CacheLayersSlice<B: Backend> {
+ pub layers: Vec<CacheLayerSlice<B>>,
}
-impl CacheLayers {
+impl<B: Backend> CacheLayers<B> {
pub fn new(
- context: &MTLContext,
+ context: &B::Context,
model_shape: &ModelShape,
max_prefix_length: usize,
max_suffix_length: usize,
@@ -108,7 +105,7 @@ impl CacheLayers {
.kv_cache_layer_shapes(max_prefix_length, max_suffix_length)
.collect();
- let data: Box<[CacheLayer]> = model_shape
+ let data: Box<[CacheLayer<B>]> = model_shape
.layer_types()
.iter()
.enumerate()
@@ -257,7 +254,7 @@ impl CacheLayers {
pub fn fill_attention_bias(
&self,
- dst: &mut HashMap<Option<usize>, MetalArray>,
+ dst: &mut HashMap<Option<usize>, Array<B>>,
suffix_token_positions: &[usize],
suffix_length: usize,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
@@ -278,10 +275,10 @@ impl CacheLayers {
pub fn fill_attention_bias_scratch(
&self,
- dst: &HashMap<Option<usize>, RefCell<MetalArray>>,
+ dst: &HashMap<Option<usize>, RefCell<Array<B>>>,
suffix_token_positions: &[usize],
suffix_length: usize,
- _context: &MTLContext,
+ _context: &B::Context,
) {
for layer in self.data.iter() {
if let CacheLayer::Transformer(layer) = layer {
@@ -301,8 +298,8 @@ impl CacheLayers {
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- kv_cache_update: &KVCacheUpdate,
+ command_buffer: &B::CommandBuffer,
+ kv_cache_update: &KVCacheUpdate<B>,
) {
for layer in self.data.iter_mut() {
if let Some(layer) = layer.as_transformer_mut() {
@@ -344,9 +341,9 @@ impl CacheLayers {
pub fn slice(
&self,
- context: &MTLContext,
+ context: &B::Context,
range: std::ops::Range<usize>,
- ) -> Option<CacheLayersSlice> {
+ ) -> Option<CacheLayersSlice<B>> {
let mut layers = Vec::with_capacity(self.data.len());
for layer in self.data.iter() {
match layer {
@@ -372,7 +369,7 @@ impl CacheLayers {
pub fn apply_slice(
&mut self,
- slice: &CacheLayersSlice,
+ slice: &CacheLayersSlice<B>,
range: Option<std::ops::Range<usize>>,
) {
for (layer, snapshot) in self.data.iter_mut().zip(slice.layers.iter()) {
@@ -381,7 +378,7 @@ impl CacheLayers {
CacheLayer::Transformer(kv),
CacheLayerSlice::Transformer(s),
) => {
- kv.apply_slice(s, range.clone());
+ kv.apply_slice(&s, range.clone());
},
(CacheLayer::StateSpace(_), CacheLayerSlice::StateSpace) => {},
(CacheLayer::ShortConv(_), CacheLayerSlice::ShortConv) => {},
@@ -392,10 +389,10 @@ impl CacheLayers {
pub fn clone(
&self,
- context: &MTLContext,
+ context: &B::Context,
) -> Self {
let mut max_prefix_capacity_across_layers = 0usize;
- let data: Box<[CacheLayer]> = self
+ let data: Box<[CacheLayer<B>]> = self
.data
.iter()
.enumerate()
diff --git a/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rs
similarity index 94%
rename from crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs
rename to crates/uzu/src/forward_pass/kv_cache_layer.rs
index 9c4e5dcf9..37348a84e 100644
--- a/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs
+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs
@@ -1,17 +1,12 @@
-use std::cell::RefCell;
-
-use super::super::{MTLContext, MetalArray};
use crate::{
- array::ArrayContextExt,
- backends::metal::{
- MTLCommandBuffer, ProtocolObject, Retained,
- kernel::{KVCacheUpdate, kv_cache_update::KVLayerData},
+ array::{Array, ArrayCell, ArrayContextExt},
+ backends::common::{
+ Backend,
+ kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},
},
utils::attention::fill_attention_bias,
};
-pub type ArrayCell = RefCell<MetalArray>;
-
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AttentionBiasUpdate {
pub key: Option<usize>,
@@ -20,7 +15,7 @@ pub struct AttentionBiasUpdate {
}
#[derive(Clone)]
-pub enum KVSlice {
+pub enum KVSlice<B: Backend> {
Full {
base_prefix_len: usize,
base_positions_len: usize,
@@ -32,8 +27,8 @@ pub enum KVSlice {
base_ring_length: usize,
slots: Vec<usize>,
positions: Vec<usize>, // per slot
- keys: MetalArray, // [num_groups, slots.len(), head_dim]
- values: MetalArray, // [num_groups, slots.len(), head_dim]
+ keys: Array<B>, // [num_groups, slots.len(), head_dim]
+ values: Array<B>, // [num_groups, slots.len(), head_dim]
},
}
@@ -55,18 +50,18 @@ pub enum KVCacheLayerState {
pub const INVALID_POSITION: usize = i32::MAX as usize;
#[derive(Debug)]
-pub struct KVCacheLayer {
+pub struct KVCacheLayer<B: Backend> {
pub state: KVCacheLayerState,
/// [num_groups, max_prefix_length + max_suffix_length, head_dim]
- pub keys: ArrayCell,
+ pub keys: ArrayCell<B>,
/// [num_groups, max_prefix_length + max_suffix_length, head_dim]
- pub values: ArrayCell,
+ pub values: ArrayCell<B>,
pub prefix_token_positions: Vec<usize>,
pub max_suffix_length: usize,
}
-impl KVCacheLayer {
+impl<B: Backend> KVCacheLayer<B> {
pub fn prefix_segment_length(&self) -> usize {
match &self.state {
KVCacheLayerState::Full {
@@ -94,10 +89,6 @@ impl KVCacheLayer {
}
}
- pub fn is_sliding_window(&self) -> bool {
- matches!(self.state, KVCacheLayerState::Windowed { .. })
- }
-
pub fn window_length(&self) -> Option<usize> {
match &self.state {
KVCacheLayerState::Full {
@@ -112,7 +103,7 @@ impl KVCacheLayer {
pub fn fill_attention_bias(
&self,
- dst: &mut MetalArray,
+ dst: &mut Array<B>,
suffix_token_positions: &[usize],
suffix_length: usize,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
@@ -181,8 +172,8 @@ impl KVCacheLayer {
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- kv_cache_update: &KVCacheUpdate,
+ command_buffer: &B::CommandBuffer,
+ kv_cache_update: &KVCacheUpdate<B>,
) {
match &mut self.state {
KVCacheLayerState::Full {
@@ -249,8 +240,8 @@ impl KVCacheLayer {
&self,
source_indices: &[usize],
destination_indices: &[usize],
- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
- kv_cache_update: &KVCacheUpdate,
+ command_buffer: &B::CommandBuffer,
+ kv_cache_update: &KVCacheUpdate<B>,
) {
if source_indices == destination_indices {
return;
@@ -351,9 +342,9 @@ impl KVCacheLayer {
pub fn slice(
&self,
- context: &MTLContext,
+ context: &B::Context,
range: std::ops::Range<usize>,
- ) -> Option<KVSlice> {
+ ) -> Option<KVSlice<B>> {
match self.state {
KVCacheLayerState::Full {
prefix_len,
@@ -427,7 +418,7 @@ impl KVCacheLayer {
pub fn apply_slice(
&mut self,
- slice: &KVSlice,
+ slice: &KVSlice<B>,
range: Option<std::ops::Range<usize>>,
) {
match (slice, &mut self.state) {
diff --git a/crates/uzu/src/forward_pass/mod.rs b/crates/uzu/src/forward_pass/mod.rs
index 4dc2d5634..cfede3e9a 100644
--- a/crates/uzu/src/forward_pass/mod.rs
+++ b/crates/uzu/src/forward_pass/mod.rs
@@ -1,3 +1,12 @@
pub mod model_shape;
+
+pub mod cache_layers;
+pub mod kv_cache_layer;
+pub mod short_conv_layer;
+pub mod ssm_layer;
+
pub mod scratch_buffers;
pub mod state;
+
+#[cfg(feature = "tracing")]
+pub mod traces;
diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rs
index 0e8df2598..32b32c0e5 100644
--- a/crates/uzu/src/forward_pass/model_shape.rs
+++ b/crates/uzu/src/forward_pass/model_shape.rs
@@ -182,22 +182,6 @@ impl ModelShape {
[suffix_length, self.vocabulary_size]
}
- pub fn embeddings_input_shape(&self) -> [usize; 2] {
- [self.vocabulary_size, self.model_dim]
- }
-
- pub fn embeddings_output_shape(&self) -> [usize; 2] {
- [self.vocabulary_size, self.model_dim]
- }
-
- pub fn quantized_embeddings_weights_shape(&self) -> [usize; 2] {
- [self.vocabulary_size, self.model_dim]
- }
-
- pub fn quantized_embeddings_scales_shape(&self) -> [usize; 1] {
- [self.vocabulary_size]
- }
-
pub fn attention_output_shape(
&self,
suffix_length: usize,
@@ -266,14 +250,6 @@ impl ModelShape {
[self.num_heads * suffix_length * TOTAL_BLOCKS_COUNT]
}
- pub fn moe_router_logits_shape(
- &self,
- suffix_length: usize,
- num_experts: usize,
- ) -> [usize; 2] {
- [suffix_length, num_experts]
- }
-
pub fn moe_topk_ids_shape(
&self,
suffix_length: usize,
diff --git a/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs b/crates/uzu/src/forward_pass/short_conv_layer.rs
similarity index 51%
rename from crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs
rename to crates/uzu/src/forward_pass/short_conv_layer.rs
index 0933ec813..7763a7eb0 100644
--- a/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs
+++ b/crates/uzu/src/forward_pass/short_conv_layer.rs
@@ -1,13 +1,13 @@
use bytemuck::fill_zeroes;
-use super::kv_cache_layer::ArrayCell;
+use crate::{array::ArrayCell, backends::common::Backend};
#[derive(Debug)]
-pub struct ShortConvLayer {
- pub conv_state: ArrayCell,
+pub struct ShortConvLayer<B: Backend> {
+ pub conv_state: ArrayCell<B>,
}
-impl ShortConvLayer {
+impl<B: Backend> ShortConvLayer<B> {
pub fn zero(&self) {
let mut conv = self.conv_state.borrow_mut();
fill_zeroes(conv.as_bytes_mut());
diff --git a/crates/uzu/src/backends/metal/forward_pass/ssm_layer.rs b/crates/uzu/src/forward_pass/ssm_layer.rs
similarity index 63%
rename from crates/uzu/src/backends/metal/forward_pass/ssm_layer.rs
rename to crates/uzu/src/forward_pass/ssm_layer.rs
index 43ede5bfc..01ba550bf 100644
--- a/crates/uzu/src/backends/metal/forward_pass/ssm_layer.rs
+++ b/crates/uzu/src/forward_pass/ssm_layer.rs
@@ -1,14 +1,14 @@
use bytemuck::fill_zeroes;
-use super::kv_cache_layer::ArrayCell;
+use crate::{array::ArrayCell, backends::common::Backend};
#[derive(Debug)]
-pub struct SSMLayer {
- pub conv_state: ArrayCell,
- pub ssm_state: ArrayCell,
+pub struct SSMLayer<B: Backend> {
+ pub conv_state: ArrayCell<B>,
+ pub ssm_state: ArrayCell<B>,
}
-impl SSMLayer {
+impl<B: Backend> SSMLayer<B> {
pub fn zero(&self) {
{
let mut conv = self.conv_state.borrow_mut();
diff --git a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
index f0ccedb7d..0323594ad 100644
--- a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
@@ -62,8 +62,4 @@ impl<B: Backend> CommonAuxBuffers<B> {
.view(&model_shape.attention_maxs_shape(suffix_length)),
}
}
-
- pub fn suffix_length(&self) -> usize {
- self.suffix_length
- }
}
diff --git a/crates/uzu/src/forward_pass/state/mod.rs b/crates/uzu/src/forward_pass/state/mod.rs
index b211f2955..3aab46cde 100644
--- a/crates/uzu/src/forward_pass/state/mod.rs
+++ b/crates/uzu/src/forward_pass/state/mod.rs
@@ -7,6 +7,8 @@ mod language_model_generator_aux_buffers;
mod rope_buffers;
mod shared_buffers;
+mod state;
+
pub use array_id::ArrayId;
pub use hash_map_id::HashMapId;
pub use rope_type::RopeType;
@@ -14,4 +16,9 @@ pub use rope_type::RopeType;
pub use common_aux_buffers::CommonAuxBuffers;
pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;
pub use rope_buffers::RopeBuffers;
-pub use shared_buffers::{MoeExpertWeights, SharedBuffers};
+pub use shared_buffers::SharedBuffers;
+
+pub use state::{
+ ClassifierModeState, ForwardPassMode, ForwardPassState,
+ LanguageModelGeneratorModeState,
+};
diff --git a/crates/uzu/src/forward_pass/state/shared_buffers.rs b/crates/uzu/src/forward_pass/state/shared_buffers.rs
index c8040d93e..fd7db21fb 100644
--- a/crates/uzu/src/forward_pass/state/shared_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/shared_buffers.rs
@@ -6,23 +6,16 @@ use crate::{
DataType,
array::{ArrayCell, ArrayContextExt},
backends::common::Backend,
- config::{DecoderConfig, MLPConfig},
+ config::DecoderConfig,
forward_pass::model_shape::ModelShape,
parameters::ParameterTree,
};
use super::RopeBuffers;
-pub struct MoeExpertWeights<B: Backend> {
- pub w1: ArrayCell<B>,
- pub w2: ArrayCell<B>,
- pub w3: ArrayCell<B>,
-}
-
pub struct SharedBuffers<B: Backend> {
pub global_rope: Option<RopeBuffers<B>>,
pub local_rope: Option<RopeBuffers<B>>,
- pub moe_expert_weights: Option<Vec<MoeExpertWeights<B>>>,
pub attention_sinks: Option<Vec<ArrayCell<B>>>,
}
@@ -42,12 +35,6 @@ impl<B: Backend> SharedBuffers<B> {
.is_some()
.then(|| RopeBuffers::new(context, model_shape));
- let moe_expert_weights = matches!(
- decoder_config.layer_config.mlp_config,
- MLPConfig::MixtureOfExperts(_)
- )
- .then(Vec::new);
-
let attention_sinks = decoder_config
.layer_config
.attention_config()
@@ -68,7 +55,6 @@ impl<B: Backend> SharedBuffers<B> {
Self {
global_rope,
local_rope,
- moe_expert_weights,
attention_sinks,
}
}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/forward_pass/state/state.rs
similarity index 85%
rename from crates/uzu/src/backends/metal/forward_pass/state/mod.rs
rename to crates/uzu/src/forward_pass/state/state.rs
index c2d61f065..7b3326c0c 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -1,26 +1,14 @@
-mod mode;
-
use std::{cell::RefCell, collections::HashMap, rc::Rc};
-pub use mode::{
- ClassifierModeState, ForwardPassMode, LanguageModelGeneratorModeState,
-};
-
-use super::cache_layers::CacheLayers;
#[cfg(feature = "tracing")]
-use super::traces::ActivationTrace;
+use crate::forward_pass::traces::ActivationTrace;
+
use crate::{
DataType, DecoderConfig,
- array::ArrayCellExt,
- backends::{
- common::Context,
- metal::{
- MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,
- MTLCommandEncoder, MTLContext, Metal, MetalArray, ProtocolObject,
- Retained,
- },
- },
+ array::{Array, ArrayCell, ArrayCellExt},
+ backends::common::{Backend, CommandBuffer, Context, CopyEncoder},
forward_pass::{
+ cache_layers::CacheLayers,
model_shape::ModelShape,
scratch_buffers::ScratchBuffers,
state::{
@@ -32,29 +20,55 @@ use crate::{
utils::attention::fill_attention_bias,
};
-pub type ArrayCell = RefCell<MetalArray>;
-
-pub struct ForwardPassState {
- context: Rc<MTLContext>,
- token_ids: ArrayCell,
- token_positions: ArrayCell,
- token_bitmask: Option<ArrayCell>,
- attention_bias: HashMap<Option<usize>, ArrayCell>,
- pub shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,
- common_aux: CommonAuxBuffers<Metal>,
- llm_aux: Option<LanguageModelGeneratorAuxBuffers<Metal>>,
- mode: ForwardPassMode,
+pub enum ForwardPassMode<B: Backend> {
+ LanguageModelGenerator(LanguageModelGeneratorModeState<B>),
+ Classifier(ClassifierModeState<B>),
+}
+
+pub struct LanguageModelGeneratorModeState<B: Backend> {
+ pub cache_layers: Rc<RefCell<CacheLayers<B>>>,
+ pub token_seeds: ArrayCell<B>,
+ pub logits: ArrayCell<B>,
+ pub sampling_output: Option<ArrayCell<B>>,
+ pub sampling_method: Option<SamplingMethod>,
+ #[cfg(feature = "tracing")]
+ pub traces: Rc<RefCell<ActivationTrace<B>>>,
+ pub active_suffix_length: usize,
+ pub sampling_start: usize,
+ pub sampling_length: usize,
+ pub is_prefilling: bool,
+}
+
+pub struct ClassifierModeState<B: Backend> {
+ pub pooling: ArrayCell<B>,
+ pub dense: ArrayCell<B>,
+ pub norm: ArrayCell<B>,
+ pub classifier_logits: ArrayCell<B>,
+ #[cfg(feature = "tracing")]
+ pub traces: Rc<RefCell<ActivationTrace<B>>>,
}
-impl ForwardPassState {
+pub struct ForwardPassState<B: Backend> {
+ context: Rc<B::Context>,
+ token_ids: ArrayCell<B>,
+ token_positions: ArrayCell<B>,
+ token_bitmask: Option<ArrayCell<B>>,
+ attention_bias: HashMap<Option<usize>, ArrayCell<B>>,
+ pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
+ common_aux: CommonAuxBuffers<B>,
+ llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,
+ mode: ForwardPassMode<B>,
+}
+
+impl<B: Backend> ForwardPassState<B> {
// ========================================================================
// Common initialization helpers
// ========================================================================
fn init_token_ids(
- scratch: &ScratchBuffers<Metal>,
+ scratch: &ScratchBuffers<B>,
token_ids: &[u64],
- ) -> ArrayCell {
+ ) -> ArrayCell<B> {
let suffix_length = token_ids.len();
let token_ids_array = scratch.token_ids.view(&[suffix_length]);
token_ids_array.borrow_mut().copy_from_view(token_ids.into());
@@ -62,9 +76,9 @@ impl ForwardPassState {
}
fn init_token_positions(
- scratch: &ScratchBuffers<Metal>,
+ scratch: &ScratchBuffers<B>,
token_positions: &[usize],
- ) -> ArrayCell {
+ ) -> ArrayCell<B> {
let suffix_length = token_positions.len();
let token_positions_array =
scratch.token_positions.view(&[suffix_length]);
@@ -82,12 +96,12 @@ impl ForwardPassState {
#[allow(clippy::too_many_arguments)]
pub fn new_llm(
- context: Rc<MTLContext>,
+ context: Rc<B::Context>,
decoder_config: &DecoderConfig,
model_shape: &ModelShape,
- scratch: &ScratchBuffers<Metal>,
- cache_layers: Rc<RefCell<CacheLayers>>,
- shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,
+ scratch: &ScratchBuffers<B>,
+ cache_layers: Rc<RefCell<CacheLayers<B>>>,
+ shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
token_ids: &[u64],
token_positions: &[usize],
token_bitmask: Option<&[u32]>,
@@ -99,11 +113,8 @@ impl ForwardPassState {
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
skip_token_ids_copy: bool,
should_fill_attention_bias: bool,
- async_positions: Option<(
- &Retained<ProtocolObject<dyn MTLBuffer>>,
- usize,
- )>,
- async_seeds: Option<(&Retained<ProtocolObject<dyn MTLBuffer>>, usize)>,
+ async_positions: Option<(&B::NativeBuffer, usize)>,
+ async_seeds: Option<(&B::NativeBuffer, usize)>,
) -> Self {
let suffix_length = token_ids.len();
assert_eq!(
@@ -127,7 +138,7 @@ impl ForwardPassState {
let token_positions_cell =
if let Some((async_buf, offset)) = async_positions {
let array = unsafe {
- MetalArray::from_parts(
+ Array::from_parts(
async_buf.clone(),
offset * std::mem::size_of::<i32>(),
&[suffix_length],
@@ -151,7 +162,7 @@ impl ForwardPassState {
// Token seeds - use async buffer if provided
let token_seeds_cell = if let Some((async_buf, offset)) = async_seeds {
let array = unsafe {
- MetalArray::from_parts(
+ Array::from_parts(
async_buf.clone(),
offset * std::mem::size_of::<u64>(),
&[suffix_length],
@@ -198,7 +209,7 @@ impl ForwardPassState {
// Traces
#[cfg(feature = "tracing")]
let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(
- &context,
+ context.as_ref(),
model_shape,
suffix_length,
)));
@@ -232,27 +243,26 @@ impl ForwardPassState {
}
fn init_llm_attention_bias(
- scratch: &ScratchBuffers<Metal>,
- cache_layers: &Rc<RefCell<CacheLayers>>,
+ scratch: &ScratchBuffers<B>,
+ cache_layers: &Rc<RefCell<CacheLayers<B>>>,
suffix_length: usize,
token_positions: &[usize],
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
should_fill_attention_bias: bool,
- ) -> HashMap<Option<usize>, ArrayCell> {
+ ) -> HashMap<Option<usize>, ArrayCell<B>> {
let cache_ref = cache_layers.borrow();
- let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =
- scratch
- .attention_window_size_to_bias
- .iter()
- .map(|(window_size, buffer)| {
- let prefix_length =
- window_size.unwrap_or(cache_ref.max_prefix_length());
- let attention_bias_shape =
- [suffix_length, suffix_length + prefix_length];
- let array = buffer.borrow().view(&attention_bias_shape);
- (*window_size, array)
- })
- .collect();
+ let mut attention_bias_map: HashMap<Option<usize>, Array<B>> = scratch
+ .attention_window_size_to_bias
+ .iter()
+ .map(|(window_size, buffer)| {
+ let prefix_length =
+ window_size.unwrap_or(cache_ref.max_prefix_length());
+ let attention_bias_shape =
+ [suffix_length, suffix_length + prefix_length];
+ let array = buffer.borrow().view(&attention_bias_shape);
+ (*window_size, array)
+ })
+ .collect();
drop(cache_ref);
// Use cache_layers' fill_attention_bias which properly handles
@@ -279,10 +289,10 @@ impl ForwardPassState {
#[allow(clippy::too_many_arguments)]
pub fn new_classifier(
- context: Rc<MTLContext>,
+ context: Rc<B::Context>,
model_shape: &ModelShape,
- scratch: &ScratchBuffers<Metal>,
- shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,
+ scratch: &ScratchBuffers<B>,
+ shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
token_ids: &[u64],
token_positions: &[usize],
bidirectional_attention: bool,
@@ -309,7 +319,7 @@ impl ForwardPassState {
// Classifier-specific buffers
let model_dim = model_shape.main_shape(1)[1];
let classifier_state = Self::init_classifier_buffers(
- &context,
+ context.as_ref(),
model_shape,
model_dim,
num_labels,
@@ -332,20 +342,19 @@ impl ForwardPassState {
}
fn init_classifier_attention_bias(
- scratch: &ScratchBuffers<Metal>,
+ scratch: &ScratchBuffers<B>,
suffix_length: usize,
bidirectional_attention: bool,
- ) -> HashMap<Option<usize>, ArrayCell> {
- let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =
- scratch
- .attention_window_size_to_bias
- .iter()
- .map(|(window_size, buffer)| {
- let attention_bias_shape = [suffix_length, suffix_length];
- let array = buffer.borrow().view(&attention_bias_shape);
- (*window_size, array)
- })
- .collect();
+ ) -> HashMap<Option<usize>, ArrayCell<B>> {
+ let mut attention_bias_map: HashMap<Option<usize>, Array<B>> = scratch
+ .attention_window_size_to_bias
+ .iter()
+ .map(|(window_size, buffer)| {
+ let attention_bias_shape = [suffix_length, suffix_length];
+ let array = buffer.borrow().view(&attention_bias_shape);
+ (*window_size, array)
+ })
+ .collect();
for (window, bias_array) in attention_bias_map.iter_mut() {
if bidirectional_attention {
@@ -386,22 +395,22 @@ impl ForwardPassState {
#[cfg(feature = "tracing")]
fn init_classifier_buffers(
- context: &MTLContext,
+ context: &B::Context,
model_shape: &ModelShape,
model_dim: usize,
num_labels: usize,
suffix_length: usize,
- ) -> ClassifierModeState {
+ ) -> ClassifierModeState<B> {
let data_type = model_shape.activation_data_type();
let batch_size = 1;
- let create_buffer = |size: usize| -> ArrayCell {
+ let create_buffer = |size: usize| -> ArrayCell<B> {
let buffer_size = size * data_type.size_in_bytes();
let buffer = context
.create_buffer(buffer_size)
.expect("Failed to create buffer");
RefCell::new(unsafe {
- MetalArray::from_parts(
+ Array::from_parts(
buffer,
0,
&[batch_size, size / batch_size],
@@ -421,7 +430,7 @@ impl ForwardPassState {
.create_buffer(buffer_size)
.expect("Failed to create buffer");
RefCell::new(unsafe {
- MetalArray::from_parts(
+ Array::from_parts(
buffer,
0,
&[batch_size, num_labels],
@@ -440,23 +449,23 @@ impl ForwardPassState {
#[cfg(not(feature = "tracing"))]
fn init_classifier_buffers(
- context: &MTLContext,
+ context: &B::Context,
model_shape: &ModelShape,
model_dim: usize,
num_labels: usize,
_suffix_length: usize,
- ) -> ClassifierModeState {
+ ) -> ClassifierModeState<B> {
let data_type = model_shape.activation_data_type();
let batch_size = 1;
- let create_buffer = |dims: &[usize]| -> ArrayCell {
+ let create_buffer = |dims: &[usize]| -> ArrayCell<B> {
let size: usize = dims.iter().product();
let buffer_size = size * data_type.size_in_bytes();
let buffer = context
.create_buffer(buffer_size)
.expect("Failed to create buffer");
RefCell::new(unsafe {
- MetalArray::from_parts(buffer, 0, dims, data_type)
+ Array::from_parts(buffer, 0, dims, data_type)
})
};
@@ -472,8 +481,8 @@ impl ForwardPassState {
// Accessors
// ========================================================================
- pub fn mtl_context(&self) -> &Rc<MTLContext> {
- &self.context
+ pub fn mtl_context(&self) -> &B::Context {
+ self.context.as_ref()
}
pub fn aux_buffers_suffix_length(&self) -> usize {
@@ -488,25 +497,25 @@ impl ForwardPassState {
matches!(self.mode, ForwardPassMode::Classifier(_))
}
- pub fn token_bitmask(&self) -> Option<&ArrayCell> {
+ pub fn token_bitmask(&self) -> Option<&ArrayCell<B>> {
self.token_bitmask.as_ref()
}
- pub fn llm_state(&self) -> &LanguageModelGeneratorModeState {
+ pub fn llm_state(&self) -> &LanguageModelGeneratorModeState<B> {
match &self.mode {
ForwardPassMode::LanguageModelGenerator(state) => state,
_ => panic!("Not in LLM mode"),
}
}
- pub fn llm_state_mut(&mut self) -> &mut LanguageModelGeneratorModeState {
+ pub fn llm_state_mut(&mut self) -> &mut LanguageModelGeneratorModeState<B> {
match &mut self.mode {
ForwardPassMode::LanguageModelGenerator(state) => state,
_ => panic!("Not in LLM mode"),
}
}
- pub fn classifier_state(&self) -> &ClassifierModeState {
+ pub fn classifier_state(&self) -> &ClassifierModeState<B> {
match &self.mode {
ForwardPassMode::Classifier(state) => state,
_ => panic!("Not in classifier mode"),
@@ -516,7 +525,7 @@ impl ForwardPassState {
pub fn array_cell(
&self,
id: ArrayId,
- ) -> ArrayCell {
+ ) -> ArrayCell<B> {
match id {
// Common arrays
ArrayId::TokenIds => self.token_ids.clone(),
@@ -752,7 +761,7 @@ impl ForwardPassState {
fn shared_buffer_array(
&self,
id: ArrayId,
- ) -> Option<ArrayCell> {
+ ) -> Option<ArrayCell<B>> {
let shared = self.shared_buffers.borrow();
match id {
ArrayId::RopeCosines(rope_type) => Some(match rope_type {
@@ -787,11 +796,11 @@ impl ForwardPassState {
}
}
- pub fn conv_padded_buffer(&self) -> Option<ArrayCell> {
+ pub fn conv_padded_buffer(&self) -> Option<ArrayCell<B>> {
self.llm_aux.as_ref().and_then(|aux| aux.ssm_conv_padded.clone())
}
- pub fn short_conv_padded_buffer(&self) -> Option<ArrayCell> {
+ pub fn short_conv_padded_buffer(&self) -> Option<ArrayCell<B>> {
self.llm_aux.as_ref().and_then(|aux| aux.short_conv_padded.clone())
}
@@ -802,14 +811,14 @@ impl ForwardPassState {
pub fn arrays(
&self,
ids: &[ArrayId],
- ) -> Box<[ArrayCell]> {
+ ) -> Box<[ArrayCell<B>]> {
ids.iter().map(|id| self.array_cell(*id)).collect()
}
pub fn hashmaps(
&self,
ids: &[HashMapId],
- ) -> Box<[HashMap<Option<usize>, ArrayCell>]> {
+ ) -> Box<[HashMap<Option<usize>, ArrayCell<B>>]> {
ids.iter()
.map(|id| match id {
HashMapId::AttentionBias => self.attention_bias.clone(),
@@ -855,7 +864,7 @@ impl ForwardPassState {
}
}
- pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers>>> {
+ pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers<B>>>> {
match &self.mode {
ForwardPassMode::LanguageModelGenerator(state) => {
Some(&state.cache_layers)
@@ -864,7 +873,7 @@ impl ForwardPassState {
}
}
- pub fn sampling_output(&self) -> Option<&ArrayCell> {
+ pub fn sampling_output(&self) -> Option<&ArrayCell<B>> {
match &self.mode {
ForwardPassMode::LanguageModelGenerator(state) => {
state.sampling_output.as_ref()
@@ -874,7 +883,7 @@ impl ForwardPassState {
}
#[cfg(feature = "tracing")]
- pub fn traces(&self) -> &Rc<RefCell<ActivationTrace>> {
+ pub fn traces(&self) -> &Rc<RefCell<ActivationTrace<B>>> {
match &self.mode {
ForwardPassMode::LanguageModelGenerator(state) => &state.traces,
ForwardPassMode::Classifier(state) => &state.traces,
@@ -904,7 +913,7 @@ impl ForwardPassState {
pub fn copy_array(
&self,
source_array_id: ArrayId,
- destination_array: RefCell<MetalArray>,
+ destination_array: RefCell<Array<B>>,
) {
destination_array
.borrow_mut()
@@ -913,30 +922,22 @@ impl ForwardPassState {
pub fn encode_copy_array(
&self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ command_buffer: &B::CommandBuffer,
source_array_id: ArrayId,
- destination_array: RefCell<MetalArray>,
+ destination_array: RefCell<Array<B>>,
) {
let source_ref = self.arrays(&[source_array_id])[0].clone();
let src_borrow = source_ref.borrow();
let dst_borrow = destination_array.borrow();
- let src_buf = src_borrow.buffer().clone();
- let dst_buf = dst_borrow.buffer().clone();
+ let src_buf = src_borrow.buffer();
+ let dst_buf = dst_borrow.buffer();
let copy_size_bytes = dst_borrow.size();
debug_assert_eq!(dst_borrow.size(), src_borrow.size());
- let blit_encoder = command_buffer
- .new_blit_command_encoder()
- .expect("Failed to create blit command encoder");
- blit_encoder.copy_buffer_to_buffer(
- &src_buf,
- 0,
- &dst_buf,
- 0,
- copy_size_bytes,
- );
- blit_encoder.end_encoding();
+ command_buffer.with_copy_encoder(|encoder| {
+ encoder.encode_copy(src_buf, dst_buf, copy_size_bytes);
+ });
}
}
diff --git a/crates/uzu/src/backends/metal/forward_pass/traces.rs b/crates/uzu/src/forward_pass/traces.rs
similarity index 78%
rename from crates/uzu/src/backends/metal/forward_pass/traces.rs
rename to crates/uzu/src/forward_pass/traces.rs
index 6ab3c8e04..2e6a4fc34 100644
--- a/crates/uzu/src/backends/metal/forward_pass/traces.rs
+++ b/crates/uzu/src/forward_pass/traces.rs
@@ -2,27 +2,25 @@ use std::{cell::RefCell, rc::Rc};
use crate::{
DataType,
- array::ArrayContextExt,
- backends::metal::{MTLContext, MetalArray},
+ array::{ArrayCell, ArrayContextExt},
+ backends::common::Backend,
forward_pass::model_shape::ModelShape,
};
-type ArrayCell = RefCell<MetalArray>;
-
-fn create_trace_array(
- context: &MTLContext,
+fn create_trace_array<B: Backend>(
+ context: &B::Context,
shape: &[usize],
data_type: DataType,
label: &str,
-) -> ArrayCell {
+) -> ArrayCell<B> {
RefCell::new(context.create_array_uninitialized(shape, data_type, label))
}
-fn create_layer_results(
- context: &MTLContext,
+fn create_layer_results<B: Backend>(
+ context: &B::Context,
model_shape: &ModelShape,
suffix_length: usize,
-) -> Vec<Rc<RefCell<LayerActivationTrace>>> {
+) -> Vec<Rc<RefCell<LayerActivationTrace<B>>>> {
(0..model_shape.num_layers)
.map(|_| {
Rc::new(RefCell::new(LayerActivationTrace::new(
@@ -34,21 +32,21 @@ fn create_layer_results(
.collect()
}
-pub struct LayerActivationTrace {
- pub inputs: ArrayCell,
- pub pre_attention_norm: ArrayCell,
- pub attention: ArrayCell,
- pub post_attention_norm: ArrayCell,
- pub mlp_inputs: ArrayCell,
- pub pre_mlp_norm: ArrayCell,
- pub mlp: ArrayCell,
- pub post_mlp_norm: ArrayCell,
- pub outputs: ArrayCell,
+pub struct LayerActivationTrace<B: Backend> {
+ pub inputs: ArrayCell<B>,
+ pub pre_attention_norm: ArrayCell<B>,
+ pub attention: ArrayCell<B>,
+ pub post_attention_norm: ArrayCell<B>,
+ pub mlp_inputs: ArrayCell<B>,
+ pub pre_mlp_norm: ArrayCell<B>,
+ pub mlp: ArrayCell<B>,
+ pub post_mlp_norm: ArrayCell<B>,
+ pub outputs: ArrayCell<B>,
}
-impl LayerActivationTrace {
+impl<B: Backend> LayerActivationTrace<B> {
pub fn new(
- context: &MTLContext,
+ context: &B::Context,
model_shape: &ModelShape,
suffix_length: usize,
) -> Self {
@@ -81,17 +79,17 @@ impl LayerActivationTrace {
}
}
-pub struct ActivationTrace {
- pub embedding_norm: Option<ArrayCell>,
- pub layer_results: Vec<Rc<RefCell<LayerActivationTrace>>>,
- pub output_norm: ArrayCell,
- pub output_pooling: Option<ArrayCell>,
- pub logits: ArrayCell,
+pub struct ActivationTrace<B: Backend> {
+ pub embedding_norm: Option<ArrayCell<B>>,
+ pub layer_results: Vec<Rc<RefCell<LayerActivationTrace<B>>>>,
+ pub output_norm: ArrayCell<B>,
+ pub output_pooling: Option<ArrayCell<B>>,
+ pub logits: ArrayCell<B>,
}
-impl ActivationTrace {
+impl<B: Backend> ActivationTrace<B> {
pub fn new_llm(
- context: &MTLContext,
+ context: &B::Context,
model_shape: &ModelShape,
suffix_length: usize,
) -> Self {
@@ -120,7 +118,7 @@ impl ActivationTrace {
}
pub fn new_classifier(
- context: &MTLContext,
+ context: &B::Context,
model_shape: &ModelShape,
suffix_length: usize,
num_labels: usize,
@@ -159,16 +157,14 @@ impl ActivationTrace {
),
}
}
-}
-impl ActivationTrace {
- pub fn embedding_norm(&self) -> &ArrayCell {
+ pub fn embedding_norm(&self) -> &ArrayCell<B> {
self.embedding_norm
.as_ref()
.expect("embedding_norm is only available for classifier traces")
}
- pub fn output_pooling(&self) -> &ArrayCell {
+ pub fn output_pooling(&self) -> &ArrayCell<B> {
self.output_pooling
.as_ref()
.expect("output_pooling is only available for classifier traces")
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index c9b42fcb1..526248a34 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -17,13 +17,14 @@ use crate::{
common::kernel::MaskUpdateKernel,
metal::{
MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt,
- MTLCommandBufferHandler, MTLCommandEncoder, MTLCommandQueue,
- forward_pass::{
- AttentionBiasUpdate, EncodableBlock, EncodingParameters,
- ForwardPassState, INVALID_POSITION,
- },
+ MTLCommandBufferHandler, MTLCommandEncoder, MTLCommandQueue, Metal,
},
},
+ encodable_block::{EncodableBlock, EncodingParameters},
+ forward_pass::{
+ kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION},
+ state::ForwardPassState,
+ },
session::{
config::DecodingConfig,
parameter::{ConfigResolvableValue, ResolvableValue, SamplingMethod},
@@ -132,7 +133,7 @@ impl LanguageModelGenerator {
.chain(flat_trie.token_seeds())
.chunks(prefill_step_size);
- let mut last_state: Option<ForwardPassState> = None;
+ let mut last_state: Option<ForwardPassState<Metal>> = None;
let mut run_times: Vec<f64> = Vec::new();
// Process each prefill step and update the KV cache.
@@ -726,7 +727,7 @@ impl LanguageModelGenerator {
allow_pre_encode: bool,
sampling_method: SamplingMethod,
should_fill_attention_bias: bool,
- ) -> (ForwardPassState, f64) {
+ ) -> (ForwardPassState<Metal>, f64) {
objc2::rc::autoreleasepool(|_pool| {
let run_start = Instant::now();
@@ -812,7 +813,7 @@ impl LanguageModelGenerator {
fn sample(
&mut self,
- state: &mut ForwardPassState,
+ state: &mut ForwardPassState<Metal>,
) -> Result<Vec<u64>, Error> {
let sampling_output = state.sampling_output()
.expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index 411195aab..f305c41bd 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -1,12 +1,13 @@
use std::mem::size_of;
use super::LanguageModelGeneratorContext;
-use crate::backends::{
- common::Context,
- metal::{
- BufferExt, MTLBuffer, ProtocolObject, Retained,
- forward_pass::{EncodableBlock, EncodingParameters, ForwardPassState},
+use crate::{
+ backends::{
+ common::Context,
+ metal::{BufferExt, MTLBuffer, Metal, ProtocolObject, Retained},
},
+ encodable_block::{EncodableBlock, EncodingParameters},
+ forward_pass::state::ForwardPassState,
};
pub struct LanguageModelGeneratorEncodedTask {
@@ -85,7 +86,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
context: &mut LanguageModelGeneratorContext,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
should_fill_attention_bias: bool,
- ) -> ForwardPassState {
+ ) -> ForwardPassState<Metal> {
ForwardPassState::new_llm(
context.mtl_context.clone(),
&context.decoder_config,
@@ -112,8 +113,8 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
pub fn build_encoded_task(
&self,
context: &LanguageModelGeneratorContext,
- state: &mut ForwardPassState,
- parameters: &EncodingParameters,
+ state: &mut ForwardPassState<Metal>,
+ parameters: &EncodingParameters<Metal>,
key: String,
) -> LanguageModelGeneratorEncodedTask {
context.executables.encode(state, &context.command_buffer, parameters);
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index 5cefdc32e..2f6c7c009 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -19,8 +19,8 @@ use tokenizers::Tokenizer;
use xgrammar::TokenizerInfo;
use crate::{
- backends::metal::forward_pass::cache_layers::CacheLayer,
config::{MixerConfig, ModelMetadata},
+ forward_pass::cache_layers::CacheLayer,
language_model::{
LanguageModelGenerator,
grammar::CompiledGrammar,
diff --git a/crates/uzu/src/session/helpers/context.rs b/crates/uzu/src/session/helpers/context.rs
index 532ff8de3..c55e0d4bb 100644
--- a/crates/uzu/src/session/helpers/context.rs
+++ b/crates/uzu/src/session/helpers/context.rs
@@ -1,18 +1,18 @@
use crate::{
- backends::metal::forward_pass::cache_layers::CacheLayers,
+ backends::metal::Metal, forward_pass::cache_layers::CacheLayers,
session::config::DecodingConfig,
};
pub struct Context {
pub tokens: Vec<u64>,
- pub cache_layers: CacheLayers,
+ pub cache_layers: CacheLayers<Metal>,
pub decoding_config: DecodingConfig,
}
impl Context {
pub fn new(
tokens: Vec<u64>,
- cache_layers: CacheLayers,
+ cache_layers: CacheLayers<Metal>,
decoding_config: DecodingConfig,
) -> Self {
Self {
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index ca5307270..f358cae2f 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -18,17 +18,18 @@ use num_traits::NumCast;
use crate::{
ArrayElement, DataType,
backends::metal::{
- CacheLayers, KVCacheUpdate, KernelDataType, MTLCommandBuffer,
- MTLCommandQueue, MTLContext, MetalArray,
- forward_pass::{
- ArrayId, EncodableBlock, EncodingParameters, ForwardPassState,
- traces::ActivationTrace,
- },
+ KVCacheUpdate, MTLCommandBuffer, MTLCommandQueue, MTLContext, Metal,
+ MetalArray,
},
classifier::Classifier,
config::ModelMetadata,
- encodable_block::Sampling,
- forward_pass::scratch_buffers::ScratchBuffers,
+ encodable_block::{EncodableBlock, EncodingParameters, Sampling},
+ forward_pass::{
+ cache_layers::CacheLayers,
+ scratch_buffers::ScratchBuffers,
+ state::{ArrayId, ForwardPassState},
+ traces::ActivationTrace,
+ },
language_model::{
LanguageModelGeneratorContext,
sampler::{ArgmaxSampler, LogitsSampler},
@@ -562,7 +563,7 @@ impl TraceValidator {
// ========================================================================
fn validate_layer_traces(
- traces: &Rc<RefCell<ActivationTrace>>,
+ traces: &Rc<RefCell<ActivationTrace<Metal>>>,
traces_view: &ParameterTree<MTLContext>,
data_type: DataType,
) -> Vec<TracerValidationResult> {
@@ -677,7 +678,7 @@ impl TraceValidator {
}
fn validate_classifier_traces(
- traces: &Rc<RefCell<ActivationTrace>>,
+ traces: &Rc<RefCell<ActivationTrace<Metal>>>,
traces_view: &ParameterTree<MTLContext>,
data_type: DataType,
) -> Vec<TracerValidationResult> {
@@ -1012,7 +1013,7 @@ impl TraceValidator {
);
context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(
- &context.mtl_context,
+ context.mtl_context.as_ref(),
&context.model_shape,
resolved_prefix_length,
desired_suffix_length,
@@ -1020,12 +1021,11 @@ impl TraceValidator {
let intermediate_dtype: DataType =
decoder_config.output_norm_config.scale_precision.into();
- let kernel_dtype: KernelDataType = intermediate_dtype.into();
context.kv_cache_update = Box::new(
KVCacheUpdate::new(
- &context.mtl_context,
- kernel_dtype,
+ context.mtl_context.as_ref(),
+ intermediate_dtype,
resolved_prefix_length,
)
.expect("Failed to create KV cache update kernel"),
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index 23a5f8577..c2ee30066 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -6,10 +6,10 @@ use uzu::{
array::ArrayContextExt,
backends::{
common::Context,
- metal::{
- KVCacheUpdate, KernelDataType, MTLContext,
- forward_pass::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},
- },
+ metal::{KVCacheUpdate, MTLContext, Metal},
+ },
+ forward_pass::kv_cache_layer::{
+ INVALID_POSITION, KVCacheLayer, KVCacheLayerState,
},
};
@@ -37,7 +37,7 @@ fn make_test_layer(
state: KVCacheLayerState,
prefix_capacity: usize,
suffix_capacity: usize,
-) -> KVCacheLayer {
+) -> KVCacheLayer<Metal> {
let total_len = match &state {
KVCacheLayerState::Full {
..
@@ -79,7 +79,7 @@ fn make_test_layer(
}
}
-fn fill_arrays(layer: &mut KVCacheLayer) -> (Vec<f32>, Vec<f32>) {
+fn fill_arrays(layer: &mut KVCacheLayer<Metal>) -> (Vec<f32>, Vec<f32>) {
let initial_keys = {
let mut keys_ref = layer.keys.borrow_mut();
let slice = keys_ref.as_slice_mut::<f32>();
@@ -143,7 +143,7 @@ fn expected_after_update(
}
fn collect_mask(
- layer: &KVCacheLayer,
+ layer: &KVCacheLayer<Metal>,
suffix_positions: &[usize],
) -> Vec<Vec<bool>> {
let suffix_length = suffix_positions.len();
@@ -205,19 +205,17 @@ fn run_scenario(
scenario.name
);
- let kv_cache_update = match KVCacheUpdate::new(
- context,
- KernelDataType::Float32,
- total_sequence_length,
- ) {
- Ok(update) => update,
- Err(e) => {
- panic!(
- "Failed to create KV cache update for scenario {}: {:?}",
- scenario.name, e
- );
- },
- };
+ let kv_cache_update =
+ match KVCacheUpdate::new(context, DataType::F32, total_sequence_length)
+ {
+ Ok(update) => update,
+ Err(e) => {
+ panic!(
+ "Failed to create KV cache update for scenario {}: {:?}",
+ scenario.name, e
+ );
+ },
+ };
let command_buffer = context
.command_queue
diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rs
index 922032495..1af8f358c 100644
--- a/crates/uzu/tests/kv_cache_update_test.rs
+++ b/crates/uzu/tests/kv_cache_update_test.rs
@@ -6,11 +6,12 @@ use metal::{
};
use ndarray::{Array, Array3, s};
use uzu::backends::{
- common::Context,
- metal::{
- KVCacheUpdate, KernelDataType, MTLContext,
- kernel::kv_cache_update::{KVLayerData, Swap, create_swaps_direct},
+ common::{
+ Context,
+ gpu_types::Swap,
+ kernel::kv_cache_update::{KVLayerData, create_swaps_direct},
},
+ metal::{KVCacheUpdate, MTLContext, Metal},
};
fn apply_swaps_3d<T: Clone>(
@@ -51,7 +52,7 @@ fn test_random_pattern(context: &MTLContext) {
let max_sequence_length = 256usize;
let kv_cache_update = match KVCacheUpdate::new(
context,
- KernelDataType::Float32,
+ uzu::DataType::F32,
max_sequence_length,
) {
Ok(update) => update,
@@ -104,7 +105,7 @@ fn test_random_pattern(context: &MTLContext) {
)
.expect("Failed to create buffer");
- let kv_layer_data = KVLayerData {
+ let kv_layer_data = KVLayerData::<Metal> {
key_buffer: key_buffer.clone(),
key_shape: [num_heads, seq_len, head_dim],
value_buffer: value_buffer.clone(),