A high-performance inference engine for AI models
generic forward pass state 2 (#161)
uuuvn23ad4f02e45f
diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rsindex 850be99b7..15452405e 100644--- a/crates/uzu/build/common/traitgen.rs+++ b/crates/uzu/build/common/traitgen.rs@@ -4,6 +4,7 @@ use anyhow::{Context, bail};use itertools::Itertools;use proc_macro2::TokenStream;use quote::{format_ident, quote};+use syn::Type;use crate::common::{codegen::write_tokens,@@ -21,7 +22,7 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {let ty = match &p.ty {KernelParameterType::DType => quote! { DataType },KernelParameterType::Specialization(ty) => {- let ty = format_ident!("{}", ty.as_ref());+ let ty: Type = syn::parse_str(ty.as_ref()).unwrap();quote! { #ty }},};@@ -39,11 +40,11 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {quote! { &<Self::Backend as Backend>::NativeBuffer }},KernelArgumentType::Constant(ty) => {- let ty = format_ident!("{}", ty.as_ref());+ let ty: Type = syn::parse_str(ty.as_ref()).unwrap();quote! { &[#ty] }},KernelArgumentType::Scalar(ty) => {- let ty = format_ident!("{}", ty.as_ref());+ let ty: Type = syn::parse_str(ty.as_ref()).unwrap();quote! { #ty }},};@@ -58,8 +59,8 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {fn new(context: &<Self::Backend as Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as Backend>::Error>;- fn encode(&self, #(#args ,)* encoder: &<Self::Backend as Backend>::EncoderRef);- fn encode_if(&self, #(#args ,)* encoder: &<Self::Backend as Backend>::EncoderRef, predicate: Option<&<Self::Backend as Backend>::NativeBuffer>);+ fn encode(&self, #(#args ,)* encoder: &<Self::Backend as Backend>::ComputeEncoder);+ fn encode_if(&self, #(#args ,)* encoder: &<Self::Backend as Backend>::ComputeEncoder, predicate: Option<&<Self::Backend as Backend>::NativeBuffer>);}};diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rsindex df079d2eb..84d2f09fa 100644--- a/crates/uzu/build/metal/ast.rs+++ b/crates/uzu/build/metal/ast.rs@@ -135,16 +135,19 @@ pub struct MetalArgument {}impl MetalArgument {- fn scalar_type_to_rust(c_type: &str) -> anyhow::Result<&'static str> {+ fn scalar_type_to_rust(c_type: &str) -> anyhow::Result<Box<str>> {let mut tokens: Vec<_> = c_type.split_whitespace().collect();if tokens.first() == Some(&"const") {tokens.remove(0);}match tokens.as_slice() {- ["bool"] => Ok("bool"),- ["uint"] | ["uint32_t"] | ["unsigned", "int"] => Ok("u32"),- ["int"] | ["int32_t"] => Ok("i32"),- ["float"] => Ok("f32"),+ ["bool"] => Ok("bool".into()),+ ["uint"] | ["uint32_t"] | ["unsigned", "int"] => Ok("u32".into()),+ ["int"] | ["int32_t"] => Ok("i32".into()),+ ["float"] => Ok("f32".into()),+ [vpath] if vpath.starts_with("uzu::") => Ok(vpath+ .replacen("uzu::", "crate::backends::common::gpu_types::", 1)+ .into()),_ => bail!("unknown scalar type: {c_type}"),}}diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex b272f900d..7b86a977b 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -4,7 +4,7 @@ use anyhow::Context;use itertools::Itertools;use proc_macro2::TokenStream;use quote::{format_ident, quote};-use syn::LitInt;+use syn::{LitInt, Type};use crate::metal::ast::MetalGroupsType;@@ -135,7 +135,7 @@ pub fn bindgen(Some((def, set, quote! { #arg_name }))}MetalArgumentType::Constant((r_type, constant_type)) => {- let arg_dtype = format_ident!("{r_type}");+ let arg_dtype: Type = syn::parse_str(&r_type).unwrap();let (def, set) = match constant_type {MetalConstantType::Scalar => (diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rsindex 17a2e9782..e0b7b3aaa 100644--- a/crates/uzu/src/backends/common/backend.rs+++ b/crates/uzu/src/backends/common/backend.rs@@ -1,12 +1,13 @@-use std::error::Error;+use std::{error::Error, fmt::Debug};-use super::{CommandBuffer, Context, Kernels, NativeBuffer};+use super::{CommandBuffer, Context, CopyEncoder, Kernels, NativeBuffer};-pub trait Backend: Clone {+pub trait Backend: Debug + Clone {type Context: Context<Backend = Self>;- type CommandBuffer: CommandBuffer<Backend = Self>;type NativeBuffer: NativeBuffer<Backend = Self>;- type EncoderRef;+ type CommandBuffer: CommandBuffer<Backend = Self>;+ type ComputeEncoder;+ type CopyEncoder: CopyEncoder<Backend = Self>;type Kernels: Kernels<Backend = Self>;- type Error: Error;+ type Error: Error + Debug;}diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex 0315e12ad..db22f269a 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -3,8 +3,13 @@ use super::Backend;pub trait CommandBuffer {type Backend: Backend;- fn with_encoder<T>(+ fn with_compute_encoder<T>(&self,- callback: impl FnOnce(&<Self::Backend as Backend>::EncoderRef) -> T,+ callback: impl FnOnce(&<Self::Backend as Backend>::ComputeEncoder) -> T,+ ) -> T;++ fn with_copy_encoder<T>(+ &self,+ callback: impl FnOnce(&<Self::Backend as Backend>::CopyEncoder) -> T,) -> T;}diff --git a/crates/uzu/src/backends/common/copy_encoder.rs b/crates/uzu/src/backends/common/copy_encoder.rsnew file mode 100644index 000000000..f68665a2f--- /dev/null+++ b/crates/uzu/src/backends/common/copy_encoder.rs@@ -0,0 +1,12 @@+use super::Backend;++pub trait CopyEncoder {+ type Backend: Backend;++ fn encode_copy(+ &self,+ src: &<Self::Backend as Backend>::NativeBuffer,+ dst: &<Self::Backend as Backend>::NativeBuffer,+ size: usize,+ );+}diff --git a/crates/uzu/src/backends/common/gpu_types/kv_cache_update.rs b/crates/uzu/src/backends/common/gpu_types/kv_cache_update.rsnew file mode 100644index 000000000..193175e2b--- /dev/null+++ b/crates/uzu/src/backends/common/gpu_types/kv_cache_update.rs@@ -0,0 +1,6 @@+#[derive(Clone, Copy, Debug)]+#[repr(C)]+pub struct Swap {+ pub source: u32,+ pub destination: u32,+}diff --git a/crates/uzu/src/backends/common/gpu_types/mod.rs b/crates/uzu/src/backends/common/gpu_types/mod.rsindex 795fdc68b..a6ba1de5c 100644--- a/crates/uzu/src/backends/common/gpu_types/mod.rs+++ b/crates/uzu/src/backends/common/gpu_types/mod.rs@@ -3,8 +3,10 @@//! These `#[repr(C)]` structs are the source of truth. The build system uses//! cbindgen to generate C headers for Metal shaders.-mod attention;-mod matmul;+pub mod attention;+pub mod kv_cache_update;+pub mod matmul;pub use attention::*;+pub use kv_cache_update::*;pub use matmul::*;diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rsnew file mode 100644index 000000000..ab6af972f--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs@@ -0,0 +1,163 @@+use thiserror::Error;++use crate::{+ DataType,+ backends::common::{+ Backend, CommandBuffer, Kernels, gpu_types::Swap,+ kernel::KVCacheUpdateKernel,+ },+};++pub struct KVLayerData<B: Backend> {+ pub key_buffer: B::NativeBuffer,+ pub key_shape: [usize; 3],+ pub value_buffer: B::NativeBuffer,+ pub value_shape: [usize; 3],+}++pub struct KVCacheUpdate<B: Backend> {+ kernel: <B::Kernels as Kernels>::KVCacheUpdateKernel,+ max_sequence_length: usize,+}++#[derive(Debug, Error)]+pub enum KVCacheUpdateError<B: Backend> {+ #[error("Source and destination indices length mismatch")]+ IndicesCountMismatch,+ #[error("Shape mismatch between key and value tensors")]+ ShapeMismatch,+ #[error("Unsupported data type")]+ UnsupportedDataType,+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Swap count exceeds maximum sequence length")]+ MaxSequenceLengthExceeded,+ #[error("Buffer creation failed")]+ BufferCreationFailed,+ #[error("Function not found: {0}")]+ FunctionNotFound(String),+}++impl<B: Backend> KVCacheUpdate<B> {+ pub fn new(+ context: &B::Context,+ data_type: DataType,+ max_sequence_length: usize,+ ) -> Result<Self, KVCacheUpdateError<B>> {+ let kernel = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(+ context, data_type,+ )+ .map_err(KVCacheUpdateError::BackendError)?;++ Ok(Self {+ kernel,+ max_sequence_length,+ })+ }++ pub fn encode(+ &self,+ in_place_data: &[KVLayerData<B>],+ source_indices: &[usize],+ destination_indices: &[usize],+ command_buffer: &B::CommandBuffer,+ ) -> Result<(), KVCacheUpdateError<B>> {+ command_buffer.with_compute_encoder(|encoder| {+ self.encode_with_encoder(+ in_place_data,+ source_indices,+ destination_indices,+ encoder,+ )+ })+ }++ /// Encode the KV cache update operation using a provided compute encoder+ pub fn encode_with_encoder(+ &self,+ in_place_data: &[KVLayerData<B>],+ source_indices: &[usize],+ destination_indices: &[usize],+ encoder: &B::ComputeEncoder,+ ) -> Result<(), KVCacheUpdateError<B>> {+ if source_indices.len() != destination_indices.len() {+ return Err(KVCacheUpdateError::IndicesCountMismatch);+ }++ let swaps = create_swaps_direct(source_indices, destination_indices);+ if swaps.len() > self.max_sequence_length {+ return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);+ }++ // For small swap counts, use set_bytes to avoid shared buffer race in async mode.+ // Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.+ let use_inline_bytes = swaps.len() <= 32;+ assert!(+ use_inline_bytes,+ "non-inline is not supported yet (and is broken anyways due to a data race)"+ );++ for layer_data in in_place_data {+ if layer_data.key_shape != layer_data.value_shape {+ return Err(KVCacheUpdateError::ShapeMismatch);+ }++ let [num_heads, max_sequence_length, head_dim] =+ layer_data.key_shape;++ self.kernel.encode(+ &layer_data.key_buffer,+ &layer_data.value_buffer,+ &swaps,+ swaps.len() as u32,+ num_heads as u32,+ max_sequence_length as u32,+ head_dim as u32,+ encoder,+ );+ }++ Ok(())+ }+}++pub fn create_swaps_direct(+ source_indices: &[usize],+ destination_indices: &[usize],+) -> Vec<Swap> {+ if source_indices.len() != destination_indices.len() {+ return Vec::new();+ }++ let mut swaps = Vec::with_capacity(source_indices.len());++ for (&src, &dst) in source_indices.iter().zip(destination_indices.iter()) {+ if src != dst {+ swaps.push(Swap {+ source: src as u32,+ destination: dst as u32,+ });+ }+ }++ swaps+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_direct_swaps() {+ let sources = [0, 2, 4];+ let destinations = [1, 3, 5];+ let swaps = create_swaps_direct(&sources, &destinations);+ assert_eq!(swaps.len(), 3);+ assert_eq!(swaps[0].source, 0);+ assert_eq!(swaps[0].destination, 1);+ assert_eq!(swaps[1].source, 2);+ assert_eq!(swaps[1].destination, 3);+ assert_eq!(swaps[2].source, 4);+ assert_eq!(swaps[2].destination, 5);+ }+}diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rsindex dc38e9b24..fb7523be0 100644--- a/crates/uzu/src/backends/common/kernel/mod.rs+++ b/crates/uzu/src/backends/common/kernel/mod.rs@@ -2,4 +2,5 @@include!(concat!(env!("OUT_DIR"), "/traits.rs"));+pub mod kv_cache_update;pub mod sampling;diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rsindex bb8737f48..5fc3cffb5 100644--- a/crates/uzu/src/backends/common/kernel/sampling.rs+++ b/crates/uzu/src/backends/common/kernel/sampling.rs@@ -184,7 +184,7 @@ impl<B: Backend> SamplingKernel<B> {sampling_method: SamplingMethod,batch_size: usize,vocab_size: usize,- compute_encoder: &B::EncoderRef,+ compute_encoder: &B::ComputeEncoder,) -> Result<(), SamplingError<B>> {if batch_size > self.max_batch_size {return Err(SamplingError::BatchSizeExceeded(diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rsindex 392acdf4c..2f561b239 100644--- a/crates/uzu/src/backends/common/mod.rs+++ b/crates/uzu/src/backends/common/mod.rs@@ -4,6 +4,7 @@ mod buffer;mod buffer_lifetime;mod command_buffer;mod context;+mod copy_encoder;pub mod gpu_types;pub mod kernel;mod native_buffer;@@ -14,5 +15,6 @@ pub use buffer::Buffer;pub use buffer_lifetime::BufferLifetime;pub use command_buffer::CommandBuffer;pub use context::Context;+pub use copy_encoder::CopyEncoder;pub use kernel::Kernels;pub use native_buffer::NativeBuffer;diff --git a/crates/uzu/src/backends/common/native_buffer.rs b/crates/uzu/src/backends/common/native_buffer.rsindex 80d09ae5c..9ea6e7ef6 100644--- a/crates/uzu/src/backends/common/native_buffer.rs+++ b/crates/uzu/src/backends/common/native_buffer.rs@@ -1,8 +1,8 @@-use std::{os::raw::c_void, ptr::NonNull};+use std::{fmt::Debug, os::raw::c_void, ptr::NonNull};use super::Backend;-pub trait NativeBuffer: Send + Sync + Clone {+pub trait NativeBuffer: Send + Sync + Debug + Clone {type Backend: Backend;fn set_label(diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rsindex 9d98fdc4a..763324f34 100644--- a/crates/uzu/src/backends/metal/backend.rs+++ b/crates/uzu/src/backends/metal/backend.rs@@ -1,4 +1,7 @@-use metal::{MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder};+use metal::{+ MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,+ MTLComputeCommandEncoder,+};use objc2::{rc::Retained, runtime::ProtocolObject};use super::{MTLContext, MTLError, MetalKernels};@@ -8,10 +11,11 @@ use crate::backends::common::Backend;pub struct Metal;impl Backend for Metal {- type NativeBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;type Context = MTLContext;+ type NativeBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;type CommandBuffer = Retained<ProtocolObject<dyn MTLCommandBuffer>>;- type EncoderRef = ProtocolObject<dyn MTLComputeCommandEncoder>;+ type ComputeEncoder = ProtocolObject<dyn MTLComputeCommandEncoder>;+ type CopyEncoder = ProtocolObject<dyn MTLBlitCommandEncoder>;type Kernels = MetalKernels;type Error = MTLError;}diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rsindex 01bcef66b..f1ccb6e94 100644--- a/crates/uzu/src/backends/metal/classifier_context.rs+++ b/crates/uzu/src/backends/metal/classifier_context.rs@@ -10,7 +10,6 @@ use super::{Pooling, Rope,transformer_layer::{embed_block, linear_block},},- forward_pass::{ArrayId, EncodableBlock, RopeType},kernel::dsl::SigmoidMetalKernel,};use crate::{@@ -20,9 +19,11 @@ use crate::{metal::error::ClassifierError,},config::{ClassifierModelConfig, ModelMetadata},+ encodable_block::EncodableBlock,forward_pass::{- model_shape::ModelShape, scratch_buffers::ScratchBuffers,- state::SharedBuffers,+ model_shape::ModelShape,+ scratch_buffers::ScratchBuffers,+ state::{ArrayId, RopeType, SharedBuffers},},parameters::ParameterLoader,session::types::Error,diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex b7cfd391a..5741b68a9 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -8,10 +8,10 @@ use super::Metal;impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {type Backend = Metal;- fn with_encoder<T>(+ fn with_compute_encoder<T>(&self,callback: impl FnOnce(- &<Self::Backend as crate::backends::common::Backend>::EncoderRef,+ &<Self::Backend as crate::backends::common::Backend>::ComputeEncoder,) -> T,) -> T {let encoder = self@@ -24,4 +24,21 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {ret}++ fn with_copy_encoder<T>(+ &self,+ callback: impl FnOnce(+ &<Self::Backend as crate::backends::common::Backend>::CopyEncoder,+ ) -> T,+ ) -> T {+ let encoder = self+ .new_blit_command_encoder()+ .expect("Failed to create blit command encoder");++ let ret = callback(&encoder);++ encoder.end_encoding();++ ret+ }}diff --git a/crates/uzu/src/backends/metal/copy_encoder.rs b/crates/uzu/src/backends/metal/copy_encoder.rsnew file mode 100644index 000000000..80394899b--- /dev/null+++ b/crates/uzu/src/backends/metal/copy_encoder.rs@@ -0,0 +1,21 @@+use metal::{MTLBlitCommandEncoder, MTLBuffer};+use objc2::runtime::ProtocolObject;++use crate::backends::common::CopyEncoder;++use super::Metal;++impl CopyEncoder for ProtocolObject<dyn MTLBlitCommandEncoder> {+ type Backend = Metal;++ fn encode_copy(+ &self,+ src: &<Self::Backend as crate::backends::common::Backend>::NativeBuffer,+ dst: &<Self::Backend as crate::backends::common::Backend>::NativeBuffer,+ size: usize,+ ) {+ assert!(src.length() <= size && dst.length() <= size);++ self.copy_buffer_to_buffer(src, 0, dst, 0, size);+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rsindex 78bd80f67..d984b3959 100644--- a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs@@ -4,15 +4,15 @@ use crate::backends::metal::{};use super::{- super::{- MTLContext, MTLError,- forward_pass::{ArrayId, ForwardPassState},- kernel::dsl::ActivationMetalKernel,- },- EncodableBlock, EncodingParameters,+ super::{MTLContext, MTLError, kernel::dsl::ActivationMetalKernel},+ EncodableBlock,};-use crate::backends::common::kernel::ActivationKernel;use crate::{DataType, config::Activation as ActivationConfig};+use crate::{+ backends::common::kernel::ActivationKernel,+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},+};pub struct Activation {kernel: ActivationMetalKernel,@@ -42,9 +42,9 @@ impl Activation {impl EncodableBlock<Metal> for Activation {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let encoder = command_buffer.new_compute_command_encoder()@@ -61,9 +61,9 @@ impl EncodableBlock<Metal> for Activation {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let input_array = arrays[0].borrow_mut();diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rsindex 8ecff4b22..8b2479303 100644--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs@@ -1,16 +1,17 @@//! Attention kernel encodable.-use super::{EncodableBlock, EncodingParameters, Metal};+use super::{EncodableBlock, Metal};use crate::backends::metal::{KernelDataType, MTLCommandBuffer, MTLCommandEncoder,MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained,- forward_pass::{ArrayId, ForwardPassState, HashMapId},kernel::attention::{AttentionError, AttentionGemmArguments, AttentionKernel,AttentionKernelVariant, AttentionSinglePassArguments,AttentionTwoPassArguments, KVCacheUpdateArguments,},};+use crate::encodable_block::EncodingParameters;+use crate::forward_pass::state::{ArrayId, ForwardPassState, HashMapId};fn env_gemm_attention_enabled() -> bool {static VALUE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();@@ -62,9 +63,9 @@ impl Attention {impl EncodableBlock<Metal> for Attention {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let compute_encoder = command_buffer.new_compute_command_encoder()@@ -84,9 +85,9 @@ impl EncodableBlock<Metal> for Attention {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let (suffix_length,@@ -317,7 +318,7 @@ impl EncodableBlock<Metal> for Attention {match variant {AttentionKernelVariant::SinglePass => {if use_gemm {- let mtl = &**state.mtl_context();+ let mtl = state.mtl_context();if let Err(e) = self.kernel.encode_gemm(mtl,compute_encoder,diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rsindex b697968c4..a258bd31a 100644--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs@@ -9,16 +9,18 @@ use crate::backends::metal::{use objc2::rc::autoreleasepool;use super::{- Attention, EncodableBlock, EncodingParameters, ForwardPassState,- Normalization, QKNorm, TensorAddSwap, TensorCopy, transformer_layer,+ Attention, EncodableBlock, Normalization, QKNorm, TensorAddSwap,+ TensorCopy, transformer_layer,};+use crate::forward_pass::state::{ArrayId, ForwardPassState};use crate::{DataType,backends::metal::{MTLContext, compilation_parameters::CompilationConfig,- forward_pass::ArrayId, kernel::KernelDataType,+ kernel::KernelDataType,},config::TransformerLayerConfig,+ encodable_block::EncodingParameters,parameters::ParameterTree,};@@ -283,9 +285,9 @@ impl ClassifierLayer {impl EncodableBlock<Metal> for ClassifierLayer {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {#[cfg(not(feature = "tracing"))]{@@ -475,9 +477,9 @@ impl EncodableBlock<Metal> for ClassifierLayer {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {debug_assert!(self.supports_shared_encoder(),diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rsindex 4f14bc79b..d56b09207 100644--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs@@ -3,9 +3,7 @@use crate::backends::metal::Metal;use std::rc::Rc;-use super::{- EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,-};+use super::{EncodableBlock, LayerExecutables, RMSNorm, Rope};use crate::{DataType, DecoderConfig,backends::metal::{@@ -13,10 +11,13 @@ use crate::{ProtocolObject, Retained,compilation_parameters::CompilationConfig,encodable_block::transformer_layer::{embed_block, readout_block},- forward_pass::{ArrayId, ForwardPassState, RopeType},},config::{DecoderLayerType, MixerConfig},- forward_pass::model_shape::ModelShape,+ encodable_block::EncodingParameters,+ forward_pass::{+ model_shape::ModelShape,+ state::{ArrayId, ForwardPassState, RopeType},+ },parameters::ParameterTree,};@@ -202,18 +203,18 @@ impl Decoder {impl EncodableBlock<Metal> for Decoder {fn encode_with_shared_encoder(&self,- _state: &mut ForwardPassState,+ _state: &mut ForwardPassState<Metal>,_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {unimplemented!("Decoder does not support shared encoder")}fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {self.embed.encode(state, command_buffer, parameters);diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rsindex 6728697e8..1085df36a 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs@@ -1,5 +1,5 @@use super::{- super::{EncodableBlock, EncodingParameters, Metal},+ super::{EncodableBlock, Metal},EmbeddingError,};use crate::{@@ -9,11 +9,11 @@ use crate::{metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,- Retained,- forward_pass::{ArrayId, ForwardPassState},- kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,+ Retained, kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,},},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -86,9 +86,9 @@ impl FullPrecisionEmbeddingLookup {impl EncodableBlock<Metal> for FullPrecisionEmbeddingLookup {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let encoder = command_buffer.new_compute_command_encoder()@@ -108,9 +108,9 @@ impl EncodableBlock<Metal> for FullPrecisionEmbeddingLookup {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);let batch_size = state.active_suffix_length();diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rsindex 4fe565361..b951ac91f 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs@@ -6,16 +6,17 @@ use crate::backends::metal::{};use super::{- super::{EncodableBlock, EncodingParameters, Metal},+ super::{EncodableBlock, Metal},EmbeddingError,};use crate::{DataType,backends::metal::{MTLContext, MTLError,- forward_pass::{ArrayId, ForwardPassState},kernel::matmul::{MatmulArguments, MatmulKernel},},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -88,9 +89,9 @@ impl FullPrecisionEmbeddingReadout {impl EncodableBlock<Metal> for FullPrecisionEmbeddingReadout {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let encoder = command_buffer.new_compute_command_encoder()@@ -110,9 +111,9 @@ impl EncodableBlock<Metal> for FullPrecisionEmbeddingReadout {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);let batch_size = state.sampling_length();diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rsindex 1fb215cb0..a8bd084f6 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs@@ -1,5 +1,5 @@use super::{- super::{EncodableBlock, EncodingParameters, Metal},+ super::{EncodableBlock, Metal},EmbeddingError,};use crate::{@@ -9,12 +9,12 @@ use crate::{metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,- Retained,- forward_pass::{ArrayId, ForwardPassState},- kernel::dsl::QuantizedEmbeddingLookupMetalKernel,+ Retained, kernel::dsl::QuantizedEmbeddingLookupMetalKernel,},},config::QuantizationMode,+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -221,9 +221,9 @@ impl QuantizedEmbeddingLookup {impl EncodableBlock<Metal> for QuantizedEmbeddingLookup {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let encoder = command_buffer.new_compute_command_encoder()@@ -243,9 +243,9 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingLookup {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);let batch_size = state.active_suffix_length();diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rsindex 576018b03..bde63ffbf 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs@@ -1,5 +1,5 @@use super::{- super::{EncodableBlock, EncodingParameters, Metal},+ super::{EncodableBlock, Metal},EmbeddingError,};use crate::{@@ -10,7 +10,6 @@ use crate::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,Retained,- forward_pass::{ArrayId, ForwardPassState},kernel::quant_matmul::{QuantizationType, QuantizedMatmulArguments,QuantizedMatmulKernel,@@ -18,6 +17,8 @@ use crate::{},},config::QuantizationMode,+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -219,9 +220,9 @@ impl QuantizedEmbeddingReadout {impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let encoder = command_buffer.new_compute_command_encoder()@@ -241,9 +242,9 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);let batch_size = state.sampling_length();diff --git a/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rs b/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rsdeleted file mode 100644index 1b28f7704..000000000--- a/crates/uzu/src/backends/metal/encodable_block/encoding_parameters.rs+++ /dev/null@@ -1,6 +0,0 @@-//! Encoding parameters for forward pass operations.--use crate::backends::metal::Metal;--pub type EncodingParameters<'a> =- crate::encodable_block::EncodingParameters<'a, Metal>;diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rsindex 7bb5a9e15..5f86335d5 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs@@ -12,20 +12,20 @@ use objc2::rc::autoreleasepool;use super::{super::{- Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm,- RMSNorm, ShortConvMixer, TensorAddSwap, TensorCopy, transformer_layer,+ Attention, EncodableBlock, MambaMixer, QKNorm, RMSNorm, ShortConvMixer,+ TensorAddSwap, TensorCopy, transformer_layer,},MixerExecutables,};use crate::{DataType, DecoderLayerConfig,backends::metal::{- MTLContext,- compilation_parameters::CompilationConfig,- forward_pass::{ArrayId, ForwardPassState},+ MTLContext, compilation_parameters::CompilationConfig,kernel::KernelDataType,},config::{DecoderLayerType, MixerConfig},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -311,9 +311,9 @@ impl LayerExecutables {impl EncodableBlock<Metal> for LayerExecutables {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {// In non-tracing builds, if every sub-block supports shared encoding,// we can run the entire layer in a single compute encoder.@@ -538,9 +538,9 @@ impl EncodableBlock<Metal> for LayerExecutables {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {debug_assert!(self.supports_shared_encoder(),diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rsindex 32d7279e4..490544054 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs@@ -7,10 +7,11 @@ use crate::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,Retained,- encodable_block::{EncodableBlock, EncodingParameters},- forward_pass::{ArrayId, ForwardPassState},+ encodable_block::EncodableBlock,kernel::matmul::{MatmulArguments, MatmulKernel},},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -106,9 +107,9 @@ impl FullPrecisionLinear {impl EncodableBlock<Metal> for FullPrecisionLinear {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();@@ -162,9 +163,9 @@ impl EncodableBlock<Metal> for FullPrecisionLinear {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rsindex 77adc96ff..357700ced 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs@@ -12,8 +12,7 @@ use crate::{DataType,backends::metal::{MTLContext, MTLError,- encodable_block::{EncodableBlock, EncodingParameters},- forward_pass::{ArrayId, ForwardPassState},+ encodable_block::EncodableBlock,kernel::{dsl::TensorAddBiasMetalKernel,quant_matmul::{@@ -23,6 +22,8 @@ use crate::{},},config::QuantizationConfig,+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -229,9 +230,9 @@ impl QuantizedLinear {impl EncodableBlock<Metal> for QuantizedLinear {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();@@ -292,9 +293,9 @@ impl EncodableBlock<Metal> for QuantizedLinear {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rsindex 99e47ffd9..f7c110a2c 100644--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs@@ -1,6 +1,6 @@//! Mamba2 SSM mixer encodable.-use super::{EncodableBlock, EncodingParameters, Metal, transformer_layer};+use super::{EncodableBlock, Metal, transformer_layer};use crate::backends::metal::kernel::ssm::ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode,};@@ -13,7 +13,6 @@ use crate::{MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,Retained,compilation_parameters::CompilationConfig,- forward_pass::{ArrayId, ForwardPassState},kernel::dsl::{SSDUpdateMetalKernel, SplitInProjMetalKernel},kernel::ssm::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,@@ -22,6 +21,8 @@ use crate::{},},config::{DecoderLayerType, Mamba2Config},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};use std::{env, rc::Rc};@@ -149,9 +150,9 @@ impl MambaMixer {fn encode_pipeline_with_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {@@ -175,7 +176,7 @@ impl MambaMixer {fn run_split_inproj(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {@@ -219,7 +220,7 @@ impl MambaMixer {fn run_conv_scan(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {@@ -335,7 +336,7 @@ impl MambaMixer {fn run_prefill_ssm(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {@@ -413,7 +414,7 @@ impl MambaMixer {fn run_decode_ssm(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {@@ -501,9 +502,9 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {impl EncodableBlock<Metal> for MambaMixer {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {let encoder = command_buffer@@ -553,9 +554,9 @@ impl EncodableBlock<Metal> for MambaMixer {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {self.encode_pipeline_with_encoder(state, encoder, parameters);}diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rsindex b32a00eec..6eb7f499f 100644--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs@@ -7,12 +7,11 @@ use crate::backends::metal::{ProtocolObject, Retained,};-use super::{EncodableBlock, EncodingParameters, Metal};+use super::{EncodableBlock, Metal};use crate::{DataType,backends::metal::{MTLContext,- forward_pass::{ArrayId, ForwardPassState},kernel::{mlp::{MlpActivationType, MlpGateActMulEncodable},mlp_fused::{MlpFusedArguments, MlpFusedKernel},@@ -23,6 +22,8 @@ use crate::{},},config::Activation,+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},};pub struct MlpBlock {@@ -48,9 +49,9 @@ impl MlpBlock {impl EncodableBlock<Metal> for MlpBlock {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- params: &EncodingParameters,+ params: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {let encoder = command_buffer@@ -97,9 +98,9 @@ impl EncodableBlock<Metal> for MlpBlock {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- params: &EncodingParameters,+ params: &EncodingParameters<Metal>,) {// Upself.up.encode_with_shared_encoder(state, encoder, params);@@ -317,9 +318,9 @@ impl MlpFusedBlock {impl EncodableBlock<Metal> for MlpFusedBlock {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- params: &EncodingParameters,+ params: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {let encoder = command_buffer@@ -361,9 +362,9 @@ impl EncodableBlock<Metal> for MlpFusedBlock {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- params: &EncodingParameters,+ params: &EncodingParameters<Metal>,) {// Fused up + activationlet arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 26e150c67..fc7339204 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -7,7 +7,6 @@//! kernels to perform operations on `ForwardPassState`.pub use super::Metal;-use super::forward_pass::ForwardPassState;pub use crate::encodable_block::EncodableBlock;mod activation;@@ -15,7 +14,6 @@ mod attention;mod classifier_layer;mod decoder;mod embedding;-mod encoding_parameters;mod layer;mod linear;mod mamba_mixer;@@ -39,7 +37,6 @@ pub use embedding::{FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,QuantizedEmbeddingReadout,};-pub use encoding_parameters::EncodingParameters;pub use layer::LayerExecutables;pub use linear::{FullPrecisionLinear, QuantizedLinear};pub(crate) use mamba_mixer::MambaMixer;diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rsindex e2bc760a1..21053540e 100644--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs@@ -3,7 +3,7 @@use std::{cell::RefCell, rc::Rc};use super::{- super::{EncodableBlock, EncodingParameters, Metal},+ super::{EncodableBlock, Metal},SharedMoeWeights,};use crate::{@@ -15,7 +15,6 @@ use crate::{KernelDataType, MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext,MetalArray, NSRange, ProtocolObject, Retained,- forward_pass::{ArrayId, ForwardPassState},kernel::{MoeGatherKernels,dsl::{@@ -32,6 +31,8 @@ use crate::{},},},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -300,9 +301,9 @@ impl MoeBlock {impl EncodableBlock<Metal> for MoeBlock {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let suffix_length = state.active_suffix_length();let arrays = state.arrays(&[@@ -644,9 +645,9 @@ impl EncodableBlock<Metal> for MoeBlock {fn encode_with_shared_encoder(&self,- _state: &mut ForwardPassState,+ _state: &mut ForwardPassState<Metal>,_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {unreachable!("MoeBlock does not support shared compute encoder");}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rsindex d88dabdee..b4fe040d8 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs@@ -1,6 +1,6 @@//! LayerNorm encodable.-use super::super::{EncodableBlock, EncodingParameters, Metal};+use super::super::{EncodableBlock, Metal};use crate::{DataType,backends::{@@ -9,11 +9,12 @@ use crate::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,MTLResourceOptions, ProtocolObject, Retained,- forward_pass::{ArrayId, ForwardPassState},kernel::dsl::LayerNormMetalKernel,},},config::{NormalizationConfig, UpcastMode},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -77,9 +78,9 @@ impl LayerNorm {impl EncodableBlock<Metal> for LayerNorm {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let compute_encoder = command_buffer.new_compute_command_encoder()@@ -99,9 +100,9 @@ impl EncodableBlock<Metal> for LayerNorm {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let input_binding = state.arrays(&[self.input_array_id]);let output_binding = state.arrays(&[self.output_array_id]);diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rsindex e892320d8..e7b65f5ea 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs@@ -8,17 +8,17 @@ pub use layer_norm::LayerNorm;pub use qk_norm::QKNorm;pub use rms_norm::RMSNorm;-use super::{EncodableBlock, EncodingParameters, Metal};+use super::{EncodableBlock, Metal};use crate::backends::metal::MTLError;use crate::{DataType,backends::metal::{MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, ProtocolObject,- Retained,- forward_pass::{ArrayId, ForwardPassState},- kernel::RMSNormError,+ Retained, kernel::RMSNormError,},config::NormalizationConfig,+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -75,9 +75,9 @@ impl Normalization {impl EncodableBlock<Metal> for Normalization {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {match self {Normalization::LayerNorm(layer_norm) => {@@ -102,9 +102,9 @@ impl EncodableBlock<Metal> for Normalization {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {match self {Normalization::LayerNorm(layer_norm) => layer_normdiff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rsindex a9694ee31..2eedae5de 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs@@ -1,19 +1,20 @@//! QK Normalization encodable.-use super::super::{EncodableBlock, EncodingParameters, Metal};+use super::super::{EncodableBlock, Metal};use crate::{DataType,backends::metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,MTLResourceOptions, ProtocolObject, Retained,- forward_pass::{ArrayId, ForwardPassState},kernel::rms_norm::{QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel,RMSNormKernelType,},},config::{NormalizationConfig, UpcastMode},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -157,9 +158,9 @@ impl QKNorm {impl EncodableBlock<Metal> for QKNorm {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let compute_encoder = command_buffer.new_compute_command_encoder()@@ -179,9 +180,9 @@ impl EncodableBlock<Metal> for QKNorm {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let qkv_binding = state.arrays(&[self.qkv_array_id]);let qkv_shape = {diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rsindex 4ec1da37c..4644e1f4a 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs@@ -1,18 +1,19 @@//! RMS Normalization encodable.-use super::super::{EncodableBlock, EncodingParameters, Metal};+use super::super::{EncodableBlock, Metal};use crate::{DataType,backends::metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,MTLResourceOptions, ProtocolObject, Retained,- forward_pass::{ArrayId, ForwardPassState},kernel::rms_norm::{RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType,},},config::{NormalizationConfig, UpcastMode},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -100,9 +101,9 @@ impl RMSNorm {impl EncodableBlock<Metal> for RMSNorm {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let compute_encoder = command_buffer.new_compute_command_encoder()@@ -122,9 +123,9 @@ impl EncodableBlock<Metal> for RMSNorm {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let input_binding = state.arrays(&[self.input_array_id]);let output_binding = state.arrays(&[self.output_array_id]);diff --git a/crates/uzu/src/backends/metal/encodable_block/pooling.rs b/crates/uzu/src/backends/metal/encodable_block/pooling.rsindex 9899cc8d2..0fe17f7b2 100644--- a/crates/uzu/src/backends/metal/encodable_block/pooling.rs+++ b/crates/uzu/src/backends/metal/encodable_block/pooling.rs@@ -1,6 +1,6 @@//! Pooling encodable for sequence-level aggregation.-use super::{EncodableBlock, EncodingParameters, Metal};+use super::{EncodableBlock, Metal};#[cfg(feature = "tracing")]use crate::backends::metal::MTLBlitCommandEncoder;use crate::{@@ -10,11 +10,12 @@ use crate::{KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,Retained,- forward_pass::{ArrayId, ForwardPassState},kernel::dsl::{PoolingClsMetalKernel, PoolingMeanMetalKernel},},},config::PoolingType,+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},};enum PoolingKernel {@@ -74,9 +75,9 @@ impl Pooling {impl EncodableBlock<Metal> for Pooling {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let encoder = command_buffer.new_compute_command_encoder()@@ -124,9 +125,9 @@ impl EncodableBlock<Metal> for Pooling {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let batch_size = 1;let seq_len = state.aux_buffers_suffix_length();diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rsindex b47b01a87..efddba52a 100644--- a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs+++ b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs@@ -1,12 +1,16 @@//! Prediction head encodable for classification output.-use super::{EncodableBlock, EncodingParameters, Metal};+use super::{EncodableBlock, Metal};#[cfg(feature = "tracing")]-use crate::backends::metal::{MTLBlitCommandEncoder, forward_pass::ArrayId};+use crate::backends::metal::MTLBlitCommandEncoder;use crate::backends::metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,- ProtocolObject, Retained, forward_pass::ForwardPassState,+ ProtocolObject, Retained,};+use crate::encodable_block::EncodingParameters;+#[cfg(feature = "tracing")]+use crate::forward_pass::state::ArrayId;+use crate::forward_pass::state::ForwardPassState;pub struct ClassifierPredictionHead {dense: Box<dyn EncodableBlock<Metal>>,@@ -38,9 +42,9 @@ impl ClassifierPredictionHead {impl EncodableBlock<Metal> for ClassifierPredictionHead {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {let encoder = command_buffer@@ -100,9 +104,9 @@ impl EncodableBlock<Metal> for ClassifierPredictionHead {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {self.dense.encode_with_shared_encoder(state, encoder, parameters);self.activation.encode_with_shared_encoder(state, encoder, parameters);diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rsindex d3b6e847a..d4671fb7a 100644--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs@@ -5,12 +5,13 @@ use crate::backends::metal::{ProtocolObject, Retained,};-use super::{EncodableBlock, EncodingParameters, Metal};+use super::{EncodableBlock, Metal};use crate::backends::metal::{KernelDataType, MTLContext,- forward_pass::{ArrayId, ForwardPassState, RopeType},kernel::rope::{RopeError, RopeKernel, RopeKernelArguments},};+use crate::encodable_block::EncodingParameters;+use crate::forward_pass::state::{ArrayId, ForwardPassState, RopeType};pub struct Rope {kernel: RopeKernel,@@ -34,9 +35,9 @@ impl Rope {impl EncodableBlock<Metal> for Rope {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let compute_encoder = command_buffer.new_compute_command_encoder()@@ -56,9 +57,9 @@ impl EncodableBlock<Metal> for Rope {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsindex 65c524b26..be4122178 100644--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs@@ -1,6 +1,6 @@use std::rc::Rc;-use super::{EncodableBlock, EncodingParameters, Metal, transformer_layer};+use super::{EncodableBlock, Metal, transformer_layer};use crate::{DataType,backends::{@@ -10,7 +10,6 @@ use crate::{MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,Retained,compilation_parameters::CompilationConfig,- forward_pass::{ArrayId, ForwardPassState},kernel::short_conv::{ShortConvDecodeArguments, ShortConvKernel,ShortConvPackArguments, ShortConvPrefillArguments,@@ -18,6 +17,8 @@ use crate::{},},config::{DecoderLayerType, ShortConvConfig},+ encodable_block::EncodingParameters,+ forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -120,9 +121,9 @@ impl ShortConvMixer {fn encode_pipeline(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {@@ -151,9 +152,9 @@ impl ShortConvMixer {fn encode_pipeline_with_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {@@ -175,7 +176,7 @@ impl ShortConvMixer {fn run_prefill_conv(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {@@ -249,7 +250,7 @@ impl ShortConvMixer {fn run_decode_conv(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,suffix_length: usize,) {@@ -300,9 +301,9 @@ impl ShortConvMixer {impl EncodableBlock<Metal> for ShortConvMixer {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {if self.supports_shared_encoder() {let encoder = command_buffer@@ -327,9 +328,9 @@ impl EncodableBlock<Metal> for ShortConvMixer {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {self.encode_pipeline_with_encoder(state, encoder, parameters);}diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rsindex 524876baa..4a8abc476 100644--- a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs@@ -8,12 +8,12 @@ use crate::backends::{},};-use super::{EncodableBlock, EncodingParameters, Metal};+use super::{EncodableBlock, Metal};use crate::backends::metal::{- MTLContext, MTLError,- forward_pass::{ArrayId, ForwardPassState},- kernel::dsl::TensorAddSwapMetalKernel,+ MTLContext, MTLError, kernel::dsl::TensorAddSwapMetalKernel,};+use crate::encodable_block::EncodingParameters;+use crate::forward_pass::state::{ArrayId, ForwardPassState};pub struct TensorAddSwap {kernel: TensorAddSwapMetalKernel,@@ -37,9 +37,9 @@ impl TensorAddSwap {impl EncodableBlock<Metal> for TensorAddSwap {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let encoder = command_buffer.new_compute_command_encoder()@@ -59,9 +59,9 @@ impl EncodableBlock<Metal> for TensorAddSwap {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&self.argument_arrays);assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rsindex 301dcfec7..916b5a1e6 100644--- a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs@@ -8,12 +8,12 @@ use crate::backends::{},};-use super::{EncodableBlock, EncodingParameters, Metal};+use super::{EncodableBlock, Metal};use crate::backends::metal::{- MTLContext, MTLError,- forward_pass::{ArrayId, ForwardPassState},- kernel::dsl::TensorCopyMetalKernel,+ MTLContext, MTLError, kernel::dsl::TensorCopyMetalKernel,};+use crate::encodable_block::EncodingParameters;+use crate::forward_pass::state::{ArrayId, ForwardPassState};pub struct TensorCopy {kernel: TensorCopyMetalKernel,@@ -37,9 +37,9 @@ impl TensorCopy {impl EncodableBlock<Metal> for TensorCopy {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- parameters: &EncodingParameters,+ parameters: &EncodingParameters<Metal>,) {let encoder = command_buffer.new_compute_command_encoder()@@ -59,9 +59,9 @@ impl EncodableBlock<Metal> for TensorCopy {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let arrays = state.arrays(&self.argument_arrays);assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 9c32dda6d..82d938c94 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -11,10 +11,10 @@ use crate::{DataType,backends::metal::{MTLContext, MTLError,- forward_pass::ArrayId,kernel::{mlp::MlpGateActMulEncodable, quant_matmul::QuantizationType},},config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},+ forward_pass::state::ArrayId,parameters::ParameterTree,};diff --git a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rsdeleted file mode 100644index 90854cadb..000000000--- a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs+++ /dev/null@@ -1,57 +0,0 @@-use super::ForwardPassState;-use crate::backends::metal::{- MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, Metal,- ProtocolObject, Retained,- encodable_block::{EncodableBlock, EncodingParameters},-};--pub struct EncoderResolver<'a> {- command_buffer: &'a Retained<ProtocolObject<dyn MTLCommandBuffer>>,- encoder: Option<Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>>,-}--impl<'a> EncoderResolver<'a> {- pub fn new(- command_buffer: &'a Retained<ProtocolObject<dyn MTLCommandBuffer>>- ) -> Self {- Self {- command_buffer,- encoder: None,- }- }-- pub fn encode(- &mut self,- block: &dyn EncodableBlock<Metal>,- state: &mut ForwardPassState,- parameters: &EncodingParameters,- ) {- if block.supports_shared_encoder() {- if self.encoder.is_none() {- self.encoder = Some(- self.command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder"),- );- }- block.encode_with_shared_encoder(- state,- self.encoder.as_ref().unwrap(),- parameters,- );- } else {- self.end_current_encoder();- block.encode(state, self.command_buffer, parameters);- }- }-- pub fn end_current_encoder(&mut self) {- if let Some(encoder) = self.encoder.take() {- encoder.end_encoding();- }- }-- pub fn finish(mut self) {- self.end_current_encoder();- }-}diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rsdeleted file mode 100644index c4f610d07..000000000--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs+++ /dev/null@@ -1,23 +0,0 @@-pub mod cache_layers;-mod encoder_resolver;-pub mod kv_cache_layer;-pub mod short_conv_layer;-pub mod ssm_layer;-pub mod state;-#[cfg(feature = "tracing")]-pub mod traces;--pub use cache_layers::{CacheLayer, CacheLayers};-pub use encoder_resolver::EncoderResolver;-pub use kv_cache_layer::{- AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,-};-pub use short_conv_layer::ShortConvLayer;-pub use ssm_layer::SSMLayer;-pub use crate::forward_pass::state::{ArrayId, HashMapId, RopeType};-pub use state::{- ArrayCell, ClassifierModeState, ForwardPassMode, ForwardPassState,- LanguageModelGeneratorModeState,-};--pub use super::encodable_block::{EncodableBlock, EncodingParameters};diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mode.rs b/crates/uzu/src/backends/metal/forward_pass/state/mode.rsdeleted file mode 100644index 1826f1189..000000000--- a/crates/uzu/src/backends/metal/forward_pass/state/mode.rs+++ /dev/null@@ -1,36 +0,0 @@-use std::{cell::RefCell, rc::Rc};--use super::super::cache_layers::CacheLayers;-#[cfg(feature = "tracing")]-use super::super::traces::ActivationTrace;-use crate::{backends::metal::MetalArray, session::parameter::SamplingMethod};--type ArrayCell = RefCell<MetalArray>;--pub enum ForwardPassMode {- LanguageModelGenerator(LanguageModelGeneratorModeState),- Classifier(ClassifierModeState),-}--pub struct LanguageModelGeneratorModeState {- pub cache_layers: Rc<RefCell<CacheLayers>>,- pub token_seeds: ArrayCell,- pub logits: ArrayCell,- pub sampling_output: Option<ArrayCell>,- pub sampling_method: Option<SamplingMethod>,- #[cfg(feature = "tracing")]- pub traces: Rc<RefCell<ActivationTrace>>,- pub active_suffix_length: usize,- pub sampling_start: usize,- pub sampling_length: usize,- pub is_prefilling: bool,-}--pub struct ClassifierModeState {- pub pooling: ArrayCell,- pub dense: ArrayCell,- pub norm: ArrayCell,- pub classifier_logits: ArrayCell,- #[cfg(feature = "tracing")]- pub traces: Rc<RefCell<ActivationTrace>>,-}diff --git a/crates/uzu/src/backends/metal/generated/kv_cache_update.h b/crates/uzu/src/backends/metal/generated/kv_cache_update.hnew file mode 100644index 000000000..fe32897f8--- /dev/null+++ b/crates/uzu/src/backends/metal/generated/kv_cache_update.h@@ -0,0 +1,27 @@+// Auto-generated from gpu_types/kv_cache_update.rs - do not edit manually+#pragma once++#ifndef UZU_KV_CACHE_UPDATE_H+#define UZU_KV_CACHE_UPDATE_H++#ifdef __METAL_VERSION__+#include <metal_stdlib>+using namespace metal;++namespace uzu {+namespace kv_cache_update {+#else+#include <stdint.h>+#endif++typedef struct {+ uint32_t source;+ uint32_t destination;+} Swap;++#ifdef __METAL_VERSION__+} // namespace kv_cache_update+} // namespace uzu+#endif++#endif // UZU_KV_CACHE_UPDATE_Hdiff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metalindex 855d5e1d1..84564431c 100644--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal@@ -1,58 +1,36 @@#include <metal_stdlib>#include "../definitions.metal"+#include "kv_cache_update.h"+template <typename T>-void swap(device T* buffer, const int sourceIdx, const int destIdx) {+void swap(device T* buffer, const uint sourceIdx, const uint destIdx) {const T temp = buffer[sourceIdx];buffer[sourceIdx] = buffer[destIdx];buffer[destIdx] = temp;}template <typename T>-void updateKVCache(- device T* inPlaceKeys,- device T* inPlaceValues,- const constant int2* swaps,- const constant int& swapCount,- const constant int& numHeads,- const constant int& maxSequenceLength,- const constant int& headDim,- const uint2 position+VARIANTS(T, float, bfloat, half)+KERNEL(KVCacheUpdate) (+ device T* in_place_keys,+ device T* in_place_values,+ const constant uzu::kv_cache_update::Swap* swaps,+ const constant uint& swap_count,+ const constant uint& num_heads,+ const constant uint& max_sequence_length,+ const constant uint& head_dim,+ const uint head_idx AXIS(num_heads, 32),+ const uint channel_idx AXIS(head_dim, 32)) {- for (int i = 0; i < swapCount; ++i) {- // [headIdx: 0..numHeads, tokenIdx: 0..maxSequenceLength, channelIdx:- // 0..headDim]- const int headOffset = position.x * maxSequenceLength * headDim;- const int channelOffset = position.y;- const int sourceIdx = headOffset + swaps[i].x * headDim + channelOffset;- const int destIdx = headOffset + swaps[i].y * headDim + channelOffset;+ for (uint i = 0; i < swap_count; ++i) {+ // [headIdx: 0..num_heads, tokenIdx: 0..max_sequence_length, channelIdx:+ // 0..head_dim]+ const uint head_offset = head_idx * max_sequence_length * head_dim;+ const uint sourceIdx = head_offset + swaps[i].source * head_dim + channel_idx;+ const uint destIdx = head_offset + swaps[i].destination * head_dim + channel_idx;- swap(inPlaceKeys, sourceIdx, destIdx);- swap(inPlaceValues, sourceIdx, destIdx);+ swap(in_place_keys, sourceIdx, destIdx);+ swap(in_place_values, sourceIdx, destIdx);}}--#define outerArguments(T) \- (device T * inPlaceKeys [[buffer(0)]], \- device T * inPlaceValues [[buffer(1)]], \- const constant int2* swaps [[buffer(2)]], \- const constant int& swapCount [[buffer(3)]], \- const constant int& numHeads [[buffer(4)]], \- const constant int& maxSequenceLength [[buffer(5)]], \- const constant int& headDim [[buffer(6)]], \- const uint2 position [[thread_position_in_grid]])--#define innerArguments \- (inPlaceKeys, \- inPlaceValues, \- swaps, \- swapCount, \- numHeads, \- maxSequenceLength, \- headDim, \- position)--generateKernels(1024, updateKVCache)--#undef outerArguments-#undef innerArgumentsdiff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs b/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rsdeleted file mode 100644index a01d915f6..000000000--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs+++ /dev/null@@ -1,262 +0,0 @@-use std::{mem::size_of, ptr::NonNull};--use thiserror::Error;--use super::{- super::MTLError, KernelDataType, MTLContext,- metal_extensions::ComputeEncoderDispatch,-};-use crate::backends::{- common::Context,- metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandEncoderExt,- MTLComputeCommandEncoder, MTLComputePipelineState, MTLSize,- ProtocolObject, Retained,- },-};--#[derive(Clone, Copy, Debug)]-#[repr(C)]-pub struct Swap {- pub source: i32,- pub destination: i32,-}--pub struct KVLayerData {- pub key_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- pub key_shape: [usize; 3],- pub value_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- pub value_shape: [usize; 3],-}--pub struct KVCacheUpdate {- pipeline_state: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- indices_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- max_sequence_length: usize,-}--#[derive(Debug, Error)]-pub enum KVCacheUpdateError {- #[error("Source and destination indices length mismatch")]- IndicesCountMismatch,- #[error("Shape mismatch between key and value tensors")]- ShapeMismatch,- #[error("Unsupported data type")]- UnsupportedDataType,- #[error("Metal error: {0}")]- MetalError(#[from] MTLError),- #[error("Swap count exceeds maximum sequence length")]- MaxSequenceLengthExceeded,- #[error("Buffer creation failed")]- BufferCreationFailed,- #[error("Function not found: {0}")]- FunctionNotFound(String),-}--impl KVCacheUpdate {- pub fn new(- context: &MTLContext,- data_type: KernelDataType,- max_sequence_length: usize,- ) -> Result<Self, KVCacheUpdateError> {- // Format function name with the appropriate suffix- let function_name =- format!("updateKVCache_{}", data_type.function_name_suffix());-- let pipeline_state = context- .compute_pipeline_state(&function_name, None)- .map_err(|e| KVCacheUpdateError::MetalError(e))?;-- let indices_buffer = context- .create_buffer(max_sequence_length * size_of::<Swap>())- .map_err(|_| KVCacheUpdateError::BufferCreationFailed)?;-- Ok(Self {- pipeline_state,- indices_buffer,- max_sequence_length,- })- }-- pub fn encode(- &self,- in_place_data: &[KVLayerData],- source_indices: &[usize],- destination_indices: &[usize],- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,- ) -> Result<(), KVCacheUpdateError> {- let compute_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder");- compute_encoder.set_label(Some("KV Cache Update"));-- self.encode_with_encoder(- in_place_data,- source_indices,- destination_indices,- &*compute_encoder,- )?;-- compute_encoder.end_encoding();- Ok(())- }-- /// Encode the KV cache update operation using a provided compute encoder- pub fn encode_with_encoder(- &self,- in_place_data: &[KVLayerData],- source_indices: &[usize],- destination_indices: &[usize],- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- ) -> Result<(), KVCacheUpdateError> {- if source_indices.len() != destination_indices.len() {- return Err(KVCacheUpdateError::IndicesCountMismatch);- }-- let swaps = create_swaps_direct(source_indices, destination_indices);- if swaps.len() > self.max_sequence_length {- return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);- }-- // For small swap counts, use set_bytes to avoid shared buffer race in async mode.- // Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.- let use_inline_bytes = swaps.len() <= 32;-- if !use_inline_bytes {- let indices_ptr =- self.indices_buffer.contents().as_ptr() as *mut Swap;- unsafe {- for (i, swap) in swaps.iter().enumerate() {- *indices_ptr.add(i) = *swap;- }- }- }-- for layer_data in in_place_data {- if layer_data.key_shape != layer_data.value_shape {- return Err(KVCacheUpdateError::ShapeMismatch);- }-- let [num_heads, max_sequence_length, head_dim] =- layer_data.key_shape;-- // Set buffers and parameters- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(&layer_data.key_buffer),- 0,- 0,- );- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(&layer_data.value_buffer),- 0,- 1,- );- if use_inline_bytes {- unsafe {- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(swaps.as_ptr() as *mut _),- swaps.len() * size_of::<Swap>(),- 2,- );- }- } else {- MTLComputeCommandEncoder::set_buffer(- compute_encoder,- Some(&self.indices_buffer),- 0,- 2,- );- }- unsafe {- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(- &(swaps.len() as i32) as *const _ as *mut _,- ),- size_of::<i32>(),- 3,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(- &(num_heads as i32) as *const _ as *mut _,- ),- size_of::<i32>(),- 4,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(- &(max_sequence_length as i32) as *const _ as *mut _,- ),- size_of::<i32>(),- 5,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(- &(head_dim as i32) as *const _ as *mut _,- ),- size_of::<i32>(),- 6,- );- }-- compute_encoder.dispatch_2d_exactly(- &self.pipeline_state,- MTLSize {- width: num_heads as usize,- height: head_dim as usize,- depth: 1,- },- None,- );- }-- Ok(())- }-}--pub fn create_swaps_direct(- source_indices: &[usize],- destination_indices: &[usize],-) -> Vec<Swap> {- if source_indices.len() != destination_indices.len() {- return Vec::new();- }-- let mut swaps = Vec::with_capacity(source_indices.len());-- for (&src, &dst) in source_indices.iter().zip(destination_indices.iter()) {- if src != dst {- swaps.push(Swap {- source: src as i32,- destination: dst as i32,- });- }- }-- swaps-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_direct_swaps() {- let sources = [0, 2, 4];- let destinations = [1, 3, 5];- let swaps = create_swaps_direct(&sources, &destinations);- assert_eq!(swaps.len(), 3);- assert_eq!(swaps[0].source, 0);- assert_eq!(swaps[0].destination, 1);- assert_eq!(swaps[1].source, 2);- assert_eq!(swaps[1].destination, 3);- assert_eq!(swaps[2].source, 4);- assert_eq!(swaps[2].destination, 5);- }-}diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rsindex 7a945dd23..ee493532d 100644--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs@@ -1,15 +1,16 @@use std::{mem, ptr::NonNull};-use crate::backends::metal::Metal;+use crate::{+ backends::metal::Metal,+ encodable_block::{EncodableBlock, EncodingParameters},+ forward_pass::state::ForwardPassState,+};use objc2::rc::Retained;use crate::backends::metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandEncoderExt,MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLSize,- ProtocolObject,- error::MTLError,- forward_pass::{EncodableBlock, EncodingParameters, ForwardPassState},- image::Image,+ ProtocolObject, error::MTLError, image::Image,metal_extensions::ComputeEncoderDispatch,};@@ -83,9 +84,9 @@ impl ScalePadNormalizeImage {impl EncodableBlock<Metal> for ScalePadNormalizeImage {fn encode(&self,- _state: &mut ForwardPassState,+ _state: &mut ForwardPassState<Metal>,command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {let _ = command_buffer;}@@ -96,9 +97,9 @@ impl EncodableBlock<Metal> for ScalePadNormalizeImage {fn encode_with_shared_encoder(&self,- _state: &mut ForwardPassState,+ _state: &mut ForwardPassState<Metal>,_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters,+ _parameters: &EncodingParameters<Metal>,) {unreachable!("ScalePadNormalizeImage does not support shared compute encoder"diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex c3c1f8d38..a24d4084a 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -6,7 +6,6 @@ pub mod dsl {pub(super) const MTLB: &[u8] =include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));-pub mod kv_cache_update;pub mod matmul;pub mod media_kernels;pub mod mlp;@@ -29,7 +28,6 @@ pub use attention::{KVCacheUpdateArguments,};pub use data_type::KernelDataType;-pub use kv_cache_update::KVCacheUpdate;pub use matmul::{MatmulArguments, MatmulKernel};pub use moe::{MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError,@@ -53,8 +51,6 @@ pub use ssm::{};pub use token_copy::TokenCopyKernel;-use super::{MTLContext, metal_extensions};-pub mod media {pub use super::media_kernels::{ExtractImagePatches, ScalePadNormalizeImage,diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rsindex 37e15d9ab..1b621d72d 100644--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs@@ -7,7 +7,7 @@ use std::{};use super::{- CacheLayers, Decoder, KVCacheUpdate, KernelDataType, MTLContext, Metal,+ Decoder, KVCacheUpdate, KernelDataType, MTLContext, Metal,compilation_parameters::CompilationConfig, kernel::TokenCopyKernel,};use crate::{@@ -23,8 +23,8 @@ use crate::{config::{DecoderConfig, LanguageModelConfig, ModelMetadata},encodable_block::Sampling,forward_pass::{- model_shape::ModelShape, scratch_buffers::ScratchBuffers,- state::SharedBuffers,+ cache_layers::CacheLayers, model_shape::ModelShape,+ scratch_buffers::ScratchBuffers, state::SharedBuffers,},language_model::rng::PRng,parameters::ParameterLoader,@@ -136,7 +136,7 @@ pub struct LanguageModelGeneratorContext {pub mtl_context: Rc<MTLContext>,pub command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,- pub cache_layers: Rc<RefCell<CacheLayers>>,+ pub cache_layers: Rc<RefCell<CacheLayers<Metal>>>,pub shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,pub scratch_buffers: ScratchBuffers<Metal>,@@ -144,7 +144,7 @@ pub struct LanguageModelGeneratorContext {pub decoder_config: Rc<DecoderConfig>,pub model_shape: ModelShape,pub executables: Decoder,- pub kv_cache_update: Box<KVCacheUpdate>,+ pub kv_cache_update: Box<KVCacheUpdate<Metal>>,pub gpu_sampler: Sampling<Metal>,pub seed: PRng,@@ -234,7 +234,7 @@ impl LanguageModelGeneratorContext {);let cache_layers = Rc::new(RefCell::new(CacheLayers::new(- &context,+ context.as_ref(),&model_shape,max_prefix_length,max_suffix_length,@@ -244,8 +244,12 @@ impl LanguageModelGeneratorContext {decoder_config.output_norm_config.scale_precision.into();let kernel_data_type: KernelDataType = intermediate_data_type.into();let kv_cache_update = Box::new(- KVCacheUpdate::new(&context, kernel_data_type, max_prefix_length)- .map_err(|_| Error::UnableToCreateMetalContext)?,+ KVCacheUpdate::new(+ context.as_ref(),+ intermediate_data_type,+ max_prefix_length,+ )+ .map_err(|_| Error::UnableToCreateMetalContext)?,);let gpu_sampler = Sampling::<Metal>::new(diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex 019247024..6ca1d398b 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -5,9 +5,9 @@ mod classifier_context;mod command_buffer;pub mod compilation_parameters;mod context;+mod copy_encoder;pub mod encodable_block;pub mod error;-pub mod forward_pass;pub mod image;pub mod kernel;mod language_model_generator_context;@@ -15,6 +15,7 @@ pub mod media_utils;pub mod metal_extensions;pub mod placement_analysis;pub mod utils;+pub use crate::backends::common::kernel::kv_cache_update::KVCacheUpdate;pub use array::MetalArray;pub use backend::Metal;pub use classifier_context::ClassifierContext;@@ -23,10 +24,7 @@ pub use context::{};pub use encodable_block::Decoder;pub use error::MTLError;-pub use forward_pass::{CacheLayers, ForwardPassState};-pub use kernel::{- KVCacheUpdate, KernelDataType, RopeKernel, dsl::MetalKernels,-};+pub use kernel::{KernelDataType, RopeKernel, dsl::MetalKernels};pub use language_model_generator_context::LanguageModelGeneratorContext;pub use media_utils::{ImagePreprocessingParams, ImagePreprocessingRequirements,diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 094325df3..7aad798f7 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -7,14 +7,12 @@ use objc2::rc::autoreleasepool;#[cfg(feature = "tracing")]use super::ActivationTrace;use super::{ClassificationOutput, ClassificationStats, ClassifierContext};+use crate::backends::metal::Metal;use crate::{DataType,- backends::metal::{- MTLCommandBuffer,- forward_pass::{- ArrayId, EncodableBlock, EncodingParameters, ForwardPassState,- },- },+ backends::metal::MTLCommandBuffer,+ encodable_block::{EncodableBlock, EncodingParameters},+ forward_pass::state::{ArrayId, ForwardPassState},session::types::Error,};@@ -83,7 +81,7 @@ impl Classifier {&mut self,token_ids: &[u64],token_positions: &[usize],- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace>>), Error> {+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<Metal>>>), Error> {self.forward_pass(token_ids, token_positions)}@@ -92,7 +90,7 @@ impl Classifier {&mut self,token_ids: &[u64],token_positions: &[usize],- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace>>), Error> {+ ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<Metal>>>), Error> {autoreleasepool(|_| {let num_labels = self.context.model_config.model_config.num_labels;let mut state = ForwardPassState::new_classifier(@@ -241,7 +239,7 @@ impl Classifier {fn copy_logits_from_state(&self,- state: &ForwardPassState,+ state: &ForwardPassState<Metal>,) -> Result<Box<[f32]>, Error> {let logits_arrays =state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rsindex 680439779..48d0f31b3 100644--- a/crates/uzu/src/classifier/mod.rs+++ b/crates/uzu/src/classifier/mod.rs@@ -11,6 +11,4 @@ pub use crate::backends::metal::encodable_block::{ClassifierLayer, ClassifierPredictionHead, Pooling,};#[cfg(feature = "tracing")]-pub use crate::backends::metal::forward_pass::traces::{- ActivationTrace, LayerActivationTrace,-};+pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 5a89ecfb4..87e85ed6f 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -1,6 +1,6 @@-use crate::backends::{- common::{Backend, CommandBuffer},- metal::ForwardPassState,+use crate::{+ backends::common::{Backend, CommandBuffer},+ forward_pass::state::ForwardPassState,};mod sampling;@@ -55,8 +55,8 @@ impl<'a, B: Backend> EncodingParameters<'a, B> {pub trait EncodableBlock<B: Backend> {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,- encoder: &B::EncoderRef,+ state: &mut ForwardPassState<B>,+ encoder: &B::ComputeEncoder,parameters: &EncodingParameters<B>,);@@ -66,11 +66,11 @@ pub trait EncodableBlock<B: Backend> {fn encode(&self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<B>,command_buffer: &B::CommandBuffer,parameters: &EncodingParameters<B>,) {- command_buffer.with_encoder(|encoder| {+ command_buffer.with_compute_encoder(|encoder| {self.encode_with_shared_encoder(state, encoder, parameters)});}diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rsindex c6cb660c2..51c775ace 100644--- a/crates/uzu/src/encodable_block/sampling.rs+++ b/crates/uzu/src/encodable_block/sampling.rs@@ -2,16 +2,11 @@use crate::{DataType,- backends::{- common::{- Backend,- kernel::sampling::{ArgmaxStrategy, SamplingError, SamplingKernel},- },- metal::{- MTLComputeCommandEncoder, Metal, ProtocolObject,- forward_pass::{ArrayId, ForwardPassState},- },+ backends::common::{+ Backend,+ kernel::sampling::{ArgmaxStrategy, SamplingError, SamplingKernel},},+ forward_pass::state::{ArrayId, ForwardPassState},};use super::{EncodableBlock, EncodingParameters};@@ -58,12 +53,12 @@ impl<B: Backend> Sampling<B> {}}-impl EncodableBlock<Metal> for Sampling<Metal> {+impl<B: Backend> EncodableBlock<B> for Sampling<B> {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- _parameters: &EncodingParameters<Metal>,+ state: &mut ForwardPassState<B>,+ encoder: &B::ComputeEncoder,+ _parameters: &EncodingParameters<B>,) {assert!(state.sampling_output().is_some(),diff --git a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rssimilarity index 91%rename from crates/uzu/src/backends/metal/forward_pass/cache_layers.rsrename to crates/uzu/src/forward_pass/cache_layers.rsindex baaf49075..b5da2272a 100644--- a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs+++ b/crates/uzu/src/forward_pass/cache_layers.rs@@ -1,33 +1,30 @@use std::{cell::RefCell, collections::HashMap};-use super::{- super::{MTLContext, MetalArray},- kv_cache_layer::{- AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,- KVSlice,- },- short_conv_layer::ShortConvLayer,- ssm_layer::SSMLayer,-};use crate::{- array::ArrayContextExt,- backends::metal::{- MTLCommandBuffer, ProtocolObject, Retained, kernel::KVCacheUpdate,- },+ array::{Array, ArrayContextExt},+ backends::common::{Backend, kernel::kv_cache_update::KVCacheUpdate},config::DecoderLayerType,- forward_pass::model_shape::ModelShape,+ forward_pass::{+ kv_cache_layer::{+ AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer,+ KVCacheLayerState, KVSlice,+ },+ model_shape::ModelShape,+ short_conv_layer::ShortConvLayer,+ ssm_layer::SSMLayer,+ },};#[derive(Debug)]-pub enum CacheLayer {- Transformer(KVCacheLayer),- StateSpace(SSMLayer),- ShortConv(ShortConvLayer),+pub enum CacheLayer<B: Backend> {+ Transformer(KVCacheLayer<B>),+ StateSpace(SSMLayer<B>),+ ShortConv(ShortConvLayer<B>),}#[derive(Clone)]-pub enum CacheLayerSlice {- Transformer(KVSlice),+pub enum CacheLayerSlice<B: Backend> {+ Transformer(KVSlice<B>),StateSpace,ShortConv,}@@ -41,43 +38,43 @@ const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str =const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str ="cache_layers_short_conv_conv_state";-impl CacheLayer {- pub fn as_transformer(&self) -> Option<&KVCacheLayer> {+impl<B: Backend> CacheLayer<B> {+ pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {match self {CacheLayer::Transformer(layer) => Some(layer),_ => None,}}- pub fn as_transformer_mut(&mut self) -> Option<&mut KVCacheLayer> {+ pub fn as_transformer_mut(&mut self) -> Option<&mut KVCacheLayer<B>> {match self {CacheLayer::Transformer(layer) => Some(layer),_ => None,}}- pub fn as_state_space(&self) -> Option<&SSMLayer> {+ pub fn as_state_space(&self) -> Option<&SSMLayer<B>> {match self {CacheLayer::StateSpace(layer) => Some(layer),_ => None,}}- pub fn as_state_space_mut(&mut self) -> Option<&mut SSMLayer> {+ pub fn as_state_space_mut(&mut self) -> Option<&mut SSMLayer<B>> {match self {CacheLayer::StateSpace(layer) => Some(layer),_ => None,}}- pub fn as_short_conv(&self) -> Option<&ShortConvLayer> {+ pub fn as_short_conv(&self) -> Option<&ShortConvLayer<B>> {match self {CacheLayer::ShortConv(layer) => Some(layer),_ => None,}}- pub fn as_short_conv_mut(&mut self) -> Option<&mut ShortConvLayer> {+ pub fn as_short_conv_mut(&mut self) -> Option<&mut ShortConvLayer<B>> {match self {CacheLayer::ShortConv(layer) => Some(layer),_ => None,@@ -85,20 +82,20 @@ impl CacheLayer {}}-pub struct CacheLayers {+pub struct CacheLayers<B: Backend> {max_suffix_length: usize,max_prefix_length: usize,- pub data: Box<[CacheLayer]>,+ pub data: Box<[CacheLayer<B>]>,}#[derive(Clone)]-pub struct CacheLayersSlice {- pub layers: Vec<CacheLayerSlice>,+pub struct CacheLayersSlice<B: Backend> {+ pub layers: Vec<CacheLayerSlice<B>>,}-impl CacheLayers {+impl<B: Backend> CacheLayers<B> {pub fn new(- context: &MTLContext,+ context: &B::Context,model_shape: &ModelShape,max_prefix_length: usize,max_suffix_length: usize,@@ -108,7 +105,7 @@ impl CacheLayers {.kv_cache_layer_shapes(max_prefix_length, max_suffix_length).collect();- let data: Box<[CacheLayer]> = model_shape+ let data: Box<[CacheLayer<B>]> = model_shape.layer_types().iter().enumerate()@@ -257,7 +254,7 @@ impl CacheLayers {pub fn fill_attention_bias(&self,- dst: &mut HashMap<Option<usize>, MetalArray>,+ dst: &mut HashMap<Option<usize>, Array<B>>,suffix_token_positions: &[usize],suffix_length: usize,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,@@ -278,10 +275,10 @@ impl CacheLayers {pub fn fill_attention_bias_scratch(&self,- dst: &HashMap<Option<usize>, RefCell<MetalArray>>,+ dst: &HashMap<Option<usize>, RefCell<Array<B>>>,suffix_token_positions: &[usize],suffix_length: usize,- _context: &MTLContext,+ _context: &B::Context,) {for layer in self.data.iter() {if let CacheLayer::Transformer(layer) = layer {@@ -301,8 +298,8 @@ impl CacheLayers {&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- kv_cache_update: &KVCacheUpdate,+ command_buffer: &B::CommandBuffer,+ kv_cache_update: &KVCacheUpdate<B>,) {for layer in self.data.iter_mut() {if let Some(layer) = layer.as_transformer_mut() {@@ -344,9 +341,9 @@ impl CacheLayers {pub fn slice(&self,- context: &MTLContext,+ context: &B::Context,range: std::ops::Range<usize>,- ) -> Option<CacheLayersSlice> {+ ) -> Option<CacheLayersSlice<B>> {let mut layers = Vec::with_capacity(self.data.len());for layer in self.data.iter() {match layer {@@ -372,7 +369,7 @@ impl CacheLayers {pub fn apply_slice(&mut self,- slice: &CacheLayersSlice,+ slice: &CacheLayersSlice<B>,range: Option<std::ops::Range<usize>>,) {for (layer, snapshot) in self.data.iter_mut().zip(slice.layers.iter()) {@@ -381,7 +378,7 @@ impl CacheLayers {CacheLayer::Transformer(kv),CacheLayerSlice::Transformer(s),) => {- kv.apply_slice(s, range.clone());+ kv.apply_slice(&s, range.clone());},(CacheLayer::StateSpace(_), CacheLayerSlice::StateSpace) => {},(CacheLayer::ShortConv(_), CacheLayerSlice::ShortConv) => {},@@ -392,10 +389,10 @@ impl CacheLayers {pub fn clone(&self,- context: &MTLContext,+ context: &B::Context,) -> Self {let mut max_prefix_capacity_across_layers = 0usize;- let data: Box<[CacheLayer]> = self+ let data: Box<[CacheLayer<B>]> = self.data.iter().enumerate()diff --git a/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rssimilarity index 94%rename from crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rsrename to crates/uzu/src/forward_pass/kv_cache_layer.rsindex 9c4e5dcf9..37348a84e 100644--- a/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs@@ -1,17 +1,12 @@-use std::cell::RefCell;--use super::super::{MTLContext, MetalArray};use crate::{- array::ArrayContextExt,- backends::metal::{- MTLCommandBuffer, ProtocolObject, Retained,- kernel::{KVCacheUpdate, kv_cache_update::KVLayerData},+ array::{Array, ArrayCell, ArrayContextExt},+ backends::common::{+ Backend,+ kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},},utils::attention::fill_attention_bias,};-pub type ArrayCell = RefCell<MetalArray>;-#[derive(Clone, Copy, Debug, PartialEq, Eq)]pub struct AttentionBiasUpdate {pub key: Option<usize>,@@ -20,7 +15,7 @@ pub struct AttentionBiasUpdate {}#[derive(Clone)]-pub enum KVSlice {+pub enum KVSlice<B: Backend> {Full {base_prefix_len: usize,base_positions_len: usize,@@ -32,8 +27,8 @@ pub enum KVSlice {base_ring_length: usize,slots: Vec<usize>,positions: Vec<usize>, // per slot- keys: MetalArray, // [num_groups, slots.len(), head_dim]- values: MetalArray, // [num_groups, slots.len(), head_dim]+ keys: Array<B>, // [num_groups, slots.len(), head_dim]+ values: Array<B>, // [num_groups, slots.len(), head_dim]},}@@ -55,18 +50,18 @@ pub enum KVCacheLayerState {pub const INVALID_POSITION: usize = i32::MAX as usize;#[derive(Debug)]-pub struct KVCacheLayer {+pub struct KVCacheLayer<B: Backend> {pub state: KVCacheLayerState,/// [num_groups, max_prefix_length + max_suffix_length, head_dim]- pub keys: ArrayCell,+ pub keys: ArrayCell<B>,/// [num_groups, max_prefix_length + max_suffix_length, head_dim]- pub values: ArrayCell,+ pub values: ArrayCell<B>,pub prefix_token_positions: Vec<usize>,pub max_suffix_length: usize,}-impl KVCacheLayer {+impl<B: Backend> KVCacheLayer<B> {pub fn prefix_segment_length(&self) -> usize {match &self.state {KVCacheLayerState::Full {@@ -94,10 +89,6 @@ impl KVCacheLayer {}}- pub fn is_sliding_window(&self) -> bool {- matches!(self.state, KVCacheLayerState::Windowed { .. })- }-pub fn window_length(&self) -> Option<usize> {match &self.state {KVCacheLayerState::Full {@@ -112,7 +103,7 @@ impl KVCacheLayer {pub fn fill_attention_bias(&self,- dst: &mut MetalArray,+ dst: &mut Array<B>,suffix_token_positions: &[usize],suffix_length: usize,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,@@ -181,8 +172,8 @@ impl KVCacheLayer {&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- kv_cache_update: &KVCacheUpdate,+ command_buffer: &B::CommandBuffer,+ kv_cache_update: &KVCacheUpdate<B>,) {match &mut self.state {KVCacheLayerState::Full {@@ -249,8 +240,8 @@ impl KVCacheLayer {&self,source_indices: &[usize],destination_indices: &[usize],- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,- kv_cache_update: &KVCacheUpdate,+ command_buffer: &B::CommandBuffer,+ kv_cache_update: &KVCacheUpdate<B>,) {if source_indices == destination_indices {return;@@ -351,9 +342,9 @@ impl KVCacheLayer {pub fn slice(&self,- context: &MTLContext,+ context: &B::Context,range: std::ops::Range<usize>,- ) -> Option<KVSlice> {+ ) -> Option<KVSlice<B>> {match self.state {KVCacheLayerState::Full {prefix_len,@@ -427,7 +418,7 @@ impl KVCacheLayer {pub fn apply_slice(&mut self,- slice: &KVSlice,+ slice: &KVSlice<B>,range: Option<std::ops::Range<usize>>,) {match (slice, &mut self.state) {diff --git a/crates/uzu/src/forward_pass/mod.rs b/crates/uzu/src/forward_pass/mod.rsindex 4dc2d5634..cfede3e9a 100644--- a/crates/uzu/src/forward_pass/mod.rs+++ b/crates/uzu/src/forward_pass/mod.rs@@ -1,3 +1,12 @@pub mod model_shape;++pub mod cache_layers;+pub mod kv_cache_layer;+pub mod short_conv_layer;+pub mod ssm_layer;+pub mod scratch_buffers;pub mod state;++#[cfg(feature = "tracing")]+pub mod traces;diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rsindex 0e8df2598..32b32c0e5 100644--- a/crates/uzu/src/forward_pass/model_shape.rs+++ b/crates/uzu/src/forward_pass/model_shape.rs@@ -182,22 +182,6 @@ impl ModelShape {[suffix_length, self.vocabulary_size]}- pub fn embeddings_input_shape(&self) -> [usize; 2] {- [self.vocabulary_size, self.model_dim]- }-- pub fn embeddings_output_shape(&self) -> [usize; 2] {- [self.vocabulary_size, self.model_dim]- }-- pub fn quantized_embeddings_weights_shape(&self) -> [usize; 2] {- [self.vocabulary_size, self.model_dim]- }-- pub fn quantized_embeddings_scales_shape(&self) -> [usize; 1] {- [self.vocabulary_size]- }-pub fn attention_output_shape(&self,suffix_length: usize,@@ -266,14 +250,6 @@ impl ModelShape {[self.num_heads * suffix_length * TOTAL_BLOCKS_COUNT]}- pub fn moe_router_logits_shape(- &self,- suffix_length: usize,- num_experts: usize,- ) -> [usize; 2] {- [suffix_length, num_experts]- }-pub fn moe_topk_ids_shape(&self,suffix_length: usize,diff --git a/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs b/crates/uzu/src/forward_pass/short_conv_layer.rssimilarity index 51%rename from crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rsrename to crates/uzu/src/forward_pass/short_conv_layer.rsindex 0933ec813..7763a7eb0 100644--- a/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs+++ b/crates/uzu/src/forward_pass/short_conv_layer.rs@@ -1,13 +1,13 @@use bytemuck::fill_zeroes;-use super::kv_cache_layer::ArrayCell;+use crate::{array::ArrayCell, backends::common::Backend};#[derive(Debug)]-pub struct ShortConvLayer {- pub conv_state: ArrayCell,+pub struct ShortConvLayer<B: Backend> {+ pub conv_state: ArrayCell<B>,}-impl ShortConvLayer {+impl<B: Backend> ShortConvLayer<B> {pub fn zero(&self) {let mut conv = self.conv_state.borrow_mut();fill_zeroes(conv.as_bytes_mut());diff --git a/crates/uzu/src/backends/metal/forward_pass/ssm_layer.rs b/crates/uzu/src/forward_pass/ssm_layer.rssimilarity index 63%rename from crates/uzu/src/backends/metal/forward_pass/ssm_layer.rsrename to crates/uzu/src/forward_pass/ssm_layer.rsindex 43ede5bfc..01ba550bf 100644--- a/crates/uzu/src/backends/metal/forward_pass/ssm_layer.rs+++ b/crates/uzu/src/forward_pass/ssm_layer.rs@@ -1,14 +1,14 @@use bytemuck::fill_zeroes;-use super::kv_cache_layer::ArrayCell;+use crate::{array::ArrayCell, backends::common::Backend};#[derive(Debug)]-pub struct SSMLayer {- pub conv_state: ArrayCell,- pub ssm_state: ArrayCell,+pub struct SSMLayer<B: Backend> {+ pub conv_state: ArrayCell<B>,+ pub ssm_state: ArrayCell<B>,}-impl SSMLayer {+impl<B: Backend> SSMLayer<B> {pub fn zero(&self) {{let mut conv = self.conv_state.borrow_mut();diff --git a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/forward_pass/state/common_aux_buffers.rsindex f0ccedb7d..0323594ad 100644--- a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs+++ b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs@@ -62,8 +62,4 @@ impl<B: Backend> CommonAuxBuffers<B> {.view(&model_shape.attention_maxs_shape(suffix_length)),}}-- pub fn suffix_length(&self) -> usize {- self.suffix_length- }}diff --git a/crates/uzu/src/forward_pass/state/mod.rs b/crates/uzu/src/forward_pass/state/mod.rsindex b211f2955..3aab46cde 100644--- a/crates/uzu/src/forward_pass/state/mod.rs+++ b/crates/uzu/src/forward_pass/state/mod.rs@@ -7,6 +7,8 @@ mod language_model_generator_aux_buffers;mod rope_buffers;mod shared_buffers;+mod state;+pub use array_id::ArrayId;pub use hash_map_id::HashMapId;pub use rope_type::RopeType;@@ -14,4 +16,9 @@ pub use rope_type::RopeType;pub use common_aux_buffers::CommonAuxBuffers;pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;pub use rope_buffers::RopeBuffers;-pub use shared_buffers::{MoeExpertWeights, SharedBuffers};+pub use shared_buffers::SharedBuffers;++pub use state::{+ ClassifierModeState, ForwardPassMode, ForwardPassState,+ LanguageModelGeneratorModeState,+};diff --git a/crates/uzu/src/forward_pass/state/shared_buffers.rs b/crates/uzu/src/forward_pass/state/shared_buffers.rsindex c8040d93e..fd7db21fb 100644--- a/crates/uzu/src/forward_pass/state/shared_buffers.rs+++ b/crates/uzu/src/forward_pass/state/shared_buffers.rs@@ -6,23 +6,16 @@ use crate::{DataType,array::{ArrayCell, ArrayContextExt},backends::common::Backend,- config::{DecoderConfig, MLPConfig},+ config::DecoderConfig,forward_pass::model_shape::ModelShape,parameters::ParameterTree,};use super::RopeBuffers;-pub struct MoeExpertWeights<B: Backend> {- pub w1: ArrayCell<B>,- pub w2: ArrayCell<B>,- pub w3: ArrayCell<B>,-}-pub struct SharedBuffers<B: Backend> {pub global_rope: Option<RopeBuffers<B>>,pub local_rope: Option<RopeBuffers<B>>,- pub moe_expert_weights: Option<Vec<MoeExpertWeights<B>>>,pub attention_sinks: Option<Vec<ArrayCell<B>>>,}@@ -42,12 +35,6 @@ impl<B: Backend> SharedBuffers<B> {.is_some().then(|| RopeBuffers::new(context, model_shape));- let moe_expert_weights = matches!(- decoder_config.layer_config.mlp_config,- MLPConfig::MixtureOfExperts(_)- )- .then(Vec::new);-let attention_sinks = decoder_config.layer_config.attention_config()@@ -68,7 +55,6 @@ impl<B: Backend> SharedBuffers<B> {Self {global_rope,local_rope,- moe_expert_weights,attention_sinks,}}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/forward_pass/state/state.rssimilarity index 85%rename from crates/uzu/src/backends/metal/forward_pass/state/mod.rsrename to crates/uzu/src/forward_pass/state/state.rsindex c2d61f065..7b3326c0c 100644--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -1,26 +1,14 @@-mod mode;-use std::{cell::RefCell, collections::HashMap, rc::Rc};-pub use mode::{- ClassifierModeState, ForwardPassMode, LanguageModelGeneratorModeState,-};--use super::cache_layers::CacheLayers;#[cfg(feature = "tracing")]-use super::traces::ActivationTrace;+use crate::forward_pass::traces::ActivationTrace;+use crate::{DataType, DecoderConfig,- array::ArrayCellExt,- backends::{- common::Context,- metal::{- MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer,- MTLCommandEncoder, MTLContext, Metal, MetalArray, ProtocolObject,- Retained,- },- },+ array::{Array, ArrayCell, ArrayCellExt},+ backends::common::{Backend, CommandBuffer, Context, CopyEncoder},forward_pass::{+ cache_layers::CacheLayers,model_shape::ModelShape,scratch_buffers::ScratchBuffers,state::{@@ -32,29 +20,55 @@ use crate::{utils::attention::fill_attention_bias,};-pub type ArrayCell = RefCell<MetalArray>;--pub struct ForwardPassState {- context: Rc<MTLContext>,- token_ids: ArrayCell,- token_positions: ArrayCell,- token_bitmask: Option<ArrayCell>,- attention_bias: HashMap<Option<usize>, ArrayCell>,- pub shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,- common_aux: CommonAuxBuffers<Metal>,- llm_aux: Option<LanguageModelGeneratorAuxBuffers<Metal>>,- mode: ForwardPassMode,+pub enum ForwardPassMode<B: Backend> {+ LanguageModelGenerator(LanguageModelGeneratorModeState<B>),+ Classifier(ClassifierModeState<B>),+}++pub struct LanguageModelGeneratorModeState<B: Backend> {+ pub cache_layers: Rc<RefCell<CacheLayers<B>>>,+ pub token_seeds: ArrayCell<B>,+ pub logits: ArrayCell<B>,+ pub sampling_output: Option<ArrayCell<B>>,+ pub sampling_method: Option<SamplingMethod>,+ #[cfg(feature = "tracing")]+ pub traces: Rc<RefCell<ActivationTrace<B>>>,+ pub active_suffix_length: usize,+ pub sampling_start: usize,+ pub sampling_length: usize,+ pub is_prefilling: bool,+}++pub struct ClassifierModeState<B: Backend> {+ pub pooling: ArrayCell<B>,+ pub dense: ArrayCell<B>,+ pub norm: ArrayCell<B>,+ pub classifier_logits: ArrayCell<B>,+ #[cfg(feature = "tracing")]+ pub traces: Rc<RefCell<ActivationTrace<B>>>,}-impl ForwardPassState {+pub struct ForwardPassState<B: Backend> {+ context: Rc<B::Context>,+ token_ids: ArrayCell<B>,+ token_positions: ArrayCell<B>,+ token_bitmask: Option<ArrayCell<B>>,+ attention_bias: HashMap<Option<usize>, ArrayCell<B>>,+ pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,+ common_aux: CommonAuxBuffers<B>,+ llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,+ mode: ForwardPassMode<B>,+}++impl<B: Backend> ForwardPassState<B> {// ========================================================================// Common initialization helpers// ========================================================================fn init_token_ids(- scratch: &ScratchBuffers<Metal>,+ scratch: &ScratchBuffers<B>,token_ids: &[u64],- ) -> ArrayCell {+ ) -> ArrayCell<B> {let suffix_length = token_ids.len();let token_ids_array = scratch.token_ids.view(&[suffix_length]);token_ids_array.borrow_mut().copy_from_view(token_ids.into());@@ -62,9 +76,9 @@ impl ForwardPassState {}fn init_token_positions(- scratch: &ScratchBuffers<Metal>,+ scratch: &ScratchBuffers<B>,token_positions: &[usize],- ) -> ArrayCell {+ ) -> ArrayCell<B> {let suffix_length = token_positions.len();let token_positions_array =scratch.token_positions.view(&[suffix_length]);@@ -82,12 +96,12 @@ impl ForwardPassState {#[allow(clippy::too_many_arguments)]pub fn new_llm(- context: Rc<MTLContext>,+ context: Rc<B::Context>,decoder_config: &DecoderConfig,model_shape: &ModelShape,- scratch: &ScratchBuffers<Metal>,- cache_layers: Rc<RefCell<CacheLayers>>,- shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,+ scratch: &ScratchBuffers<B>,+ cache_layers: Rc<RefCell<CacheLayers<B>>>,+ shared_buffers: Rc<RefCell<SharedBuffers<B>>>,token_ids: &[u64],token_positions: &[usize],token_bitmask: Option<&[u32]>,@@ -99,11 +113,8 @@ impl ForwardPassState {external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,skip_token_ids_copy: bool,should_fill_attention_bias: bool,- async_positions: Option<(- &Retained<ProtocolObject<dyn MTLBuffer>>,- usize,- )>,- async_seeds: Option<(&Retained<ProtocolObject<dyn MTLBuffer>>, usize)>,+ async_positions: Option<(&B::NativeBuffer, usize)>,+ async_seeds: Option<(&B::NativeBuffer, usize)>,) -> Self {let suffix_length = token_ids.len();assert_eq!(@@ -127,7 +138,7 @@ impl ForwardPassState {let token_positions_cell =if let Some((async_buf, offset)) = async_positions {let array = unsafe {- MetalArray::from_parts(+ Array::from_parts(async_buf.clone(),offset * std::mem::size_of::<i32>(),&[suffix_length],@@ -151,7 +162,7 @@ impl ForwardPassState {// Token seeds - use async buffer if providedlet token_seeds_cell = if let Some((async_buf, offset)) = async_seeds {let array = unsafe {- MetalArray::from_parts(+ Array::from_parts(async_buf.clone(),offset * std::mem::size_of::<u64>(),&[suffix_length],@@ -198,7 +209,7 @@ impl ForwardPassState {// Traces#[cfg(feature = "tracing")]let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(- &context,+ context.as_ref(),model_shape,suffix_length,)));@@ -232,27 +243,26 @@ impl ForwardPassState {}fn init_llm_attention_bias(- scratch: &ScratchBuffers<Metal>,- cache_layers: &Rc<RefCell<CacheLayers>>,+ scratch: &ScratchBuffers<B>,+ cache_layers: &Rc<RefCell<CacheLayers<B>>>,suffix_length: usize,token_positions: &[usize],external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,should_fill_attention_bias: bool,- ) -> HashMap<Option<usize>, ArrayCell> {+ ) -> HashMap<Option<usize>, ArrayCell<B>> {let cache_ref = cache_layers.borrow();- let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =- scratch- .attention_window_size_to_bias- .iter()- .map(|(window_size, buffer)| {- let prefix_length =- window_size.unwrap_or(cache_ref.max_prefix_length());- let attention_bias_shape =- [suffix_length, suffix_length + prefix_length];- let array = buffer.borrow().view(&attention_bias_shape);- (*window_size, array)- })- .collect();+ let mut attention_bias_map: HashMap<Option<usize>, Array<B>> = scratch+ .attention_window_size_to_bias+ .iter()+ .map(|(window_size, buffer)| {+ let prefix_length =+ window_size.unwrap_or(cache_ref.max_prefix_length());+ let attention_bias_shape =+ [suffix_length, suffix_length + prefix_length];+ let array = buffer.borrow().view(&attention_bias_shape);+ (*window_size, array)+ })+ .collect();drop(cache_ref);// Use cache_layers' fill_attention_bias which properly handles@@ -279,10 +289,10 @@ impl ForwardPassState {#[allow(clippy::too_many_arguments)]pub fn new_classifier(- context: Rc<MTLContext>,+ context: Rc<B::Context>,model_shape: &ModelShape,- scratch: &ScratchBuffers<Metal>,- shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,+ scratch: &ScratchBuffers<B>,+ shared_buffers: Rc<RefCell<SharedBuffers<B>>>,token_ids: &[u64],token_positions: &[usize],bidirectional_attention: bool,@@ -309,7 +319,7 @@ impl ForwardPassState {// Classifier-specific bufferslet model_dim = model_shape.main_shape(1)[1];let classifier_state = Self::init_classifier_buffers(- &context,+ context.as_ref(),model_shape,model_dim,num_labels,@@ -332,20 +342,19 @@ impl ForwardPassState {}fn init_classifier_attention_bias(- scratch: &ScratchBuffers<Metal>,+ scratch: &ScratchBuffers<B>,suffix_length: usize,bidirectional_attention: bool,- ) -> HashMap<Option<usize>, ArrayCell> {- let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =- scratch- .attention_window_size_to_bias- .iter()- .map(|(window_size, buffer)| {- let attention_bias_shape = [suffix_length, suffix_length];- let array = buffer.borrow().view(&attention_bias_shape);- (*window_size, array)- })- .collect();+ ) -> HashMap<Option<usize>, ArrayCell<B>> {+ let mut attention_bias_map: HashMap<Option<usize>, Array<B>> = scratch+ .attention_window_size_to_bias+ .iter()+ .map(|(window_size, buffer)| {+ let attention_bias_shape = [suffix_length, suffix_length];+ let array = buffer.borrow().view(&attention_bias_shape);+ (*window_size, array)+ })+ .collect();for (window, bias_array) in attention_bias_map.iter_mut() {if bidirectional_attention {@@ -386,22 +395,22 @@ impl ForwardPassState {#[cfg(feature = "tracing")]fn init_classifier_buffers(- context: &MTLContext,+ context: &B::Context,model_shape: &ModelShape,model_dim: usize,num_labels: usize,suffix_length: usize,- ) -> ClassifierModeState {+ ) -> ClassifierModeState<B> {let data_type = model_shape.activation_data_type();let batch_size = 1;- let create_buffer = |size: usize| -> ArrayCell {+ let create_buffer = |size: usize| -> ArrayCell<B> {let buffer_size = size * data_type.size_in_bytes();let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");RefCell::new(unsafe {- MetalArray::from_parts(+ Array::from_parts(buffer,0,&[batch_size, size / batch_size],@@ -421,7 +430,7 @@ impl ForwardPassState {.create_buffer(buffer_size).expect("Failed to create buffer");RefCell::new(unsafe {- MetalArray::from_parts(+ Array::from_parts(buffer,0,&[batch_size, num_labels],@@ -440,23 +449,23 @@ impl ForwardPassState {#[cfg(not(feature = "tracing"))]fn init_classifier_buffers(- context: &MTLContext,+ context: &B::Context,model_shape: &ModelShape,model_dim: usize,num_labels: usize,_suffix_length: usize,- ) -> ClassifierModeState {+ ) -> ClassifierModeState<B> {let data_type = model_shape.activation_data_type();let batch_size = 1;- let create_buffer = |dims: &[usize]| -> ArrayCell {+ let create_buffer = |dims: &[usize]| -> ArrayCell<B> {let size: usize = dims.iter().product();let buffer_size = size * data_type.size_in_bytes();let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");RefCell::new(unsafe {- MetalArray::from_parts(buffer, 0, dims, data_type)+ Array::from_parts(buffer, 0, dims, data_type)})};@@ -472,8 +481,8 @@ impl ForwardPassState {// Accessors// ========================================================================- pub fn mtl_context(&self) -> &Rc<MTLContext> {- &self.context+ pub fn mtl_context(&self) -> &B::Context {+ self.context.as_ref()}pub fn aux_buffers_suffix_length(&self) -> usize {@@ -488,25 +497,25 @@ impl ForwardPassState {matches!(self.mode, ForwardPassMode::Classifier(_))}- pub fn token_bitmask(&self) -> Option<&ArrayCell> {+ pub fn token_bitmask(&self) -> Option<&ArrayCell<B>> {self.token_bitmask.as_ref()}- pub fn llm_state(&self) -> &LanguageModelGeneratorModeState {+ pub fn llm_state(&self) -> &LanguageModelGeneratorModeState<B> {match &self.mode {ForwardPassMode::LanguageModelGenerator(state) => state,_ => panic!("Not in LLM mode"),}}- pub fn llm_state_mut(&mut self) -> &mut LanguageModelGeneratorModeState {+ pub fn llm_state_mut(&mut self) -> &mut LanguageModelGeneratorModeState<B> {match &mut self.mode {ForwardPassMode::LanguageModelGenerator(state) => state,_ => panic!("Not in LLM mode"),}}- pub fn classifier_state(&self) -> &ClassifierModeState {+ pub fn classifier_state(&self) -> &ClassifierModeState<B> {match &self.mode {ForwardPassMode::Classifier(state) => state,_ => panic!("Not in classifier mode"),@@ -516,7 +525,7 @@ impl ForwardPassState {pub fn array_cell(&self,id: ArrayId,- ) -> ArrayCell {+ ) -> ArrayCell<B> {match id {// Common arraysArrayId::TokenIds => self.token_ids.clone(),@@ -752,7 +761,7 @@ impl ForwardPassState {fn shared_buffer_array(&self,id: ArrayId,- ) -> Option<ArrayCell> {+ ) -> Option<ArrayCell<B>> {let shared = self.shared_buffers.borrow();match id {ArrayId::RopeCosines(rope_type) => Some(match rope_type {@@ -787,11 +796,11 @@ impl ForwardPassState {}}- pub fn conv_padded_buffer(&self) -> Option<ArrayCell> {+ pub fn conv_padded_buffer(&self) -> Option<ArrayCell<B>> {self.llm_aux.as_ref().and_then(|aux| aux.ssm_conv_padded.clone())}- pub fn short_conv_padded_buffer(&self) -> Option<ArrayCell> {+ pub fn short_conv_padded_buffer(&self) -> Option<ArrayCell<B>> {self.llm_aux.as_ref().and_then(|aux| aux.short_conv_padded.clone())}@@ -802,14 +811,14 @@ impl ForwardPassState {pub fn arrays(&self,ids: &[ArrayId],- ) -> Box<[ArrayCell]> {+ ) -> Box<[ArrayCell<B>]> {ids.iter().map(|id| self.array_cell(*id)).collect()}pub fn hashmaps(&self,ids: &[HashMapId],- ) -> Box<[HashMap<Option<usize>, ArrayCell>]> {+ ) -> Box<[HashMap<Option<usize>, ArrayCell<B>>]> {ids.iter().map(|id| match id {HashMapId::AttentionBias => self.attention_bias.clone(),@@ -855,7 +864,7 @@ impl ForwardPassState {}}- pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers>>> {+ pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers<B>>>> {match &self.mode {ForwardPassMode::LanguageModelGenerator(state) => {Some(&state.cache_layers)@@ -864,7 +873,7 @@ impl ForwardPassState {}}- pub fn sampling_output(&self) -> Option<&ArrayCell> {+ pub fn sampling_output(&self) -> Option<&ArrayCell<B>> {match &self.mode {ForwardPassMode::LanguageModelGenerator(state) => {state.sampling_output.as_ref()@@ -874,7 +883,7 @@ impl ForwardPassState {}#[cfg(feature = "tracing")]- pub fn traces(&self) -> &Rc<RefCell<ActivationTrace>> {+ pub fn traces(&self) -> &Rc<RefCell<ActivationTrace<B>>> {match &self.mode {ForwardPassMode::LanguageModelGenerator(state) => &state.traces,ForwardPassMode::Classifier(state) => &state.traces,@@ -904,7 +913,7 @@ impl ForwardPassState {pub fn copy_array(&self,source_array_id: ArrayId,- destination_array: RefCell<MetalArray>,+ destination_array: RefCell<Array<B>>,) {destination_array.borrow_mut()@@ -913,30 +922,22 @@ impl ForwardPassState {pub fn encode_copy_array(&self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ command_buffer: &B::CommandBuffer,source_array_id: ArrayId,- destination_array: RefCell<MetalArray>,+ destination_array: RefCell<Array<B>>,) {let source_ref = self.arrays(&[source_array_id])[0].clone();let src_borrow = source_ref.borrow();let dst_borrow = destination_array.borrow();- let src_buf = src_borrow.buffer().clone();- let dst_buf = dst_borrow.buffer().clone();+ let src_buf = src_borrow.buffer();+ let dst_buf = dst_borrow.buffer();let copy_size_bytes = dst_borrow.size();debug_assert_eq!(dst_borrow.size(), src_borrow.size());- let blit_encoder = command_buffer- .new_blit_command_encoder()- .expect("Failed to create blit command encoder");- blit_encoder.copy_buffer_to_buffer(- &src_buf,- 0,- &dst_buf,- 0,- copy_size_bytes,- );- blit_encoder.end_encoding();+ command_buffer.with_copy_encoder(|encoder| {+ encoder.encode_copy(src_buf, dst_buf, copy_size_bytes);+ });}}diff --git a/crates/uzu/src/backends/metal/forward_pass/traces.rs b/crates/uzu/src/forward_pass/traces.rssimilarity index 78%rename from crates/uzu/src/backends/metal/forward_pass/traces.rsrename to crates/uzu/src/forward_pass/traces.rsindex 6ab3c8e04..2e6a4fc34 100644--- a/crates/uzu/src/backends/metal/forward_pass/traces.rs+++ b/crates/uzu/src/forward_pass/traces.rs@@ -2,27 +2,25 @@ use std::{cell::RefCell, rc::Rc};use crate::{DataType,- array::ArrayContextExt,- backends::metal::{MTLContext, MetalArray},+ array::{ArrayCell, ArrayContextExt},+ backends::common::Backend,forward_pass::model_shape::ModelShape,};-type ArrayCell = RefCell<MetalArray>;--fn create_trace_array(- context: &MTLContext,+fn create_trace_array<B: Backend>(+ context: &B::Context,shape: &[usize],data_type: DataType,label: &str,-) -> ArrayCell {+) -> ArrayCell<B> {RefCell::new(context.create_array_uninitialized(shape, data_type, label))}-fn create_layer_results(- context: &MTLContext,+fn create_layer_results<B: Backend>(+ context: &B::Context,model_shape: &ModelShape,suffix_length: usize,-) -> Vec<Rc<RefCell<LayerActivationTrace>>> {+) -> Vec<Rc<RefCell<LayerActivationTrace<B>>>> {(0..model_shape.num_layers).map(|_| {Rc::new(RefCell::new(LayerActivationTrace::new(@@ -34,21 +32,21 @@ fn create_layer_results(.collect()}-pub struct LayerActivationTrace {- pub inputs: ArrayCell,- pub pre_attention_norm: ArrayCell,- pub attention: ArrayCell,- pub post_attention_norm: ArrayCell,- pub mlp_inputs: ArrayCell,- pub pre_mlp_norm: ArrayCell,- pub mlp: ArrayCell,- pub post_mlp_norm: ArrayCell,- pub outputs: ArrayCell,+pub struct LayerActivationTrace<B: Backend> {+ pub inputs: ArrayCell<B>,+ pub pre_attention_norm: ArrayCell<B>,+ pub attention: ArrayCell<B>,+ pub post_attention_norm: ArrayCell<B>,+ pub mlp_inputs: ArrayCell<B>,+ pub pre_mlp_norm: ArrayCell<B>,+ pub mlp: ArrayCell<B>,+ pub post_mlp_norm: ArrayCell<B>,+ pub outputs: ArrayCell<B>,}-impl LayerActivationTrace {+impl<B: Backend> LayerActivationTrace<B> {pub fn new(- context: &MTLContext,+ context: &B::Context,model_shape: &ModelShape,suffix_length: usize,) -> Self {@@ -81,17 +79,17 @@ impl LayerActivationTrace {}}-pub struct ActivationTrace {- pub embedding_norm: Option<ArrayCell>,- pub layer_results: Vec<Rc<RefCell<LayerActivationTrace>>>,- pub output_norm: ArrayCell,- pub output_pooling: Option<ArrayCell>,- pub logits: ArrayCell,+pub struct ActivationTrace<B: Backend> {+ pub embedding_norm: Option<ArrayCell<B>>,+ pub layer_results: Vec<Rc<RefCell<LayerActivationTrace<B>>>>,+ pub output_norm: ArrayCell<B>,+ pub output_pooling: Option<ArrayCell<B>>,+ pub logits: ArrayCell<B>,}-impl ActivationTrace {+impl<B: Backend> ActivationTrace<B> {pub fn new_llm(- context: &MTLContext,+ context: &B::Context,model_shape: &ModelShape,suffix_length: usize,) -> Self {@@ -120,7 +118,7 @@ impl ActivationTrace {}pub fn new_classifier(- context: &MTLContext,+ context: &B::Context,model_shape: &ModelShape,suffix_length: usize,num_labels: usize,@@ -159,16 +157,14 @@ impl ActivationTrace {),}}-}-impl ActivationTrace {- pub fn embedding_norm(&self) -> &ArrayCell {+ pub fn embedding_norm(&self) -> &ArrayCell<B> {self.embedding_norm.as_ref().expect("embedding_norm is only available for classifier traces")}- pub fn output_pooling(&self) -> &ArrayCell {+ pub fn output_pooling(&self) -> &ArrayCell<B> {self.output_pooling.as_ref().expect("output_pooling is only available for classifier traces")diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex c9b42fcb1..526248a34 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -17,13 +17,14 @@ use crate::{common::kernel::MaskUpdateKernel,metal::{MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt,- MTLCommandBufferHandler, MTLCommandEncoder, MTLCommandQueue,- forward_pass::{- AttentionBiasUpdate, EncodableBlock, EncodingParameters,- ForwardPassState, INVALID_POSITION,- },+ MTLCommandBufferHandler, MTLCommandEncoder, MTLCommandQueue, Metal,},},+ encodable_block::{EncodableBlock, EncodingParameters},+ forward_pass::{+ kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION},+ state::ForwardPassState,+ },session::{config::DecodingConfig,parameter::{ConfigResolvableValue, ResolvableValue, SamplingMethod},@@ -132,7 +133,7 @@ impl LanguageModelGenerator {.chain(flat_trie.token_seeds()).chunks(prefill_step_size);- let mut last_state: Option<ForwardPassState> = None;+ let mut last_state: Option<ForwardPassState<Metal>> = None;let mut run_times: Vec<f64> = Vec::new();// Process each prefill step and update the KV cache.@@ -726,7 +727,7 @@ impl LanguageModelGenerator {allow_pre_encode: bool,sampling_method: SamplingMethod,should_fill_attention_bias: bool,- ) -> (ForwardPassState, f64) {+ ) -> (ForwardPassState<Metal>, f64) {objc2::rc::autoreleasepool(|_pool| {let run_start = Instant::now();@@ -812,7 +813,7 @@ impl LanguageModelGenerator {fn sample(&mut self,- state: &mut ForwardPassState,+ state: &mut ForwardPassState<Metal>,) -> Result<Vec<u64>, Error> {let sampling_output = state.sampling_output().expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsindex 411195aab..f305c41bd 100644--- a/crates/uzu/src/language_model/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -1,12 +1,13 @@use std::mem::size_of;use super::LanguageModelGeneratorContext;-use crate::backends::{- common::Context,- metal::{- BufferExt, MTLBuffer, ProtocolObject, Retained,- forward_pass::{EncodableBlock, EncodingParameters, ForwardPassState},+use crate::{+ backends::{+ common::Context,+ metal::{BufferExt, MTLBuffer, Metal, ProtocolObject, Retained},},+ encodable_block::{EncodableBlock, EncodingParameters},+ forward_pass::state::ForwardPassState,};pub struct LanguageModelGeneratorEncodedTask {@@ -85,7 +86,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {context: &mut LanguageModelGeneratorContext,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,should_fill_attention_bias: bool,- ) -> ForwardPassState {+ ) -> ForwardPassState<Metal> {ForwardPassState::new_llm(context.mtl_context.clone(),&context.decoder_config,@@ -112,8 +113,8 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {pub fn build_encoded_task(&self,context: &LanguageModelGeneratorContext,- state: &mut ForwardPassState,- parameters: &EncodingParameters,+ state: &mut ForwardPassState<Metal>,+ parameters: &EncodingParameters<Metal>,key: String,) -> LanguageModelGeneratorEncodedTask {context.executables.encode(state, &context.command_buffer, parameters);diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex 5cefdc32e..2f6c7c009 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -19,8 +19,8 @@ use tokenizers::Tokenizer;use xgrammar::TokenizerInfo;use crate::{- backends::metal::forward_pass::cache_layers::CacheLayer,config::{MixerConfig, ModelMetadata},+ forward_pass::cache_layers::CacheLayer,language_model::{LanguageModelGenerator,grammar::CompiledGrammar,diff --git a/crates/uzu/src/session/helpers/context.rs b/crates/uzu/src/session/helpers/context.rsindex 532ff8de3..c55e0d4bb 100644--- a/crates/uzu/src/session/helpers/context.rs+++ b/crates/uzu/src/session/helpers/context.rs@@ -1,18 +1,18 @@use crate::{- backends::metal::forward_pass::cache_layers::CacheLayers,+ backends::metal::Metal, forward_pass::cache_layers::CacheLayers,session::config::DecodingConfig,};pub struct Context {pub tokens: Vec<u64>,- pub cache_layers: CacheLayers,+ pub cache_layers: CacheLayers<Metal>,pub decoding_config: DecodingConfig,}impl Context {pub fn new(tokens: Vec<u64>,- cache_layers: CacheLayers,+ cache_layers: CacheLayers<Metal>,decoding_config: DecodingConfig,) -> Self {Self {diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex ca5307270..f358cae2f 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -18,17 +18,18 @@ use num_traits::NumCast;use crate::{ArrayElement, DataType,backends::metal::{- CacheLayers, KVCacheUpdate, KernelDataType, MTLCommandBuffer,- MTLCommandQueue, MTLContext, MetalArray,- forward_pass::{- ArrayId, EncodableBlock, EncodingParameters, ForwardPassState,- traces::ActivationTrace,- },+ KVCacheUpdate, MTLCommandBuffer, MTLCommandQueue, MTLContext, Metal,+ MetalArray,},classifier::Classifier,config::ModelMetadata,- encodable_block::Sampling,- forward_pass::scratch_buffers::ScratchBuffers,+ encodable_block::{EncodableBlock, EncodingParameters, Sampling},+ forward_pass::{+ cache_layers::CacheLayers,+ scratch_buffers::ScratchBuffers,+ state::{ArrayId, ForwardPassState},+ traces::ActivationTrace,+ },language_model::{LanguageModelGeneratorContext,sampler::{ArgmaxSampler, LogitsSampler},@@ -562,7 +563,7 @@ impl TraceValidator {// ========================================================================fn validate_layer_traces(- traces: &Rc<RefCell<ActivationTrace>>,+ traces: &Rc<RefCell<ActivationTrace<Metal>>>,traces_view: &ParameterTree<MTLContext>,data_type: DataType,) -> Vec<TracerValidationResult> {@@ -677,7 +678,7 @@ impl TraceValidator {}fn validate_classifier_traces(- traces: &Rc<RefCell<ActivationTrace>>,+ traces: &Rc<RefCell<ActivationTrace<Metal>>>,traces_view: &ParameterTree<MTLContext>,data_type: DataType,) -> Vec<TracerValidationResult> {@@ -1012,7 +1013,7 @@ impl TraceValidator {);context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(- &context.mtl_context,+ context.mtl_context.as_ref(),&context.model_shape,resolved_prefix_length,desired_suffix_length,@@ -1020,12 +1021,11 @@ impl TraceValidator {let intermediate_dtype: DataType =decoder_config.output_norm_config.scale_precision.into();- let kernel_dtype: KernelDataType = intermediate_dtype.into();context.kv_cache_update = Box::new(KVCacheUpdate::new(- &context.mtl_context,- kernel_dtype,+ context.mtl_context.as_ref(),+ intermediate_dtype,resolved_prefix_length,).expect("Failed to create KV cache update kernel"),diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex 23a5f8577..c2ee30066 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -6,10 +6,10 @@ use uzu::{array::ArrayContextExt,backends::{common::Context,- metal::{- KVCacheUpdate, KernelDataType, MTLContext,- forward_pass::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},- },+ metal::{KVCacheUpdate, MTLContext, Metal},+ },+ forward_pass::kv_cache_layer::{+ INVALID_POSITION, KVCacheLayer, KVCacheLayerState,},};@@ -37,7 +37,7 @@ fn make_test_layer(state: KVCacheLayerState,prefix_capacity: usize,suffix_capacity: usize,-) -> KVCacheLayer {+) -> KVCacheLayer<Metal> {let total_len = match &state {KVCacheLayerState::Full {..@@ -79,7 +79,7 @@ fn make_test_layer(}}-fn fill_arrays(layer: &mut KVCacheLayer) -> (Vec<f32>, Vec<f32>) {+fn fill_arrays(layer: &mut KVCacheLayer<Metal>) -> (Vec<f32>, Vec<f32>) {let initial_keys = {let mut keys_ref = layer.keys.borrow_mut();let slice = keys_ref.as_slice_mut::<f32>();@@ -143,7 +143,7 @@ fn expected_after_update(}fn collect_mask(- layer: &KVCacheLayer,+ layer: &KVCacheLayer<Metal>,suffix_positions: &[usize],) -> Vec<Vec<bool>> {let suffix_length = suffix_positions.len();@@ -205,19 +205,17 @@ fn run_scenario(scenario.name);- let kv_cache_update = match KVCacheUpdate::new(- context,- KernelDataType::Float32,- total_sequence_length,- ) {- Ok(update) => update,- Err(e) => {- panic!(- "Failed to create KV cache update for scenario {}: {:?}",- scenario.name, e- );- },- };+ let kv_cache_update =+ match KVCacheUpdate::new(context, DataType::F32, total_sequence_length)+ {+ Ok(update) => update,+ Err(e) => {+ panic!(+ "Failed to create KV cache update for scenario {}: {:?}",+ scenario.name, e+ );+ },+ };let command_buffer = context.command_queuediff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rsindex 922032495..1af8f358c 100644--- a/crates/uzu/tests/kv_cache_update_test.rs+++ b/crates/uzu/tests/kv_cache_update_test.rs@@ -6,11 +6,12 @@ use metal::{};use ndarray::{Array, Array3, s};use uzu::backends::{- common::Context,- metal::{- KVCacheUpdate, KernelDataType, MTLContext,- kernel::kv_cache_update::{KVLayerData, Swap, create_swaps_direct},+ common::{+ Context,+ gpu_types::Swap,+ kernel::kv_cache_update::{KVLayerData, create_swaps_direct},},+ metal::{KVCacheUpdate, MTLContext, Metal},};fn apply_swaps_3d<T: Clone>(@@ -51,7 +52,7 @@ fn test_random_pattern(context: &MTLContext) {let max_sequence_length = 256usize;let kv_cache_update = match KVCacheUpdate::new(context,- KernelDataType::Float32,+ uzu::DataType::F32,max_sequence_length,) {Ok(update) => update,@@ -104,7 +105,7 @@ fn test_random_pattern(context: &MTLContext) {).expect("Failed to create buffer");- let kv_layer_data = KVLayerData {+ let kv_layer_data = KVLayerData::<Metal> {key_buffer: key_buffer.clone(),key_shape: [num_heads, seq_len, head_dim],value_buffer: value_buffer.clone(),