A high-performance inference engine for AI models
generic model context (#210)
uuuvnb2ef8be0de21
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rsindex e0b7b3aaa..9faeb13ab 100644--- a/crates/uzu/src/backends/common/backend.rs+++ b/crates/uzu/src/backends/common/backend.rs@@ -1,6 +1,6 @@use std::{error::Error, fmt::Debug};-use super::{CommandBuffer, Context, CopyEncoder, Kernels, NativeBuffer};+use super::{CommandBuffer, Context, CopyEncoder, Event, Kernels, NativeBuffer};pub trait Backend: Debug + Clone {type Context: Context<Backend = Self>;@@ -8,6 +8,7 @@ pub trait Backend: Debug + Clone {type CommandBuffer: CommandBuffer<Backend = Self>;type ComputeEncoder;type CopyEncoder: CopyEncoder<Backend = Self>;+ type Event: Event<Backend = Self>;type Kernels: Kernels<Backend = Self>;type Error: Error + Debug;}diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rsindex 240289230..52eda7038 100644--- a/crates/uzu/src/backends/common/context.rs+++ b/crates/uzu/src/backends/common/context.rs@@ -17,4 +17,6 @@ pub trait Context: Sized {fn create_command_buffer(&self) -> Result<<Self::Backend as Backend>::CommandBuffer, <Self::Backend as Backend>::Error>;++ fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;}diff --git a/crates/uzu/src/backends/common/event.rs b/crates/uzu/src/backends/common/event.rsnew file mode 100644index 000000000..57eeccf77--- /dev/null+++ b/crates/uzu/src/backends/common/event.rs@@ -0,0 +1,5 @@+use super::Backend;++pub trait Event {+ type Backend: Backend;+}diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rsindex 2f561b239..c5cab7be9 100644--- a/crates/uzu/src/backends/common/mod.rs+++ b/crates/uzu/src/backends/common/mod.rs@@ -5,6 +5,7 @@ mod buffer_lifetime;mod command_buffer;mod context;mod copy_encoder;+mod event;pub mod gpu_types;pub mod kernel;mod native_buffer;@@ -16,5 +17,6 @@ pub use buffer_lifetime::BufferLifetime;pub use command_buffer::CommandBuffer;pub use context::Context;pub use copy_encoder::CopyEncoder;+pub use event::Event;pub use kernel::Kernels;pub use native_buffer::NativeBuffer;diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rsindex 349ad9151..d4b7447c7 100644--- a/crates/uzu/src/backends/metal/backend.rs+++ b/crates/uzu/src/backends/metal/backend.rs@@ -1,4 +1,4 @@-use metal::{MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder};+use metal::{MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder, MTLEvent};use objc2::{rc::Retained, runtime::ProtocolObject};use super::{MetalContext, MetalError, MetalKernels};@@ -13,6 +13,7 @@ impl Backend for Metal {type CommandBuffer = Retained<ProtocolObject<dyn MTLCommandBuffer>>;type ComputeEncoder = ProtocolObject<dyn MTLComputeCommandEncoder>;type CopyEncoder = ProtocolObject<dyn MTLBlitCommandEncoder>;+ type Event = Retained<ProtocolObject<dyn MTLEvent>>;type Kernels = MetalKernels;type Error = MetalError;}diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex e653ad5cc..3b263b467 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -1,7 +1,7 @@use std::{cell::RefCell, collections::HashMap, env, rc::Rc};use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt,+ MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions,};use objc2::{rc::Retained, runtime::ProtocolObject};@@ -227,4 +227,8 @@ impl Context for MetalContext {fn create_command_buffer(&self) -> Result<Retained<ProtocolObject<dyn MTLCommandBuffer>>, MetalError> {self.command_queue.command_buffer().ok_or(MetalError::Generic("cannot create command buffer".into()))}++ fn create_event(&self) -> Result<Retained<ProtocolObject<dyn MTLEvent>>, MetalError> {+ self.device.new_event().ok_or(MetalError::Generic("cannot create event".into()))+ }}diff --git a/crates/uzu/src/backends/metal/error.rs b/crates/uzu/src/backends/metal/error.rsindex 620b39be8..5c328c7ab 100644--- a/crates/uzu/src/backends/metal/error.rs+++ b/crates/uzu/src/backends/metal/error.rs@@ -176,18 +176,3 @@ pub enum PixelFormatError {#[error("{0}")]Custom(String),}--// Classifier context errors-#[derive(Debug, Error)]-pub enum ClassifierError {- #[error("Missing required config field: {0}")]- MissingConfigField(String),- #[error("Model requires attention layers but found non-attention mixer")]- NonAttentionMixer,- #[error("Weight subtree not found: {0}")]- WeightSubtreeNotFound(String),- #[error("Kernel creation failed: {0}")]- KernelCreationFailed(String),- #[error("{0}")]- Custom(String),-}diff --git a/crates/uzu/src/backends/metal/event.rs b/crates/uzu/src/backends/metal/event.rsnew file mode 100644index 000000000..8a4e5329a--- /dev/null+++ b/crates/uzu/src/backends/metal/event.rs@@ -0,0 +1,9 @@+use metal::MTLEvent;+use objc2::{rc::Retained, runtime::ProtocolObject};++use super::Metal;+use crate::backends::common::Event;++impl Event for Retained<ProtocolObject<dyn MTLEvent>> {+ type Backend = Metal;+}diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex dd99c1aa0..71c284584 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -1,19 +1,16 @@mod backend;-mod classifier_context;mod command_buffer;mod context;mod copy_encoder;mod data_type;pub mod error;+mod event;pub mod kernel;-mod language_model_generator_context;pub mod metal_extensions;mod native_buffer;pub use backend::Metal;-pub use classifier_context::ClassifierContext;pub use context::{DeviceArchitecture, DeviceClass, DeviceGeneration, MetalContext};pub use error::MetalError;pub use kernel::dsl::MetalKernels;-pub use language_model_generator_context::LanguageModelGeneratorContext;pub use metal_extensions::{ComputeEncoderSetValue, FunctionConstantValuesSetValue};diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 5542ac7e7..fc576252c 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -17,7 +17,7 @@ use crate::{};pub struct Classifier {- pub context: ClassifierContext,+ pub context: ClassifierContext<Metal>,}impl Classifier {@@ -92,7 +92,7 @@ impl Classifier {autoreleasepool(|_| {let num_labels = self.context.model_config.model_config.num_labels;let mut state = ForwardPassState::new_classifier(- self.context.mtl_context.clone(),+ self.context.context.clone(),&self.context.model_shape,&self.context.scratch_buffers,self.context.shared_buffers.clone(),@@ -155,7 +155,7 @@ impl Classifier {autoreleasepool(|_| {let num_labels = self.context.model_config.model_config.num_labels;let mut state = ForwardPassState::new_classifier(- self.context.mtl_context.clone(),+ self.context.context.clone(),&self.context.model_shape,&self.context.scratch_buffers,self.context.shared_buffers.clone(),diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rssimilarity index 85%rename from crates/uzu/src/backends/metal/classifier_context.rsrename to crates/uzu/src/classifier/classifier_context.rsindex 951b554d2..6a95d5c34 100644--- a/crates/uzu/src/backends/metal/classifier_context.rs+++ b/crates/uzu/src/classifier/classifier_context.rs@@ -1,15 +1,12 @@use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};-use metal::MTLCommandBuffer;-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::{MetalContext, kernel::dsl::SigmoidMetalKernel};use crate::{DataType,- backends::{- common::{Context, kernel::SigmoidKernel},- metal::{Metal, error::ClassifierError},+ backends::common::{+ Backend, Context, Kernels,+ kernel::{SigmoidKernel, matmul::MatmulKernels},},+ classifier::ClassifierError,config::{ClassifierModelConfig, ModelMetadata},encodable_block::{Activation, ClassifierLayer, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope,@@ -24,34 +21,37 @@ use crate::{session::types::Error,};-pub struct ClassifierContext {- pub mtl_context: Rc<MetalContext>,- pub command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,+pub struct ClassifierContext<B: Backend> {+ pub context: Rc<B::Context>,+ pub command_buffer: B::CommandBuffer,- pub shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,- pub scratch_buffers: ScratchBuffers<Metal>,+ pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,+ pub scratch_buffers: ScratchBuffers<B>,pub model_config: ClassifierModelConfig,pub model_shape: ModelShape,- pub embed: Box<dyn EncodableBlock<Metal>>,- pub embedding_norm: Normalization<Metal>,- pub layers: Box<[ClassifierLayer<Metal>]>,- pub output_norm: Normalization<Metal>,- pub global_rope: Rc<Box<dyn EncodableBlock<Metal>>>,- pub local_rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,+ pub embed: Box<dyn EncodableBlock<B>>,+ pub embedding_norm: Normalization<B>,+ pub layers: Box<[ClassifierLayer<B>]>,+ pub output_norm: Normalization<B>,+ pub global_rope: Rc<Box<dyn EncodableBlock<B>>>,+ pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,- pub pooling: Box<dyn EncodableBlock<Metal>>,- pub prediction_head: Box<dyn EncodableBlock<Metal>>,+ pub pooling: Box<dyn EncodableBlock<B>>,+ pub prediction_head: Box<dyn EncodableBlock<B>>,- pub sigmoid_kernel: SigmoidMetalKernel,+ pub sigmoid_kernel: <B::Kernels as Kernels>::SigmoidKernel,}-impl ClassifierContext {+impl<B: Backend + 'static> ClassifierContext<B>+where+ B::Kernels: MatmulKernels,+{pub fn new(model_path: &Path) -> Result<Self, Error> {- let context = MetalContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;+ let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;- let command_buffer = context.create_command_buffer().map_err(|_| Error::UnableToCreateMetalContext)?;+ let command_buffer = context.create_command_buffer().map_err(|_| Error::UnableToCreateBackendContext)?;let config_path = model_path.join("config.json");if !config_path.exists() {@@ -201,7 +201,7 @@ impl ClassifierContext {let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))})?;- let prediction_head_dense = linear_block::<1, Metal>(+ let prediction_head_dense = linear_block::<1, B>(&prediction_head_config.dense_config,prediction_head_config.use_dense_bias,model_dim,@@ -213,7 +213,7 @@ impl ClassifierContext {);let prediction_head_activation = Box::new(- Activation::<Metal>::new(+ Activation::<B>::new(&context,prediction_head_data_type,prediction_head_config.activation.clone(),@@ -243,7 +243,7 @@ impl ClassifierContext {let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))})?;- let prediction_head_final_linear = linear_block::<1, Metal>(+ let prediction_head_final_linear = linear_block::<1, B>(&prediction_head_config.readout_config,true,model_dim,@@ -257,13 +257,13 @@ impl ClassifierContext {Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head readout: {:?}", e)))})?;- let sigmoid_kernel = SigmoidMetalKernel::new(&context, data_type.into()).map_err(|e| {+ let sigmoid_kernel = <B::Kernels as Kernels>::SigmoidKernel::new(&context, data_type.into()).map_err(|e| {eprintln!("Failed to create sigmoid kernel: {:?}", e);- Error::UnableToCreateMetalContext+ Error::UnableToCreateBackendContext})?;let pooling = Box::new(- Pooling::<Metal>::new(+ Pooling::<B>::new(context.as_ref(),data_type,classifier_model_config.model_config.classifier_pooling.clone(),@@ -271,7 +271,7 @@ impl ClassifierContext {).map_err(|e| {eprintln!("Failed to create pooling: {:?}", e);- Error::UnableToCreateMetalContext+ Error::UnableToCreateBackendContext})?,);@@ -286,7 +286,7 @@ impl ClassifierContext {));Ok(Self {- mtl_context: context,+ context,command_buffer,shared_buffers,scratch_buffers,@@ -305,18 +305,18 @@ impl ClassifierContext {}fn create_rope_block(- mtl_context: &MetalContext,+ context: &B::Context,data_type: DataType,rope_type: RopeType,- ) -> Result<Rc<Box<dyn EncodableBlock<Metal>>>, ClassifierError> {- let rotation: Box<dyn EncodableBlock<Metal>> = Box::new(- Rope::<Metal>::new(mtl_context, data_type, rope_type)+ ) -> Result<Rc<Box<dyn EncodableBlock<B>>>, ClassifierError> {+ let rotation: Box<dyn EncodableBlock<B>> = Box::new(+ Rope::<B>::new(context, data_type, rope_type).map_err(|e| ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e)))?,);Ok(Rc::new(rotation))}pub fn reset_command_buffer(&mut self) {- self.command_buffer = self.mtl_context.create_command_buffer().expect("Failed to create command buffer");+ self.command_buffer = self.context.create_command_buffer().expect("Failed to create command buffer");}}diff --git a/crates/uzu/src/classifier/error.rs b/crates/uzu/src/classifier/error.rsnew file mode 100644index 000000000..c650bcb58--- /dev/null+++ b/crates/uzu/src/classifier/error.rs@@ -0,0 +1,16 @@+use thiserror::Error;++// Classifier context errors+#[derive(Debug, Error)]+pub enum ClassifierError {+ #[error("Missing required config field: {0}")]+ MissingConfigField(String),+ #[error("Model requires attention layers but found non-attention mixer")]+ NonAttentionMixer,+ #[error("Weight subtree not found: {0}")]+ WeightSubtreeNotFound(String),+ #[error("Kernel creation failed: {0}")]+ KernelCreationFailed(String),+ #[error("{0}")]+ Custom(String),+}diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rsindex 44e22e403..a8dfc18e8 100644--- a/crates/uzu/src/classifier/mod.rs+++ b/crates/uzu/src/classifier/mod.rs@@ -1,12 +1,14 @@mod classifier;+mod classifier_context;+mod error;mod result;pub use classifier::Classifier;+pub use classifier_context::ClassifierContext;+pub use error::ClassifierError;pub use result::{ClassificationOutput, ClassificationStats};-// Re-export context from metal backend-pub use crate::backends::metal::ClassifierContext;-// Re-export encodables from metal backend for convenience+// Re-export encodables for conveniencepub use crate::encodable_block::{ClassifierLayer, ClassifierPredictionHead, Pooling};#[cfg(feature = "tracing")]pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 691622123..8c0274fbf 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -37,7 +37,7 @@ pub struct LanguageModelGenerator {pub decoding_config: DecodingConfig,pub tokens: Vec<u64>,- pub context: LanguageModelGeneratorContext,+ pub context: LanguageModelGeneratorContext<Metal>,encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask>,registered_prefix_len: usize,gpu_capture: GpuCaptureManager,@@ -173,7 +173,7 @@ impl LanguageModelGenerator {let should_capture = self.gpu_capture.should_capture_prefill(step == 0);if should_capture {- let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "prefill");+ let _ = self.gpu_capture.start_capture(&self.context.context, "prefill");}objc2::rc::autoreleasepool(|_pool| {@@ -363,7 +363,7 @@ impl LanguageModelGenerator {&self.context.scratch_buffers.attention_window_size_to_bias,&[first_decode_position],1,- &self.context.mtl_context,+ &self.context.context,);self.context.async_buffers.prepare_positions(prefill_count, tokens_to_generate);@@ -426,11 +426,11 @@ impl LanguageModelGenerator {let is_first_decode = !is_continuation;let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);if should_capture {- let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "decode");+ let _ = self.gpu_capture.start_capture(&self.context.context, "decode");}let mut state = ForwardPassState::new_llm(- self.context.mtl_context.clone(),+ self.context.context.clone(),&self.context.decoder_config,&self.context.model_shape,&self.context.scratch_buffers,@@ -616,7 +616,7 @@ impl LanguageModelGenerator {let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);if should_capture {- let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "decode");+ let _ = self.gpu_capture.start_capture(&self.context.context, "decode");}let encoded_task_key = task.encoded_task_key(self.tokens.len());@@ -703,7 +703,7 @@ impl LanguageModelGenerator {suffix_start: Option<usize>,wait_until_completed: bool,) {- let command_buffer = self.context.mtl_context.create_command_buffer().expect("Failed to create command buffer");+ let command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");let root_command_buffer = command_buffer.to_owned();{diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rssimilarity index 71%rename from crates/uzu/src/backends/metal/language_model_generator_context.rsrename to crates/uzu/src/language_model/language_model_generator_context.rsindex 20d90de05..abfbfd3dc 100644--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -6,20 +6,14 @@ use std::{rc::Rc,};-use metal::{MTLBuffer, MTLCommandBuffer, MTLDeviceExt, MTLEvent};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::{Metal, MetalContext};use crate::{DataType,- backends::{- common::{- Context,- kernel::{- MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate,- },+ backends::common::{+ Backend, Context, Kernels, NativeBuffer,+ kernel::{+ MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate,+ matmul::MatmulKernels,},- metal::kernel::dsl::{MaskUpdateMetalKernel, TokenCopySampledMetalKernel, TokenCopyToResultsMetalKernel},},config::{DecoderConfig, LanguageModelConfig, ModelMetadata},encodable_block::{Decoder, Sampling},@@ -37,18 +31,18 @@ use crate::{/// Pre-allocated buffers for async generation pipeline./// Indexed by pass_idx to avoid race conditions between GPU passes.-pub struct AsyncBuffers {+pub struct AsyncBuffers<B: Backend> {/// Positions buffer: [max_tokens] i32/// Pre-populated with [prefill_count, prefill_count+1, ...]- pub positions: Retained<ProtocolObject<dyn MTLBuffer>>,+ pub positions: B::NativeBuffer,/// Seeds buffer: [max_tokens] u64/// Pre-populated with deterministic seed sequence- pub seeds: Retained<ProtocolObject<dyn MTLBuffer>>,+ pub seeds: B::NativeBuffer,/// Results buffer: [batch_size] u32/// Each pass writes its sampled token to results[pass_idx % batch_size]- pub results: Retained<ProtocolObject<dyn MTLBuffer>>,- /// Metal event for GPU-side synchronization between passes- pub event: Retained<ProtocolObject<dyn MTLEvent>>,+ pub results: B::NativeBuffer,+ /// Event for GPU-side synchronization between passes+ pub event: B::Event,/// Current event counter (pass N waits on N, signals N+1)pub counter: Cell<u64>,/// Number of tokens after prefill (base for position calculation)@@ -57,9 +51,9 @@ pub struct AsyncBuffers {pub batch_size: usize,}-impl AsyncBuffers {+impl<B: Backend> AsyncBuffers<B> {pub fn new(- context: &MetalContext,+ context: &B::Context,max_tokens: usize,batch_size: usize,) -> Self {@@ -69,7 +63,7 @@ impl AsyncBuffers {context.create_buffer(max_tokens * std::mem::size_of::<u64>()).expect("Failed to create seeds buffer");let results =context.create_buffer(batch_size * std::mem::size_of::<u32>()).expect("Failed to create results buffer");- let event = context.device.new_event().expect("Failed to create event");+ let event = context.create_event().expect("Failed to create event");Self {positions,@@ -91,7 +85,7 @@ impl AsyncBuffers {) {self.prefill_count.set(prefill_count);let base_position = prefill_count.saturating_sub(1);- let ptr = self.positions.contents().as_ptr() as *mut i32;+ let ptr = self.positions.cpu_ptr().as_ptr() as *mut i32;for i in 0..tokens_to_generate {unsafe {*ptr.add(i) = (base_position + i) as i32;@@ -106,7 +100,7 @@ impl AsyncBuffers {prefix_len: usize,tokens_to_generate: usize,) {- let ptr = self.seeds.contents().as_ptr() as *mut u64;+ let ptr = self.seeds.cpu_ptr().as_ptr() as *mut u64;for i in 0..tokens_to_generate {unsafe {*ptr.add(i) = seed.derive((prefix_len + i - 1) as u64);@@ -124,44 +118,47 @@ impl AsyncBuffers {&self,pass_idx: usize,) -> u32 {- let ptr = self.results.contents().as_ptr() as *const u32;+ let ptr = self.results.cpu_ptr().as_ptr() as *const u32;unsafe { *ptr.add(pass_idx % self.batch_size) }}}-pub struct LanguageModelGeneratorContext {- pub mtl_context: Rc<MetalContext>,- pub command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,+pub struct LanguageModelGeneratorContext<B: Backend> {+ pub context: Rc<B::Context>,+ pub command_buffer: B::CommandBuffer,- pub cache_layers: Rc<RefCell<CacheLayers<Metal>>>,- pub shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,- pub scratch_buffers: ScratchBuffers<Metal>,+ pub cache_layers: Rc<RefCell<CacheLayers<B>>>,+ pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,+ pub scratch_buffers: ScratchBuffers<B>,pub model_config: LanguageModelConfig,pub decoder_config: Rc<DecoderConfig>,pub model_shape: ModelShape,- pub executables: Decoder<Metal>,- pub kv_cache_update: Box<KVCacheUpdate<Metal>>,- pub gpu_sampler: Sampling<Metal>,+ pub executables: Decoder<B>,+ pub kv_cache_update: Box<KVCacheUpdate<B>>,+ pub gpu_sampler: Sampling<B>,pub seed: PRng,/// Kernels for copying sampled tokens in async pipeline- pub token_copy_sampled: TokenCopySampledMetalKernel,- pub token_copy_results: TokenCopyToResultsMetalKernel,+ pub token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,+ pub token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,/// Kernel for updating attention mask between async passes- pub mask_update: Option<MaskUpdateMetalKernel>,+ pub mask_update: Option<<B::Kernels as Kernels>::MaskUpdateKernel>,/// Pre-allocated buffers for async generation- pub async_buffers: AsyncBuffers,+ pub async_buffers: AsyncBuffers<B>,}-impl LanguageModelGeneratorContext {+impl<B: Backend + 'static> LanguageModelGeneratorContext<B>+where+ B::Kernels: MatmulKernels,+{pub fn new(model_path: &Path,decoding_config: &DecodingConfig,) -> Result<Self, Error> {- let context = MetalContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;+ let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;- let command_buffer = context.create_command_buffer().expect("Failed to create command buffer").to_owned();+ let command_buffer = context.create_command_buffer().expect("Failed to create command buffer");let config_path = model_path.join("config.json");if !config_path.exists() {@@ -208,35 +205,35 @@ impl LanguageModelGeneratorContext {let intermediate_data_type: DataType = decoder_config.output_norm_config.scale_precision.into();let kv_cache_update = Box::new(KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)- .map_err(|_| Error::UnableToCreateMetalContext)?,+ .map_err(|_| Error::UnableToCreateBackendContext)?,);let gpu_sampler =- Sampling::<Metal>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)- .map_err(|_| Error::UnableToCreateMetalContext)?;+ Sampling::<B>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)+ .map_err(|_| Error::UnableToCreateBackendContext)?;- let token_copy_sampled =- TokenCopySampledMetalKernel::new(&context).map_err(|_| Error::UnableToCreateMetalContext)?;- let token_copy_results =- TokenCopyToResultsMetalKernel::new(&context).map_err(|_| Error::UnableToCreateMetalContext)?;+ let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)+ .map_err(|_| Error::UnableToCreateBackendContext)?;+ let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)+ .map_err(|_| Error::UnableToCreateBackendContext)?;// Create mask update kernel if model has attention layerslet mask_update = if decoder_config.has_attention_layers() {Some(- MaskUpdateMetalKernel::new(&context, intermediate_data_type)- .map_err(|_| Error::UnableToCreateMetalContext)?,+ <B::Kernels as Kernels>::MaskUpdateKernel::new(&context, intermediate_data_type)+ .map_err(|_| Error::UnableToCreateBackendContext)?,)} else {None};let async_batch_size = decoding_config.async_batch_size.resolve(model_path);- let async_buffers = AsyncBuffers::new(&context, max_prefix_length, async_batch_size);+ let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);let seed = PRng::new(decoding_config.sampling_seed.resolve());let context = Self {- mtl_context: context,+ context,command_buffer,cache_layers,shared_buffers,@@ -258,6 +255,6 @@ impl LanguageModelGeneratorContext {}pub fn reset_command_buffer(&mut self) {- self.command_buffer = self.mtl_context.create_command_buffer().expect("Failed to create command buffer");+ self.command_buffer = self.context.create_command_buffer().expect("Failed to create command buffer");}}diff --git a/crates/uzu/src/language_model/mod.rs b/crates/uzu/src/language_model/mod.rsindex 2366b34de..b2f74ea01 100644--- a/crates/uzu/src/language_model/mod.rs+++ b/crates/uzu/src/language_model/mod.rs@@ -2,6 +2,7 @@ pub mod gpu_capture;pub mod grammar;pub mod gumbel;pub mod language_model_generator;+pub mod language_model_generator_context;pub mod result;pub mod rng;pub mod sampler;@@ -9,5 +10,4 @@ pub mod tasks;// Re-export main typespub use language_model_generator::LanguageModelGenerator;--pub use crate::backends::metal::LanguageModelGeneratorContext;+pub use language_model_generator_context::LanguageModelGeneratorContext;diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsindex 66037297c..9604c571d 100644--- a/crates/uzu/src/language_model/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -83,12 +83,12 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {pub fn create_state(&self,- context: &mut LanguageModelGeneratorContext,+ context: &mut LanguageModelGeneratorContext<Metal>,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,should_fill_attention_bias: bool,) -> ForwardPassState<Metal> {ForwardPassState::new_llm(- context.mtl_context.clone(),+ context.context.clone(),&context.decoder_config,&context.model_shape,&context.scratch_buffers,@@ -112,7 +112,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {pub fn build_encoded_task(&self,- context: &LanguageModelGeneratorContext,+ context: &LanguageModelGeneratorContext<Metal>,state: &mut ForwardPassState<Metal>,parameters: &EncodingParameters<Metal>,key: String,@@ -122,7 +122,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {LanguageModelGeneratorEncodedTask {key,predicate_buffer: context- .mtl_context+ .context.create_buffer(size_of::<u32>()).expect("Failed to create predicate buffer"),}diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex f16f730d0..45c4ec1c1 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -438,7 +438,7 @@ impl ChatSession {let batch_submitted = batch_end - next_to_submit;let accepted_before_batch = results.len();- let cache_slice = llm.context.cache_layers.borrow().slice(&llm.context.mtl_context, 0..batch_submitted);+ let cache_slice = llm.context.cache_layers.borrow().slice(&llm.context.context, 0..batch_submitted);for idx in next_to_submit..batch_end {let batch_sender = sender.clone();@@ -568,7 +568,7 @@ impl ChatSession {fn build_context_from_language_model_generator(&self) -> Result<Context, Error> {let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;let cache_layers =- language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.mtl_context);+ language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.context);let context = Context::new(language_model_generator.tokens.clone(),cache_layers,diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rsindex 03029e4ae..4f172256f 100644--- a/crates/uzu/src/session/types/error.rs+++ b/crates/uzu/src/session/types/error.rs@@ -1,11 +1,11 @@-use crate::backends::metal::error::ClassifierError;+use crate::classifier::ClassifierError;#[derive(Debug, thiserror::Error)]pub enum Error {#[error("Model folder not found")]ModelFolderNotFound,- #[error("Unable to create Metal context")]- UnableToCreateMetalContext,+ #[error("Unable to create Backend context")]+ UnableToCreateBackendContext,#[error("Unable to load model configuration")]UnableToLoadConfig,#[error("Unable to load model weights")]diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex 789927a12..bd125e868 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -157,7 +157,7 @@ pub enum ArrayTransform {// ============================================================================enum ModelContext {- LanguageModelGenerator(LanguageModelGeneratorContext),+ LanguageModelGenerator(LanguageModelGeneratorContext<Metal>),Classifier(Classifier),}@@ -202,7 +202,7 @@ impl TraceValidator {AsyncBatchSize::default(),false,);- let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;+ let mut llm_context = LanguageModelGeneratorContext::<Metal>::new(model_path, &decoding_config)?;let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);ModelContext::LanguageModelGenerator(llm_context)@@ -240,12 +240,12 @@ impl TraceValidator {// ========================================================================fn run_llm_validation(- ctx: &LanguageModelGeneratorContext,+ ctx: &LanguageModelGeneratorContext<Metal>,traces_path: &Path,) -> Result<TracerValidationResults, Error> {let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;let traces_loader =- ParameterLoader::new(&traces_file, ctx.mtl_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;+ ParameterLoader::new(&traces_file, ctx.context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;let traces_view = traces_loader.tree();let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");@@ -253,7 +253,7 @@ impl TraceValidator {let token_seeds: Vec<u64> = vec![0; token_ids.len()];let mut state = ForwardPassState::new_llm(- ctx.mtl_context.clone(),+ ctx.context.clone(),&ctx.decoder_config,&ctx.model_shape,&ctx.scratch_buffers,@@ -274,7 +274,7 @@ impl TraceValidator {None,);- let command_buffer = ctx.mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");+ let command_buffer = ctx.context.command_queue.command_buffer().expect("Failed to create command buffer");ctx.executables.encode(&mut state, &EncodingParameters::new(false, false, false), &command_buffer);command_buffer.commit();@@ -389,9 +389,9 @@ impl TraceValidator {traces_path: &Path,) -> Result<TracerValidationResults, Error> {let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;- let mtl_context = classifier.context.mtl_context.clone();+ let context = classifier.context.context.clone();let traces_loader =- ParameterLoader::new(&traces_file, mtl_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;+ ParameterLoader::new(&traces_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;let traces_view = traces_loader.tree();let has_token_ids = traces_view.leaf("activation_trace.token_ids").is_ok();@@ -803,7 +803,7 @@ impl TraceValidator {fn ensure_llm_context_capacity(decoding_config: &DecodingConfig,desired_suffix_length: usize,- context: &mut LanguageModelGeneratorContext,+ context: &mut LanguageModelGeneratorContext<Metal>,) {let resolved_prefix_length = decoding_config.context_length.resolve(&context.model_config);let current_suffix_length = std::cmp::max(@@ -817,7 +817,7 @@ impl TraceValidator {let decoder_config = &context.decoder_config;context.scratch_buffers = ScratchBuffers::new(- context.mtl_context.as_ref(),+ context.context.as_ref(),decoder_config,&context.model_shape,resolved_prefix_length,@@ -825,7 +825,7 @@ impl TraceValidator {);context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(- context.mtl_context.as_ref(),+ context.context.as_ref(),&context.model_shape,resolved_prefix_length,desired_suffix_length,@@ -834,12 +834,12 @@ impl TraceValidator {let intermediate_dtype: DataType = decoder_config.output_norm_config.scale_precision.into();context.kv_cache_update = Box::new(- KVCacheUpdate::new(context.mtl_context.as_ref(), intermediate_dtype, resolved_prefix_length)+ KVCacheUpdate::new(context.context.as_ref(), intermediate_dtype, resolved_prefix_length).expect("Failed to create KV cache update kernel"),);context.gpu_sampler = Sampling::new(- context.mtl_context.as_ref(),+ context.context.as_ref(),intermediate_dtype,desired_suffix_length,decoder_config.vocab_size,