Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

generic model context (#210)

uuuvnb2ef8be0de21
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rs
index e0b7b3aaa..9faeb13ab 100644
--- a/crates/uzu/src/backends/common/backend.rs
+++ b/crates/uzu/src/backends/common/backend.rs
@@ -1,6 +1,6 @@
use std::{error::Error, fmt::Debug};
-use super::{CommandBuffer, Context, CopyEncoder, Kernels, NativeBuffer};
+use super::{CommandBuffer, Context, CopyEncoder, Event, Kernels, NativeBuffer};
pub trait Backend: Debug + Clone {
type Context: Context<Backend = Self>;
@@ -8,6 +8,7 @@ pub trait Backend: Debug + Clone {
type CommandBuffer: CommandBuffer<Backend = Self>;
type ComputeEncoder;
type CopyEncoder: CopyEncoder<Backend = Self>;
+ type Event: Event<Backend = Self>;
type Kernels: Kernels<Backend = Self>;
type Error: Error + Debug;
}
diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rs
index 240289230..52eda7038 100644
--- a/crates/uzu/src/backends/common/context.rs
+++ b/crates/uzu/src/backends/common/context.rs
@@ -17,4 +17,6 @@ pub trait Context: Sized {
fn create_command_buffer(
&self
) -> Result<<Self::Backend as Backend>::CommandBuffer, <Self::Backend as Backend>::Error>;
+
+ fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;
}
diff --git a/crates/uzu/src/backends/common/event.rs b/crates/uzu/src/backends/common/event.rs
new file mode 100644
index 000000000..57eeccf77
--- /dev/null
+++ b/crates/uzu/src/backends/common/event.rs
@@ -0,0 +1,5 @@
+use super::Backend;
+
+pub trait Event {
+ type Backend: Backend;
+}
diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rs
index 2f561b239..c5cab7be9 100644
--- a/crates/uzu/src/backends/common/mod.rs
+++ b/crates/uzu/src/backends/common/mod.rs
@@ -5,6 +5,7 @@ mod buffer_lifetime;
mod command_buffer;
mod context;
mod copy_encoder;
+mod event;
pub mod gpu_types;
pub mod kernel;
mod native_buffer;
@@ -16,5 +17,6 @@ pub use buffer_lifetime::BufferLifetime;
pub use command_buffer::CommandBuffer;
pub use context::Context;
pub use copy_encoder::CopyEncoder;
+pub use event::Event;
pub use kernel::Kernels;
pub use native_buffer::NativeBuffer;
diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rs
index 349ad9151..d4b7447c7 100644
--- a/crates/uzu/src/backends/metal/backend.rs
+++ b/crates/uzu/src/backends/metal/backend.rs
@@ -1,4 +1,4 @@
-use metal::{MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder};
+use metal::{MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder, MTLEvent};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{MetalContext, MetalError, MetalKernels};
@@ -13,6 +13,7 @@ impl Backend for Metal {
type CommandBuffer = Retained<ProtocolObject<dyn MTLCommandBuffer>>;
type ComputeEncoder = ProtocolObject<dyn MTLComputeCommandEncoder>;
type CopyEncoder = ProtocolObject<dyn MTLBlitCommandEncoder>;
+ type Event = Retained<ProtocolObject<dyn MTLEvent>>;
type Kernels = MetalKernels;
type Error = MetalError;
}
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index e653ad5cc..3b263b467 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -1,7 +1,7 @@
use std::{cell::RefCell, collections::HashMap, env, rc::Rc};
use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt,
+ MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,
MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
@@ -227,4 +227,8 @@ impl Context for MetalContext {
fn create_command_buffer(&self) -> Result<Retained<ProtocolObject<dyn MTLCommandBuffer>>, MetalError> {
self.command_queue.command_buffer().ok_or(MetalError::Generic("cannot create command buffer".into()))
}
+
+ fn create_event(&self) -> Result<Retained<ProtocolObject<dyn MTLEvent>>, MetalError> {
+ self.device.new_event().ok_or(MetalError::Generic("cannot create event".into()))
+ }
}
diff --git a/crates/uzu/src/backends/metal/error.rs b/crates/uzu/src/backends/metal/error.rs
index 620b39be8..5c328c7ab 100644
--- a/crates/uzu/src/backends/metal/error.rs
+++ b/crates/uzu/src/backends/metal/error.rs
@@ -176,18 +176,3 @@ pub enum PixelFormatError {
#[error("{0}")]
Custom(String),
}
-
-// Classifier context errors
-#[derive(Debug, Error)]
-pub enum ClassifierError {
- #[error("Missing required config field: {0}")]
- MissingConfigField(String),
- #[error("Model requires attention layers but found non-attention mixer")]
- NonAttentionMixer,
- #[error("Weight subtree not found: {0}")]
- WeightSubtreeNotFound(String),
- #[error("Kernel creation failed: {0}")]
- KernelCreationFailed(String),
- #[error("{0}")]
- Custom(String),
-}
diff --git a/crates/uzu/src/backends/metal/event.rs b/crates/uzu/src/backends/metal/event.rs
new file mode 100644
index 000000000..8a4e5329a
--- /dev/null
+++ b/crates/uzu/src/backends/metal/event.rs
@@ -0,0 +1,9 @@
+use metal::MTLEvent;
+use objc2::{rc::Retained, runtime::ProtocolObject};
+
+use super::Metal;
+use crate::backends::common::Event;
+
+impl Event for Retained<ProtocolObject<dyn MTLEvent>> {
+ type Backend = Metal;
+}
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index dd99c1aa0..71c284584 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -1,19 +1,16 @@
mod backend;
-mod classifier_context;
mod command_buffer;
mod context;
mod copy_encoder;
mod data_type;
pub mod error;
+mod event;
pub mod kernel;
-mod language_model_generator_context;
pub mod metal_extensions;
mod native_buffer;
pub use backend::Metal;
-pub use classifier_context::ClassifierContext;
pub use context::{DeviceArchitecture, DeviceClass, DeviceGeneration, MetalContext};
pub use error::MetalError;
pub use kernel::dsl::MetalKernels;
-pub use language_model_generator_context::LanguageModelGeneratorContext;
pub use metal_extensions::{ComputeEncoderSetValue, FunctionConstantValuesSetValue};
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 5542ac7e7..fc576252c 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -17,7 +17,7 @@ use crate::{
};
pub struct Classifier {
- pub context: ClassifierContext,
+ pub context: ClassifierContext<Metal>,
}
impl Classifier {
@@ -92,7 +92,7 @@ impl Classifier {
autoreleasepool(|_| {
let num_labels = self.context.model_config.model_config.num_labels;
let mut state = ForwardPassState::new_classifier(
- self.context.mtl_context.clone(),
+ self.context.context.clone(),
&self.context.model_shape,
&self.context.scratch_buffers,
self.context.shared_buffers.clone(),
@@ -155,7 +155,7 @@ impl Classifier {
autoreleasepool(|_| {
let num_labels = self.context.model_config.model_config.num_labels;
let mut state = ForwardPassState::new_classifier(
- self.context.mtl_context.clone(),
+ self.context.context.clone(),
&self.context.model_shape,
&self.context.scratch_buffers,
self.context.shared_buffers.clone(),
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rs
similarity index 85%
rename from crates/uzu/src/backends/metal/classifier_context.rs
rename to crates/uzu/src/classifier/classifier_context.rs
index 951b554d2..6a95d5c34 100644
--- a/crates/uzu/src/backends/metal/classifier_context.rs
+++ b/crates/uzu/src/classifier/classifier_context.rs
@@ -1,15 +1,12 @@
use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
-use metal::MTLCommandBuffer;
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::{MetalContext, kernel::dsl::SigmoidMetalKernel};
use crate::{
DataType,
- backends::{
- common::{Context, kernel::SigmoidKernel},
- metal::{Metal, error::ClassifierError},
+ backends::common::{
+ Backend, Context, Kernels,
+ kernel::{SigmoidKernel, matmul::MatmulKernels},
},
+ classifier::ClassifierError,
config::{ClassifierModelConfig, ModelMetadata},
encodable_block::{
Activation, ClassifierLayer, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope,
@@ -24,34 +21,37 @@ use crate::{
session::types::Error,
};
-pub struct ClassifierContext {
- pub mtl_context: Rc<MetalContext>,
- pub command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+pub struct ClassifierContext<B: Backend> {
+ pub context: Rc<B::Context>,
+ pub command_buffer: B::CommandBuffer,
- pub shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,
- pub scratch_buffers: ScratchBuffers<Metal>,
+ pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
+ pub scratch_buffers: ScratchBuffers<B>,
pub model_config: ClassifierModelConfig,
pub model_shape: ModelShape,
- pub embed: Box<dyn EncodableBlock<Metal>>,
- pub embedding_norm: Normalization<Metal>,
- pub layers: Box<[ClassifierLayer<Metal>]>,
- pub output_norm: Normalization<Metal>,
- pub global_rope: Rc<Box<dyn EncodableBlock<Metal>>>,
- pub local_rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,
+ pub embed: Box<dyn EncodableBlock<B>>,
+ pub embedding_norm: Normalization<B>,
+ pub layers: Box<[ClassifierLayer<B>]>,
+ pub output_norm: Normalization<B>,
+ pub global_rope: Rc<Box<dyn EncodableBlock<B>>>,
+ pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
- pub pooling: Box<dyn EncodableBlock<Metal>>,
- pub prediction_head: Box<dyn EncodableBlock<Metal>>,
+ pub pooling: Box<dyn EncodableBlock<B>>,
+ pub prediction_head: Box<dyn EncodableBlock<B>>,
- pub sigmoid_kernel: SigmoidMetalKernel,
+ pub sigmoid_kernel: <B::Kernels as Kernels>::SigmoidKernel,
}
-impl ClassifierContext {
+impl<B: Backend + 'static> ClassifierContext<B>
+where
+ B::Kernels: MatmulKernels,
+{
pub fn new(model_path: &Path) -> Result<Self, Error> {
- let context = MetalContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;
+ let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;
- let command_buffer = context.create_command_buffer().map_err(|_| Error::UnableToCreateMetalContext)?;
+ let command_buffer = context.create_command_buffer().map_err(|_| Error::UnableToCreateBackendContext)?;
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -201,7 +201,7 @@ impl ClassifierContext {
let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {
Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))
})?;
- let prediction_head_dense = linear_block::<1, Metal>(
+ let prediction_head_dense = linear_block::<1, B>(
&prediction_head_config.dense_config,
prediction_head_config.use_dense_bias,
model_dim,
@@ -213,7 +213,7 @@ impl ClassifierContext {
);
let prediction_head_activation = Box::new(
- Activation::<Metal>::new(
+ Activation::<B>::new(
&context,
prediction_head_data_type,
prediction_head_config.activation.clone(),
@@ -243,7 +243,7 @@ impl ClassifierContext {
let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {
Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))
})?;
- let prediction_head_final_linear = linear_block::<1, Metal>(
+ let prediction_head_final_linear = linear_block::<1, B>(
&prediction_head_config.readout_config,
true,
model_dim,
@@ -257,13 +257,13 @@ impl ClassifierContext {
Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head readout: {:?}", e)))
})?;
- let sigmoid_kernel = SigmoidMetalKernel::new(&context, data_type.into()).map_err(|e| {
+ let sigmoid_kernel = <B::Kernels as Kernels>::SigmoidKernel::new(&context, data_type.into()).map_err(|e| {
eprintln!("Failed to create sigmoid kernel: {:?}", e);
- Error::UnableToCreateMetalContext
+ Error::UnableToCreateBackendContext
})?;
let pooling = Box::new(
- Pooling::<Metal>::new(
+ Pooling::<B>::new(
context.as_ref(),
data_type,
classifier_model_config.model_config.classifier_pooling.clone(),
@@ -271,7 +271,7 @@ impl ClassifierContext {
)
.map_err(|e| {
eprintln!("Failed to create pooling: {:?}", e);
- Error::UnableToCreateMetalContext
+ Error::UnableToCreateBackendContext
})?,
);
@@ -286,7 +286,7 @@ impl ClassifierContext {
));
Ok(Self {
- mtl_context: context,
+ context,
command_buffer,
shared_buffers,
scratch_buffers,
@@ -305,18 +305,18 @@ impl ClassifierContext {
}
fn create_rope_block(
- mtl_context: &MetalContext,
+ context: &B::Context,
data_type: DataType,
rope_type: RopeType,
- ) -> Result<Rc<Box<dyn EncodableBlock<Metal>>>, ClassifierError> {
- let rotation: Box<dyn EncodableBlock<Metal>> = Box::new(
- Rope::<Metal>::new(mtl_context, data_type, rope_type)
+ ) -> Result<Rc<Box<dyn EncodableBlock<B>>>, ClassifierError> {
+ let rotation: Box<dyn EncodableBlock<B>> = Box::new(
+ Rope::<B>::new(context, data_type, rope_type)
.map_err(|e| ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e)))?,
);
Ok(Rc::new(rotation))
}
pub fn reset_command_buffer(&mut self) {
- self.command_buffer = self.mtl_context.create_command_buffer().expect("Failed to create command buffer");
+ self.command_buffer = self.context.create_command_buffer().expect("Failed to create command buffer");
}
}
diff --git a/crates/uzu/src/classifier/error.rs b/crates/uzu/src/classifier/error.rs
new file mode 100644
index 000000000..c650bcb58
--- /dev/null
+++ b/crates/uzu/src/classifier/error.rs
@@ -0,0 +1,16 @@
+use thiserror::Error;
+
+// Classifier context errors
+#[derive(Debug, Error)]
+pub enum ClassifierError {
+ #[error("Missing required config field: {0}")]
+ MissingConfigField(String),
+ #[error("Model requires attention layers but found non-attention mixer")]
+ NonAttentionMixer,
+ #[error("Weight subtree not found: {0}")]
+ WeightSubtreeNotFound(String),
+ #[error("Kernel creation failed: {0}")]
+ KernelCreationFailed(String),
+ #[error("{0}")]
+ Custom(String),
+}
diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rs
index 44e22e403..a8dfc18e8 100644
--- a/crates/uzu/src/classifier/mod.rs
+++ b/crates/uzu/src/classifier/mod.rs
@@ -1,12 +1,14 @@
mod classifier;
+mod classifier_context;
+mod error;
mod result;
pub use classifier::Classifier;
+pub use classifier_context::ClassifierContext;
+pub use error::ClassifierError;
pub use result::{ClassificationOutput, ClassificationStats};
-// Re-export context from metal backend
-pub use crate::backends::metal::ClassifierContext;
-// Re-export encodables from metal backend for convenience
+// Re-export encodables for convenience
pub use crate::encodable_block::{ClassifierLayer, ClassifierPredictionHead, Pooling};
#[cfg(feature = "tracing")]
pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 691622123..8c0274fbf 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -37,7 +37,7 @@ pub struct LanguageModelGenerator {
pub decoding_config: DecodingConfig,
pub tokens: Vec<u64>,
- pub context: LanguageModelGeneratorContext,
+ pub context: LanguageModelGeneratorContext<Metal>,
encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask>,
registered_prefix_len: usize,
gpu_capture: GpuCaptureManager,
@@ -173,7 +173,7 @@ impl LanguageModelGenerator {
let should_capture = self.gpu_capture.should_capture_prefill(step == 0);
if should_capture {
- let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "prefill");
+ let _ = self.gpu_capture.start_capture(&self.context.context, "prefill");
}
objc2::rc::autoreleasepool(|_pool| {
@@ -363,7 +363,7 @@ impl LanguageModelGenerator {
&self.context.scratch_buffers.attention_window_size_to_bias,
&[first_decode_position],
1,
- &self.context.mtl_context,
+ &self.context.context,
);
self.context.async_buffers.prepare_positions(prefill_count, tokens_to_generate);
@@ -426,11 +426,11 @@ impl LanguageModelGenerator {
let is_first_decode = !is_continuation;
let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
if should_capture {
- let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "decode");
+ let _ = self.gpu_capture.start_capture(&self.context.context, "decode");
}
let mut state = ForwardPassState::new_llm(
- self.context.mtl_context.clone(),
+ self.context.context.clone(),
&self.context.decoder_config,
&self.context.model_shape,
&self.context.scratch_buffers,
@@ -616,7 +616,7 @@ impl LanguageModelGenerator {
let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
if should_capture {
- let _ = self.gpu_capture.start_capture(&self.context.mtl_context, "decode");
+ let _ = self.gpu_capture.start_capture(&self.context.context, "decode");
}
let encoded_task_key = task.encoded_task_key(self.tokens.len());
@@ -703,7 +703,7 @@ impl LanguageModelGenerator {
suffix_start: Option<usize>,
wait_until_completed: bool,
) {
- let command_buffer = self.context.mtl_context.create_command_buffer().expect("Failed to create command buffer");
+ let command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");
let root_command_buffer = command_buffer.to_owned();
{
diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
similarity index 71%
rename from crates/uzu/src/backends/metal/language_model_generator_context.rs
rename to crates/uzu/src/language_model/language_model_generator_context.rs
index 20d90de05..abfbfd3dc 100644
--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -6,20 +6,14 @@ use std::{
rc::Rc,
};
-use metal::{MTLBuffer, MTLCommandBuffer, MTLDeviceExt, MTLEvent};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::{Metal, MetalContext};
use crate::{
DataType,
- backends::{
- common::{
- Context,
- kernel::{
- MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate,
- },
+ backends::common::{
+ Backend, Context, Kernels, NativeBuffer,
+ kernel::{
+ MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate,
+ matmul::MatmulKernels,
},
- metal::kernel::dsl::{MaskUpdateMetalKernel, TokenCopySampledMetalKernel, TokenCopyToResultsMetalKernel},
},
config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
encodable_block::{Decoder, Sampling},
@@ -37,18 +31,18 @@ use crate::{
/// Pre-allocated buffers for async generation pipeline.
/// Indexed by pass_idx to avoid race conditions between GPU passes.
-pub struct AsyncBuffers {
+pub struct AsyncBuffers<B: Backend> {
/// Positions buffer: [max_tokens] i32
/// Pre-populated with [prefill_count, prefill_count+1, ...]
- pub positions: Retained<ProtocolObject<dyn MTLBuffer>>,
+ pub positions: B::NativeBuffer,
/// Seeds buffer: [max_tokens] u64
/// Pre-populated with deterministic seed sequence
- pub seeds: Retained<ProtocolObject<dyn MTLBuffer>>,
+ pub seeds: B::NativeBuffer,
/// Results buffer: [batch_size] u32
/// Each pass writes its sampled token to results[pass_idx % batch_size]
- pub results: Retained<ProtocolObject<dyn MTLBuffer>>,
- /// Metal event for GPU-side synchronization between passes
- pub event: Retained<ProtocolObject<dyn MTLEvent>>,
+ pub results: B::NativeBuffer,
+ /// Event for GPU-side synchronization between passes
+ pub event: B::Event,
/// Current event counter (pass N waits on N, signals N+1)
pub counter: Cell<u64>,
/// Number of tokens after prefill (base for position calculation)
@@ -57,9 +51,9 @@ pub struct AsyncBuffers {
pub batch_size: usize,
}
-impl AsyncBuffers {
+impl<B: Backend> AsyncBuffers<B> {
pub fn new(
- context: &MetalContext,
+ context: &B::Context,
max_tokens: usize,
batch_size: usize,
) -> Self {
@@ -69,7 +63,7 @@ impl AsyncBuffers {
context.create_buffer(max_tokens * std::mem::size_of::<u64>()).expect("Failed to create seeds buffer");
let results =
context.create_buffer(batch_size * std::mem::size_of::<u32>()).expect("Failed to create results buffer");
- let event = context.device.new_event().expect("Failed to create event");
+ let event = context.create_event().expect("Failed to create event");
Self {
positions,
@@ -91,7 +85,7 @@ impl AsyncBuffers {
) {
self.prefill_count.set(prefill_count);
let base_position = prefill_count.saturating_sub(1);
- let ptr = self.positions.contents().as_ptr() as *mut i32;
+ let ptr = self.positions.cpu_ptr().as_ptr() as *mut i32;
for i in 0..tokens_to_generate {
unsafe {
*ptr.add(i) = (base_position + i) as i32;
@@ -106,7 +100,7 @@ impl AsyncBuffers {
prefix_len: usize,
tokens_to_generate: usize,
) {
- let ptr = self.seeds.contents().as_ptr() as *mut u64;
+ let ptr = self.seeds.cpu_ptr().as_ptr() as *mut u64;
for i in 0..tokens_to_generate {
unsafe {
*ptr.add(i) = seed.derive((prefix_len + i - 1) as u64);
@@ -124,44 +118,47 @@ impl AsyncBuffers {
&self,
pass_idx: usize,
) -> u32 {
- let ptr = self.results.contents().as_ptr() as *const u32;
+ let ptr = self.results.cpu_ptr().as_ptr() as *const u32;
unsafe { *ptr.add(pass_idx % self.batch_size) }
}
}
-pub struct LanguageModelGeneratorContext {
- pub mtl_context: Rc<MetalContext>,
- pub command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+pub struct LanguageModelGeneratorContext<B: Backend> {
+ pub context: Rc<B::Context>,
+ pub command_buffer: B::CommandBuffer,
- pub cache_layers: Rc<RefCell<CacheLayers<Metal>>>,
- pub shared_buffers: Rc<RefCell<SharedBuffers<Metal>>>,
- pub scratch_buffers: ScratchBuffers<Metal>,
+ pub cache_layers: Rc<RefCell<CacheLayers<B>>>,
+ pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
+ pub scratch_buffers: ScratchBuffers<B>,
pub model_config: LanguageModelConfig,
pub decoder_config: Rc<DecoderConfig>,
pub model_shape: ModelShape,
- pub executables: Decoder<Metal>,
- pub kv_cache_update: Box<KVCacheUpdate<Metal>>,
- pub gpu_sampler: Sampling<Metal>,
+ pub executables: Decoder<B>,
+ pub kv_cache_update: Box<KVCacheUpdate<B>>,
+ pub gpu_sampler: Sampling<B>,
pub seed: PRng,
/// Kernels for copying sampled tokens in async pipeline
- pub token_copy_sampled: TokenCopySampledMetalKernel,
- pub token_copy_results: TokenCopyToResultsMetalKernel,
+ pub token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
+ pub token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,
/// Kernel for updating attention mask between async passes
- pub mask_update: Option<MaskUpdateMetalKernel>,
+ pub mask_update: Option<<B::Kernels as Kernels>::MaskUpdateKernel>,
/// Pre-allocated buffers for async generation
- pub async_buffers: AsyncBuffers,
+ pub async_buffers: AsyncBuffers<B>,
}
-impl LanguageModelGeneratorContext {
+impl<B: Backend + 'static> LanguageModelGeneratorContext<B>
+where
+ B::Kernels: MatmulKernels,
+{
pub fn new(
model_path: &Path,
decoding_config: &DecodingConfig,
) -> Result<Self, Error> {
- let context = MetalContext::new().map_err(|_| Error::UnableToCreateMetalContext)?;
+ let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;
- let command_buffer = context.create_command_buffer().expect("Failed to create command buffer").to_owned();
+ let command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -208,35 +205,35 @@ impl LanguageModelGeneratorContext {
let intermediate_data_type: DataType = decoder_config.output_norm_config.scale_precision.into();
let kv_cache_update = Box::new(
KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)
- .map_err(|_| Error::UnableToCreateMetalContext)?,
+ .map_err(|_| Error::UnableToCreateBackendContext)?,
);
let gpu_sampler =
- Sampling::<Metal>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)
- .map_err(|_| Error::UnableToCreateMetalContext)?;
+ Sampling::<B>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)
+ .map_err(|_| Error::UnableToCreateBackendContext)?;
- let token_copy_sampled =
- TokenCopySampledMetalKernel::new(&context).map_err(|_| Error::UnableToCreateMetalContext)?;
- let token_copy_results =
- TokenCopyToResultsMetalKernel::new(&context).map_err(|_| Error::UnableToCreateMetalContext)?;
+ let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
+ .map_err(|_| Error::UnableToCreateBackendContext)?;
+ let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)
+ .map_err(|_| Error::UnableToCreateBackendContext)?;
// Create mask update kernel if model has attention layers
let mask_update = if decoder_config.has_attention_layers() {
Some(
- MaskUpdateMetalKernel::new(&context, intermediate_data_type)
- .map_err(|_| Error::UnableToCreateMetalContext)?,
+ <B::Kernels as Kernels>::MaskUpdateKernel::new(&context, intermediate_data_type)
+ .map_err(|_| Error::UnableToCreateBackendContext)?,
)
} else {
None
};
let async_batch_size = decoding_config.async_batch_size.resolve(model_path);
- let async_buffers = AsyncBuffers::new(&context, max_prefix_length, async_batch_size);
+ let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);
let seed = PRng::new(decoding_config.sampling_seed.resolve());
let context = Self {
- mtl_context: context,
+ context,
command_buffer,
cache_layers,
shared_buffers,
@@ -258,6 +255,6 @@ impl LanguageModelGeneratorContext {
}
pub fn reset_command_buffer(&mut self) {
- self.command_buffer = self.mtl_context.create_command_buffer().expect("Failed to create command buffer");
+ self.command_buffer = self.context.create_command_buffer().expect("Failed to create command buffer");
}
}
diff --git a/crates/uzu/src/language_model/mod.rs b/crates/uzu/src/language_model/mod.rs
index 2366b34de..b2f74ea01 100644
--- a/crates/uzu/src/language_model/mod.rs
+++ b/crates/uzu/src/language_model/mod.rs
@@ -2,6 +2,7 @@ pub mod gpu_capture;
pub mod grammar;
pub mod gumbel;
pub mod language_model_generator;
+pub mod language_model_generator_context;
pub mod result;
pub mod rng;
pub mod sampler;
@@ -9,5 +10,4 @@ pub mod tasks;
// Re-export main types
pub use language_model_generator::LanguageModelGenerator;
-
-pub use crate::backends::metal::LanguageModelGeneratorContext;
+pub use language_model_generator_context::LanguageModelGeneratorContext;
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index 66037297c..9604c571d 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -83,12 +83,12 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
pub fn create_state(
&self,
- context: &mut LanguageModelGeneratorContext,
+ context: &mut LanguageModelGeneratorContext<Metal>,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
should_fill_attention_bias: bool,
) -> ForwardPassState<Metal> {
ForwardPassState::new_llm(
- context.mtl_context.clone(),
+ context.context.clone(),
&context.decoder_config,
&context.model_shape,
&context.scratch_buffers,
@@ -112,7 +112,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
pub fn build_encoded_task(
&self,
- context: &LanguageModelGeneratorContext,
+ context: &LanguageModelGeneratorContext<Metal>,
state: &mut ForwardPassState<Metal>,
parameters: &EncodingParameters<Metal>,
key: String,
@@ -122,7 +122,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
LanguageModelGeneratorEncodedTask {
key,
predicate_buffer: context
- .mtl_context
+ .context
.create_buffer(size_of::<u32>())
.expect("Failed to create predicate buffer"),
}
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index f16f730d0..45c4ec1c1 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -438,7 +438,7 @@ impl ChatSession {
let batch_submitted = batch_end - next_to_submit;
let accepted_before_batch = results.len();
- let cache_slice = llm.context.cache_layers.borrow().slice(&llm.context.mtl_context, 0..batch_submitted);
+ let cache_slice = llm.context.cache_layers.borrow().slice(&llm.context.context, 0..batch_submitted);
for idx in next_to_submit..batch_end {
let batch_sender = sender.clone();
@@ -568,7 +568,7 @@ impl ChatSession {
fn build_context_from_language_model_generator(&self) -> Result<Context, Error> {
let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
let cache_layers =
- language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.mtl_context);
+ language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.context);
let context = Context::new(
language_model_generator.tokens.clone(),
cache_layers,
diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rs
index 03029e4ae..4f172256f 100644
--- a/crates/uzu/src/session/types/error.rs
+++ b/crates/uzu/src/session/types/error.rs
@@ -1,11 +1,11 @@
-use crate::backends::metal::error::ClassifierError;
+use crate::classifier::ClassifierError;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("Model folder not found")]
ModelFolderNotFound,
- #[error("Unable to create Metal context")]
- UnableToCreateMetalContext,
+ #[error("Unable to create Backend context")]
+ UnableToCreateBackendContext,
#[error("Unable to load model configuration")]
UnableToLoadConfig,
#[error("Unable to load model weights")]
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index 789927a12..bd125e868 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -157,7 +157,7 @@ pub enum ArrayTransform {
// ============================================================================
enum ModelContext {
- LanguageModelGenerator(LanguageModelGeneratorContext),
+ LanguageModelGenerator(LanguageModelGeneratorContext<Metal>),
Classifier(Classifier),
}
@@ -202,7 +202,7 @@ impl TraceValidator {
AsyncBatchSize::default(),
false,
);
- let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
+ let mut llm_context = LanguageModelGeneratorContext::<Metal>::new(model_path, &decoding_config)?;
let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());
Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);
ModelContext::LanguageModelGenerator(llm_context)
@@ -240,12 +240,12 @@ impl TraceValidator {
// ========================================================================
fn run_llm_validation(
- ctx: &LanguageModelGeneratorContext,
+ ctx: &LanguageModelGeneratorContext<Metal>,
traces_path: &Path,
) -> Result<TracerValidationResults, Error> {
let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
let traces_loader =
- ParameterLoader::new(&traces_file, ctx.mtl_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
+ ParameterLoader::new(&traces_file, ctx.context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let traces_view = traces_loader.tree();
let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");
@@ -253,7 +253,7 @@ impl TraceValidator {
let token_seeds: Vec<u64> = vec![0; token_ids.len()];
let mut state = ForwardPassState::new_llm(
- ctx.mtl_context.clone(),
+ ctx.context.clone(),
&ctx.decoder_config,
&ctx.model_shape,
&ctx.scratch_buffers,
@@ -274,7 +274,7 @@ impl TraceValidator {
None,
);
- let command_buffer = ctx.mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
+ let command_buffer = ctx.context.command_queue.command_buffer().expect("Failed to create command buffer");
ctx.executables.encode(&mut state, &EncodingParameters::new(false, false, false), &command_buffer);
command_buffer.commit();
@@ -389,9 +389,9 @@ impl TraceValidator {
traces_path: &Path,
) -> Result<TracerValidationResults, Error> {
let traces_file = File::open(traces_path).map_err(|_| Error::UnableToLoadWeights)?;
- let mtl_context = classifier.context.mtl_context.clone();
+ let context = classifier.context.context.clone();
let traces_loader =
- ParameterLoader::new(&traces_file, mtl_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
+ ParameterLoader::new(&traces_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let traces_view = traces_loader.tree();
let has_token_ids = traces_view.leaf("activation_trace.token_ids").is_ok();
@@ -803,7 +803,7 @@ impl TraceValidator {
fn ensure_llm_context_capacity(
decoding_config: &DecodingConfig,
desired_suffix_length: usize,
- context: &mut LanguageModelGeneratorContext,
+ context: &mut LanguageModelGeneratorContext<Metal>,
) {
let resolved_prefix_length = decoding_config.context_length.resolve(&context.model_config);
let current_suffix_length = std::cmp::max(
@@ -817,7 +817,7 @@ impl TraceValidator {
let decoder_config = &context.decoder_config;
context.scratch_buffers = ScratchBuffers::new(
- context.mtl_context.as_ref(),
+ context.context.as_ref(),
decoder_config,
&context.model_shape,
resolved_prefix_length,
@@ -825,7 +825,7 @@ impl TraceValidator {
);
context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(
- context.mtl_context.as_ref(),
+ context.context.as_ref(),
&context.model_shape,
resolved_prefix_length,
desired_suffix_length,
@@ -834,12 +834,12 @@ impl TraceValidator {
let intermediate_dtype: DataType = decoder_config.output_norm_config.scale_precision.into();
context.kv_cache_update = Box::new(
- KVCacheUpdate::new(context.mtl_context.as_ref(), intermediate_dtype, resolved_prefix_length)
+ KVCacheUpdate::new(context.context.as_ref(), intermediate_dtype, resolved_prefix_length)
.expect("Failed to create KV cache update kernel"),
);
context.gpu_sampler = Sampling::new(
- context.mtl_context.as_ref(),
+ context.context.as_ref(),
intermediate_dtype,
desired_suffix_length,
decoder_config.vocab_size,