A high-performance inference engine for AI models
add short conv
eugened773d60ca813
diff --git a/crates/uzu/examples/quick_start_chat.rs b/crates/uzu/examples/quick_start_chat.rsindex d9cd0a4ea..d75619c9a 100644--- a/crates/uzu/examples/quick_start_chat.rs+++ b/crates/uzu/examples/quick_start_chat.rs@@ -1,4 +1,4 @@-use std::{env, path::PathBuf};+use std::path::PathBuf;use uzu::session::{ChatSession,diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rsindex 0ded4ad27..c05f7b2f3 100644--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs@@ -69,6 +69,10 @@ impl Decoder {MixerConfig::Mamba(mamba_config) => {mamba_config.in_projection_config.activation_precision().into()},+ MixerConfig::ShortConv(short_conv_config) => short_conv_config+ .in_projection_config+ .activation_precision()+ .into(),};let global_rope = if decoder_config.global_rope_config.is_some() {@@ -123,6 +127,9 @@ impl Decoder {DecoderLayerType::StateSpace {..} => None,+ DecoderLayerType::ShortConv {+ ..+ } => None,};let layer_loader = decoder_weight_loaderdiff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rsindex 51ea2cad3..09b9fa29d 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs@@ -8,7 +8,7 @@ use objc2::rc::autoreleasepool;use super::{super::{Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm,- RMSNorm, TensorAddSwap, TensorCopy, transformer_layer,+ RMSNorm, ShortConvMixer, TensorAddSwap, TensorCopy, transformer_layer,},MixerExecutables,};@@ -64,6 +64,10 @@ impl LayerExecutables {MixerConfig::Mamba(mamba) => {mamba.in_projection_config.activation_precision().into()},+ MixerConfig::ShortConv(short_conv) => short_conv+ .in_projection_config+ .activation_precision()+ .into(),};let kernel_data_type: KernelDataType =intermediate_data_type.into();@@ -192,6 +196,21 @@ impl LayerExecutables {mixer,}},+ MixerConfig::ShortConv(short_conv_config) => {+ let mixer: Box<dyn EncodableBlock> =+ Box::new(ShortConvMixer::new(+ mtl_context,+ layer_type.clone(),+ short_conv_config.clone(),+ compilation_config.clone(),+ layer_index,+ model_dim,+ decoder_layer_loader,+ ));+ MixerExecutables::ShortConv {+ mixer,+ }+ },};let post_attention_norm: Option<Box<dyn EncodableBlock>> =@@ -351,6 +370,18 @@ impl EncodableBlock for LayerExecutables {);}},+ MixerExecutables::ShortConv {+ mixer,+ } => {+ mixer.encode(state, command_buffer, parameters);+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.copy_array(+ ArrayId::Main,+ layer_traces.borrow().attention.clone(),+ );+ }+ },}if let Some(post_attention_norm) = &self.post_attention_norm {diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rsindex 5a8d5e936..f4f0ef4c5 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs@@ -4,7 +4,7 @@ use std::rc::Rc;use super::super::EncodableBlock;-/// Mixer component - either attention or state space.+/// Mixer component - either attention, state space, or short conv.pub(crate) enum MixerExecutables {Attention {qkv_projection: Box<dyn EncodableBlock>,@@ -16,4 +16,7 @@ pub(crate) enum MixerExecutables {StateSpace {mixer: Box<dyn EncodableBlock>,},+ ShortConv {+ mixer: Box<dyn EncodableBlock>,+ },}diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 6a370dd85..b277e2f69 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -20,6 +20,7 @@ mod layer;mod linear;mod mamba_mixer;mod mlp;+mod short_conv_mixer;mod moe_block;mod normalization;mod pooling;@@ -42,6 +43,7 @@ pub use layer::LayerExecutables;pub use linear::QuantizedLinear;pub(crate) use mamba_mixer::MambaMixer;pub use mlp::MlpBlock;+pub(crate) use short_conv_mixer::ShortConvMixer;pub use moe_block::{MoeBlock, SharedMoeWeights};pub use normalization::{LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsnew file mode 100644index 000000000..5df658b94--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs@@ -0,0 +1,266 @@+use std::rc::Rc;++use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::{EncodableBlock, EncodingParameters, transformer_layer};+use crate::{+ DataType,+ backends::metal::{+ KernelDataType, MTLContext, MetalArray,+ compilation_parameters::CompilationConfig,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::short_conv::{+ ShortConvDecodeArguments, ShortConvKernel, ShortConvPrefillArguments,+ },+ },+ config::{DecoderLayerType, ShortConvConfig},+ parameters::ParameterTree,+};++pub(crate) struct ShortConvMixer {+ layer_index: usize,+ config: ShortConvConfig,+ model_dim: usize,+ in_projection: Box<dyn EncodableBlock>,+ out_projection: Box<dyn EncodableBlock>,+ short_conv_kernel: ShortConvKernel,+ conv_weight: MetalArray,+ conv_bias: Option<MetalArray>,+}++fn resolve_subtree<'tree>(+ tree: &'tree ParameterTree<Rc<MTLContext>>,+ candidates: &[&str],+) -> ParameterTree<'tree, Rc<MTLContext>> {+ for candidate in candidates {+ if let Ok(subtree) = tree.subtree(candidate) {+ return subtree;+ }+ }+ panic!(+ "Could not find any of {:?} in parameter tree",+ candidates+ );+}++impl ShortConvMixer {+ #[allow(clippy::too_many_arguments)]+ pub(crate) fn new(+ mtl_context: &MTLContext,+ layer_type: DecoderLayerType,+ short_conv_config: ShortConvConfig,+ compilation_config: Rc<CompilationConfig>,+ layer_index: usize,+ model_dim: usize,+ decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,+ ) -> Self {+ if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {+ panic!(+ "Layer {} marked as non-ShortConv but ShortConv config provided",+ layer_index+ );+ }++ let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);+ let conv_tree = resolve_subtree(&mixer_tree, &["conv"]);++ let data_type: DataType =+ short_conv_config.in_projection_config.activation_precision().into();+ let kernel_data_type: KernelDataType = data_type.into();++ let in_projection = transformer_layer::linear_block(+ &short_conv_config.in_projection_config,+ false,+ model_dim,+ [model_dim * 3],+ mtl_context,+ &resolve_subtree(&mixer_tree, &["in_projection", "in_proj"]),+ ArrayId::Main,+ ArrayId::SsmInProj,+ &compilation_config.descriptor_mlp,+ );++ let out_projection = transformer_layer::linear_block(+ &short_conv_config.out_projection_config,+ false,+ model_dim,+ [model_dim],+ mtl_context,+ &resolve_subtree(&mixer_tree, &["out_projection", "out_proj"]),+ ArrayId::AttentionOutput,+ ArrayId::Main,+ &compilation_config.descriptor_mlp,+ );++ let conv_weight = conv_tree.leaf("weights").unwrap().clone();+ let conv_bias = if short_conv_config.conv_config.has_biases {+ Some(conv_tree.leaf("biases").unwrap().clone())+ } else {+ None+ };++ let short_conv_kernel = ShortConvKernel::new(mtl_context, kernel_data_type)+ .expect("Failed to create short conv kernel");++ Self {+ layer_index,+ config: short_conv_config,+ model_dim,+ in_projection,+ out_projection,+ short_conv_kernel,+ conv_weight,+ conv_bias,+ }+ }++ fn encode_pipeline(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let suffix_length = state.aux_buffers_suffix_length();+ let active_suffix_length = state.active_suffix_length();+ if suffix_length == 0 || active_suffix_length == 0 {+ return;+ }++ self.in_projection.encode(state, command_buffer, parameters);++ if suffix_length == 1 {+ self.run_decode_conv(state, command_buffer, active_suffix_length);+ } else {+ self.run_prefill_conv(state, command_buffer, active_suffix_length);+ }++ self.out_projection.encode(state, command_buffer, parameters);++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn run_prefill_conv(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ suffix_length: usize,+ ) {+ let arrays = state.arrays(&[+ ArrayId::SsmInProj,+ ArrayId::ShortConvState(self.layer_index),+ ArrayId::AttentionOutput,+ ]);+ let mut in_proj = arrays[0].borrow_mut();+ let mut conv_state = arrays[1].borrow_mut();+ let mut out = arrays[2].borrow_mut();++ let in_proj_buf = unsafe { in_proj.mtl_buffer().to_owned() };+ let state_buf = unsafe { conv_state.mtl_buffer().to_owned() };+ let out_buf = unsafe { out.mtl_buffer().to_owned() };++ let mut conv_weight = self.conv_weight.clone();+ let weight_buf = unsafe { conv_weight.mtl_buffer().to_owned() };+ let bias_buf = self.conv_bias.as_ref().map(|b| {+ let mut b = b.clone();+ unsafe { b.mtl_buffer().to_owned() }+ });++ let kernel_size = self.config.kernel_size;+ let state_stride = kernel_size.saturating_sub(1);++ let mtl_command_buffer = command_buffer.root_command_buffer().to_owned();+ let compute = mtl_command_buffer.new_compute_command_encoder();++ self.short_conv_kernel+ .encode_prefill(+ &compute,+ ShortConvPrefillArguments {+ in_proj: &in_proj_buf,+ w: &weight_buf,+ b: bias_buf.as_ref(),+ state_in: &state_buf,+ out: &out_buf,+ state_out: &state_buf,+ suffix_len: suffix_length,+ kernel_size: kernel_size as i32,+ in_proj_stride: self.model_dim * 3,+ state_stride,+ model_dim: self.model_dim,+ },+ )+ .expect("Failed to encode short conv prefill kernel");++ compute.end_encoding();+ }++ fn run_decode_conv(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ suffix_length: usize,+ ) {+ let arrays = state.arrays(&[+ ArrayId::SsmInProj,+ ArrayId::ShortConvState(self.layer_index),+ ArrayId::AttentionOutput,+ ]);+ let mut in_proj = arrays[0].borrow_mut();+ let mut conv_state = arrays[1].borrow_mut();+ let mut out = arrays[2].borrow_mut();++ let in_proj_buf = unsafe { in_proj.mtl_buffer().to_owned() };+ let state_buf = unsafe { conv_state.mtl_buffer().to_owned() };+ let out_buf = unsafe { out.mtl_buffer().to_owned() };++ let mut conv_weight = self.conv_weight.clone();+ let weight_buf = unsafe { conv_weight.mtl_buffer().to_owned() };+ let bias_buf = self.conv_bias.as_ref().map(|b| {+ let mut b = b.clone();+ unsafe { b.mtl_buffer().to_owned() }+ });++ let kernel_size = self.config.kernel_size;+ let state_stride = kernel_size.saturating_sub(1);++ let mtl_command_buffer = command_buffer.root_command_buffer().to_owned();+ let compute = mtl_command_buffer.new_compute_command_encoder();++ self.short_conv_kernel+ .encode_decode(+ &compute,+ ShortConvDecodeArguments {+ in_proj: &in_proj_buf,+ w: &weight_buf,+ b: bias_buf.as_ref(),+ state: &state_buf,+ out: &out_buf,+ next_state: &state_buf,+ suffix_len: suffix_length,+ kernel_size: kernel_size as i32,+ in_proj_stride: self.model_dim * 3,+ state_stride,+ model_dim: self.model_dim,+ },+ )+ .expect("Failed to encode short conv decode kernel");++ compute.end_encoding();+ }+}++impl EncodableBlock for ShortConvMixer {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ self.encode_pipeline(state, command_buffer, parameters);+ }+}+diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 0f00b2ece..b530c06bc 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -151,11 +151,9 @@ pub fn mlp_block(| crate::config::LinearConfig::MLXQuantized(ref quant_config) =dense.linear_config{- // Quantized MLP path: up (2H) -> act+mul -> downlet dtype: DataType =dense.linear_config.activation_precision().into();- // Up fused: Main -> MlpFusedUplet up = QuantizedLinear::new(context,quant_config,@@ -167,7 +165,6 @@ pub fn mlp_block().expect("Failed to build MLP up quantized block");- // Activation+mul: MlpFusedUp -> MlpHiddenlet gate_op = MlpGateActMulEncodable::new(context,dtype,@@ -176,7 +173,6 @@ pub fn mlp_block().expect("Failed to build MLP gate activation kernel");- // Down: MlpHidden -> Mainlet down = QuantizedLinear::new(context,quant_config,@@ -193,7 +189,6 @@ pub fn mlp_block(}}- // STEP 2: Enable MOE block creation (encode will be minimal)if let crate::config::MLPConfig::MixtureOfExperts(moe) = config {let moe_block =MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)@@ -201,7 +196,6 @@ pub fn mlp_block(return Box::new(moe_block);}- // Non-quantized Dense path is handled via MPSGraph belowautoreleasepool(|_| {let graph = Graph::new();@@ -270,14 +264,14 @@ pub fn embed_block(let input_placeholder = placeholder(&graph, &input_shape, input_data_type);let input_shaped_type = shaped_type(&input_shape, input_data_type);- match config.embedding_config {+ match &config.embedding_config {EmbeddingConfig::Tied {precision,..} => {let weights_shape =[config.vocab_size as isize, config.model_dim as isize];- let weights_data_type: DataType = precision.into();+ let weights_data_type: DataType = (*precision).into();let weights_shaped_type =shaped_type(&weights_shape, weights_data_type);let weights_placeholder =@@ -322,7 +316,7 @@ pub fn embed_block(} => {let weights_shape =[config.vocab_size as isize, config.model_dim as isize];- let weights_data_type: DataType = precision.into();+ let weights_data_type: DataType = (*precision).into();let weights_shaped_type =shaped_type(&weights_shape, weights_data_type);let weights_placeholder =@@ -367,7 +361,7 @@ pub fn embed_block(} => {let weights_shape =[config.vocab_size as isize, config.model_dim as isize];- let weights_data_type: DataType = activation_precision.into();+ let weights_data_type: DataType = (*activation_precision).into();let weights_shaped_type =shaped_type(&weights_shape, weights_data_type);let weights_placeholder =@@ -410,7 +404,6 @@ pub fn embed_block(embedding_quantization_mode: _,..} => {- // Packed U8 weights with shape [vocab_size, model_dim / 2]let weights_shape =[config.vocab_size as isize, (config.model_dim / 2) as isize];let weights_data_type = DataType::U8;@@ -477,11 +470,9 @@ pub fn embed_block(embedding_quantization_mode,..} => {- // Use Metal kernel for MLX quantized embeddingslet data_type: DataType =config.output_norm_config.scale_precision.into();- // Get the embeddings subtreelet embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");@@ -491,8 +482,8 @@ pub fn embed_block(data_type,config.vocab_size,config.model_dim,- group_size,- embedding_quantization_mode,+ *group_size,+ *embedding_quantization_mode,&embeddings_tree,).expect("Failed to create quantized embedding lookup kernel");@@ -703,11 +694,9 @@ pub fn readout_block(embedding_quantization_mode,..} => {- // Use Metal kernel for MLX quantized readoutlet data_type: DataType =config.output_norm_config.scale_precision.into();- // Get the embeddings subtreelet embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");diff --git a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rsindex b0a19e413..a7b4cff7b 100644--- a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs+++ b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs@@ -6,6 +6,7 @@ use super::{super::{MTLContext, MetalArray},kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},model_shape::ModelShape,+ short_conv_layer::ShortConvLayer,ssm_layer::SSMLayer,};use crate::{@@ -17,6 +18,7 @@ use crate::{pub enum CacheLayer {Transformer(KVCacheLayer),StateSpace(SSMLayer),+ ShortConv(ShortConvLayer),}impl CacheLayer {@@ -47,6 +49,20 @@ impl CacheLayer {_ => None,}}++ pub fn as_short_conv(&self) -> Option<&ShortConvLayer> {+ match self {+ CacheLayer::ShortConv(layer) => Some(layer),+ _ => None,+ }+ }++ pub fn as_short_conv_mut(&mut self) -> Option<&mut ShortConvLayer> {+ match self {+ CacheLayer::ShortConv(layer) => Some(layer),+ _ => None,+ }+ }}pub struct CacheLayers {@@ -139,6 +155,21 @@ impl CacheLayers {),})},+ DecoderLayerType::ShortConv {+ kernel_size,+ } => {+ let conv_shape = [+ model_shape.model_dim(),+ kernel_size.saturating_sub(1),+ ];+ let dtype = model_shape.activation_data_type();++ CacheLayer::ShortConv(ShortConvLayer {+ conv_state: RefCell::new(+ context.array(&conv_shape, dtype),+ ),+ })+ },}).collect();@@ -170,6 +201,7 @@ impl CacheLayers {},},CacheLayer::StateSpace(layer) => layer.zero(),+ CacheLayer::ShortConv(layer) => layer.zero(),}}}@@ -313,6 +345,19 @@ impl CacheLayers {ssm_state: RefCell::new(new_ssm),})},+ CacheLayer::ShortConv(layer) => {+ let conv_shape = layer.conv_state.borrow().shape().to_vec();+ let conv_dtype = layer.conv_state.borrow().data_type();+ let mut new_conv = context.array(&conv_shape, conv_dtype);+ {+ let conv_src = layer.conv_state.borrow();+ new_conv.copy_from_array(&conv_src);+ }++ CacheLayer::ShortConv(ShortConvLayer {+ conv_state: RefCell::new(new_conv),+ })+ },}).collect();diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rsindex 689986567..4be2ae19b 100644--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs@@ -5,6 +5,7 @@ pub mod kv_cache_layer;mod model_shape;mod mpsgraph_block;mod scratch_buffers;+pub mod short_conv_layer;pub mod ssm_layer;pub mod state;#[cfg(feature = "tracing")]@@ -17,6 +18,7 @@ pub use kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState};pub use model_shape::ModelShape;pub use mpsgraph_block::MPSGraphBlock;pub use scratch_buffers::ScratchBuffers;+pub use short_conv_layer::ShortConvLayer;pub use ssm_layer::SSMLayer;pub use state::{ArrayCell, ArrayId, ClassifierModeState, CommonAuxBuffers,diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rsindex 4d5ddf4f5..c6ede5ae2 100644--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs@@ -124,6 +124,10 @@ impl ModelShape {self.num_groups}+ pub fn model_dim(&self) -> usize {+ self.model_dim+ }+pub fn layer_type(&self,layer_index: usize,@@ -485,6 +489,9 @@ impl ModelShape {let inner_dim = num_heads * head_dim;Some(conv_dim + inner_dim + num_heads)},+ DecoderLayerType::ShortConv { .. } => {+ Some(self.model_dim * 3)+ },_ => None,}).max()diff --git a/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs b/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rsnew file mode 100644index 000000000..0d4d5ac84--- /dev/null+++ b/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs@@ -0,0 +1,16 @@+use bytemuck::fill_zeroes;++use super::kv_cache_layer::ArrayCell;+use crate::device::array::Array;++#[derive(Debug)]+pub struct ShortConvLayer {+ pub conv_state: ArrayCell,+}++impl ShortConvLayer {+ pub fn zero(&self) {+ let mut conv = self.conv_state.borrow_mut();+ fill_zeroes(conv.buffer_mut());+ }+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs b/crates/uzu/src/backends/metal/forward_pass/state/array_id.rsindex 63f270eaf..5558fa56a 100644--- a/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs@@ -39,6 +39,7 @@ pub enum ArrayId {SsmC(usize),SsmDt(usize),SsmZ(usize),+ ShortConvState(usize),RotatedQueries,RotatedKeys,diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rsindex f7d6c5b47..64e6d1c08 100644--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs@@ -682,6 +682,14 @@ impl ForwardPassState {.as_ref().and_then(|aux| aux.ssm_z.clone()).expect("SSM z not initialized"),+ ArrayId::ShortConvState(layer_index) => {+ let cache = self.llm_state().cache_layers.borrow();+ cache.data[layer_index]+ .as_short_conv()+ .expect("Expected ShortConv layer")+ .conv_state+ .clone()+ },// MoE arrays (LLM only)ArrayId::MoeTopkIds => selfdiff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 753dc00c6..294c76591 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -11,6 +11,7 @@ pub mod quant_matmul;pub mod rms_norm;pub mod rope;pub mod sampling;+pub mod short_conv;pub mod sigmoid;pub mod ssm;mod tensor_add_bias;@@ -43,6 +44,10 @@ pub use rms_norm::{};pub use rope::{RopeError, RopeKernel, RopeKernelArguments};pub use sampling::{ArgmaxStrategy, SamplingError, SamplingKernel};+pub use short_conv::{+ ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError,+ ShortConvPrefillArguments,+};pub use sigmoid::SigmoidKernel;pub use ssm::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rsnew file mode 100644index 000000000..5b2cc91f1--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs@@ -0,0 +1,192 @@+use std::mem::size_of;++use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState, MTLSize,+};++use crate::backends::metal::{KernelDataType, MTLContext, MTLError};++#[derive(Debug, thiserror::Error)]+pub enum ShortConvKernelError {+ #[error("Metal error: {0}")]+ MetalError(#[from] MTLError),+}++fn fn_suffix(dt: KernelDataType) -> &'static str {+ dt.function_name_suffix()+}++pub struct ShortConvKernel {+ prefill_pipeline: MTLComputePipelineState,+ decode_pipeline: MTLComputePipelineState,+}++pub struct ShortConvPrefillArguments<'a> {+ pub in_proj: &'a MTLBuffer,+ pub w: &'a MTLBuffer,+ pub b: Option<&'a MTLBuffer>,+ pub state_in: &'a MTLBuffer,+ pub out: &'a MTLBuffer,+ pub state_out: &'a MTLBuffer,+ pub suffix_len: usize,+ pub kernel_size: i32,+ pub in_proj_stride: usize,+ pub state_stride: usize,+ pub model_dim: usize,+}++pub struct ShortConvDecodeArguments<'a> {+ pub in_proj: &'a MTLBuffer,+ pub w: &'a MTLBuffer,+ pub b: Option<&'a MTLBuffer>,+ pub state: &'a MTLBuffer,+ pub out: &'a MTLBuffer,+ pub next_state: &'a MTLBuffer,+ pub suffix_len: usize,+ pub kernel_size: i32,+ pub in_proj_stride: usize,+ pub state_stride: usize,+ pub model_dim: usize,+}++impl ShortConvKernel {+ pub fn new(+ context: &MTLContext,+ data_type: KernelDataType,+ ) -> Result<Self, ShortConvKernelError> {+ let prefill_name =+ format!("short_conv_prefill_kernel_{}", fn_suffix(data_type));+ let decode_name =+ format!("short_conv_decode_kernel_{}", fn_suffix(data_type));++ let prefill_pipeline = context+ .compute_pipeline_state(&prefill_name, None)+ .map_err(ShortConvKernelError::MetalError)?;+ let decode_pipeline = context+ .compute_pipeline_state(&decode_name, None)+ .map_err(ShortConvKernelError::MetalError)?;++ Ok(Self {+ prefill_pipeline,+ decode_pipeline,+ })+ }++ pub fn encode_prefill(+ &self,+ compute_encoder: &ComputeCommandEncoderRef,+ args: ShortConvPrefillArguments,+ ) -> Result<(), ShortConvKernelError> {+ if args.model_dim == 0 || args.suffix_len == 0 {+ return Ok(());+ }++ compute_encoder.set_compute_pipeline_state(&self.prefill_pipeline);+ compute_encoder.set_buffer(0, Some(args.in_proj), 0);+ compute_encoder.set_buffer(1, Some(args.w), 0);+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);+ compute_encoder.set_buffer(3, Some(args.state_in), 0);+ compute_encoder.set_buffer(4, Some(args.out), 0);+ compute_encoder.set_buffer(5, Some(args.state_out), 0);+ compute_encoder.set_bytes(+ 6,+ size_of::<usize>() as u64,+ &args.suffix_len as *const usize as *const _,+ );+ compute_encoder.set_bytes(+ 7,+ size_of::<i32>() as u64,+ &args.kernel_size as *const i32 as *const _,+ );+ compute_encoder.set_bytes(+ 8,+ size_of::<usize>() as u64,+ &args.in_proj_stride as *const usize as *const _,+ );+ compute_encoder.set_bytes(+ 9,+ size_of::<usize>() as u64,+ &args.state_stride as *const usize as *const _,+ );+ compute_encoder.set_bytes(+ 10,+ size_of::<u32>() as u64,+ &(args.model_dim as u32) as *const u32 as *const _,+ );++ let threads_per_threadgroup = MTLSize {+ width: 32,+ height: 1,+ depth: 1,+ };+ let threadgroups = MTLSize {+ width: args.suffix_len as u64,+ height: args.model_dim as u64,+ depth: 1,+ };++ compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);++ Ok(())+ }++ pub fn encode_decode(+ &self,+ compute_encoder: &ComputeCommandEncoderRef,+ args: ShortConvDecodeArguments,+ ) -> Result<(), ShortConvKernelError> {+ if args.model_dim == 0 || args.suffix_len == 0 {+ return Ok(());+ }++ compute_encoder.set_compute_pipeline_state(&self.decode_pipeline);+ compute_encoder.set_buffer(0, Some(args.in_proj), 0);+ compute_encoder.set_buffer(1, Some(args.w), 0);+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);+ compute_encoder.set_buffer(3, Some(args.state), 0);+ compute_encoder.set_buffer(4, Some(args.out), 0);+ compute_encoder.set_buffer(5, Some(args.next_state), 0);+ compute_encoder.set_bytes(+ 6,+ size_of::<usize>() as u64,+ &args.suffix_len as *const usize as *const _,+ );+ compute_encoder.set_bytes(+ 7,+ size_of::<i32>() as u64,+ &args.kernel_size as *const i32 as *const _,+ );+ compute_encoder.set_bytes(+ 8,+ size_of::<usize>() as u64,+ &args.in_proj_stride as *const usize as *const _,+ );+ compute_encoder.set_bytes(+ 9,+ size_of::<usize>() as u64,+ &args.state_stride as *const usize as *const _,+ );+ compute_encoder.set_bytes(+ 10,+ size_of::<u32>() as u64,+ &(args.model_dim as u32) as *const u32 as *const _,+ );++ let threads_per_threadgroup = MTLSize {+ width: 32,+ height: 1,+ depth: 1,+ };+ let threadgroups = MTLSize {+ width: args.suffix_len as u64,+ height: args.model_dim as u64,+ depth: 1,+ };++ compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);++ Ok(())+ }+}+diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metalnew file mode 100644index 000000000..c95655e79--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -0,0 +1,190 @@+#include <metal_stdlib>+#include "../definitions.metal"++using namespace metal;++template <typename T>+kernel void short_conv_prefill_kernel(+ device const T* in_proj [[buffer(0)]],+ device const T* w [[buffer(1)]],+ device const T* b [[buffer(2)]],+ device const T* state_in [[buffer(3)]],+ device T* out [[buffer(4)]],+ device T* state_out [[buffer(5)]],+ constant const size_t& suffix_len [[buffer(6)]],+ constant const int& kernel_size [[buffer(7)]],+ constant const size_t& in_proj_stride [[buffer(8)]],+ constant const size_t& state_stride [[buffer(9)]],+ constant const uint& model_dim [[buffer(10)]],+ uint2 grid_idx [[thread_position_in_grid]]+) {+ const uint token_idx = grid_idx.x;+ const uint channel_idx = grid_idx.y;++ if (channel_idx >= model_dim || token_idx >= suffix_len) {+ return;+ }++ const int tap_count = max(kernel_size - 1, 0);+ const size_t state_offset = size_t(channel_idx) * state_stride;+ const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);+ const bool has_bias = b != nullptr;++ size_t current_in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);+ float pre_conv_gate = float(in_proj[current_in_proj_idx]);+ float post_conv_gate = float(in_proj[current_in_proj_idx + model_dim]);+ float x_current = float(in_proj[current_in_proj_idx + 2 * model_dim]);++ float gated_input = x_current * pre_conv_gate;++ float acc = has_bias ? float(b[channel_idx]) : 0.0f;++ for (int tap = 0; tap < kernel_size; ++tap) {+ int src_token = int(token_idx) - (kernel_size - 1 - tap);+ float sample = 0.0f;++ if (src_token < 0) {+ int state_idx = tap_count + src_token;+ if (state_idx >= 0 && state_idx < tap_count) {+ sample = float(state_in[state_offset + size_t(state_idx)]);+ }+ } else if (src_token == int(token_idx)) {+ sample = gated_input;+ } else {+ size_t src_in_proj_idx = size_t(src_token) * in_proj_stride + size_t(channel_idx);+ float src_pre_gate = float(in_proj[src_in_proj_idx]);+ float src_x = float(in_proj[src_in_proj_idx + 2 * model_dim]);+ sample = src_x * src_pre_gate;+ }++ acc += float(w_row[tap]) * sample;+ }++ float gated_output = acc * post_conv_gate;++ size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);+ out[out_idx] = static_cast<T>(gated_output);++ if (tap_count > 0 && token_idx == 0) {+ for (int tap = 0; tap < tap_count; ++tap) {+ int src_token = int(suffix_len) - tap_count + tap;+ float sample = 0.0f;++ if (src_token < 0) {+ int old_state_idx = tap_count + src_token;+ if (old_state_idx >= 0 && old_state_idx < tap_count) {+ sample = float(state_in[state_offset + size_t(old_state_idx)]);+ }+ } else {+ size_t src_in_proj_idx = size_t(src_token) * in_proj_stride + size_t(channel_idx);+ float src_pre_gate = float(in_proj[src_in_proj_idx]);+ float src_x = float(in_proj[src_in_proj_idx + 2 * model_dim]);+ sample = src_x * src_pre_gate;+ }++ state_out[state_offset + size_t(tap)] = static_cast<T>(sample);+ }+ }+}++template <typename T>+kernel void short_conv_decode_kernel(+ device const T* in_proj [[buffer(0)]],+ device const T* w [[buffer(1)]],+ device const T* b [[buffer(2)]],+ device const T* state [[buffer(3)]],+ device T* out [[buffer(4)]],+ device T* next_state [[buffer(5)]],+ constant const size_t& suffix_len [[buffer(6)]],+ constant const int& kernel_size [[buffer(7)]],+ constant const size_t& in_proj_stride [[buffer(8)]],+ constant const size_t& state_stride [[buffer(9)]],+ constant const uint& model_dim [[buffer(10)]],+ uint2 grid_idx [[thread_position_in_grid]]+) {+ const uint token_idx = grid_idx.x;+ const uint channel_idx = grid_idx.y;++ if (channel_idx >= model_dim || token_idx >= suffix_len) {+ return;+ }++ const int tap_count = max(kernel_size - 1, 0);+ const size_t state_offset = size_t(channel_idx) * state_stride;+ const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);+ const bool has_bias = b != nullptr;++ size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);+ float pre_conv_gate = float(in_proj[in_proj_idx]);+ float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);+ float x_val = float(in_proj[in_proj_idx + 2 * model_dim]);++ float gated_input = x_val * pre_conv_gate;++ float acc = has_bias ? float(b[channel_idx]) : 0.0f;++ for (int tap = 0; tap < tap_count; ++tap) {+ float sample = float(state[state_offset + size_t(tap)]);+ acc += float(w_row[tap]) * sample;+ }++ acc += float(w_row[tap_count]) * gated_input;++ float gated_output = acc * post_conv_gate;++ size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);+ out[out_idx] = static_cast<T>(gated_output);++ if (tap_count > 0) {+ for (int tap = 0; tap < tap_count - 1; ++tap) {+ next_state[state_offset + size_t(tap)] = state[state_offset + size_t(tap + 1)];+ }+ next_state[state_offset + size_t(tap_count - 1)] = static_cast<T>(gated_input);+ }+}++#define instantiate_short_conv_prefill_kernel(type_name, type) \+ template [[host_name("short_conv_prefill_kernel_" #type_name)]] \+ kernel void short_conv_prefill_kernel<type>( \+ device const type* x [[buffer(0)]], \+ device const type* w [[buffer(1)]], \+ device const type* b [[buffer(2)]], \+ device const type* state_in [[buffer(3)]], \+ device type* out [[buffer(4)]], \+ device type* state_out [[buffer(5)]], \+ constant const size_t& suffix_len [[buffer(6)]], \+ constant const int& kernel_size [[buffer(7)]], \+ constant const size_t& row_stride [[buffer(8)]], \+ constant const size_t& state_stride [[buffer(9)]], \+ constant const uint& num_channels [[buffer(10)]], \+ uint2 grid_idx [[thread_position_in_grid]] \+ );++#define instantiate_short_conv_decode_kernel(type_name, type) \+ template [[host_name("short_conv_decode_kernel_" #type_name)]] \+ kernel void short_conv_decode_kernel<type>( \+ device const type* x [[buffer(0)]], \+ device const type* w [[buffer(1)]], \+ device const type* b [[buffer(2)]], \+ device const type* state [[buffer(3)]], \+ device type* out [[buffer(4)]], \+ device type* next_state [[buffer(5)]], \+ constant const size_t& suffix_len [[buffer(6)]], \+ constant const int& kernel_size [[buffer(7)]], \+ constant const size_t& row_stride [[buffer(8)]], \+ constant const size_t& state_stride [[buffer(9)]], \+ constant const uint& num_channels [[buffer(10)]], \+ uint2 grid_idx [[thread_position_in_grid]] \+ );++instantiate_short_conv_prefill_kernel(float, float);+instantiate_short_conv_prefill_kernel(bfloat, bfloat);+instantiate_short_conv_prefill_kernel(half, half);++instantiate_short_conv_decode_kernel(float, float);+instantiate_short_conv_decode_kernel(bfloat, bfloat);+instantiate_short_conv_decode_kernel(half, half);++#undef instantiate_short_conv_prefill_kernel+#undef instantiate_short_conv_decode_kernel+diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rsindex 26259a00e..7b23c74f5 100644--- a/crates/uzu/src/config/decoder.rs+++ b/crates/uzu/src/config/decoder.rs@@ -23,6 +23,10 @@ pub enum DecoderLayerType {num_groups: usize,head_dim: usize,},+ #[serde(rename = "short_conv")]+ ShortConv {+ kernel_size: usize,+ },}#[derive(Debug, Serialize, PartialEq, Clone)]@@ -223,6 +227,9 @@ fn layer_type_from_config(layer: &DecoderLayerConfig) -> DecoderLayerType {num_groups: config.num_groups,head_dim: config.head_dim,},+ MixerConfig::ShortConv(config) => DecoderLayerType::ShortConv {+ kernel_size: config.kernel_size,+ },}}impl DecoderConfig {diff --git a/crates/uzu/src/config/decoder_layer.rs b/crates/uzu/src/config/decoder_layer.rsindex 1889420cd..41328a208 100644--- a/crates/uzu/src/config/decoder_layer.rs+++ b/crates/uzu/src/config/decoder_layer.rs@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};use super::{attention::AttentionConfig, mamba::Mamba2Config, mlp::MLPConfig,- normalization::NormalizationConfig,+ normalization::NormalizationConfig, short_conv::ShortConvConfig,};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]@@ -12,6 +12,8 @@ pub enum MixerConfig {Attention(AttentionConfig),#[serde(rename = "Mamba2Config")]Mamba(Mamba2Config),+ #[serde(rename = "ShortConvConfig")]+ ShortConv(ShortConvConfig),}impl MixerConfig {@@ -29,10 +31,18 @@ impl MixerConfig {}}+ pub fn as_short_conv(&self) -> Option<&ShortConvConfig> {+ match self {+ MixerConfig::ShortConv(config) => Some(config),+ _ => None,+ }+ }+pub fn num_heads(&self) -> Option<usize> {match self {MixerConfig::Attention(config) => config.num_heads,MixerConfig::Mamba(config) => Some(config.num_heads),+ MixerConfig::ShortConv(_) => None,}}@@ -40,6 +50,7 @@ impl MixerConfig {match self {MixerConfig::Attention(config) => config.num_groups,MixerConfig::Mamba(config) => Some(config.num_groups),+ MixerConfig::ShortConv(_) => None,}}@@ -47,6 +58,7 @@ impl MixerConfig {match self {MixerConfig::Attention(config) => config.head_dim,MixerConfig::Mamba(config) => Some(config.head_dim),+ MixerConfig::ShortConv(_) => None,}}@@ -54,6 +66,7 @@ impl MixerConfig {match self {MixerConfig::Attention(config) => config.sliding_window_size,MixerConfig::Mamba(_) => None,+ MixerConfig::ShortConv(_) => None,}}@@ -61,6 +74,7 @@ impl MixerConfig {match self {MixerConfig::Attention(config) => config.scale,MixerConfig::Mamba(_) => None,+ MixerConfig::ShortConv(_) => None,}}}diff --git a/crates/uzu/src/config/language_model.rs b/crates/uzu/src/config/language_model.rsindex 373028222..7cdd741d6 100644--- a/crates/uzu/src/config/language_model.rs+++ b/crates/uzu/src/config/language_model.rs@@ -6,6 +6,13 @@ use crate::{config::ConfigError,};+struct AttentionDims {+ num_heads: usize,+ num_groups: usize,+ head_dim: usize,+ attention_scale: Option<f32>,+}+/// Inner model config matching the new lalamo export format./// Contains embedding_config at the top level, with transformer_config nested.#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]@@ -37,18 +44,7 @@ impl InnerModelConfig {post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),};- let mixer = &first_layer.mixer_config;- let num_heads =- tf.num_heads.or(mixer.num_heads()).ok_or_else(|| {- ConfigError::MissingField("num_heads".to_string())- })?;- let num_groups =- tf.num_groups.or(mixer.num_groups()).unwrap_or(num_heads);- let head_dim = tf- .head_dim- .or(mixer.head_dim())- .ok_or_else(|| ConfigError::MissingField("head_dim".to_string()))?;- let attention_scale = tf.attention_scale.or(mixer.attention_scale());+ let attention_dims = Self::derive_attention_dims(tf)?;let num_layers = tf.num_layers.unwrap_or(tf.layer_configs.len());let sliding_window_sizes: Box<[Option<usize>]> = tf@@ -94,10 +90,10 @@ impl InnerModelConfig {vocab_size: self.vocab_size,model_dim: tf.model_dim,hidden_dim: tf.hidden_dim,- num_heads,- num_groups,- head_dim,- attention_scale,+ num_heads: attention_dims.num_heads,+ num_groups: attention_dims.num_groups,+ head_dim: attention_dims.head_dim,+ attention_scale: attention_dims.attention_scale,num_layers,sliding_window_sizes: Some(sliding_window_sizes),layer_types: Some(layer_types),@@ -105,6 +101,58 @@ impl InnerModelConfig {})}+ fn derive_attention_dims(+ tf: &TransformerConfig+ ) -> Result<AttentionDims, ConfigError> {+ if let (Some(num_heads), Some(head_dim)) = (tf.num_heads, tf.head_dim) {+ return Ok(AttentionDims {+ num_heads,+ num_groups: tf.num_groups.unwrap_or(num_heads),+ head_dim,+ attention_scale: tf.attention_scale,+ });+ }++ if let Some(attn) = tf+ .layer_configs+ .iter()+ .find_map(|layer| layer.mixer_config.as_attention())+ {+ let num_heads = attn.num_heads.ok_or_else(|| {+ ConfigError::MissingField("num_heads".to_string())+ })?;+ let head_dim = attn.head_dim.ok_or_else(|| {+ ConfigError::MissingField("head_dim".to_string())+ })?;+ return Ok(AttentionDims {+ num_heads,+ num_groups: attn.num_groups.unwrap_or(num_heads),+ head_dim,+ attention_scale: attn.scale,+ });+ }++ if let Some(mamba) = tf+ .layer_configs+ .iter()+ .find_map(|layer| layer.mixer_config.as_mamba())+ {+ return Ok(AttentionDims {+ num_heads: mamba.num_heads,+ num_groups: mamba.num_groups,+ head_dim: mamba.head_dim,+ attention_scale: None,+ });+ }++ Ok(AttentionDims {+ num_heads: 1,+ num_groups: 1,+ head_dim: tf.model_dim,+ attention_scale: None,+ })+ }+fn layer_type_from_mixer(mixer: &MixerConfig) -> DecoderLayerType {match mixer {MixerConfig::Attention(_) => DecoderLayerType::Transformer,@@ -116,6 +164,9 @@ impl InnerModelConfig {num_groups: config.num_groups,head_dim: config.head_dim,},+ MixerConfig::ShortConv(config) => DecoderLayerType::ShortConv {+ kernel_size: config.kernel_size,+ },}}}diff --git a/crates/uzu/src/config/mod.rs b/crates/uzu/src/config/mod.rsindex 7abe6c828..c44e73c80 100644--- a/crates/uzu/src/config/mod.rs+++ b/crates/uzu/src/config/mod.rs@@ -17,6 +17,7 @@ mod model_config;mod model_type;mod normalization;mod rope;+mod short_conv;mod transformer;mod transformer_layer;@@ -39,5 +40,6 @@ pub use model_config::*;pub use model_type::*;pub use normalization::*;pub use rope::*;+pub use short_conv::*;pub use transformer::*;pub use transformer_layer::*;diff --git a/crates/uzu/src/config/short_conv.rs b/crates/uzu/src/config/short_conv.rsnew file mode 100644index 000000000..b44f4faa2--- /dev/null+++ b/crates/uzu/src/config/short_conv.rs@@ -0,0 +1,17 @@+use serde::{Deserialize, Serialize};++use super::{common::ConfigDataType, linear::LinearConfig};++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct SeparableCausalConvConfig {+ pub precision: ConfigDataType,+ pub has_biases: bool,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct ShortConvConfig {+ pub in_projection_config: LinearConfig,+ pub conv_config: SeparableCausalConvConfig,+ pub out_projection_config: LinearConfig,+ pub kernel_size: usize,+}diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex c90ba499f..23b19607f 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -1,4 +1,18 @@-use std::{fs::File, io::BufReader, path::PathBuf, time::Instant};+use std::{+ fs::File,+ io::BufReader,+ path::PathBuf,+ sync::{+ Arc,+ atomic::{AtomicU64, Ordering},+ mpsc,+ },+ time::Instant,+};++fn nanos_to_secs(nanos: u64) -> f64 {+ nanos as f64 / 1_000_000_000.0+}use objc2::rc::autoreleasepool;use tokenizers::Tokenizer;@@ -26,6 +40,18 @@ use crate::{},};+struct RunContext {+ eos_tokens: Vec<u64>,+ context_length: usize,+ prefix_len_before: usize,+ input_tokens_len: usize,+ tokens_limit: usize,+ prefill_result: PrefillResult,+ prefill_duration: f64,+ prefill_suffix_length: usize,+ run_start: Instant,+}+pub struct ChatSession {pub model_path: PathBuf,pub model_metadata: ModelMetadata,@@ -246,7 +272,7 @@ impl ChatSession {.map(|&id| id as u64).collect();- let language_model_config_for_context = self+ let language_model_config = self.model_metadata.model_config.as_language_model()@@ -254,68 +280,25 @@ impl ChatSession {let llm =self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- let context_length = llm- .decoding_config- .context_length- .resolve(language_model_config_for_context);+ let context_length =+ llm.decoding_config.context_length.resolve(language_model_config);if tokens.len() >= context_length {return Err(Error::ContextLengthExceeded);}- let prefix_len_before = llm.tokens.len().saturating_sub(1);- let eos_tokens: Vec<u64> = language_model_config_for_context+ let prefix_offset = llm.tokens.len();+ let prefix_len_before = prefix_offset.saturating_sub(1);++ let eos_tokens: Vec<u64> = language_model_config.generation_config.stop_token_ids.iter().map(|&x| x as u64).collect();- let finish_reason = |llm: &LanguageModelGenerator,- tokens_new: Vec<u64>|- -> Option<FinishReason> {- let start_idx = prefix_len_before + tokens.len();- let total_tokens = llm.tokens.len();- let total_new_tokens = llm.tokens[start_idx..].len();- let has_eos_token =- tokens_new.iter().any(|token| eos_tokens.contains(token));- let context_limit_reached = total_tokens >= context_length;-- if has_eos_token {- Some(FinishReason::Stop)- } else if total_new_tokens >= config.tokens_limit as usize {- Some(FinishReason::Length)- } else if context_limit_reached {- Some(FinishReason::ContextLimitReached)- } else {- None- }- };-- let build_generated_text = |llm: &LanguageModelGenerator,- tokenizer: &Tokenizer|- -> Result<String, Error> {- let start_idx = prefix_len_before + tokens.len();- let generated_tokens: Vec<u32> = llm.tokens[start_idx..]- .to_vec()- .iter()- .map(|value| *value as u32)- .collect();- let generated_text = tokenizer- .decode(&generated_tokens, true)- .map_err(|_| Error::UnableToDecodeText)?;- Ok(generated_text)- };-- let language_model_config_for_sampling = self- .model_metadata- .model_config- .as_language_model()- .ok_or(Error::UnableToLoadConfig)?;-let sampling_method =- config.sampling_policy.resolve(language_model_config_for_sampling);+ config.sampling_policy.resolve(language_model_config);- // Create grammar from config if providedlet mut compiled_grammar: Option<CompiledGrammar> =if let Some(ref grammar_config) = config.grammar_config {Some(CompiledGrammar::from_config(@@ -328,8 +311,6 @@ impl ChatSession {};let prefill_start = Instant::now();- let prefix_offset = llm.tokens.len();-let sample_suffix = config.tokens_limit > 0;let prefill_result = llm.prefill(tokens.clone(),@@ -342,41 +323,38 @@ impl ChatSession {let prefill_duration = prefill_start.elapsed().as_secs_f64();llm.clear_cache();- let prefill_finish_reason = finish_reason(llm, prefill_tokens.clone());- let prefill_generated_text =- build_generated_text(llm, &self.tokenizer)?;- let prefill_parsed_text =- self.output_parser.parse(prefill_generated_text);-- let language_model_config_for_prefill = self- .model_metadata- .model_config- .as_language_model()- .ok_or(Error::UnableToLoadConfig)?;-let prefill_suffix_length = llm.decoding_config.prefill_step_size- .resolve(language_model_config_for_prefill);- let prefill_output = Output {- text: prefill_parsed_text,- stats: Self::build_stats(- &self.model_metadata,- prefill_result.clone(),- prefill_duration,- prefill_suffix_length,- Vec::new(),- Vec::new(),- llm.decoding_config.generate_suffix_length(),- run_start.elapsed().as_secs_f64(),- tokens.len(),- llm.tokens[prefix_len_before + tokens.len()..].len(),- ),- finish_reason: prefill_finish_reason.clone(),+ .resolve(language_model_config);++ let run_context = RunContext {+ eos_tokens,+ context_length,+ prefix_len_before,+ input_tokens_len: tokens.len(),+ tokens_limit: config.tokens_limit as usize,+ prefill_result: prefill_result.clone(),+ prefill_duration,+ prefill_suffix_length,+ run_start,};- let prefill_should_continue = if let Some(progress) = &progress {- progress(prefill_output.clone())+ let prefill_finish_reason =+ Self::check_finish_reason(&run_context, llm, &prefill_tokens);+ let prefill_output = Self::build_output(+ &self.model_metadata,+ &self.tokenizer,+ &self.output_parser,+ &run_context,+ llm,+ &[],+ &[],+ prefill_finish_reason.clone(),+ )?;++ let prefill_should_continue = if let Some(ref p) = progress {+ p(prefill_output.clone())} else {true};@@ -390,60 +368,225 @@ impl ChatSession {}}+ let can_use_async = llm.decoding_config.generate_suffix_length() == 1+ && !llm.has_attention_layers()+ && compiled_grammar.is_none();++ let generate_output = if can_use_async {+ let batch_size =+ llm.decoding_config.async_batch_size.resolve(&self.model_path);+ Self::run_async_batch(+ &self.model_metadata,+ &self.tokenizer,+ &self.output_parser,+ &run_context,+ llm,+ sampling_method,+ &progress,+ batch_size,+ )?+ } else {+ Self::run_sync_generate(+ &self.model_metadata,+ &self.tokenizer,+ &self.output_parser,+ &run_context,+ llm,+ compiled_grammar.as_mut(),+ sampling_method,+ &progress,+ )?+ };++ llm.clear_cache();+ Ok(generate_output+ .clone_with_duration(run_start.elapsed().as_secs_f64()))+ }++ fn run_sync_generate<F>(+ model_metadata: &ModelMetadata,+ tokenizer: &Tokenizer,+ output_parser: &OutputParser,+ run_context: &RunContext,+ llm: &mut LanguageModelGenerator,+ compiled_grammar: Option<&mut CompiledGrammar>,+ sampling_method: super::parameter::SamplingMethod,+ progress: &Option<F>,+ ) -> Result<Output, Error>+ where+ F: Fn(Output) -> bool,+ {let mut generate_results: Vec<GenerateResult> = Vec::new();let mut generate_durations: Vec<f64> = Vec::new();- let generate_output = loop {+ let mut compiled_grammar_mut = compiled_grammar;++ loop {let generate_start = Instant::now();- let generate_result =- llm.generate(compiled_grammar.as_mut(), sampling_method)?;+ let generate_result = llm.generate(+ compiled_grammar_mut.as_deref_mut(),+ sampling_method,+ )?;let generate_tokens = generate_result.tokens.clone();let generate_duration = generate_start.elapsed().as_secs_f64();generate_results.push(generate_result);generate_durations.push(generate_duration);- let generate_finish_reason = finish_reason(llm, generate_tokens);- let generate_generated_text =- build_generated_text(llm, &self.tokenizer)?;- let generate_parsed_text =- self.output_parser.parse(generate_generated_text);-- let generate_output = Output {- text: generate_parsed_text,- stats: Self::build_stats(- &self.model_metadata,- prefill_result.clone(),- prefill_duration,- prefill_suffix_length,- generate_results.clone(),- generate_durations.clone(),- llm.decoding_config.generate_suffix_length(),- run_start.elapsed().as_secs_f64(),- tokens.len(),- llm.tokens[prefix_len_before + tokens.len()..].len(),- ),- finish_reason: generate_finish_reason.clone(),- };-- let generate_should_continue = if let Some(progress) = &progress {- progress(generate_output.clone())+ let generate_finish_reason =+ Self::check_finish_reason(run_context, llm, &generate_tokens);+ let generate_output = Self::build_output(+ model_metadata,+ tokenizer,+ output_parser,+ run_context,+ llm,+ &generate_results,+ &generate_durations,+ generate_finish_reason.clone(),+ )?;++ let generate_should_continue = if let Some(progress_fn) = progress {+ progress_fn(generate_output.clone())} else {true};if !generate_should_continue || generate_finish_reason.is_some() {if generate_should_continue {- break generate_output;+ return Ok(generate_output);} else {- break generate_output.clone_with_finish_reason(Some(+ return Ok(generate_output.clone_with_finish_reason(Some(FinishReason::Cancelled,+ )));+ }+ }+ }+ }++ fn run_async_batch<F>(+ model_metadata: &ModelMetadata,+ tokenizer: &Tokenizer,+ output_parser: &OutputParser,+ run_context: &RunContext,+ llm: &mut LanguageModelGenerator,+ sampling_method: super::parameter::SamplingMethod,+ progress: &Option<F>,+ batch_size: usize,+ ) -> Result<Output, Error>+ where+ F: Fn(Output) -> bool,+ {+ let remaining_by_limit = run_context+ .tokens_limit+ .saturating_sub(run_context.prefill_result.tokens.len());+ let remaining_by_context =+ run_context.context_length.saturating_sub(llm.tokens.len());+ let tokens_to_generate = remaining_by_limit.min(remaining_by_context);++ llm.prepare_async(tokens_to_generate);++ let (sender, receiver) = mpsc::channel::<(usize, u64, u64)>();+ let start_time = Instant::now();+ let last_nanos = Arc::new(AtomicU64::new(0));++ let mut results: Vec<GenerateResult> =+ Vec::with_capacity(tokens_to_generate);+ let mut durations: Vec<f64> = Vec::with_capacity(tokens_to_generate);+ let mut finish_reason = FinishReason::Length;+ let mut next_to_submit = 0;+ let mut in_flight = 0;++ while in_flight > 0 || next_to_submit < tokens_to_generate {+ let batch_end =+ std::cmp::min(next_to_submit + batch_size, tokens_to_generate);+ for idx in next_to_submit..batch_end {+ let batch_sender = sender.clone();+ let last_callback_nanos = last_nanos.clone();+ let batch_start_time = start_time;+ llm.async_generate(idx, sampling_method, move |token| {+ let now_nanos =+ batch_start_time.elapsed().as_nanos() as u64;+ let prev_nanos =+ last_callback_nanos.swap(now_nanos, Ordering::SeqCst);+ let _ = batch_sender.send((+ idx,+ token,+ now_nanos.saturating_sub(prev_nanos),));+ })?;+ in_flight += 1;+ }+ let batch_submitted = batch_end - next_to_submit;+ next_to_submit = batch_end;++ for _ in 0..batch_submitted {+ let (_, token, duration_nanos) =+ receiver.recv().map_err(|_| Error::SamplingFailed)?;+ in_flight -= 1;++ let duration = nanos_to_secs(duration_nanos);+ llm.tokens.push(token);+ results.push(GenerateResult {+ tokens: vec![token],+ forwardpass_duration: duration,+ });+ durations.push(duration);++ let should_stop = if run_context.eos_tokens.contains(&token) {+ finish_reason = FinishReason::Stop;+ true+ } else if llm.tokens.len() >= run_context.context_length {+ finish_reason = FinishReason::ContextLimitReached;+ true+ } else if let Some(progress_fn) = progress {+ let output = Self::build_output(+ model_metadata,+ tokenizer,+ output_parser,+ run_context,+ llm,+ &results,+ &durations,+ None,+ )?;+ if !progress_fn(output) {+ finish_reason = FinishReason::Cancelled;+ true+ } else {+ false+ }+ } else {+ false+ };++ if should_stop {+ while in_flight > 0 {+ let _ = receiver.recv();+ in_flight -= 1;+ }+ return Self::build_output(+ model_metadata,+ tokenizer,+ output_parser,+ run_context,+ llm,+ &results,+ &durations,+ Some(finish_reason),+ );}}- };+ }- llm.clear_cache();- Ok(generate_output- .clone_with_duration(run_start.elapsed().as_secs_f64()))+ Self::build_output(+ model_metadata,+ tokenizer,+ output_parser,+ run_context,+ llm,+ &results,+ &durations,+ Some(finish_reason),+ )}fn reconfigure_llm(@@ -532,6 +675,77 @@ impl ChatSession {}impl ChatSession {+ fn check_finish_reason(+ run_context: &RunContext,+ llm: &LanguageModelGenerator,+ new_tokens: &[u64],+ ) -> Option<FinishReason> {+ let start_idx =+ run_context.prefix_len_before + run_context.input_tokens_len;+ let total_new_tokens = llm.tokens.len().saturating_sub(start_idx);+ let has_eos =+ new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));+ let context_limit = llm.tokens.len() >= run_context.context_length;++ if has_eos {+ Some(FinishReason::Stop)+ } else if total_new_tokens >= run_context.tokens_limit {+ Some(FinishReason::Length)+ } else if context_limit {+ Some(FinishReason::ContextLimitReached)+ } else {+ None+ }+ }++ fn decode_generated_tokens(+ tokenizer: &Tokenizer,+ llm: &LanguageModelGenerator,+ run_context: &RunContext,+ ) -> Result<String, Error> {+ let start_idx =+ run_context.prefix_len_before + run_context.input_tokens_len;+ let generated_tokens: Vec<u32> =+ llm.tokens[start_idx..].iter().map(|&v| v as u32).collect();+ tokenizer+ .decode(&generated_tokens, true)+ .map_err(|_| Error::UnableToDecodeText)+ }++ fn build_output(+ model_metadata: &ModelMetadata,+ tokenizer: &Tokenizer,+ output_parser: &OutputParser,+ run_context: &RunContext,+ llm: &LanguageModelGenerator,+ generate_results: &[GenerateResult],+ generate_durations: &[f64],+ finish_reason: Option<FinishReason>,+ ) -> Result<Output, Error> {+ let text = Self::decode_generated_tokens(tokenizer, llm, run_context)?;+ let parsed = output_parser.parse(text);+ let start_idx =+ run_context.prefix_len_before + run_context.input_tokens_len;+ let output_tokens = llm.tokens.len().saturating_sub(start_idx);++ Ok(Output {+ text: parsed,+ stats: Self::build_stats(+ model_metadata,+ run_context.prefill_result.clone(),+ run_context.prefill_duration,+ run_context.prefill_suffix_length,+ generate_results.to_vec(),+ generate_durations.to_vec(),+ llm.decoding_config.generate_suffix_length(),+ run_context.run_start.elapsed().as_secs_f64(),+ run_context.input_tokens_len,+ output_tokens,+ ),+ finish_reason,+ })+ }+fn build_stats(model_metadata: &ModelMetadata,prefill_result: PrefillResult,