Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add short conv

eugened773d60ca813
diff --git a/crates/uzu/examples/quick_start_chat.rs b/crates/uzu/examples/quick_start_chat.rs
index d9cd0a4ea..d75619c9a 100644
--- a/crates/uzu/examples/quick_start_chat.rs
+++ b/crates/uzu/examples/quick_start_chat.rs
@@ -1,4 +1,4 @@
-use std::{env, path::PathBuf};
+use std::path::PathBuf;
use uzu::session::{
ChatSession,
diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
index 0ded4ad27..c05f7b2f3 100644
--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
@@ -69,6 +69,10 @@ impl Decoder {
MixerConfig::Mamba(mamba_config) => {
mamba_config.in_projection_config.activation_precision().into()
},
+ MixerConfig::ShortConv(short_conv_config) => short_conv_config
+ .in_projection_config
+ .activation_precision()
+ .into(),
};
let global_rope = if decoder_config.global_rope_config.is_some() {
@@ -123,6 +127,9 @@ impl Decoder {
DecoderLayerType::StateSpace {
..
} => None,
+ DecoderLayerType::ShortConv {
+ ..
+ } => None,
};
let layer_loader = decoder_weight_loader
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
index 51ea2cad3..09b9fa29d 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
@@ -8,7 +8,7 @@ use objc2::rc::autoreleasepool;
use super::{
super::{
Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm,
- RMSNorm, TensorAddSwap, TensorCopy, transformer_layer,
+ RMSNorm, ShortConvMixer, TensorAddSwap, TensorCopy, transformer_layer,
},
MixerExecutables,
};
@@ -64,6 +64,10 @@ impl LayerExecutables {
MixerConfig::Mamba(mamba) => {
mamba.in_projection_config.activation_precision().into()
},
+ MixerConfig::ShortConv(short_conv) => short_conv
+ .in_projection_config
+ .activation_precision()
+ .into(),
};
let kernel_data_type: KernelDataType =
intermediate_data_type.into();
@@ -192,6 +196,21 @@ impl LayerExecutables {
mixer,
}
},
+ MixerConfig::ShortConv(short_conv_config) => {
+ let mixer: Box<dyn EncodableBlock> =
+ Box::new(ShortConvMixer::new(
+ mtl_context,
+ layer_type.clone(),
+ short_conv_config.clone(),
+ compilation_config.clone(),
+ layer_index,
+ model_dim,
+ decoder_layer_loader,
+ ));
+ MixerExecutables::ShortConv {
+ mixer,
+ }
+ },
};
let post_attention_norm: Option<Box<dyn EncodableBlock>> =
@@ -351,6 +370,18 @@ impl EncodableBlock for LayerExecutables {
);
}
},
+ MixerExecutables::ShortConv {
+ mixer,
+ } => {
+ mixer.encode(state, command_buffer, parameters);
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.copy_array(
+ ArrayId::Main,
+ layer_traces.borrow().attention.clone(),
+ );
+ }
+ },
}
if let Some(post_attention_norm) = &self.post_attention_norm {
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs
index 5a8d5e936..f4f0ef4c5 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs
@@ -4,7 +4,7 @@ use std::rc::Rc;
use super::super::EncodableBlock;
-/// Mixer component - either attention or state space.
+/// Mixer component - either attention, state space, or short conv.
pub(crate) enum MixerExecutables {
Attention {
qkv_projection: Box<dyn EncodableBlock>,
@@ -16,4 +16,7 @@ pub(crate) enum MixerExecutables {
StateSpace {
mixer: Box<dyn EncodableBlock>,
},
+ ShortConv {
+ mixer: Box<dyn EncodableBlock>,
+ },
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 6a370dd85..b277e2f69 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -20,6 +20,7 @@ mod layer;
mod linear;
mod mamba_mixer;
mod mlp;
+mod short_conv_mixer;
mod moe_block;
mod normalization;
mod pooling;
@@ -42,6 +43,7 @@ pub use layer::LayerExecutables;
pub use linear::QuantizedLinear;
pub(crate) use mamba_mixer::MambaMixer;
pub use mlp::MlpBlock;
+pub(crate) use short_conv_mixer::ShortConvMixer;
pub use moe_block::{MoeBlock, SharedMoeWeights};
pub use normalization::{
LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
new file mode 100644
index 000000000..5df658b94
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -0,0 +1,266 @@
+use std::rc::Rc;
+
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::{EncodableBlock, EncodingParameters, transformer_layer};
+use crate::{
+ DataType,
+ backends::metal::{
+ KernelDataType, MTLContext, MetalArray,
+ compilation_parameters::CompilationConfig,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::short_conv::{
+ ShortConvDecodeArguments, ShortConvKernel, ShortConvPrefillArguments,
+ },
+ },
+ config::{DecoderLayerType, ShortConvConfig},
+ parameters::ParameterTree,
+};
+
+pub(crate) struct ShortConvMixer {
+ layer_index: usize,
+ config: ShortConvConfig,
+ model_dim: usize,
+ in_projection: Box<dyn EncodableBlock>,
+ out_projection: Box<dyn EncodableBlock>,
+ short_conv_kernel: ShortConvKernel,
+ conv_weight: MetalArray,
+ conv_bias: Option<MetalArray>,
+}
+
+fn resolve_subtree<'tree>(
+ tree: &'tree ParameterTree<Rc<MTLContext>>,
+ candidates: &[&str],
+) -> ParameterTree<'tree, Rc<MTLContext>> {
+ for candidate in candidates {
+ if let Ok(subtree) = tree.subtree(candidate) {
+ return subtree;
+ }
+ }
+ panic!(
+ "Could not find any of {:?} in parameter tree",
+ candidates
+ );
+}
+
+impl ShortConvMixer {
+ #[allow(clippy::too_many_arguments)]
+ pub(crate) fn new(
+ mtl_context: &MTLContext,
+ layer_type: DecoderLayerType,
+ short_conv_config: ShortConvConfig,
+ compilation_config: Rc<CompilationConfig>,
+ layer_index: usize,
+ model_dim: usize,
+ decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,
+ ) -> Self {
+ if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {
+ panic!(
+ "Layer {} marked as non-ShortConv but ShortConv config provided",
+ layer_index
+ );
+ }
+
+ let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);
+ let conv_tree = resolve_subtree(&mixer_tree, &["conv"]);
+
+ let data_type: DataType =
+ short_conv_config.in_projection_config.activation_precision().into();
+ let kernel_data_type: KernelDataType = data_type.into();
+
+ let in_projection = transformer_layer::linear_block(
+ &short_conv_config.in_projection_config,
+ false,
+ model_dim,
+ [model_dim * 3],
+ mtl_context,
+ &resolve_subtree(&mixer_tree, &["in_projection", "in_proj"]),
+ ArrayId::Main,
+ ArrayId::SsmInProj,
+ &compilation_config.descriptor_mlp,
+ );
+
+ let out_projection = transformer_layer::linear_block(
+ &short_conv_config.out_projection_config,
+ false,
+ model_dim,
+ [model_dim],
+ mtl_context,
+ &resolve_subtree(&mixer_tree, &["out_projection", "out_proj"]),
+ ArrayId::AttentionOutput,
+ ArrayId::Main,
+ &compilation_config.descriptor_mlp,
+ );
+
+ let conv_weight = conv_tree.leaf("weights").unwrap().clone();
+ let conv_bias = if short_conv_config.conv_config.has_biases {
+ Some(conv_tree.leaf("biases").unwrap().clone())
+ } else {
+ None
+ };
+
+ let short_conv_kernel = ShortConvKernel::new(mtl_context, kernel_data_type)
+ .expect("Failed to create short conv kernel");
+
+ Self {
+ layer_index,
+ config: short_conv_config,
+ model_dim,
+ in_projection,
+ out_projection,
+ short_conv_kernel,
+ conv_weight,
+ conv_bias,
+ }
+ }
+
+ fn encode_pipeline(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let suffix_length = state.aux_buffers_suffix_length();
+ let active_suffix_length = state.active_suffix_length();
+ if suffix_length == 0 || active_suffix_length == 0 {
+ return;
+ }
+
+ self.in_projection.encode(state, command_buffer, parameters);
+
+ if suffix_length == 1 {
+ self.run_decode_conv(state, command_buffer, active_suffix_length);
+ } else {
+ self.run_prefill_conv(state, command_buffer, active_suffix_length);
+ }
+
+ self.out_projection.encode(state, command_buffer, parameters);
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn run_prefill_conv(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ suffix_length: usize,
+ ) {
+ let arrays = state.arrays(&[
+ ArrayId::SsmInProj,
+ ArrayId::ShortConvState(self.layer_index),
+ ArrayId::AttentionOutput,
+ ]);
+ let mut in_proj = arrays[0].borrow_mut();
+ let mut conv_state = arrays[1].borrow_mut();
+ let mut out = arrays[2].borrow_mut();
+
+ let in_proj_buf = unsafe { in_proj.mtl_buffer().to_owned() };
+ let state_buf = unsafe { conv_state.mtl_buffer().to_owned() };
+ let out_buf = unsafe { out.mtl_buffer().to_owned() };
+
+ let mut conv_weight = self.conv_weight.clone();
+ let weight_buf = unsafe { conv_weight.mtl_buffer().to_owned() };
+ let bias_buf = self.conv_bias.as_ref().map(|b| {
+ let mut b = b.clone();
+ unsafe { b.mtl_buffer().to_owned() }
+ });
+
+ let kernel_size = self.config.kernel_size;
+ let state_stride = kernel_size.saturating_sub(1);
+
+ let mtl_command_buffer = command_buffer.root_command_buffer().to_owned();
+ let compute = mtl_command_buffer.new_compute_command_encoder();
+
+ self.short_conv_kernel
+ .encode_prefill(
+ &compute,
+ ShortConvPrefillArguments {
+ in_proj: &in_proj_buf,
+ w: &weight_buf,
+ b: bias_buf.as_ref(),
+ state_in: &state_buf,
+ out: &out_buf,
+ state_out: &state_buf,
+ suffix_len: suffix_length,
+ kernel_size: kernel_size as i32,
+ in_proj_stride: self.model_dim * 3,
+ state_stride,
+ model_dim: self.model_dim,
+ },
+ )
+ .expect("Failed to encode short conv prefill kernel");
+
+ compute.end_encoding();
+ }
+
+ fn run_decode_conv(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ suffix_length: usize,
+ ) {
+ let arrays = state.arrays(&[
+ ArrayId::SsmInProj,
+ ArrayId::ShortConvState(self.layer_index),
+ ArrayId::AttentionOutput,
+ ]);
+ let mut in_proj = arrays[0].borrow_mut();
+ let mut conv_state = arrays[1].borrow_mut();
+ let mut out = arrays[2].borrow_mut();
+
+ let in_proj_buf = unsafe { in_proj.mtl_buffer().to_owned() };
+ let state_buf = unsafe { conv_state.mtl_buffer().to_owned() };
+ let out_buf = unsafe { out.mtl_buffer().to_owned() };
+
+ let mut conv_weight = self.conv_weight.clone();
+ let weight_buf = unsafe { conv_weight.mtl_buffer().to_owned() };
+ let bias_buf = self.conv_bias.as_ref().map(|b| {
+ let mut b = b.clone();
+ unsafe { b.mtl_buffer().to_owned() }
+ });
+
+ let kernel_size = self.config.kernel_size;
+ let state_stride = kernel_size.saturating_sub(1);
+
+ let mtl_command_buffer = command_buffer.root_command_buffer().to_owned();
+ let compute = mtl_command_buffer.new_compute_command_encoder();
+
+ self.short_conv_kernel
+ .encode_decode(
+ &compute,
+ ShortConvDecodeArguments {
+ in_proj: &in_proj_buf,
+ w: &weight_buf,
+ b: bias_buf.as_ref(),
+ state: &state_buf,
+ out: &out_buf,
+ next_state: &state_buf,
+ suffix_len: suffix_length,
+ kernel_size: kernel_size as i32,
+ in_proj_stride: self.model_dim * 3,
+ state_stride,
+ model_dim: self.model_dim,
+ },
+ )
+ .expect("Failed to encode short conv decode kernel");
+
+ compute.end_encoding();
+ }
+}
+
+impl EncodableBlock for ShortConvMixer {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ self.encode_pipeline(state, command_buffer, parameters);
+ }
+}
+
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 0f00b2ece..b530c06bc 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -151,11 +151,9 @@ pub fn mlp_block(
| crate::config::LinearConfig::MLXQuantized(ref quant_config) =
dense.linear_config
{
- // Quantized MLP path: up (2H) -> act+mul -> down
let dtype: DataType =
dense.linear_config.activation_precision().into();
- // Up fused: Main -> MlpFusedUp
let up = QuantizedLinear::new(
context,
quant_config,
@@ -167,7 +165,6 @@ pub fn mlp_block(
)
.expect("Failed to build MLP up quantized block");
- // Activation+mul: MlpFusedUp -> MlpHidden
let gate_op = MlpGateActMulEncodable::new(
context,
dtype,
@@ -176,7 +173,6 @@ pub fn mlp_block(
)
.expect("Failed to build MLP gate activation kernel");
- // Down: MlpHidden -> Main
let down = QuantizedLinear::new(
context,
quant_config,
@@ -193,7 +189,6 @@ pub fn mlp_block(
}
}
- // STEP 2: Enable MOE block creation (encode will be minimal)
if let crate::config::MLPConfig::MixtureOfExperts(moe) = config {
let moe_block =
MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)
@@ -201,7 +196,6 @@ pub fn mlp_block(
return Box::new(moe_block);
}
- // Non-quantized Dense path is handled via MPSGraph below
autoreleasepool(|_| {
let graph = Graph::new();
@@ -270,14 +264,14 @@ pub fn embed_block(
let input_placeholder = placeholder(&graph, &input_shape, input_data_type);
let input_shaped_type = shaped_type(&input_shape, input_data_type);
- match config.embedding_config {
+ match &config.embedding_config {
EmbeddingConfig::Tied {
precision,
..
} => {
let weights_shape =
[config.vocab_size as isize, config.model_dim as isize];
- let weights_data_type: DataType = precision.into();
+ let weights_data_type: DataType = (*precision).into();
let weights_shaped_type =
shaped_type(&weights_shape, weights_data_type);
let weights_placeholder =
@@ -322,7 +316,7 @@ pub fn embed_block(
} => {
let weights_shape =
[config.vocab_size as isize, config.model_dim as isize];
- let weights_data_type: DataType = precision.into();
+ let weights_data_type: DataType = (*precision).into();
let weights_shaped_type =
shaped_type(&weights_shape, weights_data_type);
let weights_placeholder =
@@ -367,7 +361,7 @@ pub fn embed_block(
} => {
let weights_shape =
[config.vocab_size as isize, config.model_dim as isize];
- let weights_data_type: DataType = activation_precision.into();
+ let weights_data_type: DataType = (*activation_precision).into();
let weights_shaped_type =
shaped_type(&weights_shape, weights_data_type);
let weights_placeholder =
@@ -410,7 +404,6 @@ pub fn embed_block(
embedding_quantization_mode: _,
..
} => {
- // Packed U8 weights with shape [vocab_size, model_dim / 2]
let weights_shape =
[config.vocab_size as isize, (config.model_dim / 2) as isize];
let weights_data_type = DataType::U8;
@@ -477,11 +470,9 @@ pub fn embed_block(
embedding_quantization_mode,
..
} => {
- // Use Metal kernel for MLX quantized embeddings
let data_type: DataType =
config.output_norm_config.scale_precision.into();
- // Get the embeddings subtree
let embeddings_tree = parameter_tree
.subtree("embedding")
.expect("Failed to get embedding subtree");
@@ -491,8 +482,8 @@ pub fn embed_block(
data_type,
config.vocab_size,
config.model_dim,
- group_size,
- embedding_quantization_mode,
+ *group_size,
+ *embedding_quantization_mode,
&embeddings_tree,
)
.expect("Failed to create quantized embedding lookup kernel");
@@ -703,11 +694,9 @@ pub fn readout_block(
embedding_quantization_mode,
..
} => {
- // Use Metal kernel for MLX quantized readout
let data_type: DataType =
config.output_norm_config.scale_precision.into();
- // Get the embeddings subtree
let embeddings_tree = parameter_tree
.subtree("embedding")
.expect("Failed to get embedding subtree");
diff --git a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
index b0a19e413..a7b4cff7b 100644
--- a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
@@ -6,6 +6,7 @@ use super::{
super::{MTLContext, MetalArray},
kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},
model_shape::ModelShape,
+ short_conv_layer::ShortConvLayer,
ssm_layer::SSMLayer,
};
use crate::{
@@ -17,6 +18,7 @@ use crate::{
pub enum CacheLayer {
Transformer(KVCacheLayer),
StateSpace(SSMLayer),
+ ShortConv(ShortConvLayer),
}
impl CacheLayer {
@@ -47,6 +49,20 @@ impl CacheLayer {
_ => None,
}
}
+
+ pub fn as_short_conv(&self) -> Option<&ShortConvLayer> {
+ match self {
+ CacheLayer::ShortConv(layer) => Some(layer),
+ _ => None,
+ }
+ }
+
+ pub fn as_short_conv_mut(&mut self) -> Option<&mut ShortConvLayer> {
+ match self {
+ CacheLayer::ShortConv(layer) => Some(layer),
+ _ => None,
+ }
+ }
}
pub struct CacheLayers {
@@ -139,6 +155,21 @@ impl CacheLayers {
),
})
},
+ DecoderLayerType::ShortConv {
+ kernel_size,
+ } => {
+ let conv_shape = [
+ model_shape.model_dim(),
+ kernel_size.saturating_sub(1),
+ ];
+ let dtype = model_shape.activation_data_type();
+
+ CacheLayer::ShortConv(ShortConvLayer {
+ conv_state: RefCell::new(
+ context.array(&conv_shape, dtype),
+ ),
+ })
+ },
})
.collect();
@@ -170,6 +201,7 @@ impl CacheLayers {
},
},
CacheLayer::StateSpace(layer) => layer.zero(),
+ CacheLayer::ShortConv(layer) => layer.zero(),
}
}
}
@@ -313,6 +345,19 @@ impl CacheLayers {
ssm_state: RefCell::new(new_ssm),
})
},
+ CacheLayer::ShortConv(layer) => {
+ let conv_shape = layer.conv_state.borrow().shape().to_vec();
+ let conv_dtype = layer.conv_state.borrow().data_type();
+ let mut new_conv = context.array(&conv_shape, conv_dtype);
+ {
+ let conv_src = layer.conv_state.borrow();
+ new_conv.copy_from_array(&conv_src);
+ }
+
+ CacheLayer::ShortConv(ShortConvLayer {
+ conv_state: RefCell::new(new_conv),
+ })
+ },
})
.collect();
diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rs
index 689986567..4be2ae19b 100644
--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs
@@ -5,6 +5,7 @@ pub mod kv_cache_layer;
mod model_shape;
mod mpsgraph_block;
mod scratch_buffers;
+pub mod short_conv_layer;
pub mod ssm_layer;
pub mod state;
#[cfg(feature = "tracing")]
@@ -17,6 +18,7 @@ pub use kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState};
pub use model_shape::ModelShape;
pub use mpsgraph_block::MPSGraphBlock;
pub use scratch_buffers::ScratchBuffers;
+pub use short_conv_layer::ShortConvLayer;
pub use ssm_layer::SSMLayer;
pub use state::{
ArrayCell, ArrayId, ClassifierModeState, CommonAuxBuffers,
diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
index 4d5ddf4f5..c6ede5ae2 100644
--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
@@ -124,6 +124,10 @@ impl ModelShape {
self.num_groups
}
+ pub fn model_dim(&self) -> usize {
+ self.model_dim
+ }
+
pub fn layer_type(
&self,
layer_index: usize,
@@ -485,6 +489,9 @@ impl ModelShape {
let inner_dim = num_heads * head_dim;
Some(conv_dim + inner_dim + num_heads)
},
+ DecoderLayerType::ShortConv { .. } => {
+ Some(self.model_dim * 3)
+ },
_ => None,
})
.max()
diff --git a/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs b/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs
new file mode 100644
index 000000000..0d4d5ac84
--- /dev/null
+++ b/crates/uzu/src/backends/metal/forward_pass/short_conv_layer.rs
@@ -0,0 +1,16 @@
+use bytemuck::fill_zeroes;
+
+use super::kv_cache_layer::ArrayCell;
+use crate::device::array::Array;
+
+#[derive(Debug)]
+pub struct ShortConvLayer {
+ pub conv_state: ArrayCell,
+}
+
+impl ShortConvLayer {
+ pub fn zero(&self) {
+ let mut conv = self.conv_state.borrow_mut();
+ fill_zeroes(conv.buffer_mut());
+ }
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs b/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs
index 63f270eaf..5558fa56a 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/array_id.rs
@@ -39,6 +39,7 @@ pub enum ArrayId {
SsmC(usize),
SsmDt(usize),
SsmZ(usize),
+ ShortConvState(usize),
RotatedQueries,
RotatedKeys,
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
index f7d6c5b47..64e6d1c08 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
@@ -682,6 +682,14 @@ impl ForwardPassState {
.as_ref()
.and_then(|aux| aux.ssm_z.clone())
.expect("SSM z not initialized"),
+ ArrayId::ShortConvState(layer_index) => {
+ let cache = self.llm_state().cache_layers.borrow();
+ cache.data[layer_index]
+ .as_short_conv()
+ .expect("Expected ShortConv layer")
+ .conv_state
+ .clone()
+ },
// MoE arrays (LLM only)
ArrayId::MoeTopkIds => self
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 753dc00c6..294c76591 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -11,6 +11,7 @@ pub mod quant_matmul;
pub mod rms_norm;
pub mod rope;
pub mod sampling;
+pub mod short_conv;
pub mod sigmoid;
pub mod ssm;
mod tensor_add_bias;
@@ -43,6 +44,10 @@ pub use rms_norm::{
};
pub use rope::{RopeError, RopeKernel, RopeKernelArguments};
pub use sampling::{ArgmaxStrategy, SamplingError, SamplingKernel};
+pub use short_conv::{
+ ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError,
+ ShortConvPrefillArguments,
+};
pub use sigmoid::SigmoidKernel;
pub use ssm::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
new file mode 100644
index 000000000..5b2cc91f1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
@@ -0,0 +1,192 @@
+use std::mem::size_of;
+
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState, MTLSize,
+};
+
+use crate::backends::metal::{KernelDataType, MTLContext, MTLError};
+
+#[derive(Debug, thiserror::Error)]
+pub enum ShortConvKernelError {
+ #[error("Metal error: {0}")]
+ MetalError(#[from] MTLError),
+}
+
+fn fn_suffix(dt: KernelDataType) -> &'static str {
+ dt.function_name_suffix()
+}
+
+pub struct ShortConvKernel {
+ prefill_pipeline: MTLComputePipelineState,
+ decode_pipeline: MTLComputePipelineState,
+}
+
+pub struct ShortConvPrefillArguments<'a> {
+ pub in_proj: &'a MTLBuffer,
+ pub w: &'a MTLBuffer,
+ pub b: Option<&'a MTLBuffer>,
+ pub state_in: &'a MTLBuffer,
+ pub out: &'a MTLBuffer,
+ pub state_out: &'a MTLBuffer,
+ pub suffix_len: usize,
+ pub kernel_size: i32,
+ pub in_proj_stride: usize,
+ pub state_stride: usize,
+ pub model_dim: usize,
+}
+
+pub struct ShortConvDecodeArguments<'a> {
+ pub in_proj: &'a MTLBuffer,
+ pub w: &'a MTLBuffer,
+ pub b: Option<&'a MTLBuffer>,
+ pub state: &'a MTLBuffer,
+ pub out: &'a MTLBuffer,
+ pub next_state: &'a MTLBuffer,
+ pub suffix_len: usize,
+ pub kernel_size: i32,
+ pub in_proj_stride: usize,
+ pub state_stride: usize,
+ pub model_dim: usize,
+}
+
+impl ShortConvKernel {
+ pub fn new(
+ context: &MTLContext,
+ data_type: KernelDataType,
+ ) -> Result<Self, ShortConvKernelError> {
+ let prefill_name =
+ format!("short_conv_prefill_kernel_{}", fn_suffix(data_type));
+ let decode_name =
+ format!("short_conv_decode_kernel_{}", fn_suffix(data_type));
+
+ let prefill_pipeline = context
+ .compute_pipeline_state(&prefill_name, None)
+ .map_err(ShortConvKernelError::MetalError)?;
+ let decode_pipeline = context
+ .compute_pipeline_state(&decode_name, None)
+ .map_err(ShortConvKernelError::MetalError)?;
+
+ Ok(Self {
+ prefill_pipeline,
+ decode_pipeline,
+ })
+ }
+
+ pub fn encode_prefill(
+ &self,
+ compute_encoder: &ComputeCommandEncoderRef,
+ args: ShortConvPrefillArguments,
+ ) -> Result<(), ShortConvKernelError> {
+ if args.model_dim == 0 || args.suffix_len == 0 {
+ return Ok(());
+ }
+
+ compute_encoder.set_compute_pipeline_state(&self.prefill_pipeline);
+ compute_encoder.set_buffer(0, Some(args.in_proj), 0);
+ compute_encoder.set_buffer(1, Some(args.w), 0);
+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);
+ compute_encoder.set_buffer(3, Some(args.state_in), 0);
+ compute_encoder.set_buffer(4, Some(args.out), 0);
+ compute_encoder.set_buffer(5, Some(args.state_out), 0);
+ compute_encoder.set_bytes(
+ 6,
+ size_of::<usize>() as u64,
+ &args.suffix_len as *const usize as *const _,
+ );
+ compute_encoder.set_bytes(
+ 7,
+ size_of::<i32>() as u64,
+ &args.kernel_size as *const i32 as *const _,
+ );
+ compute_encoder.set_bytes(
+ 8,
+ size_of::<usize>() as u64,
+ &args.in_proj_stride as *const usize as *const _,
+ );
+ compute_encoder.set_bytes(
+ 9,
+ size_of::<usize>() as u64,
+ &args.state_stride as *const usize as *const _,
+ );
+ compute_encoder.set_bytes(
+ 10,
+ size_of::<u32>() as u64,
+ &(args.model_dim as u32) as *const u32 as *const _,
+ );
+
+ let threads_per_threadgroup = MTLSize {
+ width: 32,
+ height: 1,
+ depth: 1,
+ };
+ let threadgroups = MTLSize {
+ width: args.suffix_len as u64,
+ height: args.model_dim as u64,
+ depth: 1,
+ };
+
+ compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);
+
+ Ok(())
+ }
+
+ pub fn encode_decode(
+ &self,
+ compute_encoder: &ComputeCommandEncoderRef,
+ args: ShortConvDecodeArguments,
+ ) -> Result<(), ShortConvKernelError> {
+ if args.model_dim == 0 || args.suffix_len == 0 {
+ return Ok(());
+ }
+
+ compute_encoder.set_compute_pipeline_state(&self.decode_pipeline);
+ compute_encoder.set_buffer(0, Some(args.in_proj), 0);
+ compute_encoder.set_buffer(1, Some(args.w), 0);
+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);
+ compute_encoder.set_buffer(3, Some(args.state), 0);
+ compute_encoder.set_buffer(4, Some(args.out), 0);
+ compute_encoder.set_buffer(5, Some(args.next_state), 0);
+ compute_encoder.set_bytes(
+ 6,
+ size_of::<usize>() as u64,
+ &args.suffix_len as *const usize as *const _,
+ );
+ compute_encoder.set_bytes(
+ 7,
+ size_of::<i32>() as u64,
+ &args.kernel_size as *const i32 as *const _,
+ );
+ compute_encoder.set_bytes(
+ 8,
+ size_of::<usize>() as u64,
+ &args.in_proj_stride as *const usize as *const _,
+ );
+ compute_encoder.set_bytes(
+ 9,
+ size_of::<usize>() as u64,
+ &args.state_stride as *const usize as *const _,
+ );
+ compute_encoder.set_bytes(
+ 10,
+ size_of::<u32>() as u64,
+ &(args.model_dim as u32) as *const u32 as *const _,
+ );
+
+ let threads_per_threadgroup = MTLSize {
+ width: 32,
+ height: 1,
+ depth: 1,
+ };
+ let threadgroups = MTLSize {
+ width: args.suffix_len as u64,
+ height: args.model_dim as u64,
+ depth: 1,
+ };
+
+ compute_encoder.dispatch_threads(threadgroups, threads_per_threadgroup);
+
+ Ok(())
+ }
+}
+
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
new file mode 100644
index 000000000..c95655e79
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -0,0 +1,190 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+
+using namespace metal;
+
+template <typename T>
+kernel void short_conv_prefill_kernel(
+ device const T* in_proj [[buffer(0)]],
+ device const T* w [[buffer(1)]],
+ device const T* b [[buffer(2)]],
+ device const T* state_in [[buffer(3)]],
+ device T* out [[buffer(4)]],
+ device T* state_out [[buffer(5)]],
+ constant const size_t& suffix_len [[buffer(6)]],
+ constant const int& kernel_size [[buffer(7)]],
+ constant const size_t& in_proj_stride [[buffer(8)]],
+ constant const size_t& state_stride [[buffer(9)]],
+ constant const uint& model_dim [[buffer(10)]],
+ uint2 grid_idx [[thread_position_in_grid]]
+) {
+ const uint token_idx = grid_idx.x;
+ const uint channel_idx = grid_idx.y;
+
+ if (channel_idx >= model_dim || token_idx >= suffix_len) {
+ return;
+ }
+
+ const int tap_count = max(kernel_size - 1, 0);
+ const size_t state_offset = size_t(channel_idx) * state_stride;
+ const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);
+ const bool has_bias = b != nullptr;
+
+ size_t current_in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);
+ float pre_conv_gate = float(in_proj[current_in_proj_idx]);
+ float post_conv_gate = float(in_proj[current_in_proj_idx + model_dim]);
+ float x_current = float(in_proj[current_in_proj_idx + 2 * model_dim]);
+
+ float gated_input = x_current * pre_conv_gate;
+
+ float acc = has_bias ? float(b[channel_idx]) : 0.0f;
+
+ for (int tap = 0; tap < kernel_size; ++tap) {
+ int src_token = int(token_idx) - (kernel_size - 1 - tap);
+ float sample = 0.0f;
+
+ if (src_token < 0) {
+ int state_idx = tap_count + src_token;
+ if (state_idx >= 0 && state_idx < tap_count) {
+ sample = float(state_in[state_offset + size_t(state_idx)]);
+ }
+ } else if (src_token == int(token_idx)) {
+ sample = gated_input;
+ } else {
+ size_t src_in_proj_idx = size_t(src_token) * in_proj_stride + size_t(channel_idx);
+ float src_pre_gate = float(in_proj[src_in_proj_idx]);
+ float src_x = float(in_proj[src_in_proj_idx + 2 * model_dim]);
+ sample = src_x * src_pre_gate;
+ }
+
+ acc += float(w_row[tap]) * sample;
+ }
+
+ float gated_output = acc * post_conv_gate;
+
+ size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);
+ out[out_idx] = static_cast<T>(gated_output);
+
+ if (tap_count > 0 && token_idx == 0) {
+ for (int tap = 0; tap < tap_count; ++tap) {
+ int src_token = int(suffix_len) - tap_count + tap;
+ float sample = 0.0f;
+
+ if (src_token < 0) {
+ int old_state_idx = tap_count + src_token;
+ if (old_state_idx >= 0 && old_state_idx < tap_count) {
+ sample = float(state_in[state_offset + size_t(old_state_idx)]);
+ }
+ } else {
+ size_t src_in_proj_idx = size_t(src_token) * in_proj_stride + size_t(channel_idx);
+ float src_pre_gate = float(in_proj[src_in_proj_idx]);
+ float src_x = float(in_proj[src_in_proj_idx + 2 * model_dim]);
+ sample = src_x * src_pre_gate;
+ }
+
+ state_out[state_offset + size_t(tap)] = static_cast<T>(sample);
+ }
+ }
+}
+
+template <typename T>
+kernel void short_conv_decode_kernel(
+ device const T* in_proj [[buffer(0)]],
+ device const T* w [[buffer(1)]],
+ device const T* b [[buffer(2)]],
+ device const T* state [[buffer(3)]],
+ device T* out [[buffer(4)]],
+ device T* next_state [[buffer(5)]],
+ constant const size_t& suffix_len [[buffer(6)]],
+ constant const int& kernel_size [[buffer(7)]],
+ constant const size_t& in_proj_stride [[buffer(8)]],
+ constant const size_t& state_stride [[buffer(9)]],
+ constant const uint& model_dim [[buffer(10)]],
+ uint2 grid_idx [[thread_position_in_grid]]
+) {
+ const uint token_idx = grid_idx.x;
+ const uint channel_idx = grid_idx.y;
+
+ if (channel_idx >= model_dim || token_idx >= suffix_len) {
+ return;
+ }
+
+ const int tap_count = max(kernel_size - 1, 0);
+ const size_t state_offset = size_t(channel_idx) * state_stride;
+ const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);
+ const bool has_bias = b != nullptr;
+
+ size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);
+ float pre_conv_gate = float(in_proj[in_proj_idx]);
+ float post_conv_gate = float(in_proj[in_proj_idx + model_dim]);
+ float x_val = float(in_proj[in_proj_idx + 2 * model_dim]);
+
+ float gated_input = x_val * pre_conv_gate;
+
+ float acc = has_bias ? float(b[channel_idx]) : 0.0f;
+
+ for (int tap = 0; tap < tap_count; ++tap) {
+ float sample = float(state[state_offset + size_t(tap)]);
+ acc += float(w_row[tap]) * sample;
+ }
+
+ acc += float(w_row[tap_count]) * gated_input;
+
+ float gated_output = acc * post_conv_gate;
+
+ size_t out_idx = size_t(token_idx) * model_dim + size_t(channel_idx);
+ out[out_idx] = static_cast<T>(gated_output);
+
+ if (tap_count > 0) {
+ for (int tap = 0; tap < tap_count - 1; ++tap) {
+ next_state[state_offset + size_t(tap)] = state[state_offset + size_t(tap + 1)];
+ }
+ next_state[state_offset + size_t(tap_count - 1)] = static_cast<T>(gated_input);
+ }
+}
+
+#define instantiate_short_conv_prefill_kernel(type_name, type) \
+ template [[host_name("short_conv_prefill_kernel_" #type_name)]] \
+ kernel void short_conv_prefill_kernel<type>( \
+ device const type* x [[buffer(0)]], \
+ device const type* w [[buffer(1)]], \
+ device const type* b [[buffer(2)]], \
+ device const type* state_in [[buffer(3)]], \
+ device type* out [[buffer(4)]], \
+ device type* state_out [[buffer(5)]], \
+ constant const size_t& suffix_len [[buffer(6)]], \
+ constant const int& kernel_size [[buffer(7)]], \
+ constant const size_t& row_stride [[buffer(8)]], \
+ constant const size_t& state_stride [[buffer(9)]], \
+ constant const uint& num_channels [[buffer(10)]], \
+ uint2 grid_idx [[thread_position_in_grid]] \
+ );
+
+#define instantiate_short_conv_decode_kernel(type_name, type) \
+ template [[host_name("short_conv_decode_kernel_" #type_name)]] \
+ kernel void short_conv_decode_kernel<type>( \
+ device const type* x [[buffer(0)]], \
+ device const type* w [[buffer(1)]], \
+ device const type* b [[buffer(2)]], \
+ device const type* state [[buffer(3)]], \
+ device type* out [[buffer(4)]], \
+ device type* next_state [[buffer(5)]], \
+ constant const size_t& suffix_len [[buffer(6)]], \
+ constant const int& kernel_size [[buffer(7)]], \
+ constant const size_t& row_stride [[buffer(8)]], \
+ constant const size_t& state_stride [[buffer(9)]], \
+ constant const uint& num_channels [[buffer(10)]], \
+ uint2 grid_idx [[thread_position_in_grid]] \
+ );
+
+instantiate_short_conv_prefill_kernel(float, float);
+instantiate_short_conv_prefill_kernel(bfloat, bfloat);
+instantiate_short_conv_prefill_kernel(half, half);
+
+instantiate_short_conv_decode_kernel(float, float);
+instantiate_short_conv_decode_kernel(bfloat, bfloat);
+instantiate_short_conv_decode_kernel(half, half);
+
+#undef instantiate_short_conv_prefill_kernel
+#undef instantiate_short_conv_decode_kernel
+
diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rs
index 26259a00e..7b23c74f5 100644
--- a/crates/uzu/src/config/decoder.rs
+++ b/crates/uzu/src/config/decoder.rs
@@ -23,6 +23,10 @@ pub enum DecoderLayerType {
num_groups: usize,
head_dim: usize,
},
+ #[serde(rename = "short_conv")]
+ ShortConv {
+ kernel_size: usize,
+ },
}
#[derive(Debug, Serialize, PartialEq, Clone)]
@@ -223,6 +227,9 @@ fn layer_type_from_config(layer: &DecoderLayerConfig) -> DecoderLayerType {
num_groups: config.num_groups,
head_dim: config.head_dim,
},
+ MixerConfig::ShortConv(config) => DecoderLayerType::ShortConv {
+ kernel_size: config.kernel_size,
+ },
}
}
impl DecoderConfig {
diff --git a/crates/uzu/src/config/decoder_layer.rs b/crates/uzu/src/config/decoder_layer.rs
index 1889420cd..41328a208 100644
--- a/crates/uzu/src/config/decoder_layer.rs
+++ b/crates/uzu/src/config/decoder_layer.rs
@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
use super::{
attention::AttentionConfig, mamba::Mamba2Config, mlp::MLPConfig,
- normalization::NormalizationConfig,
+ normalization::NormalizationConfig, short_conv::ShortConvConfig,
};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
@@ -12,6 +12,8 @@ pub enum MixerConfig {
Attention(AttentionConfig),
#[serde(rename = "Mamba2Config")]
Mamba(Mamba2Config),
+ #[serde(rename = "ShortConvConfig")]
+ ShortConv(ShortConvConfig),
}
impl MixerConfig {
@@ -29,10 +31,18 @@ impl MixerConfig {
}
}
+ pub fn as_short_conv(&self) -> Option<&ShortConvConfig> {
+ match self {
+ MixerConfig::ShortConv(config) => Some(config),
+ _ => None,
+ }
+ }
+
pub fn num_heads(&self) -> Option<usize> {
match self {
MixerConfig::Attention(config) => config.num_heads,
MixerConfig::Mamba(config) => Some(config.num_heads),
+ MixerConfig::ShortConv(_) => None,
}
}
@@ -40,6 +50,7 @@ impl MixerConfig {
match self {
MixerConfig::Attention(config) => config.num_groups,
MixerConfig::Mamba(config) => Some(config.num_groups),
+ MixerConfig::ShortConv(_) => None,
}
}
@@ -47,6 +58,7 @@ impl MixerConfig {
match self {
MixerConfig::Attention(config) => config.head_dim,
MixerConfig::Mamba(config) => Some(config.head_dim),
+ MixerConfig::ShortConv(_) => None,
}
}
@@ -54,6 +66,7 @@ impl MixerConfig {
match self {
MixerConfig::Attention(config) => config.sliding_window_size,
MixerConfig::Mamba(_) => None,
+ MixerConfig::ShortConv(_) => None,
}
}
@@ -61,6 +74,7 @@ impl MixerConfig {
match self {
MixerConfig::Attention(config) => config.scale,
MixerConfig::Mamba(_) => None,
+ MixerConfig::ShortConv(_) => None,
}
}
}
diff --git a/crates/uzu/src/config/language_model.rs b/crates/uzu/src/config/language_model.rs
index 373028222..7cdd741d6 100644
--- a/crates/uzu/src/config/language_model.rs
+++ b/crates/uzu/src/config/language_model.rs
@@ -6,6 +6,13 @@ use crate::{
config::ConfigError,
};
+struct AttentionDims {
+ num_heads: usize,
+ num_groups: usize,
+ head_dim: usize,
+ attention_scale: Option<f32>,
+}
+
/// Inner model config matching the new lalamo export format.
/// Contains embedding_config at the top level, with transformer_config nested.
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
@@ -37,18 +44,7 @@ impl InnerModelConfig {
post_mlp_norm_config: first_layer.post_mlp_norm_config.clone(),
};
- let mixer = &first_layer.mixer_config;
- let num_heads =
- tf.num_heads.or(mixer.num_heads()).ok_or_else(|| {
- ConfigError::MissingField("num_heads".to_string())
- })?;
- let num_groups =
- tf.num_groups.or(mixer.num_groups()).unwrap_or(num_heads);
- let head_dim = tf
- .head_dim
- .or(mixer.head_dim())
- .ok_or_else(|| ConfigError::MissingField("head_dim".to_string()))?;
- let attention_scale = tf.attention_scale.or(mixer.attention_scale());
+ let attention_dims = Self::derive_attention_dims(tf)?;
let num_layers = tf.num_layers.unwrap_or(tf.layer_configs.len());
let sliding_window_sizes: Box<[Option<usize>]> = tf
@@ -94,10 +90,10 @@ impl InnerModelConfig {
vocab_size: self.vocab_size,
model_dim: tf.model_dim,
hidden_dim: tf.hidden_dim,
- num_heads,
- num_groups,
- head_dim,
- attention_scale,
+ num_heads: attention_dims.num_heads,
+ num_groups: attention_dims.num_groups,
+ head_dim: attention_dims.head_dim,
+ attention_scale: attention_dims.attention_scale,
num_layers,
sliding_window_sizes: Some(sliding_window_sizes),
layer_types: Some(layer_types),
@@ -105,6 +101,58 @@ impl InnerModelConfig {
})
}
+ fn derive_attention_dims(
+ tf: &TransformerConfig
+ ) -> Result<AttentionDims, ConfigError> {
+ if let (Some(num_heads), Some(head_dim)) = (tf.num_heads, tf.head_dim) {
+ return Ok(AttentionDims {
+ num_heads,
+ num_groups: tf.num_groups.unwrap_or(num_heads),
+ head_dim,
+ attention_scale: tf.attention_scale,
+ });
+ }
+
+ if let Some(attn) = tf
+ .layer_configs
+ .iter()
+ .find_map(|layer| layer.mixer_config.as_attention())
+ {
+ let num_heads = attn.num_heads.ok_or_else(|| {
+ ConfigError::MissingField("num_heads".to_string())
+ })?;
+ let head_dim = attn.head_dim.ok_or_else(|| {
+ ConfigError::MissingField("head_dim".to_string())
+ })?;
+ return Ok(AttentionDims {
+ num_heads,
+ num_groups: attn.num_groups.unwrap_or(num_heads),
+ head_dim,
+ attention_scale: attn.scale,
+ });
+ }
+
+ if let Some(mamba) = tf
+ .layer_configs
+ .iter()
+ .find_map(|layer| layer.mixer_config.as_mamba())
+ {
+ return Ok(AttentionDims {
+ num_heads: mamba.num_heads,
+ num_groups: mamba.num_groups,
+ head_dim: mamba.head_dim,
+ attention_scale: None,
+ });
+ }
+
+ Ok(AttentionDims {
+ num_heads: 1,
+ num_groups: 1,
+ head_dim: tf.model_dim,
+ attention_scale: None,
+ })
+ }
+
fn layer_type_from_mixer(mixer: &MixerConfig) -> DecoderLayerType {
match mixer {
MixerConfig::Attention(_) => DecoderLayerType::Transformer,
@@ -116,6 +164,9 @@ impl InnerModelConfig {
num_groups: config.num_groups,
head_dim: config.head_dim,
},
+ MixerConfig::ShortConv(config) => DecoderLayerType::ShortConv {
+ kernel_size: config.kernel_size,
+ },
}
}
}
diff --git a/crates/uzu/src/config/mod.rs b/crates/uzu/src/config/mod.rs
index 7abe6c828..c44e73c80 100644
--- a/crates/uzu/src/config/mod.rs
+++ b/crates/uzu/src/config/mod.rs
@@ -17,6 +17,7 @@ mod model_config;
mod model_type;
mod normalization;
mod rope;
+mod short_conv;
mod transformer;
mod transformer_layer;
@@ -39,5 +40,6 @@ pub use model_config::*;
pub use model_type::*;
pub use normalization::*;
pub use rope::*;
+pub use short_conv::*;
pub use transformer::*;
pub use transformer_layer::*;
diff --git a/crates/uzu/src/config/short_conv.rs b/crates/uzu/src/config/short_conv.rs
new file mode 100644
index 000000000..b44f4faa2
--- /dev/null
+++ b/crates/uzu/src/config/short_conv.rs
@@ -0,0 +1,17 @@
+use serde::{Deserialize, Serialize};
+
+use super::{common::ConfigDataType, linear::LinearConfig};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct SeparableCausalConvConfig {
+ pub precision: ConfigDataType,
+ pub has_biases: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct ShortConvConfig {
+ pub in_projection_config: LinearConfig,
+ pub conv_config: SeparableCausalConvConfig,
+ pub out_projection_config: LinearConfig,
+ pub kernel_size: usize,
+}
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index c90ba499f..23b19607f 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -1,4 +1,18 @@
-use std::{fs::File, io::BufReader, path::PathBuf, time::Instant};
+use std::{
+ fs::File,
+ io::BufReader,
+ path::PathBuf,
+ sync::{
+ Arc,
+ atomic::{AtomicU64, Ordering},
+ mpsc,
+ },
+ time::Instant,
+};
+
+fn nanos_to_secs(nanos: u64) -> f64 {
+ nanos as f64 / 1_000_000_000.0
+}
use objc2::rc::autoreleasepool;
use tokenizers::Tokenizer;
@@ -26,6 +40,18 @@ use crate::{
},
};
+struct RunContext {
+ eos_tokens: Vec<u64>,
+ context_length: usize,
+ prefix_len_before: usize,
+ input_tokens_len: usize,
+ tokens_limit: usize,
+ prefill_result: PrefillResult,
+ prefill_duration: f64,
+ prefill_suffix_length: usize,
+ run_start: Instant,
+}
+
pub struct ChatSession {
pub model_path: PathBuf,
pub model_metadata: ModelMetadata,
@@ -246,7 +272,7 @@ impl ChatSession {
.map(|&id| id as u64)
.collect();
- let language_model_config_for_context = self
+ let language_model_config = self
.model_metadata
.model_config
.as_language_model()
@@ -254,68 +280,25 @@ impl ChatSession {
let llm =
self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- let context_length = llm
- .decoding_config
- .context_length
- .resolve(language_model_config_for_context);
+ let context_length =
+ llm.decoding_config.context_length.resolve(language_model_config);
if tokens.len() >= context_length {
return Err(Error::ContextLengthExceeded);
}
- let prefix_len_before = llm.tokens.len().saturating_sub(1);
- let eos_tokens: Vec<u64> = language_model_config_for_context
+ let prefix_offset = llm.tokens.len();
+ let prefix_len_before = prefix_offset.saturating_sub(1);
+
+ let eos_tokens: Vec<u64> = language_model_config
.generation_config
.stop_token_ids
.iter()
.map(|&x| x as u64)
.collect();
- let finish_reason = |llm: &LanguageModelGenerator,
- tokens_new: Vec<u64>|
- -> Option<FinishReason> {
- let start_idx = prefix_len_before + tokens.len();
- let total_tokens = llm.tokens.len();
- let total_new_tokens = llm.tokens[start_idx..].len();
- let has_eos_token =
- tokens_new.iter().any(|token| eos_tokens.contains(token));
- let context_limit_reached = total_tokens >= context_length;
-
- if has_eos_token {
- Some(FinishReason::Stop)
- } else if total_new_tokens >= config.tokens_limit as usize {
- Some(FinishReason::Length)
- } else if context_limit_reached {
- Some(FinishReason::ContextLimitReached)
- } else {
- None
- }
- };
-
- let build_generated_text = |llm: &LanguageModelGenerator,
- tokenizer: &Tokenizer|
- -> Result<String, Error> {
- let start_idx = prefix_len_before + tokens.len();
- let generated_tokens: Vec<u32> = llm.tokens[start_idx..]
- .to_vec()
- .iter()
- .map(|value| *value as u32)
- .collect();
- let generated_text = tokenizer
- .decode(&generated_tokens, true)
- .map_err(|_| Error::UnableToDecodeText)?;
- Ok(generated_text)
- };
-
- let language_model_config_for_sampling = self
- .model_metadata
- .model_config
- .as_language_model()
- .ok_or(Error::UnableToLoadConfig)?;
-
let sampling_method =
- config.sampling_policy.resolve(language_model_config_for_sampling);
+ config.sampling_policy.resolve(language_model_config);
- // Create grammar from config if provided
let mut compiled_grammar: Option<CompiledGrammar> =
if let Some(ref grammar_config) = config.grammar_config {
Some(CompiledGrammar::from_config(
@@ -328,8 +311,6 @@ impl ChatSession {
};
let prefill_start = Instant::now();
- let prefix_offset = llm.tokens.len();
-
let sample_suffix = config.tokens_limit > 0;
let prefill_result = llm.prefill(
tokens.clone(),
@@ -342,41 +323,38 @@ impl ChatSession {
let prefill_duration = prefill_start.elapsed().as_secs_f64();
llm.clear_cache();
- let prefill_finish_reason = finish_reason(llm, prefill_tokens.clone());
- let prefill_generated_text =
- build_generated_text(llm, &self.tokenizer)?;
- let prefill_parsed_text =
- self.output_parser.parse(prefill_generated_text);
-
- let language_model_config_for_prefill = self
- .model_metadata
- .model_config
- .as_language_model()
- .ok_or(Error::UnableToLoadConfig)?;
-
let prefill_suffix_length = llm
.decoding_config
.prefill_step_size
- .resolve(language_model_config_for_prefill);
- let prefill_output = Output {
- text: prefill_parsed_text,
- stats: Self::build_stats(
- &self.model_metadata,
- prefill_result.clone(),
- prefill_duration,
- prefill_suffix_length,
- Vec::new(),
- Vec::new(),
- llm.decoding_config.generate_suffix_length(),
- run_start.elapsed().as_secs_f64(),
- tokens.len(),
- llm.tokens[prefix_len_before + tokens.len()..].len(),
- ),
- finish_reason: prefill_finish_reason.clone(),
+ .resolve(language_model_config);
+
+ let run_context = RunContext {
+ eos_tokens,
+ context_length,
+ prefix_len_before,
+ input_tokens_len: tokens.len(),
+ tokens_limit: config.tokens_limit as usize,
+ prefill_result: prefill_result.clone(),
+ prefill_duration,
+ prefill_suffix_length,
+ run_start,
};
- let prefill_should_continue = if let Some(progress) = &progress {
- progress(prefill_output.clone())
+ let prefill_finish_reason =
+ Self::check_finish_reason(&run_context, llm, &prefill_tokens);
+ let prefill_output = Self::build_output(
+ &self.model_metadata,
+ &self.tokenizer,
+ &self.output_parser,
+ &run_context,
+ llm,
+ &[],
+ &[],
+ prefill_finish_reason.clone(),
+ )?;
+
+ let prefill_should_continue = if let Some(ref p) = progress {
+ p(prefill_output.clone())
} else {
true
};
@@ -390,60 +368,225 @@ impl ChatSession {
}
}
+ let can_use_async = llm.decoding_config.generate_suffix_length() == 1
+ && !llm.has_attention_layers()
+ && compiled_grammar.is_none();
+
+ let generate_output = if can_use_async {
+ let batch_size =
+ llm.decoding_config.async_batch_size.resolve(&self.model_path);
+ Self::run_async_batch(
+ &self.model_metadata,
+ &self.tokenizer,
+ &self.output_parser,
+ &run_context,
+ llm,
+ sampling_method,
+ &progress,
+ batch_size,
+ )?
+ } else {
+ Self::run_sync_generate(
+ &self.model_metadata,
+ &self.tokenizer,
+ &self.output_parser,
+ &run_context,
+ llm,
+ compiled_grammar.as_mut(),
+ sampling_method,
+ &progress,
+ )?
+ };
+
+ llm.clear_cache();
+ Ok(generate_output
+ .clone_with_duration(run_start.elapsed().as_secs_f64()))
+ }
+
+ fn run_sync_generate<F>(
+ model_metadata: &ModelMetadata,
+ tokenizer: &Tokenizer,
+ output_parser: &OutputParser,
+ run_context: &RunContext,
+ llm: &mut LanguageModelGenerator,
+ compiled_grammar: Option<&mut CompiledGrammar>,
+ sampling_method: super::parameter::SamplingMethod,
+ progress: &Option<F>,
+ ) -> Result<Output, Error>
+ where
+ F: Fn(Output) -> bool,
+ {
let mut generate_results: Vec<GenerateResult> = Vec::new();
let mut generate_durations: Vec<f64> = Vec::new();
- let generate_output = loop {
+ let mut compiled_grammar_mut = compiled_grammar;
+
+ loop {
let generate_start = Instant::now();
- let generate_result =
- llm.generate(compiled_grammar.as_mut(), sampling_method)?;
+ let generate_result = llm.generate(
+ compiled_grammar_mut.as_deref_mut(),
+ sampling_method,
+ )?;
let generate_tokens = generate_result.tokens.clone();
let generate_duration = generate_start.elapsed().as_secs_f64();
generate_results.push(generate_result);
generate_durations.push(generate_duration);
- let generate_finish_reason = finish_reason(llm, generate_tokens);
- let generate_generated_text =
- build_generated_text(llm, &self.tokenizer)?;
- let generate_parsed_text =
- self.output_parser.parse(generate_generated_text);
-
- let generate_output = Output {
- text: generate_parsed_text,
- stats: Self::build_stats(
- &self.model_metadata,
- prefill_result.clone(),
- prefill_duration,
- prefill_suffix_length,
- generate_results.clone(),
- generate_durations.clone(),
- llm.decoding_config.generate_suffix_length(),
- run_start.elapsed().as_secs_f64(),
- tokens.len(),
- llm.tokens[prefix_len_before + tokens.len()..].len(),
- ),
- finish_reason: generate_finish_reason.clone(),
- };
-
- let generate_should_continue = if let Some(progress) = &progress {
- progress(generate_output.clone())
+ let generate_finish_reason =
+ Self::check_finish_reason(run_context, llm, &generate_tokens);
+ let generate_output = Self::build_output(
+ model_metadata,
+ tokenizer,
+ output_parser,
+ run_context,
+ llm,
+ &generate_results,
+ &generate_durations,
+ generate_finish_reason.clone(),
+ )?;
+
+ let generate_should_continue = if let Some(progress_fn) = progress {
+ progress_fn(generate_output.clone())
} else {
true
};
if !generate_should_continue || generate_finish_reason.is_some() {
if generate_should_continue {
- break generate_output;
+ return Ok(generate_output);
} else {
- break generate_output.clone_with_finish_reason(Some(
+ return Ok(generate_output.clone_with_finish_reason(Some(
FinishReason::Cancelled,
+ )));
+ }
+ }
+ }
+ }
+
+ fn run_async_batch<F>(
+ model_metadata: &ModelMetadata,
+ tokenizer: &Tokenizer,
+ output_parser: &OutputParser,
+ run_context: &RunContext,
+ llm: &mut LanguageModelGenerator,
+ sampling_method: super::parameter::SamplingMethod,
+ progress: &Option<F>,
+ batch_size: usize,
+ ) -> Result<Output, Error>
+ where
+ F: Fn(Output) -> bool,
+ {
+ let remaining_by_limit = run_context
+ .tokens_limit
+ .saturating_sub(run_context.prefill_result.tokens.len());
+ let remaining_by_context =
+ run_context.context_length.saturating_sub(llm.tokens.len());
+ let tokens_to_generate = remaining_by_limit.min(remaining_by_context);
+
+ llm.prepare_async(tokens_to_generate);
+
+ let (sender, receiver) = mpsc::channel::<(usize, u64, u64)>();
+ let start_time = Instant::now();
+ let last_nanos = Arc::new(AtomicU64::new(0));
+
+ let mut results: Vec<GenerateResult> =
+ Vec::with_capacity(tokens_to_generate);
+ let mut durations: Vec<f64> = Vec::with_capacity(tokens_to_generate);
+ let mut finish_reason = FinishReason::Length;
+ let mut next_to_submit = 0;
+ let mut in_flight = 0;
+
+ while in_flight > 0 || next_to_submit < tokens_to_generate {
+ let batch_end =
+ std::cmp::min(next_to_submit + batch_size, tokens_to_generate);
+ for idx in next_to_submit..batch_end {
+ let batch_sender = sender.clone();
+ let last_callback_nanos = last_nanos.clone();
+ let batch_start_time = start_time;
+ llm.async_generate(idx, sampling_method, move |token| {
+ let now_nanos =
+ batch_start_time.elapsed().as_nanos() as u64;
+ let prev_nanos =
+ last_callback_nanos.swap(now_nanos, Ordering::SeqCst);
+ let _ = batch_sender.send((
+ idx,
+ token,
+ now_nanos.saturating_sub(prev_nanos),
));
+ })?;
+ in_flight += 1;
+ }
+ let batch_submitted = batch_end - next_to_submit;
+ next_to_submit = batch_end;
+
+ for _ in 0..batch_submitted {
+ let (_, token, duration_nanos) =
+ receiver.recv().map_err(|_| Error::SamplingFailed)?;
+ in_flight -= 1;
+
+ let duration = nanos_to_secs(duration_nanos);
+ llm.tokens.push(token);
+ results.push(GenerateResult {
+ tokens: vec![token],
+ forwardpass_duration: duration,
+ });
+ durations.push(duration);
+
+ let should_stop = if run_context.eos_tokens.contains(&token) {
+ finish_reason = FinishReason::Stop;
+ true
+ } else if llm.tokens.len() >= run_context.context_length {
+ finish_reason = FinishReason::ContextLimitReached;
+ true
+ } else if let Some(progress_fn) = progress {
+ let output = Self::build_output(
+ model_metadata,
+ tokenizer,
+ output_parser,
+ run_context,
+ llm,
+ &results,
+ &durations,
+ None,
+ )?;
+ if !progress_fn(output) {
+ finish_reason = FinishReason::Cancelled;
+ true
+ } else {
+ false
+ }
+ } else {
+ false
+ };
+
+ if should_stop {
+ while in_flight > 0 {
+ let _ = receiver.recv();
+ in_flight -= 1;
+ }
+ return Self::build_output(
+ model_metadata,
+ tokenizer,
+ output_parser,
+ run_context,
+ llm,
+ &results,
+ &durations,
+ Some(finish_reason),
+ );
}
}
- };
+ }
- llm.clear_cache();
- Ok(generate_output
- .clone_with_duration(run_start.elapsed().as_secs_f64()))
+ Self::build_output(
+ model_metadata,
+ tokenizer,
+ output_parser,
+ run_context,
+ llm,
+ &results,
+ &durations,
+ Some(finish_reason),
+ )
}
fn reconfigure_llm(
@@ -532,6 +675,77 @@ impl ChatSession {
}
impl ChatSession {
+ fn check_finish_reason(
+ run_context: &RunContext,
+ llm: &LanguageModelGenerator,
+ new_tokens: &[u64],
+ ) -> Option<FinishReason> {
+ let start_idx =
+ run_context.prefix_len_before + run_context.input_tokens_len;
+ let total_new_tokens = llm.tokens.len().saturating_sub(start_idx);
+ let has_eos =
+ new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));
+ let context_limit = llm.tokens.len() >= run_context.context_length;
+
+ if has_eos {
+ Some(FinishReason::Stop)
+ } else if total_new_tokens >= run_context.tokens_limit {
+ Some(FinishReason::Length)
+ } else if context_limit {
+ Some(FinishReason::ContextLimitReached)
+ } else {
+ None
+ }
+ }
+
+ fn decode_generated_tokens(
+ tokenizer: &Tokenizer,
+ llm: &LanguageModelGenerator,
+ run_context: &RunContext,
+ ) -> Result<String, Error> {
+ let start_idx =
+ run_context.prefix_len_before + run_context.input_tokens_len;
+ let generated_tokens: Vec<u32> =
+ llm.tokens[start_idx..].iter().map(|&v| v as u32).collect();
+ tokenizer
+ .decode(&generated_tokens, true)
+ .map_err(|_| Error::UnableToDecodeText)
+ }
+
+ fn build_output(
+ model_metadata: &ModelMetadata,
+ tokenizer: &Tokenizer,
+ output_parser: &OutputParser,
+ run_context: &RunContext,
+ llm: &LanguageModelGenerator,
+ generate_results: &[GenerateResult],
+ generate_durations: &[f64],
+ finish_reason: Option<FinishReason>,
+ ) -> Result<Output, Error> {
+ let text = Self::decode_generated_tokens(tokenizer, llm, run_context)?;
+ let parsed = output_parser.parse(text);
+ let start_idx =
+ run_context.prefix_len_before + run_context.input_tokens_len;
+ let output_tokens = llm.tokens.len().saturating_sub(start_idx);
+
+ Ok(Output {
+ text: parsed,
+ stats: Self::build_stats(
+ model_metadata,
+ run_context.prefill_result.clone(),
+ run_context.prefill_duration,
+ run_context.prefill_suffix_length,
+ generate_results.to_vec(),
+ generate_durations.to_vec(),
+ llm.decoding_config.generate_suffix_length(),
+ run_context.run_start.elapsed().as_secs_f64(),
+ run_context.input_tokens_len,
+ output_tokens,
+ ),
+ finish_reason,
+ })
+ }
+
fn build_stats(
model_metadata: &ModelMetadata,
prefill_result: PrefillResult,