Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Migrate RoPE config to per layer (#405)

CCYeh95b9ffec3a43
diff --git a/Cargo.lock b/Cargo.lock
index aab2875b9..dd6d8252c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -579,6 +579,7 @@ dependencies = [
"memmap2",
"minijinja",
"minijinja-contrib",
+ "monostate",
"mtl-rs",
"ndarray",
"num-traits",
diff --git a/Cargo.toml b/Cargo.toml
index c332aa862..2bc261c56 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -62,6 +62,7 @@ ndarray = "0.17"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
serde_ignored = "0.1"
+monostate = "0.1"
thiserror = "2.0"
tokenizers = { version = "0.22", features = [
"fancy-regex",
diff --git a/crates/backend-uzu/Cargo.toml b/crates/backend-uzu/Cargo.toml
index 5acf16af8..4311f487b 100644
--- a/crates/backend-uzu/Cargo.toml
+++ b/crates/backend-uzu/Cargo.toml
@@ -17,6 +17,7 @@ ndarray.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_ignored.workspace = true
+monostate.workspace = true
thiserror.workspace = true
tokenizers.workspace = true
rangemap.workspace = true
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index 9536a958e..9093b1569 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -98,6 +98,7 @@ impl StructuredAudioCodecGraph {
transformer_config: self.config.quantizer_config.post_module_config.clone(),
vocab_size: 1,
pard_token: None,
+ ple_model_config: None,
});
let model_shape = ModelShape::from_decoder_config(&decoder_config);
@@ -114,17 +115,14 @@ impl StructuredAudioCodecGraph {
.map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;
let max_sequence_length = decoder_config.transformer_config.context_length.max(required_sequence_length.max(1));
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config);
{
let transformer_tree = root_loader_view
.subtree(transformer_subtree_name)
.map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;
- if let Some(global_rope) = &mut shared_buffers.global_rope {
- global_rope.update_data(&transformer_tree, "global_rope");
- }
- if let Some(local_rope) = &mut shared_buffers.local_rope {
- local_rope.update_data(&transformer_tree, "local_rope");
- }
+ shared_buffers.update_data_from_transformer_tree(&transformer_tree).map_err(|err| {
+ AudioError::Runtime(format!("failed to update structured audio post_module shared buffers: {err}"))
+ })?;
}
let shared_buffers = Rc::new(shared_buffers);
let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(
@@ -304,7 +302,7 @@ impl StructuredAudioCodecGraph {
.allocate_scratch(main.as_buffer_range_ref().range().len())
.map_err(|err| AudioError::Runtime(format!("post_module shortcut allocation failed: {err}")))?;
for layer in runtime.layers.iter() {
- let rope_type = layer.rope_type();
+ let rope_buffers = runtime.shared_buffers.rope_buffers_for_layer(layer.layer_index);
main = layer
.encode(
LayerArguments {
@@ -313,10 +311,7 @@ impl StructuredAudioCodecGraph {
token_parents: token_inputs.token_parents(),
token_subtrie_ranges: None,
attention_sinks: runtime.shared_buffers.attention_sinks(layer.layer_index),
- rope_cosines: rope_type.and_then(|rope_type| runtime.shared_buffers.rope_cosines(rope_type)),
- rope_sines: rope_type.and_then(|rope_type| runtime.shared_buffers.rope_sines(rope_type)),
- rope_max_sequence_length: runtime.model_shape.context_length(),
- rope_dim: runtime.model_shape.rope_dim(),
+ rope_buffers,
sampling_start: 0,
sampling_length: batch_dim,
cache_layer: None,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
index d84c4a40d..65ae02228 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
@@ -3,6 +3,7 @@ pub mod attention_gemm;
pub mod attention_single_pass;
pub mod attention_two_pass;
pub mod attention_update_kv_cache;
+pub mod qk_unpack;
pub mod sigmoid_gate;
mod mask;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/qk_unpack.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/qk_unpack.rs
new file mode 100644
index 000000000..a37b04f85
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/qk_unpack.rs
@@ -0,0 +1,55 @@
+use dsl::kernel;
+use half::{bf16, f16};
+
+use crate::ArrayElement;
+
+#[kernel(QkUnpack)]
+#[variants(T, f32, f16, bf16)]
+pub fn qk_unpack<T: ArrayElement>(
+ qkv: *const T,
+ unpacked_queries: *mut T,
+ unpacked_keys: *mut T,
+ head_dim: u32,
+ num_heads: u32,
+ num_groups: u32,
+ suffix_length: u32,
+) {
+ if num_groups == 0 || num_heads % num_groups != 0 {
+ return;
+ }
+
+ let head_dim = head_dim as usize;
+ let num_heads = num_heads as usize;
+ let num_groups = num_groups as usize;
+ let suffix_length = suffix_length as usize;
+
+ let total_heads = num_heads + 2 * num_groups;
+ let heads_per_group = num_heads / num_groups;
+
+ for head_index in 0..num_heads {
+ let group_index = head_index / heads_per_group;
+ let first_head_in_group = group_index * heads_per_group;
+
+ for token_index in 0..suffix_length {
+ for dim_index in 0..head_dim {
+ let q_val =
+ unsafe { *qkv.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index) };
+ unsafe {
+ *unpacked_queries.add(head_index * suffix_length * head_dim + token_index * head_dim + dim_index) =
+ q_val;
+ }
+
+ if head_index == first_head_in_group {
+ let key_head_index = num_heads + group_index;
+ let k_val = unsafe {
+ *qkv.add(token_index * total_heads * head_dim + key_head_index * head_dim + dim_index)
+ };
+ unsafe {
+ *unpacked_keys
+ .add(group_index * suffix_length * head_dim + token_index * head_dim + dim_index) = k_val;
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal
new file mode 100644
index 000000000..23dab48e5
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal
@@ -0,0 +1,40 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+#include "../common/tensor_view.h"
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(QkUnpack)(
+ device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]
+ device T* unpacked_queries, // [num_heads, suffix_len, head_dim]
+ device T* unpacked_keys, // [num_groups, suffix_len, head_dim]
+ constant uint& head_dim,
+ constant uint& num_heads,
+ constant uint& num_groups,
+ constant uint& suffix_length,
+ const uint head_index AXIS(num_heads, 1),
+ const uint token_index AXIS(suffix_length, 1),
+ const uint dimension_index AXIS(head_dim, 128)
+) {
+ const uint group_index = head_index / (num_heads / num_groups);
+ const uint total_heads = num_heads + 2 * num_groups;
+ const uint first_head_in_group = group_index * (num_heads / num_groups);
+
+ TensorView3D<const T> qkv_tensor_view =
+ TensorView3D<const T>(qkv).shaped(suffix_length, total_heads, head_dim);
+ TensorView3D<T> unpacked_queries_tensor_view =
+ TensorView3D<T>(unpacked_queries)
+ .shaped(num_heads, suffix_length, head_dim);
+ TensorView3D<T> unpacked_keys_tensor_view =
+ TensorView3D<T>(unpacked_keys)
+ .shaped(num_groups, suffix_length, head_dim);
+
+ unpacked_queries_tensor_view(head_index, token_index, dimension_index) =
+ qkv_tensor_view(token_index, head_index, dimension_index);
+
+ if (head_index == first_head_in_group) {
+ const uint key_head_index = num_heads + group_index;
+ unpacked_keys_tensor_view(group_index, token_index, dimension_index) =
+ qkv_tensor_view(token_index, key_head_index, dimension_index);
+ }
+}
diff --git a/crates/backend-uzu/src/classifier/classifier.rs b/crates/backend-uzu/src/classifier/classifier.rs
index 98e23913b..a9c5d6957 100644
--- a/crates/backend-uzu/src/classifier/classifier.rs
+++ b/crates/backend-uzu/src/classifier/classifier.rs
@@ -135,7 +135,7 @@ impl<B: Backend> Classifier<B> {
.allocate_scratch(main.as_buffer_range_ref().range().len())
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
for (layer_index, layer) in self.context.layers.iter().enumerate() {
- let rope_type = layer.rope_type();
+ let rope_buffers = self.context.shared_buffers.rope_buffers_for_layer(layer_index);
main = layer
.encode(
LayerArguments {
@@ -144,10 +144,7 @@ impl<B: Backend> Classifier<B> {
token_parents: token_inputs.token_parents(),
token_subtrie_ranges: None,
attention_sinks: self.context.shared_buffers.attention_sinks(layer_index),
- rope_cosines: self.context.shared_buffers.rope_cosines(rope_type),
- rope_sines: self.context.shared_buffers.rope_sines(rope_type),
- rope_max_sequence_length: self.context.model_shape.context_length(),
- rope_dim: self.context.model_shape.rope_dim(),
+ rope_buffers,
sampling_start: 0,
sampling_length: batch_dim,
cache_layer: None,
diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rs
index 382584316..f40c01f28 100644
--- a/crates/backend-uzu/src/classifier/classifier_context.rs
+++ b/crates/backend-uzu/src/classifier/classifier_context.rs
@@ -1,15 +1,16 @@
use std::{fs::File, path::Path, rc::Rc};
+#[cfg(feature = "tracing")]
+use crate::forward_pass::model_shape::ModelShape;
use crate::{
DataType,
backends::common::{Backend, Context},
classifier::ClassifierError,
config::{ClassifierModelConfig, ModelMetadata},
- encodable_block::{ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, Rope},
- forward_pass::{
- model_shape::ModelShape,
- state::{RopeType, SharedBuffers},
+ encodable_block::{
+ ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, QkUnpack, Rope,
},
+ forward_pass::state::SharedBuffers,
parameters::ParameterLoader,
session::types::Error,
};
@@ -20,6 +21,7 @@ pub struct ClassifierContext<B: Backend> {
pub shared_buffers: Rc<SharedBuffers<B>>,
pub model_config: ClassifierModelConfig,
+ #[cfg(feature = "tracing")]
pub model_shape: ModelShape,
pub embed: Embedding<B>,
@@ -43,7 +45,9 @@ impl<B: Backend> ClassifierContext<B> {
transformer_config: model_metadata.model_config.model_config.transformer_config.clone(),
vocab_size: model_metadata.model_config.model_config.vocab_size,
pard_token: None,
+ ple_model_config: None,
});
+ #[cfg(feature = "tracing")]
let model_shape = ModelShape::from_decoder_config(&decoder_config);
let weights_path = model_path.join("model.safetensors");
@@ -54,8 +58,8 @@ impl<B: Backend> ClassifierContext<B> {
let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let root_loader_view = loader.tree();
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
- shared_buffers.update_data(&root_loader_view);
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config);
+ shared_buffers.update_data(&root_loader_view)?;
let shared_buffers = Rc::new(shared_buffers);
let transformer_tree = root_loader_view
@@ -78,16 +82,14 @@ impl<B: Backend> ClassifierContext<B> {
)
.expect("Failed to create embedding");
- let global_rope = Self::create_rope_block(&context, data_type, RopeType::Global).map_err(Error::Classifier)?;
- let local_rope = model_metadata
- .model_config
- .model_config
- .transformer_config
- .local_rope_config
- .as_ref()
- .map(|_| Self::create_rope_block(&context, data_type, RopeType::Local))
- .transpose()
- .map_err(Error::Classifier)?;
+ let rope = Rc::new(
+ Rope::<B>::new(context.as_ref(), data_type)
+ .map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e))))?,
+ );
+ let qk_unpack = Rc::new(
+ QkUnpack::<B>::new(context.as_ref(), data_type)
+ .map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("QkUnpack: {:?}", e))))?,
+ );
let layers = model_metadata
.model_config
@@ -97,12 +99,7 @@ impl<B: Backend> ClassifierContext<B> {
.iter()
.enumerate()
.map(|(layer_index, layer_config)| {
- let attn = layer_config.mixer_config.as_attention().ok_or(ClassifierError::NonAttentionMixer)?;
- let rope = if attn.sliding_window_size.is_some() && local_rope.is_some() {
- local_rope.clone().unwrap()
- } else {
- global_rope.clone()
- };
+ layer_config.mixer_config.as_attention().ok_or(ClassifierError::NonAttentionMixer)?;
let layer_tree = transformer_tree
.subtree(&format!("layers.{}", layer_index))
@@ -114,7 +111,8 @@ impl<B: Backend> ClassifierContext<B> {
layer_config,
layer_index,
&layer_tree,
- rope,
+ rope.clone(),
+ qk_unpack.clone(),
))
})
.collect::<Result<Vec<_>, ClassifierError>>()
@@ -221,6 +219,7 @@ impl<B: Backend> ClassifierContext<B> {
context,
shared_buffers,
model_config: model_metadata.model_config.clone(),
+ #[cfg(feature = "tracing")]
model_shape,
embed,
embedding_norm,
@@ -230,14 +229,4 @@ impl<B: Backend> ClassifierContext<B> {
prediction_head,
})
}
-
- fn create_rope_block(
- context: &B::Context,
- data_type: DataType,
- rope_type: RopeType,
- ) -> Result<Rc<Rope<B>>, ClassifierError> {
- let rotation = Rope::<B>::new(context, data_type, rope_type)
- .map_err(|e| ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e)))?;
- Ok(Rc::new(rotation))
- }
}
diff --git a/crates/backend-uzu/src/config/attention.rs b/crates/backend-uzu/src/config/attention.rs
index 061dde769..e77a4cdc1 100644
--- a/crates/backend-uzu/src/config/attention.rs
+++ b/crates/backend-uzu/src/config/attention.rs
@@ -22,6 +22,4 @@ pub struct AttentionConfig {
pub has_qkv_biases: bool,
pub has_out_biases: bool,
pub gate_projection_config: Option<LinearConfig>,
- pub use_rope: bool,
- pub partial_rope_dim: Option<usize>,
}
diff --git a/crates/backend-uzu/src/config/decoder.rs b/crates/backend-uzu/src/config/decoder.rs
index 4cf547e35..77fd4c92c 100644
--- a/crates/backend-uzu/src/config/decoder.rs
+++ b/crates/backend-uzu/src/config/decoder.rs
@@ -1,6 +1,7 @@
use serde::{Deserialize, Serialize};
use super::{AttentionConfig, EmbeddingConfig, TransformerConfig};
+use crate::utils::strict_serde::Unsupported;
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct DecoderConfig {
@@ -8,6 +9,7 @@ pub struct DecoderConfig {
pub transformer_config: TransformerConfig,
pub vocab_size: usize,
pub pard_token: Option<usize>,
+ pub ple_model_config: Option<Unsupported>,
}
impl DecoderConfig {
diff --git a/crates/backend-uzu/src/config/generation.rs b/crates/backend-uzu/src/config/generation.rs
index badb2ce7b..e33273a1b 100644
--- a/crates/backend-uzu/src/config/generation.rs
+++ b/crates/backend-uzu/src/config/generation.rs
@@ -1,5 +1,7 @@
use serde::{Deserialize, Serialize};
+use crate::utils::strict_serde::Unsupported;
+
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct GenerationConfig {
pub stop_token_ids: Vec<u32>,
@@ -8,4 +10,7 @@ pub struct GenerationConfig {
pub top_p: Option<f32>,
pub min_p: Option<f32>,
pub banned_tokens: Option<Vec<u32>>,
+ pub repetition_penalty: Option<Unsupported>,
+ pub presence_penalty: Option<Unsupported>,
+ pub frequency_penalty: Option<Unsupported>,
}
diff --git a/crates/backend-uzu/src/config/rope.rs b/crates/backend-uzu/src/config/rope.rs
index 3300cb497..204c70a62 100644
--- a/crates/backend-uzu/src/config/rope.rs
+++ b/crates/backend-uzu/src/config/rope.rs
@@ -7,6 +7,8 @@ pub struct RopeConfigCommon {
pub precision: ConfigDataType,
pub base: f32,
pub max_sequence_length: usize,
+ pub head_dim: usize,
+ pub partial_rotary_dim: Option<usize>,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
diff --git a/crates/backend-uzu/src/config/transformer.rs b/crates/backend-uzu/src/config/transformer.rs
index 343e78958..6dc9978d8 100644
--- a/crates/backend-uzu/src/config/transformer.rs
+++ b/crates/backend-uzu/src/config/transformer.rs
@@ -1,11 +1,9 @@
use serde::{Deserialize, Serialize};
-use super::{NormalizationConfig, RoPEConfig, TransformerLayerConfig};
+use super::{NormalizationConfig, TransformerLayerConfig};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct TransformerConfig {
- pub global_rope_config: Option<RoPEConfig>,
- pub local_rope_config: Option<RoPEConfig>,
pub layer_configs: Vec<TransformerLayerConfig>,
pub output_norm_config: NormalizationConfig,
pub model_dim: usize,
diff --git a/crates/backend-uzu/src/config/transformer_layer.rs b/crates/backend-uzu/src/config/transformer_layer.rs
index b297b5a2c..67fc864ce 100644
--- a/crates/backend-uzu/src/config/transformer_layer.rs
+++ b/crates/backend-uzu/src/config/transformer_layer.rs
@@ -1,6 +1,8 @@
+use monostate::MustBe;
use serde::{Deserialize, Serialize};
-use super::{MLPConfig, MixerConfig, NormalizationConfig};
+use super::{MLPConfig, MixerConfig, NormalizationConfig, RoPEConfig};
+use crate::utils::strict_serde::Unsupported;
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct TransformerLayerConfig {
@@ -10,4 +12,9 @@ pub struct TransformerLayerConfig {
pub pre_mlp_norm_config: NormalizationConfig,
pub mlp_config: MLPConfig,
pub post_mlp_norm_config: Option<NormalizationConfig>,
+ pub rope_config: Option<RoPEConfig>,
+ pub hidden_dim: Option<usize>,
+ pub ple_config: Option<Unsupported>,
+ pub has_post_layer_scalar: MustBe!(false),
+ pub kv_source_layer: Option<Unsupported>,
}
diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
index b0a4ca4aa..e069f3cff 100644
--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
@@ -4,8 +4,9 @@ use crate::{
DataType,
backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder, Kernels, kernel::TensorAddSwapKernel},
config::{TransformerConfig, TransformerLayerConfig},
- encodable_block::{Attention, AttentionArguments, LayerArguments, Linear, Mlp, Normalization, QKNorm, Rope},
- forward_pass::state::RopeType,
+ encodable_block::{
+ Attention, AttentionArguments, LayerArguments, Linear, Mlp, Normalization, QKNorm, QkUnpack, Rope,
+ },
parameters::ParameterTree,
};
@@ -14,7 +15,7 @@ pub struct ClassifierLayer<B: Backend> {
qkv_projection: Box<dyn Linear<B>>,
qk_norm: Option<QKNorm<B>>,
rope: Rc<Rope<B>>,
- use_rope: bool,
+ qk_unpack: Rc<QkUnpack<B>>,
attention: Attention<B>,
out_projection: Box<dyn Linear<B>>,
post_attention_norm: Option<Normalization<B>>,
@@ -37,6 +38,7 @@ impl<B: Backend> ClassifierLayer<B> {
layer_index: usize,
layer_loader: &ParameterTree<B::Context>,
rope: Rc<Rope<B>>,
+ qk_unpack: Rc<QkUnpack<B>>,
) -> Self {
let attention_config = layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
@@ -144,7 +146,7 @@ impl<B: Backend> ClassifierLayer<B> {
qkv_projection,
qk_norm,
rope,
- use_rope: attention_config.use_rope,
+ qk_unpack,
attention,
out_projection,
post_attention_norm,
@@ -172,10 +174,7 @@ impl<B: Backend> ClassifierLayer<B> {
token_positions,
token_subtrie_ranges,
attention_sinks,
- rope_cosines,
- rope_sines,
- rope_max_sequence_length,
- rope_dim,
+ rope_buffers,
#[cfg(feature = "tracing")]
trace,
..
@@ -206,22 +205,22 @@ impl<B: Backend> ClassifierLayer<B> {
if let Some(ref qk_norm) = self.qk_norm {
qk_norm.encode(&mut qkv, batch_dim, encoder)?;
}
- let cosines = rope_cosines.expect("Classifier attention layer requires RoPE cosine allocation");
- let sines = rope_sines.expect("Classifier attention layer requires RoPE sine allocation");
- let (queries, rotated_keys) = self.rope.encode(
- &qkv,
- token_positions,
- cosines,
- sines,
- batch_dim,
- self.num_heads,
- self.num_groups,
- self.head_dim,
- rope_max_sequence_length,
- rope_dim,
- self.use_rope,
- encoder,
- )?;
+ let (queries, rotated_keys) = match rope_buffers {
+ Some(rope_buffers) => self.rope.encode(
+ &qkv,
+ token_positions,
+ &rope_buffers.cosines,
+ &rope_buffers.sines,
+ batch_dim,
+ self.num_heads,
+ self.num_groups,
+ self.head_dim,
+ rope_buffers.max_sequence_length(),
+ rope_buffers.dim(),
+ encoder,
+ )?,
+ None => self.qk_unpack.encode(&qkv, batch_dim, self.num_heads, self.num_groups, self.head_dim, encoder)?,
+ };
let attention_output = self.attention.encode(
AttentionArguments {
token_subtrie_ranges,
@@ -289,8 +288,4 @@ impl<B: Backend> ClassifierLayer<B> {
Ok(main)
}
-
- pub fn rope_type(&self) -> RopeType {
- self.rope.rope_type()
- }
}
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index 9cb19bf21..0baa60bb9 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -10,11 +10,10 @@ use crate::{
DataType,
backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
config::DecoderConfig,
- encodable_block::{Embedding, LayerArguments, LayerExecutables, RMSNorm, Rope, embedding::EmbeddingError},
- forward_pass::{
- cache_layers::CacheLayers,
- state::{RopeType, SharedBuffers},
+ encodable_block::{
+ Embedding, LayerArguments, LayerExecutables, QkUnpack, RMSNorm, Rope, embedding::EmbeddingError,
},
+ forward_pass::{cache_layers::CacheLayers, state::SharedBuffers},
parameters::ParameterTree,
};
@@ -42,8 +41,6 @@ pub struct DecoderArguments<'a, B: Backend> {
pub batch_dim: usize,
pub sampling_start: usize,
pub sampling_length: usize,
- pub rope_max_sequence_length: usize,
- pub rope_dim: usize,
#[cfg(feature = "tracing")]
pub trace: Option<&'a mut ActivationTrace<B>>,
}
@@ -126,28 +123,17 @@ impl<B: Backend> Decoder<B> {
let tf = &decoder_config.transformer_config;
let norm_data_type: DataType = tf.layer_configs[0].mixer_config.activation_precision().into();
-
- let global_rope =
- tf.global_rope_config.is_some().then(|| Self::create_rope_block(context, norm_data_type, RopeType::Global));
- let local_rope =
- tf.local_rope_config.is_some().then(|| Self::create_rope_block(context, norm_data_type, RopeType::Local));
+ let rope = Rc::new(Rope::<B>::new(context, norm_data_type).expect("Failed to create Rope"));
+ let qk_unpack = Rc::new(QkUnpack::<B>::new(context, norm_data_type).expect("Failed to create QkUnpack"));
let layers = tf
.layer_configs
.iter()
.enumerate()
.map(|(layer_index, layer_config)| {
- let rope_for_layer = layer_config.mixer_config.as_attention().map(|attn| {
- if attn.sliding_window_size.is_some() {
- local_rope.clone().expect("Local rope missing for sliding-window attention layer")
- } else {
- global_rope.clone().expect("Global rope missing for attention layer")
- }
- });
-
let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap();
- LayerExecutables::new(context, tf, layer_config, layer_index, &layer_loader, rope_for_layer)
+ LayerExecutables::new(context, tf, layer_config, layer_index, &layer_loader, &rope, &qk_unpack)
})
.collect::<Vec<_>>();
@@ -165,14 +151,6 @@ impl<B: Backend> Decoder<B> {
(layers.into_boxed_slice(), norm_block)
}
- fn create_rope_block(
- context: &B::Context,
- data_type: DataType,
- rope_type: RopeType,
- ) -> Rc<Rope<B>> {
- Rc::new(Rope::<B>::new(context, data_type, rope_type).expect("Failed to create Rope"))
- }
-
fn run_layers(
&self,
args: DecoderArguments<B>,
@@ -188,8 +166,6 @@ impl<B: Backend> Decoder<B> {
batch_dim,
sampling_start,
sampling_length,
- rope_max_sequence_length,
- rope_dim,
#[cfg(feature = "tracing")]
trace,
} = args;
@@ -200,9 +176,7 @@ impl<B: Backend> Decoder<B> {
encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(DecoderError::BackendError)?;
for layer in self.layers.iter() {
- let rope_type = layer.rope_type();
- let rope_cosines = rope_type.and_then(|rope_type| shared_buffers.rope_cosines(rope_type));
- let rope_sines = rope_type.and_then(|rope_type| shared_buffers.rope_sines(rope_type));
+ let rope_buffers = shared_buffers.rope_buffers_for_layer(layer.layer_index);
let attention_sinks = shared_buffers.attention_sinks(layer.layer_index);
#[cfg(feature = "tracing")]
let layer_trace = trace.as_deref_mut().map(|trace| &mut trace.layer_results[layer.layer_index]);
@@ -216,10 +190,7 @@ impl<B: Backend> Decoder<B> {
token_parents,
token_subtrie_ranges,
attention_sinks,
- rope_cosines,
- rope_sines,
- rope_max_sequence_length,
- rope_dim,
+ rope_buffers,
sampling_start,
sampling_length,
cache_layer,
@@ -274,8 +245,6 @@ impl<B: Backend> Decoder<B> {
batch_dim: args.batch_dim,
sampling_start,
sampling_length,
- rope_max_sequence_length: args.rope_max_sequence_length,
- rope_dim: args.rope_dim,
#[cfg(feature = "tracing")]
trace: trace.as_deref_mut(),
},
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index 5d8c463af..ba7a2207b 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -13,9 +13,9 @@ use crate::{
config::{MixerConfig, TransformerConfig, TransformerLayerConfig},
encodable_block::{
Attention, AttentionArguments, DeltaNetArguments, DeltaNetMixer, Linear, MambaArguments, MambaMixer, Mlp,
- QKNorm, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,
+ QKNorm, QkUnpack, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,
},
- forward_pass::{cache_layers::CacheLayer, state::RopeType},
+ forward_pass::{cache_layers::CacheLayer, state::RopeBuffers},
parameters::ParameterTree,
};
@@ -42,7 +42,8 @@ impl<B: Backend> LayerExecutables<B> {
layer_config: &TransformerLayerConfig,
layer_index: usize,
decoder_layer_loader: &ParameterTree<B::Context>,
- rope: Option<Rc<Rope<B>>>,
+ rope: &Rc<Rope<B>>,
+ qk_unpack: &Rc<QkUnpack<B>>,
) -> Self {
let intermediate_data_type: DataType = layer_config.mixer_config.activation_precision().into();
@@ -52,8 +53,6 @@ impl<B: Backend> LayerExecutables<B> {
let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {
MixerConfig::Attention(attention_config) => {
- let rope_block = rope.expect("RoPE encoder missing for attention layer");
-
let q_dim = attention_config.num_heads * attention_config.head_dim;
let kv_dim = attention_config.num_groups * attention_config.head_dim;
@@ -150,8 +149,8 @@ impl<B: Backend> LayerExecutables<B> {
qkv_projection,
gate_projection,
qk_norm,
- rope: rope_block,
- use_rope: attention_config.use_rope,
+ rope: rope.clone(),
+ qk_unpack: qk_unpack.clone(),
attention,
out_projection,
num_heads: attention_config.num_heads,
@@ -297,10 +296,7 @@ impl<B: Backend> LayerExecutables<B> {
token_parents,
token_subtrie_ranges,
attention_sinks,
- rope_cosines,
- rope_sines,
- rope_max_sequence_length,
- rope_dim,
+ rope_buffers,
sampling_start,
sampling_length,
mut cache_layer,
@@ -323,7 +319,7 @@ impl<B: Backend> LayerExecutables<B> {
gate_projection,
qk_norm,
rope,
- use_rope,
+ qk_unpack,
attention,
out_projection,
num_heads,
@@ -346,22 +342,22 @@ impl<B: Backend> LayerExecutables<B> {
if let Some(norm) = qk_norm {
norm.encode(&mut qkv, batch_dim, encoder)?;
}
- let cosines = rope_cosines.expect("Attention layer requires RoPE cosine allocation");
- let sines = rope_sines.expect("Attention layer requires RoPE sine allocation");
- let (queries, rotated_keys) = rope.encode(
- &qkv,
- token_positions,
- cosines,
- sines,
- batch_dim,
- *num_heads,
- *num_groups,
- *head_dim,
- rope_max_sequence_length,
- rope_dim,
- *use_rope,
- encoder,
- )?;
+ let (queries, rotated_keys) = match rope_buffers {
+ Some(rope_buffers) => rope.encode(
+ &qkv,
+ token_positions,
+ &rope_buffers.cosines,
+ &rope_buffers.sines,
+ batch_dim,
+ *num_heads,
+ *num_groups,
+ *head_dim,
+ rope_buffers.max_sequence_length(),
+ rope_buffers.dim(),
+ encoder,
+ )?,
+ None => qk_unpack.encode(&qkv, batch_dim, *num_heads, *num_groups, *head_dim, encoder)?,
+ };
let kv_cache_layer = cache_layer
.as_deref_mut()
.map(|layer| layer.as_transformer_mut().expect("Attention layer expects transformer cache"));
@@ -487,24 +483,6 @@ impl<B: Backend> LayerExecutables<B> {
Ok(hidden)
}
-
- pub fn rope_type(&self) -> Option<RopeType> {
- match &self.mixer {
- MixerExecutables::Attention {
- rope,
- ..
- } => Some(rope.rope_type()),
- MixerExecutables::StateSpace {
- ..
- }
- | MixerExecutables::ShortConv {
- ..
- }
- | MixerExecutables::DeltaNet {
- ..
- } => None,
- }
- }
}
pub struct LayerArguments<'a, B: Backend> {
@@ -513,10 +491,7 @@ pub struct LayerArguments<'a, B: Backend> {
pub token_parents: &'a Allocation<B>,
pub token_subtrie_ranges: Option<&'a Allocation<B>>,
pub attention_sinks: Option<&'a Allocation<B>>,
- pub rope_cosines: Option<&'a Allocation<B>>,
- pub rope_sines: Option<&'a Allocation<B>>,
- pub rope_max_sequence_length: usize,
- pub rope_dim: usize,
+ pub rope_buffers: Option<&'a RopeBuffers<B>>,
pub sampling_start: usize,
pub sampling_length: usize,
pub cache_layer: Option<&'a mut CacheLayer<B>>,
diff --git a/crates/backend-uzu/src/encodable_block/layer/mixer.rs b/crates/backend-uzu/src/encodable_block/layer/mixer.rs
index c5b0db2d9..c29a36cb5 100644
--- a/crates/backend-uzu/src/encodable_block/layer/mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/mixer.rs
@@ -4,7 +4,7 @@ use std::rc::Rc;
use crate::{
backends::common::Backend,
- encodable_block::{Attention, DeltaNetMixer, MambaMixer, QKNorm, Rope, ShortConvMixer, linear::Linear},
+ encodable_block::{Attention, DeltaNetMixer, MambaMixer, QKNorm, QkUnpack, Rope, ShortConvMixer, linear::Linear},
};
/// Mixer component - either attention, state space, short conv, or delta net.
@@ -14,7 +14,7 @@ pub(crate) enum MixerExecutables<B: Backend> {
gate_projection: Option<Box<dyn Linear<B>>>,
qk_norm: Option<QKNorm<B>>,
rope: Rc<Rope<B>>,
- use_rope: bool,
+ qk_unpack: Rc<QkUnpack<B>>,
attention: Attention<B>,
out_projection: Box<dyn Linear<B>>,
num_heads: usize,
diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rs
index 896877c45..b315743b3 100644
--- a/crates/backend-uzu/src/encodable_block/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mod.rs
@@ -12,6 +12,7 @@ mod normalization;
mod pooling;
mod prediction_head;
mod qk_norm;
+mod qk_unpack;
mod rms_norm;
mod rope;
mod sampling;
@@ -31,6 +32,7 @@ pub use normalization::Normalization;
pub use pooling::Pooling;
pub use prediction_head::ClassifierPredictionHead;
pub use qk_norm::QKNorm;
+pub use qk_unpack::QkUnpack;
pub use rms_norm::{RMSNorm, RMSNormError};
pub use rope::Rope;
pub(crate) use sampling::SamplingInputs;
diff --git a/crates/backend-uzu/src/encodable_block/qk_unpack.rs b/crates/backend-uzu/src/encodable_block/qk_unpack.rs
new file mode 100644
index 000000000..b8735ce7b
--- /dev/null
+++ b/crates/backend-uzu/src/encodable_block/qk_unpack.rs
@@ -0,0 +1,51 @@
+use crate::{
+ DataType,
+ array::size_for_shape,
+ backends::common::{
+ Allocation, Backend, Encoder,
+ kernel::{Kernels, QkUnpackKernel},
+ },
+};
+
+pub struct QkUnpack<B: Backend> {
+ unpack_kernel: <B::Kernels as Kernels>::QkUnpackKernel,
+ data_type: DataType,
+}
+
+impl<B: Backend> QkUnpack<B> {
+ pub fn new(
+ context: &B::Context,
+ data_type: DataType,
+ ) -> Result<Self, B::Error> {
+ Ok(Self {
+ unpack_kernel: <B::Kernels as Kernels>::QkUnpackKernel::new(context, data_type)?,
+ data_type,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ qkv: &Allocation<B>,
+ suffix_length: usize,
+ num_heads: usize,
+ num_groups: usize,
+ head_dim: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {
+ let mut queries =
+ encoder.allocate_scratch(size_for_shape(&[num_heads, suffix_length, head_dim], self.data_type))?;
+ let mut keys =
+ encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?;
+ self.unpack_kernel.encode(
+ qkv,
+ &mut queries,
+ &mut keys,
+ head_dim as u32,
+ num_heads as u32,
+ num_groups as u32,
+ suffix_length as u32,
+ encoder,
+ );
+ Ok((queries, keys))
+ }
+}
diff --git a/crates/backend-uzu/src/encodable_block/rope.rs b/crates/backend-uzu/src/encodable_block/rope.rs
index ed22dd8ca..5d603aa80 100644
--- a/crates/backend-uzu/src/encodable_block/rope.rs
+++ b/crates/backend-uzu/src/encodable_block/rope.rs
@@ -7,12 +7,10 @@ use crate::{
Allocation, Backend, Encoder,
kernel::{Kernels, RopeKernel},
},
- forward_pass::state::RopeType,
};
pub struct Rope<B: Backend> {
kernel: <B::Kernels as Kernels>::RopeKernel,
- rope_type: RopeType,
data_type: DataType,
}
@@ -20,11 +18,9 @@ impl<B: Backend> Rope<B> {
pub fn new(
context: &B::Context,
data_type: DataType,
- rope_type: RopeType,
) -> Result<Self, B::Error> {
Ok(Self {
kernel: <B::Kernels as Kernels>::RopeKernel::new(context, data_type)?,
- rope_type,
data_type,
})
}
@@ -41,15 +37,8 @@ impl<B: Backend> Rope<B> {
head_dim: usize,
rope_max_seq_len: usize,
rope_dim: usize,
- use_rope: bool,
encoder: &mut Encoder<B>,
) -> Result<(Allocation<B>, Allocation<B>), B::Error> {
- let applied_rope_dim = if use_rope {
- assert!(rope_dim > 0, "enabled RoPE dimension must be greater than zero");
- rope_dim
- } else {
- 0
- };
let mut rotated_queries =
encoder.allocate_scratch(size_for_shape(&[num_heads, suffix_length, head_dim], self.data_type))?;
let mut rotated_keys =
@@ -62,7 +51,7 @@ impl<B: Backend> Rope<B> {
&mut rotated_queries,
&mut rotated_keys,
head_dim as u32,
- applied_rope_dim as u32,
+ rope_dim as u32,
num_heads as u32,
num_groups as u32,
suffix_length as u32,
@@ -71,8 +60,4 @@ impl<B: Backend> Rope<B> {
);
Ok((rotated_queries, rotated_keys))
}
-
- pub fn rope_type(&self) -> RopeType {
- self.rope_type
- }
}
diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rs
index b7e531b50..9fa30e4cc 100644
--- a/crates/backend-uzu/src/forward_pass/model_shape.rs
+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs
@@ -17,7 +17,6 @@ pub struct ModelShape {
pub num_layers: usize,
pub sliding_window_length_per_layer: Box<[Option<usize>]>,
layer_mixers: Box<[MixerConfig]>,
- max_rope_dim: usize,
}
impl ModelShape {
@@ -39,18 +38,6 @@ impl ModelShape {
MLPConfig::MixtureOfExperts(m) => m.expert_config.linear_config.activation_precision().into(),
};
- let mut max_rope_dim = 0usize;
- for layer in layer_configs.iter() {
- if let MixerConfig::Attention(attn) = &layer.mixer_config
- && attn.use_rope
- {
- max_rope_dim = max_rope_dim.max(attn.partial_rope_dim.unwrap_or(attn.head_dim));
- }
- }
- if max_rope_dim == 0 {
- max_rope_dim = head_dim;
- }
-
let sliding_window_length_per_layer: Box<[Option<usize>]> =
layer_configs.iter().map(|l| l.mixer_config.sliding_window_size()).collect();
let layer_mixers: Box<[MixerConfig]> = layer_configs.iter().map(|l| l.mixer_config.clone()).collect();
@@ -66,7 +53,6 @@ impl ModelShape {
num_layers,
sliding_window_length_per_layer,
layer_mixers,
- max_rope_dim,
}
}
@@ -78,10 +64,6 @@ impl ModelShape {
self.kv_cache_type
}
- pub fn rope_dim(&self) -> usize {
- self.max_rope_dim
- }
-
pub fn num_groups(&self) -> usize {
self.num_groups
}
diff --git a/crates/backend-uzu/src/forward_pass/state/mod.rs b/crates/backend-uzu/src/forward_pass/state/mod.rs
index efa4164c8..ada5cd024 100644
--- a/crates/backend-uzu/src/forward_pass/state/mod.rs
+++ b/crates/backend-uzu/src/forward_pass/state/mod.rs
@@ -1,8 +1,5 @@
-mod rope_type;
-
mod rope_buffers;
mod shared_buffers;
pub use rope_buffers::RopeBuffers;
-pub use rope_type::RopeType;
pub use shared_buffers::SharedBuffers;
diff --git a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
index 37894d4a7..678e3c8a7 100644
--- a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
+++ b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
@@ -1,8 +1,9 @@
use crate::{
+ DataType,
array::ArrayContextExt,
backends::common::{Allocation, Backend},
- forward_pass::model_shape::ModelShape,
parameters::ParameterTree,
+ session::types::Error,
};
pub struct RopeBuffers<B: Backend> {
@@ -10,36 +11,43 @@ pub struct RopeBuffers<B: Backend> {
pub cosines: Allocation<B>,
/// [rope_max_sequence_length, rope_dim]
pub sines: Allocation<B>,
+ max_sequence_length: usize,
+ dim: usize,
}
impl<B: Backend> RopeBuffers<B> {
pub fn new(
context: &B::Context,
- model_shape: &ModelShape,
+ max_sequence_length: usize,
+ head_dim: usize,
+ data_type: DataType,
) -> Self {
- let rope_dim = model_shape.rope_dim();
- let rope_max_sequence_length = model_shape.context_length();
-
+ let shape = [max_sequence_length, head_dim];
Self {
- cosines: context
- .create_array_uninitialized(&[rope_max_sequence_length, rope_dim], model_shape.activation_data_type())
- .into_allocation(),
- sines: context
- .create_array_uninitialized(&[rope_max_sequence_length, rope_dim], model_shape.activation_data_type())
- .into_allocation(),
+ cosines: context.create_array_uninitialized(&shape, data_type).into_allocation(),
+ sines: context.create_array_uninitialized(&shape, data_type).into_allocation(),
+ max_sequence_length,
+ dim: head_dim,
}
}
pub fn update_data(
&mut self,
parameter_tree: &ParameterTree<B::Context>,
- rope_name: &str,
- ) {
- let Ok(rope_tree) = parameter_tree.subtree(rope_name) else {
- return;
- };
+ rope_index: usize,
+ ) -> Result<(), Error> {
+ let rope_tree =
+ parameter_tree.subtree(&format!("ropes.{}", rope_index)).map_err(|_| Error::UnableToLoadWeights)?;
+ self.cosines = rope_tree.leaf_allocation("cosines").map_err(|_| Error::UnableToLoadWeights)?;
+ self.sines = rope_tree.leaf_allocation("sines").map_err(|_| Error::UnableToLoadWeights)?;
+ Ok(())
+ }
+
+ pub fn max_sequence_length(&self) -> usize {
+ self.max_sequence_length
+ }
- self.cosines = rope_tree.leaf_allocation("cosines").unwrap();
- self.sines = rope_tree.leaf_allocation("sines").unwrap();
+ pub fn dim(&self) -> usize {
+ self.dim
}
}
diff --git a/crates/backend-uzu/src/forward_pass/state/rope_type.rs b/crates/backend-uzu/src/forward_pass/state/rope_type.rs
deleted file mode 100644
index 374238c75..000000000
--- a/crates/backend-uzu/src/forward_pass/state/rope_type.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
-pub enum RopeType {
- Global,
- Local,
-}
diff --git a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
index dc56d0475..37652bd64 100644
--- a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
+++ b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
@@ -1,18 +1,24 @@
use half::{bf16, f16};
-use super::{RopeBuffers, RopeType};
+use super::RopeBuffers;
use crate::{
DataType,
array::ArrayContextExt,
backends::common::{Allocation, AsBufferRangeMut, Backend, DenseBuffer},
- config::DecoderConfig,
- forward_pass::model_shape::ModelShape,
+ config::{DecoderConfig, RoPEConfig},
parameters::ParameterTree,
+ session::types::Error,
};
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum LayerRopeKind {
+ NoKernel,
+ Indexed(usize),
+}
+
pub struct SharedBuffers<B: Backend> {
- pub global_rope: Option<RopeBuffers<B>>,
- pub local_rope: Option<RopeBuffers<B>>,
+ pub rope_buffers: Box<[RopeBuffers<B>]>,
+ layer_rope_kinds: Box<[LayerRopeKind]>,
pub attention_sinks: Box<[Option<Allocation<B>>]>,
}
@@ -20,11 +26,35 @@ impl<B: Backend> SharedBuffers<B> {
pub fn new(
context: &B::Context,
decoder_config: &DecoderConfig,
- model_shape: &ModelShape,
) -> Self {
let tf = &decoder_config.transformer_config;
- let global_rope = tf.global_rope_config.is_some().then(|| RopeBuffers::new(context, model_shape));
- let local_rope = tf.local_rope_config.is_some().then(|| RopeBuffers::new(context, model_shape));
+
+ let mut configs = Vec::<RoPEConfig>::new();
+ let layer_rope_kinds: Box<[LayerRopeKind]> = tf
+ .layer_configs
+ .iter()
+ .map(|layer_config| {
+ if layer_config.mixer_config.as_attention().is_none() {
+ return LayerRopeKind::NoKernel;
+ }
+ let Some(rope_config) = &layer_config.rope_config else {
+ return LayerRopeKind::NoKernel;
+ };
+ let index = configs.iter().position(|existing| existing == rope_config).unwrap_or_else(|| {
+ configs.push(rope_config.clone());
+ configs.len() - 1
+ });
+ LayerRopeKind::Indexed(index)
+ })
+ .collect();
+
+ let rope_buffers: Box<[RopeBuffers<B>]> = configs
+ .iter()
+ .map(|config| {
+ let common = config.common();
+ RopeBuffers::new(context, common.max_sequence_length, common.head_dim, common.precision.into())
+ })
+ .collect();
let attention_sinks = tf
.layer_configs
@@ -37,8 +67,8 @@ impl<B: Backend> SharedBuffers<B> {
.collect();
Self {
- global_rope,
- local_rope,
+ rope_buffers,
+ layer_rope_kinds,
attention_sinks,
}
}
@@ -46,16 +76,19 @@ impl<B: Backend> SharedBuffers<B> {
pub fn update_data(
&mut self,
parameter_tree: &ParameterTree<B::Context>,
- ) {
- let transformer_tree = parameter_tree.subtree("transformer").expect("transformer subtree not found");
+ ) -> Result<(), Error> {
+ let transformer_tree = parameter_tree.subtree("transformer").map_err(|_| Error::UnableToLoadWeights)?;
+ self.update_data_from_transformer_tree(&transformer_tree)?;
+ Ok(())
+ }
- if let Some(global_rope) = &mut self.global_rope {
- global_rope.update_data(&transformer_tree, "global_rope");
- }
- if let Some(local_rope) = &mut self.local_rope {
- local_rope.update_data(&transformer_tree, "local_rope");
+ pub fn update_data_from_transformer_tree(
+ &mut self,
+ transformer_tree: &ParameterTree<B::Context>,
+ ) -> Result<(), Error> {
+ for (rope_index, rope_buffers) in self.rope_buffers.iter_mut().enumerate() {
+ rope_buffers.update_data(transformer_tree, rope_index)?;
}
-
for (layer_idx, sink_cell) in self.attention_sinks.iter_mut().enumerate() {
let Some(sink_cell) = sink_cell.as_mut() else {
continue;
@@ -94,25 +127,16 @@ impl<B: Backend> SharedBuffers<B> {
},
}
}
+ Ok(())
}
- pub fn rope_cosines(
+ pub fn rope_buffers_for_layer(
&self,
- rope_type: RopeType,
- ) -> Option<&Allocation<B>> {
- match rope_type {
- RopeType::Global => self.global_rope.as_ref().map(|rope| &rope.cosines),
- RopeType::Local => self.local_rope.as_ref().map(|rope| &rope.cosines),
- }
- }
-
- pub fn rope_sines(
- &self,
- rope_type: RopeType,
- ) -> Option<&Allocation<B>> {
- match rope_type {
- RopeType::Global => self.global_rope.as_ref().map(|rope| &rope.sines),
- RopeType::Local => self.local_rope.as_ref().map(|rope| &rope.sines),
+ layer_index: usize,
+ ) -> Option<&RopeBuffers<B>> {
+ match self.layer_rope_kinds[layer_index] {
+ LayerRopeKind::NoKernel => None,
+ LayerRopeKind::Indexed(index) => Some(&self.rope_buffers[index]),
}
}
diff --git a/crates/backend-uzu/src/forward_pass/token_inputs.rs b/crates/backend-uzu/src/forward_pass/token_inputs.rs
index 2c07be9f6..cc7bc3ebf 100644
--- a/crates/backend-uzu/src/forward_pass/token_inputs.rs
+++ b/crates/backend-uzu/src/forward_pass/token_inputs.rs
@@ -76,7 +76,6 @@ impl<B: Backend> TokenInputs<B> {
pub fn decoder_arguments<'a>(
&'a self,
- model_shape: &ModelShape,
shared_buffers: &'a SharedBuffers<B>,
cache_layers: Option<&'a mut CacheLayers<B>>,
batch_dim: usize,
@@ -93,8 +92,6 @@ impl<B: Backend> TokenInputs<B> {
batch_dim,
sampling_start,
sampling_length,
- rope_max_sequence_length: model_shape.context_length(),
- rope_dim: model_shape.rope_dim(),
#[cfg(feature = "tracing")]
trace,
}
diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rs
index df753ee5e..84ba67760 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs
@@ -753,7 +753,6 @@ impl<B: Backend> LanguageModelGenerator<B> {
if is_prefilling {
let mut cache_layers = context.cache_layers.borrow_mut();
let decoder_arguments = token_inputs.decoder_arguments(
- &context.model_shape,
context.shared_buffers.as_ref(),
Some(&mut *cache_layers),
batch_dim,
@@ -769,7 +768,6 @@ impl<B: Backend> LanguageModelGenerator<B> {
} else {
let mut cache_layers = context.cache_layers.borrow_mut();
let decoder_arguments = token_inputs.decoder_arguments(
- &context.model_shape,
context.shared_buffers.as_ref(),
Some(&mut *cache_layers),
batch_dim,
diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
index 10728a8b4..e331444c9 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
@@ -139,9 +139,8 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let root_loader_view = loader.tree();
- let mut shared_buffers =
- SharedBuffers::new(context.as_ref(), &model_metadata.model_config.model_config, &model_shape);
- shared_buffers.update_data(&root_loader_view);
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &model_metadata.model_config.model_config);
+ shared_buffers.update_data(&root_loader_view)?;
let shared_buffers = Rc::new(shared_buffers);
let executables = Decoder::new(context.as_ref(), &model_metadata.model_config.model_config, &root_loader_view);
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
index 9ce351393..7f3bffb8e 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
@@ -32,7 +32,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
let shared_buffers = TokenDecoderContext::<B>::build_shared_buffers(
context,
decoder_config,
- model_shape,
&root_loader_view,
transformer_subtree,
)?;
@@ -143,18 +142,12 @@ impl<B: Backend> TokenDecoderContext<B> {
fn build_shared_buffers(
context: &Rc<B::Context>,
decoder_config: &Rc<crate::config::DecoderConfig>,
- model_shape: &ModelShape,
root_loader_view: &crate::parameters::ParameterTree<B::Context>,
transformer_subtree: &str,
) -> Result<Rc<SharedBuffers<B>>, Error> {
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config, model_shape);
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config);
let transformer_tree = root_loader_view.subtree(transformer_subtree).map_err(|_| Error::UnableToLoadWeights)?;
- if let Some(global_rope) = &mut shared_buffers.global_rope {
- global_rope.update_data(&transformer_tree, "global_rope");
- }
- if let Some(local_rope) = &mut shared_buffers.local_rope {
- local_rope.update_data(&transformer_tree, "local_rope");
- }
+ shared_buffers.update_data_from_transformer_tree(&transformer_tree)?;
Ok(Rc::new(shared_buffers))
}
@@ -282,8 +275,6 @@ impl<B: Backend> TokenDecoderRunner<B> {
batch_dim,
sampling_start,
sampling_length,
- rope_max_sequence_length: self.ctx.model_shape.context_length(),
- rope_dim: self.ctx.model_shape.rope_dim(),
#[cfg(feature = "tracing")]
trace: None,
}
@@ -805,8 +796,6 @@ impl<B: Backend> TokenDecoderRunner<B> {
batch_dim: token_count,
sampling_start,
sampling_length,
- rope_max_sequence_length: self.ctx.model_shape.context_length(),
- rope_dim: self.ctx.model_shape.rope_dim(),
#[cfg(feature = "tracing")]
trace: None,
};
diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
index 3bb6aac09..3f56851ae 100644
--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs
+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
@@ -267,12 +267,14 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(
transformer_config: slow_transformer_config,
vocab_size: config.vocab_size,
pard_token: None,
+ ple_model_config: None,
});
let fast_decoder_config = Rc::new(DecoderConfig {
embedding_config: config.fast_embeddings_config.to_text_decoder_embedding_config(),
transformer_config: fast_transformer_config,
vocab_size: config.codebook_size,
pard_token: None,
+ ple_model_config: None,
});
let text_decoder_context = B::Context::new().map_err(unable_to_create_context)?;
diff --git a/crates/backend-uzu/src/tracer/trace_validator.rs b/crates/backend-uzu/src/tracer/trace_validator.rs
index c6b1eda58..1c2755b0e 100644
--- a/crates/backend-uzu/src/tracer/trace_validator.rs
+++ b/crates/backend-uzu/src/tracer/trace_validator.rs
@@ -259,7 +259,6 @@ impl<B: Backend> TraceValidator<B> {
{
let mut cache_layers = ctx.cache_layers.borrow_mut();
let decoder_arguments = token_inputs.decoder_arguments(
- &ctx.model_shape,
ctx.shared_buffers.as_ref(),
Some(&mut *cache_layers),
token_ids.len(),
diff --git a/crates/backend-uzu/src/utils/strict_serde.rs b/crates/backend-uzu/src/utils/strict_serde.rs
index 5023edb16..d822c61d1 100644
--- a/crates/backend-uzu/src/utils/strict_serde.rs
+++ b/crates/backend-uzu/src/utils/strict_serde.rs
@@ -1,6 +1,12 @@
use std::io::Read;
-use serde::de::{DeserializeOwned, Error as _};
+use serde::{
+ Deserialize, Serialize,
+ de::{DeserializeOwned, Error as _},
+};
+
+#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
+pub enum Unsupported {}
pub fn from_reader_strict<T: DeserializeOwned>(reader: impl Read) -> Result<T, serde_json::Error> {
let value: serde_json::Value = serde_json::from_reader(reader)?;
diff --git a/crates/backend-uzu/src/utils/version.rs b/crates/backend-uzu/src/utils/version.rs
index 9a87fe686..bafe1a60c 100644
--- a/crates/backend-uzu/src/utils/version.rs
+++ b/crates/backend-uzu/src/utils/version.rs
@@ -1,2 +1,2 @@
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-pub const TOOLCHAIN_VERSION: &str = "0.6.13";
+pub const TOOLCHAIN_VERSION: &str = "0.8.0";
diff --git a/crates/backend-uzu/tests/unit/kernel/mod.rs b/crates/backend-uzu/tests/unit/kernel/mod.rs
index e8c5db0e4..b94ba58df 100644
--- a/crates/backend-uzu/tests/unit/kernel/mod.rs
+++ b/crates/backend-uzu/tests/unit/kernel/mod.rs
@@ -15,6 +15,7 @@ mod matmul;
mod moe;
mod pooling;
mod qk_norm_test;
+mod qk_unpack_test;
mod quant_matmul;
mod rms_norm_test;
mod rope_test;
diff --git a/crates/backend-uzu/tests/unit/kernel/qk_unpack_test.rs b/crates/backend-uzu/tests/unit/kernel/qk_unpack_test.rs
new file mode 100644
index 000000000..7442df5ad
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/kernel/qk_unpack_test.rs
@@ -0,0 +1,78 @@
+use std::fmt::{Debug, Display};
+
+use backend_uzu::{
+ ArrayContextExt, ArrayElement,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::QkUnpackKernel},
+ cpu::Cpu,
+ },
+};
+use half::bf16;
+use num_traits::Float;
+
+use crate::{common::assert::assert_eq_float, uzu_test};
+
+fn run<T: ArrayElement + Float + Debug + Display, B: Backend>(
+ num_heads: u32,
+ num_groups: u32,
+ head_dim: u32,
+ suffix_length: u32,
+) -> (Vec<T>, Vec<T>) {
+ let total_heads = (num_heads + 2 * num_groups) as usize;
+ let qkv_len = suffix_length as usize * total_heads * head_dim as usize;
+ let qkv: Box<[T]> = (0..qkv_len).map(|index| T::from(((index as f32) * 0.1).sin() * 2.0).unwrap()).collect();
+
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::QkUnpackKernel::new(&context, T::data_type())
+ .expect("Failed to create QkUnpackKernel");
+
+ let queries_len = num_heads as usize * suffix_length as usize * head_dim as usize;
+ let keys_len = num_groups as usize * suffix_length as usize * head_dim as usize;
+ let qkv_array = context.create_array_from(&[qkv_len], &qkv);
+ let mut queries = context.create_array_uninitialized(&[queries_len], T::data_type()).into_allocation();
+ let mut keys = context.create_array_uninitialized(&[keys_len], T::data_type()).into_allocation();
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ qkv_array.allocation(),
+ &mut queries,
+ &mut keys,
+ head_dim,
+ num_heads,
+ num_groups,
+ suffix_length,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ (crate::common::helpers::allocation_to_vec(&queries), crate::common::helpers::allocation_to_vec(&keys))
+}
+
+fn check<T: ArrayElement + Float + Debug + Display>(
+ num_heads: u32,
+ num_groups: u32,
+ head_dim: u32,
+ suffix_length: u32,
+) {
+ let (expected_queries, expected_keys) = run::<T, Cpu>(num_heads, num_groups, head_dim, suffix_length);
+ for_each_non_cpu_backend!(|B| {
+ let (queries, keys) = run::<T, B>(num_heads, num_groups, head_dim, suffix_length);
+ assert_eq_float::<T>(&expected_queries, &queries, 0.0, "QkUnpack queries");
+ assert_eq_float::<T>(&expected_keys, &keys, 0.0, "QkUnpack keys");
+ });
+}
+
+#[uzu_test]
+fn test_qk_unpack_gqa_f32() {
+ check::<f32>(8, 2, 64, 4);
+}
+
+#[uzu_test]
+fn test_qk_unpack_single_token_f32() {
+ check::<f32>(16, 4, 64, 1);
+}
+
+#[uzu_test]
+fn test_qk_unpack_bf16() {
+ check::<bf16>(8, 2, 64, 4);
+}
diff --git a/tools/downloader/main.py b/tools/downloader/main.py
index c11a1d208..c960f64cb 100644
--- a/tools/downloader/main.py
+++ b/tools/downloader/main.py
@@ -1,15 +1,16 @@
import base64
import json
import os
+import platform
+import re
+import tomllib
from dataclasses import asdict
-from itertools import chain
from pathlib import Path
from typing import Annotated
import google_crc32c
import requests
-import tomllib
-from .model import BenchmarkTask, File, Message, Model, Registry, Role
+from cattrs.errors import BaseValidationError
from rich.console import Console
from rich.progress import (
BarColumn,
@@ -22,12 +23,31 @@ from rich.progress import (
)
from rich.table import Table
from typer import Argument, Option, Typer
+
+from .model import (
+ BenchmarkTask,
+ Device,
+ File,
+ Message,
+ Model,
+ Registry,
+ Role,
+ parse_registry,
+)
from .utils import download_file_with_resume
ROOT_PATH = Path(__file__).parent.parent.parent
CARGO_TOML_PATH = ROOT_PATH / "Cargo.toml"
+TOOLCHAIN_VERSION_PATH = ROOT_PATH / "crates" / "backend-uzu" / "src" / "utils" / "version.rs"
WORKSPACE_PATH = ROOT_PATH / "workspace"
MODELS_PATH = WORKSPACE_PATH / "models"
+MIRAI_API_URL = "https://sdk.trymirai.com/api/v1/fetch/models"
+USER_FACING_ERRORS = (
+ BaseValidationError,
+ OSError,
+ ValueError,
+ requests.RequestException,
+)
console = Console()
@@ -39,36 +59,53 @@ app = Typer(
)
-def crc32c_file(path, chunk_size=8 * 1024 * 1024):
+def crc32c_file(path: Path, chunk_size: int = 8 * 1024 * 1024) -> str:
checksum = google_crc32c.Checksum()
- with open(path, "rb") as file:
+ with path.open("rb") as file:
for chunk in iter(lambda: file.read(chunk_size), b""):
checksum.update(chunk)
return base64.b64encode(checksum.digest()).decode("ascii")
def get_uzu_version() -> str:
- with open(CARGO_TOML_PATH, "rb") as file:
+ with CARGO_TOML_PATH.open("rb") as file:
cargo_toml = tomllib.load(file)
return cargo_toml["workspace"]["package"]["version"]
+def get_toolchain_version() -> str:
+ version_source = TOOLCHAIN_VERSION_PATH.read_text()
+ match = re.search(r'pub const TOOLCHAIN_VERSION: &str = "([^"]+)";', version_source)
+ if match is None:
+ raise ValueError(f"Unable to read TOOLCHAIN_VERSION from {TOOLCHAIN_VERSION_PATH}")
+ return match.group(1)
+
+
+def build_device() -> Device:
+ try:
+ memory_total: int | None = os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")
+ except (AttributeError, ValueError, OSError):
+ memory_total = None
+ return Device(
+ os_name=platform.platform(),
+ cpu_name=platform.processor() or None,
+ memory_total=memory_total,
+ )
+
+
def load_registry() -> Registry:
- types = ["language_model", "classifier_model", "tts_model"]
- url = f"https://sdk.trymirai.com/api/v1/models/list/uzu/{get_uzu_version()}?includeTraces=true"
- models: [Model] = []
- for type in types:
- url = f"https://sdk.trymirai.com/api/v1/models/list/uzu/{get_uzu_version()}?includeTraces=true&type={type}"
- response = requests.get(url)
- response.raise_for_status()
- data = response.json()
- registry = Registry.from_dict(data)
- models.extend(registry.models)
- return Registry(models=models)
-
-
[email protected](help="List models")
-def list(
+ payload = {
+ "device": {key: value for key, value in asdict(build_device()).items() if value is not None},
+ "backends": [{"identifier": "uzu", "version": get_toolchain_version()}],
+ "include_traces": True,
+ }
+ response = requests.post(MIRAI_API_URL, json=payload, timeout=10)
+ response.raise_for_status()
+ return parse_registry(response.json())
+
+
[email protected](name="list", help="List models")
+def list_models(
repos_only: Annotated[
bool,
Option("--repos", help="Print only model repository IDs."),
@@ -87,18 +124,18 @@ def list(
if repos_only:
for model in registry.models:
- console.print(model.repod_id, markup=False, highlight=False)
+ console.print(model.repo_id, markup=False, highlight=False)
return
table = Table()
table.add_column("Repo", style="cyan")
table.add_column("Name", style="green")
for model in registry.models:
- table.add_row(model.repod_id, model.name)
+ table.add_row(model.repo_id, model.name)
console.print(table)
- except Exception as e:
- err_console.print(f"[red]Error: {e}[/red]")
+ except USER_FACING_ERRORS as error:
+ err_console.print(f"[red]Error: {error}[/red]")
@app.command(help="Download model")
@@ -106,12 +143,10 @@ def download(
model_repo: Annotated[
str,
Argument(
- help=(
- "Hugging Face model repo. Example: [cyan]'meta-llama/Llama-3.2-1B-Instruct'[/cyan]."
- ),
+ help=("Hugging Face model repo. Example: [cyan]'meta-llama/Llama-3.2-1B-Instruct'[/cyan]."),
),
],
-):
+) -> None:
try:
with Progress(
SpinnerColumn(),
@@ -123,15 +158,9 @@ def download(
registry = load_registry()
progress.update(task, completed=True)
- model = None
- for current_model in registry.models:
- if current_model.repod_id == model_repo:
- model = current_model
- break
+ model = next((m for m in registry.models if model_repo in {m.id, m.repo_id, m.name}), None)
if not model:
- err_console.print(
- f"[red]Error: Model '{model_repo}' not found in registry[/red]"
- )
+ err_console.print(f"[red]Error: Model '{model_repo}' not found in registry[/red]")
return
engine_version = get_uzu_version()
@@ -150,11 +179,11 @@ def download(
files_to_download = []
def need_to_download(file: File, file_path: Path) -> bool:
- if file_path.exists():
- if file.crc32c != crc32c_file(file_path):
- os.remove(file_path)
- else:
- return False
+ if not file_path.exists():
+ return True
+ if file.crc32c == crc32c_file(file_path):
+ return False
+ file_path.unlink()
return True
for file in model.files:
@@ -165,28 +194,11 @@ def download(
else:
files_to_download.append((file, file_path))
- speculators_path = model_path / "speculators"
- speculators_path.mkdir(parents=True, exist_ok=True)
-
- for speculator in model.speculators:
- speculator_path = speculators_path / speculator.use_case
- speculator_path.mkdir(parents=True, exist_ok=True)
-
- for file in speculator.files:
- file_path = speculator_path / file.name
- need_to_download_file = need_to_download(file, file_path)
- if not need_to_download_file:
- console.print(f"[green]✓[/green] {file.name} already exists")
- else:
- files_to_download.append((file, file_path))
-
if not files_to_download:
console.print("[green]All files already downloaded![/green]")
return
- console.print(
- f"Downloading {len(files_to_download)} file(s) for [cyan]{model.name}[/cyan]..."
- )
+ console.print(f"Downloading {len(files_to_download)} file(s) for [cyan]{model.name}[/cyan]...")
with Progress(
TextColumn("[progress.description]{task.description}"),
BarColumn(),
@@ -201,34 +213,33 @@ def download(
try:
download_file_with_resume(file.url, file_path, progress, task)
progress.update(task, description=f"[green]✓[/green] {file.name}")
- except Exception as e:
+ except USER_FACING_ERRORS:
progress.update(task, description=f"[red]✗[/red] {file.name}")
- raise e
+ raise
console.print(f"[green]Successfully downloaded model: {model.name}[/green]")
- except Exception as e:
- err_console.print(f"[red]Error: {e}[/red]")
+ except USER_FACING_ERRORS as error:
+ err_console.print(f"[red]Error: {error}[/red]")
def generate_benchmark_task(model: Model) -> BenchmarkTask:
- messages: [Message] = [
+ messages: list[Message] = [
Message(role=Role.SYSTEM, content="Summarize user's input"),
Message(
role=Role.USER,
- content="Large language models, commonly referred to as LLMs, are a class of artificial intelligence systems designed to understand, generate, and manipulate human language at scale. They are built using deep learning techniques, most notably transformer architectures, and are trained on vast collections of text data drawn from books, articles, websites, and other written sources. Through this training process, LLMs learn statistical patterns in language, allowing them to predict likely sequences of words and produce coherent, contextually appropriate responses to prompts. At their core, LLMs operate by representing words or subword units as numerical vectors in a high-dimensional space. These representations capture semantic and syntactic relationships, such that words with similar meanings or grammatical roles tend to have similar vector representations. The transformer architecture enables the model to process entire sequences of text simultaneously rather than sequentially, using mechanisms such as self-attention to determine which parts of the input are most relevant at any given moment. This allows LLMs to handle long-range dependencies in text, such as references made many sentences earlier, more effectively than earlier generations of language models. Training an LLM typically involves two main phases. The first is pretraining, during which the model learns general language patterns by predicting missing or next tokens in large, mostly uncurated text corpora. This phase gives the model broad linguistic competence and general world knowledge as reflected in its training data. The second phase often involves fine-tuning, where the model is further trained on more specific datasets, such as question-answer pairs, instructional content, or conversational examples. Fine-tuning helps align the model’s behavior with particular tasks or desired styles of interaction. One of the most notable characteristics of LLMs is their versatility. A single model can perform a wide range of language-related tasks, including text generation, summarization, translation, classification, question answering, and code generation, often without task-specific retraining. This flexibility arises from the model’s general-purpose training and its ability to condition its outputs on the instructions or context provided in the prompt. As a result, LLMs are increasingly used as foundational models that can be adapted to many applications across different domains. Despite their impressive capabilities, LLMs do not possess true understanding or consciousness. They do not have beliefs, intentions, or awareness in the human sense. Instead, they generate responses based on learned correlations in data. This limitation can lead to errors such as producing confident-sounding but incorrect information, sometimes referred to as hallucinations. Because LLMs rely on patterns in their training data, they may also reflect biases, inaccuracies, or gaps present in that data. Addressing these issues is an ongoing area of research and development. The computational cost of training and running LLMs is another significant consideration. Training state-of-the-art models can require enormous amounts of computing power, energy, and financial investment. This has implications for environmental sustainability and for who can realistically develop and deploy such models. As a result, there is growing interest in techniques that improve efficiency, such as model compression, distillation, sparse architectures, and more efficient training algorithms, as well as in smaller models that can perform well on specific tasks. LLMs also raise important ethical and social questions. Their ability to generate human-like text can be beneficial in areas such as education, accessibility, and productivity, but it can also be misused for purposes like misinformation, plagiarism, or automated spam. Ensuring responsible use involves a combination of technical safeguards, policy decisions, and user education. Researchers and developers are actively exploring methods for making LLMs more transparent, controllable, and aligned with human values. As research continues, LLMs are likely to become more capable, more efficient, and more integrated into everyday tools and workflows. Future developments may involve better reasoning abilities, improved factual reliability, stronger multimodal integration with images, audio, and video, and more personalized interactions that adapt to individual users while respecting privacy. While LLMs are not a replacement for human judgment or creativity, they represent a powerful technology that, when used thoughtfully, can augment human capabilities and transform how people interact with information and with machines.",
+ content="Large language models, commonly referred to as LLMs, are a class of artificial intelligence systems designed to understand, generate, and manipulate human language at scale. They are built using deep learning techniques, most notably transformer architectures, and are trained on vast collections of text data drawn from books, articles, websites, and other written sources. Through this training process, LLMs learn statistical patterns in language, allowing them to predict likely sequences of words and produce coherent, contextually appropriate responses to prompts. At their core, LLMs operate by representing words or subword units as numerical vectors in a high-dimensional space. These representations capture semantic and syntactic relationships, such that words with similar meanings or grammatical roles tend to have similar vector representations. The transformer architecture enables the model to process entire sequences of text simultaneously rather than sequentially, using mechanisms such as self-attention to determine which parts of the input are most relevant at any given moment. This allows LLMs to handle long-range dependencies in text, such as references made many sentences earlier, more effectively than earlier generations of language models. Training an LLM typically involves two main phases. The first is pretraining, during which the model learns general language patterns by predicting missing or next tokens in large, mostly uncurated text corpora. This phase gives the model broad linguistic competence and general world knowledge as reflected in its training data. The second phase often involves fine-tuning, where the model is further trained on more specific datasets, such as question-answer pairs, instructional content, or conversational examples. Fine-tuning helps align the model’s behavior with particular tasks or desired styles of interaction. One of the most notable characteristics of LLMs is their versatility. A single model can perform a wide range of language-related tasks, including text generation, summarization, translation, classification, question answering, and code generation, often without task-specific retraining. This flexibility arises from the model’s general-purpose training and its ability to condition its outputs on the instructions or context provided in the prompt. As a result, LLMs are increasingly used as foundational models that can be adapted to many applications across different domains. Despite their impressive capabilities, LLMs do not possess true understanding or consciousness. They do not have beliefs, intentions, or awareness in the human sense. Instead, they generate responses based on learned correlations in data. This limitation can lead to errors such as producing confident-sounding but incorrect information, sometimes referred to as hallucinations. Because LLMs rely on patterns in their training data, they may also reflect biases, inaccuracies, or gaps present in that data. Addressing these issues is an ongoing area of research and development. The computational cost of training and running LLMs is another significant consideration. Training state-of-the-art models can require enormous amounts of computing power, energy, and financial investment. This has implications for environmental sustainability and for who can realistically develop and deploy such models. As a result, there is growing interest in techniques that improve efficiency, such as model compression, distillation, sparse architectures, and more efficient training algorithms, as well as in smaller models that can perform well on specific tasks. LLMs also raise important ethical and social questions. Their ability to generate human-like text can be beneficial in areas such as education, accessibility, and productivity, but it can also be misused for purposes like misinformation, plagiarism, or automated spam. Ensuring responsible use involves a combination of technical safeguards, policy decisions, and user education. Researchers and developers are actively exploring methods for making LLMs more transparent, controllable, and aligned with human values. As research continues, LLMs are likely to become more capable, more efficient, and more integrated into everyday tools and workflows. Future developments may involve better reasoning abilities, improved factual reliability, stronger multimodal integration with images, audio, and video, and more personalized interactions that adapt to individual users while respecting privacy. While LLMs are not a replacement for human judgment or creativity, they represent a powerful technology that, when used thoughtfully, can augment human capabilities and transform how people interact with information and with machines.", # noqa: E501, RUF001
),
]
- task = BenchmarkTask(
+ return BenchmarkTask(
identifier=model.name.lower(),
- repo_id=model.repod_id,
+ repo_id=model.repo_id,
number_of_runs=15,
tokens_limit=512,
messages=messages,
greedy=False,
)
- return task
if __name__ == "__main__":
diff --git a/tools/downloader/model.py b/tools/downloader/model.py
index 40d06be03..47b622188 100644
--- a/tools/downloader/model.py
+++ b/tools/downloader/model.py
@@ -1,71 +1,82 @@
from dataclasses import dataclass
-from enum import Enum
-from typing import List
+from enum import StrEnum
+from typing import Any
+
+import cattrs
@dataclass(frozen=True)
-class File:
+class _Hash:
+ method: str
+ value: str
+
+
+@dataclass(frozen=True)
+class _Repository:
+ identifier: str
+
+
+@dataclass(frozen=True)
+class _ReferenceFile:
name: str
url: str
- crc32c: str
+ size: int
+ hashes: list[_Hash]
- @classmethod
- def from_dict(cls, data: dict) -> "File":
- return cls(name=data["name"], url=data["url"], crc32c=data["crc32c"])
+
+@dataclass(frozen=True)
+class _Reference:
+ type: str
+ toolchain_version: str
+ files: list[_ReferenceFile]
+ source_repository: _Repository | None = None
+ repository: _Repository | None = None
@dataclass(frozen=True)
-class Speculator:
+class _Accessibility:
+ type: str
+ reference: _Reference | None = None
+
+
+@dataclass(frozen=True)
+class _MiraiModel:
id: str
- title: str
- description: str
- use_case: str
- files: List[File]
-
- @classmethod
- def from_dict(cls, data: dict) -> "Speculator":
- return cls(
- id=data["id"],
- title=data["title"],
- description=data["description"],
- use_case=data["useCase"],
- files=[File.from_dict(file) for file in data["files"]],
- )
+ accessibility: _Accessibility
+
+
+_CONVERTER = cattrs.Converter()
+
+
+@dataclass(frozen=True)
+class Device:
+ os_name: str
+ cpu_name: str | None
+ memory_total: int | None = None
+
+
+@dataclass(frozen=True)
+class File:
+ name: str
+ url: str
+ crc32c: str
@dataclass(frozen=True)
class Model:
id: str
toolchain_version: str
- repod_id: str
+ repo_id: str
name: str
- speculators: List[Speculator]
- files: List[File]
-
- @classmethod
- def from_dict(cls, data: dict) -> "Model":
- return cls(
- id=data["id"],
- toolchain_version=data["toolchainVersion"],
- repod_id=data["repoId"],
- name=data["name"],
- speculators=[
- Speculator.from_dict(speculator) for speculator in data["speculators"]
- ],
- files=[File.from_dict(file) for file in data["files"]],
- )
+ files: list[File]
@dataclass(frozen=True)
class Registry:
- models: List[Model]
+ models: list[Model]
- @classmethod
- def from_dict(cls, data: dict) -> "Registry":
- return cls(models=[Model.from_dict(model) for model in data["models"]])
-
-class Role(str, Enum):
+class Role(StrEnum):
SYSTEM = "system"
USER = "user"
ASSISTANT = "assistant"
@@ -83,5 +94,40 @@ class BenchmarkTask:
repo_id: str
number_of_runs: int
tokens_limit: int
- messages: List[Message]
+ messages: list[Message]
greedy: bool
+
+
+def parse_registry(payload: dict[str, Any]) -> Registry:
+ raw_models = _CONVERTER.structure(payload["models"], list[_MiraiModel])
+ models: list[Model] = []
+ for raw in raw_models:
+ if raw.accessibility.type != "local":
+ continue
+ reference = raw.accessibility.reference
+ if reference is None or reference.type != "mirai":
+ continue
+ repository = reference.source_repository or reference.repository
+ if repository is None:
+ continue
+ repo_id = repository.identifier
+ files = [
+ File(name=raw_file.name, url=raw_file.url, crc32c=_find_crc32c(raw_file)) for raw_file in reference.files
+ ]
+ models.append(
+ Model(
+ id=raw.id,
+ toolchain_version=reference.toolchain_version,
+ repo_id=repo_id,
+ name=repo_id.split("/")[-1],
+ files=files,
+ ),
+ )
+ return Registry(models=models)
+
+
+def _find_crc32c(raw_file: _ReferenceFile) -> str:
+ crc32c = next((h.value for h in raw_file.hashes if h.method == "crc32c"), None)
+ if crc32c is None:
+ raise ValueError(f"File {raw_file.name!r} is missing a crc32c hash")
+ return crc32c
diff --git a/tools/downloader/utils.py b/tools/downloader/utils.py
index e4f70bea7..4f1489594 100644
--- a/tools/downloader/utils.py
+++ b/tools/downloader/utils.py
@@ -4,9 +4,7 @@ import requests
from rich.progress import Progress
-def download_file_with_resume(
- url: str, file_path: Path, progress: Progress, task_id
-) -> None:
+def download_file_with_resume(url: str, file_path: Path, progress: Progress, task_id) -> None:
part_file_path = file_path.with_suffix(file_path.suffix + ".part")
resume_position = 0