A high-performance inference engine for AI models
Per-forward RoPE (#491)
uuuvn4ec41e1fa4c3
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime.rs b/crates/backend-uzu/src/audio/nanocodec/runtime.rsindex 665b81c6b..059da2d5a 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime.rs@@ -24,6 +24,7 @@ use crate::{decoder::DecoderConfig,embedding::{AnyEmbeddingConfig, untied_embedding::UntiedEmbeddingConfig},normalization::NormalizationConfig,+ rope::AnyRoPEConfig,tts::{TTSConfig,audio_decoder::{AnyTTSAudioDecoderConfig, descript_audio_codec::DescriptAudioCodecConfig},@@ -31,7 +32,7 @@ use crate::{},data_type::DataType,encodable_block::{Decoder, LayerExecutables, RMSNorm},- forward_pass::{model_shape::ModelShape, state::SharedBuffers},+ forward_pass::model_shape::ModelShape,parameters::ParameterLoader,};diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rsindex df09c9349..8a77982f6 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs@@ -12,7 +12,7 @@ pub(super) fn structured_audio_dtype_key(data_type: DataType) -> u8 {pub(super) struct StructuredAudioPostModuleRuntime<B: Backend> {pub(super) context: Rc<B::Context>,pub(super) model_shape: ModelShape,- pub(super) shared_buffers: Rc<SharedBuffers<B>>,+ pub(super) rope_configs: Box<[AnyRoPEConfig]>,pub(super) layers: Box<[LayerExecutables<B>]>,pub(super) output_norm: RMSNorm<B>,pub(super) max_sequence_length: usize,diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rsindex 763848eac..351756762 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs@@ -2,8 +2,8 @@ use super::*;use crate::{array::{Array, ArrayContextExt},backends::common::{Allocation, AsBufferRangeRef},- encodable_block::LayerArguments,- forward_pass::token_inputs::TokenInputs,+ encodable_block::{LayerArguments, PrecalculatedRope},+ forward_pass::{rope::precalculate_rope, token_inputs::TokenInputs},};impl StructuredAudioCodecGraph {@@ -108,17 +108,7 @@ impl StructuredAudioCodecGraph {let max_sequence_length =decoder_config.transformer_config.max_sequence_length().unwrap_or(0).max(required_sequence_length.max(1));- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);- {- let transformer_tree = root_loader_view- .subtree(transformer_subtree_name)- .map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;- shared_buffers.update_data_from_transformer_tree(&transformer_tree).map_err(|err| {- AudioError::Runtime(format!("failed to update structured audio post_module shared buffers: {err}"))- })?;- }- let shared_buffers = Rc::new(shared_buffers);- let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(+ let (rope_configs, layers, output_norm) = Decoder::build_transformer_layers_and_norm(context.as_ref(),&decoder_config,&root_loader_view,@@ -131,7 +121,7 @@ impl StructuredAudioCodecGraph {Ok(StructuredAudioPostModuleRuntime {context,model_shape,- shared_buffers,+ rope_configs,layers,output_norm,max_sequence_length,@@ -281,16 +271,34 @@ impl StructuredAudioCodecGraph {let mut shortcut = encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(|err| AudioError::Runtime(format!("post_module shortcut allocation failed: {err}")))?;+ let ropes = runtime+ .rope_configs+ .iter()+ .map(|rope_config| {+ let (sines, cosines) = precalculate_rope(rope_config, token_inputs.token_positions());+ let mut sines_allocation = encoder+ .allocate_constant(sines.len() * std::mem::size_of::<f32>())+ .map_err(|err| AudioError::Runtime(format!("post_module rope sines allocation failed: {err}")))?;+ sines_allocation.copyin(sines.as_ref());+ let mut cosines_allocation = encoder+ .allocate_constant(cosines.len() * std::mem::size_of::<f32>())+ .map_err(|err| AudioError::Runtime(format!("post_module rope cosines allocation failed: {err}")))?;+ cosines_allocation.copyin(cosines.as_ref());+ Ok(PrecalculatedRope {+ cosines: cosines_allocation,+ sines: sines_allocation,+ dim: *rope_config.head_dim(),+ })+ })+ .collect::<AudioResult<Box<[_]>>>()?;for layer in runtime.layers.iter() {- let rope_buffers = runtime.shared_buffers.rope_buffers_for_layer(layer.layer_index);main = layer.encode(LayerArguments {batch_dim,- token_positions: token_inputs.token_positions(),token_parents: token_inputs.token_parents(),token_subtrie_ranges: None,- rope_buffers,+ rope: layer.rope_kind.get(&ropes),per_layer_inputs: None,sampling_start: 0,sampling_length: batch_dim,diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rsindex d317b6cdf..f977f213a 100644--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs@@ -21,6 +21,18 @@ pub struct Allocation<B: Backend> {}impl<B: Backend> Allocation<B> {+ pub fn copyin<T: ArrayElement>(+ &mut self,+ data: &[T],+ ) {+ let buffer_range = self.as_buffer_range_mut();+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());+ let bytes = unsafe {+ std::slice::from_raw_parts_mut((buffer.cpu_ptr().as_ptr() as *mut u8).add(range.start), range.len())+ };+ bytemuck::cast_slice_mut::<u8, T>(bytes).copy_from_slice(data);+ }+pub fn copyout<T: ArrayElement>(&self) -> Vec<T> {let buffer_range = self.as_buffer_range_ref();let (buffer, range) = (buffer_range.buffer(), buffer_range.range());diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs b/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rsindex 00afc9a55..aecc03c90 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs@@ -33,7 +33,6 @@ pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(qkv: *const ElementT,cosines: *const RopeT,sines: *const RopeT,- token_positions: *const i32,rotated_queries: *mut ElementT,#[optional(!query_only)] rotated_keys: Option<*mut ElementT>,head_dim: u32,@@ -41,7 +40,6 @@ pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(num_heads: u32,#[optional(!query_only)] num_groups: Option<u32>,suffix_length: u32,- max_sequence_length: u32,#[specialize] query_only: bool,) {// Query-only projections carry no K/V heads, i.e. num_groups == 0.@@ -57,7 +55,6 @@ pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(let rope_dim = rope_dim as usize;let num_heads = num_heads as usize;let suffix_length = suffix_length as usize;- let max_sequence_length = max_sequence_length as usize;let half_rope_dim = rope_dim / 2;let total_heads = num_heads + 2 * num_groups;@@ -67,17 +64,10 @@ pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(let group_index = head_index / heads_per_group;for token_index in 0..suffix_length {- let raw_position = unsafe { *token_positions.add(token_index) } as usize;- let absolute_position = if raw_position >= max_sequence_length {- 0- } else {- raw_position- };-// Rotated dimensions: apply RoPE to dims 0..rope_dimfor dim_index in 0..rope_dim {- let cos_val = unsafe { (*cosines.add(absolute_position * rope_dim + dim_index)).to_f32().unwrap() };- let sin_val = unsafe { (*sines.add(absolute_position * rope_dim + dim_index)).to_f32().unwrap() };+ let cos_val = unsafe { (*cosines.add(token_index * rope_dim + dim_index)).to_f32().unwrap() };+ let sin_val = unsafe { (*sines.add(token_index * rope_dim + dim_index)).to_f32().unwrap() };// Query rotationlet q_val = unsafe {diff --git a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metalindex b596ba19d..670208e68 100644--- a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal@@ -25,9 +25,8 @@ VARIANTS(ElementT, float, half, bfloat)VARIANTS(RopeT, float, half, bfloat)PUBLIC KERNEL(Rope)(device const ElementT* qkv, // [suffix_len, total_heads, head_dim]- device const RopeT* cosines, // [max_seq_len, rope_dim]- device const RopeT* sines, // [max_seq_len, rope_dim]- device const int* token_positions, // [suffix_len]+ device const RopeT* cosines, // [suffix_len, rope_dim]+ device const RopeT* sines, // [suffix_len, rope_dim]device ElementT* rotated_queries, // [num_heads, suffix_len, head_dim]device ElementT* rotated_keys OPTIONAL(!query_only),constant uint& head_dim,@@ -35,7 +34,6 @@ PUBLIC KERNEL(Rope)(constant uint& num_heads,constant uint& num_groups OPTIONAL(!query_only),constant uint& suffix_length,- constant uint& max_sequence_length,const bool query_only SPECIALIZE,const uint head_index AXIS(num_heads, 1),const uint token_index AXIS(suffix_length, 1),@@ -54,27 +52,24 @@ PUBLIC KERNEL(Rope)(return;const uint total_heads = query_only ? num_heads : num_heads + 2 * num_groups;- const uint raw_position = token_positions[token_index];- const uint absolute_position =- raw_position >= max_sequence_length ? 0 : raw_position;const uint half_rope_dim = rope_dim / 2;TensorView3D<const ElementT> qkv_tensor_view =TensorView3D<const ElementT>(qkv).shaped(suffix_length, total_heads, head_dim);TensorView2D<const RopeT> cosines_tensor_view =- TensorView2D<const RopeT>(cosines).shaped(max_sequence_length, rope_dim);+ TensorView2D<const RopeT>(cosines).shaped(suffix_length, rope_dim);TensorView2D<const RopeT> sines_tensor_view =- TensorView2D<const RopeT>(sines).shaped(max_sequence_length, rope_dim);+ TensorView2D<const RopeT>(sines).shaped(suffix_length, rope_dim);TensorView3D<ElementT> rotated_queries_tensor_view =TensorView3D<ElementT>(rotated_queries).shaped(num_heads, suffix_length, head_dim);if (dimension_index < rope_dim) {const float cos_val =- float(cosines_tensor_view(absolute_position, dimension_index));+ float(cosines_tensor_view(token_index, dimension_index));const float sin_val =- float(sines_tensor_view(absolute_position, dimension_index));+ float(sines_tensor_view(token_index, dimension_index));rotated_queries_tensor_view(head_index, token_index, dimension_index) =applyRopeTransform(qkv_tensor_view,@@ -104,9 +99,9 @@ PUBLIC KERNEL(Rope)(const uint key_head_index = num_heads + group_index;if (dimension_index < rope_dim) {const float cos_val =- float(cosines_tensor_view(absolute_position, dimension_index));+ float(cosines_tensor_view(token_index, dimension_index));const float sin_val =- float(sines_tensor_view(absolute_position, dimension_index));+ float(sines_tensor_view(token_index, dimension_index));rotated_keys_tensor_view(group_index, token_index, dimension_index) =applyRopeTransform(qkv_tensor_view,diff --git a/crates/backend-uzu/src/classifier/classifier.rs b/crates/backend-uzu/src/classifier/classifier.rsindex 4193a642c..ac8b2a066 100644--- a/crates/backend-uzu/src/classifier/classifier.rs+++ b/crates/backend-uzu/src/classifier/classifier.rs@@ -7,8 +7,8 @@ use crate::{backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},config::model::classifier_model::ClassifierModelConfig,data_type::DataType,- encodable_block::LayerArguments,- forward_pass::token_inputs::TokenInputs,+ encodable_block::{LayerArguments, PrecalculatedRope},+ forward_pass::{rope::precalculate_rope, token_inputs::TokenInputs},session::types::Error,};@@ -134,22 +134,38 @@ impl<B: Backend> Classifier<B> {let mut shortcut = encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- for (layer_index, layer) in self.context.layers.iter().enumerate() {- let rope_buffers = self.context.shared_buffers.rope_buffers_for_layer(layer_index);+ let ropes = self+ .context+ .rope_configs+ .iter()+ .map(|rope_config| {+ let (sines, cosines) = precalculate_rope(rope_config, token_inputs.token_positions());+ let mut sines_allocation = encoder.allocate_constant(sines.len() * std::mem::size_of::<f32>())?;+ sines_allocation.copyin(sines.as_ref());+ let mut cosines_allocation = encoder.allocate_constant(cosines.len() * std::mem::size_of::<f32>())?;+ cosines_allocation.copyin(cosines.as_ref());+ Ok(PrecalculatedRope {+ cosines: cosines_allocation,+ sines: sines_allocation,+ dim: *rope_config.head_dim(),+ })+ })+ .collect::<Result<Box<[_]>, B::Error>>()+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ for layer in self.context.layers.iter() {main = layer.encode(LayerArguments {batch_dim,- token_positions: token_inputs.token_positions(),token_parents: token_inputs.token_parents(),token_subtrie_ranges: None,- rope_buffers,+ rope: layer.rope_kind.get(&ropes),per_layer_inputs: None,sampling_start: 0,sampling_length: batch_dim,cache_access: None,#[cfg(feature = "tracing")]- trace: trace.as_deref_mut().map(|traces| &mut traces.layer_results[layer_index]),+ trace: trace.as_deref_mut().map(|traces| &mut traces.layer_results[layer.layer_index]),},main,&mut shortcut,diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rsindex 81c8492c0..6896a1d75 100644--- a/crates/backend-uzu/src/classifier/classifier_context.rs+++ b/crates/backend-uzu/src/classifier/classifier_context.rs@@ -3,12 +3,13 @@ use std::{fs::File, path::Path, rc::Rc};use crate::{backends::common::{Backend, Context},classifier::ClassifierError,- config::{decoder::DecoderConfig, model::classifier_model::ClassifierModelConfig},+ config::{decoder::DecoderConfig, model::classifier_model::ClassifierModelConfig, rope::AnyRoPEConfig},data_type::DataType,encodable_block::{- ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, QkUnpack, Rope,+ ClassifierLayer, ClassifierPredictionHead, Embedding, LayerRopeKind, Linear, Normalization, Pooling, QkUnpack,+ Rope,},- forward_pass::{model_shape::ModelShape, state::SharedBuffers},+ forward_pass::model_shape::ModelShape,parameters::ParameterLoader,session::types::Error,};@@ -16,9 +17,8 @@ use crate::{pub struct ClassifierContext<B: Backend> {pub context: Rc<B::Context>,- pub shared_buffers: Rc<SharedBuffers<B>>,-pub model_config: ClassifierModelConfig,+ pub rope_configs: Box<[AnyRoPEConfig]>,#[cfg(feature = "tracing")]pub model_shape: ModelShape,@@ -57,12 +57,45 @@ impl<B: Backend> ClassifierContext<B> {let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::BF16);- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);- shared_buffers.update_data(&root_loader_view)?;- let shared_buffers = Rc::new(shared_buffers);-let transformer_tree =root_loader_view.subtree("transformer").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let mut rope_configs = Vec::<(AnyRoPEConfig, usize)>::new();+ let layer_rope_kinds: Box<[LayerRopeKind]> = classifier_config+ .transformer_config+ .layer_configs+ .iter()+ .map(|layer_config| {+ let Some(rope_config) = &layer_config.rope_config else {+ return LayerRopeKind::NoKernel;+ };+ let head_dim = *rope_config.head_dim();+ let index = rope_configs+ .iter()+ .position(|(existing_config, existing_head_dim)| {+ existing_config == rope_config && *existing_head_dim == head_dim+ })+ .unwrap_or_else(|| {+ rope_configs.push((rope_config.clone(), head_dim));+ rope_configs.len() - 1+ });+ LayerRopeKind::Indexed(index)+ })+ .collect();++ for (rope_index, (rope_config, head_dim)) in rope_configs.iter().enumerate() {+ let rope_tree = transformer_tree+ .subtree(&format!("ropes.{}", rope_index))+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let shape = [*rope_config.max_sequence_length(), *head_dim];+ rope_tree+ .leaf("cosines")+ .and_then(|leaf| leaf.validate(&shape, model_shape.rope_data_type))+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ rope_tree+ .leaf("sines")+ .and_then(|leaf| leaf.validate(&shape, model_shape.rope_data_type))+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ }let output_norm_tree =transformer_tree.subtree("output_norm").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;@@ -105,6 +138,8 @@ impl<B: Backend> ClassifierContext<B> {context.as_ref(),&classifier_config.transformer_config,layer_config,+ layer_index,+ layer_rope_kinds[layer_index],&layer_tree,rope.clone(),qk_unpack.clone(),@@ -207,8 +242,8 @@ impl<B: Backend> ClassifierContext<B> {Ok(Self {context,- shared_buffers,model_config: model_config.clone(),+ rope_configs: rope_configs.into_iter().map(|(config, _)| config).collect(),logits_data_type: model_shape.data_type,#[cfg(feature = "tracing")]model_shape,diff --git a/crates/backend-uzu/src/encodable_block/attention/mod.rs b/crates/backend-uzu/src/encodable_block/attention/mod.rsindex 05f8f0294..cb7211e04 100644--- a/crates/backend-uzu/src/encodable_block/attention/mod.rs+++ b/crates/backend-uzu/src/encodable_block/attention/mod.rs@@ -8,7 +8,7 @@ use gemm::{AttentionGemmArguments, AttentionGemmBlock};use itertools::iproduct;use thiserror::Error;-use super::{Linear, QKVNorm, QkUnpack, Rope, linear::LinearBlockError, qkv_norm::QKVNormError};+use super::{Linear, PrecalculatedRope, QKVNorm, QkUnpack, Rope, linear::LinearBlockError, qkv_norm::QKVNormError};use crate::{array::size_for_shape,backends::common::{@@ -26,7 +26,6 @@ use crate::{forward_pass::{cache_layers::LayerCacheAccess,kv_cache_layer::{KVCacheLayer, KVCacheLayerState},- state::RopeBuffers,},parameters::{ParameterLoaderError, ParameterTree},};@@ -330,9 +329,8 @@ impl<B: Backend> Attention<B> {pub fn encode(&self,- token_positions: &Allocation<B>,token_subtrie_ranges: Option<&Allocation<B>>,- rope_buffers: Option<&RopeBuffers<B>>,+ rope: Option<&PrecalculatedRope<B>>,cache_access: Option<LayerCacheAccess<B>>,hidden: Allocation<B>,suffix_length: usize,@@ -352,20 +350,10 @@ impl<B: Backend> Attention<B> {if let Some(qkv_norm) = &self.qkv_norm {qkv_norm.encode(&mut qkv, suffix_length, encoder)?;}- let (queries, rotated_keys) = match rope_buffers {- Some(rope_buffers) => self.rope.encode(- &qkv,- token_positions,- &rope_buffers.cosines,- &rope_buffers.sines,- suffix_length,- self.num_heads,- self.num_groups,- self.head_dim,- rope_buffers.max_sequence_length(),- rope_buffers.dim(),- encoder,- )?,+ let (queries, rotated_keys) = match rope {+ Some(rope) => {+ self.rope.encode(&qkv, rope, suffix_length, self.num_heads, self.num_groups, self.head_dim, encoder)?+ },None => {let (queries, keys) = self.qk_unpack.encode(&qkv,diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rsindex 168ae5e25..fa9fa3823 100644--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs@@ -7,8 +7,8 @@ use crate::{config::{transformer::TransformerConfig, transformer_layer::TransformerLayerConfig},data_type::DataType,encodable_block::{- Attention, AttentionError, LayerArguments, Mlp, MlpBlockError, Normalization, NormalizationError, QkUnpack,- Rope,+ Attention, AttentionError, LayerArguments, LayerRopeKind, Mlp, MlpBlockError, Normalization,+ NormalizationError, QkUnpack, Rope,},parameters::{ParameterLoaderError, ParameterTree},};@@ -34,6 +34,9 @@ pub enum ClassifierLayerError<B: Backend> {}pub struct ClassifierLayer<B: Backend> {+ #[cfg(feature = "tracing")]+ pub layer_index: usize,+ pub rope_kind: LayerRopeKind,pre_attention_norm: Option<Normalization<B>>,attention: Attention<B>,post_attention_norm: Option<Normalization<B>>,@@ -50,6 +53,8 @@ impl<B: Backend> ClassifierLayer<B> {context: &B::Context,transformer_config: &TransformerConfig,layer_config: &TransformerLayerConfig,+ #[cfg_attr(not(feature = "tracing"), allow(unused_variables))] layer_index: usize,+ rope_kind: LayerRopeKind,layer_loader: &ParameterTree<B>,rope: Rc<Rope<B>>,qk_unpack: Rc<QkUnpack<B>>,@@ -137,6 +142,9 @@ impl<B: Backend> ClassifierLayer<B> {.map_err(ClassifierLayerError::BackendError)?;Ok(Self {+ #[cfg(feature = "tracing")]+ layer_index,+ rope_kind,pre_attention_norm,attention,post_attention_norm,@@ -158,9 +166,8 @@ impl<B: Backend> ClassifierLayer<B> {) -> Result<Allocation<B>, B::Error> {let LayerArguments {batch_dim,- token_positions,token_subtrie_ranges,- rope_buffers,+ rope,#[cfg(feature = "tracing")]trace,..@@ -187,15 +194,7 @@ impl<B: Backend> ClassifierLayer<B> {encoder.encode_copy(&main, .., layer_traces.pre_attention_norm.allocation_mut(), ..);}- main = self.attention.encode(- token_positions,- token_subtrie_ranges,- rope_buffers,- None,- main,- batch_dim,- encoder,- )?;+ main = self.attention.encode(token_subtrie_ranges, rope, None, main, batch_dim, encoder)?;#[cfg(feature = "tracing")]if let Some(layer_traces) = layer_traces.as_deref_mut() {encoder.encode_copy(&main, .., layer_traces.attention.allocation_mut(), ..);diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex c64d6e70c..d99f02637 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -8,13 +8,14 @@ use thiserror::Error;use crate::forward_pass::traces::ActivationTrace;use crate::{backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},- config::decoder::DecoderConfig,+ config::{decoder::DecoderConfig, rope::AnyRoPEConfig},data_type::DataType,encodable_block::{- Embedding, LayerArguments, LayerExecutables, PerLayerEmbedding, PostLayerScalar, QkUnpack, RMSNorm,- RMSNormError, Rope, embedding::EmbeddingError, layer::LayerExecutablesError,+ Embedding, LayerArguments, LayerExecutables, LayerRopeKind, PerLayerEmbedding, PostLayerScalar,+ PrecalculatedRope, QkUnpack, RMSNorm, RMSNormError, Rope, embedding::EmbeddingError,+ layer::LayerExecutablesError,},- forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, rope::precalculate_rope},parameters::{ParameterLoaderError, ParameterTree},};@@ -37,16 +38,16 @@ pub enum DecoderError<B: Backend> {/// Full decoder executable with all layers and components.pub struct Decoder<B: Backend> {pub embed: Embedding<B>,+ pub rope_configs: Box<[AnyRoPEConfig]>,pub layers: Box<[LayerExecutables<B>]>,pub norm: RMSNorm<B>,pub per_layer_embedding: Option<PerLayerEmbedding<B>>,}pub struct DecoderArguments<'a, B: Backend> {- pub token_positions: &'a Allocation<B>,+ pub token_positions: &'a [usize],pub token_parents: &'a Allocation<B>,pub token_subtrie_ranges: Option<&'a Allocation<B>>,- pub shared_buffers: &'a SharedBuffers<B>,pub cache_layers: Option<&'a mut CacheLayers<B>>,pub batch_dim: usize,pub sampling_start: usize,@@ -79,7 +80,7 @@ impl<B: Backend> Decoder<B> {model_shape,)?;- let (layers, norm) = Self::build_transformer_layers_and_norm(+ let (rope_configs, layers, norm) = Self::build_transformer_layers_and_norm(context,decoder_config,root_weight_loader,@@ -93,6 +94,7 @@ impl<B: Backend> Decoder<B> {Ok(Self {embed,+ rope_configs,layers,norm,per_layer_embedding,@@ -121,7 +123,7 @@ impl<B: Backend> Decoder<B> {model_shape,)?;- let (layers, norm) = Self::build_transformer_layers_and_norm(+ let (rope_configs, layers, norm) = Self::build_transformer_layers_and_norm(context,decoder_config,root_weight_loader,@@ -135,6 +137,7 @@ impl<B: Backend> Decoder<B> {Ok(Self {embed,+ rope_configs,layers,norm,per_layer_embedding,@@ -148,10 +151,42 @@ impl<B: Backend> Decoder<B> {transformer_subtree: &str,output_norm_hadamard_factors: Option<Allocation<B>>,model_shape: &ModelShape,- ) -> Result<(Box<[LayerExecutables<B>]>, RMSNorm<B>), DecoderError<B>> {+ ) -> Result<(Box<[AnyRoPEConfig]>, Box<[LayerExecutables<B>]>, RMSNorm<B>), DecoderError<B>> {let decoder_weight_loader = root_weight_loader.subtree(transformer_subtree)?;let tf = &decoder_config.transformer_config;+ let mut rope_configs = Vec::<(AnyRoPEConfig, usize)>::new();+ let layer_rope_kinds: Box<[LayerRopeKind]> = tf+ .layer_configs+ .iter()+ .map(|layer_config| {+ if layer_config.mixer_config.as_attention().is_none() {+ return LayerRopeKind::NoKernel;+ }+ let Some(rope_config) = &layer_config.rope_config else {+ return LayerRopeKind::NoKernel;+ };+ let head_dim = *rope_config.head_dim();+ let index = rope_configs+ .iter()+ .position(|(existing_config, existing_head_dim)| {+ existing_config == rope_config && *existing_head_dim == head_dim+ })+ .unwrap_or_else(|| {+ rope_configs.push((rope_config.clone(), head_dim));+ rope_configs.len() - 1+ });+ LayerRopeKind::Indexed(index)+ })+ .collect();++ for (rope_index, (rope_config, head_dim)) in rope_configs.iter().enumerate() {+ let rope_tree = decoder_weight_loader.subtree(&format!("ropes.{}", rope_index))?;+ let shape = [*rope_config.max_sequence_length(), *head_dim];+ rope_tree.leaf("cosines")?.validate(&shape, model_shape.rope_data_type)?;+ rope_tree.leaf("sines")?.validate(&shape, model_shape.rope_data_type)?;+ }+let rope = Rc::new(Rope::<B>::new(context, model_shape, false).map_err(DecoderError::BackendError)?);// Built only when some layer projects queries only (reuses an earlier layer's KV cache).let rope_query_only = if tf.layer_configs.iter().any(|l| l.kv_source_layer_index.is_some()) {@@ -179,6 +214,7 @@ impl<B: Backend> Decoder<B> {tf,layer_config,layer_index,+ layer_rope_kinds[layer_index],&layer_loader,layer_rope,&qk_unpack,@@ -200,7 +236,7 @@ impl<B: Backend> Decoder<B> {PostLayerScalar::None,)?;- Ok((layers.into_boxed_slice(), norm_block))+ Ok((rope_configs.into_iter().map(|(config, _)| config).collect(), layers.into_boxed_slice(), norm_block))}fn build_per_layer_embedding(@@ -255,7 +291,6 @@ impl<B: Backend> Decoder<B> {token_positions,token_parents,token_subtrie_ranges,- shared_buffers,mut cache_layers,batch_dim,sampling_start,@@ -268,9 +303,25 @@ impl<B: Backend> Decoder<B> {let mut shortcut =encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(DecoderError::BackendError)?;+ let ropes = self+ .rope_configs+ .iter()+ .map(|rope_config| {+ let (sines, cosines) = precalculate_rope(rope_config, token_positions);+ let mut sines_allocation = encoder.allocate_constant(sines.len() * std::mem::size_of::<f32>())?;+ sines_allocation.copyin(sines.as_ref());+ let mut cosines_allocation = encoder.allocate_constant(cosines.len() * std::mem::size_of::<f32>())?;+ cosines_allocation.copyin(cosines.as_ref());+ Ok(PrecalculatedRope {+ cosines: cosines_allocation,+ sines: sines_allocation,+ dim: *rope_config.head_dim(),+ })+ })+ .collect::<Result<Box<[_]>, B::Error>>()+ .map_err(DecoderError::BackendError)?;for layer in self.layers.iter() {- let rope_buffers = shared_buffers.rope_buffers_for_layer(layer.layer_index);#[cfg(feature = "tracing")]let layer_trace = trace.as_deref_mut().map(|trace| &mut trace.layer_results[layer.layer_index]);@@ -280,10 +331,9 @@ impl<B: Backend> Decoder<B> {.encode(LayerArguments {batch_dim,- token_positions,token_parents,token_subtrie_ranges,- rope_buffers,+ rope: layer.rope_kind.get(&ropes),per_layer_inputs,sampling_start,sampling_length,@@ -345,7 +395,6 @@ impl<B: Backend> Decoder<B> {token_positions: args.token_positions,token_parents: args.token_parents,token_subtrie_ranges: args.token_subtrie_ranges,- shared_buffers: args.shared_buffers,cache_layers: args.cache_layers,batch_dim: args.batch_dim,sampling_start,diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rsindex b5981c917..633273c49 100644--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs@@ -12,10 +12,10 @@ use crate::{data_type::DataType,encodable_block::{Attention, AttentionError, DeltaNetArguments, DeltaNetMixer, DeltaNetMixerError, MambaArguments, MambaMixer,- MambaMixerError, Mlp, MlpBlockError, PerLayerEmbeddingProjection, PostLayerScalar, QkUnpack, RMSNorm,- RMSNormError, Rope, ShortConvArguments, ShortConvMixer, ShortConvMixerError, layer::MixerExecutables,+ MambaMixerError, Mlp, MlpBlockError, PerLayerEmbeddingProjection, PostLayerScalar, PrecalculatedRope, QkUnpack,+ RMSNorm, RMSNormError, Rope, ShortConvArguments, ShortConvMixer, ShortConvMixerError, layer::MixerExecutables,},- forward_pass::{cache_layers::LayerCacheAccess, state::RopeBuffers},+ forward_pass::cache_layers::LayerCacheAccess,parameters::{ParameterLoaderError, ParameterTree},};#[cfg(feature = "tracing")]@@ -55,9 +55,28 @@ pub enum LayerExecutablesError<B: Backend> {MissingPreMixerNormConfig,}+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum LayerRopeKind {+ NoKernel,+ Indexed(usize),+}++impl LayerRopeKind {+ pub fn get<B: Backend>(+ self,+ ropes: &[PrecalculatedRope<B>],+ ) -> Option<&PrecalculatedRope<B>> {+ match self {+ LayerRopeKind::NoKernel => None,+ LayerRopeKind::Indexed(index) => Some(&ropes[index]),+ }+ }+}+/// A single decoder layer with all its components.pub struct LayerExecutables<B: Backend> {pub layer_index: usize,+ pub rope_kind: LayerRopeKind,#[cfg(feature = "tracing")]pub tensor_add: <B::Kernels as Kernels>::TensorAddBiasKernel,pub pre_mixer_norm: RMSNorm<B>,@@ -77,6 +96,7 @@ impl<B: Backend> LayerExecutables<B> {transformer_config: &TransformerConfig,layer_config: &TransformerLayerConfig,layer_index: usize,+ rope_kind: LayerRopeKind,parameter_tree: &ParameterTree<B>,rope: &Rc<Rope<B>>,qk_unpack: &Rc<QkUnpack<B>>,@@ -261,6 +281,7 @@ impl<B: Backend> LayerExecutables<B> {Ok(Self {layer_index,+ rope_kind,#[cfg(feature = "tracing")]tensor_add,pre_mixer_norm,@@ -284,10 +305,9 @@ impl<B: Backend> LayerExecutables<B> {) -> Result<Allocation<B>, B::Error> {let LayerArguments {batch_dim,- token_positions,token_parents,token_subtrie_ranges,- rope_buffers,+ rope,per_layer_inputs,sampling_start,sampling_length,@@ -308,15 +328,7 @@ impl<B: Backend> LayerExecutables<B> {hidden = match &self.mixer {MixerExecutables::Attention {attention,- } => attention.encode(- token_positions,- token_subtrie_ranges,- rope_buffers,- cache_access,- hidden,- batch_dim,- encoder,- )?,+ } => attention.encode(token_subtrie_ranges, rope, cache_access, hidden, batch_dim, encoder)?,MixerExecutables::StateSpace {mixer,} => {@@ -437,10 +449,9 @@ impl<B: Backend> LayerExecutables<B> {pub struct LayerArguments<'a, B: Backend> {pub batch_dim: usize,- pub token_positions: &'a Allocation<B>,pub token_parents: &'a Allocation<B>,pub token_subtrie_ranges: Option<&'a Allocation<B>>,- pub rope_buffers: Option<&'a RopeBuffers<B>>,+ pub rope: Option<&'a PrecalculatedRope<B>>,pub per_layer_inputs: Option<&'a Allocation<B>>,pub sampling_start: usize,pub sampling_length: usize,diff --git a/crates/backend-uzu/src/encodable_block/layer/mod.rs b/crates/backend-uzu/src/encodable_block/layer/mod.rsindex 4898e3959..a7949b2b8 100644--- a/crates/backend-uzu/src/encodable_block/layer/mod.rs+++ b/crates/backend-uzu/src/encodable_block/layer/mod.rs@@ -3,5 +3,5 @@mod executables;mod mixer;-pub use executables::{LayerArguments, LayerExecutables, LayerExecutablesError};+pub use executables::{LayerArguments, LayerExecutables, LayerExecutablesError, LayerRopeKind};use mixer::MixerExecutables;diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rsindex ea6a68f65..dfb965035 100644--- a/crates/backend-uzu/src/encodable_block/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mod.rs@@ -26,7 +26,7 @@ pub use decoder::{Decoder, DecoderArguments, DecoderDecodeInput};pub use delta_net_mixer::{DeltaNetArguments, DeltaNetMixer, DeltaNetMixerError};pub use embedding::Embedding;pub use kv_cache_update::{KVCacheUpdate, KVLayerData};-pub use layer::{LayerArguments, LayerExecutables};+pub use layer::{LayerArguments, LayerExecutables, LayerRopeKind};pub use layer_norm::{LayerNorm, LayerNormError};pub use linear::Linear;pub use mamba_mixer::{MambaArguments, MambaMixer, MambaMixerError};@@ -38,6 +38,6 @@ pub use prediction_head::ClassifierPredictionHead;pub use qk_unpack::QkUnpack;pub use qkv_norm::QKVNorm;pub use rms_norm::{PostLayerScalar, RMSNorm, RMSNormError};-pub use rope::Rope;+pub use rope::{PrecalculatedRope, Rope};pub use sampling::{Sampling, SamplingArguments, SamplingInputs};pub use short_conv_mixer::{ShortConvArguments, ShortConvMixer, ShortConvMixerError};diff --git a/crates/backend-uzu/src/encodable_block/rope.rs b/crates/backend-uzu/src/encodable_block/rope.rsindex 472a4607d..94750e750 100644--- a/crates/backend-uzu/src/encodable_block/rope.rs+++ b/crates/backend-uzu/src/encodable_block/rope.rs@@ -16,6 +16,12 @@ pub struct Rope<B: Backend> {query_only: bool,}+pub struct PrecalculatedRope<B: Backend> {+ pub cosines: Allocation<B>,+ pub sines: Allocation<B>,+ pub dim: usize,+}+impl<B: Backend> Rope<B> {pub fn new(context: &B::Context,@@ -37,15 +43,11 @@ impl<B: Backend> Rope<B> {pub fn encode(&self,qkv: &Allocation<B>,- token_positions: &Allocation<B>,- cosines: &Allocation<B>,- sines: &Allocation<B>,+ rope: &PrecalculatedRope<B>,suffix_length: usize,num_heads: usize,num_groups: usize,head_dim: usize,- rope_max_seq_len: usize,- rope_dim: usize,encoder: &mut Encoder<B>,) -> Result<(Allocation<B>, Option<Allocation<B>>), B::Error> {let mut rotated_queries =@@ -57,17 +59,15 @@ impl<B: Backend> Rope<B> {};self.kernel.encode(qkv,- cosines,- sines,- token_positions,+ &rope.cosines,+ &rope.sines,&mut rotated_queries,rotated_keys.as_mut(),head_dim as u32,- rope_dim as u32,+ rope.dim as u32,num_heads as u32,(!self.query_only).then_some(num_groups as u32),suffix_length as u32,- rope_max_seq_len as u32,encoder,);Ok((rotated_queries, rotated_keys))diff --git a/crates/backend-uzu/src/forward_pass/mod.rs b/crates/backend-uzu/src/forward_pass/mod.rsindex 3f599a5ec..2bbab2cbb 100644--- a/crates/backend-uzu/src/forward_pass/mod.rs+++ b/crates/backend-uzu/src/forward_pass/mod.rs@@ -1,4 +1,5 @@pub mod model_shape;+pub mod rope;pub mod cache_layers;pub mod delta_net_layer;@@ -7,7 +8,5 @@ pub mod short_conv_layer;pub mod ssm_layer;pub mod token_inputs;-pub mod state;-#[cfg(feature = "tracing")]pub mod traces;diff --git a/crates/backend-uzu/src/forward_pass/rope.rs b/crates/backend-uzu/src/forward_pass/rope.rsnew file mode 100644index 000000000..af21e1b24--- /dev/null+++ b/crates/backend-uzu/src/forward_pass/rope.rs@@ -0,0 +1,81 @@+use crate::config::rope::AnyRoPEConfig;++pub fn precalculate_rope(+ rope_config: &AnyRoPEConfig,+ token_positions: &[usize],+) -> (Box<[f32]>, Box<[f32]>) {+ let head_dim = *rope_config.head_dim();+ assert!(head_dim > 0 && head_dim.is_multiple_of(2), "RoPE head_dim must be positive and even");+ let half_dim = head_dim / 2;+ let attention_scaling_factor = match rope_config {+ AnyRoPEConfig::YARNRoPEConfig(config) => 0.1 * config.scaling_factor.ln() + 1.0,+ _ => 1.0,+ };++ let element_count = token_positions.len() * head_dim;+ let mut sines = vec![0.0; element_count];+ let mut cosines = vec![0.0; element_count];+ for pair_index in 0..half_dim {+ let channel_index = pair_index * 2;+ let inverse_frequency = 1.0 / rope_config.base().powf(channel_index as f32 / head_dim as f32);+ let inverse_frequency = match rope_config {+ AnyRoPEConfig::UnscaledRoPEConfig(_) => inverse_frequency,+ AnyRoPEConfig::LinearScalingRoPEConfig(config) => inverse_frequency / config.scaling_factor,+ AnyRoPEConfig::LlamaRoPEConfig(config) => {+ let low_frequency_wavelength = config.original_context_length as f32 / config.low_frequency_factor;+ let high_frequency_wavelength = config.original_context_length as f32 / config.high_frequency_factor;+ let wavelength = 2.0 * std::f32::consts::PI / inverse_frequency;+ let scaled_frequency = inverse_frequency / config.scaling_factor;++ if wavelength < high_frequency_wavelength {+ inverse_frequency+ } else if wavelength > low_frequency_wavelength {+ scaled_frequency+ } else {+ let smoothing_factor =+ config.original_context_length as f32 / wavelength - config.low_frequency_factor;+ let smoothing_factor =+ smoothing_factor / (config.high_frequency_factor - config.low_frequency_factor);+ smoothing_factor * inverse_frequency + (1.0 - smoothing_factor) * scaled_frequency+ }+ },+ AnyRoPEConfig::YARNRoPEConfig(config) => {+ let dim = config.head_dim as f64;+ let base = config.base as f64;+ let original_context_length = config.original_context_length as f64;+ let beta_fast = config.beta_fast as f64;+ let beta_slow = config.beta_slow as f64;+ let mut low =+ dim * (original_context_length / (beta_fast * 2.0 * std::f64::consts::PI)).ln() / (2.0 * base.ln());+ let mut high =+ dim * (original_context_length / (beta_slow * 2.0 * std::f64::consts::PI)).ln() / (2.0 * base.ln());+ if config.truncate {+ low = low.floor();+ high = high.ceil();+ }+ let low = low.max(0.0) as f32;+ let mut high = high.min((config.head_dim - 1) as f64) as f32;+ if low == high {+ high += 0.001;+ }+ let ramp = ((pair_index as f32 - low) / (high - low)).clamp(0.0, 1.0);+ let smoothing_factor = 1.0 - ramp;+ let scaled_frequency = inverse_frequency / config.scaling_factor;+ scaled_frequency * (1.0 - smoothing_factor) + inverse_frequency * smoothing_factor+ },+ };++ for (token_index, token_position) in token_positions.iter().enumerate() {+ let embedding = *token_position as f32 * inverse_frequency;+ let sine = embedding.sin() * attention_scaling_factor;+ let cosine = embedding.cos() * attention_scaling_factor;+ let row_offset = token_index * head_dim;+ sines[row_offset + pair_index] = sine;+ sines[row_offset + half_dim + pair_index] = sine;+ cosines[row_offset + pair_index] = cosine;+ cosines[row_offset + half_dim + pair_index] = cosine;+ }+ }++ (sines.into_boxed_slice(), cosines.into_boxed_slice())+}diff --git a/crates/backend-uzu/src/forward_pass/state/mod.rs b/crates/backend-uzu/src/forward_pass/state/mod.rsdeleted file mode 100644index ada5cd024..000000000--- a/crates/backend-uzu/src/forward_pass/state/mod.rs+++ /dev/null@@ -1,5 +0,0 @@-mod rope_buffers;-mod shared_buffers;--pub use rope_buffers::RopeBuffers;-pub use shared_buffers::SharedBuffers;diff --git a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rsdeleted file mode 100644index c58553d02..000000000--- a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs+++ /dev/null@@ -1,65 +0,0 @@-use crate::{- array::ArrayContextExt,- backends::common::{Allocation, Backend},- data_type::DataType,- parameters::ParameterTree,- session::types::Error,-};--pub struct RopeBuffers<B: Backend> {- /// [rope_max_sequence_length, rope_dim]- pub cosines: Allocation<B>,- /// [rope_max_sequence_length, rope_dim]- pub sines: Allocation<B>,- max_sequence_length: usize,- dim: usize,- data_type: DataType,-}--impl<B: Backend> RopeBuffers<B> {- pub fn new(- context: &B::Context,- max_sequence_length: usize,- head_dim: usize,- data_type: DataType,- ) -> Self {- let shape = [max_sequence_length, head_dim];- Self {- cosines: context.create_array_uninitialized(&shape, data_type).into_allocation(),- sines: context.create_array_uninitialized(&shape, data_type).into_allocation(),- max_sequence_length,- dim: head_dim,- data_type,- }- }-- pub fn update_data(- &mut self,- parameter_tree: &ParameterTree<B>,- rope_index: usize,- ) -> Result<(), Error> {- let rope_tree = parameter_tree- .subtree(&format!("ropes.{}", rope_index))- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- let cosines_leaf = rope_tree.leaf("cosines").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- let cosines_leaf = cosines_leaf- .validate(&[self.max_sequence_length, self.dim], self.data_type)- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- self.cosines = cosines_leaf.read_allocation().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;-- let sines_leaf = rope_tree.leaf("sines").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- let sines_leaf = sines_leaf- .validate(&[self.max_sequence_length, self.dim], self.data_type)- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- self.sines = sines_leaf.read_allocation().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- Ok(())- }-- pub fn max_sequence_length(&self) -> usize {- self.max_sequence_length- }-- pub fn dim(&self) -> usize {- self.dim- }-}diff --git a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rsdeleted file mode 100644index 4799ea6fa..000000000--- a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs+++ /dev/null@@ -1,96 +0,0 @@-use super::RopeBuffers;-use crate::{- backends::common::Backend,- config::{decoder::DecoderConfig, rope::AnyRoPEConfig},- forward_pass::model_shape::ModelShape,- parameters::ParameterTree,- session::types::Error,-};--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-enum LayerRopeKind {- NoKernel,- Indexed(usize),-}--pub struct SharedBuffers<B: Backend> {- pub rope_buffers: Box<[RopeBuffers<B>]>,- layer_rope_kinds: Box<[LayerRopeKind]>,-}--impl<B: Backend> SharedBuffers<B> {- pub fn new(- context: &B::Context,- decoder_config: &DecoderConfig,- model_shape: &ModelShape,- ) -> Self {- let tf = &decoder_config.transformer_config;-- let mut configs = Vec::<(AnyRoPEConfig, usize)>::new();- let layer_rope_kinds: Box<[LayerRopeKind]> = tf- .layer_configs- .iter()- .map(|layer_config| {- if layer_config.mixer_config.as_attention().is_none() {- return LayerRopeKind::NoKernel;- }- let Some(rope_config) = &layer_config.rope_config else {- return LayerRopeKind::NoKernel;- };- let head_dim = *rope_config.head_dim();- let index = configs- .iter()- .position(|(existing_config, existing_head_dim)| {- existing_config == rope_config && *existing_head_dim == head_dim- })- .unwrap_or_else(|| {- configs.push((rope_config.clone(), head_dim));- configs.len() - 1- });- LayerRopeKind::Indexed(index)- })- .collect();-- let rope_buffers: Box<[RopeBuffers<B>]> = configs- .iter()- .map(|(config, head_dim)| {- RopeBuffers::new(context, *config.max_sequence_length(), *head_dim, model_shape.rope_data_type)- })- .collect();-- Self {- rope_buffers,- layer_rope_kinds,- }- }-- pub fn update_data(- &mut self,- parameter_tree: &ParameterTree<B>,- ) -> Result<(), Error> {- let transformer_tree =- parameter_tree.subtree("transformer").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- self.update_data_from_transformer_tree(&transformer_tree)?;- Ok(())- }-- pub fn update_data_from_transformer_tree(- &mut self,- transformer_tree: &ParameterTree<B>,- ) -> Result<(), Error> {- for (rope_index, rope_buffers) in self.rope_buffers.iter_mut().enumerate() {- rope_buffers.update_data(transformer_tree, rope_index)?;- }- Ok(())- }-- pub fn rope_buffers_for_layer(- &self,- layer_index: usize,- ) -> Option<&RopeBuffers<B>> {- match self.layer_rope_kinds[layer_index] {- LayerRopeKind::NoKernel => None,- LayerRopeKind::Indexed(index) => Some(&self.rope_buffers[index]),- }- }-}diff --git a/crates/backend-uzu/src/forward_pass/token_inputs.rs b/crates/backend-uzu/src/forward_pass/token_inputs.rsindex 49945c030..091f20ec0 100644--- a/crates/backend-uzu/src/forward_pass/token_inputs.rs+++ b/crates/backend-uzu/src/forward_pass/token_inputs.rs@@ -7,13 +7,13 @@ use crate::{backends::common::{Allocation, Backend},data_type::DataType,encodable_block::DecoderArguments,- forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape},};pub struct TokenInputs<B: Backend> {token_ids: Array<B>,token_subtrie_ranges: Option<Array<B>>,- token_positions: Array<B>,+ token_positions: Box<[usize]>,token_parents: Array<B>,}@@ -25,7 +25,6 @@ impl<B: Backend> TokenInputs<B> {token_subtrie_ranges: Option<&[[u32; 3]]>,token_positions: &[usize],token_ids_array: Option<Array<B>>,- token_positions_array: Option<Array<B>>,sampling_start: usize,sampling_length: usize,) -> Self {@@ -36,8 +35,7 @@ impl<B: Backend> TokenInputs<B> {token_ids: token_ids_array.unwrap_or_else(|| Self::init_token_ids(context, token_ids)),token_subtrie_ranges: token_subtrie_ranges.map(|ranges| Self::init_token_subtrie_ranges(context, model_shape, suffix_length, ranges)),- token_positions: token_positions_array- .unwrap_or_else(|| Self::init_token_positions(context, token_positions)),+ token_positions: token_positions.to_vec().into_boxed_slice(),token_parents: Self::init_token_parents(context, token_positions, sampling_start, sampling_length),}}@@ -53,7 +51,7 @@ impl<B: Backend> TokenInputs<B> {Self {token_ids: Self::init_token_ids(context, token_ids),token_subtrie_ranges: None,- token_positions: Self::init_token_positions(context, token_positions),+ token_positions: token_positions.to_vec().into_boxed_slice(),token_parents: Self::init_token_parents(context, token_positions, 0, 0),}}@@ -62,8 +60,8 @@ impl<B: Backend> TokenInputs<B> {self.token_ids.allocation()}- pub fn token_positions(&self) -> &Allocation<B> {- self.token_positions.allocation()+ pub fn token_positions(&self) -> &[usize] {+ &self.token_positions}pub fn token_parents(&self) -> &Allocation<B> {@@ -76,7 +74,6 @@ impl<B: Backend> TokenInputs<B> {pub fn decoder_arguments<'a>(&'a self,- shared_buffers: &'a SharedBuffers<B>,cache_layers: Option<&'a mut CacheLayers<B>>,batch_dim: usize,sampling_start: usize,@@ -87,7 +84,6 @@ impl<B: Backend> TokenInputs<B> {token_positions: self.token_positions(),token_parents: self.token_parents(),token_subtrie_ranges: self.token_subtrie_ranges(),- shared_buffers,cache_layers,batch_dim,sampling_start,@@ -104,14 +100,6 @@ impl<B: Backend> TokenInputs<B> {context.create_array_from(&[token_ids.len()], token_ids)}- fn init_token_positions(- context: &B::Context,- token_positions: &[usize],- ) -> Array<B> {- let positions_i32: Box<[i32]> = token_positions.iter().map(|position| *position as i32).collect();- context.create_array_from(&[token_positions.len()], positions_i32.as_ref())- }-fn init_token_parents(context: &B::Context,token_positions: &[usize],diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rsindex 101b9b5b5..35ca5bfa5 100644--- a/crates/backend-uzu/src/language_model/language_model_generator.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs@@ -465,7 +465,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let batch_dim = task.active_row_count;let sampling_start = task.sampling_start;let sampling_length = task.sampling_length;- let token_inputs = self.build_token_inputs(task, token_ids_array, None);+ let token_inputs = self.build_token_inputs(task, token_ids_array);let mut encoder = Encoder::<B>::new(self.context.context.as_ref()).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;let resources = Self::encode_forward_pass_on(@@ -728,7 +728,7 @@ impl<B: Backend> LanguageModelGenerator<B> {let is_first_decode = task.token_ids.len() == 1;let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);- let token_inputs = self.build_token_inputs(task, None, None);+ let token_inputs = self.build_token_inputs(task, None);if should_capture {self.gpu_capture@@ -796,7 +796,6 @@ impl<B: Backend> LanguageModelGenerator<B> {let mut cache_layers = context.cache_layers.borrow_mut();cache_layers.prepare_for_forward_pass(&context.context, batch_dim);let decoder_arguments = token_inputs.decoder_arguments(- context.shared_buffers.as_ref(),Some(&mut *cache_layers),batch_dim,sampling_start,@@ -812,7 +811,6 @@ impl<B: Backend> LanguageModelGenerator<B> {let mut cache_layers = context.cache_layers.borrow_mut();cache_layers.prepare_for_forward_pass(&context.context, batch_dim);let decoder_arguments = token_inputs.decoder_arguments(- context.shared_buffers.as_ref(),Some(&mut *cache_layers),batch_dim,sampling_start,@@ -854,7 +852,6 @@ impl<B: Backend> LanguageModelGenerator<B> {&self,task: Task,token_ids_array: Option<Array<B>>,- token_positions_array: Option<Array<B>>,) -> TokenInputs<B> {TokenInputs::new_llm(self.context.context.as_ref(),@@ -863,7 +860,6 @@ impl<B: Backend> LanguageModelGenerator<B> {task.token_subtrie_ranges.as_deref(),&task.token_positions,token_ids_array,- token_positions_array,task.sampling_start,task.sampling_length,)diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rsindex b46fb7cfd..dca924e6d 100644--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs@@ -12,7 +12,7 @@ use crate::{config::{decoder::DecoderConfig, model::language_model::LanguageModelConfig},data_type::DataType,encodable_block::{Decoder, KVCacheUpdate, Sampling},- forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape},language_model::rng::PRng,parameters::ParameterLoader,prelude::ContextLength,@@ -96,7 +96,6 @@ pub struct LanguageModelGeneratorContext<B: Backend> {pub context: Rc<B::Context>,pub cache_layers: Rc<RefCell<CacheLayers<B>>>,- pub shared_buffers: Rc<SharedBuffers<B>>,pub model_config: LanguageModelConfig,pub model_shape: ModelShape,@@ -133,10 +132,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {loader.tree().subtree("decoder").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;let model_shape = ModelShape::from_decoder_config(&model_config.decoder_config, DataType::BF16);- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &model_config.decoder_config, &model_shape);- shared_buffers.update_data(&root_loader_view)?;- let shared_buffers = Rc::new(shared_buffers);-let executables = Decoder::new(context.as_ref(), &model_config.decoder_config, &root_loader_view, &model_shape).map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;@@ -171,7 +166,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {Ok(Self {context,cache_layers,- shared_buffers,model_config: model_config.clone(),model_shape,executables,diff --git a/crates/backend-uzu/src/session/tts_session.rs b/crates/backend-uzu/src/session/tts_session.rsindex 0df79dde5..889186467 100644--- a/crates/backend-uzu/src/session/tts_session.rs+++ b/crates/backend-uzu/src/session/tts_session.rs@@ -39,7 +39,7 @@ use crate::{config::{model::tts_model::TTSModelConfig, token_codec::tts_codec::TTSCodecConfig},data_type::DataType,encodable_block::{Decoder, KVCacheUpdate, Sampling as GpuSampling},- forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape},parameters::ParameterLoader,session::{config::{TextDecoderRuntimeConfig, TextSamplingConfig, TtsChunkPolicy, TtsRunConfig, TtsSessionOptions},diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rsindex 93666d948..77a5c7d53 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs@@ -3,13 +3,14 @@ use crate::{array::{Array, ArrayContextExt},backends::common::{Allocation, AsBufferRangeRef},config::decoder::DecoderConfig,- encodable_block::{DecoderArguments, DecoderDecodeInput, LayerArguments, Linear, SamplingInputs},- forward_pass::token_inputs::TokenInputs,+ encodable_block::{+ DecoderArguments, DecoderDecodeInput, LayerArguments, Linear, PrecalculatedRope, SamplingInputs,+ },+ forward_pass::{rope::precalculate_rope, token_inputs::TokenInputs},session::types::TtsModelConfigError,};struct TokenDecoderLoadedModel<B: Backend> {- shared_buffers: Rc<SharedBuffers<B>>,executables: Decoder<B>,readout: Box<dyn Linear<B>>,sampler: GpuSampling<B>,@@ -26,13 +27,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {embedding_subtree: &str,readout_subtree: &str,) -> Result<Self, Error> {- let shared_buffers = TokenDecoderContext::<B>::build_shared_buffers(- context,- decoder_config,- model_shape,- root_loader_view,- transformer_subtree,- )?;let executables = Decoder::new_with_embedding_subtree(context.as_ref(),decoder_config,@@ -58,7 +52,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {<B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;Ok(Self {- shared_buffers,executables,readout,sampler,@@ -70,7 +63,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {struct TokenDecoderContext<B: Backend> {context: Rc<B::Context>,cache_layers: Rc<RefCell<CacheLayers<B>>>,- shared_buffers: Rc<SharedBuffers<B>>,model_shape: ModelShape,decoder_config: Rc<DecoderConfig>,runtime_config: TextDecoderRuntimeConfig,@@ -137,7 +129,6 @@ impl<B: Backend> TokenDecoderContext<B> {Ok(Self {context,cache_layers,- shared_buffers: loaded_model.shared_buffers,model_shape,decoder_config,runtime_config: runtime_config.clone(),@@ -154,21 +145,6 @@ impl<B: Backend> TokenDecoderContext<B> {})}- fn build_shared_buffers(- context: &Rc<B::Context>,- decoder_config: &Rc<DecoderConfig>,- model_shape: &ModelShape,- root_loader_view: &crate::parameters::ParameterTree<B>,- transformer_subtree: &str,- ) -> Result<Rc<SharedBuffers<B>>, Error> {- let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config, model_shape);- let transformer_tree = root_loader_view- .subtree(transformer_subtree)- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;- shared_buffers.update_data_from_transformer_tree(&transformer_tree)?;- Ok(Rc::new(shared_buffers))- }-fn build_async_chain_buffers(context: &Rc<B::Context>,async_chain_capacity: usize,@@ -259,7 +235,6 @@ impl<B: Backend> TokenDecoderRunner<B> {token_ids: &[u64],token_positions: &[usize],token_ids_array: Option<Array<B>>,- token_positions_array: Option<Array<B>>,sampling_start: usize,sampling_length: usize,) -> TokenInputs<B> {@@ -270,7 +245,6 @@ impl<B: Backend> TokenDecoderRunner<B> {None,token_positions,token_ids_array,- token_positions_array,sampling_start,sampling_length,)@@ -288,7 +262,6 @@ impl<B: Backend> TokenDecoderRunner<B> {token_positions: token_inputs.token_positions(),token_parents: token_inputs.token_parents(),token_subtrie_ranges: token_inputs.token_subtrie_ranges(),- shared_buffers: self.ctx.shared_buffers.as_ref(),cache_layers: Some(cache_layers),batch_dim,sampling_start,@@ -378,7 +351,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let token_count = token_ids.len();let positions: Vec<usize> = (self.next_position..self.next_position + token_count).collect();- let token_inputs = self.llm_token_inputs(token_ids, &positions, None, None, 0, 0);+ let token_inputs = self.llm_token_inputs(token_ids, &positions, None, 0, 0);let context = Rc::clone(&self.ctx.context);let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;@@ -585,7 +558,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let seed = self.ctx.async_chain_seeds.as_slice::<u64>()[pass];let sampling_inputs = self.create_sampling_inputs_from_slices(&[seed], token_bitmask.as_deref());let token_inputs =- self.llm_token_inputs(&token_ids, &[self.next_position + pass], next_token_ids.take(), None, 0, 1);+ self.llm_token_inputs(&token_ids, &[self.next_position + pass], next_token_ids.take(), 0, 1);pending_token_inputs.push(token_inputs);let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");@@ -732,15 +705,34 @@ impl<B: Backend> TokenDecoderRunner<B> {{let mut cache_layers = self.ctx.cache_layers.borrow_mut();cache_layers.prepare_for_forward_pass(&self.ctx.context, batch_dim);+ let ropes = self+ .ctx+ .executables+ .rope_configs+ .iter()+ .map(|rope_config| {+ let (sines, cosines) = precalculate_rope(rope_config, token_inputs.token_positions());+ let mut sines_allocation = encoder.allocate_constant(sines.len() * std::mem::size_of::<f32>())?;+ sines_allocation.copyin(sines.as_ref());+ let mut cosines_allocation =+ encoder.allocate_constant(cosines.len() * std::mem::size_of::<f32>())?;+ cosines_allocation.copyin(cosines.as_ref());+ Ok(PrecalculatedRope {+ cosines: cosines_allocation,+ sines: sines_allocation,+ dim: *rope_config.head_dim(),+ })+ })+ .collect::<Result<Box<[_]>, B::Error>>()+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;for layer in self.ctx.executables.layers.iter() {main = layer.encode(LayerArguments {batch_dim,- token_positions: token_inputs.token_positions(),token_parents: token_inputs.token_parents(),token_subtrie_ranges: token_inputs.token_subtrie_ranges(),- rope_buffers: self.ctx.shared_buffers.rope_buffers_for_layer(layer.layer_index),+ rope: layer.rope_kind.get(&ropes),per_layer_inputs: None,sampling_start,sampling_length,@@ -855,7 +847,7 @@ impl<B: Backend> TokenDecoderRunner<B> {&mask[row_start..row_end]});let sampling_inputs = self.create_sampling_inputs_from_slices(sampling_token_seeds, sampling_token_bitmask);- let token_inputs = self.llm_token_inputs(token_ids, positions, None, None, sampling_start, sampling_length);+ let token_inputs = self.llm_token_inputs(token_ids, positions, None, sampling_start, sampling_length);pending_token_inputs.push(token_inputs);let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rsindex b3d55326b..5a0b2c215 100644--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs@@ -322,7 +322,6 @@ impl<B: Backend> TraceValidator<B> {None,&token_positions,None,- None,/*sampling_start=*/ 0,/*sampling_length=*/ suffix_length,);@@ -333,7 +332,6 @@ impl<B: Backend> TraceValidator<B> {{let mut cache_layers = ctx.cache_layers.borrow_mut();let decoder_arguments = token_inputs.decoder_arguments(- ctx.shared_buffers.as_ref(),Some(&mut *cache_layers),suffix_length,/*sampling_start=*/ 0,diff --git a/crates/backend-uzu/tests/unit/kernel/rope_test.rs b/crates/backend-uzu/tests/unit/kernel/rope_test.rsindex 64dfa8c96..6da80abdf 100644--- a/crates/backend-uzu/tests/unit/kernel/rope_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/rope_test.rs@@ -83,18 +83,31 @@ fn get_output<T: ArrayElement + Float, B: Backend>(let total_heads = (input.num_heads + 2 * input.num_groups) as usize;let qkv_len = input.suffix_length as usize * total_heads * input.head_dim as usize;- let cos_sin_len = input.max_sequence_length as usize * input.rope_dim as usize;+ let cos_sin_len = input.suffix_length as usize * input.rope_dim as usize;let queries_len = input.num_heads as usize * input.suffix_length as usize * input.head_dim as usize;let keys_len = input.num_groups as usize * input.suffix_length as usize * input.head_dim as usize;+ let mut cosines = Vec::with_capacity(cos_sin_len);+ let mut sines = Vec::with_capacity(cos_sin_len);+ for token_index in 0..input.suffix_length as usize {+ let raw_position = input.token_positions[token_index] as usize;+ let absolute_position = if raw_position >= input.max_sequence_length as usize {+ 0+ } else {+ raw_position+ };+ let offset = absolute_position * input.rope_dim as usize;+ let end = offset + input.rope_dim as usize;+ cosines.extend_from_slice(&input.cosines[offset..end]);+ sines.extend_from_slice(&input.sines[offset..end]);+ }let qkv_array = if query_only {context.create_array_from(&[queries_len], &input.qkv[..queries_len])} else {context.create_array_from(&[qkv_len], &input.qkv)};- let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines);- let sines_array = context.create_array_from(&[cos_sin_len], &input.sines);- let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions);+ let cosines_array = context.create_array_from(&[cos_sin_len], &cosines);+ let sines_array = context.create_array_from(&[cos_sin_len], &sines);let mut rotated_queries = context.create_array_uninitialized(&[queries_len], T::data_type()).into_allocation();let mut rotated_keys = context.create_array_uninitialized(&[keys_len], T::data_type()).into_allocation();@@ -103,7 +116,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(qkv_array.allocation(),cosines_array.allocation(),sines_array.allocation(),- token_positions_array.allocation(),&mut rotated_queries,if query_only {None::<&mut backend_uzu::backends::common::Allocation<B>>@@ -119,7 +131,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(Some(input.num_groups)},input.suffix_length,- input.max_sequence_length,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();