Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Per-forward RoPE (#491)

uuuvn4ec41e1fa4c3
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime.rs b/crates/backend-uzu/src/audio/nanocodec/runtime.rs
index 665b81c6b..059da2d5a 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime.rs
@@ -24,6 +24,7 @@ use crate::{
decoder::DecoderConfig,
embedding::{AnyEmbeddingConfig, untied_embedding::UntiedEmbeddingConfig},
normalization::NormalizationConfig,
+ rope::AnyRoPEConfig,
tts::{
TTSConfig,
audio_decoder::{AnyTTSAudioDecoderConfig, descript_audio_codec::DescriptAudioCodecConfig},
@@ -31,7 +32,7 @@ use crate::{
},
data_type::DataType,
encodable_block::{Decoder, LayerExecutables, RMSNorm},
- forward_pass::{model_shape::ModelShape, state::SharedBuffers},
+ forward_pass::model_shape::ModelShape,
parameters::ParameterLoader,
};
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
index df09c9349..8a77982f6 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
@@ -12,7 +12,7 @@ pub(super) fn structured_audio_dtype_key(data_type: DataType) -> u8 {
pub(super) struct StructuredAudioPostModuleRuntime<B: Backend> {
pub(super) context: Rc<B::Context>,
pub(super) model_shape: ModelShape,
- pub(super) shared_buffers: Rc<SharedBuffers<B>>,
+ pub(super) rope_configs: Box<[AnyRoPEConfig]>,
pub(super) layers: Box<[LayerExecutables<B>]>,
pub(super) output_norm: RMSNorm<B>,
pub(super) max_sequence_length: usize,
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index 763848eac..351756762 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -2,8 +2,8 @@ use super::*;
use crate::{
array::{Array, ArrayContextExt},
backends::common::{Allocation, AsBufferRangeRef},
- encodable_block::LayerArguments,
- forward_pass::token_inputs::TokenInputs,
+ encodable_block::{LayerArguments, PrecalculatedRope},
+ forward_pass::{rope::precalculate_rope, token_inputs::TokenInputs},
};
impl StructuredAudioCodecGraph {
@@ -108,17 +108,7 @@ impl StructuredAudioCodecGraph {
let max_sequence_length =
decoder_config.transformer_config.max_sequence_length().unwrap_or(0).max(required_sequence_length.max(1));
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
- {
- let transformer_tree = root_loader_view
- .subtree(transformer_subtree_name)
- .map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;
- shared_buffers.update_data_from_transformer_tree(&transformer_tree).map_err(|err| {
- AudioError::Runtime(format!("failed to update structured audio post_module shared buffers: {err}"))
- })?;
- }
- let shared_buffers = Rc::new(shared_buffers);
- let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(
+ let (rope_configs, layers, output_norm) = Decoder::build_transformer_layers_and_norm(
context.as_ref(),
&decoder_config,
&root_loader_view,
@@ -131,7 +121,7 @@ impl StructuredAudioCodecGraph {
Ok(StructuredAudioPostModuleRuntime {
context,
model_shape,
- shared_buffers,
+ rope_configs,
layers,
output_norm,
max_sequence_length,
@@ -281,16 +271,34 @@ impl StructuredAudioCodecGraph {
let mut shortcut = encoder
.allocate_scratch(main.as_buffer_range_ref().range().len())
.map_err(|err| AudioError::Runtime(format!("post_module shortcut allocation failed: {err}")))?;
+ let ropes = runtime
+ .rope_configs
+ .iter()
+ .map(|rope_config| {
+ let (sines, cosines) = precalculate_rope(rope_config, token_inputs.token_positions());
+ let mut sines_allocation = encoder
+ .allocate_constant(sines.len() * std::mem::size_of::<f32>())
+ .map_err(|err| AudioError::Runtime(format!("post_module rope sines allocation failed: {err}")))?;
+ sines_allocation.copyin(sines.as_ref());
+ let mut cosines_allocation = encoder
+ .allocate_constant(cosines.len() * std::mem::size_of::<f32>())
+ .map_err(|err| AudioError::Runtime(format!("post_module rope cosines allocation failed: {err}")))?;
+ cosines_allocation.copyin(cosines.as_ref());
+ Ok(PrecalculatedRope {
+ cosines: cosines_allocation,
+ sines: sines_allocation,
+ dim: *rope_config.head_dim(),
+ })
+ })
+ .collect::<AudioResult<Box<[_]>>>()?;
for layer in runtime.layers.iter() {
- let rope_buffers = runtime.shared_buffers.rope_buffers_for_layer(layer.layer_index);
main = layer
.encode(
LayerArguments {
batch_dim,
- token_positions: token_inputs.token_positions(),
token_parents: token_inputs.token_parents(),
token_subtrie_ranges: None,
- rope_buffers,
+ rope: layer.rope_kind.get(&ropes),
per_layer_inputs: None,
sampling_start: 0,
sampling_length: batch_dim,
diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
index d317b6cdf..f977f213a 100644
--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs
+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
@@ -21,6 +21,18 @@ pub struct Allocation<B: Backend> {
}
impl<B: Backend> Allocation<B> {
+ pub fn copyin<T: ArrayElement>(
+ &mut self,
+ data: &[T],
+ ) {
+ let buffer_range = self.as_buffer_range_mut();
+ let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
+ let bytes = unsafe {
+ std::slice::from_raw_parts_mut((buffer.cpu_ptr().as_ptr() as *mut u8).add(range.start), range.len())
+ };
+ bytemuck::cast_slice_mut::<u8, T>(bytes).copy_from_slice(data);
+ }
+
pub fn copyout<T: ArrayElement>(&self) -> Vec<T> {
let buffer_range = self.as_buffer_range_ref();
let (buffer, range) = (buffer_range.buffer(), buffer_range.range());
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs b/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs
index 00afc9a55..aecc03c90 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/rope/rope.rs
@@ -33,7 +33,6 @@ pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(
qkv: *const ElementT,
cosines: *const RopeT,
sines: *const RopeT,
- token_positions: *const i32,
rotated_queries: *mut ElementT,
#[optional(!query_only)] rotated_keys: Option<*mut ElementT>,
head_dim: u32,
@@ -41,7 +40,6 @@ pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(
num_heads: u32,
#[optional(!query_only)] num_groups: Option<u32>,
suffix_length: u32,
- max_sequence_length: u32,
#[specialize] query_only: bool,
) {
// Query-only projections carry no K/V heads, i.e. num_groups == 0.
@@ -57,7 +55,6 @@ pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(
let rope_dim = rope_dim as usize;
let num_heads = num_heads as usize;
let suffix_length = suffix_length as usize;
- let max_sequence_length = max_sequence_length as usize;
let half_rope_dim = rope_dim / 2;
let total_heads = num_heads + 2 * num_groups;
@@ -67,17 +64,10 @@ pub fn rope<ElementT: ArrayElement + Float, RopeT: ArrayElement + Float>(
let group_index = head_index / heads_per_group;
for token_index in 0..suffix_length {
- let raw_position = unsafe { *token_positions.add(token_index) } as usize;
- let absolute_position = if raw_position >= max_sequence_length {
- 0
- } else {
- raw_position
- };
-
// Rotated dimensions: apply RoPE to dims 0..rope_dim
for dim_index in 0..rope_dim {
- let cos_val = unsafe { (*cosines.add(absolute_position * rope_dim + dim_index)).to_f32().unwrap() };
- let sin_val = unsafe { (*sines.add(absolute_position * rope_dim + dim_index)).to_f32().unwrap() };
+ let cos_val = unsafe { (*cosines.add(token_index * rope_dim + dim_index)).to_f32().unwrap() };
+ let sin_val = unsafe { (*sines.add(token_index * rope_dim + dim_index)).to_f32().unwrap() };
// Query rotation
let q_val = unsafe {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal
index b596ba19d..670208e68 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal
@@ -25,9 +25,8 @@ VARIANTS(ElementT, float, half, bfloat)
VARIANTS(RopeT, float, half, bfloat)
PUBLIC KERNEL(Rope)(
device const ElementT* qkv, // [suffix_len, total_heads, head_dim]
- device const RopeT* cosines, // [max_seq_len, rope_dim]
- device const RopeT* sines, // [max_seq_len, rope_dim]
- device const int* token_positions, // [suffix_len]
+ device const RopeT* cosines, // [suffix_len, rope_dim]
+ device const RopeT* sines, // [suffix_len, rope_dim]
device ElementT* rotated_queries, // [num_heads, suffix_len, head_dim]
device ElementT* rotated_keys OPTIONAL(!query_only),
constant uint& head_dim,
@@ -35,7 +34,6 @@ PUBLIC KERNEL(Rope)(
constant uint& num_heads,
constant uint& num_groups OPTIONAL(!query_only),
constant uint& suffix_length,
- constant uint& max_sequence_length,
const bool query_only SPECIALIZE,
const uint head_index AXIS(num_heads, 1),
const uint token_index AXIS(suffix_length, 1),
@@ -54,27 +52,24 @@ PUBLIC KERNEL(Rope)(
return;
const uint total_heads = query_only ? num_heads : num_heads + 2 * num_groups;
- const uint raw_position = token_positions[token_index];
- const uint absolute_position =
- raw_position >= max_sequence_length ? 0 : raw_position;
const uint half_rope_dim = rope_dim / 2;
TensorView3D<const ElementT> qkv_tensor_view =
TensorView3D<const ElementT>(qkv)
.shaped(suffix_length, total_heads, head_dim);
TensorView2D<const RopeT> cosines_tensor_view =
- TensorView2D<const RopeT>(cosines).shaped(max_sequence_length, rope_dim);
+ TensorView2D<const RopeT>(cosines).shaped(suffix_length, rope_dim);
TensorView2D<const RopeT> sines_tensor_view =
- TensorView2D<const RopeT>(sines).shaped(max_sequence_length, rope_dim);
+ TensorView2D<const RopeT>(sines).shaped(suffix_length, rope_dim);
TensorView3D<ElementT> rotated_queries_tensor_view =
TensorView3D<ElementT>(rotated_queries)
.shaped(num_heads, suffix_length, head_dim);
if (dimension_index < rope_dim) {
const float cos_val =
- float(cosines_tensor_view(absolute_position, dimension_index));
+ float(cosines_tensor_view(token_index, dimension_index));
const float sin_val =
- float(sines_tensor_view(absolute_position, dimension_index));
+ float(sines_tensor_view(token_index, dimension_index));
rotated_queries_tensor_view(head_index, token_index, dimension_index) =
applyRopeTransform(
qkv_tensor_view,
@@ -104,9 +99,9 @@ PUBLIC KERNEL(Rope)(
const uint key_head_index = num_heads + group_index;
if (dimension_index < rope_dim) {
const float cos_val =
- float(cosines_tensor_view(absolute_position, dimension_index));
+ float(cosines_tensor_view(token_index, dimension_index));
const float sin_val =
- float(sines_tensor_view(absolute_position, dimension_index));
+ float(sines_tensor_view(token_index, dimension_index));
rotated_keys_tensor_view(group_index, token_index, dimension_index) =
applyRopeTransform(
qkv_tensor_view,
diff --git a/crates/backend-uzu/src/classifier/classifier.rs b/crates/backend-uzu/src/classifier/classifier.rs
index 4193a642c..ac8b2a066 100644
--- a/crates/backend-uzu/src/classifier/classifier.rs
+++ b/crates/backend-uzu/src/classifier/classifier.rs
@@ -7,8 +7,8 @@ use crate::{
backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
config::model::classifier_model::ClassifierModelConfig,
data_type::DataType,
- encodable_block::LayerArguments,
- forward_pass::token_inputs::TokenInputs,
+ encodable_block::{LayerArguments, PrecalculatedRope},
+ forward_pass::{rope::precalculate_rope, token_inputs::TokenInputs},
session::types::Error,
};
@@ -134,22 +134,38 @@ impl<B: Backend> Classifier<B> {
let mut shortcut = encoder
.allocate_scratch(main.as_buffer_range_ref().range().len())
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- for (layer_index, layer) in self.context.layers.iter().enumerate() {
- let rope_buffers = self.context.shared_buffers.rope_buffers_for_layer(layer_index);
+ let ropes = self
+ .context
+ .rope_configs
+ .iter()
+ .map(|rope_config| {
+ let (sines, cosines) = precalculate_rope(rope_config, token_inputs.token_positions());
+ let mut sines_allocation = encoder.allocate_constant(sines.len() * std::mem::size_of::<f32>())?;
+ sines_allocation.copyin(sines.as_ref());
+ let mut cosines_allocation = encoder.allocate_constant(cosines.len() * std::mem::size_of::<f32>())?;
+ cosines_allocation.copyin(cosines.as_ref());
+ Ok(PrecalculatedRope {
+ cosines: cosines_allocation,
+ sines: sines_allocation,
+ dim: *rope_config.head_dim(),
+ })
+ })
+ .collect::<Result<Box<[_]>, B::Error>>()
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ for layer in self.context.layers.iter() {
main = layer
.encode(
LayerArguments {
batch_dim,
- token_positions: token_inputs.token_positions(),
token_parents: token_inputs.token_parents(),
token_subtrie_ranges: None,
- rope_buffers,
+ rope: layer.rope_kind.get(&ropes),
per_layer_inputs: None,
sampling_start: 0,
sampling_length: batch_dim,
cache_access: None,
#[cfg(feature = "tracing")]
- trace: trace.as_deref_mut().map(|traces| &mut traces.layer_results[layer_index]),
+ trace: trace.as_deref_mut().map(|traces| &mut traces.layer_results[layer.layer_index]),
},
main,
&mut shortcut,
diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rs
index 81c8492c0..6896a1d75 100644
--- a/crates/backend-uzu/src/classifier/classifier_context.rs
+++ b/crates/backend-uzu/src/classifier/classifier_context.rs
@@ -3,12 +3,13 @@ use std::{fs::File, path::Path, rc::Rc};
use crate::{
backends::common::{Backend, Context},
classifier::ClassifierError,
- config::{decoder::DecoderConfig, model::classifier_model::ClassifierModelConfig},
+ config::{decoder::DecoderConfig, model::classifier_model::ClassifierModelConfig, rope::AnyRoPEConfig},
data_type::DataType,
encodable_block::{
- ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, QkUnpack, Rope,
+ ClassifierLayer, ClassifierPredictionHead, Embedding, LayerRopeKind, Linear, Normalization, Pooling, QkUnpack,
+ Rope,
},
- forward_pass::{model_shape::ModelShape, state::SharedBuffers},
+ forward_pass::model_shape::ModelShape,
parameters::ParameterLoader,
session::types::Error,
};
@@ -16,9 +17,8 @@ use crate::{
pub struct ClassifierContext<B: Backend> {
pub context: Rc<B::Context>,
- pub shared_buffers: Rc<SharedBuffers<B>>,
-
pub model_config: ClassifierModelConfig,
+ pub rope_configs: Box<[AnyRoPEConfig]>,
#[cfg(feature = "tracing")]
pub model_shape: ModelShape,
@@ -57,12 +57,45 @@ impl<B: Backend> ClassifierContext<B> {
let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::BF16);
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
- shared_buffers.update_data(&root_loader_view)?;
- let shared_buffers = Rc::new(shared_buffers);
-
let transformer_tree =
root_loader_view.subtree("transformer").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let mut rope_configs = Vec::<(AnyRoPEConfig, usize)>::new();
+ let layer_rope_kinds: Box<[LayerRopeKind]> = classifier_config
+ .transformer_config
+ .layer_configs
+ .iter()
+ .map(|layer_config| {
+ let Some(rope_config) = &layer_config.rope_config else {
+ return LayerRopeKind::NoKernel;
+ };
+ let head_dim = *rope_config.head_dim();
+ let index = rope_configs
+ .iter()
+ .position(|(existing_config, existing_head_dim)| {
+ existing_config == rope_config && *existing_head_dim == head_dim
+ })
+ .unwrap_or_else(|| {
+ rope_configs.push((rope_config.clone(), head_dim));
+ rope_configs.len() - 1
+ });
+ LayerRopeKind::Indexed(index)
+ })
+ .collect();
+
+ for (rope_index, (rope_config, head_dim)) in rope_configs.iter().enumerate() {
+ let rope_tree = transformer_tree
+ .subtree(&format!("ropes.{}", rope_index))
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let shape = [*rope_config.max_sequence_length(), *head_dim];
+ rope_tree
+ .leaf("cosines")
+ .and_then(|leaf| leaf.validate(&shape, model_shape.rope_data_type))
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ rope_tree
+ .leaf("sines")
+ .and_then(|leaf| leaf.validate(&shape, model_shape.rope_data_type))
+ .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ }
let output_norm_tree =
transformer_tree.subtree("output_norm").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
@@ -105,6 +138,8 @@ impl<B: Backend> ClassifierContext<B> {
context.as_ref(),
&classifier_config.transformer_config,
layer_config,
+ layer_index,
+ layer_rope_kinds[layer_index],
&layer_tree,
rope.clone(),
qk_unpack.clone(),
@@ -207,8 +242,8 @@ impl<B: Backend> ClassifierContext<B> {
Ok(Self {
context,
- shared_buffers,
model_config: model_config.clone(),
+ rope_configs: rope_configs.into_iter().map(|(config, _)| config).collect(),
logits_data_type: model_shape.data_type,
#[cfg(feature = "tracing")]
model_shape,
diff --git a/crates/backend-uzu/src/encodable_block/attention/mod.rs b/crates/backend-uzu/src/encodable_block/attention/mod.rs
index 05f8f0294..cb7211e04 100644
--- a/crates/backend-uzu/src/encodable_block/attention/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/attention/mod.rs
@@ -8,7 +8,7 @@ use gemm::{AttentionGemmArguments, AttentionGemmBlock};
use itertools::iproduct;
use thiserror::Error;
-use super::{Linear, QKVNorm, QkUnpack, Rope, linear::LinearBlockError, qkv_norm::QKVNormError};
+use super::{Linear, PrecalculatedRope, QKVNorm, QkUnpack, Rope, linear::LinearBlockError, qkv_norm::QKVNormError};
use crate::{
array::size_for_shape,
backends::common::{
@@ -26,7 +26,6 @@ use crate::{
forward_pass::{
cache_layers::LayerCacheAccess,
kv_cache_layer::{KVCacheLayer, KVCacheLayerState},
- state::RopeBuffers,
},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -330,9 +329,8 @@ impl<B: Backend> Attention<B> {
pub fn encode(
&self,
- token_positions: &Allocation<B>,
token_subtrie_ranges: Option<&Allocation<B>>,
- rope_buffers: Option<&RopeBuffers<B>>,
+ rope: Option<&PrecalculatedRope<B>>,
cache_access: Option<LayerCacheAccess<B>>,
hidden: Allocation<B>,
suffix_length: usize,
@@ -352,20 +350,10 @@ impl<B: Backend> Attention<B> {
if let Some(qkv_norm) = &self.qkv_norm {
qkv_norm.encode(&mut qkv, suffix_length, encoder)?;
}
- let (queries, rotated_keys) = match rope_buffers {
- Some(rope_buffers) => self.rope.encode(
- &qkv,
- token_positions,
- &rope_buffers.cosines,
- &rope_buffers.sines,
- suffix_length,
- self.num_heads,
- self.num_groups,
- self.head_dim,
- rope_buffers.max_sequence_length(),
- rope_buffers.dim(),
- encoder,
- )?,
+ let (queries, rotated_keys) = match rope {
+ Some(rope) => {
+ self.rope.encode(&qkv, rope, suffix_length, self.num_heads, self.num_groups, self.head_dim, encoder)?
+ },
None => {
let (queries, keys) = self.qk_unpack.encode(
&qkv,
diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
index 168ae5e25..fa9fa3823 100644
--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
@@ -7,8 +7,8 @@ use crate::{
config::{transformer::TransformerConfig, transformer_layer::TransformerLayerConfig},
data_type::DataType,
encodable_block::{
- Attention, AttentionError, LayerArguments, Mlp, MlpBlockError, Normalization, NormalizationError, QkUnpack,
- Rope,
+ Attention, AttentionError, LayerArguments, LayerRopeKind, Mlp, MlpBlockError, Normalization,
+ NormalizationError, QkUnpack, Rope,
},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -34,6 +34,9 @@ pub enum ClassifierLayerError<B: Backend> {
}
pub struct ClassifierLayer<B: Backend> {
+ #[cfg(feature = "tracing")]
+ pub layer_index: usize,
+ pub rope_kind: LayerRopeKind,
pre_attention_norm: Option<Normalization<B>>,
attention: Attention<B>,
post_attention_norm: Option<Normalization<B>>,
@@ -50,6 +53,8 @@ impl<B: Backend> ClassifierLayer<B> {
context: &B::Context,
transformer_config: &TransformerConfig,
layer_config: &TransformerLayerConfig,
+ #[cfg_attr(not(feature = "tracing"), allow(unused_variables))] layer_index: usize,
+ rope_kind: LayerRopeKind,
layer_loader: &ParameterTree<B>,
rope: Rc<Rope<B>>,
qk_unpack: Rc<QkUnpack<B>>,
@@ -137,6 +142,9 @@ impl<B: Backend> ClassifierLayer<B> {
.map_err(ClassifierLayerError::BackendError)?;
Ok(Self {
+ #[cfg(feature = "tracing")]
+ layer_index,
+ rope_kind,
pre_attention_norm,
attention,
post_attention_norm,
@@ -158,9 +166,8 @@ impl<B: Backend> ClassifierLayer<B> {
) -> Result<Allocation<B>, B::Error> {
let LayerArguments {
batch_dim,
- token_positions,
token_subtrie_ranges,
- rope_buffers,
+ rope,
#[cfg(feature = "tracing")]
trace,
..
@@ -187,15 +194,7 @@ impl<B: Backend> ClassifierLayer<B> {
encoder.encode_copy(&main, .., layer_traces.pre_attention_norm.allocation_mut(), ..);
}
- main = self.attention.encode(
- token_positions,
- token_subtrie_ranges,
- rope_buffers,
- None,
- main,
- batch_dim,
- encoder,
- )?;
+ main = self.attention.encode(token_subtrie_ranges, rope, None, main, batch_dim, encoder)?;
#[cfg(feature = "tracing")]
if let Some(layer_traces) = layer_traces.as_deref_mut() {
encoder.encode_copy(&main, .., layer_traces.attention.allocation_mut(), ..);
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index c64d6e70c..d99f02637 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -8,13 +8,14 @@ use thiserror::Error;
use crate::forward_pass::traces::ActivationTrace;
use crate::{
backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
- config::decoder::DecoderConfig,
+ config::{decoder::DecoderConfig, rope::AnyRoPEConfig},
data_type::DataType,
encodable_block::{
- Embedding, LayerArguments, LayerExecutables, PerLayerEmbedding, PostLayerScalar, QkUnpack, RMSNorm,
- RMSNormError, Rope, embedding::EmbeddingError, layer::LayerExecutablesError,
+ Embedding, LayerArguments, LayerExecutables, LayerRopeKind, PerLayerEmbedding, PostLayerScalar,
+ PrecalculatedRope, QkUnpack, RMSNorm, RMSNormError, Rope, embedding::EmbeddingError,
+ layer::LayerExecutablesError,
},
- forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, rope::precalculate_rope},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -37,16 +38,16 @@ pub enum DecoderError<B: Backend> {
/// Full decoder executable with all layers and components.
pub struct Decoder<B: Backend> {
pub embed: Embedding<B>,
+ pub rope_configs: Box<[AnyRoPEConfig]>,
pub layers: Box<[LayerExecutables<B>]>,
pub norm: RMSNorm<B>,
pub per_layer_embedding: Option<PerLayerEmbedding<B>>,
}
pub struct DecoderArguments<'a, B: Backend> {
- pub token_positions: &'a Allocation<B>,
+ pub token_positions: &'a [usize],
pub token_parents: &'a Allocation<B>,
pub token_subtrie_ranges: Option<&'a Allocation<B>>,
- pub shared_buffers: &'a SharedBuffers<B>,
pub cache_layers: Option<&'a mut CacheLayers<B>>,
pub batch_dim: usize,
pub sampling_start: usize,
@@ -79,7 +80,7 @@ impl<B: Backend> Decoder<B> {
model_shape,
)?;
- let (layers, norm) = Self::build_transformer_layers_and_norm(
+ let (rope_configs, layers, norm) = Self::build_transformer_layers_and_norm(
context,
decoder_config,
root_weight_loader,
@@ -93,6 +94,7 @@ impl<B: Backend> Decoder<B> {
Ok(Self {
embed,
+ rope_configs,
layers,
norm,
per_layer_embedding,
@@ -121,7 +123,7 @@ impl<B: Backend> Decoder<B> {
model_shape,
)?;
- let (layers, norm) = Self::build_transformer_layers_and_norm(
+ let (rope_configs, layers, norm) = Self::build_transformer_layers_and_norm(
context,
decoder_config,
root_weight_loader,
@@ -135,6 +137,7 @@ impl<B: Backend> Decoder<B> {
Ok(Self {
embed,
+ rope_configs,
layers,
norm,
per_layer_embedding,
@@ -148,10 +151,42 @@ impl<B: Backend> Decoder<B> {
transformer_subtree: &str,
output_norm_hadamard_factors: Option<Allocation<B>>,
model_shape: &ModelShape,
- ) -> Result<(Box<[LayerExecutables<B>]>, RMSNorm<B>), DecoderError<B>> {
+ ) -> Result<(Box<[AnyRoPEConfig]>, Box<[LayerExecutables<B>]>, RMSNorm<B>), DecoderError<B>> {
let decoder_weight_loader = root_weight_loader.subtree(transformer_subtree)?;
let tf = &decoder_config.transformer_config;
+ let mut rope_configs = Vec::<(AnyRoPEConfig, usize)>::new();
+ let layer_rope_kinds: Box<[LayerRopeKind]> = tf
+ .layer_configs
+ .iter()
+ .map(|layer_config| {
+ if layer_config.mixer_config.as_attention().is_none() {
+ return LayerRopeKind::NoKernel;
+ }
+ let Some(rope_config) = &layer_config.rope_config else {
+ return LayerRopeKind::NoKernel;
+ };
+ let head_dim = *rope_config.head_dim();
+ let index = rope_configs
+ .iter()
+ .position(|(existing_config, existing_head_dim)| {
+ existing_config == rope_config && *existing_head_dim == head_dim
+ })
+ .unwrap_or_else(|| {
+ rope_configs.push((rope_config.clone(), head_dim));
+ rope_configs.len() - 1
+ });
+ LayerRopeKind::Indexed(index)
+ })
+ .collect();
+
+ for (rope_index, (rope_config, head_dim)) in rope_configs.iter().enumerate() {
+ let rope_tree = decoder_weight_loader.subtree(&format!("ropes.{}", rope_index))?;
+ let shape = [*rope_config.max_sequence_length(), *head_dim];
+ rope_tree.leaf("cosines")?.validate(&shape, model_shape.rope_data_type)?;
+ rope_tree.leaf("sines")?.validate(&shape, model_shape.rope_data_type)?;
+ }
+
let rope = Rc::new(Rope::<B>::new(context, model_shape, false).map_err(DecoderError::BackendError)?);
// Built only when some layer projects queries only (reuses an earlier layer's KV cache).
let rope_query_only = if tf.layer_configs.iter().any(|l| l.kv_source_layer_index.is_some()) {
@@ -179,6 +214,7 @@ impl<B: Backend> Decoder<B> {
tf,
layer_config,
layer_index,
+ layer_rope_kinds[layer_index],
&layer_loader,
layer_rope,
&qk_unpack,
@@ -200,7 +236,7 @@ impl<B: Backend> Decoder<B> {
PostLayerScalar::None,
)?;
- Ok((layers.into_boxed_slice(), norm_block))
+ Ok((rope_configs.into_iter().map(|(config, _)| config).collect(), layers.into_boxed_slice(), norm_block))
}
fn build_per_layer_embedding(
@@ -255,7 +291,6 @@ impl<B: Backend> Decoder<B> {
token_positions,
token_parents,
token_subtrie_ranges,
- shared_buffers,
mut cache_layers,
batch_dim,
sampling_start,
@@ -268,9 +303,25 @@ impl<B: Backend> Decoder<B> {
let mut shortcut =
encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(DecoderError::BackendError)?;
+ let ropes = self
+ .rope_configs
+ .iter()
+ .map(|rope_config| {
+ let (sines, cosines) = precalculate_rope(rope_config, token_positions);
+ let mut sines_allocation = encoder.allocate_constant(sines.len() * std::mem::size_of::<f32>())?;
+ sines_allocation.copyin(sines.as_ref());
+ let mut cosines_allocation = encoder.allocate_constant(cosines.len() * std::mem::size_of::<f32>())?;
+ cosines_allocation.copyin(cosines.as_ref());
+ Ok(PrecalculatedRope {
+ cosines: cosines_allocation,
+ sines: sines_allocation,
+ dim: *rope_config.head_dim(),
+ })
+ })
+ .collect::<Result<Box<[_]>, B::Error>>()
+ .map_err(DecoderError::BackendError)?;
for layer in self.layers.iter() {
- let rope_buffers = shared_buffers.rope_buffers_for_layer(layer.layer_index);
#[cfg(feature = "tracing")]
let layer_trace = trace.as_deref_mut().map(|trace| &mut trace.layer_results[layer.layer_index]);
@@ -280,10 +331,9 @@ impl<B: Backend> Decoder<B> {
.encode(
LayerArguments {
batch_dim,
- token_positions,
token_parents,
token_subtrie_ranges,
- rope_buffers,
+ rope: layer.rope_kind.get(&ropes),
per_layer_inputs,
sampling_start,
sampling_length,
@@ -345,7 +395,6 @@ impl<B: Backend> Decoder<B> {
token_positions: args.token_positions,
token_parents: args.token_parents,
token_subtrie_ranges: args.token_subtrie_ranges,
- shared_buffers: args.shared_buffers,
cache_layers: args.cache_layers,
batch_dim: args.batch_dim,
sampling_start,
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index b5981c917..633273c49 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -12,10 +12,10 @@ use crate::{
data_type::DataType,
encodable_block::{
Attention, AttentionError, DeltaNetArguments, DeltaNetMixer, DeltaNetMixerError, MambaArguments, MambaMixer,
- MambaMixerError, Mlp, MlpBlockError, PerLayerEmbeddingProjection, PostLayerScalar, QkUnpack, RMSNorm,
- RMSNormError, Rope, ShortConvArguments, ShortConvMixer, ShortConvMixerError, layer::MixerExecutables,
+ MambaMixerError, Mlp, MlpBlockError, PerLayerEmbeddingProjection, PostLayerScalar, PrecalculatedRope, QkUnpack,
+ RMSNorm, RMSNormError, Rope, ShortConvArguments, ShortConvMixer, ShortConvMixerError, layer::MixerExecutables,
},
- forward_pass::{cache_layers::LayerCacheAccess, state::RopeBuffers},
+ forward_pass::cache_layers::LayerCacheAccess,
parameters::{ParameterLoaderError, ParameterTree},
};
#[cfg(feature = "tracing")]
@@ -55,9 +55,28 @@ pub enum LayerExecutablesError<B: Backend> {
MissingPreMixerNormConfig,
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LayerRopeKind {
+ NoKernel,
+ Indexed(usize),
+}
+
+impl LayerRopeKind {
+ pub fn get<B: Backend>(
+ self,
+ ropes: &[PrecalculatedRope<B>],
+ ) -> Option<&PrecalculatedRope<B>> {
+ match self {
+ LayerRopeKind::NoKernel => None,
+ LayerRopeKind::Indexed(index) => Some(&ropes[index]),
+ }
+ }
+}
+
/// A single decoder layer with all its components.
pub struct LayerExecutables<B: Backend> {
pub layer_index: usize,
+ pub rope_kind: LayerRopeKind,
#[cfg(feature = "tracing")]
pub tensor_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
pub pre_mixer_norm: RMSNorm<B>,
@@ -77,6 +96,7 @@ impl<B: Backend> LayerExecutables<B> {
transformer_config: &TransformerConfig,
layer_config: &TransformerLayerConfig,
layer_index: usize,
+ rope_kind: LayerRopeKind,
parameter_tree: &ParameterTree<B>,
rope: &Rc<Rope<B>>,
qk_unpack: &Rc<QkUnpack<B>>,
@@ -261,6 +281,7 @@ impl<B: Backend> LayerExecutables<B> {
Ok(Self {
layer_index,
+ rope_kind,
#[cfg(feature = "tracing")]
tensor_add,
pre_mixer_norm,
@@ -284,10 +305,9 @@ impl<B: Backend> LayerExecutables<B> {
) -> Result<Allocation<B>, B::Error> {
let LayerArguments {
batch_dim,
- token_positions,
token_parents,
token_subtrie_ranges,
- rope_buffers,
+ rope,
per_layer_inputs,
sampling_start,
sampling_length,
@@ -308,15 +328,7 @@ impl<B: Backend> LayerExecutables<B> {
hidden = match &self.mixer {
MixerExecutables::Attention {
attention,
- } => attention.encode(
- token_positions,
- token_subtrie_ranges,
- rope_buffers,
- cache_access,
- hidden,
- batch_dim,
- encoder,
- )?,
+ } => attention.encode(token_subtrie_ranges, rope, cache_access, hidden, batch_dim, encoder)?,
MixerExecutables::StateSpace {
mixer,
} => {
@@ -437,10 +449,9 @@ impl<B: Backend> LayerExecutables<B> {
pub struct LayerArguments<'a, B: Backend> {
pub batch_dim: usize,
- pub token_positions: &'a Allocation<B>,
pub token_parents: &'a Allocation<B>,
pub token_subtrie_ranges: Option<&'a Allocation<B>>,
- pub rope_buffers: Option<&'a RopeBuffers<B>>,
+ pub rope: Option<&'a PrecalculatedRope<B>>,
pub per_layer_inputs: Option<&'a Allocation<B>>,
pub sampling_start: usize,
pub sampling_length: usize,
diff --git a/crates/backend-uzu/src/encodable_block/layer/mod.rs b/crates/backend-uzu/src/encodable_block/layer/mod.rs
index 4898e3959..a7949b2b8 100644
--- a/crates/backend-uzu/src/encodable_block/layer/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/mod.rs
@@ -3,5 +3,5 @@
mod executables;
mod mixer;
-pub use executables::{LayerArguments, LayerExecutables, LayerExecutablesError};
+pub use executables::{LayerArguments, LayerExecutables, LayerExecutablesError, LayerRopeKind};
use mixer::MixerExecutables;
diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rs
index ea6a68f65..dfb965035 100644
--- a/crates/backend-uzu/src/encodable_block/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mod.rs
@@ -26,7 +26,7 @@ pub use decoder::{Decoder, DecoderArguments, DecoderDecodeInput};
pub use delta_net_mixer::{DeltaNetArguments, DeltaNetMixer, DeltaNetMixerError};
pub use embedding::Embedding;
pub use kv_cache_update::{KVCacheUpdate, KVLayerData};
-pub use layer::{LayerArguments, LayerExecutables};
+pub use layer::{LayerArguments, LayerExecutables, LayerRopeKind};
pub use layer_norm::{LayerNorm, LayerNormError};
pub use linear::Linear;
pub use mamba_mixer::{MambaArguments, MambaMixer, MambaMixerError};
@@ -38,6 +38,6 @@ pub use prediction_head::ClassifierPredictionHead;
pub use qk_unpack::QkUnpack;
pub use qkv_norm::QKVNorm;
pub use rms_norm::{PostLayerScalar, RMSNorm, RMSNormError};
-pub use rope::Rope;
+pub use rope::{PrecalculatedRope, Rope};
pub use sampling::{Sampling, SamplingArguments, SamplingInputs};
pub use short_conv_mixer::{ShortConvArguments, ShortConvMixer, ShortConvMixerError};
diff --git a/crates/backend-uzu/src/encodable_block/rope.rs b/crates/backend-uzu/src/encodable_block/rope.rs
index 472a4607d..94750e750 100644
--- a/crates/backend-uzu/src/encodable_block/rope.rs
+++ b/crates/backend-uzu/src/encodable_block/rope.rs
@@ -16,6 +16,12 @@ pub struct Rope<B: Backend> {
query_only: bool,
}
+pub struct PrecalculatedRope<B: Backend> {
+ pub cosines: Allocation<B>,
+ pub sines: Allocation<B>,
+ pub dim: usize,
+}
+
impl<B: Backend> Rope<B> {
pub fn new(
context: &B::Context,
@@ -37,15 +43,11 @@ impl<B: Backend> Rope<B> {
pub fn encode(
&self,
qkv: &Allocation<B>,
- token_positions: &Allocation<B>,
- cosines: &Allocation<B>,
- sines: &Allocation<B>,
+ rope: &PrecalculatedRope<B>,
suffix_length: usize,
num_heads: usize,
num_groups: usize,
head_dim: usize,
- rope_max_seq_len: usize,
- rope_dim: usize,
encoder: &mut Encoder<B>,
) -> Result<(Allocation<B>, Option<Allocation<B>>), B::Error> {
let mut rotated_queries =
@@ -57,17 +59,15 @@ impl<B: Backend> Rope<B> {
};
self.kernel.encode(
qkv,
- cosines,
- sines,
- token_positions,
+ &rope.cosines,
+ &rope.sines,
&mut rotated_queries,
rotated_keys.as_mut(),
head_dim as u32,
- rope_dim as u32,
+ rope.dim as u32,
num_heads as u32,
(!self.query_only).then_some(num_groups as u32),
suffix_length as u32,
- rope_max_seq_len as u32,
encoder,
);
Ok((rotated_queries, rotated_keys))
diff --git a/crates/backend-uzu/src/forward_pass/mod.rs b/crates/backend-uzu/src/forward_pass/mod.rs
index 3f599a5ec..2bbab2cbb 100644
--- a/crates/backend-uzu/src/forward_pass/mod.rs
+++ b/crates/backend-uzu/src/forward_pass/mod.rs
@@ -1,4 +1,5 @@
pub mod model_shape;
+pub mod rope;
pub mod cache_layers;
pub mod delta_net_layer;
@@ -7,7 +8,5 @@ pub mod short_conv_layer;
pub mod ssm_layer;
pub mod token_inputs;
-pub mod state;
-
#[cfg(feature = "tracing")]
pub mod traces;
diff --git a/crates/backend-uzu/src/forward_pass/rope.rs b/crates/backend-uzu/src/forward_pass/rope.rs
new file mode 100644
index 000000000..af21e1b24
--- /dev/null
+++ b/crates/backend-uzu/src/forward_pass/rope.rs
@@ -0,0 +1,81 @@
+use crate::config::rope::AnyRoPEConfig;
+
+pub fn precalculate_rope(
+ rope_config: &AnyRoPEConfig,
+ token_positions: &[usize],
+) -> (Box<[f32]>, Box<[f32]>) {
+ let head_dim = *rope_config.head_dim();
+ assert!(head_dim > 0 && head_dim.is_multiple_of(2), "RoPE head_dim must be positive and even");
+ let half_dim = head_dim / 2;
+ let attention_scaling_factor = match rope_config {
+ AnyRoPEConfig::YARNRoPEConfig(config) => 0.1 * config.scaling_factor.ln() + 1.0,
+ _ => 1.0,
+ };
+
+ let element_count = token_positions.len() * head_dim;
+ let mut sines = vec![0.0; element_count];
+ let mut cosines = vec![0.0; element_count];
+ for pair_index in 0..half_dim {
+ let channel_index = pair_index * 2;
+ let inverse_frequency = 1.0 / rope_config.base().powf(channel_index as f32 / head_dim as f32);
+ let inverse_frequency = match rope_config {
+ AnyRoPEConfig::UnscaledRoPEConfig(_) => inverse_frequency,
+ AnyRoPEConfig::LinearScalingRoPEConfig(config) => inverse_frequency / config.scaling_factor,
+ AnyRoPEConfig::LlamaRoPEConfig(config) => {
+ let low_frequency_wavelength = config.original_context_length as f32 / config.low_frequency_factor;
+ let high_frequency_wavelength = config.original_context_length as f32 / config.high_frequency_factor;
+ let wavelength = 2.0 * std::f32::consts::PI / inverse_frequency;
+ let scaled_frequency = inverse_frequency / config.scaling_factor;
+
+ if wavelength < high_frequency_wavelength {
+ inverse_frequency
+ } else if wavelength > low_frequency_wavelength {
+ scaled_frequency
+ } else {
+ let smoothing_factor =
+ config.original_context_length as f32 / wavelength - config.low_frequency_factor;
+ let smoothing_factor =
+ smoothing_factor / (config.high_frequency_factor - config.low_frequency_factor);
+ smoothing_factor * inverse_frequency + (1.0 - smoothing_factor) * scaled_frequency
+ }
+ },
+ AnyRoPEConfig::YARNRoPEConfig(config) => {
+ let dim = config.head_dim as f64;
+ let base = config.base as f64;
+ let original_context_length = config.original_context_length as f64;
+ let beta_fast = config.beta_fast as f64;
+ let beta_slow = config.beta_slow as f64;
+ let mut low =
+ dim * (original_context_length / (beta_fast * 2.0 * std::f64::consts::PI)).ln() / (2.0 * base.ln());
+ let mut high =
+ dim * (original_context_length / (beta_slow * 2.0 * std::f64::consts::PI)).ln() / (2.0 * base.ln());
+ if config.truncate {
+ low = low.floor();
+ high = high.ceil();
+ }
+ let low = low.max(0.0) as f32;
+ let mut high = high.min((config.head_dim - 1) as f64) as f32;
+ if low == high {
+ high += 0.001;
+ }
+ let ramp = ((pair_index as f32 - low) / (high - low)).clamp(0.0, 1.0);
+ let smoothing_factor = 1.0 - ramp;
+ let scaled_frequency = inverse_frequency / config.scaling_factor;
+ scaled_frequency * (1.0 - smoothing_factor) + inverse_frequency * smoothing_factor
+ },
+ };
+
+ for (token_index, token_position) in token_positions.iter().enumerate() {
+ let embedding = *token_position as f32 * inverse_frequency;
+ let sine = embedding.sin() * attention_scaling_factor;
+ let cosine = embedding.cos() * attention_scaling_factor;
+ let row_offset = token_index * head_dim;
+ sines[row_offset + pair_index] = sine;
+ sines[row_offset + half_dim + pair_index] = sine;
+ cosines[row_offset + pair_index] = cosine;
+ cosines[row_offset + half_dim + pair_index] = cosine;
+ }
+ }
+
+ (sines.into_boxed_slice(), cosines.into_boxed_slice())
+}
diff --git a/crates/backend-uzu/src/forward_pass/state/mod.rs b/crates/backend-uzu/src/forward_pass/state/mod.rs
deleted file mode 100644
index ada5cd024..000000000
--- a/crates/backend-uzu/src/forward_pass/state/mod.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-mod rope_buffers;
-mod shared_buffers;
-
-pub use rope_buffers::RopeBuffers;
-pub use shared_buffers::SharedBuffers;
diff --git a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
deleted file mode 100644
index c58553d02..000000000
--- a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
+++ /dev/null
@@ -1,65 +0,0 @@
-use crate::{
- array::ArrayContextExt,
- backends::common::{Allocation, Backend},
- data_type::DataType,
- parameters::ParameterTree,
- session::types::Error,
-};
-
-pub struct RopeBuffers<B: Backend> {
- /// [rope_max_sequence_length, rope_dim]
- pub cosines: Allocation<B>,
- /// [rope_max_sequence_length, rope_dim]
- pub sines: Allocation<B>,
- max_sequence_length: usize,
- dim: usize,
- data_type: DataType,
-}
-
-impl<B: Backend> RopeBuffers<B> {
- pub fn new(
- context: &B::Context,
- max_sequence_length: usize,
- head_dim: usize,
- data_type: DataType,
- ) -> Self {
- let shape = [max_sequence_length, head_dim];
- Self {
- cosines: context.create_array_uninitialized(&shape, data_type).into_allocation(),
- sines: context.create_array_uninitialized(&shape, data_type).into_allocation(),
- max_sequence_length,
- dim: head_dim,
- data_type,
- }
- }
-
- pub fn update_data(
- &mut self,
- parameter_tree: &ParameterTree<B>,
- rope_index: usize,
- ) -> Result<(), Error> {
- let rope_tree = parameter_tree
- .subtree(&format!("ropes.{}", rope_index))
- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- let cosines_leaf = rope_tree.leaf("cosines").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- let cosines_leaf = cosines_leaf
- .validate(&[self.max_sequence_length, self.dim], self.data_type)
- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- self.cosines = cosines_leaf.read_allocation().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
-
- let sines_leaf = rope_tree.leaf("sines").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- let sines_leaf = sines_leaf
- .validate(&[self.max_sequence_length, self.dim], self.data_type)
- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- self.sines = sines_leaf.read_allocation().map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- Ok(())
- }
-
- pub fn max_sequence_length(&self) -> usize {
- self.max_sequence_length
- }
-
- pub fn dim(&self) -> usize {
- self.dim
- }
-}
diff --git a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
deleted file mode 100644
index 4799ea6fa..000000000
--- a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
+++ /dev/null
@@ -1,96 +0,0 @@
-use super::RopeBuffers;
-use crate::{
- backends::common::Backend,
- config::{decoder::DecoderConfig, rope::AnyRoPEConfig},
- forward_pass::model_shape::ModelShape,
- parameters::ParameterTree,
- session::types::Error,
-};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum LayerRopeKind {
- NoKernel,
- Indexed(usize),
-}
-
-pub struct SharedBuffers<B: Backend> {
- pub rope_buffers: Box<[RopeBuffers<B>]>,
- layer_rope_kinds: Box<[LayerRopeKind]>,
-}
-
-impl<B: Backend> SharedBuffers<B> {
- pub fn new(
- context: &B::Context,
- decoder_config: &DecoderConfig,
- model_shape: &ModelShape,
- ) -> Self {
- let tf = &decoder_config.transformer_config;
-
- let mut configs = Vec::<(AnyRoPEConfig, usize)>::new();
- let layer_rope_kinds: Box<[LayerRopeKind]> = tf
- .layer_configs
- .iter()
- .map(|layer_config| {
- if layer_config.mixer_config.as_attention().is_none() {
- return LayerRopeKind::NoKernel;
- }
- let Some(rope_config) = &layer_config.rope_config else {
- return LayerRopeKind::NoKernel;
- };
- let head_dim = *rope_config.head_dim();
- let index = configs
- .iter()
- .position(|(existing_config, existing_head_dim)| {
- existing_config == rope_config && *existing_head_dim == head_dim
- })
- .unwrap_or_else(|| {
- configs.push((rope_config.clone(), head_dim));
- configs.len() - 1
- });
- LayerRopeKind::Indexed(index)
- })
- .collect();
-
- let rope_buffers: Box<[RopeBuffers<B>]> = configs
- .iter()
- .map(|(config, head_dim)| {
- RopeBuffers::new(context, *config.max_sequence_length(), *head_dim, model_shape.rope_data_type)
- })
- .collect();
-
- Self {
- rope_buffers,
- layer_rope_kinds,
- }
- }
-
- pub fn update_data(
- &mut self,
- parameter_tree: &ParameterTree<B>,
- ) -> Result<(), Error> {
- let transformer_tree =
- parameter_tree.subtree("transformer").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- self.update_data_from_transformer_tree(&transformer_tree)?;
- Ok(())
- }
-
- pub fn update_data_from_transformer_tree(
- &mut self,
- transformer_tree: &ParameterTree<B>,
- ) -> Result<(), Error> {
- for (rope_index, rope_buffers) in self.rope_buffers.iter_mut().enumerate() {
- rope_buffers.update_data(transformer_tree, rope_index)?;
- }
- Ok(())
- }
-
- pub fn rope_buffers_for_layer(
- &self,
- layer_index: usize,
- ) -> Option<&RopeBuffers<B>> {
- match self.layer_rope_kinds[layer_index] {
- LayerRopeKind::NoKernel => None,
- LayerRopeKind::Indexed(index) => Some(&self.rope_buffers[index]),
- }
- }
-}
diff --git a/crates/backend-uzu/src/forward_pass/token_inputs.rs b/crates/backend-uzu/src/forward_pass/token_inputs.rs
index 49945c030..091f20ec0 100644
--- a/crates/backend-uzu/src/forward_pass/token_inputs.rs
+++ b/crates/backend-uzu/src/forward_pass/token_inputs.rs
@@ -7,13 +7,13 @@ use crate::{
backends::common::{Allocation, Backend},
data_type::DataType,
encodable_block::DecoderArguments,
- forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape},
};
pub struct TokenInputs<B: Backend> {
token_ids: Array<B>,
token_subtrie_ranges: Option<Array<B>>,
- token_positions: Array<B>,
+ token_positions: Box<[usize]>,
token_parents: Array<B>,
}
@@ -25,7 +25,6 @@ impl<B: Backend> TokenInputs<B> {
token_subtrie_ranges: Option<&[[u32; 3]]>,
token_positions: &[usize],
token_ids_array: Option<Array<B>>,
- token_positions_array: Option<Array<B>>,
sampling_start: usize,
sampling_length: usize,
) -> Self {
@@ -36,8 +35,7 @@ impl<B: Backend> TokenInputs<B> {
token_ids: token_ids_array.unwrap_or_else(|| Self::init_token_ids(context, token_ids)),
token_subtrie_ranges: token_subtrie_ranges
.map(|ranges| Self::init_token_subtrie_ranges(context, model_shape, suffix_length, ranges)),
- token_positions: token_positions_array
- .unwrap_or_else(|| Self::init_token_positions(context, token_positions)),
+ token_positions: token_positions.to_vec().into_boxed_slice(),
token_parents: Self::init_token_parents(context, token_positions, sampling_start, sampling_length),
}
}
@@ -53,7 +51,7 @@ impl<B: Backend> TokenInputs<B> {
Self {
token_ids: Self::init_token_ids(context, token_ids),
token_subtrie_ranges: None,
- token_positions: Self::init_token_positions(context, token_positions),
+ token_positions: token_positions.to_vec().into_boxed_slice(),
token_parents: Self::init_token_parents(context, token_positions, 0, 0),
}
}
@@ -62,8 +60,8 @@ impl<B: Backend> TokenInputs<B> {
self.token_ids.allocation()
}
- pub fn token_positions(&self) -> &Allocation<B> {
- self.token_positions.allocation()
+ pub fn token_positions(&self) -> &[usize] {
+ &self.token_positions
}
pub fn token_parents(&self) -> &Allocation<B> {
@@ -76,7 +74,6 @@ impl<B: Backend> TokenInputs<B> {
pub fn decoder_arguments<'a>(
&'a self,
- shared_buffers: &'a SharedBuffers<B>,
cache_layers: Option<&'a mut CacheLayers<B>>,
batch_dim: usize,
sampling_start: usize,
@@ -87,7 +84,6 @@ impl<B: Backend> TokenInputs<B> {
token_positions: self.token_positions(),
token_parents: self.token_parents(),
token_subtrie_ranges: self.token_subtrie_ranges(),
- shared_buffers,
cache_layers,
batch_dim,
sampling_start,
@@ -104,14 +100,6 @@ impl<B: Backend> TokenInputs<B> {
context.create_array_from(&[token_ids.len()], token_ids)
}
- fn init_token_positions(
- context: &B::Context,
- token_positions: &[usize],
- ) -> Array<B> {
- let positions_i32: Box<[i32]> = token_positions.iter().map(|position| *position as i32).collect();
- context.create_array_from(&[token_positions.len()], positions_i32.as_ref())
- }
-
fn init_token_parents(
context: &B::Context,
token_positions: &[usize],
diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rs
index 101b9b5b5..35ca5bfa5 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs
@@ -465,7 +465,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let batch_dim = task.active_row_count;
let sampling_start = task.sampling_start;
let sampling_length = task.sampling_length;
- let token_inputs = self.build_token_inputs(task, token_ids_array, None);
+ let token_inputs = self.build_token_inputs(task, token_ids_array);
let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
.map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
let resources = Self::encode_forward_pass_on(
@@ -728,7 +728,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
let is_first_decode = task.token_ids.len() == 1;
let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
- let token_inputs = self.build_token_inputs(task, None, None);
+ let token_inputs = self.build_token_inputs(task, None);
if should_capture {
self.gpu_capture
@@ -796,7 +796,6 @@ impl<B: Backend> LanguageModelGenerator<B> {
let mut cache_layers = context.cache_layers.borrow_mut();
cache_layers.prepare_for_forward_pass(&context.context, batch_dim);
let decoder_arguments = token_inputs.decoder_arguments(
- context.shared_buffers.as_ref(),
Some(&mut *cache_layers),
batch_dim,
sampling_start,
@@ -812,7 +811,6 @@ impl<B: Backend> LanguageModelGenerator<B> {
let mut cache_layers = context.cache_layers.borrow_mut();
cache_layers.prepare_for_forward_pass(&context.context, batch_dim);
let decoder_arguments = token_inputs.decoder_arguments(
- context.shared_buffers.as_ref(),
Some(&mut *cache_layers),
batch_dim,
sampling_start,
@@ -854,7 +852,6 @@ impl<B: Backend> LanguageModelGenerator<B> {
&self,
task: Task,
token_ids_array: Option<Array<B>>,
- token_positions_array: Option<Array<B>>,
) -> TokenInputs<B> {
TokenInputs::new_llm(
self.context.context.as_ref(),
@@ -863,7 +860,6 @@ impl<B: Backend> LanguageModelGenerator<B> {
task.token_subtrie_ranges.as_deref(),
&task.token_positions,
token_ids_array,
- token_positions_array,
task.sampling_start,
task.sampling_length,
)
diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
index b46fb7cfd..dca924e6d 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
@@ -12,7 +12,7 @@ use crate::{
config::{decoder::DecoderConfig, model::language_model::LanguageModelConfig},
data_type::DataType,
encodable_block::{Decoder, KVCacheUpdate, Sampling},
- forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape},
language_model::rng::PRng,
parameters::ParameterLoader,
prelude::ContextLength,
@@ -96,7 +96,6 @@ pub struct LanguageModelGeneratorContext<B: Backend> {
pub context: Rc<B::Context>,
pub cache_layers: Rc<RefCell<CacheLayers<B>>>,
- pub shared_buffers: Rc<SharedBuffers<B>>,
pub model_config: LanguageModelConfig,
pub model_shape: ModelShape,
@@ -133,10 +132,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
loader.tree().subtree("decoder").map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
let model_shape = ModelShape::from_decoder_config(&model_config.decoder_config, DataType::BF16);
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), &model_config.decoder_config, &model_shape);
- shared_buffers.update_data(&root_loader_view)?;
- let shared_buffers = Rc::new(shared_buffers);
-
let executables = Decoder::new(context.as_ref(), &model_config.decoder_config, &root_loader_view, &model_shape)
.map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;
@@ -171,7 +166,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
Ok(Self {
context,
cache_layers,
- shared_buffers,
model_config: model_config.clone(),
model_shape,
executables,
diff --git a/crates/backend-uzu/src/session/tts_session.rs b/crates/backend-uzu/src/session/tts_session.rs
index 0df79dde5..889186467 100644
--- a/crates/backend-uzu/src/session/tts_session.rs
+++ b/crates/backend-uzu/src/session/tts_session.rs
@@ -39,7 +39,7 @@ use crate::{
config::{model::tts_model::TTSModelConfig, token_codec::tts_codec::TTSCodecConfig},
data_type::DataType,
encodable_block::{Decoder, KVCacheUpdate, Sampling as GpuSampling},
- forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape},
parameters::ParameterLoader,
session::{
config::{TextDecoderRuntimeConfig, TextSamplingConfig, TtsChunkPolicy, TtsRunConfig, TtsSessionOptions},
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
index 93666d948..77a5c7d53 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
@@ -3,13 +3,14 @@ use crate::{
array::{Array, ArrayContextExt},
backends::common::{Allocation, AsBufferRangeRef},
config::decoder::DecoderConfig,
- encodable_block::{DecoderArguments, DecoderDecodeInput, LayerArguments, Linear, SamplingInputs},
- forward_pass::token_inputs::TokenInputs,
+ encodable_block::{
+ DecoderArguments, DecoderDecodeInput, LayerArguments, Linear, PrecalculatedRope, SamplingInputs,
+ },
+ forward_pass::{rope::precalculate_rope, token_inputs::TokenInputs},
session::types::TtsModelConfigError,
};
struct TokenDecoderLoadedModel<B: Backend> {
- shared_buffers: Rc<SharedBuffers<B>>,
executables: Decoder<B>,
readout: Box<dyn Linear<B>>,
sampler: GpuSampling<B>,
@@ -26,13 +27,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
embedding_subtree: &str,
readout_subtree: &str,
) -> Result<Self, Error> {
- let shared_buffers = TokenDecoderContext::<B>::build_shared_buffers(
- context,
- decoder_config,
- model_shape,
- root_loader_view,
- transformer_subtree,
- )?;
let executables = Decoder::new_with_embedding_subtree(
context.as_ref(),
decoder_config,
@@ -58,7 +52,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
<B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;
Ok(Self {
- shared_buffers,
executables,
readout,
sampler,
@@ -70,7 +63,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
struct TokenDecoderContext<B: Backend> {
context: Rc<B::Context>,
cache_layers: Rc<RefCell<CacheLayers<B>>>,
- shared_buffers: Rc<SharedBuffers<B>>,
model_shape: ModelShape,
decoder_config: Rc<DecoderConfig>,
runtime_config: TextDecoderRuntimeConfig,
@@ -137,7 +129,6 @@ impl<B: Backend> TokenDecoderContext<B> {
Ok(Self {
context,
cache_layers,
- shared_buffers: loaded_model.shared_buffers,
model_shape,
decoder_config,
runtime_config: runtime_config.clone(),
@@ -154,21 +145,6 @@ impl<B: Backend> TokenDecoderContext<B> {
})
}
- fn build_shared_buffers(
- context: &Rc<B::Context>,
- decoder_config: &Rc<DecoderConfig>,
- model_shape: &ModelShape,
- root_loader_view: &crate::parameters::ParameterTree<B>,
- transformer_subtree: &str,
- ) -> Result<Rc<SharedBuffers<B>>, Error> {
- let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config, model_shape);
- let transformer_tree = root_loader_view
- .subtree(transformer_subtree)
- .map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
- shared_buffers.update_data_from_transformer_tree(&transformer_tree)?;
- Ok(Rc::new(shared_buffers))
- }
-
fn build_async_chain_buffers(
context: &Rc<B::Context>,
async_chain_capacity: usize,
@@ -259,7 +235,6 @@ impl<B: Backend> TokenDecoderRunner<B> {
token_ids: &[u64],
token_positions: &[usize],
token_ids_array: Option<Array<B>>,
- token_positions_array: Option<Array<B>>,
sampling_start: usize,
sampling_length: usize,
) -> TokenInputs<B> {
@@ -270,7 +245,6 @@ impl<B: Backend> TokenDecoderRunner<B> {
None,
token_positions,
token_ids_array,
- token_positions_array,
sampling_start,
sampling_length,
)
@@ -288,7 +262,6 @@ impl<B: Backend> TokenDecoderRunner<B> {
token_positions: token_inputs.token_positions(),
token_parents: token_inputs.token_parents(),
token_subtrie_ranges: token_inputs.token_subtrie_ranges(),
- shared_buffers: self.ctx.shared_buffers.as_ref(),
cache_layers: Some(cache_layers),
batch_dim,
sampling_start,
@@ -378,7 +351,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let token_count = token_ids.len();
let positions: Vec<usize> = (self.next_position..self.next_position + token_count).collect();
- let token_inputs = self.llm_token_inputs(token_ids, &positions, None, None, 0, 0);
+ let token_inputs = self.llm_token_inputs(token_ids, &positions, None, 0, 0);
let context = Rc::clone(&self.ctx.context);
let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;
@@ -585,7 +558,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let seed = self.ctx.async_chain_seeds.as_slice::<u64>()[pass];
let sampling_inputs = self.create_sampling_inputs_from_slices(&[seed], token_bitmask.as_deref());
let token_inputs =
- self.llm_token_inputs(&token_ids, &[self.next_position + pass], next_token_ids.take(), None, 0, 1);
+ self.llm_token_inputs(&token_ids, &[self.next_position + pass], next_token_ids.take(), 0, 1);
pending_token_inputs.push(token_inputs);
let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");
@@ -732,15 +705,34 @@ impl<B: Backend> TokenDecoderRunner<B> {
{
let mut cache_layers = self.ctx.cache_layers.borrow_mut();
cache_layers.prepare_for_forward_pass(&self.ctx.context, batch_dim);
+ let ropes = self
+ .ctx
+ .executables
+ .rope_configs
+ .iter()
+ .map(|rope_config| {
+ let (sines, cosines) = precalculate_rope(rope_config, token_inputs.token_positions());
+ let mut sines_allocation = encoder.allocate_constant(sines.len() * std::mem::size_of::<f32>())?;
+ sines_allocation.copyin(sines.as_ref());
+ let mut cosines_allocation =
+ encoder.allocate_constant(cosines.len() * std::mem::size_of::<f32>())?;
+ cosines_allocation.copyin(cosines.as_ref());
+ Ok(PrecalculatedRope {
+ cosines: cosines_allocation,
+ sines: sines_allocation,
+ dim: *rope_config.head_dim(),
+ })
+ })
+ .collect::<Result<Box<[_]>, B::Error>>()
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
for layer in self.ctx.executables.layers.iter() {
main = layer
.encode(
LayerArguments {
batch_dim,
- token_positions: token_inputs.token_positions(),
token_parents: token_inputs.token_parents(),
token_subtrie_ranges: token_inputs.token_subtrie_ranges(),
- rope_buffers: self.ctx.shared_buffers.rope_buffers_for_layer(layer.layer_index),
+ rope: layer.rope_kind.get(&ropes),
per_layer_inputs: None,
sampling_start,
sampling_length,
@@ -855,7 +847,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
&mask[row_start..row_end]
});
let sampling_inputs = self.create_sampling_inputs_from_slices(sampling_token_seeds, sampling_token_bitmask);
- let token_inputs = self.llm_token_inputs(token_ids, positions, None, None, sampling_start, sampling_length);
+ let token_inputs = self.llm_token_inputs(token_ids, positions, None, sampling_start, sampling_length);
pending_token_inputs.push(token_inputs);
let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");
diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
index b3d55326b..5a0b2c215 100644
--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
@@ -322,7 +322,6 @@ impl<B: Backend> TraceValidator<B> {
None,
&token_positions,
None,
- None,
/*sampling_start=*/ 0,
/*sampling_length=*/ suffix_length,
);
@@ -333,7 +332,6 @@ impl<B: Backend> TraceValidator<B> {
{
let mut cache_layers = ctx.cache_layers.borrow_mut();
let decoder_arguments = token_inputs.decoder_arguments(
- ctx.shared_buffers.as_ref(),
Some(&mut *cache_layers),
suffix_length,
/*sampling_start=*/ 0,
diff --git a/crates/backend-uzu/tests/unit/kernel/rope_test.rs b/crates/backend-uzu/tests/unit/kernel/rope_test.rs
index 64dfa8c96..6da80abdf 100644
--- a/crates/backend-uzu/tests/unit/kernel/rope_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/rope_test.rs
@@ -83,18 +83,31 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
let total_heads = (input.num_heads + 2 * input.num_groups) as usize;
let qkv_len = input.suffix_length as usize * total_heads * input.head_dim as usize;
- let cos_sin_len = input.max_sequence_length as usize * input.rope_dim as usize;
+ let cos_sin_len = input.suffix_length as usize * input.rope_dim as usize;
let queries_len = input.num_heads as usize * input.suffix_length as usize * input.head_dim as usize;
let keys_len = input.num_groups as usize * input.suffix_length as usize * input.head_dim as usize;
+ let mut cosines = Vec::with_capacity(cos_sin_len);
+ let mut sines = Vec::with_capacity(cos_sin_len);
+ for token_index in 0..input.suffix_length as usize {
+ let raw_position = input.token_positions[token_index] as usize;
+ let absolute_position = if raw_position >= input.max_sequence_length as usize {
+ 0
+ } else {
+ raw_position
+ };
+ let offset = absolute_position * input.rope_dim as usize;
+ let end = offset + input.rope_dim as usize;
+ cosines.extend_from_slice(&input.cosines[offset..end]);
+ sines.extend_from_slice(&input.sines[offset..end]);
+ }
let qkv_array = if query_only {
context.create_array_from(&[queries_len], &input.qkv[..queries_len])
} else {
context.create_array_from(&[qkv_len], &input.qkv)
};
- let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines);
- let sines_array = context.create_array_from(&[cos_sin_len], &input.sines);
- let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions);
+ let cosines_array = context.create_array_from(&[cos_sin_len], &cosines);
+ let sines_array = context.create_array_from(&[cos_sin_len], &sines);
let mut rotated_queries = context.create_array_uninitialized(&[queries_len], T::data_type()).into_allocation();
let mut rotated_keys = context.create_array_uninitialized(&[keys_len], T::data_type()).into_allocation();
@@ -103,7 +116,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
qkv_array.allocation(),
cosines_array.allocation(),
sines_array.allocation(),
- token_positions_array.allocation(),
&mut rotated_queries,
if query_only {
None::<&mut backend_uzu::backends::common::Allocation<B>>
@@ -119,7 +131,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
Some(input.num_groups)
},
input.suffix_length,
- input.max_sequence_length,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();