A high-performance inference engine for AI models
Add LongRoPE config variant (#604)
Dmitry Barakhovb8e766f22cd7
diff --git a/crates/backend-uzu/src/config/rope/longrope.rs b/crates/backend-uzu/src/config/rope/longrope.rsnew file mode 100644index 000000000..403ef4581--- /dev/null+++ b/crates/backend-uzu/src/config/rope/longrope.rs@@ -0,0 +1,9 @@+use proc_macros::uzu_config;++#[uzu_config(super::RoPEConfig)]+pub struct LongRoPEConfig {+ pub short_factor: Box<[f32]>,+ pub long_factor: Box<[f32]>,+ pub original_context_length: usize,+ pub scaling_factor: f32,+}diff --git a/crates/backend-uzu/src/config/rope/mod.rs b/crates/backend-uzu/src/config/rope/mod.rsindex ef12fe9b9..e493f5ab7 100644--- a/crates/backend-uzu/src/config/rope/mod.rs+++ b/crates/backend-uzu/src/config/rope/mod.rs@@ -2,6 +2,7 @@ use proc_macros::uzu_config_abstract;pub mod linear_scaling_rope;pub mod llama_rope;+pub mod longrope;pub mod unscaled_rope;pub mod yarn_rope;@@ -9,7 +10,8 @@ pub mod yarn_rope;unscaled_rope::UnscaledRoPEConfig,llama_rope::LlamaRoPEConfig,yarn_rope::YARNRoPEConfig,- linear_scaling_rope::LinearScalingRoPEConfig+ linear_scaling_rope::LinearScalingRoPEConfig,+ longrope::LongRoPEConfig)]pub struct RoPEConfig {pub base: f32,diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rsindex 2e89c39d2..b873b2d89 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs@@ -20,6 +20,9 @@ impl<B: Backend> PrecalculatedRoPE<B> {let half_dim = head_dim / 2;let attention_scaling_factor = match rope_config {AnyRoPEConfig::YARNRoPEConfig(config) => 0.1 * config.scaling_factor.ln() + 1.0,+ AnyRoPEConfig::LongRoPEConfig(config) if config.scaling_factor > 1.0 => {+ (1.0 + config.scaling_factor.ln() / (config.original_context_length as f32).ln()).sqrt()+ },_ => 1.0,};@@ -75,6 +78,14 @@ impl<B: Backend> PrecalculatedRoPE<B> {let scaled_frequency = inverse_frequency / config.scaling_factor;scaled_frequency * (1.0 - smoothing_factor) + inverse_frequency * smoothing_factor},+ AnyRoPEConfig::LongRoPEConfig(config) => {+ let factors = if config.max_sequence_length > config.original_context_length {+ &config.long_factor+ } else {+ &config.short_factor+ };+ inverse_frequency / factors[pair_index]+ },};for (token_index, token_position) in token_positions.iter().enumerate() {