Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add LongRoPE config variant (#604)

Dmitry Barakhovb8e766f22cd7
diff --git a/crates/backend-uzu/src/config/rope/longrope.rs b/crates/backend-uzu/src/config/rope/longrope.rs
new file mode 100644
index 000000000..403ef4581
--- /dev/null
+++ b/crates/backend-uzu/src/config/rope/longrope.rs
@@ -0,0 +1,9 @@
+use proc_macros::uzu_config;
+
+#[uzu_config(super::RoPEConfig)]
+pub struct LongRoPEConfig {
+ pub short_factor: Box<[f32]>,
+ pub long_factor: Box<[f32]>,
+ pub original_context_length: usize,
+ pub scaling_factor: f32,
+}
diff --git a/crates/backend-uzu/src/config/rope/mod.rs b/crates/backend-uzu/src/config/rope/mod.rs
index ef12fe9b9..e493f5ab7 100644
--- a/crates/backend-uzu/src/config/rope/mod.rs
+++ b/crates/backend-uzu/src/config/rope/mod.rs
@@ -2,6 +2,7 @@ use proc_macros::uzu_config_abstract;
pub mod linear_scaling_rope;
pub mod llama_rope;
+pub mod longrope;
pub mod unscaled_rope;
pub mod yarn_rope;
@@ -9,7 +10,8 @@ pub mod yarn_rope;
unscaled_rope::UnscaledRoPEConfig,
llama_rope::LlamaRoPEConfig,
yarn_rope::YARNRoPEConfig,
- linear_scaling_rope::LinearScalingRoPEConfig
+ linear_scaling_rope::LinearScalingRoPEConfig,
+ longrope::LongRoPEConfig
)]
pub struct RoPEConfig {
pub base: f32,
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs
index 2e89c39d2..b873b2d89 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/rope.rs
@@ -20,6 +20,9 @@ impl<B: Backend> PrecalculatedRoPE<B> {
let half_dim = head_dim / 2;
let attention_scaling_factor = match rope_config {
AnyRoPEConfig::YARNRoPEConfig(config) => 0.1 * config.scaling_factor.ln() + 1.0,
+ AnyRoPEConfig::LongRoPEConfig(config) if config.scaling_factor > 1.0 => {
+ (1.0 + config.scaling_factor.ln() / (config.original_context_length as f32).ln()).sqrt()
+ },
_ => 1.0,
};
@@ -75,6 +78,14 @@ impl<B: Backend> PrecalculatedRoPE<B> {
let scaled_frequency = inverse_frequency / config.scaling_factor;
scaled_frequency * (1.0 - smoothing_factor) + inverse_frequency * smoothing_factor
},
+ AnyRoPEConfig::LongRoPEConfig(config) => {
+ let factors = if config.max_sequence_length > config.original_context_length {
+ &config.long_factor
+ } else {
+ &config.short_factor
+ };
+ inverse_frequency / factors[pair_index]
+ },
};
for (token_index, token_position) in token_positions.iter().enumerate() {