Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support ring KV caches for noncausal attention (#761)

CCYeh74fa465a4439
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention.rs b/crates/uzu-engine/src/backends/metal/kernel/attention.rs
index 92b80f3be..dfb95d2b6 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention.rs
@@ -143,8 +143,8 @@ impl AttentionGemmCore for AttentionGemmMetalCore {
gqa_factor: self.num_q_heads / self.num_groups,
scale: self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),
q_len: arguments.suffix_length,
- k_len: arguments.state_type.physical_prefix_length() + arguments.suffix_length,
- q_off: arguments.state_type.physical_prefix_length(),
+ k_len: arguments.cache.prefix_len() + arguments.suffix_length,
+ q_off: arguments.cache.prefix_len(),
nq_aligned: 0,
q_rem: 0,
nk: 0,
@@ -167,7 +167,7 @@ impl AttentionGemmCore for AttentionGemmMetalCore {
arguments.values,
&mut output,
params,
- arguments.state_type.ring_params(),
+ arguments.cache.ring_params(),
arguments.trie,
self.sliding_window_size,
arguments.sinks,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs
index 37fb77d5d..bb97d7057 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs
@@ -70,7 +70,7 @@ impl<B: Backend> AttentionFallbackCore<B> {
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let suffix_length = arguments.suffix_length;
- let sequence_length = arguments.state_type.physical_prefix_length() + suffix_length;
+ let sequence_length = arguments.cache.prefix_len() + suffix_length;
let gqa_factor = self.num_q_heads / self.num_groups;
let scale = self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt());
@@ -112,7 +112,7 @@ impl<B: Backend> AttentionFallbackCore<B> {
self.scatter_scores.encode(
&group_scores,
&mut scores,
- arguments.state_type.ring_params(),
+ arguments.cache.ring_params(),
None::<&Allocation<B>>,
self.sliding_window_size,
group_index,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs
index 94622ac1b..1bb3f9278 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs
@@ -5,8 +5,8 @@ use crate::{
},
data_type::DataType,
encodable_block::mixer::attention::{
+ KVCacheView,
core::{fallback::AttentionFallbackCore, single_pass::AttentionSinglePassCore, two_pass::AttentionTwoPassCore},
- state::AttentionStateType,
},
};
@@ -34,7 +34,7 @@ pub struct AttentionCoreEncodeArguments<'a, B: Backend, KT: BufferArg<'a, B>, VT
pub suffix_length: u32,
pub trie: Option<&'a Allocation<B>>,
pub sinks: Option<&'a Allocation<B>>,
- pub state_type: &'a AttentionStateType,
+ pub cache: KVCacheView,
}
pub struct AttentionCores<B: Backend> {
@@ -86,7 +86,7 @@ impl<B: Backend> AttentionCores<B> {
&& let Some(fallback) = &self.fallback
{
fallback.encode(arguments, encoder)
- } else if arguments.state_type.physical_prefix_length() + arguments.suffix_length > 1024 {
+ } else if arguments.cache.prefix_len() + arguments.suffix_length > 1024 {
self.two_pass.encode(arguments, encoder)
} else {
self.single_pass.encode(arguments, encoder)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs
index a2e89b301..bffd0153b 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs
@@ -54,12 +54,12 @@ impl<B: Backend> AttentionSinglePassCore<B> {
arguments.values,
&mut output,
self.num_q_heads / self.num_groups,
- arguments.state_type.physical_prefix_length() + arguments.suffix_length,
+ arguments.cache.prefix_len() + arguments.suffix_length,
self.head_dim,
self.num_groups * self.head_dim,
self.head_dim,
self.num_groups * self.head_dim,
- arguments.state_type.ring_params(),
+ arguments.cache.ring_params(),
self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),
arguments.trie,
self.sliding_window_size,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs
index 305c184dd..ca072829b 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs
@@ -78,12 +78,12 @@ impl<B: Backend> AttentionTwoPassCore<B> {
&mut sums,
&mut maxs,
self.num_q_heads / self.num_groups,
- arguments.state_type.physical_prefix_length() + arguments.suffix_length,
+ arguments.cache.prefix_len() + arguments.suffix_length,
self.head_dim,
self.num_groups * self.head_dim,
self.head_dim,
self.num_groups * self.head_dim,
- arguments.state_type.ring_params(),
+ arguments.cache.ring_params(),
self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),
self.num_q_heads,
arguments.suffix_length,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
index 4d8c641fe..1aef18ff2 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
@@ -24,12 +24,12 @@ use crate::{
utils::maybe_mut::MaybeMut,
};
-pub(crate) mod core;
+pub mod core;
mod mode;
mod qkv_norm;
mod state;
-pub(crate) use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, AttentionStateType};
+pub use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, KVCacheView};
pub mod rope;
@@ -37,8 +37,7 @@ pub struct Attention<B: Backend> {
head_dim: u32,
num_q_heads: u32,
num_kv_heads: Option<u32>,
- is_causal: bool,
- sliding_window_size: Option<u32>,
+ ring_capacity: Option<u32>,
max_rope_length: Option<u32>,
data_type: DataType,
qkv: LinearProjection<B>,
@@ -164,7 +163,9 @@ impl<B: Backend> Attention<B> {
.then(|| parameter_tree.leaf("sinks")?.validate(&[num_q_heads], data_type)?.read_allocation())
.transpose()?;
- let is_kv_cache_ring = is_causal && sliding_window_size.is_some();
+ assert!(sliding_window_size.is_none_or(|size| size > 0), "zero sliding window size");
+ let ring_capacity = sliding_window_size;
+ let is_kv_cache_ring = ring_capacity.is_some();
let flat_core = AttentionCores::new(
AttentionCoreNewArguments {
@@ -219,8 +220,7 @@ impl<B: Backend> Attention<B> {
head_dim,
num_q_heads,
num_kv_heads,
- is_causal,
- sliding_window_size,
+ ring_capacity,
max_rope_length,
data_type,
qkv: LinearProjection {
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
index 1fc1284ea..218cbb124 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
@@ -10,11 +10,8 @@ use crate::{
mixer::{
MixerState,
attention::{
- Attention,
- core::AttentionCoreEncodeArguments,
- qkv_norm::QKVNorm,
- rope::PrecalculatedRoPE,
- state::{AttentionState, AttentionStateType},
+ Attention, KVCacheView, core::AttentionCoreEncodeArguments, qkv_norm::QKVNorm, rope::PrecalculatedRoPE,
+ state::AttentionState,
},
},
},
@@ -63,11 +60,12 @@ impl<B: Backend> Attention<B> {
let mut attention_output = match state {
Some(MaybeMut::Mut(state)) => {
let qkv = self.qkv.project(hidden, batch_dim.size(), encoder)?;
+ let prefix_len = state.view().prefix_len();
let queries = self.prepare_kv_and_queries(
&qkv,
state.keys.as_mut(),
state.values.as_mut(),
- state.state_type.physical_prefix_length(),
+ prefix_len,
self.num_q_heads,
precalculated_rope,
batch_dim.size(),
@@ -104,18 +102,7 @@ impl<B: Backend> Attention<B> {
encoder,
)?;
- // HACK: state_type should be Option.
- let state_type = if self.sliding_window_size.is_some() {
- AttentionStateType::Ring {
- offset: 0,
- length: 0,
- max_length: 0,
- }
- } else {
- AttentionStateType::Full {
- length: 0,
- }
- };
+ let cache = self.ring_capacity.map_or_else(|| KVCacheView::full(0), |_| KVCacheView::ring(0, 0));
self.flat_core.encode(
AttentionCoreEncodeArguments {
@@ -125,7 +112,7 @@ impl<B: Backend> Attention<B> {
suffix_length: batch_dim.size(),
trie: None,
sinks: self.sinks.as_ref(),
- state_type: &state_type,
+ cache,
},
encoder,
)?
@@ -154,11 +141,12 @@ impl<B: Backend> Attention<B> {
if let Some(norm) = &self.qkv.norm {
norm.encode_key_value(&mut key_value, batch_dim, encoder)?;
}
+ let prefix_len = state.view().prefix_len();
self.prepare_kv_and_queries(
&key_value,
state.keys.as_mut(),
state.values.as_mut(),
- state.state_type.physical_prefix_length(),
+ prefix_len,
0,
Some(precalculated_rope),
batch_dim,
@@ -191,7 +179,7 @@ impl<B: Backend> Attention<B> {
suffix_length: batch_dim.size(),
trie: trie.as_ref(),
sinks: self.sinks.as_ref(),
- state_type: &state.state_type,
+ cache: state.view(),
},
encoder,
)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
index ea48bc691..56abbb3c9 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs
@@ -11,55 +11,166 @@ use crate::{
encodable_block::mixer::{MixerState, attention::Attention},
};
-pub(crate) const ATTENTION_SUFFIX_CAPACITY: u32 = 1024; // TODO: remove hardcoded suffix capacity
+pub const ATTENTION_SUFFIX_CAPACITY: u32 = 1024; // TODO: remove hardcoded suffix capacity
-pub enum AttentionStateType {
+#[derive(Clone, Copy)]
+pub struct KVCacheView {
+ prefix_len: u32,
+ ring: Option<RingParams>,
+}
+
+impl KVCacheView {
+ pub fn full(prefix_len: u32) -> Self {
+ Self {
+ prefix_len,
+ ring: None,
+ }
+ }
+
+ pub fn ring(
+ prefix_len: u32,
+ offset: u32,
+ ) -> Self {
+ Self {
+ prefix_len,
+ ring: Some(RingParams {
+ ring_offset: offset,
+ ring_length: prefix_len,
+ }),
+ }
+ }
+
+ pub fn prefix_len(self) -> u32 {
+ self.prefix_len
+ }
+
+ pub fn ring_params(self) -> Option<RingParams> {
+ self.ring
+ }
+}
+
+enum KVCacheState {
Full {
length: u32,
},
Ring {
offset: u32,
length: u32,
- max_length: u32,
+ capacity: u32,
},
}
-impl AttentionStateType {
- pub fn physical_prefix_length(&self) -> u32 {
+impl KVCacheState {
+ fn full() -> Self {
+ Self::Full {
+ length: 0,
+ }
+ }
+
+ fn ring(capacity: u32) -> Self {
+ assert!(capacity > 0, "zero ring capacity");
+ Self::Ring {
+ offset: 0,
+ length: 0,
+ capacity,
+ }
+ }
+
+ pub fn view(&self) -> KVCacheView {
+ match self {
+ Self::Full {
+ length,
+ } => KVCacheView::full(*length),
+ Self::Ring {
+ offset,
+ length,
+ capacity: _,
+ } => KVCacheView::ring(*length, *offset),
+ }
+ }
+
+ fn required_prefix_len(
+ &self,
+ context_length: u32,
+ ) -> u32 {
match self {
Self::Full {
length,
- } => *length,
+ } => {
+ assert_eq!(*length, context_length, "cache length mismatch");
+ context_length
+ },
Self::Ring {
- max_length,
+ length,
+ capacity,
..
- } => *max_length,
+ } => {
+ let required = context_length.min(*capacity);
+ assert_eq!(*length, required, "cache length mismatch");
+ required
+ },
}
}
- pub fn ring_params(&self) -> Option<RingParams> {
- let Self::Ring {
- offset,
- length,
- max_length: _,
- } = self
- else {
- return None;
- };
+ fn accept(
+ &mut self,
+ accepted_indices: &[u32],
+ ) -> Vec<Copy> {
+ assert!(accepted_indices.is_sorted_by(|a, b| a < b), "unsorted accept");
+ let suffix_base = self.view().prefix_len();
- Some(RingParams {
- ring_offset: *offset,
- ring_length: *length,
- })
+ match self {
+ Self::Full {
+ length,
+ } => {
+ let mut copies = Vec::with_capacity(accepted_indices.len());
+ for (destination_index, accepted_index) in accepted_indices.iter().copied().enumerate() {
+ let source = suffix_base + accepted_index;
+ let destination = suffix_base + destination_index as u32;
+ if source != destination {
+ copies.push(Copy {
+ source,
+ destination,
+ });
+ }
+ }
+ *length += accepted_indices.len() as u32;
+ copies
+ },
+ Self::Ring {
+ offset,
+ length,
+ capacity,
+ } => {
+ assert!(accepted_indices.len() as u32 <= *capacity, "accept exceeds ring capacity");
+ let mut copies = Vec::with_capacity(accepted_indices.len());
+ for accepted_index in accepted_indices {
+ let source = suffix_base + *accepted_index;
+ let destination = (*offset + *length) % *capacity;
+ if source != destination {
+ copies.push(Copy {
+ source,
+ destination,
+ });
+ }
+
+ if length < capacity {
+ *length += 1;
+ } else {
+ *offset = (*offset + 1) % *capacity;
+ }
+ }
+ copies
+ },
+ }
}
}
pub struct AttentionState<B: Backend> {
- pub cur_context_length: u32,
pub elements_prepared: u32,
pub element_dim: u32,
pub data_type: DataType,
- pub state_type: AttentionStateType,
+ cache: KVCacheState,
pub is_sparse: bool,
pub keys: Box<dyn Buffer<Backend = B>>,
pub values: Box<dyn Buffer<Backend = B>>,
@@ -67,6 +178,10 @@ pub struct AttentionState<B: Backend> {
}
impl<B: Backend> AttentionState<B> {
+ pub fn view(&self) -> KVCacheView {
+ self.cache.view()
+ }
+
pub fn create_empty(
attention: &Attention<B>,
max_context_length: Option<u32>,
@@ -75,42 +190,25 @@ impl<B: Backend> AttentionState<B> {
if let Some(max_context_length) = max_context_length {
assert!(
attention.max_rope_length.is_none_or(|max_rope_length| max_context_length <= max_rope_length),
- "Attention state max_prefix_elements overflows RoPE"
+ "context exceeds RoPE length"
);
}
let data_type = attention.data_type;
- let max_prefix_elements = if attention.is_causal
- && let Some(sliding_window_size) = attention.sliding_window_size
- {
- sliding_window_size
- } else if let Some(max_context_length) = max_context_length {
+ let max_prefix_elements = attention.ring_capacity.unwrap_or_else(|| {
max_context_length
- } else {
- attention
- .max_rope_length
+ .or(attention.max_rope_length)
.expect("Cannot create full attention state with unlimited length for with no RoPE")
- };
+ });
- let state_type = if attention.is_causal && attention.sliding_window_size.is_some() {
- AttentionStateType::Ring {
- offset: 0,
- length: 0,
- max_length: max_prefix_elements,
- }
- } else {
- AttentionStateType::Full {
- length: 0,
- }
- };
+ let cache = attention.ring_capacity.map_or_else(KVCacheState::full, KVCacheState::ring);
let max_elements = max_prefix_elements + ATTENTION_SUFFIX_CAPACITY;
let element_size = attention.num_kv_heads.unwrap() * attention.head_dim;
let kv_buffer_bytes = size_for_shape(&[max_elements, element_size], data_type);
- let is_ring = matches!(state_type, AttentionStateType::Ring { .. });
- let is_sparse = !is_ring && context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS);
+ let is_sparse = context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS);
let (keys, values): (Box<dyn Buffer<Backend = B>>, Box<dyn Buffer<Backend = B>>) = if is_sparse {
(
@@ -124,11 +222,10 @@ impl<B: Backend> AttentionState<B> {
let kv_cache_update = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(context, data_type)?;
Ok(Self {
- cur_context_length: 0,
elements_prepared: 0,
element_dim: element_size,
data_type,
- state_type,
+ cache,
is_sparse,
keys,
values,
@@ -148,8 +245,8 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
return Ok(());
}
- assert!(suffix_length <= ATTENTION_SUFFIX_CAPACITY, "attention suffix length exceeds hardcoded capacity");
- let elements_required = context_length + suffix_length;
+ assert!(suffix_length <= ATTENTION_SUFFIX_CAPACITY, "suffix exceeds capacity");
+ let elements_required = self.cache.required_prefix_len(context_length) + suffix_length;
let bytes_required = size_for_shape(&[elements_required, self.element_dim], self.data_type);
let bytes_prepared = size_for_shape(&[self.elements_prepared, self.element_dim], self.data_type);
@@ -176,48 +273,7 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
accepted_indices: &[u32],
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- assert!(accepted_indices.is_sorted_by(|a, b| a < b), "invalid accepted indicies");
-
- let copies = match &mut self.state_type {
- AttentionStateType::Full {
- length,
- } => {
- let copies = accepted_indices
- .iter()
- .copied()
- .zip(0u32..)
- .filter(|&(accepted_index, index)| index != accepted_index)
- .map(|(accepted_index, index)| Copy {
- source: *length + accepted_index,
- destination: *length + index,
- })
- .collect::<Vec<Copy>>();
-
- *length += accepted_indices.len() as u32;
-
- copies
- },
- AttentionStateType::Ring {
- offset,
- length,
- max_length,
- } => {
- let mut copies = Vec::new();
- for accepted_index in accepted_indices {
- copies.push(Copy {
- source: *max_length + *accepted_index,
- destination: (*offset + *length) % *max_length,
- });
-
- if length < max_length {
- *length += 1;
- } else {
- *offset = (*offset + 1) % *max_length;
- }
- }
- copies
- },
- };
+ let copies = self.cache.accept(accepted_indices);
for copies_chunk in copies.chunks(B::MAX_INLINE_BYTES / size_of::<Copy>()) {
self.kv_cache_update.encode(
@@ -230,8 +286,10 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {
);
}
- self.cur_context_length += accepted_indices.len() as u32;
-
Ok(())
}
}
+
+#[cfg(test)]
+#[path = "../../../../unit/encodable_block/kv_cache_state_test.rs"]
+mod tests;
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index bda6fcc37..309210ccd 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -18,7 +18,7 @@ use crate::{
encodable_block::{
embedding::{Embedding, EmbeddingError},
linear::{Linear, LinearBlockError},
- mixer::attention::{AttentionStateType, core::AttentionCoreEncodeArguments, rope::PrecalculatedRoPE},
+ mixer::attention::{KVCacheView, core::AttentionCoreEncodeArguments, rope::PrecalculatedRoPE},
mlp::MlpBlockError,
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
weaver_layer::{PreparedPrefixAttention, WeaverLayer},
@@ -325,9 +325,7 @@ impl<B: Backend> Weaver<B> {
} = layer
.encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, encoder)
.map_err(WeaverEncodeError::Backend)?;
- let state_type = AttentionStateType::Full {
- length: 0,
- };
+ let cache = KVCacheView::full(0);
let kv_plane_bytes = size_for_shape(&[depth, self.model_dim], DATA_TYPE);
let attention_output = layer
.prefix_attention
@@ -339,7 +337,7 @@ impl<B: Backend> Weaver<B> {
suffix_length: depth,
trie: None,
sinks: None,
- state_type: &state_type,
+ cache,
},
encoder,
)
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs
index bab360961..bda3b9bf2 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs
@@ -7,7 +7,10 @@ use uzu_engine_macros::uzu_test;
use crate::{
array::ArrayElement,
backends::{
- common::{Allocation, Backend, Context, Encoder, Kernels, kernel::AttentionSinglePassKernel},
+ common::{
+ Allocation, Backend, Context, Encoder, Kernels, gpu_types::ring::RingParams,
+ kernel::AttentionSinglePassKernel,
+ },
cpu::Cpu,
},
data_type::DataType,
@@ -28,6 +31,8 @@ struct Input<T: ArrayElement + Float> {
head_dim: u32,
scale: f32,
do_causal: bool,
+ ring_params: Option<RingParams>,
+ sliding_window_size: Option<u32>,
}
fn get_input<T: ArrayElement + Float>(
@@ -74,6 +79,8 @@ fn get_input<T: ArrayElement + Float>(
head_dim,
scale,
do_causal,
+ ring_params: None,
+ sliding_window_size: None,
}
}
@@ -85,10 +92,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
T::data_type(),
input.head_dim,
false,
- false,
+ input.ring_params.is_some(),
input.do_causal,
false,
- false,
+ input.sliding_window_size.is_some(),
)
.expect("Failed to create AttentionSinglePassKernel");
@@ -111,10 +118,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
input.head_dim,
input.sequence_length * input.head_dim,
input.head_dim,
- None,
+ input.ring_params,
input.scale,
None::<&Allocation<B>>,
- None,
+ input.sliding_window_size,
None::<&Allocation<B>>,
input.num_heads,
input.suffix_length,
@@ -186,6 +193,17 @@ fn test_gqa<T: ArrayElement + Float + Debug + Display>() {
test_internal(&input, &expected);
}
+fn test_noncausal_ring<T: ArrayElement + Float + Debug + Display>() {
+ let mut input = get_input::<T>(8, 2, 8, 4, 64, false);
+ input.ring_params = Some(RingParams {
+ ring_offset: 2,
+ ring_length: 4,
+ });
+ input.sliding_window_size = Some(4);
+ let expected = get_output::<T, Cpu>(&input);
+ test_internal(&input, &expected);
+}
+
fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {
let input = get_input::<T>(4, 4, 8, 2, head_dim, true);
let expected = get_output::<T, Cpu>(&input);
@@ -240,6 +258,11 @@ fn test_gqa_bf16() {
test_gqa::<bf16>();
}
+#[uzu_test]
+fn test_noncausal_ring_bf16() {
+ test_noncausal_ring::<bf16>();
+}
+
// Head dim 128
#[uzu_test]
fn test_head_dim_128_f32() {
diff --git a/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs b/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs
index 90de7b14f..d37dcb66e 100644
--- a/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs
@@ -12,8 +12,8 @@ use crate::{
},
data_type::DataType,
encodable_block::mixer::attention::{
+ KVCacheView,
core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},
- state::AttentionStateType,
},
tests::{
assert::assert_eq_float,
@@ -100,10 +100,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let values_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.values);
let segment_prefix_length = input.sequence_length - input.suffix_length;
- let state_type = AttentionStateType::Full {
- length: segment_prefix_length as u32,
- };
-
let args = AttentionCoreEncodeArguments {
queries: &queries_allocation,
keys: &keys_allocation,
@@ -111,7 +107,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
suffix_length: input.suffix_length as u32,
trie: None,
sinks: None,
- state_type: &state_type,
+ cache: KVCacheView::full(segment_prefix_length as u32),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
diff --git a/crates/uzu-engine/unit/encodable_block/attention_test.rs b/crates/uzu-engine/unit/encodable_block/attention_test.rs
index 7b7624dfa..52981e5ba 100644
--- a/crates/uzu-engine/unit/encodable_block/attention_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/attention_test.rs
@@ -17,8 +17,8 @@ use crate::{
},
data_type::DataType,
encodable_block::mixer::attention::{
+ KVCacheView,
core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
- state::AttentionStateType,
},
tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, submit_encoder},
};
@@ -339,10 +339,6 @@ fn run_gemm_attention(
let value_allocation = create_attention_cache_allocation(values, seq_len, context);
let sinks_allocation = sinks.map(|sinks| create_sinks_allocation(sinks, context));
- let state_type = AttentionStateType::Full {
- length: 0,
- };
-
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
let args = AttentionCoreEncodeArguments {
@@ -352,7 +348,7 @@ fn run_gemm_attention(
suffix_length: seq_len as u32,
trie: None,
sinks: sinks_allocation.as_ref(),
- state_type: &state_type,
+ cache: KVCacheView::full(0),
};
let pooled_output = core.encode(args, &mut encoder)?;
diff --git a/crates/uzu-engine/unit/encodable_block/kv_cache_state_test.rs b/crates/uzu-engine/unit/encodable_block/kv_cache_state_test.rs
new file mode 100644
index 000000000..51cc00745
--- /dev/null
+++ b/crates/uzu-engine/unit/encodable_block/kv_cache_state_test.rs
@@ -0,0 +1,25 @@
+use uzu_engine_macros::uzu_test;
+
+use super::KVCacheState;
+
+fn accept(
+ state: &mut KVCacheState,
+ indices: &[u32],
+) -> Vec<(u32, u32)> {
+ state.accept(indices).into_iter().map(|copy| (copy.source, copy.destination)).collect()
+}
+
+#[uzu_test]
+fn ring_accepts_append_then_wrap() {
+ let mut state = KVCacheState::ring(2);
+
+ assert_eq!(accept(&mut state, &[1]), [(1, 0)]);
+ assert_eq!(state.view().prefix_len(), 1);
+
+ assert!(accept(&mut state, &[0]).is_empty());
+ assert_eq!(state.view().prefix_len(), 2);
+
+ assert_eq!(accept(&mut state, &[0]), [(2, 0)]);
+ assert_eq!(state.view().ring_params().unwrap().ring_offset, 1);
+ assert_eq!(state.required_prefix_len(99), 2);
+}