A high-performance inference engine for AI models
Support ring KV caches for noncausal attention (#761)
CCYeh74fa465a4439
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention.rs b/crates/uzu-engine/src/backends/metal/kernel/attention.rsindex 92b80f3be..dfb95d2b6 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention.rs@@ -143,8 +143,8 @@ impl AttentionGemmCore for AttentionGemmMetalCore {gqa_factor: self.num_q_heads / self.num_groups,scale: self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),q_len: arguments.suffix_length,- k_len: arguments.state_type.physical_prefix_length() + arguments.suffix_length,- q_off: arguments.state_type.physical_prefix_length(),+ k_len: arguments.cache.prefix_len() + arguments.suffix_length,+ q_off: arguments.cache.prefix_len(),nq_aligned: 0,q_rem: 0,nk: 0,@@ -167,7 +167,7 @@ impl AttentionGemmCore for AttentionGemmMetalCore {arguments.values,&mut output,params,- arguments.state_type.ring_params(),+ arguments.cache.ring_params(),arguments.trie,self.sliding_window_size,arguments.sinks,diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rsindex 37fb77d5d..bb97d7057 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs@@ -70,7 +70,7 @@ impl<B: Backend> AttentionFallbackCore<B> {encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let suffix_length = arguments.suffix_length;- let sequence_length = arguments.state_type.physical_prefix_length() + suffix_length;+ let sequence_length = arguments.cache.prefix_len() + suffix_length;let gqa_factor = self.num_q_heads / self.num_groups;let scale = self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt());@@ -112,7 +112,7 @@ impl<B: Backend> AttentionFallbackCore<B> {self.scatter_scores.encode(&group_scores,&mut scores,- arguments.state_type.ring_params(),+ arguments.cache.ring_params(),None::<&Allocation<B>>,self.sliding_window_size,group_index,diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rsindex 94622ac1b..1bb3f9278 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs@@ -5,8 +5,8 @@ use crate::{},data_type::DataType,encodable_block::mixer::attention::{+ KVCacheView,core::{fallback::AttentionFallbackCore, single_pass::AttentionSinglePassCore, two_pass::AttentionTwoPassCore},- state::AttentionStateType,},};@@ -34,7 +34,7 @@ pub struct AttentionCoreEncodeArguments<'a, B: Backend, KT: BufferArg<'a, B>, VTpub suffix_length: u32,pub trie: Option<&'a Allocation<B>>,pub sinks: Option<&'a Allocation<B>>,- pub state_type: &'a AttentionStateType,+ pub cache: KVCacheView,}pub struct AttentionCores<B: Backend> {@@ -86,7 +86,7 @@ impl<B: Backend> AttentionCores<B> {&& let Some(fallback) = &self.fallback{fallback.encode(arguments, encoder)- } else if arguments.state_type.physical_prefix_length() + arguments.suffix_length > 1024 {+ } else if arguments.cache.prefix_len() + arguments.suffix_length > 1024 {self.two_pass.encode(arguments, encoder)} else {self.single_pass.encode(arguments, encoder)diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rsindex a2e89b301..bffd0153b 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs@@ -54,12 +54,12 @@ impl<B: Backend> AttentionSinglePassCore<B> {arguments.values,&mut output,self.num_q_heads / self.num_groups,- arguments.state_type.physical_prefix_length() + arguments.suffix_length,+ arguments.cache.prefix_len() + arguments.suffix_length,self.head_dim,self.num_groups * self.head_dim,self.head_dim,self.num_groups * self.head_dim,- arguments.state_type.ring_params(),+ arguments.cache.ring_params(),self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),arguments.trie,self.sliding_window_size,diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rsindex 305c184dd..ca072829b 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs@@ -78,12 +78,12 @@ impl<B: Backend> AttentionTwoPassCore<B> {&mut sums,&mut maxs,self.num_q_heads / self.num_groups,- arguments.state_type.physical_prefix_length() + arguments.suffix_length,+ arguments.cache.prefix_len() + arguments.suffix_length,self.head_dim,self.num_groups * self.head_dim,self.head_dim,self.num_groups * self.head_dim,- arguments.state_type.ring_params(),+ arguments.cache.ring_params(),self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),self.num_q_heads,arguments.suffix_length,diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rsindex 4d8c641fe..1aef18ff2 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs@@ -24,12 +24,12 @@ use crate::{utils::maybe_mut::MaybeMut,};-pub(crate) mod core;+pub mod core;mod mode;mod qkv_norm;mod state;-pub(crate) use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, AttentionStateType};+pub use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, KVCacheView};pub mod rope;@@ -37,8 +37,7 @@ pub struct Attention<B: Backend> {head_dim: u32,num_q_heads: u32,num_kv_heads: Option<u32>,- is_causal: bool,- sliding_window_size: Option<u32>,+ ring_capacity: Option<u32>,max_rope_length: Option<u32>,data_type: DataType,qkv: LinearProjection<B>,@@ -164,7 +163,9 @@ impl<B: Backend> Attention<B> {.then(|| parameter_tree.leaf("sinks")?.validate(&[num_q_heads], data_type)?.read_allocation()).transpose()?;- let is_kv_cache_ring = is_causal && sliding_window_size.is_some();+ assert!(sliding_window_size.is_none_or(|size| size > 0), "zero sliding window size");+ let ring_capacity = sliding_window_size;+ let is_kv_cache_ring = ring_capacity.is_some();let flat_core = AttentionCores::new(AttentionCoreNewArguments {@@ -219,8 +220,7 @@ impl<B: Backend> Attention<B> {head_dim,num_q_heads,num_kv_heads,- is_causal,- sliding_window_size,+ ring_capacity,max_rope_length,data_type,qkv: LinearProjection {diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rsindex 1fc1284ea..218cbb124 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs@@ -10,11 +10,8 @@ use crate::{mixer::{MixerState,attention::{- Attention,- core::AttentionCoreEncodeArguments,- qkv_norm::QKVNorm,- rope::PrecalculatedRoPE,- state::{AttentionState, AttentionStateType},+ Attention, KVCacheView, core::AttentionCoreEncodeArguments, qkv_norm::QKVNorm, rope::PrecalculatedRoPE,+ state::AttentionState,},},},@@ -63,11 +60,12 @@ impl<B: Backend> Attention<B> {let mut attention_output = match state {Some(MaybeMut::Mut(state)) => {let qkv = self.qkv.project(hidden, batch_dim.size(), encoder)?;+ let prefix_len = state.view().prefix_len();let queries = self.prepare_kv_and_queries(&qkv,state.keys.as_mut(),state.values.as_mut(),- state.state_type.physical_prefix_length(),+ prefix_len,self.num_q_heads,precalculated_rope,batch_dim.size(),@@ -104,18 +102,7 @@ impl<B: Backend> Attention<B> {encoder,)?;- // HACK: state_type should be Option.- let state_type = if self.sliding_window_size.is_some() {- AttentionStateType::Ring {- offset: 0,- length: 0,- max_length: 0,- }- } else {- AttentionStateType::Full {- length: 0,- }- };+ let cache = self.ring_capacity.map_or_else(|| KVCacheView::full(0), |_| KVCacheView::ring(0, 0));self.flat_core.encode(AttentionCoreEncodeArguments {@@ -125,7 +112,7 @@ impl<B: Backend> Attention<B> {suffix_length: batch_dim.size(),trie: None,sinks: self.sinks.as_ref(),- state_type: &state_type,+ cache,},encoder,)?@@ -154,11 +141,12 @@ impl<B: Backend> Attention<B> {if let Some(norm) = &self.qkv.norm {norm.encode_key_value(&mut key_value, batch_dim, encoder)?;}+ let prefix_len = state.view().prefix_len();self.prepare_kv_and_queries(&key_value,state.keys.as_mut(),state.values.as_mut(),- state.state_type.physical_prefix_length(),+ prefix_len,0,Some(precalculated_rope),batch_dim,@@ -191,7 +179,7 @@ impl<B: Backend> Attention<B> {suffix_length: batch_dim.size(),trie: trie.as_ref(),sinks: self.sinks.as_ref(),- state_type: &state.state_type,+ cache: state.view(),},encoder,)diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rsindex ea48bc691..56abbb3c9 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/state.rs@@ -11,55 +11,166 @@ use crate::{encodable_block::mixer::{MixerState, attention::Attention},};-pub(crate) const ATTENTION_SUFFIX_CAPACITY: u32 = 1024; // TODO: remove hardcoded suffix capacity+pub const ATTENTION_SUFFIX_CAPACITY: u32 = 1024; // TODO: remove hardcoded suffix capacity-pub enum AttentionStateType {+#[derive(Clone, Copy)]+pub struct KVCacheView {+ prefix_len: u32,+ ring: Option<RingParams>,+}++impl KVCacheView {+ pub fn full(prefix_len: u32) -> Self {+ Self {+ prefix_len,+ ring: None,+ }+ }++ pub fn ring(+ prefix_len: u32,+ offset: u32,+ ) -> Self {+ Self {+ prefix_len,+ ring: Some(RingParams {+ ring_offset: offset,+ ring_length: prefix_len,+ }),+ }+ }++ pub fn prefix_len(self) -> u32 {+ self.prefix_len+ }++ pub fn ring_params(self) -> Option<RingParams> {+ self.ring+ }+}++enum KVCacheState {Full {length: u32,},Ring {offset: u32,length: u32,- max_length: u32,+ capacity: u32,},}-impl AttentionStateType {- pub fn physical_prefix_length(&self) -> u32 {+impl KVCacheState {+ fn full() -> Self {+ Self::Full {+ length: 0,+ }+ }++ fn ring(capacity: u32) -> Self {+ assert!(capacity > 0, "zero ring capacity");+ Self::Ring {+ offset: 0,+ length: 0,+ capacity,+ }+ }++ pub fn view(&self) -> KVCacheView {+ match self {+ Self::Full {+ length,+ } => KVCacheView::full(*length),+ Self::Ring {+ offset,+ length,+ capacity: _,+ } => KVCacheView::ring(*length, *offset),+ }+ }++ fn required_prefix_len(+ &self,+ context_length: u32,+ ) -> u32 {match self {Self::Full {length,- } => *length,+ } => {+ assert_eq!(*length, context_length, "cache length mismatch");+ context_length+ },Self::Ring {- max_length,+ length,+ capacity,..- } => *max_length,+ } => {+ let required = context_length.min(*capacity);+ assert_eq!(*length, required, "cache length mismatch");+ required+ },}}- pub fn ring_params(&self) -> Option<RingParams> {- let Self::Ring {- offset,- length,- max_length: _,- } = self- else {- return None;- };+ fn accept(+ &mut self,+ accepted_indices: &[u32],+ ) -> Vec<Copy> {+ assert!(accepted_indices.is_sorted_by(|a, b| a < b), "unsorted accept");+ let suffix_base = self.view().prefix_len();- Some(RingParams {- ring_offset: *offset,- ring_length: *length,- })+ match self {+ Self::Full {+ length,+ } => {+ let mut copies = Vec::with_capacity(accepted_indices.len());+ for (destination_index, accepted_index) in accepted_indices.iter().copied().enumerate() {+ let source = suffix_base + accepted_index;+ let destination = suffix_base + destination_index as u32;+ if source != destination {+ copies.push(Copy {+ source,+ destination,+ });+ }+ }+ *length += accepted_indices.len() as u32;+ copies+ },+ Self::Ring {+ offset,+ length,+ capacity,+ } => {+ assert!(accepted_indices.len() as u32 <= *capacity, "accept exceeds ring capacity");+ let mut copies = Vec::with_capacity(accepted_indices.len());+ for accepted_index in accepted_indices {+ let source = suffix_base + *accepted_index;+ let destination = (*offset + *length) % *capacity;+ if source != destination {+ copies.push(Copy {+ source,+ destination,+ });+ }++ if length < capacity {+ *length += 1;+ } else {+ *offset = (*offset + 1) % *capacity;+ }+ }+ copies+ },+ }}}pub struct AttentionState<B: Backend> {- pub cur_context_length: u32,pub elements_prepared: u32,pub element_dim: u32,pub data_type: DataType,- pub state_type: AttentionStateType,+ cache: KVCacheState,pub is_sparse: bool,pub keys: Box<dyn Buffer<Backend = B>>,pub values: Box<dyn Buffer<Backend = B>>,@@ -67,6 +178,10 @@ pub struct AttentionState<B: Backend> {}impl<B: Backend> AttentionState<B> {+ pub fn view(&self) -> KVCacheView {+ self.cache.view()+ }+pub fn create_empty(attention: &Attention<B>,max_context_length: Option<u32>,@@ -75,42 +190,25 @@ impl<B: Backend> AttentionState<B> {if let Some(max_context_length) = max_context_length {assert!(attention.max_rope_length.is_none_or(|max_rope_length| max_context_length <= max_rope_length),- "Attention state max_prefix_elements overflows RoPE"+ "context exceeds RoPE length");}let data_type = attention.data_type;- let max_prefix_elements = if attention.is_causal- && let Some(sliding_window_size) = attention.sliding_window_size- {- sliding_window_size- } else if let Some(max_context_length) = max_context_length {+ let max_prefix_elements = attention.ring_capacity.unwrap_or_else(|| {max_context_length- } else {- attention- .max_rope_length+ .or(attention.max_rope_length).expect("Cannot create full attention state with unlimited length for with no RoPE")- };+ });- let state_type = if attention.is_causal && attention.sliding_window_size.is_some() {- AttentionStateType::Ring {- offset: 0,- length: 0,- max_length: max_prefix_elements,- }- } else {- AttentionStateType::Full {- length: 0,- }- };+ let cache = attention.ring_capacity.map_or_else(KVCacheState::full, KVCacheState::ring);let max_elements = max_prefix_elements + ATTENTION_SUFFIX_CAPACITY;let element_size = attention.num_kv_heads.unwrap() * attention.head_dim;let kv_buffer_bytes = size_for_shape(&[max_elements, element_size], data_type);- let is_ring = matches!(state_type, AttentionStateType::Ring { .. });- let is_sparse = !is_ring && context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS);+ let is_sparse = context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS);let (keys, values): (Box<dyn Buffer<Backend = B>>, Box<dyn Buffer<Backend = B>>) = if is_sparse {(@@ -124,11 +222,10 @@ impl<B: Backend> AttentionState<B> {let kv_cache_update = <B::Kernels as Kernels>::KVCacheUpdateKernel::new(context, data_type)?;Ok(Self {- cur_context_length: 0,elements_prepared: 0,element_dim: element_size,data_type,- state_type,+ cache,is_sparse,keys,values,@@ -148,8 +245,8 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {return Ok(());}- assert!(suffix_length <= ATTENTION_SUFFIX_CAPACITY, "attention suffix length exceeds hardcoded capacity");- let elements_required = context_length + suffix_length;+ assert!(suffix_length <= ATTENTION_SUFFIX_CAPACITY, "suffix exceeds capacity");+ let elements_required = self.cache.required_prefix_len(context_length) + suffix_length;let bytes_required = size_for_shape(&[elements_required, self.element_dim], self.data_type);let bytes_prepared = size_for_shape(&[self.elements_prepared, self.element_dim], self.data_type);@@ -176,48 +273,7 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {accepted_indices: &[u32],encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- assert!(accepted_indices.is_sorted_by(|a, b| a < b), "invalid accepted indicies");-- let copies = match &mut self.state_type {- AttentionStateType::Full {- length,- } => {- let copies = accepted_indices- .iter()- .copied()- .zip(0u32..)- .filter(|&(accepted_index, index)| index != accepted_index)- .map(|(accepted_index, index)| Copy {- source: *length + accepted_index,- destination: *length + index,- })- .collect::<Vec<Copy>>();-- *length += accepted_indices.len() as u32;-- copies- },- AttentionStateType::Ring {- offset,- length,- max_length,- } => {- let mut copies = Vec::new();- for accepted_index in accepted_indices {- copies.push(Copy {- source: *max_length + *accepted_index,- destination: (*offset + *length) % *max_length,- });-- if length < max_length {- *length += 1;- } else {- *offset = (*offset + 1) % *max_length;- }- }- copies- },- };+ let copies = self.cache.accept(accepted_indices);for copies_chunk in copies.chunks(B::MAX_INLINE_BYTES / size_of::<Copy>()) {self.kv_cache_update.encode(@@ -230,8 +286,10 @@ impl<B: Backend> MixerState<B> for AttentionState<B> {);}- self.cur_context_length += accepted_indices.len() as u32;-Ok(())}}++#[cfg(test)]+#[path = "../../../../unit/encodable_block/kv_cache_state_test.rs"]+mod tests;diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex bda6fcc37..309210ccd 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -18,7 +18,7 @@ use crate::{encodable_block::{embedding::{Embedding, EmbeddingError},linear::{Linear, LinearBlockError},- mixer::attention::{AttentionStateType, core::AttentionCoreEncodeArguments, rope::PrecalculatedRoPE},+ mixer::attention::{KVCacheView, core::AttentionCoreEncodeArguments, rope::PrecalculatedRoPE},mlp::MlpBlockError,normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},weaver_layer::{PreparedPrefixAttention, WeaverLayer},@@ -325,9 +325,7 @@ impl<B: Backend> Weaver<B> {} = layer.encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, encoder).map_err(WeaverEncodeError::Backend)?;- let state_type = AttentionStateType::Full {- length: 0,- };+ let cache = KVCacheView::full(0);let kv_plane_bytes = size_for_shape(&[depth, self.model_dim], DATA_TYPE);let attention_output = layer.prefix_attention@@ -339,7 +337,7 @@ impl<B: Backend> Weaver<B> {suffix_length: depth,trie: None,sinks: None,- state_type: &state_type,+ cache,},encoder,)diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rsindex bab360961..bda3b9bf2 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs@@ -7,7 +7,10 @@ use uzu_engine_macros::uzu_test;use crate::{array::ArrayElement,backends::{- common::{Allocation, Backend, Context, Encoder, Kernels, kernel::AttentionSinglePassKernel},+ common::{+ Allocation, Backend, Context, Encoder, Kernels, gpu_types::ring::RingParams,+ kernel::AttentionSinglePassKernel,+ },cpu::Cpu,},data_type::DataType,@@ -28,6 +31,8 @@ struct Input<T: ArrayElement + Float> {head_dim: u32,scale: f32,do_causal: bool,+ ring_params: Option<RingParams>,+ sliding_window_size: Option<u32>,}fn get_input<T: ArrayElement + Float>(@@ -74,6 +79,8 @@ fn get_input<T: ArrayElement + Float>(head_dim,scale,do_causal,+ ring_params: None,+ sliding_window_size: None,}}@@ -85,10 +92,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {T::data_type(),input.head_dim,false,- false,+ input.ring_params.is_some(),input.do_causal,false,- false,+ input.sliding_window_size.is_some(),).expect("Failed to create AttentionSinglePassKernel");@@ -111,10 +118,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {input.head_dim,input.sequence_length * input.head_dim,input.head_dim,- None,+ input.ring_params,input.scale,None::<&Allocation<B>>,- None,+ input.sliding_window_size,None::<&Allocation<B>>,input.num_heads,input.suffix_length,@@ -186,6 +193,17 @@ fn test_gqa<T: ArrayElement + Float + Debug + Display>() {test_internal(&input, &expected);}+fn test_noncausal_ring<T: ArrayElement + Float + Debug + Display>() {+ let mut input = get_input::<T>(8, 2, 8, 4, 64, false);+ input.ring_params = Some(RingParams {+ ring_offset: 2,+ ring_length: 4,+ });+ input.sliding_window_size = Some(4);+ let expected = get_output::<T, Cpu>(&input);+ test_internal(&input, &expected);+}+fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {let input = get_input::<T>(4, 4, 8, 2, head_dim, true);let expected = get_output::<T, Cpu>(&input);@@ -240,6 +258,11 @@ fn test_gqa_bf16() {test_gqa::<bf16>();}+#[uzu_test]+fn test_noncausal_ring_bf16() {+ test_noncausal_ring::<bf16>();+}+// Head dim 128#[uzu_test]fn test_head_dim_128_f32() {diff --git a/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs b/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rsindex 90de7b14f..d37dcb66e 100644--- a/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs+++ b/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs@@ -12,8 +12,8 @@ use crate::{},data_type::DataType,encodable_block::mixer::attention::{+ KVCacheView,core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},- state::AttentionStateType,},tests::{assert::assert_eq_float,@@ -100,10 +100,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let values_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.values);let segment_prefix_length = input.sequence_length - input.suffix_length;- let state_type = AttentionStateType::Full {- length: segment_prefix_length as u32,- };-let args = AttentionCoreEncodeArguments {queries: &queries_allocation,keys: &keys_allocation,@@ -111,7 +107,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {suffix_length: input.suffix_length as u32,trie: None,sinks: None,- state_type: &state_type,+ cache: KVCacheView::full(segment_prefix_length as u32),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");diff --git a/crates/uzu-engine/unit/encodable_block/attention_test.rs b/crates/uzu-engine/unit/encodable_block/attention_test.rsindex 7b7624dfa..52981e5ba 100644--- a/crates/uzu-engine/unit/encodable_block/attention_test.rs+++ b/crates/uzu-engine/unit/encodable_block/attention_test.rs@@ -17,8 +17,8 @@ use crate::{},data_type::DataType,encodable_block::mixer::attention::{+ KVCacheView,core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},- state::AttentionStateType,},tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, submit_encoder},};@@ -339,10 +339,6 @@ fn run_gemm_attention(let value_allocation = create_attention_cache_allocation(values, seq_len, context);let sinks_allocation = sinks.map(|sinks| create_sinks_allocation(sinks, context));- let state_type = AttentionStateType::Full {- length: 0,- };-let mut encoder = Encoder::new(context).expect("Failed to create encoder");let args = AttentionCoreEncodeArguments {@@ -352,7 +348,7 @@ fn run_gemm_attention(suffix_length: seq_len as u32,trie: None,sinks: sinks_allocation.as_ref(),- state_type: &state_type,+ cache: KVCacheView::full(0),};let pooled_output = core.encode(args, &mut encoder)?;diff --git a/crates/uzu-engine/unit/encodable_block/kv_cache_state_test.rs b/crates/uzu-engine/unit/encodable_block/kv_cache_state_test.rsnew file mode 100644index 000000000..51cc00745--- /dev/null+++ b/crates/uzu-engine/unit/encodable_block/kv_cache_state_test.rs@@ -0,0 +1,25 @@+use uzu_engine_macros::uzu_test;++use super::KVCacheState;++fn accept(+ state: &mut KVCacheState,+ indices: &[u32],+) -> Vec<(u32, u32)> {+ state.accept(indices).into_iter().map(|copy| (copy.source, copy.destination)).collect()+}++#[uzu_test]+fn ring_accepts_append_then_wrap() {+ let mut state = KVCacheState::ring(2);++ assert_eq!(accept(&mut state, &[1]), [(1, 0)]);+ assert_eq!(state.view().prefix_len(), 1);++ assert!(accept(&mut state, &[0]).is_empty());+ assert_eq!(state.view().prefix_len(), 2);++ assert_eq!(accept(&mut state, &[0]), [(2, 0)]);+ assert_eq!(state.view().ring_params().unwrap().ring_offset, 1);+ assert_eq!(state.required_prefix_len(99), 2);+}