Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Cross-layer KV cache sharing (#428)

CCYeh153837f0ef99
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index 9093b1569..4b3293b3e 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -314,7 +314,7 @@ impl StructuredAudioCodecGraph {
rope_buffers,
sampling_start: 0,
sampling_length: batch_dim,
- cache_layer: None,
+ cache_access: None,
#[cfg(feature = "tracing")]
trace: None,
},
diff --git a/crates/backend-uzu/src/classifier/classifier.rs b/crates/backend-uzu/src/classifier/classifier.rs
index a9c5d6957..f9ed6ab53 100644
--- a/crates/backend-uzu/src/classifier/classifier.rs
+++ b/crates/backend-uzu/src/classifier/classifier.rs
@@ -147,7 +147,7 @@ impl<B: Backend> Classifier<B> {
rope_buffers,
sampling_start: 0,
sampling_length: batch_dim,
- cache_layer: None,
+ cache_access: None,
#[cfg(feature = "tracing")]
trace: trace.as_deref_mut().map(|traces| &mut traces.layer_results[layer_index]),
},
diff --git a/crates/backend-uzu/src/config/transformer_layer.rs b/crates/backend-uzu/src/config/transformer_layer.rs
index b6510e6ab..12c88ffcb 100644
--- a/crates/backend-uzu/src/config/transformer_layer.rs
+++ b/crates/backend-uzu/src/config/transformer_layer.rs
@@ -15,5 +15,5 @@ pub struct TransformerLayerConfig {
pub hidden_dim: Option<usize>,
pub ple_config: Option<Unsupported>,
pub has_post_layer_scalar: bool,
- pub kv_source_layer: Option<Unsupported>,
+ pub kv_source_layer: Option<usize>,
}
diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rs
index b9152eff9..945528bcb 100644
--- a/crates/backend-uzu/src/encodable_block/attention.rs
+++ b/crates/backend-uzu/src/encodable_block/attention.rs
@@ -19,7 +19,10 @@ use crate::{
},
},
config::AttentionConfig,
- forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},
+ forward_pass::{
+ cache_layers::LayerCacheAccess,
+ kv_cache_layer::KVCacheLayerState,
+ },
};
fn env_gemm_attention_enabled() -> bool {
@@ -60,7 +63,7 @@ pub struct Attention<B: Backend> {
pub struct AttentionArguments<'a, B: Backend> {
pub token_subtrie_ranges: Option<&'a Allocation<B>>,
pub attention_sinks: Option<&'a Allocation<B>>,
- pub kv_cache_layer: Option<&'a mut KVCacheLayer<B>>,
+ pub cache_access: Option<LayerCacheAccess<'a, B>>,
}
impl<B: Backend> Attention<B> {
@@ -200,7 +203,7 @@ impl<B: Backend> Attention<B> {
pub fn encode(
&self,
- mut args: AttentionArguments<B>,
+ args: AttentionArguments<B>,
qkv: &Allocation<B>,
queries: &Allocation<B>,
mut rotated_keys: Allocation<B>,
@@ -213,8 +216,20 @@ impl<B: Backend> Attention<B> {
) -> Result<Allocation<B>, B::Error> {
let is_trie = args.token_subtrie_ranges.is_some();
+ let cache_layer = args.cache_access.as_ref().map(|a| {
+ match a {
+ LayerCacheAccess::Owned {
+ entry,
+ } => entry.as_transformer(),
+ LayerCacheAccess::Shared {
+ source,
+ } => source.as_transformer(),
+ }
+ .expect("attention expects transformer cache")
+ });
+
let (max_sequence_length, segment_prefix_length, ring_params) =
- if let Some(layer) = args.kv_cache_layer.as_deref() {
+ if let Some(layer) = cache_layer {
let max_sequence_length = layer.shape[0];
let ring_params = match layer.state {
KVCacheLayerState::Windowed {
@@ -255,8 +270,7 @@ impl<B: Backend> Attention<B> {
let sinks_allocation = args.attention_sinks;
let trie_allocation = args.token_subtrie_ranges;
- // Get KV cache buffers only if KV cache exists (LLM mode)
- let has_kv_cache = args.kv_cache_layer.is_some();
+ let has_kv_cache = args.cache_access.is_some();
let extracted_values = (!has_kv_cache)
.then(|| {
let mut values =
@@ -298,23 +312,36 @@ impl<B: Backend> Attention<B> {
is_kv_cache_ring,
};
- let (keys, values) = if let Some(layer) = args.kv_cache_layer.as_deref_mut() {
- self.update_kv_cache_kernel.encode(
- Some(&rotated_keys),
- qkv,
- &mut layer.keys,
- &mut layer.values,
- num_groups as u32,
- num_heads as u32,
- head_dim as u32,
- suffix_length as u32,
- segment_prefix_length as u32,
- max_sequence_length as u32,
- encoder,
- );
- (&layer.keys, &layer.values)
- } else {
- (&rotated_keys, extracted_values.as_ref().expect("Missing extracted values for classifier attention"))
+ let (keys, values) = match args.cache_access {
+ Some(LayerCacheAccess::Shared {
+ source,
+ }) => {
+ let source = source.as_transformer().expect("kv_source must be a transformer cache");
+ (&source.keys, &source.values)
+ },
+ Some(LayerCacheAccess::Owned {
+ entry,
+ }) => {
+ let layer = entry.as_transformer_mut().expect("Attention layer expects transformer cache");
+ self.update_kv_cache_kernel.encode(
+ Some(&rotated_keys),
+ qkv,
+ &mut layer.keys,
+ &mut layer.values,
+ num_groups as u32,
+ num_heads as u32,
+ head_dim as u32,
+ suffix_length as u32,
+ segment_prefix_length as u32,
+ max_sequence_length as u32,
+ encoder,
+ );
+ (&layer.keys, &layer.values)
+ },
+ None => (
+ &rotated_keys,
+ extracted_values.as_ref().expect("Missing extracted values for classifier attention"),
+ ),
};
let mut attention_output = self.encode_attention_variant(
diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
index 5a2dfc8b5..8697ac102 100644
--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
@@ -230,7 +230,7 @@ impl<B: Backend> ClassifierLayer<B> {
AttentionArguments {
token_subtrie_ranges,
attention_sinks,
- kv_cache_layer: None,
+ cache_access: None,
},
&qkv,
&queries,
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index 474236279..def9102ae 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -183,7 +183,7 @@ impl<B: Backend> Decoder<B> {
#[cfg(feature = "tracing")]
let layer_trace = trace.as_deref_mut().map(|trace| &mut trace.layer_results[layer.layer_index]);
- let cache_layer = cache_layers.as_deref_mut().map(|cache_layers| &mut cache_layers.data[layer.layer_index]);
+ let cache_access = cache_layers.as_deref_mut().map(|cache_layers| cache_layers.cache_for_layer(layer.layer_index));
main = layer
.encode(
LayerArguments {
@@ -195,7 +195,7 @@ impl<B: Backend> Decoder<B> {
rope_buffers,
sampling_start,
sampling_length,
- cache_layer,
+ cache_access,
#[cfg(feature = "tracing")]
trace: layer_trace,
},
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index 07ed4e5bf..f554f04b6 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -17,7 +17,7 @@ use crate::{
Attention, AttentionArguments, DeltaNetArguments, DeltaNetMixer, Linear, MambaArguments, MambaMixer, Mlp,
PostLayerScalar, QKVNorm, QkUnpack, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,
},
- forward_pass::{cache_layers::CacheLayer, state::RopeBuffers},
+ forward_pass::{cache_layers::LayerCacheAccess, state::RopeBuffers},
parameters::ParameterTree,
};
@@ -330,7 +330,7 @@ impl<B: Backend> LayerExecutables<B> {
rope_buffers,
sampling_start,
sampling_length,
- mut cache_layer,
+ cache_access,
#[cfg(feature = "tracing")]
trace,
} = args;
@@ -389,14 +389,11 @@ impl<B: Backend> LayerExecutables<B> {
)?,
None => qk_unpack.encode(&qkv, batch_dim, *num_heads, *num_groups, *head_dim, encoder)?,
};
- let kv_cache_layer = cache_layer
- .as_deref_mut()
- .map(|layer| layer.as_transformer_mut().expect("Attention layer expects transformer cache"));
let attention_output = attention.encode(
AttentionArguments {
token_subtrie_ranges,
attention_sinks,
- kv_cache_layer,
+ cache_access,
},
&qkv,
&queries,
@@ -413,11 +410,12 @@ impl<B: Backend> LayerExecutables<B> {
MixerExecutables::StateSpace {
mixer,
} => {
- let layer = cache_layer
- .as_deref_mut()
- .expect("State-space layer requires cache state")
- .as_state_space_mut()
- .expect("State-space mixer expects SSM cache layer");
+ let Some(LayerCacheAccess::Owned {
+ entry,
+ }) = cache_access else {
+ panic!("State-space layer requires writable cache state");
+ };
+ let layer = entry.as_state_space_mut().expect("State-space mixer expects SSM cache layer");
mixer.encode(
MambaArguments {
active_row_count: batch_dim,
@@ -430,11 +428,12 @@ impl<B: Backend> LayerExecutables<B> {
MixerExecutables::ShortConv {
mixer,
} => {
- let layer = cache_layer
- .as_deref_mut()
- .expect("ShortConv layer requires cache state")
- .as_short_conv_mut()
- .expect("ShortConv mixer expects ShortConv cache layer");
+ let Some(LayerCacheAccess::Owned {
+ entry,
+ }) = cache_access else {
+ panic!("ShortConv layer requires writable cache state");
+ };
+ let layer = entry.as_short_conv_mut().expect("ShortConv mixer expects ShortConv cache layer");
mixer.encode(
ShortConvArguments {
active_row_count: batch_dim,
@@ -450,11 +449,12 @@ impl<B: Backend> LayerExecutables<B> {
MixerExecutables::DeltaNet {
mixer,
} => {
- let layer = cache_layer
- .as_deref_mut()
- .expect("DeltaNet layer requires cache state")
- .as_delta_net_mut()
- .expect("DeltaNet mixer expects DeltaNet cache layer");
+ let Some(LayerCacheAccess::Owned {
+ entry,
+ }) = cache_access else {
+ panic!("DeltaNet layer requires writable cache state");
+ };
+ let layer = entry.as_delta_net_mut().expect("DeltaNet mixer expects DeltaNet cache layer");
mixer.encode(
DeltaNetArguments {
active_row_count: batch_dim,
@@ -525,7 +525,7 @@ pub struct LayerArguments<'a, B: Backend> {
pub rope_buffers: Option<&'a RopeBuffers<B>>,
pub sampling_start: usize,
pub sampling_length: usize,
- pub cache_layer: Option<&'a mut CacheLayer<B>>,
+ pub cache_access: Option<LayerCacheAccess<'a, B>>,
#[cfg(feature = "tracing")]
pub trace: Option<&'a mut LayerActivationTrace<B>>,
}
diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rs
index 7b904cd03..8eda52178 100644
--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs
+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs
@@ -88,13 +88,38 @@ impl<B: Backend> CacheLayer<B> {
pub struct CacheLayers<B: Backend> {
max_suffix_length: usize,
max_prefix_length: usize,
- pub data: Box<[CacheLayer<B>]>,
+ entries: Box<[CacheLayer<B>]>,
+ bindings: Box<[LayerCacheBinding]>,
}
pub struct CacheLayersSlice<B: Backend> {
pub layers: Vec<CacheLayerSlice<B>>,
}
+#[derive(Clone, Copy)]
+struct CacheEntryIndex {
+ index: usize,
+}
+
+#[derive(Clone, Copy)]
+enum LayerCacheBinding {
+ Owned {
+ entry: CacheEntryIndex,
+ },
+ Shared {
+ source: CacheEntryIndex,
+ },
+}
+
+pub enum LayerCacheAccess<'a, B: Backend> {
+ Owned {
+ entry: &'a mut CacheLayer<B>,
+ },
+ Shared {
+ source: &'a CacheLayer<B>,
+ },
+}
+
impl<B: Backend> CacheLayers<B> {
pub fn new(
context: &B::Context,
@@ -104,10 +129,32 @@ impl<B: Backend> CacheLayers<B> {
) -> Self {
let total_context_length = max_prefix_length.max(max_suffix_length);
- let mut data: Box<[CacheLayer<B>]> = model_shape
- .layer_mixers()
- .iter()
- .map(|mixer| match mixer {
+ let kv_source_layers = model_shape.kv_source_layers();
+
+ let mut entries: Vec<CacheLayer<B>> = Vec::with_capacity(model_shape.layer_mixers().len());
+ let mut bindings: Vec<LayerCacheBinding> = Vec::with_capacity(model_shape.layer_mixers().len());
+ for (layer_index, mixer) in model_shape.layer_mixers().iter().enumerate() {
+ if let Some(source_layer_index) = kv_source_layers.get(layer_index).copied().flatten() {
+ let LayerCacheBinding::Owned {
+ entry: source,
+ } = bindings[source_layer_index] else {
+ panic!("kv_source_layer {source_layer_index} (layer {layer_index}) must be a prior owner");
+ };
+ let geometry = |m: &MixerConfig| {
+ m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size))
+ };
+ assert_eq!(
+ geometry(&model_shape.layer_mixers()[source_layer_index]),
+ geometry(mixer),
+ "KV cache sharing requires identical attention geometry"
+ );
+ bindings.push(LayerCacheBinding::Shared {
+ source,
+ });
+ continue;
+ }
+
+ let layer = match mixer {
MixerConfig::Attention(attn) => {
let sliding_window = attn.sliding_window_size;
let length = sliding_window.unwrap_or(max_prefix_length);
@@ -129,7 +176,7 @@ impl<B: Backend> CacheLayers<B> {
let kv_bytes = size_for_shape(&shape, model_shape.kv_cache_data_type());
CacheLayer::Transformer(KVCacheLayer {
- state: state.clone(),
+ state,
keys: context
.create_allocation(kv_bytes, AllocationType::Global)
.expect("Failed to create kv keys allocation"),
@@ -202,11 +249,17 @@ impl<B: Backend> CacheLayers<B> {
data_type: dtype,
})
},
- })
- .collect();
+ };
+ bindings.push(LayerCacheBinding::Owned {
+ entry: CacheEntryIndex { index: entries.len() },
+ });
+ entries.push(layer);
+ }
+ let mut entries = entries.into_boxed_slice();
+ let bindings = bindings.into_boxed_slice();
let mut encoder: Encoder<B> = Encoder::new(context).expect("Failed to create cache initialization encoder");
- for layer in data.iter_mut() {
+ for layer in entries.iter_mut() {
match layer {
CacheLayer::Transformer(layer) => {
encoder.encode_fill(&mut layer.keys, 0);
@@ -233,16 +286,46 @@ impl<B: Backend> CacheLayers<B> {
Self {
max_suffix_length,
max_prefix_length,
- data,
+ entries,
+ bindings,
}
}
+ pub fn cache_for_layer(
+ &mut self,
+ layer_index: usize,
+ ) -> LayerCacheAccess<'_, B> {
+ match self.bindings[layer_index] {
+ LayerCacheBinding::Owned {
+ entry,
+ } => LayerCacheAccess::Owned {
+ entry: &mut self.entries[entry.index],
+ },
+ LayerCacheBinding::Shared {
+ source,
+ } => LayerCacheAccess::Shared {
+ source: &self.entries[source.index],
+ },
+ }
+ }
+
+ pub fn iter_layers(&self) -> impl Iterator<Item = (usize, &CacheLayer<B>)> {
+ self.bindings.iter().enumerate().filter_map(|(index, binding)| match binding {
+ LayerCacheBinding::Owned {
+ entry,
+ } => Some((index, &self.entries[entry.index])),
+ LayerCacheBinding::Shared {
+ ..
+ } => None,
+ })
+ }
+
pub fn clear(
&mut self,
context: &B::Context,
) {
let mut encoder: Option<Encoder<B>> = None;
- for layer in self.data.iter_mut() {
+ for layer in self.entries.iter_mut() {
match layer {
CacheLayer::Transformer(layer) => match &mut layer.state {
KVCacheLayerState::Full {
@@ -303,7 +386,7 @@ impl<B: Backend> CacheLayers<B> {
kv_cache_update: &KVCacheUpdate<B>,
) {
let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);
- for layer in self.data.iter_mut() {
+ for layer in self.entries.iter_mut() {
if let Some(layer) = layer.as_transformer_mut() {
layer.update_after_acceptance(accepted_suffix_indices, suffix_start, encoder, kv_cache_update);
} else if let Some(layer) = layer.as_short_conv_mut() {
@@ -316,7 +399,7 @@ impl<B: Backend> CacheLayers<B> {
&mut self,
number_of_accepted_tokens: usize,
) {
- for layer in self.data.iter_mut() {
+ for layer in self.entries.iter_mut() {
if let Some(layer) = layer.as_transformer_mut() {
layer.register_accepted_tokens(number_of_accepted_tokens);
}
@@ -328,8 +411,8 @@ impl<B: Backend> CacheLayers<B> {
context: &B::Context,
range: std::ops::Range<usize>,
) -> Option<CacheLayersSlice<B>> {
- let mut layers = Vec::with_capacity(self.data.len());
- for layer in self.data.iter() {
+ let mut layers = Vec::with_capacity(self.entries.len());
+ for layer in self.entries.iter() {
match layer {
CacheLayer::Transformer(kv) => {
let Some(slice) = kv.slice(context, range.clone()) else {
@@ -353,7 +436,7 @@ impl<B: Backend> CacheLayers<B> {
slice: &CacheLayersSlice<B>,
range: Option<std::ops::Range<usize>>,
) {
- for (layer, snapshot) in self.data.iter_mut().zip(slice.layers.iter()) {
+ for (layer, snapshot) in self.entries.iter_mut().zip(slice.layers.iter()) {
match (layer, snapshot) {
(CacheLayer::Transformer(kv), CacheLayerSlice::Transformer(s)) => {
kv.apply_slice(&s, range.clone());
@@ -382,9 +465,9 @@ impl<B: Backend> CacheLayers<B> {
source: &Self,
context: &B::Context,
) {
- assert_eq!(source.data.len(), self.data.len(), "cache layer count mismatch");
+ assert_eq!(source.entries.len(), self.entries.len(), "cache layer count mismatch");
- for (source_layer, destination_layer) in source.data.iter().zip(self.data.iter_mut()) {
+ for (source_layer, destination_layer) in source.entries.iter().zip(self.entries.iter_mut()) {
match (source_layer, destination_layer) {
(CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {
let copy_rows = source.prefix_segment_length();
@@ -409,9 +492,9 @@ impl<B: Backend> CacheLayers<B> {
source: &Self,
encoder: &mut Encoder<B>,
) {
- assert_eq!(source.data.len(), self.data.len(), "cache layer count mismatch");
+ assert_eq!(source.entries.len(), self.entries.len(), "cache layer count mismatch");
- for (source_layer, destination_layer) in source.data.iter().zip(self.data.iter_mut()) {
+ for (source_layer, destination_layer) in source.entries.iter().zip(self.entries.iter_mut()) {
match (source_layer, destination_layer) {
(CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {
if matches!(source.state, KVCacheLayerState::Full { .. }) {
@@ -445,8 +528,8 @@ impl<B: Backend> CacheLayers<B> {
context: &B::Context,
) -> Self {
let mut max_prefix_capacity_across_layers = 0usize;
- let mut data: Box<[CacheLayer<B>]> = self
- .data
+ let mut entries: Box<[CacheLayer<B>]> = self
+ .entries
.iter()
.map(|layer| match layer {
CacheLayer::Transformer(layer) => {
@@ -539,7 +622,7 @@ impl<B: Backend> CacheLayers<B> {
.collect();
let mut zero_encoder: Encoder<B> = Encoder::new(context).expect("Failed to create cache clone zero encoder");
- for layer in data.iter_mut() {
+ for layer in entries.iter_mut() {
match layer {
CacheLayer::Transformer(layer) => {
zero_encoder.encode_fill(&mut layer.keys, 0);
@@ -556,7 +639,8 @@ impl<B: Backend> CacheLayers<B> {
let mut cloned = Self {
max_suffix_length: self.max_suffix_length,
max_prefix_length: max_prefix_capacity_across_layers,
- data,
+ entries,
+ bindings: self.bindings.clone(),
};
cloned.copy_from(self, context);
cloned
diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rs
index 0d2e95887..8efdfe4fc 100644
--- a/crates/backend-uzu/src/forward_pass/model_shape.rs
+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs
@@ -15,6 +15,7 @@ pub struct ModelShape {
num_groups: usize,
pub num_layers: usize,
layer_mixers: Box<[MixerConfig]>,
+ kv_source_layers: Box<[Option<usize>]>,
}
impl ModelShape {
@@ -36,6 +37,7 @@ impl ModelShape {
};
let layer_mixers: Box<[MixerConfig]> = layer_configs.iter().map(|l| l.mixer_config.clone()).collect();
+ let kv_source_layers: Box<[Option<usize>]> = layer_configs.iter().map(|l| l.kv_source_layer).collect();
Self {
activation_type,
@@ -46,6 +48,7 @@ impl ModelShape {
num_groups,
num_layers,
layer_mixers,
+ kv_source_layers,
}
}
@@ -65,6 +68,10 @@ impl ModelShape {
self.model_dim
}
+ pub fn kv_source_layers(&self) -> &[Option<usize>] {
+ &self.kv_source_layers
+ }
+
pub fn layer_mixers(&self) -> &[MixerConfig] {
&self.layer_mixers
}
diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
index 018a6bbaf..058590d23 100644
--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
@@ -306,14 +306,11 @@ impl<B: Backend> TraceValidator<B> {
let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);
// LLM-specific: KV cache validation
- let transformer_layers: Vec<usize> = {
- let cache = ctx.cache_layers.borrow();
- cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_transformer().map(|_| index)).collect()
- };
-
- for index in transformer_layers {
- let cache = ctx.cache_layers.borrow();
- let kv = cache.data[index].as_transformer().expect("Expected transformer layer");
+ let cache = ctx.cache_layers.borrow();
+ for (index, layer) in cache.iter_layers() {
+ let Some(kv) = layer.as_transformer() else {
+ continue;
+ };
if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {
results.push(TracerValidationResult {
@@ -343,14 +340,10 @@ impl<B: Backend> TraceValidator<B> {
}
// LLM-specific: SSM state validation
- let ssm_layers: Vec<usize> = {
- let cache = ctx.cache_layers.borrow();
- cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_state_space().map(|_| index)).collect()
- };
-
- for index in ssm_layers {
- let cache = ctx.cache_layers.borrow();
- let ssm = cache.data[index].as_state_space().expect("Expected SSM layer");
+ for (index, layer) in cache.iter_layers() {
+ let Some(ssm) = layer.as_state_space() else {
+ continue;
+ };
for path in [
format!("updated_state.{}.conv_state", index),
@@ -384,14 +377,10 @@ impl<B: Backend> TraceValidator<B> {
}
// LLM-specific: DeltaNet state validation
- let delta_net_layers: Vec<usize> = {
- let cache = ctx.cache_layers.borrow();
- cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_delta_net().map(|_| index)).collect()
- };
-
- for index in delta_net_layers {
- let cache = ctx.cache_layers.borrow();
- let delta = cache.data[index].as_delta_net().expect("Expected DeltaNet layer");
+ for (index, layer) in cache.iter_layers() {
+ let Some(delta) = layer.as_delta_net() else {
+ continue;
+ };
for path in [
format!("updated_state.{}.conv_state", index),