A high-performance inference engine for AI models
Cross-layer KV cache sharing (#428)
CCYeh153837f0ef99
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rsindex 9093b1569..4b3293b3e 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs@@ -314,7 +314,7 @@ impl StructuredAudioCodecGraph {rope_buffers,sampling_start: 0,sampling_length: batch_dim,- cache_layer: None,+ cache_access: None,#[cfg(feature = "tracing")]trace: None,},diff --git a/crates/backend-uzu/src/classifier/classifier.rs b/crates/backend-uzu/src/classifier/classifier.rsindex a9c5d6957..f9ed6ab53 100644--- a/crates/backend-uzu/src/classifier/classifier.rs+++ b/crates/backend-uzu/src/classifier/classifier.rs@@ -147,7 +147,7 @@ impl<B: Backend> Classifier<B> {rope_buffers,sampling_start: 0,sampling_length: batch_dim,- cache_layer: None,+ cache_access: None,#[cfg(feature = "tracing")]trace: trace.as_deref_mut().map(|traces| &mut traces.layer_results[layer_index]),},diff --git a/crates/backend-uzu/src/config/transformer_layer.rs b/crates/backend-uzu/src/config/transformer_layer.rsindex b6510e6ab..12c88ffcb 100644--- a/crates/backend-uzu/src/config/transformer_layer.rs+++ b/crates/backend-uzu/src/config/transformer_layer.rs@@ -15,5 +15,5 @@ pub struct TransformerLayerConfig {pub hidden_dim: Option<usize>,pub ple_config: Option<Unsupported>,pub has_post_layer_scalar: bool,- pub kv_source_layer: Option<Unsupported>,+ pub kv_source_layer: Option<usize>,}diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rsindex b9152eff9..945528bcb 100644--- a/crates/backend-uzu/src/encodable_block/attention.rs+++ b/crates/backend-uzu/src/encodable_block/attention.rs@@ -19,7 +19,10 @@ use crate::{},},config::AttentionConfig,- forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},+ forward_pass::{+ cache_layers::LayerCacheAccess,+ kv_cache_layer::KVCacheLayerState,+ },};fn env_gemm_attention_enabled() -> bool {@@ -60,7 +63,7 @@ pub struct Attention<B: Backend> {pub struct AttentionArguments<'a, B: Backend> {pub token_subtrie_ranges: Option<&'a Allocation<B>>,pub attention_sinks: Option<&'a Allocation<B>>,- pub kv_cache_layer: Option<&'a mut KVCacheLayer<B>>,+ pub cache_access: Option<LayerCacheAccess<'a, B>>,}impl<B: Backend> Attention<B> {@@ -200,7 +203,7 @@ impl<B: Backend> Attention<B> {pub fn encode(&self,- mut args: AttentionArguments<B>,+ args: AttentionArguments<B>,qkv: &Allocation<B>,queries: &Allocation<B>,mut rotated_keys: Allocation<B>,@@ -213,8 +216,20 @@ impl<B: Backend> Attention<B> {) -> Result<Allocation<B>, B::Error> {let is_trie = args.token_subtrie_ranges.is_some();+ let cache_layer = args.cache_access.as_ref().map(|a| {+ match a {+ LayerCacheAccess::Owned {+ entry,+ } => entry.as_transformer(),+ LayerCacheAccess::Shared {+ source,+ } => source.as_transformer(),+ }+ .expect("attention expects transformer cache")+ });+let (max_sequence_length, segment_prefix_length, ring_params) =- if let Some(layer) = args.kv_cache_layer.as_deref() {+ if let Some(layer) = cache_layer {let max_sequence_length = layer.shape[0];let ring_params = match layer.state {KVCacheLayerState::Windowed {@@ -255,8 +270,7 @@ impl<B: Backend> Attention<B> {let sinks_allocation = args.attention_sinks;let trie_allocation = args.token_subtrie_ranges;- // Get KV cache buffers only if KV cache exists (LLM mode)- let has_kv_cache = args.kv_cache_layer.is_some();+ let has_kv_cache = args.cache_access.is_some();let extracted_values = (!has_kv_cache).then(|| {let mut values =@@ -298,23 +312,36 @@ impl<B: Backend> Attention<B> {is_kv_cache_ring,};- let (keys, values) = if let Some(layer) = args.kv_cache_layer.as_deref_mut() {- self.update_kv_cache_kernel.encode(- Some(&rotated_keys),- qkv,- &mut layer.keys,- &mut layer.values,- num_groups as u32,- num_heads as u32,- head_dim as u32,- suffix_length as u32,- segment_prefix_length as u32,- max_sequence_length as u32,- encoder,- );- (&layer.keys, &layer.values)- } else {- (&rotated_keys, extracted_values.as_ref().expect("Missing extracted values for classifier attention"))+ let (keys, values) = match args.cache_access {+ Some(LayerCacheAccess::Shared {+ source,+ }) => {+ let source = source.as_transformer().expect("kv_source must be a transformer cache");+ (&source.keys, &source.values)+ },+ Some(LayerCacheAccess::Owned {+ entry,+ }) => {+ let layer = entry.as_transformer_mut().expect("Attention layer expects transformer cache");+ self.update_kv_cache_kernel.encode(+ Some(&rotated_keys),+ qkv,+ &mut layer.keys,+ &mut layer.values,+ num_groups as u32,+ num_heads as u32,+ head_dim as u32,+ suffix_length as u32,+ segment_prefix_length as u32,+ max_sequence_length as u32,+ encoder,+ );+ (&layer.keys, &layer.values)+ },+ None => (+ &rotated_keys,+ extracted_values.as_ref().expect("Missing extracted values for classifier attention"),+ ),};let mut attention_output = self.encode_attention_variant(diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rsindex 5a2dfc8b5..8697ac102 100644--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs@@ -230,7 +230,7 @@ impl<B: Backend> ClassifierLayer<B> {AttentionArguments {token_subtrie_ranges,attention_sinks,- kv_cache_layer: None,+ cache_access: None,},&qkv,&queries,diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex 474236279..def9102ae 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -183,7 +183,7 @@ impl<B: Backend> Decoder<B> {#[cfg(feature = "tracing")]let layer_trace = trace.as_deref_mut().map(|trace| &mut trace.layer_results[layer.layer_index]);- let cache_layer = cache_layers.as_deref_mut().map(|cache_layers| &mut cache_layers.data[layer.layer_index]);+ let cache_access = cache_layers.as_deref_mut().map(|cache_layers| cache_layers.cache_for_layer(layer.layer_index));main = layer.encode(LayerArguments {@@ -195,7 +195,7 @@ impl<B: Backend> Decoder<B> {rope_buffers,sampling_start,sampling_length,- cache_layer,+ cache_access,#[cfg(feature = "tracing")]trace: layer_trace,},diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rsindex 07ed4e5bf..f554f04b6 100644--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs@@ -17,7 +17,7 @@ use crate::{Attention, AttentionArguments, DeltaNetArguments, DeltaNetMixer, Linear, MambaArguments, MambaMixer, Mlp,PostLayerScalar, QKVNorm, QkUnpack, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,},- forward_pass::{cache_layers::CacheLayer, state::RopeBuffers},+ forward_pass::{cache_layers::LayerCacheAccess, state::RopeBuffers},parameters::ParameterTree,};@@ -330,7 +330,7 @@ impl<B: Backend> LayerExecutables<B> {rope_buffers,sampling_start,sampling_length,- mut cache_layer,+ cache_access,#[cfg(feature = "tracing")]trace,} = args;@@ -389,14 +389,11 @@ impl<B: Backend> LayerExecutables<B> {)?,None => qk_unpack.encode(&qkv, batch_dim, *num_heads, *num_groups, *head_dim, encoder)?,};- let kv_cache_layer = cache_layer- .as_deref_mut()- .map(|layer| layer.as_transformer_mut().expect("Attention layer expects transformer cache"));let attention_output = attention.encode(AttentionArguments {token_subtrie_ranges,attention_sinks,- kv_cache_layer,+ cache_access,},&qkv,&queries,@@ -413,11 +410,12 @@ impl<B: Backend> LayerExecutables<B> {MixerExecutables::StateSpace {mixer,} => {- let layer = cache_layer- .as_deref_mut()- .expect("State-space layer requires cache state")- .as_state_space_mut()- .expect("State-space mixer expects SSM cache layer");+ let Some(LayerCacheAccess::Owned {+ entry,+ }) = cache_access else {+ panic!("State-space layer requires writable cache state");+ };+ let layer = entry.as_state_space_mut().expect("State-space mixer expects SSM cache layer");mixer.encode(MambaArguments {active_row_count: batch_dim,@@ -430,11 +428,12 @@ impl<B: Backend> LayerExecutables<B> {MixerExecutables::ShortConv {mixer,} => {- let layer = cache_layer- .as_deref_mut()- .expect("ShortConv layer requires cache state")- .as_short_conv_mut()- .expect("ShortConv mixer expects ShortConv cache layer");+ let Some(LayerCacheAccess::Owned {+ entry,+ }) = cache_access else {+ panic!("ShortConv layer requires writable cache state");+ };+ let layer = entry.as_short_conv_mut().expect("ShortConv mixer expects ShortConv cache layer");mixer.encode(ShortConvArguments {active_row_count: batch_dim,@@ -450,11 +449,12 @@ impl<B: Backend> LayerExecutables<B> {MixerExecutables::DeltaNet {mixer,} => {- let layer = cache_layer- .as_deref_mut()- .expect("DeltaNet layer requires cache state")- .as_delta_net_mut()- .expect("DeltaNet mixer expects DeltaNet cache layer");+ let Some(LayerCacheAccess::Owned {+ entry,+ }) = cache_access else {+ panic!("DeltaNet layer requires writable cache state");+ };+ let layer = entry.as_delta_net_mut().expect("DeltaNet mixer expects DeltaNet cache layer");mixer.encode(DeltaNetArguments {active_row_count: batch_dim,@@ -525,7 +525,7 @@ pub struct LayerArguments<'a, B: Backend> {pub rope_buffers: Option<&'a RopeBuffers<B>>,pub sampling_start: usize,pub sampling_length: usize,- pub cache_layer: Option<&'a mut CacheLayer<B>>,+ pub cache_access: Option<LayerCacheAccess<'a, B>>,#[cfg(feature = "tracing")]pub trace: Option<&'a mut LayerActivationTrace<B>>,}diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rsindex 7b904cd03..8eda52178 100644--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs@@ -88,13 +88,38 @@ impl<B: Backend> CacheLayer<B> {pub struct CacheLayers<B: Backend> {max_suffix_length: usize,max_prefix_length: usize,- pub data: Box<[CacheLayer<B>]>,+ entries: Box<[CacheLayer<B>]>,+ bindings: Box<[LayerCacheBinding]>,}pub struct CacheLayersSlice<B: Backend> {pub layers: Vec<CacheLayerSlice<B>>,}+#[derive(Clone, Copy)]+struct CacheEntryIndex {+ index: usize,+}++#[derive(Clone, Copy)]+enum LayerCacheBinding {+ Owned {+ entry: CacheEntryIndex,+ },+ Shared {+ source: CacheEntryIndex,+ },+}++pub enum LayerCacheAccess<'a, B: Backend> {+ Owned {+ entry: &'a mut CacheLayer<B>,+ },+ Shared {+ source: &'a CacheLayer<B>,+ },+}+impl<B: Backend> CacheLayers<B> {pub fn new(context: &B::Context,@@ -104,10 +129,32 @@ impl<B: Backend> CacheLayers<B> {) -> Self {let total_context_length = max_prefix_length.max(max_suffix_length);- let mut data: Box<[CacheLayer<B>]> = model_shape- .layer_mixers()- .iter()- .map(|mixer| match mixer {+ let kv_source_layers = model_shape.kv_source_layers();++ let mut entries: Vec<CacheLayer<B>> = Vec::with_capacity(model_shape.layer_mixers().len());+ let mut bindings: Vec<LayerCacheBinding> = Vec::with_capacity(model_shape.layer_mixers().len());+ for (layer_index, mixer) in model_shape.layer_mixers().iter().enumerate() {+ if let Some(source_layer_index) = kv_source_layers.get(layer_index).copied().flatten() {+ let LayerCacheBinding::Owned {+ entry: source,+ } = bindings[source_layer_index] else {+ panic!("kv_source_layer {source_layer_index} (layer {layer_index}) must be a prior owner");+ };+ let geometry = |m: &MixerConfig| {+ m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size))+ };+ assert_eq!(+ geometry(&model_shape.layer_mixers()[source_layer_index]),+ geometry(mixer),+ "KV cache sharing requires identical attention geometry"+ );+ bindings.push(LayerCacheBinding::Shared {+ source,+ });+ continue;+ }++ let layer = match mixer {MixerConfig::Attention(attn) => {let sliding_window = attn.sliding_window_size;let length = sliding_window.unwrap_or(max_prefix_length);@@ -129,7 +176,7 @@ impl<B: Backend> CacheLayers<B> {let kv_bytes = size_for_shape(&shape, model_shape.kv_cache_data_type());CacheLayer::Transformer(KVCacheLayer {- state: state.clone(),+ state,keys: context.create_allocation(kv_bytes, AllocationType::Global).expect("Failed to create kv keys allocation"),@@ -202,11 +249,17 @@ impl<B: Backend> CacheLayers<B> {data_type: dtype,})},- })- .collect();+ };+ bindings.push(LayerCacheBinding::Owned {+ entry: CacheEntryIndex { index: entries.len() },+ });+ entries.push(layer);+ }+ let mut entries = entries.into_boxed_slice();+ let bindings = bindings.into_boxed_slice();let mut encoder: Encoder<B> = Encoder::new(context).expect("Failed to create cache initialization encoder");- for layer in data.iter_mut() {+ for layer in entries.iter_mut() {match layer {CacheLayer::Transformer(layer) => {encoder.encode_fill(&mut layer.keys, 0);@@ -233,16 +286,46 @@ impl<B: Backend> CacheLayers<B> {Self {max_suffix_length,max_prefix_length,- data,+ entries,+ bindings,}}+ pub fn cache_for_layer(+ &mut self,+ layer_index: usize,+ ) -> LayerCacheAccess<'_, B> {+ match self.bindings[layer_index] {+ LayerCacheBinding::Owned {+ entry,+ } => LayerCacheAccess::Owned {+ entry: &mut self.entries[entry.index],+ },+ LayerCacheBinding::Shared {+ source,+ } => LayerCacheAccess::Shared {+ source: &self.entries[source.index],+ },+ }+ }++ pub fn iter_layers(&self) -> impl Iterator<Item = (usize, &CacheLayer<B>)> {+ self.bindings.iter().enumerate().filter_map(|(index, binding)| match binding {+ LayerCacheBinding::Owned {+ entry,+ } => Some((index, &self.entries[entry.index])),+ LayerCacheBinding::Shared {+ ..+ } => None,+ })+ }+pub fn clear(&mut self,context: &B::Context,) {let mut encoder: Option<Encoder<B>> = None;- for layer in self.data.iter_mut() {+ for layer in self.entries.iter_mut() {match layer {CacheLayer::Transformer(layer) => match &mut layer.state {KVCacheLayerState::Full {@@ -303,7 +386,7 @@ impl<B: Backend> CacheLayers<B> {kv_cache_update: &KVCacheUpdate<B>,) {let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);- for layer in self.data.iter_mut() {+ for layer in self.entries.iter_mut() {if let Some(layer) = layer.as_transformer_mut() {layer.update_after_acceptance(accepted_suffix_indices, suffix_start, encoder, kv_cache_update);} else if let Some(layer) = layer.as_short_conv_mut() {@@ -316,7 +399,7 @@ impl<B: Backend> CacheLayers<B> {&mut self,number_of_accepted_tokens: usize,) {- for layer in self.data.iter_mut() {+ for layer in self.entries.iter_mut() {if let Some(layer) = layer.as_transformer_mut() {layer.register_accepted_tokens(number_of_accepted_tokens);}@@ -328,8 +411,8 @@ impl<B: Backend> CacheLayers<B> {context: &B::Context,range: std::ops::Range<usize>,) -> Option<CacheLayersSlice<B>> {- let mut layers = Vec::with_capacity(self.data.len());- for layer in self.data.iter() {+ let mut layers = Vec::with_capacity(self.entries.len());+ for layer in self.entries.iter() {match layer {CacheLayer::Transformer(kv) => {let Some(slice) = kv.slice(context, range.clone()) else {@@ -353,7 +436,7 @@ impl<B: Backend> CacheLayers<B> {slice: &CacheLayersSlice<B>,range: Option<std::ops::Range<usize>>,) {- for (layer, snapshot) in self.data.iter_mut().zip(slice.layers.iter()) {+ for (layer, snapshot) in self.entries.iter_mut().zip(slice.layers.iter()) {match (layer, snapshot) {(CacheLayer::Transformer(kv), CacheLayerSlice::Transformer(s)) => {kv.apply_slice(&s, range.clone());@@ -382,9 +465,9 @@ impl<B: Backend> CacheLayers<B> {source: &Self,context: &B::Context,) {- assert_eq!(source.data.len(), self.data.len(), "cache layer count mismatch");+ assert_eq!(source.entries.len(), self.entries.len(), "cache layer count mismatch");- for (source_layer, destination_layer) in source.data.iter().zip(self.data.iter_mut()) {+ for (source_layer, destination_layer) in source.entries.iter().zip(self.entries.iter_mut()) {match (source_layer, destination_layer) {(CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {let copy_rows = source.prefix_segment_length();@@ -409,9 +492,9 @@ impl<B: Backend> CacheLayers<B> {source: &Self,encoder: &mut Encoder<B>,) {- assert_eq!(source.data.len(), self.data.len(), "cache layer count mismatch");+ assert_eq!(source.entries.len(), self.entries.len(), "cache layer count mismatch");- for (source_layer, destination_layer) in source.data.iter().zip(self.data.iter_mut()) {+ for (source_layer, destination_layer) in source.entries.iter().zip(self.entries.iter_mut()) {match (source_layer, destination_layer) {(CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {if matches!(source.state, KVCacheLayerState::Full { .. }) {@@ -445,8 +528,8 @@ impl<B: Backend> CacheLayers<B> {context: &B::Context,) -> Self {let mut max_prefix_capacity_across_layers = 0usize;- let mut data: Box<[CacheLayer<B>]> = self- .data+ let mut entries: Box<[CacheLayer<B>]> = self+ .entries.iter().map(|layer| match layer {CacheLayer::Transformer(layer) => {@@ -539,7 +622,7 @@ impl<B: Backend> CacheLayers<B> {.collect();let mut zero_encoder: Encoder<B> = Encoder::new(context).expect("Failed to create cache clone zero encoder");- for layer in data.iter_mut() {+ for layer in entries.iter_mut() {match layer {CacheLayer::Transformer(layer) => {zero_encoder.encode_fill(&mut layer.keys, 0);@@ -556,7 +639,8 @@ impl<B: Backend> CacheLayers<B> {let mut cloned = Self {max_suffix_length: self.max_suffix_length,max_prefix_length: max_prefix_capacity_across_layers,- data,+ entries,+ bindings: self.bindings.clone(),};cloned.copy_from(self, context);cloneddiff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rsindex 0d2e95887..8efdfe4fc 100644--- a/crates/backend-uzu/src/forward_pass/model_shape.rs+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs@@ -15,6 +15,7 @@ pub struct ModelShape {num_groups: usize,pub num_layers: usize,layer_mixers: Box<[MixerConfig]>,+ kv_source_layers: Box<[Option<usize>]>,}impl ModelShape {@@ -36,6 +37,7 @@ impl ModelShape {};let layer_mixers: Box<[MixerConfig]> = layer_configs.iter().map(|l| l.mixer_config.clone()).collect();+ let kv_source_layers: Box<[Option<usize>]> = layer_configs.iter().map(|l| l.kv_source_layer).collect();Self {activation_type,@@ -46,6 +48,7 @@ impl ModelShape {num_groups,num_layers,layer_mixers,+ kv_source_layers,}}@@ -65,6 +68,10 @@ impl ModelShape {self.model_dim}+ pub fn kv_source_layers(&self) -> &[Option<usize>] {+ &self.kv_source_layers+ }+pub fn layer_mixers(&self) -> &[MixerConfig] {&self.layer_mixers}diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rsindex 018a6bbaf..058590d23 100644--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs@@ -306,14 +306,11 @@ impl<B: Backend> TraceValidator<B> {let mut results = Self::validate_layer_traces(&traces, &traces_view, data_type);// LLM-specific: KV cache validation- let transformer_layers: Vec<usize> = {- let cache = ctx.cache_layers.borrow();- cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_transformer().map(|_| index)).collect()- };-- for index in transformer_layers {- let cache = ctx.cache_layers.borrow();- let kv = cache.data[index].as_transformer().expect("Expected transformer layer");+ let cache = ctx.cache_layers.borrow();+ for (index, layer) in cache.iter_layers() {+ let Some(kv) = layer.as_transformer() else {+ continue;+ };if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {results.push(TracerValidationResult {@@ -343,14 +340,10 @@ impl<B: Backend> TraceValidator<B> {}// LLM-specific: SSM state validation- let ssm_layers: Vec<usize> = {- let cache = ctx.cache_layers.borrow();- cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_state_space().map(|_| index)).collect()- };-- for index in ssm_layers {- let cache = ctx.cache_layers.borrow();- let ssm = cache.data[index].as_state_space().expect("Expected SSM layer");+ for (index, layer) in cache.iter_layers() {+ let Some(ssm) = layer.as_state_space() else {+ continue;+ };for path in [format!("updated_state.{}.conv_state", index),@@ -384,14 +377,10 @@ impl<B: Backend> TraceValidator<B> {}// LLM-specific: DeltaNet state validation- let delta_net_layers: Vec<usize> = {- let cache = ctx.cache_layers.borrow();- cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_delta_net().map(|_| index)).collect()- };-- for index in delta_net_layers {- let cache = ctx.cache_layers.borrow();- let delta = cache.data[index].as_delta_net().expect("Expected DeltaNet layer");+ for (index, layer) in cache.iter_layers() {+ let Some(delta) = layer.as_delta_net() else {+ continue;+ };for path in [format!("updated_state.{}.conv_state", index),