Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix Gemma4 KeyError sliding_attention issue (#1839)

Liang Lv09bc743716e8
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 1d94f6354..453149f4e 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -114,6 +114,14 @@ def prepare_special_model_block_inputs(block, rotary_input, input_others, positi
return input_others, positional_inputs
+def _get_gemma4_shared_kv_states_global(block):
+ """Return the shared KV states dict for Gemma4 block-wise quantization."""
+ ref = getattr(block, "_shared_kv_states_global_ref", None)
+ if ref is not None:
+ return ref
+ return {}
+
+
def _get_gemma4_rotary_emb(block, default_rotary_emb=None):
rotary_emb_ref = getattr(block, "_rotary_emb_ref", None)
if rotary_emb_ref:
@@ -152,7 +160,12 @@ def _prepare_gemma4_replay_inputs(
head_dim = getattr(attn, "head_dim", None)
if attn is not None and hasattr(attn, "store_full_length_kv") and shared_kv_states is None:
- shared_kv_states = default_shared_kv_states if default_shared_kv_states is not None else {}
+ if default_shared_kv_states is not None:
+ shared_kv_states = default_shared_kv_states
+ else:
+ shared_kv_states = _get_gemma4_shared_kv_states_global(block)
+ if getattr(block, "layer_idx", None) == 0:
+ shared_kv_states.clear()
need_position_embeddings = position_embeddings is None
if isinstance(position_embeddings, dict):
@@ -1154,10 +1167,15 @@ def _attach_gemma4_rotary_emb(model):
if text_model is None:
return
+ # Create a single shared dict to propagate KV state between anchor/sharer layers.
+ # Gemma4TextModel.forward in newer transformers uses the same pattern.
+ shared_kv_states_global = {}
+
for layer in text_model.layers:
- # Store in a plain list to prevent nn.Module from registering rotary_emb
- # as a child submodule (which would cause meta-tensor errors during .to(device)).
+ # Store in a plain list to prevent nn.Module from registering these
+ # as child submodules (which would cause meta-tensor errors during .to(device)).
object.__setattr__(layer, "_rotary_emb_ref", [text_model.rotary_emb])
+ object.__setattr__(layer, "_shared_kv_states_global_ref", shared_kv_states_global)
object.__setattr__(layer, "_autoround_special_replay", "gemma4")
object.__setattr__(layer, "_gemma4_config_ref", text_model.config)