Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: update scale handling in QuantizedKVParameterCache to use amax of all batches (#2533)

Xin He512872d077d5
diff --git a/auto_round/experimental/kv_cache.py b/auto_round/experimental/kv_cache.py
index 83bef24f3..0148a024e 100644
--- a/auto_round/experimental/kv_cache.py
+++ b/auto_round/experimental/kv_cache.py
@@ -197,7 +197,10 @@ class QuantizedKVParameterCache(DynamicCache):
qdq_tensor, scale = fp8_qdq(tensor, granularity=self.granularity)
# Detach scale to prevent holding computation graph references
- _pad_and_append_at_idx_(scales, layer_idx, scale.reshape(-1).detach())
+ scale = scale.reshape(-1).detach()
+ if layer_idx < len(scales) and scales[layer_idx] is not None:
+ scale = torch.maximum(scales[layer_idx], scale)
+ _pad_and_append_at_idx_(scales, layer_idx, scale)
return qdq_tensor
def _nvfp4_track_amax(self, tensor: torch.Tensor, kv_type: KVCacheScaleType, layer_idx: int):
@@ -347,6 +350,9 @@ def kvcache_quant_context(
model: torch.nn.Module, static_kv_dtype=torch.float8_e4m3fn, static_kv_granularity: str = "tensor"
):
"""Context manager for static KV cache quantization (FP8 or NVFP4) operations."""
+ hook_handles = []
+ QuantizedKVParameterCache._instance = None
+ QuantizedKVParameterCache._initialized = False
try:
# Setup phase: Initialize KV cache for quantization
static_kv_dtype = normalize_static_kv_dtype(static_kv_dtype)
@@ -368,11 +374,18 @@ def kvcache_quant_context(
)
if static_kv_dtype in (torch.float8_e4m3fn, NVFP4_KV_DTYPE):
model.apply(initialize_fn)
- model.apply(prep_attention_module_for_calibration)
+ for module in model.modules():
+ if is_attention_module(module):
+ hook_handles.append(
+ module.register_forward_pre_hook(calibrate_kv_cache_input_hook, with_kwargs=True)
+ )
+ hook_handles.append(module.register_forward_hook(calibrate_kv_cache_output_hook))
# Provide the model to the with block
yield model
finally:
# Cleanup phase: Freeze quantization parameters
+ for handle in hook_handles:
+ handle.remove()
model.apply(freeze_module_quantization_)
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index 22af10d1e..66a9ac20e 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -311,6 +311,13 @@ def pack_layer(name, model, device=None):
delattr(layer, "scale")
_compress_and_set_format(layer, scheme, device)
+ if hasattr(layer, "weight_zero_point"):
+ zero_point = layer.weight_zero_point.data.contiguous()
+ if scheme.weights.num_bits == 4 and zero_point.ndim >= 2 and zero_point.shape[-2] == 1:
+ stride = list(zero_point.stride())
+ stride[-2] = 1
+ zero_point = zero_point.as_strided(zero_point.shape, stride)
+ layer.weight_zero_point.data = zero_point
@torch.no_grad()
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index ffb135acf..7fdbd19b8 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -454,7 +454,8 @@ def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):
"""Get block names for Qwen3-Omni MoE model.
Qwen3-Omni has the following structure:
- - thinker: Contains audio_tower, visual, model (text decoder)
+ - thinker.model: Contains visual/audio towers and the text decoder in newer Transformers releases
+ - thinker: Contains visual/audio towers directly in older Transformers releases
- talker: Contains model (talker decoder), code_predictor
- code2wav: Audio decoder
@@ -466,26 +467,56 @@ def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):
degrade audio quality in long-form generation .
"""
block_names = []
+ thinker = getattr(model, "thinker", None)
+ thinker_model = getattr(thinker, "model", None)
# Quantize visual encoder blocks if quant_vision is enabled
if quant_vision:
- # Vision encoder blocks
- if hasattr(model, "thinker") and hasattr(model.thinker, "visual") and hasattr(model.thinker.visual, "blocks"):
- block_names.append([f"thinker.visual.blocks.{i}" for i in range(len(model.thinker.visual.blocks))])
- # Audio encoder layers
- if hasattr(model, "thinker") and hasattr(model.thinker, "audio_tower"):
- if hasattr(model.thinker.audio_tower, "layers"):
- block_names.append(
- [f"thinker.audio_tower.layers.{i}" for i in range(len(model.thinker.audio_tower.layers))]
- )
-
- # Thinker text model layers (main LLM decoder)
- if hasattr(model, "thinker") and hasattr(model.thinker, "model") and hasattr(model.thinker.model, "layers"):
- block_names.append([f"thinker.model.layers.{i}" for i in range(len(model.thinker.model.layers))])
+ visual = getattr(thinker_model, "visual", None)
+ visual_blocks = getattr(visual, "blocks", None)
+ visual_prefix = "thinker.model.visual"
+ if visual_blocks is None or len(visual_blocks) == 0:
+ visual = getattr(thinker, "visual", None)
+ visual_blocks = getattr(visual, "blocks", None)
+ visual_prefix = "thinker.visual"
+ if visual_blocks is not None:
+ block_names.append([f"{visual_prefix}.blocks.{i}" for i in range(len(visual_blocks))])
+
+ audio_tower = getattr(thinker_model, "audio_tower", None)
+ audio_layers = getattr(audio_tower, "layers", None)
+ audio_prefix = "thinker.model.audio_tower"
+ if audio_layers is None or len(audio_layers) == 0:
+ audio_tower = getattr(thinker, "audio_tower", None)
+ audio_layers = getattr(audio_tower, "layers", None)
+ audio_prefix = "thinker.audio_tower"
+ if audio_layers is not None:
+ block_names.append([f"{audio_prefix}.layers.{i}" for i in range(len(audio_layers))])
+
+ # Thinker text model layers (main LLM decoder). Transformers may wrap the
+ # decoder in `language_model` in newer Qwen3-Omni releases.
+ layer_prefix, layers = _get_qwen3_omni_thinker_layer_info(model)
+ if layers is not None:
+ block_names.append([f"{layer_prefix}.{i}" for i in range(len(layers))])
return block_names
+def _get_qwen3_omni_thinker_layer_info(model):
+ """Return the module path and layers for either supported thinker layout."""
+ thinker_model = getattr(getattr(model, "thinker", None), "model", None)
+ if thinker_model is None:
+ return None, None
+
+ if hasattr(thinker_model, "layers"):
+ return "thinker.model.layers", thinker_model.layers
+
+ language_model = getattr(thinker_model, "language_model", None)
+ if language_model is not None and hasattr(language_model, "layers"):
+ return "thinker.model.language_model.layers", language_model.layers
+
+ return None, None
+
+
def _get_glm_image_multimodal_block(model, quant_vision=False):
"""Get block names for GLM-Image AR model.
diff --git a/test/unit/common/models/test_omni_model.py b/test/unit/common/models/test_omni_model.py
index ddf417f66..8b33c21ef 100644
--- a/test/unit/common/models/test_omni_model.py
+++ b/test/unit/common/models/test_omni_model.py
@@ -36,6 +36,14 @@ pytestmark = pytest.mark.skipif(
)
+def _qwen3_omni_thinker_layers(model):
+ from auto_round.special_model_handler import _get_qwen3_omni_thinker_layer_info
+
+ layer_prefix, layers = _get_qwen3_omni_thinker_layer_info(model)
+ assert layers is not None
+ return layer_prefix, layers
+
+
# ========================= Qwen2.5-Omni Tests =============================
# NOTE: Tests use the `tiny_qwen2_5_omni` session-scoped fixture from fixtures.py
# (real config, reduced layers, random weights). Skipped if model not available.
@@ -123,6 +131,17 @@ class TestQwen2_5OmniProcessor:
class TestQwen3OmniMoeBlockNames:
"""Test block name discovery for Qwen3-Omni-MoE."""
+ def test_block_names_language_model_nested_layers(self):
+ from types import SimpleNamespace
+
+ from auto_round.special_model_handler import _get_qwen3_omni_moe_multimodal_block
+
+ model = SimpleNamespace(
+ thinker=SimpleNamespace(model=SimpleNamespace(language_model=SimpleNamespace(layers=[object(), object()])))
+ )
+ block_names = _get_qwen3_omni_moe_multimodal_block(model)
+ assert block_names == [["thinker.model.language_model.layers.0", "thinker.model.language_model.layers.1"]]
+
def test_block_names_default(self):
"""Test that get_block_names returns thinker layers only by default."""
from auto_round.utils import get_block_names
@@ -131,9 +150,8 @@ class TestQwen3OmniMoeBlockNames:
model = Qwen3OmniMoeForConditionalGeneration(config)
block_names = get_block_names(model, quant_vision=False)
- assert any(
- "thinker.model.layers" in str(b) for b in block_names
- ), f"Expected thinker.model.layers, got: {block_names}"
+ layer_prefix, _ = _qwen3_omni_thinker_layers(model)
+ assert any(layer_prefix in str(b) for b in block_names), f"Expected {layer_prefix}, got: {block_names}"
assert not any(
"talker.model.layers" in str(b) for b in block_names
), f"Did not expect talker.model.layers by default, got: {block_names}"
@@ -205,7 +223,8 @@ class TestQwen3OmniMoeReplacement:
model = apply_replacements(model)
# Check that thinker MoE was custom-replaced
- thinker_mlp = model.thinker.model.layers[0].mlp
+ _, thinker_layers = _qwen3_omni_thinker_layers(model)
+ thinker_mlp = thinker_layers[0].mlp
assert (
"LinearQwen3OmniThinker" in thinker_mlp.__class__.__name__
), f"Expected LinearQwen3OmniThinker, got {thinker_mlp.__class__.__name__}"
@@ -238,8 +257,9 @@ class TestQwen3OmniMoeReplacement:
model = Qwen3OmniMoeForConditionalGeneration(config)
# Save original fused thinker weights
- thinker_gate_up = model.thinker.model.layers[0].mlp.experts.gate_up_proj.data.clone()
- thinker_down = model.thinker.model.layers[0].mlp.experts.down_proj.data.clone()
+ _, thinker_layers = _qwen3_omni_thinker_layers(model)
+ thinker_gate_up = thinker_layers[0].mlp.experts.gate_up_proj.data.clone()
+ thinker_down = thinker_layers[0].mlp.experts.down_proj.data.clone()
model = apply_replacements(model)
materialize_model_(model)
@@ -249,7 +269,8 @@ class TestQwen3OmniMoeReplacement:
# Use equal_nan=True because fused expert parameters may contain
# uninitialized memory with NaN bit patterns, and NaN != NaN in IEEE 754.
for i in range(4):
- expert = model.thinker.model.layers[0].mlp.experts[i]
+ _, thinker_layers = _qwen3_omni_thinker_layers(model)
+ expert = thinker_layers[0].mlp.experts[i]
assert_same_weights(expert.gate_proj.weight.data, thinker_gate_up[i, :intermediate, :])
assert_same_weights(expert.up_proj.weight.data, thinker_gate_up[i, intermediate:, :])
assert_same_weights(expert.down_proj.weight.data, thinker_down[i])
@@ -264,16 +285,18 @@ class TestQwen3OmniMoeReplacement:
torch.manual_seed(0)
config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
+ _, thinker_layers = _qwen3_omni_thinker_layers(model)
x = torch.randn(1, 4, 64) * 0.1
with torch.no_grad():
- orig_thinker_out = model.thinker.model.layers[0].mlp(x)
+ orig_thinker_out = thinker_layers[0].mlp(x)
model = apply_replacements(model)
materialize_model_(model)
+ _, thinker_layers = _qwen3_omni_thinker_layers(model)
with torch.no_grad():
- new_thinker_out = model.thinker.model.layers[0].mlp(x)
+ new_thinker_out = thinker_layers[0].mlp(x)
assert orig_thinker_out.shape == new_thinker_out.shape, "Thinker shape mismatch"
orig_nan = torch.isnan(orig_thinker_out)
@@ -308,7 +331,8 @@ class TestQwen3OmniMoeUtils:
config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
- moe_block = model.thinker.model.layers[0].mlp
+ _, thinker_layers = _qwen3_omni_thinker_layers(model)
+ moe_block = thinker_layers[0].mlp
assert is_moe_layer(moe_block), f"Thinker MoE block ({moe_block.__class__.__name__}) should be detected as MoE"
def test_is_moe_layer_talker(self):
@@ -325,7 +349,8 @@ class TestQwen3OmniMoeUtils:
config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
- thinker_mlp = model.thinker.model.layers[0].mlp
+ _, thinker_layers = _qwen3_omni_thinker_layers(model)
+ thinker_mlp = thinker_layers[0].mlp
names = get_expert_linear_names(thinker_mlp)
assert set(names) == {"gate_proj", "down_proj", "up_proj"}
@@ -339,7 +364,8 @@ class TestQwen3OmniMoeUtils:
config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
- thinker_mlp = model.thinker.model.layers[0].mlp
+ _, thinker_layers = _qwen3_omni_thinker_layers(model)
+ thinker_mlp = thinker_layers[0].mlp
names = get_expert_input_proj_names(thinker_mlp)
assert set(names) == {"gate_proj", "up_proj"}
@@ -349,9 +375,10 @@ class TestQwen3OmniMoeUtils:
config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
+ layer_prefix, _ = _qwen3_omni_thinker_layers(model)
ignore_layers = get_predefined_ignore_layers(model)
assert ignore_layers == [
- "thinker.model.layers.0.mlp.gate",
+ f"{layer_prefix}.0.mlp.gate",
"talker.model.layers.0.mlp.gate",
"talker.model.layers.0.mlp.shared_expert_gate",
], f"Expected mlp.gate in ignore_layers for qwen3_omni_moe, got: {ignore_layers}"
diff --git a/test/unit/common/models/test_special_model_handler.py b/test/unit/common/models/test_special_model_handler.py
index 5bfcebe97..dc6b635b8 100644
--- a/test/unit/common/models/test_special_model_handler.py
+++ b/test/unit/common/models/test_special_model_handler.py
@@ -727,6 +727,36 @@ class TestGetQwen25OmniMultimodalBlock:
class TestGetQwen3OmniMoeMultimodalBlock:
"""Test _get_qwen3_omni_moe_multimodal_block function."""
+ def test_with_nested_vision_and_audio_towers(self):
+ from types import SimpleNamespace
+
+ from auto_round.special_model_handler import _get_qwen3_omni_moe_multimodal_block
+
+ model = SimpleNamespace(
+ thinker=SimpleNamespace(
+ model=SimpleNamespace(
+ visual=SimpleNamespace(blocks=[MagicMock() for _ in range(2)]),
+ audio_tower=SimpleNamespace(layers=[MagicMock() for _ in range(3)]),
+ language_model=SimpleNamespace(layers=[MagicMock() for _ in range(4)]),
+ )
+ )
+ )
+ block_names = _get_qwen3_omni_moe_multimodal_block(model, quant_vision=True)
+ assert block_names == [
+ ["thinker.model.visual.blocks.0", "thinker.model.visual.blocks.1"],
+ [
+ "thinker.model.audio_tower.layers.0",
+ "thinker.model.audio_tower.layers.1",
+ "thinker.model.audio_tower.layers.2",
+ ],
+ [
+ "thinker.model.language_model.layers.0",
+ "thinker.model.language_model.layers.1",
+ "thinker.model.language_model.layers.2",
+ "thinker.model.language_model.layers.3",
+ ],
+ ]
+
def test_returns_thinker_layers(self):
from auto_round.special_model_handler import _get_qwen3_omni_moe_multimodal_block
diff --git a/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py b/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py
index 535c6273a..eda62a8a8 100644
--- a/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py
+++ b/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py
@@ -14,7 +14,6 @@ from auto_round.experimental.kv_cache import QuantizedKVParameterCache
def _reset_singleton():
- # ``reset`` is an instance method; clear the class state directly.
QuantizedKVParameterCache._instance = None
QuantizedKVParameterCache._initialized = False
@@ -125,3 +124,207 @@ def test_nvfp4_kv_incompatible_head_dim_raises_at_update():
bad = torch.randn(1, 1, 16, 88)
with pytest.raises(ValueError, match="divisible by 16"):
cache.update(bad, bad.clone(), layer_idx=0)
+
+
+class ObservedAttention(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer_idx = 0
+ self.head_dim = 16
+ self.k_proj = torch.nn.Linear(16, 16, bias=False)
+ self.v_proj = torch.nn.Linear(16, 16, bias=False)
+
+ def forward(self, hidden_states, past_key_values=None, use_cache=False):
+ key_states = self.k_proj(hidden_states).unsqueeze(1)
+ value_states = self.v_proj(hidden_states).unsqueeze(1)
+ if past_key_values is not None:
+ key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)
+ query_states = hidden_states.unsqueeze(1)
+ return torch.nn.functional.scaled_dot_product_attention(query_states, key_states, value_states)
+
+
+def test_nvfp4_observer_preserves_optimization_trajectory():
+ import copy
+ from contextlib import nullcontext
+
+ from auto_round.experimental.kv_cache import kvcache_quant_context
+
+ torch.manual_seed(42)
+ baseline = ObservedAttention()
+ observed = copy.deepcopy(baseline)
+ inputs = [torch.randn(2, 4, 16) for _ in range(3)]
+ targets = [torch.randn(2, 1, 4, 16) for _ in inputs]
+ trajectories = []
+ for model, context in ((baseline, nullcontext()), (observed, kvcache_quant_context(observed, "nvfp4"))):
+ trajectory = []
+ parameters = [model.k_proj.weight, model.v_proj.weight]
+ with context:
+ for hidden_states, target in zip(inputs, targets):
+ model.zero_grad(set_to_none=True)
+ output = model(hidden_states, past_key_values=None)
+ loss = torch.nn.functional.mse_loss(output, target)
+ loss.backward()
+ gradients = [parameter.grad.detach().clone() for parameter in parameters]
+ with torch.no_grad():
+ for parameter in parameters:
+ parameter.sub_(0.01 * parameter.grad.sign())
+ trajectory.append((output.detach(), loss.detach(), gradients))
+ trajectories.append(trajectory)
+ for expected, actual in zip(*trajectories):
+ torch.testing.assert_close(actual[0], expected[0])
+ torch.testing.assert_close(actual[1], expected[1])
+ for expected_gradient, actual_gradient in zip(expected[2], actual[2]):
+ torch.testing.assert_close(actual_gradient, expected_gradient)
+ torch.testing.assert_close(actual_gradient.sign(), expected_gradient.sign(), rtol=0, atol=0)
+ for name, expected in baseline.named_parameters():
+ torch.testing.assert_close(observed.get_parameter(name), expected, rtol=0, atol=0)
+
+
[email protected]("dtype", ["nvfp4", "fp8"])
+def test_kv_context_removes_its_hooks(dtype):
+ from auto_round.experimental.kv_cache import kvcache_quant_context
+
+ model = ObservedAttention()
+ inputs = torch.randn(1, 4, 16)
+ expected = model(inputs, past_key_values=None)
+ with kvcache_quant_context(model, dtype):
+ model(inputs, past_key_values=None)
+ actual = model(inputs, past_key_values=None)
+ torch.testing.assert_close(actual, expected)
+
+
+def test_kv_context_does_not_reuse_previous_amax():
+ from auto_round.experimental.kv_cache import kvcache_quant_context
+
+ model = ObservedAttention()
+ with kvcache_quant_context(model, "nvfp4"):
+ model.kv_cache.update(torch.full((1, 1, 1, 16), 100.0), torch.ones(1, 1, 1, 16), 0)
+ with kvcache_quant_context(model, "nvfp4"):
+ model.kv_cache.update(torch.ones(1, 1, 1, 16), torch.ones(1, 1, 1, 16), 0)
+ assert model.kv_cache.k_amax[0] == 1.0
+
+
+def test_kv_context_cleans_up_after_exception():
+ from auto_round.experimental.kv_cache import kvcache_quant_context
+
+ model = ObservedAttention()
+ with pytest.raises(RuntimeError, match="calibration failed"), kvcache_quant_context(model, "nvfp4"):
+ raise RuntimeError("calibration failed")
+ assert not hasattr(model, "kv_cache")
+ model(torch.randn(1, 4, 16), past_key_values=None)
+
+
+def test_kv_context_preserves_unrelated_hooks():
+ from auto_round.experimental.kv_cache import kvcache_quant_context
+
+ model = ObservedAttention()
+ calls = []
+ handle = model.register_forward_hook(lambda module, args, output: calls.append(True))
+ with kvcache_quant_context(model, "nvfp4"):
+ model(torch.randn(1, 4, 16), past_key_values=None)
+ model(torch.randn(1, 4, 16), past_key_values=None)
+ assert calls == [True, True]
+ handle.remove()
+
+
[email protected]("granularity", ["tensor", "head"])
+def test_fp8_update_quantizes_and_keeps_running_max_scale(granularity):
+ cache = QuantizedKVParameterCache(dtype="fp8", granularity=granularity)
+ key_states = torch.linspace(-3.1, 2.7, 64).reshape(1, 2, 2, 16)
+ value_states = key_states * 2.7
+ actual_keys, actual_values = cache.update(key_states, value_states, 0)
+ assert not torch.equal(actual_keys, key_states)
+ assert not torch.equal(actual_values, value_states)
+ initial_k_scale = cache.k_scales[0].clone()
+ initial_v_scale = cache.v_scales[0].clone()
+ cache.update(key_states / 4, value_states / 4, 0)
+ torch.testing.assert_close(cache.k_scales[0], initial_k_scale)
+ torch.testing.assert_close(cache.v_scales[0], initial_v_scale)
+
+
[email protected]("granularity", ["tensor", "head"])
+def test_fp8_update_running_max_per_side_head_and_layer(granularity):
+ cache = QuantizedKVParameterCache(dtype="fp8", granularity=granularity)
+ first_keys = torch.tensor([8.0, 1.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)
+ first_values = torch.tensor([2.0, 16.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)
+ cache.update(first_keys, first_values, 2)
+ second_keys = torch.tensor([2.0, 4.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)
+ second_values = torch.tensor([32.0, 1.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)
+ cache.reset_states()
+ cache.update(second_keys, second_values, 2)
+ expected_keys = torch.tensor([8.0, 4.0] if granularity == "head" else [8.0]) / 448
+ expected_values = torch.tensor([32.0, 16.0] if granularity == "head" else [32.0]) / 448
+ torch.testing.assert_close(cache.k_scales[2], expected_keys)
+ torch.testing.assert_close(cache.v_scales[2], expected_values)
+ assert not cache.k_scales[2].requires_grad
+ assert not cache.v_scales[2].requires_grad
+
+ cache.update(second_keys / 8, second_values / 8, 0)
+ expected_layer_keys = torch.tensor([0.25, 0.5] if granularity == "head" else [0.5]) / 448
+ expected_layer_values = torch.tensor([4.0, 0.125] if granularity == "head" else [4.0]) / 448
+ torch.testing.assert_close(cache.k_scales[0], expected_layer_keys)
+ torch.testing.assert_close(cache.v_scales[0], expected_layer_values)
+ torch.testing.assert_close(cache.k_scales[2], expected_keys)
+ torch.testing.assert_close(cache.v_scales[2], expected_values)
+
+
[email protected]("scheme", ["W4A16", "NVFP4"])
+def test_nvfp4_observer_preserves_autoround_tuned_weights(scheme):
+ import copy
+ from contextlib import nullcontext
+
+ from tokenizers import Tokenizer
+ from tokenizers.models import WordLevel
+ from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedTokenizerFast
+
+ from auto_round import AutoRound
+ from auto_round.experimental.kv_cache import kvcache_quant_context
+
+ torch.manual_seed(42)
+ config = LlamaConfig(
+ vocab_size=16,
+ hidden_size=32,
+ intermediate_size=32,
+ num_hidden_layers=1,
+ num_attention_heads=2,
+ num_key_value_heads=2,
+ max_position_embeddings=16,
+ attn_implementation="eager",
+ )
+ baseline = LlamaForCausalLM(config).eval()
+ observed = copy.deepcopy(baseline)
+ original_weight = baseline.model.layers[0].self_attn.k_proj.weight.detach().clone()
+ tokenizer = PreTrainedTokenizerFast(
+ tokenizer_object=Tokenizer(WordLevel({str(index): index for index in range(config.vocab_size)}, unk_token="0")),
+ unk_token="0",
+ pad_token="0",
+ eos_token="1",
+ )
+ dataset = torch.randint(0, config.vocab_size, (2, 4))
+ for model, dtype in ((baseline, None), (observed, "nvfp4")):
+ compressor = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ scheme=scheme,
+ group_size=16,
+ iters=2,
+ batch_size=1,
+ nsamples=2,
+ seqlen=4,
+ dataset=[{"input_ids": sample.unsqueeze(0).clone()} for sample in dataset],
+ device_map="cpu",
+ amp=False,
+ seed=42,
+ static_kv_dtype=dtype,
+ )
+ context = nullcontext() if dtype is None else kvcache_quant_context(model, dtype)
+ with context:
+ compressor.quantize()
+ for name, expected in baseline.named_parameters():
+ torch.testing.assert_close(observed.get_parameter(name), expected)
+ assert not torch.equal(baseline.model.layers[0].self_attn.k_proj.weight, original_weight)
+ for layer in observed.model.layers:
+ assert layer.self_attn.k_global_scale.item() > 0
+ assert layer.self_attn.v_global_scale.item() > 0
+ with torch.no_grad():
+ torch.testing.assert_close(observed(dataset).logits, baseline(dataset).logits)
diff --git a/test/unit/test_cpu/export/test_llmc_format.py b/test/unit/test_cpu/export/test_llmc_format.py
index e98155176..98d9dcc78 100644
--- a/test/unit/test_cpu/export/test_llmc_format.py
+++ b/test/unit/test_cpu/export/test_llmc_format.py
@@ -129,6 +129,7 @@ class TestLLMCZpInt8:
model = nn.Sequential()
model.add_module("q_proj", layer)
pack_layer("q_proj", model, device="cpu")
+ assert layer.weight_zero_point.stride(-2) == 1
# AutoRound's own fake-quantized weight (unsigned convention)
s = scale.repeat_interleave(gs, dim=1)