PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: update scale handling in QuantizedKVParameterCache to use amax of all batches (#2533)
Xin He512872d077d5
diff --git a/auto_round/experimental/kv_cache.py b/auto_round/experimental/kv_cache.pyindex 83bef24f3..0148a024e 100644--- a/auto_round/experimental/kv_cache.py+++ b/auto_round/experimental/kv_cache.py@@ -197,7 +197,10 @@ class QuantizedKVParameterCache(DynamicCache):qdq_tensor, scale = fp8_qdq(tensor, granularity=self.granularity)# Detach scale to prevent holding computation graph references- _pad_and_append_at_idx_(scales, layer_idx, scale.reshape(-1).detach())+ scale = scale.reshape(-1).detach()+ if layer_idx < len(scales) and scales[layer_idx] is not None:+ scale = torch.maximum(scales[layer_idx], scale)+ _pad_and_append_at_idx_(scales, layer_idx, scale)return qdq_tensordef _nvfp4_track_amax(self, tensor: torch.Tensor, kv_type: KVCacheScaleType, layer_idx: int):@@ -347,6 +350,9 @@ def kvcache_quant_context(model: torch.nn.Module, static_kv_dtype=torch.float8_e4m3fn, static_kv_granularity: str = "tensor"):"""Context manager for static KV cache quantization (FP8 or NVFP4) operations."""+ hook_handles = []+ QuantizedKVParameterCache._instance = None+ QuantizedKVParameterCache._initialized = Falsetry:# Setup phase: Initialize KV cache for quantizationstatic_kv_dtype = normalize_static_kv_dtype(static_kv_dtype)@@ -368,11 +374,18 @@ def kvcache_quant_context()if static_kv_dtype in (torch.float8_e4m3fn, NVFP4_KV_DTYPE):model.apply(initialize_fn)- model.apply(prep_attention_module_for_calibration)+ for module in model.modules():+ if is_attention_module(module):+ hook_handles.append(+ module.register_forward_pre_hook(calibrate_kv_cache_input_hook, with_kwargs=True)+ )+ hook_handles.append(module.register_forward_hook(calibrate_kv_cache_output_hook))# Provide the model to the with blockyield modelfinally:# Cleanup phase: Freeze quantization parameters+ for handle in hook_handles:+ handle.remove()model.apply(freeze_module_quantization_)diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex 22af10d1e..66a9ac20e 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -311,6 +311,13 @@ def pack_layer(name, model, device=None):delattr(layer, "scale")_compress_and_set_format(layer, scheme, device)+ if hasattr(layer, "weight_zero_point"):+ zero_point = layer.weight_zero_point.data.contiguous()+ if scheme.weights.num_bits == 4 and zero_point.ndim >= 2 and zero_point.shape[-2] == 1:+ stride = list(zero_point.stride())+ stride[-2] = 1+ zero_point = zero_point.as_strided(zero_point.shape, stride)+ layer.weight_zero_point.data = zero_point@torch.no_grad()diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex ffb135acf..7fdbd19b8 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -454,7 +454,8 @@ def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):"""Get block names for Qwen3-Omni MoE model.Qwen3-Omni has the following structure:- - thinker: Contains audio_tower, visual, model (text decoder)+ - thinker.model: Contains visual/audio towers and the text decoder in newer Transformers releases+ - thinker: Contains visual/audio towers directly in older Transformers releases- talker: Contains model (talker decoder), code_predictor- code2wav: Audio decoder@@ -466,26 +467,56 @@ def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):degrade audio quality in long-form generation ."""block_names = []+ thinker = getattr(model, "thinker", None)+ thinker_model = getattr(thinker, "model", None)# Quantize visual encoder blocks if quant_vision is enabledif quant_vision:- # Vision encoder blocks- if hasattr(model, "thinker") and hasattr(model.thinker, "visual") and hasattr(model.thinker.visual, "blocks"):- block_names.append([f"thinker.visual.blocks.{i}" for i in range(len(model.thinker.visual.blocks))])- # Audio encoder layers- if hasattr(model, "thinker") and hasattr(model.thinker, "audio_tower"):- if hasattr(model.thinker.audio_tower, "layers"):- block_names.append(- [f"thinker.audio_tower.layers.{i}" for i in range(len(model.thinker.audio_tower.layers))]- )-- # Thinker text model layers (main LLM decoder)- if hasattr(model, "thinker") and hasattr(model.thinker, "model") and hasattr(model.thinker.model, "layers"):- block_names.append([f"thinker.model.layers.{i}" for i in range(len(model.thinker.model.layers))])+ visual = getattr(thinker_model, "visual", None)+ visual_blocks = getattr(visual, "blocks", None)+ visual_prefix = "thinker.model.visual"+ if visual_blocks is None or len(visual_blocks) == 0:+ visual = getattr(thinker, "visual", None)+ visual_blocks = getattr(visual, "blocks", None)+ visual_prefix = "thinker.visual"+ if visual_blocks is not None:+ block_names.append([f"{visual_prefix}.blocks.{i}" for i in range(len(visual_blocks))])++ audio_tower = getattr(thinker_model, "audio_tower", None)+ audio_layers = getattr(audio_tower, "layers", None)+ audio_prefix = "thinker.model.audio_tower"+ if audio_layers is None or len(audio_layers) == 0:+ audio_tower = getattr(thinker, "audio_tower", None)+ audio_layers = getattr(audio_tower, "layers", None)+ audio_prefix = "thinker.audio_tower"+ if audio_layers is not None:+ block_names.append([f"{audio_prefix}.layers.{i}" for i in range(len(audio_layers))])++ # Thinker text model layers (main LLM decoder). Transformers may wrap the+ # decoder in `language_model` in newer Qwen3-Omni releases.+ layer_prefix, layers = _get_qwen3_omni_thinker_layer_info(model)+ if layers is not None:+ block_names.append([f"{layer_prefix}.{i}" for i in range(len(layers))])return block_names+def _get_qwen3_omni_thinker_layer_info(model):+ """Return the module path and layers for either supported thinker layout."""+ thinker_model = getattr(getattr(model, "thinker", None), "model", None)+ if thinker_model is None:+ return None, None++ if hasattr(thinker_model, "layers"):+ return "thinker.model.layers", thinker_model.layers++ language_model = getattr(thinker_model, "language_model", None)+ if language_model is not None and hasattr(language_model, "layers"):+ return "thinker.model.language_model.layers", language_model.layers++ return None, None++def _get_glm_image_multimodal_block(model, quant_vision=False):"""Get block names for GLM-Image AR model.diff --git a/test/unit/common/models/test_omni_model.py b/test/unit/common/models/test_omni_model.pyindex ddf417f66..8b33c21ef 100644--- a/test/unit/common/models/test_omni_model.py+++ b/test/unit/common/models/test_omni_model.py@@ -36,6 +36,14 @@ pytestmark = pytest.mark.skipif()+def _qwen3_omni_thinker_layers(model):+ from auto_round.special_model_handler import _get_qwen3_omni_thinker_layer_info++ layer_prefix, layers = _get_qwen3_omni_thinker_layer_info(model)+ assert layers is not None+ return layer_prefix, layers++# ========================= Qwen2.5-Omni Tests =============================# NOTE: Tests use the `tiny_qwen2_5_omni` session-scoped fixture from fixtures.py# (real config, reduced layers, random weights). Skipped if model not available.@@ -123,6 +131,17 @@ class TestQwen2_5OmniProcessor:class TestQwen3OmniMoeBlockNames:"""Test block name discovery for Qwen3-Omni-MoE."""+ def test_block_names_language_model_nested_layers(self):+ from types import SimpleNamespace++ from auto_round.special_model_handler import _get_qwen3_omni_moe_multimodal_block++ model = SimpleNamespace(+ thinker=SimpleNamespace(model=SimpleNamespace(language_model=SimpleNamespace(layers=[object(), object()])))+ )+ block_names = _get_qwen3_omni_moe_multimodal_block(model)+ assert block_names == [["thinker.model.language_model.layers.0", "thinker.model.language_model.layers.1"]]+def test_block_names_default(self):"""Test that get_block_names returns thinker layers only by default."""from auto_round.utils import get_block_names@@ -131,9 +150,8 @@ class TestQwen3OmniMoeBlockNames:model = Qwen3OmniMoeForConditionalGeneration(config)block_names = get_block_names(model, quant_vision=False)- assert any(- "thinker.model.layers" in str(b) for b in block_names- ), f"Expected thinker.model.layers, got: {block_names}"+ layer_prefix, _ = _qwen3_omni_thinker_layers(model)+ assert any(layer_prefix in str(b) for b in block_names), f"Expected {layer_prefix}, got: {block_names}"assert not any("talker.model.layers" in str(b) for b in block_names), f"Did not expect talker.model.layers by default, got: {block_names}"@@ -205,7 +223,8 @@ class TestQwen3OmniMoeReplacement:model = apply_replacements(model)# Check that thinker MoE was custom-replaced- thinker_mlp = model.thinker.model.layers[0].mlp+ _, thinker_layers = _qwen3_omni_thinker_layers(model)+ thinker_mlp = thinker_layers[0].mlpassert ("LinearQwen3OmniThinker" in thinker_mlp.__class__.__name__), f"Expected LinearQwen3OmniThinker, got {thinker_mlp.__class__.__name__}"@@ -238,8 +257,9 @@ class TestQwen3OmniMoeReplacement:model = Qwen3OmniMoeForConditionalGeneration(config)# Save original fused thinker weights- thinker_gate_up = model.thinker.model.layers[0].mlp.experts.gate_up_proj.data.clone()- thinker_down = model.thinker.model.layers[0].mlp.experts.down_proj.data.clone()+ _, thinker_layers = _qwen3_omni_thinker_layers(model)+ thinker_gate_up = thinker_layers[0].mlp.experts.gate_up_proj.data.clone()+ thinker_down = thinker_layers[0].mlp.experts.down_proj.data.clone()model = apply_replacements(model)materialize_model_(model)@@ -249,7 +269,8 @@ class TestQwen3OmniMoeReplacement:# Use equal_nan=True because fused expert parameters may contain# uninitialized memory with NaN bit patterns, and NaN != NaN in IEEE 754.for i in range(4):- expert = model.thinker.model.layers[0].mlp.experts[i]+ _, thinker_layers = _qwen3_omni_thinker_layers(model)+ expert = thinker_layers[0].mlp.experts[i]assert_same_weights(expert.gate_proj.weight.data, thinker_gate_up[i, :intermediate, :])assert_same_weights(expert.up_proj.weight.data, thinker_gate_up[i, intermediate:, :])assert_same_weights(expert.down_proj.weight.data, thinker_down[i])@@ -264,16 +285,18 @@ class TestQwen3OmniMoeReplacement:torch.manual_seed(0)config = make_tiny_qwen3_omni_moe_config()model = Qwen3OmniMoeForConditionalGeneration(config)+ _, thinker_layers = _qwen3_omni_thinker_layers(model)x = torch.randn(1, 4, 64) * 0.1with torch.no_grad():- orig_thinker_out = model.thinker.model.layers[0].mlp(x)+ orig_thinker_out = thinker_layers[0].mlp(x)model = apply_replacements(model)materialize_model_(model)+ _, thinker_layers = _qwen3_omni_thinker_layers(model)with torch.no_grad():- new_thinker_out = model.thinker.model.layers[0].mlp(x)+ new_thinker_out = thinker_layers[0].mlp(x)assert orig_thinker_out.shape == new_thinker_out.shape, "Thinker shape mismatch"orig_nan = torch.isnan(orig_thinker_out)@@ -308,7 +331,8 @@ class TestQwen3OmniMoeUtils:config = make_tiny_qwen3_omni_moe_config()model = Qwen3OmniMoeForConditionalGeneration(config)- moe_block = model.thinker.model.layers[0].mlp+ _, thinker_layers = _qwen3_omni_thinker_layers(model)+ moe_block = thinker_layers[0].mlpassert is_moe_layer(moe_block), f"Thinker MoE block ({moe_block.__class__.__name__}) should be detected as MoE"def test_is_moe_layer_talker(self):@@ -325,7 +349,8 @@ class TestQwen3OmniMoeUtils:config = make_tiny_qwen3_omni_moe_config()model = Qwen3OmniMoeForConditionalGeneration(config)- thinker_mlp = model.thinker.model.layers[0].mlp+ _, thinker_layers = _qwen3_omni_thinker_layers(model)+ thinker_mlp = thinker_layers[0].mlpnames = get_expert_linear_names(thinker_mlp)assert set(names) == {"gate_proj", "down_proj", "up_proj"}@@ -339,7 +364,8 @@ class TestQwen3OmniMoeUtils:config = make_tiny_qwen3_omni_moe_config()model = Qwen3OmniMoeForConditionalGeneration(config)- thinker_mlp = model.thinker.model.layers[0].mlp+ _, thinker_layers = _qwen3_omni_thinker_layers(model)+ thinker_mlp = thinker_layers[0].mlpnames = get_expert_input_proj_names(thinker_mlp)assert set(names) == {"gate_proj", "up_proj"}@@ -349,9 +375,10 @@ class TestQwen3OmniMoeUtils:config = make_tiny_qwen3_omni_moe_config()model = Qwen3OmniMoeForConditionalGeneration(config)+ layer_prefix, _ = _qwen3_omni_thinker_layers(model)ignore_layers = get_predefined_ignore_layers(model)assert ignore_layers == [- "thinker.model.layers.0.mlp.gate",+ f"{layer_prefix}.0.mlp.gate","talker.model.layers.0.mlp.gate","talker.model.layers.0.mlp.shared_expert_gate",], f"Expected mlp.gate in ignore_layers for qwen3_omni_moe, got: {ignore_layers}"diff --git a/test/unit/common/models/test_special_model_handler.py b/test/unit/common/models/test_special_model_handler.pyindex 5bfcebe97..dc6b635b8 100644--- a/test/unit/common/models/test_special_model_handler.py+++ b/test/unit/common/models/test_special_model_handler.py@@ -727,6 +727,36 @@ class TestGetQwen25OmniMultimodalBlock:class TestGetQwen3OmniMoeMultimodalBlock:"""Test _get_qwen3_omni_moe_multimodal_block function."""+ def test_with_nested_vision_and_audio_towers(self):+ from types import SimpleNamespace++ from auto_round.special_model_handler import _get_qwen3_omni_moe_multimodal_block++ model = SimpleNamespace(+ thinker=SimpleNamespace(+ model=SimpleNamespace(+ visual=SimpleNamespace(blocks=[MagicMock() for _ in range(2)]),+ audio_tower=SimpleNamespace(layers=[MagicMock() for _ in range(3)]),+ language_model=SimpleNamespace(layers=[MagicMock() for _ in range(4)]),+ )+ )+ )+ block_names = _get_qwen3_omni_moe_multimodal_block(model, quant_vision=True)+ assert block_names == [+ ["thinker.model.visual.blocks.0", "thinker.model.visual.blocks.1"],+ [+ "thinker.model.audio_tower.layers.0",+ "thinker.model.audio_tower.layers.1",+ "thinker.model.audio_tower.layers.2",+ ],+ [+ "thinker.model.language_model.layers.0",+ "thinker.model.language_model.layers.1",+ "thinker.model.language_model.layers.2",+ "thinker.model.language_model.layers.3",+ ],+ ]+def test_returns_thinker_layers(self):from auto_round.special_model_handler import _get_qwen3_omni_moe_multimodal_blockdiff --git a/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py b/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.pyindex 535c6273a..eda62a8a8 100644--- a/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py+++ b/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py@@ -14,7 +14,6 @@ from auto_round.experimental.kv_cache import QuantizedKVParameterCachedef _reset_singleton():- # ``reset`` is an instance method; clear the class state directly.QuantizedKVParameterCache._instance = NoneQuantizedKVParameterCache._initialized = False@@ -125,3 +124,207 @@ def test_nvfp4_kv_incompatible_head_dim_raises_at_update():bad = torch.randn(1, 1, 16, 88)with pytest.raises(ValueError, match="divisible by 16"):cache.update(bad, bad.clone(), layer_idx=0)+++class ObservedAttention(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layer_idx = 0+ self.head_dim = 16+ self.k_proj = torch.nn.Linear(16, 16, bias=False)+ self.v_proj = torch.nn.Linear(16, 16, bias=False)++ def forward(self, hidden_states, past_key_values=None, use_cache=False):+ key_states = self.k_proj(hidden_states).unsqueeze(1)+ value_states = self.v_proj(hidden_states).unsqueeze(1)+ if past_key_values is not None:+ key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)+ query_states = hidden_states.unsqueeze(1)+ return torch.nn.functional.scaled_dot_product_attention(query_states, key_states, value_states)+++def test_nvfp4_observer_preserves_optimization_trajectory():+ import copy+ from contextlib import nullcontext++ from auto_round.experimental.kv_cache import kvcache_quant_context++ torch.manual_seed(42)+ baseline = ObservedAttention()+ observed = copy.deepcopy(baseline)+ inputs = [torch.randn(2, 4, 16) for _ in range(3)]+ targets = [torch.randn(2, 1, 4, 16) for _ in inputs]+ trajectories = []+ for model, context in ((baseline, nullcontext()), (observed, kvcache_quant_context(observed, "nvfp4"))):+ trajectory = []+ parameters = [model.k_proj.weight, model.v_proj.weight]+ with context:+ for hidden_states, target in zip(inputs, targets):+ model.zero_grad(set_to_none=True)+ output = model(hidden_states, past_key_values=None)+ loss = torch.nn.functional.mse_loss(output, target)+ loss.backward()+ gradients = [parameter.grad.detach().clone() for parameter in parameters]+ with torch.no_grad():+ for parameter in parameters:+ parameter.sub_(0.01 * parameter.grad.sign())+ trajectory.append((output.detach(), loss.detach(), gradients))+ trajectories.append(trajectory)+ for expected, actual in zip(*trajectories):+ torch.testing.assert_close(actual[0], expected[0])+ torch.testing.assert_close(actual[1], expected[1])+ for expected_gradient, actual_gradient in zip(expected[2], actual[2]):+ torch.testing.assert_close(actual_gradient, expected_gradient)+ torch.testing.assert_close(actual_gradient.sign(), expected_gradient.sign(), rtol=0, atol=0)+ for name, expected in baseline.named_parameters():+ torch.testing.assert_close(observed.get_parameter(name), expected, rtol=0, atol=0)++[email protected]("dtype", ["nvfp4", "fp8"])+def test_kv_context_removes_its_hooks(dtype):+ from auto_round.experimental.kv_cache import kvcache_quant_context++ model = ObservedAttention()+ inputs = torch.randn(1, 4, 16)+ expected = model(inputs, past_key_values=None)+ with kvcache_quant_context(model, dtype):+ model(inputs, past_key_values=None)+ actual = model(inputs, past_key_values=None)+ torch.testing.assert_close(actual, expected)+++def test_kv_context_does_not_reuse_previous_amax():+ from auto_round.experimental.kv_cache import kvcache_quant_context++ model = ObservedAttention()+ with kvcache_quant_context(model, "nvfp4"):+ model.kv_cache.update(torch.full((1, 1, 1, 16), 100.0), torch.ones(1, 1, 1, 16), 0)+ with kvcache_quant_context(model, "nvfp4"):+ model.kv_cache.update(torch.ones(1, 1, 1, 16), torch.ones(1, 1, 1, 16), 0)+ assert model.kv_cache.k_amax[0] == 1.0+++def test_kv_context_cleans_up_after_exception():+ from auto_round.experimental.kv_cache import kvcache_quant_context++ model = ObservedAttention()+ with pytest.raises(RuntimeError, match="calibration failed"), kvcache_quant_context(model, "nvfp4"):+ raise RuntimeError("calibration failed")+ assert not hasattr(model, "kv_cache")+ model(torch.randn(1, 4, 16), past_key_values=None)+++def test_kv_context_preserves_unrelated_hooks():+ from auto_round.experimental.kv_cache import kvcache_quant_context++ model = ObservedAttention()+ calls = []+ handle = model.register_forward_hook(lambda module, args, output: calls.append(True))+ with kvcache_quant_context(model, "nvfp4"):+ model(torch.randn(1, 4, 16), past_key_values=None)+ model(torch.randn(1, 4, 16), past_key_values=None)+ assert calls == [True, True]+ handle.remove()++[email protected]("granularity", ["tensor", "head"])+def test_fp8_update_quantizes_and_keeps_running_max_scale(granularity):+ cache = QuantizedKVParameterCache(dtype="fp8", granularity=granularity)+ key_states = torch.linspace(-3.1, 2.7, 64).reshape(1, 2, 2, 16)+ value_states = key_states * 2.7+ actual_keys, actual_values = cache.update(key_states, value_states, 0)+ assert not torch.equal(actual_keys, key_states)+ assert not torch.equal(actual_values, value_states)+ initial_k_scale = cache.k_scales[0].clone()+ initial_v_scale = cache.v_scales[0].clone()+ cache.update(key_states / 4, value_states / 4, 0)+ torch.testing.assert_close(cache.k_scales[0], initial_k_scale)+ torch.testing.assert_close(cache.v_scales[0], initial_v_scale)++[email protected]("granularity", ["tensor", "head"])+def test_fp8_update_running_max_per_side_head_and_layer(granularity):+ cache = QuantizedKVParameterCache(dtype="fp8", granularity=granularity)+ first_keys = torch.tensor([8.0, 1.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)+ first_values = torch.tensor([2.0, 16.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)+ cache.update(first_keys, first_values, 2)+ second_keys = torch.tensor([2.0, 4.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)+ second_values = torch.tensor([32.0, 1.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)+ cache.reset_states()+ cache.update(second_keys, second_values, 2)+ expected_keys = torch.tensor([8.0, 4.0] if granularity == "head" else [8.0]) / 448+ expected_values = torch.tensor([32.0, 16.0] if granularity == "head" else [32.0]) / 448+ torch.testing.assert_close(cache.k_scales[2], expected_keys)+ torch.testing.assert_close(cache.v_scales[2], expected_values)+ assert not cache.k_scales[2].requires_grad+ assert not cache.v_scales[2].requires_grad++ cache.update(second_keys / 8, second_values / 8, 0)+ expected_layer_keys = torch.tensor([0.25, 0.5] if granularity == "head" else [0.5]) / 448+ expected_layer_values = torch.tensor([4.0, 0.125] if granularity == "head" else [4.0]) / 448+ torch.testing.assert_close(cache.k_scales[0], expected_layer_keys)+ torch.testing.assert_close(cache.v_scales[0], expected_layer_values)+ torch.testing.assert_close(cache.k_scales[2], expected_keys)+ torch.testing.assert_close(cache.v_scales[2], expected_values)++[email protected]("scheme", ["W4A16", "NVFP4"])+def test_nvfp4_observer_preserves_autoround_tuned_weights(scheme):+ import copy+ from contextlib import nullcontext++ from tokenizers import Tokenizer+ from tokenizers.models import WordLevel+ from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedTokenizerFast++ from auto_round import AutoRound+ from auto_round.experimental.kv_cache import kvcache_quant_context++ torch.manual_seed(42)+ config = LlamaConfig(+ vocab_size=16,+ hidden_size=32,+ intermediate_size=32,+ num_hidden_layers=1,+ num_attention_heads=2,+ num_key_value_heads=2,+ max_position_embeddings=16,+ attn_implementation="eager",+ )+ baseline = LlamaForCausalLM(config).eval()+ observed = copy.deepcopy(baseline)+ original_weight = baseline.model.layers[0].self_attn.k_proj.weight.detach().clone()+ tokenizer = PreTrainedTokenizerFast(+ tokenizer_object=Tokenizer(WordLevel({str(index): index for index in range(config.vocab_size)}, unk_token="0")),+ unk_token="0",+ pad_token="0",+ eos_token="1",+ )+ dataset = torch.randint(0, config.vocab_size, (2, 4))+ for model, dtype in ((baseline, None), (observed, "nvfp4")):+ compressor = AutoRound(+ model,+ tokenizer=tokenizer,+ scheme=scheme,+ group_size=16,+ iters=2,+ batch_size=1,+ nsamples=2,+ seqlen=4,+ dataset=[{"input_ids": sample.unsqueeze(0).clone()} for sample in dataset],+ device_map="cpu",+ amp=False,+ seed=42,+ static_kv_dtype=dtype,+ )+ context = nullcontext() if dtype is None else kvcache_quant_context(model, dtype)+ with context:+ compressor.quantize()+ for name, expected in baseline.named_parameters():+ torch.testing.assert_close(observed.get_parameter(name), expected)+ assert not torch.equal(baseline.model.layers[0].self_attn.k_proj.weight, original_weight)+ for layer in observed.model.layers:+ assert layer.self_attn.k_global_scale.item() > 0+ assert layer.self_attn.v_global_scale.item() > 0+ with torch.no_grad():+ torch.testing.assert_close(observed(dataset).logits, baseline(dataset).logits)diff --git a/test/unit/test_cpu/export/test_llmc_format.py b/test/unit/test_cpu/export/test_llmc_format.pyindex e98155176..98d9dcc78 100644--- a/test/unit/test_cpu/export/test_llmc_format.py+++ b/test/unit/test_cpu/export/test_llmc_format.py@@ -129,6 +129,7 @@ class TestLLMCZpInt8:model = nn.Sequential()model.add_module("q_proj", layer)pack_layer("q_proj", model, device="cpu")+ assert layer.weight_zero_point.stride(-2) == 1# AutoRound's own fake-quantized weight (unsigned convention)s = scale.repeat_interleave(gs, dim=1)