PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enable q_scale calibration for deepseek v2 on Gaudi (#1299)
Wang, Mengnid8ed35a8e778
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.ymlindex 7fdb74b45..0bf1a31d6 100644--- a/.azure-pipelines/template/ut-template.yml+++ b/.azure-pipelines/template/ut-template.yml@@ -54,6 +54,7 @@ steps:docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \&& python setup.py bdist_wheel lib \&& pip install dist/*.whl \+ && pip install -r test/test_hpu/requirements.txt \&& pip list"displayName: "HPU Env Setup"diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 0a17a05d0..e3592e406 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -159,6 +159,12 @@ class BasicArgumentParser(argparse.ArgumentParser):basic.add_argument("--enable_torch_compile", action="store_true", help="Enable PyTorch compilation for faster execution. ")+ basic.add_argument(+ "--disable_trust_remote_code",+ action="store_true",+ help="Disable trusting remote code when loading models. "+ "Use for security if you don't trust the model source.",+ )tuning = self.add_argument_group("Tuning Arguments")tuning.add_argument(@@ -349,12 +355,6 @@ class BasicArgumentParser(argparse.ArgumentParser):## ======================= eval =======================eval_args = self.add_argument_group("eval arguments")- eval_args.add_argument(- "--disable_trust_remote_code",- action="store_true",- help="Disable trusting remote code when loading models. "- "Use for security if you don't trust the model source.",- )eval_args.add_argument("--tasks","--task",@@ -661,6 +661,7 @@ def tune(args):layer_config=layer_config,model_dtype=args.model_dtype,momentum=args.momentum,+ trust_remote_code=not args.disable_trust_remote_code,)model_name = args.model.rstrip("/")diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 5d7ba6371..accbc946b 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -256,6 +256,7 @@ class BaseCompressor(object):# Model relatedmodel_dtype = kwargs.pop("model_dtype", None)self.mllm = kwargs.pop("mllm") if "mllm" in kwargs else False+ self.trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else Trueself.diffusion = kwargs.pop("diffusion") if "diffusion" in kwargs else Falseself.quantized = Falseif isinstance(model, str):@@ -264,6 +265,7 @@ class BaseCompressor(object):platform=platform,device="cpu", # always load cpu firstmodel_dtype=model_dtype,+ trust_remote_code=self.trust_remote_code,)elif tokenizer is None and not self.diffusion and iters > 0:raise ValueError("A tokenizer must be set for non-str model input")@@ -1458,7 +1460,7 @@ class BaseCompressor(object):formats = self.formats if hasattr(self, "formats") else None# It is best to modify the model structure in the quantize function and check the format,# because it may cause the gguf format to not be exported normally.- self.model = update_module(self.model, formats=formats)+ self.model = update_module(self.model, formats=formats, trust_remote_code=self.trust_remote_code)# Temporary names must be assigned after handle_moe_model;# placing them earlier would cause them to be removed when the module is replaced.diff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.pyindex bf813e967..348d5524d 100644--- a/auto_round/experimental/attention.py+++ b/auto_round/experimental/attention.py@@ -95,7 +95,7 @@ class QuantizedAttentionImpl(torch.nn.Module):cur_query_max.detach().to(getattr(module, QUERY_MAX_NAME).data.device),)update_parameter_data(module, query_max, QUERY_MAX_NAME)- query, query_scale = per_tensor_fp8_qdq(query, tensor_max=query_max)+ _, query_scale = per_tensor_fp8_qdq(query, tensor_max=query_max)update_parameter_data(module, query_scale.squeeze(0), QUERY_SCALE_NAME)# original attentionreturn ALL_ATTENTION_FUNCTIONS[self._original_impl](diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex a162a0cf9..c9c749324 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -266,7 +266,12 @@ def save_quantized_as_static_fp(quantization_config = QuantizationConfig(config_groups=config_groups,kv_cache_scheme=(- _construct_kv_scheme() if _use_fp8_kv(serialization_dict.get("static_kv_dtype", None)) else None+ _construct_kv_scheme()+ if (+ _use_fp8_kv(serialization_dict.get("static_kv_dtype", None))+ or _use_fp8_kv(serialization_dict.get("static_attention_dtype", None))+ )+ else None),quantization_status=QuantizationStatus.COMPRESSED,ignore=ignore,diff --git a/auto_round/modelling/deepseek_v2.py b/auto_round/modelling/deepseek_v2.pynew file mode 100644index 000000000..6b47a2866--- /dev/null+++ b/auto_round/modelling/deepseek_v2.py@@ -0,0 +1,208 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import inspect+import warnings+from functools import partial+from typing import Callable, Optional++import torch+from transformers.cache_utils import Cache+from transformers.modeling_rope_utils import dynamic_rope_update+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS+from transformers.models.deepseek_v2.configuration_deepseek_v2 import DeepseekV2Config+from transformers.models.deepseek_v2.modeling_deepseek_v2 import (+ eager_attention_forward,+)++from auto_round.modelling.replace_modules import ReplacementModuleBase+from auto_round.utils import is_hpex_available+++def apply_rotary_emb(+ xq: torch.Tensor,+ xk: torch.Tensor,+ cos: torch.Tensor,+ sin: torch.Tensor,+) -> tuple[torch.Tensor, torch.Tensor]:+ cos = cos.unsqueeze(1)+ sin = sin.unsqueeze(1)++ real_xq = xq[:, :, :, ::2] * cos - xq[:, :, :, 1::2] * sin+ img_xq = xq[:, :, :, ::2] * sin + xq[:, :, :, 1::2] * cos++ real_xk = xk[:, :, :, ::2] * cos - xk[:, :, :, 1::2] * sin+ img_xk = xk[:, :, :, ::2] * sin + xk[:, :, :, 1::2] * cos++ xq_out = torch.stack([real_xq, img_xq], dim=-1).flatten(3).type_as(xq)+ xk_out = torch.stack([real_xk, img_xk], dim=-1).flatten(3).type_as(xk)++ return xq_out, xk_out++[email protected]_grad()+@dynamic_rope_update+def rotary_emb_forward(module, x, position_ids):+ inv_freq_expanded = module.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)+ position_ids_expanded = position_ids[:, None, :].float()++ device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"+ with torch.autocast(device_type=device_type, enabled=False): # Force float32+ freqs = (inv_freq_expanded.to(x.device) @ position_ids_expanded).transpose(1, 2)+ cos = torch.cos(freqs) * module.attention_scaling+ sin = torch.sin(freqs) * module.attention_scaling+ return cos, sin+++def attn_forward(+ module,+ hidden_states: torch.Tensor,+ attention_mask: Optional[torch.Tensor] = None,+ cache_position: Optional[torch.LongTensor] = None,+ position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,+ position_ids: Optional[torch.Tensor] = None,+ **kwargs,+) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:+ if "padding_mask" in kwargs:+ warnings.warn(+ "Passing `padding_mask` is deprecated and will be removed in v4.37. "+ "Please make sure use `attention_mask` instead.`"+ )+ batch_size, seq_length = hidden_states.shape[:-1]+ query_shape = (batch_size, seq_length, -1, module.qk_head_dim)+ key_shape = (batch_size, seq_length, -1, module.qk_nope_head_dim + module.v_head_dim)++ if module.q_lora_rank is None:+ q = module.q_proj(hidden_states)+ else:+ q = module.q_b_proj(module.q_a_layernorm(module.q_a_proj(hidden_states)))+ q = q.view(query_shape).transpose(1, 2)+ q_nope, q_pe = torch.split(q, [module.qk_nope_head_dim, module.qk_rope_head_dim], dim=-1)++ compressed_kv = module.kv_a_proj_with_mqa(hidden_states)+ k_nope, k_pe = torch.split(compressed_kv, [module.kv_lora_rank, module.qk_rope_head_dim], dim=-1)+ k_nope = module.kv_b_proj(module.kv_a_layernorm(k_nope)).view(key_shape).transpose(1, 2)+ k_nope, value_states = torch.split(k_nope, [module.qk_nope_head_dim, module.v_head_dim], dim=-1)++ k_pe = k_pe.view(batch_size, 1, seq_length, module.qk_rope_head_dim)+ cos, sin = position_embeddings+ q_pe, k_pe = apply_rotary_emb(q_pe, k_pe, cos.to(q_pe.device), sin.to(q_pe.device))++ k_pe = k_pe.expand(*k_nope.shape[:-1], -1)+ query_states = torch.cat((q_nope, q_pe), dim=-1)+ key_states = torch.cat((k_nope, k_pe), dim=-1)++ past_key_values = None++ # transformers <= 4.55.4 is past_key_value+ if "past_key_values" in kwargs:+ past_key_values = kwargs["past_key_values"]+ elif "past_key_value" in kwargs:+ past_key_values = kwargs["past_key_value"]++ if past_key_values is not None:+ # sin and cos are specific to RoPE models; cache_position needed for the static cache+ cache_kwargs = {"cache_position": cache_position}+ key_states, value_states = past_key_values.update(key_states, value_states, module.layer_idx, cache_kwargs)++ if module.config._attn_implementation == "flash_attention_2" and module.qk_head_dim != module.v_head_dim:+ value_states = torch.nn.functional.pad(value_states, [0, module.qk_head_dim - module.v_head_dim])++ attention_interface: Callable = eager_attention_forward+ if module.config._attn_implementation != "eager":+ attention_interface = ALL_ATTENTION_FUNCTIONS[module.config._attn_implementation]++ attn_output, attn_weights = attention_interface(+ module,+ query_states,+ key_states,+ value_states,+ attention_mask,+ dropout=0.0 if not module.training else module.attention_dropout,+ scaling=module.scaling,+ **kwargs,+ )++ if module.config._attn_implementation == "flash_attention_2" and module.qk_head_dim != module.v_head_dim:+ attn_output = attn_output[:, :, :, : module.v_head_dim]++ attn_output = attn_output.reshape(batch_size, seq_length, -1).contiguous()+ attn_output = module.o_proj(attn_output)+ return attn_output, attn_weights+++class DeepseekV2RotaryEmbedding(ReplacementModuleBase):+ def __init__(self, original, config):+ super().__init__()++ @classmethod+ def original_module_class(cls) -> str:+ """Return the class name of the module this replaces."""+ return "DeepseekV2RotaryEmbedding"++ @classmethod+ def from_original(+ cls,+ original: torch.nn.Module,+ config: DeepseekV2Config,+ **kwargs,+ ) -> "DeepseekV2RotaryEmbedding":+ """Create an instance from the original module."""+ original.forward = partial(rotary_emb_forward, original)+ return original++ @classmethod+ def is_to_be_replaced(+ cls,+ original: torch.nn.Module,+ ) -> bool:+ """Determine if the given module should be replaced."""+ return (+ cls.is_registered(original.__class__.__name__)+ and is_hpex_available()+ and inspect.getfile(original.__class__) == inspect.getfile(eager_attention_forward)+ )+++class DeepseekV2Attention(ReplacementModuleBase):+ def __init__(self, original, config):+ super().__init__()++ @classmethod+ def original_module_class(cls) -> str:+ """Return the class name of the module this replaces."""+ return "DeepseekV2Attention"++ @classmethod+ def from_original(+ cls,+ original: torch.nn.Module,+ config: DeepseekV2Config,+ **kwargs,+ ) -> "DeepseekV2Attention":+ """Create an instance from the original module."""+ original.forward = partial(attn_forward, original)+ return original++ @classmethod+ def is_to_be_replaced(+ cls,+ original: torch.nn.Module,+ ) -> bool:+ """Determine if the given module should be replaced."""+ return (+ cls.is_registered(original.__class__.__name__)+ and is_hpex_available()+ and inspect.getfile(original.__class__) == inspect.getfile(eager_attention_forward)+ )diff --git a/auto_round/modelling/replace_modules.py b/auto_round/modelling/replace_modules.pyindex c8e34954f..591ed891d 100644--- a/auto_round/modelling/replace_modules.py+++ b/auto_round/modelling/replace_modules.py@@ -25,6 +25,7 @@ BUILTIN_MODULES = {"Llama4TextMoe": LazyImport("auto_round.modelling.llama4"),"GptOssMLP": LazyImport("auto_round.modelling.gpt_oss"),"Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modelling.qwen3_vl_moe"),+ "DeepseekV2Attention": LazyImport("auto_round.modelling.deepseek_v2"),}@@ -94,13 +95,12 @@ class ReplacementModuleBase(ABC, torch.nn.Module):def is_to_be_replaced(cls,original: torch.nn.Module,- module_class_name: str,) -> bool:"""Determine if the given module should be replaced.Users can extend this method to add custom logic for replacement."""- return cls.is_registered(module_class_name)+ return cls.is_registered(original.__class__.__name__)@classmethoddef get_registered_modules(cls) -> list:@@ -156,7 +156,9 @@ def apply_replacements(if isinstance(module, ReplacementModuleBase):continueclass_name = module.__class__.__name__- if ReplacementModuleBase.is_to_be_replaced(module, class_name):+ if ReplacementModuleBase.is_registered(class_name) and ReplacementModuleBase.get_replacement_class(+ class_name+ ).is_to_be_replaced(module):modules_to_replace.append((name, module, class_name))# Step 2: Replace modulesdiff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 3bb60bfb0..c7e5b5d90 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -52,7 +52,7 @@ def _handle_special_model(model):return model-def update_module(model, formats: list[OutputFormat] = None):+def update_module(model, formats: list[OutputFormat] = None, trust_remote_code: bool = True):if formats is not None and any([format_.is_gguf() for format_ in formats]):return modeldiff --git a/test/test_cpu/quantization/test_statc_attn.py b/test/test_cpu/quantization/test_statc_attn.pynew file mode 100644index 000000000..8a133cc02--- /dev/null+++ b/test/test_cpu/quantization/test_statc_attn.py@@ -0,0 +1,80 @@+import shutil++import pytest+import torch+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound++from ...helpers import get_model_path++deepseekv2_model_name = get_model_path("deepseek-ai/DeepSeek-V2-Lite-Chat")+deepseekv3_model_name = get_model_path("tflsxyy/DeepSeek-V3-bf16-4layers")+++def setup_deepseekv2():+ """Fixture to set up model and tokenizer."""+ model_name = deepseekv2_model_name+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=False)+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)+ config.num_hidden_layers = 1 # Reduce layers for testing+ model = AutoModelForCausalLM.from_config(config)+ output_dir = "./tmp/test_quantized_deepseekv2"+ return model, tokenizer, output_dir, config+++def setup_deepseekv3():+ """Fixture to set up model and tokenizer."""+ model_name = deepseekv3_model_name+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=False)+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)+ config.num_hidden_layers = 1 # Reduce layers for testing+ model = AutoModelForCausalLM.from_config(config)+ output_dir = "./tmp/test_quantized_deepseekv3"+ return model, tokenizer, output_dir, config+++def test_deepseek_v2(setup_deepseekv2):+ model, tokenizer, output_dir, config = setup_deepseekv2+ autoround = AutoRound(+ model,+ tokenizer,+ scheme="FP8_STATIC",+ static_attention_dtype="fp8",+ iters=0,+ seqlen=2,+ trust_remote_code=False,+ )+ quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)+ assert quantized_model is not None, "Expected quantized_model to be not None"+ device = quantized_model.model.layers[0].self_attn.q_scale.device+ assert not all(+ quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)+ ), "q_scale is not collected"++ # clean the output directory after test+ shutil.rmtree(output_dir, ignore_errors=True)+++def test_deepseek_v3(setup_deepseekv3):+ model, tokenizer, output_dir, config = setup_deepseekv3+ autoround = AutoRound(+ model,+ tokenizer,+ scheme="FP8_STATIC",+ static_attention_dtype="fp8",+ iters=0,+ seqlen=2,+ trust_remote_code=False,+ )+ quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)+ assert quantized_model is not None, "Expected quantized_model to be not None"+ device = quantized_model.model.layers[0].self_attn.q_scale.device+ assert not all(+ quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)+ ), "q_scale is not collected"++ # clean the output directory after test+ shutil.rmtree(output_dir, ignore_errors=True)diff --git a/test/test_hpu/requirements.txt b/test/test_hpu/requirements.txtindex 8b1378917..8a05844b9 100644--- a/test/test_hpu/requirements.txt+++ b/test/test_hpu/requirements.txt@@ -1 +1 @@-+compressed_tensorsdiff --git a/test/test_hpu/test_statc_attn.py b/test/test_hpu/test_statc_attn.pynew file mode 100644index 000000000..ac4845ceb--- /dev/null+++ b/test/test_hpu/test_statc_attn.py@@ -0,0 +1,48 @@+import shutil++import pytest+import torch+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from auto_round.utils import is_hpex_available++from ..helpers import get_model_path, is_pytest_mode_lazy++deepseekv2_model_name = get_model_path("deepseek-ai/DeepSeek-V2-Lite-Chat")+++def setup_deepseekv2():+ """Fixture to set up model and tokenizer."""+ model_name = deepseekv2_model_name+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=False)+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)+ config.num_hidden_layers = 1 # Reduce layers for testing+ model = AutoModelForCausalLM.from_config(config)+ output_dir = "./tmp/test_quantized_deepseekv2"+ return model, tokenizer, output_dir, config++[email protected](not is_hpex_available(), reason="HPU is not supported")[email protected](not is_pytest_mode_lazy(), reason="Only for lazy mode")+def test_deepseek_v2_on_hpu(setup_deepseekv2):+ model, tokenizer, output_dir, config = setup_deepseekv2+ autoround = AutoRound(+ model,+ tokenizer,+ scheme="FP8_STATIC",+ static_attention_dtype="fp8",+ iters=0,+ seqlen=2,+ trust_remote_code=False,+ )+ quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)+ assert quantized_model is not None, "Expected quantized_model to be not None"+ device = quantized_model.model.layers[0].self_attn.q_scale.device+ assert not all(+ quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)+ ), "q_scale is not collected"++ # clean the output directory after test+ shutil.rmtree(output_dir, ignore_errors=True)