Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enable q_scale calibration for deepseek v2 on Gaudi (#1299)

Wang, Mengnid8ed35a8e778
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index 7fdb74b45..0bf1a31d6 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -54,6 +54,7 @@ steps:
docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \
&& python setup.py bdist_wheel lib \
&& pip install dist/*.whl \
+ && pip install -r test/test_hpu/requirements.txt \
&& pip list"
displayName: "HPU Env Setup"
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 0a17a05d0..e3592e406 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -159,6 +159,12 @@ class BasicArgumentParser(argparse.ArgumentParser):
basic.add_argument(
"--enable_torch_compile", action="store_true", help="Enable PyTorch compilation for faster execution. "
)
+ basic.add_argument(
+ "--disable_trust_remote_code",
+ action="store_true",
+ help="Disable trusting remote code when loading models. "
+ "Use for security if you don't trust the model source.",
+ )
tuning = self.add_argument_group("Tuning Arguments")
tuning.add_argument(
@@ -349,12 +355,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
## ======================= eval =======================
eval_args = self.add_argument_group("eval arguments")
- eval_args.add_argument(
- "--disable_trust_remote_code",
- action="store_true",
- help="Disable trusting remote code when loading models. "
- "Use for security if you don't trust the model source.",
- )
eval_args.add_argument(
"--tasks",
"--task",
@@ -661,6 +661,7 @@ def tune(args):
layer_config=layer_config,
model_dtype=args.model_dtype,
momentum=args.momentum,
+ trust_remote_code=not args.disable_trust_remote_code,
)
model_name = args.model.rstrip("/")
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 5d7ba6371..accbc946b 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -256,6 +256,7 @@ class BaseCompressor(object):
# Model related
model_dtype = kwargs.pop("model_dtype", None)
self.mllm = kwargs.pop("mllm") if "mllm" in kwargs else False
+ self.trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else True
self.diffusion = kwargs.pop("diffusion") if "diffusion" in kwargs else False
self.quantized = False
if isinstance(model, str):
@@ -264,6 +265,7 @@ class BaseCompressor(object):
platform=platform,
device="cpu", # always load cpu first
model_dtype=model_dtype,
+ trust_remote_code=self.trust_remote_code,
)
elif tokenizer is None and not self.diffusion and iters > 0:
raise ValueError("A tokenizer must be set for non-str model input")
@@ -1458,7 +1460,7 @@ class BaseCompressor(object):
formats = self.formats if hasattr(self, "formats") else None
# It is best to modify the model structure in the quantize function and check the format,
# because it may cause the gguf format to not be exported normally.
- self.model = update_module(self.model, formats=formats)
+ self.model = update_module(self.model, formats=formats, trust_remote_code=self.trust_remote_code)
# Temporary names must be assigned after handle_moe_model;
# placing them earlier would cause them to be removed when the module is replaced.
diff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.py
index bf813e967..348d5524d 100644
--- a/auto_round/experimental/attention.py
+++ b/auto_round/experimental/attention.py
@@ -95,7 +95,7 @@ class QuantizedAttentionImpl(torch.nn.Module):
cur_query_max.detach().to(getattr(module, QUERY_MAX_NAME).data.device),
)
update_parameter_data(module, query_max, QUERY_MAX_NAME)
- query, query_scale = per_tensor_fp8_qdq(query, tensor_max=query_max)
+ _, query_scale = per_tensor_fp8_qdq(query, tensor_max=query_max)
update_parameter_data(module, query_scale.squeeze(0), QUERY_SCALE_NAME)
# original attention
return ALL_ATTENTION_FUNCTIONS[self._original_impl](
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index a162a0cf9..c9c749324 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -266,7 +266,12 @@ def save_quantized_as_static_fp(
quantization_config = QuantizationConfig(
config_groups=config_groups,
kv_cache_scheme=(
- _construct_kv_scheme() if _use_fp8_kv(serialization_dict.get("static_kv_dtype", None)) else None
+ _construct_kv_scheme()
+ if (
+ _use_fp8_kv(serialization_dict.get("static_kv_dtype", None))
+ or _use_fp8_kv(serialization_dict.get("static_attention_dtype", None))
+ )
+ else None
),
quantization_status=QuantizationStatus.COMPRESSED,
ignore=ignore,
diff --git a/auto_round/modelling/deepseek_v2.py b/auto_round/modelling/deepseek_v2.py
new file mode 100644
index 000000000..6b47a2866
--- /dev/null
+++ b/auto_round/modelling/deepseek_v2.py
@@ -0,0 +1,208 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import warnings
+from functools import partial
+from typing import Callable, Optional
+
+import torch
+from transformers.cache_utils import Cache
+from transformers.modeling_rope_utils import dynamic_rope_update
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+from transformers.models.deepseek_v2.configuration_deepseek_v2 import DeepseekV2Config
+from transformers.models.deepseek_v2.modeling_deepseek_v2 import (
+ eager_attention_forward,
+)
+
+from auto_round.modelling.replace_modules import ReplacementModuleBase
+from auto_round.utils import is_hpex_available
+
+
+def apply_rotary_emb(
+ xq: torch.Tensor,
+ xk: torch.Tensor,
+ cos: torch.Tensor,
+ sin: torch.Tensor,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ cos = cos.unsqueeze(1)
+ sin = sin.unsqueeze(1)
+
+ real_xq = xq[:, :, :, ::2] * cos - xq[:, :, :, 1::2] * sin
+ img_xq = xq[:, :, :, ::2] * sin + xq[:, :, :, 1::2] * cos
+
+ real_xk = xk[:, :, :, ::2] * cos - xk[:, :, :, 1::2] * sin
+ img_xk = xk[:, :, :, ::2] * sin + xk[:, :, :, 1::2] * cos
+
+ xq_out = torch.stack([real_xq, img_xq], dim=-1).flatten(3).type_as(xq)
+ xk_out = torch.stack([real_xk, img_xk], dim=-1).flatten(3).type_as(xk)
+
+ return xq_out, xk_out
+
+
+@dynamic_rope_update
+def rotary_emb_forward(module, x, position_ids):
+ inv_freq_expanded = module.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
+ position_ids_expanded = position_ids[:, None, :].float()
+
+ device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+ with torch.autocast(device_type=device_type, enabled=False): # Force float32
+ freqs = (inv_freq_expanded.to(x.device) @ position_ids_expanded).transpose(1, 2)
+ cos = torch.cos(freqs) * module.attention_scaling
+ sin = torch.sin(freqs) * module.attention_scaling
+ return cos, sin
+
+
+def attn_forward(
+ module,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ **kwargs,
+) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:
+ if "padding_mask" in kwargs:
+ warnings.warn(
+ "Passing `padding_mask` is deprecated and will be removed in v4.37. "
+ "Please make sure use `attention_mask` instead.`"
+ )
+ batch_size, seq_length = hidden_states.shape[:-1]
+ query_shape = (batch_size, seq_length, -1, module.qk_head_dim)
+ key_shape = (batch_size, seq_length, -1, module.qk_nope_head_dim + module.v_head_dim)
+
+ if module.q_lora_rank is None:
+ q = module.q_proj(hidden_states)
+ else:
+ q = module.q_b_proj(module.q_a_layernorm(module.q_a_proj(hidden_states)))
+ q = q.view(query_shape).transpose(1, 2)
+ q_nope, q_pe = torch.split(q, [module.qk_nope_head_dim, module.qk_rope_head_dim], dim=-1)
+
+ compressed_kv = module.kv_a_proj_with_mqa(hidden_states)
+ k_nope, k_pe = torch.split(compressed_kv, [module.kv_lora_rank, module.qk_rope_head_dim], dim=-1)
+ k_nope = module.kv_b_proj(module.kv_a_layernorm(k_nope)).view(key_shape).transpose(1, 2)
+ k_nope, value_states = torch.split(k_nope, [module.qk_nope_head_dim, module.v_head_dim], dim=-1)
+
+ k_pe = k_pe.view(batch_size, 1, seq_length, module.qk_rope_head_dim)
+ cos, sin = position_embeddings
+ q_pe, k_pe = apply_rotary_emb(q_pe, k_pe, cos.to(q_pe.device), sin.to(q_pe.device))
+
+ k_pe = k_pe.expand(*k_nope.shape[:-1], -1)
+ query_states = torch.cat((q_nope, q_pe), dim=-1)
+ key_states = torch.cat((k_nope, k_pe), dim=-1)
+
+ past_key_values = None
+
+ # transformers <= 4.55.4 is past_key_value
+ if "past_key_values" in kwargs:
+ past_key_values = kwargs["past_key_values"]
+ elif "past_key_value" in kwargs:
+ past_key_values = kwargs["past_key_value"]
+
+ if past_key_values is not None:
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
+ cache_kwargs = {"cache_position": cache_position}
+ key_states, value_states = past_key_values.update(key_states, value_states, module.layer_idx, cache_kwargs)
+
+ if module.config._attn_implementation == "flash_attention_2" and module.qk_head_dim != module.v_head_dim:
+ value_states = torch.nn.functional.pad(value_states, [0, module.qk_head_dim - module.v_head_dim])
+
+ attention_interface: Callable = eager_attention_forward
+ if module.config._attn_implementation != "eager":
+ attention_interface = ALL_ATTENTION_FUNCTIONS[module.config._attn_implementation]
+
+ attn_output, attn_weights = attention_interface(
+ module,
+ query_states,
+ key_states,
+ value_states,
+ attention_mask,
+ dropout=0.0 if not module.training else module.attention_dropout,
+ scaling=module.scaling,
+ **kwargs,
+ )
+
+ if module.config._attn_implementation == "flash_attention_2" and module.qk_head_dim != module.v_head_dim:
+ attn_output = attn_output[:, :, :, : module.v_head_dim]
+
+ attn_output = attn_output.reshape(batch_size, seq_length, -1).contiguous()
+ attn_output = module.o_proj(attn_output)
+ return attn_output, attn_weights
+
+
+class DeepseekV2RotaryEmbedding(ReplacementModuleBase):
+ def __init__(self, original, config):
+ super().__init__()
+
+ @classmethod
+ def original_module_class(cls) -> str:
+ """Return the class name of the module this replaces."""
+ return "DeepseekV2RotaryEmbedding"
+
+ @classmethod
+ def from_original(
+ cls,
+ original: torch.nn.Module,
+ config: DeepseekV2Config,
+ **kwargs,
+ ) -> "DeepseekV2RotaryEmbedding":
+ """Create an instance from the original module."""
+ original.forward = partial(rotary_emb_forward, original)
+ return original
+
+ @classmethod
+ def is_to_be_replaced(
+ cls,
+ original: torch.nn.Module,
+ ) -> bool:
+ """Determine if the given module should be replaced."""
+ return (
+ cls.is_registered(original.__class__.__name__)
+ and is_hpex_available()
+ and inspect.getfile(original.__class__) == inspect.getfile(eager_attention_forward)
+ )
+
+
+class DeepseekV2Attention(ReplacementModuleBase):
+ def __init__(self, original, config):
+ super().__init__()
+
+ @classmethod
+ def original_module_class(cls) -> str:
+ """Return the class name of the module this replaces."""
+ return "DeepseekV2Attention"
+
+ @classmethod
+ def from_original(
+ cls,
+ original: torch.nn.Module,
+ config: DeepseekV2Config,
+ **kwargs,
+ ) -> "DeepseekV2Attention":
+ """Create an instance from the original module."""
+ original.forward = partial(attn_forward, original)
+ return original
+
+ @classmethod
+ def is_to_be_replaced(
+ cls,
+ original: torch.nn.Module,
+ ) -> bool:
+ """Determine if the given module should be replaced."""
+ return (
+ cls.is_registered(original.__class__.__name__)
+ and is_hpex_available()
+ and inspect.getfile(original.__class__) == inspect.getfile(eager_attention_forward)
+ )
diff --git a/auto_round/modelling/replace_modules.py b/auto_round/modelling/replace_modules.py
index c8e34954f..591ed891d 100644
--- a/auto_round/modelling/replace_modules.py
+++ b/auto_round/modelling/replace_modules.py
@@ -25,6 +25,7 @@ BUILTIN_MODULES = {
"Llama4TextMoe": LazyImport("auto_round.modelling.llama4"),
"GptOssMLP": LazyImport("auto_round.modelling.gpt_oss"),
"Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modelling.qwen3_vl_moe"),
+ "DeepseekV2Attention": LazyImport("auto_round.modelling.deepseek_v2"),
}
@@ -94,13 +95,12 @@ class ReplacementModuleBase(ABC, torch.nn.Module):
def is_to_be_replaced(
cls,
original: torch.nn.Module,
- module_class_name: str,
) -> bool:
"""Determine if the given module should be replaced.
Users can extend this method to add custom logic for replacement.
"""
- return cls.is_registered(module_class_name)
+ return cls.is_registered(original.__class__.__name__)
@classmethod
def get_registered_modules(cls) -> list:
@@ -156,7 +156,9 @@ def apply_replacements(
if isinstance(module, ReplacementModuleBase):
continue
class_name = module.__class__.__name__
- if ReplacementModuleBase.is_to_be_replaced(module, class_name):
+ if ReplacementModuleBase.is_registered(class_name) and ReplacementModuleBase.get_replacement_class(
+ class_name
+ ).is_to_be_replaced(module):
modules_to_replace.append((name, module, class_name))
# Step 2: Replace modules
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 3bb60bfb0..c7e5b5d90 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -52,7 +52,7 @@ def _handle_special_model(model):
return model
-def update_module(model, formats: list[OutputFormat] = None):
+def update_module(model, formats: list[OutputFormat] = None, trust_remote_code: bool = True):
if formats is not None and any([format_.is_gguf() for format_ in formats]):
return model
diff --git a/test/test_cpu/quantization/test_statc_attn.py b/test/test_cpu/quantization/test_statc_attn.py
new file mode 100644
index 000000000..8a133cc02
--- /dev/null
+++ b/test/test_cpu/quantization/test_statc_attn.py
@@ -0,0 +1,80 @@
+import shutil
+
+import pytest
+import torch
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+
+from ...helpers import get_model_path
+
+deepseekv2_model_name = get_model_path("deepseek-ai/DeepSeek-V2-Lite-Chat")
+deepseekv3_model_name = get_model_path("tflsxyy/DeepSeek-V3-bf16-4layers")
+
+
+def setup_deepseekv2():
+ """Fixture to set up model and tokenizer."""
+ model_name = deepseekv2_model_name
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=False)
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
+ config.num_hidden_layers = 1 # Reduce layers for testing
+ model = AutoModelForCausalLM.from_config(config)
+ output_dir = "./tmp/test_quantized_deepseekv2"
+ return model, tokenizer, output_dir, config
+
+
+def setup_deepseekv3():
+ """Fixture to set up model and tokenizer."""
+ model_name = deepseekv3_model_name
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=False)
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
+ config.num_hidden_layers = 1 # Reduce layers for testing
+ model = AutoModelForCausalLM.from_config(config)
+ output_dir = "./tmp/test_quantized_deepseekv3"
+ return model, tokenizer, output_dir, config
+
+
+def test_deepseek_v2(setup_deepseekv2):
+ model, tokenizer, output_dir, config = setup_deepseekv2
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ scheme="FP8_STATIC",
+ static_attention_dtype="fp8",
+ iters=0,
+ seqlen=2,
+ trust_remote_code=False,
+ )
+ quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)
+ assert quantized_model is not None, "Expected quantized_model to be not None"
+ device = quantized_model.model.layers[0].self_attn.q_scale.device
+ assert not all(
+ quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)
+ ), "q_scale is not collected"
+
+ # clean the output directory after test
+ shutil.rmtree(output_dir, ignore_errors=True)
+
+
+def test_deepseek_v3(setup_deepseekv3):
+ model, tokenizer, output_dir, config = setup_deepseekv3
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ scheme="FP8_STATIC",
+ static_attention_dtype="fp8",
+ iters=0,
+ seqlen=2,
+ trust_remote_code=False,
+ )
+ quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)
+ assert quantized_model is not None, "Expected quantized_model to be not None"
+ device = quantized_model.model.layers[0].self_attn.q_scale.device
+ assert not all(
+ quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)
+ ), "q_scale is not collected"
+
+ # clean the output directory after test
+ shutil.rmtree(output_dir, ignore_errors=True)
diff --git a/test/test_hpu/requirements.txt b/test/test_hpu/requirements.txt
index 8b1378917..8a05844b9 100644
--- a/test/test_hpu/requirements.txt
+++ b/test/test_hpu/requirements.txt
@@ -1 +1 @@
-
+compressed_tensors
diff --git a/test/test_hpu/test_statc_attn.py b/test/test_hpu/test_statc_attn.py
new file mode 100644
index 000000000..ac4845ceb
--- /dev/null
+++ b/test/test_hpu/test_statc_attn.py
@@ -0,0 +1,48 @@
+import shutil
+
+import pytest
+import torch
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round.utils import is_hpex_available
+
+from ..helpers import get_model_path, is_pytest_mode_lazy
+
+deepseekv2_model_name = get_model_path("deepseek-ai/DeepSeek-V2-Lite-Chat")
+
+
+def setup_deepseekv2():
+ """Fixture to set up model and tokenizer."""
+ model_name = deepseekv2_model_name
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=False)
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
+ config.num_hidden_layers = 1 # Reduce layers for testing
+ model = AutoModelForCausalLM.from_config(config)
+ output_dir = "./tmp/test_quantized_deepseekv2"
+ return model, tokenizer, output_dir, config
+
+
[email protected](not is_hpex_available(), reason="HPU is not supported")
[email protected](not is_pytest_mode_lazy(), reason="Only for lazy mode")
+def test_deepseek_v2_on_hpu(setup_deepseekv2):
+ model, tokenizer, output_dir, config = setup_deepseekv2
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ scheme="FP8_STATIC",
+ static_attention_dtype="fp8",
+ iters=0,
+ seqlen=2,
+ trust_remote_code=False,
+ )
+ quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)
+ assert quantized_model is not None, "Expected quantized_model to be not None"
+ device = quantized_model.model.layers[0].self_attn.q_scale.device
+ assert not all(
+ quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)
+ ), "q_scale is not collected"
+
+ # clean the output directory after test
+ shutil.rmtree(output_dir, ignore_errors=True)