Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Disable replace `FP8Expert` (#1379)

Yi Liu137da5510be1
diff --git a/auto_round/experimental/qmodules/fp4_utils.py b/auto_round/experimental/qmodules/fp4_utils.py
index e755a2b38..aa9cae421 100644
--- a/auto_round/experimental/qmodules/fp4_utils.py
+++ b/auto_round/experimental/qmodules/fp4_utils.py
@@ -65,7 +65,7 @@ def _unpack_fp4_from_uint8_cpu(
return _unpack_fp4_from_uint8(a, m, n, dtype)
[email protected](fullgraph=True, dynamic=True)
+# @torch.compile(fullgraph=True, dynamic=True)
def _unpack_fp4_from_uint8_cuda(
a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16
) -> torch.Tensor:
diff --git a/auto_round/modeling/__init__.py b/auto_round/modeling/__init__.py
index 14a492441..d1bc25269 100644
--- a/auto_round/modeling/__init__.py
+++ b/auto_round/modeling/__init__.py
@@ -11,3 +11,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+from .fp8_quant import *
diff --git a/auto_round/modeling/fp8_quant.py b/auto_round/modeling/fp8_quant.py
new file mode 100644
index 000000000..da4cae175
--- /dev/null
+++ b/auto_round/modeling/fp8_quant.py
@@ -0,0 +1,114 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from transformers.quantizers.quantizer_finegrained_fp8 import (
+ FineGrainedFP8HfQuantizer as OriginalFineGrainedFP8HfQuantizer,
+)
+
+from auto_round.utils import is_transformers_version_greater_or_equal_5
+from auto_round.utils import logger as auto_round_logger
+from auto_round.utils.device import override_cuda_device_capability
+
+
+# Patching replace_with_fp8_linear to disable expert replacement
+# https://github.com/huggingface/transformers/blob/78bb85146c59258a0710c8d08311d98d52303c38/src/transformers/integrations/finegrained_fp8.py#L720
+def oot_replace_with_fp8_linear(
+ model, modules_to_not_convert: list[str] | None = None, quantization_config=None, pre_quantized=False
+):
+ """
+ A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.
+
+ Parameters:
+ model (`torch.nn.Module`):
+ Input model or `torch.nn.Module` as the function is run recursively.
+ modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
+ Names of the modules to not convert. In practice we keep the `lm_head`
+ in full precision for numerical stability reasons.
+ quantization_config (`FbgemmFp8Config`):
+ The quantization config object that contains the quantization parameters.
+ pre_quantized (`bool`, defaults to `False`):
+ Whether the model is pre-quantized or not
+ """
+ from transformers.integrations.finegrained_fp8 import (
+ FP8Linear,
+ logger,
+ should_convert_module,
+ )
+
+ if quantization_config.dequantize:
+ return model
+
+ has_been_replaced = False
+ for module_name, module in model.named_modules():
+ if not should_convert_module(module_name, modules_to_not_convert):
+ continue
+ # we need this to correctly materialize the weights during quantization
+ module_kwargs = {} if pre_quantized else {"dtype": None}
+ new_module = None
+ with torch.device("meta"):
+ # Note: Disable replacing experts, as we do not want concatenated experts
+ # if module_name.endswith(".experts"):
+ # new_module = FP8Expert(
+ # config=model.config, block_size=quantization_config.weight_block_size, **module_kwargs
+ # )
+ # elif isinstance(module, nn.Linear):
+ if isinstance(module, torch.nn.Linear):
+ new_module = FP8Linear(
+ in_features=module.in_features,
+ out_features=module.out_features,
+ bias=module.bias is not None,
+ activation_scheme=quantization_config.activation_scheme,
+ block_size=quantization_config.weight_block_size,
+ **module_kwargs,
+ )
+ if new_module is not None:
+ model.set_submodule(module_name, new_module)
+ has_been_replaced = True
+
+ if not has_been_replaced:
+ logger.warning(
+ "You are loading your model using fp8 but no linear modules were found in your model."
+ " Please double check your model architecture."
+ )
+ return model
+
+
+_orig_validate_environment = OriginalFineGrainedFP8HfQuantizer.validate_environment
+
+
+@override_cuda_device_capability()
+def oot_validate_environment(self, *args, **kwargs):
+ return _orig_validate_environment(self, *args, **kwargs)
+
+
+def apply_fp8_expert_replacement_patch():
+ if is_transformers_version_greater_or_equal_5() and torch.cuda.is_available():
+ try:
+ import transformers.integrations.finegrained_fp8 as transformers_fp8
+
+ transformers_fp8.replace_with_fp8_linear = oot_replace_with_fp8_linear
+ auto_round_logger.debug("Applied FP8 expert replacement patch to transformers.")
+ OriginalFineGrainedFP8HfQuantizer.validate_environment = oot_validate_environment
+ auto_round_logger.debug(
+ (
+ "Patched FineGrainedFP8HfQuantizer.validate_environment to bypass device "
+ "capability check for loading FP8 models on unsupported GPUs."
+ )
+ )
+ except ImportError as e:
+ auto_round_logger.warning(f"Could not apply FP8 expert replacement patch as {e}.")
+
+
+apply_fp8_expert_replacement_patch()
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index b8d7ab9fb..2ae2a78e4 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -390,3 +390,11 @@ class GlobalState:
global_state = GlobalState()
+
+
+@lru_cache(None)
+def is_transformers_version_greater_or_equal_5():
+ import transformers
+ from packaging import version
+
+ return version.parse(transformers.__version__) >= version.parse("5.0.0")
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index b25a6888e..a16f441bf 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -15,7 +15,7 @@ import functools
import gc
import os
import re
-from contextlib import contextmanager
+from contextlib import ContextDecorator, contextmanager
from functools import lru_cache
from itertools import combinations
from threading import Lock
@@ -315,17 +315,28 @@ def set_cuda_visible_devices(device):
os.environ["CUDA_VISIBLE_DEVICES"] = device
-def set_fake_cuda_device_capability(func=None):
- if func is not None:
- torch.cuda.get_device_capability = func
- return func
+class override_cuda_device_capability(ContextDecorator):
+ """Context manager/decorator to temporarily override CUDA capability checks."""
- def fake_cuda():
- return 100, 1
+ def __init__(self, major: int = 100, minor: int = 1) -> None:
+ self.major = major
+ self.minor = minor
+ self._orig_func = None
- orig_func = torch.cuda.get_device_capability
- torch.cuda.get_device_capability = fake_cuda
- return orig_func
+ def __enter__(self):
+ self._orig_func = torch.cuda.get_device_capability
+
+ def _override_capability(*_args, **_kwargs):
+ return self.major, self.minor
+
+ torch.cuda.get_device_capability = _override_capability
+ return self
+
+ def __exit__(self, exc_type, exc, exc_tb):
+ if self._orig_func is not None:
+ torch.cuda.get_device_capability = self._orig_func
+ self._orig_func = None
+ return False
def get_packing_device(device: str | torch.device | None = "auto") -> torch.device:
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 0371caf2f..513d23e68 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -256,6 +256,7 @@ def llm_load_model(
device: str = "cpu",
**kwargs,
):
+
assert platform.lower() in [
"hf",
"model_scope",
@@ -269,11 +270,10 @@ def llm_load_model(
from modelscope import AutoModel, AutoModelForCausalLM, AutoTokenizer # pylint: disable=E0401
else:
from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer
-
from auto_round.utils.device import (
_use_hpu_compile_mode,
get_device_and_parallelism,
- set_fake_cuda_device_capability,
+ override_cuda_device_capability,
)
device_str, use_auto_mapping = get_device_and_parallelism(device)
@@ -306,14 +306,13 @@ def llm_load_model(
)
except ValueError as e:
if "FP8 quantized" in str(e):
- orig_func = set_fake_cuda_device_capability()
- model = model_cls.from_pretrained(
- pretrained_model_name_or_path,
- torch_dtype=torch_dtype,
- trust_remote_code=trust_remote_code,
- device_map="auto" if use_auto_mapping else None,
- )
- torch.cuda.get_device_capability = orig_func
+ with override_cuda_device_capability():
+ model = model_cls.from_pretrained(
+ pretrained_model_name_or_path,
+ torch_dtype=torch_dtype,
+ trust_remote_code=trust_remote_code,
+ device_map="auto" if use_auto_mapping else None,
+ )
logger.warning("the support for fp8 model as input is experimental, please use with caution.")
else:
raise
@@ -366,7 +365,7 @@ def mllm_load_model(
base_lib = transformers
- from auto_round.utils.device import get_device_and_parallelism, set_fake_cuda_device_capability
+ from auto_round.utils.device import get_device_and_parallelism, override_cuda_device_capability
device_str, use_auto_mapping = get_device_and_parallelism(device)
torch_dtype = "auto"
@@ -439,14 +438,13 @@ def mllm_load_model(
)
except ValueError as e:
if "FP8 quantized" in str(e):
- orig_func = set_fake_cuda_device_capability()
- model = cls.from_pretrained(
- pretrained_model_name_or_path,
- trust_remote_code=trust_remote_code,
- torch_dtype=torch_dtype,
- device_map="auto" if use_auto_mapping else None,
- )
- torch.cuda.get_device_capability = orig_func
+ with override_cuda_device_capability():
+ model = cls.from_pretrained(
+ pretrained_model_name_or_path,
+ trust_remote_code=trust_remote_code,
+ torch_dtype=torch_dtype,
+ device_map="auto" if use_auto_mapping else None,
+ )
logger.warning("the support for fp8 model as input is experimental, please use with caution.")
else:
raise
diff --git a/test/test_cuda/models/test_fp8_model.py b/test/test_cuda/models/test_fp8_model.py
new file mode 100644
index 000000000..5b7962f7b
--- /dev/null
+++ b/test/test_cuda/models/test_fp8_model.py
@@ -0,0 +1,55 @@
+import shutil
+
+import pytest
+import torch
+import torch.nn as nn
+from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
+
+from auto_round import AutoRound
+
+
+def setup_qwen_fp8():
+ """Fixture to set up the GPT-OSS model and tokenizer."""
+ model_name = "INC4AI/Qwen3-30B-A3B-Instruct-2507-FP8-2Layers"
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
+ # model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+ output_dir = "test_quantized_qwen3_fp8_moe_mxfp"
+ return tokenizer, output_dir, config, model_name
+
+
+def test_qwen3_fp8_moe_mxfp(setup_qwen_fp8):
+ tokenizer, output_dir, config, model_name = setup_qwen_fp8
+ autoround = AutoRound(
+ model_name,
+ scheme="MXFP4",
+ nsamples=2,
+ seqlen=32,
+ iters=0,
+ )
+ quantized_model, _ = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)
+ assert quantized_model is not None, "Quantized model should not be None."
+ loaded_model = AutoModelForCausalLM.from_pretrained(output_dir)
+ loaded_model.to("cuda")
+ quantized_model.to("cuda")
+ for n, m in quantized_model.named_modules():
+ if m.__class__.__name__ == "QuantLinear":
+ loaded_m = loaded_model.get_submodule(n)
+ assert (loaded_m.weight_packed == m.weight_packed).all()
+ # Expect all linear in experts are quantized
+ for n, m in quantized_model.named_modules():
+ if "experts" in m.__class__.__name__.lower():
+ for sub_n, sub_m in m.named_modules():
+ assert sub_m.__class__.__name__ == "QuantLinear", f"Module {n}.{sub_n} is not quantized."
+ inp = torch.randint(0, 100, (1, 64)).to("cuda")
+ with torch.inference_mode():
+ loaded_out = loaded_model(inp)
+
+ # test generation
+ tokenizer = AutoTokenizer.from_pretrained(output_dir)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(device=loaded_model.device)
+ print(tokenizer.decode(loaded_model.generate(**inputs, max_new_tokens=50)[0]))
+ # clean the output directory after test
+ shutil.rmtree(output_dir, ignore_errors=True)