PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Disable replace `FP8Expert` (#1379)
Yi Liu137da5510be1
diff --git a/auto_round/experimental/qmodules/fp4_utils.py b/auto_round/experimental/qmodules/fp4_utils.pyindex e755a2b38..aa9cae421 100644--- a/auto_round/experimental/qmodules/fp4_utils.py+++ b/auto_round/experimental/qmodules/fp4_utils.py@@ -65,7 +65,7 @@ def _unpack_fp4_from_uint8_cpu(return _unpack_fp4_from_uint8(a, m, n, dtype)[email protected](fullgraph=True, dynamic=True)+# @torch.compile(fullgraph=True, dynamic=True)def _unpack_fp4_from_uint8_cuda(a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16) -> torch.Tensor:diff --git a/auto_round/modeling/__init__.py b/auto_round/modeling/__init__.pyindex 14a492441..d1bc25269 100644--- a/auto_round/modeling/__init__.py+++ b/auto_round/modeling/__init__.py@@ -11,3 +11,5 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.++from .fp8_quant import *diff --git a/auto_round/modeling/fp8_quant.py b/auto_round/modeling/fp8_quant.pynew file mode 100644index 000000000..da4cae175--- /dev/null+++ b/auto_round/modeling/fp8_quant.py@@ -0,0 +1,114 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch+from transformers.quantizers.quantizer_finegrained_fp8 import (+ FineGrainedFP8HfQuantizer as OriginalFineGrainedFP8HfQuantizer,+)++from auto_round.utils import is_transformers_version_greater_or_equal_5+from auto_round.utils import logger as auto_round_logger+from auto_round.utils.device import override_cuda_device_capability+++# Patching replace_with_fp8_linear to disable expert replacement+# https://github.com/huggingface/transformers/blob/78bb85146c59258a0710c8d08311d98d52303c38/src/transformers/integrations/finegrained_fp8.py#L720+def oot_replace_with_fp8_linear(+ model, modules_to_not_convert: list[str] | None = None, quantization_config=None, pre_quantized=False+):+ """+ A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.++ Parameters:+ model (`torch.nn.Module`):+ Input model or `torch.nn.Module` as the function is run recursively.+ modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):+ Names of the modules to not convert. In practice we keep the `lm_head`+ in full precision for numerical stability reasons.+ quantization_config (`FbgemmFp8Config`):+ The quantization config object that contains the quantization parameters.+ pre_quantized (`bool`, defaults to `False`):+ Whether the model is pre-quantized or not+ """+ from transformers.integrations.finegrained_fp8 import (+ FP8Linear,+ logger,+ should_convert_module,+ )++ if quantization_config.dequantize:+ return model++ has_been_replaced = False+ for module_name, module in model.named_modules():+ if not should_convert_module(module_name, modules_to_not_convert):+ continue+ # we need this to correctly materialize the weights during quantization+ module_kwargs = {} if pre_quantized else {"dtype": None}+ new_module = None+ with torch.device("meta"):+ # Note: Disable replacing experts, as we do not want concatenated experts+ # if module_name.endswith(".experts"):+ # new_module = FP8Expert(+ # config=model.config, block_size=quantization_config.weight_block_size, **module_kwargs+ # )+ # elif isinstance(module, nn.Linear):+ if isinstance(module, torch.nn.Linear):+ new_module = FP8Linear(+ in_features=module.in_features,+ out_features=module.out_features,+ bias=module.bias is not None,+ activation_scheme=quantization_config.activation_scheme,+ block_size=quantization_config.weight_block_size,+ **module_kwargs,+ )+ if new_module is not None:+ model.set_submodule(module_name, new_module)+ has_been_replaced = True++ if not has_been_replaced:+ logger.warning(+ "You are loading your model using fp8 but no linear modules were found in your model."+ " Please double check your model architecture."+ )+ return model+++_orig_validate_environment = OriginalFineGrainedFP8HfQuantizer.validate_environment+++@override_cuda_device_capability()+def oot_validate_environment(self, *args, **kwargs):+ return _orig_validate_environment(self, *args, **kwargs)+++def apply_fp8_expert_replacement_patch():+ if is_transformers_version_greater_or_equal_5() and torch.cuda.is_available():+ try:+ import transformers.integrations.finegrained_fp8 as transformers_fp8++ transformers_fp8.replace_with_fp8_linear = oot_replace_with_fp8_linear+ auto_round_logger.debug("Applied FP8 expert replacement patch to transformers.")+ OriginalFineGrainedFP8HfQuantizer.validate_environment = oot_validate_environment+ auto_round_logger.debug(+ (+ "Patched FineGrainedFP8HfQuantizer.validate_environment to bypass device "+ "capability check for loading FP8 models on unsupported GPUs."+ )+ )+ except ImportError as e:+ auto_round_logger.warning(f"Could not apply FP8 expert replacement patch as {e}.")+++apply_fp8_expert_replacement_patch()diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex b8d7ab9fb..2ae2a78e4 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -390,3 +390,11 @@ class GlobalState:global_state = GlobalState()+++@lru_cache(None)+def is_transformers_version_greater_or_equal_5():+ import transformers+ from packaging import version++ return version.parse(transformers.__version__) >= version.parse("5.0.0")diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex b25a6888e..a16f441bf 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -15,7 +15,7 @@ import functoolsimport gcimport osimport re-from contextlib import contextmanager+from contextlib import ContextDecorator, contextmanagerfrom functools import lru_cachefrom itertools import combinationsfrom threading import Lock@@ -315,17 +315,28 @@ def set_cuda_visible_devices(device):os.environ["CUDA_VISIBLE_DEVICES"] = device-def set_fake_cuda_device_capability(func=None):- if func is not None:- torch.cuda.get_device_capability = func- return func+class override_cuda_device_capability(ContextDecorator):+ """Context manager/decorator to temporarily override CUDA capability checks."""- def fake_cuda():- return 100, 1+ def __init__(self, major: int = 100, minor: int = 1) -> None:+ self.major = major+ self.minor = minor+ self._orig_func = None- orig_func = torch.cuda.get_device_capability- torch.cuda.get_device_capability = fake_cuda- return orig_func+ def __enter__(self):+ self._orig_func = torch.cuda.get_device_capability++ def _override_capability(*_args, **_kwargs):+ return self.major, self.minor++ torch.cuda.get_device_capability = _override_capability+ return self++ def __exit__(self, exc_type, exc, exc_tb):+ if self._orig_func is not None:+ torch.cuda.get_device_capability = self._orig_func+ self._orig_func = None+ return Falsedef get_packing_device(device: str | torch.device | None = "auto") -> torch.device:diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 0371caf2f..513d23e68 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -256,6 +256,7 @@ def llm_load_model(device: str = "cpu",**kwargs,):+assert platform.lower() in ["hf","model_scope",@@ -269,11 +270,10 @@ def llm_load_model(from modelscope import AutoModel, AutoModelForCausalLM, AutoTokenizer # pylint: disable=E0401else:from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer-from auto_round.utils.device import (_use_hpu_compile_mode,get_device_and_parallelism,- set_fake_cuda_device_capability,+ override_cuda_device_capability,)device_str, use_auto_mapping = get_device_and_parallelism(device)@@ -306,14 +306,13 @@ def llm_load_model()except ValueError as e:if "FP8 quantized" in str(e):- orig_func = set_fake_cuda_device_capability()- model = model_cls.from_pretrained(- pretrained_model_name_or_path,- torch_dtype=torch_dtype,- trust_remote_code=trust_remote_code,- device_map="auto" if use_auto_mapping else None,- )- torch.cuda.get_device_capability = orig_func+ with override_cuda_device_capability():+ model = model_cls.from_pretrained(+ pretrained_model_name_or_path,+ torch_dtype=torch_dtype,+ trust_remote_code=trust_remote_code,+ device_map="auto" if use_auto_mapping else None,+ )logger.warning("the support for fp8 model as input is experimental, please use with caution.")else:raise@@ -366,7 +365,7 @@ def mllm_load_model(base_lib = transformers- from auto_round.utils.device import get_device_and_parallelism, set_fake_cuda_device_capability+ from auto_round.utils.device import get_device_and_parallelism, override_cuda_device_capabilitydevice_str, use_auto_mapping = get_device_and_parallelism(device)torch_dtype = "auto"@@ -439,14 +438,13 @@ def mllm_load_model()except ValueError as e:if "FP8 quantized" in str(e):- orig_func = set_fake_cuda_device_capability()- model = cls.from_pretrained(- pretrained_model_name_or_path,- trust_remote_code=trust_remote_code,- torch_dtype=torch_dtype,- device_map="auto" if use_auto_mapping else None,- )- torch.cuda.get_device_capability = orig_func+ with override_cuda_device_capability():+ model = cls.from_pretrained(+ pretrained_model_name_or_path,+ trust_remote_code=trust_remote_code,+ torch_dtype=torch_dtype,+ device_map="auto" if use_auto_mapping else None,+ )logger.warning("the support for fp8 model as input is experimental, please use with caution.")else:raisediff --git a/test/test_cuda/models/test_fp8_model.py b/test/test_cuda/models/test_fp8_model.pynew file mode 100644index 000000000..5b7962f7b--- /dev/null+++ b/test/test_cuda/models/test_fp8_model.py@@ -0,0 +1,55 @@+import shutil++import pytest+import torch+import torch.nn as nn+from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor, AutoTokenizer++from auto_round import AutoRound+++def setup_qwen_fp8():+ """Fixture to set up the GPT-OSS model and tokenizer."""+ model_name = "INC4AI/Qwen3-30B-A3B-Instruct-2507-FP8-2Layers"+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)+ # model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)+ output_dir = "test_quantized_qwen3_fp8_moe_mxfp"+ return tokenizer, output_dir, config, model_name+++def test_qwen3_fp8_moe_mxfp(setup_qwen_fp8):+ tokenizer, output_dir, config, model_name = setup_qwen_fp8+ autoround = AutoRound(+ model_name,+ scheme="MXFP4",+ nsamples=2,+ seqlen=32,+ iters=0,+ )+ quantized_model, _ = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)+ assert quantized_model is not None, "Quantized model should not be None."+ loaded_model = AutoModelForCausalLM.from_pretrained(output_dir)+ loaded_model.to("cuda")+ quantized_model.to("cuda")+ for n, m in quantized_model.named_modules():+ if m.__class__.__name__ == "QuantLinear":+ loaded_m = loaded_model.get_submodule(n)+ assert (loaded_m.weight_packed == m.weight_packed).all()+ # Expect all linear in experts are quantized+ for n, m in quantized_model.named_modules():+ if "experts" in m.__class__.__name__.lower():+ for sub_n, sub_m in m.named_modules():+ assert sub_m.__class__.__name__ == "QuantLinear", f"Module {n}.{sub_n} is not quantized."+ inp = torch.randint(0, 100, (1, 64)).to("cuda")+ with torch.inference_mode():+ loaded_out = loaded_model(inp)++ # test generation+ tokenizer = AutoTokenizer.from_pretrained(output_dir)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(device=loaded_model.device)+ print(tokenizer.decode(loaded_model.generate(**inputs, max_new_tokens=50)[0]))+ # clean the output directory after test+ shutil.rmtree(output_dir, ignore_errors=True)