PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Bug fix: Replace WAWrapperLinear with FakeActQuantLinear instead of Linear (#2245)
Xin He6b29e9c7602f
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex c01e5feb7..9145ce7d5 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -441,8 +441,11 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):eval_folder = folders[-1] if folders else Noneelse:eval_folder = folders+# set model_type for ModelFreeCompressor.- model_type = detect_model_type(eval_folder)+ # Fake format now always saves, so eval_folder is the canonical probe.+ type_probe = eval_folder+ model_type = detect_model_type(type_probe)if hasattr(autoround, "model_context") and model_type in ("mllm", "diffusion"):setattr(autoround.model_context, f"is_{model_type}", True)else:@@ -496,8 +499,12 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):logger.warning("set add_bos_token=True for llama model.")args.add_bos_token = True- # Check if GGUF model- eval_gguf_model = any(file.endswith("gguf") for file in os.listdir(eval_folder))+ # Check if GGUF model in exported eval folder.+ eval_gguf_model = (+ eval_folder is not None+ and os.path.isdir(eval_folder)+ and any(file.endswith("gguf") for file in os.listdir(eval_folder))+ )# Determine if model instance evaluation is neededneed_model_instance = formats[-1] == "fake" or eval_gguf_modeldiff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.pyindex 069d1aeae..d29e131f8 100644--- a/auto_round/export/formats/backends/fake.py+++ b/auto_round/export/formats/backends/fake.py@@ -13,6 +13,8 @@# limitations under the License.import copy+import glob+import osfrom typing import Any, Callable, Unionimport torch@@ -33,6 +35,51 @@ def _serialize_quantization_config_value(value):return value+def _normalize_state_dict_keys(state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], bool]:+ """Collapse wrapped-layer keys like ``*.orig_layer.weight`` to ``*.weight``."""+ normalized = {}+ changed = False+ for key, value in state_dict.items():+ new_key = key.replace(".orig_layer.", ".")+ if new_key != key:+ changed = True+ # Prefer already-normalized keys if both happen to exist.+ if new_key not in normalized:+ normalized[new_key] = value+ return normalized, changed+++def _rewrite_saved_weights_without_orig_layer(output_dir: str) -> None:+ """Rewrite saved checkpoint shards in-place so no ``.orig_layer.`` keys remain."""+ if not os.path.isdir(output_dir):+ return++ safetensor_files = sorted(glob.glob(os.path.join(output_dir, "*.safetensors")))+ for file_path in safetensor_files:+ try:+ from safetensors.torch import load_file as safe_load_file+ from safetensors.torch import save_file as safe_save_file++ state = safe_load_file(file_path)+ normalized, changed = _normalize_state_dict_keys(state)+ if changed:+ safe_save_file(normalized, file_path)+ except Exception as exc:+ logger.warning("Failed to normalize safetensors keys for %s: %s", file_path, exc)++ bin_files = sorted(glob.glob(os.path.join(output_dir, "*.bin")))+ for file_path in bin_files:+ try:+ state = torch.load(file_path, weights_only=True)+ if not isinstance(state, dict):+ continue+ normalized, changed = _normalize_state_dict_keys(state)+ if changed:+ torch.save(normalized, file_path)+ except Exception as exc:+ logger.warning("Failed to normalize pytorch checkpoint keys for %s: %s", file_path, exc)++@OutputFormat.register("fake")class FakeFormat(OutputFormat):support_schemes = None@@ -58,44 +105,30 @@ class FakeFormat(OutputFormat):serialization_dict: dict = None,**kwargs,):+ has_fake_act_quant = False+ logger.warning(+ "Saving fake-quantized model to disk. "+ "Linear replacement is deferred to load-time (via auto_round:fake backend); "+ "save-time now keeps the in-memory quantized model structure unchanged."+ )has_meta_device = unsupported_meta_device(model)if not inplace and not has_meta_device:model = copy.deepcopy(model.to("cpu"))- from auto_round.utils.model import set_module- from auto_round.wrapper import WrapperLinear, WrapperWALayer+ config_act_bits = (serialization_dict or {}).get("act_bits")+ if config_act_bits is not None and config_act_bits <= 8:+ has_fake_act_quant = True- if not has_meta_device:- wrapped_modules = [- (name, module)- for name, module in model.named_modules()- if name and isinstance(module, (WrapperLinear, WrapperWALayer))- ]- wrapped_names = {name for name, _ in wrapped_modules}- for name, module in wrapped_modules:- if any(name.startswith(f"{parent}.") for parent in wrapped_names if parent != name):- continue- orig_layer = module.orig_layer- while isinstance(orig_layer, (WrapperLinear, WrapperWALayer)):- orig_layer = orig_layer.orig_layer- for attr_name in ("act_min_scale", "act_max_scale", "act_scale"):- orig_layer._parameters.pop(attr_name, None)- orig_layer._buffers.pop(attr_name, None)- if hasattr(orig_layer, attr_name):- delattr(orig_layer, attr_name)- set_module(model, name, orig_layer.to("cpu"))-- is_nvfp4_v2 = (serialization_dict or {}).get("data_type") == "nvfp4_v2"- if is_nvfp4_v2:- quantization_config = _serialize_quantization_config_value(dict(serialization_dict or {}))- quantization_config["quant_method"] = "auto-round"+ quantization_config = _serialize_quantization_config_value(dict(serialization_dict or {}))+ quantization_config["quant_method"] = "auto-round"+ if has_fake_act_quant:quantization_config["packing_format"] = "auto_round:fake"- quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)- from auto_round.export.utils import filter_quantization_config+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)+ from auto_round.export.utils import filter_quantization_config- filter_quantization_config(quantization_config)- if hasattr(model, "config") and model.config is not None:- model.config.quantization_config = quantization_config+ filter_quantization_config(quantization_config)+ if hasattr(model, "config") and model.config is not None:+ model.config.quantization_config = quantization_configif not has_meta_device:model = model.to("cpu")@@ -103,6 +136,11 @@ class FakeFormat(OutputFormat):elif hasattr(model, "config") and model.config is not None:model.config.save_pretrained(output_dir)+ # Some save flows write wrapper keys first; normalize to plain Linear keys+ # so HF loading does not report UNEXPECTED/MISSING pairs.+ if has_fake_act_quant:+ _rewrite_saved_weights_without_orig_layer(output_dir)+if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):tokenizer.save_pretrained(output_dir)processor = kwargs.get("processor", None)diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 289e163de..1ef8ba9b1 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -211,7 +211,7 @@ GPTQ_FORMAT = ["auto_round:auto_gptq"] # zp+-1GPTQ_FORMAT_NO_ZP = ["auto_round", "auto_round:gptqmodel"]AWQ_FORMAT = ["auto_round:auto_awq"]LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]-FAKE_FORMAT = ["auto_round:fake"]+FAKE_FORMAT = ["auto_round:fake", "auto_round:auto_gptq", "auto_round", "auto_round:auto_awq"]NVFP4_E5M3_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3"]WOQ_DEFAULT_ACT_BITS = [None, 16, 32]@@ -351,17 +351,11 @@ BackendInfos["auto_round:torch_mxint4"] = BackendInfo(BackendInfos["auto_round:fake"] = BackendInfo(device=["xpu", "cuda", "cpu"],packing_format=FAKE_FORMAT,- sym=[True],compute_dtype=["float32", "float16", "bfloat16"],- data_type=["nvfp4_v2"],- group_size=[16],- bits=[4],- act_bits=[4],- act_group_size=[16],- act_sym=[True],- act_data_type=["nvfp4_v2"],- priority=0,- checkers=[mxfp_nvfp_feature_checker],+ bits=[1, 2, 3, 4, 5, 6, 7, 8],+ sym=[True, False],+ # Keep fake backend as a fallback path; prefer real kernels when available.+ priority=-1,alias=["auto_round", "torch"],requirements=["auto-round>0.12.0"],)diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 6f738e2cd..1239fd81f 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1314,9 +1314,7 @@ def is_diffusion_model(model_or_path: Union[str, object], trust_remote_code: booif model_type == "nextstep":return Trueexcept:- logger.warning(- f"Failed to load config for {model_or_path}, trying to check model_index.json for diffusion pipeline."- )+ passindex_file = Noneif not os.path.isdir(model_or_path):try:diff --git a/test/unit/test_cpu/export/test_fake_format.py b/test/unit/test_cpu/export/test_fake_format.pyindex 10b064e4a..b4d3a3842 100644--- a/test/unit/test_cpu/export/test_fake_format.py+++ b/test/unit/test_cpu/export/test_fake_format.py@@ -20,12 +20,23 @@ import torchfrom transformers import AutoModelForCausalLM, OPTConfig, OPTForCausalLMfrom auto_round.experimental.qmodules.fake import FakeActQuantLinear+from auto_round.export.formats.backends.fake import (+ _normalize_state_dict_keys,+ _rewrite_saved_weights_without_orig_layer,+)from auto_round.formats import FakeFormat+from auto_round.inference.backend import get_layer_backendfrom auto_round.inference.convert_model import convert_hf_modelfrom auto_round.schemes import PRESET_SCHEMESfrom auto_round.wrapper import WrapperWALayer+def _assert_has_act_hook(layer):+ assert isinstance(layer, FakeActQuantLinear)+ assert hasattr(layer, "qdq_input")+ assert callable(layer.qdq_input)++class _WrappedLinear(WrapperWALayer):def __init__(self, linear):torch.nn.Module.__init__(self)@@ -56,7 +67,7 @@ def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):expected_weight = model.linear.orig_layer.weight.detach().clone()output_dir = str(tmp_path / "fake_model")- FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(+ saved_model = FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(output_dir=output_dir,model=model,inplace=False,@@ -69,18 +80,30 @@ def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):"act_group_size": 16,"act_sym": True,"act_data_type": "nvfp4_v2",+ "to_quant_block_names": ["block"],"supported_types": [torch.nn.Linear],},)state_dict = torch.load(os.path.join(output_dir, "pytorch_model.bin"), weights_only=True)- assert set(state_dict) == {"linear.weight", "linear.bias"}+ assert set(state_dict) == {"linear.weight", "linear.bias", "linear.act_max_scale"}assert torch.equal(state_dict["linear.weight"], expected_weight)- assert hasattr(model.linear, "orig_layer")+ # Save-time should keep in-memory wrappers unchanged; replacement happens on load.+ assert hasattr(saved_model.linear, "orig_layer")with open(os.path.join(output_dir, "config.json")) as config_file:quantization_config = json.load(config_file)["quantization_config"]assert "supported_types" not in quantization_configassert quantization_config["packing_format"] == "auto_round:fake"+ assert quantization_config["quant_method"] == "auto-round"+ assert quantization_config["act_bits"] == 4+ assert quantization_config["block_name_to_quantize"] == ["block"]++ loaded_model = _TinyLoadModel(SimpleNamespace(**quantization_config))+ loaded_model, used_backends = convert_hf_model(loaded_model, target_device="cpu")+ assert used_backends == ["auto_round:fake"]+ _assert_has_act_hook(loaded_model.block.linear)+ roundtrip_activation = torch.randn(2, 3, 16)+ assert not torch.equal(loaded_model.block.linear.qdq_input(roundtrip_activation), roundtrip_activation)class _TinyLoadModel(torch.nn.Module):@@ -117,7 +140,7 @@ def test_fake_config_replaces_linear_and_qdq_activation_on_load():model, used_backends = convert_hf_model(model, target_device="cpu")assert used_backends == ["auto_round:fake"]- assert isinstance(model.block.linear, FakeActQuantLinear)+ _assert_has_act_hook(model.block.linear)assert torch.equal(model.block.linear.weight, original_weight)qdq_activation = model.block.linear.qdq_input(activation)assert not torch.equal(qdq_activation, activation)@@ -183,6 +206,218 @@ def test_transformers_load_replaces_fake_linear(tmp_path):loaded_model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="cpu")q_proj = loaded_model.model.decoder.layers[0].self_attn.q_proj- assert isinstance(q_proj, FakeActQuantLinear)+ _assert_has_act_hook(q_proj)activation = torch.randn(1, 2, 16)assert not torch.equal(q_proj.qdq_input(activation), activation)+++def test_fake_format_keeps_woq_packing_format(tmp_path):+ model = _SaveableModel()+ output_dir = str(tmp_path / "woq_model")++ FakeFormat("fake", PRESET_SCHEMES["INT4"], SimpleNamespace(mllm=False)).save_quantized(+ output_dir=output_dir,+ model=model,+ inplace=False,+ serialization_dict={+ "bits": 4,+ "group_size": 128,+ "sym": True,+ "data_type": "int",+ "quant_method": "auto-round",+ "packing_format": "auto_round:auto_gptq",+ "to_quant_block_names": ["block"],+ "supported_types": [torch.nn.Linear],+ },+ )++ with open(os.path.join(output_dir, "config.json")) as config_file:+ quantization_config = json.load(config_file)["quantization_config"]++ assert quantization_config["packing_format"] == "auto_round:auto_gptq"+ assert "act_bits" not in quantization_config+++def test_fake_format_still_saves_when_env_disabled(tmp_path):+ model = _SaveableModel()+ output_dir = str(tmp_path / "save_model_even_when_env_disabled")++ returned = FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(+ output_dir=output_dir,+ model=model,+ inplace=False,+ serialization_dict={+ "bits": 4,+ "group_size": 16,+ "sym": True,+ "data_type": "nvfp4_v2",+ "act_bits": 4,+ "act_group_size": 16,+ "act_sym": True,+ "act_data_type": "nvfp4_v2",+ "to_quant_block_names": ["block"],+ "supported_types": [torch.nn.Linear],+ },+ )+ assert returned is not None+ assert os.path.exists(output_dir)+ assert os.path.exists(os.path.join(output_dir, "config.json"))+++def test_fake_backend_accepts_mxfp_roundtrip_config():+ layer_backend = get_layer_backend(+ "cpu",+ "auto",+ "auto_round:fake",+ {+ "bits": 4,+ "group_size": 32,+ "sym": True,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_group_size": 32,+ "act_sym": True,+ "act_data_type": "mx_fp",+ "act_dynamic": True,+ },+ 32,+ 32,+ )++ assert layer_backend == "auto_round:fake"+++def test_fake_backend_accepts_mxfp8_roundtrip_config():+ layer_backend = get_layer_backend(+ "cpu",+ "auto",+ "auto_round:fake",+ {+ "bits": 8,+ "group_size": 32,+ "sym": True,+ "data_type": "mx_fp",+ "act_bits": 8,+ "act_group_size": 32,+ "act_sym": True,+ "act_data_type": "mx_fp",+ "act_dynamic": True,+ },+ 32,+ 32,+ )++ assert layer_backend == "auto_round:fake"+++def test_fake_backend_accepts_nvfp_roundtrip_config():+ layer_backend = get_layer_backend(+ "cpu",+ "auto",+ "auto_round:fake",+ {+ "bits": 4,+ "group_size": 16,+ "sym": True,+ "data_type": "nv_fp",+ "act_bits": 4,+ "act_group_size": 16,+ "act_sym": True,+ "act_data_type": "nv_fp",+ "act_dynamic": True,+ },+ 16,+ 16,+ )++ assert layer_backend == "auto_round:fake"+++def test_fake_backend_fallback_for_woq_w4a8_dynamic():+ """WOQ packing with unsupported act scheme should gracefully fallback to fake backend."""+ layer_backend = get_layer_backend(+ "cpu",+ "auto",+ "auto_round:auto_gptq",+ {+ "bits": 4,+ "group_size": 128,+ "sym": True,+ "data_type": "int",+ "act_bits": 8,+ "act_group_size": 32,+ "act_sym": True,+ "act_data_type": "int",+ "act_dynamic": True,+ },+ 3072,+ 768,+ )++ assert layer_backend == "auto_round:fake"+++def test_fake_format_keeps_in_memory_wrapper_structure_on_save(tmp_path):+ model = _SaveableModel()+ output_dir = str(tmp_path / "fake_keep_wrapper_model")++ saved_model = FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(+ output_dir=output_dir,+ model=model,+ inplace=False,+ serialization_dict={+ "bits": 4,+ "group_size": 16,+ "sym": True,+ "data_type": "nvfp4_v2",+ "act_bits": 4,+ "act_group_size": 16,+ "act_sym": True,+ "act_data_type": "nvfp4_v2",+ "to_quant_block_names": ["block"],+ },+ )++ assert hasattr(saved_model.linear, "orig_layer")+ assert not isinstance(saved_model.linear, FakeActQuantLinear)+++def test_normalize_state_dict_keys_strips_orig_layer_segment():+ state = {+ "model.layers.0.fc1.orig_layer.weight": torch.ones(1),+ "model.layers.0.fc1.orig_layer.bias": torch.zeros(1),+ "model.layers.0.fc2.weight": torch.randn(1),+ }++ normalized, changed = _normalize_state_dict_keys(state)++ assert changed is True+ assert "model.layers.0.fc1.weight" in normalized+ assert "model.layers.0.fc1.bias" in normalized+ assert "model.layers.0.fc1.orig_layer.weight" not in normalized+ assert "model.layers.0.fc2.weight" in normalized+++def test_rewrite_saved_weights_without_orig_layer_for_safetensors(tmp_path):+ from safetensors.torch import load_file as safe_load_file+ from safetensors.torch import save_file as safe_save_file++ ckpt_dir = tmp_path / "fake_ckpt"+ ckpt_dir.mkdir(parents=True)+ ckpt_path = ckpt_dir / "model.safetensors"++ safe_save_file(+ {+ "model.layers.0.fc1.orig_layer.weight": torch.randn(2, 2),+ "model.layers.0.fc1.orig_layer.bias": torch.randn(2),+ "model.layers.0.fc2.weight": torch.randn(2, 2),+ },+ str(ckpt_path),+ )++ _rewrite_saved_weights_without_orig_layer(str(ckpt_dir))++ rewritten = safe_load_file(str(ckpt_path))+ assert "model.layers.0.fc1.orig_layer.weight" not in rewritten+ assert "model.layers.0.fc1.weight" in rewritten+ assert "model.layers.0.fc1.bias" in rewrittendiff --git a/test/unit/test_cuda/quantization/test_act_quantization.py b/test/unit/test_cuda/quantization/test_act_quantization.pyindex f100ae4c9..295510c32 100644--- a/test/unit/test_cuda/quantization/test_act_quantization.py+++ b/test/unit/test_cuda/quantization/test_act_quantization.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import osimport shutilimport pytest@@ -19,6 +20,8 @@ import torchfrom transformers import AutoModelForCausalLMfrom auto_round import AutoRound+from auto_round.experimental.qmodules.fake import FakeActQuantLinear+from auto_round.wrapper import WrapperWALayerpytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires CUDA")@@ -132,3 +135,74 @@ class TestActQuantizationGpu:)with pytest.raises(ValueError, match="does not support exporting"):autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ @pytest.mark.timeout(180)+ def test_w4a8_reload_layer_type_diff_but_output_equal(self, tiny_opt_model_path):+ """Save path keeps wrappers; reload path materializes FakeActQuantLinear, outputs should match."""+ old_offload = os.environ.get("AR_DISABLE_OFFLOAD")+ os.environ["AR_DISABLE_OFFLOAD"] = "1"+ try:+ autoround = AutoRound(+ tiny_opt_model_path,+ bits=4,+ act_bits=8,+ group_size=128,+ act_group_size=32,+ act_dynamic=True,+ sym=True,+ iters=0,+ disable_opt_rtn=True,+ nsamples=2,+ seqlen=16,+ )+ quantized_model, quantized_model_path = autoround.quantize_and_save(+ output_dir=self.save_dir, format="auto_round"+ )++ quantized_model.eval()+ reloaded_model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, device_map="cuda:0", trust_remote_code=True+ )+ reloaded_model.eval()++ src_layer = quantized_model.model.decoder.layers[0].fc2+ dst_layer = reloaded_model.model.decoder.layers[0].fc2+ assert isinstance(src_layer, WrapperWALayer)+ assert isinstance(dst_layer, FakeActQuantLinear)+ assert type(src_layer) is not type(dst_layer)++ state_path = os.path.join(quantized_model_path, "model.safetensors")+ if os.path.exists(state_path):+ from safetensors.torch import load_file as load_state_dict++ state_dict = load_state_dict(state_path)+ else:+ state_dict = torch.load(os.path.join(quantized_model_path, "pytorch_model.bin"), weights_only=True)++ weight_key = "model.decoder.layers.0.fc2.weight"+ bias_key = "model.decoder.layers.0.fc2.bias"+ assert weight_key in state_dict+ assert bias_key in state_dict+ assert "model.decoder.layers.0.fc2.orig_layer.weight" not in state_dict++ torch.manual_seed(7)+ probe = torch.randn(2, 3, dst_layer.in_features, device="cuda:0", dtype=dst_layer.weight.dtype)+ with torch.no_grad():+ qdq_probe = dst_layer.qdq_input(probe)+ src_out = (+ torch.nn.functional.linear(+ qdq_probe,+ state_dict[weight_key].to(qdq_probe.device, qdq_probe.dtype),+ state_dict[bias_key].to(qdq_probe.device, qdq_probe.dtype),+ )+ .float()+ .cpu()+ )+ dst_out = dst_layer(probe).float().cpu()++ torch.testing.assert_close(src_out, dst_out, rtol=1e-3, atol=1e-3)+ finally:+ if old_offload is None:+ os.environ.pop("AR_DISABLE_OFFLOAD", None)+ else:+ os.environ["AR_DISABLE_OFFLOAD"] = old_offload