PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix FP8 CT export metadata for KV cache and attention (#1752)
Yi Liu24df42ea643c
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex 3f5ee91f3..c220dfe8a 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -62,15 +62,14 @@ def pack_layer(layer_name: str, model: torch.nn.Module, data_type: str, device:fp8_pack_layer(layer_name, model, data_type, device, unsqueeze=True)-def _construct_kv_scheme():- """Construct the default KV cache quantization scheme for FP8_STATIC export."""+def _construct_fp8_args():from compressed_tensors.quantization import ( # pylint: disable=E0401QuantizationArgs,QuantizationStrategy,QuantizationType,)- default_kv_scheme = QuantizationArgs(+ return QuantizationArgs(num_bits=8,type=QuantizationType.FLOAT,strategy=QuantizationStrategy.TENSOR,@@ -78,6 +77,11 @@ def _construct_kv_scheme():dynamic=False,)++def _construct_kv_scheme():+ """Construct the default KV cache quantization scheme for FP8_STATIC export."""+ default_kv_scheme = _construct_fp8_args()+logger.warning_once("Using default KV cache scheme: %s. ""Currently, only this KV cache scheme is supported for FP8_STATIC + FP8 KV.",@@ -95,6 +99,42 @@ def _use_fp8_kv(static_kv_dtype: str | None) -> bool:return False+def _use_fp8_attention(static_attention_dtype: str | None) -> bool:+ """Return True if static attention should use FP8."""+ if static_attention_dtype in ("fp8", "float8_e4m3fn"):+ logger.warning_once("Exporting model with static attention in FP8 dtype.")+ return True+ return False+++def _get_attention_targets(model: torch.nn.Module) -> list[str]:+ from compressed_tensors.quantization.lifecycle.initialize import is_attention_module # pylint: disable=E0401++ attention_targets = []+ for module in model.modules():+ if is_attention_module(module):+ target = type(module).__name__+ if target not in attention_targets:+ attention_targets.append(target)+ return attention_targets+++def _append_attention_group(config_groups: dict, model: torch.nn.Module) -> None:+ from compressed_tensors.quantization import QuantizationScheme # pylint: disable=E0401++ attention_targets = _get_attention_targets(model)+ if len(attention_targets) == 0:+ logger.warning("Skipping FP8 attention config group because no attention modules were detected.")+ return++ group_name = f"group_{len(config_groups)}"+ config_groups[group_name] = QuantizationScheme(+ targets=attention_targets,+ input_activations=_construct_fp8_args(),+ weights=None,+ )++def _configure_gaudi2_fp8_dtype(quantization_config: dict) -> None:"""Configure FP8 dtype flavor for Intel Gaudi2 hardware compatibility."""_GAUDI2_FP8_DTYPE_FLAVOR = "float8_e4m3fnuz"@@ -184,13 +224,7 @@ def save_quantized_as_static_fp(symmetric=True,dynamic=False,),- input_activations=QuantizationArgs(- num_bits=8,- type=QuantizationType.FLOAT,- strategy=QuantizationStrategy.TENSOR,- symmetric=True,- dynamic=False,- ),+ input_activations=_construct_fp8_args(),)targets = ["Linear"]ignore = []@@ -200,13 +234,15 @@ def save_quantized_as_static_fp(config_groups = {}scheme = QuantizationScheme(targets=targets, **scheme_args)config_groups["group_0"] = scheme+ if _use_fp8_attention(serialization_dict.get("static_attention_dtype", None)):+ _append_attention_group(config_groups, model)quantization_config = QuantizationConfig(config_groups=config_groups,kv_cache_scheme=(_construct_kv_scheme()if (_use_fp8_kv(serialization_dict.get("static_kv_dtype", None))- or _use_fp8_kv(serialization_dict.get("static_attention_dtype", None))+ or _use_fp8_attention(serialization_dict.get("static_attention_dtype", None)))else None),diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.pyindex c00e5e6df..d9e79b631 100644--- a/test/test_cpu/export/test_llmc_format.py+++ b/test/test_cpu/export/test_llmc_format.py@@ -1,3 +1,4 @@+import jsonimport osimport shutil@@ -164,13 +165,14 @@ class TestLLMC:), f"Invalid mixed precision quantization configuration: {quantization_config}"-def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, tmp_path, monkeypatch):+def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, dataloader, tmp_path, monkeypatch):autoround = AutoRound(tiny_opt_model_path,scheme="FP8_STATIC",seqlen=8,nsamples=2,iters=0,+ dataset=dataloader,)autoround.quantize()forbid_threaded_packing(monkeypatch, llmc_static_fp_export)@@ -178,6 +180,85 @@ def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, tmp_assert os.path.exists(os.path.join(tmp_path, "config.json"))+def test_llmcompressor_static_fp8_kv_config(tiny_opt_model_path, dataloader, tmp_path):+ autoround = AutoRound(+ tiny_opt_model_path,+ scheme="FP8_STATIC",+ seqlen=8,+ nsamples=2,+ iters=0,+ dataset=dataloader,+ static_kv_dtype="fp8",+ )+ _, quantized_model_path = autoround.quantize_and_save(tmp_path, format="llm_compressor")++ with open(os.path.join(quantized_model_path, "config.json")) as f:+ config = json.load(f)+ kv_cache_scheme = config["quantization_config"]["kv_cache_scheme"]+ assert kv_cache_scheme is not None+ assert kv_cache_scheme["num_bits"] == 8+ assert kv_cache_scheme["type"] == "float"+ assert kv_cache_scheme["strategy"] == "tensor"+ assert kv_cache_scheme["dynamic"] is False+ assert kv_cache_scheme["symmetric"] is True+++def test_llmcompressor_static_fp8_attention_config(dataloader, tmp_path):+ model_name = get_model_path("stas/tiny-random-llama-2")+ autoround = AutoRound(+ model_name,+ scheme="FP8_STATIC",+ seqlen=8,+ nsamples=2,+ iters=0,+ dataset=dataloader,+ static_attention_dtype="fp8",+ )+ _, quantized_model_path = autoround.quantize_and_save(tmp_path, format="llm_compressor")++ with open(os.path.join(quantized_model_path, "config.json")) as f:+ saved_config = json.load(f)+ saved_groups = saved_config["quantization_config"]["config_groups"]+ attention_group = None+ for group in saved_groups.values():+ if "Linear" not in group["targets"]:+ attention_group = group+ break++ assert attention_group is not None+ assert attention_group["weights"] is None+ assert attention_group["input_activations"]["num_bits"] == 8+ assert attention_group["input_activations"]["type"] == "float"+ assert attention_group["input_activations"]["strategy"] == "tensor"+ assert attention_group["input_activations"]["dynamic"] is False+ assert attention_group["input_activations"]["symmetric"] is True+ assert saved_config["quantization_config"]["kv_cache_scheme"] is not None++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", trust_remote_code=True)+ quantization_config = model.config.quantization_config+ config = quantization_config.to_dict() if hasattr(quantization_config, "to_dict") else quantization_config+ config_groups = config["config_groups"]++ assert "group_0" in config_groups+ attention_group = None+ for group in config_groups.values():+ targets = group["targets"]+ if "Linear" not in targets:+ attention_group = group+ break++ assert attention_group is not None+ assert attention_group["targets"] == [model.model.layers[0].self_attn.__class__.__name__]+ assert attention_group["weights"] is None+ assert attention_group["input_activations"]["num_bits"] == 8+ assert attention_group["input_activations"]["type"] == "float"+ assert attention_group["input_activations"]["strategy"] == "tensor"+ assert attention_group["input_activations"]["dynamic"] is False+ assert attention_group["input_activations"]["symmetric"] is True+ assert getattr(model.model.layers[0].self_attn, "q_scale", None) is not None+ assert config["kv_cache_scheme"] is not None++def test_llmcompressor_mxfp8_export_packs_serially(tmp_path, monkeypatch):autoround = AutoRound(model=opt_name_or_path,