PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix device mismatch issue for unifying qkv scale (#1175)
Xin He13ba747f45df
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex d8b29364c..401617d94 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -660,6 +660,10 @@ class BaseCompressor(object):if self.enable_torch_compile and is_wfp8afp8(self) and not is_hpex_available():self.enable_torch_compile = Falselogger.warning("reset enable_torch_compile to `False` as fp8 is enabled")+ # TODO: fix https://github.com/intel/auto-round/issues/1109+ if self.enable_torch_compile and is_nv_fp(self.act_data_type):+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as nvfp4 is enabled")def _dq_check(self) -> None:"""Reset the default value of super_bits and super_group_size"""diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 890329f5c..820ec4bde 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -290,32 +290,31 @@ def update_fused_layer_global_scales(hasattr(module, "gate_proj") and hasattr(module, "up_proj"))- # ---------------- Attention ----------------- if _is_attention_module(submodule):- # Already fused- if hasattr(submodule, "qkv_proj"):- return-- scales = _collect_scales([submodule.q_proj, submodule.k_proj, submodule.v_proj])+ def _update_global_scales(modules: List[Module]):+ """Update global scales for a list of modules."""+ scales = _collect_scales(modules)if not scales:return- global_scale = torch.min(torch.stack(scales), dim=0).values+ # Move all scales to the same device before stacking+ target_device = scales[0].device+ scales_on_device = [s.to(target_device) for s in scales]+ global_scale = torch.min(torch.stack(scales_on_device), dim=0).values- for proj in (submodule.q_proj, submodule.k_proj, submodule.v_proj):+ for proj in modules:if hasattr(proj, global_scale_name):- setattr(proj, global_scale_name, global_scale.clone())+ # Move global_scale to the same device as the projection's current scale+ proj_scale = getattr(proj, global_scale_name)+ setattr(proj, global_scale_name, global_scale.clone().to(proj_scale.device))+ # ---------------- Attention ----------------+ if _is_attention_module(submodule):+ # Already fused+ if hasattr(submodule, "qkv_proj"):+ return+ _update_global_scales([submodule.q_proj, submodule.k_proj, submodule.v_proj])return# ---------------- MLP ----------------if _is_mlp_module(submodule):- scales = _collect_scales([submodule.gate_proj, submodule.up_proj])- if not scales:- return-- global_scale = torch.min(torch.stack(scales), dim=0).values-- for proj in (submodule.gate_proj, submodule.up_proj):- if hasattr(proj, global_scale_name):- setattr(proj, global_scale_name, global_scale.clone())+ _update_global_scales([submodule.gate_proj, submodule.up_proj])diff --git a/test/test_cpu/test_init.py b/test/test_cpu/test_init.pynew file mode 100644index 000000000..6ebee954d--- /dev/null+++ b/test/test_cpu/test_init.py@@ -0,0 +1,8 @@+from auto_round import AutoRound+++def test_torch_compile():+ ar = AutoRound(model="facebook/opt-125m", scheme="NVFP4", enable_torch_compile=True)+ assert not ar.enable_torch_compile, "NVFP4 cannot work with torch.compile."+ ar = AutoRound(model="facebook/opt-125m", scheme="FP8_STATIC", enable_torch_compile=True)+ assert not ar.enable_torch_compile, "FP8_STATIC cannot work with torch.compile."diff --git a/test/test_cuda/test_multiple_card_calib.py b/test/test_cuda/test_multiple_card_calib.pyindex 490193532..8d97046fb 100644--- a/test/test_cuda/test_multiple_card_calib.py+++ b/test/test_cuda/test_multiple_card_calib.py@@ -41,6 +41,17 @@ class TestAutoRound(unittest.TestCase):if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"+ @multi_card+ def test_multiple_card_nvfp4(self):+ python_path = sys.executable++ ##test llm script+ res = os.system(+ f"cd ../.. && {python_path} -m auto_round --model facebook/opt-125m --scheme NVFP4 --devices '0,1' --iters 1 --nsamples 1 --enable_torch_compile --low_gpu_mem_usage"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+if __name__ == "__main__":unittest.main()