Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix device mismatch issue for unifying qkv scale (#1175)

Xin He13ba747f45df
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index d8b29364c..401617d94 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -660,6 +660,10 @@ class BaseCompressor(object):
if self.enable_torch_compile and is_wfp8afp8(self) and not is_hpex_available():
self.enable_torch_compile = False
logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")
+ # TODO: fix https://github.com/intel/auto-round/issues/1109
+ if self.enable_torch_compile and is_nv_fp(self.act_data_type):
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as nvfp4 is enabled")
def _dq_check(self) -> None:
"""Reset the default value of super_bits and super_group_size"""
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 890329f5c..820ec4bde 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -290,32 +290,31 @@ def update_fused_layer_global_scales(
hasattr(module, "gate_proj") and hasattr(module, "up_proj")
)
- # ---------------- Attention ----------------
- if _is_attention_module(submodule):
- # Already fused
- if hasattr(submodule, "qkv_proj"):
- return
-
- scales = _collect_scales([submodule.q_proj, submodule.k_proj, submodule.v_proj])
+ def _update_global_scales(modules: List[Module]):
+ """Update global scales for a list of modules."""
+ scales = _collect_scales(modules)
if not scales:
return
- global_scale = torch.min(torch.stack(scales), dim=0).values
+ # Move all scales to the same device before stacking
+ target_device = scales[0].device
+ scales_on_device = [s.to(target_device) for s in scales]
+ global_scale = torch.min(torch.stack(scales_on_device), dim=0).values
- for proj in (submodule.q_proj, submodule.k_proj, submodule.v_proj):
+ for proj in modules:
if hasattr(proj, global_scale_name):
- setattr(proj, global_scale_name, global_scale.clone())
+ # Move global_scale to the same device as the projection's current scale
+ proj_scale = getattr(proj, global_scale_name)
+ setattr(proj, global_scale_name, global_scale.clone().to(proj_scale.device))
+ # ---------------- Attention ----------------
+ if _is_attention_module(submodule):
+ # Already fused
+ if hasattr(submodule, "qkv_proj"):
+ return
+ _update_global_scales([submodule.q_proj, submodule.k_proj, submodule.v_proj])
return
# ---------------- MLP ----------------
if _is_mlp_module(submodule):
- scales = _collect_scales([submodule.gate_proj, submodule.up_proj])
- if not scales:
- return
-
- global_scale = torch.min(torch.stack(scales), dim=0).values
-
- for proj in (submodule.gate_proj, submodule.up_proj):
- if hasattr(proj, global_scale_name):
- setattr(proj, global_scale_name, global_scale.clone())
+ _update_global_scales([submodule.gate_proj, submodule.up_proj])
diff --git a/test/test_cpu/test_init.py b/test/test_cpu/test_init.py
new file mode 100644
index 000000000..6ebee954d
--- /dev/null
+++ b/test/test_cpu/test_init.py
@@ -0,0 +1,8 @@
+from auto_round import AutoRound
+
+
+def test_torch_compile():
+ ar = AutoRound(model="facebook/opt-125m", scheme="NVFP4", enable_torch_compile=True)
+ assert not ar.enable_torch_compile, "NVFP4 cannot work with torch.compile."
+ ar = AutoRound(model="facebook/opt-125m", scheme="FP8_STATIC", enable_torch_compile=True)
+ assert not ar.enable_torch_compile, "FP8_STATIC cannot work with torch.compile."
diff --git a/test/test_cuda/test_multiple_card_calib.py b/test/test_cuda/test_multiple_card_calib.py
index 490193532..8d97046fb 100644
--- a/test/test_cuda/test_multiple_card_calib.py
+++ b/test/test_cuda/test_multiple_card_calib.py
@@ -41,6 +41,17 @@ class TestAutoRound(unittest.TestCase):
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
+ @multi_card
+ def test_multiple_card_nvfp4(self):
+ python_path = sys.executable
+
+ ##test llm script
+ res = os.system(
+ f"cd ../.. && {python_path} -m auto_round --model facebook/opt-125m --scheme NVFP4 --devices '0,1' --iters 1 --nsamples 1 --enable_torch_compile --low_gpu_mem_usage"
+ )
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+
if __name__ == "__main__":
unittest.main()