PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Reduce xpu memory usage with patch_xpu_sdpa_drop_causal_mask (#1716)
Xin Hedc8a6a4428c7
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 3b88c0e60..9a751968f 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -353,7 +353,7 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_devblock_inputs[module_name] = input_infomodule.to("cpu")- # torch.cuda.empty_cache()+ clear_memory()# Enable gradients for the output of the last blockif module.tmp_name == block_names[-1]:@@ -410,6 +410,8 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):passcurrent_grad = last_grad_input+ del output, data+ clear_memory()# Manually compute gradients block by blocklast_block_backward_hook.remove()@@ -468,8 +470,9 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):print(f"Warning: No suitable input gradient found for {block_name}")break+ del block_output, main_output, block_input_args, block_input_kwargsblock_module.to("cpu")- # clear_memory()+ clear_memory()pbar.update(1)diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex bbeb85d0d..238df50e5 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -117,6 +117,7 @@ from auto_round.utils.device import (clear_memory_if_reached_threshold,get_major_device,parse_available_devices,+ patch_xpu_sdpa_drop_causal_mask,set_auto_device_map_for_block_with_tuning,set_non_auto_device_map,)@@ -401,6 +402,11 @@ class BaseCompressor(object):else:torch.use_deterministic_algorithms(True, warn_only=True)+ # XPU SDPA workaround: drop pure causal masks so FLASH backend is used,+ # and set torch.use_deterministic_algorithms(False)+ # instead of MATH (avoids ~10x peak-VRAM blow-up during block tuning).+ patch_xpu_sdpa_drop_causal_mask()+self.to_quant_block_names = to_quant_block_namesif not hasattr(self, "quant_block_list"):all_blocks = get_block_names(model)diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex f38e233be..a4727a996 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -185,6 +185,82 @@ def is_hpex_available():return _hpex_available+_xpu_sdpa_patched = False+++# TODO: This is a workaround for the XPU SDPA memory blow-up issue.+# We should remove this patch after the issue is fixed in XPU side.+# https://github.com/intel/auto-round/issues/990+def patch_xpu_sdpa_drop_causal_mask():+ """Workaround for XPU peak-VRAM blow-up in SDPA.++ On Intel XPU, ``torch.nn.functional.scaled_dot_product_attention`` falls back+ to the MATH backend whenever ``attn_mask`` is non-None (FLASH on XPU does+ not support explicit attn_mask, EFFICIENT/CUDNN are not implemented).+ The MATH backend materializes the full ``(B, H, S, S)`` probability matrix+ in both forward and backward, costing several GB at typical+ ``batch_size=8, seqlen=2048``.++ HuggingFace transformers happily passes a *pure causal* 4D mask, even+ though the same effect is achievable via ``is_causal=True`` (which the+ FLASH backend supports and which uses ~12x less memory).++ This monkey-patch detects a pure causal mask, drops it, and forwards+ ``is_causal=True`` instead -- only on XPU and only when no real mask info+ would be lost. CPU/CUDA/HPU paths are left untouched.++ Idempotent. Safe to call multiple times.+ """+ global _xpu_sdpa_patched+ if _xpu_sdpa_patched:+ return+ if not (hasattr(torch, "xpu") and torch.xpu.is_available()):+ return+ torch.use_deterministic_algorithms(False)++ _orig_sdpa = torch.nn.functional.scaled_dot_product_attention++ def _is_pure_causal_mask(mask: torch.Tensor, s: int) -> bool:+ # Cheap shape check first+ if mask.shape[-1] != s or mask.shape[-2] != s:+ return False+ if mask.dtype not in (torch.float16, torch.bfloat16, torch.float32):+ return False+ # Pick the first (B,H) slice; HF mask is broadcast across batch/heads.+ m2d = mask.reshape(-1, s, s)[0]+ tri_up = torch.triu(torch.ones(s, s, dtype=torch.bool, device=mask.device), 1)+ return bool(torch.isinf(m2d[tri_up]).all().item()) and bool((m2d[~tri_up] == 0).all().item())++ def _patched_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None, **kwargs):+ if (+ query.device.type == "xpu"+ and attn_mask is not None+ and not is_causal+ and query.shape[-2] == key.shape[-2] # square QK length (no kv-cache)+ and _is_pure_causal_mask(attn_mask, query.shape[-2])+ ):+ attn_mask = None+ is_causal = True+ return _orig_sdpa(+ query,+ key,+ value,+ attn_mask=attn_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ **kwargs,+ )++ torch.nn.functional.scaled_dot_product_attention = _patched_sdpa+ _xpu_sdpa_patched = True+ logger.info("torch.use_deterministic_algorithms(False) is set for XPU.")+ logger.info(+ "Patched torch SDPA on XPU to use is_causal=True for pure causal masks "+ "(avoids ~10x peak-VRAM blow-up from MATH backend)."+ )++def check_is_cpu(device):"""Check if the device is a CPU.@@ -1121,15 +1197,6 @@ def estimate_tuning_block_mem(# and multiple expert activations. Here we use a conservative estimate.moe_additional_memory = additional_memory * 6 # GBadditional_memory += moe_additional_memory- if torch.xpu.is_available():- # https://github.com/intel/torch-xpu-ops/issues/2232- # TODO: XPU takes more memory than expected. for llama 8B, it's about 12 GB- xpu_additional_memory = 12 # GB- additional_memory += xpu_additional_memory- # logger.warning_once(- # "[Memory Estimation]: If there is an abnormal memory issue, please collect log with "- # + "AR_LOG_LEVEL=debug and raise issue to us."- # )return layer_memory_dict, layer_activation_memory, block_input_output_memory, additional_memorydiff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 7ed9133f7..3f67deb63 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -796,27 +796,26 @@ def is_gguf_model(model_path: Union[str, torch.nn.Module]) -> bool:return is_gguf_file-def is_diffusion_model(model_or_path: Union[str, object]) -> bool:+def is_diffusion_model(model_or_path: Union[str, object], trust_remote_code: bool = True) -> bool:from auto_round.utils.common import LazyImport- # First check if it's a known diffusion pipeline by config/model_type- # to avoid unnecessary imports and file checks for non-diffusion models, which can be time-consuming.- try:- from transformers import AutoConfig-- config = AutoConfig.from_pretrained(model_or_path, trust_remote_code=True)- model_type = getattr(config, "model_type", "")- # A special case for NextStep- if model_type == "nextstep":- return True- except:- logger.warning(- f"Failed to load config for {model_or_path}, trying to check model_index.json for diffusion pipeline."- )-# Then check if model_index.json exists for diffusion pipeline,# which is a strong signal of being a diffusion pipeline.if isinstance(model_or_path, str):+ # First check if it's a known diffusion pipeline by config/model_type+ # to avoid unnecessary imports and file checks for non-diffusion models, which can be time-consuming.+ try:+ from transformers import AutoConfig++ config = AutoConfig.from_pretrained(model_or_path, trust_remote_code=trust_remote_code)+ model_type = getattr(config, "model_type", "")+ # A special case for NextStep+ if model_type == "nextstep":+ return True+ except:+ logger.warning(+ f"Failed to load config for {model_or_path}, trying to check model_index.json for diffusion pipeline."+ )index_file = Noneif not os.path.isdir(model_or_path):try:diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 4ca147926..53acef0bc 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -476,17 +476,14 @@ The 3B and 14B models were evaluated on Qwen 2.5, the 8X7B model is Mixtral, whi| 2.5 w/o torch compile | 8min<br/>10GB | 16min<br/>20GB | 30min<br/>25GB | 140min<br/>49GB | 50min<br/>49GB |W4G128 Quantization Time and Memory Usage (Intel GPU B60 24G)-Testing was conducted on the Intel GPU B60 24G using the release version of PyTorch 2.11.0+xpu. Please note that data loading and packing costs have been excluded from the evaluation. Time and memory usage were measured using Qwen3-series models.--| Torch version/Config W4G128 | 0.6B | 1.7B | 4B | 8B | 30B-A3B |-|------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------|---------------------|---------------------|---------------------|-| 2.11.0+xpu with torch compile | 20min<br/>10.7GB | 26min<br/>13.2GB | 58min<br/>22.8GB | OOM | OOM |-| 2.11.0+xpu with torch compile<br/>low_gpu_mem_usage=True | 29min<br/>9.5GB | 38min<br/>9.8GB | 1h 23min<br/>19.4GB | 1h 32min<br/>20.1GB | 5h 33min<br/>22.8GB |-| 2.11.0+xpu with torch compile<br/>low_gpu_mem_usage=True<br/>gradient_accumulate_steps=8,bs=1 | 41min<br/>1.3GB | 42min<br/>1.8GB | 1h 29min<br/>3.6GB | 2h 4min<br/>4.6GB | 21h 41min<br/>10.2GB |-| 2.11.0+xpu w/o torch compile | 20min<br/>10.9GB | 28min<br/>13.2GB | OOM | OOM | OOM |---+Testing was conducted on the Intel GPU B60 24G using the release version of PyTorch 2.11.0+xpu. Please note that data loading and packing costs have been excluded from the evaluation. Time and memory usage were measured using Qwen2.5-series models.++| Torch version/Config W4G128 | 0.5B | 1.5B | 3B | 7B |+|------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------|---------------------|---------------------|+| 2.11.0+xpu with torch compile | 6min<br/>2.9GB | 13min<br/>5.4GB | 22min<br/>7.1GB | 40min<br/>14.9GB |+| 2.11.0+xpu with torch compile<br/>low_gpu_mem_usage=True | 10min<br/>1.7GB | 17min<br/>3.3GB | 30min<br/>4.3GB | 50min<br/>8.5GB |+| 2.11.0+xpu with torch compile<br/>low_gpu_mem_usage=True<br/>gradient_accumulate_steps=8,bs=1 | 14min<br/>0.4GB | 22min<br/>1.1GB | 38min<br/>1.5GB | 1h 4min<br/>4.1GB |+| 2.11.0+xpu w/o torch compile | 6min<br/>2.9GB | 14min<br/>5.7GB | 26min<br/>7.6GB | 51min<br/>15.5GB |### Device/Multi-GPU setting in Quantization**The tuning device is specified using the `device_map` argument in AutoRound API, _not_ through the `device_map`diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 0e2c6bec1..5eddd22d3 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -462,14 +462,13 @@ ar.quantize_and_save(output_dir, format="gguf:q4_k_m") # gguf:q*_k_s、gguf:q*_| 2.5 + 关闭 torch compile | 8min<br/>10GB | 16min<br/>20GB | 30min<br/>25GB | 140min<br/>49GB | 50min<br/>49GB |W4G128 量化耗时与显存占用(英特尔 GPU B60 24G)-(测试环境:英特尔 GPU B60 24G,PyTorch 2.11.0+xpu 正式版。注意评测未计入数据加载和打包耗时。所有测试均使用 Qwen3 系列模型。)--| Torch version/Config W4G128 | 0.6B | 1.7B | 4B | 8B | 30B-A3B |-|------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------|---------------------|---------------------|---------------------|-| 2.11.0+xpu 开启 torch compile | 20min<br/>10.7GB | 26min<br/>13.2GB | 58min<br/>22.8GB | OOM | OOM |-| 2.11.0+xpu 开启 torch compile<br/>low_gpu_mem_usage=True | 29min<br/>9.5GB | 38min<br/>9.8GB | 1h 23min<br/>19.4GB | 1h 32min<br/>20.1GB | 5h 33min<br/>22.8GB |-| 2.11.0+xpu 开启 torch compile<br/>low_gpu_mem_usage=True<br/>gradient_accumulate_steps=8,bs=1 | 41min<br/>1.3GB | 42min<br/>1.8GB | 1h 29min<br/>3.6GB | 2h 4min<br/>4.6GB | 21h 41min<br/>10.2GB |-| 2.11.0+xpu 关闭 torch compile | 20min<br/>10.9GB | 28min<br/>13.2GB | OOM | OOM | OOM |+(测试环境:英特尔 GPU B60 24G,PyTorch 2.11.0+xpu 正式版。注意评测未计入数据加载和打包耗时。所有测试均使用 Qwen2.5 系列模型。)+| Torch version/Config W4G128 | 0.5B | 1.5B | 3B | 7B |+|------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------|---------------------|---------------------|+| 2.11.0+xpu 开启 torch compile | 6min<br/>2.9GB | 13min<br/>5.4GB | 22min<br/>7.1GB | 40min<br/>14.9GB |+| 2.11.0+xpu 开启 torch compile<br/>low_gpu_mem_usage=True | 10min<br/>1.7GB | 17min<br/>3.3GB | 30min<br/>4.3GB | 50min<br/>8.5GB |+| 2.11.0+xpu 开启 torch compile<br/>low_gpu_mem_usage=True<br/>gradient_accumulate_steps=8,bs=1 | 14min<br/>0.4GB | 22min<br/>1.1GB | 38min<br/>1.5GB | 1h 4min<br/>4.1GB |+| 2.11.0+xpu 关闭 torch compile | 6min<br/>2.9GB | 14min<br/>5.7GB | 26min<br/>7.6GB | 51min<br/>15.5GB |### 设备及多卡量化设置diff --git a/test/test_xpu/test_xpu_sdpa_patch.py b/test/test_xpu/test_xpu_sdpa_patch.pynew file mode 100644index 000000000..c0201af2f--- /dev/null+++ b/test/test_xpu/test_xpu_sdpa_patch.py@@ -0,0 +1,50 @@+import pytest+import torch+import torch.nn.functional as F++from auto_round.utils.device import patch_xpu_sdpa_drop_causal_mask++[email protected](not (hasattr(torch, "xpu") and torch.xpu.is_available()), reason="XPU not available")+def test_patch_xpu_sdpa_drop_causal_mask():+ """Test that patch_xpu_sdpa_drop_causal_mask correctly identifies and replaces causal masks on XPU."""+ patch_xpu_sdpa_drop_causal_mask()++ s = 128+ query = torch.randn(1, 8, s, 64, device="xpu", dtype=torch.float16)+ key = torch.randn(1, 8, s, 64, device="xpu", dtype=torch.float16)+ value = torch.randn(1, 8, s, 64, device="xpu", dtype=torch.float16)++ # Create a pure causal mask (HF style: small values for non-masked, -inf for masked)+ mask = torch.zeros(1, 1, s, s, device="xpu", dtype=torch.float16)+ tri_up = torch.triu(torch.ones(s, s, dtype=torch.bool, device="xpu"), 1)+ mask.reshape(-1, s, s)[0][tri_up] = float("-inf")++ # We can't easily check internal calls without complex mocking,+ # but we can verify it doesn't crash and produces same results as standard causal=True++ # 1. Test with causal mask (should be patched to is_causal=True internally)+ output_with_mask = F.scaled_dot_product_attention(query, key, value, attn_mask=mask, is_causal=False)++ # 2. Test with is_causal=True directly+ output_is_causal = F.scaled_dot_product_attention(query, key, value, attn_mask=None, is_causal=True)++ torch.xpu.synchronize()+ assert torch.allclose(output_with_mask, output_is_causal, atol=1e-3)++ # 3. Test with a non-causal mask (should NOT be patched)+ non_causal_mask = torch.zeros(1, 1, s, s, device="xpu", dtype=torch.float16)+ non_causal_mask.reshape(-1, s, s)[0][:, 0] = float("-inf") # Mask first column++ output_non_causal = F.scaled_dot_product_attention(query, key, value, attn_mask=non_causal_mask, is_causal=False)+ # This should be different from a simple causal mask+ assert not torch.allclose(output_with_mask, output_non_causal, atol=1e-3)+++if __name__ == "__main__":+ # For manual testing+ if hasattr(torch, "xpu") and torch.xpu.is_available():+ test_patch_xpu_sdpa_drop_causal_mask()+ print("XPU SDPA Patch Test Passed!")+ else:+ print("XPU not available, skipping test.")