Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Reduce xpu memory usage with patch_xpu_sdpa_drop_causal_mask (#1716)

Xin Hedc8a6a4428c7
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index 3b88c0e60..9a751968f 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -353,7 +353,7 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_dev
block_inputs[module_name] = input_info
module.to("cpu")
- # torch.cuda.empty_cache()
+ clear_memory()
# Enable gradients for the output of the last block
if module.tmp_name == block_names[-1]:
@@ -410,6 +410,8 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):
pass
current_grad = last_grad_input
+ del output, data
+ clear_memory()
# Manually compute gradients block by block
last_block_backward_hook.remove()
@@ -468,8 +470,9 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):
print(f"Warning: No suitable input gradient found for {block_name}")
break
+ del block_output, main_output, block_input_args, block_input_kwargs
block_module.to("cpu")
- # clear_memory()
+ clear_memory()
pbar.update(1)
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index bbeb85d0d..238df50e5 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -117,6 +117,7 @@ from auto_round.utils.device import (
clear_memory_if_reached_threshold,
get_major_device,
parse_available_devices,
+ patch_xpu_sdpa_drop_causal_mask,
set_auto_device_map_for_block_with_tuning,
set_non_auto_device_map,
)
@@ -401,6 +402,11 @@ class BaseCompressor(object):
else:
torch.use_deterministic_algorithms(True, warn_only=True)
+ # XPU SDPA workaround: drop pure causal masks so FLASH backend is used,
+ # and set torch.use_deterministic_algorithms(False)
+ # instead of MATH (avoids ~10x peak-VRAM blow-up during block tuning).
+ patch_xpu_sdpa_drop_causal_mask()
+
self.to_quant_block_names = to_quant_block_names
if not hasattr(self, "quant_block_list"):
all_blocks = get_block_names(model)
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index f38e233be..a4727a996 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -185,6 +185,82 @@ def is_hpex_available():
return _hpex_available
+_xpu_sdpa_patched = False
+
+
+# TODO: This is a workaround for the XPU SDPA memory blow-up issue.
+# We should remove this patch after the issue is fixed in XPU side.
+# https://github.com/intel/auto-round/issues/990
+def patch_xpu_sdpa_drop_causal_mask():
+ """Workaround for XPU peak-VRAM blow-up in SDPA.
+
+ On Intel XPU, ``torch.nn.functional.scaled_dot_product_attention`` falls back
+ to the MATH backend whenever ``attn_mask`` is non-None (FLASH on XPU does
+ not support explicit attn_mask, EFFICIENT/CUDNN are not implemented).
+ The MATH backend materializes the full ``(B, H, S, S)`` probability matrix
+ in both forward and backward, costing several GB at typical
+ ``batch_size=8, seqlen=2048``.
+
+ HuggingFace transformers happily passes a *pure causal* 4D mask, even
+ though the same effect is achievable via ``is_causal=True`` (which the
+ FLASH backend supports and which uses ~12x less memory).
+
+ This monkey-patch detects a pure causal mask, drops it, and forwards
+ ``is_causal=True`` instead -- only on XPU and only when no real mask info
+ would be lost. CPU/CUDA/HPU paths are left untouched.
+
+ Idempotent. Safe to call multiple times.
+ """
+ global _xpu_sdpa_patched
+ if _xpu_sdpa_patched:
+ return
+ if not (hasattr(torch, "xpu") and torch.xpu.is_available()):
+ return
+ torch.use_deterministic_algorithms(False)
+
+ _orig_sdpa = torch.nn.functional.scaled_dot_product_attention
+
+ def _is_pure_causal_mask(mask: torch.Tensor, s: int) -> bool:
+ # Cheap shape check first
+ if mask.shape[-1] != s or mask.shape[-2] != s:
+ return False
+ if mask.dtype not in (torch.float16, torch.bfloat16, torch.float32):
+ return False
+ # Pick the first (B,H) slice; HF mask is broadcast across batch/heads.
+ m2d = mask.reshape(-1, s, s)[0]
+ tri_up = torch.triu(torch.ones(s, s, dtype=torch.bool, device=mask.device), 1)
+ return bool(torch.isinf(m2d[tri_up]).all().item()) and bool((m2d[~tri_up] == 0).all().item())
+
+ def _patched_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None, **kwargs):
+ if (
+ query.device.type == "xpu"
+ and attn_mask is not None
+ and not is_causal
+ and query.shape[-2] == key.shape[-2] # square QK length (no kv-cache)
+ and _is_pure_causal_mask(attn_mask, query.shape[-2])
+ ):
+ attn_mask = None
+ is_causal = True
+ return _orig_sdpa(
+ query,
+ key,
+ value,
+ attn_mask=attn_mask,
+ dropout_p=dropout_p,
+ is_causal=is_causal,
+ scale=scale,
+ **kwargs,
+ )
+
+ torch.nn.functional.scaled_dot_product_attention = _patched_sdpa
+ _xpu_sdpa_patched = True
+ logger.info("torch.use_deterministic_algorithms(False) is set for XPU.")
+ logger.info(
+ "Patched torch SDPA on XPU to use is_causal=True for pure causal masks "
+ "(avoids ~10x peak-VRAM blow-up from MATH backend)."
+ )
+
+
def check_is_cpu(device):
"""Check if the device is a CPU.
@@ -1121,15 +1197,6 @@ def estimate_tuning_block_mem(
# and multiple expert activations. Here we use a conservative estimate.
moe_additional_memory = additional_memory * 6 # GB
additional_memory += moe_additional_memory
- if torch.xpu.is_available():
- # https://github.com/intel/torch-xpu-ops/issues/2232
- # TODO: XPU takes more memory than expected. for llama 8B, it's about 12 GB
- xpu_additional_memory = 12 # GB
- additional_memory += xpu_additional_memory
- # logger.warning_once(
- # "[Memory Estimation]: If there is an abnormal memory issue, please collect log with "
- # + "AR_LOG_LEVEL=debug and raise issue to us."
- # )
return layer_memory_dict, layer_activation_memory, block_input_output_memory, additional_memory
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 7ed9133f7..3f67deb63 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -796,27 +796,26 @@ def is_gguf_model(model_path: Union[str, torch.nn.Module]) -> bool:
return is_gguf_file
-def is_diffusion_model(model_or_path: Union[str, object]) -> bool:
+def is_diffusion_model(model_or_path: Union[str, object], trust_remote_code: bool = True) -> bool:
from auto_round.utils.common import LazyImport
- # First check if it's a known diffusion pipeline by config/model_type
- # to avoid unnecessary imports and file checks for non-diffusion models, which can be time-consuming.
- try:
- from transformers import AutoConfig
-
- config = AutoConfig.from_pretrained(model_or_path, trust_remote_code=True)
- model_type = getattr(config, "model_type", "")
- # A special case for NextStep
- if model_type == "nextstep":
- return True
- except:
- logger.warning(
- f"Failed to load config for {model_or_path}, trying to check model_index.json for diffusion pipeline."
- )
-
# Then check if model_index.json exists for diffusion pipeline,
# which is a strong signal of being a diffusion pipeline.
if isinstance(model_or_path, str):
+ # First check if it's a known diffusion pipeline by config/model_type
+ # to avoid unnecessary imports and file checks for non-diffusion models, which can be time-consuming.
+ try:
+ from transformers import AutoConfig
+
+ config = AutoConfig.from_pretrained(model_or_path, trust_remote_code=trust_remote_code)
+ model_type = getattr(config, "model_type", "")
+ # A special case for NextStep
+ if model_type == "nextstep":
+ return True
+ except:
+ logger.warning(
+ f"Failed to load config for {model_or_path}, trying to check model_index.json for diffusion pipeline."
+ )
index_file = None
if not os.path.isdir(model_or_path):
try:
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 4ca147926..53acef0bc 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -476,17 +476,14 @@ The 3B and 14B models were evaluated on Qwen 2.5, the 8X7B model is Mixtral, whi
| 2.5 w/o torch compile | 8min<br/>10GB | 16min<br/>20GB | 30min<br/>25GB | 140min<br/>49GB | 50min<br/>49GB |
W4G128 Quantization Time and Memory Usage (Intel GPU B60 24G)
-Testing was conducted on the Intel GPU B60 24G using the release version of PyTorch 2.11.0+xpu. Please note that data loading and packing costs have been excluded from the evaluation. Time and memory usage were measured using Qwen3-series models.
-
-| Torch version/Config W4G128 | 0.6B | 1.7B | 4B | 8B | 30B-A3B |
-|------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------|---------------------|---------------------|---------------------|
-| 2.11.0+xpu with torch compile | 20min<br/>10.7GB | 26min<br/>13.2GB | 58min<br/>22.8GB | OOM | OOM |
-| 2.11.0+xpu with torch compile<br/>low_gpu_mem_usage=True | 29min<br/>9.5GB | 38min<br/>9.8GB | 1h 23min<br/>19.4GB | 1h 32min<br/>20.1GB | 5h 33min<br/>22.8GB |
-| 2.11.0+xpu with torch compile<br/>low_gpu_mem_usage=True<br/>gradient_accumulate_steps=8,bs=1 | 41min<br/>1.3GB | 42min<br/>1.8GB | 1h 29min<br/>3.6GB | 2h 4min<br/>4.6GB | 21h 41min<br/>10.2GB |
-| 2.11.0+xpu w/o torch compile | 20min<br/>10.9GB | 28min<br/>13.2GB | OOM | OOM | OOM |
-
-
-
+Testing was conducted on the Intel GPU B60 24G using the release version of PyTorch 2.11.0+xpu. Please note that data loading and packing costs have been excluded from the evaluation. Time and memory usage were measured using Qwen2.5-series models.
+
+| Torch version/Config W4G128 | 0.5B | 1.5B | 3B | 7B |
+|------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------|---------------------|---------------------|
+| 2.11.0+xpu with torch compile | 6min<br/>2.9GB | 13min<br/>5.4GB | 22min<br/>7.1GB | 40min<br/>14.9GB |
+| 2.11.0+xpu with torch compile<br/>low_gpu_mem_usage=True | 10min<br/>1.7GB | 17min<br/>3.3GB | 30min<br/>4.3GB | 50min<br/>8.5GB |
+| 2.11.0+xpu with torch compile<br/>low_gpu_mem_usage=True<br/>gradient_accumulate_steps=8,bs=1 | 14min<br/>0.4GB | 22min<br/>1.1GB | 38min<br/>1.5GB | 1h 4min<br/>4.1GB |
+| 2.11.0+xpu w/o torch compile | 6min<br/>2.9GB | 14min<br/>5.7GB | 26min<br/>7.6GB | 51min<br/>15.5GB |
### Device/Multi-GPU setting in Quantization
**The tuning device is specified using the `device_map` argument in AutoRound API, _not_ through the `device_map`
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 0e2c6bec1..5eddd22d3 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -462,14 +462,13 @@ ar.quantize_and_save(output_dir, format="gguf:q4_k_m") # gguf:q*_k_s、gguf:q*_
| 2.5 + 关闭 torch compile | 8min<br/>10GB | 16min<br/>20GB | 30min<br/>25GB | 140min<br/>49GB | 50min<br/>49GB |
W4G128 量化耗时与显存占用(英特尔 GPU B60 24G)
-(测试环境:英特尔 GPU B60 24G,PyTorch 2.11.0+xpu 正式版。注意评测未计入数据加载和打包耗时。所有测试均使用 Qwen3 系列模型。)
-
-| Torch version/Config W4G128 | 0.6B | 1.7B | 4B | 8B | 30B-A3B |
-|------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------|---------------------|---------------------|---------------------|
-| 2.11.0+xpu 开启 torch compile | 20min<br/>10.7GB | 26min<br/>13.2GB | 58min<br/>22.8GB | OOM | OOM |
-| 2.11.0+xpu 开启 torch compile<br/>low_gpu_mem_usage=True | 29min<br/>9.5GB | 38min<br/>9.8GB | 1h 23min<br/>19.4GB | 1h 32min<br/>20.1GB | 5h 33min<br/>22.8GB |
-| 2.11.0+xpu 开启 torch compile<br/>low_gpu_mem_usage=True<br/>gradient_accumulate_steps=8,bs=1 | 41min<br/>1.3GB | 42min<br/>1.8GB | 1h 29min<br/>3.6GB | 2h 4min<br/>4.6GB | 21h 41min<br/>10.2GB |
-| 2.11.0+xpu 关闭 torch compile | 20min<br/>10.9GB | 28min<br/>13.2GB | OOM | OOM | OOM |
+(测试环境:英特尔 GPU B60 24G,PyTorch 2.11.0+xpu 正式版。注意评测未计入数据加载和打包耗时。所有测试均使用 Qwen2.5 系列模型。)
+| Torch version/Config W4G128 | 0.5B | 1.5B | 3B | 7B |
+|------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------|---------------------|---------------------|
+| 2.11.0+xpu 开启 torch compile | 6min<br/>2.9GB | 13min<br/>5.4GB | 22min<br/>7.1GB | 40min<br/>14.9GB |
+| 2.11.0+xpu 开启 torch compile<br/>low_gpu_mem_usage=True | 10min<br/>1.7GB | 17min<br/>3.3GB | 30min<br/>4.3GB | 50min<br/>8.5GB |
+| 2.11.0+xpu 开启 torch compile<br/>low_gpu_mem_usage=True<br/>gradient_accumulate_steps=8,bs=1 | 14min<br/>0.4GB | 22min<br/>1.1GB | 38min<br/>1.5GB | 1h 4min<br/>4.1GB |
+| 2.11.0+xpu 关闭 torch compile | 6min<br/>2.9GB | 14min<br/>5.7GB | 26min<br/>7.6GB | 51min<br/>15.5GB |
### 设备及多卡量化设置
diff --git a/test/test_xpu/test_xpu_sdpa_patch.py b/test/test_xpu/test_xpu_sdpa_patch.py
new file mode 100644
index 000000000..c0201af2f
--- /dev/null
+++ b/test/test_xpu/test_xpu_sdpa_patch.py
@@ -0,0 +1,50 @@
+import pytest
+import torch
+import torch.nn.functional as F
+
+from auto_round.utils.device import patch_xpu_sdpa_drop_causal_mask
+
+
[email protected](not (hasattr(torch, "xpu") and torch.xpu.is_available()), reason="XPU not available")
+def test_patch_xpu_sdpa_drop_causal_mask():
+ """Test that patch_xpu_sdpa_drop_causal_mask correctly identifies and replaces causal masks on XPU."""
+ patch_xpu_sdpa_drop_causal_mask()
+
+ s = 128
+ query = torch.randn(1, 8, s, 64, device="xpu", dtype=torch.float16)
+ key = torch.randn(1, 8, s, 64, device="xpu", dtype=torch.float16)
+ value = torch.randn(1, 8, s, 64, device="xpu", dtype=torch.float16)
+
+ # Create a pure causal mask (HF style: small values for non-masked, -inf for masked)
+ mask = torch.zeros(1, 1, s, s, device="xpu", dtype=torch.float16)
+ tri_up = torch.triu(torch.ones(s, s, dtype=torch.bool, device="xpu"), 1)
+ mask.reshape(-1, s, s)[0][tri_up] = float("-inf")
+
+ # We can't easily check internal calls without complex mocking,
+ # but we can verify it doesn't crash and produces same results as standard causal=True
+
+ # 1. Test with causal mask (should be patched to is_causal=True internally)
+ output_with_mask = F.scaled_dot_product_attention(query, key, value, attn_mask=mask, is_causal=False)
+
+ # 2. Test with is_causal=True directly
+ output_is_causal = F.scaled_dot_product_attention(query, key, value, attn_mask=None, is_causal=True)
+
+ torch.xpu.synchronize()
+ assert torch.allclose(output_with_mask, output_is_causal, atol=1e-3)
+
+ # 3. Test with a non-causal mask (should NOT be patched)
+ non_causal_mask = torch.zeros(1, 1, s, s, device="xpu", dtype=torch.float16)
+ non_causal_mask.reshape(-1, s, s)[0][:, 0] = float("-inf") # Mask first column
+
+ output_non_causal = F.scaled_dot_product_attention(query, key, value, attn_mask=non_causal_mask, is_causal=False)
+ # This should be different from a simple causal mask
+ assert not torch.allclose(output_with_mask, output_non_causal, atol=1e-3)
+
+
+if __name__ == "__main__":
+ # For manual testing
+ if hasattr(torch, "xpu") and torch.xpu.is_available():
+ test_patch_xpu_sdpa_drop_causal_mask()
+ print("XPU SDPA Patch Test Passed!")
+ else:
+ print("XPU not available, skipping test.")