Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix(xpu): make ARK SDPA graph-capture safe for torch.xpu.XPUGraph (#2416)

Luo Yua4f01f8584a6
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index 33bb1f38d..be679a144 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -949,6 +949,18 @@ def sdpa(
return O
+def _xpu_capturing() -> bool:
+ """True if the current XPU stream is inside a torch.xpu graph capture.
+
+ Used to skip host-side device->host syncs (.item() / .cpu()) that are
+ illegal while a SyclTensor command graph is being recorded.
+ """
+ try:
+ return bool(torch.xpu.is_current_stream_capturing())
+ except Exception:
+ return False
+
+
def sdpa_varlen(
query: torch.Tensor,
key: torch.Tensor,
@@ -1015,12 +1027,17 @@ def sdpa_varlen(
total_kv, Hkv, Dk = key.shape
total_kv_v, Hkv2, Dv = value.shape
- if total_q != cu_seqlens_q[-1].item():
- raise ValueError(f"Q dim-0 ({total_q}) != cu_seqlens_q[-1] ({cu_seqlens_q[-1].item()})")
- if total_kv != cu_seqlens_k[-1].item():
- raise ValueError(f"K dim-0 ({total_kv}) != cu_seqlens_k[-1] ({cu_seqlens_k[-1].item()})")
- if total_kv_v != cu_seqlens_k[-1].item():
- raise ValueError(f"V dim-0 ({total_kv_v}) != cu_seqlens_k[-1] ({cu_seqlens_k[-1].item()})")
+ if not _xpu_capturing():
+ # Cross-check flat layout against the cumulative sequence-length
+ # boundaries. These .item() calls force a device->host sync, which is
+ # illegal while a command graph is being captured, so the check is
+ # skipped during capture (the caller must keep buffer geometry stable).
+ if total_q != int(cu_seqlens_q[-1].item()):
+ raise ValueError(f"Q dim-0 ({total_q}) != cu_seqlens_q[-1]")
+ if total_kv != int(cu_seqlens_k[-1].item()):
+ raise ValueError(f"K dim-0 ({total_kv}) != cu_seqlens_k[-1]")
+ if total_kv_v != int(cu_seqlens_k[-1].item()):
+ raise ValueError(f"V dim-0 ({total_kv_v}) != cu_seqlens_k[-1]")
if Hkv != Hkv2 or Dk != Dv:
raise ValueError("K/V shape mismatch")
if Dk != D:
@@ -1054,7 +1071,8 @@ def sdpa_varlen(
O = torch.empty(total_q, Hq, D, dtype=value.dtype, device=query.device)
if return_lse:
- max_q = int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())
+ # .item() is a device->host sync; illegal during graph capture.
+ max_q = max_seqlen_q if _xpu_capturing() else int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())
if max_seqlen_q < max_q:
raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")
LSE = torch.full(
@@ -2273,7 +2291,8 @@ def sageattn_varlen(
O = torch.empty(total_q, Hq, D, dtype=v.dtype, device=q.device)
if return_lse:
- max_q = int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())
+ # .item() is a device->host sync; illegal during graph capture.
+ max_q = max_seqlen_q if _xpu_capturing() else int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())
if max_seqlen_q < max_q:
raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")
LSE = torch.full(
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp
index 5ecf3b7d5..775107e25 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp
@@ -21,7 +21,6 @@
#include <type_traits>
-#include "cutlass/util/device_memory.h"
#include "cutlass/util/packed_stride.hpp"
// FlashAttention-v2 (Xe) building blocks (from sycl-tla).
@@ -47,6 +46,7 @@
// CUTLASS-SYCL/CuTe compat layer
#include "cute/util/compat.hpp"
#include "cute/tensor.hpp"
+#include "utils.hpp"
// sycl-tla example utilities
#include "helper.h"
@@ -66,6 +66,8 @@ namespace detail {
// Import cute namespace for convenience in kernel builders
using namespace cute;
+inline constexpr size_t kPrefillScratchLoc = 12;
+
// Helper to get subgroup size from kernel
template <typename Kernel, typename = void>
struct KernelSubgroupSize {
@@ -362,13 +364,16 @@ inline void run_prefill_impl(
// Get workspace size and allocate
size_t workspace_size = Kernel::get_workspace_size(args);
- cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
+ void* workspace = DeviceMemoryPool::Instance()->get_scratch_mem(workspace_size, kPrefillScratchLoc, q);
// Initialize workspace if needed
- Kernel::initialize_workspace(args, workspace.get());
+ Kernel::initialize_workspace(args, workspace);
- typename Kernel::Params params = Kernel::to_underlying_arguments(args, workspace.get());
- launch_prefill_kernel<Kernel>(params).wait();
+ typename Kernel::Params params = Kernel::to_underlying_arguments(args, workspace);
+ launch_prefill_kernel<Kernel>(params);
+ // No host-side wait here: the caller (ARK wrapper / torch.xpu stream)
+ // is responsible for synchronization. A blocking wait would also be
+ // illegal during torch.xpu CUDAGraph capture.
}
template <int HeadDim, bool Causal>
@@ -478,7 +483,9 @@ inline void run_decode_impl(
{(float*)O_ptr, stride_O}};
typename Kernel::Params params = Kernel::to_underlying_arguments(args);
- launch_decode_kernel<Kernel>(params).wait();
+ launch_decode_kernel<Kernel>(params);
+ // No host-side wait: the caller handles synchronization via the torch
+ // stream. A blocking wait here would abort torch.xpu graph capture.
}
template <int HeadDim, bool Causal>
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
index c7c0862ff..09f650332 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
@@ -216,8 +216,9 @@ class DeviceMemoryPool {
// weight-repack and activation-sum buffers
// (`moe_decode_detail::kInt4RepackScratchLoc` / `kActGroupSumScratchLoc`);
// slot 11 is the XMX Hadamard staging buffer
- // (`kHadamardStagingScratchLoc` in `ark.cpp`).
- static constexpr int MaxLocNum = 12;
+ // (`kHadamardStagingScratchLoc` in `ark.cpp`); slot 12 is the SYCL-TLA
+ // prefill workspace (`kPrefillScratchLoc` in `sycl_tla_wrapper.hpp`).
+ static constexpr int MaxLocNum = 13;
using SizeMap = std::unordered_map<size_t, size_t>;
using PtrMap = std::unordered_map<size_t, int8_t*>;
diff --git a/auto_round_extension/ark/test/test_ark_sdpa_graph_capture.py b/auto_round_extension/ark/test/test_ark_sdpa_graph_capture.py
new file mode 100644
index 000000000..9f14d4885
--- /dev/null
+++ b/auto_round_extension/ark/test/test_ark_sdpa_graph_capture.py
@@ -0,0 +1,314 @@
+# Copyright (c) 2025 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests that ARK SDPA kernels can be captured in a torch.xpu.XPUGraph.
+
+Run from this directory:
+
+ python -m pytest test_ark_sdpa_graph_capture.py -v -s
+
+Requires the ARK extension (``auto_round_kernel``) to be built for the
+current XPU runtime and an Intel GPU to be available.
+
+What this exercises
+-------------------
+* Dense SDPA (``ark.sdpa``) — prefill (causal / non-causal, MHA & GQA) and
+ decode. The output tensor returned by ``ark.sdpa`` during capture lives
+ in the graph's capture pool and keeps a stable device pointer across
+ replays. After mutating Q in-place and calling ``graph.replay()``, the
+ same output buffer must hold the result for the new Q.
+* Varlen (``ark.sdpa_varlen``) — currently only asserts that capture and
+ one replay complete without raising. Replay-numeric verification is not
+ yet exercised because the varlen scratch allocation (``DeviceMemoryPool``)
+ and grid sizing interact with graph replay in a way that requires
+ further kernel-side validation.
+"""
+
+import math
+import sys
+from pathlib import Path
+
+import pytest
+import torch
+
+sys.path.append(str(Path(__file__).resolve().parent))
+import auto_round_kernel as ark # noqa: E402
+from ut_utils import is_xpu_available, reference_sdpa # noqa: E402
+
+
+def _graph_supported() -> bool:
+ return all(hasattr(torch.xpu, n) for n in ("XPUGraph", "graph", "Stream"))
+
+
+requires_graph = pytest.mark.skipif(
+ not (is_xpu_available() and _graph_supported()),
+ reason="requires XPU with torch.xpu.XPUGraph support",
+)
+
+
+# ---------------------------------------------------------------------------
+# Capture helper
+# ---------------------------------------------------------------------------
+
+
+def _capture_graph(fn):
+ """Generic capture helper: warm up then capture ``fn`` into an XPUGraph.
+
+ Returns (graph, output_tensor). The output tensor lives in the
+ XPU graph's capture pool and therefore retains a *stable* device
+ address across replays, making it safe to compare after ``replay()``.
+ """
+ with torch.no_grad():
+ fn()
+ torch.xpu.synchronize()
+
+ g = torch.xpu.XPUGraph()
+ side = torch.xpu.Stream()
+ cur = torch.xpu.current_stream()
+ side.wait_stream(cur)
+ with torch.no_grad(), torch.xpu.stream(side):
+ torch.xpu.synchronize()
+ with torch.xpu.graph(g):
+ out = fn()
+ cur.wait_stream(side)
+ torch.xpu.synchronize()
+ return g, out
+
+
+def _capture_dense_sdpa(q, k, v, is_causal, scale):
+ """Warm up, then capture ark.sdpa into an XPUGraph.
+
+ Returns (graph, output_tensor). The output tensor lives in the
+ XPU graph's capture pool and therefore retains a *stable* device
+ address across replays, making it safe to compare after ``replay()``.
+ """
+ return _capture_graph(lambda: ark.sdpa(q, k, v, is_causal=is_causal, scale=scale))
+
+
+def _run_dense_case(b, hq, hkv, sq, skv, d, dtype, is_causal):
+ torch.manual_seed(0)
+ scale = 1.0 / math.sqrt(d)
+ dev = "xpu:0"
+
+ q = torch.randn(b, hq, sq, d, device=dev, dtype=dtype)
+ k = torch.randn(b, hkv, skv, d, device=dev, dtype=dtype)
+ v = torch.randn(b, hkv, skv, d, device=dev, dtype=dtype)
+ q_new = torch.randn(b, hq, sq, d, device=dev, dtype=dtype)
+
+ g, out = _capture_dense_sdpa(q, k, v, is_causal, scale)
+
+ # --- replay with changed input ---
+ q.copy_(q_new)
+ torch.xpu.synchronize()
+ g.replay()
+ torch.xpu.synchronize()
+
+ ref_new = reference_sdpa(q, k, v, is_causal=is_causal, scale=scale)
+ diff = (out.float() - ref_new.float()).abs().max().item()
+ assert torch.allclose(
+ out.float(), ref_new.float(), atol=1e-2, rtol=1e-2
+ ), f"dense SDPA graph replay mismatch: max_abs_diff={diff:.6f}"
+ g.reset()
+
+
+# ---------------------------------------------------------------------------
+# Varlen helpers
+# ---------------------------------------------------------------------------
+
+
+def _build_varlen(batch, lengths_q, lengths_kv, hq, hkv, d, dtype):
+ """Build varlen inputs: packed Q/K/V + cu_seqlens + max seqlens."""
+ dev = "xpu:0"
+ total_q = sum(lengths_q)
+ total_kv = sum(lengths_kv)
+ max_q = max(lengths_q)
+ max_kv = max(lengths_kv)
+
+ q = torch.randn(total_q, hq, d, device=dev, dtype=dtype)
+ k = torch.randn(total_kv, hkv, d, device=dev, dtype=dtype)
+ v = torch.randn(total_kv, hkv, d, device=dev, dtype=dtype)
+
+ cu_q = torch.tensor([0] + list(torch.cumsum(torch.tensor(lengths_q), 0).tolist()), device=dev, dtype=torch.int64)
+ cu_kv = torch.tensor([0] + list(torch.cumsum(torch.tensor(lengths_kv), 0).tolist()), device=dev, dtype=torch.int64)
+ return q, k, v, cu_q, cu_kv, max_q, max_kv
+
+
+def _run_varlen_capture(dtype, lengths_q, lengths_kv, hq=8, hkv=4, d=128):
+ """Capture varlen SDPA in a graph and replay once — only checks no-throw."""
+ torch.manual_seed(0)
+ q, k, v, cu_q, cu_kv, max_q, max_kv = _build_varlen(len(lengths_q), lengths_q, lengths_kv, hq, hkv, d, dtype)
+ scale = 1.0 / math.sqrt(d)
+
+ def call():
+ return ark.sdpa_varlen(q, k, v, cu_q, cu_kv, max_q, max_kv, is_causal=True, scale=scale)
+
+ # Eager warm-up
+ with torch.no_grad():
+ _ = call()
+ torch.xpu.synchronize()
+
+ # Capture
+ g = torch.xpu.XPUGraph()
+ side, cur = torch.xpu.Stream(), torch.xpu.current_stream()
+ side.wait_stream(cur)
+ with torch.no_grad(), torch.xpu.stream(side):
+ torch.xpu.synchronize()
+ with torch.xpu.graph(g):
+ call()
+ cur.wait_stream(side)
+ torch.xpu.synchronize()
+
+ # One replay must not throw
+ g.replay()
+ torch.xpu.synchronize()
+ g.reset()
+
+
+# ---------------------------------------------------------------------------
+# Dense SDPA graph-capture tests
+# ---------------------------------------------------------------------------
+
+
+@requires_graph
[email protected]("dtype", [torch.float16, torch.bfloat16])
[email protected]("head_dim", [64, 128])
+class TestSDPAGraphCapture:
+ def test_prefill_causal(self, dtype, head_dim):
+ _run_dense_case(2, 8, 8, 128, 128, head_dim, dtype, is_causal=True)
+
+ def test_prefill_noncausal(self, dtype, head_dim):
+ _run_dense_case(2, 8, 8, 128, 192, head_dim, dtype, is_causal=False)
+
+ def test_prefill_gqa(self, dtype, head_dim):
+ _run_dense_case(2, 32, 4, 64, 256, head_dim, dtype, is_causal=True)
+
+ def test_decode(self, dtype, head_dim):
+ _run_dense_case(2, 16, 4, 1, 64, head_dim, dtype, is_causal=True)
+
+
+# ---------------------------------------------------------------------------
+# Varlen graph-capture smoke tests
+# ---------------------------------------------------------------------------
+
+
+@requires_graph
[email protected]("dtype", [torch.float16, torch.bfloat16])
+class TestSDPAVarlenGraphCapture:
+ def test_capture_replay_smoke(self, dtype):
+ """Capture + replay must not throw; numeric check TODO."""
+ _run_varlen_capture(dtype, [128, 64, 96], [128, 64, 96])
+
+
+# ---------------------------------------------------------------------------
+# GEMM graph-capture smoke tests
+#
+# ``matmul_sycl_tla`` (dense float GEMM) shares the same
+# ``compat::set_default_queue`` code path as SDPA, so it must also be
+# graph-capture safe. Verify capture → mutate input in-place → replay →
+# output matches a fresh eager call on the mutated inputs.
+# ---------------------------------------------------------------------------
+
+
+@requires_graph
[email protected]("dtype", [torch.float16, torch.bfloat16])
+class TestGEMMGraphCapture:
+ def test_capture_replay(self, dtype):
+ """Dense GEMM (matmul_sycl_tla) must survive capture + replay."""
+ M, N, K = 128, 256, 512
+ dev = "xpu:0"
+ A = torch.randn(M, K, device=dev, dtype=dtype)
+ B = torch.randn(N, K, device=dev, dtype=dtype)
+ bias = torch.randn(N, device=dev, dtype=dtype)
+
+ def call():
+ return ark.matmul_sycl_tla(A, B, bias)
+
+ with torch.no_grad():
+ _ = call() # warm-up / JIT
+ torch.xpu.synchronize()
+
+ g, out = _capture_graph(lambda: ark.matmul_sycl_tla(A, B, bias))
+
+ # --- mutate inputs in-place, replay, compare to fresh eager ---
+ A.copy_(torch.randn_like(A))
+ bias.copy_(torch.randn_like(bias))
+ torch.xpu.synchronize()
+
+ g.replay()
+ torch.xpu.synchronize()
+
+ # Self-consistency: replay must match a fresh eager call on same inputs.
+ with torch.no_grad():
+ eager = ark.matmul_sycl_tla(A, B, bias)
+ torch.xpu.synchronize()
+ atol = 2e-2 if dtype == torch.bfloat16 else 1e-3
+ diff = (out.float() - eager.float()).abs().max().item()
+ assert diff < atol, f"GEMM graph replay mismatch: max_abs_diff={diff:.6f}"
+
+
+# ---------------------------------------------------------------------------
+# SDPA with LSE graph-capture tests
+#
+# return_lse=True allocates an LSE tensor inside ark.sdpa() during capture.
+# The LSE buffer lives in the capture pool and must be correctly updated on
+# each replay.
+# ---------------------------------------------------------------------------
+
+
+@requires_graph
[email protected]("dtype", [torch.float16, torch.bfloat16])
[email protected]("head_dim", [64, 128])
+class TestSDPALSEGraphCapture:
+ def test_capture_replay_lse(self, dtype, head_dim):
+ """SDPA with return_lse=True must survive capture + replay.
+
+ LSE is allocated inside the capture region, so its buffer lives in
+ the graph pool and is stable across replays.
+ """
+ B, Hq, Hkv, Sq, Skv, D = 2, 8, 8, 128, 128, head_dim
+ scale = 1.0 / math.sqrt(D)
+ q = torch.randn(B, Sq, Hq, D, device="xpu:0", dtype=dtype)
+ k = torch.randn(B, Skv, Hkv, D, device="xpu:0", dtype=dtype)
+ v = torch.randn(B, Skv, Hkv, D, device="xpu:0", dtype=dtype)
+
+ def call():
+ return ark.sdpa(q, k, v, scale=scale, is_causal=True, return_lse=True)
+
+ with torch.no_grad():
+ _ = call() # warm-up / JIT
+ torch.xpu.synchronize()
+
+ g, out = _capture_graph(call)
+ # out should be a tuple (O, LSE)
+ assert isinstance(out, (tuple, list)) and len(out) == 2, f"Expected (O, LSE) tuple, got {type(out)}"
+ o_out, lse_out = out[0], out[1]
+
+ # --- mutate Q in-place, replay, verify both O and LSE ---
+ q_new = torch.randn_like(q)
+ q.copy_(q_new)
+ torch.xpu.synchronize()
+ g.replay()
+ torch.xpu.synchronize()
+
+ # Self-consistency: compare replay outputs to fresh eager call
+ with torch.no_grad():
+ o_eager, lse_eager = ark.sdpa(q, k, v, scale=scale, is_causal=True, return_lse=True)
+ torch.xpu.synchronize()
+
+ o_diff = (o_out.float() - o_eager.float()).abs().max().item()
+ assert o_diff < 2e-2, f"LSE test: O mismatch after replay: {o_diff:.6f}"
+
+ # LSE is in fp32, should be bit-identical or very close
+ lse_diff = (lse_out.float() - lse_eager.float()).abs().max().item()
+ assert lse_diff < 1e-4, f"LSE graph replay mismatch: max_abs_diff={lse_diff:.6f}"
+ g.reset()