PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix(xpu): make ARK SDPA graph-capture safe for torch.xpu.XPUGraph (#2416)
Luo Yua4f01f8584a6
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex 33bb1f38d..be679a144 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -949,6 +949,18 @@ def sdpa(return O+def _xpu_capturing() -> bool:+ """True if the current XPU stream is inside a torch.xpu graph capture.++ Used to skip host-side device->host syncs (.item() / .cpu()) that are+ illegal while a SyclTensor command graph is being recorded.+ """+ try:+ return bool(torch.xpu.is_current_stream_capturing())+ except Exception:+ return False++def sdpa_varlen(query: torch.Tensor,key: torch.Tensor,@@ -1015,12 +1027,17 @@ def sdpa_varlen(total_kv, Hkv, Dk = key.shapetotal_kv_v, Hkv2, Dv = value.shape- if total_q != cu_seqlens_q[-1].item():- raise ValueError(f"Q dim-0 ({total_q}) != cu_seqlens_q[-1] ({cu_seqlens_q[-1].item()})")- if total_kv != cu_seqlens_k[-1].item():- raise ValueError(f"K dim-0 ({total_kv}) != cu_seqlens_k[-1] ({cu_seqlens_k[-1].item()})")- if total_kv_v != cu_seqlens_k[-1].item():- raise ValueError(f"V dim-0 ({total_kv_v}) != cu_seqlens_k[-1] ({cu_seqlens_k[-1].item()})")+ if not _xpu_capturing():+ # Cross-check flat layout against the cumulative sequence-length+ # boundaries. These .item() calls force a device->host sync, which is+ # illegal while a command graph is being captured, so the check is+ # skipped during capture (the caller must keep buffer geometry stable).+ if total_q != int(cu_seqlens_q[-1].item()):+ raise ValueError(f"Q dim-0 ({total_q}) != cu_seqlens_q[-1]")+ if total_kv != int(cu_seqlens_k[-1].item()):+ raise ValueError(f"K dim-0 ({total_kv}) != cu_seqlens_k[-1]")+ if total_kv_v != int(cu_seqlens_k[-1].item()):+ raise ValueError(f"V dim-0 ({total_kv_v}) != cu_seqlens_k[-1]")if Hkv != Hkv2 or Dk != Dv:raise ValueError("K/V shape mismatch")if Dk != D:@@ -1054,7 +1071,8 @@ def sdpa_varlen(O = torch.empty(total_q, Hq, D, dtype=value.dtype, device=query.device)if return_lse:- max_q = int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())+ # .item() is a device->host sync; illegal during graph capture.+ max_q = max_seqlen_q if _xpu_capturing() else int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())if max_seqlen_q < max_q:raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")LSE = torch.full(@@ -2273,7 +2291,8 @@ def sageattn_varlen(O = torch.empty(total_q, Hq, D, dtype=v.dtype, device=q.device)if return_lse:- max_q = int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())+ # .item() is a device->host sync; illegal during graph capture.+ max_q = max_seqlen_q if _xpu_capturing() else int((cu_seqlens_q_i32[1:] - cu_seqlens_q_i32[:-1]).max().item())if max_seqlen_q < max_q:raise ValueError(f"max_seqlen_q ({max_seqlen_q}) < max sequence length in cu_seqlens_q ({max_q})")LSE = torch.full(diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hppindex 5ecf3b7d5..775107e25 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp@@ -21,7 +21,6 @@#include <type_traits>-#include "cutlass/util/device_memory.h"#include "cutlass/util/packed_stride.hpp"// FlashAttention-v2 (Xe) building blocks (from sycl-tla).@@ -47,6 +46,7 @@// CUTLASS-SYCL/CuTe compat layer#include "cute/util/compat.hpp"#include "cute/tensor.hpp"+#include "utils.hpp"// sycl-tla example utilities#include "helper.h"@@ -66,6 +66,8 @@ namespace detail {// Import cute namespace for convenience in kernel buildersusing namespace cute;+inline constexpr size_t kPrefillScratchLoc = 12;+// Helper to get subgroup size from kerneltemplate <typename Kernel, typename = void>struct KernelSubgroupSize {@@ -362,13 +364,16 @@ inline void run_prefill_impl(// Get workspace size and allocatesize_t workspace_size = Kernel::get_workspace_size(args);- cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);+ void* workspace = DeviceMemoryPool::Instance()->get_scratch_mem(workspace_size, kPrefillScratchLoc, q);// Initialize workspace if needed- Kernel::initialize_workspace(args, workspace.get());+ Kernel::initialize_workspace(args, workspace);- typename Kernel::Params params = Kernel::to_underlying_arguments(args, workspace.get());- launch_prefill_kernel<Kernel>(params).wait();+ typename Kernel::Params params = Kernel::to_underlying_arguments(args, workspace);+ launch_prefill_kernel<Kernel>(params);+ // No host-side wait here: the caller (ARK wrapper / torch.xpu stream)+ // is responsible for synchronization. A blocking wait would also be+ // illegal during torch.xpu CUDAGraph capture.}template <int HeadDim, bool Causal>@@ -478,7 +483,9 @@ inline void run_decode_impl({(float*)O_ptr, stride_O}};typename Kernel::Params params = Kernel::to_underlying_arguments(args);- launch_decode_kernel<Kernel>(params).wait();+ launch_decode_kernel<Kernel>(params);+ // No host-side wait: the caller handles synchronization via the torch+ // stream. A blocking wait here would abort torch.xpu graph capture.}template <int HeadDim, bool Causal>diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hppindex c7c0862ff..09f650332 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp@@ -216,8 +216,9 @@ class DeviceMemoryPool {// weight-repack and activation-sum buffers// (`moe_decode_detail::kInt4RepackScratchLoc` / `kActGroupSumScratchLoc`);// slot 11 is the XMX Hadamard staging buffer- // (`kHadamardStagingScratchLoc` in `ark.cpp`).- static constexpr int MaxLocNum = 12;+ // (`kHadamardStagingScratchLoc` in `ark.cpp`); slot 12 is the SYCL-TLA+ // prefill workspace (`kPrefillScratchLoc` in `sycl_tla_wrapper.hpp`).+ static constexpr int MaxLocNum = 13;using SizeMap = std::unordered_map<size_t, size_t>;using PtrMap = std::unordered_map<size_t, int8_t*>;diff --git a/auto_round_extension/ark/test/test_ark_sdpa_graph_capture.py b/auto_round_extension/ark/test/test_ark_sdpa_graph_capture.pynew file mode 100644index 000000000..9f14d4885--- /dev/null+++ b/auto_round_extension/ark/test/test_ark_sdpa_graph_capture.py@@ -0,0 +1,314 @@+# Copyright (c) 2025 Intel Corporation.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Tests that ARK SDPA kernels can be captured in a torch.xpu.XPUGraph.++Run from this directory:++ python -m pytest test_ark_sdpa_graph_capture.py -v -s++Requires the ARK extension (``auto_round_kernel``) to be built for the+current XPU runtime and an Intel GPU to be available.++What this exercises+-------------------+* Dense SDPA (``ark.sdpa``) — prefill (causal / non-causal, MHA & GQA) and+ decode. The output tensor returned by ``ark.sdpa`` during capture lives+ in the graph's capture pool and keeps a stable device pointer across+ replays. After mutating Q in-place and calling ``graph.replay()``, the+ same output buffer must hold the result for the new Q.+* Varlen (``ark.sdpa_varlen``) — currently only asserts that capture and+ one replay complete without raising. Replay-numeric verification is not+ yet exercised because the varlen scratch allocation (``DeviceMemoryPool``)+ and grid sizing interact with graph replay in a way that requires+ further kernel-side validation.+"""++import math+import sys+from pathlib import Path++import pytest+import torch++sys.path.append(str(Path(__file__).resolve().parent))+import auto_round_kernel as ark # noqa: E402+from ut_utils import is_xpu_available, reference_sdpa # noqa: E402+++def _graph_supported() -> bool:+ return all(hasattr(torch.xpu, n) for n in ("XPUGraph", "graph", "Stream"))+++requires_graph = pytest.mark.skipif(+ not (is_xpu_available() and _graph_supported()),+ reason="requires XPU with torch.xpu.XPUGraph support",+)+++# ---------------------------------------------------------------------------+# Capture helper+# ---------------------------------------------------------------------------+++def _capture_graph(fn):+ """Generic capture helper: warm up then capture ``fn`` into an XPUGraph.++ Returns (graph, output_tensor). The output tensor lives in the+ XPU graph's capture pool and therefore retains a *stable* device+ address across replays, making it safe to compare after ``replay()``.+ """+ with torch.no_grad():+ fn()+ torch.xpu.synchronize()++ g = torch.xpu.XPUGraph()+ side = torch.xpu.Stream()+ cur = torch.xpu.current_stream()+ side.wait_stream(cur)+ with torch.no_grad(), torch.xpu.stream(side):+ torch.xpu.synchronize()+ with torch.xpu.graph(g):+ out = fn()+ cur.wait_stream(side)+ torch.xpu.synchronize()+ return g, out+++def _capture_dense_sdpa(q, k, v, is_causal, scale):+ """Warm up, then capture ark.sdpa into an XPUGraph.++ Returns (graph, output_tensor). The output tensor lives in the+ XPU graph's capture pool and therefore retains a *stable* device+ address across replays, making it safe to compare after ``replay()``.+ """+ return _capture_graph(lambda: ark.sdpa(q, k, v, is_causal=is_causal, scale=scale))+++def _run_dense_case(b, hq, hkv, sq, skv, d, dtype, is_causal):+ torch.manual_seed(0)+ scale = 1.0 / math.sqrt(d)+ dev = "xpu:0"++ q = torch.randn(b, hq, sq, d, device=dev, dtype=dtype)+ k = torch.randn(b, hkv, skv, d, device=dev, dtype=dtype)+ v = torch.randn(b, hkv, skv, d, device=dev, dtype=dtype)+ q_new = torch.randn(b, hq, sq, d, device=dev, dtype=dtype)++ g, out = _capture_dense_sdpa(q, k, v, is_causal, scale)++ # --- replay with changed input ---+ q.copy_(q_new)+ torch.xpu.synchronize()+ g.replay()+ torch.xpu.synchronize()++ ref_new = reference_sdpa(q, k, v, is_causal=is_causal, scale=scale)+ diff = (out.float() - ref_new.float()).abs().max().item()+ assert torch.allclose(+ out.float(), ref_new.float(), atol=1e-2, rtol=1e-2+ ), f"dense SDPA graph replay mismatch: max_abs_diff={diff:.6f}"+ g.reset()+++# ---------------------------------------------------------------------------+# Varlen helpers+# ---------------------------------------------------------------------------+++def _build_varlen(batch, lengths_q, lengths_kv, hq, hkv, d, dtype):+ """Build varlen inputs: packed Q/K/V + cu_seqlens + max seqlens."""+ dev = "xpu:0"+ total_q = sum(lengths_q)+ total_kv = sum(lengths_kv)+ max_q = max(lengths_q)+ max_kv = max(lengths_kv)++ q = torch.randn(total_q, hq, d, device=dev, dtype=dtype)+ k = torch.randn(total_kv, hkv, d, device=dev, dtype=dtype)+ v = torch.randn(total_kv, hkv, d, device=dev, dtype=dtype)++ cu_q = torch.tensor([0] + list(torch.cumsum(torch.tensor(lengths_q), 0).tolist()), device=dev, dtype=torch.int64)+ cu_kv = torch.tensor([0] + list(torch.cumsum(torch.tensor(lengths_kv), 0).tolist()), device=dev, dtype=torch.int64)+ return q, k, v, cu_q, cu_kv, max_q, max_kv+++def _run_varlen_capture(dtype, lengths_q, lengths_kv, hq=8, hkv=4, d=128):+ """Capture varlen SDPA in a graph and replay once — only checks no-throw."""+ torch.manual_seed(0)+ q, k, v, cu_q, cu_kv, max_q, max_kv = _build_varlen(len(lengths_q), lengths_q, lengths_kv, hq, hkv, d, dtype)+ scale = 1.0 / math.sqrt(d)++ def call():+ return ark.sdpa_varlen(q, k, v, cu_q, cu_kv, max_q, max_kv, is_causal=True, scale=scale)++ # Eager warm-up+ with torch.no_grad():+ _ = call()+ torch.xpu.synchronize()++ # Capture+ g = torch.xpu.XPUGraph()+ side, cur = torch.xpu.Stream(), torch.xpu.current_stream()+ side.wait_stream(cur)+ with torch.no_grad(), torch.xpu.stream(side):+ torch.xpu.synchronize()+ with torch.xpu.graph(g):+ call()+ cur.wait_stream(side)+ torch.xpu.synchronize()++ # One replay must not throw+ g.replay()+ torch.xpu.synchronize()+ g.reset()+++# ---------------------------------------------------------------------------+# Dense SDPA graph-capture tests+# ---------------------------------------------------------------------------+++@requires_graph[email protected]("dtype", [torch.float16, torch.bfloat16])[email protected]("head_dim", [64, 128])+class TestSDPAGraphCapture:+ def test_prefill_causal(self, dtype, head_dim):+ _run_dense_case(2, 8, 8, 128, 128, head_dim, dtype, is_causal=True)++ def test_prefill_noncausal(self, dtype, head_dim):+ _run_dense_case(2, 8, 8, 128, 192, head_dim, dtype, is_causal=False)++ def test_prefill_gqa(self, dtype, head_dim):+ _run_dense_case(2, 32, 4, 64, 256, head_dim, dtype, is_causal=True)++ def test_decode(self, dtype, head_dim):+ _run_dense_case(2, 16, 4, 1, 64, head_dim, dtype, is_causal=True)+++# ---------------------------------------------------------------------------+# Varlen graph-capture smoke tests+# ---------------------------------------------------------------------------+++@requires_graph[email protected]("dtype", [torch.float16, torch.bfloat16])+class TestSDPAVarlenGraphCapture:+ def test_capture_replay_smoke(self, dtype):+ """Capture + replay must not throw; numeric check TODO."""+ _run_varlen_capture(dtype, [128, 64, 96], [128, 64, 96])+++# ---------------------------------------------------------------------------+# GEMM graph-capture smoke tests+#+# ``matmul_sycl_tla`` (dense float GEMM) shares the same+# ``compat::set_default_queue`` code path as SDPA, so it must also be+# graph-capture safe. Verify capture → mutate input in-place → replay →+# output matches a fresh eager call on the mutated inputs.+# ---------------------------------------------------------------------------+++@requires_graph[email protected]("dtype", [torch.float16, torch.bfloat16])+class TestGEMMGraphCapture:+ def test_capture_replay(self, dtype):+ """Dense GEMM (matmul_sycl_tla) must survive capture + replay."""+ M, N, K = 128, 256, 512+ dev = "xpu:0"+ A = torch.randn(M, K, device=dev, dtype=dtype)+ B = torch.randn(N, K, device=dev, dtype=dtype)+ bias = torch.randn(N, device=dev, dtype=dtype)++ def call():+ return ark.matmul_sycl_tla(A, B, bias)++ with torch.no_grad():+ _ = call() # warm-up / JIT+ torch.xpu.synchronize()++ g, out = _capture_graph(lambda: ark.matmul_sycl_tla(A, B, bias))++ # --- mutate inputs in-place, replay, compare to fresh eager ---+ A.copy_(torch.randn_like(A))+ bias.copy_(torch.randn_like(bias))+ torch.xpu.synchronize()++ g.replay()+ torch.xpu.synchronize()++ # Self-consistency: replay must match a fresh eager call on same inputs.+ with torch.no_grad():+ eager = ark.matmul_sycl_tla(A, B, bias)+ torch.xpu.synchronize()+ atol = 2e-2 if dtype == torch.bfloat16 else 1e-3+ diff = (out.float() - eager.float()).abs().max().item()+ assert diff < atol, f"GEMM graph replay mismatch: max_abs_diff={diff:.6f}"+++# ---------------------------------------------------------------------------+# SDPA with LSE graph-capture tests+#+# return_lse=True allocates an LSE tensor inside ark.sdpa() during capture.+# The LSE buffer lives in the capture pool and must be correctly updated on+# each replay.+# ---------------------------------------------------------------------------+++@requires_graph[email protected]("dtype", [torch.float16, torch.bfloat16])[email protected]("head_dim", [64, 128])+class TestSDPALSEGraphCapture:+ def test_capture_replay_lse(self, dtype, head_dim):+ """SDPA with return_lse=True must survive capture + replay.++ LSE is allocated inside the capture region, so its buffer lives in+ the graph pool and is stable across replays.+ """+ B, Hq, Hkv, Sq, Skv, D = 2, 8, 8, 128, 128, head_dim+ scale = 1.0 / math.sqrt(D)+ q = torch.randn(B, Sq, Hq, D, device="xpu:0", dtype=dtype)+ k = torch.randn(B, Skv, Hkv, D, device="xpu:0", dtype=dtype)+ v = torch.randn(B, Skv, Hkv, D, device="xpu:0", dtype=dtype)++ def call():+ return ark.sdpa(q, k, v, scale=scale, is_causal=True, return_lse=True)++ with torch.no_grad():+ _ = call() # warm-up / JIT+ torch.xpu.synchronize()++ g, out = _capture_graph(call)+ # out should be a tuple (O, LSE)+ assert isinstance(out, (tuple, list)) and len(out) == 2, f"Expected (O, LSE) tuple, got {type(out)}"+ o_out, lse_out = out[0], out[1]++ # --- mutate Q in-place, replay, verify both O and LSE ---+ q_new = torch.randn_like(q)+ q.copy_(q_new)+ torch.xpu.synchronize()+ g.replay()+ torch.xpu.synchronize()++ # Self-consistency: compare replay outputs to fresh eager call+ with torch.no_grad():+ o_eager, lse_eager = ark.sdpa(q, k, v, scale=scale, is_causal=True, return_lse=True)+ torch.xpu.synchronize()++ o_diff = (o_out.float() - o_eager.float()).abs().max().item()+ assert o_diff < 2e-2, f"LSE test: O mismatch after replay: {o_diff:.6f}"++ # LSE is in fp32, should be bit-identical or very close+ lse_diff = (lse_out.float() - lse_eager.float()).abs().max().item()+ assert lse_diff < 1e-4, f"LSE graph replay mismatch: max_abs_diff={lse_diff:.6f}"+ g.reset()