PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Sage fast sfm (#1843)
Luo Yud52c2a1589b9
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex 831d3714a..bd844b656 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -447,8 +447,8 @@ class ARK:- value: [B, Hkv, Skv, D]Args:- - scale: Attention scale. Uses 1.0 when None.- - scale_block_size: Block size for qscale and kscale.+ - scale: Attention scale. Uses 1 / sqrt(D) when None.+ - quant_block_size: Block size for qscale and kscale.Returns:- O: [B, Hq, Sq, D] (same dtype as value)@@ -502,7 +502,91 @@ class ARK:Sq,Skv,D,- float(scale) if scale is not None else 1.0,+ float(scale) if scale is not None else 1.0 / (D**0.5),+ bool(is_causal),+ )+ return O++ def sage_pvi8(+ self,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+ qscale: torch.Tensor = None,+ kscale: torch.Tensor = None,+ vscale: torch.Tensor = None,+ out_dtype: torch.dtype = torch.float16,+ ) -> torch.Tensor:+ """Low-level SAGE attention with pre-quantized INT8 Q/K/V and PV int8.++ Expects contiguous layouts:+ - query: [B, Hq, Sq, D] int8+ - key: [B, Hkv, Skv, D] int8+ - value: [B, Hkv, Skv, D] int8+ - qscale: [B, Hq, ceil(Sq / quant_block_size), 1] float32+ - kscale/vscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32++ Returns:+ - O: [B, Hq, Sq, D] float16+ """+ if query.device.type != "xpu":+ raise NotImplementedError("sage_pvi8 is only supported on XPU")+ if query.dtype != torch.int8 or key.dtype != torch.int8 or value.dtype != torch.int8:+ raise ValueError(f"Q/K/V must be int8, got Q={query.dtype}, K={key.dtype}, V={value.dtype}")+ if out_dtype != torch.float16:+ raise ValueError(f"sage_pvi8 output must be float16, got {out_dtype}")+ if qscale is None or kscale is None or vscale is None:+ raise ValueError("qscale, kscale and vscale must be provided for sage_pvi8")++ if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:+ raise ValueError("Q/K/V must be 4D tensors")++ if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():+ raise ValueError("Q/K/V must be contiguous")++ B, Hq, Sq, D = query.shape+ Bk, Hkv, Skv, Dk = key.shape+ Bv, Hkv2, Skv2, Dv = value.shape++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")++ lib = self.get_lib(query)+ stream = get_stream(query)+ O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=out_dtype)+ lib.sage_pvi8(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ quant_block_size,+ qscale.data_ptr(),+ kscale.data_ptr(),+ vscale.data_ptr(),+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0 / (D**0.5),bool(is_causal),)return O@@ -527,7 +611,7 @@ class ARK:- value: [B, Hkv, Skv, D]Args:- - scale: Attention scale. Uses 1.0 when None.+ - scale: Attention scale. Uses 1 / sqrt(D) when None.- quant_block_size: Quantization block size used by the kernel.Returns:@@ -590,7 +674,86 @@ class ARK:Sq,Skv,D,- float(scale) if scale is not None else 1.0,+ float(scale) if scale is not None else 1.0 / (D**0.5),+ bool(is_causal),+ )+ return O++ def sagev1_pvi8(+ self,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+ ) -> torch.Tensor:+ """SAGE v1 attention with PV int8 path.++ Expects FP16 Q/K/V input and quantizes Q/K/V internally before calling+ the PV int8 kernel.+ """+ if quant_block_size <= 0:+ return self.sdpa(+ query=query,+ key=key,+ value=value,+ attn_mask=attn_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ )+ if query.device.type != "xpu":+ raise NotImplementedError("sagev1_pvi8 is only supported on XPU")+ if query.dtype not in (torch.float16,):+ raise ValueError(f"Q must be float16, got {query.dtype}")+ if key.dtype != query.dtype or value.dtype != query.dtype:+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")++ if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:+ raise ValueError("Q/K/V must be 4D tensors")++ if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():+ raise ValueError("Q/K/V must be contiguous")++ B, Hq, Sq, D = query.shape+ Bk, Hkv, Skv, Dk = key.shape+ Bv, Hkv2, Skv2, Dv = value.shape++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")++ lib = self.get_lib(query)+ stream = get_stream(query)+ O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)+ lib.sagev1_pvi8(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ quant_block_size,+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(value.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0 / (D**0.5),bool(is_causal),)return O@@ -636,7 +799,7 @@ class ARK:# block_size=0 means per-tokenblock_size = quant_block_size if quant_block_size > 0 else 1- # SAGE V1 kernel uses K-tile size=32; scale_block_size must be 1 or >=32+ # SAGE V1 kernel uses K-tile size=32; quant_block_size must be 1 or >=32if block_size != 1 and block_size < 32:raise ValueError(f"quant_block_size={block_size} is not supported. "@@ -700,7 +863,7 @@ class ARK:)k_scale = k_scale.reshape(B, Hkv, Skv_pad // block_size, 1)- # Call SAGE v1 with matching scale_block_size+ # Call SAGE v1 with matching quant_block_sizeout = self.sage(q_i8,k_i8,@@ -709,7 +872,7 @@ class ARK:is_causal=is_causal,scale=scale,enable_gqa=enable_gqa,- scale_block_size=block_size,+ quant_block_size=block_size,qscale=q_scale,kscale=k_scale,)diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex 86d983f90..b623ed437 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -114,9 +114,10 @@ static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);}-static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,- int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q,- int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,+ int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal, bool use_int8_pv) {if (mask && is_causal) {throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");}@@ -127,13 +128,34 @@ static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torcthrow std::invalid_argument("ark::sagev1: only F16 is supported for q_dtype");}#ifdef ARK_XPU- XpuWrapper::sagev1((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, scale_block_size, batch,- num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ if (use_int8_pv) {+ XpuWrapper::sagev1_pvi8((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,+ scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal);+ } else {+ XpuWrapper::sagev1((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, scale_block_size,+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ }#elsethrow std::runtime_error("ark::sagev1 is only supported on XPU");#endif}+static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,+ int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q,+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch, num_heads_q,+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false);+}++static void sagev1_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,+ int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal) {+ sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch, num_heads_q,+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true);+}+static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,int scale_block_size, torch_ptr qscale, torch_ptr kscale, int q_dtype, int k_dtype, int o_dtype,int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,@@ -146,6 +168,18 @@ static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);}+static void sage_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,+ int scale_block_size, torch_ptr qscale, torch_ptr kscale, torch_ptr vscale, int q_dtype,+ int k_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ if (mask && is_causal) {+ throw std::invalid_argument("ark::sage_pvi8: mask and is_causal cannot both be set");+ }+ ark::sdpa_impl_qks8_pvi8((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,+ scale_block_size, (void*)qscale, (void*)kscale, (void*)vscale, batch, num_heads_q,+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+}+static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr weights, torch_ptr scales,torch_ptr outputs, int dtype, int N, int K, torch_ptr num_tokens_per_expert,int num_experts) {@@ -289,7 +323,11 @@ PYBIND11_MODULE(PY_NAME, m) {#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)m.def("sdpa", &ark::sdpa);m.def("sagev1", &ark::sagev1);+ // High-level SAGEV1 PVi8 API: input Q/K/V are FP16 and quantized internally.+ m.def("sagev1_pvi8", &ark::sagev1_pvi8);m.def("sage", &ark::sage);+ // Low-level SAGE PVi8 API: input Q/K/V are pre-quantized int8 with qscale/kscale/vscale.+ m.def("sage_pvi8", &ark::sage_pvi8);m.def("sage_dynamic_quant", &ark::sage_dynamic_quant);m.def("moe_gemm", &ark::moe_gemm_wrapper);#endifdiff --git a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa.cppindex 990c1c9d4..f5e30e7aa 100644--- a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp+++ b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp@@ -36,10 +36,39 @@ int launch_prefill_kernel_f16_128_sage(detail::Options const& options) {return launch_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::half_t>(options);}+int launch_prefill_kernel_f16_128_sage_i8pv(detail::Options const& options) {+ return launch_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::int8_t, cute::half_t, true, true, true>(+ options);+}+int launch_prefill_kernel_f16_64_sage(detail::Options const& options) {return launch_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::half_t>(options);}+int launch_prefill_kernel_f16_64_sage_i8pv(detail::Options const& options) {+ return launch_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::int8_t, cute::half_t, true, true, true>(+ options);+}++KernelLauncher select_sage_prefill_launcher(BTLA_DTYPE dtype, int head_dim, bool use_int8_pv) {+ switch (dtype) {+ case BTLA_DTYPE::S8:+ switch (head_dim) {+ case 128:+ if (!use_int8_pv) return launch_prefill_kernel_f16_128_sage;+ return launch_prefill_kernel_f16_128_sage_i8pv;+ case 64:+ if (!use_int8_pv) return launch_prefill_kernel_f16_64_sage;+ return launch_prefill_kernel_f16_64_sage_i8pv;+ default:+ return nullptr;+ }++ default:+ return nullptr;+ }+}+KernelLauncher select_sage_prefill_launcher(BTLA_DTYPE dtype, int head_dim) {switch (dtype) {case BTLA_DTYPE::S8:@@ -148,16 +177,18 @@ detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void*} // namespacevoid sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,- void* kscale, BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ void* kscale, void* vscale, bool use_int8_pv,+ BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q, num_heads_kv,seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);options.scale_block_size = scale_block_size;options.qscale = qscale;options.kscale = kscale;+ options.vscale = vscale;compat::set_default_queue(*q);- KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, head_dim);+ KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, head_dim, use_int8_pv);if (launcher == nullptr) {throw std::runtime_error("Unsupported dtype or head dimension for sage_prefill / SAGE (only F16/BF16 and 64/96/128/192 are supported)");@@ -238,10 +269,36 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr}- sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q,+ sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, nullptr, false,+ BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q,seq_len_kv, head_dim, softmax_scale, is_causal);}+void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ int scale_block_size, void* qscale, void* kscale, void* vscale, int batch, int num_heads_q,+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal) {+ if (mask && is_causal) {+ throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");+ }++ if (seq_len_q <= 0 || seq_len_kv <= 0) {+ throw std::invalid_argument("sdpa_impl: seq_len_q and seq_len_kv must be greater than 0");+ }+ if (vscale == nullptr) {+ throw std::invalid_argument("sdpa_impl_qks8_pvi8: vscale must be provided for int8 PV");+ }+ if (seq_len_q == 1) {+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, BTLA_DTYPE::F16, batch, num_heads_q, num_heads_kv,+ seq_len_kv, head_dim, softmax_scale, is_causal);+ return;+ }++ sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, vscale, true,+ BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,+ is_causal);+}+} // namespace ark#endif // ARK_XPU && ARK_SYCL_TLAdiff --git a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.pyindex 7b51f96db..aa9f2ac30 100644--- a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py+++ b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py@@ -33,10 +33,10 @@ def _log_fallback_warning_once(backend: str, error: Exception) -> None:def _validate_backend(backend: str, quant_block_size: int) -> str:backend = backend.lower()- if backend not in {"sdpa", "sagev1"}:+ if backend not in {"sdpa", "sagev1", "sagev1_pvi8"}:raise ValueError(f"Unsupported ARK attention backend: {backend}")- if backend == "sagev1" and quant_block_size <= 0:- raise ValueError("quant_block_size must be > 0 when backend='sagev1'")+ if backend in {"sagev1", "sagev1_pvi8"} and quant_block_size <= 0:+ raise ValueError(f"quant_block_size must be > 0 when backend='{backend}'")return backend@@ -113,7 +113,7 @@ def _can_use_ark_attention(128,192,)- if backend == "sagev1":+ if backend in {"sagev1", "sagev1_pvi8"}:return query.dtype == torch.float16 and query.shape[-1] in (64, 128)return True@@ -195,7 +195,18 @@ def patch_torch_sdpa_with_ark(*, strict: bool = False, backend: str = "sdpa", quis_causal=is_causal,scale=scale,)- return ark.sagev1(+ if backend == "sagev1":+ return ark.sagev1(+ query.contiguous(),+ key.contiguous(),+ value.contiguous(),+ attn_mask=normalized_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ quant_block_size=quant_block_size,+ )+ return ark.sagev1_pvi8(query.contiguous(),key.contiguous(),value.contiguous(),diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hppindex ff0d4bee3..f80f08cc9 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp@@ -269,6 +269,9 @@ class XeSageFwdKernel {auto scaleK = params.mainloop.scale_block_size? (float*)params.mainloop.kscale + (idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad): nullptr; // per head kscale+ auto scaleV = params.mainloop.scale_block_size && params.mainloop.vscale+ ? (float*)params.mainloop.vscale + (idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad)+ : nullptr; // per head vscaleauto ptrO = p.O + offset_o;auto stride_q = is_var_len ? cutlass::make_cute_packed_stride(StrideQ{}, shape_Q) : p.dQ;@@ -293,7 +296,8 @@ class XeSageFwdKernel {int l_coord = is_var_len ? 0 : idx_b;CollectiveMainloop mainloop(params.mainloop, shared_storage.mainloop);mainloop(Q(_, _, head_q, l_coord), K(_, _, head, l_coord), V(_, _, head, l_coord), tArA, tA_max, tA_sum, blk_qv,- 0, k_blocks, k_blocks, thr_id, seq_len, seq_len_kv_cache, idx_b, scaleQ, scaleK, full_tile_offset,+ 0, k_blocks, k_blocks, thr_id, seq_len, seq_len_kv_cache, idx_b, scaleQ, scaleK, scaleV,+ full_tile_offset,discard_seq_coord, K_cache(_, _, head, l_coord), V_cache(_, _, head, l_coord));if constexpr (!is_empty_v<MainloopSharedStorage> && !is_empty_v<EpilogueSharedStorage>) {diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hppindex d4a6eaa5a..0762423ff 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp@@ -42,6 +42,7 @@#include "cute/atom/mma_atom.hpp"#include "flash_attention_v2/collective/fmha_fusion.hpp"#include <cmath>+#include <type_traits>#include <cute/util/xe_split_barrier.hpp>namespace cutlass::sage {@@ -56,7 +57,8 @@ using namespace cute;/////////////////////////////////////////////////////////////////////////////////////////////////-template <class DispatchPolicy_, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_,+template <class DispatchPolicy_, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_, bool UseInt8PV_,+ bool WriteBackInt8PV_, bool ExecuteInt8PV_,class TiledMMAQK_, // Tiling for Q*K GEMMclass TiledMMAPV_, // Tiling for P*V GEMMint VTiles_, // # of tiles in V dimension@@ -73,13 +75,15 @@ struct SAGEV1FwdMainloop {/////////////////////////////////////////////////////////////////////////////////////////////////-template <int Stages, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_, class TiledMMAQK_,+template <int Stages, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_, bool UseInt8PV_,+ bool WriteBackInt8PV_, bool ExecuteInt8PV_, class TiledMMAQK_,class TiledMMAPV_, int VTiles_, class TensorQ_, class TensorK_, class TensorV_, class TensorK_cache_,class TensorV_cache_, class TiledCopyQ_, class TiledCopyK_, class TiledCopyV_, class TiledCopyK_cache_,class TiledCopyV_cache_>-struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV_, PagedKV_, TiledMMAQK_, TiledMMAPV_,- VTiles_, TensorQ_, TensorK_, TensorV_, TensorK_cache_, TensorV_cache_, TiledCopyQ_,- TiledCopyK_, TiledCopyV_, TiledCopyK_cache_, TiledCopyV_cache_> {+struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV_, PagedKV_, UseInt8PV_,+ WriteBackInt8PV_, ExecuteInt8PV_, TiledMMAQK_, TiledMMAPV_, VTiles_, TensorQ_, TensorK_,+ TensorV_, TensorK_cache_, TensorV_cache_, TiledCopyQ_, TiledCopyK_, TiledCopyV_,+ TiledCopyK_cache_, TiledCopyV_cache_> {//// Type Aliases//@@ -89,6 +93,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cachedusing TileShapePV = decltype(TiledMMAPV{}.tile_mnk());static constexpr int VTiles = VTiles_;using SubgroupLayoutQK = decltype(TiledMMAQK{}.get_atom_layout_mnk());+ using SubgroupLayoutPV = decltype(TiledMMAPV{}.get_atom_layout_mnk());using SGPerWG = decltype(product(take<1, 4>(shape(typename TiledMMAQK::ThrLayoutVMNK{}))));using TensorQ = TensorQ_;@@ -141,14 +146,29 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cacheddecltype(FragS{}.tv_layout()){}));using FragPRow = decltype(reduce<1>(FragP{}, sycl::plus<void>{}));- using SingleFragA = FragC<TiledMMAPV>; // (atom val,q',v')+ using SingleFragPV = FragC<TiledMMAPV>; // (atom val,q',v')+ using SingleFragAFloat = std::remove_reference_t<decltype(make_subgroup_tensor(+ make_fragment_like<float>(typename SingleFragPV::Base{}), decltype(SingleFragPV{}.tv_layout()){}))>;+ using SingleFragA = conditional_t<UseInt8PV_, SingleFragAFloat, SingleFragPV>;using FragA = expand_sg_fragment_t<SingleFragA, 1, VTiles>; // (atom val,q',v',VV)using FragARow = decltype(reduce<1>(FragA{}, sycl::plus<void>{}));- using ElementA = typename TiledMMAPV::ValTypeD;+ using ElementA = typename SingleFragA::value_type;++ static constexpr int SGTilePVQ = get<0>(shape_div(TileShapePV{}, shape(SubgroupLayoutPV{})))();+ using QuantizedPVOperation = XE_DPAS_TT<cute::gcd(SGTilePVQ, 8), int32_t, int8_t, int8_t>;+ using QuantizedTiledMMAPV =+ typename TiledMMAHelper<MMA_Atom<QuantizedPVOperation>, Layout<TileShapePV>, SubgroupLayoutPV>::TiledMMA;+ using QuantizedSingleFragA = FragC<QuantizedTiledMMAPV>;++ static_assert(size(SingleFragA{}.shape()) == size(QuantizedSingleFragA{}.shape()),+ "Quantized PV accumulator must match float PV tile size");static constexpr bool CausalMask = CausalMask_;static constexpr bool CachedKV = CachedKV_;static constexpr bool PagedKV = PagedKV_;+ static constexpr bool UseInt8PV = UseInt8PV_;+ static constexpr bool WriteBackInt8PV = WriteBackInt8PV_;+ static constexpr bool ExecuteInt8PV = ExecuteInt8PV_;// User-facing argumentsstruct Arguments {@@ -157,6 +177,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cachedint scale_block_size = 0; // if non-zero, apply scaling in blocks of this size (for block-sparse attention)float const* qscale = nullptr;float const* kscale = nullptr;+ float const* vscale = nullptr;int const* ptr_page_table = nullptr;int page_size = 0;int const* num_pages_per_seq = nullptr;@@ -179,12 +200,38 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cachedstatic constexpr Params to_underlying_arguments(Arguments const& args, void* /* workspace */) {constexpr double kLog2e = 1.4426950408889634074; // log_2(e)float val = args.scale * static_cast<float>(kLog2e);- return Params{val, args.mask, args.scale_block_size, args.qscale,- args.kscale, args.ptr_page_table, args.page_size, args.num_pages_per_seq};+ return Params{val, args.mask, args.scale_block_size, args.qscale, args.kscale, args.vscale,+ args.ptr_page_table, args.page_size, args.num_pages_per_seq};}CUTLASS_HOST_DEVICE static bool can_implement(Arguments const&) { return true; }+ // DPAS kernels run as SIMD16, so pack two SIMD16 vectors into one temporary SIMD32 vector+ // before issuing a single vISA SIMD32 instruction.+ CUTLASS_DEVICE static void exp2_pair_simd32_asm(ElementP x0, ElementP x1, ElementP& y0, ElementP& y1) {+#if defined(__SYCL_DEVICE_ONLY__) && defined(SYCL_INTEL_TARGET)+ asm(+ "{\n"+ ".decl OUT0 v_type=G type=F num_elts=16 alias=<%0,0>\n"+ ".decl OUT1 v_type=G type=F num_elts=16 alias=<%1,0>\n"+ ".decl IN0 v_type=G type=F num_elts=16 alias=<%2,0>\n"+ ".decl IN1 v_type=G type=F num_elts=16 alias=<%3,0>\n"+ ".decl TMP_IN v_type=G type=F num_elts=32 align=64\n"+ ".decl TMP_OUT v_type=G type=F num_elts=32 align=64\n"+ "mov (M1_NM, 16) TMP_IN(0,0)<1> IN0(0,0)<1;1,0>\n"+ "mov (M1_NM, 16) TMP_IN(1,0)<1> IN1(0,0)<1;1,0>\n"+ "exp (M1_NM, 32) TMP_OUT(0,0)<1> TMP_IN(0,0)<1;1,0>\n"+ "mov (M1_NM, 16) OUT0(0,0)<1> TMP_OUT(0,0)<1;1,0>\n"+ "mov (M1_NM, 16) OUT1(0,0)<1> TMP_OUT(1,0)<1;1,0>\n"+ "}\n"+ : "=rw"(y0), "=rw"(y1)+ : "rw"(x0), "rw"(x1));+#else+ y0 = sycl::native::exp2(x0);+ y1 = sycl::native::exp2(x1);+#endif+ }+CUTLASS_DEVICEint get_physical_k_tile(int K, int l_coord, int seq_len_kv_cache) {int next_page_logical_idx = K * get<1>(TileShapeQK{}) / params.page_size;@@ -209,7 +256,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cachedint blk_k1,int total_blk, // Total # of K blocksint thr_id, int seq_len, int seq_len_kv_cache, int l_coord, float* scaleQ,- float* scaleK, int full_tile_offset, int discard_seq_coord,+ float* scaleK, float* scaleV, int full_tile_offset, int discard_seq_coord,TensorK_cache2D const& K_cache_2D = TensorK_cache2D{},TensorV_cache2D const& V_cache_2D = TensorV_cache2D{}) {using namespace sycl::ext::oneapi::this_work_item;@@ -253,6 +300,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached/* Create MMAs */TiledMMAQK mma_qk{};TiledMMAPV mma_pv{};+ QuantizedTiledMMAPV mma_pv_i8{};/* Slice TiledCopy/TiledMMA operations down to to work-item level */auto thr_copy_q = copy_q.get_slice(thr_id);@@ -262,6 +310,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cachedauto thr_copy_v_cache = copy_v_cache.get_slice(thr_id);auto thr_mma_qk = mma_qk.get_slice(thr_id);auto thr_mma_pv = mma_pv.get_slice(thr_id);+ auto thr_mma_pv_i8 = mma_pv_i8.get_slice(thr_id);/* Partition coordinate tensors for copy */auto tQgQ = thr_copy_q.partition_S(gQ); // (atom_val,q',d',D)@@ -282,6 +331,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, CachedFragP tPrS = make_subgroup_tensor(make_fragment_like<ElementP>(tSrS.tensor()), tSrS.tv_layout());auto tArP = thr_mma_pv.partition_sg_fragment_A(cP);+ auto tArP_i8 = thr_mma_pv_i8.partition_sg_fragment_A(cP);auto tVrV = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));auto tArV = thr_mma_pv.partition_sg_fragment_B(gV_split(_, _, 0, 0));@@ -467,20 +517,57 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached/* Apply softmax and scaling (tA rescaling fused into GEMM2 VTile loop) */auto rescale = softmax(K == blk_k0, tPrS, tA_max, tA_sum);- reorder(tPrS, tArP);+ if constexpr (!UseInt8PV) {+ reorder(tPrS, tArP);+ } else {+ reorder(tPrS, tArP_i8);+ }/* GEMM 2: A += P * V, split in v dimension.tArA rescaling is fused to per-VTile */- CUTLASS_PRAGMA_UNROLL- for (int VV = 0; VV < VTiles; VV++) {- copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV);- reorder(tVrV, tArV);- if (K != blk_k0) {- CUTLASS_PRAGMA_UNROLL- for (int i = 0; i < tArA.size() / VTiles; i++) tArA(_, _, _, VV)(i) *= broadcast<0>(rescale, tArA, i);+ if constexpr (!UseInt8PV) {+ CUTLASS_PRAGMA_UNROLL+ for (int VV = 0; VV < VTiles; VV++) {+ copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV);+ reorder(tVrV, tArV);+ if (K != blk_k0) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tArA.size() / VTiles; i++) tArA(_, _, _, VV)(i) *= broadcast<0>(rescale, tArA, i);+ }++ cute::gemm(mma_pv, tArP, tArV, tArA(_, _, _, VV));}+ } else {+ ElementA pv_dequant_scale = ElementA(0.0f);+ if constexpr (WriteBackInt8PV) {+ pv_dequant_scale = ElementA(scaleV[scalek_idx]) * (ElementA(1.0f) / ElementA(127.0f));+ }+ CUTLASS_PRAGMA_UNROLL+ for (int VV = 0; VV < VTiles; VV++) {+ auto tArA_v = tArA(_, _, _, VV);+ if (K != blk_k0) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tArA.size() / VTiles; i++) tArA_v(i) *= broadcast<0>(rescale, tArA, i);+ }- cute::gemm(mma_pv, tArP, tArV, tArA(_, _, _, VV));+ auto tArAcc = thr_mma_pv_i8.partition_sg_fragment_C(make_identity_tensor(select<0, 1>(TileShapePV{})));+ auto tVrV_i8 = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));+ auto tArV_i8 = thr_mma_pv_i8.partition_sg_fragment_B(gV_split(_, _, 0, 0));+ copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV_i8);+ reorder(tVrV_i8, tArV_i8);+ clear(tArAcc);++ if constexpr (ExecuteInt8PV) {+ cute::gemm(mma_pv_i8, tArP_i8, tArV_i8, tArAcc);+ }++ if constexpr (WriteBackInt8PV) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tArAcc.size(); ++i) {+ tArA_v(i) += ElementA(tArAcc(i)) * pv_dequant_scale;+ }+ }+ }}/* K prefetch */@@ -525,11 +612,10 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, CachedFragARow softmax(bool first_block, // First softmax block?FragP& tS, // Softmax src/dst blockFragARow& tS_max, // Softmax row-wise max accumulator- FragARow& tS_sum) { // Softmax row-wise sum accumulator- /* Compute row-wise maxima for this block */+ FragARow& tS_sum) { // Softmax row-wise sum accumulator+ /* Compute row-wise maxima for this block */auto tS_bmax = reduce<1>(tS, sycl::maximum{});-FragARow rescale;CUTLASS_PRAGMA_UNROLLfor (int i = 0; i < tS_max.size(); i++) {@@ -539,8 +625,20 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached}/* Scale S and subtract maxima, then exponentiate */+ static_assert(FragP{}.size() % 2 == 0, "FragP size must be even for pairwise SIMD32 exp.");CUTLASS_PRAGMA_UNROLL- for (int i = 0; i < tS.size(); i++) tS(i) = sycl::native::exp2(tS(i) - broadcast<0>(tS_max, tS, i));+ for (int i = 0; i < tS.size(); i += 2) {+ ElementP x0 = tS(i) - broadcast<0>(tS_max, tS, i);+ ElementP x1 = tS(i + 1) - broadcast<0>(tS_max, tS, i + 1);+ exp2_pair_simd32_asm(x0, x1, tS(i), tS(i + 1));+ }++ if constexpr (UseInt8PV) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS.size(); i++) {+ tS(i) *= ElementP(127.0f);+ }+ }/* Rescale existing S sums */if (!first_block) {@@ -552,8 +650,14 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached/* Update sums */auto tS_bsum = reduce<1>(tS, sycl::plus<void>{});- CUTLASS_PRAGMA_UNROLL- for (int i = 0; i < tS_sum.size(); i++) tS_sum(i) += tS_bsum(i);+ if constexpr (UseInt8PV) {+ constexpr ElementA kInvPQuantScale = ElementA(1.0f / 127.0f);+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_sum.size(); i++) tS_sum(i) += ElementA(tS_bsum(i)) * kInvPQuantScale;+ } else {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_sum.size(); i++) tS_sum(i) += tS_bsum(i);+ }return rescale;}diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hppindex 37752f79a..62850e46b 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp@@ -64,6 +64,11 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptvoid sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,void* kscale, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,int seq_len_kv, int head_dim, float softmax_scale, bool is_causal);++void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ int scale_block_size, void* qscale, void* kscale, void* vscale, int batch, int num_heads_q,+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,+ bool is_causal);#endif // ARK_SYCL_TLA} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hppindex 9aaac19f3..6c4cdb006 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp@@ -68,7 +68,7 @@ struct Options {void* mask = nullptr;void* o = nullptr;int scale_block_size = 0;- const void *qscale = nullptr, *kscale = nullptr;+ const void *qscale = nullptr, *kscale = nullptr, *vscale = nullptr;const void *block_K = nullptr, *block_V = nullptr;const int *page_table = nullptr, *num_pages_per_seq = nullptr;bool is_causal = false;@@ -609,6 +609,7 @@ struct SageKernelRunner {},{options.softmax_scale, static_cast<float*>(options.mask), options.scale_block_size,static_cast<const float*>(options.qscale), static_cast<const float*>(options.kscale),+ static_cast<const float*>(options.vscale),options.use_paged_kv ? options.page_table : nullptr, options.use_paged_kv ? options.page_size : 0,options.use_paged_kv ? options.num_pages_per_seq : nullptr},{},@@ -635,7 +636,8 @@ struct SageKernelRunner {}};-template <bool Causal, typename TileShapeQK, typename TileShapePV, typename TileShapeOutput, typename SubgroupLayoutQK,+template <bool Causal, bool UseInt8PV, bool WriteBackInt8PV, bool ExecuteInt8PV, typename TileShapeQK,+ typename TileShapePV, typename TileShapeOutput, typename SubgroupLayoutQK,typename SubgroupLayoutPV_, /* void -> default */int PipelineStages, bool persistent, typename ElementQ = bfloat16_t, typename ElementK = bfloat16_t,typename ElementV = bfloat16_t, typename ElementO = ElementQ,@@ -693,7 +695,8 @@ struct SageConfig {if constexpr (Causal) {using CollectiveMainloop =cutlass::fmha::collective::SAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,- TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,+ TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;@@ -712,7 +715,8 @@ struct SageConfig {if (options.mask) {using CollectiveMainloop =cutlass::fmha::collective::SAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, true, CachedKV, PagedKV,- TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,+ TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;@@ -730,7 +734,8 @@ struct SageConfig {} else {using CollectiveMainloop =cutlass::fmha::collective::SAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,- TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,+ TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;@@ -803,7 +808,8 @@ inline int launch_prefill_kernel_128(Options const& options) {/*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);}-template <typename ElementQ, typename ElementK, typename ElementV>+template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV, bool UseInt8PV = false,+ bool WriteBackInt8PV = true, bool ExecuteInt8PV = true>inline int launch_sage_prefill_kernel_128(Options const& options) {constexpr int PipelineStages = 1;constexpr int PipelineStages1 = 1;@@ -815,13 +821,16 @@ inline int launch_sage_prefill_kernel_128(Options const& options) {using ShapePV1 = Shape<_256, _32, _64>;using ShapeOut1 = Shape<_256, _128>;using SubgroupLayoutQK1 = Layout<Shape<_16, _1, _1>>;- return options.is_causal ? SageConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,- /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options)- : SageConfig<false, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void, PipelineStages1,- /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options);+ return options.is_causal ? SageConfig<true, UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, ShapeQK, ShapePV, ShapeOut,+ SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementO>::run(options)+ : SageConfig<false, UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, ShapeQK1, ShapePV1,+ ShapeOut1, SubgroupLayoutQK1, void, PipelineStages1,+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementO>::run(options);}-template <typename ElementQ, typename ElementK, typename ElementV>+template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV, bool UseInt8PV = false,+ bool WriteBackInt8PV = true, bool ExecuteInt8PV = true>inline int launch_sage_prefill_kernel_64(Options const& options) {constexpr int PipelineStages = 2;constexpr int PipelineStages1 = 2;@@ -831,10 +840,12 @@ inline int launch_sage_prefill_kernel_64(Options const& options) {using SubgroupLayoutQK = Layout<Shape<_8, _1, _1>>;using SubgroupLayoutPV = void;return options.is_causal- ? SageConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages,- /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options)- : SageConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages1,- /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options);+ ? SageConfig<true, UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, ShapeQK, ShapePV, ShapeOut,+ SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages,+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementO>::run(options)+ : SageConfig<false, UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, ShapeQK, ShapePV, ShapeOut,+ SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages1,+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementO>::run(options);}template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hppindex a7dec5b4b..13a3ef875 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp@@ -1041,9 +1041,9 @@ class XpuWrapper {}}- static void sagev1(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,- int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,- int head_dim, float softmax_scale, bool is_causal) {+ static void sagev1_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv) {bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;size_t seq_q_blk = (seq_len_q + scale_block_size - 1) / scale_block_size;size_t seq_kv_blk = (seq_len_kv + scale_block_size - 1) / scale_block_size;@@ -1051,16 +1051,25 @@ class XpuWrapper {size_t q_scale_size = num_heads_q * seq_q_blk * batch;size_t k_size = num_heads_kv * seq_len_kv * head_dim * batch;size_t k_scale_size = num_heads_kv * seq_kv_blk * batch;+ size_t v_tmp_size = use_int8_pv ? k_size : 0;+ size_t v_scale_size = use_int8_pv ? k_scale_size * sizeof(float) : 0;size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(sycl::half) : 0;- size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + k_bias_size;+ size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size ++ v_scale_size + k_bias_size;auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);auto q_out_ptr = (int8_t*)ptr;auto k_out_ptr = (int8_t*)ptr + q_size;auto qscale = (float*)((int8_t*)ptr + q_size + k_size);auto kscale = (float*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float));+ auto v_out_ptr = use_int8_pv ? (int8_t*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) ++ k_scale_size * sizeof(float))+ : nullptr;+ auto vscale = use_int8_pv ? (float*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) ++ k_scale_size * sizeof(float) + k_size)+ : nullptr;auto kbias = use_mean_bias? (sycl::half*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) +- k_scale_size * sizeof(float))+ k_scale_size * sizeof(float) + v_tmp_size + v_scale_size): nullptr;sage_dynamic_quant<sycl::half>(q, (sycl::half*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch * num_heads_q,seq_len_q, seq_q_blk, head_dim, scale_block_size);@@ -1072,8 +1081,31 @@ class XpuWrapper {}sage_dynamic_quant<sycl::half>(q, (sycl::half*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);- ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, batch,- num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ if (use_int8_pv) {+ sage_dynamic_quant<sycl::half>(q, (sycl::half*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale, batch * num_heads_kv,+ seq_len_kv, seq_kv_blk, head_dim, scale_block_size);+ ark::sdpa_impl_qks8_pvi8(q, q_out_ptr, k_out_ptr, v_out_ptr, O_ptr, mask, scale_block_size, qscale, kscale,+ vscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,+ softmax_scale, is_causal);+ } else {+ ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale,+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,+ is_causal);+ }+ }++ static void sagev1(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim, float softmax_scale, bool is_causal) {+ sagev1_impl(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal, false);+ }++ static void sagev1_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ sagev1_impl(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal, true);}};diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hppindex b2ee33c4d..a42dfc3b9 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp@@ -20,7 +20,9 @@ struct TestSDPA {#if defined(ARK_XPU) && defined(ARK_SYCL_TLA) && ARK_XPU && ARK_SYCL_TLA// test_sagev1_accuracy("prefill_gqa_128_bias", 1, 32, 8, 256, 512, 128, 128, false, true);// test_sagev1_accuracy("prefill_gqa_128_no_bias", 1, 32, 8, 256, 512, 128, 128, false, false);- benchmark_sagev1("bench_prefill_gqa_128", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);+ test_sagev1_accuracy("prefill_gqa_128_causal_no_bias", 1, 32, 8, 256, 512, 128, 128, true, false);+ benchmark_sagev1("bench_prefill_gqa_128_s4096", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);+ benchmark_sagev1("bench_bmg_hq96_d128_k8192", 1, 96, 8, 4096, 8192, 128, 128, false, 5, 20);#elsestd::cout << "[sagev1] skipped: requires ARK_XPU and ARK_SYCL_TLA\n";#endif@@ -191,6 +193,9 @@ struct TestSDPA {size_t q_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;size_t k_count = size_t(batch) * num_heads_kv * seq_len_kv * head_dim;size_t o_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;+ size_t q_scale_count = size_t(batch) * num_heads_q * ((seq_len_q + scale_block_size - 1) / scale_block_size);+ size_t k_scale_count = size_t(batch) * num_heads_kv * ((seq_len_kv + scale_block_size - 1) / scale_block_size);+ size_t v_scale_count = k_scale_count;float softmax_scale = 1.0f / std::sqrt(float(head_dim));auto host_q = to_fp16_vector(make_random_vector(q_count, -1.0f, 1.0f, 901u + uint32_t(seq_len_q)));@@ -201,10 +206,28 @@ struct TestSDPA {auto* dev_k = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));auto* dev_v = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));auto* dev_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));+ auto* dev_kernel_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));+ auto* dev_pvi8_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));+ auto* dev_qi8 = reinterpret_cast<int8_t*>(ctx->allocate(q_count * sizeof(int8_t)));+ auto* dev_ki8 = reinterpret_cast<int8_t*>(ctx->allocate(k_count * sizeof(int8_t)));+ auto* dev_vi8 = reinterpret_cast<int8_t*>(ctx->allocate(k_count * sizeof(int8_t)));+ auto* dev_qscale = reinterpret_cast<float*>(ctx->allocate(q_scale_count * sizeof(float)));+ auto* dev_kscale = reinterpret_cast<float*>(ctx->allocate(k_scale_count * sizeof(float)));+ auto* dev_vscale = reinterpret_cast<float*>(ctx->allocate(v_scale_count * sizeof(float)));q->memcpy(dev_q, host_q.data(), q_count * sizeof(sycl::half)).wait();q->memcpy(dev_k, host_k.data(), k_count * sizeof(sycl::half)).wait();q->memcpy(dev_v, host_v.data(), k_count * sizeof(sycl::half)).wait();+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_q, dev_qi8, dev_qscale, batch * num_heads_q, seq_len_q,+ (seq_len_q + scale_block_size - 1) / scale_block_size, head_dim,+ scale_block_size);+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_k, dev_ki8, dev_kscale, batch * num_heads_kv, seq_len_kv,+ (seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,+ scale_block_size);+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_v, dev_vi8, dev_vscale, batch * num_heads_kv, seq_len_kv,+ (seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,+ scale_block_size);+double sage_ms = run_bench([&]() {ark::XpuWrapper::sagev1(q, dev_q, dev_k, dev_v, dev_out, nullptr, scale_block_size, batch, num_heads_q,@@ -212,18 +235,52 @@ struct TestSDPA {},q, warmup, iters);+ double kernel_pvhalf_ms = run_bench(+ [&]() {+ ark::sdpa_impl_qks8_pvhalf(q, dev_qi8, dev_ki8, dev_v, dev_kernel_out, nullptr, scale_block_size,+ dev_qscale, dev_kscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,+ head_dim, softmax_scale, is_causal);+ },+ q, warmup, iters);++ double kernel_pvi8_ms = run_bench(+ [&]() {+ ark::sdpa_impl_qks8_pvi8(q, dev_qi8, dev_ki8, dev_vi8, dev_pvi8_out, nullptr, scale_block_size, dev_qscale,+ dev_kscale, dev_vscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,+ head_dim, softmax_scale, is_causal);+ },+ q, warmup, iters);+double flops = compute_sdpa_flops(batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim);double io_bytes = compute_sdpa_io_bytes(batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim);double sage_tflops = flops / (sage_ms * 1e-3) / 1e12;double sage_gbps = io_bytes / (sage_ms * 1e-3) / 1e9;+ double kernel_pvhalf_tflops = flops / (kernel_pvhalf_ms * 1e-3) / 1e12;+ double kernel_pvhalf_gbps = io_bytes / (kernel_pvhalf_ms * 1e-3) / 1e9;+ double kernel_pvi8_tflops = flops / (kernel_pvi8_ms * 1e-3) / 1e12;+ double kernel_pvi8_gbps = io_bytes / (kernel_pvi8_ms * 1e-3) / 1e9;std::cout << std::fixed << std::setprecision(3) << "[sagev1][bench] " << name << " sage_ms=" << sage_ms<< " sage_TFLOPS=" << sage_tflops << " sage_GBps=" << sage_gbps << "\n";+ std::cout << std::fixed << std::setprecision(3) << "[sagev1][kernel_bench] " << name+ << " pvhalf_ms=" << kernel_pvhalf_ms << " pvhalf_TFLOPS=" << kernel_pvhalf_tflops+ << " pvhalf_GBps=" << kernel_pvhalf_gbps << "\n";+ std::cout << std::fixed << std::setprecision(3) << "[sagev1][kernel_bench] " << name+ << " pvi8_ms=" << kernel_pvi8_ms << " pvi8_TFLOPS=" << kernel_pvi8_tflops+ << " pvi8_GBps=" << kernel_pvi8_gbps << "\n";ctx->deallocate(dev_q);ctx->deallocate(dev_k);ctx->deallocate(dev_v);ctx->deallocate(dev_out);+ ctx->deallocate(dev_kernel_out);+ ctx->deallocate(dev_pvi8_out);+ ctx->deallocate(dev_qi8);+ ctx->deallocate(dev_ki8);+ ctx->deallocate(dev_vi8);+ ctx->deallocate(dev_qscale);+ ctx->deallocate(dev_kscale);+ ctx->deallocate(dev_vscale);}#endif-};\ No newline at end of file+};diff --git a/auto_round_extension/ark/test/test_sage_dynquant.py b/auto_round_extension/ark/test/test_sage_dynquant.pyindex 54e3e9832..37a90a0e9 100644--- a/auto_round_extension/ark/test/test_sage_dynquant.py+++ b/auto_round_extension/ark/test/test_sage_dynquant.py@@ -66,6 +66,30 @@ def run_benchmark(batch=2, seq=17776, h_q=30, h_kv=30, H=128, H_v=128, dt=torch.f" SageDynQuan block_size={bs:3d}: {t:8.1f} ms diff max={dff.max().item():.6f} mean={dff.mean().item():.6f}")+ # --- SAGE v1 high-level API comparison: PV half vs PV int8/PVS8 ---+ sagev1_pvhalf_results = {}+ sagev1_pvs8_results = {}+ for bs in block_sizes:+ pvhalf_out = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs)+ pvhalf_diff = (ref - pvhalf_out).abs()+ pvhalf_t = bench(lambda bs=bs: get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs))+ sagev1_pvhalf_results[bs] = (pvhalf_t, pvhalf_diff.max().item(), pvhalf_diff.mean().item())+ print(+ f" SAGE v1 pvhalf block_size={bs:3d}: {pvhalf_t:8.1f} ms "+ f"diff max={pvhalf_diff.max().item():.6f} mean={pvhalf_diff.mean().item():.6f}"+ )++ pvs8_out = get_ark().sagev1_pvi8(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs)+ pvs8_diff = (ref - pvs8_out).abs()+ pvs8_t = bench(+ lambda bs=bs: get_ark().sagev1_pvi8(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs)+ )+ sagev1_pvs8_results[bs] = (pvs8_t, pvs8_diff.max().item(), pvs8_diff.mean().item())+ print(+ f" SAGE v1 pvs8 block_size={bs:3d}: {pvs8_t:8.1f} ms "+ f"diff max={pvs8_diff.max().item():.6f} mean={pvs8_diff.mean().item():.6f}"+ )+# --- SAGE v1 reference for each block_size (pre-quantized INT8) ---sage_results = {}for bs in block_sizes:@@ -75,19 +99,27 @@ def run_benchmark(batch=2, seq=17776, h_q=30, h_kv=30, H=128, H_v=128, dt=torch.ks = torch.randn(batch, h_kv, seq // bs, 1, dtype=torch.float32, device=dev) / 100 + 0.001sage_t = bench(lambda bs=bs, q_i8=q_i8, k_i8=k_i8, qs=qs, ks=ks: get_ark().sage(- q_i8, k_i8, v, scale_block_size=bs, qscale=qs, kscale=ks, scale=scale, is_causal=is_causal+ q_i8, k_i8, v, quant_block_size=bs, qscale=qs, kscale=ks, scale=scale, is_causal=is_causal))sage_results[bs] = sage_tprint(f" SAGE v1 bs={bs:3d}: {sage_t:8.1f} ms")print()- print("--- Summary (dynquant vs SAGE v1 at same block_size) ---")+ print("--- Summary (dynquant / sagev1 pvhalf / sagev1 pvs8 / pre-quantized SAGE at same block_size) ---")for bs in block_sizes:t, mx, mn = results[bs]+ pvhalf_t, pvhalf_mx, pvhalf_mn = sagev1_pvhalf_results[bs]+ pvs8_t, pvs8_mx, pvs8_mn = sagev1_pvs8_results[bs]sage_t = sage_results[bs]overhead = (t / sage_t - 1) * 100- print(f" block_size={bs:3d}: dynquant {t:8.1f} ms vs sage_v1 {sage_t:8.1f} ms (+{overhead:.0f}%)")+ pvs8_over_pvhalf = (pvs8_t / pvhalf_t - 1) * 100+ print(+ f" block_size={bs:3d}: dynquant {t:8.1f} ms "+ f"pvhalf {pvhalf_t:8.1f} ms pvs8 {pvs8_t:8.1f} ms "+ f"prequant_sage {sage_t:8.1f} ms "+ f"dynquant_vs_prequant={overhead:+.0f}% pvs8_vs_pvhalf={pvs8_over_pvhalf:+.0f}%"+ )if __name__ == "__main__":