Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Sage fast sfm (#1843)

Luo Yud52c2a1589b9
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index 831d3714a..bd844b656 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -447,8 +447,8 @@ class ARK:
- value: [B, Hkv, Skv, D]
Args:
- - scale: Attention scale. Uses 1.0 when None.
- - scale_block_size: Block size for qscale and kscale.
+ - scale: Attention scale. Uses 1 / sqrt(D) when None.
+ - quant_block_size: Block size for qscale and kscale.
Returns:
- O: [B, Hq, Sq, D] (same dtype as value)
@@ -502,7 +502,91 @@ class ARK:
Sq,
Skv,
D,
- float(scale) if scale is not None else 1.0,
+ float(scale) if scale is not None else 1.0 / (D**0.5),
+ bool(is_causal),
+ )
+ return O
+
+ def sage_pvi8(
+ self,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attn_mask: torch.Tensor | None = None,
+ dropout_p: float = 0.0,
+ is_causal: bool = False,
+ scale: float | None = None,
+ enable_gqa: bool = False,
+ quant_block_size: int = 64,
+ qscale: torch.Tensor = None,
+ kscale: torch.Tensor = None,
+ vscale: torch.Tensor = None,
+ out_dtype: torch.dtype = torch.float16,
+ ) -> torch.Tensor:
+ """Low-level SAGE attention with pre-quantized INT8 Q/K/V and PV int8.
+
+ Expects contiguous layouts:
+ - query: [B, Hq, Sq, D] int8
+ - key: [B, Hkv, Skv, D] int8
+ - value: [B, Hkv, Skv, D] int8
+ - qscale: [B, Hq, ceil(Sq / quant_block_size), 1] float32
+ - kscale/vscale: [B, Hkv, ceil(Skv / quant_block_size), 1] float32
+
+ Returns:
+ - O: [B, Hq, Sq, D] float16
+ """
+ if query.device.type != "xpu":
+ raise NotImplementedError("sage_pvi8 is only supported on XPU")
+ if query.dtype != torch.int8 or key.dtype != torch.int8 or value.dtype != torch.int8:
+ raise ValueError(f"Q/K/V must be int8, got Q={query.dtype}, K={key.dtype}, V={value.dtype}")
+ if out_dtype != torch.float16:
+ raise ValueError(f"sage_pvi8 output must be float16, got {out_dtype}")
+ if qscale is None or kscale is None or vscale is None:
+ raise ValueError("qscale, kscale and vscale must be provided for sage_pvi8")
+
+ if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
+ raise ValueError("Q/K/V must be 4D tensors")
+
+ if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():
+ raise ValueError("Q/K/V must be contiguous")
+
+ B, Hq, Sq, D = query.shape
+ Bk, Hkv, Skv, Dk = key.shape
+ Bv, Hkv2, Skv2, Dv = value.shape
+
+ if Bk != B or Bv != B:
+ raise ValueError("Batch size mismatch between Q/K/V")
+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
+ raise ValueError("K/V shape mismatch")
+ if Dk != D:
+ raise ValueError("Head dim mismatch between Q and K/V")
+ if D not in (64, 128):
+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
+
+ lib = self.get_lib(query)
+ stream = get_stream(query)
+ O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=out_dtype)
+ lib.sage_pvi8(
+ stream,
+ query.data_ptr(),
+ key.data_ptr(),
+ value.data_ptr(),
+ O.data_ptr(),
+ attn_mask.data_ptr() if attn_mask is not None else 0,
+ quant_block_size,
+ qscale.data_ptr(),
+ kscale.data_ptr(),
+ vscale.data_ptr(),
+ cvt_dtype(query.dtype),
+ cvt_dtype(key.dtype),
+ cvt_dtype(O.dtype),
+ B,
+ Hq,
+ Hkv,
+ Sq,
+ Skv,
+ D,
+ float(scale) if scale is not None else 1.0 / (D**0.5),
bool(is_causal),
)
return O
@@ -527,7 +611,7 @@ class ARK:
- value: [B, Hkv, Skv, D]
Args:
- - scale: Attention scale. Uses 1.0 when None.
+ - scale: Attention scale. Uses 1 / sqrt(D) when None.
- quant_block_size: Quantization block size used by the kernel.
Returns:
@@ -590,7 +674,86 @@ class ARK:
Sq,
Skv,
D,
- float(scale) if scale is not None else 1.0,
+ float(scale) if scale is not None else 1.0 / (D**0.5),
+ bool(is_causal),
+ )
+ return O
+
+ def sagev1_pvi8(
+ self,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attn_mask: torch.Tensor | None = None,
+ dropout_p: float = 0.0,
+ is_causal: bool = False,
+ scale: float | None = None,
+ enable_gqa: bool = False,
+ quant_block_size: int = 64,
+ ) -> torch.Tensor:
+ """SAGE v1 attention with PV int8 path.
+
+ Expects FP16 Q/K/V input and quantizes Q/K/V internally before calling
+ the PV int8 kernel.
+ """
+ if quant_block_size <= 0:
+ return self.sdpa(
+ query=query,
+ key=key,
+ value=value,
+ attn_mask=attn_mask,
+ dropout_p=dropout_p,
+ is_causal=is_causal,
+ scale=scale,
+ )
+ if query.device.type != "xpu":
+ raise NotImplementedError("sagev1_pvi8 is only supported on XPU")
+ if query.dtype not in (torch.float16,):
+ raise ValueError(f"Q must be float16, got {query.dtype}")
+ if key.dtype != query.dtype or value.dtype != query.dtype:
+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")
+
+ if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
+ raise ValueError("Q/K/V must be 4D tensors")
+
+ if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():
+ raise ValueError("Q/K/V must be contiguous")
+
+ B, Hq, Sq, D = query.shape
+ Bk, Hkv, Skv, Dk = key.shape
+ Bv, Hkv2, Skv2, Dv = value.shape
+
+ if Bk != B or Bv != B:
+ raise ValueError("Batch size mismatch between Q/K/V")
+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:
+ raise ValueError("K/V shape mismatch")
+ if Dk != D:
+ raise ValueError("Head dim mismatch between Q and K/V")
+ if D not in (64, 128):
+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")
+
+ lib = self.get_lib(query)
+ stream = get_stream(query)
+ O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)
+ lib.sagev1_pvi8(
+ stream,
+ query.data_ptr(),
+ key.data_ptr(),
+ value.data_ptr(),
+ O.data_ptr(),
+ attn_mask.data_ptr() if attn_mask is not None else 0,
+ quant_block_size,
+ cvt_dtype(query.dtype),
+ cvt_dtype(key.dtype),
+ cvt_dtype(value.dtype),
+ cvt_dtype(O.dtype),
+ B,
+ Hq,
+ Hkv,
+ Sq,
+ Skv,
+ D,
+ float(scale) if scale is not None else 1.0 / (D**0.5),
bool(is_causal),
)
return O
@@ -636,7 +799,7 @@ class ARK:
# block_size=0 means per-token
block_size = quant_block_size if quant_block_size > 0 else 1
- # SAGE V1 kernel uses K-tile size=32; scale_block_size must be 1 or >=32
+ # SAGE V1 kernel uses K-tile size=32; quant_block_size must be 1 or >=32
if block_size != 1 and block_size < 32:
raise ValueError(
f"quant_block_size={block_size} is not supported. "
@@ -700,7 +863,7 @@ class ARK:
)
k_scale = k_scale.reshape(B, Hkv, Skv_pad // block_size, 1)
- # Call SAGE v1 with matching scale_block_size
+ # Call SAGE v1 with matching quant_block_size
out = self.sage(
q_i8,
k_i8,
@@ -709,7 +872,7 @@ class ARK:
is_causal=is_causal,
scale=scale,
enable_gqa=enable_gqa,
- scale_block_size=block_size,
+ quant_block_size=block_size,
qscale=q_scale,
kscale=k_scale,
)
diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cpp
index 86d983f90..b623ed437 100755
--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp
@@ -114,9 +114,10 @@ static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_
batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
}
-static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
- int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q,
- int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+static void sagev1_impl(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
+ int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch,
+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
+ float softmax_scale, bool is_causal, bool use_int8_pv) {
if (mask && is_causal) {
throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");
}
@@ -127,13 +128,34 @@ static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torc
throw std::invalid_argument("ark::sagev1: only F16 is supported for q_dtype");
}
#ifdef ARK_XPU
- XpuWrapper::sagev1((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, scale_block_size, batch,
- num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ if (use_int8_pv) {
+ XpuWrapper::sagev1_pvi8((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,
+ scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
+ softmax_scale, is_causal);
+ } else {
+ XpuWrapper::sagev1((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, scale_block_size,
+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ }
#else
throw std::runtime_error("ark::sagev1 is only supported on XPU");
#endif
}
+static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
+ int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q,
+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+ sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch, num_heads_q,
+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, false);
+}
+
+static void sagev1_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
+ int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch,
+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
+ float softmax_scale, bool is_causal) {
+ sagev1_impl(stream, Q, K, V, O, mask, scale_block_size, q_dtype, k_dtype, v_dtype, o_dtype, batch, num_heads_q,
+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal, true);
+}
+
static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
int scale_block_size, torch_ptr qscale, torch_ptr kscale, int q_dtype, int k_dtype, int o_dtype,
int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,
@@ -146,6 +168,18 @@ static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_
seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
}
+static void sage_pvi8(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,
+ int scale_block_size, torch_ptr qscale, torch_ptr kscale, torch_ptr vscale, int q_dtype,
+ int k_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+ if (mask && is_causal) {
+ throw std::invalid_argument("ark::sage_pvi8: mask and is_causal cannot both be set");
+ }
+ ark::sdpa_impl_qks8_pvi8((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,
+ scale_block_size, (void*)qscale, (void*)kscale, (void*)vscale, batch, num_heads_q,
+ num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+}
+
static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr weights, torch_ptr scales,
torch_ptr outputs, int dtype, int N, int K, torch_ptr num_tokens_per_expert,
int num_experts) {
@@ -289,7 +323,11 @@ PYBIND11_MODULE(PY_NAME, m) {
#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)
m.def("sdpa", &ark::sdpa);
m.def("sagev1", &ark::sagev1);
+ // High-level SAGEV1 PVi8 API: input Q/K/V are FP16 and quantized internally.
+ m.def("sagev1_pvi8", &ark::sagev1_pvi8);
m.def("sage", &ark::sage);
+ // Low-level SAGE PVi8 API: input Q/K/V are pre-quantized int8 with qscale/kscale/vscale.
+ m.def("sage_pvi8", &ark::sage_pvi8);
m.def("sage_dynamic_quant", &ark::sage_dynamic_quant);
m.def("moe_gemm", &ark::moe_gemm_wrapper);
#endif
diff --git a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
index 990c1c9d4..f5e30e7aa 100644
--- a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp
@@ -36,10 +36,39 @@ int launch_prefill_kernel_f16_128_sage(detail::Options const& options) {
return launch_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::half_t>(options);
}
+int launch_prefill_kernel_f16_128_sage_i8pv(detail::Options const& options) {
+ return launch_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::int8_t, cute::half_t, true, true, true>(
+ options);
+}
+
int launch_prefill_kernel_f16_64_sage(detail::Options const& options) {
return launch_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::half_t>(options);
}
+int launch_prefill_kernel_f16_64_sage_i8pv(detail::Options const& options) {
+ return launch_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::int8_t, cute::half_t, true, true, true>(
+ options);
+}
+
+KernelLauncher select_sage_prefill_launcher(BTLA_DTYPE dtype, int head_dim, bool use_int8_pv) {
+ switch (dtype) {
+ case BTLA_DTYPE::S8:
+ switch (head_dim) {
+ case 128:
+ if (!use_int8_pv) return launch_prefill_kernel_f16_128_sage;
+ return launch_prefill_kernel_f16_128_sage_i8pv;
+ case 64:
+ if (!use_int8_pv) return launch_prefill_kernel_f16_64_sage;
+ return launch_prefill_kernel_f16_64_sage_i8pv;
+ default:
+ return nullptr;
+ }
+
+ default:
+ return nullptr;
+ }
+}
+
KernelLauncher select_sage_prefill_launcher(BTLA_DTYPE dtype, int head_dim) {
switch (dtype) {
case BTLA_DTYPE::S8:
@@ -148,16 +177,18 @@ detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void*
} // namespace
void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,
- void* kscale, BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
+ void* kscale, void* vscale, bool use_int8_pv,
+ BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q, num_heads_kv,
seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
options.scale_block_size = scale_block_size;
options.qscale = qscale;
options.kscale = kscale;
+ options.vscale = vscale;
compat::set_default_queue(*q);
- KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, head_dim);
+ KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, head_dim, use_int8_pv);
if (launcher == nullptr) {
throw std::runtime_error(
"Unsupported dtype or head dimension for sage_prefill / SAGE (only F16/BF16 and 64/96/128/192 are supported)");
@@ -238,10 +269,36 @@ void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr
}
- sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q,
+ sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, nullptr, false,
+ BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q,
seq_len_kv, head_dim, softmax_scale, is_causal);
}
+void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
+ int scale_block_size, void* qscale, void* kscale, void* vscale, int batch, int num_heads_q,
+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,
+ bool is_causal) {
+ if (mask && is_causal) {
+ throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");
+ }
+
+ if (seq_len_q <= 0 || seq_len_kv <= 0) {
+ throw std::invalid_argument("sdpa_impl: seq_len_q and seq_len_kv must be greater than 0");
+ }
+ if (vscale == nullptr) {
+ throw std::invalid_argument("sdpa_impl_qks8_pvi8: vscale must be provided for int8 PV");
+ }
+ if (seq_len_q == 1) {
+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, BTLA_DTYPE::F16, batch, num_heads_q, num_heads_kv,
+ seq_len_kv, head_dim, softmax_scale, is_causal);
+ return;
+ }
+
+ sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, vscale, true,
+ BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
+ is_causal);
+}
+
} // namespace ark
#endif // ARK_XPU && ARK_SYCL_TLA
diff --git a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py
index 7b51f96db..aa9f2ac30 100644
--- a/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py
+++ b/auto_round_extension/ark/auto_round_kernel/torch_sdpa_patch.py
@@ -33,10 +33,10 @@ def _log_fallback_warning_once(backend: str, error: Exception) -> None:
def _validate_backend(backend: str, quant_block_size: int) -> str:
backend = backend.lower()
- if backend not in {"sdpa", "sagev1"}:
+ if backend not in {"sdpa", "sagev1", "sagev1_pvi8"}:
raise ValueError(f"Unsupported ARK attention backend: {backend}")
- if backend == "sagev1" and quant_block_size <= 0:
- raise ValueError("quant_block_size must be > 0 when backend='sagev1'")
+ if backend in {"sagev1", "sagev1_pvi8"} and quant_block_size <= 0:
+ raise ValueError(f"quant_block_size must be > 0 when backend='{backend}'")
return backend
@@ -113,7 +113,7 @@ def _can_use_ark_attention(
128,
192,
)
- if backend == "sagev1":
+ if backend in {"sagev1", "sagev1_pvi8"}:
return query.dtype == torch.float16 and query.shape[-1] in (64, 128)
return True
@@ -195,7 +195,18 @@ def patch_torch_sdpa_with_ark(*, strict: bool = False, backend: str = "sdpa", qu
is_causal=is_causal,
scale=scale,
)
- return ark.sagev1(
+ if backend == "sagev1":
+ return ark.sagev1(
+ query.contiguous(),
+ key.contiguous(),
+ value.contiguous(),
+ attn_mask=normalized_mask,
+ dropout_p=dropout_p,
+ is_causal=is_causal,
+ scale=scale,
+ quant_block_size=quant_block_size,
+ )
+ return ark.sagev1_pvi8(
query.contiguous(),
key.contiguous(),
value.contiguous(),
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp
index ff0d4bee3..f80f08cc9 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp
@@ -269,6 +269,9 @@ class XeSageFwdKernel {
auto scaleK = params.mainloop.scale_block_size
? (float*)params.mainloop.kscale + (idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad)
: nullptr; // per head kscale
+ auto scaleV = params.mainloop.scale_block_size && params.mainloop.vscale
+ ? (float*)params.mainloop.vscale + (idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad)
+ : nullptr; // per head vscale
auto ptrO = p.O + offset_o;
auto stride_q = is_var_len ? cutlass::make_cute_packed_stride(StrideQ{}, shape_Q) : p.dQ;
@@ -293,7 +296,8 @@ class XeSageFwdKernel {
int l_coord = is_var_len ? 0 : idx_b;
CollectiveMainloop mainloop(params.mainloop, shared_storage.mainloop);
mainloop(Q(_, _, head_q, l_coord), K(_, _, head, l_coord), V(_, _, head, l_coord), tArA, tA_max, tA_sum, blk_qv,
- 0, k_blocks, k_blocks, thr_id, seq_len, seq_len_kv_cache, idx_b, scaleQ, scaleK, full_tile_offset,
+ 0, k_blocks, k_blocks, thr_id, seq_len, seq_len_kv_cache, idx_b, scaleQ, scaleK, scaleV,
+ full_tile_offset,
discard_seq_coord, K_cache(_, _, head, l_coord), V_cache(_, _, head, l_coord));
if constexpr (!is_empty_v<MainloopSharedStorage> && !is_empty_v<EpilogueSharedStorage>) {
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp
index d4a6eaa5a..0762423ff 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp
@@ -42,6 +42,7 @@
#include "cute/atom/mma_atom.hpp"
#include "flash_attention_v2/collective/fmha_fusion.hpp"
#include <cmath>
+#include <type_traits>
#include <cute/util/xe_split_barrier.hpp>
namespace cutlass::sage {
@@ -56,7 +57,8 @@ using namespace cute;
/////////////////////////////////////////////////////////////////////////////////////////////////
-template <class DispatchPolicy_, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_,
+template <class DispatchPolicy_, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_, bool UseInt8PV_,
+ bool WriteBackInt8PV_, bool ExecuteInt8PV_,
class TiledMMAQK_, // Tiling for Q*K GEMM
class TiledMMAPV_, // Tiling for P*V GEMM
int VTiles_, // # of tiles in V dimension
@@ -73,13 +75,15 @@ struct SAGEV1FwdMainloop {
/////////////////////////////////////////////////////////////////////////////////////////////////
-template <int Stages, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_, class TiledMMAQK_,
+template <int Stages, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_, bool UseInt8PV_,
+ bool WriteBackInt8PV_, bool ExecuteInt8PV_, class TiledMMAQK_,
class TiledMMAPV_, int VTiles_, class TensorQ_, class TensorK_, class TensorV_, class TensorK_cache_,
class TensorV_cache_, class TiledCopyQ_, class TiledCopyK_, class TiledCopyV_, class TiledCopyK_cache_,
class TiledCopyV_cache_>
-struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV_, PagedKV_, TiledMMAQK_, TiledMMAPV_,
- VTiles_, TensorQ_, TensorK_, TensorV_, TensorK_cache_, TensorV_cache_, TiledCopyQ_,
- TiledCopyK_, TiledCopyV_, TiledCopyK_cache_, TiledCopyV_cache_> {
+struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV_, PagedKV_, UseInt8PV_,
+ WriteBackInt8PV_, ExecuteInt8PV_, TiledMMAQK_, TiledMMAPV_, VTiles_, TensorQ_, TensorK_,
+ TensorV_, TensorK_cache_, TensorV_cache_, TiledCopyQ_, TiledCopyK_, TiledCopyV_,
+ TiledCopyK_cache_, TiledCopyV_cache_> {
//
// Type Aliases
//
@@ -89,6 +93,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
using TileShapePV = decltype(TiledMMAPV{}.tile_mnk());
static constexpr int VTiles = VTiles_;
using SubgroupLayoutQK = decltype(TiledMMAQK{}.get_atom_layout_mnk());
+ using SubgroupLayoutPV = decltype(TiledMMAPV{}.get_atom_layout_mnk());
using SGPerWG = decltype(product(take<1, 4>(shape(typename TiledMMAQK::ThrLayoutVMNK{}))));
using TensorQ = TensorQ_;
@@ -141,14 +146,29 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
decltype(FragS{}.tv_layout()){}));
using FragPRow = decltype(reduce<1>(FragP{}, sycl::plus<void>{}));
- using SingleFragA = FragC<TiledMMAPV>; // (atom val,q',v')
+ using SingleFragPV = FragC<TiledMMAPV>; // (atom val,q',v')
+ using SingleFragAFloat = std::remove_reference_t<decltype(make_subgroup_tensor(
+ make_fragment_like<float>(typename SingleFragPV::Base{}), decltype(SingleFragPV{}.tv_layout()){}))>;
+ using SingleFragA = conditional_t<UseInt8PV_, SingleFragAFloat, SingleFragPV>;
using FragA = expand_sg_fragment_t<SingleFragA, 1, VTiles>; // (atom val,q',v',VV)
using FragARow = decltype(reduce<1>(FragA{}, sycl::plus<void>{}));
- using ElementA = typename TiledMMAPV::ValTypeD;
+ using ElementA = typename SingleFragA::value_type;
+
+ static constexpr int SGTilePVQ = get<0>(shape_div(TileShapePV{}, shape(SubgroupLayoutPV{})))();
+ using QuantizedPVOperation = XE_DPAS_TT<cute::gcd(SGTilePVQ, 8), int32_t, int8_t, int8_t>;
+ using QuantizedTiledMMAPV =
+ typename TiledMMAHelper<MMA_Atom<QuantizedPVOperation>, Layout<TileShapePV>, SubgroupLayoutPV>::TiledMMA;
+ using QuantizedSingleFragA = FragC<QuantizedTiledMMAPV>;
+
+ static_assert(size(SingleFragA{}.shape()) == size(QuantizedSingleFragA{}.shape()),
+ "Quantized PV accumulator must match float PV tile size");
static constexpr bool CausalMask = CausalMask_;
static constexpr bool CachedKV = CachedKV_;
static constexpr bool PagedKV = PagedKV_;
+ static constexpr bool UseInt8PV = UseInt8PV_;
+ static constexpr bool WriteBackInt8PV = WriteBackInt8PV_;
+ static constexpr bool ExecuteInt8PV = ExecuteInt8PV_;
// User-facing arguments
struct Arguments {
@@ -157,6 +177,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
int scale_block_size = 0; // if non-zero, apply scaling in blocks of this size (for block-sparse attention)
float const* qscale = nullptr;
float const* kscale = nullptr;
+ float const* vscale = nullptr;
int const* ptr_page_table = nullptr;
int page_size = 0;
int const* num_pages_per_seq = nullptr;
@@ -179,12 +200,38 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
static constexpr Params to_underlying_arguments(Arguments const& args, void* /* workspace */) {
constexpr double kLog2e = 1.4426950408889634074; // log_2(e)
float val = args.scale * static_cast<float>(kLog2e);
- return Params{val, args.mask, args.scale_block_size, args.qscale,
- args.kscale, args.ptr_page_table, args.page_size, args.num_pages_per_seq};
+ return Params{val, args.mask, args.scale_block_size, args.qscale, args.kscale, args.vscale,
+ args.ptr_page_table, args.page_size, args.num_pages_per_seq};
}
CUTLASS_HOST_DEVICE static bool can_implement(Arguments const&) { return true; }
+ // DPAS kernels run as SIMD16, so pack two SIMD16 vectors into one temporary SIMD32 vector
+ // before issuing a single vISA SIMD32 instruction.
+ CUTLASS_DEVICE static void exp2_pair_simd32_asm(ElementP x0, ElementP x1, ElementP& y0, ElementP& y1) {
+#if defined(__SYCL_DEVICE_ONLY__) && defined(SYCL_INTEL_TARGET)
+ asm(
+ "{\n"
+ ".decl OUT0 v_type=G type=F num_elts=16 alias=<%0,0>\n"
+ ".decl OUT1 v_type=G type=F num_elts=16 alias=<%1,0>\n"
+ ".decl IN0 v_type=G type=F num_elts=16 alias=<%2,0>\n"
+ ".decl IN1 v_type=G type=F num_elts=16 alias=<%3,0>\n"
+ ".decl TMP_IN v_type=G type=F num_elts=32 align=64\n"
+ ".decl TMP_OUT v_type=G type=F num_elts=32 align=64\n"
+ "mov (M1_NM, 16) TMP_IN(0,0)<1> IN0(0,0)<1;1,0>\n"
+ "mov (M1_NM, 16) TMP_IN(1,0)<1> IN1(0,0)<1;1,0>\n"
+ "exp (M1_NM, 32) TMP_OUT(0,0)<1> TMP_IN(0,0)<1;1,0>\n"
+ "mov (M1_NM, 16) OUT0(0,0)<1> TMP_OUT(0,0)<1;1,0>\n"
+ "mov (M1_NM, 16) OUT1(0,0)<1> TMP_OUT(1,0)<1;1,0>\n"
+ "}\n"
+ : "=rw"(y0), "=rw"(y1)
+ : "rw"(x0), "rw"(x1));
+#else
+ y0 = sycl::native::exp2(x0);
+ y1 = sycl::native::exp2(x1);
+#endif
+ }
+
CUTLASS_DEVICE
int get_physical_k_tile(int K, int l_coord, int seq_len_kv_cache) {
int next_page_logical_idx = K * get<1>(TileShapeQK{}) / params.page_size;
@@ -209,7 +256,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
int blk_k1,
int total_blk, // Total # of K blocks
int thr_id, int seq_len, int seq_len_kv_cache, int l_coord, float* scaleQ,
- float* scaleK, int full_tile_offset, int discard_seq_coord,
+ float* scaleK, float* scaleV, int full_tile_offset, int discard_seq_coord,
TensorK_cache2D const& K_cache_2D = TensorK_cache2D{},
TensorV_cache2D const& V_cache_2D = TensorV_cache2D{}) {
using namespace sycl::ext::oneapi::this_work_item;
@@ -253,6 +300,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
/* Create MMAs */
TiledMMAQK mma_qk{};
TiledMMAPV mma_pv{};
+ QuantizedTiledMMAPV mma_pv_i8{};
/* Slice TiledCopy/TiledMMA operations down to to work-item level */
auto thr_copy_q = copy_q.get_slice(thr_id);
@@ -262,6 +310,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
auto thr_copy_v_cache = copy_v_cache.get_slice(thr_id);
auto thr_mma_qk = mma_qk.get_slice(thr_id);
auto thr_mma_pv = mma_pv.get_slice(thr_id);
+ auto thr_mma_pv_i8 = mma_pv_i8.get_slice(thr_id);
/* Partition coordinate tensors for copy */
auto tQgQ = thr_copy_q.partition_S(gQ); // (atom_val,q',d',D)
@@ -282,6 +331,7 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
FragP tPrS = make_subgroup_tensor(make_fragment_like<ElementP>(tSrS.tensor()), tSrS.tv_layout());
auto tArP = thr_mma_pv.partition_sg_fragment_A(cP);
+ auto tArP_i8 = thr_mma_pv_i8.partition_sg_fragment_A(cP);
auto tVrV = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));
auto tArV = thr_mma_pv.partition_sg_fragment_B(gV_split(_, _, 0, 0));
@@ -467,20 +517,57 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
/* Apply softmax and scaling (tA rescaling fused into GEMM2 VTile loop) */
auto rescale = softmax(K == blk_k0, tPrS, tA_max, tA_sum);
- reorder(tPrS, tArP);
+ if constexpr (!UseInt8PV) {
+ reorder(tPrS, tArP);
+ } else {
+ reorder(tPrS, tArP_i8);
+ }
/* GEMM 2: A += P * V, split in v dimension.
tArA rescaling is fused to per-VTile */
- CUTLASS_PRAGMA_UNROLL
- for (int VV = 0; VV < VTiles; VV++) {
- copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV);
- reorder(tVrV, tArV);
- if (K != blk_k0) {
- CUTLASS_PRAGMA_UNROLL
- for (int i = 0; i < tArA.size() / VTiles; i++) tArA(_, _, _, VV)(i) *= broadcast<0>(rescale, tArA, i);
+ if constexpr (!UseInt8PV) {
+ CUTLASS_PRAGMA_UNROLL
+ for (int VV = 0; VV < VTiles; VV++) {
+ copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV);
+ reorder(tVrV, tArV);
+ if (K != blk_k0) {
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 0; i < tArA.size() / VTiles; i++) tArA(_, _, _, VV)(i) *= broadcast<0>(rescale, tArA, i);
+ }
+
+ cute::gemm(mma_pv, tArP, tArV, tArA(_, _, _, VV));
}
+ } else {
+ ElementA pv_dequant_scale = ElementA(0.0f);
+ if constexpr (WriteBackInt8PV) {
+ pv_dequant_scale = ElementA(scaleV[scalek_idx]) * (ElementA(1.0f) / ElementA(127.0f));
+ }
+ CUTLASS_PRAGMA_UNROLL
+ for (int VV = 0; VV < VTiles; VV++) {
+ auto tArA_v = tArA(_, _, _, VV);
+ if (K != blk_k0) {
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 0; i < tArA.size() / VTiles; i++) tArA_v(i) *= broadcast<0>(rescale, tArA, i);
+ }
- cute::gemm(mma_pv, tArP, tArV, tArA(_, _, _, VV));
+ auto tArAcc = thr_mma_pv_i8.partition_sg_fragment_C(make_identity_tensor(select<0, 1>(TileShapePV{})));
+ auto tVrV_i8 = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));
+ auto tArV_i8 = thr_mma_pv_i8.partition_sg_fragment_B(gV_split(_, _, 0, 0));
+ copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV_i8);
+ reorder(tVrV_i8, tArV_i8);
+ clear(tArAcc);
+
+ if constexpr (ExecuteInt8PV) {
+ cute::gemm(mma_pv_i8, tArP_i8, tArV_i8, tArAcc);
+ }
+
+ if constexpr (WriteBackInt8PV) {
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 0; i < tArAcc.size(); ++i) {
+ tArA_v(i) += ElementA(tArAcc(i)) * pv_dequant_scale;
+ }
+ }
+ }
}
/* K prefetch */
@@ -525,11 +612,10 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
FragARow softmax(bool first_block, // First softmax block?
FragP& tS, // Softmax src/dst block
FragARow& tS_max, // Softmax row-wise max accumulator
- FragARow& tS_sum) { // Softmax row-wise sum accumulator
- /* Compute row-wise maxima for this block */
+ FragARow& tS_sum) { // Softmax row-wise sum accumulator
+ /* Compute row-wise maxima for this block */
auto tS_bmax = reduce<1>(tS, sycl::maximum{});
-
FragARow rescale;
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < tS_max.size(); i++) {
@@ -539,8 +625,20 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
}
/* Scale S and subtract maxima, then exponentiate */
+ static_assert(FragP{}.size() % 2 == 0, "FragP size must be even for pairwise SIMD32 exp.");
CUTLASS_PRAGMA_UNROLL
- for (int i = 0; i < tS.size(); i++) tS(i) = sycl::native::exp2(tS(i) - broadcast<0>(tS_max, tS, i));
+ for (int i = 0; i < tS.size(); i += 2) {
+ ElementP x0 = tS(i) - broadcast<0>(tS_max, tS, i);
+ ElementP x1 = tS(i + 1) - broadcast<0>(tS_max, tS, i + 1);
+ exp2_pair_simd32_asm(x0, x1, tS(i), tS(i + 1));
+ }
+
+ if constexpr (UseInt8PV) {
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 0; i < tS.size(); i++) {
+ tS(i) *= ElementP(127.0f);
+ }
+ }
/* Rescale existing S sums */
if (!first_block) {
@@ -552,8 +650,14 @@ struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, Cached
/* Update sums */
auto tS_bsum = reduce<1>(tS, sycl::plus<void>{});
- CUTLASS_PRAGMA_UNROLL
- for (int i = 0; i < tS_sum.size(); i++) tS_sum(i) += tS_bsum(i);
+ if constexpr (UseInt8PV) {
+ constexpr ElementA kInvPQuantScale = ElementA(1.0f / 127.0f);
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 0; i < tS_sum.size(); i++) tS_sum(i) += ElementA(tS_bsum(i)) * kInvPQuantScale;
+ } else {
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 0; i < tS_sum.size(); i++) tS_sum(i) += tS_bsum(i);
+ }
return rescale;
}
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp
index 37752f79a..62850e46b 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp
@@ -64,6 +64,11 @@ void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_pt
void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,
void* kscale, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
int seq_len_kv, int head_dim, float softmax_scale, bool is_causal);
+
+void sdpa_impl_qks8_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
+ int scale_block_size, void* qscale, void* kscale, void* vscale, int batch, int num_heads_q,
+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale,
+ bool is_causal);
#endif // ARK_SYCL_TLA
} // namespace ark
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp
index 9aaac19f3..6c4cdb006 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp
@@ -68,7 +68,7 @@ struct Options {
void* mask = nullptr;
void* o = nullptr;
int scale_block_size = 0;
- const void *qscale = nullptr, *kscale = nullptr;
+ const void *qscale = nullptr, *kscale = nullptr, *vscale = nullptr;
const void *block_K = nullptr, *block_V = nullptr;
const int *page_table = nullptr, *num_pages_per_seq = nullptr;
bool is_causal = false;
@@ -609,6 +609,7 @@ struct SageKernelRunner {
},
{options.softmax_scale, static_cast<float*>(options.mask), options.scale_block_size,
static_cast<const float*>(options.qscale), static_cast<const float*>(options.kscale),
+ static_cast<const float*>(options.vscale),
options.use_paged_kv ? options.page_table : nullptr, options.use_paged_kv ? options.page_size : 0,
options.use_paged_kv ? options.num_pages_per_seq : nullptr},
{},
@@ -635,7 +636,8 @@ struct SageKernelRunner {
}
};
-template <bool Causal, typename TileShapeQK, typename TileShapePV, typename TileShapeOutput, typename SubgroupLayoutQK,
+template <bool Causal, bool UseInt8PV, bool WriteBackInt8PV, bool ExecuteInt8PV, typename TileShapeQK,
+ typename TileShapePV, typename TileShapeOutput, typename SubgroupLayoutQK,
typename SubgroupLayoutPV_, /* void -> default */
int PipelineStages, bool persistent, typename ElementQ = bfloat16_t, typename ElementK = bfloat16_t,
typename ElementV = bfloat16_t, typename ElementO = ElementQ,
@@ -693,7 +695,8 @@ struct SageConfig {
if constexpr (Causal) {
using CollectiveMainloop =
cutlass::fmha::collective::SAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,
- TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,
+ UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,
+ TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,
TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,
GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;
@@ -712,7 +715,8 @@ struct SageConfig {
if (options.mask) {
using CollectiveMainloop =
cutlass::fmha::collective::SAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, true, CachedKV, PagedKV,
- TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,
+ UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,
+ TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,
TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,
GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;
@@ -730,7 +734,8 @@ struct SageConfig {
} else {
using CollectiveMainloop =
cutlass::fmha::collective::SAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,
- TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,
+ UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, TiledMMAQK,
+ TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,
TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,
GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;
@@ -803,7 +808,8 @@ inline int launch_prefill_kernel_128(Options const& options) {
/*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);
}
-template <typename ElementQ, typename ElementK, typename ElementV>
+template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV, bool UseInt8PV = false,
+ bool WriteBackInt8PV = true, bool ExecuteInt8PV = true>
inline int launch_sage_prefill_kernel_128(Options const& options) {
constexpr int PipelineStages = 1;
constexpr int PipelineStages1 = 1;
@@ -815,13 +821,16 @@ inline int launch_sage_prefill_kernel_128(Options const& options) {
using ShapePV1 = Shape<_256, _32, _64>;
using ShapeOut1 = Shape<_256, _128>;
using SubgroupLayoutQK1 = Layout<Shape<_16, _1, _1>>;
- return options.is_causal ? SageConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,
- /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options)
- : SageConfig<false, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void, PipelineStages1,
- /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options);
+ return options.is_causal ? SageConfig<true, UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, ShapeQK, ShapePV, ShapeOut,
+ SubgroupLayoutQK, void, PipelineStages,
+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementO>::run(options)
+ : SageConfig<false, UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, ShapeQK1, ShapePV1,
+ ShapeOut1, SubgroupLayoutQK1, void, PipelineStages1,
+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementO>::run(options);
}
-template <typename ElementQ, typename ElementK, typename ElementV>
+template <typename ElementQ, typename ElementK, typename ElementV, typename ElementO = ElementV, bool UseInt8PV = false,
+ bool WriteBackInt8PV = true, bool ExecuteInt8PV = true>
inline int launch_sage_prefill_kernel_64(Options const& options) {
constexpr int PipelineStages = 2;
constexpr int PipelineStages1 = 2;
@@ -831,10 +840,12 @@ inline int launch_sage_prefill_kernel_64(Options const& options) {
using SubgroupLayoutQK = Layout<Shape<_8, _1, _1>>;
using SubgroupLayoutPV = void;
return options.is_causal
- ? SageConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages,
- /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options)
- : SageConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages1,
- /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options);
+ ? SageConfig<true, UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, ShapeQK, ShapePV, ShapeOut,
+ SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages,
+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementO>::run(options)
+ : SageConfig<false, UseInt8PV, WriteBackInt8PV, ExecuteInt8PV, ShapeQK, ShapePV, ShapeOut,
+ SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages1,
+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementO>::run(options);
}
template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
index a7dec5b4b..13a3ef875 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp
@@ -1041,9 +1041,9 @@ class XpuWrapper {
}
}
- static void sagev1(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
- int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
- int head_dim, float softmax_scale, bool is_causal) {
+ static void sagev1_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
+ int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal, bool use_int8_pv) {
bool use_mean_bias = env_params::Instance()->sage_use_mean_bias != 0;
size_t seq_q_blk = (seq_len_q + scale_block_size - 1) / scale_block_size;
size_t seq_kv_blk = (seq_len_kv + scale_block_size - 1) / scale_block_size;
@@ -1051,16 +1051,25 @@ class XpuWrapper {
size_t q_scale_size = num_heads_q * seq_q_blk * batch;
size_t k_size = num_heads_kv * seq_len_kv * head_dim * batch;
size_t k_scale_size = num_heads_kv * seq_kv_blk * batch;
+ size_t v_tmp_size = use_int8_pv ? k_size : 0;
+ size_t v_scale_size = use_int8_pv ? k_scale_size * sizeof(float) : 0;
size_t k_bias_size = use_mean_bias ? num_heads_kv * head_dim * batch * sizeof(sycl::half) : 0;
- size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + k_bias_size;
+ size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float) + v_tmp_size +
+ v_scale_size + k_bias_size;
auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);
auto q_out_ptr = (int8_t*)ptr;
auto k_out_ptr = (int8_t*)ptr + q_size;
auto qscale = (float*)((int8_t*)ptr + q_size + k_size);
auto kscale = (float*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float));
+ auto v_out_ptr = use_int8_pv ? (int8_t*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) +
+ k_scale_size * sizeof(float))
+ : nullptr;
+ auto vscale = use_int8_pv ? (float*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) +
+ k_scale_size * sizeof(float) + k_size)
+ : nullptr;
auto kbias = use_mean_bias
? (sycl::half*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float) +
- k_scale_size * sizeof(float))
+ k_scale_size * sizeof(float) + v_tmp_size + v_scale_size)
: nullptr;
sage_dynamic_quant<sycl::half>(q, (sycl::half*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch * num_heads_q,
seq_len_q, seq_q_blk, head_dim, scale_block_size);
@@ -1072,8 +1081,31 @@ class XpuWrapper {
}
sage_dynamic_quant<sycl::half>(q, (sycl::half*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,
seq_len_kv, seq_kv_blk, head_dim, scale_block_size, kbias);
- ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, batch,
- num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);
+ if (use_int8_pv) {
+ sage_dynamic_quant<sycl::half>(q, (sycl::half*)V_ptr, (int8_t*)v_out_ptr, (float*)vscale, batch * num_heads_kv,
+ seq_len_kv, seq_kv_blk, head_dim, scale_block_size);
+ ark::sdpa_impl_qks8_pvi8(q, q_out_ptr, k_out_ptr, v_out_ptr, O_ptr, mask, scale_block_size, qscale, kscale,
+ vscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim,
+ softmax_scale, is_causal);
+ } else {
+ ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale,
+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale,
+ is_causal);
+ }
+ }
+
+ static void sagev1(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
+ int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,
+ int head_dim, float softmax_scale, bool is_causal) {
+ sagev1_impl(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q,
+ seq_len_kv, head_dim, softmax_scale, is_causal, false);
+ }
+
+ static void sagev1_pvi8(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,
+ int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,
+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {
+ sagev1_impl(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, batch, num_heads_q, num_heads_kv, seq_len_q,
+ seq_len_kv, head_dim, softmax_scale, is_causal, true);
}
};
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
index b2ee33c4d..a42dfc3b9 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_sdpa.hpp
@@ -20,7 +20,9 @@ struct TestSDPA {
#if defined(ARK_XPU) && defined(ARK_SYCL_TLA) && ARK_XPU && ARK_SYCL_TLA
// test_sagev1_accuracy("prefill_gqa_128_bias", 1, 32, 8, 256, 512, 128, 128, false, true);
// test_sagev1_accuracy("prefill_gqa_128_no_bias", 1, 32, 8, 256, 512, 128, 128, false, false);
- benchmark_sagev1("bench_prefill_gqa_128", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);
+ test_sagev1_accuracy("prefill_gqa_128_causal_no_bias", 1, 32, 8, 256, 512, 128, 128, true, false);
+ benchmark_sagev1("bench_prefill_gqa_128_s4096", 1, 32, 8, 4096, 4096, 128, 128, false, 5, 20);
+ benchmark_sagev1("bench_bmg_hq96_d128_k8192", 1, 96, 8, 4096, 8192, 128, 128, false, 5, 20);
#else
std::cout << "[sagev1] skipped: requires ARK_XPU and ARK_SYCL_TLA\n";
#endif
@@ -191,6 +193,9 @@ struct TestSDPA {
size_t q_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;
size_t k_count = size_t(batch) * num_heads_kv * seq_len_kv * head_dim;
size_t o_count = size_t(batch) * num_heads_q * seq_len_q * head_dim;
+ size_t q_scale_count = size_t(batch) * num_heads_q * ((seq_len_q + scale_block_size - 1) / scale_block_size);
+ size_t k_scale_count = size_t(batch) * num_heads_kv * ((seq_len_kv + scale_block_size - 1) / scale_block_size);
+ size_t v_scale_count = k_scale_count;
float softmax_scale = 1.0f / std::sqrt(float(head_dim));
auto host_q = to_fp16_vector(make_random_vector(q_count, -1.0f, 1.0f, 901u + uint32_t(seq_len_q)));
@@ -201,10 +206,28 @@ struct TestSDPA {
auto* dev_k = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));
auto* dev_v = reinterpret_cast<sycl::half*>(ctx->allocate(k_count * sizeof(sycl::half)));
auto* dev_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));
+ auto* dev_kernel_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));
+ auto* dev_pvi8_out = reinterpret_cast<sycl::half*>(ctx->allocate(o_count * sizeof(sycl::half)));
+ auto* dev_qi8 = reinterpret_cast<int8_t*>(ctx->allocate(q_count * sizeof(int8_t)));
+ auto* dev_ki8 = reinterpret_cast<int8_t*>(ctx->allocate(k_count * sizeof(int8_t)));
+ auto* dev_vi8 = reinterpret_cast<int8_t*>(ctx->allocate(k_count * sizeof(int8_t)));
+ auto* dev_qscale = reinterpret_cast<float*>(ctx->allocate(q_scale_count * sizeof(float)));
+ auto* dev_kscale = reinterpret_cast<float*>(ctx->allocate(k_scale_count * sizeof(float)));
+ auto* dev_vscale = reinterpret_cast<float*>(ctx->allocate(v_scale_count * sizeof(float)));
q->memcpy(dev_q, host_q.data(), q_count * sizeof(sycl::half)).wait();
q->memcpy(dev_k, host_k.data(), k_count * sizeof(sycl::half)).wait();
q->memcpy(dev_v, host_v.data(), k_count * sizeof(sycl::half)).wait();
+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_q, dev_qi8, dev_qscale, batch * num_heads_q, seq_len_q,
+ (seq_len_q + scale_block_size - 1) / scale_block_size, head_dim,
+ scale_block_size);
+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_k, dev_ki8, dev_kscale, batch * num_heads_kv, seq_len_kv,
+ (seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,
+ scale_block_size);
+ ark::XpuWrapper::sage_dynamic_quant<sycl::half>(q, dev_v, dev_vi8, dev_vscale, batch * num_heads_kv, seq_len_kv,
+ (seq_len_kv + scale_block_size - 1) / scale_block_size, head_dim,
+ scale_block_size);
+
double sage_ms = run_bench(
[&]() {
ark::XpuWrapper::sagev1(q, dev_q, dev_k, dev_v, dev_out, nullptr, scale_block_size, batch, num_heads_q,
@@ -212,18 +235,52 @@ struct TestSDPA {
},
q, warmup, iters);
+ double kernel_pvhalf_ms = run_bench(
+ [&]() {
+ ark::sdpa_impl_qks8_pvhalf(q, dev_qi8, dev_ki8, dev_v, dev_kernel_out, nullptr, scale_block_size,
+ dev_qscale, dev_kscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,
+ head_dim, softmax_scale, is_causal);
+ },
+ q, warmup, iters);
+
+ double kernel_pvi8_ms = run_bench(
+ [&]() {
+ ark::sdpa_impl_qks8_pvi8(q, dev_qi8, dev_ki8, dev_vi8, dev_pvi8_out, nullptr, scale_block_size, dev_qscale,
+ dev_kscale, dev_vscale, batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv,
+ head_dim, softmax_scale, is_causal);
+ },
+ q, warmup, iters);
+
double flops = compute_sdpa_flops(batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim);
double io_bytes = compute_sdpa_io_bytes(batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim);
double sage_tflops = flops / (sage_ms * 1e-3) / 1e12;
double sage_gbps = io_bytes / (sage_ms * 1e-3) / 1e9;
+ double kernel_pvhalf_tflops = flops / (kernel_pvhalf_ms * 1e-3) / 1e12;
+ double kernel_pvhalf_gbps = io_bytes / (kernel_pvhalf_ms * 1e-3) / 1e9;
+ double kernel_pvi8_tflops = flops / (kernel_pvi8_ms * 1e-3) / 1e12;
+ double kernel_pvi8_gbps = io_bytes / (kernel_pvi8_ms * 1e-3) / 1e9;
std::cout << std::fixed << std::setprecision(3) << "[sagev1][bench] " << name << " sage_ms=" << sage_ms
<< " sage_TFLOPS=" << sage_tflops << " sage_GBps=" << sage_gbps << "\n";
+ std::cout << std::fixed << std::setprecision(3) << "[sagev1][kernel_bench] " << name
+ << " pvhalf_ms=" << kernel_pvhalf_ms << " pvhalf_TFLOPS=" << kernel_pvhalf_tflops
+ << " pvhalf_GBps=" << kernel_pvhalf_gbps << "\n";
+ std::cout << std::fixed << std::setprecision(3) << "[sagev1][kernel_bench] " << name
+ << " pvi8_ms=" << kernel_pvi8_ms << " pvi8_TFLOPS=" << kernel_pvi8_tflops
+ << " pvi8_GBps=" << kernel_pvi8_gbps << "\n";
ctx->deallocate(dev_q);
ctx->deallocate(dev_k);
ctx->deallocate(dev_v);
ctx->deallocate(dev_out);
+ ctx->deallocate(dev_kernel_out);
+ ctx->deallocate(dev_pvi8_out);
+ ctx->deallocate(dev_qi8);
+ ctx->deallocate(dev_ki8);
+ ctx->deallocate(dev_vi8);
+ ctx->deallocate(dev_qscale);
+ ctx->deallocate(dev_kscale);
+ ctx->deallocate(dev_vscale);
}
#endif
-};
\ No newline at end of file
+};
diff --git a/auto_round_extension/ark/test/test_sage_dynquant.py b/auto_round_extension/ark/test/test_sage_dynquant.py
index 54e3e9832..37a90a0e9 100644
--- a/auto_round_extension/ark/test/test_sage_dynquant.py
+++ b/auto_round_extension/ark/test/test_sage_dynquant.py
@@ -66,6 +66,30 @@ def run_benchmark(batch=2, seq=17776, h_q=30, h_kv=30, H=128, H_v=128, dt=torch.
f" SageDynQuan block_size={bs:3d}: {t:8.1f} ms diff max={dff.max().item():.6f} mean={dff.mean().item():.6f}"
)
+ # --- SAGE v1 high-level API comparison: PV half vs PV int8/PVS8 ---
+ sagev1_pvhalf_results = {}
+ sagev1_pvs8_results = {}
+ for bs in block_sizes:
+ pvhalf_out = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs)
+ pvhalf_diff = (ref - pvhalf_out).abs()
+ pvhalf_t = bench(lambda bs=bs: get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs))
+ sagev1_pvhalf_results[bs] = (pvhalf_t, pvhalf_diff.max().item(), pvhalf_diff.mean().item())
+ print(
+ f" SAGE v1 pvhalf block_size={bs:3d}: {pvhalf_t:8.1f} ms "
+ f"diff max={pvhalf_diff.max().item():.6f} mean={pvhalf_diff.mean().item():.6f}"
+ )
+
+ pvs8_out = get_ark().sagev1_pvi8(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs)
+ pvs8_diff = (ref - pvs8_out).abs()
+ pvs8_t = bench(
+ lambda bs=bs: get_ark().sagev1_pvi8(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs)
+ )
+ sagev1_pvs8_results[bs] = (pvs8_t, pvs8_diff.max().item(), pvs8_diff.mean().item())
+ print(
+ f" SAGE v1 pvs8 block_size={bs:3d}: {pvs8_t:8.1f} ms "
+ f"diff max={pvs8_diff.max().item():.6f} mean={pvs8_diff.mean().item():.6f}"
+ )
+
# --- SAGE v1 reference for each block_size (pre-quantized INT8) ---
sage_results = {}
for bs in block_sizes:
@@ -75,19 +99,27 @@ def run_benchmark(batch=2, seq=17776, h_q=30, h_kv=30, H=128, H_v=128, dt=torch.
ks = torch.randn(batch, h_kv, seq // bs, 1, dtype=torch.float32, device=dev) / 100 + 0.001
sage_t = bench(
lambda bs=bs, q_i8=q_i8, k_i8=k_i8, qs=qs, ks=ks: get_ark().sage(
- q_i8, k_i8, v, scale_block_size=bs, qscale=qs, kscale=ks, scale=scale, is_causal=is_causal
+ q_i8, k_i8, v, quant_block_size=bs, qscale=qs, kscale=ks, scale=scale, is_causal=is_causal
)
)
sage_results[bs] = sage_t
print(f" SAGE v1 bs={bs:3d}: {sage_t:8.1f} ms")
print()
- print("--- Summary (dynquant vs SAGE v1 at same block_size) ---")
+ print("--- Summary (dynquant / sagev1 pvhalf / sagev1 pvs8 / pre-quantized SAGE at same block_size) ---")
for bs in block_sizes:
t, mx, mn = results[bs]
+ pvhalf_t, pvhalf_mx, pvhalf_mn = sagev1_pvhalf_results[bs]
+ pvs8_t, pvs8_mx, pvs8_mn = sagev1_pvs8_results[bs]
sage_t = sage_results[bs]
overhead = (t / sage_t - 1) * 100
- print(f" block_size={bs:3d}: dynquant {t:8.1f} ms vs sage_v1 {sage_t:8.1f} ms (+{overhead:.0f}%)")
+ pvs8_over_pvhalf = (pvs8_t / pvhalf_t - 1) * 100
+ print(
+ f" block_size={bs:3d}: dynquant {t:8.1f} ms "
+ f"pvhalf {pvhalf_t:8.1f} ms pvs8 {pvs8_t:8.1f} ms "
+ f"prequant_sage {sage_t:8.1f} ms "
+ f"dynquant_vs_prequant={overhead:+.0f}% pvs8_vs_pvhalf={pvs8_over_pvhalf:+.0f}%"
+ )
if __name__ == "__main__":