Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

modified repack fp16-bf16 route tests

jijiaz0474fab9e629
diff --git a/auto_round_extension/ark/test/test_ark_cpu_mixed_bestla_sdpa.py b/auto_round_extension/ark/test/test_ark_cpu_mixed_bestla_sdpa.py
index 7749a326a..79cd352a9 100644
--- a/auto_round_extension/ark/test/test_ark_cpu_mixed_bestla_sdpa.py
+++ b/auto_round_extension/ark/test/test_ark_cpu_mixed_bestla_sdpa.py
@@ -40,6 +40,26 @@ def _mixed_sdpa(q, k, v, scale, is_causal, layout):
return auto_round_kernel.sdpa(q, k, v, scale=scale, is_causal=is_causal, tensor_layout=layout)
+def _kernel_view_kv(k, v, seq_q):
+ """Mirror the CPU mixed runtime's fp16->bf16 conversion for fp16 prefill.
+
+ The packed mixed path (``_cpu_public_mixed_sdpa_packed``) converts fp16 K/V
+ to bf16 on prefill (query seq_len > 1) when the BF16 route is compiled, so
+ the kernel computes from bf16-rounded K/V. Build the fp32 reference from the
+ same data the kernel will see; decode (seq_q == 1) and native-bf16 K/V are
+ left untouched. The caller must keep the original fp16 K/V for the kernel
+ call and use the returned tensors only for the reference.
+ """
+ if (
+ k.dtype == torch.float16
+ and seq_q > 1
+ and getattr(auto_round_kernel.cpu_lib, "ARK_CPU_SDPA_BUILD_HAS_BF16_ROUTE", False)
+ ):
+ k = k.to(dtype=torch.bfloat16)
+ v = v.to(dtype=torch.bfloat16)
+ return k, v
+
+
def test_mixed_dtype_sdpa_routes_to_mixed_path():
"""Verify mixed-dtype SDPA is dispatched to the BestLA mixed path by default."""
torch.manual_seed(4001)
@@ -47,7 +67,10 @@ def test_mixed_dtype_sdpa_routes_to_mixed_path():
k = torch.randn(1, 2, 16, 64, dtype=torch.float16)
v = torch.randn(1, 2, 16, 64, dtype=torch.float16)
scale = 1 / math.sqrt(64)
- expected = torch.nn.functional.scaled_dot_product_attention(q, k.float(), v.float(), scale=scale, enable_gqa=True)
+ k_ref, v_ref = _kernel_view_kv(k, v, seq_q=16)
+ expected = torch.nn.functional.scaled_dot_product_attention(
+ q, k_ref.float(), v_ref.float(), scale=scale, enable_gqa=True
+ )
out = auto_round_kernel.sdpa(q, k, v, scale=scale)
@@ -67,8 +90,9 @@ def test_bestla_mixed_sdpa_matches_torch(kv_dtype, is_causal, layout):
k = torch.randn(batch, heads_kv, seq, head_dim, dtype=kv_dtype)
v = torch.randn(batch, heads_kv, seq, head_dim, dtype=kv_dtype)
+ k_ref, v_ref = _kernel_view_kv(k, v, seq_q=seq)
expected_hnd = torch.nn.functional.scaled_dot_product_attention(
- q, k.float(), v.float(), scale=scale, enable_gqa=True, is_causal=is_causal
+ q, k_ref.float(), v_ref.float(), scale=scale, enable_gqa=True, is_causal=is_causal
)
try:
actual = _mixed_sdpa(
@@ -94,8 +118,9 @@ def test_bestla_mixed_sdpa_gqa_ratio(kv_dtype, gqa_ratio):
k = torch.randn(batch, heads_kv, seq, head_dim, dtype=kv_dtype)
v = torch.randn(batch, heads_kv, seq, head_dim, dtype=kv_dtype)
+ k_ref, v_ref = _kernel_view_kv(k, v, seq_q=seq)
expected = torch.nn.functional.scaled_dot_product_attention(
- q, k.float(), v.float(), scale=scale, enable_gqa=True, is_causal=False
+ q, k_ref.float(), v_ref.float(), scale=scale, enable_gqa=True, is_causal=False
)
try:
actual = _mixed_sdpa(q, k, v, scale, False, "HND")
@@ -169,8 +194,9 @@ def test_mixed_batched_gqa_prefill_runs_repeatedly(kv_dtype):
route = auto_round_kernel.debug_cpu_sdpa_route(q, k, v, scale=scale, is_causal=True, tensor_layout="HND")
assert route == auto_round_kernel.cpu_lib.ARK_CPU_SDPA_ROUTE_MIXED_RAW
+ k_ref, v_ref = _kernel_view_kv(k, v, seq_q=seq)
expected = torch.nn.functional.scaled_dot_product_attention(
- q, k.float(), v.float(), scale=scale, enable_gqa=True, is_causal=True
+ q, k_ref.float(), v_ref.float(), scale=scale, enable_gqa=True, is_causal=True
)
for _ in range(20):
actual = _mixed_sdpa(q, k, v, scale, True, "HND")
@@ -210,8 +236,9 @@ def test_mixed_llm_shape_accuracy(label, batch, heads_q, heads_kv, head_dim, seq
k = torch.randn(batch, heads_kv, seq_kv, head_dim, dtype=kv_dtype)
v = torch.randn(batch, heads_kv, seq_kv, head_dim, dtype=kv_dtype)
+ k_ref, v_ref = _kernel_view_kv(k, v, seq_q=seq_q)
expected = torch.nn.functional.scaled_dot_product_attention(
- q, k.float(), v.float(), scale=scale, enable_gqa=True, is_causal=is_causal
+ q, k_ref.float(), v_ref.float(), scale=scale, enable_gqa=True, is_causal=is_causal
)
try:
actual = _mixed_sdpa(q, k, v, scale, is_causal, "HND")