Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat(ark): fused Hadamard + MXFP4 activation quantization kernel

jijiazecc5b0053a2b
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.py
index d22b47c2d..cc73da5c0 100644
--- a/auto_round_extension/ark/auto_round_kernel/__init__.py
+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py
@@ -3868,6 +3868,19 @@ if torch.xpu.is_available():
except ImportError as _e:
print(f"ARK is unable to load XPU lib: {_e}")
+# Activation fused HMT + MXFP4 quantization (XPU). Imported last so the lib
+# handles above are already bound when the submodule looks them up.
+from .mxfp4_hadamard import ( # noqa: E402
+ HADAMARD_DIM,
+ HADAMARD_DIM_128,
+ MAX_LANES_PER_ROW,
+ SUPPORTED_HADAMARD_DIMS,
+ get_hadamard_matrix,
+ mxfp4_hadamard_quant,
+ mxfp4_hadamard_quant_reference,
+ mxfp4_quant_reference,
+ mxfp4_stream_reference,
+)
if __name__ == "__main__":
print(cpu_lib is None, xpu_lib is None)
diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cpp
index affb8b5bb..870cc924a 100755
--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp
@@ -28,7 +28,10 @@ typedef uintptr_t torch_ptr;
#include "xpu_wrapper.hpp"
#include "sycl_s8_wrapper.hpp"
#include "utils.hpp"
+#include "xpu_mxfp4_hadamard.hpp"
#if ARK_SYCL_TLA
+#include "xpu_mxfp4_hadamard_xmx.hpp"
+// Only include declarations, implementations are in separate .cpp files
#include "sycl_tla_common.hpp"
#endif
#else
@@ -776,6 +779,143 @@ static void sage_dynamic_quant_v_layout(torch_ptr stream, torch_ptr input, torch
}
}
+#if defined(ARK_SYCL_TLA)
+// Scratch-pool slot for the ``[D, D]`` Hadamard staging buffer the XMX path
+// below needs in the activation dtype. ``DeviceMemoryPool`` (``utils.hpp``)
+// hands out one grow-on-demand slab per (slot, device UUID, context, queue)
+// key, reused across every later call on that queue; slots 0-10 are already
+// claimed (see the slot table on ``DeviceMemoryPool::MaxLocNum``), so this one
+// takes 11 rather than aliasing a live scratch buffer.
+constexpr size_t kHadamardStagingScratchLoc = 11;
+
+// Body of the XMX path, templated over the activation dtype
+// (``cute::half_t`` / ``cute::bfloat16_t``) so the FP16 and BF16 entry points
+// share a single implementation.
+//
+// ``h_ptr`` is the caller's FP32 Hadamard matrix; the staging buffer holds its
+// transpose in ``T``, which is the layout the ``C = A @ B^T`` GEMM needs (see
+// ``convert_hadamard_to_dtype``). Every call rewrites all ``D * D`` elements,
+// so reusing one slab across calls with different matrices is safe: the two
+// launches that touch it go to the same queue, and the extension's queues are
+// in-order (see the note in `DeviceMemoryPool::get_scratch_ptr`).
+template <typename T>
+static void xmx_hadamard_quant_impl(sycl::queue* q, torch_ptr x, const float* h_ptr, uint8_t* codes_ptr,
+ uint8_t* scale_ptr, int64_t total_groups) {
+ constexpr size_t kElems =
+ static_cast<size_t>(XpuMxfp4Hadamard::kHadamardDim) * static_cast<size_t>(XpuMxfp4Hadamard::kHadamardDim);
+ // The request size is a compile-time constant, so the slab is allocated on
+ // the first call for a queue and then never grows -- the pool's grow path
+ // (which frees the old slab in place) is unreachable from here.
+ auto* pool = DeviceMemoryPool::Instance();
+ auto* h_t = static_cast<T*>(pool->get_scratch_mem(sizeof(T) * kElems, kHadamardStagingScratchLoc, q));
+ if (h_t == nullptr) {
+ throw std::runtime_error("ark::mxfp4_hadamard_quant: failed to acquire the Hadamard staging buffer");
+ }
+ ark::xmx_hadamard_detail::convert_hadamard_to_dtype<T>(q, h_ptr, h_t);
+ ark::xmx_hadamard_detail::mxfp4_hadamard_quant_xmx<T>(q, reinterpret_cast<const T*>(x), h_t, codes_ptr, scale_ptr,
+ total_groups);
+}
+#endif // ARK_SYCL_TLA
+
+// Activation-only fused kernel: normalized Hadamard + MXFP4 quant.
+// x: [num_rows, k] FP16 or BF16
+// hadamard: [D, D] FP32, row major, already normalized by 1/sqrt(D)
+// hadamard_dim: D, one of 32 (GEMM activations), 64, 128 (attention head dim),
+// 256 or 512. The quantization group stays at 32 for all of them, so
+// a D = 128 row yields 4 MXFP4 groups. Every D > 32 implements the
+// FWHT path only.
+// use_fwht: true when hadamard is the normalized Sylvester matrix, which is the
+// only matrix the butterfly network implements. The caller decides so
+// that the hot path does not pay for a device-side comparison.
+// use_xmx: opt-in XMX path (requires an ARK_SYCL_TLA build). Uses the
+// relaxed numerical contract of xpu_mxfp4_hadamard_xmx.hpp (H stored
+// in the activation dtype, DPAS accumulation); tolerance-based, not
+// bit-exact. D = 32 only.
+// use_quant_only: strip the Hadamard transform and quantize the raw activation
+// (quant-only baseline: byte-identical traffic to the fused path, so
+// the bandwidth ratio isolates the cost of the transform). Ignored
+// on the XMX path.
+// use_stream_only: strip the transform *and* the quantization math, keeping the
+// loads, the packing shape and the stores (traffic-matched roofline
+// baseline). Mutually exclusive with use_quant_only.
+// out_codes: [num_rows, k / 2] uint8, two packed FP4 codes per byte
+// out_scale: [num_rows, k / 32] uint8, one E8M0 exponent per 32-element group
+static void mxfp4_hadamard_quant(torch_ptr stream, torch_ptr x, torch_ptr hadamard, torch_ptr out_codes,
+ torch_ptr out_scale, int64_t num_rows, int64_t k, int in_dtype, bool use_fwht,
+ bool use_xmx, bool use_quant_only, int64_t hadamard_dim, bool use_stream_only) {
+ if (!stream) {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: stream must not be null");
+ }
+ if (!x || !hadamard || !out_codes || !out_scale) {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: input/output pointers must not be null");
+ }
+ if (num_rows <= 0 || k <= 0) {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: num_rows and k must be positive");
+ }
+ if (k % ark::XpuMxfp4Hadamard::kGroupSize != 0) {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: k must be a multiple of 32");
+ }
+ if (use_quant_only && use_stream_only) {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: use_quant_only and use_stream_only are exclusive");
+ }
+ const bool baseline = use_quant_only || use_stream_only;
+ // The baselines run on the flat [total_groups, 32] view, so the transform
+ // dimension is irrelevant to them and is not validated in that case.
+ if (!baseline) {
+ if (!ark::XpuMxfp4Hadamard::is_supported_hadamard_dim(hadamard_dim)) {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: hadamard_dim must be 32, 64, 128, 256 or 512");
+ }
+ if (k % hadamard_dim != 0) {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: k must be a multiple of hadamard_dim");
+ }
+ if (hadamard_dim > ark::XpuMxfp4Hadamard::kGroupSize) {
+ if (!use_fwht) {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: hadamard_dim > 32 requires the Sylvester matrix");
+ }
+ if (use_xmx) {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: use_xmx is not supported for hadamard_dim > 32");
+ }
+ }
+ }
+ auto* q = (sycl::queue*)stream;
+ auto* h_ptr = (const float*)hadamard;
+ auto* codes_ptr = (uint8_t*)out_codes;
+ auto* scale_ptr = (uint8_t*)out_scale;
+ const auto dtype = (BTLA_DTYPE)in_dtype;
+ const int64_t total_groups = num_rows * (k / ark::XpuMxfp4Hadamard::kGroupSize);
+
+ if (use_xmx && !baseline) {
+#if defined(ARK_SYCL_TLA)
+ // XMX path: H is converted to the activation dtype (lossless) and the
+ // transform runs on DPAS. x (sycl bf16/half) is layout-identical to
+ // cute::bfloat16_t / cute::half_t, so the pointers are reinterpreted.
+ if (dtype == BTLA_DTYPE::F16) {
+ xmx_hadamard_quant_impl<cute::half_t>(q, x, h_ptr, codes_ptr, scale_ptr, total_groups);
+ } else if (dtype == BTLA_DTYPE::BF16) {
+ xmx_hadamard_quant_impl<cute::bfloat16_t>(q, x, h_ptr, codes_ptr, scale_ptr, total_groups);
+ } else {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: only FP16 and BF16 activations are supported");
+ }
+#else
+ (void)total_groups;
+ throw std::runtime_error("ark::mxfp4_hadamard_quant: use_xmx requires an ARK_SYCL_TLA build");
+#endif
+ return;
+ }
+
+ if (dtype == BTLA_DTYPE::F16) {
+ ark::XpuMxfp4Hadamard::mxfp4_hadamard_quant<sycl::half>(q, (const sycl::half*)x, h_ptr, codes_ptr, scale_ptr,
+ num_rows, k, use_fwht, use_quant_only, hadamard_dim,
+ use_stream_only);
+ } else if (dtype == BTLA_DTYPE::BF16) {
+ ark::XpuMxfp4Hadamard::mxfp4_hadamard_quant<sycl::ext::oneapi::bfloat16>(
+ q, (const sycl::ext::oneapi::bfloat16*)x, h_ptr, codes_ptr, scale_ptr, num_rows, k, use_fwht,
+ use_quant_only, hadamard_dim, use_stream_only);
+ } else {
+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: only FP16 and BF16 activations are supported");
+ }
+}
+
#elif !defined(ARK_XPU)
enum class CpuSdpaRoute {
@@ -1424,6 +1564,11 @@ PYBIND11_MODULE(PY_NAME, m) {
m.def("sage_compute_seq_mean_bias_layout", &ark::sage_compute_seq_mean_bias_layout);
m.def("sage_dynamic_quant_layout", &ark::sage_dynamic_quant_layout);
m.def("sage_dynamic_quant_v_layout", &ark::sage_dynamic_quant_v_layout);
+ m.def("mxfp4_hadamard_quant", &ark::mxfp4_hadamard_quant, pybind11::arg("stream"), pybind11::arg("x"),
+ pybind11::arg("hadamard"), pybind11::arg("out_codes"), pybind11::arg("out_scale"),
+ pybind11::arg("num_rows"), pybind11::arg("k"), pybind11::arg("in_dtype"), pybind11::arg("use_fwht") = true,
+ pybind11::arg("use_xmx") = false, pybind11::arg("use_quant_only") = false,
+ pybind11::arg("hadamard_dim") = 32, pybind11::arg("use_stream_only") = false);
m.def("moe_gemm", &ark::moe_gemm_wrapper);
m.def("moe_gemm_decode", &ark::moe_gemm_decode_wrapper);
m.def("moe_decode_release_scratch", &ark::moe_decode_release_scratch);
diff --git a/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py b/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py
new file mode 100644
index 000000000..e6ab5d2d9
--- /dev/null
+++ b/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py
@@ -0,0 +1,583 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Activation fused HMT + MXFP4 quantization (XPU).
+
+Pipeline::
+
+ FP16/BF16 activation -> 32-point normalized Hadamard -> MXFP4 quantization
+ -> packed FP4 codes + E8M0 scales
+
+Frozen MVP contract (Phase 0):
+
+* ``hadamard_dim == group_size == 32`` and ``K % 32 == 0``;
+* ``H`` is a normalized 32x32 Hadamard matrix (already contains ``1/sqrt(32)``),
+ so ``y = reshape(x, [-1, 32]) @ H`` needs no extra scaling;
+* per 32-element group ``amax = max(|y_g|)``,
+ ``e8m0 = clamp(floor(log2(amax)) - 2 + 127, 0, 254)`` and the effective scale
+ is ``2 ** (e8m0 - 127)`` (standard E8M0, always a power of two);
+* ``q = y * 2 ** -(e8m0 - 127)`` is encoded as ``signbit(q) << 3 | magnitude``
+ with FP4 (E2M1) magnitude levels ``0, 0.5, 1, 1.5, 2, 3, 4, 6`` and the
+ nearest-even thresholds of ``vllm_ext/fp4_utils.py::cast_to_fp4``;
+* **zero is canonicalised**: whenever the magnitude index is 0 the sign bit is
+ dropped, so the code is ``0x0`` and never ``0x8`` (negative zero). See
+ "Canonical zero" below for why this rule is required rather than optional;
+* two codes share one byte, the even element occupying the low nibble
+ (identical to ``vllm_ext/fp4_utils.py::pack_fp4_to_uint8``);
+* an all-zero group produces ``e8m0 = 0`` and all-zero codes;
+* NaN/Inf are outside the supported input domain. The reference always rejects
+ them; the XPU entry point only does so under ``check_finite=True``, because
+ the scan reads the whole activation and syncs, costing several times the
+ fused kernel itself.
+
+FP32 transform contract (Phase 2, revised in Phase 3)
+-----------------------------------------------------
+
+Bit-exactness between the SYCL kernel and this reference requires a *defined*
+summation order for ``y = x_g @ H``. There are two paths, and each is bit-exact
+against its own reference; they are deliberately *not* bit-exact against each
+other, because a butterfly network and a 32-term dot product round differently.
+:func:`transform_reference` mirrors the choice the wrapper makes.
+
+**FWHT (default, used for the normalized Sylvester matrix).** Five butterfly
+stages; stage ``s`` pairs each lane with ``lane ^ (1 << s)`` and the lane
+holding the high half of the pair computes the difference. A single final
+multiply by ``H[0][0] == 1/sqrt(32)`` applies the normalization. Only adds and
+subtracts occur, so there is no multiply-add for the compiler to contract, and
+the order is fully determined by the stage index.
+
+This path exists for performance and is not merely an optimization detail. The
+kernel is intended to be memory bound, but the O(D^2) path below costs 32
+multiplies plus 32 adds per lane with FMA disabled, which caps effective
+bandwidth at roughly 60% of the measured streaming-copy baseline on Arc Pro B60
+*before* accounting for shuffles and matrix loads. The butterfly costs 5 adds
+plus one scale, moving the bottleneck back to memory.
+
+**Path A (only for a caller-supplied non-Sylvester matrix).** ``acc`` starts at
+``+0.0`` and is updated as ``acc = fp32_add(acc, fp32_mul(x[j], H[j][i]))`` for
+``j = 0 .. 31`` in increasing ``j``, with a separate FP32 rounding after the
+multiply and after the add (no fused multiply-add, no reassociation). The
+kernel enforces this with ``#pragma clang fp contract(off)``.
+``torch.matmul`` is deliberately not used for either path because its blocking,
+FMA usage and reassociation are unspecified and would make the bit-exact
+acceptance criterion untestable.
+
+Canonical zero (Phase 2)
+------------------------
+
+There is one quantity the accumulation contract above cannot pin down: the
+*sign* of a result that is mathematically zero. When a group of 32 inputs is
+constant, every output column except the first cancels exactly, and the residue
+left by FP32 rounding is on the order of ``1e-8`` with an order-dependent sign.
+Device-side flush-to-zero of FP32 subnormals produces the same ambiguity for
+very small inputs, where the CPU reference keeps the subnormal but the GPU
+returns a signed zero.
+
+Such a value always quantizes to FP4 magnitude index 0, so the ambiguity can
+only ever affect the sign bit, turning ``0x0`` into ``0x8`` (negative zero).
+Because ``0x8`` and ``0x0`` dequantize to the same number, no information is
+lost by forbidding ``0x8``, and doing so makes the encoding a total function of
+the mathematical value rather than of the rounding residue. Both this reference
+and the kernel therefore drop the sign bit whenever the magnitude index is 0.
+
+This is a deliberate, documented deviation from
+``vllm_ext/fp4_utils.py::pack_fp4_to_uint8``, which applies ``signbit``
+unconditionally: that helper encodes already-clean dequantized values, where a
+negative zero can only appear if the caller supplied one.
+"""
+
+from __future__ import annotations
+
+import torch
+
+HADAMARD_DIM = 32
+GROUP_SIZE = 32
+
+# Attention head dim. The quantization group stays at ``GROUP_SIZE`` for every
+# transform size, so a 128-element row produces 4 independent MXFP4 groups and
+# the output layout is identical to the 32-point case.
+HADAMARD_DIM_128 = 128
+
+# Transform sizes the XPU kernel implements: ``GROUP_SIZE * L`` for a power-of-
+# two lane count ``L``. Every size above 32 is FWHT-only (Sylvester matrix) and
+# fans a row out over ``L`` cooperating sub-group lanes, so ``L`` must divide the
+# sub-group size of 32 -- see ``fwht_quant_cooperative`` in
+# xpu_mxfp4_hadamard.hpp. 512 is the largest instantiated size; nothing in the
+# design stops 1024, it simply has no consumer.
+MAX_LANES_PER_ROW = 16
+SUPPORTED_HADAMARD_DIMS = tuple(GROUP_SIZE * (1 << i) for i in range(MAX_LANES_PER_ROW.bit_length()))
+
+# FP4 (E2M1) magnitude levels.
+E2M1_VALUES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
+
+# (threshold, is_closed_interval, magnitude_index), first match wins, mirroring
+# the early-out ladder of ``cast_to_fp4`` including its alternating ``<=`` / ``<``
+# boundary operators. ``_encode_fp4`` walks the table in reverse with
+# ``torch.where``, so each earlier entry overrides every later one.
+_E2M1_THRESHOLDS = (
+ (0.25, True, 0),
+ (0.75, False, 1),
+ (1.25, True, 2),
+ (1.75, False, 3),
+ (2.5, True, 4),
+ (3.5, False, 5),
+ (5.0, True, 6),
+)
+
+_HADAMARD_CACHE: dict[tuple[int, str], torch.Tensor] = {}
+
+
+def _sylvester_hadamard(dim: int) -> torch.Tensor:
+ if dim < 1 or (dim & (dim - 1)) != 0:
+ raise ValueError(f"Hadamard dimension must be a power of two, got {dim}")
+ h = torch.ones(1, 1, dtype=torch.float32)
+ while h.shape[0] < dim:
+ h = torch.cat([torch.cat([h, h], dim=1), torch.cat([h, -h], dim=1)], dim=0)
+ return h
+
+
+def get_hadamard_matrix(dim: int = HADAMARD_DIM, device: torch.device | str = "cpu") -> torch.Tensor:
+ """Return the normalized ``dim x dim`` Hadamard matrix (FP32, contiguous)."""
+ key = (dim, str(torch.device(device)))
+ cached = _HADAMARD_CACHE.get(key)
+ if cached is None:
+ cached = (_sylvester_hadamard(dim) / (dim**0.5)).to(device=device).contiguous()
+ _HADAMARD_CACHE[key] = cached
+ return cached
+
+
+def _validate_hadamard(hadamard_matrix: torch.Tensor, *, check_finite: bool = True) -> int:
+ """Validate the matrix and return its dimension ``D``.
+
+ ``D`` is read from the matrix rather than passed in, so a caller selects the
+ transform size simply by handing over a ``get_hadamard_matrix(D)`` tensor.
+ """
+ if not isinstance(hadamard_matrix, torch.Tensor):
+ raise TypeError(f"hadamard_matrix must be a torch.Tensor, got {type(hadamard_matrix)}")
+ shape = tuple(hadamard_matrix.shape)
+ if len(shape) != 2 or shape[0] != shape[1] or shape[0] not in SUPPORTED_HADAMARD_DIMS:
+ allowed = " or ".join(f"({d}, {d})" for d in SUPPORTED_HADAMARD_DIMS)
+ raise ValueError(f"hadamard_matrix must have shape {allowed}, got {shape}")
+ if hadamard_matrix.dtype not in (torch.float32, torch.float64):
+ raise ValueError(f"hadamard_matrix must be float32 or float64, got {hadamard_matrix.dtype}")
+ # Unlike the checks above, this one reads a device tensor and forces a
+ # device->host sync, which costs about as much as the fused kernel itself.
+ # Callers that have already established the matrix is the known-good default
+ # pass check_finite=False.
+ if check_finite and not torch.isfinite(hadamard_matrix).all():
+ raise ValueError("hadamard_matrix must contain only finite values")
+ return shape[0]
+
+
+def _require_activation_tensor(x: torch.Tensor) -> None:
+ """Reject a non-tensor ``x`` before anything dereferences it.
+
+ Both entry points resolve the Hadamard matrix (and hence ``x.device``)
+ before the full activation validation runs, so this cheap check has to
+ happen first for a bad ``x`` to raise TypeError rather than AttributeError.
+ """
+ if not isinstance(x, torch.Tensor):
+ raise TypeError(f"x must be a torch.Tensor, got {type(x)}")
+
+
+def _validate_activation(
+ x: torch.Tensor, *, require_xpu: bool, check_finite: bool = True, hadamard_dim: int = HADAMARD_DIM
+) -> tuple[int, int]:
+ _require_activation_tensor(x)
+ if x.dtype not in (torch.float16, torch.bfloat16):
+ raise ValueError(f"x must be float16 or bfloat16, got {x.dtype}")
+ if x.ndim < 1:
+ raise ValueError("x must have at least one dimension")
+ if require_xpu and x.device.type != "xpu":
+ raise ValueError(f"mxfp4_hadamard_quant is only supported on XPU, got device {x.device}")
+ k = x.shape[-1]
+ if k == 0 or x.numel() == 0:
+ raise ValueError("x must not be empty")
+ # The transform dimension is the stricter of the two constraints (it is a
+ # multiple of GROUP_SIZE), so checking it alone is sufficient.
+ if k % hadamard_dim != 0:
+ raise ValueError(f"the last dimension of x must be a multiple of {hadamard_dim}, got {k}")
+ # This scan reads all of x and then forces a device->host sync on the
+ # result, which on XPU costs roughly 4x the fused kernel itself. It is a
+ # debugging aid, not part of the numerical contract, so the device entry
+ # point leaves it off by default (see ``check_finite`` there).
+ if check_finite and not torch.isfinite(x).all():
+ raise ValueError("x must contain only finite values (NaN/Inf are not supported)")
+ return x.numel() // k, k
+
+
+def hadamard_transform_reference(x_groups: torch.Tensor, h: torch.Tensor) -> torch.Tensor:
+ """``x_groups [G, 32] @ h [32, 32]`` under the Path A FP32 accumulation contract.
+
+ Sums over ``j`` in increasing order with a separate FP32 rounding after each
+ multiply and each add, matching the kernel loop exactly. ``torch.matmul`` is
+ intentionally avoided (unspecified blocking / FMA / reassociation).
+
+ Used only for a caller-supplied non-Sylvester matrix; the default matrix
+ goes through :func:`fwht_transform_reference`.
+ """
+ x_groups = x_groups.to(torch.float32)
+ h = h.to(torch.float32)
+ acc = torch.zeros_like(x_groups)
+ for j in range(h.shape[0]):
+ acc = acc + x_groups[:, j : j + 1] * h[j]
+ return acc
+
+
+def fwht_transform_reference(x_groups: torch.Tensor, norm: torch.Tensor, *, norm_last: bool = False) -> torch.Tensor:
+ """D-point fast Walsh-Hadamard transform under the frozen butterfly contract.
+
+ Computes the same mathematical result as ``x_groups @ H`` for the normalized
+ Sylvester matrix ``H`` of dimension ``D``, but in ``log2(D)`` butterfly
+ stages instead of ``D`` multiply-accumulates. ``norm = H[0][0] =
+ 1/sqrt(D)`` is applied *first*, then stage ``s`` pairs each lane with
+ ``lane ^ (1 << s)``:
+
+ ``acc = (lane & h) ? (partner - acc) : (acc + partner)``
+
+ Normalizing up front rather than at the end costs the same single multiply
+ but bounds the intermediates by ``sqrt(D) * max|x|`` instead of
+ ``D * max|x|``, keeping the safe input range identical to Path A. With the
+ scale applied last, inputs above ``FP32_MAX / D`` overflow to infinity even
+ though the mathematical result is perfectly representable.
+
+ Taking ``norm`` from the matrix itself (rather than recomputing
+ ``1/sqrt(D)``) guarantees the kernel and this reference scale by the
+ identical FP32 value.
+
+ Every intermediate is a plain FP32 add or subtract, so there is nothing for
+ the compiler to contract into an FMA and the order is fully determined by
+ the stage index -- which is what keeps this bit-exact against the kernel.
+
+ ``norm_last`` moves the scaling after the butterflies, mirroring the
+ cooperative ``D > 32`` kernel. That kernel cannot scale on load: doing so
+ leaves a multiply feeding the first stage's add, which the GPU backend fuses
+ into an FMA that no compiler flag or pragma was able to suppress, breaking
+ bit-exactness. See the normalization note above ``fwht_quant_cooperative``
+ in xpu_mxfp4_hadamard.hpp. The two orders round differently and are
+ deliberately not bit-exact against each other.
+ """
+ dim = x_groups.shape[-1]
+ scale = norm.to(device=x_groups.device, dtype=torch.float32)
+ acc = x_groups.to(torch.float32)
+ if not norm_last:
+ acc = acc * scale
+ lanes = torch.arange(dim, device=acc.device)
+ for stage in range(dim.bit_length() - 1):
+ h = 1 << stage
+ partner = acc[:, lanes ^ h]
+ acc = torch.where((lanes & h) != 0, partner - acc, acc + partner)
+ if norm_last:
+ acc = acc * scale
+ return acc
+
+
+def is_default_hadamard(hadamard_matrix: torch.Tensor) -> bool:
+ """True if ``hadamard_matrix`` is exactly the normalized Sylvester matrix.
+
+ Only that matrix may take the FWHT path, because the butterfly network
+ implements the Sylvester ordering specifically.
+ """
+ dim = hadamard_matrix.shape[0]
+ # Callers normally pass the tensor returned by get_hadamard_matrix, which is
+ # cached per device; recognising it by identity avoids a device->host copy
+ # and the sync that torch.equal would impose on every quantization call.
+ if hadamard_matrix is _HADAMARD_CACHE.get((dim, str(hadamard_matrix.device))):
+ return True
+ h = hadamard_matrix.to(torch.float32)
+ return bool(torch.equal(h.cpu(), get_hadamard_matrix(dim, "cpu")))
+
+
+def transform_reference(x_groups: torch.Tensor, h: torch.Tensor) -> torch.Tensor:
+ """Dispatch to the FWHT or Path A reference, mirroring the kernel's choice."""
+ if is_default_hadamard(h):
+ # D > 32 runs the cooperative kernel, which normalizes after the
+ # butterflies rather than on load; the reference has to match.
+ return fwht_transform_reference(x_groups, h.reshape(-1)[0], norm_last=h.shape[0] > GROUP_SIZE)
+ return hadamard_transform_reference(x_groups, h)
+
+
+def _e8m0_and_quantized(y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ """Return ``(e8m0 [G], q [G, 32])`` for FP32 transformed groups ``y [G, 32]``."""
+ amax = y.abs().amax(dim=-1)
+ # frexp: amax = mantissa * 2 ** exp with mantissa in [0.5, 1)
+ # => floor(log2(amax)) == exp - 1 (exact, also for exact powers of two).
+ _, exp = torch.frexp(amax)
+ scale_exp = exp.to(torch.int32) - 1 - 2
+ e8m0 = torch.clamp(scale_exp + 127, 0, 254)
+ # torch.frexp(inf) returns exponent 0, which would silently yield e8m0 = 124.
+ # The kernel uses ilogb, which saturates, so an overflowed group must clamp
+ # to 254 here too or the two would disagree. y can only be non-finite when
+ # the transform of a finite input overflowed FP32, i.e. for inputs beyond
+ # the documented safe range; the group is garbage either way, but the two
+ # implementations must still agree on it.
+ e8m0 = torch.where(torch.isfinite(amax), e8m0, torch.full_like(e8m0, 254))
+ zero_group = amax == 0
+ e8m0 = torch.where(zero_group, torch.zeros_like(e8m0), e8m0)
+ q = torch.ldexp(y, -(e8m0 - 127).unsqueeze(-1))
+ q = torch.where(zero_group.unsqueeze(-1), torch.zeros_like(q), q)
+ return e8m0.to(torch.uint8), q
+
+
+def _encode_fp4(q: torch.Tensor) -> torch.Tensor:
+ """Encode FP32 values into 4-bit ``sign << 3 | magnitude_index`` codes."""
+ a = q.abs()
+ idx = torch.full_like(a, len(E2M1_VALUES) - 1, dtype=torch.int32)
+ for threshold, closed, value in reversed(_E2M1_THRESHOLDS):
+ hit = a <= threshold if closed else a < threshold
+ idx = torch.where(hit, torch.full_like(idx, value), idx)
+ sign = torch.signbit(q).to(torch.int32) << 3
+ # Canonical zero: magnitude 0 always encodes as 0x0, never 0x8 (negative
+ # zero). The sign of a value that rounds to zero is not reproducible across
+ # implementations, so it must not reach the output. See module docstring.
+ return torch.where(idx == 0, idx, idx | sign).to(torch.uint8)
+
+
+def pack_codes(codes: torch.Tensor) -> torch.Tensor:
+ """Pack 4-bit codes ``[M, K]`` into bytes ``[M, K // 2]`` (even element = low nibble)."""
+ low = codes[..., 0::2].to(torch.uint8) & 0x0F
+ high = codes[..., 1::2].to(torch.uint8) & 0x0F
+ return low | (high << 4)
+
+
+def mxfp4_hadamard_quant_reference(
+ x: torch.Tensor, hadamard_matrix: torch.Tensor | None = None
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Pure PyTorch FP32 reference for :func:`mxfp4_hadamard_quant`.
+
+ Runs on any device (including CPU) and defines the frozen numerical contract.
+ The transform dimension is taken from ``hadamard_matrix`` (any of
+ :data:`SUPPORTED_HADAMARD_DIMS`); the quantization group is always 32, so a
+ 128-point row yields 4 groups.
+ """
+ _require_activation_tensor(x)
+ if hadamard_matrix is None:
+ hadamard_matrix = get_hadamard_matrix(HADAMARD_DIM, x.device)
+ # The matrix is validated next because it determines the divisibility
+ # constraint the activation is then checked against.
+ dim = _validate_hadamard(hadamard_matrix)
+ num_rows, k = _validate_activation(x, require_xpu=False, hadamard_dim=dim)
+
+ h = hadamard_matrix.to(device=x.device, dtype=torch.float32).contiguous()
+ y = transform_reference(x.contiguous().reshape(-1, dim), h)
+ # The transform works on D-element rows, the quantizer on 32-element groups.
+ e8m0, q = _e8m0_and_quantized(y.reshape(-1, GROUP_SIZE))
+ codes = _encode_fp4(q).reshape(num_rows, k)
+ return pack_codes(codes), e8m0.reshape(num_rows, k // GROUP_SIZE)
+
+
+def mxfp4_quant_reference(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ """Pure PyTorch FP32 reference for the **quant-only** baseline.
+
+ Quantizes the *raw* activation (no Hadamard transform) with the same frozen
+ MXFP4 contract as :func:`mxfp4_hadamard_quant_reference`: per-32-element
+ E8M0 scale + packed FP4 codes. It is the reference for the quant-only path
+ of :func:`mxfp4_hadamard_quant` (``_quant_only=True``), the Hadamard
+ ablation of the bandwidth benchmark: that mode moves byte-identical traffic
+ and only drops the transform, so the measured ratio isolates its cost.
+
+ Mathematically it equals ``mxfp4_hadamard_quant_reference(x, I)``: running
+ the reference transform with the identity matrix is bit-exact with applying
+ no transform at all (it only ever adds exact zeros), so the two agree byte
+ for byte.
+ """
+ num_rows, k = _validate_activation(x, require_xpu=False)
+ x_groups = x.contiguous().reshape(-1, GROUP_SIZE).to(torch.float32)
+ e8m0, q = _e8m0_and_quantized(x_groups)
+ codes = _encode_fp4(q).reshape(num_rows, k)
+ return pack_codes(codes), e8m0.reshape(num_rows, k // GROUP_SIZE)
+
+
+def mxfp4_stream_reference(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ """Pure PyTorch reference for the **stream-only** roofline baseline.
+
+ The stream-only mode exists to answer "how fast could *any* kernel with this
+ access pattern be?". It keeps the fused kernel's loads, packing shape and
+ stores but replaces the transform and the quantization math with the
+ cheapest data-dependent function available: the low nibble of each element's
+ raw bit pattern, packed under the usual even-element-low-nibble rule, plus
+ an XOR fold for the scale byte. The output is not a quantization of anything
+ and must never be consumed as one.
+
+ This reference is not part of the numerical contract -- it exists so a test
+ can prove the baseline kernel actually reads every input element. Without
+ it, dead-code elimination of the loads would turn the baseline into an empty
+ kernel reporting an unreachable bandwidth, silently inflating every ratio
+ measured against it.
+
+ The scale byte folds the four code bytes that hold elements
+ ``{0, 1}, {8, 9}, {16, 17}, {24, 25}`` of the group, which is exactly the
+ low byte of the kernel's XOR over its four packed 32-bit words.
+ """
+ num_rows, k = _validate_activation(x, require_xpu=False, check_finite=False)
+ # view() reinterprets the 16-bit float as raw bits without converting.
+ bits = x.contiguous().view(torch.int16).reshape(-1, GROUP_SIZE).to(torch.int32) & 0x0F
+ codes = pack_codes(bits.to(torch.uint8)).reshape(num_rows, k // 2)
+ groups = codes.reshape(-1, GROUP_SIZE // 2)
+ fold = groups[:, 0] ^ groups[:, 4] ^ groups[:, 8] ^ groups[:, 12]
+ return codes, fold.reshape(num_rows, k // GROUP_SIZE)
+
+
+_XMX_SUPPORTED: bool | None = None
+
+
+def _xmx_supported() -> bool:
+ """True when the current XPU build exposes the XMX path.
+
+ Probes once by forcing the XMX path on a tiny tensor; the C++ binding raises
+ ``RuntimeError`` when ARK_SYCL_TLA is not compiled in. The result is cached.
+ """
+ global _XMX_SUPPORTED
+ if _XMX_SUPPORTED is None:
+ try:
+ x = torch.zeros(1, GROUP_SIZE, dtype=torch.float16, device="xpu")
+ mxfp4_hadamard_quant(x, _force_xmx=True)
+ _XMX_SUPPORTED = True
+ except (RuntimeError, ValueError, NotImplementedError):
+ _XMX_SUPPORTED = False
+ return _XMX_SUPPORTED
+
+
+def mxfp4_hadamard_quant(
+ x: torch.Tensor,
+ hadamard_matrix: torch.Tensor | None = None,
+ *,
+ check_finite: bool = False,
+ _force_xmx: bool | None = None,
+ _quant_only: bool = False,
+ _stream_only: bool = False,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Fused Hadamard transform + MXFP4 quantization on XPU.
+
+ Args:
+ x: FP16/BF16 XPU activation with ``x.shape[-1] % D == 0``, where ``D``
+ is the dimension of ``hadamard_matrix``.
+ hadamard_matrix: normalized ``D x D`` Hadamard matrix with ``D`` in
+ :data:`SUPPORTED_HADAMARD_DIMS`, i.e. ``32 * 2**n`` for ``n = 0..4``.
+ Defaults to the ``32 x 32`` Sylvester matrix returned by
+ :func:`get_hadamard_matrix`. ``D = 32`` runs one work-item per
+ quantization group; every larger ``D`` is implemented by a
+ cooperative ``D / 32``-lane FWHT and supports the Sylvester matrix
+ only. Pass ``get_hadamard_matrix(128, dev)`` for the attention
+ head-dim transform.
+ check_finite: reject NaN/Inf in ``x`` before launching. Off by default:
+ the check reads all of ``x`` and syncs on the result, which costs
+ several times the fused kernel itself. NaN/Inf are still outside the
+ supported input domain -- the kernel simply does not police it on the
+ hot path. :func:`mxfp4_hadamard_quant_reference` always checks.
+ _force_xmx: private override used by tests/benchmarks (None = auto).
+ _quant_only: private quant-only baseline override (default False). When
+ True, the Hadamard transform is stripped and the *raw* activation is
+ quantized (see :func:`mxfp4_quant_reference`), with identical memory
+ traffic to the fused path -- the C++ dispatcher ignores
+ use_fwht/use_xmx in this mode and always uses the per-item FWHT
+ layout. This is the quant-only baseline: identical traffic, no
+ transform, so the benchmark's ``f/q`` ratio isolates its cost.
+ _stream_only: private stream-only baseline override (default False).
+ When True, both the transform *and* the quantization math are
+ stripped, leaving the loads, the packing shape and the stores. The
+ byte traffic and the item mapping are identical to the fused path,
+ so ``BW_fused / BW_stream_only`` is the kernel's utilization of the
+ traffic-matched roofline -- the denominator the benchmark gates on.
+ The output is *not* a quantization; see
+ :func:`mxfp4_stream_reference`. Mutually exclusive with
+ ``_quant_only``.
+
+ Routing is automatic: the normalized Sylvester matrix always takes the
+ bit-exact FWHT path (first priority); any other Hadamard matrix falls
+ back to the XMX path when the build supports it (relaxed contract:
+ H stored in the activation dtype, DPAS accumulation, tolerance-based
+ acceptance), otherwise to the bit-exact Path A.
+
+ Returns:
+ ``(out_codes, out_scale)`` where ``out_codes`` is ``uint8 [M, K // 2]``
+ (two packed FP4 codes per byte) and ``out_scale`` is
+ ``uint8 [M, K // 32]`` (one E8M0 exponent per group), with
+ ``M = x.numel() // K``.
+ """
+ from . import cvt_dtype, get_lib, get_stream
+
+ if _quant_only and _stream_only:
+ raise ValueError("_quant_only and _stream_only are mutually exclusive")
+ _require_activation_tensor(x)
+
+ hadamard_dim = HADAMARD_DIM
+ if _quant_only or _stream_only:
+ # Baselines run on the flat [total_groups, 32] view, so no Hadamard
+ # matrix is involved: no validation and no device comparison. A
+ # (default) matrix is still materialised so the pointer argument passed
+ # to the C++ binding stays valid; the dispatcher never dereferences it
+ # in either baseline mode.
+ hadamard_matrix = get_hadamard_matrix(HADAMARD_DIM, x.device)
+ use_fwht = True
+ use_xmx = False
+ elif hadamard_matrix is None:
+ # The default matrix is known to be the Sylvester one, so the FWHT path
+ # is taken without paying for a comparison on the hot path.
+ hadamard_matrix = get_hadamard_matrix(HADAMARD_DIM, x.device)
+ use_fwht = True
+ else:
+ # Structural checks are cheap. The finiteness check is not: it syncs on
+ # the device every call. The default matrix is known finite, so only a
+ # caller-supplied one pays for it.
+ hadamard_dim = _validate_hadamard(hadamard_matrix, check_finite=False)
+ use_fwht = is_default_hadamard(hadamard_matrix)
+ if not use_fwht:
+ if hadamard_dim != HADAMARD_DIM:
+ # Only the butterfly network is implemented above D = 32; there
+ # is no O(D^2) fallback and no XMX path for it.
+ raise NotImplementedError(f"hadamard_dim {hadamard_dim} supports the normalized Sylvester matrix only")
+ _validate_hadamard(hadamard_matrix)
+
+ num_rows, k = _validate_activation(x, require_xpu=True, check_finite=check_finite, hadamard_dim=hadamard_dim)
+
+ # Path resolution (auto-router): FWHT has first priority for the Sylvester
+ # matrix; any other (custom) matrix falls back to the XMX path when the
+ # build supports it (relaxed contract), otherwise to Path A. ``_force_xmx``
+ # is a private override used by tests/benchmarks. ``_quant_only`` always
+ # disables XMX: it is implemented on the shared per-item FWHT layout, which
+ # is exactly the layout the quant-only baseline must match.
+ if not (_quant_only or _stream_only):
+ if _force_xmx is not None:
+ use_xmx = bool(_force_xmx)
+ elif use_fwht or hadamard_dim != HADAMARD_DIM:
+ use_xmx = False
+ else:
+ use_xmx = _xmx_supported()
+
+ lib = get_lib(x)
+ if lib is None or not hasattr(lib, "mxfp4_hadamard_quant"):
+ raise NotImplementedError("Current XPU build does not expose mxfp4_hadamard_quant")
+
+ x_arg = x.contiguous().reshape(num_rows, k)
+ h_arg = hadamard_matrix.to(device=x.device, dtype=torch.float32).contiguous()
+ out_codes = torch.empty((num_rows, k // 2), dtype=torch.uint8, device=x.device)
+ out_scale = torch.empty((num_rows, k // GROUP_SIZE), dtype=torch.uint8, device=x.device)
+
+ lib.mxfp4_hadamard_quant(
+ get_stream(x_arg),
+ x_arg.data_ptr(),
+ h_arg.data_ptr(),
+ out_codes.data_ptr(),
+ out_scale.data_ptr(),
+ num_rows,
+ k,
+ cvt_dtype(x_arg.dtype),
+ use_fwht,
+ use_xmx,
+ _quant_only,
+ hadamard_dim,
+ _stream_only,
+ )
+ return out_codes, out_scale
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
index efa54a48d..f7f166a58 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
@@ -214,8 +214,10 @@ class DeviceMemoryPool {
// SDPA kernels; slot 8 is the MoE DPAS grouped-GEMM work-group counter
// (`moe_dpas_fp8::kAtomicScratchLoc`); slots 9 and 10 are the int4 decode
// weight-repack and activation-sum buffers
- // (`moe_decode_detail::kInt4RepackScratchLoc` / `kActGroupSumScratchLoc`).
- static constexpr int MaxLocNum = 11;
+ // (`moe_decode_detail::kInt4RepackScratchLoc` / `kActGroupSumScratchLoc`);
+ // slot 11 is the XMX Hadamard staging buffer
+ // (`kHadamardStagingScratchLoc` in `ark.cpp`).
+ static constexpr int MaxLocNum = 12;
using SizeMap = std::unordered_map<size_t, size_t>;
using PtrMap = std::unordered_map<size_t, int8_t*>;
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp
new file mode 100644
index 000000000..00b34c6f8
--- /dev/null
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp
@@ -0,0 +1,701 @@
+//
+// Copyright (c) 2026 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Activation fused kernel: FP16/BF16 activation -> normalized Hadamard
+// transform (D = 32, 64, 128, 256 or 512) -> MXFP4 quantization (packed FP4
+// codes + E8M0 scales).
+//
+// Numerical contract:
+// * hadamard_dim == group_size == 32, K % 32 == 0
+// * H is a *normalized* 32x32 Hadamard matrix (already contains 1/sqrt(32))
+// * y = reshape(x, [-1, 32]) @ H (FP32 accumulation)
+// * amax = max(|y_g|) per 32-element group
+// * e8m0 = clamp(floor(log2(amax)) - 2 + 127, 0, 254)
+// * q = y * 2^-(e8m0 - 127)
+// * code = signbit(q) << 3 | e2m1_magnitude_index(|q|)
+// * canonical zero: magnitude index 0 always encodes as 0x0, never 0x8
+// * two codes per byte, even element in the low nibble
+// * all-zero group -> e8m0 = 0 and all codes = 0
+//
+// FP32 transform contract: two paths exist and each is bit-exact against its own
+// PyTorch reference.
+//
+// FWHT (default): log2(D) butterfly stages over lane ^ (1 << stage), plus a
+// single multiply by H[0][0] == 1/sqrt(D). The butterflies are pure adds and
+// subtracts, so their order is fixed by the stage index; the only contractable
+// operation is that normalization multiply, and *where* it sits decides
+// whether the kernel is reproducible. See the normalization note above
+// fwht_quant_cooperative. This is the performance path: the O(D^2) dot product
+// below is compute bound well below streaming-copy bandwidth on this device,
+// the butterfly is memory bound.
+//
+// Path A (non-Sylvester matrix only): sums over j in increasing order with a
+// separate FP32 rounding after each multiply and each add (no FMA, no
+// reassociation), enforced by #pragma clang fp contract(off).
+//
+// The two paths round differently and are deliberately *not* bit-exact against
+// each other; the wrapper picks one and the reference mirrors that choice.
+//
+// D > 32 (e.g. 128, the attention head dim). Only the FWHT path exists: H is
+// required to be the normalized Sylvester matrix. The quantization group stays
+// at 32, so a D-element row produces D/32 independent MXFP4 groups. See
+// fwht_quant_cooperative below for the two-level (lane-local + cross-lane)
+// decomposition and why a single work-item per row is not viable.
+//
+// Baseline modes. Two controlled ablations share the fused kernel's exact
+// memory traffic and item mapping, so their bandwidths are directly
+// comparable:
+// * quant-only -- drops the Hadamard transform, keeps the quantization.
+// * stream-only -- drops the transform *and* the quantization math, keeping
+// only the loads, the packing shape and the stores. This is
+// the traffic-matched roofline: the highest bandwidth any
+// kernel with this read:write ratio and this access pattern
+// can reach on the device, and therefore the only
+// self-calibrating denominator for the fused kernel.
+
+#pragma once
+
+#include <cstdint>
+#include <stdexcept>
+
+#if defined(ARK_XPU)
+#include <sycl/sycl.hpp>
+
+namespace ark {
+
+class XpuMxfp4Hadamard {
+ public:
+ static constexpr int kHadamardDim = 32;
+ static constexpr int kGroupSize = 32;
+ static constexpr int kSubGroupSize = 32;
+ static constexpr int kWorkGroupSize = 256;
+ // log2(32): number of FWHT butterfly stages.
+ static constexpr int kNumFwhtStages = 5;
+
+ // Cooperative FWHT for D > 32: L lanes per row, each owning one 32-element
+ // quantization group, so D = 32 * L. The sub-group is pinned so that the L
+ // lanes of a row always land in one sub-group, which is what makes the
+ // cross-lane butterfly shuffles legal: sub-groups start at multiples of their
+ // size and rows are L-lane aligned, so any size divisible by L is correct.
+ //
+ // 32 rather than 16. Both are legal, but requesting 16 forces the kernel to
+ // be compiled SIMD16, which needs twice as many instructions to retire the
+ // same work as SIMD32 and roughly halves ALU throughput. This kernel has
+ // little slack, so that is not free: measured 207 GB/s at SIMD16 against
+ // 357 GB/s at SIMD32 on Arc Pro B60, same code otherwise.
+ static constexpr int kSubGroupSizeCoop = 32;
+ // log2(32): butterfly stages that stay inside a lane's own 32 elements. The
+ // remaining log2(L) stages cross lanes.
+ static constexpr int kNumFwhtStagesLocal = 5;
+
+ // Largest supported cooperative fan-out. L must divide the sub-group size,
+ // so L <= 32; L = 32 would put one whole sub-group on a single row and is not
+ // instantiated because no current model needs D = 1024.
+ static constexpr int kMaxLanesPerRow = 16;
+ static constexpr int kMaxHadamardDim = kGroupSize * kMaxLanesPerRow;
+
+ static constexpr int ct_log2(int n) { return n <= 1 ? 0 : 1 + ct_log2(n / 2); }
+
+ // A transform size is supported iff it is 32 * L for a power-of-two L that
+ // divides the sub-group size and is within the instantiated range.
+ static constexpr bool is_supported_hadamard_dim(int64_t dim) {
+ return dim >= kGroupSize && dim <= kMaxHadamardDim && dim % kGroupSize == 0 &&
+ (dim / kGroupSize) * kGroupSize == dim && ((dim / kGroupSize) & (dim / kGroupSize - 1)) == 0;
+ }
+
+ // FP4 (E2M1) magnitude levels: 0, 0.5, 1, 1.5, 2, 3, 4, 6.
+ // Thresholds and boundary comparison operators are taken verbatim from the
+ // PyTorch reference (auto_round_extension/vllm_ext/fp4_utils.py::cast_to_fp4)
+ // so that the kernel is bit-exact with it.
+ //
+ // Written branchless, as a descending count rather than the reference's
+ // early-out if-ladder, because this runs 32 times per work-item on data whose
+ // magnitudes are unrelated across neighbouring SIMD lanes. An if-ladder there
+ // diverges on essentially every group and the hardware has to execute all
+ // eight arms under mask, which is enough to turn this memory-bound kernel
+ // ALU-bound: measured 386 -> 397 GB/s at D = 32 and 357 -> 397 GB/s at
+ // D = 128 on Arc Pro B60.
+ //
+ // The descending form is required, not merely convenient. Counting *up* with
+ // the complementary predicates (a > 0.25f, a >= 0.75f, ...) agrees with the
+ // ladder on every ordered input but maps NaN to 0, whereas the ladder falls
+ // through to 7. Counting down with the ladder's own predicates reproduces
+ // that: for NaN every comparison is false, giving 7 - 0. Verified exhaustively
+ // against the if-ladder over all 2^32 float bit patterns: zero mismatches.
+ static inline int e2m1_magnitude_index(float a) {
+ return 7 - (static_cast<int>(a <= 5.0f) + static_cast<int>(a < 3.5f) + static_cast<int>(a <= 2.5f) +
+ static_cast<int>(a < 1.75f) + static_cast<int>(a <= 1.25f) + static_cast<int>(a < 0.75f) +
+ static_cast<int>(a <= 0.25f));
+ }
+
+ // Quantize one 32-element group held in registers: E8M0 exponent + the 16
+ // packed code bytes as four 32-bit words.
+ //
+ // Shared by every path that owns a whole group in a single work-item (the
+ // D = 32 per-item kernel and each lane of the cooperative kernel at any
+ // D > 32), so that all of them are bit-exact against the same PyTorch
+ // reference. The operation order is part of the frozen contract: ilogb for the
+ // exponent, a per-element ldexp for the scaling, and the branchless threshold
+ // count above -- never a direct read of the exponent field, which handles a
+ // non-finite amax differently (see xpu_mxfp4_hadamard_xmx.hpp).
+ static inline void quant_group32(const float* v, sycl::vec<uint32_t, 4>& packed, uint8_t& e8m0) {
+ float amax = 0.0f;
+#pragma unroll
+ for (int i = 0; i < kGroupSize; ++i) {
+ amax = sycl::fmax(amax, sycl::fabs(v[i]));
+ }
+
+ e8m0 = 0;
+ int exp_shift = 0;
+ if (amax > 0.0f) {
+ int biased = sycl::ilogb(amax) - 2 + 127;
+ biased = biased < 0 ? 0 : (biased > 254 ? 254 : biased);
+ e8m0 = static_cast<uint8_t>(biased);
+ exp_shift = biased - 127;
+ }
+
+ packed = sycl::vec<uint32_t, 4>(0u);
+ if (amax > 0.0f) {
+#pragma unroll
+ for (int i = 0; i < kGroupSize; ++i) {
+ const float qv = sycl::ldexp(v[i], -exp_shift);
+ const int idx = e2m1_magnitude_index(sycl::fabs(qv));
+ // Canonical zero: never emit 0x8 (negative zero). A value that rounds
+ // to magnitude 0 may carry either sign depending on FP32 rounding
+ // residue and on flush-to-zero, so the sign is dropped.
+ const int code = (idx == 0) ? 0 : ((sycl::signbit(qv) ? 8 : 0) | idx);
+ // Even element -> low nibble of its byte.
+ packed[i >> 3] |= static_cast<uint32_t>(code & 0xF) << ((i & 7) * 4);
+ }
+ }
+ }
+
+ // Fast path: one work-item owns one full 32-element group.
+ //
+ // The original design gave each *lane* one element, so a 32-lane sub-group
+ // handled a single group: 134M work-items each moving 2 bytes, and 16 separate
+ // 1-byte stores per group. Byte-granularity stores force read-modify-write on
+ // cache lines and the per-work-item overhead dwarfs the actual work, which
+ // pinned the kernel near 33 GB/s regardless of how cheap the transform was --
+ // replacing the O(D^2) dot product with the butterfly changed nothing.
+ //
+ // Here a work-item loads its whole group as 16-byte vectors, runs the
+ // butterflies entirely in registers (no sub-group shuffles at all), and emits
+ // the 16 packed code bytes as a single aligned 16-byte store. Because the
+ // activation is contiguous, group g occupies exactly x[g*32 .. g*32+31],
+ // codes bytes [g*16 .. g*16+15] and scale byte g.
+ //
+ // The group is read straight from global memory with two 16-byte vector loads
+ // per work-item. An earlier revision staged the work-group's whole slab
+ // through SLM first, on the theory that a work-item reading 64 contiguous
+ // bytes makes neighbouring work-items stride 64B apart and therefore
+ // uncoalesced. Measured on Arc Pro B60 that staging *cost* 2.3x: the transpose
+ // read tile[lt * 32 + i] has every lane hitting the same SLM bank for a given
+ // i (stride 64B = 16 dwords = the bank count), so it serializes 16-32 ways;
+ // padding the slot to 34 elements removes the conflict but the extra SLM
+ // round trip plus the barrier still leaves it slower than not staging at all.
+ // The direct path is fine because a 64B-per-work-item stride is exactly one
+ // cache line per work-item: the loads are already at full line granularity, so
+ // there is nothing for a staging buffer to coalesce. Direct + vector loads
+ // measures 1.01-1.03x the streaming-copy baseline versus 0.31-0.43x staged.
+ //
+ // Bit-exactness is unaffected: the butterfly order is identical to the SLM
+ // version, to the sub-group version and to fwht_transform_reference.
+ //
+ // Quant-only mode (``quant_only = true``) is the controlled baseline used to
+ // attribute bandwidth: it strips the Hadamard transform (forces ``norm`` to
+ // 1 and skips the butterfly stages) while keeping the exact same loads,
+ // packing and stores. The raw activation is then quantized directly, so the
+ // memory traffic -- and therefore the bandwidth -- is directly comparable to
+ // the fused path.
+ template <typename T>
+ static void fwht_quant_per_item(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,
+ uint8_t* out_scale, int64_t total_groups, bool quant_only = false) {
+ const int64_t num_wg = (total_groups + kWorkGroupSize - 1) / kWorkGroupSize;
+ const size_t global_size = static_cast<size_t>(num_wg) * kWorkGroupSize;
+ // 16-byte vector loads: 8 halves per chunk, 4 chunks per 32-element group.
+ constexpr int kLoadVecElems = 8;
+ constexpr int kLoadVecCount = kGroupSize / kLoadVecElems;
+
+ q->parallel_for(sycl::nd_range<1>(global_size, kWorkGroupSize), [=](sycl::nd_item<1> item) {
+ const int64_t gid = static_cast<int64_t>(item.get_global_id(0));
+ if (gid >= total_groups) {
+ return;
+ }
+
+ // hadamard[0] == H[0][0] == 1/sqrt(32); applied before the butterflies so
+ // intermediates stay bounded by sqrt(32)*max|x| (see the header comment).
+ // Quant-only mode forces norm to 1 (hadamard is *not* dereferenced: the
+ // ternary only evaluates the taken branch) and skips the butterflies
+ // below, so the raw activation is quantized directly.
+ const float norm = quant_only ? 1.0f : hadamard[0];
+
+ // x + gid * 32 is 64-byte aligned for T = half/bfloat16, so each chunk
+ // load is an aligned 16-byte access.
+ const auto* src = reinterpret_cast<const sycl::vec<T, kLoadVecElems>*>(x + gid * kGroupSize);
+
+ float v[kGroupSize];
+#pragma unroll
+ for (int c = 0; c < kLoadVecCount; ++c) {
+ const sycl::vec<T, kLoadVecElems> chunk = src[c];
+#pragma unroll
+ for (int i = 0; i < kLoadVecElems; ++i) {
+ v[c * kLoadVecElems + i] = static_cast<float>(chunk[i]) * norm;
+ }
+ }
+
+ if (!quant_only) {
+#pragma unroll
+ for (int stage = 0; stage < kNumFwhtStages; ++stage) {
+ const int h = 1 << stage;
+#pragma unroll
+ for (int i = 0; i < kGroupSize; ++i) {
+ if ((i & h) == 0) {
+ const float a = v[i];
+ const float b = v[i ^ h];
+ v[i] = a + b;
+ v[i ^ h] = a - b;
+ }
+ }
+ }
+ }
+
+ // Pack 32 codes into 16 bytes, emitted as four 32-bit words. gid*16 is
+ // 16-byte aligned, so this is a single aligned vector store.
+ sycl::vec<uint32_t, 4> packed;
+ uint8_t e8m0;
+ quant_group32(v, packed, e8m0);
+
+ auto* dst = reinterpret_cast<sycl::vec<uint32_t, 4>*>(out_codes + gid * (kGroupSize / 2));
+ *dst = packed;
+ out_scale[gid] = e8m0;
+ });
+ }
+
+ // Stream-only baseline: the traffic-matched roofline for every fused path.
+ //
+ // Keeps the fused kernel's item mapping (one work-item = 32 contiguous
+ // elements = 16 code bytes + 1 scale byte), its two 16-byte vector loads and
+ // its single 16-byte vector store, but replaces the transform *and* the
+ // quantization math with the cheapest possible data-dependent function: the
+ // low nibble of each element's raw bit pattern, packed with the same
+ // even-element-low-nibble rule, and an XOR fold for the scale byte.
+ //
+ // Why this and not ``dst.copy_(src)``. A streaming copy has a 1:1 read:write
+ // ratio; the fused kernel reads 64 B and writes 17 B per group, i.e. W/R =
+ // 0.266. Measured on Arc Pro B60 the device's cost per 64 B read is strongly
+ // non-linear in W/R (0.144 ns at W/R = 0, 0.187 at 0.25, 0.414 at 1.0), and a
+ // hand-written 1:1 kernel reaches only 310 GB/s where ``memcpy`` reaches 407
+ // -- so a copy differs from the fused kernel in *both* the traffic ratio and
+ // the instruction mix, and its bandwidth is not an attainable target. This
+ // baseline differs in neither: it is the same kernel with the arithmetic
+ // removed, so ``BW_fused / BW_stream`` is a true utilization figure and stays
+ // meaningful when the dtype, the shape or the Hadamard dimension changes.
+ //
+ // The result is deterministic and has a PyTorch reference
+ // (``mxfp4_stream_reference``) precisely so that a test can prove the loads
+ // were not optimized away -- a dead-code-eliminated baseline would report an
+ // absurd bandwidth and silently invalidate every ratio computed against it.
+ template <typename T>
+ static void stream_baseline_per_item(sycl::queue* q, const T* x, uint8_t* out_codes, uint8_t* out_scale,
+ int64_t total_groups) {
+ const int64_t num_wg = (total_groups + kWorkGroupSize - 1) / kWorkGroupSize;
+ const size_t global_size = static_cast<size_t>(num_wg) * kWorkGroupSize;
+ constexpr int kLoadVecElems = 8;
+ constexpr int kLoadVecCount = kGroupSize / kLoadVecElems;
+
+ q->parallel_for(sycl::nd_range<1>(global_size, kWorkGroupSize), [=](sycl::nd_item<1> item) {
+ const int64_t gid = static_cast<int64_t>(item.get_global_id(0));
+ if (gid >= total_groups) {
+ return;
+ }
+
+ const auto* src = reinterpret_cast<const sycl::vec<T, kLoadVecElems>*>(x + gid * kGroupSize);
+
+ sycl::vec<uint32_t, 4> packed(0u);
+ uint32_t fold = 0u;
+#pragma unroll
+ for (int c = 0; c < kLoadVecCount; ++c) {
+ const sycl::vec<T, kLoadVecElems> chunk = src[c];
+ uint32_t word = 0u;
+#pragma unroll
+ for (int i = 0; i < kLoadVecElems; ++i) {
+ const uint32_t bits = static_cast<uint32_t>(sycl::bit_cast<uint16_t>(chunk[i]));
+ word |= (bits & 0xFu) << (i * 4);
+ }
+ packed[c] = word;
+ // Every loaded element feeds the stored result, so nothing can be
+ // eliminated; the fold also produces the scale byte.
+ fold ^= word;
+ }
+
+ auto* dst = reinterpret_cast<sycl::vec<uint32_t, 4>*>(out_codes + gid * (kGroupSize / 2));
+ *dst = packed;
+ out_scale[gid] = static_cast<uint8_t>(fold & 0xFFu);
+ });
+ }
+
+ // D = 128 FWHT + MXFP4, four cooperating lanes per row.
+ //
+ // A 128-point transform does not fit the one-work-item-per-row shape that
+ // works so well at D = 32: 128 live FP32 values exceed the default per-thread
+ // register budget and the kernel spills catastrophically (measured 140 GB/s,
+ // a third of the roofline; raising the GRF to 256 recovers only 317 GB/s
+ // because occupancy drops).
+ //
+ // Instead the transform is factored as H_D = H32 (x) H_L, with L = D / 32
+ // lanes per row. Lane l of a group of L owns the contiguous chunk
+ // [l*32, l*32+32):
+ //
+ // * stages 0..4 (h = 1, 2, 4, 8, 16) are entirely inside the lane's own 32
+ // values -- bit-for-bit the same butterfly as the D = 32 kernel;
+ // * stages 5..(5 + log2(L) - 1) (h = 32, 64, ...) become cross-lane
+ // butterflies against lane ^1, ^2, ^4, ..., i.e. log2(L) * 32 sub-group
+ // shuffles.
+ //
+ // The 32-element quantization group is exactly one lane's chunk, so absmax,
+ // E8M0, encoding, packing and the 16-byte store are all lane-local: no SLM,
+ // no barriers, and no cross-lane reduction anywhere in the quantizer. This is
+ // the property that makes the fan-out cheap, and it holds for every L: the
+ // per-lane register footprint is a constant 32 floats, so unlike a
+ // one-work-item-per-row formulation nothing spills as D grows. Only the
+ // shuffle count grows, and it grows as log2(L).
+ //
+ // Measured on Arc Pro B60 at the attention QKV shape ([1, 75600, 40, 128],
+ // bf16): 395 GB/s against a 391 GB/s stream-only roofline, versus 140 GB/s
+ // for one work-item per row (register spill) and 318 GB/s for two lanes per
+ // row. Note that raising the GRF size *hurts* here (276 GB/s), the opposite
+ // of the single-work-item variant.
+ //
+ // Convergence. The cross-lane stages are sub-group collectives, so every lane
+ // of the sub-group must reach them. Out-of-range lanes in the tail therefore
+ // do *not* return early: they clamp their row index to 0, participate in the
+ // shuffles with valid data, and are masked off only at the store. Because
+ // rows are L-lane aligned and the sub-group size is a multiple of L, the L
+ // lanes of a row always live in the same sub-group.
+ //
+ // L = 1 instantiates correctly (zero cross-lane stages, degenerating to the
+ // D = 32 butterfly) but is not what the dispatcher routes D = 32 to: that
+ // case goes to fwht_quant_per_item, which also carries the quant-only and
+ // stream-only ablations and must keep the exact item mapping they compare
+ // against.
+ //
+ // Normalization placement. Unlike the D = 32 kernel, which scales by
+ // 1/sqrt(D) on load, this kernel scales *after* the butterflies. Scaling on
+ // load would leave a multiply feeding directly into the first stage's add,
+ // and the Intel GPU compiler fuses that pair into an FMA. The fused form
+ // rounds differently from the reference's separate multiply and add, and the
+ // resulting one-ULP error propagates through every later stage: measured on
+ // Arc Pro B60, 60% of the transform outputs differed at D = 128 and D = 512,
+ // which showed up downstream as 17 and 25 wrong MXFP4 codes per 67 M -- only
+ // groups whose scaled magnitude landed within an ULP of an E2M1 threshold
+ // flipped, which is exactly the kind of rate that small unit-test shapes miss.
+ //
+ // Neither -ffp-contract=off nor a #pragma clang fp contract(off) suppresses
+ // it (both were measured to change nothing: the fusion happens in the GPU
+ // backend, below the point those act on), and a bit_cast round trip through
+ // uint32_t does not either. Moving the multiply past the butterflies removes
+ // the mul/add pair altogether: what remains multiplies into fabs, fmax and
+ // the comparison ladder of the quantizer, none of which is an add. That is a
+ // structural fix rather than a request the compiler may ignore, and it costs
+ // nothing measurable (235 vs 237 GB/s).
+ //
+ // Note this is *why* D = 64 and D = 256 never failed: their 1/sqrt(D) is
+ // exactly 1/8 and 1/16, so the load-time product was exact and fusing it
+ // changed nothing. Only the irrational norms exposed the bug.
+ //
+ // The cost is the intermediate range: values now grow to D*max|x| before
+ // being scaled down, rather than staying at sqrt(D)*max|x|. In FP32 that is
+ // immaterial here -- FP16 activations cap the intermediate at 512*65504 =
+ // 3.4e7 -- but it does mean this path and the D = 32 path are not bit-exact
+ // against each other, and the torch reference mirrors the split.
+ template <typename T, int kLanes>
+ static void fwht_quant_cooperative(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,
+ uint8_t* out_scale, int64_t num_rows) {
+ static_assert(kLanes >= 1 && kLanes <= kMaxLanesPerRow, "lanes per row out of range");
+ static_assert((kLanes & (kLanes - 1)) == 0, "lanes per row must be a power of two");
+ static_assert(kSubGroupSizeCoop % kLanes == 0,
+ "lanes per row must divide the sub-group size, otherwise a row's lanes can straddle two "
+ "sub-groups and the cross-lane butterfly reads the wrong partner");
+
+ constexpr int kDim = kGroupSize * kLanes;
+ constexpr int kCrossStages = ct_log2(kLanes);
+
+ const int64_t total_items = num_rows * kLanes;
+ const int64_t num_wg = (total_items + kWorkGroupSize - 1) / kWorkGroupSize;
+ const size_t global_size = static_cast<size_t>(num_wg) * kWorkGroupSize;
+ constexpr int kLoadVecElems = 8;
+ constexpr int kLoadVecCount = kGroupSize / kLoadVecElems;
+
+ q->parallel_for(
+ sycl::nd_range<1>(global_size, kWorkGroupSize),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(kSubGroupSizeCoop)]] {
+ auto sg = item.get_sub_group();
+ const int64_t gid = static_cast<int64_t>(item.get_global_id(0));
+ const int64_t row = gid / kLanes;
+ const int lane = static_cast<int>(gid % kLanes);
+ const int sg_lane = static_cast<int>(sg.get_local_id()[0]);
+
+ // Tail lanes stay converged for the sub-group shuffles below by
+ // re-reading row 0; only their store is suppressed.
+ const bool active = row < num_rows;
+ const int64_t safe_row = active ? row : 0;
+
+ // hadamard[0] == H[0][0] == 1/sqrt(D), applied *after* the butterflies
+ // -- see the normalization-placement note above the function.
+ const float norm = hadamard[0];
+
+ const auto* src =
+ reinterpret_cast<const sycl::vec<T, kLoadVecElems>*>(x + safe_row * kDim + lane * kGroupSize);
+
+ float v[kGroupSize];
+#pragma unroll
+ for (int c = 0; c < kLoadVecCount; ++c) {
+ const sycl::vec<T, kLoadVecElems> chunk = src[c];
+#pragma unroll
+ for (int i = 0; i < kLoadVecElems; ++i) {
+ v[c * kLoadVecElems + i] = static_cast<float>(chunk[i]);
+ }
+ }
+
+ // Stages 0..4: h < 32, both partners live in this lane.
+#pragma unroll
+ for (int stage = 0; stage < kNumFwhtStagesLocal; ++stage) {
+ const int h = 1 << stage;
+#pragma unroll
+ for (int i = 0; i < kGroupSize; ++i) {
+ if ((i & h) == 0) {
+ const float a = v[i];
+ const float b = v[i ^ h];
+ v[i] = a + b;
+ v[i ^ h] = a - b;
+ }
+ }
+ }
+
+ // Remaining stages: h = 32, 64, ... The partner element lives in lane
+ // ^1, ^2, ^4, ... at the same offset within the chunk. The lane
+ // holding the high half of the pair computes the difference, matching
+ // the D = 32 butterfly's orientation.
+#pragma unroll
+ for (int stage = 0; stage < kCrossStages; ++stage) {
+ const int lane_h = 1 << stage;
+ const int partner = sg_lane ^ lane_h;
+ const bool is_low = (lane & lane_h) == 0;
+#pragma unroll
+ for (int i = 0; i < kGroupSize; ++i) {
+ const float other = sycl::select_from_group(sg, v[i], partner);
+ v[i] = is_low ? (v[i] + other) : (other - v[i]);
+ }
+ }
+
+ // Normalization, applied last (see the note above the function). The
+ // product feeds fabs/fmax and the comparison ladder inside the
+ // quantizer, never an add, so there is nothing here to contract.
+#pragma unroll
+ for (int i = 0; i < kGroupSize; ++i) {
+ v[i] *= norm;
+ }
+
+ sycl::vec<uint32_t, 4> packed;
+ uint8_t e8m0;
+ quant_group32(v, packed, e8m0);
+
+ if (active) {
+ // Group index within the flattened [total_groups, 32] view: the
+ // row's L groups are contiguous, so this matches the D = 32 layout.
+ const int64_t group_id = row * kLanes + lane;
+ auto* dst = reinterpret_cast<sycl::vec<uint32_t, 4>*>(out_codes + group_id * (kGroupSize / 2));
+ *dst = packed;
+ out_scale[group_id] = e8m0;
+ }
+ });
+ }
+
+ // Runtime dim -> compile-time lane count. Every cooperative D (i.e. every
+ // supported size above 32) gets its own fully unrolled instantiation; an
+ // unsupported one is rejected here as a backstop, the wrapper and the binding
+ // having already validated it.
+ template <typename T>
+ static bool dispatch_cooperative(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,
+ uint8_t* out_scale, int64_t num_rows, int64_t hadamard_dim) {
+ switch (hadamard_dim) {
+ case kGroupSize * 2:
+ fwht_quant_cooperative<T, 2>(q, x, hadamard, out_codes, out_scale, num_rows);
+ return true;
+ case kGroupSize * 4:
+ fwht_quant_cooperative<T, 4>(q, x, hadamard, out_codes, out_scale, num_rows);
+ return true;
+ case kGroupSize * 8:
+ fwht_quant_cooperative<T, 8>(q, x, hadamard, out_codes, out_scale, num_rows);
+ return true;
+ case kGroupSize * 16:
+ fwht_quant_cooperative<T, 16>(q, x, hadamard, out_codes, out_scale, num_rows);
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ // in: x [num_rows, k] (T = sycl::half or bfloat16)
+ // hadamard [32, 32] (FP32, row major, normalized)
+ // out: codes [num_rows, k / 2] (uint8, two FP4 codes per byte)
+ // scale [num_rows, k / 32] (uint8, one E8M0 exponent per group)
+ //
+ // Path A fallback for a caller-supplied non-Sylvester matrix. Bit-exact
+ // against hadamard_transform_reference; deliberately *not* bit-exact against
+ // the FWHT path above, because a butterfly network and a 32-term dot product
+ // round differently.
+ template <typename T>
+ static void mxfp4_hadamard_quant_impl(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,
+ uint8_t* out_scale, int64_t num_rows, int64_t k) {
+ constexpr int groups_per_wg = kWorkGroupSize / kSubGroupSize;
+ const int64_t groups_per_row = k / kGroupSize;
+ const int64_t total_groups = num_rows * groups_per_row;
+ if (total_groups <= 0) {
+ return;
+ }
+ const int64_t num_wg = (total_groups + groups_per_wg - 1) / groups_per_wg;
+ const size_t global_size = static_cast<size_t>(num_wg) * kWorkGroupSize;
+
+ q->parallel_for(sycl::nd_range<1>(global_size, kWorkGroupSize),
+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(kSubGroupSize)]] {
+ auto sg = item.get_sub_group();
+ const int lane = static_cast<int>(sg.get_local_id()[0]);
+ const int64_t group_id = static_cast<int64_t>(item.get_group(0)) * groups_per_wg +
+ static_cast<int64_t>(sg.get_group_id()[0]);
+ // Tail work-groups: the whole sub-group exits together, so the
+ // sub-group collectives below stay converged.
+ if (group_id >= total_groups) {
+ return;
+ }
+
+ const int64_t row = group_id / groups_per_row;
+ const int64_t group_in_row = group_id % groups_per_row;
+ const int64_t base = row * k + group_in_row * kGroupSize;
+
+ // Path A: generic 32x32 matrix multiply. Lane i owns column i of
+ // H, values of x are broadcast one by one inside the sub-group.
+ //
+ // The Path A accumulation contract (see mxfp4_hadamard.py)
+ // requires increasing j, no reassociation and a separate FP32
+ // rounding after the multiply and after the add. Contracting
+ // into an FMA would change near-threshold elements and break
+ // bit-exactness with the reference, so it is disabled here.
+ const float xv = static_cast<float>(x[base + lane]);
+ float acc = 0.0f;
+#pragma unroll
+ for (int j = 0; j < kHadamardDim; ++j) {
+#if defined(__clang__)
+#pragma clang fp contract(off)
+#endif
+ const float xj = sycl::select_from_group(sg, xv, j);
+ acc += xj * hadamard[j * kHadamardDim + lane];
+ }
+
+ const float amax = sycl::reduce_over_group(sg, sycl::fabs(acc), sycl::maximum<float>{});
+
+ uint8_t e8m0 = 0;
+ int code = 0;
+ if (amax > 0.0f) {
+ // floor(log2(amax)) is exact through ilogb, including for
+ // exact powers of two and subnormal inputs.
+ int biased = sycl::ilogb(amax) - 2 + 127;
+ biased = biased < 0 ? 0 : (biased > 254 ? 254 : biased);
+ e8m0 = static_cast<uint8_t>(biased);
+ const float qv = sycl::ldexp(acc, -(biased - 127));
+ const int idx = e2m1_magnitude_index(sycl::fabs(qv));
+ const int sign = sycl::signbit(qv) ? 1 : 0;
+ // Canonical zero: never emit 0x8 (negative zero). A value
+ // that rounds to magnitude 0 may carry either sign
+ // depending on FP32 accumulation residue and on whether
+ // the device flushes subnormals, so the sign is dropped.
+ code = (idx == 0) ? 0 : ((sign << 3) | idx);
+ }
+
+ // Even lane keeps the low nibble, its odd neighbour the high one.
+ const int partner = ((lane & 1) == 0) ? (lane + 1) : lane;
+ const int hi_code = sycl::select_from_group(sg, code, partner);
+ if ((lane & 1) == 0) {
+ const int64_t byte_idx = (base + lane) >> 1;
+ out_codes[byte_idx] = static_cast<uint8_t>((code & 0xF) | ((hi_code & 0xF) << 4));
+ }
+ if (lane == 0) {
+ out_scale[row * groups_per_row + group_in_row] = e8m0;
+ }
+ });
+ }
+
+ // Dispatcher.
+ //
+ // ``hadamard_dim`` selects the transform size: 32 (the GEMM activation case,
+ // one work-item per group) or 32 * L for L in {2, 4, 8, 16} -- i.e. 64, 128,
+ // 256, 512 -- which fan a row out over L cooperating lanes. Every size
+ // quantizes in groups of 32 and shares the output layout, so the two baseline
+ // modes below are dimension-agnostic: they only ever see a flat sequence of
+ // 32-element groups.
+ //
+ // ``stream_only`` and ``quant_only`` are mutually exclusive; the caller is
+ // expected to have rejected the combination already.
+ template <typename T>
+ static void mxfp4_hadamard_quant(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,
+ uint8_t* out_scale, int64_t num_rows, int64_t k, bool use_fwht,
+ bool quant_only = false, int64_t hadamard_dim = kHadamardDim,
+ bool stream_only = false) {
+ const int64_t total_groups = num_rows * (k / kGroupSize);
+ if (total_groups <= 0) {
+ return;
+ }
+
+ if (stream_only) {
+ // Traffic-matched roofline: same loads, same packing shape, same stores,
+ // no transform and no quantization math. hadamard is not read.
+ stream_baseline_per_item<T>(q, x, out_codes, out_scale, total_groups);
+ return;
+ }
+ if (quant_only) {
+ // Quant-only baseline: strip the Hadamard transform entirely. The
+ // per-item layout and memory traffic are identical to the fused FWHT
+ // path, so bandwidths are directly comparable; hadamard is not read.
+ fwht_quant_per_item<T>(q, x, nullptr, out_codes, out_scale, total_groups, /*quant_only=*/true);
+ return;
+ }
+
+ if (hadamard_dim > kGroupSize) {
+ // D > 32 supports the Sylvester matrix only; the caller enforces this.
+ // The dispatch result is checked rather than discarded: on an unsupported
+ // dim no kernel is enqueued, and silently returning would leave
+ // out_codes / out_scale holding whatever the caller allocated -- no
+ // error, just wrong bytes.
+ const bool dispatched = dispatch_cooperative<T>(q, x, hadamard, out_codes, out_scale,
+ num_rows * (k / hadamard_dim), hadamard_dim);
+ if (!dispatched) {
+ throw std::invalid_argument(
+ "ark::XpuMxfp4Hadamard: hadamard_dim is not a dispatched cooperative transform size");
+ }
+ return;
+ }
+
+ if (use_fwht) {
+ fwht_quant_per_item<T>(q, x, hadamard, out_codes, out_scale, total_groups);
+ } else {
+ mxfp4_hadamard_quant_impl<T>(q, x, hadamard, out_codes, out_scale, num_rows, k);
+ }
+ }
+};
+
+} // namespace ark
+
+#endif // ARK_XPU
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard_xmx.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard_xmx.hpp
new file mode 100644
index 000000000..a871f48ea
--- /dev/null
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard_xmx.hpp
@@ -0,0 +1,272 @@
+//
+// Copyright (c) 2026 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// XMX (Xe matrix / DPAS) path for the activation fused 32-point Hadamard
+// transform + MXFP4 quantization. Opt-in via ``use_xmx``; deliberately a
+// *relaxed* numerical contract: the Hadamard matrix is stored in FP16/BF16
+// (same dtype as the activation) and the transform runs on XMX DPAS with FP32
+// accumulation, which is neither bit-exact with the FWHT path nor with Path A.
+// Acceptance is tolerance based, and is enforced by
+// test/test_mxfp4_hadamard.py::TestXpuKernelXmx: SQNR >= 15 dB, no code off by
+// more than one E2M1 level, and a code-disagreement rate below 0.1%.
+//
+// Non-finite amax is out of contract and deliberately unspecified. This path
+// reads the FP32 exponent field rather than calling ilogb, so amax = +inf gives
+// e8m0 = 253 instead of the 254 that the ilogb-based FWHT paths and the PyTorch
+// reference saturate to, and NaN fails the ``amax > 0`` test and falls through
+// as an all-zero group. The two quantizer ladders also disagree on NaN by
+// construction -- the reference initialises the magnitude index to 7 and only
+// lowers it on a threshold hit, while this one sums the comparisons, so NaN
+// lands on 0 -- which puts byte-parity on non-finite input out of reach without
+// changing the bit-exact ladder the other paths rely on. Reaching it takes an
+// already-non-finite activation: with fp16/bf16 operands the 32-term FP32
+// accumulation peaks around 1.4e11, well inside the finite range.
+//
+// This is not a bandwidth optimization: it exists for non-Sylvester matrices,
+// where the butterfly (and therefore the FWHT path) is unavailable, and it is
+// not uniformly faster than FWHT on the shapes measured so far.
+//
+// The fused kernel computes, for each 32-element group ``g`` (one row of the
+// flattened activation ``[total_groups, 32]``):
+//
+// y[g][i] = sum_j H_T[j][i] * x[g][j] (DPAS, FP32 accumulate)
+//
+// i.e. ``y = H_T @ x^T`` with H_T the Hadamard matrix stored as T (fp16/bf16).
+// Each XMX lane owns one full group (its 32 outputs live entirely in the lane's
+// fragment), so absmax / E8M0 / E2M1 / packing / writeback are all lane-local:
+// no SLM, no barriers, no cross-lane shuffles. The MMA's real N-tile is 64 (not
+// the 32 passed to choose_tiled_mma_tile), so the global group index uses
+// ``get<1>(mma.tile_mnk())`` as the per-workgroup stride.
+//
+// Measured on Arc Pro B60 (m = 262144 groups, BF16, memory-bound): ~335-418 GB/s
+// vs ~186 GB/s Path A and ~395 GB/s streaming-copy baseline.
+
+#pragma once
+
+#include <cstdint>
+
+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)
+
+#include <sycl/sycl.hpp>
+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>
+
+// Only the cute layer is needed (XE_DPAS atoms, block-2D copies, tiled-MMA
+// helper). The cutlass-facing sycl_tla_dense_gemm.hpp is deliberately NOT
+// included: pulling in cutlass headers makes cute::half_t resolve to
+// cutlass::half_t and breaks the bf16 reorder (ambiguous conversion).
+// cute/tensor.hpp brings in the XE_DPAS traits; do not include
+// cute/atom/mma_traits_xe.hpp explicitly (it pulls in cutlass::half_t).
+#include "cute/tensor.hpp"
+
+namespace ark {
+namespace xmx_hadamard_detail {
+
+using namespace cute;
+
+// The DPAS atom is pinned to half input: bf16 values (8-bit mantissa) convert
+// losslessly to half (10-bit mantissa) in range, so one atom serves both
+// activation dtypes. Verified against the fp32 reference for both fp16 and bf16
+// on Arc Pro B60 by
+// test/test_mxfp4_hadamard.py::TestXpuKernelXmx::test_xmx_matches_reference_within_tolerance.
+template <typename TA, typename TB, typename TC>
+auto choose_mma_op() {
+ return XE_DPAS_TT<8, float, cute::half_t>{};
+}
+
+template <int TileM, int TileN, class SGLayout, class ATensor, class BTensor, class CTensor>
+auto choose_tiled_mma_tile(ATensor const& A, BTensor const& B, CTensor const&) {
+ using TA = typename ATensor::element_type;
+ using TB = typename BTensor::element_type;
+ using TC = typename CTensor::element_type;
+
+ auto op = choose_mma_op<TA, TB, TC>();
+
+ constexpr bool byte = (cute::max(sizeof_bits_v<TA>, sizeof_bits_v<TB>) <= 8);
+ constexpr bool a_t = is_constant_v<1, decltype(stride<0>(A))>;
+ constexpr bool b_n = is_constant_v<1, decltype(stride<0>(B))>;
+ constexpr bool use_1x_dpas_per_k = a_t || (byte && b_n);
+
+ using _K = conditional_t<use_1x_dpas_per_k, C<op.K>, C<op.K * 2>>;
+ using WGTile = Shape<Int<TileM>, Int<TileN>, _K>;
+ using MMA = typename TiledMMAHelper<MMA_Atom<decltype(op)>, Layout<WGTile>, SGLayout>::TiledMMA;
+
+ return MMA{};
+}
+
+// A = H_T [32, 32] (row-major, T = fp16/bf16), B = activation [m, 32] (row-major).
+// Each workgroup handles one n-tile of tile_n = 64 groups; each lane owns one
+// full group (a 32-row column of the C tile), so the whole quant is lane-local.
+template <int TileM, int TileN, class ATensor, class BTensor, class TiledMMA>
+void fused_core_xmx(ATensor const& A, BTensor const& B, uint8_t* out_codes, uint8_t* out_scale, int64_t m,
+ TiledMMA const& mma) {
+ auto item = sycl::ext::oneapi::this_work_item::get_nd_item<2>();
+ auto wg_m = int(item.get_group(1));
+ auto wg_n = int(item.get_group(0));
+ auto local_id = int(item.get_local_id(0));
+
+ auto C = make_tensor(make_gmem_ptr(static_cast<float*>(nullptr)), make_shape(m, Int<TileN>{}),
+ make_stride(Int<TileN>{}, Int<1>{}));
+
+ Tensor cA = make_identity_tensor(A.shape());
+ Tensor cB = make_identity_tensor(B.shape());
+ Tensor cC = make_identity_tensor(C.shape());
+
+ auto wg_tile = mma.tile_mnk();
+ auto wg_coord = make_coord(wg_m, wg_n, 0);
+
+ Tensor gA = local_tile(cA, select<0, 2>(wg_tile), make_coord(wg_m, _));
+ Tensor gB = local_tile(cB, select<1, 2>(wg_tile), make_coord(wg_n, _));
+ Tensor gC = local_tile(cC, wg_tile, wg_coord, Step<_1, _1, X>{});
+
+ auto copy_a = make_block_2d_copy_A(mma, A);
+ auto copy_b = make_block_2d_copy_B(mma, B);
+
+ auto thr_mma = mma.get_slice(local_id);
+ auto thr_copy_a = copy_a.get_slice(local_id);
+ auto thr_copy_b = copy_b.get_slice(local_id);
+
+ auto tCrA = thr_mma.partition_sg_fragment_A(gA(_, _, 0));
+ auto tCrB = thr_mma.partition_sg_fragment_B(gB(_, _, 0));
+ auto tArA = thr_copy_a.partition_sg_fragment_D(gA(_, _, 0));
+ auto tBrB = thr_copy_b.partition_sg_fragment_D(gB(_, _, 0));
+
+ Tensor tAgA = thr_copy_a.partition_S(gA);
+ Tensor tBgB = thr_copy_b.partition_S(gB);
+
+ Tensor tCrC = partition_fragment_C(mma, select<0, 1>(wg_tile));
+ Tensor tCgC = thr_mma.partition_C(gC);
+
+ clear(tCrC);
+ copy(copy_a, tAgA(_, _, _, 0), tArA);
+ copy(copy_b, tBgB(_, _, _, 0), tBrB);
+ reorder(tArA, tCrA);
+ reorder(tBrB, tCrB);
+ gemm(mma, tCrA, tCrB, tCrC);
+
+ // ---- lane-local quant (1 lane = 1 full group) ----
+ const int col = get<1>(tCgC(0)); // group within the workgroup tile
+ const int tile_n = get<1>(mma.tile_mnk()); // 64, NOT the template TileN
+ const int64_t g = (int64_t)wg_n * tile_n + col;
+
+ // absmax over the lane's 32 outputs (4 x float8 SIMD).
+ float amax = 0.0f;
+ for (int c = 0; c < 4; ++c) {
+ sycl::vec<float, 8> v;
+ for (int t = 0; t < 8; ++t) v[t] = tCrC(c * 8 + t);
+ const sycl::vec<float, 8> av = sycl::fabs(v);
+ amax = sycl::fmax(amax, sycl::fmax(sycl::fmax(av[0], av[1]), sycl::fmax(av[2], av[3])));
+ amax = sycl::fmax(amax, sycl::fmax(sycl::fmax(av[4], av[5]), sycl::fmax(av[6], av[7])));
+ }
+
+ uint8_t e8m0 = 0;
+ float inv = 0.0f;
+ if (amax > 0.0f) {
+ // e8m0 = biased_exponent - 2, read straight from the fp32 bits (no ilogb).
+ const int b = int(sycl::bit_cast<uint32_t>(amax) >> 23) - 2;
+ const int bcl = b < 0 ? 0 : (b > 254 ? 254 : b);
+ e8m0 = static_cast<uint8_t>(bcl);
+ inv = sycl::bit_cast<float>((254u - static_cast<uint32_t>(bcl)) << 23); // 2^(127-b)
+ }
+
+ uint32_t packed[4] = {0u, 0u, 0u, 0u};
+ if (amax > 0.0f) {
+ for (int c = 0; c < 4; ++c) {
+ sycl::vec<float, 8> v;
+ for (int t = 0; t < 8; ++t) v[t] = tCrC(c * 8 + t);
+ const sycl::vec<float, 8> aq = sycl::fabs(v) * inv;
+ // The same piecewise thresholds and alternating <= / < operators as
+ // XpuMxfp4Hadamard::e2m1_magnitude_index, but counted up from 0 instead of
+ // down from 7. On this target vec comparisons return -1 (true) / 0
+ // (false), so the negated sum is the magnitude index.
+ const sycl::vec<int, 8> idx =
+ -(aq > 0.25f) - (aq >= 0.75f) - (aq > 1.25f) - (aq >= 1.75f) -
+ (aq > 2.5f) - (aq >= 3.5f) - (aq > 5.0f); // 0..7
+ const sycl::vec<uint32_t, 8> u = sycl::bit_cast<sycl::vec<uint32_t, 8>>(v);
+ const sycl::vec<uint32_t, 8> iu = idx.template convert<uint32_t>();
+ const sycl::vec<uint32_t, 8> iszero =
+ sycl::vec<int, 8>(-(idx == sycl::vec<int, 8>(0))).template convert<uint32_t>();
+ // Canonical zero: magnitude index 0 always encodes 0x0, never 0x8.
+ const sycl::vec<uint32_t, 8> code = (iu | (((u >> 31) & 1u) << 3)) & ~(0u - iszero);
+ for (int t = 0; t < 8; ++t) {
+ const int row = c * 8 + t;
+ packed[row >> 3] |= (code[t] & 0xF) << ((row & 7) * 4);
+ }
+ }
+ }
+ if (g < m) {
+ auto* dst4 = reinterpret_cast<sycl::vec<uint32_t, 4>*>(out_codes + g * (32 / 2));
+ *dst4 = sycl::vec<uint32_t, 4>{packed[0], packed[1], packed[2], packed[3]};
+ out_scale[g] = e8m0;
+ }
+}
+
+template <int TileM, int TileN, class SGLayout, class Element>
+void fused_launch_xmx(sycl::queue* q, int64_t m, const Element* h_ptr, const Element* a_ptr, uint8_t* out_codes,
+ uint8_t* out_scale) {
+ // A = H_T [32,32] (row-major), B = activation [m,32] (row-major).
+ auto A = make_tensor(make_gmem_ptr(const_cast<Element*>(h_ptr)), make_shape(32, 32), make_stride(32, _1{}));
+ auto B = make_tensor(make_gmem_ptr(const_cast<Element*>(a_ptr)), make_shape(m, 32), make_stride(32, _1{}));
+ auto C = make_tensor(make_gmem_ptr(static_cast<float*>(nullptr)), make_shape(TileM, TileN), make_stride(TileN, _1{}));
+ auto mma = choose_tiled_mma_tile<TileM, TileN, SGLayout>(A, B, C);
+
+ sycl::range<2> local = {size(mma), 1};
+ sycl::range<2> global = {local[0] * ceil_div(shape<0>(B), get<1>(mma.tile_mnk())),
+ local[1] * ceil_div(shape<0>(A), get<0>(mma.tile_mnk()))};
+
+ namespace syclex = sycl::ext::oneapi::experimental;
+ namespace intelex = sycl::ext::intel::experimental;
+ syclex::properties kernel_props{syclex::sub_group_size<16>, intelex::grf_size<256>};
+
+ q->parallel_for(sycl::nd_range<2>(global, local), kernel_props,
+ [=](sycl::nd_item<2>) { fused_core_xmx<TileM, TileN>(A, B, out_codes, out_scale, m, mma); });
+}
+
+// Convert the FP32 Hadamard matrix to the activation dtype AND transpose it.
+// The fused GEMM is ``C = A @ B^T`` with A = the Hadamard matrix, so ``C[r][c]``
+// = ``sum_k A[r][k] * x[c][k]``. We need ``y[group c][element r] = sum_k x[c][k]
+// * H[k][r]``, which forces ``A[r][k] = H[k][r]`` -- i.e. A must be H^T. The
+// transpose is applied here (once per call, 1024 elements, negligible) so the
+// kernel entry point simply takes the logical (row-major) Hadamard matrix and
+// always produces ``y = x @ H`` regardless of symmetry. The normalized Sylvester
+// matrix is symmetric so a missing transpose is silently masked; a non-symmetric
+// custom matrix is what exposes a missing transpose.
+template <typename T>
+void convert_hadamard_to_dtype(sycl::queue* q, const float* h_fp32, T* h_t) {
+ q->parallel_for(32 * 32, [=](sycl::id<1> i) {
+ const int r = int(i) / 32; // row of H
+ const int c = int(i) % 32; // column of H
+ h_t[c * 32 + r] = static_cast<T>(h_fp32[i]); // write H^T
+ });
+}
+
+// Public entry point. x is the activation [num_rows, k] flattened to
+// [total_groups, 32]; h is the Hadamard matrix [32,32] in the *same* dtype as
+// the activation (T), already normalized by 1/sqrt(32), row-major. The kernel
+// computes y = x @ h (the transpose for the C = A @ B^T GEMM convention is the
+// caller's responsibility -- see convert_hadamard_to_dtype).
+template <typename T>
+void mxfp4_hadamard_quant_xmx(sycl::queue* q, const T* x, const T* h, uint8_t* out_codes, uint8_t* out_scale,
+ int64_t total_groups) {
+ if (total_groups <= 0) {
+ return;
+ }
+ using SmallTileSG = Layout<Shape<_1, _4, _1>, Stride<_0, _1, _0>>;
+ fused_launch_xmx<32, 32, SmallTileSG, T>(q, total_groups, h, x, out_codes, out_scale);
+}
+
+} // namespace xmx_hadamard_detail
+} // namespace ark
+
+#endif // ARK_XPU && ARK_SYCL_TLA
diff --git a/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.py b/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.py
new file mode 100644
index 000000000..04af7f551
--- /dev/null
+++ b/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.py
@@ -0,0 +1,461 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Bandwidth benchmark for the fused Hadamard + MXFP4 quantization XPU kernel.
+
+ bytes = M*K*sizeof(input) + M*K/2 + M*K/32
+ BW = bytes / latency
+
+Choosing the denominator
+------------------------
+
+A bandwidth number only means something relative to what the device could
+actually deliver *for this kernel*, and picking that ceiling is the whole
+difficulty. Three candidates, in increasing order of usefulness:
+
+``copy_same_shape`` (``dst.copy_(src)``) was the original acceptance baseline
+and is now reported for context only. It is not an attainable target, for two
+independent reasons. First, the traffic ratio is wrong: a copy reads and writes
+in a 1:1 ratio, while the fused kernel reads 64 B and writes 17 B per group
+(W/R = 0.266), and the device's cost per byte read is strongly non-linear in
+that ratio -- measured on Arc Pro B60, 0.144 ns per 64 B read at W/R = 0, 0.187
+at W/R = 0.25 and 0.414 at W/R = 1.0. Second, the instruction mix is wrong:
+``memcpy`` reaches 407 GB/s where a hand-written 1:1 SYCL kernel reaches only
+310, so even at a matched ratio the two are not comparable. The copy baseline is
+in fact the faster of the two, so ``f/cp`` reads systematically below the
+stream-only figure on every configuration measured; that is why it is reported
+for context but never gated on.
+
+``quant_only`` strips the Hadamard transform but keeps the quantization, with
+byte-identical traffic. It is a clean *ablation* and answers "what does the
+transform cost?" -- but it is not a roofline, because it is itself subject to
+whatever limits the fused kernel.
+
+``stream_only`` is the baseline this benchmark gates on. It keeps the fused
+kernel's loads, its packing shape and its stores, and removes only the
+transform and the quantization arithmetic. Same bytes, same access pattern,
+same item mapping, no math -- so it *is* the traffic-matched roofline, and
+``BW_fused / BW_stream`` is a true utilization figure. It also re-derives itself
+automatically when the dtype, the shape or the Hadamard dimension changes,
+which none of the alternatives do.
+
+Cache residency
+---------------
+
+At small ``M*K`` the working set fits in device cache and absolute bandwidths
+stop reflecting DRAM at all -- on Arc Pro B60 the 8 MB configurations measure
+over 1 TB/s, which no memory controller on this part can deliver.
+
+Residency is detected from ``stream_only`` itself, not from the copy. The two
+have different footprints for the same ``[M, K]`` (the copy touches
+``4 B/element``, the kernel ``2.53 B/element``), so the kernel stays partly
+resident at sizes where the copy no longer is, and a copy-based test misses it.
+Any configuration whose stream baseline beats a sustained DRAM copy by more
+than ``CACHE_TOLERANCE`` is flagged.
+
+Flagged configurations are reported but excluded from the gate, and the reason
+is worth stating precisely, because it is not merely "the numbers are big".
+Once the data is resident the roofline rises to cache bandwidth, and the
+transform and quantization arithmetic no longer fits underneath it -- the
+kernel stops being memory bound and becomes math bound. Measured: at
+``[2048, 2048]`` bf16 the stream baseline reaches ~1050 GB/s while the fused
+kernel reaches ~420, an ``f/s`` of 0.40, whereas the same kernel sits at
+0.98-1.01 once the working set spills to DRAM. Both numbers are real; they
+answer different questions, and only the DRAM-bound one answers "does this
+kernel saturate memory?", which is what the gate is for.
+"""
+
+from __future__ import annotations
+
+import argparse
+import statistics
+import sys
+import time
+from pathlib import Path
+
+import torch
+
+# Import the *installed* auto_round_kernel: the benchmark must exercise the
+# compiled XPU extension, and the in-tree source directory has no .so beside it.
+# Only fall back to the source tree if the package is not installed at all.
+try:
+ import auto_round_kernel # noqa: F401
+except ImportError: # pragma: no cover - developer convenience
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from auto_round_kernel.mxfp4_hadamard import ( # noqa: E402
+ GROUP_SIZE,
+ HADAMARD_DIM,
+ HADAMARD_DIM_128,
+ SUPPORTED_HADAMARD_DIMS,
+ get_hadamard_matrix,
+ mxfp4_hadamard_quant,
+ mxfp4_hadamard_quant_reference,
+ mxfp4_quant_reference,
+ mxfp4_stream_reference,
+)
+
+DTYPES = {"fp16": torch.float16, "bf16": torch.bfloat16}
+
+# Typical prefill shapes: token counts 2K/4K/8K/16K (M=2048 is the smallest
+# DRAM-bound prefill config; 1024 is too small to be representative).
+DEFAULT_M = (2048, 4096, 8192, 16384)
+DEFAULT_K = (2048, 4096, 8192)
+
+# Primary gate: BW(fused) / BW(stream-only). The stream-only baseline is the
+# traffic-matched roofline (same loads/stores, no math), so this is the
+# kernel's utilization of what the device can actually deliver for this access
+# pattern. Measured on Arc Pro B60: 1.011-1.012 on the WAN shapes and a mean of
+# 0.998-0.999 over the DRAM-bound prefill sweep, for both D = 32 and D = 128.
+TARGET_STREAM_RATIO = 0.95
+
+# Secondary gate: BW(HMT+quant) / BW(quant-only) -- the Hadamard ablation.
+# Quant-only moves byte-identical traffic and only drops the transform, so this
+# isolates the transform's cost: it runs in-register and is expected to be
+# hidden behind memory traffic, i.e. near 1.0.
+TARGET_QUANT_RATIO = 0.95
+
+# [# CRI-WAN-HMT] WAN per-op activation shapes [M, K] = the input activation of
+# each listed GEMM, i.e. the tensor the HMT+quant kernel processes. Confirmed
+# scope: bf16 + FWHT + the full (global) M, since bandwidth is M-invariant
+# while the kernel is DRAM-bound. ``--wan`` benchmarks exactly these pairs
+# (text projection / text-encoder FFN included per the colleague's update).
+WAN_SHAPES = [(75600, 5120), (75600, 13824), (512, 5120), (512, 4096)]
+WAN_DTYPE = "bf16"
+
+# A configuration is treated as cache-resident, and therefore excluded from the
+# DRAM-bandwidth gate, once its stream-only baseline exceeds the sustained DRAM
+# copy by this factor. The margin absorbs run-to-run noise and the fact that a
+# partially resident working set still gets some cache benefit.
+CACHE_TOLERANCE = 1.15
+
+# Buffer size for the sustained DRAM copy baseline. Far larger than any cache on
+# a current Intel discrete GPU, so the copy has to reach memory.
+DRAM_PROBE_BYTES = 1 << 29 # 512 MiB per buffer
+
+
+def is_xpu_available() -> bool:
+ return hasattr(torch, "xpu") and torch.xpu.is_available()
+
+
+def bench(fn, warmup: int, iters: int, reps: int = 5) -> float:
+ """Return the median latency of ``fn`` in milliseconds.
+
+ ``torch.xpu.synchronize()`` is called on both boundaries of each rep, so a
+ measured window contains exactly ``iters`` completed kernel executions.
+
+ The median over ``reps`` windows, rather than a single mean, is what makes
+ the reported ratios stable: the device is often shared, and a single
+ interfering process inflates one window by several percent. A mean absorbs
+ that spike into the result; a median discards it as long as most windows
+ are clean.
+ """
+ for _ in range(warmup):
+ out = fn()
+ del out
+ samples = []
+ for _ in range(reps):
+ torch.xpu.synchronize()
+ start = time.perf_counter()
+ for _ in range(iters):
+ out = fn()
+ del out
+ torch.xpu.synchronize()
+ samples.append((time.perf_counter() - start) * 1000.0 / float(iters))
+ return statistics.median(samples)
+
+
+def fused_bytes(m: int, k: int, dtype: torch.dtype) -> int:
+ """Bytes moved by the fused kernel: read activation, write codes + scales."""
+ itemsize = torch.empty((), dtype=dtype).element_size()
+ return m * k * itemsize + m * k // 2 + m * k // GROUP_SIZE
+
+
+def to_gbps(nbytes: int, latency_ms: float) -> float:
+ return nbytes / (latency_ms * 1.0e-3) / 1.0e9
+
+
+def measure_copy_same_shape(m: int, k: int, dtype: torch.dtype, warmup: int, iters: int) -> tuple[float, int]:
+ src = torch.randn((m, k), dtype=dtype, device="xpu")
+ dst = torch.empty_like(src)
+ latency = bench(lambda: dst.copy_(src), warmup, iters)
+ return latency, 2 * src.numel() * src.element_size()
+
+
+def measure_sustained_dram_copy(dtype: torch.dtype, warmup: int, iters: int) -> float:
+ """Copy bandwidth on a buffer too large to cache, in GB/s.
+
+ This is the highest bandwidth the device can actually sustain from memory,
+ and therefore the ceiling any DRAM-bound kernel is measured against. It is
+ used only to decide which per-configuration baselines are cache-resident.
+ """
+ itemsize = torch.empty((), dtype=dtype).element_size()
+ numel = DRAM_PROBE_BYTES // itemsize
+ src = torch.randn(numel, dtype=dtype, device="xpu")
+ dst = torch.empty_like(src)
+ latency = bench(lambda: dst.copy_(src), warmup, iters)
+ gbps = to_gbps(2 * src.numel() * itemsize, latency)
+ # Release both buffers (~2x DRAM_PROBE_BYTES) before the caller allocates
+ # again. Rebinding is equivalent to ``del`` here but does not trip ruff's
+ # F821 for a name captured by the lambda above.
+ src = dst = None
+ torch.xpu.empty_cache()
+ return gbps
+
+
+def _dequantize(codes: torch.Tensor, scale: torch.Tensor, k: int) -> torch.Tensor:
+ """Unpack (codes, e8m0) back to FP32, for the tolerance check."""
+ levels = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32, device=codes.device)
+ flat = codes.reshape(-1, k // 2).to(torch.int32)
+ low = flat & 0x0F
+ high = (flat >> 4) & 0x0F
+ nibbles = torch.stack((low, high), dim=-1).reshape(-1, k)
+ values = levels[nibbles & 0x07] * torch.where((nibbles & 0x08) != 0, -1.0, 1.0)
+ exp = scale.reshape(-1, k // GROUP_SIZE).to(torch.int32) - 127
+ return torch.ldexp(values.reshape(-1, GROUP_SIZE), exp.reshape(-1, 1)).reshape(-1, k)
+
+
+def verify_once(
+ x: torch.Tensor,
+ hadamard: torch.Tensor,
+ rows: int,
+ *,
+ use_xmx: bool = False,
+ mode: str = "fused",
+) -> bool:
+ """Spot-check the first ``rows`` rows against the CPU reference.
+
+ A benchmark that measures an incorrect kernel is worthless, so every
+ configuration is validated before it is timed. Only a slice is checked
+ because the reference is a slow elementwise implementation.
+
+ ``mode`` selects which contract is checked:
+
+ * ``fused`` -- the default FWHT/Path A path (bit-exact) or, with
+ ``use_xmx``, the relaxed XMX path (SQNR >= 15 dB).
+ * ``quant`` -- the quant-only ablation, bit-exact against
+ :func:`mxfp4_quant_reference`.
+ * ``stream`` -- the stream-only roofline baseline, bit-exact against
+ :func:`mxfp4_stream_reference`. Verifying this one is not optional: if
+ the compiler eliminated its loads, the baseline would report an
+ unreachable bandwidth and every ratio computed against it would be wrong.
+ """
+ sub = x[:rows].contiguous()
+ if mode == "quant":
+ codes, scale = mxfp4_hadamard_quant(sub, _quant_only=True)
+ ref_codes, ref_scale = mxfp4_quant_reference(sub.cpu())
+ return torch.equal(codes.cpu(), ref_codes) and torch.equal(scale.cpu(), ref_scale)
+ if mode == "stream":
+ codes, scale = mxfp4_hadamard_quant(sub, _stream_only=True)
+ ref_codes, ref_scale = mxfp4_stream_reference(sub.cpu())
+ return torch.equal(codes.cpu(), ref_codes) and torch.equal(scale.cpu(), ref_scale)
+
+ codes, scale = mxfp4_hadamard_quant(sub, hadamard, _force_xmx=use_xmx or None)
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(sub.cpu(), hadamard.cpu())
+ if not use_xmx:
+ return torch.equal(codes.cpu(), ref_codes) and torch.equal(scale.cpu(), ref_scale)
+ k = sub.shape[-1]
+ deq = _dequantize(codes.cpu(), scale.cpu(), k).double()
+ ref = _dequantize(ref_codes, ref_scale, k).double()
+ err = deq - ref
+ sqnr = float(10.0 * torch.log10((ref * ref).sum() / (err * err).sum().clamp_min(1e-30)))
+ return sqnr >= 15.0
+
+
+def run_case(m: int, k: int, dtype: torch.dtype, args: argparse.Namespace, dram_gbps: float) -> dict:
+ torch.manual_seed(20260611)
+ x = torch.randn((m, k), dtype=dtype, device="xpu")
+ hadamard = get_hadamard_matrix(args.hadamard_dim, x.device)
+
+ verify_rows = min(args.verify_rows, m)
+ if args.no_verify:
+ correct = correct_qo = correct_so = None
+ else:
+ correct = verify_once(x, hadamard, verify_rows, use_xmx=args.xmx)
+ correct_qo = verify_once(x, hadamard, verify_rows, mode="quant")
+ correct_so = verify_once(x, hadamard, verify_rows, mode="stream")
+
+ # All three modes move exactly the same bytes (fused_bytes), so their
+ # bandwidths are directly comparable and the ratios below are meaningful.
+ latency = bench(lambda: mxfp4_hadamard_quant(x, hadamard, _force_xmx=args.xmx or None), args.warmup, args.iters)
+ latency_qo = bench(lambda: mxfp4_hadamard_quant(x, _quant_only=True), args.warmup, args.iters)
+ latency_so = bench(lambda: mxfp4_hadamard_quant(x, _stream_only=True), args.warmup, args.iters)
+ nbytes = fused_bytes(m, k, dtype)
+ bw_fused = to_gbps(nbytes, latency)
+ bw_qo = to_gbps(nbytes, latency_qo)
+ bw_so = to_gbps(nbytes, latency_so)
+
+ copy_latency, copy_bytes = measure_copy_same_shape(m, k, dtype, args.warmup, args.iters)
+ bw_copy = to_gbps(copy_bytes, copy_latency)
+
+ def _ok(v: bool | None) -> str:
+ return "" if v is None else ("pass" if v else "FAIL")
+
+ return {
+ "M": m,
+ "K": k,
+ "dtype": str(dtype).replace("torch.", ""),
+ "correct": _ok(correct),
+ "correct_qo": _ok(correct_qo),
+ "correct_so": _ok(correct_so),
+ "bytes": nbytes,
+ "latency_ms": latency,
+ "latency_qo_ms": latency_qo,
+ "latency_so_ms": latency_so,
+ "BW_fused_GBps": bw_fused,
+ "BW_quant_GBps": bw_qo,
+ "BW_stream_GBps": bw_so,
+ # Primary metric: utilization of the traffic-matched roofline.
+ "ratio_fused_stream": bw_fused / bw_so if bw_so > 0 else float("nan"),
+ # Secondary metric: cost of the Hadamard transform alone.
+ "ratio_fused_quant": bw_fused / bw_qo if bw_qo > 0 else float("nan"),
+ # Reported for context only; a copy is neither traffic-matched nor
+ # instruction-matched to this kernel, so it is not gated on.
+ "BW_copy_GBps": bw_copy,
+ "ratio_copy": bw_fused / bw_copy if bw_copy > 0 else float("nan"),
+ # Detected on the stream baseline, which is both the gate's denominator
+ # and a closer proxy for the fused kernel's footprint than the copy is.
+ "cached": bw_so > dram_gbps * CACHE_TOLERANCE,
+ }
+
+
+def format_table(rows: list[dict]) -> str:
+ header = (
+ f"{'M':>7} {'K':>7} {'dtype':>5} {'ok':>4} {'okQ':>4} {'okS':>4} "
+ f"{'BW_fused':>9} {'BW_stream':>9} {'f/s':>6} {'BW_quant':>9} {'f/q':>6} "
+ f"{'BW_copy':>9} {'f/cp':>6} {'note':>7}"
+ )
+ lines = [header, "-" * len(header)]
+ for r in rows:
+ lines.append(
+ f"{r['M']:>7} {r['K']:>7} {r['dtype']:>5} {r['correct']:>4} {r['correct_qo']:>4} "
+ f"{r['correct_so']:>4} {r['BW_fused_GBps']:>9.1f} {r['BW_stream_GBps']:>9.1f} "
+ f"{r['ratio_fused_stream']:>6.3f} {r['BW_quant_GBps']:>9.1f} {r['ratio_fused_quant']:>6.3f} "
+ f"{r['BW_copy_GBps']:>9.1f} {r['ratio_copy']:>6.3f} "
+ f"{'cached' if r['cached'] else '':>7}"
+ )
+ return "\n".join(lines)
+
+
+def parse_args() -> argparse.Namespace:
+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument(
+ "--wan",
+ action="store_true",
+ help="benchmark the [# CRI-WAN-HMT] activation shapes (bf16, full M), overriding --m/--k defaults",
+ )
+ p.add_argument("--m", type=int, nargs="+", default=None, help="M values (default prefill set)")
+ p.add_argument("--k", type=int, nargs="+", default=None, help="K values (default prefill set)")
+ p.add_argument("--dtype", nargs="+", choices=sorted(DTYPES), default=None, help="default: fp16 bf16")
+ p.add_argument("--warmup", type=int, default=20)
+ p.add_argument("--iters", type=int, default=100)
+ p.add_argument("--verify-rows", type=int, default=64, help="rows spot-checked against the CPU reference")
+ p.add_argument("--no-verify", action="store_true", help="skip the correctness spot check")
+ p.add_argument(
+ "--hadamard-dim",
+ type=int,
+ default=HADAMARD_DIM,
+ choices=SUPPORTED_HADAMARD_DIMS,
+ help="transform size: 32 (GEMM activations, one work-item per group) or "
+ "32*L for L in {2,4,8,16} (cooperative L-lane FWHT; 128 is the attention head dim)",
+ )
+ p.add_argument("--target-stream-ratio", type=float, default=TARGET_STREAM_RATIO, help="min BW_fused/BW_stream")
+ p.add_argument("--target-quant-ratio", type=float, default=TARGET_QUANT_RATIO, help="min BW_fused/BW_quant-only")
+ p.add_argument("--xmx", action="store_true", help="force the XMX path (auto-routed otherwise)")
+ return p.parse_args()
+
+
+def main() -> int:
+ args = parse_args()
+ if not is_xpu_available():
+ print("XPU is not available; nothing to benchmark.")
+ return 1
+ if args.xmx and args.hadamard_dim != HADAMARD_DIM:
+ print(f"--xmx is not supported for hadamard_dim {args.hadamard_dim}.")
+ return 1
+
+ if args.wan:
+ # WAN deliverable: exact per-op (M, K) pairs, bf16, full global M.
+ dtypes = [WAN_DTYPE] if not args.dtype else args.dtype
+ combos = list(WAN_SHAPES)
+ else:
+ dtypes = args.dtype or list(DTYPES)
+ combos = [(m, k) for m in (args.m or list(DEFAULT_M)) for k in (args.k or list(DEFAULT_K))]
+
+ print(f"device: {torch.xpu.get_device_name(0)}")
+ print(f"warmup={args.warmup} iters={args.iters} (median of 5 windows)")
+ print(f"hadamard_dim={args.hadamard_dim} path={'XMX (forced)' if args.xmx else 'default (FWHT/Path A)'}")
+ print(f"bytes = M*K*sizeof(input) + M*K/{2} + M*K/{GROUP_SIZE}")
+ print(f"primary gate: f/s = BW_fused / BW_stream-only >= {args.target_stream_ratio}")
+ print(f"secondary gate: f/q = BW_fused / BW_quant-only >= {args.target_quant_ratio}")
+ print("f/cp (vs dst.copy_) is reported for context only and is NOT gated on.")
+
+ dram_gbps = {name: measure_sustained_dram_copy(DTYPES[name], args.warmup, args.iters) for name in dtypes}
+ for name, gbps in dram_gbps.items():
+ print(f"sustained DRAM copy ({name}): {gbps:.1f} GB/s")
+ print()
+
+ rows: list[dict] = []
+ for name in dtypes:
+ dtype = DTYPES[name]
+ for m, k in combos:
+ if k % args.hadamard_dim != 0:
+ print(f"skipping K={k}: not a multiple of {args.hadamard_dim}")
+ continue
+ rows.append(run_case(m, k, dtype, args, dram_gbps[name]))
+ torch.xpu.empty_cache()
+
+ if not rows:
+ print("No configuration was benchmarked.")
+ return 1
+
+ print(format_table(rows))
+
+ failed_correctness = [r for r in rows if "FAIL" in (r["correct"], r["correct_qo"], r["correct_so"])]
+ if failed_correctness:
+ print(f"\nCORRECTNESS FAILED for {len(failed_correctness)} configuration(s); timings are meaningless.")
+ return 1
+
+ # Cache-resident configurations are reported but not gated: with the data in
+ # cache the roofline rises and the kernel becomes math bound rather than
+ # memory bound, so a DRAM-derived target does not apply to them.
+ gated = [r for r in rows if not r["cached"]]
+ cached = len(rows) - len(gated)
+ if cached:
+ print(
+ f"\n{cached} of {len(rows)} configuration(s) marked 'cached' and excluded from the gate: "
+ "with the working set resident the roofline is cache bandwidth, not DRAM, and the kernel "
+ "becomes math bound (see the module docstring)."
+ )
+ if not gated:
+ print("No DRAM-bound configuration was measured; increase M/K.")
+ return 1
+
+ ok = True
+ for key, target, label in (
+ ("ratio_fused_stream", args.target_stream_ratio, "fused/stream-only"),
+ ("ratio_fused_quant", args.target_quant_ratio, "fused/quant-only"),
+ ):
+ values = [r[key] for r in gated]
+ below = [r for r in gated if r[key] < target]
+ print(f"\nmean {label} ratio over DRAM-bound configurations = {sum(values) / len(values):.3f}")
+ print(f"min {label} ratio over DRAM-bound configurations = {min(values):.3f} (target {target})")
+ if below:
+ worst = min(below, key=lambda r: r[key])
+ print(
+ f"FAIL: {len(below)} of {len(gated)} DRAM-bound configuration(s) below the {label} target "
+ f"(worst: M={worst['M']} K={worst['K']} {worst['dtype']} at {worst[key]:.3f})."
+ )
+ ok = False
+
+ if not ok:
+ return 1
+ print("\nPASS: all configurations meet the bandwidth targets.")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/auto_round_extension/ark/test/test_mxfp4_hadamard.py b/auto_round_extension/ark/test/test_mxfp4_hadamard.py
new file mode 100644
index 000000000..218fadcb7
--- /dev/null
+++ b/auto_round_extension/ark/test/test_mxfp4_hadamard.py
@@ -0,0 +1,1126 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Correctness tests for the fused HMT + MXFP4 activation quantization kernel.
+
+The suite is organised by what each class proves:
+
+* ``TestReferenceContract`` -- the frozen quantizer contract (FP4 packing,
+ E8M0 scales, zero groups, input validation). CPU only.
+* ``TestXpuKernel`` -- the basic ``D = 32`` contract: random input, zero
+ groups, signs, extremes, nibble order.
+* ``TestXpuKernelLayouts`` -- input layouts, work-group tails, row counts.
+* ``TestXpuKernelNumerics`` -- dtype dispatch, exponent boundaries and custom
+ Hadamard matrices.
+* ``TestXpuQuantOnly`` -- the quant-only baseline (Hadamard stripped).
+* ``TestXpuKernelErrors`` -- the validation contract.
+* ``TestXpuKernelXmx`` -- the opt-in XMX path (tolerance-based).
+* ``TestStreamOnlyBaseline`` -- the stream-only roofline baseline, and the
+ guarantee that it really reads every input element.
+* ``TestHadamardDimReferenceParity`` -- the reference itself is correct at every
+ supported transform size ``D``, pinned against an independent construction.
+ CPU only.
+* ``TestXpuKernelAllDims`` -- the cooperative lane-count template, ``D > 32``.
+
+The two reference classes run everywhere, including on CPU-only machines; the
+XPU classes are skipped when no XPU is available.
+
+Acceptance criteria are bit-exact except on the XMX path: E8M0 scale bytes and
+packed FP4 code bytes must be **equal** to the PyTorch FP32 reference, no
+mismatch tolerance.
+"""
+
+import math
+
+import pytest
+import torch
+from auto_round_kernel.mxfp4_hadamard import (
+ GROUP_SIZE,
+ HADAMARD_DIM,
+ HADAMARD_DIM_128,
+ MAX_LANES_PER_ROW,
+ SUPPORTED_HADAMARD_DIMS,
+ _e8m0_and_quantized,
+ _encode_fp4,
+ _xmx_supported,
+ get_hadamard_matrix,
+ hadamard_transform_reference,
+ mxfp4_hadamard_quant,
+ mxfp4_hadamard_quant_reference,
+ mxfp4_quant_reference,
+ mxfp4_stream_reference,
+ pack_codes,
+)
+
+XPU_AVAILABLE = hasattr(torch, "xpu") and torch.xpu.is_available()
+requires_xpu = pytest.mark.skipif(not XPU_AVAILABLE, reason="XPU is not available")
+
+SHAPES = [(1, 32), (1, 128), (17, 256)]
+DTYPES = [torch.float16, torch.bfloat16]
+
+# A 32-point Hadamard transform can amplify a group by at most
+# 32 / sqrt(32) = sqrt(32), so inputs above FP32_MAX / sqrt(32) would overflow
+# the FP32 accumulator. Tests that probe "as large as possible" stay below it.
+MAX_SAFE_INPUT = 3.4e38 / 32.0**0.5
+
+# Group counts that stress the work-group tail. The D = 32 per-item kernel gives
+# one work-item one quantization group and runs kWorkGroupSize = 256 work-items
+# per work-group, so a count that is not a multiple of 256 leaves a partial
+# trailing work-group whose idle work-items must exit without writing. All four
+# counts below are below 256, i.e. every launch here is a single partial
+# work-group.
+TAIL_SHAPES = [(1, 32), (9, 32), (5, 96), (13, 160)]
+
+
+def _dequantize(codes: torch.Tensor, scale: torch.Tensor, k: int) -> torch.Tensor:
+ """Unpack (codes, e8m0) back to FP32 values, for readability of failures."""
+ levels = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32, device=codes.device)
+ flat = codes.reshape(-1, k // 2).to(torch.int32)
+ low = flat & 0x0F
+ high = (flat >> 4) & 0x0F
+ nibbles = torch.stack((low, high), dim=-1).reshape(-1, k)
+ values = levels[nibbles & 0x07] * torch.where((nibbles & 0x08) != 0, -1.0, 1.0)
+ exp = scale.reshape(-1, k // GROUP_SIZE).to(torch.int32) - 127
+ return torch.ldexp(values.reshape(-1, GROUP_SIZE), exp.reshape(-1, 1)).reshape(-1, k)
+
+
+class TestReferenceContract:
+ """The frozen reference / packing / E8M0 contract. Runs everywhere.
+
+ Orthogonality of the Hadamard matrices themselves is covered for every
+ supported D by :class:`TestHadamardDimReferenceParity`.
+ """
+
+ def test_packing_matches_vllm_ext_fp4_utils(self):
+ fp4_utils = pytest.importorskip("auto_round_extension.vllm_ext.fp4_utils")
+ torch.manual_seed(0)
+ codes = torch.randint(0, 16, (4, 64), dtype=torch.uint8)
+ # Build the FP4 values the codes represent, then pack them with the
+ # reference packer and compare byte by byte.
+ levels = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32)
+ values = levels[(codes & 0x07).long()] * torch.where((codes & 0x08) != 0, -1.0, 1.0)
+ # -0.0 keeps the sign bit, matching ``signbit`` based encoding.
+ values = torch.where((codes & 0x08) != 0, -values.abs(), values.abs())
+ expected = fp4_utils.pack_fp4_to_uint8(values)
+ assert torch.equal(pack_codes(codes), expected)
+
+ def test_zero_group_contract(self):
+ x = torch.zeros(2, 64, dtype=torch.float16)
+ codes, scale = mxfp4_hadamard_quant_reference(x)
+ assert torch.all(codes == 0)
+ assert torch.all(scale == 0)
+
+ def test_e8m0_matches_floor_log2_contract(self):
+ torch.manual_seed(0)
+ x = torch.randn(4, 64, dtype=torch.float16)
+ _, scale = mxfp4_hadamard_quant_reference(x)
+ y = hadamard_transform_reference(x.reshape(-1, HADAMARD_DIM), get_hadamard_matrix(HADAMARD_DIM))
+ amax = y.abs().amax(dim=-1)
+ for group, group_amax in enumerate(amax.tolist()):
+ expected = min(max(int(math.floor(math.log2(group_amax))) - 2 + 127, 0), 254)
+ assert scale.reshape(-1)[group].item() == expected
+ # amax / scale must land in [4, 8): the E8M0 exponent is standard.
+ ratio = group_amax / (2.0 ** (expected - 127))
+ assert 4.0 <= ratio < 8.0
+
+ def test_encode_fp4_threshold_boundaries(self):
+ # Exact boundary values exercise the alternating <= / < comparisons of
+ # cast_to_fp4. Anything at or below 0.25 -> 0, 0.75 is *not* below the
+ # 0.5 level, and so on.
+ cases = {
+ 0.0: 0,
+ 0.25: 0,
+ 0.2500001: 1,
+ 0.75: 2,
+ 1.25: 2,
+ 1.2500001: 3,
+ 1.75: 4,
+ 2.5: 4,
+ 2.5000005: 5,
+ 3.5: 6,
+ 5.0: 6,
+ 5.0000005: 7,
+ 7.9: 7,
+ }
+ values = torch.tensor(list(cases), dtype=torch.float32)
+ expected = torch.tensor(list(cases.values()), dtype=torch.uint8)
+ assert torch.equal(_encode_fp4(values), expected)
+ # The sign bit is bit 3 and is taken from signbit, except that magnitude
+ # 0 is canonicalised to 0x0 rather than 0x8 (see canonical zero rule).
+ negated = torch.where(expected == 0, expected, expected | 0x08)
+ assert torch.equal(_encode_fp4(-values), negated)
+
+ def test_canonical_zero_never_encodes_negative_zero(self):
+ # Any value that rounds to FP4 magnitude 0 must encode as 0x0. The sign
+ # of such a value comes from FP32 rounding residue (or from device-side
+ # flush-to-zero of subnormals) and is not reproducible, so it must not
+ # be observable in the output.
+ values = torch.tensor([-0.0, 0.0, -1e-30, 1e-30, -0.25, 0.25, -1e-8], dtype=torch.float32)
+ assert torch.equal(_encode_fp4(values), torch.zeros(7, dtype=torch.uint8))
+ # Sanity check that the rule is narrow: the smallest non-zero magnitude
+ # still carries its sign.
+ assert _encode_fp4(torch.tensor([-0.2500001])).item() == 0x09
+
+ def test_canonical_zero_survives_full_pipeline(self):
+ # A constant group cancels exactly in every Hadamard column but the
+ # first, which is precisely where a negative zero would appear.
+ x = torch.full((1, 32), -1.0, dtype=torch.float16)
+ codes, _ = mxfp4_hadamard_quant_reference(x)
+ assert torch.all(codes[0, 1:] == 0)
+ assert (codes & 0x08 != 0).sum() + (codes & 0x80 != 0).sum() <= 1
+
+ def test_accumulation_contract_is_order_defined(self):
+ # hadamard_transform_reference must be reproducible bit for bit and must
+ # not silently fall back to torch.matmul semantics.
+ torch.manual_seed(0)
+ x = torch.randn(64, HADAMARD_DIM, dtype=torch.float16).to(torch.float32)
+ h = get_hadamard_matrix(HADAMARD_DIM)
+ a = hadamard_transform_reference(x, h)
+ b = hadamard_transform_reference(x, h)
+ assert torch.equal(a, b)
+ manual = torch.zeros_like(a)
+ for j in range(HADAMARD_DIM):
+ manual = manual + x[:, j : j + 1] * h[j]
+ assert torch.equal(a, manual)
+
+ def test_custom_hadamard_matrix_is_honored(self):
+ # A sign-flipped Hadamard matrix is still orthogonal; the reference must
+ # use the matrix it is given rather than the cached default.
+ h = get_hadamard_matrix(HADAMARD_DIM).clone()
+ h[:, 0] = -h[:, 0]
+ torch.manual_seed(0)
+ x = torch.randn(4, 32, dtype=torch.float16)
+ codes_default, _ = mxfp4_hadamard_quant_reference(x)
+ codes_custom, _ = mxfp4_hadamard_quant_reference(x, h)
+ assert not torch.equal(codes_default, codes_custom)
+
+ def test_reference_roundtrip_is_close(self):
+ torch.manual_seed(0)
+ x = torch.randn(8, 256, dtype=torch.float16)
+ codes, scale = mxfp4_hadamard_quant_reference(x)
+ deq = _dequantize(codes, scale, 256)
+ y = hadamard_transform_reference(x.reshape(-1, HADAMARD_DIM), get_hadamard_matrix(HADAMARD_DIM))
+ y = y.reshape(8, 256)
+ rel = (deq - y).abs().max() / y.abs().max()
+ assert rel < 0.2
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_reference_accepts_bf16_and_fp16(self, dtype):
+ torch.manual_seed(0)
+ x = torch.randn(4, 128, dtype=dtype)
+ codes, scale = mxfp4_hadamard_quant_reference(x)
+ assert codes.shape == (4, 64) and scale.shape == (4, 4)
+ assert codes.dtype == torch.uint8 and scale.dtype == torch.uint8
+
+ # One case per validation branch: not a tensor, non-floating dtype,
+ # K not a multiple of 32, empty, non-finite. The XPU entry point runs the
+ # same checks, so the remaining variants live there.
+ @pytest.mark.parametrize(
+ "bad_input, error",
+ [
+ ("not a tensor", TypeError),
+ (torch.randn(1, 32, dtype=torch.float32), ValueError),
+ (torch.randint(0, 4, (1, 32), dtype=torch.int8), ValueError),
+ (torch.randn(1, 48, dtype=torch.float16), ValueError),
+ (torch.randn(1, 0, dtype=torch.float16), ValueError),
+ (torch.full((1, 32), float("nan"), dtype=torch.float16), ValueError),
+ ],
+ )
+ def test_reference_rejects_invalid_input(self, bad_input, error):
+ with pytest.raises(error):
+ mxfp4_hadamard_quant_reference(bad_input)
+
+ @pytest.mark.parametrize(
+ "bad_matrix",
+ [
+ torch.eye(16, dtype=torch.float32),
+ torch.eye(HADAMARD_DIM, dtype=torch.int32),
+ torch.full((HADAMARD_DIM, HADAMARD_DIM), float("nan"), dtype=torch.float32),
+ ],
+ )
+ def test_reference_rejects_invalid_hadamard(self, bad_matrix):
+ x = torch.randn(1, 32, dtype=torch.float16)
+ with pytest.raises(ValueError):
+ mxfp4_hadamard_quant_reference(x, bad_matrix)
+
+ # ---- quant-only reference (no Hadamard transform) ---------------------
+
+ def test_quant_only_reference_matches_fused_with_identity(self):
+ # Quantizing the raw activation is mathematically identical to fusing an
+ # identity Hadamard matrix: the Path A transform with H = I only ever
+ # adds exact zeros, so the two references must agree byte for byte.
+ torch.manual_seed(0)
+ x = torch.randn(4, 64, dtype=torch.float16)
+ codes, scale = mxfp4_quant_reference(x)
+ identity = torch.eye(HADAMARD_DIM, dtype=torch.float32)
+ fused_codes, fused_scale = mxfp4_hadamard_quant_reference(x, identity)
+ assert torch.equal(codes, fused_codes)
+ assert torch.equal(scale, fused_scale)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_quant_only_reference_shape_dtype_and_zero_group(self, dtype):
+ x = torch.zeros(2, 64, dtype=dtype)
+ codes, scale = mxfp4_quant_reference(x)
+ assert codes.shape == (2, 32)
+ assert scale.shape == (2, 2)
+ assert codes.dtype == torch.uint8
+ assert scale.dtype == torch.uint8
+ assert torch.all(codes == 0)
+ assert torch.all(scale == 0)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_quant_only_reference_differs_from_fused_hmt(self, dtype):
+ # A real Hadamard transform scrambles values across the group, so the
+ # quant-only output must differ from the fused output in general. This
+ # guards against quant-only accidentally still applying the transform.
+ torch.manual_seed(0)
+ x = torch.randn(8, 128, dtype=dtype)
+ q_codes, q_scale = mxfp4_quant_reference(x)
+ f_codes, f_scale = mxfp4_hadamard_quant_reference(x)
+ assert not torch.equal(q_codes, f_codes) or not torch.equal(q_scale, f_scale)
+
+
+def _assert_bit_exact(x: torch.Tensor):
+ """Run the kernel and the reference on ``x`` and require byte equality."""
+ codes, scale = mxfp4_hadamard_quant(x)
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())
+ num_rows, k = x.numel() // x.shape[-1], x.shape[-1]
+ assert codes.shape == ref_codes.shape == (num_rows, k // 2)
+ assert scale.shape == ref_scale.shape == (num_rows, k // GROUP_SIZE)
+ assert codes.dtype == torch.uint8 and scale.dtype == torch.uint8
+ assert codes.device.type == scale.device.type == "xpu"
+ assert torch.equal(scale.cpu(), ref_scale)
+ assert torch.equal(codes.cpu(), ref_codes)
+ return codes.cpu(), scale.cpu()
+
+
+@requires_xpu
+class TestXpuKernel:
+ """The ``D = 32`` per-item kernel: bit-exact against the reference."""
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ @pytest.mark.parametrize("shape", SHAPES)
+ def test_random_finite_input(self, dtype, shape):
+ torch.manual_seed(0)
+ _assert_bit_exact(torch.randn(*shape, dtype=dtype, device="xpu"))
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_zero_group(self, dtype):
+ x = torch.zeros(4, 64, dtype=dtype, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x)
+ assert torch.all(codes.cpu() == 0)
+ assert torch.all(scale.cpu() == 0)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_mixed_sign_and_partial_zero_groups(self, dtype):
+ torch.manual_seed(1)
+ x = torch.randn(5, 128, dtype=dtype, device="xpu")
+ x[1, :32] = 0
+ x[3, 64:96] = 0
+ x[2] = -x[2].abs()
+ codes, scale = mxfp4_hadamard_quant(x)
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())
+ assert torch.equal(scale.cpu(), ref_scale)
+ assert torch.equal(codes.cpu(), ref_codes)
+ assert torch.all(scale.cpu()[1, 0] == 0)
+ assert torch.all(codes.cpu()[1, :16] == 0)
+
+ def test_extreme_finite_values(self):
+ x = torch.zeros(2, 32, dtype=torch.float16, device="xpu")
+ x[0] = 65504.0 # FP16 max
+ x[1] = 6.1e-5 # smallest FP16 normal
+ codes, scale = mxfp4_hadamard_quant(x)
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())
+ assert torch.equal(scale.cpu(), ref_scale)
+ assert torch.equal(codes.cpu(), ref_codes)
+
+ def test_nibble_order(self):
+ torch.manual_seed(3)
+ x = torch.randn(1, 64, dtype=torch.float16, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x)
+ # Rebuild the per-element 4-bit codes from the reference and check that
+ # element 2i sits in the low nibble of byte i, element 2i+1 in the high one.
+ y = hadamard_transform_reference(x.cpu().reshape(-1, HADAMARD_DIM), get_hadamard_matrix(HADAMARD_DIM))
+ e8m0, q = _e8m0_and_quantized(y)
+ nibbles = _encode_fp4(q).reshape(1, 64)
+ packed = codes.cpu().to(torch.int32)
+ assert torch.equal((packed & 0x0F).to(torch.uint8), nibbles[:, 0::2])
+ assert ((packed >> 4) & 0x0F).to(torch.uint8).equal(nibbles[:, 1::2])
+ assert torch.equal(scale.cpu(), e8m0.reshape(1, 2))
+
+ def test_non_finite_input_is_not_scanned_by_default(self):
+ # The finiteness scan is a debugging aid, not part of the contract: it
+ # costs more than the kernel, so the hot path must not pay for it.
+ x = torch.randn(1, 32, dtype=torch.float16, device="xpu")
+ x[0, 0] = float("nan")
+ mxfp4_hadamard_quant(x)
+
+
+@requires_xpu
+class TestXpuKernelLayouts:
+ """How arbitrary input layouts and row counts map onto rows and work-groups."""
+
+ # ---- input shapes and work-group tails -------------------------------
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ @pytest.mark.parametrize("shape", TAIL_SHAPES)
+ def test_work_group_tail(self, dtype, shape):
+ torch.manual_seed(shape[0])
+ x = torch.randn(*shape, dtype=dtype, device="xpu")
+ _assert_bit_exact(x)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ @pytest.mark.parametrize("shape", [(1024, 2048)])
+ def test_large_multi_row(self, dtype, shape):
+ # A 2-D tensor with many rows and a large K: the row -> work-group
+ # mapping is no longer trivial, unlike in TAIL_SHAPES.
+ torch.manual_seed(7)
+ x = torch.randn(*shape, dtype=dtype, device="xpu")
+ _assert_bit_exact(x)
+
+ @pytest.mark.parametrize("shape", [(64,), (2, 3, 64), (2, 3, 5, 96)])
+ def test_input_shapes_are_flattened_to_rows(self, shape):
+ # 1-D, 3-D and 4-D inputs all flatten to ``[-1, K]`` with K the last
+ # dim; the output is always the 2-D ``[rows, ...]`` form.
+ torch.manual_seed(2)
+ x = torch.randn(*shape, dtype=torch.float16, device="xpu")
+ rows, k = x.numel() // shape[-1], shape[-1]
+ codes, scale = _assert_bit_exact(x)
+ assert codes.shape == (rows, k // 2)
+ assert scale.shape == (rows, k // GROUP_SIZE)
+
+ def test_non_contiguous_input_is_materialized(self):
+ torch.manual_seed(0)
+ base = torch.randn(64, 128, dtype=torch.float16, device="xpu")
+ view = base[:, ::2] # stride-2 columns, [64, 64], non-contiguous
+ assert not view.is_contiguous()
+ codes, scale = mxfp4_hadamard_quant(view)
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(view.cpu())
+ assert torch.equal(codes.cpu(), ref_codes)
+ assert torch.equal(scale.cpu(), ref_scale)
+ # A contiguous copy of the same values must give the same bytes.
+ codes_contig, scale_contig = mxfp4_hadamard_quant(view.contiguous())
+ assert torch.equal(codes.cpu(), codes_contig.cpu())
+ assert torch.equal(scale.cpu(), scale_contig.cpu())
+
+
+@requires_xpu
+class TestXpuKernelNumerics:
+ """D = 32 numerics: dtype dispatch, exponent boundaries, Hadamard handling."""
+
+ # ---- BF16 -------------------------------------------------------------
+
+ def test_bf16_and_fp16_agree_on_exactly_representable_values(self):
+ # Values that are exact in both BF16 and FP16 must produce identical
+ # codes and scales, proving the two dispatch paths share the FP32 math.
+ torch.manual_seed(0)
+ base = torch.randint(-8, 9, (16, 128), dtype=torch.int32).to(torch.float32) / 4.0
+ codes_fp16, scale_fp16 = mxfp4_hadamard_quant(base.to(torch.float16).to("xpu"))
+ codes_bf16, scale_bf16 = mxfp4_hadamard_quant(base.to(torch.bfloat16).to("xpu"))
+ assert torch.equal(scale_fp16.cpu(), scale_bf16.cpu())
+ assert torch.equal(codes_fp16.cpu(), codes_bf16.cpu())
+
+ def test_bf16_subnormal_clamp_range(self):
+ # BF16 has FP32's exponent range, so tiny values drive
+ # floor(log2(amax)) - 2 + 127 below 0 and must clamp to e8m0 = 0.
+ # 1e-39 is chosen so the clamp fires while the rescaled values are still
+ # large enough to encode as non-zero FP4 codes, i.e. this is the clamp
+ # path and not the all-zero-group path.
+ x = torch.full((2, 32), 1e-39, dtype=torch.bfloat16, device="xpu")
+ x[1] = -1e-39
+ codes, scale = _assert_bit_exact(x)
+ assert torch.all(scale == 0)
+ assert torch.any(codes != 0)
+ # Far below the clamp the rescaled values fall under the first FP4
+ # threshold, so codes become zero while e8m0 stays clamped at 0.
+ x = torch.full((1, 32), 1e-43, dtype=torch.bfloat16, device="xpu")
+ codes, scale = _assert_bit_exact(x)
+ assert torch.all(scale == 0)
+ assert torch.all(codes == 0)
+
+ # ---- numerical boundaries --------------------------------------------
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_row_and_group_mapping(self, dtype):
+ # Each row gets a distinct magnitude so a row/column mix-up in the
+ # scale layout is detected, not just an average mismatch.
+ rows, k = 12, 128
+ x = torch.zeros(rows, k, dtype=dtype, device="xpu")
+ for r in range(rows):
+ for g in range(k // GROUP_SIZE):
+ x[r, g * GROUP_SIZE] = float(2 ** (r - 6 + g))
+ codes, scale = _assert_bit_exact(x)
+ # Scales must be strictly increasing along both axes by one octave.
+ scale_i = scale.to(torch.int32)
+ assert torch.all(scale_i[1:, :] - scale_i[:-1, :] == 1)
+ assert torch.all(scale_i[:, 1:] - scale_i[:, :-1] == 1)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_single_spike_per_group(self, dtype):
+ # A one-hot group makes every transformed element equal in magnitude,
+ # which puts them exactly on an FP4 level rather than between levels.
+ x = torch.zeros(8, 32, dtype=dtype, device="xpu")
+ for r in range(8):
+ x[r, r * 4] = 1.0 if r % 2 == 0 else -1.0
+ _assert_bit_exact(x)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_alternating_extremes(self, dtype):
+ finfo = torch.finfo(dtype)
+ big = min(finfo.max, MAX_SAFE_INPUT)
+ x = torch.zeros(4, 64, dtype=dtype, device="xpu")
+ x[0] = big
+ x[1, ::2] = big
+ x[1, 1::2] = -big
+ x[2] = finfo.tiny
+ x[3, ::2] = finfo.tiny
+ _assert_bit_exact(x)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_cancelling_group_yields_canonical_zero(self, dtype):
+ # A constant group cancels exactly in Hadamard columns 1..31. On device
+ # the FP32 residue of that cancellation, and any flush-to-zero of
+ # subnormals, may be negatively signed; the canonical zero rule must
+ # keep that out of the codes so the kernel still matches the reference.
+ x = torch.zeros(4, 32, dtype=dtype, device="xpu")
+ x[0] = 1.0
+ x[1] = -1.0
+ x[2] = torch.finfo(dtype).tiny
+ x[3] = -torch.finfo(dtype).tiny
+ codes, _ = _assert_bit_exact(x)
+ # Columns 1..31 of every row are exact zeros, i.e. bytes 1..15.
+ assert torch.all(codes[:, 1:].cpu() == 0)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_quantized_grid_values_stress_thresholds(self, dtype):
+ # Coarse, exactly representable inputs make transformed values land on
+ # or extremely close to the FP4 decision thresholds, which is where an
+ # accumulation-order mismatch between kernel and reference would show.
+ torch.manual_seed(11)
+ x = (torch.randint(-4, 5, (256, 128), device="xpu").to(torch.float32) / 4.0).to(dtype)
+ _assert_bit_exact(x)
+
+ @pytest.mark.parametrize("seed", [0, 4, 7])
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_random_fuzz(self, dtype, seed):
+ # The seeds sweep 1e-4, 1 and 1e3, i.e. the E8M0 exponent range; more
+ # seeds only resample the same magnitudes.
+ torch.manual_seed(seed)
+ x = (torch.randn(128, 256, device="xpu") * (10.0 ** (seed - 4))).to(dtype)
+ _assert_bit_exact(x)
+
+ # ---- Hadamard matrix handling ----------------------------------------
+
+ def test_custom_hadamard_matrix(self):
+ # A sign-flipped Hadamard matrix is still orthogonal, so it must be
+ # honored rather than silently replaced by the cached default. The
+ # forced Path A call pins the bit-exact branch; the auto-routed call
+ # exercises whatever the dispatcher picks (the XMX path when it is
+ # built, which has a relaxed, tolerance-based contract).
+ h = get_hadamard_matrix(HADAMARD_DIM).clone()
+ h[:, 0] = -h[:, 0]
+ torch.manual_seed(0)
+ x = torch.randn(16, 128, dtype=torch.float16, device="xpu")
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), h.cpu())
+
+ codes, scale = mxfp4_hadamard_quant(x, h, _force_xmx=False)
+ assert torch.equal(codes.cpu(), ref_codes)
+ assert torch.equal(scale.cpu(), ref_scale)
+
+ routed_codes, routed_scale = mxfp4_hadamard_quant(x, h)
+ if _xmx_supported():
+ deq = _dequantize(routed_codes.cpu(), routed_scale.cpu(), 128)
+ ref_deq = _dequantize(ref_codes, ref_scale, 128)
+ sqnr_db, _, _ = _precision_metrics(deq, ref_deq, ref_scale)
+ assert sqnr_db >= 15.0, f"SQNR {sqnr_db:.2f} dB < 15 dB"
+ else:
+ assert torch.equal(routed_codes.cpu(), ref_codes)
+ assert torch.equal(routed_scale.cpu(), ref_scale)
+
+ default_codes, _ = mxfp4_hadamard_quant(x)
+ assert not torch.equal(routed_codes.cpu(), default_codes.cpu())
+
+ def test_hadamard_matrix_input_is_normalized(self):
+ # The matrix is coerced to a contiguous FP32 tensor on the activation's
+ # device; a CPU host matrix and an FP64 matrix must give the same bytes.
+ torch.manual_seed(0)
+ x = torch.randn(4, 64, dtype=torch.float16, device="xpu")
+ h_cpu = get_hadamard_matrix(HADAMARD_DIM, "cpu")
+ assert h_cpu.device.type == "cpu"
+ h64 = get_hadamard_matrix(HADAMARD_DIM).to(torch.float64)
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())
+ for h in (h_cpu, h64):
+ codes, scale = mxfp4_hadamard_quant(x, h)
+ assert torch.equal(codes.cpu(), ref_codes)
+ assert torch.equal(scale.cpu(), ref_scale)
+
+
+@requires_xpu
+class TestXpuQuantOnly:
+ """The quant-only baseline: the fused kernel with the Hadamard stripped.
+
+ It must be bit-exact with :func:`mxfp4_quant_reference`, the raw-activation
+ reference, on the same per-item layout. That equivalence is what makes the
+ fused/quant-only bandwidth ratio a valid ablation of the transform.
+ """
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ @pytest.mark.parametrize("shape", [(1, 32), (64, 512), (4, 13824)])
+ def test_quant_only_matches_reference(self, dtype, shape):
+ # The quant-only device path must be bit-exact with mxfp4_quant_reference
+ # (raw activation, no transform), on the same per-item layout.
+ torch.manual_seed(shape[0])
+ x = torch.randn(*shape, dtype=dtype, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x, _quant_only=True)
+ ref_codes, ref_scale = mxfp4_quant_reference(x.cpu())
+ assert codes.shape == ref_codes.shape == (shape[0], shape[1] // 2)
+ assert scale.shape == ref_scale.shape == (shape[0], shape[1] // GROUP_SIZE)
+ assert codes.dtype == torch.uint8 and scale.dtype == torch.uint8
+ assert torch.equal(scale.cpu(), ref_scale)
+ assert torch.equal(codes.cpu(), ref_codes)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_quant_only_zero_group(self, dtype):
+ x = torch.zeros(2, 64, dtype=dtype, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x, _quant_only=True)
+ assert torch.all(codes.cpu() == 0)
+ assert torch.all(scale.cpu() == 0)
+
+ @pytest.mark.parametrize("seed", [0, 2])
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_quant_only_fuzz(self, dtype, seed):
+ # The two seeds span 1e-2 and 1e2, the magnitudes the reference cannot
+ # reach through the shape cases above.
+ torch.manual_seed(seed)
+ x = (torch.randn(64, 256, device="xpu") * (10.0 ** (seed - 2))).to(dtype)
+ codes, scale = mxfp4_hadamard_quant(x, _quant_only=True)
+ ref_codes, ref_scale = mxfp4_quant_reference(x.cpu())
+ assert torch.equal(codes.cpu(), ref_codes)
+ assert torch.equal(scale.cpu(), ref_scale)
+
+
+@requires_xpu
+class TestXpuKernelErrors:
+ """The validation contract: bad inputs are rejected up front, never run."""
+
+ @pytest.mark.parametrize("bad_dtype", [torch.float32, torch.int8, torch.bool])
+ def test_rejects_unsupported_dtype(self, bad_dtype):
+ x = torch.zeros(1, 32, dtype=bad_dtype, device="xpu")
+ with pytest.raises(ValueError, match="float16 or bfloat16"):
+ mxfp4_hadamard_quant(x)
+
+ @pytest.mark.parametrize("k", [1, 33, 48])
+ def test_rejects_k_not_multiple_of_32(self, k):
+ x = torch.randn(2, k, dtype=torch.float16, device="xpu")
+ with pytest.raises(ValueError, match="multiple of 32"):
+ mxfp4_hadamard_quant(x)
+
+ def test_rejects_empty_tensor(self):
+ with pytest.raises(ValueError, match="must not be empty"):
+ mxfp4_hadamard_quant(torch.randn(0, 32, dtype=torch.float16, device="xpu"))
+ with pytest.raises(ValueError, match="must not be empty"):
+ mxfp4_hadamard_quant(torch.randn(2, 0, dtype=torch.float16, device="xpu"))
+
+ def test_rejects_cpu_tensor(self):
+ x = torch.randn(1, 32, dtype=torch.float16)
+ with pytest.raises(ValueError, match="only supported on XPU"):
+ mxfp4_hadamard_quant(x)
+
+ def test_rejects_non_tensor(self):
+ with pytest.raises(TypeError):
+ mxfp4_hadamard_quant([0.0] * 32)
+
+ @pytest.mark.parametrize("bad_value", [float("nan"), float("inf")])
+ def test_rejects_non_finite(self, bad_value):
+ x = torch.randn(2, 64, dtype=torch.float16, device="xpu")
+ x[1, 17] = bad_value
+ with pytest.raises(ValueError, match="finite"):
+ mxfp4_hadamard_quant(x, check_finite=True)
+
+ @pytest.mark.parametrize(
+ "bad_matrix_factory",
+ [
+ # Wrong shape, degenerate (singular) values, wrong dtype and
+ # non-finite entries: one case per validation branch. A power-of-two
+ # D above the largest instantiated lane count is covered by
+ # TestXpuKernelAllDims::test_rejects_dim_above_max_lane_count.
+ lambda: torch.eye(16, dtype=torch.float32, device="xpu"),
+ lambda: torch.zeros(HADAMARD_DIM, dtype=torch.float32, device="xpu"),
+ lambda: torch.eye(HADAMARD_DIM, dtype=torch.int32, device="xpu"),
+ lambda: torch.full((HADAMARD_DIM, HADAMARD_DIM), float("inf"), dtype=torch.float32, device="xpu"),
+ ],
+ )
+ def test_rejects_invalid_hadamard_matrix(self, bad_matrix_factory):
+ x = torch.randn(1, 32, dtype=torch.float16, device="xpu")
+ with pytest.raises(ValueError):
+ mxfp4_hadamard_quant(x, bad_matrix_factory())
+
+ def test_rejects_non_tensor_hadamard_matrix(self):
+ x = torch.randn(1, 32, dtype=torch.float16, device="xpu")
+ with pytest.raises(TypeError):
+ mxfp4_hadamard_quant(x, [[0.0] * 32] * 32)
+
+ def test_state_is_intact_after_rejected_call(self):
+ torch.manual_seed(0)
+ good = torch.randn(4, 64, dtype=torch.float16, device="xpu")
+ expected = mxfp4_hadamard_quant(good)
+ bad = torch.randn(2, 48, dtype=torch.float16, device="xpu")
+ with pytest.raises(ValueError):
+ mxfp4_hadamard_quant(bad)
+ actual = mxfp4_hadamard_quant(good)
+ assert torch.equal(expected[0].cpu(), actual[0].cpu())
+ assert torch.equal(expected[1].cpu(), actual[1].cpu())
+
+
+def _xmx_path_available() -> bool:
+ """True when the current XPU build exposes the opt-in XMX path."""
+ return _xmx_supported()
+
+
+# One E2M1 level step at the top of a group, expressed as a fraction of the
+# group peak: the levels are 0, .5, 1, 1.5, 2, 3, 4, 6, so the widest single-step
+# gap is 6 - 4 = 2 against a peak of 6. A code that lands one level off cannot
+# exceed this; anything larger means the code moved by two or more levels, or the
+# E8M0 scale bucket disagrees, both of which are real bugs rather than rounding.
+ONE_E2M1_LEVEL_REL = 2.0 / 6.0
+
+
+def _precision_metrics(deq: torch.Tensor, ref: torch.Tensor, ref_scale: torch.Tensor) -> tuple[float, float, float]:
+ """``(sqnr_db, max_rel, flip_rate)`` of ``deq`` vs ``ref`` (both FP32).
+
+ SQNR is the standard signal-to-quantization-noise ratio in dB. Relative
+ errors are measured **per group against the group's peak magnitude**
+ (``amax = 6 * 2**(e8m0-127)``, the largest FP4 level): per-element relative
+ error is meaningless near zero (FP4 alone allows unbounded relative error
+ for tiny values), while the per-group bound is inherent to E2M1.
+
+ ``max_rel`` is the strict worst case. It is **discrete**, not continuous:
+ the two paths either agree on a code or they do not, so the only attainable
+ values are the E2M1 level gaps over the group peak (0, 1/12, 1/6, 1/3). A
+ threshold set between two of those -- 0.25, say -- therefore does not express
+ a tolerance at all; it silently demands bit-exactness of the top level, which
+ the XMX path (bf16 H + DPAS) never promised.
+
+ ``flip_rate`` is the fraction of elements whose code differs at all. This is
+ what actually carries regression signal: a percentile of ``rel`` is useless
+ here because disagreements are so rare (~3e-4) that every percentile up to
+ p999 sits inside the identical majority and reads exactly 0.
+ """
+ deq = deq.double()
+ ref = ref.double()
+ err = deq - ref
+ signal = (ref * ref).sum()
+ noise = (err * err).sum()
+ sqnr_db = float(10.0 * torch.log10(signal / noise.clamp_min(1e-30)))
+ amax = (6.0 * torch.pow(2.0, ref_scale.double() - 127.0)).reshape(-1, 1)
+ rel = (err.abs().reshape(-1, GROUP_SIZE) / amax).flatten()
+ max_rel = float(rel.max())
+ flip_rate = float((err != 0).double().mean())
+ return sqnr_db, max_rel, flip_rate
+
+
+@requires_xpu
+class TestXpuKernelXmx:
+ """Opt-in XMX path, tolerance-based acceptance.
+
+ The XMX path is *not* bit-exact: the Hadamard matrix is stored in the
+ activation dtype (fp16/bf16) and the transform runs on XMX DPAS with FP32
+ accumulation (relaxed contract). Acceptance: SQNR >= 15 dB, no code off by
+ more than one E2M1 level, and a code-disagreement rate below 0.1%.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _require_xmx(self):
+ if not _xmx_path_available():
+ pytest.skip("XMX path not available in this build (ARK_SYCL_TLA)")
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ @pytest.mark.parametrize("shape", [(1, 32), (17, 256)])
+ def test_xmx_matches_reference_within_tolerance(self, dtype, shape):
+ torch.manual_seed(shape[0])
+ x = torch.randn(*shape, dtype=dtype, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x, _force_xmx=True)
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())
+
+ num_rows, k = x.numel() // x.shape[-1], x.shape[-1]
+ assert codes.shape == ref_codes.shape == (num_rows, k // 2)
+ assert scale.shape == ref_scale.shape == (num_rows, k // GROUP_SIZE)
+ assert codes.dtype == torch.uint8 and scale.dtype == torch.uint8
+
+ deq_xmx = _dequantize(codes.cpu(), scale.cpu(), k)
+ deq_ref = _dequantize(ref_codes, ref_scale, k)
+ sqnr_db, max_rel, flip_rate = _precision_metrics(deq_xmx, deq_ref, ref_scale)
+ assert sqnr_db >= 15.0, f"SQNR {sqnr_db:.2f} dB < 15 dB"
+ # Both bounds are on the same quantity -- how far a code may move -- so
+ # they stay consistent as the path changes: max_rel can never exceed one
+ # E2M1 level step, and flip_rate stays a small fraction of a percent
+ # (measured ~2e-4 .. 5e-4 on the shapes below).
+ assert max_rel <= ONE_E2M1_LEVEL_REL + 1e-6, f"max relative error {max_rel:.4f} exceeds one E2M1 level"
+ assert flip_rate < 1e-3, f"code disagreement rate {flip_rate:.2e} >= 1e-3"
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_xmx_scales_are_close_to_reference(self, dtype):
+ # E8M0 scales are octave buckets; the XMX transform (fp16/bf16 H) can
+ # shift a group's max by at most a couple of dB, so at most one bucket.
+ torch.manual_seed(3)
+ x = torch.randn(64, 256, dtype=dtype, device="xpu")
+ _, scale = mxfp4_hadamard_quant(x, _force_xmx=True)
+ _, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())
+ diff = (scale.cpu().to(torch.int32) - ref_scale.to(torch.int32)).abs()
+ assert torch.all(diff <= 1), f"E8M0 scales diverge by more than 1: max={diff.max().item()}"
+
+
+@requires_xpu
+class TestStreamOnlyBaseline:
+ """The stream-only roofline baseline (``_stream_only=True``).
+
+ This mode is a measurement instrument, not a quantizer: it keeps the fused
+ kernel's loads, packing shape and stores but drops the transform and the
+ quantization math. The tests below exist for one reason above all -- to
+ prove the kernel really touches every input element. A compiler that sank
+ the loads would leave an empty kernel reporting an unreachable bandwidth,
+ and every ratio measured against it would be silently wrong.
+ """
+
+ # The stream-only path reuses the fused kernel's layout, packing and stores,
+ # so it only needs one shape per distinct work-group situation (a single
+ # group, a full work-group and a partial trailing one), not the full matrix.
+ @pytest.mark.parametrize("dtype", DTYPES)
+ @pytest.mark.parametrize("num_rows,k", [(1, 32), (17, 256), (9, 32), (13, 160)])
+ def test_matches_reference(self, dtype, num_rows, k):
+ torch.manual_seed(11)
+ x = torch.randn(num_rows, k, dtype=dtype, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x, _stream_only=True)
+ ref_codes, ref_scale = mxfp4_stream_reference(x.cpu())
+ assert torch.equal(codes.cpu(), ref_codes)
+ assert torch.equal(scale.cpu(), ref_scale)
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_every_input_element_is_read(self, dtype):
+ """Perturbing any single element must change the output.
+
+ Each element contributes its low nibble to one code byte and (for a
+ quarter of the positions) to the scale fold, so a one-element change is
+ always observable. This is the anti-dead-code-elimination guarantee the
+ whole baseline rests on.
+ """
+ torch.manual_seed(12)
+ x = torch.randn(4, 64, dtype=dtype, device="xpu")
+ # Set the low nibble of every element to zero, then flip one bit at a
+ # time; using raw bits keeps the test independent of float semantics.
+ bits = x.view(torch.int16)
+ bits &= ~0x0F
+ base_codes, _ = mxfp4_hadamard_quant(x, _stream_only=True)
+ for idx in range(x.numel()):
+ probe = x.clone()
+ probe.view(torch.int16).reshape(-1)[idx] |= 0x0F
+ codes, _ = mxfp4_hadamard_quant(probe, _stream_only=True)
+ assert not torch.equal(codes.cpu(), base_codes.cpu()), f"element {idx} was not read"
+
+ def test_shares_byte_traffic_with_fused(self):
+ """Identical output shapes/dtypes, i.e. identical bytes written.
+
+ The bandwidth ratio is only meaningful because both modes move exactly
+ the same number of bytes.
+ """
+ x = torch.randn(8, 256, dtype=torch.bfloat16, device="xpu")
+ stream = mxfp4_hadamard_quant(x, _stream_only=True)
+ fused = mxfp4_hadamard_quant(x)
+ for a, b in zip(stream, fused):
+ assert a.shape == b.shape and a.dtype == b.dtype
+
+ def test_rejects_combination_with_quant_only(self):
+ x = torch.randn(1, 32, dtype=torch.bfloat16, device="xpu")
+ with pytest.raises(ValueError):
+ mxfp4_hadamard_quant(x, _quant_only=True, _stream_only=True)
+
+
+# Every cooperative size, i.e. everything the lane-count template instantiates
+# except D = 32, which the dispatcher routes to the per-item kernel instead.
+COOPERATIVE_DIMS = [d for d in SUPPORTED_HADAMARD_DIMS if d > GROUP_SIZE]
+
+
+class TestHadamardDimReferenceParity:
+ """The torch reference must itself be correct at every supported D.
+
+ Bit-exactness against the reference is worthless if the reference is wrong,
+ so this pins it against an independent construction: the transform is a
+ matrix multiply by the normalized Sylvester matrix, and that matrix is its
+ own inverse up to the normalization.
+ """
+
+ @pytest.mark.parametrize("dim", SUPPORTED_HADAMARD_DIMS)
+ def test_matrix_is_orthogonal(self, dim):
+ h = get_hadamard_matrix(dim, "cpu")
+ assert h.shape == (dim, dim)
+ assert h.dtype == torch.float32
+ identity = h @ h.T
+ assert torch.allclose(identity, torch.eye(dim, dtype=torch.float32), atol=1e-5)
+
+ @pytest.mark.parametrize("dim", SUPPORTED_HADAMARD_DIMS)
+ def test_reference_matches_matmul(self, dim):
+ """hadamard_transform_reference takes an already-grouped [-1, D] view."""
+ torch.manual_seed(31)
+ x = torch.randn(12, dim, dtype=torch.float32)
+ h = get_hadamard_matrix(dim, "cpu")
+ assert torch.allclose(hadamard_transform_reference(x, h), x @ h, atol=1e-5)
+
+ @pytest.mark.parametrize("dim", SUPPORTED_HADAMARD_DIMS)
+ def test_output_group_count_is_dim_independent(self, dim):
+ """Whatever D is, the output is still one E8M0 per 32 elements."""
+ x = torch.randn(4, 1024, dtype=torch.bfloat16)
+ codes, scale = mxfp4_hadamard_quant_reference(x, get_hadamard_matrix(dim, "cpu"))
+ assert codes.shape == (4, 1024 // 2)
+ assert scale.shape == (4, 1024 // GROUP_SIZE)
+
+ def test_fwht128_matches_dense_matmul(self):
+ """The 7-stage butterfly computes ``x @ H128`` up to FP32 rounding.
+
+ This is the one reference routine that is not a plain matmul, so it
+ needs its own independent construction.
+ """
+ from auto_round_kernel.mxfp4_hadamard import fwht_transform_reference
+
+ torch.manual_seed(21)
+ x = torch.randn(64, HADAMARD_DIM_128, dtype=torch.float32)
+ h = get_hadamard_matrix(HADAMARD_DIM_128, "cpu")
+ got = fwht_transform_reference(x, h.reshape(-1)[0])
+ assert torch.allclose(got, x @ h, atol=1e-5, rtol=1e-5)
+
+
+@requires_xpu
+class TestXpuKernelAllDims:
+ """The lane-count template at every instantiated D, not just 128.
+
+ D = 32 * L uses L cooperating lanes, so the tail-convergence behaviour and
+ the rows-per-work-group ratio differ for each L. Row counts here are chosen
+ to straddle the 256/L rows that fill one work-group.
+ """
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ @pytest.mark.parametrize("dim", COOPERATIVE_DIMS)
+ def test_bit_exact_against_reference(self, dtype, dim):
+ # Rows per work-group is 256 / L, so 33 rows spans more than one
+ # work-group only at D = 256 (L = 8) and D = 512 (L = 16); at L = 2 and
+ # L = 4 it is a single partial work-group.
+ # test_partial_work_group_tail owns the exact boundary row counts, in
+ # one dtype.
+ torch.manual_seed(32)
+ x = torch.randn(33, dim, dtype=dtype, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(dim, "cpu"))
+ assert torch.equal(codes.cpu(), ref_codes)
+ assert torch.equal(scale.cpu(), ref_scale)
+
+ @pytest.mark.parametrize("dim", COOPERATIVE_DIMS)
+ def test_partial_work_group_tail(self, dim):
+ """Rows that leave a partial trailing work-group.
+
+ Inactive lanes in that work-group must still reach the cross-lane
+ shuffles; if they returned early the collective would be undefined and
+ the *active* lanes of the same row would read garbage.
+ """
+ rows_per_wg = 256 // (dim // GROUP_SIZE)
+ for num_rows in (1, rows_per_wg - 1, rows_per_wg, rows_per_wg + 1):
+ torch.manual_seed(33)
+ x = torch.randn(num_rows, dim, dtype=torch.bfloat16, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(dim, "cpu"))
+ assert torch.equal(codes.cpu(), ref_codes), f"codes mismatch at D={dim} rows={num_rows}"
+ assert torch.equal(scale.cpu(), ref_scale), f"scale mismatch at D={dim} rows={num_rows}"
+
+ @pytest.mark.parametrize("dim", COOPERATIVE_DIMS)
+ def test_multiple_transforms_per_row(self, dim):
+ """K = 4*D exercises the row -> group index arithmetic in the store."""
+ torch.manual_seed(34)
+ x = torch.randn(5, dim * 4, dtype=torch.bfloat16, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(dim, "cpu"))
+ assert torch.equal(codes.cpu(), ref_codes)
+ assert torch.equal(scale.cpu(), ref_scale)
+
+ def test_all_zero_rows(self):
+ # All-zero groups take the e8m0 = 0 clamp on every lane count, so one
+ # loop covers what used to be one case per dim.
+ for dim in COOPERATIVE_DIMS:
+ x = torch.zeros(3, dim, dtype=torch.bfloat16, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))
+ assert torch.all(codes.cpu() == 0) and torch.all(scale.cpu() == 0), f"D={dim}"
+
+ def test_rejects_k_not_multiple_of_dim(self):
+ for dim in COOPERATIVE_DIMS:
+ x = torch.randn(2, dim + GROUP_SIZE, dtype=torch.bfloat16, device="xpu")
+ with pytest.raises(ValueError, match=f"multiple of {dim}"):
+ mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))
+
+ def test_rejects_non_sylvester(self):
+ """There is no O(D^2) fallback above D = 32, so a custom matrix is refused."""
+ for dim in COOPERATIVE_DIMS:
+ h = get_hadamard_matrix(dim, "xpu").clone()
+ h[0, 1] = -h[0, 1]
+ x = torch.randn(2, dim, dtype=torch.bfloat16, device="xpu")
+ with pytest.raises(NotImplementedError):
+ mxfp4_hadamard_quant(x, h)
+
+ def test_transform_size_changes_the_output(self):
+ """Each D is a genuinely different transform, not a relabelled D = 32.
+
+ The output *shapes* are D-independent -- everything quantizes in groups
+ of 32, so downstream consumers and the bandwidth accounting are
+ unaffected by the transform size -- while the values must differ. The
+ value check is what stops a dispatcher bug that silently fell back to
+ the 32-point butterfly from comparing the reference against itself.
+ """
+ torch.manual_seed(35)
+ x = torch.randn(16, 512, dtype=torch.bfloat16, device="xpu")
+ c32, s32 = mxfp4_hadamard_quant(x)
+ for dim in COOPERATIVE_DIMS:
+ cd, sd = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))
+ assert c32.shape == cd.shape and s32.shape == sd.shape, f"D={dim}"
+ assert not torch.equal(c32.cpu(), cd.cpu()), f"D={dim} fell back to the 32-point butterfly"
+
+ @pytest.mark.parametrize("dim", SUPPORTED_HADAMARD_DIMS)
+ def test_bit_exact_at_scale(self, dim):
+ """Large-shape bit-exactness, which small shapes cannot establish.
+
+ The FMA-contraction bug this guards against flipped roughly 1 code in
+ 4 million -- only groups whose scaled magnitude landed within an ULP of
+ an E2M1 threshold -- so every shape elsewhere in this file passed while
+ the kernel was wrong. Catching it needs enough codes for the tail to
+ show up, hence 16 M here rather than the few thousand used above.
+
+ It is also why D = 64 and D = 256 must be covered and not assumed safe
+ by analogy: their 1/sqrt(D) is an exact power of two, which made the
+ load-time product exact and hid the bug for those two sizes alone.
+ """
+ torch.manual_seed(36)
+ x = torch.randn(4096, 4096, dtype=torch.bfloat16, device="xpu")
+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(dim, "cpu"))
+ bad_codes = int((codes.cpu() != ref_codes).sum())
+ bad_scale = int((scale.cpu() != ref_scale).sum())
+ assert bad_codes == 0, f"D={dim}: {bad_codes} of {ref_codes.numel()} codes differ"
+ assert bad_scale == 0, f"D={dim}: {bad_scale} of {ref_scale.numel()} scales differ"
+
+ def test_normalization_is_applied_last(self):
+ """The cooperative path must match the norm-last reference, not norm-first.
+
+ These two orders differ only in rounding, so this pins the choice that
+ keeps the kernel free of a contractable multiply-add (see the
+ normalization note in xpu_mxfp4_hadamard.hpp). For D whose 1/sqrt(D) is
+ an exact power of two the two orders coincide, so only the others can
+ actually discriminate.
+ """
+ from auto_round_kernel.mxfp4_hadamard import fwht_transform_reference
+
+ for dim in COOPERATIVE_DIMS:
+ h = get_hadamard_matrix(dim, "cpu")
+ norm = h.reshape(-1)[0]
+ torch.manual_seed(37)
+ x = torch.randn(2048, dim, dtype=torch.float32)
+ last = fwht_transform_reference(x, norm, norm_last=True)
+ first = fwht_transform_reference(x, norm, norm_last=False)
+ assert torch.allclose(last, first, atol=1e-4), f"D={dim}"
+ # 1/sqrt(D) is an exact power of two only for D = 64 (1/8) and
+ # D = 256 (1/16); a frexp mantissa of exactly 0.5 is the precise test
+ # for that. Everything else -- 128 included -- must discriminate.
+ if math.frexp(float(norm))[0] != 0.5:
+ assert not torch.equal(last, first), f"D={dim}: the two orders must be distinguishable"
+
+ def test_lane_count_divides_sub_group(self):
+ """The correctness precondition for the cross-lane butterfly.
+
+ A row's lanes must share one sub-group, which holds only while the lane
+ count divides the sub-group size of 32.
+ """
+ for dim in COOPERATIVE_DIMS:
+ assert 32 % (dim // GROUP_SIZE) == 0
+ assert max(SUPPORTED_HADAMARD_DIMS) // GROUP_SIZE == MAX_LANES_PER_ROW
+
+ @pytest.mark.parametrize("dtype", DTYPES)
+ def test_attention_head_dim_layout(self, dtype):
+ """A [B, S, H, D] attention tensor transforms along the head dim.
+
+ The kernel only requires the transform dimension to be the innermost
+ one, which is exactly how QKV activations are laid out.
+ """
+ torch.manual_seed(23)
+ x = torch.randn(1, 96, 4, HADAMARD_DIM_128, dtype=dtype, device="xpu")
+ h = get_hadamard_matrix(HADAMARD_DIM_128, x.device)
+ codes, scale = mxfp4_hadamard_quant(x, h)
+ num_rows = x.numel() // HADAMARD_DIM_128
+ assert codes.shape == (num_rows, HADAMARD_DIM_128 // 2)
+ assert scale.shape == (num_rows, 4)
+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(HADAMARD_DIM_128, "cpu"))
+ assert torch.equal(codes.cpu(), ref_codes)
+ assert torch.equal(scale.cpu(), ref_scale)
+
+ def test_rejects_xmx_for_cooperative_dims(self):
+ """The XMX path is D = 32 only."""
+ for dim in COOPERATIVE_DIMS:
+ x = torch.randn(2, dim, dtype=torch.bfloat16, device="xpu")
+ with pytest.raises((RuntimeError, ValueError)):
+ mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device), _force_xmx=True)
+
+ def test_rejects_dim_above_max_lane_count(self):
+ """A power-of-two D beyond 32 * MAX_LANES_PER_ROW has no instantiation.
+
+ These would need more than 16 lanes per row; 1024 is the first size that
+ needs a full 32-lane sub-group per row, which is legal in principle but
+ deliberately not built.
+ """
+ for dim in (1024, 2048):
+ x = torch.randn(2, dim, dtype=torch.bfloat16, device="xpu")
+ with pytest.raises(ValueError):
+ mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))
+
+ def test_rejects_non_power_of_two_dim(self):
+ """D must be 32 * 2^n: the butterfly has no non-power-of-two form."""
+ for dim in (96, 160):
+ x = torch.randn(2, dim * 2, dtype=torch.bfloat16, device="xpu")
+ with pytest.raises(ValueError):
+ mxfp4_hadamard_quant(x, torch.eye(dim, dtype=torch.float32, device="xpu"))
+
+ def test_deterministic_across_modes(self):
+ """Every mode must be bit-for-bit repeatable.
+
+ The modes are separate kernels behind separate dispatch decisions, but
+ the property is the same one, so a single test covers all of them
+ instead of one near-identical test per class.
+ """
+ torch.manual_seed(24)
+ x32 = torch.randn(256, 512, dtype=torch.float16, device="xpu")
+ x128 = torch.randn(70, 256, dtype=torch.float16, device="xpu")
+ h128 = get_hadamard_matrix(HADAMARD_DIM_128, x128.device)
+ cases = [
+ ("fused D=32", lambda: mxfp4_hadamard_quant(x32)),
+ ("cooperative D=128", lambda: mxfp4_hadamard_quant(x128, h128)),
+ ("quant-only", lambda: mxfp4_hadamard_quant(x32, _quant_only=True)),
+ ("stream-only", lambda: mxfp4_hadamard_quant(x32, _stream_only=True)),
+ ]
+ if _xmx_supported():
+ cases.append(("xmx", lambda: mxfp4_hadamard_quant(x32, _force_xmx=True)))
+ for name, run in cases:
+ first, first_scale = run()
+ for _ in range(2):
+ again, again_scale = run()
+ assert torch.equal(first.cpu(), again.cpu()), name
+ assert torch.equal(first_scale.cpu(), again_scale.cpu()), name
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])