PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat(ark): fused Hadamard + MXFP4 activation quantization kernel
jijiazecc5b0053a2b
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex d22b47c2d..cc73da5c0 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -3868,6 +3868,19 @@ if torch.xpu.is_available():except ImportError as _e:print(f"ARK is unable to load XPU lib: {_e}")+# Activation fused HMT + MXFP4 quantization (XPU). Imported last so the lib+# handles above are already bound when the submodule looks them up.+from .mxfp4_hadamard import ( # noqa: E402+ HADAMARD_DIM,+ HADAMARD_DIM_128,+ MAX_LANES_PER_ROW,+ SUPPORTED_HADAMARD_DIMS,+ get_hadamard_matrix,+ mxfp4_hadamard_quant,+ mxfp4_hadamard_quant_reference,+ mxfp4_quant_reference,+ mxfp4_stream_reference,+)if __name__ == "__main__":print(cpu_lib is None, xpu_lib is None)diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex affb8b5bb..870cc924a 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -28,7 +28,10 @@ typedef uintptr_t torch_ptr;#include "xpu_wrapper.hpp"#include "sycl_s8_wrapper.hpp"#include "utils.hpp"+#include "xpu_mxfp4_hadamard.hpp"#if ARK_SYCL_TLA+#include "xpu_mxfp4_hadamard_xmx.hpp"+// Only include declarations, implementations are in separate .cpp files#include "sycl_tla_common.hpp"#endif#else@@ -776,6 +779,143 @@ static void sage_dynamic_quant_v_layout(torch_ptr stream, torch_ptr input, torch}}+#if defined(ARK_SYCL_TLA)+// Scratch-pool slot for the ``[D, D]`` Hadamard staging buffer the XMX path+// below needs in the activation dtype. ``DeviceMemoryPool`` (``utils.hpp``)+// hands out one grow-on-demand slab per (slot, device UUID, context, queue)+// key, reused across every later call on that queue; slots 0-10 are already+// claimed (see the slot table on ``DeviceMemoryPool::MaxLocNum``), so this one+// takes 11 rather than aliasing a live scratch buffer.+constexpr size_t kHadamardStagingScratchLoc = 11;++// Body of the XMX path, templated over the activation dtype+// (``cute::half_t`` / ``cute::bfloat16_t``) so the FP16 and BF16 entry points+// share a single implementation.+//+// ``h_ptr`` is the caller's FP32 Hadamard matrix; the staging buffer holds its+// transpose in ``T``, which is the layout the ``C = A @ B^T`` GEMM needs (see+// ``convert_hadamard_to_dtype``). Every call rewrites all ``D * D`` elements,+// so reusing one slab across calls with different matrices is safe: the two+// launches that touch it go to the same queue, and the extension's queues are+// in-order (see the note in `DeviceMemoryPool::get_scratch_ptr`).+template <typename T>+static void xmx_hadamard_quant_impl(sycl::queue* q, torch_ptr x, const float* h_ptr, uint8_t* codes_ptr,+ uint8_t* scale_ptr, int64_t total_groups) {+ constexpr size_t kElems =+ static_cast<size_t>(XpuMxfp4Hadamard::kHadamardDim) * static_cast<size_t>(XpuMxfp4Hadamard::kHadamardDim);+ // The request size is a compile-time constant, so the slab is allocated on+ // the first call for a queue and then never grows -- the pool's grow path+ // (which frees the old slab in place) is unreachable from here.+ auto* pool = DeviceMemoryPool::Instance();+ auto* h_t = static_cast<T*>(pool->get_scratch_mem(sizeof(T) * kElems, kHadamardStagingScratchLoc, q));+ if (h_t == nullptr) {+ throw std::runtime_error("ark::mxfp4_hadamard_quant: failed to acquire the Hadamard staging buffer");+ }+ ark::xmx_hadamard_detail::convert_hadamard_to_dtype<T>(q, h_ptr, h_t);+ ark::xmx_hadamard_detail::mxfp4_hadamard_quant_xmx<T>(q, reinterpret_cast<const T*>(x), h_t, codes_ptr, scale_ptr,+ total_groups);+}+#endif // ARK_SYCL_TLA++// Activation-only fused kernel: normalized Hadamard + MXFP4 quant.+// x: [num_rows, k] FP16 or BF16+// hadamard: [D, D] FP32, row major, already normalized by 1/sqrt(D)+// hadamard_dim: D, one of 32 (GEMM activations), 64, 128 (attention head dim),+// 256 or 512. The quantization group stays at 32 for all of them, so+// a D = 128 row yields 4 MXFP4 groups. Every D > 32 implements the+// FWHT path only.+// use_fwht: true when hadamard is the normalized Sylvester matrix, which is the+// only matrix the butterfly network implements. The caller decides so+// that the hot path does not pay for a device-side comparison.+// use_xmx: opt-in XMX path (requires an ARK_SYCL_TLA build). Uses the+// relaxed numerical contract of xpu_mxfp4_hadamard_xmx.hpp (H stored+// in the activation dtype, DPAS accumulation); tolerance-based, not+// bit-exact. D = 32 only.+// use_quant_only: strip the Hadamard transform and quantize the raw activation+// (quant-only baseline: byte-identical traffic to the fused path, so+// the bandwidth ratio isolates the cost of the transform). Ignored+// on the XMX path.+// use_stream_only: strip the transform *and* the quantization math, keeping the+// loads, the packing shape and the stores (traffic-matched roofline+// baseline). Mutually exclusive with use_quant_only.+// out_codes: [num_rows, k / 2] uint8, two packed FP4 codes per byte+// out_scale: [num_rows, k / 32] uint8, one E8M0 exponent per 32-element group+static void mxfp4_hadamard_quant(torch_ptr stream, torch_ptr x, torch_ptr hadamard, torch_ptr out_codes,+ torch_ptr out_scale, int64_t num_rows, int64_t k, int in_dtype, bool use_fwht,+ bool use_xmx, bool use_quant_only, int64_t hadamard_dim, bool use_stream_only) {+ if (!stream) {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: stream must not be null");+ }+ if (!x || !hadamard || !out_codes || !out_scale) {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: input/output pointers must not be null");+ }+ if (num_rows <= 0 || k <= 0) {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: num_rows and k must be positive");+ }+ if (k % ark::XpuMxfp4Hadamard::kGroupSize != 0) {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: k must be a multiple of 32");+ }+ if (use_quant_only && use_stream_only) {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: use_quant_only and use_stream_only are exclusive");+ }+ const bool baseline = use_quant_only || use_stream_only;+ // The baselines run on the flat [total_groups, 32] view, so the transform+ // dimension is irrelevant to them and is not validated in that case.+ if (!baseline) {+ if (!ark::XpuMxfp4Hadamard::is_supported_hadamard_dim(hadamard_dim)) {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: hadamard_dim must be 32, 64, 128, 256 or 512");+ }+ if (k % hadamard_dim != 0) {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: k must be a multiple of hadamard_dim");+ }+ if (hadamard_dim > ark::XpuMxfp4Hadamard::kGroupSize) {+ if (!use_fwht) {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: hadamard_dim > 32 requires the Sylvester matrix");+ }+ if (use_xmx) {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: use_xmx is not supported for hadamard_dim > 32");+ }+ }+ }+ auto* q = (sycl::queue*)stream;+ auto* h_ptr = (const float*)hadamard;+ auto* codes_ptr = (uint8_t*)out_codes;+ auto* scale_ptr = (uint8_t*)out_scale;+ const auto dtype = (BTLA_DTYPE)in_dtype;+ const int64_t total_groups = num_rows * (k / ark::XpuMxfp4Hadamard::kGroupSize);++ if (use_xmx && !baseline) {+#if defined(ARK_SYCL_TLA)+ // XMX path: H is converted to the activation dtype (lossless) and the+ // transform runs on DPAS. x (sycl bf16/half) is layout-identical to+ // cute::bfloat16_t / cute::half_t, so the pointers are reinterpreted.+ if (dtype == BTLA_DTYPE::F16) {+ xmx_hadamard_quant_impl<cute::half_t>(q, x, h_ptr, codes_ptr, scale_ptr, total_groups);+ } else if (dtype == BTLA_DTYPE::BF16) {+ xmx_hadamard_quant_impl<cute::bfloat16_t>(q, x, h_ptr, codes_ptr, scale_ptr, total_groups);+ } else {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: only FP16 and BF16 activations are supported");+ }+#else+ (void)total_groups;+ throw std::runtime_error("ark::mxfp4_hadamard_quant: use_xmx requires an ARK_SYCL_TLA build");+#endif+ return;+ }++ if (dtype == BTLA_DTYPE::F16) {+ ark::XpuMxfp4Hadamard::mxfp4_hadamard_quant<sycl::half>(q, (const sycl::half*)x, h_ptr, codes_ptr, scale_ptr,+ num_rows, k, use_fwht, use_quant_only, hadamard_dim,+ use_stream_only);+ } else if (dtype == BTLA_DTYPE::BF16) {+ ark::XpuMxfp4Hadamard::mxfp4_hadamard_quant<sycl::ext::oneapi::bfloat16>(+ q, (const sycl::ext::oneapi::bfloat16*)x, h_ptr, codes_ptr, scale_ptr, num_rows, k, use_fwht,+ use_quant_only, hadamard_dim, use_stream_only);+ } else {+ throw std::invalid_argument("ark::mxfp4_hadamard_quant: only FP16 and BF16 activations are supported");+ }+}+#elif !defined(ARK_XPU)enum class CpuSdpaRoute {@@ -1424,6 +1564,11 @@ PYBIND11_MODULE(PY_NAME, m) {m.def("sage_compute_seq_mean_bias_layout", &ark::sage_compute_seq_mean_bias_layout);m.def("sage_dynamic_quant_layout", &ark::sage_dynamic_quant_layout);m.def("sage_dynamic_quant_v_layout", &ark::sage_dynamic_quant_v_layout);+ m.def("mxfp4_hadamard_quant", &ark::mxfp4_hadamard_quant, pybind11::arg("stream"), pybind11::arg("x"),+ pybind11::arg("hadamard"), pybind11::arg("out_codes"), pybind11::arg("out_scale"),+ pybind11::arg("num_rows"), pybind11::arg("k"), pybind11::arg("in_dtype"), pybind11::arg("use_fwht") = true,+ pybind11::arg("use_xmx") = false, pybind11::arg("use_quant_only") = false,+ pybind11::arg("hadamard_dim") = 32, pybind11::arg("use_stream_only") = false);m.def("moe_gemm", &ark::moe_gemm_wrapper);m.def("moe_gemm_decode", &ark::moe_gemm_decode_wrapper);m.def("moe_decode_release_scratch", &ark::moe_decode_release_scratch);diff --git a/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py b/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.pynew file mode 100644index 000000000..e6ab5d2d9--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py@@ -0,0 +1,583 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Activation fused HMT + MXFP4 quantization (XPU).++Pipeline::++ FP16/BF16 activation -> 32-point normalized Hadamard -> MXFP4 quantization+ -> packed FP4 codes + E8M0 scales++Frozen MVP contract (Phase 0):++* ``hadamard_dim == group_size == 32`` and ``K % 32 == 0``;+* ``H`` is a normalized 32x32 Hadamard matrix (already contains ``1/sqrt(32)``),+ so ``y = reshape(x, [-1, 32]) @ H`` needs no extra scaling;+* per 32-element group ``amax = max(|y_g|)``,+ ``e8m0 = clamp(floor(log2(amax)) - 2 + 127, 0, 254)`` and the effective scale+ is ``2 ** (e8m0 - 127)`` (standard E8M0, always a power of two);+* ``q = y * 2 ** -(e8m0 - 127)`` is encoded as ``signbit(q) << 3 | magnitude``+ with FP4 (E2M1) magnitude levels ``0, 0.5, 1, 1.5, 2, 3, 4, 6`` and the+ nearest-even thresholds of ``vllm_ext/fp4_utils.py::cast_to_fp4``;+* **zero is canonicalised**: whenever the magnitude index is 0 the sign bit is+ dropped, so the code is ``0x0`` and never ``0x8`` (negative zero). See+ "Canonical zero" below for why this rule is required rather than optional;+* two codes share one byte, the even element occupying the low nibble+ (identical to ``vllm_ext/fp4_utils.py::pack_fp4_to_uint8``);+* an all-zero group produces ``e8m0 = 0`` and all-zero codes;+* NaN/Inf are outside the supported input domain. The reference always rejects+ them; the XPU entry point only does so under ``check_finite=True``, because+ the scan reads the whole activation and syncs, costing several times the+ fused kernel itself.++FP32 transform contract (Phase 2, revised in Phase 3)+-----------------------------------------------------++Bit-exactness between the SYCL kernel and this reference requires a *defined*+summation order for ``y = x_g @ H``. There are two paths, and each is bit-exact+against its own reference; they are deliberately *not* bit-exact against each+other, because a butterfly network and a 32-term dot product round differently.+:func:`transform_reference` mirrors the choice the wrapper makes.++**FWHT (default, used for the normalized Sylvester matrix).** Five butterfly+stages; stage ``s`` pairs each lane with ``lane ^ (1 << s)`` and the lane+holding the high half of the pair computes the difference. A single final+multiply by ``H[0][0] == 1/sqrt(32)`` applies the normalization. Only adds and+subtracts occur, so there is no multiply-add for the compiler to contract, and+the order is fully determined by the stage index.++This path exists for performance and is not merely an optimization detail. The+kernel is intended to be memory bound, but the O(D^2) path below costs 32+multiplies plus 32 adds per lane with FMA disabled, which caps effective+bandwidth at roughly 60% of the measured streaming-copy baseline on Arc Pro B60+*before* accounting for shuffles and matrix loads. The butterfly costs 5 adds+plus one scale, moving the bottleneck back to memory.++**Path A (only for a caller-supplied non-Sylvester matrix).** ``acc`` starts at+``+0.0`` and is updated as ``acc = fp32_add(acc, fp32_mul(x[j], H[j][i]))`` for+``j = 0 .. 31`` in increasing ``j``, with a separate FP32 rounding after the+multiply and after the add (no fused multiply-add, no reassociation). The+kernel enforces this with ``#pragma clang fp contract(off)``.+``torch.matmul`` is deliberately not used for either path because its blocking,+FMA usage and reassociation are unspecified and would make the bit-exact+acceptance criterion untestable.++Canonical zero (Phase 2)+------------------------++There is one quantity the accumulation contract above cannot pin down: the+*sign* of a result that is mathematically zero. When a group of 32 inputs is+constant, every output column except the first cancels exactly, and the residue+left by FP32 rounding is on the order of ``1e-8`` with an order-dependent sign.+Device-side flush-to-zero of FP32 subnormals produces the same ambiguity for+very small inputs, where the CPU reference keeps the subnormal but the GPU+returns a signed zero.++Such a value always quantizes to FP4 magnitude index 0, so the ambiguity can+only ever affect the sign bit, turning ``0x0`` into ``0x8`` (negative zero).+Because ``0x8`` and ``0x0`` dequantize to the same number, no information is+lost by forbidding ``0x8``, and doing so makes the encoding a total function of+the mathematical value rather than of the rounding residue. Both this reference+and the kernel therefore drop the sign bit whenever the magnitude index is 0.++This is a deliberate, documented deviation from+``vllm_ext/fp4_utils.py::pack_fp4_to_uint8``, which applies ``signbit``+unconditionally: that helper encodes already-clean dequantized values, where a+negative zero can only appear if the caller supplied one.+"""++from __future__ import annotations++import torch++HADAMARD_DIM = 32+GROUP_SIZE = 32++# Attention head dim. The quantization group stays at ``GROUP_SIZE`` for every+# transform size, so a 128-element row produces 4 independent MXFP4 groups and+# the output layout is identical to the 32-point case.+HADAMARD_DIM_128 = 128++# Transform sizes the XPU kernel implements: ``GROUP_SIZE * L`` for a power-of-+# two lane count ``L``. Every size above 32 is FWHT-only (Sylvester matrix) and+# fans a row out over ``L`` cooperating sub-group lanes, so ``L`` must divide the+# sub-group size of 32 -- see ``fwht_quant_cooperative`` in+# xpu_mxfp4_hadamard.hpp. 512 is the largest instantiated size; nothing in the+# design stops 1024, it simply has no consumer.+MAX_LANES_PER_ROW = 16+SUPPORTED_HADAMARD_DIMS = tuple(GROUP_SIZE * (1 << i) for i in range(MAX_LANES_PER_ROW.bit_length()))++# FP4 (E2M1) magnitude levels.+E2M1_VALUES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)++# (threshold, is_closed_interval, magnitude_index), first match wins, mirroring+# the early-out ladder of ``cast_to_fp4`` including its alternating ``<=`` / ``<``+# boundary operators. ``_encode_fp4`` walks the table in reverse with+# ``torch.where``, so each earlier entry overrides every later one.+_E2M1_THRESHOLDS = (+ (0.25, True, 0),+ (0.75, False, 1),+ (1.25, True, 2),+ (1.75, False, 3),+ (2.5, True, 4),+ (3.5, False, 5),+ (5.0, True, 6),+)++_HADAMARD_CACHE: dict[tuple[int, str], torch.Tensor] = {}+++def _sylvester_hadamard(dim: int) -> torch.Tensor:+ if dim < 1 or (dim & (dim - 1)) != 0:+ raise ValueError(f"Hadamard dimension must be a power of two, got {dim}")+ h = torch.ones(1, 1, dtype=torch.float32)+ while h.shape[0] < dim:+ h = torch.cat([torch.cat([h, h], dim=1), torch.cat([h, -h], dim=1)], dim=0)+ return h+++def get_hadamard_matrix(dim: int = HADAMARD_DIM, device: torch.device | str = "cpu") -> torch.Tensor:+ """Return the normalized ``dim x dim`` Hadamard matrix (FP32, contiguous)."""+ key = (dim, str(torch.device(device)))+ cached = _HADAMARD_CACHE.get(key)+ if cached is None:+ cached = (_sylvester_hadamard(dim) / (dim**0.5)).to(device=device).contiguous()+ _HADAMARD_CACHE[key] = cached+ return cached+++def _validate_hadamard(hadamard_matrix: torch.Tensor, *, check_finite: bool = True) -> int:+ """Validate the matrix and return its dimension ``D``.++ ``D`` is read from the matrix rather than passed in, so a caller selects the+ transform size simply by handing over a ``get_hadamard_matrix(D)`` tensor.+ """+ if not isinstance(hadamard_matrix, torch.Tensor):+ raise TypeError(f"hadamard_matrix must be a torch.Tensor, got {type(hadamard_matrix)}")+ shape = tuple(hadamard_matrix.shape)+ if len(shape) != 2 or shape[0] != shape[1] or shape[0] not in SUPPORTED_HADAMARD_DIMS:+ allowed = " or ".join(f"({d}, {d})" for d in SUPPORTED_HADAMARD_DIMS)+ raise ValueError(f"hadamard_matrix must have shape {allowed}, got {shape}")+ if hadamard_matrix.dtype not in (torch.float32, torch.float64):+ raise ValueError(f"hadamard_matrix must be float32 or float64, got {hadamard_matrix.dtype}")+ # Unlike the checks above, this one reads a device tensor and forces a+ # device->host sync, which costs about as much as the fused kernel itself.+ # Callers that have already established the matrix is the known-good default+ # pass check_finite=False.+ if check_finite and not torch.isfinite(hadamard_matrix).all():+ raise ValueError("hadamard_matrix must contain only finite values")+ return shape[0]+++def _require_activation_tensor(x: torch.Tensor) -> None:+ """Reject a non-tensor ``x`` before anything dereferences it.++ Both entry points resolve the Hadamard matrix (and hence ``x.device``)+ before the full activation validation runs, so this cheap check has to+ happen first for a bad ``x`` to raise TypeError rather than AttributeError.+ """+ if not isinstance(x, torch.Tensor):+ raise TypeError(f"x must be a torch.Tensor, got {type(x)}")+++def _validate_activation(+ x: torch.Tensor, *, require_xpu: bool, check_finite: bool = True, hadamard_dim: int = HADAMARD_DIM+) -> tuple[int, int]:+ _require_activation_tensor(x)+ if x.dtype not in (torch.float16, torch.bfloat16):+ raise ValueError(f"x must be float16 or bfloat16, got {x.dtype}")+ if x.ndim < 1:+ raise ValueError("x must have at least one dimension")+ if require_xpu and x.device.type != "xpu":+ raise ValueError(f"mxfp4_hadamard_quant is only supported on XPU, got device {x.device}")+ k = x.shape[-1]+ if k == 0 or x.numel() == 0:+ raise ValueError("x must not be empty")+ # The transform dimension is the stricter of the two constraints (it is a+ # multiple of GROUP_SIZE), so checking it alone is sufficient.+ if k % hadamard_dim != 0:+ raise ValueError(f"the last dimension of x must be a multiple of {hadamard_dim}, got {k}")+ # This scan reads all of x and then forces a device->host sync on the+ # result, which on XPU costs roughly 4x the fused kernel itself. It is a+ # debugging aid, not part of the numerical contract, so the device entry+ # point leaves it off by default (see ``check_finite`` there).+ if check_finite and not torch.isfinite(x).all():+ raise ValueError("x must contain only finite values (NaN/Inf are not supported)")+ return x.numel() // k, k+++def hadamard_transform_reference(x_groups: torch.Tensor, h: torch.Tensor) -> torch.Tensor:+ """``x_groups [G, 32] @ h [32, 32]`` under the Path A FP32 accumulation contract.++ Sums over ``j`` in increasing order with a separate FP32 rounding after each+ multiply and each add, matching the kernel loop exactly. ``torch.matmul`` is+ intentionally avoided (unspecified blocking / FMA / reassociation).++ Used only for a caller-supplied non-Sylvester matrix; the default matrix+ goes through :func:`fwht_transform_reference`.+ """+ x_groups = x_groups.to(torch.float32)+ h = h.to(torch.float32)+ acc = torch.zeros_like(x_groups)+ for j in range(h.shape[0]):+ acc = acc + x_groups[:, j : j + 1] * h[j]+ return acc+++def fwht_transform_reference(x_groups: torch.Tensor, norm: torch.Tensor, *, norm_last: bool = False) -> torch.Tensor:+ """D-point fast Walsh-Hadamard transform under the frozen butterfly contract.++ Computes the same mathematical result as ``x_groups @ H`` for the normalized+ Sylvester matrix ``H`` of dimension ``D``, but in ``log2(D)`` butterfly+ stages instead of ``D`` multiply-accumulates. ``norm = H[0][0] =+ 1/sqrt(D)`` is applied *first*, then stage ``s`` pairs each lane with+ ``lane ^ (1 << s)``:++ ``acc = (lane & h) ? (partner - acc) : (acc + partner)``++ Normalizing up front rather than at the end costs the same single multiply+ but bounds the intermediates by ``sqrt(D) * max|x|`` instead of+ ``D * max|x|``, keeping the safe input range identical to Path A. With the+ scale applied last, inputs above ``FP32_MAX / D`` overflow to infinity even+ though the mathematical result is perfectly representable.++ Taking ``norm`` from the matrix itself (rather than recomputing+ ``1/sqrt(D)``) guarantees the kernel and this reference scale by the+ identical FP32 value.++ Every intermediate is a plain FP32 add or subtract, so there is nothing for+ the compiler to contract into an FMA and the order is fully determined by+ the stage index -- which is what keeps this bit-exact against the kernel.++ ``norm_last`` moves the scaling after the butterflies, mirroring the+ cooperative ``D > 32`` kernel. That kernel cannot scale on load: doing so+ leaves a multiply feeding the first stage's add, which the GPU backend fuses+ into an FMA that no compiler flag or pragma was able to suppress, breaking+ bit-exactness. See the normalization note above ``fwht_quant_cooperative``+ in xpu_mxfp4_hadamard.hpp. The two orders round differently and are+ deliberately not bit-exact against each other.+ """+ dim = x_groups.shape[-1]+ scale = norm.to(device=x_groups.device, dtype=torch.float32)+ acc = x_groups.to(torch.float32)+ if not norm_last:+ acc = acc * scale+ lanes = torch.arange(dim, device=acc.device)+ for stage in range(dim.bit_length() - 1):+ h = 1 << stage+ partner = acc[:, lanes ^ h]+ acc = torch.where((lanes & h) != 0, partner - acc, acc + partner)+ if norm_last:+ acc = acc * scale+ return acc+++def is_default_hadamard(hadamard_matrix: torch.Tensor) -> bool:+ """True if ``hadamard_matrix`` is exactly the normalized Sylvester matrix.++ Only that matrix may take the FWHT path, because the butterfly network+ implements the Sylvester ordering specifically.+ """+ dim = hadamard_matrix.shape[0]+ # Callers normally pass the tensor returned by get_hadamard_matrix, which is+ # cached per device; recognising it by identity avoids a device->host copy+ # and the sync that torch.equal would impose on every quantization call.+ if hadamard_matrix is _HADAMARD_CACHE.get((dim, str(hadamard_matrix.device))):+ return True+ h = hadamard_matrix.to(torch.float32)+ return bool(torch.equal(h.cpu(), get_hadamard_matrix(dim, "cpu")))+++def transform_reference(x_groups: torch.Tensor, h: torch.Tensor) -> torch.Tensor:+ """Dispatch to the FWHT or Path A reference, mirroring the kernel's choice."""+ if is_default_hadamard(h):+ # D > 32 runs the cooperative kernel, which normalizes after the+ # butterflies rather than on load; the reference has to match.+ return fwht_transform_reference(x_groups, h.reshape(-1)[0], norm_last=h.shape[0] > GROUP_SIZE)+ return hadamard_transform_reference(x_groups, h)+++def _e8m0_and_quantized(y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ """Return ``(e8m0 [G], q [G, 32])`` for FP32 transformed groups ``y [G, 32]``."""+ amax = y.abs().amax(dim=-1)+ # frexp: amax = mantissa * 2 ** exp with mantissa in [0.5, 1)+ # => floor(log2(amax)) == exp - 1 (exact, also for exact powers of two).+ _, exp = torch.frexp(amax)+ scale_exp = exp.to(torch.int32) - 1 - 2+ e8m0 = torch.clamp(scale_exp + 127, 0, 254)+ # torch.frexp(inf) returns exponent 0, which would silently yield e8m0 = 124.+ # The kernel uses ilogb, which saturates, so an overflowed group must clamp+ # to 254 here too or the two would disagree. y can only be non-finite when+ # the transform of a finite input overflowed FP32, i.e. for inputs beyond+ # the documented safe range; the group is garbage either way, but the two+ # implementations must still agree on it.+ e8m0 = torch.where(torch.isfinite(amax), e8m0, torch.full_like(e8m0, 254))+ zero_group = amax == 0+ e8m0 = torch.where(zero_group, torch.zeros_like(e8m0), e8m0)+ q = torch.ldexp(y, -(e8m0 - 127).unsqueeze(-1))+ q = torch.where(zero_group.unsqueeze(-1), torch.zeros_like(q), q)+ return e8m0.to(torch.uint8), q+++def _encode_fp4(q: torch.Tensor) -> torch.Tensor:+ """Encode FP32 values into 4-bit ``sign << 3 | magnitude_index`` codes."""+ a = q.abs()+ idx = torch.full_like(a, len(E2M1_VALUES) - 1, dtype=torch.int32)+ for threshold, closed, value in reversed(_E2M1_THRESHOLDS):+ hit = a <= threshold if closed else a < threshold+ idx = torch.where(hit, torch.full_like(idx, value), idx)+ sign = torch.signbit(q).to(torch.int32) << 3+ # Canonical zero: magnitude 0 always encodes as 0x0, never 0x8 (negative+ # zero). The sign of a value that rounds to zero is not reproducible across+ # implementations, so it must not reach the output. See module docstring.+ return torch.where(idx == 0, idx, idx | sign).to(torch.uint8)+++def pack_codes(codes: torch.Tensor) -> torch.Tensor:+ """Pack 4-bit codes ``[M, K]`` into bytes ``[M, K // 2]`` (even element = low nibble)."""+ low = codes[..., 0::2].to(torch.uint8) & 0x0F+ high = codes[..., 1::2].to(torch.uint8) & 0x0F+ return low | (high << 4)+++def mxfp4_hadamard_quant_reference(+ x: torch.Tensor, hadamard_matrix: torch.Tensor | None = None+) -> tuple[torch.Tensor, torch.Tensor]:+ """Pure PyTorch FP32 reference for :func:`mxfp4_hadamard_quant`.++ Runs on any device (including CPU) and defines the frozen numerical contract.+ The transform dimension is taken from ``hadamard_matrix`` (any of+ :data:`SUPPORTED_HADAMARD_DIMS`); the quantization group is always 32, so a+ 128-point row yields 4 groups.+ """+ _require_activation_tensor(x)+ if hadamard_matrix is None:+ hadamard_matrix = get_hadamard_matrix(HADAMARD_DIM, x.device)+ # The matrix is validated next because it determines the divisibility+ # constraint the activation is then checked against.+ dim = _validate_hadamard(hadamard_matrix)+ num_rows, k = _validate_activation(x, require_xpu=False, hadamard_dim=dim)++ h = hadamard_matrix.to(device=x.device, dtype=torch.float32).contiguous()+ y = transform_reference(x.contiguous().reshape(-1, dim), h)+ # The transform works on D-element rows, the quantizer on 32-element groups.+ e8m0, q = _e8m0_and_quantized(y.reshape(-1, GROUP_SIZE))+ codes = _encode_fp4(q).reshape(num_rows, k)+ return pack_codes(codes), e8m0.reshape(num_rows, k // GROUP_SIZE)+++def mxfp4_quant_reference(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ """Pure PyTorch FP32 reference for the **quant-only** baseline.++ Quantizes the *raw* activation (no Hadamard transform) with the same frozen+ MXFP4 contract as :func:`mxfp4_hadamard_quant_reference`: per-32-element+ E8M0 scale + packed FP4 codes. It is the reference for the quant-only path+ of :func:`mxfp4_hadamard_quant` (``_quant_only=True``), the Hadamard+ ablation of the bandwidth benchmark: that mode moves byte-identical traffic+ and only drops the transform, so the measured ratio isolates its cost.++ Mathematically it equals ``mxfp4_hadamard_quant_reference(x, I)``: running+ the reference transform with the identity matrix is bit-exact with applying+ no transform at all (it only ever adds exact zeros), so the two agree byte+ for byte.+ """+ num_rows, k = _validate_activation(x, require_xpu=False)+ x_groups = x.contiguous().reshape(-1, GROUP_SIZE).to(torch.float32)+ e8m0, q = _e8m0_and_quantized(x_groups)+ codes = _encode_fp4(q).reshape(num_rows, k)+ return pack_codes(codes), e8m0.reshape(num_rows, k // GROUP_SIZE)+++def mxfp4_stream_reference(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ """Pure PyTorch reference for the **stream-only** roofline baseline.++ The stream-only mode exists to answer "how fast could *any* kernel with this+ access pattern be?". It keeps the fused kernel's loads, packing shape and+ stores but replaces the transform and the quantization math with the+ cheapest data-dependent function available: the low nibble of each element's+ raw bit pattern, packed under the usual even-element-low-nibble rule, plus+ an XOR fold for the scale byte. The output is not a quantization of anything+ and must never be consumed as one.++ This reference is not part of the numerical contract -- it exists so a test+ can prove the baseline kernel actually reads every input element. Without+ it, dead-code elimination of the loads would turn the baseline into an empty+ kernel reporting an unreachable bandwidth, silently inflating every ratio+ measured against it.++ The scale byte folds the four code bytes that hold elements+ ``{0, 1}, {8, 9}, {16, 17}, {24, 25}`` of the group, which is exactly the+ low byte of the kernel's XOR over its four packed 32-bit words.+ """+ num_rows, k = _validate_activation(x, require_xpu=False, check_finite=False)+ # view() reinterprets the 16-bit float as raw bits without converting.+ bits = x.contiguous().view(torch.int16).reshape(-1, GROUP_SIZE).to(torch.int32) & 0x0F+ codes = pack_codes(bits.to(torch.uint8)).reshape(num_rows, k // 2)+ groups = codes.reshape(-1, GROUP_SIZE // 2)+ fold = groups[:, 0] ^ groups[:, 4] ^ groups[:, 8] ^ groups[:, 12]+ return codes, fold.reshape(num_rows, k // GROUP_SIZE)+++_XMX_SUPPORTED: bool | None = None+++def _xmx_supported() -> bool:+ """True when the current XPU build exposes the XMX path.++ Probes once by forcing the XMX path on a tiny tensor; the C++ binding raises+ ``RuntimeError`` when ARK_SYCL_TLA is not compiled in. The result is cached.+ """+ global _XMX_SUPPORTED+ if _XMX_SUPPORTED is None:+ try:+ x = torch.zeros(1, GROUP_SIZE, dtype=torch.float16, device="xpu")+ mxfp4_hadamard_quant(x, _force_xmx=True)+ _XMX_SUPPORTED = True+ except (RuntimeError, ValueError, NotImplementedError):+ _XMX_SUPPORTED = False+ return _XMX_SUPPORTED+++def mxfp4_hadamard_quant(+ x: torch.Tensor,+ hadamard_matrix: torch.Tensor | None = None,+ *,+ check_finite: bool = False,+ _force_xmx: bool | None = None,+ _quant_only: bool = False,+ _stream_only: bool = False,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Fused Hadamard transform + MXFP4 quantization on XPU.++ Args:+ x: FP16/BF16 XPU activation with ``x.shape[-1] % D == 0``, where ``D``+ is the dimension of ``hadamard_matrix``.+ hadamard_matrix: normalized ``D x D`` Hadamard matrix with ``D`` in+ :data:`SUPPORTED_HADAMARD_DIMS`, i.e. ``32 * 2**n`` for ``n = 0..4``.+ Defaults to the ``32 x 32`` Sylvester matrix returned by+ :func:`get_hadamard_matrix`. ``D = 32`` runs one work-item per+ quantization group; every larger ``D`` is implemented by a+ cooperative ``D / 32``-lane FWHT and supports the Sylvester matrix+ only. Pass ``get_hadamard_matrix(128, dev)`` for the attention+ head-dim transform.+ check_finite: reject NaN/Inf in ``x`` before launching. Off by default:+ the check reads all of ``x`` and syncs on the result, which costs+ several times the fused kernel itself. NaN/Inf are still outside the+ supported input domain -- the kernel simply does not police it on the+ hot path. :func:`mxfp4_hadamard_quant_reference` always checks.+ _force_xmx: private override used by tests/benchmarks (None = auto).+ _quant_only: private quant-only baseline override (default False). When+ True, the Hadamard transform is stripped and the *raw* activation is+ quantized (see :func:`mxfp4_quant_reference`), with identical memory+ traffic to the fused path -- the C++ dispatcher ignores+ use_fwht/use_xmx in this mode and always uses the per-item FWHT+ layout. This is the quant-only baseline: identical traffic, no+ transform, so the benchmark's ``f/q`` ratio isolates its cost.+ _stream_only: private stream-only baseline override (default False).+ When True, both the transform *and* the quantization math are+ stripped, leaving the loads, the packing shape and the stores. The+ byte traffic and the item mapping are identical to the fused path,+ so ``BW_fused / BW_stream_only`` is the kernel's utilization of the+ traffic-matched roofline -- the denominator the benchmark gates on.+ The output is *not* a quantization; see+ :func:`mxfp4_stream_reference`. Mutually exclusive with+ ``_quant_only``.++ Routing is automatic: the normalized Sylvester matrix always takes the+ bit-exact FWHT path (first priority); any other Hadamard matrix falls+ back to the XMX path when the build supports it (relaxed contract:+ H stored in the activation dtype, DPAS accumulation, tolerance-based+ acceptance), otherwise to the bit-exact Path A.++ Returns:+ ``(out_codes, out_scale)`` where ``out_codes`` is ``uint8 [M, K // 2]``+ (two packed FP4 codes per byte) and ``out_scale`` is+ ``uint8 [M, K // 32]`` (one E8M0 exponent per group), with+ ``M = x.numel() // K``.+ """+ from . import cvt_dtype, get_lib, get_stream++ if _quant_only and _stream_only:+ raise ValueError("_quant_only and _stream_only are mutually exclusive")+ _require_activation_tensor(x)++ hadamard_dim = HADAMARD_DIM+ if _quant_only or _stream_only:+ # Baselines run on the flat [total_groups, 32] view, so no Hadamard+ # matrix is involved: no validation and no device comparison. A+ # (default) matrix is still materialised so the pointer argument passed+ # to the C++ binding stays valid; the dispatcher never dereferences it+ # in either baseline mode.+ hadamard_matrix = get_hadamard_matrix(HADAMARD_DIM, x.device)+ use_fwht = True+ use_xmx = False+ elif hadamard_matrix is None:+ # The default matrix is known to be the Sylvester one, so the FWHT path+ # is taken without paying for a comparison on the hot path.+ hadamard_matrix = get_hadamard_matrix(HADAMARD_DIM, x.device)+ use_fwht = True+ else:+ # Structural checks are cheap. The finiteness check is not: it syncs on+ # the device every call. The default matrix is known finite, so only a+ # caller-supplied one pays for it.+ hadamard_dim = _validate_hadamard(hadamard_matrix, check_finite=False)+ use_fwht = is_default_hadamard(hadamard_matrix)+ if not use_fwht:+ if hadamard_dim != HADAMARD_DIM:+ # Only the butterfly network is implemented above D = 32; there+ # is no O(D^2) fallback and no XMX path for it.+ raise NotImplementedError(f"hadamard_dim {hadamard_dim} supports the normalized Sylvester matrix only")+ _validate_hadamard(hadamard_matrix)++ num_rows, k = _validate_activation(x, require_xpu=True, check_finite=check_finite, hadamard_dim=hadamard_dim)++ # Path resolution (auto-router): FWHT has first priority for the Sylvester+ # matrix; any other (custom) matrix falls back to the XMX path when the+ # build supports it (relaxed contract), otherwise to Path A. ``_force_xmx``+ # is a private override used by tests/benchmarks. ``_quant_only`` always+ # disables XMX: it is implemented on the shared per-item FWHT layout, which+ # is exactly the layout the quant-only baseline must match.+ if not (_quant_only or _stream_only):+ if _force_xmx is not None:+ use_xmx = bool(_force_xmx)+ elif use_fwht or hadamard_dim != HADAMARD_DIM:+ use_xmx = False+ else:+ use_xmx = _xmx_supported()++ lib = get_lib(x)+ if lib is None or not hasattr(lib, "mxfp4_hadamard_quant"):+ raise NotImplementedError("Current XPU build does not expose mxfp4_hadamard_quant")++ x_arg = x.contiguous().reshape(num_rows, k)+ h_arg = hadamard_matrix.to(device=x.device, dtype=torch.float32).contiguous()+ out_codes = torch.empty((num_rows, k // 2), dtype=torch.uint8, device=x.device)+ out_scale = torch.empty((num_rows, k // GROUP_SIZE), dtype=torch.uint8, device=x.device)++ lib.mxfp4_hadamard_quant(+ get_stream(x_arg),+ x_arg.data_ptr(),+ h_arg.data_ptr(),+ out_codes.data_ptr(),+ out_scale.data_ptr(),+ num_rows,+ k,+ cvt_dtype(x_arg.dtype),+ use_fwht,+ use_xmx,+ _quant_only,+ hadamard_dim,+ _stream_only,+ )+ return out_codes, out_scalediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hppindex efa54a48d..f7f166a58 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp@@ -214,8 +214,10 @@ class DeviceMemoryPool {// SDPA kernels; slot 8 is the MoE DPAS grouped-GEMM work-group counter// (`moe_dpas_fp8::kAtomicScratchLoc`); slots 9 and 10 are the int4 decode// weight-repack and activation-sum buffers- // (`moe_decode_detail::kInt4RepackScratchLoc` / `kActGroupSumScratchLoc`).- static constexpr int MaxLocNum = 11;+ // (`moe_decode_detail::kInt4RepackScratchLoc` / `kActGroupSumScratchLoc`);+ // slot 11 is the XMX Hadamard staging buffer+ // (`kHadamardStagingScratchLoc` in `ark.cpp`).+ static constexpr int MaxLocNum = 12;using SizeMap = std::unordered_map<size_t, size_t>;using PtrMap = std::unordered_map<size_t, int8_t*>;diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hppnew file mode 100644index 000000000..00b34c6f8--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp@@ -0,0 +1,701 @@+//+// Copyright (c) 2026 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//+// Activation fused kernel: FP16/BF16 activation -> normalized Hadamard+// transform (D = 32, 64, 128, 256 or 512) -> MXFP4 quantization (packed FP4+// codes + E8M0 scales).+//+// Numerical contract:+// * hadamard_dim == group_size == 32, K % 32 == 0+// * H is a *normalized* 32x32 Hadamard matrix (already contains 1/sqrt(32))+// * y = reshape(x, [-1, 32]) @ H (FP32 accumulation)+// * amax = max(|y_g|) per 32-element group+// * e8m0 = clamp(floor(log2(amax)) - 2 + 127, 0, 254)+// * q = y * 2^-(e8m0 - 127)+// * code = signbit(q) << 3 | e2m1_magnitude_index(|q|)+// * canonical zero: magnitude index 0 always encodes as 0x0, never 0x8+// * two codes per byte, even element in the low nibble+// * all-zero group -> e8m0 = 0 and all codes = 0+//+// FP32 transform contract: two paths exist and each is bit-exact against its own+// PyTorch reference.+//+// FWHT (default): log2(D) butterfly stages over lane ^ (1 << stage), plus a+// single multiply by H[0][0] == 1/sqrt(D). The butterflies are pure adds and+// subtracts, so their order is fixed by the stage index; the only contractable+// operation is that normalization multiply, and *where* it sits decides+// whether the kernel is reproducible. See the normalization note above+// fwht_quant_cooperative. This is the performance path: the O(D^2) dot product+// below is compute bound well below streaming-copy bandwidth on this device,+// the butterfly is memory bound.+//+// Path A (non-Sylvester matrix only): sums over j in increasing order with a+// separate FP32 rounding after each multiply and each add (no FMA, no+// reassociation), enforced by #pragma clang fp contract(off).+//+// The two paths round differently and are deliberately *not* bit-exact against+// each other; the wrapper picks one and the reference mirrors that choice.+//+// D > 32 (e.g. 128, the attention head dim). Only the FWHT path exists: H is+// required to be the normalized Sylvester matrix. The quantization group stays+// at 32, so a D-element row produces D/32 independent MXFP4 groups. See+// fwht_quant_cooperative below for the two-level (lane-local + cross-lane)+// decomposition and why a single work-item per row is not viable.+//+// Baseline modes. Two controlled ablations share the fused kernel's exact+// memory traffic and item mapping, so their bandwidths are directly+// comparable:+// * quant-only -- drops the Hadamard transform, keeps the quantization.+// * stream-only -- drops the transform *and* the quantization math, keeping+// only the loads, the packing shape and the stores. This is+// the traffic-matched roofline: the highest bandwidth any+// kernel with this read:write ratio and this access pattern+// can reach on the device, and therefore the only+// self-calibrating denominator for the fused kernel.++#pragma once++#include <cstdint>+#include <stdexcept>++#if defined(ARK_XPU)+#include <sycl/sycl.hpp>++namespace ark {++class XpuMxfp4Hadamard {+ public:+ static constexpr int kHadamardDim = 32;+ static constexpr int kGroupSize = 32;+ static constexpr int kSubGroupSize = 32;+ static constexpr int kWorkGroupSize = 256;+ // log2(32): number of FWHT butterfly stages.+ static constexpr int kNumFwhtStages = 5;++ // Cooperative FWHT for D > 32: L lanes per row, each owning one 32-element+ // quantization group, so D = 32 * L. The sub-group is pinned so that the L+ // lanes of a row always land in one sub-group, which is what makes the+ // cross-lane butterfly shuffles legal: sub-groups start at multiples of their+ // size and rows are L-lane aligned, so any size divisible by L is correct.+ //+ // 32 rather than 16. Both are legal, but requesting 16 forces the kernel to+ // be compiled SIMD16, which needs twice as many instructions to retire the+ // same work as SIMD32 and roughly halves ALU throughput. This kernel has+ // little slack, so that is not free: measured 207 GB/s at SIMD16 against+ // 357 GB/s at SIMD32 on Arc Pro B60, same code otherwise.+ static constexpr int kSubGroupSizeCoop = 32;+ // log2(32): butterfly stages that stay inside a lane's own 32 elements. The+ // remaining log2(L) stages cross lanes.+ static constexpr int kNumFwhtStagesLocal = 5;++ // Largest supported cooperative fan-out. L must divide the sub-group size,+ // so L <= 32; L = 32 would put one whole sub-group on a single row and is not+ // instantiated because no current model needs D = 1024.+ static constexpr int kMaxLanesPerRow = 16;+ static constexpr int kMaxHadamardDim = kGroupSize * kMaxLanesPerRow;++ static constexpr int ct_log2(int n) { return n <= 1 ? 0 : 1 + ct_log2(n / 2); }++ // A transform size is supported iff it is 32 * L for a power-of-two L that+ // divides the sub-group size and is within the instantiated range.+ static constexpr bool is_supported_hadamard_dim(int64_t dim) {+ return dim >= kGroupSize && dim <= kMaxHadamardDim && dim % kGroupSize == 0 &&+ (dim / kGroupSize) * kGroupSize == dim && ((dim / kGroupSize) & (dim / kGroupSize - 1)) == 0;+ }++ // FP4 (E2M1) magnitude levels: 0, 0.5, 1, 1.5, 2, 3, 4, 6.+ // Thresholds and boundary comparison operators are taken verbatim from the+ // PyTorch reference (auto_round_extension/vllm_ext/fp4_utils.py::cast_to_fp4)+ // so that the kernel is bit-exact with it.+ //+ // Written branchless, as a descending count rather than the reference's+ // early-out if-ladder, because this runs 32 times per work-item on data whose+ // magnitudes are unrelated across neighbouring SIMD lanes. An if-ladder there+ // diverges on essentially every group and the hardware has to execute all+ // eight arms under mask, which is enough to turn this memory-bound kernel+ // ALU-bound: measured 386 -> 397 GB/s at D = 32 and 357 -> 397 GB/s at+ // D = 128 on Arc Pro B60.+ //+ // The descending form is required, not merely convenient. Counting *up* with+ // the complementary predicates (a > 0.25f, a >= 0.75f, ...) agrees with the+ // ladder on every ordered input but maps NaN to 0, whereas the ladder falls+ // through to 7. Counting down with the ladder's own predicates reproduces+ // that: for NaN every comparison is false, giving 7 - 0. Verified exhaustively+ // against the if-ladder over all 2^32 float bit patterns: zero mismatches.+ static inline int e2m1_magnitude_index(float a) {+ return 7 - (static_cast<int>(a <= 5.0f) + static_cast<int>(a < 3.5f) + static_cast<int>(a <= 2.5f) ++ static_cast<int>(a < 1.75f) + static_cast<int>(a <= 1.25f) + static_cast<int>(a < 0.75f) ++ static_cast<int>(a <= 0.25f));+ }++ // Quantize one 32-element group held in registers: E8M0 exponent + the 16+ // packed code bytes as four 32-bit words.+ //+ // Shared by every path that owns a whole group in a single work-item (the+ // D = 32 per-item kernel and each lane of the cooperative kernel at any+ // D > 32), so that all of them are bit-exact against the same PyTorch+ // reference. The operation order is part of the frozen contract: ilogb for the+ // exponent, a per-element ldexp for the scaling, and the branchless threshold+ // count above -- never a direct read of the exponent field, which handles a+ // non-finite amax differently (see xpu_mxfp4_hadamard_xmx.hpp).+ static inline void quant_group32(const float* v, sycl::vec<uint32_t, 4>& packed, uint8_t& e8m0) {+ float amax = 0.0f;+#pragma unroll+ for (int i = 0; i < kGroupSize; ++i) {+ amax = sycl::fmax(amax, sycl::fabs(v[i]));+ }++ e8m0 = 0;+ int exp_shift = 0;+ if (amax > 0.0f) {+ int biased = sycl::ilogb(amax) - 2 + 127;+ biased = biased < 0 ? 0 : (biased > 254 ? 254 : biased);+ e8m0 = static_cast<uint8_t>(biased);+ exp_shift = biased - 127;+ }++ packed = sycl::vec<uint32_t, 4>(0u);+ if (amax > 0.0f) {+#pragma unroll+ for (int i = 0; i < kGroupSize; ++i) {+ const float qv = sycl::ldexp(v[i], -exp_shift);+ const int idx = e2m1_magnitude_index(sycl::fabs(qv));+ // Canonical zero: never emit 0x8 (negative zero). A value that rounds+ // to magnitude 0 may carry either sign depending on FP32 rounding+ // residue and on flush-to-zero, so the sign is dropped.+ const int code = (idx == 0) ? 0 : ((sycl::signbit(qv) ? 8 : 0) | idx);+ // Even element -> low nibble of its byte.+ packed[i >> 3] |= static_cast<uint32_t>(code & 0xF) << ((i & 7) * 4);+ }+ }+ }++ // Fast path: one work-item owns one full 32-element group.+ //+ // The original design gave each *lane* one element, so a 32-lane sub-group+ // handled a single group: 134M work-items each moving 2 bytes, and 16 separate+ // 1-byte stores per group. Byte-granularity stores force read-modify-write on+ // cache lines and the per-work-item overhead dwarfs the actual work, which+ // pinned the kernel near 33 GB/s regardless of how cheap the transform was --+ // replacing the O(D^2) dot product with the butterfly changed nothing.+ //+ // Here a work-item loads its whole group as 16-byte vectors, runs the+ // butterflies entirely in registers (no sub-group shuffles at all), and emits+ // the 16 packed code bytes as a single aligned 16-byte store. Because the+ // activation is contiguous, group g occupies exactly x[g*32 .. g*32+31],+ // codes bytes [g*16 .. g*16+15] and scale byte g.+ //+ // The group is read straight from global memory with two 16-byte vector loads+ // per work-item. An earlier revision staged the work-group's whole slab+ // through SLM first, on the theory that a work-item reading 64 contiguous+ // bytes makes neighbouring work-items stride 64B apart and therefore+ // uncoalesced. Measured on Arc Pro B60 that staging *cost* 2.3x: the transpose+ // read tile[lt * 32 + i] has every lane hitting the same SLM bank for a given+ // i (stride 64B = 16 dwords = the bank count), so it serializes 16-32 ways;+ // padding the slot to 34 elements removes the conflict but the extra SLM+ // round trip plus the barrier still leaves it slower than not staging at all.+ // The direct path is fine because a 64B-per-work-item stride is exactly one+ // cache line per work-item: the loads are already at full line granularity, so+ // there is nothing for a staging buffer to coalesce. Direct + vector loads+ // measures 1.01-1.03x the streaming-copy baseline versus 0.31-0.43x staged.+ //+ // Bit-exactness is unaffected: the butterfly order is identical to the SLM+ // version, to the sub-group version and to fwht_transform_reference.+ //+ // Quant-only mode (``quant_only = true``) is the controlled baseline used to+ // attribute bandwidth: it strips the Hadamard transform (forces ``norm`` to+ // 1 and skips the butterfly stages) while keeping the exact same loads,+ // packing and stores. The raw activation is then quantized directly, so the+ // memory traffic -- and therefore the bandwidth -- is directly comparable to+ // the fused path.+ template <typename T>+ static void fwht_quant_per_item(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,+ uint8_t* out_scale, int64_t total_groups, bool quant_only = false) {+ const int64_t num_wg = (total_groups + kWorkGroupSize - 1) / kWorkGroupSize;+ const size_t global_size = static_cast<size_t>(num_wg) * kWorkGroupSize;+ // 16-byte vector loads: 8 halves per chunk, 4 chunks per 32-element group.+ constexpr int kLoadVecElems = 8;+ constexpr int kLoadVecCount = kGroupSize / kLoadVecElems;++ q->parallel_for(sycl::nd_range<1>(global_size, kWorkGroupSize), [=](sycl::nd_item<1> item) {+ const int64_t gid = static_cast<int64_t>(item.get_global_id(0));+ if (gid >= total_groups) {+ return;+ }++ // hadamard[0] == H[0][0] == 1/sqrt(32); applied before the butterflies so+ // intermediates stay bounded by sqrt(32)*max|x| (see the header comment).+ // Quant-only mode forces norm to 1 (hadamard is *not* dereferenced: the+ // ternary only evaluates the taken branch) and skips the butterflies+ // below, so the raw activation is quantized directly.+ const float norm = quant_only ? 1.0f : hadamard[0];++ // x + gid * 32 is 64-byte aligned for T = half/bfloat16, so each chunk+ // load is an aligned 16-byte access.+ const auto* src = reinterpret_cast<const sycl::vec<T, kLoadVecElems>*>(x + gid * kGroupSize);++ float v[kGroupSize];+#pragma unroll+ for (int c = 0; c < kLoadVecCount; ++c) {+ const sycl::vec<T, kLoadVecElems> chunk = src[c];+#pragma unroll+ for (int i = 0; i < kLoadVecElems; ++i) {+ v[c * kLoadVecElems + i] = static_cast<float>(chunk[i]) * norm;+ }+ }++ if (!quant_only) {+#pragma unroll+ for (int stage = 0; stage < kNumFwhtStages; ++stage) {+ const int h = 1 << stage;+#pragma unroll+ for (int i = 0; i < kGroupSize; ++i) {+ if ((i & h) == 0) {+ const float a = v[i];+ const float b = v[i ^ h];+ v[i] = a + b;+ v[i ^ h] = a - b;+ }+ }+ }+ }++ // Pack 32 codes into 16 bytes, emitted as four 32-bit words. gid*16 is+ // 16-byte aligned, so this is a single aligned vector store.+ sycl::vec<uint32_t, 4> packed;+ uint8_t e8m0;+ quant_group32(v, packed, e8m0);++ auto* dst = reinterpret_cast<sycl::vec<uint32_t, 4>*>(out_codes + gid * (kGroupSize / 2));+ *dst = packed;+ out_scale[gid] = e8m0;+ });+ }++ // Stream-only baseline: the traffic-matched roofline for every fused path.+ //+ // Keeps the fused kernel's item mapping (one work-item = 32 contiguous+ // elements = 16 code bytes + 1 scale byte), its two 16-byte vector loads and+ // its single 16-byte vector store, but replaces the transform *and* the+ // quantization math with the cheapest possible data-dependent function: the+ // low nibble of each element's raw bit pattern, packed with the same+ // even-element-low-nibble rule, and an XOR fold for the scale byte.+ //+ // Why this and not ``dst.copy_(src)``. A streaming copy has a 1:1 read:write+ // ratio; the fused kernel reads 64 B and writes 17 B per group, i.e. W/R =+ // 0.266. Measured on Arc Pro B60 the device's cost per 64 B read is strongly+ // non-linear in W/R (0.144 ns at W/R = 0, 0.187 at 0.25, 0.414 at 1.0), and a+ // hand-written 1:1 kernel reaches only 310 GB/s where ``memcpy`` reaches 407+ // -- so a copy differs from the fused kernel in *both* the traffic ratio and+ // the instruction mix, and its bandwidth is not an attainable target. This+ // baseline differs in neither: it is the same kernel with the arithmetic+ // removed, so ``BW_fused / BW_stream`` is a true utilization figure and stays+ // meaningful when the dtype, the shape or the Hadamard dimension changes.+ //+ // The result is deterministic and has a PyTorch reference+ // (``mxfp4_stream_reference``) precisely so that a test can prove the loads+ // were not optimized away -- a dead-code-eliminated baseline would report an+ // absurd bandwidth and silently invalidate every ratio computed against it.+ template <typename T>+ static void stream_baseline_per_item(sycl::queue* q, const T* x, uint8_t* out_codes, uint8_t* out_scale,+ int64_t total_groups) {+ const int64_t num_wg = (total_groups + kWorkGroupSize - 1) / kWorkGroupSize;+ const size_t global_size = static_cast<size_t>(num_wg) * kWorkGroupSize;+ constexpr int kLoadVecElems = 8;+ constexpr int kLoadVecCount = kGroupSize / kLoadVecElems;++ q->parallel_for(sycl::nd_range<1>(global_size, kWorkGroupSize), [=](sycl::nd_item<1> item) {+ const int64_t gid = static_cast<int64_t>(item.get_global_id(0));+ if (gid >= total_groups) {+ return;+ }++ const auto* src = reinterpret_cast<const sycl::vec<T, kLoadVecElems>*>(x + gid * kGroupSize);++ sycl::vec<uint32_t, 4> packed(0u);+ uint32_t fold = 0u;+#pragma unroll+ for (int c = 0; c < kLoadVecCount; ++c) {+ const sycl::vec<T, kLoadVecElems> chunk = src[c];+ uint32_t word = 0u;+#pragma unroll+ for (int i = 0; i < kLoadVecElems; ++i) {+ const uint32_t bits = static_cast<uint32_t>(sycl::bit_cast<uint16_t>(chunk[i]));+ word |= (bits & 0xFu) << (i * 4);+ }+ packed[c] = word;+ // Every loaded element feeds the stored result, so nothing can be+ // eliminated; the fold also produces the scale byte.+ fold ^= word;+ }++ auto* dst = reinterpret_cast<sycl::vec<uint32_t, 4>*>(out_codes + gid * (kGroupSize / 2));+ *dst = packed;+ out_scale[gid] = static_cast<uint8_t>(fold & 0xFFu);+ });+ }++ // D = 128 FWHT + MXFP4, four cooperating lanes per row.+ //+ // A 128-point transform does not fit the one-work-item-per-row shape that+ // works so well at D = 32: 128 live FP32 values exceed the default per-thread+ // register budget and the kernel spills catastrophically (measured 140 GB/s,+ // a third of the roofline; raising the GRF to 256 recovers only 317 GB/s+ // because occupancy drops).+ //+ // Instead the transform is factored as H_D = H32 (x) H_L, with L = D / 32+ // lanes per row. Lane l of a group of L owns the contiguous chunk+ // [l*32, l*32+32):+ //+ // * stages 0..4 (h = 1, 2, 4, 8, 16) are entirely inside the lane's own 32+ // values -- bit-for-bit the same butterfly as the D = 32 kernel;+ // * stages 5..(5 + log2(L) - 1) (h = 32, 64, ...) become cross-lane+ // butterflies against lane ^1, ^2, ^4, ..., i.e. log2(L) * 32 sub-group+ // shuffles.+ //+ // The 32-element quantization group is exactly one lane's chunk, so absmax,+ // E8M0, encoding, packing and the 16-byte store are all lane-local: no SLM,+ // no barriers, and no cross-lane reduction anywhere in the quantizer. This is+ // the property that makes the fan-out cheap, and it holds for every L: the+ // per-lane register footprint is a constant 32 floats, so unlike a+ // one-work-item-per-row formulation nothing spills as D grows. Only the+ // shuffle count grows, and it grows as log2(L).+ //+ // Measured on Arc Pro B60 at the attention QKV shape ([1, 75600, 40, 128],+ // bf16): 395 GB/s against a 391 GB/s stream-only roofline, versus 140 GB/s+ // for one work-item per row (register spill) and 318 GB/s for two lanes per+ // row. Note that raising the GRF size *hurts* here (276 GB/s), the opposite+ // of the single-work-item variant.+ //+ // Convergence. The cross-lane stages are sub-group collectives, so every lane+ // of the sub-group must reach them. Out-of-range lanes in the tail therefore+ // do *not* return early: they clamp their row index to 0, participate in the+ // shuffles with valid data, and are masked off only at the store. Because+ // rows are L-lane aligned and the sub-group size is a multiple of L, the L+ // lanes of a row always live in the same sub-group.+ //+ // L = 1 instantiates correctly (zero cross-lane stages, degenerating to the+ // D = 32 butterfly) but is not what the dispatcher routes D = 32 to: that+ // case goes to fwht_quant_per_item, which also carries the quant-only and+ // stream-only ablations and must keep the exact item mapping they compare+ // against.+ //+ // Normalization placement. Unlike the D = 32 kernel, which scales by+ // 1/sqrt(D) on load, this kernel scales *after* the butterflies. Scaling on+ // load would leave a multiply feeding directly into the first stage's add,+ // and the Intel GPU compiler fuses that pair into an FMA. The fused form+ // rounds differently from the reference's separate multiply and add, and the+ // resulting one-ULP error propagates through every later stage: measured on+ // Arc Pro B60, 60% of the transform outputs differed at D = 128 and D = 512,+ // which showed up downstream as 17 and 25 wrong MXFP4 codes per 67 M -- only+ // groups whose scaled magnitude landed within an ULP of an E2M1 threshold+ // flipped, which is exactly the kind of rate that small unit-test shapes miss.+ //+ // Neither -ffp-contract=off nor a #pragma clang fp contract(off) suppresses+ // it (both were measured to change nothing: the fusion happens in the GPU+ // backend, below the point those act on), and a bit_cast round trip through+ // uint32_t does not either. Moving the multiply past the butterflies removes+ // the mul/add pair altogether: what remains multiplies into fabs, fmax and+ // the comparison ladder of the quantizer, none of which is an add. That is a+ // structural fix rather than a request the compiler may ignore, and it costs+ // nothing measurable (235 vs 237 GB/s).+ //+ // Note this is *why* D = 64 and D = 256 never failed: their 1/sqrt(D) is+ // exactly 1/8 and 1/16, so the load-time product was exact and fusing it+ // changed nothing. Only the irrational norms exposed the bug.+ //+ // The cost is the intermediate range: values now grow to D*max|x| before+ // being scaled down, rather than staying at sqrt(D)*max|x|. In FP32 that is+ // immaterial here -- FP16 activations cap the intermediate at 512*65504 =+ // 3.4e7 -- but it does mean this path and the D = 32 path are not bit-exact+ // against each other, and the torch reference mirrors the split.+ template <typename T, int kLanes>+ static void fwht_quant_cooperative(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,+ uint8_t* out_scale, int64_t num_rows) {+ static_assert(kLanes >= 1 && kLanes <= kMaxLanesPerRow, "lanes per row out of range");+ static_assert((kLanes & (kLanes - 1)) == 0, "lanes per row must be a power of two");+ static_assert(kSubGroupSizeCoop % kLanes == 0,+ "lanes per row must divide the sub-group size, otherwise a row's lanes can straddle two "+ "sub-groups and the cross-lane butterfly reads the wrong partner");++ constexpr int kDim = kGroupSize * kLanes;+ constexpr int kCrossStages = ct_log2(kLanes);++ const int64_t total_items = num_rows * kLanes;+ const int64_t num_wg = (total_items + kWorkGroupSize - 1) / kWorkGroupSize;+ const size_t global_size = static_cast<size_t>(num_wg) * kWorkGroupSize;+ constexpr int kLoadVecElems = 8;+ constexpr int kLoadVecCount = kGroupSize / kLoadVecElems;++ q->parallel_for(+ sycl::nd_range<1>(global_size, kWorkGroupSize),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(kSubGroupSizeCoop)]] {+ auto sg = item.get_sub_group();+ const int64_t gid = static_cast<int64_t>(item.get_global_id(0));+ const int64_t row = gid / kLanes;+ const int lane = static_cast<int>(gid % kLanes);+ const int sg_lane = static_cast<int>(sg.get_local_id()[0]);++ // Tail lanes stay converged for the sub-group shuffles below by+ // re-reading row 0; only their store is suppressed.+ const bool active = row < num_rows;+ const int64_t safe_row = active ? row : 0;++ // hadamard[0] == H[0][0] == 1/sqrt(D), applied *after* the butterflies+ // -- see the normalization-placement note above the function.+ const float norm = hadamard[0];++ const auto* src =+ reinterpret_cast<const sycl::vec<T, kLoadVecElems>*>(x + safe_row * kDim + lane * kGroupSize);++ float v[kGroupSize];+#pragma unroll+ for (int c = 0; c < kLoadVecCount; ++c) {+ const sycl::vec<T, kLoadVecElems> chunk = src[c];+#pragma unroll+ for (int i = 0; i < kLoadVecElems; ++i) {+ v[c * kLoadVecElems + i] = static_cast<float>(chunk[i]);+ }+ }++ // Stages 0..4: h < 32, both partners live in this lane.+#pragma unroll+ for (int stage = 0; stage < kNumFwhtStagesLocal; ++stage) {+ const int h = 1 << stage;+#pragma unroll+ for (int i = 0; i < kGroupSize; ++i) {+ if ((i & h) == 0) {+ const float a = v[i];+ const float b = v[i ^ h];+ v[i] = a + b;+ v[i ^ h] = a - b;+ }+ }+ }++ // Remaining stages: h = 32, 64, ... The partner element lives in lane+ // ^1, ^2, ^4, ... at the same offset within the chunk. The lane+ // holding the high half of the pair computes the difference, matching+ // the D = 32 butterfly's orientation.+#pragma unroll+ for (int stage = 0; stage < kCrossStages; ++stage) {+ const int lane_h = 1 << stage;+ const int partner = sg_lane ^ lane_h;+ const bool is_low = (lane & lane_h) == 0;+#pragma unroll+ for (int i = 0; i < kGroupSize; ++i) {+ const float other = sycl::select_from_group(sg, v[i], partner);+ v[i] = is_low ? (v[i] + other) : (other - v[i]);+ }+ }++ // Normalization, applied last (see the note above the function). The+ // product feeds fabs/fmax and the comparison ladder inside the+ // quantizer, never an add, so there is nothing here to contract.+#pragma unroll+ for (int i = 0; i < kGroupSize; ++i) {+ v[i] *= norm;+ }++ sycl::vec<uint32_t, 4> packed;+ uint8_t e8m0;+ quant_group32(v, packed, e8m0);++ if (active) {+ // Group index within the flattened [total_groups, 32] view: the+ // row's L groups are contiguous, so this matches the D = 32 layout.+ const int64_t group_id = row * kLanes + lane;+ auto* dst = reinterpret_cast<sycl::vec<uint32_t, 4>*>(out_codes + group_id * (kGroupSize / 2));+ *dst = packed;+ out_scale[group_id] = e8m0;+ }+ });+ }++ // Runtime dim -> compile-time lane count. Every cooperative D (i.e. every+ // supported size above 32) gets its own fully unrolled instantiation; an+ // unsupported one is rejected here as a backstop, the wrapper and the binding+ // having already validated it.+ template <typename T>+ static bool dispatch_cooperative(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,+ uint8_t* out_scale, int64_t num_rows, int64_t hadamard_dim) {+ switch (hadamard_dim) {+ case kGroupSize * 2:+ fwht_quant_cooperative<T, 2>(q, x, hadamard, out_codes, out_scale, num_rows);+ return true;+ case kGroupSize * 4:+ fwht_quant_cooperative<T, 4>(q, x, hadamard, out_codes, out_scale, num_rows);+ return true;+ case kGroupSize * 8:+ fwht_quant_cooperative<T, 8>(q, x, hadamard, out_codes, out_scale, num_rows);+ return true;+ case kGroupSize * 16:+ fwht_quant_cooperative<T, 16>(q, x, hadamard, out_codes, out_scale, num_rows);+ return true;+ default:+ return false;+ }+ }++ // in: x [num_rows, k] (T = sycl::half or bfloat16)+ // hadamard [32, 32] (FP32, row major, normalized)+ // out: codes [num_rows, k / 2] (uint8, two FP4 codes per byte)+ // scale [num_rows, k / 32] (uint8, one E8M0 exponent per group)+ //+ // Path A fallback for a caller-supplied non-Sylvester matrix. Bit-exact+ // against hadamard_transform_reference; deliberately *not* bit-exact against+ // the FWHT path above, because a butterfly network and a 32-term dot product+ // round differently.+ template <typename T>+ static void mxfp4_hadamard_quant_impl(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,+ uint8_t* out_scale, int64_t num_rows, int64_t k) {+ constexpr int groups_per_wg = kWorkGroupSize / kSubGroupSize;+ const int64_t groups_per_row = k / kGroupSize;+ const int64_t total_groups = num_rows * groups_per_row;+ if (total_groups <= 0) {+ return;+ }+ const int64_t num_wg = (total_groups + groups_per_wg - 1) / groups_per_wg;+ const size_t global_size = static_cast<size_t>(num_wg) * kWorkGroupSize;++ q->parallel_for(sycl::nd_range<1>(global_size, kWorkGroupSize),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(kSubGroupSize)]] {+ auto sg = item.get_sub_group();+ const int lane = static_cast<int>(sg.get_local_id()[0]);+ const int64_t group_id = static_cast<int64_t>(item.get_group(0)) * groups_per_wg ++ static_cast<int64_t>(sg.get_group_id()[0]);+ // Tail work-groups: the whole sub-group exits together, so the+ // sub-group collectives below stay converged.+ if (group_id >= total_groups) {+ return;+ }++ const int64_t row = group_id / groups_per_row;+ const int64_t group_in_row = group_id % groups_per_row;+ const int64_t base = row * k + group_in_row * kGroupSize;++ // Path A: generic 32x32 matrix multiply. Lane i owns column i of+ // H, values of x are broadcast one by one inside the sub-group.+ //+ // The Path A accumulation contract (see mxfp4_hadamard.py)+ // requires increasing j, no reassociation and a separate FP32+ // rounding after the multiply and after the add. Contracting+ // into an FMA would change near-threshold elements and break+ // bit-exactness with the reference, so it is disabled here.+ const float xv = static_cast<float>(x[base + lane]);+ float acc = 0.0f;+#pragma unroll+ for (int j = 0; j < kHadamardDim; ++j) {+#if defined(__clang__)+#pragma clang fp contract(off)+#endif+ const float xj = sycl::select_from_group(sg, xv, j);+ acc += xj * hadamard[j * kHadamardDim + lane];+ }++ const float amax = sycl::reduce_over_group(sg, sycl::fabs(acc), sycl::maximum<float>{});++ uint8_t e8m0 = 0;+ int code = 0;+ if (amax > 0.0f) {+ // floor(log2(amax)) is exact through ilogb, including for+ // exact powers of two and subnormal inputs.+ int biased = sycl::ilogb(amax) - 2 + 127;+ biased = biased < 0 ? 0 : (biased > 254 ? 254 : biased);+ e8m0 = static_cast<uint8_t>(biased);+ const float qv = sycl::ldexp(acc, -(biased - 127));+ const int idx = e2m1_magnitude_index(sycl::fabs(qv));+ const int sign = sycl::signbit(qv) ? 1 : 0;+ // Canonical zero: never emit 0x8 (negative zero). A value+ // that rounds to magnitude 0 may carry either sign+ // depending on FP32 accumulation residue and on whether+ // the device flushes subnormals, so the sign is dropped.+ code = (idx == 0) ? 0 : ((sign << 3) | idx);+ }++ // Even lane keeps the low nibble, its odd neighbour the high one.+ const int partner = ((lane & 1) == 0) ? (lane + 1) : lane;+ const int hi_code = sycl::select_from_group(sg, code, partner);+ if ((lane & 1) == 0) {+ const int64_t byte_idx = (base + lane) >> 1;+ out_codes[byte_idx] = static_cast<uint8_t>((code & 0xF) | ((hi_code & 0xF) << 4));+ }+ if (lane == 0) {+ out_scale[row * groups_per_row + group_in_row] = e8m0;+ }+ });+ }++ // Dispatcher.+ //+ // ``hadamard_dim`` selects the transform size: 32 (the GEMM activation case,+ // one work-item per group) or 32 * L for L in {2, 4, 8, 16} -- i.e. 64, 128,+ // 256, 512 -- which fan a row out over L cooperating lanes. Every size+ // quantizes in groups of 32 and shares the output layout, so the two baseline+ // modes below are dimension-agnostic: they only ever see a flat sequence of+ // 32-element groups.+ //+ // ``stream_only`` and ``quant_only`` are mutually exclusive; the caller is+ // expected to have rejected the combination already.+ template <typename T>+ static void mxfp4_hadamard_quant(sycl::queue* q, const T* x, const float* hadamard, uint8_t* out_codes,+ uint8_t* out_scale, int64_t num_rows, int64_t k, bool use_fwht,+ bool quant_only = false, int64_t hadamard_dim = kHadamardDim,+ bool stream_only = false) {+ const int64_t total_groups = num_rows * (k / kGroupSize);+ if (total_groups <= 0) {+ return;+ }++ if (stream_only) {+ // Traffic-matched roofline: same loads, same packing shape, same stores,+ // no transform and no quantization math. hadamard is not read.+ stream_baseline_per_item<T>(q, x, out_codes, out_scale, total_groups);+ return;+ }+ if (quant_only) {+ // Quant-only baseline: strip the Hadamard transform entirely. The+ // per-item layout and memory traffic are identical to the fused FWHT+ // path, so bandwidths are directly comparable; hadamard is not read.+ fwht_quant_per_item<T>(q, x, nullptr, out_codes, out_scale, total_groups, /*quant_only=*/true);+ return;+ }++ if (hadamard_dim > kGroupSize) {+ // D > 32 supports the Sylvester matrix only; the caller enforces this.+ // The dispatch result is checked rather than discarded: on an unsupported+ // dim no kernel is enqueued, and silently returning would leave+ // out_codes / out_scale holding whatever the caller allocated -- no+ // error, just wrong bytes.+ const bool dispatched = dispatch_cooperative<T>(q, x, hadamard, out_codes, out_scale,+ num_rows * (k / hadamard_dim), hadamard_dim);+ if (!dispatched) {+ throw std::invalid_argument(+ "ark::XpuMxfp4Hadamard: hadamard_dim is not a dispatched cooperative transform size");+ }+ return;+ }++ if (use_fwht) {+ fwht_quant_per_item<T>(q, x, hadamard, out_codes, out_scale, total_groups);+ } else {+ mxfp4_hadamard_quant_impl<T>(q, x, hadamard, out_codes, out_scale, num_rows, k);+ }+ }+};++} // namespace ark++#endif // ARK_XPUdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard_xmx.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard_xmx.hppnew file mode 100644index 000000000..a871f48ea--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard_xmx.hpp@@ -0,0 +1,272 @@+//+// Copyright (c) 2026 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//+// XMX (Xe matrix / DPAS) path for the activation fused 32-point Hadamard+// transform + MXFP4 quantization. Opt-in via ``use_xmx``; deliberately a+// *relaxed* numerical contract: the Hadamard matrix is stored in FP16/BF16+// (same dtype as the activation) and the transform runs on XMX DPAS with FP32+// accumulation, which is neither bit-exact with the FWHT path nor with Path A.+// Acceptance is tolerance based, and is enforced by+// test/test_mxfp4_hadamard.py::TestXpuKernelXmx: SQNR >= 15 dB, no code off by+// more than one E2M1 level, and a code-disagreement rate below 0.1%.+//+// Non-finite amax is out of contract and deliberately unspecified. This path+// reads the FP32 exponent field rather than calling ilogb, so amax = +inf gives+// e8m0 = 253 instead of the 254 that the ilogb-based FWHT paths and the PyTorch+// reference saturate to, and NaN fails the ``amax > 0`` test and falls through+// as an all-zero group. The two quantizer ladders also disagree on NaN by+// construction -- the reference initialises the magnitude index to 7 and only+// lowers it on a threshold hit, while this one sums the comparisons, so NaN+// lands on 0 -- which puts byte-parity on non-finite input out of reach without+// changing the bit-exact ladder the other paths rely on. Reaching it takes an+// already-non-finite activation: with fp16/bf16 operands the 32-term FP32+// accumulation peaks around 1.4e11, well inside the finite range.+//+// This is not a bandwidth optimization: it exists for non-Sylvester matrices,+// where the butterfly (and therefore the FWHT path) is unavailable, and it is+// not uniformly faster than FWHT on the shapes measured so far.+//+// The fused kernel computes, for each 32-element group ``g`` (one row of the+// flattened activation ``[total_groups, 32]``):+//+// y[g][i] = sum_j H_T[j][i] * x[g][j] (DPAS, FP32 accumulate)+//+// i.e. ``y = H_T @ x^T`` with H_T the Hadamard matrix stored as T (fp16/bf16).+// Each XMX lane owns one full group (its 32 outputs live entirely in the lane's+// fragment), so absmax / E8M0 / E2M1 / packing / writeback are all lane-local:+// no SLM, no barriers, no cross-lane shuffles. The MMA's real N-tile is 64 (not+// the 32 passed to choose_tiled_mma_tile), so the global group index uses+// ``get<1>(mma.tile_mnk())`` as the per-workgroup stride.+//+// Measured on Arc Pro B60 (m = 262144 groups, BF16, memory-bound): ~335-418 GB/s+// vs ~186 GB/s Path A and ~395 GB/s streaming-copy baseline.++#pragma once++#include <cstdint>++#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)++#include <sycl/sycl.hpp>+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>++// Only the cute layer is needed (XE_DPAS atoms, block-2D copies, tiled-MMA+// helper). The cutlass-facing sycl_tla_dense_gemm.hpp is deliberately NOT+// included: pulling in cutlass headers makes cute::half_t resolve to+// cutlass::half_t and breaks the bf16 reorder (ambiguous conversion).+// cute/tensor.hpp brings in the XE_DPAS traits; do not include+// cute/atom/mma_traits_xe.hpp explicitly (it pulls in cutlass::half_t).+#include "cute/tensor.hpp"++namespace ark {+namespace xmx_hadamard_detail {++using namespace cute;++// The DPAS atom is pinned to half input: bf16 values (8-bit mantissa) convert+// losslessly to half (10-bit mantissa) in range, so one atom serves both+// activation dtypes. Verified against the fp32 reference for both fp16 and bf16+// on Arc Pro B60 by+// test/test_mxfp4_hadamard.py::TestXpuKernelXmx::test_xmx_matches_reference_within_tolerance.+template <typename TA, typename TB, typename TC>+auto choose_mma_op() {+ return XE_DPAS_TT<8, float, cute::half_t>{};+}++template <int TileM, int TileN, class SGLayout, class ATensor, class BTensor, class CTensor>+auto choose_tiled_mma_tile(ATensor const& A, BTensor const& B, CTensor const&) {+ using TA = typename ATensor::element_type;+ using TB = typename BTensor::element_type;+ using TC = typename CTensor::element_type;++ auto op = choose_mma_op<TA, TB, TC>();++ constexpr bool byte = (cute::max(sizeof_bits_v<TA>, sizeof_bits_v<TB>) <= 8);+ constexpr bool a_t = is_constant_v<1, decltype(stride<0>(A))>;+ constexpr bool b_n = is_constant_v<1, decltype(stride<0>(B))>;+ constexpr bool use_1x_dpas_per_k = a_t || (byte && b_n);++ using _K = conditional_t<use_1x_dpas_per_k, C<op.K>, C<op.K * 2>>;+ using WGTile = Shape<Int<TileM>, Int<TileN>, _K>;+ using MMA = typename TiledMMAHelper<MMA_Atom<decltype(op)>, Layout<WGTile>, SGLayout>::TiledMMA;++ return MMA{};+}++// A = H_T [32, 32] (row-major, T = fp16/bf16), B = activation [m, 32] (row-major).+// Each workgroup handles one n-tile of tile_n = 64 groups; each lane owns one+// full group (a 32-row column of the C tile), so the whole quant is lane-local.+template <int TileM, int TileN, class ATensor, class BTensor, class TiledMMA>+void fused_core_xmx(ATensor const& A, BTensor const& B, uint8_t* out_codes, uint8_t* out_scale, int64_t m,+ TiledMMA const& mma) {+ auto item = sycl::ext::oneapi::this_work_item::get_nd_item<2>();+ auto wg_m = int(item.get_group(1));+ auto wg_n = int(item.get_group(0));+ auto local_id = int(item.get_local_id(0));++ auto C = make_tensor(make_gmem_ptr(static_cast<float*>(nullptr)), make_shape(m, Int<TileN>{}),+ make_stride(Int<TileN>{}, Int<1>{}));++ Tensor cA = make_identity_tensor(A.shape());+ Tensor cB = make_identity_tensor(B.shape());+ Tensor cC = make_identity_tensor(C.shape());++ auto wg_tile = mma.tile_mnk();+ auto wg_coord = make_coord(wg_m, wg_n, 0);++ Tensor gA = local_tile(cA, select<0, 2>(wg_tile), make_coord(wg_m, _));+ Tensor gB = local_tile(cB, select<1, 2>(wg_tile), make_coord(wg_n, _));+ Tensor gC = local_tile(cC, wg_tile, wg_coord, Step<_1, _1, X>{});++ auto copy_a = make_block_2d_copy_A(mma, A);+ auto copy_b = make_block_2d_copy_B(mma, B);++ auto thr_mma = mma.get_slice(local_id);+ auto thr_copy_a = copy_a.get_slice(local_id);+ auto thr_copy_b = copy_b.get_slice(local_id);++ auto tCrA = thr_mma.partition_sg_fragment_A(gA(_, _, 0));+ auto tCrB = thr_mma.partition_sg_fragment_B(gB(_, _, 0));+ auto tArA = thr_copy_a.partition_sg_fragment_D(gA(_, _, 0));+ auto tBrB = thr_copy_b.partition_sg_fragment_D(gB(_, _, 0));++ Tensor tAgA = thr_copy_a.partition_S(gA);+ Tensor tBgB = thr_copy_b.partition_S(gB);++ Tensor tCrC = partition_fragment_C(mma, select<0, 1>(wg_tile));+ Tensor tCgC = thr_mma.partition_C(gC);++ clear(tCrC);+ copy(copy_a, tAgA(_, _, _, 0), tArA);+ copy(copy_b, tBgB(_, _, _, 0), tBrB);+ reorder(tArA, tCrA);+ reorder(tBrB, tCrB);+ gemm(mma, tCrA, tCrB, tCrC);++ // ---- lane-local quant (1 lane = 1 full group) ----+ const int col = get<1>(tCgC(0)); // group within the workgroup tile+ const int tile_n = get<1>(mma.tile_mnk()); // 64, NOT the template TileN+ const int64_t g = (int64_t)wg_n * tile_n + col;++ // absmax over the lane's 32 outputs (4 x float8 SIMD).+ float amax = 0.0f;+ for (int c = 0; c < 4; ++c) {+ sycl::vec<float, 8> v;+ for (int t = 0; t < 8; ++t) v[t] = tCrC(c * 8 + t);+ const sycl::vec<float, 8> av = sycl::fabs(v);+ amax = sycl::fmax(amax, sycl::fmax(sycl::fmax(av[0], av[1]), sycl::fmax(av[2], av[3])));+ amax = sycl::fmax(amax, sycl::fmax(sycl::fmax(av[4], av[5]), sycl::fmax(av[6], av[7])));+ }++ uint8_t e8m0 = 0;+ float inv = 0.0f;+ if (amax > 0.0f) {+ // e8m0 = biased_exponent - 2, read straight from the fp32 bits (no ilogb).+ const int b = int(sycl::bit_cast<uint32_t>(amax) >> 23) - 2;+ const int bcl = b < 0 ? 0 : (b > 254 ? 254 : b);+ e8m0 = static_cast<uint8_t>(bcl);+ inv = sycl::bit_cast<float>((254u - static_cast<uint32_t>(bcl)) << 23); // 2^(127-b)+ }++ uint32_t packed[4] = {0u, 0u, 0u, 0u};+ if (amax > 0.0f) {+ for (int c = 0; c < 4; ++c) {+ sycl::vec<float, 8> v;+ for (int t = 0; t < 8; ++t) v[t] = tCrC(c * 8 + t);+ const sycl::vec<float, 8> aq = sycl::fabs(v) * inv;+ // The same piecewise thresholds and alternating <= / < operators as+ // XpuMxfp4Hadamard::e2m1_magnitude_index, but counted up from 0 instead of+ // down from 7. On this target vec comparisons return -1 (true) / 0+ // (false), so the negated sum is the magnitude index.+ const sycl::vec<int, 8> idx =+ -(aq > 0.25f) - (aq >= 0.75f) - (aq > 1.25f) - (aq >= 1.75f) -+ (aq > 2.5f) - (aq >= 3.5f) - (aq > 5.0f); // 0..7+ const sycl::vec<uint32_t, 8> u = sycl::bit_cast<sycl::vec<uint32_t, 8>>(v);+ const sycl::vec<uint32_t, 8> iu = idx.template convert<uint32_t>();+ const sycl::vec<uint32_t, 8> iszero =+ sycl::vec<int, 8>(-(idx == sycl::vec<int, 8>(0))).template convert<uint32_t>();+ // Canonical zero: magnitude index 0 always encodes 0x0, never 0x8.+ const sycl::vec<uint32_t, 8> code = (iu | (((u >> 31) & 1u) << 3)) & ~(0u - iszero);+ for (int t = 0; t < 8; ++t) {+ const int row = c * 8 + t;+ packed[row >> 3] |= (code[t] & 0xF) << ((row & 7) * 4);+ }+ }+ }+ if (g < m) {+ auto* dst4 = reinterpret_cast<sycl::vec<uint32_t, 4>*>(out_codes + g * (32 / 2));+ *dst4 = sycl::vec<uint32_t, 4>{packed[0], packed[1], packed[2], packed[3]};+ out_scale[g] = e8m0;+ }+}++template <int TileM, int TileN, class SGLayout, class Element>+void fused_launch_xmx(sycl::queue* q, int64_t m, const Element* h_ptr, const Element* a_ptr, uint8_t* out_codes,+ uint8_t* out_scale) {+ // A = H_T [32,32] (row-major), B = activation [m,32] (row-major).+ auto A = make_tensor(make_gmem_ptr(const_cast<Element*>(h_ptr)), make_shape(32, 32), make_stride(32, _1{}));+ auto B = make_tensor(make_gmem_ptr(const_cast<Element*>(a_ptr)), make_shape(m, 32), make_stride(32, _1{}));+ auto C = make_tensor(make_gmem_ptr(static_cast<float*>(nullptr)), make_shape(TileM, TileN), make_stride(TileN, _1{}));+ auto mma = choose_tiled_mma_tile<TileM, TileN, SGLayout>(A, B, C);++ sycl::range<2> local = {size(mma), 1};+ sycl::range<2> global = {local[0] * ceil_div(shape<0>(B), get<1>(mma.tile_mnk())),+ local[1] * ceil_div(shape<0>(A), get<0>(mma.tile_mnk()))};++ namespace syclex = sycl::ext::oneapi::experimental;+ namespace intelex = sycl::ext::intel::experimental;+ syclex::properties kernel_props{syclex::sub_group_size<16>, intelex::grf_size<256>};++ q->parallel_for(sycl::nd_range<2>(global, local), kernel_props,+ [=](sycl::nd_item<2>) { fused_core_xmx<TileM, TileN>(A, B, out_codes, out_scale, m, mma); });+}++// Convert the FP32 Hadamard matrix to the activation dtype AND transpose it.+// The fused GEMM is ``C = A @ B^T`` with A = the Hadamard matrix, so ``C[r][c]``+// = ``sum_k A[r][k] * x[c][k]``. We need ``y[group c][element r] = sum_k x[c][k]+// * H[k][r]``, which forces ``A[r][k] = H[k][r]`` -- i.e. A must be H^T. The+// transpose is applied here (once per call, 1024 elements, negligible) so the+// kernel entry point simply takes the logical (row-major) Hadamard matrix and+// always produces ``y = x @ H`` regardless of symmetry. The normalized Sylvester+// matrix is symmetric so a missing transpose is silently masked; a non-symmetric+// custom matrix is what exposes a missing transpose.+template <typename T>+void convert_hadamard_to_dtype(sycl::queue* q, const float* h_fp32, T* h_t) {+ q->parallel_for(32 * 32, [=](sycl::id<1> i) {+ const int r = int(i) / 32; // row of H+ const int c = int(i) % 32; // column of H+ h_t[c * 32 + r] = static_cast<T>(h_fp32[i]); // write H^T+ });+}++// Public entry point. x is the activation [num_rows, k] flattened to+// [total_groups, 32]; h is the Hadamard matrix [32,32] in the *same* dtype as+// the activation (T), already normalized by 1/sqrt(32), row-major. The kernel+// computes y = x @ h (the transpose for the C = A @ B^T GEMM convention is the+// caller's responsibility -- see convert_hadamard_to_dtype).+template <typename T>+void mxfp4_hadamard_quant_xmx(sycl::queue* q, const T* x, const T* h, uint8_t* out_codes, uint8_t* out_scale,+ int64_t total_groups) {+ if (total_groups <= 0) {+ return;+ }+ using SmallTileSG = Layout<Shape<_1, _4, _1>, Stride<_0, _1, _0>>;+ fused_launch_xmx<32, 32, SmallTileSG, T>(q, total_groups, h, x, out_codes, out_scale);+}++} // namespace xmx_hadamard_detail+} // namespace ark++#endif // ARK_XPU && ARK_SYCL_TLAdiff --git a/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.py b/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.pynew file mode 100644index 000000000..04af7f551--- /dev/null+++ b/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.py@@ -0,0 +1,461 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-++# Copyright (C) 2026 Intel Corporation+# SPDX-License-Identifier: Apache-2.0++"""Bandwidth benchmark for the fused Hadamard + MXFP4 quantization XPU kernel.++ bytes = M*K*sizeof(input) + M*K/2 + M*K/32+ BW = bytes / latency++Choosing the denominator+------------------------++A bandwidth number only means something relative to what the device could+actually deliver *for this kernel*, and picking that ceiling is the whole+difficulty. Three candidates, in increasing order of usefulness:++``copy_same_shape`` (``dst.copy_(src)``) was the original acceptance baseline+and is now reported for context only. It is not an attainable target, for two+independent reasons. First, the traffic ratio is wrong: a copy reads and writes+in a 1:1 ratio, while the fused kernel reads 64 B and writes 17 B per group+(W/R = 0.266), and the device's cost per byte read is strongly non-linear in+that ratio -- measured on Arc Pro B60, 0.144 ns per 64 B read at W/R = 0, 0.187+at W/R = 0.25 and 0.414 at W/R = 1.0. Second, the instruction mix is wrong:+``memcpy`` reaches 407 GB/s where a hand-written 1:1 SYCL kernel reaches only+310, so even at a matched ratio the two are not comparable. The copy baseline is+in fact the faster of the two, so ``f/cp`` reads systematically below the+stream-only figure on every configuration measured; that is why it is reported+for context but never gated on.++``quant_only`` strips the Hadamard transform but keeps the quantization, with+byte-identical traffic. It is a clean *ablation* and answers "what does the+transform cost?" -- but it is not a roofline, because it is itself subject to+whatever limits the fused kernel.++``stream_only`` is the baseline this benchmark gates on. It keeps the fused+kernel's loads, its packing shape and its stores, and removes only the+transform and the quantization arithmetic. Same bytes, same access pattern,+same item mapping, no math -- so it *is* the traffic-matched roofline, and+``BW_fused / BW_stream`` is a true utilization figure. It also re-derives itself+automatically when the dtype, the shape or the Hadamard dimension changes,+which none of the alternatives do.++Cache residency+---------------++At small ``M*K`` the working set fits in device cache and absolute bandwidths+stop reflecting DRAM at all -- on Arc Pro B60 the 8 MB configurations measure+over 1 TB/s, which no memory controller on this part can deliver.++Residency is detected from ``stream_only`` itself, not from the copy. The two+have different footprints for the same ``[M, K]`` (the copy touches+``4 B/element``, the kernel ``2.53 B/element``), so the kernel stays partly+resident at sizes where the copy no longer is, and a copy-based test misses it.+Any configuration whose stream baseline beats a sustained DRAM copy by more+than ``CACHE_TOLERANCE`` is flagged.++Flagged configurations are reported but excluded from the gate, and the reason+is worth stating precisely, because it is not merely "the numbers are big".+Once the data is resident the roofline rises to cache bandwidth, and the+transform and quantization arithmetic no longer fits underneath it -- the+kernel stops being memory bound and becomes math bound. Measured: at+``[2048, 2048]`` bf16 the stream baseline reaches ~1050 GB/s while the fused+kernel reaches ~420, an ``f/s`` of 0.40, whereas the same kernel sits at+0.98-1.01 once the working set spills to DRAM. Both numbers are real; they+answer different questions, and only the DRAM-bound one answers "does this+kernel saturate memory?", which is what the gate is for.+"""++from __future__ import annotations++import argparse+import statistics+import sys+import time+from pathlib import Path++import torch++# Import the *installed* auto_round_kernel: the benchmark must exercise the+# compiled XPU extension, and the in-tree source directory has no .so beside it.+# Only fall back to the source tree if the package is not installed at all.+try:+ import auto_round_kernel # noqa: F401+except ImportError: # pragma: no cover - developer convenience+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from auto_round_kernel.mxfp4_hadamard import ( # noqa: E402+ GROUP_SIZE,+ HADAMARD_DIM,+ HADAMARD_DIM_128,+ SUPPORTED_HADAMARD_DIMS,+ get_hadamard_matrix,+ mxfp4_hadamard_quant,+ mxfp4_hadamard_quant_reference,+ mxfp4_quant_reference,+ mxfp4_stream_reference,+)++DTYPES = {"fp16": torch.float16, "bf16": torch.bfloat16}++# Typical prefill shapes: token counts 2K/4K/8K/16K (M=2048 is the smallest+# DRAM-bound prefill config; 1024 is too small to be representative).+DEFAULT_M = (2048, 4096, 8192, 16384)+DEFAULT_K = (2048, 4096, 8192)++# Primary gate: BW(fused) / BW(stream-only). The stream-only baseline is the+# traffic-matched roofline (same loads/stores, no math), so this is the+# kernel's utilization of what the device can actually deliver for this access+# pattern. Measured on Arc Pro B60: 1.011-1.012 on the WAN shapes and a mean of+# 0.998-0.999 over the DRAM-bound prefill sweep, for both D = 32 and D = 128.+TARGET_STREAM_RATIO = 0.95++# Secondary gate: BW(HMT+quant) / BW(quant-only) -- the Hadamard ablation.+# Quant-only moves byte-identical traffic and only drops the transform, so this+# isolates the transform's cost: it runs in-register and is expected to be+# hidden behind memory traffic, i.e. near 1.0.+TARGET_QUANT_RATIO = 0.95++# [# CRI-WAN-HMT] WAN per-op activation shapes [M, K] = the input activation of+# each listed GEMM, i.e. the tensor the HMT+quant kernel processes. Confirmed+# scope: bf16 + FWHT + the full (global) M, since bandwidth is M-invariant+# while the kernel is DRAM-bound. ``--wan`` benchmarks exactly these pairs+# (text projection / text-encoder FFN included per the colleague's update).+WAN_SHAPES = [(75600, 5120), (75600, 13824), (512, 5120), (512, 4096)]+WAN_DTYPE = "bf16"++# A configuration is treated as cache-resident, and therefore excluded from the+# DRAM-bandwidth gate, once its stream-only baseline exceeds the sustained DRAM+# copy by this factor. The margin absorbs run-to-run noise and the fact that a+# partially resident working set still gets some cache benefit.+CACHE_TOLERANCE = 1.15++# Buffer size for the sustained DRAM copy baseline. Far larger than any cache on+# a current Intel discrete GPU, so the copy has to reach memory.+DRAM_PROBE_BYTES = 1 << 29 # 512 MiB per buffer+++def is_xpu_available() -> bool:+ return hasattr(torch, "xpu") and torch.xpu.is_available()+++def bench(fn, warmup: int, iters: int, reps: int = 5) -> float:+ """Return the median latency of ``fn`` in milliseconds.++ ``torch.xpu.synchronize()`` is called on both boundaries of each rep, so a+ measured window contains exactly ``iters`` completed kernel executions.++ The median over ``reps`` windows, rather than a single mean, is what makes+ the reported ratios stable: the device is often shared, and a single+ interfering process inflates one window by several percent. A mean absorbs+ that spike into the result; a median discards it as long as most windows+ are clean.+ """+ for _ in range(warmup):+ out = fn()+ del out+ samples = []+ for _ in range(reps):+ torch.xpu.synchronize()+ start = time.perf_counter()+ for _ in range(iters):+ out = fn()+ del out+ torch.xpu.synchronize()+ samples.append((time.perf_counter() - start) * 1000.0 / float(iters))+ return statistics.median(samples)+++def fused_bytes(m: int, k: int, dtype: torch.dtype) -> int:+ """Bytes moved by the fused kernel: read activation, write codes + scales."""+ itemsize = torch.empty((), dtype=dtype).element_size()+ return m * k * itemsize + m * k // 2 + m * k // GROUP_SIZE+++def to_gbps(nbytes: int, latency_ms: float) -> float:+ return nbytes / (latency_ms * 1.0e-3) / 1.0e9+++def measure_copy_same_shape(m: int, k: int, dtype: torch.dtype, warmup: int, iters: int) -> tuple[float, int]:+ src = torch.randn((m, k), dtype=dtype, device="xpu")+ dst = torch.empty_like(src)+ latency = bench(lambda: dst.copy_(src), warmup, iters)+ return latency, 2 * src.numel() * src.element_size()+++def measure_sustained_dram_copy(dtype: torch.dtype, warmup: int, iters: int) -> float:+ """Copy bandwidth on a buffer too large to cache, in GB/s.++ This is the highest bandwidth the device can actually sustain from memory,+ and therefore the ceiling any DRAM-bound kernel is measured against. It is+ used only to decide which per-configuration baselines are cache-resident.+ """+ itemsize = torch.empty((), dtype=dtype).element_size()+ numel = DRAM_PROBE_BYTES // itemsize+ src = torch.randn(numel, dtype=dtype, device="xpu")+ dst = torch.empty_like(src)+ latency = bench(lambda: dst.copy_(src), warmup, iters)+ gbps = to_gbps(2 * src.numel() * itemsize, latency)+ # Release both buffers (~2x DRAM_PROBE_BYTES) before the caller allocates+ # again. Rebinding is equivalent to ``del`` here but does not trip ruff's+ # F821 for a name captured by the lambda above.+ src = dst = None+ torch.xpu.empty_cache()+ return gbps+++def _dequantize(codes: torch.Tensor, scale: torch.Tensor, k: int) -> torch.Tensor:+ """Unpack (codes, e8m0) back to FP32, for the tolerance check."""+ levels = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32, device=codes.device)+ flat = codes.reshape(-1, k // 2).to(torch.int32)+ low = flat & 0x0F+ high = (flat >> 4) & 0x0F+ nibbles = torch.stack((low, high), dim=-1).reshape(-1, k)+ values = levels[nibbles & 0x07] * torch.where((nibbles & 0x08) != 0, -1.0, 1.0)+ exp = scale.reshape(-1, k // GROUP_SIZE).to(torch.int32) - 127+ return torch.ldexp(values.reshape(-1, GROUP_SIZE), exp.reshape(-1, 1)).reshape(-1, k)+++def verify_once(+ x: torch.Tensor,+ hadamard: torch.Tensor,+ rows: int,+ *,+ use_xmx: bool = False,+ mode: str = "fused",+) -> bool:+ """Spot-check the first ``rows`` rows against the CPU reference.++ A benchmark that measures an incorrect kernel is worthless, so every+ configuration is validated before it is timed. Only a slice is checked+ because the reference is a slow elementwise implementation.++ ``mode`` selects which contract is checked:++ * ``fused`` -- the default FWHT/Path A path (bit-exact) or, with+ ``use_xmx``, the relaxed XMX path (SQNR >= 15 dB).+ * ``quant`` -- the quant-only ablation, bit-exact against+ :func:`mxfp4_quant_reference`.+ * ``stream`` -- the stream-only roofline baseline, bit-exact against+ :func:`mxfp4_stream_reference`. Verifying this one is not optional: if+ the compiler eliminated its loads, the baseline would report an+ unreachable bandwidth and every ratio computed against it would be wrong.+ """+ sub = x[:rows].contiguous()+ if mode == "quant":+ codes, scale = mxfp4_hadamard_quant(sub, _quant_only=True)+ ref_codes, ref_scale = mxfp4_quant_reference(sub.cpu())+ return torch.equal(codes.cpu(), ref_codes) and torch.equal(scale.cpu(), ref_scale)+ if mode == "stream":+ codes, scale = mxfp4_hadamard_quant(sub, _stream_only=True)+ ref_codes, ref_scale = mxfp4_stream_reference(sub.cpu())+ return torch.equal(codes.cpu(), ref_codes) and torch.equal(scale.cpu(), ref_scale)++ codes, scale = mxfp4_hadamard_quant(sub, hadamard, _force_xmx=use_xmx or None)+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(sub.cpu(), hadamard.cpu())+ if not use_xmx:+ return torch.equal(codes.cpu(), ref_codes) and torch.equal(scale.cpu(), ref_scale)+ k = sub.shape[-1]+ deq = _dequantize(codes.cpu(), scale.cpu(), k).double()+ ref = _dequantize(ref_codes, ref_scale, k).double()+ err = deq - ref+ sqnr = float(10.0 * torch.log10((ref * ref).sum() / (err * err).sum().clamp_min(1e-30)))+ return sqnr >= 15.0+++def run_case(m: int, k: int, dtype: torch.dtype, args: argparse.Namespace, dram_gbps: float) -> dict:+ torch.manual_seed(20260611)+ x = torch.randn((m, k), dtype=dtype, device="xpu")+ hadamard = get_hadamard_matrix(args.hadamard_dim, x.device)++ verify_rows = min(args.verify_rows, m)+ if args.no_verify:+ correct = correct_qo = correct_so = None+ else:+ correct = verify_once(x, hadamard, verify_rows, use_xmx=args.xmx)+ correct_qo = verify_once(x, hadamard, verify_rows, mode="quant")+ correct_so = verify_once(x, hadamard, verify_rows, mode="stream")++ # All three modes move exactly the same bytes (fused_bytes), so their+ # bandwidths are directly comparable and the ratios below are meaningful.+ latency = bench(lambda: mxfp4_hadamard_quant(x, hadamard, _force_xmx=args.xmx or None), args.warmup, args.iters)+ latency_qo = bench(lambda: mxfp4_hadamard_quant(x, _quant_only=True), args.warmup, args.iters)+ latency_so = bench(lambda: mxfp4_hadamard_quant(x, _stream_only=True), args.warmup, args.iters)+ nbytes = fused_bytes(m, k, dtype)+ bw_fused = to_gbps(nbytes, latency)+ bw_qo = to_gbps(nbytes, latency_qo)+ bw_so = to_gbps(nbytes, latency_so)++ copy_latency, copy_bytes = measure_copy_same_shape(m, k, dtype, args.warmup, args.iters)+ bw_copy = to_gbps(copy_bytes, copy_latency)++ def _ok(v: bool | None) -> str:+ return "" if v is None else ("pass" if v else "FAIL")++ return {+ "M": m,+ "K": k,+ "dtype": str(dtype).replace("torch.", ""),+ "correct": _ok(correct),+ "correct_qo": _ok(correct_qo),+ "correct_so": _ok(correct_so),+ "bytes": nbytes,+ "latency_ms": latency,+ "latency_qo_ms": latency_qo,+ "latency_so_ms": latency_so,+ "BW_fused_GBps": bw_fused,+ "BW_quant_GBps": bw_qo,+ "BW_stream_GBps": bw_so,+ # Primary metric: utilization of the traffic-matched roofline.+ "ratio_fused_stream": bw_fused / bw_so if bw_so > 0 else float("nan"),+ # Secondary metric: cost of the Hadamard transform alone.+ "ratio_fused_quant": bw_fused / bw_qo if bw_qo > 0 else float("nan"),+ # Reported for context only; a copy is neither traffic-matched nor+ # instruction-matched to this kernel, so it is not gated on.+ "BW_copy_GBps": bw_copy,+ "ratio_copy": bw_fused / bw_copy if bw_copy > 0 else float("nan"),+ # Detected on the stream baseline, which is both the gate's denominator+ # and a closer proxy for the fused kernel's footprint than the copy is.+ "cached": bw_so > dram_gbps * CACHE_TOLERANCE,+ }+++def format_table(rows: list[dict]) -> str:+ header = (+ f"{'M':>7} {'K':>7} {'dtype':>5} {'ok':>4} {'okQ':>4} {'okS':>4} "+ f"{'BW_fused':>9} {'BW_stream':>9} {'f/s':>6} {'BW_quant':>9} {'f/q':>6} "+ f"{'BW_copy':>9} {'f/cp':>6} {'note':>7}"+ )+ lines = [header, "-" * len(header)]+ for r in rows:+ lines.append(+ f"{r['M']:>7} {r['K']:>7} {r['dtype']:>5} {r['correct']:>4} {r['correct_qo']:>4} "+ f"{r['correct_so']:>4} {r['BW_fused_GBps']:>9.1f} {r['BW_stream_GBps']:>9.1f} "+ f"{r['ratio_fused_stream']:>6.3f} {r['BW_quant_GBps']:>9.1f} {r['ratio_fused_quant']:>6.3f} "+ f"{r['BW_copy_GBps']:>9.1f} {r['ratio_copy']:>6.3f} "+ f"{'cached' if r['cached'] else '':>7}"+ )+ return "\n".join(lines)+++def parse_args() -> argparse.Namespace:+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ p.add_argument(+ "--wan",+ action="store_true",+ help="benchmark the [# CRI-WAN-HMT] activation shapes (bf16, full M), overriding --m/--k defaults",+ )+ p.add_argument("--m", type=int, nargs="+", default=None, help="M values (default prefill set)")+ p.add_argument("--k", type=int, nargs="+", default=None, help="K values (default prefill set)")+ p.add_argument("--dtype", nargs="+", choices=sorted(DTYPES), default=None, help="default: fp16 bf16")+ p.add_argument("--warmup", type=int, default=20)+ p.add_argument("--iters", type=int, default=100)+ p.add_argument("--verify-rows", type=int, default=64, help="rows spot-checked against the CPU reference")+ p.add_argument("--no-verify", action="store_true", help="skip the correctness spot check")+ p.add_argument(+ "--hadamard-dim",+ type=int,+ default=HADAMARD_DIM,+ choices=SUPPORTED_HADAMARD_DIMS,+ help="transform size: 32 (GEMM activations, one work-item per group) or "+ "32*L for L in {2,4,8,16} (cooperative L-lane FWHT; 128 is the attention head dim)",+ )+ p.add_argument("--target-stream-ratio", type=float, default=TARGET_STREAM_RATIO, help="min BW_fused/BW_stream")+ p.add_argument("--target-quant-ratio", type=float, default=TARGET_QUANT_RATIO, help="min BW_fused/BW_quant-only")+ p.add_argument("--xmx", action="store_true", help="force the XMX path (auto-routed otherwise)")+ return p.parse_args()+++def main() -> int:+ args = parse_args()+ if not is_xpu_available():+ print("XPU is not available; nothing to benchmark.")+ return 1+ if args.xmx and args.hadamard_dim != HADAMARD_DIM:+ print(f"--xmx is not supported for hadamard_dim {args.hadamard_dim}.")+ return 1++ if args.wan:+ # WAN deliverable: exact per-op (M, K) pairs, bf16, full global M.+ dtypes = [WAN_DTYPE] if not args.dtype else args.dtype+ combos = list(WAN_SHAPES)+ else:+ dtypes = args.dtype or list(DTYPES)+ combos = [(m, k) for m in (args.m or list(DEFAULT_M)) for k in (args.k or list(DEFAULT_K))]++ print(f"device: {torch.xpu.get_device_name(0)}")+ print(f"warmup={args.warmup} iters={args.iters} (median of 5 windows)")+ print(f"hadamard_dim={args.hadamard_dim} path={'XMX (forced)' if args.xmx else 'default (FWHT/Path A)'}")+ print(f"bytes = M*K*sizeof(input) + M*K/{2} + M*K/{GROUP_SIZE}")+ print(f"primary gate: f/s = BW_fused / BW_stream-only >= {args.target_stream_ratio}")+ print(f"secondary gate: f/q = BW_fused / BW_quant-only >= {args.target_quant_ratio}")+ print("f/cp (vs dst.copy_) is reported for context only and is NOT gated on.")++ dram_gbps = {name: measure_sustained_dram_copy(DTYPES[name], args.warmup, args.iters) for name in dtypes}+ for name, gbps in dram_gbps.items():+ print(f"sustained DRAM copy ({name}): {gbps:.1f} GB/s")+ print()++ rows: list[dict] = []+ for name in dtypes:+ dtype = DTYPES[name]+ for m, k in combos:+ if k % args.hadamard_dim != 0:+ print(f"skipping K={k}: not a multiple of {args.hadamard_dim}")+ continue+ rows.append(run_case(m, k, dtype, args, dram_gbps[name]))+ torch.xpu.empty_cache()++ if not rows:+ print("No configuration was benchmarked.")+ return 1++ print(format_table(rows))++ failed_correctness = [r for r in rows if "FAIL" in (r["correct"], r["correct_qo"], r["correct_so"])]+ if failed_correctness:+ print(f"\nCORRECTNESS FAILED for {len(failed_correctness)} configuration(s); timings are meaningless.")+ return 1++ # Cache-resident configurations are reported but not gated: with the data in+ # cache the roofline rises and the kernel becomes math bound rather than+ # memory bound, so a DRAM-derived target does not apply to them.+ gated = [r for r in rows if not r["cached"]]+ cached = len(rows) - len(gated)+ if cached:+ print(+ f"\n{cached} of {len(rows)} configuration(s) marked 'cached' and excluded from the gate: "+ "with the working set resident the roofline is cache bandwidth, not DRAM, and the kernel "+ "becomes math bound (see the module docstring)."+ )+ if not gated:+ print("No DRAM-bound configuration was measured; increase M/K.")+ return 1++ ok = True+ for key, target, label in (+ ("ratio_fused_stream", args.target_stream_ratio, "fused/stream-only"),+ ("ratio_fused_quant", args.target_quant_ratio, "fused/quant-only"),+ ):+ values = [r[key] for r in gated]+ below = [r for r in gated if r[key] < target]+ print(f"\nmean {label} ratio over DRAM-bound configurations = {sum(values) / len(values):.3f}")+ print(f"min {label} ratio over DRAM-bound configurations = {min(values):.3f} (target {target})")+ if below:+ worst = min(below, key=lambda r: r[key])+ print(+ f"FAIL: {len(below)} of {len(gated)} DRAM-bound configuration(s) below the {label} target "+ f"(worst: M={worst['M']} K={worst['K']} {worst['dtype']} at {worst[key]:.3f})."+ )+ ok = False++ if not ok:+ return 1+ print("\nPASS: all configurations meet the bandwidth targets.")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/auto_round_extension/ark/test/test_mxfp4_hadamard.py b/auto_round_extension/ark/test/test_mxfp4_hadamard.pynew file mode 100644index 000000000..218fadcb7--- /dev/null+++ b/auto_round_extension/ark/test/test_mxfp4_hadamard.py@@ -0,0 +1,1126 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-+#+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Correctness tests for the fused HMT + MXFP4 activation quantization kernel.++The suite is organised by what each class proves:++* ``TestReferenceContract`` -- the frozen quantizer contract (FP4 packing,+ E8M0 scales, zero groups, input validation). CPU only.+* ``TestXpuKernel`` -- the basic ``D = 32`` contract: random input, zero+ groups, signs, extremes, nibble order.+* ``TestXpuKernelLayouts`` -- input layouts, work-group tails, row counts.+* ``TestXpuKernelNumerics`` -- dtype dispatch, exponent boundaries and custom+ Hadamard matrices.+* ``TestXpuQuantOnly`` -- the quant-only baseline (Hadamard stripped).+* ``TestXpuKernelErrors`` -- the validation contract.+* ``TestXpuKernelXmx`` -- the opt-in XMX path (tolerance-based).+* ``TestStreamOnlyBaseline`` -- the stream-only roofline baseline, and the+ guarantee that it really reads every input element.+* ``TestHadamardDimReferenceParity`` -- the reference itself is correct at every+ supported transform size ``D``, pinned against an independent construction.+ CPU only.+* ``TestXpuKernelAllDims`` -- the cooperative lane-count template, ``D > 32``.++The two reference classes run everywhere, including on CPU-only machines; the+XPU classes are skipped when no XPU is available.++Acceptance criteria are bit-exact except on the XMX path: E8M0 scale bytes and+packed FP4 code bytes must be **equal** to the PyTorch FP32 reference, no+mismatch tolerance.+"""++import math++import pytest+import torch+from auto_round_kernel.mxfp4_hadamard import (+ GROUP_SIZE,+ HADAMARD_DIM,+ HADAMARD_DIM_128,+ MAX_LANES_PER_ROW,+ SUPPORTED_HADAMARD_DIMS,+ _e8m0_and_quantized,+ _encode_fp4,+ _xmx_supported,+ get_hadamard_matrix,+ hadamard_transform_reference,+ mxfp4_hadamard_quant,+ mxfp4_hadamard_quant_reference,+ mxfp4_quant_reference,+ mxfp4_stream_reference,+ pack_codes,+)++XPU_AVAILABLE = hasattr(torch, "xpu") and torch.xpu.is_available()+requires_xpu = pytest.mark.skipif(not XPU_AVAILABLE, reason="XPU is not available")++SHAPES = [(1, 32), (1, 128), (17, 256)]+DTYPES = [torch.float16, torch.bfloat16]++# A 32-point Hadamard transform can amplify a group by at most+# 32 / sqrt(32) = sqrt(32), so inputs above FP32_MAX / sqrt(32) would overflow+# the FP32 accumulator. Tests that probe "as large as possible" stay below it.+MAX_SAFE_INPUT = 3.4e38 / 32.0**0.5++# Group counts that stress the work-group tail. The D = 32 per-item kernel gives+# one work-item one quantization group and runs kWorkGroupSize = 256 work-items+# per work-group, so a count that is not a multiple of 256 leaves a partial+# trailing work-group whose idle work-items must exit without writing. All four+# counts below are below 256, i.e. every launch here is a single partial+# work-group.+TAIL_SHAPES = [(1, 32), (9, 32), (5, 96), (13, 160)]+++def _dequantize(codes: torch.Tensor, scale: torch.Tensor, k: int) -> torch.Tensor:+ """Unpack (codes, e8m0) back to FP32 values, for readability of failures."""+ levels = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32, device=codes.device)+ flat = codes.reshape(-1, k // 2).to(torch.int32)+ low = flat & 0x0F+ high = (flat >> 4) & 0x0F+ nibbles = torch.stack((low, high), dim=-1).reshape(-1, k)+ values = levels[nibbles & 0x07] * torch.where((nibbles & 0x08) != 0, -1.0, 1.0)+ exp = scale.reshape(-1, k // GROUP_SIZE).to(torch.int32) - 127+ return torch.ldexp(values.reshape(-1, GROUP_SIZE), exp.reshape(-1, 1)).reshape(-1, k)+++class TestReferenceContract:+ """The frozen reference / packing / E8M0 contract. Runs everywhere.++ Orthogonality of the Hadamard matrices themselves is covered for every+ supported D by :class:`TestHadamardDimReferenceParity`.+ """++ def test_packing_matches_vllm_ext_fp4_utils(self):+ fp4_utils = pytest.importorskip("auto_round_extension.vllm_ext.fp4_utils")+ torch.manual_seed(0)+ codes = torch.randint(0, 16, (4, 64), dtype=torch.uint8)+ # Build the FP4 values the codes represent, then pack them with the+ # reference packer and compare byte by byte.+ levels = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32)+ values = levels[(codes & 0x07).long()] * torch.where((codes & 0x08) != 0, -1.0, 1.0)+ # -0.0 keeps the sign bit, matching ``signbit`` based encoding.+ values = torch.where((codes & 0x08) != 0, -values.abs(), values.abs())+ expected = fp4_utils.pack_fp4_to_uint8(values)+ assert torch.equal(pack_codes(codes), expected)++ def test_zero_group_contract(self):+ x = torch.zeros(2, 64, dtype=torch.float16)+ codes, scale = mxfp4_hadamard_quant_reference(x)+ assert torch.all(codes == 0)+ assert torch.all(scale == 0)++ def test_e8m0_matches_floor_log2_contract(self):+ torch.manual_seed(0)+ x = torch.randn(4, 64, dtype=torch.float16)+ _, scale = mxfp4_hadamard_quant_reference(x)+ y = hadamard_transform_reference(x.reshape(-1, HADAMARD_DIM), get_hadamard_matrix(HADAMARD_DIM))+ amax = y.abs().amax(dim=-1)+ for group, group_amax in enumerate(amax.tolist()):+ expected = min(max(int(math.floor(math.log2(group_amax))) - 2 + 127, 0), 254)+ assert scale.reshape(-1)[group].item() == expected+ # amax / scale must land in [4, 8): the E8M0 exponent is standard.+ ratio = group_amax / (2.0 ** (expected - 127))+ assert 4.0 <= ratio < 8.0++ def test_encode_fp4_threshold_boundaries(self):+ # Exact boundary values exercise the alternating <= / < comparisons of+ # cast_to_fp4. Anything at or below 0.25 -> 0, 0.75 is *not* below the+ # 0.5 level, and so on.+ cases = {+ 0.0: 0,+ 0.25: 0,+ 0.2500001: 1,+ 0.75: 2,+ 1.25: 2,+ 1.2500001: 3,+ 1.75: 4,+ 2.5: 4,+ 2.5000005: 5,+ 3.5: 6,+ 5.0: 6,+ 5.0000005: 7,+ 7.9: 7,+ }+ values = torch.tensor(list(cases), dtype=torch.float32)+ expected = torch.tensor(list(cases.values()), dtype=torch.uint8)+ assert torch.equal(_encode_fp4(values), expected)+ # The sign bit is bit 3 and is taken from signbit, except that magnitude+ # 0 is canonicalised to 0x0 rather than 0x8 (see canonical zero rule).+ negated = torch.where(expected == 0, expected, expected | 0x08)+ assert torch.equal(_encode_fp4(-values), negated)++ def test_canonical_zero_never_encodes_negative_zero(self):+ # Any value that rounds to FP4 magnitude 0 must encode as 0x0. The sign+ # of such a value comes from FP32 rounding residue (or from device-side+ # flush-to-zero of subnormals) and is not reproducible, so it must not+ # be observable in the output.+ values = torch.tensor([-0.0, 0.0, -1e-30, 1e-30, -0.25, 0.25, -1e-8], dtype=torch.float32)+ assert torch.equal(_encode_fp4(values), torch.zeros(7, dtype=torch.uint8))+ # Sanity check that the rule is narrow: the smallest non-zero magnitude+ # still carries its sign.+ assert _encode_fp4(torch.tensor([-0.2500001])).item() == 0x09++ def test_canonical_zero_survives_full_pipeline(self):+ # A constant group cancels exactly in every Hadamard column but the+ # first, which is precisely where a negative zero would appear.+ x = torch.full((1, 32), -1.0, dtype=torch.float16)+ codes, _ = mxfp4_hadamard_quant_reference(x)+ assert torch.all(codes[0, 1:] == 0)+ assert (codes & 0x08 != 0).sum() + (codes & 0x80 != 0).sum() <= 1++ def test_accumulation_contract_is_order_defined(self):+ # hadamard_transform_reference must be reproducible bit for bit and must+ # not silently fall back to torch.matmul semantics.+ torch.manual_seed(0)+ x = torch.randn(64, HADAMARD_DIM, dtype=torch.float16).to(torch.float32)+ h = get_hadamard_matrix(HADAMARD_DIM)+ a = hadamard_transform_reference(x, h)+ b = hadamard_transform_reference(x, h)+ assert torch.equal(a, b)+ manual = torch.zeros_like(a)+ for j in range(HADAMARD_DIM):+ manual = manual + x[:, j : j + 1] * h[j]+ assert torch.equal(a, manual)++ def test_custom_hadamard_matrix_is_honored(self):+ # A sign-flipped Hadamard matrix is still orthogonal; the reference must+ # use the matrix it is given rather than the cached default.+ h = get_hadamard_matrix(HADAMARD_DIM).clone()+ h[:, 0] = -h[:, 0]+ torch.manual_seed(0)+ x = torch.randn(4, 32, dtype=torch.float16)+ codes_default, _ = mxfp4_hadamard_quant_reference(x)+ codes_custom, _ = mxfp4_hadamard_quant_reference(x, h)+ assert not torch.equal(codes_default, codes_custom)++ def test_reference_roundtrip_is_close(self):+ torch.manual_seed(0)+ x = torch.randn(8, 256, dtype=torch.float16)+ codes, scale = mxfp4_hadamard_quant_reference(x)+ deq = _dequantize(codes, scale, 256)+ y = hadamard_transform_reference(x.reshape(-1, HADAMARD_DIM), get_hadamard_matrix(HADAMARD_DIM))+ y = y.reshape(8, 256)+ rel = (deq - y).abs().max() / y.abs().max()+ assert rel < 0.2++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_reference_accepts_bf16_and_fp16(self, dtype):+ torch.manual_seed(0)+ x = torch.randn(4, 128, dtype=dtype)+ codes, scale = mxfp4_hadamard_quant_reference(x)+ assert codes.shape == (4, 64) and scale.shape == (4, 4)+ assert codes.dtype == torch.uint8 and scale.dtype == torch.uint8++ # One case per validation branch: not a tensor, non-floating dtype,+ # K not a multiple of 32, empty, non-finite. The XPU entry point runs the+ # same checks, so the remaining variants live there.+ @pytest.mark.parametrize(+ "bad_input, error",+ [+ ("not a tensor", TypeError),+ (torch.randn(1, 32, dtype=torch.float32), ValueError),+ (torch.randint(0, 4, (1, 32), dtype=torch.int8), ValueError),+ (torch.randn(1, 48, dtype=torch.float16), ValueError),+ (torch.randn(1, 0, dtype=torch.float16), ValueError),+ (torch.full((1, 32), float("nan"), dtype=torch.float16), ValueError),+ ],+ )+ def test_reference_rejects_invalid_input(self, bad_input, error):+ with pytest.raises(error):+ mxfp4_hadamard_quant_reference(bad_input)++ @pytest.mark.parametrize(+ "bad_matrix",+ [+ torch.eye(16, dtype=torch.float32),+ torch.eye(HADAMARD_DIM, dtype=torch.int32),+ torch.full((HADAMARD_DIM, HADAMARD_DIM), float("nan"), dtype=torch.float32),+ ],+ )+ def test_reference_rejects_invalid_hadamard(self, bad_matrix):+ x = torch.randn(1, 32, dtype=torch.float16)+ with pytest.raises(ValueError):+ mxfp4_hadamard_quant_reference(x, bad_matrix)++ # ---- quant-only reference (no Hadamard transform) ---------------------++ def test_quant_only_reference_matches_fused_with_identity(self):+ # Quantizing the raw activation is mathematically identical to fusing an+ # identity Hadamard matrix: the Path A transform with H = I only ever+ # adds exact zeros, so the two references must agree byte for byte.+ torch.manual_seed(0)+ x = torch.randn(4, 64, dtype=torch.float16)+ codes, scale = mxfp4_quant_reference(x)+ identity = torch.eye(HADAMARD_DIM, dtype=torch.float32)+ fused_codes, fused_scale = mxfp4_hadamard_quant_reference(x, identity)+ assert torch.equal(codes, fused_codes)+ assert torch.equal(scale, fused_scale)++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_quant_only_reference_shape_dtype_and_zero_group(self, dtype):+ x = torch.zeros(2, 64, dtype=dtype)+ codes, scale = mxfp4_quant_reference(x)+ assert codes.shape == (2, 32)+ assert scale.shape == (2, 2)+ assert codes.dtype == torch.uint8+ assert scale.dtype == torch.uint8+ assert torch.all(codes == 0)+ assert torch.all(scale == 0)++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_quant_only_reference_differs_from_fused_hmt(self, dtype):+ # A real Hadamard transform scrambles values across the group, so the+ # quant-only output must differ from the fused output in general. This+ # guards against quant-only accidentally still applying the transform.+ torch.manual_seed(0)+ x = torch.randn(8, 128, dtype=dtype)+ q_codes, q_scale = mxfp4_quant_reference(x)+ f_codes, f_scale = mxfp4_hadamard_quant_reference(x)+ assert not torch.equal(q_codes, f_codes) or not torch.equal(q_scale, f_scale)+++def _assert_bit_exact(x: torch.Tensor):+ """Run the kernel and the reference on ``x`` and require byte equality."""+ codes, scale = mxfp4_hadamard_quant(x)+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())+ num_rows, k = x.numel() // x.shape[-1], x.shape[-1]+ assert codes.shape == ref_codes.shape == (num_rows, k // 2)+ assert scale.shape == ref_scale.shape == (num_rows, k // GROUP_SIZE)+ assert codes.dtype == torch.uint8 and scale.dtype == torch.uint8+ assert codes.device.type == scale.device.type == "xpu"+ assert torch.equal(scale.cpu(), ref_scale)+ assert torch.equal(codes.cpu(), ref_codes)+ return codes.cpu(), scale.cpu()+++@requires_xpu+class TestXpuKernel:+ """The ``D = 32`` per-item kernel: bit-exact against the reference."""++ @pytest.mark.parametrize("dtype", DTYPES)+ @pytest.mark.parametrize("shape", SHAPES)+ def test_random_finite_input(self, dtype, shape):+ torch.manual_seed(0)+ _assert_bit_exact(torch.randn(*shape, dtype=dtype, device="xpu"))++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_zero_group(self, dtype):+ x = torch.zeros(4, 64, dtype=dtype, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x)+ assert torch.all(codes.cpu() == 0)+ assert torch.all(scale.cpu() == 0)++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_mixed_sign_and_partial_zero_groups(self, dtype):+ torch.manual_seed(1)+ x = torch.randn(5, 128, dtype=dtype, device="xpu")+ x[1, :32] = 0+ x[3, 64:96] = 0+ x[2] = -x[2].abs()+ codes, scale = mxfp4_hadamard_quant(x)+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())+ assert torch.equal(scale.cpu(), ref_scale)+ assert torch.equal(codes.cpu(), ref_codes)+ assert torch.all(scale.cpu()[1, 0] == 0)+ assert torch.all(codes.cpu()[1, :16] == 0)++ def test_extreme_finite_values(self):+ x = torch.zeros(2, 32, dtype=torch.float16, device="xpu")+ x[0] = 65504.0 # FP16 max+ x[1] = 6.1e-5 # smallest FP16 normal+ codes, scale = mxfp4_hadamard_quant(x)+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())+ assert torch.equal(scale.cpu(), ref_scale)+ assert torch.equal(codes.cpu(), ref_codes)++ def test_nibble_order(self):+ torch.manual_seed(3)+ x = torch.randn(1, 64, dtype=torch.float16, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x)+ # Rebuild the per-element 4-bit codes from the reference and check that+ # element 2i sits in the low nibble of byte i, element 2i+1 in the high one.+ y = hadamard_transform_reference(x.cpu().reshape(-1, HADAMARD_DIM), get_hadamard_matrix(HADAMARD_DIM))+ e8m0, q = _e8m0_and_quantized(y)+ nibbles = _encode_fp4(q).reshape(1, 64)+ packed = codes.cpu().to(torch.int32)+ assert torch.equal((packed & 0x0F).to(torch.uint8), nibbles[:, 0::2])+ assert ((packed >> 4) & 0x0F).to(torch.uint8).equal(nibbles[:, 1::2])+ assert torch.equal(scale.cpu(), e8m0.reshape(1, 2))++ def test_non_finite_input_is_not_scanned_by_default(self):+ # The finiteness scan is a debugging aid, not part of the contract: it+ # costs more than the kernel, so the hot path must not pay for it.+ x = torch.randn(1, 32, dtype=torch.float16, device="xpu")+ x[0, 0] = float("nan")+ mxfp4_hadamard_quant(x)+++@requires_xpu+class TestXpuKernelLayouts:+ """How arbitrary input layouts and row counts map onto rows and work-groups."""++ # ---- input shapes and work-group tails -------------------------------++ @pytest.mark.parametrize("dtype", DTYPES)+ @pytest.mark.parametrize("shape", TAIL_SHAPES)+ def test_work_group_tail(self, dtype, shape):+ torch.manual_seed(shape[0])+ x = torch.randn(*shape, dtype=dtype, device="xpu")+ _assert_bit_exact(x)++ @pytest.mark.parametrize("dtype", DTYPES)+ @pytest.mark.parametrize("shape", [(1024, 2048)])+ def test_large_multi_row(self, dtype, shape):+ # A 2-D tensor with many rows and a large K: the row -> work-group+ # mapping is no longer trivial, unlike in TAIL_SHAPES.+ torch.manual_seed(7)+ x = torch.randn(*shape, dtype=dtype, device="xpu")+ _assert_bit_exact(x)++ @pytest.mark.parametrize("shape", [(64,), (2, 3, 64), (2, 3, 5, 96)])+ def test_input_shapes_are_flattened_to_rows(self, shape):+ # 1-D, 3-D and 4-D inputs all flatten to ``[-1, K]`` with K the last+ # dim; the output is always the 2-D ``[rows, ...]`` form.+ torch.manual_seed(2)+ x = torch.randn(*shape, dtype=torch.float16, device="xpu")+ rows, k = x.numel() // shape[-1], shape[-1]+ codes, scale = _assert_bit_exact(x)+ assert codes.shape == (rows, k // 2)+ assert scale.shape == (rows, k // GROUP_SIZE)++ def test_non_contiguous_input_is_materialized(self):+ torch.manual_seed(0)+ base = torch.randn(64, 128, dtype=torch.float16, device="xpu")+ view = base[:, ::2] # stride-2 columns, [64, 64], non-contiguous+ assert not view.is_contiguous()+ codes, scale = mxfp4_hadamard_quant(view)+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(view.cpu())+ assert torch.equal(codes.cpu(), ref_codes)+ assert torch.equal(scale.cpu(), ref_scale)+ # A contiguous copy of the same values must give the same bytes.+ codes_contig, scale_contig = mxfp4_hadamard_quant(view.contiguous())+ assert torch.equal(codes.cpu(), codes_contig.cpu())+ assert torch.equal(scale.cpu(), scale_contig.cpu())+++@requires_xpu+class TestXpuKernelNumerics:+ """D = 32 numerics: dtype dispatch, exponent boundaries, Hadamard handling."""++ # ---- BF16 -------------------------------------------------------------++ def test_bf16_and_fp16_agree_on_exactly_representable_values(self):+ # Values that are exact in both BF16 and FP16 must produce identical+ # codes and scales, proving the two dispatch paths share the FP32 math.+ torch.manual_seed(0)+ base = torch.randint(-8, 9, (16, 128), dtype=torch.int32).to(torch.float32) / 4.0+ codes_fp16, scale_fp16 = mxfp4_hadamard_quant(base.to(torch.float16).to("xpu"))+ codes_bf16, scale_bf16 = mxfp4_hadamard_quant(base.to(torch.bfloat16).to("xpu"))+ assert torch.equal(scale_fp16.cpu(), scale_bf16.cpu())+ assert torch.equal(codes_fp16.cpu(), codes_bf16.cpu())++ def test_bf16_subnormal_clamp_range(self):+ # BF16 has FP32's exponent range, so tiny values drive+ # floor(log2(amax)) - 2 + 127 below 0 and must clamp to e8m0 = 0.+ # 1e-39 is chosen so the clamp fires while the rescaled values are still+ # large enough to encode as non-zero FP4 codes, i.e. this is the clamp+ # path and not the all-zero-group path.+ x = torch.full((2, 32), 1e-39, dtype=torch.bfloat16, device="xpu")+ x[1] = -1e-39+ codes, scale = _assert_bit_exact(x)+ assert torch.all(scale == 0)+ assert torch.any(codes != 0)+ # Far below the clamp the rescaled values fall under the first FP4+ # threshold, so codes become zero while e8m0 stays clamped at 0.+ x = torch.full((1, 32), 1e-43, dtype=torch.bfloat16, device="xpu")+ codes, scale = _assert_bit_exact(x)+ assert torch.all(scale == 0)+ assert torch.all(codes == 0)++ # ---- numerical boundaries --------------------------------------------++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_row_and_group_mapping(self, dtype):+ # Each row gets a distinct magnitude so a row/column mix-up in the+ # scale layout is detected, not just an average mismatch.+ rows, k = 12, 128+ x = torch.zeros(rows, k, dtype=dtype, device="xpu")+ for r in range(rows):+ for g in range(k // GROUP_SIZE):+ x[r, g * GROUP_SIZE] = float(2 ** (r - 6 + g))+ codes, scale = _assert_bit_exact(x)+ # Scales must be strictly increasing along both axes by one octave.+ scale_i = scale.to(torch.int32)+ assert torch.all(scale_i[1:, :] - scale_i[:-1, :] == 1)+ assert torch.all(scale_i[:, 1:] - scale_i[:, :-1] == 1)++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_single_spike_per_group(self, dtype):+ # A one-hot group makes every transformed element equal in magnitude,+ # which puts them exactly on an FP4 level rather than between levels.+ x = torch.zeros(8, 32, dtype=dtype, device="xpu")+ for r in range(8):+ x[r, r * 4] = 1.0 if r % 2 == 0 else -1.0+ _assert_bit_exact(x)++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_alternating_extremes(self, dtype):+ finfo = torch.finfo(dtype)+ big = min(finfo.max, MAX_SAFE_INPUT)+ x = torch.zeros(4, 64, dtype=dtype, device="xpu")+ x[0] = big+ x[1, ::2] = big+ x[1, 1::2] = -big+ x[2] = finfo.tiny+ x[3, ::2] = finfo.tiny+ _assert_bit_exact(x)++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_cancelling_group_yields_canonical_zero(self, dtype):+ # A constant group cancels exactly in Hadamard columns 1..31. On device+ # the FP32 residue of that cancellation, and any flush-to-zero of+ # subnormals, may be negatively signed; the canonical zero rule must+ # keep that out of the codes so the kernel still matches the reference.+ x = torch.zeros(4, 32, dtype=dtype, device="xpu")+ x[0] = 1.0+ x[1] = -1.0+ x[2] = torch.finfo(dtype).tiny+ x[3] = -torch.finfo(dtype).tiny+ codes, _ = _assert_bit_exact(x)+ # Columns 1..31 of every row are exact zeros, i.e. bytes 1..15.+ assert torch.all(codes[:, 1:].cpu() == 0)++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_quantized_grid_values_stress_thresholds(self, dtype):+ # Coarse, exactly representable inputs make transformed values land on+ # or extremely close to the FP4 decision thresholds, which is where an+ # accumulation-order mismatch between kernel and reference would show.+ torch.manual_seed(11)+ x = (torch.randint(-4, 5, (256, 128), device="xpu").to(torch.float32) / 4.0).to(dtype)+ _assert_bit_exact(x)++ @pytest.mark.parametrize("seed", [0, 4, 7])+ @pytest.mark.parametrize("dtype", DTYPES)+ def test_random_fuzz(self, dtype, seed):+ # The seeds sweep 1e-4, 1 and 1e3, i.e. the E8M0 exponent range; more+ # seeds only resample the same magnitudes.+ torch.manual_seed(seed)+ x = (torch.randn(128, 256, device="xpu") * (10.0 ** (seed - 4))).to(dtype)+ _assert_bit_exact(x)++ # ---- Hadamard matrix handling ----------------------------------------++ def test_custom_hadamard_matrix(self):+ # A sign-flipped Hadamard matrix is still orthogonal, so it must be+ # honored rather than silently replaced by the cached default. The+ # forced Path A call pins the bit-exact branch; the auto-routed call+ # exercises whatever the dispatcher picks (the XMX path when it is+ # built, which has a relaxed, tolerance-based contract).+ h = get_hadamard_matrix(HADAMARD_DIM).clone()+ h[:, 0] = -h[:, 0]+ torch.manual_seed(0)+ x = torch.randn(16, 128, dtype=torch.float16, device="xpu")+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), h.cpu())++ codes, scale = mxfp4_hadamard_quant(x, h, _force_xmx=False)+ assert torch.equal(codes.cpu(), ref_codes)+ assert torch.equal(scale.cpu(), ref_scale)++ routed_codes, routed_scale = mxfp4_hadamard_quant(x, h)+ if _xmx_supported():+ deq = _dequantize(routed_codes.cpu(), routed_scale.cpu(), 128)+ ref_deq = _dequantize(ref_codes, ref_scale, 128)+ sqnr_db, _, _ = _precision_metrics(deq, ref_deq, ref_scale)+ assert sqnr_db >= 15.0, f"SQNR {sqnr_db:.2f} dB < 15 dB"+ else:+ assert torch.equal(routed_codes.cpu(), ref_codes)+ assert torch.equal(routed_scale.cpu(), ref_scale)++ default_codes, _ = mxfp4_hadamard_quant(x)+ assert not torch.equal(routed_codes.cpu(), default_codes.cpu())++ def test_hadamard_matrix_input_is_normalized(self):+ # The matrix is coerced to a contiguous FP32 tensor on the activation's+ # device; a CPU host matrix and an FP64 matrix must give the same bytes.+ torch.manual_seed(0)+ x = torch.randn(4, 64, dtype=torch.float16, device="xpu")+ h_cpu = get_hadamard_matrix(HADAMARD_DIM, "cpu")+ assert h_cpu.device.type == "cpu"+ h64 = get_hadamard_matrix(HADAMARD_DIM).to(torch.float64)+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())+ for h in (h_cpu, h64):+ codes, scale = mxfp4_hadamard_quant(x, h)+ assert torch.equal(codes.cpu(), ref_codes)+ assert torch.equal(scale.cpu(), ref_scale)+++@requires_xpu+class TestXpuQuantOnly:+ """The quant-only baseline: the fused kernel with the Hadamard stripped.++ It must be bit-exact with :func:`mxfp4_quant_reference`, the raw-activation+ reference, on the same per-item layout. That equivalence is what makes the+ fused/quant-only bandwidth ratio a valid ablation of the transform.+ """++ @pytest.mark.parametrize("dtype", DTYPES)+ @pytest.mark.parametrize("shape", [(1, 32), (64, 512), (4, 13824)])+ def test_quant_only_matches_reference(self, dtype, shape):+ # The quant-only device path must be bit-exact with mxfp4_quant_reference+ # (raw activation, no transform), on the same per-item layout.+ torch.manual_seed(shape[0])+ x = torch.randn(*shape, dtype=dtype, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x, _quant_only=True)+ ref_codes, ref_scale = mxfp4_quant_reference(x.cpu())+ assert codes.shape == ref_codes.shape == (shape[0], shape[1] // 2)+ assert scale.shape == ref_scale.shape == (shape[0], shape[1] // GROUP_SIZE)+ assert codes.dtype == torch.uint8 and scale.dtype == torch.uint8+ assert torch.equal(scale.cpu(), ref_scale)+ assert torch.equal(codes.cpu(), ref_codes)++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_quant_only_zero_group(self, dtype):+ x = torch.zeros(2, 64, dtype=dtype, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x, _quant_only=True)+ assert torch.all(codes.cpu() == 0)+ assert torch.all(scale.cpu() == 0)++ @pytest.mark.parametrize("seed", [0, 2])+ @pytest.mark.parametrize("dtype", DTYPES)+ def test_quant_only_fuzz(self, dtype, seed):+ # The two seeds span 1e-2 and 1e2, the magnitudes the reference cannot+ # reach through the shape cases above.+ torch.manual_seed(seed)+ x = (torch.randn(64, 256, device="xpu") * (10.0 ** (seed - 2))).to(dtype)+ codes, scale = mxfp4_hadamard_quant(x, _quant_only=True)+ ref_codes, ref_scale = mxfp4_quant_reference(x.cpu())+ assert torch.equal(codes.cpu(), ref_codes)+ assert torch.equal(scale.cpu(), ref_scale)+++@requires_xpu+class TestXpuKernelErrors:+ """The validation contract: bad inputs are rejected up front, never run."""++ @pytest.mark.parametrize("bad_dtype", [torch.float32, torch.int8, torch.bool])+ def test_rejects_unsupported_dtype(self, bad_dtype):+ x = torch.zeros(1, 32, dtype=bad_dtype, device="xpu")+ with pytest.raises(ValueError, match="float16 or bfloat16"):+ mxfp4_hadamard_quant(x)++ @pytest.mark.parametrize("k", [1, 33, 48])+ def test_rejects_k_not_multiple_of_32(self, k):+ x = torch.randn(2, k, dtype=torch.float16, device="xpu")+ with pytest.raises(ValueError, match="multiple of 32"):+ mxfp4_hadamard_quant(x)++ def test_rejects_empty_tensor(self):+ with pytest.raises(ValueError, match="must not be empty"):+ mxfp4_hadamard_quant(torch.randn(0, 32, dtype=torch.float16, device="xpu"))+ with pytest.raises(ValueError, match="must not be empty"):+ mxfp4_hadamard_quant(torch.randn(2, 0, dtype=torch.float16, device="xpu"))++ def test_rejects_cpu_tensor(self):+ x = torch.randn(1, 32, dtype=torch.float16)+ with pytest.raises(ValueError, match="only supported on XPU"):+ mxfp4_hadamard_quant(x)++ def test_rejects_non_tensor(self):+ with pytest.raises(TypeError):+ mxfp4_hadamard_quant([0.0] * 32)++ @pytest.mark.parametrize("bad_value", [float("nan"), float("inf")])+ def test_rejects_non_finite(self, bad_value):+ x = torch.randn(2, 64, dtype=torch.float16, device="xpu")+ x[1, 17] = bad_value+ with pytest.raises(ValueError, match="finite"):+ mxfp4_hadamard_quant(x, check_finite=True)++ @pytest.mark.parametrize(+ "bad_matrix_factory",+ [+ # Wrong shape, degenerate (singular) values, wrong dtype and+ # non-finite entries: one case per validation branch. A power-of-two+ # D above the largest instantiated lane count is covered by+ # TestXpuKernelAllDims::test_rejects_dim_above_max_lane_count.+ lambda: torch.eye(16, dtype=torch.float32, device="xpu"),+ lambda: torch.zeros(HADAMARD_DIM, dtype=torch.float32, device="xpu"),+ lambda: torch.eye(HADAMARD_DIM, dtype=torch.int32, device="xpu"),+ lambda: torch.full((HADAMARD_DIM, HADAMARD_DIM), float("inf"), dtype=torch.float32, device="xpu"),+ ],+ )+ def test_rejects_invalid_hadamard_matrix(self, bad_matrix_factory):+ x = torch.randn(1, 32, dtype=torch.float16, device="xpu")+ with pytest.raises(ValueError):+ mxfp4_hadamard_quant(x, bad_matrix_factory())++ def test_rejects_non_tensor_hadamard_matrix(self):+ x = torch.randn(1, 32, dtype=torch.float16, device="xpu")+ with pytest.raises(TypeError):+ mxfp4_hadamard_quant(x, [[0.0] * 32] * 32)++ def test_state_is_intact_after_rejected_call(self):+ torch.manual_seed(0)+ good = torch.randn(4, 64, dtype=torch.float16, device="xpu")+ expected = mxfp4_hadamard_quant(good)+ bad = torch.randn(2, 48, dtype=torch.float16, device="xpu")+ with pytest.raises(ValueError):+ mxfp4_hadamard_quant(bad)+ actual = mxfp4_hadamard_quant(good)+ assert torch.equal(expected[0].cpu(), actual[0].cpu())+ assert torch.equal(expected[1].cpu(), actual[1].cpu())+++def _xmx_path_available() -> bool:+ """True when the current XPU build exposes the opt-in XMX path."""+ return _xmx_supported()+++# One E2M1 level step at the top of a group, expressed as a fraction of the+# group peak: the levels are 0, .5, 1, 1.5, 2, 3, 4, 6, so the widest single-step+# gap is 6 - 4 = 2 against a peak of 6. A code that lands one level off cannot+# exceed this; anything larger means the code moved by two or more levels, or the+# E8M0 scale bucket disagrees, both of which are real bugs rather than rounding.+ONE_E2M1_LEVEL_REL = 2.0 / 6.0+++def _precision_metrics(deq: torch.Tensor, ref: torch.Tensor, ref_scale: torch.Tensor) -> tuple[float, float, float]:+ """``(sqnr_db, max_rel, flip_rate)`` of ``deq`` vs ``ref`` (both FP32).++ SQNR is the standard signal-to-quantization-noise ratio in dB. Relative+ errors are measured **per group against the group's peak magnitude**+ (``amax = 6 * 2**(e8m0-127)``, the largest FP4 level): per-element relative+ error is meaningless near zero (FP4 alone allows unbounded relative error+ for tiny values), while the per-group bound is inherent to E2M1.++ ``max_rel`` is the strict worst case. It is **discrete**, not continuous:+ the two paths either agree on a code or they do not, so the only attainable+ values are the E2M1 level gaps over the group peak (0, 1/12, 1/6, 1/3). A+ threshold set between two of those -- 0.25, say -- therefore does not express+ a tolerance at all; it silently demands bit-exactness of the top level, which+ the XMX path (bf16 H + DPAS) never promised.++ ``flip_rate`` is the fraction of elements whose code differs at all. This is+ what actually carries regression signal: a percentile of ``rel`` is useless+ here because disagreements are so rare (~3e-4) that every percentile up to+ p999 sits inside the identical majority and reads exactly 0.+ """+ deq = deq.double()+ ref = ref.double()+ err = deq - ref+ signal = (ref * ref).sum()+ noise = (err * err).sum()+ sqnr_db = float(10.0 * torch.log10(signal / noise.clamp_min(1e-30)))+ amax = (6.0 * torch.pow(2.0, ref_scale.double() - 127.0)).reshape(-1, 1)+ rel = (err.abs().reshape(-1, GROUP_SIZE) / amax).flatten()+ max_rel = float(rel.max())+ flip_rate = float((err != 0).double().mean())+ return sqnr_db, max_rel, flip_rate+++@requires_xpu+class TestXpuKernelXmx:+ """Opt-in XMX path, tolerance-based acceptance.++ The XMX path is *not* bit-exact: the Hadamard matrix is stored in the+ activation dtype (fp16/bf16) and the transform runs on XMX DPAS with FP32+ accumulation (relaxed contract). Acceptance: SQNR >= 15 dB, no code off by+ more than one E2M1 level, and a code-disagreement rate below 0.1%.+ """++ @pytest.fixture(autouse=True)+ def _require_xmx(self):+ if not _xmx_path_available():+ pytest.skip("XMX path not available in this build (ARK_SYCL_TLA)")++ @pytest.mark.parametrize("dtype", DTYPES)+ @pytest.mark.parametrize("shape", [(1, 32), (17, 256)])+ def test_xmx_matches_reference_within_tolerance(self, dtype, shape):+ torch.manual_seed(shape[0])+ x = torch.randn(*shape, dtype=dtype, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x, _force_xmx=True)+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())++ num_rows, k = x.numel() // x.shape[-1], x.shape[-1]+ assert codes.shape == ref_codes.shape == (num_rows, k // 2)+ assert scale.shape == ref_scale.shape == (num_rows, k // GROUP_SIZE)+ assert codes.dtype == torch.uint8 and scale.dtype == torch.uint8++ deq_xmx = _dequantize(codes.cpu(), scale.cpu(), k)+ deq_ref = _dequantize(ref_codes, ref_scale, k)+ sqnr_db, max_rel, flip_rate = _precision_metrics(deq_xmx, deq_ref, ref_scale)+ assert sqnr_db >= 15.0, f"SQNR {sqnr_db:.2f} dB < 15 dB"+ # Both bounds are on the same quantity -- how far a code may move -- so+ # they stay consistent as the path changes: max_rel can never exceed one+ # E2M1 level step, and flip_rate stays a small fraction of a percent+ # (measured ~2e-4 .. 5e-4 on the shapes below).+ assert max_rel <= ONE_E2M1_LEVEL_REL + 1e-6, f"max relative error {max_rel:.4f} exceeds one E2M1 level"+ assert flip_rate < 1e-3, f"code disagreement rate {flip_rate:.2e} >= 1e-3"++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_xmx_scales_are_close_to_reference(self, dtype):+ # E8M0 scales are octave buckets; the XMX transform (fp16/bf16 H) can+ # shift a group's max by at most a couple of dB, so at most one bucket.+ torch.manual_seed(3)+ x = torch.randn(64, 256, dtype=dtype, device="xpu")+ _, scale = mxfp4_hadamard_quant(x, _force_xmx=True)+ _, ref_scale = mxfp4_hadamard_quant_reference(x.cpu())+ diff = (scale.cpu().to(torch.int32) - ref_scale.to(torch.int32)).abs()+ assert torch.all(diff <= 1), f"E8M0 scales diverge by more than 1: max={diff.max().item()}"+++@requires_xpu+class TestStreamOnlyBaseline:+ """The stream-only roofline baseline (``_stream_only=True``).++ This mode is a measurement instrument, not a quantizer: it keeps the fused+ kernel's loads, packing shape and stores but drops the transform and the+ quantization math. The tests below exist for one reason above all -- to+ prove the kernel really touches every input element. A compiler that sank+ the loads would leave an empty kernel reporting an unreachable bandwidth,+ and every ratio measured against it would be silently wrong.+ """++ # The stream-only path reuses the fused kernel's layout, packing and stores,+ # so it only needs one shape per distinct work-group situation (a single+ # group, a full work-group and a partial trailing one), not the full matrix.+ @pytest.mark.parametrize("dtype", DTYPES)+ @pytest.mark.parametrize("num_rows,k", [(1, 32), (17, 256), (9, 32), (13, 160)])+ def test_matches_reference(self, dtype, num_rows, k):+ torch.manual_seed(11)+ x = torch.randn(num_rows, k, dtype=dtype, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x, _stream_only=True)+ ref_codes, ref_scale = mxfp4_stream_reference(x.cpu())+ assert torch.equal(codes.cpu(), ref_codes)+ assert torch.equal(scale.cpu(), ref_scale)++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_every_input_element_is_read(self, dtype):+ """Perturbing any single element must change the output.++ Each element contributes its low nibble to one code byte and (for a+ quarter of the positions) to the scale fold, so a one-element change is+ always observable. This is the anti-dead-code-elimination guarantee the+ whole baseline rests on.+ """+ torch.manual_seed(12)+ x = torch.randn(4, 64, dtype=dtype, device="xpu")+ # Set the low nibble of every element to zero, then flip one bit at a+ # time; using raw bits keeps the test independent of float semantics.+ bits = x.view(torch.int16)+ bits &= ~0x0F+ base_codes, _ = mxfp4_hadamard_quant(x, _stream_only=True)+ for idx in range(x.numel()):+ probe = x.clone()+ probe.view(torch.int16).reshape(-1)[idx] |= 0x0F+ codes, _ = mxfp4_hadamard_quant(probe, _stream_only=True)+ assert not torch.equal(codes.cpu(), base_codes.cpu()), f"element {idx} was not read"++ def test_shares_byte_traffic_with_fused(self):+ """Identical output shapes/dtypes, i.e. identical bytes written.++ The bandwidth ratio is only meaningful because both modes move exactly+ the same number of bytes.+ """+ x = torch.randn(8, 256, dtype=torch.bfloat16, device="xpu")+ stream = mxfp4_hadamard_quant(x, _stream_only=True)+ fused = mxfp4_hadamard_quant(x)+ for a, b in zip(stream, fused):+ assert a.shape == b.shape and a.dtype == b.dtype++ def test_rejects_combination_with_quant_only(self):+ x = torch.randn(1, 32, dtype=torch.bfloat16, device="xpu")+ with pytest.raises(ValueError):+ mxfp4_hadamard_quant(x, _quant_only=True, _stream_only=True)+++# Every cooperative size, i.e. everything the lane-count template instantiates+# except D = 32, which the dispatcher routes to the per-item kernel instead.+COOPERATIVE_DIMS = [d for d in SUPPORTED_HADAMARD_DIMS if d > GROUP_SIZE]+++class TestHadamardDimReferenceParity:+ """The torch reference must itself be correct at every supported D.++ Bit-exactness against the reference is worthless if the reference is wrong,+ so this pins it against an independent construction: the transform is a+ matrix multiply by the normalized Sylvester matrix, and that matrix is its+ own inverse up to the normalization.+ """++ @pytest.mark.parametrize("dim", SUPPORTED_HADAMARD_DIMS)+ def test_matrix_is_orthogonal(self, dim):+ h = get_hadamard_matrix(dim, "cpu")+ assert h.shape == (dim, dim)+ assert h.dtype == torch.float32+ identity = h @ h.T+ assert torch.allclose(identity, torch.eye(dim, dtype=torch.float32), atol=1e-5)++ @pytest.mark.parametrize("dim", SUPPORTED_HADAMARD_DIMS)+ def test_reference_matches_matmul(self, dim):+ """hadamard_transform_reference takes an already-grouped [-1, D] view."""+ torch.manual_seed(31)+ x = torch.randn(12, dim, dtype=torch.float32)+ h = get_hadamard_matrix(dim, "cpu")+ assert torch.allclose(hadamard_transform_reference(x, h), x @ h, atol=1e-5)++ @pytest.mark.parametrize("dim", SUPPORTED_HADAMARD_DIMS)+ def test_output_group_count_is_dim_independent(self, dim):+ """Whatever D is, the output is still one E8M0 per 32 elements."""+ x = torch.randn(4, 1024, dtype=torch.bfloat16)+ codes, scale = mxfp4_hadamard_quant_reference(x, get_hadamard_matrix(dim, "cpu"))+ assert codes.shape == (4, 1024 // 2)+ assert scale.shape == (4, 1024 // GROUP_SIZE)++ def test_fwht128_matches_dense_matmul(self):+ """The 7-stage butterfly computes ``x @ H128`` up to FP32 rounding.++ This is the one reference routine that is not a plain matmul, so it+ needs its own independent construction.+ """+ from auto_round_kernel.mxfp4_hadamard import fwht_transform_reference++ torch.manual_seed(21)+ x = torch.randn(64, HADAMARD_DIM_128, dtype=torch.float32)+ h = get_hadamard_matrix(HADAMARD_DIM_128, "cpu")+ got = fwht_transform_reference(x, h.reshape(-1)[0])+ assert torch.allclose(got, x @ h, atol=1e-5, rtol=1e-5)+++@requires_xpu+class TestXpuKernelAllDims:+ """The lane-count template at every instantiated D, not just 128.++ D = 32 * L uses L cooperating lanes, so the tail-convergence behaviour and+ the rows-per-work-group ratio differ for each L. Row counts here are chosen+ to straddle the 256/L rows that fill one work-group.+ """++ @pytest.mark.parametrize("dtype", DTYPES)+ @pytest.mark.parametrize("dim", COOPERATIVE_DIMS)+ def test_bit_exact_against_reference(self, dtype, dim):+ # Rows per work-group is 256 / L, so 33 rows spans more than one+ # work-group only at D = 256 (L = 8) and D = 512 (L = 16); at L = 2 and+ # L = 4 it is a single partial work-group.+ # test_partial_work_group_tail owns the exact boundary row counts, in+ # one dtype.+ torch.manual_seed(32)+ x = torch.randn(33, dim, dtype=dtype, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(dim, "cpu"))+ assert torch.equal(codes.cpu(), ref_codes)+ assert torch.equal(scale.cpu(), ref_scale)++ @pytest.mark.parametrize("dim", COOPERATIVE_DIMS)+ def test_partial_work_group_tail(self, dim):+ """Rows that leave a partial trailing work-group.++ Inactive lanes in that work-group must still reach the cross-lane+ shuffles; if they returned early the collective would be undefined and+ the *active* lanes of the same row would read garbage.+ """+ rows_per_wg = 256 // (dim // GROUP_SIZE)+ for num_rows in (1, rows_per_wg - 1, rows_per_wg, rows_per_wg + 1):+ torch.manual_seed(33)+ x = torch.randn(num_rows, dim, dtype=torch.bfloat16, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(dim, "cpu"))+ assert torch.equal(codes.cpu(), ref_codes), f"codes mismatch at D={dim} rows={num_rows}"+ assert torch.equal(scale.cpu(), ref_scale), f"scale mismatch at D={dim} rows={num_rows}"++ @pytest.mark.parametrize("dim", COOPERATIVE_DIMS)+ def test_multiple_transforms_per_row(self, dim):+ """K = 4*D exercises the row -> group index arithmetic in the store."""+ torch.manual_seed(34)+ x = torch.randn(5, dim * 4, dtype=torch.bfloat16, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(dim, "cpu"))+ assert torch.equal(codes.cpu(), ref_codes)+ assert torch.equal(scale.cpu(), ref_scale)++ def test_all_zero_rows(self):+ # All-zero groups take the e8m0 = 0 clamp on every lane count, so one+ # loop covers what used to be one case per dim.+ for dim in COOPERATIVE_DIMS:+ x = torch.zeros(3, dim, dtype=torch.bfloat16, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))+ assert torch.all(codes.cpu() == 0) and torch.all(scale.cpu() == 0), f"D={dim}"++ def test_rejects_k_not_multiple_of_dim(self):+ for dim in COOPERATIVE_DIMS:+ x = torch.randn(2, dim + GROUP_SIZE, dtype=torch.bfloat16, device="xpu")+ with pytest.raises(ValueError, match=f"multiple of {dim}"):+ mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))++ def test_rejects_non_sylvester(self):+ """There is no O(D^2) fallback above D = 32, so a custom matrix is refused."""+ for dim in COOPERATIVE_DIMS:+ h = get_hadamard_matrix(dim, "xpu").clone()+ h[0, 1] = -h[0, 1]+ x = torch.randn(2, dim, dtype=torch.bfloat16, device="xpu")+ with pytest.raises(NotImplementedError):+ mxfp4_hadamard_quant(x, h)++ def test_transform_size_changes_the_output(self):+ """Each D is a genuinely different transform, not a relabelled D = 32.++ The output *shapes* are D-independent -- everything quantizes in groups+ of 32, so downstream consumers and the bandwidth accounting are+ unaffected by the transform size -- while the values must differ. The+ value check is what stops a dispatcher bug that silently fell back to+ the 32-point butterfly from comparing the reference against itself.+ """+ torch.manual_seed(35)+ x = torch.randn(16, 512, dtype=torch.bfloat16, device="xpu")+ c32, s32 = mxfp4_hadamard_quant(x)+ for dim in COOPERATIVE_DIMS:+ cd, sd = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))+ assert c32.shape == cd.shape and s32.shape == sd.shape, f"D={dim}"+ assert not torch.equal(c32.cpu(), cd.cpu()), f"D={dim} fell back to the 32-point butterfly"++ @pytest.mark.parametrize("dim", SUPPORTED_HADAMARD_DIMS)+ def test_bit_exact_at_scale(self, dim):+ """Large-shape bit-exactness, which small shapes cannot establish.++ The FMA-contraction bug this guards against flipped roughly 1 code in+ 4 million -- only groups whose scaled magnitude landed within an ULP of+ an E2M1 threshold -- so every shape elsewhere in this file passed while+ the kernel was wrong. Catching it needs enough codes for the tail to+ show up, hence 16 M here rather than the few thousand used above.++ It is also why D = 64 and D = 256 must be covered and not assumed safe+ by analogy: their 1/sqrt(D) is an exact power of two, which made the+ load-time product exact and hid the bug for those two sizes alone.+ """+ torch.manual_seed(36)+ x = torch.randn(4096, 4096, dtype=torch.bfloat16, device="xpu")+ codes, scale = mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(dim, "cpu"))+ bad_codes = int((codes.cpu() != ref_codes).sum())+ bad_scale = int((scale.cpu() != ref_scale).sum())+ assert bad_codes == 0, f"D={dim}: {bad_codes} of {ref_codes.numel()} codes differ"+ assert bad_scale == 0, f"D={dim}: {bad_scale} of {ref_scale.numel()} scales differ"++ def test_normalization_is_applied_last(self):+ """The cooperative path must match the norm-last reference, not norm-first.++ These two orders differ only in rounding, so this pins the choice that+ keeps the kernel free of a contractable multiply-add (see the+ normalization note in xpu_mxfp4_hadamard.hpp). For D whose 1/sqrt(D) is+ an exact power of two the two orders coincide, so only the others can+ actually discriminate.+ """+ from auto_round_kernel.mxfp4_hadamard import fwht_transform_reference++ for dim in COOPERATIVE_DIMS:+ h = get_hadamard_matrix(dim, "cpu")+ norm = h.reshape(-1)[0]+ torch.manual_seed(37)+ x = torch.randn(2048, dim, dtype=torch.float32)+ last = fwht_transform_reference(x, norm, norm_last=True)+ first = fwht_transform_reference(x, norm, norm_last=False)+ assert torch.allclose(last, first, atol=1e-4), f"D={dim}"+ # 1/sqrt(D) is an exact power of two only for D = 64 (1/8) and+ # D = 256 (1/16); a frexp mantissa of exactly 0.5 is the precise test+ # for that. Everything else -- 128 included -- must discriminate.+ if math.frexp(float(norm))[0] != 0.5:+ assert not torch.equal(last, first), f"D={dim}: the two orders must be distinguishable"++ def test_lane_count_divides_sub_group(self):+ """The correctness precondition for the cross-lane butterfly.++ A row's lanes must share one sub-group, which holds only while the lane+ count divides the sub-group size of 32.+ """+ for dim in COOPERATIVE_DIMS:+ assert 32 % (dim // GROUP_SIZE) == 0+ assert max(SUPPORTED_HADAMARD_DIMS) // GROUP_SIZE == MAX_LANES_PER_ROW++ @pytest.mark.parametrize("dtype", DTYPES)+ def test_attention_head_dim_layout(self, dtype):+ """A [B, S, H, D] attention tensor transforms along the head dim.++ The kernel only requires the transform dimension to be the innermost+ one, which is exactly how QKV activations are laid out.+ """+ torch.manual_seed(23)+ x = torch.randn(1, 96, 4, HADAMARD_DIM_128, dtype=dtype, device="xpu")+ h = get_hadamard_matrix(HADAMARD_DIM_128, x.device)+ codes, scale = mxfp4_hadamard_quant(x, h)+ num_rows = x.numel() // HADAMARD_DIM_128+ assert codes.shape == (num_rows, HADAMARD_DIM_128 // 2)+ assert scale.shape == (num_rows, 4)+ ref_codes, ref_scale = mxfp4_hadamard_quant_reference(x.cpu(), get_hadamard_matrix(HADAMARD_DIM_128, "cpu"))+ assert torch.equal(codes.cpu(), ref_codes)+ assert torch.equal(scale.cpu(), ref_scale)++ def test_rejects_xmx_for_cooperative_dims(self):+ """The XMX path is D = 32 only."""+ for dim in COOPERATIVE_DIMS:+ x = torch.randn(2, dim, dtype=torch.bfloat16, device="xpu")+ with pytest.raises((RuntimeError, ValueError)):+ mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device), _force_xmx=True)++ def test_rejects_dim_above_max_lane_count(self):+ """A power-of-two D beyond 32 * MAX_LANES_PER_ROW has no instantiation.++ These would need more than 16 lanes per row; 1024 is the first size that+ needs a full 32-lane sub-group per row, which is legal in principle but+ deliberately not built.+ """+ for dim in (1024, 2048):+ x = torch.randn(2, dim, dtype=torch.bfloat16, device="xpu")+ with pytest.raises(ValueError):+ mxfp4_hadamard_quant(x, get_hadamard_matrix(dim, x.device))++ def test_rejects_non_power_of_two_dim(self):+ """D must be 32 * 2^n: the butterfly has no non-power-of-two form."""+ for dim in (96, 160):+ x = torch.randn(2, dim * 2, dtype=torch.bfloat16, device="xpu")+ with pytest.raises(ValueError):+ mxfp4_hadamard_quant(x, torch.eye(dim, dtype=torch.float32, device="xpu"))++ def test_deterministic_across_modes(self):+ """Every mode must be bit-for-bit repeatable.++ The modes are separate kernels behind separate dispatch decisions, but+ the property is the same one, so a single test covers all of them+ instead of one near-identical test per class.+ """+ torch.manual_seed(24)+ x32 = torch.randn(256, 512, dtype=torch.float16, device="xpu")+ x128 = torch.randn(70, 256, dtype=torch.float16, device="xpu")+ h128 = get_hadamard_matrix(HADAMARD_DIM_128, x128.device)+ cases = [+ ("fused D=32", lambda: mxfp4_hadamard_quant(x32)),+ ("cooperative D=128", lambda: mxfp4_hadamard_quant(x128, h128)),+ ("quant-only", lambda: mxfp4_hadamard_quant(x32, _quant_only=True)),+ ("stream-only", lambda: mxfp4_hadamard_quant(x32, _stream_only=True)),+ ]+ if _xmx_supported():+ cases.append(("xmx", lambda: mxfp4_hadamard_quant(x32, _force_xmx=True)))+ for name, run in cases:+ first, first_scale = run()+ for _ in range(2):+ again, again_scale = run()+ assert torch.equal(first.cpu(), again.cpu()), name+ assert torch.equal(first_scale.cpu(), again_scale.cpu()), name+++if __name__ == "__main__":+ pytest.main([__file__, "-v"])