Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

patch triangular_solve for Qwen3.8 model for hpu (#2377)

lkk8cd9b87841a3
diff --git a/auto_round/modeling/hpu_patch.py b/auto_round/modeling/hpu_patch.py
index a62298f61..b03f88e18 100644
--- a/auto_round/modeling/hpu_patch.py
+++ b/auto_round/modeling/hpu_patch.py
@@ -73,5 +73,86 @@ def patch_finegrained_fp8():
logger.warning(f"Failed to patch finegrained_fp8: {e}")
-# Apply patch on import if HPU is available
+def _make_solve_triangular_wrapper(orig):
+ """Build a drop-in replacement for ``torch.linalg.solve_triangular`` that
+ probes for a native kernel at runtime.
+
+ Some ``(device, dtype)`` combinations have no native triangular-solve
+ kernel: the call falls back to the generic CPU implementation, which
+ raises ``NotImplementedError`` (e.g. bf16, observed on HPU). Instead of
+ hard-coding a list of broken devices, the wrapper tries the original
+ dtype first and, on ``NotImplementedError`` only, solves in fp32 and
+ casts the result back to ``torch.result_type(A, B)``. The outcome is
+ cached per ``(device.type, A.dtype, B.dtype)`` so the probe cost is paid
+ exactly once per combination.
+ """
+ import torch
+
+ float32 = torch.float32
+ result_type = torch.result_type
+ support_cache = {}
+
+ def solve_triangular_compat(A, B, **kwargs):
+ # A caller-provided out= tensor must be written in its own dtype; the
+ # fp32 detour cannot satisfy it, so pass through untouched.
+ if kwargs.get("out") is not None:
+ return orig(A, B, **kwargs)
+
+ key = (A.device.type, A.dtype, B.dtype)
+ if support_cache.get(key) is not False:
+ try:
+ result = orig(A, B, **kwargs)
+ except NotImplementedError:
+ if key not in support_cache:
+ support_cache[key] = False
+ logger.info(
+ "torch.linalg.solve_triangular has no native kernel for"
+ f" device '{key[0]}' A.dtype={key[1]} B.dtype={key[2]};"
+ " falling back to fp32 computation."
+ )
+ else:
+ support_cache.setdefault(key, True)
+ return result
+ # No native kernel: solve in fp32, then cast back to the promoted dtype.
+ result = orig(A.to(float32), B.to(float32), **kwargs)
+ return result.to(result_type(A, B))
+
+ solve_triangular_compat._ar_solve_triangular_patched = True
+ solve_triangular_compat._ar_orig_solve_triangular = orig
+ solve_triangular_compat._ar_support_cache = support_cache
+ return solve_triangular_compat
+
+
+def patch_solve_triangular():
+ """Install the runtime-probed fp32 fallback for ``torch.linalg.solve_triangular``.
+
+ ``transformers`` calls this API at runtime (e.g. Qwen3.5 linear-attention
+ state cleanup), so patching the torch entry point covers all callers and
+ survives transformers upgrades. The wrapper is only installed on hosts
+ where HPEX is available; once installed it is process-wide, which also
+ keeps CPU fallback calls made from HPU code paths correct.
+ """
+ try:
+ from auto_round.utils import is_hpex_available
+
+ if not is_hpex_available():
+ return # No patching needed on non-HPU devices
+
+ import torch
+
+ current = torch.linalg.solve_triangular
+ if getattr(current, "_ar_solve_triangular_patched", False):
+ return # already patched
+
+ torch.linalg.solve_triangular = _make_solve_triangular_wrapper(current)
+ logger.info(
+ "✓ Patched torch.linalg.solve_triangular with a runtime-probed fp32 fallback"
+ " for (device, dtype) pairs lacking a native kernel."
+ )
+ except Exception as e:
+ logger.warning(f"Failed to patch torch.linalg.solve_triangular: {e}")
+
+
+# Apply patches on import if HPU is available
patch_finegrained_fp8()
+patch_solve_triangular()
diff --git a/test/unit/common/utils/test_hpu_patch.py b/test/unit/common/utils/test_hpu_patch.py
index 891a84263..d8fc0fecb 100644
--- a/test/unit/common/utils/test_hpu_patch.py
+++ b/test/unit/common/utils/test_hpu_patch.py
@@ -27,6 +27,14 @@ We cover both paths with ``unittest.mock``:
* **transformers < 4.0** (mocked): the helper should log a warning and
return early without touching the upstream module.
+
+* **solve_triangular fallback**: the module also installs a wrapper around
+ ``torch.linalg.solve_triangular`` (``patch_solve_triangular()``) that
+ probes for a native kernel at runtime and, for ``(device, dtype)``
+ combinations that raise ``NotImplementedError``, falls back to an fp32
+ solve cast back to the promoted dtype. The wrapper factory
+ ``_make_solve_triangular_wrapper()`` is tested directly against fakes so
+ no real broken kernel is required.
"""
import importlib
@@ -255,3 +263,195 @@ def test_hpu_patch_handles_generic_exception(fresh_hpu_patch, monkeypatch):
# Should swallow the RuntimeError.
assert hpu_patch.patch_finegrained_fp8() is None
+
+
+# ---------------------------------------------------------------------------
+# torch.linalg.solve_triangular runtime-probed fp32 fallback
+# ---------------------------------------------------------------------------
+
+
+def clean_solve_triangular(monkeypatch):
+ """Guarantee ``torch.linalg.solve_triangular`` is the pristine torch
+ implementation for the duration of the test, and restore the previous
+ state afterwards.
+
+ The wrapper installed by ``patch_solve_triangular()`` carries an
+ ``_ar_orig_solve_triangular`` back-reference, so any number of wrapping
+ layers can be unwound to reach the original builtin.
+ """
+ import torch
+
+ current = torch.linalg.solve_triangular
+ while getattr(current, "_ar_solve_triangular_patched", False):
+ current = current._ar_orig_solve_triangular
+ monkeypatch.setattr(torch.linalg, "solve_triangular", current)
+ yield current
+
+
+def _make_lower_triangular(n, dtype, device="cpu"):
+ """Unit lower-triangular matrix: identity with ones below the diagonal."""
+ import torch
+
+ return torch.eye(n, dtype=dtype, device=device) + torch.tril(
+ torch.ones(n, n, dtype=dtype, device=device), diagonal=-1
+ )
+
+
+def test_solve_triangular_not_patched_when_hpu_unavailable(fresh_hpu_patch, clean_solve_triangular, monkeypatch):
+ """On a non-HPU host the wrapper must not be installed."""
+ import torch
+
+ monkeypatch.setattr("auto_round.utils.is_hpex_available", lambda: False)
+
+ import auto_round.modeling.hpu_patch # noqa: F401
+
+ assert not getattr(torch.linalg.solve_triangular, "_ar_solve_triangular_patched", False)
+ assert torch.linalg.solve_triangular is clean_solve_triangular
+
+
+def test_solve_triangular_patch_is_idempotent(fresh_hpu_patch, clean_solve_triangular, monkeypatch):
+ """Repeated ``patch_solve_triangular()`` calls must install exactly one
+ wrapper layer whose back-reference reaches the original builtin."""
+ import torch
+
+ monkeypatch.setattr("auto_round.utils.is_hpex_available", lambda: True)
+
+ import auto_round.modeling.hpu_patch as hpu_patch
+
+ hpu_patch.patch_solve_triangular()
+ wrapper1 = torch.linalg.solve_triangular
+ hpu_patch.patch_solve_triangular()
+ wrapper2 = torch.linalg.solve_triangular
+
+ assert wrapper1 is wrapper2
+ assert wrapper1._ar_solve_triangular_patched is True
+ # The import-time call (HPU mocked available above) and the explicit
+ # calls must converge on the same single wrapper.
+ assert wrapper1._ar_orig_solve_triangular is clean_solve_triangular
+
+
+def test_solve_triangular_supported_dtype_uses_native_path(fresh_hpu_patch, clean_solve_triangular, monkeypatch):
+ """When the native kernel handles the (device, dtype) pair, results and
+ dtypes pass through untouched and the pair is cached as supported."""
+ import torch
+
+ real = clean_solve_triangular
+ seen = []
+
+ def spy(A, B, **kwargs):
+ seen.append((A.dtype, B.dtype))
+ return real(A, B, **kwargs)
+
+ import auto_round.modeling.hpu_patch as hpu_patch
+
+ wrapper = hpu_patch._make_solve_triangular_wrapper(spy)
+ monkeypatch.setattr(torch.linalg, "solve_triangular", wrapper)
+
+ A = _make_lower_triangular(4, torch.float32)
+ B = torch.randn(4, 2, dtype=torch.float32)
+ result = torch.linalg.solve_triangular(A, B, upper=False, unitriangular=True)
+ again = torch.linalg.solve_triangular(A, B, upper=False, unitriangular=True)
+
+ assert result.dtype == torch.float32
+ assert torch.allclose(result, real(A, B, upper=False, unitriangular=True))
+ assert torch.allclose(again, result)
+ # Each call invokes the original exactly once, always in the original dtype.
+ assert seen == [(torch.float32, torch.float32)] * 2
+ assert wrapper._ar_support_cache[("cpu", torch.float32, torch.float32)] is True
+
+
+def test_solve_triangular_falls_back_to_fp32_when_kernel_missing(fresh_hpu_patch, clean_solve_triangular, monkeypatch):
+ """When the native kernel is missing for a (device, dtype) pair, the
+ wrapper solves in fp32 and casts back to the promoted dtype; the negative
+ result is cached so later calls skip the probe."""
+ import torch
+
+ real = clean_solve_triangular
+ seen = []
+
+ def fake_orig(A, B, **kwargs):
+ seen.append(A.dtype)
+ if A.dtype == torch.bfloat16:
+ raise NotImplementedError('triangular_solve_cpu not implemented for "BFloat16"')
+ return real(A, B, **kwargs)
+
+ import auto_round.modeling.hpu_patch as hpu_patch
+
+ wrapper = hpu_patch._make_solve_triangular_wrapper(fake_orig)
+ monkeypatch.setattr(torch.linalg, "solve_triangular", wrapper)
+
+ A = _make_lower_triangular(4, torch.bfloat16)
+ B = torch.randn(4, 2, dtype=torch.float32).to(torch.bfloat16)
+ result = torch.linalg.solve_triangular(A, B, upper=False, unitriangular=True)
+
+ expected = real(A.float(), B.float(), upper=False, unitriangular=True).to(torch.bfloat16)
+ assert result.dtype == torch.bfloat16
+ assert torch.allclose(result, expected, rtol=1e-2, atol=1e-2)
+ # First call: the probe with bf16 failed, then the fp32 solve ran.
+ assert seen == [torch.bfloat16, torch.float32]
+ assert wrapper._ar_support_cache[("cpu", torch.bfloat16, torch.bfloat16)] is False
+
+ # Second call: probe skipped, straight to the fp32 solve.
+ seen.clear()
+ again = torch.linalg.solve_triangular(A, B, upper=False, unitriangular=True)
+ assert torch.allclose(again, expected, rtol=1e-2, atol=1e-2)
+ assert seen == [torch.float32]
+
+
+def test_solve_triangular_out_kwarg_passes_through(fresh_hpu_patch, clean_solve_triangular, monkeypatch):
+ """With a caller-provided out= tensor the wrapper must not take the fp32
+ detour (it could not write into the caller's tensor); the call is passed
+ through untouched."""
+ import torch
+
+ captured = {}
+ sentinel = object()
+
+ def fake_orig(A, B, **kwargs):
+ captured["A"] = A
+ captured["B"] = B
+ captured["kwargs"] = kwargs
+ return sentinel
+
+ import auto_round.modeling.hpu_patch as hpu_patch
+
+ wrapper = hpu_patch._make_solve_triangular_wrapper(fake_orig)
+
+ A = _make_lower_triangular(3, torch.bfloat16)
+ B = torch.randn(3, 2, dtype=torch.float32).to(torch.bfloat16)
+ out = torch.empty(3, 2, dtype=torch.bfloat16)
+
+ result = wrapper(A, B, upper=False, unitriangular=True, out=out)
+
+ assert result is sentinel
+ assert captured["A"] is A
+ assert captured["B"] is B
+ assert captured["kwargs"]["out"] is out
+ assert wrapper._ar_support_cache == {} # never probed
+
+
+def test_solve_triangular_runtime_error_propagates_and_is_not_cached(
+ fresh_hpu_patch, clean_solve_triangular, monkeypatch
+):
+ """Errors other than NotImplementedError (e.g. a singular matrix) must
+ propagate to the caller and must not poison the capability cache."""
+ import torch
+
+ calls = []
+
+ def fake_orig(A, B, **kwargs):
+ calls.append(A.dtype)
+ raise RuntimeError("simulated singular matrix")
+
+ import auto_round.modeling.hpu_patch as hpu_patch
+
+ wrapper = hpu_patch._make_solve_triangular_wrapper(fake_orig)
+ A = _make_lower_triangular(3, torch.float32)
+ B = torch.randn(3, 2, dtype=torch.float32)
+
+ with pytest.raises(RuntimeError, match="singular matrix"):
+ wrapper(A, B, upper=False, unitriangular=True)
+
+ assert calls == [torch.float32] # no fp32 retry
+ assert wrapper._ar_support_cache == {}