PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
patch triangular_solve for Qwen3.8 model for hpu (#2377)
lkk8cd9b87841a3
diff --git a/auto_round/modeling/hpu_patch.py b/auto_round/modeling/hpu_patch.pyindex a62298f61..b03f88e18 100644--- a/auto_round/modeling/hpu_patch.py+++ b/auto_round/modeling/hpu_patch.py@@ -73,5 +73,86 @@ def patch_finegrained_fp8():logger.warning(f"Failed to patch finegrained_fp8: {e}")-# Apply patch on import if HPU is available+def _make_solve_triangular_wrapper(orig):+ """Build a drop-in replacement for ``torch.linalg.solve_triangular`` that+ probes for a native kernel at runtime.++ Some ``(device, dtype)`` combinations have no native triangular-solve+ kernel: the call falls back to the generic CPU implementation, which+ raises ``NotImplementedError`` (e.g. bf16, observed on HPU). Instead of+ hard-coding a list of broken devices, the wrapper tries the original+ dtype first and, on ``NotImplementedError`` only, solves in fp32 and+ casts the result back to ``torch.result_type(A, B)``. The outcome is+ cached per ``(device.type, A.dtype, B.dtype)`` so the probe cost is paid+ exactly once per combination.+ """+ import torch++ float32 = torch.float32+ result_type = torch.result_type+ support_cache = {}++ def solve_triangular_compat(A, B, **kwargs):+ # A caller-provided out= tensor must be written in its own dtype; the+ # fp32 detour cannot satisfy it, so pass through untouched.+ if kwargs.get("out") is not None:+ return orig(A, B, **kwargs)++ key = (A.device.type, A.dtype, B.dtype)+ if support_cache.get(key) is not False:+ try:+ result = orig(A, B, **kwargs)+ except NotImplementedError:+ if key not in support_cache:+ support_cache[key] = False+ logger.info(+ "torch.linalg.solve_triangular has no native kernel for"+ f" device '{key[0]}' A.dtype={key[1]} B.dtype={key[2]};"+ " falling back to fp32 computation."+ )+ else:+ support_cache.setdefault(key, True)+ return result+ # No native kernel: solve in fp32, then cast back to the promoted dtype.+ result = orig(A.to(float32), B.to(float32), **kwargs)+ return result.to(result_type(A, B))++ solve_triangular_compat._ar_solve_triangular_patched = True+ solve_triangular_compat._ar_orig_solve_triangular = orig+ solve_triangular_compat._ar_support_cache = support_cache+ return solve_triangular_compat+++def patch_solve_triangular():+ """Install the runtime-probed fp32 fallback for ``torch.linalg.solve_triangular``.++ ``transformers`` calls this API at runtime (e.g. Qwen3.5 linear-attention+ state cleanup), so patching the torch entry point covers all callers and+ survives transformers upgrades. The wrapper is only installed on hosts+ where HPEX is available; once installed it is process-wide, which also+ keeps CPU fallback calls made from HPU code paths correct.+ """+ try:+ from auto_round.utils import is_hpex_available++ if not is_hpex_available():+ return # No patching needed on non-HPU devices++ import torch++ current = torch.linalg.solve_triangular+ if getattr(current, "_ar_solve_triangular_patched", False):+ return # already patched++ torch.linalg.solve_triangular = _make_solve_triangular_wrapper(current)+ logger.info(+ "✓ Patched torch.linalg.solve_triangular with a runtime-probed fp32 fallback"+ " for (device, dtype) pairs lacking a native kernel."+ )+ except Exception as e:+ logger.warning(f"Failed to patch torch.linalg.solve_triangular: {e}")+++# Apply patches on import if HPU is availablepatch_finegrained_fp8()+patch_solve_triangular()diff --git a/test/unit/common/utils/test_hpu_patch.py b/test/unit/common/utils/test_hpu_patch.pyindex 891a84263..d8fc0fecb 100644--- a/test/unit/common/utils/test_hpu_patch.py+++ b/test/unit/common/utils/test_hpu_patch.py@@ -27,6 +27,14 @@ We cover both paths with ``unittest.mock``:* **transformers < 4.0** (mocked): the helper should log a warning andreturn early without touching the upstream module.++* **solve_triangular fallback**: the module also installs a wrapper around+ ``torch.linalg.solve_triangular`` (``patch_solve_triangular()``) that+ probes for a native kernel at runtime and, for ``(device, dtype)``+ combinations that raise ``NotImplementedError``, falls back to an fp32+ solve cast back to the promoted dtype. The wrapper factory+ ``_make_solve_triangular_wrapper()`` is tested directly against fakes so+ no real broken kernel is required."""import importlib@@ -255,3 +263,195 @@ def test_hpu_patch_handles_generic_exception(fresh_hpu_patch, monkeypatch):# Should swallow the RuntimeError.assert hpu_patch.patch_finegrained_fp8() is None+++# ---------------------------------------------------------------------------+# torch.linalg.solve_triangular runtime-probed fp32 fallback+# ---------------------------------------------------------------------------+++def clean_solve_triangular(monkeypatch):+ """Guarantee ``torch.linalg.solve_triangular`` is the pristine torch+ implementation for the duration of the test, and restore the previous+ state afterwards.++ The wrapper installed by ``patch_solve_triangular()`` carries an+ ``_ar_orig_solve_triangular`` back-reference, so any number of wrapping+ layers can be unwound to reach the original builtin.+ """+ import torch++ current = torch.linalg.solve_triangular+ while getattr(current, "_ar_solve_triangular_patched", False):+ current = current._ar_orig_solve_triangular+ monkeypatch.setattr(torch.linalg, "solve_triangular", current)+ yield current+++def _make_lower_triangular(n, dtype, device="cpu"):+ """Unit lower-triangular matrix: identity with ones below the diagonal."""+ import torch++ return torch.eye(n, dtype=dtype, device=device) + torch.tril(+ torch.ones(n, n, dtype=dtype, device=device), diagonal=-1+ )+++def test_solve_triangular_not_patched_when_hpu_unavailable(fresh_hpu_patch, clean_solve_triangular, monkeypatch):+ """On a non-HPU host the wrapper must not be installed."""+ import torch++ monkeypatch.setattr("auto_round.utils.is_hpex_available", lambda: False)++ import auto_round.modeling.hpu_patch # noqa: F401++ assert not getattr(torch.linalg.solve_triangular, "_ar_solve_triangular_patched", False)+ assert torch.linalg.solve_triangular is clean_solve_triangular+++def test_solve_triangular_patch_is_idempotent(fresh_hpu_patch, clean_solve_triangular, monkeypatch):+ """Repeated ``patch_solve_triangular()`` calls must install exactly one+ wrapper layer whose back-reference reaches the original builtin."""+ import torch++ monkeypatch.setattr("auto_round.utils.is_hpex_available", lambda: True)++ import auto_round.modeling.hpu_patch as hpu_patch++ hpu_patch.patch_solve_triangular()+ wrapper1 = torch.linalg.solve_triangular+ hpu_patch.patch_solve_triangular()+ wrapper2 = torch.linalg.solve_triangular++ assert wrapper1 is wrapper2+ assert wrapper1._ar_solve_triangular_patched is True+ # The import-time call (HPU mocked available above) and the explicit+ # calls must converge on the same single wrapper.+ assert wrapper1._ar_orig_solve_triangular is clean_solve_triangular+++def test_solve_triangular_supported_dtype_uses_native_path(fresh_hpu_patch, clean_solve_triangular, monkeypatch):+ """When the native kernel handles the (device, dtype) pair, results and+ dtypes pass through untouched and the pair is cached as supported."""+ import torch++ real = clean_solve_triangular+ seen = []++ def spy(A, B, **kwargs):+ seen.append((A.dtype, B.dtype))+ return real(A, B, **kwargs)++ import auto_round.modeling.hpu_patch as hpu_patch++ wrapper = hpu_patch._make_solve_triangular_wrapper(spy)+ monkeypatch.setattr(torch.linalg, "solve_triangular", wrapper)++ A = _make_lower_triangular(4, torch.float32)+ B = torch.randn(4, 2, dtype=torch.float32)+ result = torch.linalg.solve_triangular(A, B, upper=False, unitriangular=True)+ again = torch.linalg.solve_triangular(A, B, upper=False, unitriangular=True)++ assert result.dtype == torch.float32+ assert torch.allclose(result, real(A, B, upper=False, unitriangular=True))+ assert torch.allclose(again, result)+ # Each call invokes the original exactly once, always in the original dtype.+ assert seen == [(torch.float32, torch.float32)] * 2+ assert wrapper._ar_support_cache[("cpu", torch.float32, torch.float32)] is True+++def test_solve_triangular_falls_back_to_fp32_when_kernel_missing(fresh_hpu_patch, clean_solve_triangular, monkeypatch):+ """When the native kernel is missing for a (device, dtype) pair, the+ wrapper solves in fp32 and casts back to the promoted dtype; the negative+ result is cached so later calls skip the probe."""+ import torch++ real = clean_solve_triangular+ seen = []++ def fake_orig(A, B, **kwargs):+ seen.append(A.dtype)+ if A.dtype == torch.bfloat16:+ raise NotImplementedError('triangular_solve_cpu not implemented for "BFloat16"')+ return real(A, B, **kwargs)++ import auto_round.modeling.hpu_patch as hpu_patch++ wrapper = hpu_patch._make_solve_triangular_wrapper(fake_orig)+ monkeypatch.setattr(torch.linalg, "solve_triangular", wrapper)++ A = _make_lower_triangular(4, torch.bfloat16)+ B = torch.randn(4, 2, dtype=torch.float32).to(torch.bfloat16)+ result = torch.linalg.solve_triangular(A, B, upper=False, unitriangular=True)++ expected = real(A.float(), B.float(), upper=False, unitriangular=True).to(torch.bfloat16)+ assert result.dtype == torch.bfloat16+ assert torch.allclose(result, expected, rtol=1e-2, atol=1e-2)+ # First call: the probe with bf16 failed, then the fp32 solve ran.+ assert seen == [torch.bfloat16, torch.float32]+ assert wrapper._ar_support_cache[("cpu", torch.bfloat16, torch.bfloat16)] is False++ # Second call: probe skipped, straight to the fp32 solve.+ seen.clear()+ again = torch.linalg.solve_triangular(A, B, upper=False, unitriangular=True)+ assert torch.allclose(again, expected, rtol=1e-2, atol=1e-2)+ assert seen == [torch.float32]+++def test_solve_triangular_out_kwarg_passes_through(fresh_hpu_patch, clean_solve_triangular, monkeypatch):+ """With a caller-provided out= tensor the wrapper must not take the fp32+ detour (it could not write into the caller's tensor); the call is passed+ through untouched."""+ import torch++ captured = {}+ sentinel = object()++ def fake_orig(A, B, **kwargs):+ captured["A"] = A+ captured["B"] = B+ captured["kwargs"] = kwargs+ return sentinel++ import auto_round.modeling.hpu_patch as hpu_patch++ wrapper = hpu_patch._make_solve_triangular_wrapper(fake_orig)++ A = _make_lower_triangular(3, torch.bfloat16)+ B = torch.randn(3, 2, dtype=torch.float32).to(torch.bfloat16)+ out = torch.empty(3, 2, dtype=torch.bfloat16)++ result = wrapper(A, B, upper=False, unitriangular=True, out=out)++ assert result is sentinel+ assert captured["A"] is A+ assert captured["B"] is B+ assert captured["kwargs"]["out"] is out+ assert wrapper._ar_support_cache == {} # never probed+++def test_solve_triangular_runtime_error_propagates_and_is_not_cached(+ fresh_hpu_patch, clean_solve_triangular, monkeypatch+):+ """Errors other than NotImplementedError (e.g. a singular matrix) must+ propagate to the caller and must not poison the capability cache."""+ import torch++ calls = []++ def fake_orig(A, B, **kwargs):+ calls.append(A.dtype)+ raise RuntimeError("simulated singular matrix")++ import auto_round.modeling.hpu_patch as hpu_patch++ wrapper = hpu_patch._make_solve_triangular_wrapper(fake_orig)+ A = _make_lower_triangular(3, torch.float32)+ B = torch.randn(3, 2, dtype=torch.float32)++ with pytest.raises(RuntimeError, match="singular matrix"):+ wrapper(A, B, upper=False, unitriangular=True)++ assert calls == [torch.float32] # no fp32 retry+ assert wrapper._ar_support_cache == {}