Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(amp): resolve autocast/GradScaler device type for non-CUDA accelerators (#3735)
li-lizhe7f85ddd29923
diff --git a/funasr/utils/amp.py b/funasr/utils/amp.pyindex 2469d12cc..e2ffe450b 100644--- a/funasr/utils/amp.py+++ b/funasr/utils/amp.py@@ -14,26 +14,198 @@ older torch versions. Import it instead of ``torch.cuda.amp`` directly:Note: ``torch.amp.autocast`` requires a ``device_type`` argument (e.g.``'cuda'``) that the deprecated ``torch.cuda.amp.autocast`` did not, so-the shim supplies ``device_type="cuda"`` by default for callers that use-the old signature (``with autocast(enabled=..., dtype=...)``).+the shim resolves ``device_type`` for callers that use the old signature+(``with autocast(enabled=..., dtype=...)``) — see+``resolve_amp_device_type``. Callers that know the device can still pass+``device_type=...`` explicitly.++Device type resolution+----------------------++The deprecated ``torch.cuda.amp`` API could only target CUDA, so the first+version of this shim pinned ``device_type="cuda"``. Off CUDA that is not an+error: ``torch.amp`` only warns++ UserWarning: CUDA is not available or torch_xla is imported. Disabling autocast.++and the context becomes a no-op, so ``use_fp16``/``use_bf16`` training silently+ran in float32 on Ascend NPU / XPU / MPS builds. ``GradScaler(enabled=True)``+was silently disabled the same way, so loss scaling was missing too.++Both now default to the accelerator that is actually active+(``torch.accelerator``) — the same device types that+``funasr/models/fun_asr_nano/device_utils.py`` accepts for the Nano inference+path — and keep the historical ``"cuda"`` (a no-op on CPU-only builds) when no+accelerator is available, so CPU behaviour is unchanged.++An explicit device always wins over the detected accelerator:+``GradScaler(device="cpu", enabled=False)`` and ``GradScaler("cpu", ...)``+keep whichever implementation understands that device, exactly as the+``torch.amp.GradScaler`` export they replace did. On an accelerator that ships+its own scaler the ``device`` argument is consumed by this shim (that+constructor takes no ``device``) while every other argument is translated by+name, because those constructors do not share the generic positional order."""import torchtorch_amp = getattr(torch, "amp", None)++# Device types that FunASR is willing to enter an AMP context on; anything else+# keeps the historical "cuda" no-op instead of risking an unsupported context.+_ACCELERATOR_DEVICE_TYPES = ("cuda", "npu", "xpu", "mps")++# Backends that ship their own AMP scaler. Everything else — including ``cpu``,+# whose ``torch.cpu.amp`` is only a deprecated re-export of the generic scaler —+# goes through ``torch.amp.GradScaler``.+_NATIVE_SCALER_DEVICE_TYPES = ("npu", "xpu")++# Parameter names of ``torch.amp.GradScaler``: ``device`` first, ``enabled``+# last. A call written against the generic signature is bound by name before it+# is dispatched to an accelerator-specific scaler, whose parameter order is not+# the same (``torch_npu`` inserts ``dynamic`` before ``enabled``).+_GENERIC_SCALER_PARAMETERS = (+ "device",+ "init_scale",+ "growth_factor",+ "backoff_factor",+ "growth_interval",+ "enabled",+)+++def resolve_amp_device_type():+ """Return the ``torch.amp`` device type of the active accelerator.++ ``torch.accelerator`` reports the accelerator torch was built/registered+ for (``cuda``/``npu``/``xpu``/``mps``); it is ``None`` on CPU-only builds,+ where ``"cuda"`` is kept as the historical no-op default.+ """+ accelerator = getattr(torch, "accelerator", None)+ if accelerator is not None:+ try:+ current = accelerator.current_accelerator()+ except (AttributeError, RuntimeError):+ current = None+ device_type = getattr(current, "type", None) if current is not None else None+ if device_type and str(device_type).lower() in _ACCELERATOR_DEVICE_TYPES:+ return str(device_type).lower()+ return "cuda"+++def _resolve_device_type(device):+ """Return the device type of an explicitly passed device, or ``None``.++ Accepts whatever ``torch.amp.GradScaler`` accepts (``str``, ``torch.device``),+ so ``"npu"``, ``"cuda:0"`` and ``torch.device("cpu")`` all resolve.+ """+ if device is None:+ return None+ device_type = getattr(device, "type", None)+ if device_type is None:+ device_type = str(device).split(":")[0]+ device_type = str(device_type).strip().lower()+ return device_type or None++if torch_amp is not None and hasattr(torch_amp, "autocast") and hasattr(torch_amp, "GradScaler"):+ from torch.amp import GradScaler as _amp_grad_scalerfrom torch.amp import autocast as _amp_autocast- from torch.amp import GradScaler- def autocast(enabled=True, dtype=None, cache_enabled=True):+ def autocast(enabled=True, dtype=None, cache_enabled=True, device_type=None):"""torch.amp.autocast with the legacy CUDA autocast signature."""+ if device_type is None:+ device_type = resolve_amp_device_type()return _amp_autocast(- "cuda", dtype=dtype, enabled=enabled, cache_enabled=cache_enabled+ device_type, dtype=dtype, enabled=enabled, cache_enabled=cache_enabled+ )++ def _grad_scaler_class(device_type):+ """GradScaler implementation to use for ``device_type``.++ Accelerators that ship their own AMP scaler (``torch.npu.amp.GradScaler``+ from torch_npu, ``torch.xpu.amp.GradScaler``) are preferred: the generic+ ``torch.amp.GradScaler("npu")`` builds its scale on the device but then+ fails in ``update()`` (``aclnnAmpUpdateScale`` error 561002 on CANN 9.1.0+ with torch_npu 2.15), while ``torch.npu.amp.GradScaler`` runs a full+ fp16 step. CUDA and CPU stay on ``torch.amp.GradScaler``, i.e. the+ non-deprecated API this shim exists for — note that ``torch.cpu.amp``+ does exist on recent torch but is a deprecated re-export of the generic+ scaler with the device pinned, so it must not be picked up here.+ """+ if device_type in _NATIVE_SCALER_DEVICE_TYPES:+ amp_module = getattr(getattr(torch, device_type, None), "amp", None)+ grad_scaler = getattr(amp_module, "GradScaler", None)+ if grad_scaler is not None:+ return grad_scaler+ return _amp_grad_scaler++ def _generic_arguments(args, kwargs):+ """Bind a ``torch.amp.GradScaler`` call to its parameter names.++ Returns the arguments keyed by generic parameter name, or ``None`` when+ the call does not match the generic signature (unknown or repeated+ parameters). Those calls keep the plain export behaviour: an explicit+ ``device`` is dropped and the rest is forwarded untouched, so the+ *generic* scaler rejects them as ``torch.amp`` would, while an+ accelerator-specific scaler validates the remaining arguments against+ its own signature (``torch_npu`` takes one parameter more, so a call+ the generic scaler would reject can still bind natively).+ """+ if len(args) > len(_GENERIC_SCALER_PARAMETERS):+ return None+ arguments = dict(kwargs)+ for name, value in zip(_GENERIC_SCALER_PARAMETERS, args):+ if name in arguments:+ return None+ arguments[name] = value+ if any(name not in _GENERIC_SCALER_PARAMETERS for name in arguments):+ return None+ return arguments++ def GradScaler(*args, **kwargs):+ """Return the AMP ``GradScaler`` for the requested / active device.++ ``torch.amp.GradScaler``-compatible: an explicit ``device`` (positional+ or keyword) selects the implementation for that device and is forwarded+ untouched to the generic scaler. Without an explicit device the active+ accelerator decides, and the device is filled in for the generic+ implementation only — accelerator-specific scalers default to their own+ device and their constructor takes no ``device`` argument.++ The accelerator implementations do not share the generic positional+ order (``torch_npu`` inserts ``dynamic`` before ``enabled``), so their+ arguments are translated by name: ``GradScaler("npu", 1024.0, 2.0, 0.5,+ 2000, False)`` reaches ``torch.npu.amp.GradScaler`` with+ ``enabled=False`` instead of filling the trailing positionals into the+ native parameters one slot early and enabling a static scaler.+ """+ arguments = _generic_arguments(args, kwargs)+ device = None if arguments is None else arguments.get("device")+ device_type = (+ resolve_amp_device_type() if device is None else _resolve_device_type(device))+ scaler_class = _grad_scaler_class(device_type)+ if scaler_class is _amp_grad_scaler:+ if device is None:+ kwargs["device"] = device_type+ return scaler_class(*args, **kwargs)+ if arguments is None:+ # Not the generic signature: keep the plain export behaviour and+ # only drop the device the caller spelled out, if any.+ if "device" in kwargs:+ kwargs.pop("device", None)+ elif args:+ args = args[1:]+ return scaler_class(*args, **kwargs)+ # The accelerator implementation owns its device and has no ``device``+ # parameter, so drop ours and forward every remaining argument by name.+ arguments.pop("device", None)+ return scaler_class(**arguments)else:from torch.cuda.amp import autocast, GradScaler-__all__ = ["autocast", "GradScaler"]+__all__ = ["autocast", "GradScaler", "resolve_amp_device_type"]diff --git a/tests/test_amp_device_type.py b/tests/test_amp_device_type.pynew file mode 100644index 000000000..3ee6debde--- /dev/null+++ b/tests/test_amp_device_type.py@@ -0,0 +1,369 @@+"""Regression tests for the device type used by the AMP shim.++The shim used to pin ``device_type="cuda"``, which ``torch.amp`` turns into a+silent no-op on any other accelerator (it only warns "CUDA is not available ...+Disabling autocast"), so ``use_fp16``/``use_bf16`` training ran in float32 on+Ascend NPU / XPU / MPS builds, and ``GradScaler(enabled=True)`` was silently+disabled as well.+"""++import importlib.util+import types+from pathlib import Path++import pytest+import torch++AMP_PATH = Path(__file__).resolve().parents[1] / "funasr" / "utils" / "amp.py"+++def _load_amp():+ spec = importlib.util.spec_from_file_location("funasr_amp_device_type", AMP_PATH)+ module = importlib.util.module_from_spec(spec)+ assert spec.loader is not None+ spec.loader.exec_module(module)+ return module+++class _Device:+ """Stand-in for a ``torch.device`` of a backend that may be unavailable."""++ def __init__(self, device_type):+ self.type = device_type+++class _Accelerator:+ def __init__(self, device_type, error=None):+ self._device_type = device_type+ self._error = error++ def current_accelerator(self):+ if self._error is not None:+ raise self._error+ if self._device_type is None:+ return None+ return _Device(self._device_type)+++class _Recorder:+ def __init__(self, result=None):+ self.calls = []+ self._result = result++ def __call__(self, *args, **kwargs):+ self.calls.append((args, kwargs))+ return self._result+++def _patch_accelerator(monkeypatch, device_type, error=None):+ monkeypatch.setattr(+ torch, "accelerator", _Accelerator(device_type, error=error), raising=False+ )+++ "device_type,expected",+ [("cuda", "cuda"), ("npu", "npu"), ("xpu", "xpu"), ("mps", "mps")],+)+def test_resolve_amp_device_type_keeps_the_active_accelerator(monkeypatch, device_type, expected):+ amp = _load_amp()+ _patch_accelerator(monkeypatch, device_type)++ assert amp.resolve_amp_device_type() == expected++[email protected]("device_type", [None, "mtia", "meta"])+def test_resolve_amp_device_type_falls_back_to_cuda(monkeypatch, device_type):+ """No accelerator (CPU-only builds) keeps the historical no-op default."""+ amp = _load_amp()+ _patch_accelerator(monkeypatch, device_type)++ assert amp.resolve_amp_device_type() == "cuda"+++def test_resolve_amp_device_type_survives_a_missing_or_failing_accelerator(monkeypatch):+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu", error=RuntimeError("no accelerator compiled"))+ assert amp.resolve_amp_device_type() == "cuda"++ monkeypatch.delattr(torch, "accelerator", raising=False)+ assert amp.resolve_amp_device_type() == "cuda"+++def test_autocast_resolves_the_device_type_for_the_legacy_signature(monkeypatch):+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ recorder = _Recorder(result="ctx")+ monkeypatch.setattr(amp, "_amp_autocast", recorder)++ assert amp.autocast(enabled=True, dtype=torch.float16) == "ctx"+ assert recorder.calls == [+ (("npu",), {"dtype": torch.float16, "enabled": True, "cache_enabled": True})+ ]+++def test_autocast_keeps_an_explicit_device_type(monkeypatch):+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ recorder = _Recorder(result="ctx")+ monkeypatch.setattr(amp, "_amp_autocast", recorder)++ amp.autocast(False)+ amp.autocast(True, torch.bfloat16, False, "cuda")+ amp.autocast(enabled=False, device_type="cuda")++ assert [args[0] for args, _ in recorder.calls] == ["npu", "cuda", "cuda"]+++def test_grad_scaler_prefers_the_accelerator_amp_module(monkeypatch):+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ accelerator_scaler = _Recorder()+ monkeypatch.setattr(+ torch,+ "npu",+ types.SimpleNamespace(amp=types.SimpleNamespace(GradScaler=accelerator_scaler)),+ raising=False,+ )+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ amp.GradScaler(enabled=True)++ assert generic_scaler.calls == []+ assert accelerator_scaler.calls == [((), {"enabled": True})]+++def test_grad_scaler_passes_the_resolved_device_to_the_generic_implementation(monkeypatch):+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "mps")+ # a backend without its own amp module must not shadow the generic scaler+ monkeypatch.setattr(torch, "mps", types.SimpleNamespace(), raising=False)+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ amp.GradScaler(enabled=True)+ amp.GradScaler("cuda", enabled=False)++ assert generic_scaler.calls == [+ ((), {"enabled": True, "device": "mps"}),+ (("cuda",), {"enabled": False}),+ ]+++def test_grad_scaler_keeps_cuda_and_cpu_on_the_generic_implementation(monkeypatch):+ amp = _load_amp()+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ _patch_accelerator(monkeypatch, "cuda")+ amp.GradScaler(enabled=True)+ assert generic_scaler.calls == [((), {"enabled": True, "device": "cuda"})]++ _patch_accelerator(monkeypatch, None)+ amp.GradScaler(enabled=True)+ assert generic_scaler.calls[-1] == ((), {"enabled": True, "device": "cuda"})+++class _NpuGradScalerStub:+ """``torch.npu.amp.GradScaler`` stand-in with torch_npu's real signature.++ The parameter order matters: torch_npu inserts ``dynamic`` before+ ``enabled``+ (https://github.com/Ascend/pytorch/blob/6d59edda7452b26267eecbb6c328d876a6521ada/torch_npu/npu/amp/grad_scaler.py#L93-L99),+ so a ``*args, **kwargs`` recorder cannot tell a correctly translated call+ from one whose positional tail landed one parameter too early.+ """++ def __init__(+ self,+ init_scale=2.0 ** 16,+ growth_factor=2.0,+ backoff_factor=0.5,+ growth_interval=2000,+ dynamic=True,+ enabled=True,+ ):+ self.init_scale = init_scale+ self.growth_factor = growth_factor+ self.backoff_factor = backoff_factor+ self.growth_interval = growth_interval+ self.dynamic = dynamic+ self._enabled = enabled++ def is_enabled(self):+ return self._enabled+++def _patch_npu_scaler(monkeypatch, scaler=None):+ """Register an NPU AMP scaler (an argument recorder by default)."""+ if scaler is None:+ scaler = _Recorder()+ monkeypatch.setattr(+ torch,+ "npu",+ types.SimpleNamespace(amp=types.SimpleNamespace(GradScaler=scaler)),+ raising=False,+ )+ return scaler+++def test_grad_scaler_honours_an_explicit_keyword_device(monkeypatch):+ """``GradScaler(device="cpu", ...)`` must not reach the NPU implementation.++ The accelerator scaler has no ``device`` parameter, so forwarding it would+ raise ``TypeError``; the generic implementation must be selected instead.+ """+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ accelerator_scaler = _patch_npu_scaler(monkeypatch)+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ amp.GradScaler(device="cpu", enabled=False)++ assert accelerator_scaler.calls == []+ assert generic_scaler.calls == [((), {"device": "cpu", "enabled": False})]+++def test_grad_scaler_honours_an_explicit_positional_device(monkeypatch):+ """``GradScaler("cpu", ...)`` keeps ``"cpu"`` as the device, not as ``init_scale``."""+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ accelerator_scaler = _patch_npu_scaler(monkeypatch)+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ amp.GradScaler("cpu", enabled=False)++ assert accelerator_scaler.calls == []+ assert generic_scaler.calls == [(("cpu",), {"enabled": False})]+++def test_grad_scaler_honours_an_explicit_torch_device(monkeypatch):+ """A ``torch.device`` is resolved by its type, like ``torch.amp.GradScaler``."""+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ accelerator_scaler = _patch_npu_scaler(monkeypatch)+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ device = torch.device("cpu")+ amp.GradScaler(device, enabled=False)++ assert accelerator_scaler.calls == []+ assert generic_scaler.calls == [((device,), {"enabled": False})]+++def test_grad_scaler_drops_the_device_for_the_accelerator_implementation(monkeypatch):+ """An explicit accelerator device selects its scaler without forwarding ``device``."""+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ accelerator_scaler = _patch_npu_scaler(monkeypatch)+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ amp.GradScaler(device="npu", enabled=True)+ amp.GradScaler("npu", 1024.0)++ assert generic_scaler.calls == []+ assert accelerator_scaler.calls == [+ ((), {"enabled": True}),+ ((), {"init_scale": 1024.0}),+ ]++[email protected]("enabled", [False, True])+def test_grad_scaler_translates_the_generic_positional_tail_for_the_npu_scaler(+ monkeypatch, enabled+):+ """``GradScaler("npu", 1024.0, 2.0, 0.5, 2000, enabled)`` keeps ``enabled``.++ The last parameter of the generic signature is ``enabled``, but torch_npu+ inserts ``dynamic`` before it, so forwarding the positional tail unchanged+ hands ``False`` to ``dynamic`` and leaves the scaler enabled — a caller+ asking to disable scaling gets an enabled static scaler instead.+ """+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ _patch_npu_scaler(monkeypatch, scaler=_NpuGradScalerStub)+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ scaler = amp.GradScaler("npu", 1024.0, 2.0, 0.5, 2000, enabled)++ assert generic_scaler.calls == []+ assert isinstance(scaler, _NpuGradScalerStub)+ assert scaler.is_enabled() is enabled+ assert scaler.init_scale == 1024.0+ assert scaler.growth_factor == 2.0+ assert scaler.backoff_factor == 0.5+ assert scaler.growth_interval == 2000+ assert scaler.dynamic is True+++def test_grad_scaler_translates_arguments_by_name_before_dispatch(monkeypatch):+ """The accelerator scaler is called with parameter names, not a shifted tail."""+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ accelerator_scaler = _patch_npu_scaler(monkeypatch)+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ amp.GradScaler("npu", 1024.0, 2.0, 0.5, 2000, False)++ assert generic_scaler.calls == []+ assert accelerator_scaler.calls == [+ (+ (),+ {+ "init_scale": 1024.0,+ "growth_factor": 2.0,+ "backoff_factor": 0.5,+ "growth_interval": 2000,+ "enabled": False,+ },+ )+ ]+++def test_grad_scaler_keeps_a_keyword_enabled_flag_for_the_npu_scaler(monkeypatch):+ """The keyword form keeps meaning the same thing on every scaler."""+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ _patch_npu_scaler(monkeypatch, scaler=_NpuGradScalerStub)++ assert amp.GradScaler(enabled=False).is_enabled() is False+ assert amp.GradScaler(device="npu", enabled=False).is_enabled() is False+ assert amp.GradScaler("npu", enabled=True).is_enabled() is True+++def test_grad_scaler_keeps_an_explicit_cuda_device_on_the_generic_implementation(monkeypatch):+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ accelerator_scaler = _patch_npu_scaler(monkeypatch)+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ amp.GradScaler(device="cuda")+ amp.GradScaler("cuda:0", enabled=False)++ assert accelerator_scaler.calls == []+ assert generic_scaler.calls == [+ ((), {"device": "cuda"}),+ (("cuda:0",), {"enabled": False}),+ ]+++def test_grad_scaler_without_a_device_still_prefers_the_accelerator(monkeypatch):+ """No explicit device keeps the previous behaviour: the active accelerator wins."""+ amp = _load_amp()+ _patch_accelerator(monkeypatch, "npu")+ accelerator_scaler = _patch_npu_scaler(monkeypatch)+ generic_scaler = _Recorder()+ monkeypatch.setattr(amp, "_amp_grad_scaler", generic_scaler)++ amp.GradScaler(enabled=True)++ assert generic_scaler.calls == []+ assert accelerator_scaler.calls == [((), {"enabled": True})]