Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(train): bind distributed training to the active accelerator (#3771)
li-lizhee5be9c9827ee
diff --git a/funasr/bin/train.py b/funasr/bin/train.pyindex d7c20f8f0..51e0c51e0 100644--- a/funasr/bin/train.py+++ b/funasr/bin/train.py@@ -32,6 +32,12 @@ from funasr.train_utils.initialize import initializefrom funasr.download.download_model_from_hub import download_modelfrom funasr.models.lora.utils import mark_only_lora_as_trainablefrom funasr.train_utils.set_all_random_seed import set_all_random_seed+from funasr.train_utils.device_funcs import (+ accelerator_device,+ clear_accelerator_cache,+ get_default_dist_backend,+ set_accelerator_device,+)from funasr.train_utils.load_pretrained_model import load_pretrained_modelfrom funasr.utils.misc import prepare_model_dirfrom funasr.train_utils.model_summary import model_summary@@ -81,8 +87,11 @@ def main(**kwargs):use_fsdp = kwargs.get("use_fsdp", False)# use_ddp = False if use_fsdp else use_fsdpif use_ddp or use_fsdp:- dist.init_process_group(backend=kwargs.get("backend", "nccl"), init_method="env://")- torch.cuda.set_device(local_rank)+ dist.init_process_group(+ backend=kwargs.get("backend", get_default_dist_backend()),+ init_method="env://",+ )+ set_accelerator_device(local_rank)logging.info("Build model, frontend, tokenizer")device = kwargs.get("device", "cuda")@@ -128,7 +137,7 @@ def main(**kwargs):logging.info(f"{model_summary(model)}")if use_ddp:- model = model.cuda(local_rank)+ model = model.to(accelerator_device(local_rank))model = DDP(model,device_ids=[local_rank],@@ -161,12 +170,12 @@ def main(**kwargs):# Configure a custom `min_num_params`my_auto_wrap_policy = functools.partial(custom_auto_wrap_policy, min_num_params=int(1e5))- torch.cuda.set_device(local_rank)+ set_accelerator_device(local_rank)model = FSDP(model,auto_wrap_policy=custom_auto_wrap_policy,mixed_precision=None,- device_id=torch.cuda.current_device(),+ device_id=local_rank,)else:model = model.to(device=kwargs.get("device", "cuda"))@@ -246,9 +255,7 @@ def main(**kwargs):trainer.start_step = 0device = next(model.parameters()).device- if device.type == "cuda":- with torch.cuda.device(device):- torch.cuda.empty_cache()+ clear_accelerator_cache(device)time_escaped = (time.perf_counter() - time_slice_i) / 3600.0logging.info(diff --git a/funasr/bin/train_ds.py b/funasr/bin/train_ds.pyindex 1b18fe1b6..f6d2dde23 100644--- a/funasr/bin/train_ds.py+++ b/funasr/bin/train_ds.py@@ -30,6 +30,11 @@ from funasr.train_utils.initialize import initializefrom funasr.download.download_model_from_hub import download_modelfrom funasr.models.lora.utils import mark_only_lora_as_trainablefrom funasr.train_utils.set_all_random_seed import set_all_random_seed+from funasr.train_utils.device_funcs import (+ clear_accelerator_cache,+ get_default_dist_backend,+ set_accelerator_device,+)from funasr.train_utils.load_pretrained_model import load_pretrained_modelfrom funasr.utils.misc import prepare_model_dirfrom funasr.train_utils.model_summary import model_summary@@ -121,16 +126,17 @@ def main(**kwargs):deepspeed_config,trainer_conf,) = _resolve_distributed_config(kwargs, world_size)+ dist_backend = kwargs.get("backend", get_default_dist_backend())if use_deepspeed:logging.info(f"use_deepspeed: {use_deepspeed}")- deepspeed.init_distributed(dist_backend=kwargs.get("backend", "nccl"))+ deepspeed.init_distributed(dist_backend=dist_backend)elif use_ddp or use_fsdp:logging.info(f"use_ddp: {use_ddp}, use_fsdp: {use_fsdp}")dist.init_process_group(- backend=kwargs.get("backend", "nccl"),+ backend=dist_backend,init_method="env://",)- torch.cuda.set_device(local_rank)+ set_accelerator_device(local_rank)# rank = dist.get_rank()@@ -241,9 +247,7 @@ def main(**kwargs):trainer.start_step = 0device = next(model.parameters()).device- if device.type == "cuda":- with torch.cuda.device(device):- torch.cuda.empty_cache()+ clear_accelerator_cache(device)time_escaped = (time.perf_counter() - time_slice_i) / 3600.0logging.info(diff --git a/funasr/train_utils/device_funcs.py b/funasr/train_utils/device_funcs.pyindex dd7fc76a2..6cafb8a59 100644--- a/funasr/train_utils/device_funcs.py+++ b/funasr/train_utils/device_funcs.py@@ -62,3 +62,131 @@ def force_gatherable(data, device):else:warnings.warn(f"{type(data)} may not be gatherable by DataParallel")return data+++_ACCELERATOR_DEVICE_TYPES = ("cuda", "npu", "xpu")++#: Default ``torch.distributed`` backend per accelerator, most preferred first.+#: Only Ascend NPU is listed. Every other device keeps the historical ``"nccl"``+#: default, exactly as before this change, so the backend this module selects at+#: runtime is always one that has been exercised on real hardware. XPU does use+#: XCCL (the former oneCCL name ``"ccl"``), but selecting that backend is not+#: covered by hardware here and is tracked separately, outside this change.+_ACCELERATOR_DIST_BACKENDS = {"npu": ("hccl",)}+++def _accelerator_module(device_type):+ """Return the torch submodule owning ``device_type``, or ``None``.++ ``torch_npu`` registers ``torch.npu`` lazily, so it is imported on demand;+ builds without it simply have no ``torch.npu`` attribute. ``torch.cuda`` and+ ``torch.xpu`` exist on every build, but their ``is_available()`` reports+ ``False`` when that backend is missing.+ """+ if device_type == "npu":+ try:+ import torch_npu # noqa: F401+ except ImportError:+ return None+ return getattr(torch, device_type, None)+++def get_accelerator_device_type():+ """Return the device type of the active accelerator, or ``"cpu"``.++ Probes the same backends FunASR accepts for ``AutoModel`` device fallback+ (CUDA, then Ascend NPU, then XPU). CPU-only torch builds report ``"cpu"`` so+ callers can keep a device-agnostic, no-op codepath.+ """+ for device_type in _ACCELERATOR_DEVICE_TYPES:+ module = _accelerator_module(device_type)+ if module is None:+ continue+ try:+ if module.is_available():+ return device_type+ except Exception:+ continue+ return "cpu"+++def accelerator_device(local_rank=0):+ """Return the ``torch.device`` for ``local_rank`` on the active accelerator."""+ device_type = get_accelerator_device_type()+ if device_type == "cpu":+ return torch.device("cpu")+ return torch.device(device_type, int(local_rank))+++def set_accelerator_device(local_rank=0):+ """Bind this process to ``local_rank`` on the active accelerator.++ Replaces the CUDA-only ``torch.cuda.set_device(local_rank)``, which raises+ (``module 'torch._C' has no attribute '_cuda_setDevice'``) on a build+ without CUDA even when an Ascend NPU is present. Returns the resolved device+ type; for ``"cpu"`` nothing is bound, matching the previous CPU behaviour.+ """+ device_type = get_accelerator_device_type()+ if device_type not in _ACCELERATOR_DEVICE_TYPES:+ return device_type+ module = _accelerator_module(device_type)+ set_device = getattr(module, "set_device", None)+ if set_device is not None:+ set_device(int(local_rank))+ return device_type+++def _dist_backend_available(backend):+ """Ask torch whether ``backend`` is usable: ``is_<backend>_available()``.++ A build that does not expose the probe cannot confirm anything, so it+ reports ``False`` and the caller keeps the accelerator's canonical backend+ name instead of inventing availability.+ """+ distributed = getattr(torch, "distributed", None)+ probe = getattr(distributed, "is_{}_available".format(backend), None)+ if probe is None:+ return False+ try:+ return bool(probe())+ except Exception:+ return False+++def get_default_dist_backend():+ """Return the ``torch.distributed`` backend for the active accelerator.++ CUDA keeps the historical ``"nccl"`` and Ascend NPU uses HCCL, probed with+ ``torch.distributed.is_hccl_available()`` where the build exposes it so the+ selected backend matches the build. A non-CUDA accelerator never falls back+ to NCCL. This is only the default — an explicit ``backend=...`` still wins.++ XPU is deliberately absent: it is a valid ``torch.distributed`` backend+ (XCCL / the former oneCCL name ``"ccl"``), but no XPU hardware is available+ to this change, so XPU keeps the historical ``"nccl"`` value here rather+ than advertising a runtime path nobody can exercise. Selecting XCCL is a+ separate follow-up.+ """+ candidates = _ACCELERATOR_DIST_BACKENDS.get(get_accelerator_device_type())+ if not candidates:+ return "nccl"+ for backend in candidates:+ if _dist_backend_available(backend):+ return backend+ return candidates[0]+++def clear_accelerator_cache(device=None):+ """Release cached allocator memory for ``device`` (no-op on CPU).++ ``torch.cuda.empty_cache()`` is a silent no-op on a build without CUDA, so+ the end-of-epoch release used by the training entry points did nothing on+ Ascend NPU / XPU. Dispatch to the accelerator that owns ``device`` instead.+ """+ device_type = getattr(device, "type", None) or get_accelerator_device_type()+ if device_type not in _ACCELERATOR_DEVICE_TYPES:+ return+ module = _accelerator_module(device_type)+ empty_cache = getattr(module, "empty_cache", None)+ if empty_cache is not None:+ empty_cache()diff --git a/funasr/train_utils/trainer_ds.py b/funasr/train_utils/trainer_ds.pyindex fc2f2b660..eb1ded1d2 100644--- a/funasr/train_utils/trainer_ds.py+++ b/funasr/train_utils/trainer_ds.py@@ -11,7 +11,7 @@ from contextlib import nullcontext, contextmanagerfrom pathlib import Pathfrom torch.nn.parallel import DistributedDataParallel as DDPfrom torch.distributed.fsdp import FullyShardedDataParallel as FSDP-from funasr.train_utils.device_funcs import to_device+from funasr.train_utils.device_funcs import accelerator_device, to_devicefrom funasr.train_utils.recursive_op import recursive_averagefrom funasr.train_utils.average_nbest_models import average_checkpointsfrom funasr.train_utils.checkpoint_metrics import (@@ -1027,7 +1027,7 @@ class Trainer:elif self.use_ddp:local_rank = int(os.environ.get("LOCAL_RANK", 0))- model = model.cuda(local_rank)+ model = model.to(accelerator_device(local_rank))model = DDP(model,device_ids=[local_rank],diff --git a/tests/test_distributed_device_setup.py b/tests/test_distributed_device_setup.pynew file mode 100644index 000000000..4695e0757--- /dev/null+++ b/tests/test_distributed_device_setup.py@@ -0,0 +1,111 @@+import unittest+from unittest import mock++import torch++from funasr.train_utils import device_funcs+++class _FakeAccelerator:+ """Stand-in for ``torch.cuda`` / ``torch.npu`` / ``torch.xpu``."""++ def __init__(self, available):+ self._available = available+ self.device_calls = []+ self.empty_cache_calls = 0++ def is_available(self):+ return self._available++ def set_device(self, index):+ self.device_calls.append(index)++ def empty_cache(self):+ self.empty_cache_calls += 1+++def _fake_modules(**availability):+ return {+ device_type: _FakeAccelerator(availability.get(device_type, False))+ for device_type in device_funcs._ACCELERATOR_DEVICE_TYPES+ }+++class TestAcceleratorResolution(unittest.TestCase):+ """The training entry points must bind devices without assuming CUDA."""++ def _patch(self, **availability):+ modules = _fake_modules(**availability)+ patcher = mock.patch.object(+ device_funcs,+ "_accelerator_module",+ side_effect=lambda device_type: modules.get(device_type),+ )+ patcher.start()+ self.addCleanup(patcher.stop)+ return modules++ def test_device_type_prefers_cuda_then_npu_then_xpu(self):+ self._patch(cuda=True, npu=True, xpu=True)+ self.assertEqual(device_funcs.get_accelerator_device_type(), "cuda")+ self._patch(npu=True, xpu=True)+ self.assertEqual(device_funcs.get_accelerator_device_type(), "npu")+ self._patch(xpu=True)+ self.assertEqual(device_funcs.get_accelerator_device_type(), "xpu")+ self._patch()+ self.assertEqual(device_funcs.get_accelerator_device_type(), "cpu")++ def test_accelerator_device_follows_active_backend(self):+ self._patch()+ self.assertEqual(device_funcs.accelerator_device(0), torch.device("cpu"))+ self._patch(cuda=True)+ self.assertEqual(device_funcs.accelerator_device(3), torch.device("cuda", 3))++ def test_set_accelerator_device_binds_local_rank(self):+ modules = self._patch(npu=True)+ resolved = device_funcs.set_accelerator_device(3)+ self.assertEqual(resolved, "npu")+ self.assertEqual(modules["npu"].device_calls, [3])++ def test_set_accelerator_device_is_noop_without_accelerator(self):+ modules = self._patch()+ self.assertEqual(device_funcs.set_accelerator_device(1), "cpu")+ self.assertTrue(all(not module.device_calls for module in modules.values()))++ def test_default_dist_backend_matches_accelerator(self):+ self._patch(npu=True)+ self.assertEqual(device_funcs.get_default_dist_backend(), "hccl")+ self._patch(cuda=True)+ self.assertEqual(device_funcs.get_default_dist_backend(), "nccl")+ self._patch()+ self.assertEqual(device_funcs.get_default_dist_backend(), "nccl")++ def test_npu_backend_needs_no_capability_probe(self):+ """HCCL is the canonical backend name for Ascend NPU, so a build that+ exposes no ``is_hccl_available()`` probe still resolves to it."""+ self._patch(npu=True)+ with mock.patch.object(device_funcs, "_dist_backend_available", return_value=False):+ self.assertEqual(device_funcs.get_default_dist_backend(), "hccl")++ def test_xpu_keeps_the_historical_nccl_default(self):+ """XPU is a valid ``torch.distributed`` backend (XCCL / oneCCL), but this+ change has no XPU hardware to exercise it, so it must not advertise XCCL:+ the value for an XPU run stays exactly what it was before the change."""+ self._patch(xpu=True)+ with mock.patch.object(+ device_funcs,+ "_dist_backend_available",+ side_effect=lambda backend: backend == "xccl",+ ):+ self.assertEqual(device_funcs.get_default_dist_backend(), "nccl")++ def test_clear_accelerator_cache_dispatches_to_active_backend(self):+ modules = self._patch(npu=True)+ device_funcs.clear_accelerator_cache(torch.device("cpu"))+ self.assertEqual(modules["npu"].empty_cache_calls, 0)+ device_funcs.clear_accelerator_cache(None)+ self.assertEqual(modules["npu"].empty_cache_calls, 1)+++if __name__ == "__main__":+ unittest.main()