Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add `unsloth doctor --gpus` multi-GPU preflight and UNSLOTH_DEBUG_DISTRIBUTED (#13884)

Daniel Hane61d5295606b
diff --git a/tests/test_debug_distributed_env.py b/tests/test_debug_distributed_env.py
new file mode 100644
index 00000000000..a96b97b9c27
--- /dev/null
+++ b/tests/test_debug_distributed_env.py
@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import ast
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+_UNSLOTH = Path(__file__).resolve().parents[1] / "unsloth"
+
+
+def _module():
+ # By path: importing the unsloth package needs a GPU.
+ spec = importlib.util.spec_from_file_location(
+ "_unsloth_distributed_debug", _UNSLOTH / "distributed_debug.py"
+ )
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ return mod
+
+
+def test_off_by_default():
+ env = {}
+ assert _module().apply_debug_distributed_env(env) == []
+ assert env == {}
+
+
[email protected]("flag", ["1", "true", "ON"])
+def test_sets_nccl_and_both_flight_recorder_names(flag):
+ env = {"UNSLOTH_DEBUG_DISTRIBUTED": flag}
+ _module().apply_debug_distributed_env(env)
+ assert env["TORCH_NCCL_DESYNC_DEBUG"] == "1"
+ assert env["TORCH_NCCL_DUMP_ON_TIMEOUT"] == "1"
+ assert env["NCCL_DEBUG"] == "INFO"
+ # ProcessGroupNCCL reads only the old name for dump on timeout, the recorder the new one.
+ assert env["TORCH_FR_BUFFER_SIZE"] == "2000"
+ assert env["TORCH_NCCL_TRACE_BUFFER_SIZE"] == "2000"
+
+
+def test_user_values_win_and_old_buffer_name_is_respected():
+ env = {
+ "UNSLOTH_DEBUG_DISTRIBUTED": "1",
+ "NCCL_DEBUG": "WARN",
+ "TORCH_NCCL_TRACE_BUFFER_SIZE": "0",
+ }
+ changed = _module().apply_debug_distributed_env(env)
+ assert env["NCCL_DEBUG"] == "WARN"
+ assert env["TORCH_NCCL_TRACE_BUFFER_SIZE"] == "0"
+ assert env["TORCH_FR_BUFFER_SIZE"] == "0"
+ assert set(changed) == {
+ "TORCH_NCCL_DESYNC_DEBUG",
+ "TORCH_NCCL_DUMP_ON_TIMEOUT",
+ "TORCH_FR_BUFFER_SIZE",
+ }
+
+
+def test_new_buffer_name_is_mirrored_to_the_one_process_group_nccl_reads():
+ env = {"UNSLOTH_DEBUG_DISTRIBUTED": "1", "TORCH_FR_BUFFER_SIZE": "5000"}
+ _module().apply_debug_distributed_env(env)
+ assert env["TORCH_NCCL_TRACE_BUFFER_SIZE"] == "5000"
+
+
+def test_hook_runs_before_torch_is_imported():
+ tree = ast.parse((_UNSLOTH / "__init__.py").read_text(encoding = "utf-8"))
+ hook = first_torch = None
+ for node in tree.body:
+ src = ast.unparse(node)
+ if hook is None and "apply_debug_distributed_env" in src:
+ hook = node.lineno
+ if first_torch is None and ("import torch" in src or "from torch" in src):
+ first_torch = node.lineno
+ assert hook is not None and (first_torch is None or hook < first_torch)
+
+
+def test_existing_process_group_leaves_the_environment_untouched(monkeypatch, capsys):
+ import sys
+ import types
+
+ dist = types.SimpleNamespace(is_available = lambda: True, is_initialized = lambda: True)
+ monkeypatch.setitem(sys.modules, "torch.distributed", dist)
+ env = {"UNSLOTH_DEBUG_DISTRIBUTED": "1"}
+ assert _module().apply_debug_distributed_env(env) == []
+ assert env == {"UNSLOTH_DEBUG_DISTRIBUTED": "1"}
+ assert "already exists" in capsys.readouterr().err
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index 3d4cedda2a9..2a1e70f35f7 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -18,6 +18,12 @@ os.environ["UNSLOTH_IS_PRESENT"] = "1"
# an explicit override, including "0".
os.environ.setdefault("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "1")
+# Before anything can create a process group: c10d and NCCL read these at init.
+if os.environ.get("UNSLOTH_DEBUG_DISTRIBUTED"):
+ from .distributed_debug import apply_debug_distributed_env as _apply_debug_distributed_env
+ _apply_debug_distributed_env()
+ del _apply_debug_distributed_env
+
# UNSLOTH_FSDP2=1 under plain torchrun: export accelerate's FSDP2 launch env before transformers or
# accelerate read it. A conflicting launch (DeepSpeed, FSDP_VERSION=1) raises here, on every rank.
from .fsdp2_launch import apply_unsloth_fsdp2_launch_env as _apply_fsdp2_env
diff --git a/unsloth/distributed_debug.py b/unsloth/distributed_debug.py
new file mode 100644
index 00000000000..b539b7e2c1c
--- /dev/null
+++ b/unsloth/distributed_debug.py
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import os
+import sys
+
+__all__ = ["apply_debug_distributed_env"]
+
+# Read at process group / communicator creation, so they only need to precede init_process_group.
+_DEBUG_DISTRIBUTED_ENV = (
+ ("TORCH_NCCL_DESYNC_DEBUG", "1"),
+ ("TORCH_NCCL_DUMP_ON_TIMEOUT", "1"),
+ ("NCCL_DEBUG", "INFO"),
+)
+# Flight recorder ring size. Both names are needed: the recorder reads TORCH_FR_BUFFER_SIZE first
+# (torch >= 2.8), but ProcessGroupNCCL gates dump on timeout on TORCH_NCCL_TRACE_BUFFER_SIZE alone
+# (ProcessGroupNCCL.hpp, through 2.13), the only name torch 2.6 / 2.7 know. Costs torch >= 2.8 one
+# deprecation warning.
+_FR_BUFFER_NAMES = ("TORCH_FR_BUFFER_SIZE", "TORCH_NCCL_TRACE_BUFFER_SIZE")
+_FR_BUFFER_SIZE = "2000"
+
+
+def apply_debug_distributed_env(environ = None):
+ """Under UNSLOTH_DEBUG_DISTRIBUTED=1, enable NCCL / c10d hang diagnostics.
+
+ Variables the user already set are left alone. The two flight recorder names are
+ one setting: a value given under either is copied to the other, so an explicit
+ TORCH_NCCL_TRACE_BUFFER_SIZE=0 is not overridden. Returns the names that were set.
+ """
+ environ = os.environ if environ is None else environ
+ if environ.get("UNSLOTH_DEBUG_DISTRIBUTED", "0").lower() not in ("1", "true", "yes", "on"):
+ return []
+ dist = sys.modules.get("torch.distributed")
+ try:
+ initialized = dist is not None and dist.is_available() and dist.is_initialized()
+ except Exception:
+ initialized = False
+ if initialized:
+ # Too late to take effect, and ProcessGroupNCCL threads read the environment
+ # concurrently (setenv races getenv), so leave it untouched.
+ print(
+ "Unsloth: UNSLOTH_DEBUG_DISTRIBUTED=1 is set but the process group already exists, "
+ "so it was not applied. Import unsloth before init_process_group, or export "
+ "the variables in the launcher.",
+ file = sys.stderr,
+ )
+ return []
+ changed = []
+ for name, value in _DEBUG_DISTRIBUTED_ENV:
+ if name not in environ:
+ environ[name] = value
+ changed.append(name)
+ given = [environ[name] for name in _FR_BUFFER_NAMES if name in environ]
+ for name in _FR_BUFFER_NAMES:
+ if name not in environ:
+ environ[name] = given[0] if given else _FR_BUFFER_SIZE
+ changed.append(name)
+ return changed
diff --git a/unsloth_cli/__init__.py b/unsloth_cli/__init__.py
index 809e82f3482..2a351b1088b 100644
--- a/unsloth_cli/__init__.py
+++ b/unsloth_cli/__init__.py
@@ -100,6 +100,7 @@ else:
from unsloth_cli.commands.start import start_app
from unsloth_cli.commands.export import export, list_checkpoints
from unsloth_cli.commands.eval import evaluate as eval_command
+ from unsloth_cli.commands.doctor import doctor
from unsloth_cli.commands.studio import (
run as studio_run,
studio_app,
@@ -235,6 +236,7 @@ if not _windows_studio_mutation_entry:
app.command()(export)
app.command("eval")(eval_command)
app.command("list-checkpoints")(list_checkpoints)
+ app.command("doctor")(doctor)
app.add_typer(
start_app,
name = "start",
diff --git a/unsloth_cli/commands/doctor.py b/unsloth_cli/commands/doctor.py
new file mode 100644
index 00000000000..6de9be9c80a
--- /dev/null
+++ b/unsloth_cli/commands/doctor.py
@@ -0,0 +1,583 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""`unsloth doctor --gpus`: a multi-GPU preflight to run before a distributed job.
+
+Reports the GPUs, versions and interconnect, checks that peer copies keep their
+data, runs a short all_reduce across every visible GPU, and ends with a verdict
+and the settings to try. torch is imported lazily so `unsloth --help` stays fast.
+"""
+
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+from typing import List, Optional, Tuple
+
+import typer
+
+# NVML topology levels (nvmlGpuTopologyLevel_t), closest first.
+_TOPOLOGY_LABELS = {0: "SAME", 10: "PIX", 20: "PXB", 30: "PHB", 40: "NODE", 50: "SYS"}
+_PCIE_HOST_LEVELS = {"PHB", "NODE", "SYS"}
+_SENTINEL = -7.0
+
+_SMOKE_WORKER = r"""
+import datetime, os, sys, time
+import torch
+import torch.distributed as dist
+
+rank = int(os.environ["RANK"])
+world = int(os.environ["WORLD_SIZE"])
+backend = os.environ["UNSLOTH_DOCTOR_BACKEND"]
+if backend == "nccl":
+ torch.cuda.set_device(rank)
+ device = torch.device("cuda", rank)
+else:
+ device = torch.device("cpu")
+dist.init_process_group(
+ backend,
+ timeout = datetime.timedelta(seconds = float(os.environ["UNSLOTH_DOCTOR_TIMEOUT"])),
+)
+numel = int(os.environ["UNSLOTH_DOCTOR_NUMEL"])
+x = torch.full((numel,), float(rank + 1), device = device)
+expected = world * (world + 1) / 2
+ok = True
+dist.all_reduce(x)
+ok = ok and bool((x == expected).all())
+iters = 5
+dist.barrier()
+if device.type == "cuda":
+ torch.cuda.synchronize()
+start = time.perf_counter()
+for _ in range(iters):
+ x.fill_(1.0)
+ dist.all_reduce(x)
+if device.type == "cuda":
+ torch.cuda.synchronize()
+elapsed = (time.perf_counter() - start) / iters
+ok = ok and bool((x == world).all())
+# nccl-tests busbw convention for all_reduce: algbw * 2 (n - 1) / n.
+busbw = numel * x.element_size() / elapsed * 2 * (world - 1) / world / 1e9
+if rank == 0:
+ print(f"UNSLOTH_DOCTOR_RESULT ok={int(ok)} busbw_gbps={busbw:.2f}", flush = True)
+dist.destroy_process_group()
+sys.exit(0 if ok else 3)
+"""
+
+
+def _free_port() -> int:
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def collect_gpus(torch) -> dict:
+ hip = getattr(torch.version, "hip", None)
+ # AMD SDK wheels leave version.hip unset and only say "rocm" in __version__.
+ if hip is None and "rocm" in str(torch.__version__).lower():
+ hip = str(torch.__version__).lower().split("rocm", 1)[1] or "unknown"
+ info = {
+ "torch": torch.__version__,
+ "cuda": getattr(torch.version, "cuda", None),
+ "hip": hip,
+ "nccl": None,
+ "gpus": [],
+ }
+ if not torch.cuda.is_available():
+ return info
+ try:
+ nccl = torch.cuda.nccl.version()
+ info["nccl"] = ".".join(str(v) for v in nccl) if isinstance(nccl, tuple) else str(nccl)
+ except Exception:
+ pass
+ for i in range(torch.cuda.device_count()):
+ p = torch.cuda.get_device_properties(i)
+ pci = None
+ if getattr(p, "pci_bus_id", None) is not None:
+ pci = (
+ f"{getattr(p, 'pci_domain_id', 0):08x}:{p.pci_bus_id:02x}:"
+ f"{getattr(p, 'pci_device_id', 0):02x}.0"
+ )
+ # torch < 2.8 has no pci_bus_id; the UUID still identifies the card to NVML.
+ uuid = getattr(p, "uuid", None)
+ uuid = str(uuid) if uuid is not None and not hip else None
+ info["gpus"].append(
+ {
+ "index": i,
+ "name": p.name,
+ "capability": f"{p.major}.{p.minor}",
+ "memory_gib": round(p.total_memory / 1024**3, 1),
+ "pci": pci,
+ "uuid": uuid,
+ }
+ )
+ return info
+
+
+def driver_version(pynvml = None) -> Optional[str]:
+ if pynvml is not None:
+ try:
+ pynvml.nvmlInit()
+ try:
+ v = pynvml.nvmlSystemGetDriverVersion()
+ return v.decode() if isinstance(v, bytes) else str(v)
+ finally:
+ pynvml.nvmlShutdown()
+ except Exception:
+ pass
+ if shutil.which("nvidia-smi") is None:
+ return None
+ try:
+ out = subprocess.run(
+ ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"],
+ capture_output = True,
+ text = True,
+ timeout = 20,
+ )
+ lines = out.stdout.strip().splitlines()
+ return lines[0].strip() if out.returncode == 0 and lines else None
+ except Exception:
+ return None
+
+
+def nvml_topology(pynvml, gpus: List[dict]):
+ """({(a, b): {"level": "PHB", "nvlink": bool, "p2p_read": bool}} keyed by torch index, error).
+
+ Handles are looked up by PCI bus id (else device UUID), since NVML numbers GPUs in
+ PCI order and ignores CUDA_VISIBLE_DEVICES / CUDA_DEVICE_ORDER, which renumber them.
+ A query a virtualised GPU refuses degrades that field instead of the whole table.
+ """
+ if pynvml is None:
+ return None, "nvidia-ml-py is not installed"
+ if len(gpus) < 2 or any(g["pci"] is None and not g.get("uuid") for g in gpus):
+ return None, "torch reports neither PCI bus ids nor device UUIDs"
+ read_idx = getattr(pynvml, "NVML_P2P_CAPS_INDEX_READ", 0)
+ nvlink_idx = getattr(pynvml, "NVML_P2P_CAPS_INDEX_NVLINK", 2)
+ status_ok = getattr(pynvml, "NVML_P2P_STATUS_OK", 0)
+
+ def query(fn, *args):
+ try:
+ return fn(*args)
+ except Exception:
+ return None
+
+ try:
+ pynvml.nvmlInit()
+ except Exception as e:
+ return None, f"nvmlInit failed: {e}"
+ try:
+ handles = []
+ for g in gpus:
+ if g["pci"] is not None:
+ handles.append(pynvml.nvmlDeviceGetHandleByPciBusId(g["pci"].encode()))
+ else:
+ uuid = g["uuid"]
+ if not uuid.startswith(("GPU-", "MIG-")):
+ uuid = "GPU-" + uuid
+ handles.append(pynvml.nvmlDeviceGetHandleByUUID(uuid.encode()))
+ out = {}
+ for a in range(len(gpus)):
+ for b in range(a + 1, len(gpus)):
+ ha, hb = handles[a], handles[b]
+ level = query(pynvml.nvmlDeviceGetTopologyCommonAncestor, ha, hb)
+ out[(a, b)] = {
+ "level": "?" if level is None else _TOPOLOGY_LABELS.get(int(level), str(level)),
+ "nvlink": query(pynvml.nvmlDeviceGetP2PStatus, ha, hb, nvlink_idx) == status_ok,
+ # Directional: a virtualised or IOMMU-restricted host can grant one way only.
+ "p2p_read": all(
+ query(pynvml.nvmlDeviceGetP2PStatus, x, y, read_idx) == status_ok
+ for x, y in ((ha, hb), (hb, ha))
+ ),
+ }
+ return out, None
+ except Exception as e:
+ return None, f"NVML query failed: {type(e).__name__}: {e}"
+ finally:
+ try:
+ pynvml.nvmlShutdown()
+ except Exception:
+ pass
+
+
+def p2p_copy_probe_with_deadline(torch, count: int, timeout: float):
+ """p2p_copy_probe in a daemon thread: a wedged peer copy cannot hold up the verdict.
+ Returns (results, hung)."""
+ import threading
+
+ out = {}
+ active = []
+ crashed = []
+
+ def run():
+ try:
+ p2p_copy_probe(torch, count, out = out, active = active)
+ except Exception as e:
+ crashed.append(f"{type(e).__name__}: {e}")
+
+ worker = threading.Thread(target = run, daemon = True)
+ worker.start()
+ worker.join(timeout)
+ hung = worker.is_alive()
+ # Snapshot: a hung worker may still write. Pairs finished before the deadline keep their result.
+ results = dict(out)
+ for a in range(count):
+ for b in range(count):
+ if a == b or (a, b) in results:
+ continue
+ if hung and active and active[-1] == (a, b):
+ results[(a, b)] = f"error: peer copy did not finish within {timeout:.0f}s (hung)"
+ elif hung:
+ results[(a, b)] = "inconclusive: not reached before the probe deadline"
+ elif crashed:
+ results[(a, b)] = f"inconclusive: probe stopped: {crashed[0]}"
+ return results, hung
+
+
+def p2p_copy_probe(
+ torch,
+ count: int,
+ mib: int = 16,
+ out: Optional[dict] = None,
+ active: Optional[list] = None,
+) -> dict:
+ """{(src, dst): "ok" | "corrupt" | "error: ..." | "inconclusive: ..."} per ordered GPU pair.
+
+ The destination is sentinel-filled first: on a translating IOMMU a PCIe peer copy
+ can be dropped while CUDA reports success (scripts/p2p_integrity_probe.py).
+ """
+ out = {} if out is None else out
+ numel = mib * 1024 * 1024 // 4
+ for src in range(count):
+ for dst in range(count):
+ if src == dst:
+ continue
+ if active is not None:
+ active.append((src, dst))
+ s = d = None
+ try:
+ s = torch.arange(numel, dtype = torch.float32, device = f"cuda:{src}")
+ d = torch.full((numel,), _SENTINEL, dtype = torch.float32, device = f"cuda:{dst}")
+ except Exception as e:
+ # No copy was attempted, so this says nothing about peer copies.
+ out[(src, dst)] = f"inconclusive: could not allocate: {e}"
+ del s, d
+ continue
+ try:
+ d.copy_(s)
+ torch.cuda.synchronize(src)
+ torch.cuda.synchronize(dst)
+ out[(src, dst)] = "ok" if torch.equal(d.cpu(), s.cpu()) else "corrupt"
+ del s, d
+ except Exception as e:
+ out[(src, dst)] = f"error: {e}"
+ torch.cuda.empty_cache()
+ return out
+
+
+def _nccl_p2p_disabled() -> bool:
+ """True when the inherited NCCL_P2P_DISABLE already turns P2P off.
+
+ NCCL's ncclGetLevel (src/graph/paths.cc) parses it with strtol and disables only on 1.
+ """
+ try:
+ return int(os.environ.get("NCCL_P2P_DISABLE", "").strip() or "0", 0) == 1
+ except ValueError:
+ return False
+
+
+def allreduce_smoke(
+ world: int,
+ backend: str,
+ timeout: float,
+ extra_env: Optional[dict] = None,
+ numel: int = 16 * 1024 * 1024,
+) -> dict:
+ """Run one all_reduce over `world` local ranks in fresh processes.
+
+ Separate interpreters rather than threads: a hung NCCL communicator can only be
+ recovered by killing its process, and the doctor must answer instead of hanging.
+ """
+ env = dict(os.environ)
+ env.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(_free_port()),
+ WORLD_SIZE = str(world),
+ UNSLOTH_DOCTOR_BACKEND = backend,
+ UNSLOTH_DOCTOR_TIMEOUT = str(timeout),
+ UNSLOTH_DOCTOR_NUMEL = str(numel),
+ )
+ env.update(extra_env or {})
+ procs = []
+ logs = []
+ try:
+ # Files, not pipes: an undrained pipe (e.g. NCCL_DEBUG=INFO on rank 1) blocks that
+ # rank inside the collective and the doctor would report a healthy system as hung.
+ for rank in range(world):
+ logs.append(tempfile.TemporaryFile(mode = "w+"))
+ rank_env = dict(env, RANK = str(rank), LOCAL_RANK = str(rank))
+ procs.append(
+ subprocess.Popen(
+ [sys.executable, "-c", _SMOKE_WORKER],
+ env = rank_env,
+ stdout = logs[rank],
+ stderr = subprocess.STDOUT,
+ text = True,
+ )
+ )
+ except Exception as e:
+ # Started ranks would wait in init_process_group for ranks that never exist.
+ for q in procs:
+ q.kill()
+ q.wait()
+ for f in logs:
+ f.close()
+ return {
+ "ok": False,
+ "hung": False,
+ "busbw_gbps": None,
+ "log": f"could not start rank {len(procs)} of {world}: {type(e).__name__}: {e}",
+ }
+ # Startup (torch import, communicator init) is not covered by the collective timeout.
+ deadline = time.monotonic() + timeout + 60
+ hung = False
+ outputs = []
+ try:
+ for p in procs:
+ try:
+ p.wait(timeout = max(1.0, deadline - time.monotonic()))
+ except subprocess.TimeoutExpired:
+ hung = True
+ for q in procs:
+ q.kill()
+ break
+ for f in logs:
+ f.seek(0)
+ outputs.append(f.read())
+ finally:
+ # Ctrl-C on a hung run must not leave ranks holding GPU contexts behind. Bounded:
+ # a rank stuck in an uninterruptible driver call may never be reaped.
+ for q in procs:
+ if q.poll() is None:
+ q.kill()
+ for q in procs:
+ try:
+ q.wait(timeout = 10)
+ except subprocess.TimeoutExpired:
+ hung = True
+ for f in logs:
+ f.close()
+ result = {"ok": False, "hung": hung, "busbw_gbps": None, "log": ""}
+ for line in outputs[0].splitlines():
+ if line.startswith("UNSLOTH_DOCTOR_RESULT"):
+ fields = dict(kv.split("=", 1) for kv in line.split()[1:])
+ result["busbw_gbps"] = float(fields["busbw_gbps"])
+ result["ok"] = fields["ok"] == "1"
+ result["ok"] = result["ok"] and not hung and all(p.returncode == 0 for p in procs)
+ if not result["ok"]:
+ result["log"] = "\n".join(o[-2000:] for o in outputs)
+ return result
+
+
+def verdict(
+ info: dict,
+ topology: Optional[dict],
+ p2p: Optional[dict],
+ smoke: Optional[dict],
+ smoke_no_p2p: Optional[dict],
+) -> Tuple[bool, List[str]]:
+ """(healthy, hints). Pure, so every branch is testable without GPUs."""
+ hints = []
+ gpus = info["gpus"]
+ if not gpus:
+ return False, [
+ "No GPU is visible to torch. Check `nvidia-smi`, CUDA_VISIBLE_DEVICES, and "
+ "that torch was installed with CUDA or ROCm support."
+ ]
+ if len(gpus) == 1:
+ return True, ["One GPU visible: multi-GPU checks skipped."]
+ healthy = True
+ if len({g["capability"] for g in gpus}) > 1 or len({g["name"] for g in gpus}) > 1:
+ hints.append(
+ "The GPUs differ in model or compute capability. Distributed training runs at "
+ "the pace of the slowest card, and bf16 / FlashAttention need every card to "
+ "support them."
+ )
+ bad_pairs = sorted(k for k, v in (p2p or {}).items() if v == "corrupt" or v.startswith("error"))
+ unchecked = sorted(k for k, v in (p2p or {}).items() if v.startswith("inconclusive"))
+ if bad_pairs:
+ healthy = False
+ hints.append(
+ f"Peer-to-peer copies lost data or failed on {bad_pairs}. Set "
+ "NCCL_P2P_DISABLE=1 for every distributed run on this host, and check the "
+ "kernel log for IOMMU faults: dmesg | grep -i -E 'DMAR|AMD-Vi'."
+ )
+ if unchecked:
+ reason = p2p[unchecked[0]][len("inconclusive: ") :]
+ hints.append(
+ f"Peer copies on {unchecked} were not tested ({reason}). Free GPU memory and "
+ "re-run `unsloth doctor --gpus`."
+ )
+ if smoke is not None and smoke.get("unverified"):
+ healthy = False
+ hints.append(
+ "This PyTorch build has no NCCL backend (for example on Windows), so GPU "
+ "collectives could not be tested and multi-GPU training here is unverified."
+ )
+ elif smoke is not None and not smoke["ok"]:
+ healthy = False
+ what = "hung" if smoke["hung"] else "failed"
+ if smoke_no_p2p is not None and smoke_no_p2p["ok"]:
+ hints.append(
+ f"The all_reduce smoke test {what}, and passed with NCCL_P2P_DISABLE=1. "
+ "Export NCCL_P2P_DISABLE=1 before launching distributed training here."
+ )
+ else:
+ hints.append(
+ f"The all_reduce smoke test {what}"
+ + (" even with NCCL_P2P_DISABLE=1" if smoke_no_p2p is not None else "")
+ + ". Re-run your job with UNSLOTH_DEBUG_DISTRIBUTED=1 (NCCL_DEBUG=INFO, "
+ "flight recorder, desync debug) to see where it stops."
+ )
+ if topology:
+ nvlinked = all(v["nvlink"] for v in topology.values())
+ far = sorted(
+ k for k, v in topology.items() if not v["nvlink"] and v["level"] in _PCIE_HOST_LEVELS
+ )
+ if far:
+ hints.append(
+ f"GPU pairs {far} talk through the PCIe host bridge or the CPU "
+ "interconnect (PHB / NODE / SYS), with no NVLink. For LoRA prefer DDP: "
+ "it only all-reduces the small adapter gradients, while FSDP / ZeRO-3 "
+ "and tensor parallel move the full weights every step over this link."
+ )
+ elif nvlinked:
+ hints.append("Every GPU pair is NVLinked: FSDP and tensor parallel are viable.")
+ no_p2p = sorted(k for k, v in topology.items() if not v["p2p_read"])
+ if no_p2p and not bad_pairs:
+ hints.append(
+ f"NVML does not confirm P2P reads for {no_p2p} (common on cloud VMs). "
+ "If NCCL is slow there it is staging transfers through host memory."
+ )
+ if healthy and not hints:
+ hints.append("No problems found.")
+ return healthy, hints
+
+
+def _load_pynvml():
+ try:
+ import pynvml
+ return pynvml
+ except Exception:
+ return None
+
+
+def doctor(
+ gpus: bool = typer.Option(
+ True,
+ "--gpus/--no-gpus",
+ help = "Check GPUs, interconnect and a multi-GPU all_reduce (the default check).",
+ ),
+ timeout: float = typer.Option(
+ 60.0,
+ "--timeout",
+ help = "Seconds before the all_reduce smoke test is declared hung.",
+ ),
+):
+ """Preflight checks for this machine. Exit status 1 means a problem was found."""
+ if not gpus:
+ typer.echo("Nothing to check.")
+ raise typer.Exit()
+ try:
+ import torch
+ except ImportError:
+ typer.echo("torch is not installed; nothing to check.")
+ raise typer.Exit(code = 1)
+ except Exception as e:
+ # A missing or mismatched CUDA / DLL dependency surfaces as OSError, not ImportError.
+ typer.echo(f"torch failed to import: {type(e).__name__}: {e}")
+ typer.echo("\n=== Verdict: PROBLEMS FOUND ===")
+ typer.echo("- Reinstall torch with a build matching this driver and CUDA / ROCm runtime.")
+ raise typer.Exit(code = 1)
+
+ try:
+ info = collect_gpus(torch)
+ except Exception as e:
+ # A driver / runtime mismatch or a faulted GPU fails here, after a clean torch import.
+ typer.echo(f"GPU enumeration failed: {type(e).__name__}: {e}")
+ typer.echo("\n=== Verdict: PROBLEMS FOUND ===")
+ typer.echo(
+ "- Check that the driver matches this torch build and that every visible GPU is healthy."
+ )
+ raise typer.Exit(code = 1)
+ pynvml = _load_pynvml()
+ probe_hung = False
+ typer.echo("=== Versions ===")
+ typer.echo(f"torch {info['torch']}")
+ if info["hip"]:
+ typer.echo(f"ROCm (HIP) {info['hip']}")
+ else:
+ typer.echo(f"CUDA (torch build) {info['cuda']}")
+ typer.echo(f"driver {driver_version(pynvml) or 'unknown'}")
+ typer.echo(f"NCCL {info['nccl'] or 'unavailable'}")
+ typer.echo(f"\n=== GPUs ({len(info['gpus'])}) ===")
+ for g in info["gpus"]:
+ typer.echo(
+ f" cuda:{g['index']} {g['name']} sm{g['capability'].replace('.', '')} "
+ f"{g['memory_gib']} GiB {g['pci'] or ''}"
+ )
+
+ topology = p2p = smoke = smoke_no_p2p = None
+ count = len(info["gpus"])
+ if count > 1:
+ topology, topology_error = nvml_topology(pynvml, info["gpus"])
+ typer.echo("\n=== Topology (NVML) ===")
+ if topology is None:
+ typer.echo(f" unavailable: {topology_error}")
+ else:
+ for (a, b), v in sorted(topology.items()):
+ typer.echo(
+ f" {a}<->{b} {v['level']:<4} nvlink={'yes' if v['nvlink'] else 'no'} "
+ f"p2p_read={'yes' if v['p2p_read'] else 'no'}"
+ )
+ typer.echo("\n=== Peer copy integrity ===")
+ if info["hip"]:
+ typer.echo(" skipped on ROCm")
+ else:
+ p2p, probe_hung = p2p_copy_probe_with_deadline(torch, count, timeout)
+ for (a, b), v in sorted(p2p.items()):
+ if probe_hung:
+ typer.echo(f" {a}->{b} {v}")
+ continue
+ peer = "yes" if torch.cuda.can_device_access_peer(a, b) else "no"
+ typer.echo(f" {a}->{b} peer_access={peer} {v}")
+ typer.echo(f"\n=== all_reduce smoke ({count} ranks, nccl) ===")
+ if not torch.distributed.is_nccl_available():
+ # A gloo run would put the tensors on the CPU and prove nothing about the GPUs.
+ smoke = {"ok": False, "hung": False, "unverified": True, "busbw_gbps": None, "log": ""}
+ typer.echo(" skipped: this PyTorch build has no NCCL backend")
+ else:
+ smoke = allreduce_smoke(count, "nccl", timeout)
+ if smoke["ok"]:
+ typer.echo(f" ok busbw {smoke['busbw_gbps']:.2f} GB/s (64 MiB fp32)")
+ else:
+ typer.echo(f" {'HUNG' if smoke['hung'] else 'FAILED'}\n{smoke['log']}")
+ if not _nccl_p2p_disabled():
+ smoke_no_p2p = allreduce_smoke(
+ count, "nccl", timeout, extra_env = {"NCCL_P2P_DISABLE": "1"}
+ )
+ typer.echo(
+ f" retry with NCCL_P2P_DISABLE=1: {'ok' if smoke_no_p2p['ok'] else 'failed'}"
+ )
+
+ healthy, hints = verdict(info, topology, p2p, smoke, smoke_no_p2p)
+ typer.echo(f"\n=== Verdict: {'OK' if healthy else 'PROBLEMS FOUND'} ===")
+ for h in hints:
+ typer.echo(f"- {h}")
+ if probe_hung:
+ # The wedged CUDA call would also hang interpreter shutdown.
+ sys.stdout.flush()
+ os._exit(1)
+ raise typer.Exit(code = 0 if healthy else 1)
diff --git a/unsloth_cli/tests/test_doctor_gpus.py b/unsloth_cli/tests/test_doctor_gpus.py
new file mode 100644
index 00000000000..a63159453b8
--- /dev/null
+++ b/unsloth_cli/tests/test_doctor_gpus.py
@@ -0,0 +1,468 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import sys
+import threading
+from pathlib import Path
+from types import SimpleNamespace
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+import pytest
+import typer
+from typer.testing import CliRunner
+
+import unsloth_cli
+import unsloth_cli.commands.doctor as doctor_mod
+
+
+def _gpu(
+ index,
+ name = "Tesla T4",
+ cap = "7.5",
+ pci = None,
+):
+ return {"index": index, "name": name, "capability": cap, "memory_gib": 15.0, "pci": pci}
+
+
+def _info(*gpus):
+ return {"torch": "2.x", "cuda": "12.4", "hip": None, "nccl": "2.21.5", "gpus": list(gpus)}
+
+
+def _smoke(ok, hung = False):
+ return {"ok": ok, "hung": hung, "busbw_gbps": 5.0 if ok else None, "log": ""}
+
+
+def test_doctor_is_registered_on_the_cli():
+ result = CliRunner().invoke(unsloth_cli.app, ["doctor", "--help"])
+ assert result.exit_code == 0, result.output
+ assert "--gpus" in result.output
+
+
+def test_verdict_no_gpu_and_single_gpu():
+ healthy, hints = doctor_mod.verdict(_info(), None, None, None, None)
+ assert not healthy and "No GPU" in hints[0]
+ healthy, hints = doctor_mod.verdict(_info(_gpu(0)), None, None, None, None)
+ assert healthy and "skipped" in hints[0]
+
+
+def test_verdict_p2p_corruption_recommends_nccl_p2p_disable():
+ p2p = {(0, 1): "corrupt", (1, 0): "ok"}
+ healthy, hints = doctor_mod.verdict(_info(_gpu(0), _gpu(1)), None, p2p, _smoke(True), None)
+ assert not healthy
+ assert any("NCCL_P2P_DISABLE=1" in h and "(0, 1)" in h for h in hints)
+
+
+def test_verdict_smoke_fixed_by_disabling_p2p():
+ healthy, hints = doctor_mod.verdict(
+ _info(_gpu(0), _gpu(1)), None, None, _smoke(False, hung = True), _smoke(True)
+ )
+ assert not healthy
+ assert any("hung" in h and "passed with NCCL_P2P_DISABLE=1" in h for h in hints)
+
+
+def test_verdict_smoke_failing_regardless_points_at_debug_env():
+ healthy, hints = doctor_mod.verdict(
+ _info(_gpu(0), _gpu(1)), None, None, _smoke(False), _smoke(False)
+ )
+ assert not healthy
+ assert any("UNSLOTH_DEBUG_DISTRIBUTED=1" in h for h in hints)
+
+
[email protected]("level", ["PHB", "NODE", "SYS"])
+def test_verdict_pcie_host_topology_prefers_ddp_for_lora(level):
+ topo = {(0, 1): {"level": level, "nvlink": False, "p2p_read": True}}
+ healthy, hints = doctor_mod.verdict(_info(_gpu(0), _gpu(1)), topo, {}, _smoke(True), None)
+ assert healthy
+ assert any("prefer DDP" in h for h in hints)
+
+
+def test_verdict_nvlink_and_mixed_gpus():
+ topo = {(0, 1): {"level": "SYS", "nvlink": True, "p2p_read": True}}
+ healthy, hints = doctor_mod.verdict(
+ _info(_gpu(0), _gpu(1, name = "NVIDIA L4", cap = "8.9")), topo, {}, _smoke(True), None
+ )
+ assert healthy
+ assert not any("prefer DDP" in h for h in hints)
+ assert any("NVLinked" in h for h in hints)
+ assert any("compute capability" in h for h in hints)
+
+
+class _FakeNvml:
+ NVML_P2P_CAPS_INDEX_READ = 0
+ NVML_P2P_CAPS_INDEX_NVLINK = 2
+ NVML_P2P_STATUS_OK = 0
+
+ def __init__(self):
+ self.shutdown = False
+
+ def nvmlInit(self):
+ pass
+
+ def nvmlShutdown(self):
+ self.shutdown = True
+
+ def nvmlDeviceGetHandleByPciBusId(self, bus):
+ return bus.decode()
+
+ def nvmlDeviceGetTopologyCommonAncestor(self, a, b):
+ return 50 if {a, b} == {"bus-a", "bus-c"} else 30
+
+ def nvmlDeviceGetP2PStatus(self, a, b, idx):
+ if idx == self.NVML_P2P_CAPS_INDEX_NVLINK:
+ return 0 if {a, b} == {"bus-a", "bus-b"} else 1
+ return 0
+
+
+def test_nvml_topology_keys_by_torch_index_via_pci_bus():
+ nvml = _FakeNvml()
+ gpus = [_gpu(0, pci = "bus-c"), _gpu(1, pci = "bus-a"), _gpu(2, pci = "bus-b")]
+ topo, err = doctor_mod.nvml_topology(nvml, gpus)
+ assert err is None
+ assert topo[(0, 1)] == {"level": "SYS", "nvlink": False, "p2p_read": True}
+ assert topo[(1, 2)] == {"level": "PHB", "nvlink": True, "p2p_read": True}
+ assert nvml.shutdown
+
+
+def test_nvml_topology_absent_without_pynvml_or_pci():
+ topo, err = doctor_mod.nvml_topology(None, [_gpu(0, pci = "a"), _gpu(1, pci = "b")])
+ assert topo is None and "nvidia-ml-py" in err
+ topo, err = doctor_mod.nvml_topology(_FakeNvml(), [_gpu(0), _gpu(1)])
+ assert topo is None and "PCI" in err
+
+
+def test_nvml_topology_survives_a_refused_query():
+ class Refusing(_FakeNvml):
+ def nvmlDeviceGetP2PStatus(self, a, b, idx):
+ raise RuntimeError("Not Supported")
+
+ topo, err = doctor_mod.nvml_topology(Refusing(), [_gpu(0, pci = "bus-a"), _gpu(1, pci = "bus-c")])
+ assert err is None
+ assert topo[(0, 1)] == {"level": "SYS", "nvlink": False, "p2p_read": False}
+
+
+def test_allreduce_smoke_gloo_two_ranks():
+ pytest.importorskip("torch.distributed")
+ result = doctor_mod.allreduce_smoke(2, "gloo", timeout = 60, numel = 1024)
+ assert result["ok"], result["log"]
+ assert result["busbw_gbps"] is not None
+
+
+def test_allreduce_smoke_reports_a_hang(monkeypatch):
+ monkeypatch.setattr(doctor_mod, "_SMOKE_WORKER", "import time; time.sleep(60)")
+ result = doctor_mod.allreduce_smoke(2, "gloo", timeout = -59, numel = 1024)
+ assert result["hung"] and not result["ok"]
+
+
+def _fake_torch(count):
+ props = SimpleNamespace(
+ name = "Tesla T4", major = 7, minor = 5, total_memory = 15 * 1024**3, pci_bus_id = None
+ )
+ cuda = SimpleNamespace(
+ is_available = lambda: count > 0,
+ device_count = lambda: count,
+ get_device_properties = lambda i: props,
+ can_device_access_peer = lambda a, b: False,
+ nccl = SimpleNamespace(version = lambda: (2, 21, 5)),
+ )
+ return SimpleNamespace(
+ __version__ = "2.x",
+ version = SimpleNamespace(cuda = "12.4", hip = None),
+ cuda = cuda,
+ distributed = SimpleNamespace(is_nccl_available = lambda: True),
+ )
+
+
+def _run_doctor(
+ monkeypatch,
+ torch,
+ smoke_results,
+ p2p_disable = None,
+):
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ monkeypatch.setattr(doctor_mod, "_load_pynvml", lambda: None)
+ monkeypatch.setattr(doctor_mod, "driver_version", lambda pynvml = None: "550.1")
+ monkeypatch.setattr(doctor_mod, "p2p_copy_probe", lambda t, n: {(0, 1): "ok", (1, 0): "ok"})
+ calls = []
+
+ def fake_smoke(
+ world,
+ backend,
+ timeout,
+ extra_env = None,
+ ):
+ calls.append((world, backend, extra_env))
+ return smoke_results[len(calls) - 1]
+
+ monkeypatch.setattr(doctor_mod, "allreduce_smoke", fake_smoke)
+ monkeypatch.delenv("NCCL_P2P_DISABLE", raising = False)
+ if p2p_disable is not None:
+ monkeypatch.setenv("NCCL_P2P_DISABLE", p2p_disable)
+ cli = typer.Typer()
+ cli.command()(doctor_mod.doctor)
+ return CliRunner().invoke(cli, ["--gpus"]), calls
+
+
+def test_doctor_two_gpus_healthy(monkeypatch):
+ result, calls = _run_doctor(monkeypatch, _fake_torch(2), [_smoke(True)])
+ assert result.exit_code == 0, result.output
+ assert calls == [(2, "nccl", None)]
+ assert "driver 550.1" in result.output
+ assert "Verdict: OK" in result.output
+
+
+def test_doctor_retries_smoke_without_p2p(monkeypatch):
+ result, calls = _run_doctor(monkeypatch, _fake_torch(2), [_smoke(False), _smoke(True)])
+ assert result.exit_code == 1, result.output
+ assert calls[1] == (2, "nccl", {"NCCL_P2P_DISABLE": "1"})
+ assert "PROBLEMS FOUND" in result.output
+ assert "passed with NCCL_P2P_DISABLE=1" in result.output
+
+
+def test_doctor_without_gpus_exits_nonzero(monkeypatch):
+ result, calls = _run_doctor(monkeypatch, _fake_torch(0), [])
+ assert result.exit_code == 1 and calls == []
+ assert "No GPU" in result.output
+
+
+def test_doctor_without_nccl_is_unverified_not_a_cpu_gloo_pass(monkeypatch):
+ torch = _fake_torch(2)
+ torch.distributed = SimpleNamespace(is_nccl_available = lambda: False)
+ result, calls = _run_doctor(monkeypatch, torch, [_smoke(True)])
+ assert calls == []
+ assert result.exit_code == 1, result.output
+ assert "no NCCL backend" in result.output and "Verdict: OK" not in result.output
+
+
+def test_p2p_allocation_failure_is_inconclusive_not_corruption():
+ class OomTorch:
+ float32 = "f32"
+
+ @staticmethod
+ def arange(*a, device, **k):
+ if device == "cuda:1":
+ raise RuntimeError("CUDA out of memory")
+ return SimpleNamespace()
+
+ full = arange
+ cuda = SimpleNamespace(empty_cache = lambda: None)
+
+ p2p = doctor_mod.p2p_copy_probe(OomTorch, 2, mib = 1)
+ assert all(v.startswith("inconclusive") for v in p2p.values()), p2p
+ healthy, hints = doctor_mod.verdict(_info(_gpu(0), _gpu(1)), None, p2p, _smoke(True), None)
+ assert healthy
+ assert not any("NCCL_P2P_DISABLE" in h for h in hints)
+ assert any("not tested" in h for h in hints)
+
+
+def test_rocm_wheel_without_version_hip_is_rocm():
+ torch = _fake_torch(0)
+ torch.__version__ = "2.5.1+rocm6.2"
+ torch.version = SimpleNamespace(cuda = None, hip = None)
+ assert doctor_mod.collect_gpus(torch)["hip"] == "6.2"
+
+
+def test_allreduce_smoke_survives_a_chatty_nonzero_rank(monkeypatch, tmp_path):
+ # Rank 1 writes far more than a pipe buffer (64 KiB) before rank 0 can finish.
+ worker = (
+ "import os, sys, time\n"
+ "if os.environ['RANK'] == '1':\n"
+ " sys.stdout.write('x' * (4 << 20)); sys.stdout.flush(); sys.exit(0)\n"
+ "while not os.path.exists(os.environ['DONE']): time.sleep(0.05)\n"
+ "print('UNSLOTH_DOCTOR_RESULT ok=1 busbw_gbps=1.00', flush = True)\n"
+ )
+ done = tmp_path / "rank1_done"
+ monkeypatch.setattr(
+ doctor_mod, "_SMOKE_WORKER", worker.replace("os.environ['DONE']", repr(str(done)))
+ )
+ real_popen = doctor_mod.subprocess.Popen
+ procs = []
+
+ def popen(*a, **k):
+ p = real_popen(*a, **k)
+ procs.append(p)
+ if len(procs) == 2:
+ # Rank 0 only finishes once rank 1 has exited, like a collective would.
+ threading.Thread(target = lambda: (p.wait(), done.touch()), daemon = True).start()
+ return p
+
+ monkeypatch.setattr(doctor_mod.subprocess, "Popen", popen)
+ result = doctor_mod.allreduce_smoke(2, "gloo", timeout = -45, numel = 1024)
+ assert result["ok"] and not result["hung"], result["log"][-300:]
+
+
+def test_driver_version_initialises_nvml():
+ class Nvml:
+ ready = False
+
+ def nvmlInit(self):
+ self.ready = True
+
+ def nvmlShutdown(self):
+ self.ready = False
+
+ def nvmlSystemGetDriverVersion(self):
+ if not self.ready:
+ raise RuntimeError("Uninitialized")
+ return b"580.65"
+
+ nvml = Nvml()
+ assert doctor_mod.driver_version(nvml) == "580.65"
+ assert not nvml.ready
+
+
+def test_nvml_topology_falls_back_to_uuid_without_pci_bus_id():
+ class ByUuid(_FakeNvml):
+ def nvmlDeviceGetHandleByPciBusId(self, bus):
+ raise AssertionError("no PCI id to look up")
+
+ def nvmlDeviceGetHandleByUUID(self, uuid):
+ return {b"GPU-u0": "bus-a", b"GPU-u1": "bus-c"}[uuid]
+
+ torch = _fake_torch(2)
+ props = SimpleNamespace(name = "Tesla T4", major = 7, minor = 5, total_memory = 15 * 1024**3)
+ torch.cuda.get_device_properties = lambda i: SimpleNamespace(**vars(props), uuid = f"u{i}")
+ gpus = doctor_mod.collect_gpus(torch)["gpus"]
+ assert [g["pci"] for g in gpus] == [None, None]
+ topo, err = doctor_mod.nvml_topology(ByUuid(), gpus)
+ assert err is None
+ assert topo[(0, 1)]["level"] == "SYS"
+
+
+def test_allreduce_smoke_kills_started_ranks_when_a_spawn_fails(monkeypatch):
+ real_popen = doctor_mod.subprocess.Popen
+ started = []
+
+ def flaky_popen(*args, **kwargs):
+ if started:
+ raise OSError("Resource temporarily unavailable")
+ started.append(real_popen(*args, **kwargs))
+ return started[-1]
+
+ monkeypatch.setattr(doctor_mod, "_SMOKE_WORKER", "import time; time.sleep(60)")
+ monkeypatch.setattr(doctor_mod.subprocess, "Popen", flaky_popen)
+ result = doctor_mod.allreduce_smoke(2, "gloo", timeout = 60, numel = 1024)
+ assert not result["ok"] and not result["hung"]
+ assert "could not start rank 1" in result["log"]
+ assert started[0].returncode is not None
+
+
+def test_doctor_reports_a_torch_that_fails_to_load(monkeypatch):
+ import builtins
+
+ real_import = builtins.__import__
+
+ def broken(name, *args, **kwargs):
+ if name == "torch":
+ raise OSError("libcudnn.so.9: cannot open shared object file")
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.delitem(sys.modules, "torch", raising = False)
+ monkeypatch.setattr(builtins, "__import__", broken)
+ cli = typer.Typer()
+ cli.command()(doctor_mod.doctor)
+ result = CliRunner().invoke(cli, ["--gpus"])
+ assert result.exit_code == 1
+ assert result.exception is None or isinstance(result.exception, SystemExit)
+ assert "libcudnn" in result.output and "PROBLEMS FOUND" in result.output
+
+
[email protected]("value, retried", [("0", True), ("", True), ("1", False)])
+def test_doctor_retry_honours_the_inherited_nccl_p2p_disable_value(monkeypatch, value, retried):
+ result, calls = _run_doctor(
+ monkeypatch, _fake_torch(2), [_smoke(False), _smoke(True)], p2p_disable = value
+ )
+ assert result.exit_code == 1, result.output
+ assert len(calls) == (2 if retried else 1)
+
+
+def test_nvml_topology_requires_p2p_read_in_both_directions():
+ class OneWay(_FakeNvml):
+ def nvmlDeviceGetP2PStatus(self, a, b, idx):
+ if idx == self.NVML_P2P_CAPS_INDEX_READ:
+ return 0 if (a, b) == ("bus-a", "bus-c") else 1
+ return 1
+
+ gpus = [_gpu(0, pci = "bus-a"), _gpu(1, pci = "bus-c")]
+ topo, err = doctor_mod.nvml_topology(OneWay(), gpus)
+ assert err is None
+ assert topo[(0, 1)]["p2p_read"] is False
+
+
+def test_allreduce_smoke_reaps_ranks_when_the_wait_is_interrupted(monkeypatch):
+ real_popen = doctor_mod.subprocess.Popen
+ started = []
+
+ def popen(*args, **kwargs):
+ started.append(real_popen(*args, **kwargs))
+ return started[-1]
+
+ interrupted = []
+
+ def interrupted_wait(self, timeout = None):
+ if timeout is not None and not interrupted:
+ interrupted.append(True)
+ raise KeyboardInterrupt
+ return real_wait(self, timeout = timeout)
+
+ real_wait = doctor_mod.subprocess.Popen.wait
+ monkeypatch.setattr(doctor_mod, "_SMOKE_WORKER", "import time; time.sleep(60)")
+ monkeypatch.setattr(doctor_mod.subprocess, "Popen", popen)
+ monkeypatch.setattr(real_popen, "wait", interrupted_wait)
+ with pytest.raises(KeyboardInterrupt):
+ doctor_mod.allreduce_smoke(2, "gloo", timeout = 60, numel = 1024)
+ assert len(started) == 2
+ assert all(p.poll() is not None for p in started)
+
+
+def test_peer_copy_probe_has_a_deadline(monkeypatch):
+ release = threading.Event()
+
+ def wedged(torch, count, out, active):
+ out[(0, 1)] = "ok"
+ active.append((0, 2))
+ release.wait(30)
+ return out
+
+ monkeypatch.setattr(doctor_mod, "p2p_copy_probe", wedged)
+ out, hung = doctor_mod.p2p_copy_probe_with_deadline(None, 3, 0.2)
+ release.set()
+ assert hung and len(out) == 6
+ # Finished pairs keep their result; only the wedged pair is an error, the rest untested.
+ assert out[(0, 1)] == "ok"
+ assert out[(0, 2)].startswith("error:") and "hung" in out[(0, 2)]
+ assert all(v.startswith("inconclusive") for k, v in out.items() if k not in ((0, 1), (0, 2)))
+ healthy, hints = doctor_mod.verdict(
+ _info(_gpu(0), _gpu(1), _gpu(2)), None, out, _smoke(True), None
+ )
+ assert not healthy and any("[(0, 2)]" in h and "NCCL_P2P_DISABLE" in h for h in hints)
+
+
+def test_peer_copy_probe_crash_is_inconclusive_not_ok(monkeypatch):
+ def crashing(torch, count, out, active):
+ out[(0, 1)] = "ok"
+ raise RuntimeError("CUDA error: an illegal memory access was encountered")
+
+ monkeypatch.setattr(doctor_mod, "p2p_copy_probe", crashing)
+ out, hung = doctor_mod.p2p_copy_probe_with_deadline(None, 2, 5)
+ assert not hung and out[(0, 1)] == "ok"
+ assert out[(1, 0)].startswith("inconclusive") and "illegal memory access" in out[(1, 0)]
+ _, hints = doctor_mod.verdict(_info(_gpu(0), _gpu(1)), None, out, _smoke(True), None)
+ assert any("not tested" in h and "illegal memory access" in h for h in hints)
+
+
+def test_doctor_reports_a_gpu_enumeration_failure(monkeypatch):
+ torch = _fake_torch(2)
+
+ def fault(i):
+ raise RuntimeError("CUDA error: unspecified launch failure")
+
+ torch.cuda.get_device_properties = fault
+ result, _ = _run_doctor(monkeypatch, torch, [_smoke(True)])
+ assert result.exit_code == 1
+ assert result.exception is None or isinstance(result.exception, SystemExit)
+ assert "GPU enumeration failed" in result.output and "PROBLEMS FOUND" in result.output