Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add `unsloth doctor --gpus` multi-GPU preflight and UNSLOTH_DEBUG_DISTRIBUTED (#13884)
Daniel Hane61d5295606b
diff --git a/tests/test_debug_distributed_env.py b/tests/test_debug_distributed_env.pynew file mode 100644index 00000000000..a96b97b9c27--- /dev/null+++ b/tests/test_debug_distributed_env.py@@ -0,0 +1,85 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import ast+import importlib.util+from pathlib import Path++import pytest++_UNSLOTH = Path(__file__).resolve().parents[1] / "unsloth"+++def _module():+ # By path: importing the unsloth package needs a GPU.+ spec = importlib.util.spec_from_file_location(+ "_unsloth_distributed_debug", _UNSLOTH / "distributed_debug.py"+ )+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ return mod+++def test_off_by_default():+ env = {}+ assert _module().apply_debug_distributed_env(env) == []+ assert env == {}++[email protected]("flag", ["1", "true", "ON"])+def test_sets_nccl_and_both_flight_recorder_names(flag):+ env = {"UNSLOTH_DEBUG_DISTRIBUTED": flag}+ _module().apply_debug_distributed_env(env)+ assert env["TORCH_NCCL_DESYNC_DEBUG"] == "1"+ assert env["TORCH_NCCL_DUMP_ON_TIMEOUT"] == "1"+ assert env["NCCL_DEBUG"] == "INFO"+ # ProcessGroupNCCL reads only the old name for dump on timeout, the recorder the new one.+ assert env["TORCH_FR_BUFFER_SIZE"] == "2000"+ assert env["TORCH_NCCL_TRACE_BUFFER_SIZE"] == "2000"+++def test_user_values_win_and_old_buffer_name_is_respected():+ env = {+ "UNSLOTH_DEBUG_DISTRIBUTED": "1",+ "NCCL_DEBUG": "WARN",+ "TORCH_NCCL_TRACE_BUFFER_SIZE": "0",+ }+ changed = _module().apply_debug_distributed_env(env)+ assert env["NCCL_DEBUG"] == "WARN"+ assert env["TORCH_NCCL_TRACE_BUFFER_SIZE"] == "0"+ assert env["TORCH_FR_BUFFER_SIZE"] == "0"+ assert set(changed) == {+ "TORCH_NCCL_DESYNC_DEBUG",+ "TORCH_NCCL_DUMP_ON_TIMEOUT",+ "TORCH_FR_BUFFER_SIZE",+ }+++def test_new_buffer_name_is_mirrored_to_the_one_process_group_nccl_reads():+ env = {"UNSLOTH_DEBUG_DISTRIBUTED": "1", "TORCH_FR_BUFFER_SIZE": "5000"}+ _module().apply_debug_distributed_env(env)+ assert env["TORCH_NCCL_TRACE_BUFFER_SIZE"] == "5000"+++def test_hook_runs_before_torch_is_imported():+ tree = ast.parse((_UNSLOTH / "__init__.py").read_text(encoding = "utf-8"))+ hook = first_torch = None+ for node in tree.body:+ src = ast.unparse(node)+ if hook is None and "apply_debug_distributed_env" in src:+ hook = node.lineno+ if first_torch is None and ("import torch" in src or "from torch" in src):+ first_torch = node.lineno+ assert hook is not None and (first_torch is None or hook < first_torch)+++def test_existing_process_group_leaves_the_environment_untouched(monkeypatch, capsys):+ import sys+ import types++ dist = types.SimpleNamespace(is_available = lambda: True, is_initialized = lambda: True)+ monkeypatch.setitem(sys.modules, "torch.distributed", dist)+ env = {"UNSLOTH_DEBUG_DISTRIBUTED": "1"}+ assert _module().apply_debug_distributed_env(env) == []+ assert env == {"UNSLOTH_DEBUG_DISTRIBUTED": "1"}+ assert "already exists" in capsys.readouterr().errdiff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex 3d4cedda2a9..2a1e70f35f7 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -18,6 +18,12 @@ os.environ["UNSLOTH_IS_PRESENT"] = "1"# an explicit override, including "0".os.environ.setdefault("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "1")+# Before anything can create a process group: c10d and NCCL read these at init.+if os.environ.get("UNSLOTH_DEBUG_DISTRIBUTED"):+ from .distributed_debug import apply_debug_distributed_env as _apply_debug_distributed_env+ _apply_debug_distributed_env()+ del _apply_debug_distributed_env+# UNSLOTH_FSDP2=1 under plain torchrun: export accelerate's FSDP2 launch env before transformers or# accelerate read it. A conflicting launch (DeepSpeed, FSDP_VERSION=1) raises here, on every rank.from .fsdp2_launch import apply_unsloth_fsdp2_launch_env as _apply_fsdp2_envdiff --git a/unsloth/distributed_debug.py b/unsloth/distributed_debug.pynew file mode 100644index 00000000000..b539b7e2c1c--- /dev/null+++ b/unsloth/distributed_debug.py@@ -0,0 +1,58 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import os+import sys++__all__ = ["apply_debug_distributed_env"]++# Read at process group / communicator creation, so they only need to precede init_process_group.+_DEBUG_DISTRIBUTED_ENV = (+ ("TORCH_NCCL_DESYNC_DEBUG", "1"),+ ("TORCH_NCCL_DUMP_ON_TIMEOUT", "1"),+ ("NCCL_DEBUG", "INFO"),+)+# Flight recorder ring size. Both names are needed: the recorder reads TORCH_FR_BUFFER_SIZE first+# (torch >= 2.8), but ProcessGroupNCCL gates dump on timeout on TORCH_NCCL_TRACE_BUFFER_SIZE alone+# (ProcessGroupNCCL.hpp, through 2.13), the only name torch 2.6 / 2.7 know. Costs torch >= 2.8 one+# deprecation warning.+_FR_BUFFER_NAMES = ("TORCH_FR_BUFFER_SIZE", "TORCH_NCCL_TRACE_BUFFER_SIZE")+_FR_BUFFER_SIZE = "2000"+++def apply_debug_distributed_env(environ = None):+ """Under UNSLOTH_DEBUG_DISTRIBUTED=1, enable NCCL / c10d hang diagnostics.++ Variables the user already set are left alone. The two flight recorder names are+ one setting: a value given under either is copied to the other, so an explicit+ TORCH_NCCL_TRACE_BUFFER_SIZE=0 is not overridden. Returns the names that were set.+ """+ environ = os.environ if environ is None else environ+ if environ.get("UNSLOTH_DEBUG_DISTRIBUTED", "0").lower() not in ("1", "true", "yes", "on"):+ return []+ dist = sys.modules.get("torch.distributed")+ try:+ initialized = dist is not None and dist.is_available() and dist.is_initialized()+ except Exception:+ initialized = False+ if initialized:+ # Too late to take effect, and ProcessGroupNCCL threads read the environment+ # concurrently (setenv races getenv), so leave it untouched.+ print(+ "Unsloth: UNSLOTH_DEBUG_DISTRIBUTED=1 is set but the process group already exists, "+ "so it was not applied. Import unsloth before init_process_group, or export "+ "the variables in the launcher.",+ file = sys.stderr,+ )+ return []+ changed = []+ for name, value in _DEBUG_DISTRIBUTED_ENV:+ if name not in environ:+ environ[name] = value+ changed.append(name)+ given = [environ[name] for name in _FR_BUFFER_NAMES if name in environ]+ for name in _FR_BUFFER_NAMES:+ if name not in environ:+ environ[name] = given[0] if given else _FR_BUFFER_SIZE+ changed.append(name)+ return changeddiff --git a/unsloth_cli/__init__.py b/unsloth_cli/__init__.pyindex 809e82f3482..2a351b1088b 100644--- a/unsloth_cli/__init__.py+++ b/unsloth_cli/__init__.py@@ -100,6 +100,7 @@ else:from unsloth_cli.commands.start import start_appfrom unsloth_cli.commands.export import export, list_checkpointsfrom unsloth_cli.commands.eval import evaluate as eval_command+ from unsloth_cli.commands.doctor import doctorfrom unsloth_cli.commands.studio import (run as studio_run,studio_app,@@ -235,6 +236,7 @@ if not _windows_studio_mutation_entry:app.command()(export)app.command("eval")(eval_command)app.command("list-checkpoints")(list_checkpoints)+ app.command("doctor")(doctor)app.add_typer(start_app,name = "start",diff --git a/unsloth_cli/commands/doctor.py b/unsloth_cli/commands/doctor.pynew file mode 100644index 00000000000..6de9be9c80a--- /dev/null+++ b/unsloth_cli/commands/doctor.py@@ -0,0 +1,583 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""`unsloth doctor --gpus`: a multi-GPU preflight to run before a distributed job.++Reports the GPUs, versions and interconnect, checks that peer copies keep their+data, runs a short all_reduce across every visible GPU, and ends with a verdict+and the settings to try. torch is imported lazily so `unsloth --help` stays fast.+"""++import os+import shutil+import socket+import subprocess+import sys+import tempfile+import time+from typing import List, Optional, Tuple++import typer++# NVML topology levels (nvmlGpuTopologyLevel_t), closest first.+_TOPOLOGY_LABELS = {0: "SAME", 10: "PIX", 20: "PXB", 30: "PHB", 40: "NODE", 50: "SYS"}+_PCIE_HOST_LEVELS = {"PHB", "NODE", "SYS"}+_SENTINEL = -7.0++_SMOKE_WORKER = r"""+import datetime, os, sys, time+import torch+import torch.distributed as dist++rank = int(os.environ["RANK"])+world = int(os.environ["WORLD_SIZE"])+backend = os.environ["UNSLOTH_DOCTOR_BACKEND"]+if backend == "nccl":+ torch.cuda.set_device(rank)+ device = torch.device("cuda", rank)+else:+ device = torch.device("cpu")+dist.init_process_group(+ backend,+ timeout = datetime.timedelta(seconds = float(os.environ["UNSLOTH_DOCTOR_TIMEOUT"])),+)+numel = int(os.environ["UNSLOTH_DOCTOR_NUMEL"])+x = torch.full((numel,), float(rank + 1), device = device)+expected = world * (world + 1) / 2+ok = True+dist.all_reduce(x)+ok = ok and bool((x == expected).all())+iters = 5+dist.barrier()+if device.type == "cuda":+ torch.cuda.synchronize()+start = time.perf_counter()+for _ in range(iters):+ x.fill_(1.0)+ dist.all_reduce(x)+if device.type == "cuda":+ torch.cuda.synchronize()+elapsed = (time.perf_counter() - start) / iters+ok = ok and bool((x == world).all())+# nccl-tests busbw convention for all_reduce: algbw * 2 (n - 1) / n.+busbw = numel * x.element_size() / elapsed * 2 * (world - 1) / world / 1e9+if rank == 0:+ print(f"UNSLOTH_DOCTOR_RESULT ok={int(ok)} busbw_gbps={busbw:.2f}", flush = True)+dist.destroy_process_group()+sys.exit(0 if ok else 3)+"""+++def _free_port() -> int:+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def collect_gpus(torch) -> dict:+ hip = getattr(torch.version, "hip", None)+ # AMD SDK wheels leave version.hip unset and only say "rocm" in __version__.+ if hip is None and "rocm" in str(torch.__version__).lower():+ hip = str(torch.__version__).lower().split("rocm", 1)[1] or "unknown"+ info = {+ "torch": torch.__version__,+ "cuda": getattr(torch.version, "cuda", None),+ "hip": hip,+ "nccl": None,+ "gpus": [],+ }+ if not torch.cuda.is_available():+ return info+ try:+ nccl = torch.cuda.nccl.version()+ info["nccl"] = ".".join(str(v) for v in nccl) if isinstance(nccl, tuple) else str(nccl)+ except Exception:+ pass+ for i in range(torch.cuda.device_count()):+ p = torch.cuda.get_device_properties(i)+ pci = None+ if getattr(p, "pci_bus_id", None) is not None:+ pci = (+ f"{getattr(p, 'pci_domain_id', 0):08x}:{p.pci_bus_id:02x}:"+ f"{getattr(p, 'pci_device_id', 0):02x}.0"+ )+ # torch < 2.8 has no pci_bus_id; the UUID still identifies the card to NVML.+ uuid = getattr(p, "uuid", None)+ uuid = str(uuid) if uuid is not None and not hip else None+ info["gpus"].append(+ {+ "index": i,+ "name": p.name,+ "capability": f"{p.major}.{p.minor}",+ "memory_gib": round(p.total_memory / 1024**3, 1),+ "pci": pci,+ "uuid": uuid,+ }+ )+ return info+++def driver_version(pynvml = None) -> Optional[str]:+ if pynvml is not None:+ try:+ pynvml.nvmlInit()+ try:+ v = pynvml.nvmlSystemGetDriverVersion()+ return v.decode() if isinstance(v, bytes) else str(v)+ finally:+ pynvml.nvmlShutdown()+ except Exception:+ pass+ if shutil.which("nvidia-smi") is None:+ return None+ try:+ out = subprocess.run(+ ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"],+ capture_output = True,+ text = True,+ timeout = 20,+ )+ lines = out.stdout.strip().splitlines()+ return lines[0].strip() if out.returncode == 0 and lines else None+ except Exception:+ return None+++def nvml_topology(pynvml, gpus: List[dict]):+ """({(a, b): {"level": "PHB", "nvlink": bool, "p2p_read": bool}} keyed by torch index, error).++ Handles are looked up by PCI bus id (else device UUID), since NVML numbers GPUs in+ PCI order and ignores CUDA_VISIBLE_DEVICES / CUDA_DEVICE_ORDER, which renumber them.+ A query a virtualised GPU refuses degrades that field instead of the whole table.+ """+ if pynvml is None:+ return None, "nvidia-ml-py is not installed"+ if len(gpus) < 2 or any(g["pci"] is None and not g.get("uuid") for g in gpus):+ return None, "torch reports neither PCI bus ids nor device UUIDs"+ read_idx = getattr(pynvml, "NVML_P2P_CAPS_INDEX_READ", 0)+ nvlink_idx = getattr(pynvml, "NVML_P2P_CAPS_INDEX_NVLINK", 2)+ status_ok = getattr(pynvml, "NVML_P2P_STATUS_OK", 0)++ def query(fn, *args):+ try:+ return fn(*args)+ except Exception:+ return None++ try:+ pynvml.nvmlInit()+ except Exception as e:+ return None, f"nvmlInit failed: {e}"+ try:+ handles = []+ for g in gpus:+ if g["pci"] is not None:+ handles.append(pynvml.nvmlDeviceGetHandleByPciBusId(g["pci"].encode()))+ else:+ uuid = g["uuid"]+ if not uuid.startswith(("GPU-", "MIG-")):+ uuid = "GPU-" + uuid+ handles.append(pynvml.nvmlDeviceGetHandleByUUID(uuid.encode()))+ out = {}+ for a in range(len(gpus)):+ for b in range(a + 1, len(gpus)):+ ha, hb = handles[a], handles[b]+ level = query(pynvml.nvmlDeviceGetTopologyCommonAncestor, ha, hb)+ out[(a, b)] = {+ "level": "?" if level is None else _TOPOLOGY_LABELS.get(int(level), str(level)),+ "nvlink": query(pynvml.nvmlDeviceGetP2PStatus, ha, hb, nvlink_idx) == status_ok,+ # Directional: a virtualised or IOMMU-restricted host can grant one way only.+ "p2p_read": all(+ query(pynvml.nvmlDeviceGetP2PStatus, x, y, read_idx) == status_ok+ for x, y in ((ha, hb), (hb, ha))+ ),+ }+ return out, None+ except Exception as e:+ return None, f"NVML query failed: {type(e).__name__}: {e}"+ finally:+ try:+ pynvml.nvmlShutdown()+ except Exception:+ pass+++def p2p_copy_probe_with_deadline(torch, count: int, timeout: float):+ """p2p_copy_probe in a daemon thread: a wedged peer copy cannot hold up the verdict.+ Returns (results, hung)."""+ import threading++ out = {}+ active = []+ crashed = []++ def run():+ try:+ p2p_copy_probe(torch, count, out = out, active = active)+ except Exception as e:+ crashed.append(f"{type(e).__name__}: {e}")++ worker = threading.Thread(target = run, daemon = True)+ worker.start()+ worker.join(timeout)+ hung = worker.is_alive()+ # Snapshot: a hung worker may still write. Pairs finished before the deadline keep their result.+ results = dict(out)+ for a in range(count):+ for b in range(count):+ if a == b or (a, b) in results:+ continue+ if hung and active and active[-1] == (a, b):+ results[(a, b)] = f"error: peer copy did not finish within {timeout:.0f}s (hung)"+ elif hung:+ results[(a, b)] = "inconclusive: not reached before the probe deadline"+ elif crashed:+ results[(a, b)] = f"inconclusive: probe stopped: {crashed[0]}"+ return results, hung+++def p2p_copy_probe(+ torch,+ count: int,+ mib: int = 16,+ out: Optional[dict] = None,+ active: Optional[list] = None,+) -> dict:+ """{(src, dst): "ok" | "corrupt" | "error: ..." | "inconclusive: ..."} per ordered GPU pair.++ The destination is sentinel-filled first: on a translating IOMMU a PCIe peer copy+ can be dropped while CUDA reports success (scripts/p2p_integrity_probe.py).+ """+ out = {} if out is None else out+ numel = mib * 1024 * 1024 // 4+ for src in range(count):+ for dst in range(count):+ if src == dst:+ continue+ if active is not None:+ active.append((src, dst))+ s = d = None+ try:+ s = torch.arange(numel, dtype = torch.float32, device = f"cuda:{src}")+ d = torch.full((numel,), _SENTINEL, dtype = torch.float32, device = f"cuda:{dst}")+ except Exception as e:+ # No copy was attempted, so this says nothing about peer copies.+ out[(src, dst)] = f"inconclusive: could not allocate: {e}"+ del s, d+ continue+ try:+ d.copy_(s)+ torch.cuda.synchronize(src)+ torch.cuda.synchronize(dst)+ out[(src, dst)] = "ok" if torch.equal(d.cpu(), s.cpu()) else "corrupt"+ del s, d+ except Exception as e:+ out[(src, dst)] = f"error: {e}"+ torch.cuda.empty_cache()+ return out+++def _nccl_p2p_disabled() -> bool:+ """True when the inherited NCCL_P2P_DISABLE already turns P2P off.++ NCCL's ncclGetLevel (src/graph/paths.cc) parses it with strtol and disables only on 1.+ """+ try:+ return int(os.environ.get("NCCL_P2P_DISABLE", "").strip() or "0", 0) == 1+ except ValueError:+ return False+++def allreduce_smoke(+ world: int,+ backend: str,+ timeout: float,+ extra_env: Optional[dict] = None,+ numel: int = 16 * 1024 * 1024,+) -> dict:+ """Run one all_reduce over `world` local ranks in fresh processes.++ Separate interpreters rather than threads: a hung NCCL communicator can only be+ recovered by killing its process, and the doctor must answer instead of hanging.+ """+ env = dict(os.environ)+ env.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(_free_port()),+ WORLD_SIZE = str(world),+ UNSLOTH_DOCTOR_BACKEND = backend,+ UNSLOTH_DOCTOR_TIMEOUT = str(timeout),+ UNSLOTH_DOCTOR_NUMEL = str(numel),+ )+ env.update(extra_env or {})+ procs = []+ logs = []+ try:+ # Files, not pipes: an undrained pipe (e.g. NCCL_DEBUG=INFO on rank 1) blocks that+ # rank inside the collective and the doctor would report a healthy system as hung.+ for rank in range(world):+ logs.append(tempfile.TemporaryFile(mode = "w+"))+ rank_env = dict(env, RANK = str(rank), LOCAL_RANK = str(rank))+ procs.append(+ subprocess.Popen(+ [sys.executable, "-c", _SMOKE_WORKER],+ env = rank_env,+ stdout = logs[rank],+ stderr = subprocess.STDOUT,+ text = True,+ )+ )+ except Exception as e:+ # Started ranks would wait in init_process_group for ranks that never exist.+ for q in procs:+ q.kill()+ q.wait()+ for f in logs:+ f.close()+ return {+ "ok": False,+ "hung": False,+ "busbw_gbps": None,+ "log": f"could not start rank {len(procs)} of {world}: {type(e).__name__}: {e}",+ }+ # Startup (torch import, communicator init) is not covered by the collective timeout.+ deadline = time.monotonic() + timeout + 60+ hung = False+ outputs = []+ try:+ for p in procs:+ try:+ p.wait(timeout = max(1.0, deadline - time.monotonic()))+ except subprocess.TimeoutExpired:+ hung = True+ for q in procs:+ q.kill()+ break+ for f in logs:+ f.seek(0)+ outputs.append(f.read())+ finally:+ # Ctrl-C on a hung run must not leave ranks holding GPU contexts behind. Bounded:+ # a rank stuck in an uninterruptible driver call may never be reaped.+ for q in procs:+ if q.poll() is None:+ q.kill()+ for q in procs:+ try:+ q.wait(timeout = 10)+ except subprocess.TimeoutExpired:+ hung = True+ for f in logs:+ f.close()+ result = {"ok": False, "hung": hung, "busbw_gbps": None, "log": ""}+ for line in outputs[0].splitlines():+ if line.startswith("UNSLOTH_DOCTOR_RESULT"):+ fields = dict(kv.split("=", 1) for kv in line.split()[1:])+ result["busbw_gbps"] = float(fields["busbw_gbps"])+ result["ok"] = fields["ok"] == "1"+ result["ok"] = result["ok"] and not hung and all(p.returncode == 0 for p in procs)+ if not result["ok"]:+ result["log"] = "\n".join(o[-2000:] for o in outputs)+ return result+++def verdict(+ info: dict,+ topology: Optional[dict],+ p2p: Optional[dict],+ smoke: Optional[dict],+ smoke_no_p2p: Optional[dict],+) -> Tuple[bool, List[str]]:+ """(healthy, hints). Pure, so every branch is testable without GPUs."""+ hints = []+ gpus = info["gpus"]+ if not gpus:+ return False, [+ "No GPU is visible to torch. Check `nvidia-smi`, CUDA_VISIBLE_DEVICES, and "+ "that torch was installed with CUDA or ROCm support."+ ]+ if len(gpus) == 1:+ return True, ["One GPU visible: multi-GPU checks skipped."]+ healthy = True+ if len({g["capability"] for g in gpus}) > 1 or len({g["name"] for g in gpus}) > 1:+ hints.append(+ "The GPUs differ in model or compute capability. Distributed training runs at "+ "the pace of the slowest card, and bf16 / FlashAttention need every card to "+ "support them."+ )+ bad_pairs = sorted(k for k, v in (p2p or {}).items() if v == "corrupt" or v.startswith("error"))+ unchecked = sorted(k for k, v in (p2p or {}).items() if v.startswith("inconclusive"))+ if bad_pairs:+ healthy = False+ hints.append(+ f"Peer-to-peer copies lost data or failed on {bad_pairs}. Set "+ "NCCL_P2P_DISABLE=1 for every distributed run on this host, and check the "+ "kernel log for IOMMU faults: dmesg | grep -i -E 'DMAR|AMD-Vi'."+ )+ if unchecked:+ reason = p2p[unchecked[0]][len("inconclusive: ") :]+ hints.append(+ f"Peer copies on {unchecked} were not tested ({reason}). Free GPU memory and "+ "re-run `unsloth doctor --gpus`."+ )+ if smoke is not None and smoke.get("unverified"):+ healthy = False+ hints.append(+ "This PyTorch build has no NCCL backend (for example on Windows), so GPU "+ "collectives could not be tested and multi-GPU training here is unverified."+ )+ elif smoke is not None and not smoke["ok"]:+ healthy = False+ what = "hung" if smoke["hung"] else "failed"+ if smoke_no_p2p is not None and smoke_no_p2p["ok"]:+ hints.append(+ f"The all_reduce smoke test {what}, and passed with NCCL_P2P_DISABLE=1. "+ "Export NCCL_P2P_DISABLE=1 before launching distributed training here."+ )+ else:+ hints.append(+ f"The all_reduce smoke test {what}"+ + (" even with NCCL_P2P_DISABLE=1" if smoke_no_p2p is not None else "")+ + ". Re-run your job with UNSLOTH_DEBUG_DISTRIBUTED=1 (NCCL_DEBUG=INFO, "+ "flight recorder, desync debug) to see where it stops."+ )+ if topology:+ nvlinked = all(v["nvlink"] for v in topology.values())+ far = sorted(+ k for k, v in topology.items() if not v["nvlink"] and v["level"] in _PCIE_HOST_LEVELS+ )+ if far:+ hints.append(+ f"GPU pairs {far} talk through the PCIe host bridge or the CPU "+ "interconnect (PHB / NODE / SYS), with no NVLink. For LoRA prefer DDP: "+ "it only all-reduces the small adapter gradients, while FSDP / ZeRO-3 "+ "and tensor parallel move the full weights every step over this link."+ )+ elif nvlinked:+ hints.append("Every GPU pair is NVLinked: FSDP and tensor parallel are viable.")+ no_p2p = sorted(k for k, v in topology.items() if not v["p2p_read"])+ if no_p2p and not bad_pairs:+ hints.append(+ f"NVML does not confirm P2P reads for {no_p2p} (common on cloud VMs). "+ "If NCCL is slow there it is staging transfers through host memory."+ )+ if healthy and not hints:+ hints.append("No problems found.")+ return healthy, hints+++def _load_pynvml():+ try:+ import pynvml+ return pynvml+ except Exception:+ return None+++def doctor(+ gpus: bool = typer.Option(+ True,+ "--gpus/--no-gpus",+ help = "Check GPUs, interconnect and a multi-GPU all_reduce (the default check).",+ ),+ timeout: float = typer.Option(+ 60.0,+ "--timeout",+ help = "Seconds before the all_reduce smoke test is declared hung.",+ ),+):+ """Preflight checks for this machine. Exit status 1 means a problem was found."""+ if not gpus:+ typer.echo("Nothing to check.")+ raise typer.Exit()+ try:+ import torch+ except ImportError:+ typer.echo("torch is not installed; nothing to check.")+ raise typer.Exit(code = 1)+ except Exception as e:+ # A missing or mismatched CUDA / DLL dependency surfaces as OSError, not ImportError.+ typer.echo(f"torch failed to import: {type(e).__name__}: {e}")+ typer.echo("\n=== Verdict: PROBLEMS FOUND ===")+ typer.echo("- Reinstall torch with a build matching this driver and CUDA / ROCm runtime.")+ raise typer.Exit(code = 1)++ try:+ info = collect_gpus(torch)+ except Exception as e:+ # A driver / runtime mismatch or a faulted GPU fails here, after a clean torch import.+ typer.echo(f"GPU enumeration failed: {type(e).__name__}: {e}")+ typer.echo("\n=== Verdict: PROBLEMS FOUND ===")+ typer.echo(+ "- Check that the driver matches this torch build and that every visible GPU is healthy."+ )+ raise typer.Exit(code = 1)+ pynvml = _load_pynvml()+ probe_hung = False+ typer.echo("=== Versions ===")+ typer.echo(f"torch {info['torch']}")+ if info["hip"]:+ typer.echo(f"ROCm (HIP) {info['hip']}")+ else:+ typer.echo(f"CUDA (torch build) {info['cuda']}")+ typer.echo(f"driver {driver_version(pynvml) or 'unknown'}")+ typer.echo(f"NCCL {info['nccl'] or 'unavailable'}")+ typer.echo(f"\n=== GPUs ({len(info['gpus'])}) ===")+ for g in info["gpus"]:+ typer.echo(+ f" cuda:{g['index']} {g['name']} sm{g['capability'].replace('.', '')} "+ f"{g['memory_gib']} GiB {g['pci'] or ''}"+ )++ topology = p2p = smoke = smoke_no_p2p = None+ count = len(info["gpus"])+ if count > 1:+ topology, topology_error = nvml_topology(pynvml, info["gpus"])+ typer.echo("\n=== Topology (NVML) ===")+ if topology is None:+ typer.echo(f" unavailable: {topology_error}")+ else:+ for (a, b), v in sorted(topology.items()):+ typer.echo(+ f" {a}<->{b} {v['level']:<4} nvlink={'yes' if v['nvlink'] else 'no'} "+ f"p2p_read={'yes' if v['p2p_read'] else 'no'}"+ )+ typer.echo("\n=== Peer copy integrity ===")+ if info["hip"]:+ typer.echo(" skipped on ROCm")+ else:+ p2p, probe_hung = p2p_copy_probe_with_deadline(torch, count, timeout)+ for (a, b), v in sorted(p2p.items()):+ if probe_hung:+ typer.echo(f" {a}->{b} {v}")+ continue+ peer = "yes" if torch.cuda.can_device_access_peer(a, b) else "no"+ typer.echo(f" {a}->{b} peer_access={peer} {v}")+ typer.echo(f"\n=== all_reduce smoke ({count} ranks, nccl) ===")+ if not torch.distributed.is_nccl_available():+ # A gloo run would put the tensors on the CPU and prove nothing about the GPUs.+ smoke = {"ok": False, "hung": False, "unverified": True, "busbw_gbps": None, "log": ""}+ typer.echo(" skipped: this PyTorch build has no NCCL backend")+ else:+ smoke = allreduce_smoke(count, "nccl", timeout)+ if smoke["ok"]:+ typer.echo(f" ok busbw {smoke['busbw_gbps']:.2f} GB/s (64 MiB fp32)")+ else:+ typer.echo(f" {'HUNG' if smoke['hung'] else 'FAILED'}\n{smoke['log']}")+ if not _nccl_p2p_disabled():+ smoke_no_p2p = allreduce_smoke(+ count, "nccl", timeout, extra_env = {"NCCL_P2P_DISABLE": "1"}+ )+ typer.echo(+ f" retry with NCCL_P2P_DISABLE=1: {'ok' if smoke_no_p2p['ok'] else 'failed'}"+ )++ healthy, hints = verdict(info, topology, p2p, smoke, smoke_no_p2p)+ typer.echo(f"\n=== Verdict: {'OK' if healthy else 'PROBLEMS FOUND'} ===")+ for h in hints:+ typer.echo(f"- {h}")+ if probe_hung:+ # The wedged CUDA call would also hang interpreter shutdown.+ sys.stdout.flush()+ os._exit(1)+ raise typer.Exit(code = 0 if healthy else 1)diff --git a/unsloth_cli/tests/test_doctor_gpus.py b/unsloth_cli/tests/test_doctor_gpus.pynew file mode 100644index 00000000000..a63159453b8--- /dev/null+++ b/unsloth_cli/tests/test_doctor_gpus.py@@ -0,0 +1,468 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import sys+import threading+from pathlib import Path+from types import SimpleNamespace++_REPO_ROOT = Path(__file__).resolve().parents[2]+if str(_REPO_ROOT) not in sys.path:+ sys.path.insert(0, str(_REPO_ROOT))++import pytest+import typer+from typer.testing import CliRunner++import unsloth_cli+import unsloth_cli.commands.doctor as doctor_mod+++def _gpu(+ index,+ name = "Tesla T4",+ cap = "7.5",+ pci = None,+):+ return {"index": index, "name": name, "capability": cap, "memory_gib": 15.0, "pci": pci}+++def _info(*gpus):+ return {"torch": "2.x", "cuda": "12.4", "hip": None, "nccl": "2.21.5", "gpus": list(gpus)}+++def _smoke(ok, hung = False):+ return {"ok": ok, "hung": hung, "busbw_gbps": 5.0 if ok else None, "log": ""}+++def test_doctor_is_registered_on_the_cli():+ result = CliRunner().invoke(unsloth_cli.app, ["doctor", "--help"])+ assert result.exit_code == 0, result.output+ assert "--gpus" in result.output+++def test_verdict_no_gpu_and_single_gpu():+ healthy, hints = doctor_mod.verdict(_info(), None, None, None, None)+ assert not healthy and "No GPU" in hints[0]+ healthy, hints = doctor_mod.verdict(_info(_gpu(0)), None, None, None, None)+ assert healthy and "skipped" in hints[0]+++def test_verdict_p2p_corruption_recommends_nccl_p2p_disable():+ p2p = {(0, 1): "corrupt", (1, 0): "ok"}+ healthy, hints = doctor_mod.verdict(_info(_gpu(0), _gpu(1)), None, p2p, _smoke(True), None)+ assert not healthy+ assert any("NCCL_P2P_DISABLE=1" in h and "(0, 1)" in h for h in hints)+++def test_verdict_smoke_fixed_by_disabling_p2p():+ healthy, hints = doctor_mod.verdict(+ _info(_gpu(0), _gpu(1)), None, None, _smoke(False, hung = True), _smoke(True)+ )+ assert not healthy+ assert any("hung" in h and "passed with NCCL_P2P_DISABLE=1" in h for h in hints)+++def test_verdict_smoke_failing_regardless_points_at_debug_env():+ healthy, hints = doctor_mod.verdict(+ _info(_gpu(0), _gpu(1)), None, None, _smoke(False), _smoke(False)+ )+ assert not healthy+ assert any("UNSLOTH_DEBUG_DISTRIBUTED=1" in h for h in hints)++[email protected]("level", ["PHB", "NODE", "SYS"])+def test_verdict_pcie_host_topology_prefers_ddp_for_lora(level):+ topo = {(0, 1): {"level": level, "nvlink": False, "p2p_read": True}}+ healthy, hints = doctor_mod.verdict(_info(_gpu(0), _gpu(1)), topo, {}, _smoke(True), None)+ assert healthy+ assert any("prefer DDP" in h for h in hints)+++def test_verdict_nvlink_and_mixed_gpus():+ topo = {(0, 1): {"level": "SYS", "nvlink": True, "p2p_read": True}}+ healthy, hints = doctor_mod.verdict(+ _info(_gpu(0), _gpu(1, name = "NVIDIA L4", cap = "8.9")), topo, {}, _smoke(True), None+ )+ assert healthy+ assert not any("prefer DDP" in h for h in hints)+ assert any("NVLinked" in h for h in hints)+ assert any("compute capability" in h for h in hints)+++class _FakeNvml:+ NVML_P2P_CAPS_INDEX_READ = 0+ NVML_P2P_CAPS_INDEX_NVLINK = 2+ NVML_P2P_STATUS_OK = 0++ def __init__(self):+ self.shutdown = False++ def nvmlInit(self):+ pass++ def nvmlShutdown(self):+ self.shutdown = True++ def nvmlDeviceGetHandleByPciBusId(self, bus):+ return bus.decode()++ def nvmlDeviceGetTopologyCommonAncestor(self, a, b):+ return 50 if {a, b} == {"bus-a", "bus-c"} else 30++ def nvmlDeviceGetP2PStatus(self, a, b, idx):+ if idx == self.NVML_P2P_CAPS_INDEX_NVLINK:+ return 0 if {a, b} == {"bus-a", "bus-b"} else 1+ return 0+++def test_nvml_topology_keys_by_torch_index_via_pci_bus():+ nvml = _FakeNvml()+ gpus = [_gpu(0, pci = "bus-c"), _gpu(1, pci = "bus-a"), _gpu(2, pci = "bus-b")]+ topo, err = doctor_mod.nvml_topology(nvml, gpus)+ assert err is None+ assert topo[(0, 1)] == {"level": "SYS", "nvlink": False, "p2p_read": True}+ assert topo[(1, 2)] == {"level": "PHB", "nvlink": True, "p2p_read": True}+ assert nvml.shutdown+++def test_nvml_topology_absent_without_pynvml_or_pci():+ topo, err = doctor_mod.nvml_topology(None, [_gpu(0, pci = "a"), _gpu(1, pci = "b")])+ assert topo is None and "nvidia-ml-py" in err+ topo, err = doctor_mod.nvml_topology(_FakeNvml(), [_gpu(0), _gpu(1)])+ assert topo is None and "PCI" in err+++def test_nvml_topology_survives_a_refused_query():+ class Refusing(_FakeNvml):+ def nvmlDeviceGetP2PStatus(self, a, b, idx):+ raise RuntimeError("Not Supported")++ topo, err = doctor_mod.nvml_topology(Refusing(), [_gpu(0, pci = "bus-a"), _gpu(1, pci = "bus-c")])+ assert err is None+ assert topo[(0, 1)] == {"level": "SYS", "nvlink": False, "p2p_read": False}+++def test_allreduce_smoke_gloo_two_ranks():+ pytest.importorskip("torch.distributed")+ result = doctor_mod.allreduce_smoke(2, "gloo", timeout = 60, numel = 1024)+ assert result["ok"], result["log"]+ assert result["busbw_gbps"] is not None+++def test_allreduce_smoke_reports_a_hang(monkeypatch):+ monkeypatch.setattr(doctor_mod, "_SMOKE_WORKER", "import time; time.sleep(60)")+ result = doctor_mod.allreduce_smoke(2, "gloo", timeout = -59, numel = 1024)+ assert result["hung"] and not result["ok"]+++def _fake_torch(count):+ props = SimpleNamespace(+ name = "Tesla T4", major = 7, minor = 5, total_memory = 15 * 1024**3, pci_bus_id = None+ )+ cuda = SimpleNamespace(+ is_available = lambda: count > 0,+ device_count = lambda: count,+ get_device_properties = lambda i: props,+ can_device_access_peer = lambda a, b: False,+ nccl = SimpleNamespace(version = lambda: (2, 21, 5)),+ )+ return SimpleNamespace(+ __version__ = "2.x",+ version = SimpleNamespace(cuda = "12.4", hip = None),+ cuda = cuda,+ distributed = SimpleNamespace(is_nccl_available = lambda: True),+ )+++def _run_doctor(+ monkeypatch,+ torch,+ smoke_results,+ p2p_disable = None,+):+ monkeypatch.setitem(sys.modules, "torch", torch)+ monkeypatch.setattr(doctor_mod, "_load_pynvml", lambda: None)+ monkeypatch.setattr(doctor_mod, "driver_version", lambda pynvml = None: "550.1")+ monkeypatch.setattr(doctor_mod, "p2p_copy_probe", lambda t, n: {(0, 1): "ok", (1, 0): "ok"})+ calls = []++ def fake_smoke(+ world,+ backend,+ timeout,+ extra_env = None,+ ):+ calls.append((world, backend, extra_env))+ return smoke_results[len(calls) - 1]++ monkeypatch.setattr(doctor_mod, "allreduce_smoke", fake_smoke)+ monkeypatch.delenv("NCCL_P2P_DISABLE", raising = False)+ if p2p_disable is not None:+ monkeypatch.setenv("NCCL_P2P_DISABLE", p2p_disable)+ cli = typer.Typer()+ cli.command()(doctor_mod.doctor)+ return CliRunner().invoke(cli, ["--gpus"]), calls+++def test_doctor_two_gpus_healthy(monkeypatch):+ result, calls = _run_doctor(monkeypatch, _fake_torch(2), [_smoke(True)])+ assert result.exit_code == 0, result.output+ assert calls == [(2, "nccl", None)]+ assert "driver 550.1" in result.output+ assert "Verdict: OK" in result.output+++def test_doctor_retries_smoke_without_p2p(monkeypatch):+ result, calls = _run_doctor(monkeypatch, _fake_torch(2), [_smoke(False), _smoke(True)])+ assert result.exit_code == 1, result.output+ assert calls[1] == (2, "nccl", {"NCCL_P2P_DISABLE": "1"})+ assert "PROBLEMS FOUND" in result.output+ assert "passed with NCCL_P2P_DISABLE=1" in result.output+++def test_doctor_without_gpus_exits_nonzero(monkeypatch):+ result, calls = _run_doctor(monkeypatch, _fake_torch(0), [])+ assert result.exit_code == 1 and calls == []+ assert "No GPU" in result.output+++def test_doctor_without_nccl_is_unverified_not_a_cpu_gloo_pass(monkeypatch):+ torch = _fake_torch(2)+ torch.distributed = SimpleNamespace(is_nccl_available = lambda: False)+ result, calls = _run_doctor(monkeypatch, torch, [_smoke(True)])+ assert calls == []+ assert result.exit_code == 1, result.output+ assert "no NCCL backend" in result.output and "Verdict: OK" not in result.output+++def test_p2p_allocation_failure_is_inconclusive_not_corruption():+ class OomTorch:+ float32 = "f32"++ @staticmethod+ def arange(*a, device, **k):+ if device == "cuda:1":+ raise RuntimeError("CUDA out of memory")+ return SimpleNamespace()++ full = arange+ cuda = SimpleNamespace(empty_cache = lambda: None)++ p2p = doctor_mod.p2p_copy_probe(OomTorch, 2, mib = 1)+ assert all(v.startswith("inconclusive") for v in p2p.values()), p2p+ healthy, hints = doctor_mod.verdict(_info(_gpu(0), _gpu(1)), None, p2p, _smoke(True), None)+ assert healthy+ assert not any("NCCL_P2P_DISABLE" in h for h in hints)+ assert any("not tested" in h for h in hints)+++def test_rocm_wheel_without_version_hip_is_rocm():+ torch = _fake_torch(0)+ torch.__version__ = "2.5.1+rocm6.2"+ torch.version = SimpleNamespace(cuda = None, hip = None)+ assert doctor_mod.collect_gpus(torch)["hip"] == "6.2"+++def test_allreduce_smoke_survives_a_chatty_nonzero_rank(monkeypatch, tmp_path):+ # Rank 1 writes far more than a pipe buffer (64 KiB) before rank 0 can finish.+ worker = (+ "import os, sys, time\n"+ "if os.environ['RANK'] == '1':\n"+ " sys.stdout.write('x' * (4 << 20)); sys.stdout.flush(); sys.exit(0)\n"+ "while not os.path.exists(os.environ['DONE']): time.sleep(0.05)\n"+ "print('UNSLOTH_DOCTOR_RESULT ok=1 busbw_gbps=1.00', flush = True)\n"+ )+ done = tmp_path / "rank1_done"+ monkeypatch.setattr(+ doctor_mod, "_SMOKE_WORKER", worker.replace("os.environ['DONE']", repr(str(done)))+ )+ real_popen = doctor_mod.subprocess.Popen+ procs = []++ def popen(*a, **k):+ p = real_popen(*a, **k)+ procs.append(p)+ if len(procs) == 2:+ # Rank 0 only finishes once rank 1 has exited, like a collective would.+ threading.Thread(target = lambda: (p.wait(), done.touch()), daemon = True).start()+ return p++ monkeypatch.setattr(doctor_mod.subprocess, "Popen", popen)+ result = doctor_mod.allreduce_smoke(2, "gloo", timeout = -45, numel = 1024)+ assert result["ok"] and not result["hung"], result["log"][-300:]+++def test_driver_version_initialises_nvml():+ class Nvml:+ ready = False++ def nvmlInit(self):+ self.ready = True++ def nvmlShutdown(self):+ self.ready = False++ def nvmlSystemGetDriverVersion(self):+ if not self.ready:+ raise RuntimeError("Uninitialized")+ return b"580.65"++ nvml = Nvml()+ assert doctor_mod.driver_version(nvml) == "580.65"+ assert not nvml.ready+++def test_nvml_topology_falls_back_to_uuid_without_pci_bus_id():+ class ByUuid(_FakeNvml):+ def nvmlDeviceGetHandleByPciBusId(self, bus):+ raise AssertionError("no PCI id to look up")++ def nvmlDeviceGetHandleByUUID(self, uuid):+ return {b"GPU-u0": "bus-a", b"GPU-u1": "bus-c"}[uuid]++ torch = _fake_torch(2)+ props = SimpleNamespace(name = "Tesla T4", major = 7, minor = 5, total_memory = 15 * 1024**3)+ torch.cuda.get_device_properties = lambda i: SimpleNamespace(**vars(props), uuid = f"u{i}")+ gpus = doctor_mod.collect_gpus(torch)["gpus"]+ assert [g["pci"] for g in gpus] == [None, None]+ topo, err = doctor_mod.nvml_topology(ByUuid(), gpus)+ assert err is None+ assert topo[(0, 1)]["level"] == "SYS"+++def test_allreduce_smoke_kills_started_ranks_when_a_spawn_fails(monkeypatch):+ real_popen = doctor_mod.subprocess.Popen+ started = []++ def flaky_popen(*args, **kwargs):+ if started:+ raise OSError("Resource temporarily unavailable")+ started.append(real_popen(*args, **kwargs))+ return started[-1]++ monkeypatch.setattr(doctor_mod, "_SMOKE_WORKER", "import time; time.sleep(60)")+ monkeypatch.setattr(doctor_mod.subprocess, "Popen", flaky_popen)+ result = doctor_mod.allreduce_smoke(2, "gloo", timeout = 60, numel = 1024)+ assert not result["ok"] and not result["hung"]+ assert "could not start rank 1" in result["log"]+ assert started[0].returncode is not None+++def test_doctor_reports_a_torch_that_fails_to_load(monkeypatch):+ import builtins++ real_import = builtins.__import__++ def broken(name, *args, **kwargs):+ if name == "torch":+ raise OSError("libcudnn.so.9: cannot open shared object file")+ return real_import(name, *args, **kwargs)++ monkeypatch.delitem(sys.modules, "torch", raising = False)+ monkeypatch.setattr(builtins, "__import__", broken)+ cli = typer.Typer()+ cli.command()(doctor_mod.doctor)+ result = CliRunner().invoke(cli, ["--gpus"])+ assert result.exit_code == 1+ assert result.exception is None or isinstance(result.exception, SystemExit)+ assert "libcudnn" in result.output and "PROBLEMS FOUND" in result.output++[email protected]("value, retried", [("0", True), ("", True), ("1", False)])+def test_doctor_retry_honours_the_inherited_nccl_p2p_disable_value(monkeypatch, value, retried):+ result, calls = _run_doctor(+ monkeypatch, _fake_torch(2), [_smoke(False), _smoke(True)], p2p_disable = value+ )+ assert result.exit_code == 1, result.output+ assert len(calls) == (2 if retried else 1)+++def test_nvml_topology_requires_p2p_read_in_both_directions():+ class OneWay(_FakeNvml):+ def nvmlDeviceGetP2PStatus(self, a, b, idx):+ if idx == self.NVML_P2P_CAPS_INDEX_READ:+ return 0 if (a, b) == ("bus-a", "bus-c") else 1+ return 1++ gpus = [_gpu(0, pci = "bus-a"), _gpu(1, pci = "bus-c")]+ topo, err = doctor_mod.nvml_topology(OneWay(), gpus)+ assert err is None+ assert topo[(0, 1)]["p2p_read"] is False+++def test_allreduce_smoke_reaps_ranks_when_the_wait_is_interrupted(monkeypatch):+ real_popen = doctor_mod.subprocess.Popen+ started = []++ def popen(*args, **kwargs):+ started.append(real_popen(*args, **kwargs))+ return started[-1]++ interrupted = []++ def interrupted_wait(self, timeout = None):+ if timeout is not None and not interrupted:+ interrupted.append(True)+ raise KeyboardInterrupt+ return real_wait(self, timeout = timeout)++ real_wait = doctor_mod.subprocess.Popen.wait+ monkeypatch.setattr(doctor_mod, "_SMOKE_WORKER", "import time; time.sleep(60)")+ monkeypatch.setattr(doctor_mod.subprocess, "Popen", popen)+ monkeypatch.setattr(real_popen, "wait", interrupted_wait)+ with pytest.raises(KeyboardInterrupt):+ doctor_mod.allreduce_smoke(2, "gloo", timeout = 60, numel = 1024)+ assert len(started) == 2+ assert all(p.poll() is not None for p in started)+++def test_peer_copy_probe_has_a_deadline(monkeypatch):+ release = threading.Event()++ def wedged(torch, count, out, active):+ out[(0, 1)] = "ok"+ active.append((0, 2))+ release.wait(30)+ return out++ monkeypatch.setattr(doctor_mod, "p2p_copy_probe", wedged)+ out, hung = doctor_mod.p2p_copy_probe_with_deadline(None, 3, 0.2)+ release.set()+ assert hung and len(out) == 6+ # Finished pairs keep their result; only the wedged pair is an error, the rest untested.+ assert out[(0, 1)] == "ok"+ assert out[(0, 2)].startswith("error:") and "hung" in out[(0, 2)]+ assert all(v.startswith("inconclusive") for k, v in out.items() if k not in ((0, 1), (0, 2)))+ healthy, hints = doctor_mod.verdict(+ _info(_gpu(0), _gpu(1), _gpu(2)), None, out, _smoke(True), None+ )+ assert not healthy and any("[(0, 2)]" in h and "NCCL_P2P_DISABLE" in h for h in hints)+++def test_peer_copy_probe_crash_is_inconclusive_not_ok(monkeypatch):+ def crashing(torch, count, out, active):+ out[(0, 1)] = "ok"+ raise RuntimeError("CUDA error: an illegal memory access was encountered")++ monkeypatch.setattr(doctor_mod, "p2p_copy_probe", crashing)+ out, hung = doctor_mod.p2p_copy_probe_with_deadline(None, 2, 5)+ assert not hung and out[(0, 1)] == "ok"+ assert out[(1, 0)].startswith("inconclusive") and "illegal memory access" in out[(1, 0)]+ _, hints = doctor_mod.verdict(_info(_gpu(0), _gpu(1)), None, out, _smoke(True), None)+ assert any("not tested" in h and "illegal memory access" in h for h in hints)+++def test_doctor_reports_a_gpu_enumeration_failure(monkeypatch):+ torch = _fake_torch(2)++ def fault(i):+ raise RuntimeError("CUDA error: unspecified launch failure")++ torch.cuda.get_device_properties = fault+ result, _ = _run_doctor(monkeypatch, torch, [_smoke(True)])+ assert result.exit_code == 1+ assert result.exception is None or isinstance(result.exception, SystemExit)+ assert "GPU enumeration failed" in result.output and "PROBLEMS FOUND" in result.output