Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""soup doctor — check dependency compatibility and system health."""
from __future__ import annotations
import os
import platform
import re
import sys
import typer
from rich.console import Console
from rich.markup import escape
from rich.panel import Panel
from rich.table import Table
from soup_cli.utils.constants import GITHUB_URL
from soup_cli.utils.torchao_compat import TORCHAO_MIN_VERSION
console = Console()
# Dependencies to check: (import_name, package_name, min_version, required)
#
# A core-only install (`pip install soup-cli`) is intentionally light: the CLI,
# config system, and data tools — no PyTorch. Only the core rows below are
# required; the heavy training stack lives in EXTRA_GROUPS as one optional
# extra, so a healthy core-only install reports no failures (#828).
DEPS = [
("pydantic", "pydantic", "2.0.0", True),
# A copy of the floor pyproject.toml declares, pinned to it by
# tests/test_cli_dependency_floors.py (#1802).
("typer", "typer", "0.15.4", True),
("rich", "rich", "13.0.0", True),
("yaml", "pyyaml", "6.0", True),
("plotext", "plotext", "5.2.0", True),
# Optional
("fastapi", "fastapi", "0.104.0", False),
("uvicorn", "uvicorn", "0.24.0", False),
("datasketch", "datasketch", "1.6.0", False),
("lm_eval", "lm-eval", "0.4.0", False),
("wandb", "wandb", "0.15.0", False),
("deepspeed", "deepspeed", "0.12.0", False),
("httpx", "httpx", "0.24.0", False),
("unsloth", "unsloth", "2024.8", False),
("PIL", "Pillow", "9.0.0", False),
("torchao", "torchao", TORCHAO_MIN_VERSION, False),
("sglang", "sglang", "0.5.1", False),
("librosa", "librosa", "0.10.0", False),
]
# Extra groups: (extra_name, [(import_name, package_name, min_version), ...])
EXTRA_GROUPS: list[tuple[str, list[tuple[str, str, str]]]] = [
(
"train",
[
# The torch floor is declared once, in pyproject.toml's [train] extra.
# This literal is a copy, pinned to the declaration by
# tests/test_issue636_torch_floor.py — reading installed metadata instead
# would report the install's history, not the declaration (#636).
("torch", "torch", "2.6.0"),
("transformers", "transformers", "5.16.1"),
("peft", "peft", "0.20.0"),
("trl", "trl", "0.29.0"),
("datasets", "datasets", "2.14.0"),
("bitsandbytes", "bitsandbytes", "0.41.0"),
("accelerate", "accelerate", "0.27.0"),
],
),
]
# Packages whose declared breaking-major ceiling must be reported as
# incompatible instead of silently green-lighted by ``soup doctor``.
_MAX_EXCLUSIVE: dict[str, str] = {
"transformers": "6.0.0",
"peft": "1.0.0",
"trl": "1.0.0",
}
def doctor(
nccl: bool = typer.Option(
False, "--nccl", help="Measure NCCL bandwidth and check against reference table."
),
disk: bool = typer.Option(
False,
"--disk",
help=(
"Detect the disk media type (NVMe / SSD / HDD). Layer streaming's "
"disk overflow tier needs NVMe. Off by default: the probe costs ~9 s "
"on Windows, and on Linux it WRITES a ~64 MiB scratch file "
"(.soup-diskprobe-*, git-ignored) into the current directory to "
"measure sequential read throughput."
),
),
config: str | None = typer.Option(
None,
"--config",
"-c",
help=(
"Also check a soup.yaml: report which of the settings it actually "
"sets are not read on its task and backend (#755)."
),
),
):
"""Check system dependencies, GPU, and compatibility."""
console.print("[bold]Soup Doctor[/] - checking your environment...\n")
# System info
dual_python_advisory = _detect_dual_python_interpreters()
panel_body = (
f"Python: [bold]{sys.version.split()[0]}[/]\n"
f"Platform: [bold]{platform.system()} {platform.release()}[/]\n"
f"Arch: [bold]{platform.machine()}[/]"
)
if dual_python_advisory:
panel_body += f"\n[yellow]{dual_python_advisory}[/]"
console.print(Panel(panel_body, title="System"))
# GPU check
_check_gpu()
# MLX (Apple Silicon) check
_check_mlx()
issues: list[str] = []
# Resources check
_check_resources(probe_disk=disk, issues=issues)
# Dependencies table
table = Table(title="Dependencies")
table.add_column("Package", style="bold")
table.add_column("Required", justify="center")
table.add_column("Installed", justify="center")
table.add_column("Min Version")
table.add_column("Status")
# Actionable install specs for the trailing "Fix all" line. Only entries
# that are actually missing or out of range land here — never the full
# required set, and never a bare per-package floor for an extra group.
fix_parts: list[str] = []
fix_pre: list[str] = []
# A missing or incompatible core dependency turns doctor into a gate:
# exit non-zero at the end. An installed package beyond its declared
# ceiling is blocking wherever it is found, extra groups included (#874).
# Advisory issues (optional packages, a missing or outdated [train]
# member, torchvision skew) never touch the exit code.
core_broken = False
for import_name, pkg_name, min_ver, required in DEPS:
try:
mod = __import__(import_name)
version = getattr(mod, "__version__", getattr(mod, "VERSION", None))
if version is None:
# v0.40.1 Part D / M1 — some installs (notably ``rich``)
# don't export ``__version__`` on the package surface;
# importlib.metadata is canonical and works everywhere.
try:
from importlib.metadata import (
PackageNotFoundError,
)
from importlib.metadata import (
version as _pkgver,
)
version = _pkgver(pkg_name)
except (PackageNotFoundError, ImportError):
version = "?"
version_str = str(version)
# Flag versions beyond Soup's validated compatibility band.
max_excl = _MAX_EXCLUSIVE.get(pkg_name)
if max_excl and _version_ge(version_str, max_excl):
status = f"[red]INCOMPATIBLE (need <{max_excl})[/]"
issues.append(
f'Downgrade {pkg_name}: pip install "{pkg_name}>={min_ver},<{max_excl}"'
)
fix_parts.append(f'"{pkg_name}>={min_ver},<{max_excl}"')
# Asymmetry, on purpose (#874): here only a *required* DEPS row
# past its ceiling blocks, whereas the extra-group check below
# blocks unconditionally. An optional DEPS row past a ceiling
# would exit 0 while the same package in an extra group exits 1.
# Unreachable today (_MAX_EXCLUSIVE and the optional DEPS rows
# do not intersect), but written down so it is not rediscovered.
if required:
core_broken = True
elif _version_ok(version_str, min_ver):
status = "[green]OK[/]"
else:
status = f"[yellow]outdated (need >={min_ver})[/]"
issues.append(f'Upgrade {pkg_name}: pip install "{pkg_name}>={min_ver}"')
fix_parts.append(f'"{pkg_name}>={min_ver}"')
table.add_row(
pkg_name,
"yes" if required else "optional",
version_str,
f">={min_ver}",
status,
)
except ImportError:
if required:
status = "[red]MISSING[/]"
issues.append(f'Install {pkg_name}: pip install "{pkg_name}>={min_ver}"')
fix_parts.append(f'"{pkg_name}>={min_ver}"')
core_broken = True
else:
status = "[dim]not installed[/]"
table.add_row(
pkg_name,
"yes" if required else "optional",
"-",
f">={min_ver}",
status,
)
# Extra groups render in the same table as optional rows. A missing group
# member is advisory (status only, no per-package issue); a group with at
# least one missing member contributes exactly one issue pointing at the
# extra, so the suggestion keeps the declared ceilings and the platform
# torch index instead of bare per-package floors.
for extra_name, members in EXTRA_GROUPS:
missing_pkgs: list[str] = []
for import_name, pkg_name, min_ver in members:
version_str = _installed_version_str(import_name, pkg_name)
if version_str is None:
table.add_row(
pkg_name,
escape(f"[{extra_name}]"),
"-",
f">={min_ver}",
"[dim]not installed[/]",
)
missing_pkgs.append(pkg_name)
continue
max_excl = _MAX_EXCLUSIVE.get(pkg_name)
if max_excl and _version_ge(version_str, max_excl):
status = f"[red]INCOMPATIBLE (need <{max_excl})[/]"
issues.append(
f'Downgrade {pkg_name}: pip install "{pkg_name}>={min_ver},<{max_excl}"'
)
fix_parts.append(f'"{pkg_name}>={min_ver},<{max_excl}"')
core_broken = True
elif _version_ok(version_str, min_ver):
status = "[green]OK[/]"
else:
status = f"[yellow]outdated (need >={min_ver})[/]"
issues.append(f'Upgrade {pkg_name}: pip install "{pkg_name}>={min_ver}"')
fix_parts.append(f'"{pkg_name}>={min_ver}"')
table.add_row(pkg_name, escape(f"[{extra_name}]"), version_str, f">={min_ver}", status)
if missing_pkgs:
all_missing = len(missing_pkgs) == len(members)
missing_list = ", ".join(missing_pkgs)
if extra_name == "train":
driver = _nvidia_smi_cuda_version()
torch_missing = "torch" in missing_pkgs
# Single call site, gated on torch itself being missing.
tag = (
_torch_cuda_wheel_tag(driver)
if driver is not None and torch_missing
else None
)
url = f"https://download.pytorch.org/whl/{tag}" if tag else None
if all_missing:
if url is not None:
# ``--index-url`` replaces PyPI, so torch must come from the
# CUDA wheel index in its own step; the ``[train]`` extra is
# then resolved against PyPI with torch already satisfied.
issues.append(
"Training stack not installed:\n"
f" pip install torch --index-url {url}\n"
' pip install "soup-cli[train]"'
)
fix_pre.append(f"pip install torch --index-url {url}")
else:
issues.append(
'Training stack not installed: pip install "soup-cli[train]"'
)
elif url is not None:
issues.append(
f"Training stack incomplete, missing: {missing_list}\n"
f" pip install torch --index-url {url}\n"
' pip install "soup-cli[train]"'
)
fix_pre.append(f"pip install torch --index-url {url}")
else:
issues.append(
f"Training stack incomplete, missing: {missing_list}\n"
' pip install "soup-cli[train]"'
)
fix_parts.append('"soup-cli[train]"')
elif all_missing:
issues.append(
f"{extra_name} stack not installed: "
f'pip install "soup-cli[{extra_name}]"'
)
fix_parts.append(f'"soup-cli[{extra_name}]"')
else:
issues.append(
f"{extra_name} stack incomplete, missing: {missing_list}\n"
f' pip install "soup-cli[{extra_name}]"'
)
fix_parts.append(f'"soup-cli[{extra_name}]"')
console.print(table)
# Check torchvision + torch compatibility
_check_torchvision_compat(issues)
if nccl:
_run_nccl_check()
# Summary
if issues:
# Issue/fix text may contain "[train]"-style brackets, which Rich
# would otherwise swallow as markup tags — escape so the suggestion
# renders literally (cmd.exe-safe double quotes included). highlight
# is off so Rich does not colour-wrap the quoted specs mid-command.
console.print(f"\n[yellow]Found {len(issues)} issue(s):[/]")
for issue in issues:
console.print(f" [red]>[/] {escape(issue)}", highlight=False)
if fix_pre or fix_parts:
console.print("\n[dim]Fix all:[/]")
for step in fix_pre:
console.print(f"[dim] {escape(step)}[/]", highlight=False)
if fix_parts:
console.print(
f"[dim] pip install {escape(' '.join(fix_parts))}[/]", highlight=False
)
loaded_cfg = None
if config:
# Load and validate the config before the environment summary (#2100).
# If the config is refused or cannot be loaded, doctor terminates with exit 2
# before asserting that "All checks passed!".
loaded_cfg = _load_doctor_config(config)
if not issues:
console.print("\n[bold green]All checks passed![/] Your environment is ready.")
# After the environment summary on purpose: "All checks passed" reports on
# the environment, and printing config findings above it read as though the
# green line covered them too.
if loaded_cfg is not None:
_report_config_support(loaded_cfg)
console.print(f"\n[dim]GitHub: [link={GITHUB_URL}]{GITHUB_URL}[/link][/]")
if core_broken:
raise typer.Exit(code=1)
def _installed_version_str(import_name: str, pkg_name: str) -> str | None:
"""Return the installed version string, or None when not importable."""
try:
mod = __import__(import_name)
except ImportError:
return None
version = getattr(mod, "__version__", getattr(mod, "VERSION", None))
if version is None:
try:
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as _pkgver
version = _pkgver(pkg_name)
except (PackageNotFoundError, ImportError):
version = "?"
return str(version)
def _load_doctor_config(config_path: str):
"""Load config for doctor, exiting with code 2 on refusal or load error (#2100)."""
try:
from soup_cli.config.loader import load_config
return load_config(config_path)
except FileNotFoundError as exc:
console.print(f"\n[red]Config not found:[/] {config_path}")
# Non-zero deliberately: this leg is meant to be gate-able in CI, and a
# config that cannot be read is not a clean bill of health. `doctor`
# without --config keeps its old exit status.
raise typer.Exit(2) from exc
except SystemExit as exc:
# load_config prints its own diagnosis and raises SystemExit(1) for a
# schema-invalid config. SystemExit is a BaseException, so it walks
# past `except Exception` and the exit code contradicted the 2
# documented in docs/commands.md. Re-raised as 2 so all three unreadable
# shapes -- missing, unparseable, schema-invalid -- agree.
console.print("\n[red]Config could not be loaded (see above).[/]")
raise typer.Exit(2) from exc
except Exception as exc: # invalid YAML, unreadable file
console.print(f"\n[red]Config could not be loaded:[/] {exc}")
raise typer.Exit(2) from exc
def _report_config_support(cfg) -> None:
"""#755 — report the settings this config sets that its backend never reads.
Only fields the user wrote and switched on are listed (a ``false`` or unset
value is not; ``seed: 0`` is). A wall of 275 rows is not a pre-flight check,
and the fields sitting at their schema default or written in their off
position are not what anyone came here to ask about.
"""
from soup_cli.config.backend_support import (
DEFAULT_BACKEND,
DEFAULT_MODALITY,
check_config,
unsupported_for,
)
backend = getattr(cfg, "backend", DEFAULT_BACKEND)
modality = getattr(cfg, "modality", DEFAULT_MODALITY)
gaps = check_config(cfg)
modality_str = f" modality=[bold]{modality}[/]" if modality != DEFAULT_MODALITY else ""
console.print(
f"\n[bold]Config check[/] - task=[bold]{cfg.task}[/] "
f"backend=[bold]{backend}[/]{modality_str}"
)
if not gaps:
known = unsupported_for(cfg.task, backend, modality)
modality_msg = f" modality={modality}" if modality != DEFAULT_MODALITY else ""
console.print(
f" [green]None of the {len(known)} setting(s) known to be unread "
f"on task={cfg.task} backend={backend}{modality_msg} is switched on in this config.[/]"
)
return
table = Table(title=None, show_header=True)
# overflow="fold" on both text columns: a dotted field name is one
# unbreakable word, so Rich ellipsises it on a narrow terminal and the row
# says a setting is ignored without saying which one. Folding keeps the
# name and the reason legible at any width.
table.add_column("setting", style="bold", overflow="fold")
table.add_column("status", justify="center")
table.add_column("why", overflow="fold")
for entry in gaps:
table.add_row(entry.field, f"[yellow]{entry.status}[/]", entry.describe())
console.print(table)
modality_summary = f" (modality={modality})" if modality != DEFAULT_MODALITY else ""
console.print(
f" [yellow]{len(gaps)} setting(s) switched on here are not read on "
f"backend={backend}{modality_summary}.[/]"
)
def _get_mlx_info() -> dict:
"""Surface MLX info in the doctor report (never crashes on non-Apple)."""
try:
from soup_cli.utils.mlx import get_mlx_info
except ImportError:
return {"available": False}
try:
return get_mlx_info()
except Exception: # noqa: BLE001
return {"available": False}
def _check_mlx():
"""Report the MLX (Apple Silicon) backend in ``soup doctor``.
MLX is an Apple Silicon-only stack, so the panel is informational rather
than a pass/fail dependency: it shows the installed version and hardware
when present, and says so plainly when MLX is missing. It must never crash
the report (the info helper degrades to ``available=False`` on any error).
"""
info = _get_mlx_info()
if info.get("available"):
mem_bytes = info.get("unified_memory_bytes")
mem_str = f"{mem_bytes / (1024**3):.0f} GB" if mem_bytes else "unknown"
chip = (info.get("chip") or {}).get("chip")
console.print(
Panel(
f"Version: [bold green]{info.get('version') or 'unknown'}[/]\n"
f"Chip: [bold]{chip or 'Apple Silicon'}[/]\n"
f"Memory: [bold]{mem_str}[/] unified",
title="MLX",
)
)
elif info.get("apple_silicon"):
console.print(
Panel(
"Status: [yellow]not installed[/]\n"
"Install: [dim]pip install \"soup-cli\\[mlx]\"[/]",
title="MLX",
)
)
def _gpu_architecture_name(major: int, minor: int) -> str:
"""Return the NVIDIA architecture family for a compute capability."""
if major >= 12 or major in (10,):
return "Blackwell"
if major == 9:
return "Hopper"
if major == 8 and minor == 9:
return "Ada"
if major == 8:
return "Ampere"
if major == 7 and minor in (5,):
return "Turing"
if major == 7 and minor in (0, 2):
return "Volta"
if major == 6 and minor in (0, 1):
return "Pascal"
return "Unknown"
def _torch_gpu_arch_supported(torch, major: int, minor: int) -> bool:
"""Return whether Torch has native or PTX coverage for this GPU."""
try:
arch_list = torch.cuda.get_arch_list()
except (AttributeError, RuntimeError):
return False
target = major * 10 + minor
for arch in arch_list:
if arch.startswith("sm_"):
sm_target = arch.removeprefix("sm_")
suffixed = sm_target.endswith("a")
sm_target = sm_target.rstrip("a")
try:
target_major = int(sm_target) // 10
target_minor = int(sm_target) % 10
except ValueError:
continue
if suffixed and sm_target == f"{major}{minor}":
return True
if (
not suffixed
and target_major == major
and target_minor <= minor
):
return True
if arch.startswith("compute_"):
try:
ptx_target = int(arch.removeprefix("compute_").rstrip("a"))
except ValueError:
continue
if ptx_target <= target:
return True
return False
def _format_gpu_capability(torch, idx: int) -> tuple[str, bool]:
"""Return ``(capability_text, supported_by_torch)`` for one GPU."""
try:
major, minor = torch.cuda.get_device_capability(idx)
except (AttributeError, RuntimeError, AssertionError):
return "unknown", False
architecture = _gpu_architecture_name(major, minor)
supported = _torch_gpu_arch_supported(torch, major, minor)
return f"sm_{major}{minor} ({architecture})", supported
def _get_precision_capabilities(torch) -> dict[str, tuple[bool, bool | None]]:
"""Return hardware/software support for the reported precision features."""
from soup_cli.utils.advanced_precision import (
_torchao_available,
is_blackwell_gpu,
is_nvfp4_software_supported,
)
from soup_cli.utils.fp8 import is_fp8_gpu_supported
try:
from soup_cli.utils.gpu import cuda_supports_bf16
bf16_supported = bool(cuda_supports_bf16())
except (AttributeError, RuntimeError, AssertionError):
bf16_supported = False
torchao_available = bool(_torchao_available())
nvfp4_software = bool(is_nvfp4_software_supported())
return {
"BF16": (bf16_supported, None),
"FP8": (bool(is_fp8_gpu_supported()), torchao_available),
"NVFP4": (bool(is_blackwell_gpu()), nvfp4_software),
}
def _check_gpu():
"""Check GPU availability and display info."""
try:
import torch
if torch.cuda.is_available():
gpu_count = torch.cuda.device_count()
gpus = []
for idx in range(gpu_count):
name = torch.cuda.get_device_name(idx)
mem = torch.cuda.get_device_properties(idx)
total_gb = getattr(mem, "total_memory", getattr(mem, "total_mem", 0))
total_gb = total_gb / (1024**3)
capability, torch_arch_supported = _format_gpu_capability(torch, idx)
arch_warning = ""
if not torch_arch_supported:
advisory = _detect_gpu_arch_mismatch_advisory()
arch_warning = (
" [bold red]Torch build does not include this GPU "
"architecture[/]"
)
if advisory:
arch_warning += f" [dim]{advisory}[/]"
gpus.append(
f" GPU {idx}: [bold]{name}[/] "
f"({total_gb:.1f} GB) — {capability}{arch_warning}"
)
gpu_info = "\n".join(gpus)
cuda_ver = torch.version.cuda or "N/A"
precision = _get_precision_capabilities(torch)
precision_lines = []
for feature, (hardware, software) in precision.items():
hw_status = "[green]yes[/]" if hardware else "[red]no[/]"
if software is None:
precision_lines.append(f" {feature}: hardware={hw_status}")
else:
sw_status = "[green]yes[/]" if software else "[yellow]no[/]"
precision_lines.append(
f" {feature}: hardware={hw_status}, software={sw_status}"
)
precision_info = "\n".join(precision_lines)
console.print(
Panel(
f"CUDA: [bold green]available[/] (v{cuda_ver})\n"
f"GPUs: [bold]{gpu_count}[/]\n{gpu_info}\n\n"
"[bold]Precision features[/]\n"
f"{precision_info}",
title="GPU",
)
)
elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
from soup_cli.utils.terminal import for_terminal
async_load = os.environ.get("HF_DEACTIVATE_ASYNC_LOAD")
if async_load is None:
async_desc = "not set (soup train sets it to 1 on MPS)"
else:
async_desc = for_terminal(async_load)
console.print(
Panel(
"Backend: [bold green]MPS (Apple Silicon)[/]\n"
"Status: [bold green]available[/]\n"
f"HF_DEACTIVATE_ASYNC_LOAD: [dim]{async_desc}[/]",
title="GPU",
)
)
else:
# v0.40.1 Part C / N3 — distinguish "no GPU hardware" from
# "GPU hardware present, wrong torch wheel". When nvidia-smi
# reports a GPU but torch lacks CUDA, the user installed the
# CPU-only wheel — point them at the right reinstall command.
advisory = _detect_gpu_hw_without_torch_cuda()
console.print(
Panel(
"Backend: [bold yellow]CPU only[/]\n"
"Warning: Training will be slow without GPU."
+ (f"\n[dim]{advisory}[/]" if advisory else ""),
title="GPU",
)
)
except ImportError:
console.print(
Panel(
"Backend: [red]unknown (torch not installed)[/]",
title="GPU",
)
)
# Newest-first PyTorch CUDA wheel tags. A driver that advertises CUDA N.M can
# load any wheel whose CUDA is <= N.M (driver backward compatibility).
_TORCH_CUDA_WHEELS: tuple[tuple[int, int, str], ...] = (
(13, 2, "cu132"),
(13, 0, "cu130"),
(12, 8, "cu128"),
(12, 6, "cu126"),
(12, 4, "cu124"),
(12, 1, "cu121"),
(11, 8, "cu118"),
)
# PyTorch CUDA indexes that carry a torch release satisfying Soup's
# ``[train]`` floor (torch>=2.6.0). Keep this table explicit: not every
# CUDA version reported by a driver has a corresponding PyTorch index.
_SUPPORTED_TORCH_CUDA_WHEELS = frozenset(
{"cu132", "cu130", "cu128", "cu126", "cu124", "cu118"}
)
def _parse_cuda_version(text: str) -> tuple[int, int] | None:
"""Extract ``(major, minor)`` from an nvidia-smi CUDA version header."""
match = re.search(r"CUDA(?:\s+UMD)?\s+Version:\s*(\d+)\.(\d+)", text or "")
if match is None:
return None
return int(match.group(1)), int(match.group(2))
def _torch_cuda_wheel_tag(driver: tuple[int, int] | None) -> str | None:
"""Pick the newest supported PyTorch CUDA wheel for a driver version.
A missing driver version is not safe to guess from, so it returns ``None``.
"""
if driver is None:
return None
for major, minor, tag in _TORCH_CUDA_WHEELS:
if driver >= (major, minor) and tag in _SUPPORTED_TORCH_CUDA_WHEELS:
return tag
return None
def _nvidia_smi_executable() -> str | None:
"""Absolute path to nvidia-smi, or None.
Bare ``nvidia-smi`` is never handed to ``subprocess`` (CWE-427):
Windows ``CreateProcess`` searches the current directory before PATH.
"""
import shutil
return shutil.which("nvidia-smi")
def _nvidia_smi_cuda_version() -> tuple[int, int] | None:
"""Read the driver's max CUDA version from ``nvidia-smi`` header output."""
import subprocess
nvidia_smi = _nvidia_smi_executable()
if nvidia_smi is None:
return None
try:
completed = subprocess.run( # noqa: S603 — argv list, no shell
[nvidia_smi],
capture_output=True,
text=True,
timeout=5,
)
except (OSError, subprocess.TimeoutExpired):
return None
if completed.returncode != 0:
return None
return _parse_cuda_version((completed.stdout or "") + (completed.stderr or ""))
def _detect_gpu_arch_mismatch_advisory() -> str:
"""Return a CUDA wheel reinstall hint for an unsupported GPU architecture."""
driver_cuda = _nvidia_smi_cuda_version()
if driver_cuda is None:
return (
"Try reinstalling a CUDA-enabled PyTorch build that supports "
"your GPU architecture."
)
wheel = _torch_cuda_wheel_tag(driver_cuda)
if wheel is None:
return (
"Try reinstalling a CUDA-enabled PyTorch build that supports "
"your GPU architecture. Run `nvidia-smi` and install a PyTorch "
"CUDA wheel compatible with the reported driver."
)
index_url = f"https://download.pytorch.org/whl/{wheel}"
return (
"Reinstall a CUDA-enabled PyTorch build for your driver: "
f"`pip install torch --index-url {index_url}`"
)
def _detect_gpu_hw_without_torch_cuda() -> str:
"""v0.40.1 Part C / N3 — return advisory string if nvidia-smi succeeds
but torch lacks CUDA (i.e. user installed the CPU-only wheel).
"""
import subprocess
nvidia_smi = _nvidia_smi_executable()
if nvidia_smi is None:
return ""
try:
completed = subprocess.run( # noqa: S603 — argv list, no shell
[nvidia_smi, "--query-gpu=name", "--format=csv,noheader"],
capture_output=True,
text=True,
timeout=5,
)
except (OSError, subprocess.TimeoutExpired):
return ""
if completed.returncode != 0:
return ""
gpu_name = (completed.stdout or "").strip().splitlines()[:1]
raw_label = gpu_name[0] if gpu_name else "GPU"
# v0.40.1 review fix — security: nvidia-smi output is embedded in a
# Rich-markup string at the call site; escape `[`/`]` so a GPU name like
# "NVIDIA Quadro [T4]" cannot break or inject markup.
from rich.markup import escape as _markup_escape
gpu_label = _markup_escape(raw_label)
try:
from importlib.metadata import version as _pkgver
torch_version = _pkgver("torch")
except Exception: # noqa: BLE001
torch_version = "?"
try:
import torch
torch_cuda_version = getattr(torch.version, "cuda", None)
except Exception: # noqa: BLE001
torch_cuda_version = None
wheel = _torch_cuda_wheel_tag(_nvidia_smi_cuda_version())
windows_note = ""
if platform.system() == "Windows":
windows_note = " On Windows, PyPI's torch wheel is CPU-only."
if wheel is None:
return (
f"GPU hardware present ({gpu_label}) but torch CUDA could not "
f"be confirmed. The installed torch is {torch_version}. "
"Run `nvidia-smi` and install a PyTorch CUDA wheel compatible "
"with the reported driver."
f"{windows_note}"
)
index_url = f"https://download.pytorch.org/whl/{wheel}"
if torch_cuda_version:
build_status = (
f"torch CUDA build ({torch_cuda_version}) could not initialise"
)
else:
build_status = "torch is the CPU build"
return (
f"GPU hardware present ({gpu_label}) but {build_status} "
f"(torch {torch_version}). To enable your GPU: "
f"`pip install --force-reinstall \"torch>=2.6.0\" "
f"--index-url {index_url}`"
f"{windows_note}"
)
def _detect_dual_python_interpreters() -> str:
"""v0.40.1 Part C / N4 — flag when ``soup`` runs under one Python and
``python`` on the user's PATH is a different interpreter.
"""
import os
import shutil
soup_python = sys.executable
path_python = shutil.which("python") or shutil.which("python3")
if not path_python:
return ""
# v0.40.1 review fix — use os.path.realpath, not Path.resolve(), so
# Windows 8.3 short names don't produce a false-positive advisory.
try:
if os.path.realpath(path_python) == os.path.realpath(soup_python):
return ""
except OSError:
return ""
return (
f"`soup` runs under {soup_python}; `python` on your PATH is "
f"{path_python}. site-packages may differ — for any `python -c` "
f"check use the soup interpreter explicitly."
)
_GB = 1024**3
def _get_ram_gb() -> str:
"""Get total system RAM in GB, with cross-platform fallbacks."""
# Prefer psutil if installed
try:
import psutil
return f"{psutil.virtual_memory().total / _GB:.0f} GB"
except ImportError:
pass
system = platform.system()
if system == "Linux":
try:
with open("/proc/meminfo", encoding="utf-8") as fh:
for line in fh:
if line.startswith("MemTotal:"):
kb = int(line.split()[1])
return f"{kb * 1024 / _GB:.0f} GB"
except (OSError, ValueError):
pass
elif system == "Darwin":
try:
import subprocess
res = subprocess.run(
["sysctl", "-n", "hw.memsize"],
capture_output=True,
text=True,
timeout=5,
check=False,
)
if res.returncode == 0:
return f"{int(res.stdout.strip()) / _GB:.0f} GB"
except (OSError, ValueError, subprocess.TimeoutExpired):
pass
elif system == "Windows":
try:
import ctypes
class MEMORYSTATUSEX(ctypes.Structure):
_fields_ = [
("dwLength", ctypes.c_ulong),
("dwMemoryLoad", ctypes.c_ulong),
("ullTotalPhys", ctypes.c_ulonglong),
("ullAvailPhys", ctypes.c_ulonglong),
("ullTotalPageFile", ctypes.c_ulonglong),
("ullAvailPageFile", ctypes.c_ulonglong),
("ullTotalVirtual", ctypes.c_ulonglong),
("ullAvailVirtual", ctypes.c_ulonglong),
("sullAvailExtendedVirtual", ctypes.c_ulonglong),
]
stat = MEMORYSTATUSEX()
stat.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
return f"{stat.ullTotalPhys / _GB:.0f} GB"
except (OSError, AttributeError):
pass
return "Unknown"
def _check_resources(probe_disk: bool = False, issues: list[str] | None = None):
"""Check RAM and Disk space and display info."""
import shutil
table = Table(title="System Resources")
table.add_column("Resource", style="bold")
table.add_column("Value", overflow="fold")
table.add_row("RAM", _get_ram_gb())
try:
usage = shutil.disk_usage(".")
disk_str = f"{usage.free / _GB:.0f} GB free"
except OSError:
disk_str = "Unknown"
table.add_row("Disk", disk_str)
# v0.72.3 — layer streaming's disk overflow tier needs NVMe: on a spinning
# disk each step costs 2 seeks per layer (plan P11), which does not merely
# run slower, it thrashes. Reported here so the refusal is not the first
# time an operator hears about it.
#
# Opt-in behind --disk, matching this command's own --nccl convention for
# expensive probes. Measured on the dev box: the Windows PowerShell query
# costs ~9 s cold and ~2.4 s warm (17.6 s -> 20.0 s on `soup doctor`), paid
# by every user including the majority who never touch layer streaming. A
# streaming run does its own lazy probe when the tier decision actually
# depends on the answer, so nothing is lost by defaulting this off.
if probe_disk:
try:
from soup_cli.utils.layer_stream import detect_disk_kind
kind = detect_disk_kind(".")
except Exception: # noqa: BLE001 — a diagnostic must never crash the report
kind = "unknown"
verdict = {
"nvme": "[green]NVMe[/] — layer streaming can use the disk overflow tier",
"ssd": "[yellow]SATA SSD[/] — layer streaming refuses the disk tier (RAM only)",
"hdd": "[red]HDD[/] — layer streaming refuses the disk tier (RAM only)",
}.get(kind, "[yellow]Unknown[/] — layer streaming will refuse the disk tier")
table.add_row("Disk type", verdict)
stripe_var = os.environ.get("SOUP_LAYER_STREAM_STRIPE_DIRS")
if stripe_var:
from soup_cli.utils.layer_shard import resolve_cache_root
from soup_cli.utils.stripe_roots import (
STRIPE_DIRS_ENV,
iter_early_stripe_roots,
parse_stripe_dirs,
)
from soup_cli.utils.terminal import for_terminal
stripe_entries = parse_stripe_dirs(stripe_var)
primary = resolve_cache_root()
has_refused = False
for entry, resolved, reason in iter_early_stripe_roots(primary, stripe_entries):
row_label = "Stripe root"
if reason is not None:
has_refused = True
row_val = f"{for_terminal(entry)} — [red]Refused[/]: {for_terminal(reason)}"
elif probe_disk:
from soup_cli.utils.layer_stream import detect_disk_kind
try:
kind = detect_disk_kind(resolved)
except Exception: # noqa: BLE001
kind = "unknown"
if kind != "nvme":
has_refused = True
reason_disk = (
"the disk tier streams from NVMe only, and this volume classifies "
f"as {kind!r}. If the probe is wrong, "
"training.stream_disk_kind overrides it."
)
refused_msg = for_terminal(reason_disk)
row_val = f"{for_terminal(entry)} — [red]Refused[/]: {refused_msg}"
else:
row_val = f"{for_terminal(entry)} — [green]NVMe[/]"
else:
row_val = f"{for_terminal(entry)} — [green]OK[/]"
table.add_row(row_label, row_val)
if has_refused and issues is not None:
issues.append(
f"{STRIPE_DIRS_ENV}: one or more stripe roots are refused "
"(see System Resources above)"
)
console.print(table)
console.print()
def _check_torchvision_compat(issues: list):
"""Check that torchvision version is compatible with torch."""
try:
import torch
import torchvision
torch_ver = torch.__version__.split("+")[0]
tv_ver = torchvision.__version__.split("+")[0]
torch_minor = ".".join(torch_ver.split(".")[:2])
tv_minor = ".".join(tv_ver.split(".")[:2])
# Known compatible pairs (torch minor -> torchvision minor)
compat = {
"2.6": "0.21",
"2.5": "0.20",
"2.4": "0.19",
"2.3": "0.18",
"2.2": "0.17",
"2.1": "0.16",
"2.0": "0.15",
}
expected_tv = compat.get(torch_minor)
if expected_tv and not tv_minor.startswith(expected_tv):
msg = (
f"torchvision {tv_ver} may be incompatible with torch {torch_ver}. "
f"Expected torchvision {expected_tv}.x"
)
console.print(f" [yellow]Warning:[/] {msg}")
issues.append(msg)
except (ImportError, AttributeError):
# AttributeError: torchvision circular import on some platforms
pass
def _version_ok(installed: str, minimum: str) -> bool:
"""Check if installed version meets minimum requirement."""
try:
inst_parts = [int(x) for x in installed.split(".")[:3]]
min_parts = [int(x) for x in minimum.split(".")[:3]]
# Pad to same length
while len(inst_parts) < 3:
inst_parts.append(0)
while len(min_parts) < 3:
min_parts.append(0)
return inst_parts >= min_parts
except (ValueError, AttributeError):
return True # Can't parse, assume OK
def _version_ge(installed: str, threshold: str) -> bool:
"""v0.40.1 Part C / C5 — return True iff installed >= threshold.
Used to flag major-version upgrades we haven't migrated to. Robust to
suffixes like ``5.0.0.dev0`` (split on ``.``, parse leading ints only).
"""
try:
inst_parts: list[int] = []
for chunk in installed.split(".")[:3]:
digits = "".join(c for c in chunk if c.isdigit())
inst_parts.append(int(digits) if digits else 0)
thr_parts = [int(x) for x in threshold.split(".")[:3]]
while len(inst_parts) < 3:
inst_parts.append(0)
while len(thr_parts) < 3:
thr_parts.append(0)
return inst_parts >= thr_parts
except (ValueError, AttributeError):
return False
def _run_nccl_check():
"""Run NCCL bandwidth test if requested."""
try:
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from soup_cli.utils.profiling_v0_43 import nccl_bandwidth_check
from soup_cli.utils.topology import detect_topology
except ImportError:
console.print("\n[yellow]NCCL bandwidth check requires torch and torch.distributed.[/]")
return
if not torch.cuda.is_available() or not dist.is_available():
console.print("\n[yellow]NCCL bandwidth check requires CUDA and torch.distributed.[/]")
return
topo = detect_topology()
gpu_count = topo["gpu_count"]
if gpu_count < 2:
console.print("\n[yellow]NCCL bandwidth requires >=2 GPUs[/]")
return
name = torch.cuda.get_device_name(0).lower()
if "h100" in name:
gpu = "h100"
elif "a100" in name:
gpu = "a100"
elif "v100" in name:
gpu = "v100"
elif "4090" in name:
gpu = "rtx4090"
elif "3090" in name:
gpu = "rtx3090"
else:
gpu = "unknown"
link = topo["interconnect"]
manager = mp.Manager()
return_dict = manager.dict()
console.print(
"\n[bold]Measuring NCCL bandwidth[/] "
"(100 MB all_reduce, 3 warmup + 10 timed iters, median GB/s)..."
)
try:
mp.spawn(_nccl_worker, args=(return_dict,), nprocs=2, join=True)
except Exception as e:
console.print(f"[red]Failed to measure NCCL bandwidth:[/] {e}")
return
if "gb_per_sec" in return_dict:
measured = return_dict["gb_per_sec"]
res = nccl_bandwidth_check(gpu=gpu, link=link, measured_gb_per_sec=measured)
status = res["status"]
if status == "OK":
color = "green"
elif status == "MINOR":
color = "yellow"
else:
color = "red"
expected = res.get("expected_gb_per_sec")
expected_str = f" vs expected {expected:.1f}" if expected else ""
console.print(
f" Result ({gpu.upper()} over {link.upper()}): "
f"[{color}]{status}[/] ({measured:.1f} GB/s{expected_str})"
)
else:
console.print("[red]Failed to capture NCCL bandwidth measurement.[/]")
# NCCL benchmark constants. 100 MB matches typical gradient-bucket size for
# medium models (7B fp16 ≈ 100-200 MB per bucket) so the measurement
# reflects real all-reduce traffic, not artificially small messages.
_NCCL_BENCHMARK_TENSOR_BYTES = 100 * 1024 * 1024 # 100 MB
_NCCL_BENCHMARK_WARMUP_ITERS = 3 # 3 warmups to amortise CUDA kernel JIT
_NCCL_BENCHMARK_TIMED_ITERS = 10 # 10 timed runs; report the median (robust to outliers)
def _nccl_worker(rank: int, return_dict):
import os
import statistics
import time
import torch
import torch.distributed as dist
# Snapshot prior env so we can restore after the spawn finishes —
# avoids leaking MASTER_ADDR/MASTER_PORT into the parent doctor process
# (matters when tests run multiple doctor invocations in one session).
prior_addr = os.environ.get("MASTER_ADDR")
prior_port = os.environ.get("MASTER_PORT")
os.environ["MASTER_ADDR"] = "localhost"
os.environ["MASTER_PORT"] = "29500"
try:
dist.init_process_group("nccl", rank=rank, world_size=2)
# 100 MB tensor of float32 (4 bytes per element).
num_elements = _NCCL_BENCHMARK_TENSOR_BYTES // 4
tensor = torch.ones(num_elements, dtype=torch.float32, device=f"cuda:{rank}")
# Warmup — runs CUDA kernel JIT + first-NCCL-collective handshake
# out of the timing window. 3 iters is enough to stabilise.
for _ in range(_NCCL_BENCHMARK_WARMUP_ITERS):
dist.all_reduce(tensor)
torch.cuda.synchronize(device=f"cuda:{rank}")
# Per-iteration timing (10 samples). Median is more robust to
# one-off jitter (kernel preemption, swap, etc.) than mean.
per_iter_sec: list[float] = []
for _ in range(_NCCL_BENCHMARK_TIMED_ITERS):
start = time.perf_counter()
dist.all_reduce(tensor)
torch.cuda.synchronize(device=f"cuda:{rank}")
per_iter_sec.append(time.perf_counter() - start)
if rank == 0:
median_elapsed = statistics.median(per_iter_sec)
size_gb = (tensor.element_size() * tensor.numel()) / 1e9
return_dict["gb_per_sec"] = size_gb / median_elapsed
dist.destroy_process_group()
finally:
# Restore prior env exactly as we found it (or remove if missing).
if prior_addr is None:
os.environ.pop("MASTER_ADDR", None)
else:
os.environ["MASTER_ADDR"] = prior_addr
if prior_port is None:
os.environ.pop("MASTER_PORT", None)
else:
os.environ["MASTER_PORT"] = prior_port