Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

38.7 KB
Raw Blame
"""Tests for soup doctor command."""

import sys
from unittest.mock import patch

from typer.testing import CliRunner

from soup_cli.cli import app
from soup_cli.commands.doctor import _version_ok
from tests.conftest import strip_ansi

runner = CliRunner()

def _strip_ansi(text: str) -> str:
    return strip_ansi(text)


def _plain(text: str) -> str:
    return " ".join(strip_ansi(text).split())


# --- _version_ok tests ---


def test_version_ok_exact():
    assert _version_ok("2.0.0", "2.0.0") is True


def test_version_ok_higher():
    assert _version_ok("2.1.0", "2.0.0") is True


def test_version_ok_lower():
    assert _version_ok("1.9.0", "2.0.0") is False


def test_version_ok_patch():
    assert _version_ok("2.0.1", "2.0.0") is True


def test_version_ok_major_higher():
    assert _version_ok("3.0.0", "2.0.0") is True


def test_version_ok_two_part():
    assert _version_ok("6.0", "6.0") is True


def test_version_ok_unparseable():
    """Unparseable versions should return True (assume OK)."""
    assert _version_ok("unknown", "2.0.0") is True


def test_version_ok_dev_suffix():
    """Version with dev suffix (can't fully parse)."""
    assert _version_ok("2.1.0.dev0", "2.0.0") is True


# --- doctor CLI tests ---


def test_doctor_runs():
    """soup doctor runs without crashing."""
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    assert "Soup Doctor" in _strip_ansi(result.output)


def test_doctor_shows_system_info():
    """soup doctor shows system info panel."""
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    output = _strip_ansi(result.output)
    assert "Python" in output
    assert "Platform" in output


def test_doctor_shows_dependencies():
    """soup doctor shows dependency table."""
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    output = _strip_ansi(result.output)
    assert "Dependencies" in output
    assert "Package" in output


def test_gpu_arch_mismatch_advisory_uses_driver_cuda_wheel(monkeypatch):
    from soup_cli.commands.doctor import _detect_gpu_arch_mismatch_advisory

    monkeypatch.setattr(
        "soup_cli.commands.doctor._nvidia_smi_cuda_version",
        lambda: (13, 2),
    )

    advisory = _detect_gpu_arch_mismatch_advisory()

    assert "download.pytorch.org/whl/cu132" in advisory
    assert "pip install torch" in advisory


def test_gpu_arch_mismatch_advisory_falls_back_for_unsupported_driver(
    monkeypatch,
):
    from soup_cli.commands.doctor import _detect_gpu_arch_mismatch_advisory

    monkeypatch.setattr(
        "soup_cli.commands.doctor._nvidia_smi_cuda_version",
        lambda: (11, 7),
    )

    advisory = _detect_gpu_arch_mismatch_advisory()

    assert "whl/None" not in advisory
    assert "CUDA-enabled PyTorch build" in advisory


def test_format_gpu_capability_handles_missing_torch_cuda_attribute():
    from types import SimpleNamespace

    from soup_cli.commands.doctor import _format_gpu_capability

    fake_torch = SimpleNamespace(cuda=SimpleNamespace())

    assert _format_gpu_capability(fake_torch, 0) == ("unknown", False)


def test_gpu_architecture_name_maps_legacy_nvidia_architectures():
    from soup_cli.commands.doctor import _gpu_architecture_name

    assert _gpu_architecture_name(6, 0) == "Pascal"
    assert _gpu_architecture_name(6, 1) == "Pascal"
    assert _gpu_architecture_name(7, 0) == "Volta"
    assert _gpu_architecture_name(7, 2) == "Volta"
    assert _gpu_architecture_name(7, 5) == "Turing"


def test_torch_gpu_arch_supported_accepts_suffixed_sm_target():
    from types import SimpleNamespace

    from soup_cli.commands.doctor import _torch_gpu_arch_supported

    fake_torch = SimpleNamespace(
        cuda=SimpleNamespace(
            get_arch_list=lambda: ["sm_90a", "sm_100a"],
        )
    )

    assert _torch_gpu_arch_supported(fake_torch, 9, 0) is True
    assert _torch_gpu_arch_supported(fake_torch, 10, 0) is True


def test_torch_gpu_arch_supported_accepts_exact_sm_target():
    from types import SimpleNamespace

    from soup_cli.commands.doctor import _torch_gpu_arch_supported

    fake_torch = SimpleNamespace(
        cuda=SimpleNamespace(
            get_arch_list=lambda: ["sm_90", "sm_120"],
        )
    )

    assert _torch_gpu_arch_supported(fake_torch, 12, 0) is True


def test_torch_gpu_arch_supported_accepts_later_minor_same_major():
    from types import SimpleNamespace

    from soup_cli.commands.doctor import _torch_gpu_arch_supported

    fake_torch = SimpleNamespace(
        cuda=SimpleNamespace(
            get_arch_list=lambda: ["sm_75", "sm_80", "sm_86", "sm_90"],
        )
    )

    assert _torch_gpu_arch_supported(fake_torch, 8, 9) is True


def test_torch_gpu_arch_supported_rejects_earlier_minor_same_major():
    from types import SimpleNamespace

    from soup_cli.commands.doctor import _torch_gpu_arch_supported

    fake_torch = SimpleNamespace(
        cuda=SimpleNamespace(get_arch_list=lambda: ["sm_86"])
    )

    assert _torch_gpu_arch_supported(fake_torch, 8, 0) is False


def test_torch_gpu_arch_supported_keeps_suffixed_arch_exact():
    from types import SimpleNamespace

    from soup_cli.commands.doctor import _torch_gpu_arch_supported

    fake_torch = SimpleNamespace(
        cuda=SimpleNamespace(get_arch_list=lambda: ["sm_100a"])
    )

    assert _torch_gpu_arch_supported(fake_torch, 10, 3) is False


def test_torch_gpu_arch_supported_accepts_lower_ptx_target():
    from types import SimpleNamespace

    from soup_cli.commands.doctor import _torch_gpu_arch_supported

    fake_torch = SimpleNamespace(
        cuda=SimpleNamespace(
            get_arch_list=lambda: ["sm_90", "compute_90"],
        )
    )

    assert _torch_gpu_arch_supported(fake_torch, 12, 0) is True


def test_torch_gpu_arch_supported_rejects_higher_only_ptx_target():
    from types import SimpleNamespace

    from soup_cli.commands.doctor import _torch_gpu_arch_supported

    fake_torch = SimpleNamespace(
        cuda=SimpleNamespace(
            get_arch_list=lambda: ["sm_80", "compute_120"],
        )
    )

    assert _torch_gpu_arch_supported(fake_torch, 9, 0) is False


def test_torch_gpu_arch_supported_accepts_a_suffixed_architectures():
    from types import SimpleNamespace

    from soup_cli.commands.doctor import _torch_gpu_arch_supported

    fake_torch = SimpleNamespace(
        cuda=SimpleNamespace(
            get_arch_list=lambda: ["sm_90a", "sm_100a"],
        )
    )

    assert _torch_gpu_arch_supported(fake_torch, 9, 0) is True
    assert _torch_gpu_arch_supported(fake_torch, 10, 0) is True


def test_doctor_gpu_panel_shows_compute_capability_and_precision(monkeypatch):
    """GPU panel reports compute capability and precision feature gates."""
    import importlib.machinery
    import types

    fake_cuda = types.SimpleNamespace(
        is_available=lambda: True,
        device_count=lambda: 1,
        get_device_name=lambda idx: "NVIDIA GeForce RTX 5070 Laptop GPU",
        get_device_properties=lambda idx: types.SimpleNamespace(
            total_memory=8 * 1024**3
        ),
        get_device_capability=lambda idx: (12, 0),
        get_arch_list=lambda: ["sm_75", "sm_80", "sm_90", "sm_120"],
        is_bf16_supported=lambda including_emulation=True: True,
    )

    fake_torch = types.SimpleNamespace(
        __spec__=importlib.machinery.ModuleSpec("torch", None),
        cuda=fake_cuda,
        version=types.SimpleNamespace(cuda="13.0"),
    )

    monkeypatch.setitem(__import__("sys").modules, "torch", fake_torch)
    monkeypatch.setattr(
        "soup_cli.commands.doctor._installed_version_str",
        lambda import_name, pkg_name: None,
    )
    monkeypatch.setattr(
        "soup_cli.utils.fp8.is_fp8_gpu_supported",
        lambda: True,
    )
    monkeypatch.setattr(
        "soup_cli.utils.fp8.is_fp8_available",
        lambda: True,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision.is_blackwell_gpu",
        lambda: True,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision._torchao_available",
        lambda: True,
    )
    monkeypatch.setattr(
        "soup_cli.utils.torchao_compat.resolve_torchao_class",
        lambda key: object,
    )

    fake_torchao = types.ModuleType("torchao")
    fake_quantization = types.ModuleType("torchao.quantization")
    fake_quantization.NVFP4Config = object
    fake_quantization.quantize_ = lambda *args, **kwargs: None
    fake_torchao.quantization = fake_quantization

    monkeypatch.setitem(__import__("sys").modules, "torchao", fake_torchao)
    monkeypatch.setitem(
        __import__("sys").modules,
        "torchao.quantization",
        fake_quantization,
    )

    result = runner.invoke(
        app,
        ["doctor"],
        env={"COLUMNS": "120", "FORCE_COLOR": "1"},
    )
    out = _plain(result.output)

    assert result.exit_code == 0
    assert "sm_120 (Blackwell)" in out
    assert "Precision features" in out
    assert "BF16: hardware=yes" in out
    assert "FP8: hardware=yes, software=yes" in out
    assert "NVFP4: hardware=yes, software=yes" in out


def test_get_precision_capabilities_reports_negative_nvfp4_software(
    monkeypatch,
):
    """NVFP4 software support must report false when the runtime gate is absent."""
    import types

    fake_torch = types.SimpleNamespace(
        cuda=types.SimpleNamespace(
            is_bf16_supported=lambda including_emulation=True: True,
        )
    )

    monkeypatch.setattr(
        "soup_cli.utils.fp8.is_fp8_gpu_supported",
        lambda: False,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision._torchao_available",
        lambda: True,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision.is_blackwell_gpu",
        lambda: True,
    )
    fake_torchao = types.ModuleType("torchao")
    fake_quantization = types.ModuleType("torchao.quantization")
    fake_quantization.quantize_ = lambda *args, **kwargs: None
    fake_torchao.quantization = fake_quantization
    monkeypatch.setitem(sys.modules, "torchao", fake_torchao)
    monkeypatch.setitem(sys.modules, "torchao.quantization", fake_quantization)

    from soup_cli.commands.doctor import _get_precision_capabilities

    result = _get_precision_capabilities(fake_torch)

    assert result["NVFP4"] == (True, False)


def test_is_nvfp4_software_supported_requires_nvfp4_training_config(monkeypatch):
    from tests.test_v07121 import _install_fake_torchao_quant

    _install_fake_torchao_quant(monkeypatch, {}, with_nvfp4=False)

    from soup_cli.utils.advanced_precision import is_nvfp4_software_supported

    assert is_nvfp4_software_supported() is False


def test_is_nvfp4_software_supported_accepts_nvfp4_training_config(monkeypatch):
    from tests.test_v07121 import _install_fake_torchao_quant

    _install_fake_torchao_quant(monkeypatch, {})

    from soup_cli.utils.advanced_precision import is_nvfp4_software_supported

    assert is_nvfp4_software_supported() is True


def test_is_nvfp4_software_supported_requires_quantize(monkeypatch):
    from tests.test_v07121 import _install_fake_torchao_quant

    _install_fake_torchao_quant(monkeypatch, {})

    fake_quantization = sys.modules["torchao.quantization"]
    del fake_quantization.quantize_

    from soup_cli.utils.advanced_precision import is_nvfp4_software_supported

    assert is_nvfp4_software_supported() is False


def test_get_precision_capabilities_requires_torchao_for_fp8_attention(
    monkeypatch,
):
    """FP8 software support follows the torchao-specific attention gate."""
    import types

    fake_torch = types.SimpleNamespace(
        cuda=types.SimpleNamespace(
            is_bf16_supported=lambda including_emulation=True: True,
        )
    )

    monkeypatch.setattr(
        "soup_cli.utils.fp8.is_fp8_gpu_supported",
        lambda: True,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision._torchao_available",
        lambda: False,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision.is_blackwell_gpu",
        lambda: False,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision.is_nvfp4_software_supported",
        lambda: False,
    )

    from soup_cli.commands.doctor import _get_precision_capabilities

    result = _get_precision_capabilities(fake_torch)

    assert result["FP8"] == (True, False)


def test_get_precision_capabilities_reports_negative_bf16(monkeypatch):
    """BF16 hardware support must reflect the real Torch probe."""
    import importlib.machinery
    import types

    fake_cuda = types.SimpleNamespace(
        is_bf16_supported=lambda including_emulation=True: False,
    )
    fake_torch = types.SimpleNamespace(
        __spec__=importlib.machinery.ModuleSpec("torch", None),
        cuda=fake_cuda,
    )
    monkeypatch.setitem(sys.modules, "torch", fake_torch)

    monkeypatch.setattr(
        "soup_cli.utils.fp8.is_fp8_gpu_supported",
        lambda: False,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision.is_blackwell_gpu",
        lambda: False,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision._torchao_available",
        lambda: False,
    )

    from soup_cli.commands.doctor import _get_precision_capabilities

    result = _get_precision_capabilities(fake_torch)

    assert result["BF16"] == (False, None)


def test_get_precision_capabilities_bf16_matches_trainer_gate_on_pre_ampere(
    monkeypatch,
):
    """Doctor must match the trainer's non-emulated BF16 hardware gate."""
    import importlib.machinery
    import types

    def is_bf16_supported(including_emulation=True):
        return including_emulation

    fake_torch = types.SimpleNamespace(
        __spec__=importlib.machinery.ModuleSpec("torch", None),
        cuda=types.SimpleNamespace(
            is_available=lambda: True,
            get_device_capability=lambda idx=0: (7, 5),
            is_bf16_supported=is_bf16_supported,
        ),
    )
    monkeypatch.setitem(sys.modules, "torch", fake_torch)
    monkeypatch.setattr(
        "soup_cli.utils.fp8.is_fp8_gpu_supported",
        lambda: False,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision.is_blackwell_gpu",
        lambda: False,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision._torchao_available",
        lambda: False,
    )

    from soup_cli.commands.doctor import _get_precision_capabilities
    from soup_cli.utils.gpu import cuda_supports_bf16

    assert cuda_supports_bf16() is False
    assert _get_precision_capabilities(fake_torch)["BF16"] == (False, None)


def test_doctor_gpu_panel_warns_when_torch_lacks_gpu_arch(monkeypatch):
    """GPU panel warns when Torch does not include the detected GPU architecture."""
    import importlib.machinery
    import types

    fake_cuda = types.SimpleNamespace(
        is_available=lambda: True,
        device_count=lambda: 1,
        get_device_name=lambda idx: "NVIDIA GeForce RTX 5070 Laptop GPU",
        get_device_properties=lambda idx: types.SimpleNamespace(
            total_memory=8 * 1024**3
        ),
        get_device_capability=lambda idx: (12, 0),
        get_arch_list=lambda: ["sm_75", "sm_80", "sm_90"],
        is_bf16_supported=lambda including_emulation=True: True,
    )

    fake_torch = types.SimpleNamespace(
        __spec__=importlib.machinery.ModuleSpec("torch", None),
        cuda=fake_cuda,
        version=types.SimpleNamespace(cuda="13.0"),
    )

    monkeypatch.setitem(__import__("sys").modules, "torch", fake_torch)
    monkeypatch.setattr(
        "soup_cli.commands.doctor._installed_version_str",
        lambda import_name, pkg_name: None,
    )
    monkeypatch.setattr(
        "soup_cli.utils.fp8.is_fp8_gpu_supported",
        lambda: True,
    )
    monkeypatch.setattr(
        "soup_cli.utils.fp8.is_fp8_available",
        lambda: True,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision.is_blackwell_gpu",
        lambda: True,
    )
    monkeypatch.setattr(
        "soup_cli.utils.advanced_precision._torchao_available",
        lambda: True,
    )

    result = runner.invoke(
        app,
        ["doctor"],
        env={"COLUMNS": "120", "FORCE_COLOR": "1"},
    )

    assert result.exit_code == 0
    normalized = _plain(result.output)
    assert "Torch build" in normalized
    assert "architecture" in normalized


def test_doctor_shows_gpu_section():
    """soup doctor shows GPU section."""
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    assert "GPU" in _strip_ansi(result.output)


def test_doctor_shows_system_resources():
    """soup doctor shows System Resources section."""
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    output = _strip_ansi(result.output)
    assert "System Resources" in output
    assert "RAM" in output
    assert "Disk" in output


def test_doctor_checks_torch():
    """soup doctor checks for torch."""
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    assert "torch" in _strip_ansi(result.output)


def test_doctor_checks_pydantic():
    """soup doctor checks for pydantic."""
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    assert "pydantic" in _strip_ansi(result.output)


def test_doctor_checks_optional_deps():
    """soup doctor shows optional deps."""
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    assert "optional" in _strip_ansi(result.output)


def test_doctor_missing_dep():
    """soup doctor reports a missing required dep and exits non-zero (#828)."""
    with patch(
        "soup_cli.commands.doctor.DEPS",
        [
            ("nonexistent_fake_pkg_xyz", "nonexistent-pkg", "1.0.0", True),
        ],
    ):
        result = runner.invoke(app, ["doctor"])
        assert result.exit_code == 1
        assert "MISSING" in _strip_ansi(result.output)


def test_doctor_outdated_dep():
    """soup doctor reports outdated dep."""
    with patch(
        "soup_cli.commands.doctor.DEPS",
        [
            ("sys", "sys", "999.0.0", True),  # sys has no __version__ but import won't fail
        ],
    ):
        result = runner.invoke(app, ["doctor"])
        assert result.exit_code == 0
        # Either outdated or OK (depends on version attr presence)


# --- NCCL Check tests ---


def test_doctor_missing_train_extra_suggests_extra(monkeypatch):
    """A core-only install suggests the [train] extra, not bare floors (#828)."""
    for name in (
        "torch",
        "transformers",
        "peft",
        "trl",
        "datasets",
        "bitsandbytes",
        "accelerate",
    ):
        monkeypatch.setitem(sys.modules, name, None)
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    out = _strip_ansi(result.output)
    assert 'pip install "soup-cli[train]"' in out
    assert "torch>=2.6.0" not in out
    assert "transformers>=5.16.1" not in out
    assert "peft>=0.20.0" not in out
    assert "trl>=0.29.0" not in out
    assert "datasets>=2.14.0" not in out
    assert "bitsandbytes>=0.41.0" not in out
    assert "accelerate>=0.27.0" not in out


def test_doctor_partial_train_extra_names_missing_members(monkeypatch):
    """A partial [train] install names the absent members, not the whole stack (#875)."""
    installed = {
        "torch": "2.6.0",
        "transformers": "5.16.1",
        "peft": "0.20.0",
        "trl": "0.29.0",
        "datasets": "2.14.0",
        "accelerate": "0.27.0",
    }

    def _fake_version(import_name, pkg_name):
        return installed.get(pkg_name)

    monkeypatch.setattr("soup_cli.commands.doctor._installed_version_str", _fake_version)
    monkeypatch.setattr("soup_cli.commands.doctor._nvidia_smi_cuda_version", lambda: None)
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    out = _strip_ansi(result.output)
    assert "Training stack not installed" not in out
    assert "Training stack incomplete, missing: bitsandbytes" in out
    assert 'pip install "soup-cli[train]"' in out
    assert "All checks passed!" not in out


def test_doctor_missing_train_extra_message_unchanged_when_none_installed(monkeypatch):
    """With no [train] members, the existing not-installed message is kept (#875)."""
    monkeypatch.setattr(
        "soup_cli.commands.doctor._installed_version_str", lambda import_name, pkg_name: None
    )
    monkeypatch.setattr("soup_cli.commands.doctor._nvidia_smi_cuda_version", lambda: None)
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    out = _strip_ansi(result.output)
    assert 'Training stack not installed: pip install "soup-cli[train]"' in out
    assert "Training stack incomplete" not in out
    assert "All checks passed!" not in out


def test_doctor_missing_train_extra_with_unsupported_driver_falls_back_to_extra(monkeypatch):
    """Do not construct a whl/None URL when the driver has no supported wheel."""
    for name in (
        "torch",
        "transformers",
        "peft",
        "trl",
        "datasets",
        "bitsandbytes",
        "accelerate",
    ):
        monkeypatch.setitem(sys.modules, name, None)

    monkeypatch.setattr(
        "soup_cli.commands.doctor._nvidia_smi_cuda_version",
        lambda: (11, 7),
    )
    result = runner.invoke(app, ["doctor"])

    assert result.exit_code == 0
    out = _strip_ansi(result.output)
    assert 'pip install "soup-cli[train]"' in out
    assert "download.pytorch.org/whl/" not in out
    assert "whl/None" not in out


def test_doctor_suggestion_is_colour_safe(monkeypatch):
    """The [train] suggestion survives Rich highlighting (#828 review)."""
    from rich.console import Console

    monkeypatch.setattr("soup_cli.commands.doctor.console", Console(force_terminal=True))
    for name in (
        "torch",
        "transformers",
        "peft",
        "trl",
        "datasets",
        "bitsandbytes",
        "accelerate",
    ):
        monkeypatch.setitem(sys.modules, name, None)
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    assert 'pip install "soup-cli[train]"' in _strip_ansi(result.output)


def test_doctor_fix_all_line_keeps_extra_marker(monkeypatch):
    """The Fix all line must not drop [train] to Rich markup (#828 review)."""
    for name in (
        "torch",
        "transformers",
        "peft",
        "trl",
        "datasets",
        "bitsandbytes",
        "accelerate",
    ):
        monkeypatch.setitem(sys.modules, name, None)
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    out = _strip_ansi(result.output)
    assert "Fix all:" in out
    after = out.split("Fix all:", 1)[1]
    assert 'pip install "soup-cli[train]"' in after


def test_doctor_table_keeps_extra_marker(monkeypatch):
    """The Required column must render [train] literally, not blank (#828 review)."""
    for name in (
        "torch",
        "transformers",
        "peft",
        "trl",
        "datasets",
        "bitsandbytes",
        "accelerate",
    ):
        monkeypatch.setitem(sys.modules, name, None)
    result = runner.invoke(app, ["doctor"])
    out = _strip_ansi(result.output)
    torch_rows = [line for line in out.splitlines() if "torch" in line]
    assert torch_rows, "expected a table row for torch"
    assert any("[train]" in line for line in torch_rows)


def test_doctor_incompatible_train_member_is_reported(monkeypatch):
    """A [train] member past its breaking-major ceiling must add an issue (#828 review)."""
    monkeypatch.setattr(
        "soup_cli.commands.doctor.EXTRA_GROUPS",
        [("train", [("transformers", "transformers", "5.16.1")])],
    )
    monkeypatch.setattr(
        "soup_cli.commands.doctor._installed_version_str", lambda import_name, pkg_name: "6.1.0"
    )
    result = runner.invoke(app, ["doctor"])
    out = _strip_ansi(result.output)
    assert "INCOMPATIBLE" in out
    assert 'Downgrade transformers: pip install "transformers>=5.16.1,<6.0.0"' in out
    assert "All checks passed!" not in out


def test_doctor_out_of_range_train_member_is_reported(monkeypatch):
    """An installed-but-out-of-range [train] member must add an issue (#828 review)."""
    monkeypatch.setattr(
        "soup_cli.commands.doctor.EXTRA_GROUPS",
        [("train", [("pydantic", "pydantic", "999.0.0")])],
    )
    result = runner.invoke(app, ["doctor"])
    out = _strip_ansi(result.output)
    assert "outdated" in out
    assert "All checks passed!" not in out


def test_doctor_incompatible_train_member_exits_nonzero(monkeypatch):
    """A [train] member past its ceiling makes `soup doctor` exit non-zero (#874)."""
    monkeypatch.setattr(
        "soup_cli.commands.doctor.EXTRA_GROUPS",
        [("train", [("transformers", "transformers", "5.16.1")])],
    )
    monkeypatch.setattr(
        "soup_cli.commands.doctor._installed_version_str", lambda import_name, pkg_name: "6.1.0"
    )
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 1
    assert "INCOMPATIBLE" in _strip_ansi(result.output)


def test_doctor_incompatible_core_dependency_exits_nonzero(monkeypatch):
    """A core dependency past its ceiling makes `soup doctor` exit non-zero (#874)."""
    monkeypatch.setattr("soup_cli.commands.doctor.DEPS", [("typer", "typer", "0.1.0", True)])
    monkeypatch.setattr("soup_cli.commands.doctor._MAX_EXCLUSIVE", {"typer": "0.2.0"})
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 1
    assert "INCOMPATIBLE" in _strip_ansi(result.output)


def test_doctor_outdated_train_member_exits_zero(monkeypatch):
    """An outdated [train] member stays advisory: only beyond-ceiling blocks (#874)."""
    monkeypatch.setattr(
        "soup_cli.commands.doctor.EXTRA_GROUPS",
        [("train", [("transformers", "transformers", "5.16.1")])],
    )
    monkeypatch.setattr(
        "soup_cli.commands.doctor._installed_version_str", lambda import_name, pkg_name: "4.0.0"
    )
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    assert "outdated" in _strip_ansi(result.output)


def test_doctor_partial_train_group_in_range_exits_zero(monkeypatch):
    """A partially installed [train] group with in-range members stays exit 0 (#874)."""
    monkeypatch.setattr(
        "soup_cli.commands.doctor.EXTRA_GROUPS",
        [
            (
                "train",
                [
                    ("transformers", "transformers", "5.16.1"),
                    ("bitsandbytes", "bitsandbytes", "0.41.0"),
                ],
            )
        ],
    )
    monkeypatch.setattr(
        "soup_cli.commands.doctor._installed_version_str",
        lambda import_name, pkg_name: "5.16.1" if pkg_name == "transformers" else None,
    )
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    out = _strip_ansi(result.output)
    assert "OK" in out
    assert "not installed" in out


def test_doctor_nvidia_train_suggestion_is_two_step(monkeypatch):
    """On an NVIDIA box the [train] suggestion installs torch from its own index (#828 review)."""
    monkeypatch.setattr(
        "soup_cli.commands.doctor._nvidia_smi_cuda_version", lambda: (13, 0)
    )
    for name in ("torch", "transformers", "peft", "trl", "datasets", "bitsandbytes", "accelerate"):
        monkeypatch.setitem(sys.modules, name, None)
    result = runner.invoke(app, ["doctor"])
    out = _strip_ansi(result.output)
    assert "pip install torch --index-url https://download.pytorch.org/whl/" in out
    assert 'pip install "soup-cli[train]"' in out
    # The broken single-step form must be gone.
    assert '["soup-cli[train]" --index-url' not in out and '[train]" --index-url' not in out


def test_doctor_nvidia_partial_stack_with_torch_missing(monkeypatch):
    """NVIDIA + partial [train] + torch missing keeps the two-step index URL (#884)."""
    installed = {
        "transformers": "5.16.1",
        "peft": "0.20.0",
        "trl": "0.29.0",
        "datasets": "2.14.0",
        "bitsandbytes": "0.41.0",
        "accelerate": "0.27.0",
    }

    def _fake_version(import_name, pkg_name):
        return installed.get(pkg_name)

    monkeypatch.setattr("soup_cli.commands.doctor._installed_version_str", _fake_version)
    monkeypatch.setattr(
        "soup_cli.commands.doctor._nvidia_smi_cuda_version", lambda: (13, 0)
    )
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    out = _strip_ansi(result.output)
    assert "Training stack incomplete, missing: torch" in out
    assert "pip install torch --index-url https://download.pytorch.org/whl/" in out
    assert 'pip install "soup-cli[train]"' in out
    assert "All checks passed!" not in out
    assert "Training stack not installed" not in out


def test_doctor_nvidia_partial_stack_with_torch_present(monkeypatch):
    """NVIDIA + partial [train] + torch installed drops the index URL (#884)."""
    installed = {
        "torch": "2.6.0",
        "transformers": "5.16.1",
        "peft": "0.20.0",
        "trl": "0.29.0",
        "datasets": "2.14.0",
        "accelerate": "0.27.0",
    }  # bitsandbytes missing, torch present
    monkeypatch.setattr(
        "soup_cli.commands.doctor._installed_version_str",
        lambda import_name, pkg_name: installed.get(pkg_name),
    )
    monkeypatch.setattr(
        "soup_cli.commands.doctor._nvidia_smi_cuda_version", lambda: (13, 0)
    )
    # The GPU panel runs the REAL nvidia-smi when torch has no CUDA; on a box
    # with a GPU that prints its own reinstall hint (with --index-url), which is
    # not the partial-stack hint under test. Pin it off so every box agrees.
    monkeypatch.setattr("soup_cli.commands.doctor._nvidia_smi_executable", lambda: None)
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0
    out = _strip_ansi(result.output)
    assert "Training stack incomplete, missing: bitsandbytes" in out
    assert "index-url" not in out


def test_doctor_missing_core_dependency_exits_nonzero(monkeypatch):
    """A missing core dependency makes `soup doctor` exit non-zero (#828)."""
    monkeypatch.setitem(sys.modules, "plotext", None)
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code != 0
    assert "MISSING" in _strip_ansi(result.output)


def test_doctor_full_install_exits_zero():
    """Runs against the full dev install: missing [train] stays advisory (#828)."""
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0


def test_installed_extras_lists_train_and_mcp(monkeypatch):
    """_installed_extras() derives train and mcp from dist metadata (#828)."""
    import importlib.metadata

    class _FakeMeta:
        def get_all(self, key):
            if key == "Provides-Extra":
                return ["train", "mcp", "serve", "data"]
            return None

    requires = [
        "torch>=2.6.0; extra == 'train'",
        "transformers>=5.16.1; extra == 'train'",
        "mcp>=1.10.0; extra == 'mcp'",
        "fastapi>=0.104.0; extra == 'serve'",
        "scikit-learn>=1.3.0; extra == 'data'",
    ]
    present = {"torch", "transformers", "mcp", "fastapi", "scikit-learn"}

    def _distribution(name):
        if name not in present:
            raise importlib.metadata.PackageNotFoundError(name)
        return object()

    monkeypatch.setattr(importlib.metadata, "metadata", lambda name: _FakeMeta())
    monkeypatch.setattr(importlib.metadata, "requires", lambda name: list(requires))
    monkeypatch.setattr(importlib.metadata, "distribution", _distribution)

    from soup_cli.cli import _installed_extras

    extras = _installed_extras()
    assert "train" in extras
    assert "mcp" in extras
    assert "data" in extras


def test_doctor_nccl_no_gpu():
    """--nccl with <2 GPUs prints a skip message."""
    with (
        patch("torch.cuda.is_available", return_value=True),
        patch("torch.distributed.is_available", return_value=True),
        patch(
            "soup_cli.utils.topology.detect_topology",
            return_value={"gpu_count": 1, "nvlink_pairs": 0, "interconnect": "single"},
        ),
    ):
        result = runner.invoke(app, ["doctor", "--nccl"])
        assert result.exit_code == 0
        # _strip_ansi, like the newer tests in this file: under colour Rich
        # splits the message with SGR codes and the substring match fails
        # for a reason unrelated to NCCL (#886). These two simply predate
        # the helper defined at the top of this module.
        assert "NCCL bandwidth requires >=2 GPUs" in _strip_ansi(result.output)


def test_doctor_nccl_mocked_success():
    """--nccl with 2 GPUs runs the check and displays result."""

    # We mock mp.spawn to just set a value in the return_dict instead of actually running processes.
    def mock_spawn(func, args, nprocs, join):
        return_dict = args[0]
        return_dict["gb_per_sec"] = 350.0  # mock value

    with (
        patch("torch.cuda.is_available", return_value=True),
        patch("torch.distributed.is_available", return_value=True),
        patch(
            "soup_cli.utils.topology.detect_topology",
            return_value={"gpu_count": 2, "nvlink_pairs": 1, "interconnect": "nvlink"},
        ),
        patch("torch.cuda.get_device_name", return_value="NVIDIA H100 80GB HBM3"),
        patch("torch.multiprocessing.spawn", side_effect=mock_spawn),
    ):
        result = runner.invoke(app, ["doctor", "--nccl"])
        assert result.exit_code == 0
        assert "Measuring NCCL bandwidth" in _strip_ansi(result.output)
        assert "Result (H100 over NVLINK)" in _strip_ansi(result.output)


# --- Doctor --config tests (#2100) ---


def _mock_clean_doctor_env(monkeypatch, missing: tuple[str, ...] | list[str] = ()):
    installed = {
        "pydantic": "2.10.0",
        "typer": "0.15.4",
        "rich": "13.0.0",
        "pyyaml": "6.0",
        "plotext": "5.2.0",
        "torch": "2.6.0",
        "transformers": "5.16.1",
        "peft": "0.20.0",
        "trl": "0.29.0",
        "datasets": "2.14.0",
        "bitsandbytes": "0.41.0",
        "accelerate": "0.27.0",
    }
    for pkg in missing:
        installed.pop(pkg, None)
    monkeypatch.setattr(
        "soup_cli.commands.doctor._installed_version_str",
        lambda imp, pkg: installed.get(pkg),
    )
    monkeypatch.setattr(
        "soup_cli.commands.doctor._check_torchvision_compat",
        lambda issues: None,
    )


def test_doctor_refused_config_does_not_print_all_checks_passed(tmp_path, monkeypatch):
    """A refused config must not print 'All checks passed!' before exit 2 (#2100)."""
    _mock_clean_doctor_env(monkeypatch)
    bad_config = tmp_path / "soup.yaml"
    bad_config.write_text(
        "base: meta-llama/Llama-3.1-8B-Instruct\nunknown_key: true\n",
        encoding="utf-8",
    )
    result = runner.invoke(app, ["doctor", "--config", str(bad_config)])
    assert result.exit_code == 2, (result.output, repr(result.exception))
    out = _strip_ansi(result.output)
    assert "All checks passed!" not in out
    assert "unknown config key 'unknown_key'" in out
    assert "Config could not be loaded" in out


def test_doctor_missing_package_and_refused_config_shows_issues_and_refusal(tmp_path, monkeypatch):
    """Environment issues are not hidden when a refused config exits 2 (#2100)."""
    _mock_clean_doctor_env(monkeypatch, missing=["peft"])
    bad_config = tmp_path / "soup.yaml"
    bad_config.write_text(
        "base: meta-llama/Llama-3.1-8B-Instruct\nunknown_key: true\n",
        encoding="utf-8",
    )
    result = runner.invoke(app, ["doctor", "--config", str(bad_config)])
    assert result.exit_code == 2, (result.output, repr(result.exception))
    out = _strip_ansi(result.output)
    assert "Found 1 issue(s):" in out
    assert 'pip install "soup-cli[train]"' in out
    assert "unknown config key 'unknown_key'" in out
    assert "Config could not be loaded" in out
    assert "All checks passed!" not in out


def test_doctor_missing_package_and_valid_config_omits_summary_and_reports_config(
    sample_config, monkeypatch
):
    """When an issue exists, a valid config is reported but summary is omitted (#2100)."""
    _mock_clean_doctor_env(monkeypatch, missing=["peft"])
    result = runner.invoke(app, ["doctor", "--config", str(sample_config)])
    assert result.exit_code == 0, (result.output, repr(result.exception))
    out = _strip_ansi(result.output)
    assert "Found 1 issue(s):" in out
    assert "All checks passed!" not in out
    assert "Config check" in out
    assert out.index("Found 1 issue(s):") < out.index("Config check")


def test_doctor_valid_config_prints_all_checks_passed(sample_config, monkeypatch):
    """A valid config check allows 'All checks passed!' on a healthy environment (#2100)."""
    _mock_clean_doctor_env(monkeypatch)
    result = runner.invoke(app, ["doctor", "--config", str(sample_config)])
    assert result.exit_code == 0, (result.output, repr(result.exception))
    out = _strip_ansi(result.output)
    assert "All checks passed!" in out
    assert "Config check" in out


def test_doctor_loadable_config_with_gaps_prints_findings_after_summary(tmp_path, monkeypatch):
    """Config check findings for a loadable config are printed after summary (#2100)."""
    _mock_clean_doctor_env(monkeypatch)
    gap_config = tmp_path / "soup.yaml"
    gap_config.write_text(
        "base: Qwen/Qwen2-VL-7B-Instruct\n"
        "task: sft\n"
        "modality: vision\n"
        "data:\n"
        "  train: ./data.jsonl\n"
        "  format: llava\n"
        "  mask_history: true\n",
        encoding="utf-8",
    )
    result = runner.invoke(app, ["doctor", "--config", str(gap_config)])
    assert result.exit_code == 0, (result.output, repr(result.exception))
    out = _strip_ansi(result.output)
    assert "All checks passed!" in out
    assert "Config check" in out
    assert out.index("All checks passed!") < out.index("Config check")


def test_doctor_without_config_unchanged(monkeypatch):
    """Without --config, doctor still prints 'All checks passed!' and exits 0 (#2100)."""
    _mock_clean_doctor_env(monkeypatch)
    result = runner.invoke(app, ["doctor"])
    assert result.exit_code == 0, (result.output, repr(result.exception))
    out = _strip_ansi(result.output)
    assert "All checks passed!" in out


def test_doctor_nonexistent_config_exits_2(tmp_path, monkeypatch):
    """A nonexistent config path prints 'Config not found' and exits 2 (#2100)."""
    _mock_clean_doctor_env(monkeypatch)
    missing_config = tmp_path / "does_not_exist.yaml"
    result = runner.invoke(app, ["doctor", "--config", str(missing_config)])
    assert result.exit_code == 2, (result.output, repr(result.exception))
    out = _strip_ansi(result.output)
    assert "Config not found:" in out
    assert "All checks passed!" not in out


def test_doctor_load_error_exception_exits_2(monkeypatch):
    """An unexpected exception during config load prints error detail and exits 2 (#2100)."""
    _mock_clean_doctor_env(monkeypatch)

    def _fail_load(path):
        raise RuntimeError("disk read failed")

    monkeypatch.setattr("soup_cli.config.loader.load_config", _fail_load)
    result = runner.invoke(app, ["doctor", "--config", "unreadable.yaml"])
    assert result.exit_code == 2, (result.output, repr(result.exception))
    out = _strip_ansi(result.output)
    assert "Config could not be loaded: disk read failed" in out
    assert "All checks passed!" not in out