Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

41.9 KB
Raw Blame
"""Tests for v0.27.0 Multi-GPU Mastery.

Covers: topology, launcher, ZeRO++, FSDP+compile, MII, pipeline, recipes.
"""

from __future__ import annotations

import json
import os
import re
import shlex
import sys
from unittest.mock import MagicMock, patch

import pytest
import yaml

# =============================================================================
# Part A: Topology detector + --gpus flag
# =============================================================================

class TestResolveNumGpus:
    """Test --gpus resolution: int, 'auto', None."""

    def test_resolve_integer(self):
        from soup_cli.utils.topology import resolve_num_gpus

        assert resolve_num_gpus("4") == 4
        assert resolve_num_gpus(2) == 2

    def test_resolve_auto_uses_detected(self):
        from soup_cli.utils.topology import resolve_num_gpus

        with patch(
            "soup_cli.utils.topology._detected_gpu_count", return_value=8
        ):
            assert resolve_num_gpus("auto") == 8

    def test_resolve_auto_cpu_returns_zero(self):
        from soup_cli.utils.topology import resolve_num_gpus

        with patch(
            "soup_cli.utils.topology._detected_gpu_count", return_value=0
        ):
            assert resolve_num_gpus("auto") == 0

    def test_resolve_none_returns_none(self):
        from soup_cli.utils.topology import resolve_num_gpus

        assert resolve_num_gpus(None) is None

    def test_resolve_invalid_negative(self):
        from soup_cli.utils.topology import resolve_num_gpus

        with pytest.raises(ValueError, match="must be"):
            resolve_num_gpus("-1")

    def test_resolve_invalid_zero(self):
        from soup_cli.utils.topology import resolve_num_gpus

        with pytest.raises(ValueError, match="must be"):
            resolve_num_gpus("0")

    def test_resolve_invalid_string(self):
        from soup_cli.utils.topology import resolve_num_gpus

        with pytest.raises(ValueError, match="Invalid"):
            resolve_num_gpus("all")

    def test_resolve_out_of_bounds(self):
        from soup_cli.utils.topology import resolve_num_gpus

        with pytest.raises(ValueError, match="exceeds"):
            resolve_num_gpus("1024")


class TestDetectTopology:
    """Test topology sniffing (NVLink / PCIe) — all mocked."""

    def test_no_cuda_returns_none_interconnect(self):
        from soup_cli.utils.topology import detect_topology

        with patch(
            "soup_cli.utils.topology._detected_gpu_count", return_value=0
        ):
            topo = detect_topology()
            assert topo["gpu_count"] == 0
            assert topo["interconnect"] == "none"
            assert topo["nvlink_pairs"] == 0

    def test_single_gpu_has_no_interconnect(self):
        from soup_cli.utils.topology import detect_topology

        with patch(
            "soup_cli.utils.topology._detected_gpu_count", return_value=1
        ):
            topo = detect_topology()
            assert topo["gpu_count"] == 1
            assert topo["interconnect"] == "single"
            assert topo["nvlink_pairs"] == 0

    def test_multi_gpu_nvlink_detected(self):
        from soup_cli.utils.topology import detect_topology

        # Mock: 4 GPUs, all connected via NVLink
        with (
            patch(
                "soup_cli.utils.topology._detected_gpu_count", return_value=4
            ),
            patch(
                "soup_cli.utils.topology._count_nvlink_pairs", return_value=6
            ),
        ):
            topo = detect_topology()
            assert topo["gpu_count"] == 4
            assert topo["interconnect"] == "nvlink"
            assert topo["nvlink_pairs"] == 6

    def test_multi_gpu_pcie_fallback(self):
        from soup_cli.utils.topology import detect_topology

        # 2 GPUs, no NVLink
        with (
            patch(
                "soup_cli.utils.topology._detected_gpu_count", return_value=2
            ),
            patch(
                "soup_cli.utils.topology._count_nvlink_pairs", return_value=0
            ),
        ):
            topo = detect_topology()
            assert topo["gpu_count"] == 2
            assert topo["interconnect"] == "pcie"

    def test_nccl_env_suggestions(self):
        from soup_cli.utils.topology import suggest_nccl_env

        env = suggest_nccl_env(gpu_count=4, interconnect="nvlink")
        assert env["NCCL_P2P_DISABLE"] == "0"
        assert env["NCCL_IB_DISABLE"] == "1"  # no IB in local multi-GPU

        env_pcie = suggest_nccl_env(gpu_count=2, interconnect="pcie")
        assert env_pcie["NCCL_P2P_DISABLE"] == "0"

        env_single = suggest_nccl_env(gpu_count=1, interconnect="single")
        assert env_single == {}


class TestSuggestStrategy:
    """Test distributed-strategy suggestion based on model size + GPU count."""

    def test_single_gpu_no_strategy(self):
        from soup_cli.utils.topology import suggest_strategy

        assert suggest_strategy(gpu_count=1, model_size_b=7.0)["strategy"] == "single"

    def test_small_model_multi_gpu_ddp(self):
        from soup_cli.utils.topology import suggest_strategy

        rec = suggest_strategy(gpu_count=2, model_size_b=3.0)
        assert rec["strategy"] in ("ddp", "zero2")

    def test_mid_model_multi_gpu_zero2(self):
        from soup_cli.utils.topology import suggest_strategy

        rec = suggest_strategy(gpu_count=4, model_size_b=13.0)
        assert rec["strategy"] in ("zero2", "zero3", "fsdp_full_shard")

    def test_huge_model_zero3_or_fsdp(self):
        from soup_cli.utils.topology import suggest_strategy

        rec = suggest_strategy(gpu_count=8, model_size_b=70.0)
        assert rec["strategy"] in ("zero3", "fsdp_full_shard", "zero2_offload")

    def test_cpu_suggests_none(self):
        from soup_cli.utils.topology import suggest_strategy

        rec = suggest_strategy(gpu_count=0, model_size_b=7.0)
        assert rec["strategy"] == "none"


class TestTrainGpusFlag:
    """Test --gpus flag in train command."""

    def test_train_help_shows_gpus(self):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        result = runner.invoke(app, ["train", "--help"])
        # Strip ANSI + whitespace per v0.26.0 CI hardening pattern
        out = re.sub(r"\x1b\[[0-9;]*m", "", result.output)
        out = re.sub(r"\s+", " ", out)
        assert "--gpus" in out


# =============================================================================
# Part B: Accelerate launcher wrapper
# =============================================================================

class TestLauncherArgv:
    """Test accelerate-launch argv construction."""

    def test_single_process_no_launcher(self):
        from soup_cli.utils.launcher import build_accelerate_argv

        # num_processes=1 → no wrapper needed
        argv = build_accelerate_argv(num_processes=1, script_args=["soup", "train"])
        assert argv == ["soup", "train"]

    def test_multi_process_wraps(self):
        from soup_cli.utils.launcher import build_accelerate_argv

        argv = build_accelerate_argv(num_processes=4, script_args=["soup", "train"])
        assert argv[0] == "accelerate"
        assert argv[1] == "launch"
        assert "--num_processes" in argv
        # num_processes value follows the flag
        idx = argv.index("--num_processes")
        assert argv[idx + 1] == "4"

    def test_multi_process_mixed_precision(self):
        from soup_cli.utils.launcher import build_accelerate_argv

        argv = build_accelerate_argv(
            num_processes=4, script_args=["soup", "train"], mixed_precision="bf16"
        )
        assert "--mixed_precision" in argv
        idx = argv.index("--mixed_precision")
        assert argv[idx + 1] == "bf16"

    def test_invalid_num_processes(self):
        from soup_cli.utils.launcher import build_accelerate_argv

        with pytest.raises(ValueError, match="num_processes"):
            build_accelerate_argv(num_processes=0, script_args=["soup", "train"])

    def test_invalid_mixed_precision(self):
        from soup_cli.utils.launcher import build_accelerate_argv

        with pytest.raises(ValueError, match="mixed_precision"):
            build_accelerate_argv(
                num_processes=2, script_args=["soup", "train"], mixed_precision="int8"
            )

    def test_invalid_num_machines_low(self):
        from soup_cli.utils.launcher import build_accelerate_argv

        with pytest.raises(ValueError, match="num_machines"):
            build_accelerate_argv(
                num_processes=2, script_args=["soup", "train"], num_machines=0
            )

    def test_invalid_num_machines_high(self):
        from soup_cli.utils.launcher import build_accelerate_argv

        with pytest.raises(ValueError, match="num_machines"):
            build_accelerate_argv(
                num_processes=2, script_args=["soup", "train"], num_machines=10_000
            )


class TestMultiNodeLauncher:
    """CPU-only launch contracts for #40; no distributed training is run."""

    @staticmethod
    def _prefix(rank=0, processes=16, port=29500):
        return [
            "accelerate", "launch", "--num_processes", str(processes),
            "--multi_gpu", "--num_machines", "2", "--machine_rank", str(rank),
            "--main_process_ip", "head.internal", "--main_process_port", str(port),
            "--rdzv_backend", "static",
        ]

    @staticmethod
    def _output(result):
        from tests.conftest import strip_ansi

        return " ".join(strip_ansi(result.output).replace("│", " ").split())

    @pytest.mark.parametrize("rank", [0, 1])
    def test_exact_argv_for_each_node(self, rank):
        from soup_cli.utils.launcher import build_accelerate_argv

        argv = build_accelerate_argv(
            16, [sys.executable, "-m", "soup_cli.cli", "train"],
            num_machines=2, machine_rank=rank,
            main_process_ip="head.internal", main_process_port=29500,
        )
        assert argv == self._prefix(rank) + ["--module", "soup_cli.cli", "train"]

    def test_defaults_and_mixed_precision(self):
        from soup_cli.utils.launcher import build_accelerate_argv

        assert build_accelerate_argv(
            16, ["train.py"], num_machines=2,
            main_process_ip="head.internal", mixed_precision="bf16",
        ) == self._prefix() + ["--mixed_precision", "bf16", "train.py"]

    @pytest.mark.parametrize("processes", [1, 3, 15])
    def test_process_count_must_divide_evenly_between_nodes(self, processes):
        from soup_cli.utils.launcher import build_accelerate_argv

        with pytest.raises(ValueError, match="num_processes"):
            build_accelerate_argv(
                processes, ["train.py"], num_machines=2,
                main_process_ip="head.internal",
            )

    @pytest.mark.parametrize("options, field", [
        ({"num_machines": True}, "num_machines"),
        ({"num_machines": 1.5}, "num_machines"),
        ({"machine_rank": -1}, "machine_rank"),
        ({"machine_rank": 2}, "machine_rank"),
        ({"machine_rank": True}, "machine_rank"),
        ({"machine_rank": "1"}, "machine_rank"),
        ({"main_process_ip": None}, "main_process_ip"),
        ({"main_process_ip": ""}, "main_process_ip"),
        ({"main_process_ip": "bad host"}, "main_process_ip"),
        ({"main_process_ip": "head\x00host"}, "main_process_ip"),
        ({"main_process_ip": "--bad"}, "main_process_ip"),
        ({"main_process_ip": "http://head"}, "main_process_ip"),
        ({"main_process_port": 0}, "main_process_port"),
        ({"main_process_port": 65536}, "main_process_port"),
        ({"main_process_port": True}, "main_process_port"),
        ({"main_process_port": "29500"}, "main_process_port"),
    ])
    def test_rejects_invalid_node_settings(self, options, field):
        from soup_cli.utils.launcher import build_accelerate_argv

        kwargs = {"num_machines": 2, "main_process_ip": "head.internal", **options}
        with pytest.raises(ValueError, match=field):
            build_accelerate_argv(16, ["train.py"], **kwargs)

    @pytest.mark.parametrize("processes", [1, 4])
    def test_single_node_control_is_unchanged(self, processes):
        from soup_cli.utils.launcher import build_accelerate_argv

        expected = ["train.py"] if processes == 1 else [
            "accelerate", "launch", "--num_processes", "4", "train.py",
        ]
        assert build_accelerate_argv(processes, ["train.py"]) == expected

    @pytest.fixture
    def launch_cli(self, tmp_path, monkeypatch):
        from rich.console import Console
        from typer.testing import CliRunner

        from soup_cli.cli import app
        from soup_cli.commands import train as train_cmd
        from soup_cli.utils import topology

        for key in (
            "RANK", "WORLD_SIZE", "LOCAL_RANK", "ACCELERATE_MIXED_PRECISION",
            "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP",
            "NCCL_IB_DISABLE", "NCCL_P2P_DISABLE", "NCCL_NVLS_ENABLE",
        ):
            monkeypatch.delenv(key, raising=False)
        monkeypatch.chdir(tmp_path)
        monkeypatch.setenv("COLUMNS", "1000")
        monkeypatch.setenv("FORCE_COLOR", "1")
        (tmp_path / "soup.yaml").write_text(
            "base: test/model\ntask: sft\n"
            "data: {train: data.jsonl, format: alpaca}\n"
            "training: {epochs: 1, lr: 1e-4, batch_size: 1}\n",
            encoding="utf-8",
        )
        monkeypatch.setattr(topology, "_detected_gpu_count", lambda: 8)
        monkeypatch.setattr(topology, "detect_topology", lambda: {
            "gpu_count": 8, "interconnect": "pcie",
        })
        monkeypatch.setattr(train_cmd, "console", Console(width=1000, force_terminal=True))
        monkeypatch.setattr(train_cmd, "detect_device", lambda **kw: ("cpu", "CPU"))
        monkeypatch.setattr(train_cmd, "get_gpu_info", lambda **kw: {"memory_total": "0 GB"})
        loader = MagicMock(return_value={"train": [{}]})
        monkeypatch.setattr(train_cmd, "load_dataset", loader)
        captured = []

        def fake_run_launcher(argv):
            captured.append((argv[0], list(argv), {
                key: value for key, value in os.environ.items() if key.startswith("NCCL_")
            }))
            raise SystemExit(99)

        monkeypatch.setattr("soup_cli.utils.launcher.run_launcher", fake_run_launcher)

        def invoke(args):
            return CliRunner().invoke(
                app, ["train", "--config", "soup.yaml", "--yes", *args], color=True,
            )

        return invoke, captured, loader

    @pytest.mark.parametrize("rank", [0, 1])
    @pytest.mark.parametrize("gpus", ["8", "1", "auto"])
    def test_cli_launches_exact_command_and_env(self, launch_cli, rank, gpus):
        invoke, captured, loader = launch_cli
        result = invoke([
            "--gpus", gpus, "--nodes", "2", "--node-rank", str(rank),
            "--master-addr", "head.internal", "--master-port", "29601",
        ])
        assert result.exit_code == 99, (self._output(result), result.exception)
        expected = self._prefix(rank, 2 if gpus == "1" else 16, 29601) + [
            "--module", "soup_cli.cli", "train", "--config", "soup.yaml",
            "--no-reexec", "--yes",
        ]
        assert captured == [("accelerate", expected, {
            "NCCL_P2P_DISABLE": "0", "NCCL_IB_DISABLE": "0",
        })]
        loader.assert_not_called()

    def test_no_reexec_prints_the_same_argv_and_does_not_mutate_env(self, launch_cli):
        invoke, captured, loader = launch_cli
        args = ["--gpus", "8", "--nodes", "2", "--node-rank", "1",
                "--master-addr", "head.internal", "--name", "space in name"]
        result = invoke([*args, "--no-reexec"])
        out = self._output(result)
        assert result.exit_code == 1, (out, result.exception)
        assert not captured
        assert "NCCL_IB_DISABLE=0" in out
        assert "NCCL_IB_DISABLE" not in os.environ
        loader.assert_not_called()
        result = invoke(args)
        assert result.exit_code == 99, (self._output(result), result.exception)
        printed = out[out.index("accelerate launch "):].split("Run this command", 1)[0]
        assert shlex.split(printed) == captured[0][1]

    @pytest.mark.parametrize("no_reexec", [False, True])
    def test_dry_run_validates_data_without_launching_or_changing_env(
        self, launch_cli, no_reexec,
    ):
        invoke, captured, loader = launch_cli
        args = ["--gpus", "8", "--nodes", "2", "--master-addr", "head.internal",
                "--dry-run"]
        result = invoke(args + (["--no-reexec"] if no_reexec else []))
        out = self._output(result)
        assert result.exit_code == 0, (out, result.exception)
        assert "--num_processes 16" in out
        assert "Config valid" in out
        assert not captured
        assert "NCCL_IB_DISABLE" not in os.environ
        loader.assert_called_once()

    @pytest.mark.parametrize("args, message", [
        (["--gpus", "8", "--nodes", "2"], "--master-addr"),
        (["--nodes", "2", "--master-addr", "head.internal"], "--gpus"),
        (["--gpus", "8", "--nodes", "0"], "--nodes"),
        (["--gpus", "8", "--nodes", "257"], "--nodes"),
        (["--gpus", "8", "--nodes", "2", "--node-rank", "2",
          "--master-addr", "head.internal"], "--node-rank"),
        (["--gpus", "8", "--nodes", "2", "--master-addr", "head.internal",
          "--master-port", "0"], "--master-port"),
        (["--gpus", "8", "--master-addr", "head.internal"], "--nodes"),
        (["--gpus", "8", "--nodes", "2", "--master-addr", "head.internal",
          "--cloud", "modal"], "--cloud"),
        (["--gpus", "8", "--nodes", "2", "--master-addr", "head.internal",
          "--find-lr"], "--find-lr"),
    ])
    def test_invalid_cli_settings_fail_before_loading_data(self, launch_cli, args, message):
        invoke, captured, loader = launch_cli
        result = invoke(args)
        assert result.exit_code != 0
        assert message in self._output(result), result.exception
        assert not captured
        loader.assert_not_called()

    def test_auto_without_a_local_gpu_refuses_multi_node(self, launch_cli, monkeypatch):
        from soup_cli.utils import topology

        invoke, captured, loader = launch_cli
        monkeypatch.setattr(topology, "_detected_gpu_count", lambda: 0)
        result = invoke(["--gpus", "auto", "--nodes", "2", "--master-addr", "head.internal"])
        assert result.exit_code == 1, result.exception
        assert "at least one GPU per node" in self._output(result)
        assert not captured
        loader.assert_not_called()

    def test_user_nccl_overrides_survive_launch(self, launch_cli, monkeypatch):
        invoke, captured, _ = launch_cli
        monkeypatch.setenv("NCCL_IB_DISABLE", "1")
        monkeypatch.setenv("NCCL_P2P_DISABLE", "1")
        result = invoke(["--gpus", "8", "--nodes", "2", "--master-addr", "head.internal"])
        assert result.exit_code == 99, result.exception
        assert captured[0][2] == {"NCCL_IB_DISABLE": "1", "NCCL_P2P_DISABLE": "1"}

    def test_existing_rank_does_not_launch_again(self, launch_cli, monkeypatch):
        invoke, captured, _ = launch_cli
        monkeypatch.setenv("RANK", "8")
        monkeypatch.setenv("WORLD_SIZE", "16")
        result = invoke([
            "--gpus", "8", "--nodes", "2", "--node-rank", "1",
            "--master-addr", "head.internal", "--dry-run",
        ])
        assert result.exit_code == 0, (self._output(result), result.exception)
        assert not captured
        assert "NCCL_IB_DISABLE" not in os.environ

    def test_cli_help_survives_rich_ansi(self, launch_cli):
        invoke, _, _ = launch_cli
        result = invoke(["--help"])
        assert result.exit_code == 0
        for flag in ("--nodes", "--node-rank", "--master-addr", "--master-port"):
            assert flag in self._output(result)


class TestLauncherDetectActive:
    """Test detection of whether we're running under accelerate/torchrun.

    Uses ``clear=False`` + explicit key deletion so Windows CI keeps
    SYSTEMROOT / PATH / TEMP available for the rest of the process.
    """

    _DIST_KEYS = (
        "RANK",
        "WORLD_SIZE",
        "LOCAL_RANK",
        "ACCELERATE_MIXED_PRECISION",
        "ACCELERATE_USE_DEEPSPEED",
        "ACCELERATE_USE_FSDP",
    )

    def test_not_in_distributed(self):
        from soup_cli.utils.launcher import is_in_distributed

        # Delete any pre-existing distributed markers without wiping env.
        stripped_env = {
            key: val for key, val in os.environ.items() if key not in self._DIST_KEYS
        }
        with patch.dict(os.environ, stripped_env, clear=True):
            assert is_in_distributed() is False

    def test_in_torchrun(self):
        from soup_cli.utils.launcher import is_in_distributed

        env = {"RANK": "0", "WORLD_SIZE": "4", "LOCAL_RANK": "0"}
        with patch.dict(os.environ, env, clear=False):
            assert is_in_distributed() is True

    def test_in_accelerate(self):
        from soup_cli.utils.launcher import is_in_distributed

        env = {"ACCELERATE_MIXED_PRECISION": "bf16", "RANK": "0", "WORLD_SIZE": "2"}
        with patch.dict(os.environ, env, clear=False):
            assert is_in_distributed() is True


class TestLauncherAdvice:
    """Test that launcher prints useful advice message when N>1 and not distributed."""

    def test_format_advice_command(self):
        from soup_cli.utils.launcher import format_advice

        text = format_advice(num_processes=4, script_args=["soup", "train", "-c", "soup.yaml"])
        assert "accelerate launch" in text
        assert "--num_processes 4" in text
        assert "soup train" in text


# =============================================================================
# Part C: ZeRO++ DeepSpeed template
# =============================================================================

class TestZeroPlusPlus:
    """Test ZeRO++ (zero_plus_plus / zero++) config template."""

    def test_zero_pp_config_exists(self):
        from soup_cli.utils.deepspeed import CONFIGS

        assert "zero++" in CONFIGS or "zero_pp" in CONFIGS

    def test_zero_pp_has_stage_3_base(self):
        from soup_cli.utils.deepspeed import get_deepspeed_config

        cfg = get_deepspeed_config("zero++")
        assert cfg["zero_optimization"]["stage"] == 3

    def test_zero_pp_has_hierarchical_comms(self):
        from soup_cli.utils.deepspeed import get_deepspeed_config

        cfg = get_deepspeed_config("zero++")
        zopt = cfg["zero_optimization"]
        # ZeRO++ distinguishing features: quantized weights/gradients + hpz
        assert "zero_quantized_weights" in zopt
        assert "zero_hpz_partition_size" in zopt
        assert "zero_quantized_gradients" in zopt

    def test_zero_pp_write_file(self):
        """#336 — the written config is the *resolved* one.

        This test used to assert ``zero_quantized_weights is True`` on the
        written file. That is the setting that crashed on real hardware: the
        preset enables bf16 and DeepSpeed's quantiser is the fp16 kernel, so
        the dequantised all-gather returned Half into a BFloat16 activation.
        The key is still present — it is a ZeRO++ knob, see
        ``test_zero_pp_has_hierarchical_comms`` — but resolved to ``False``
        for a bf16 run. ``tests/test_issue336_deepspeed_lora.py`` carries the
        fp16 control proving quantisation is a dtype decision, not a blanket
        disable.
        """
        from soup_cli.utils.deepspeed import write_deepspeed_config

        path = write_deepspeed_config("zero++")
        try:
            assert os.path.exists(path)
            with open(path) as f:
                cfg = json.load(f)
            assert cfg["zero_optimization"]["zero_quantized_weights"] is False
            assert "zero_hpz_partition_size" in cfg["zero_optimization"]
        finally:
            if os.path.exists(path):
                os.unlink(path)

    def test_resolve_zero_pp_in_train(self):
        from soup_cli.commands.train import _resolve_deepspeed

        path = _resolve_deepspeed("zero++")
        try:
            assert os.path.exists(path)
        finally:
            if os.path.exists(path):
                os.unlink(path)


# =============================================================================
# Part D: FSDP2 + torch.compile
# =============================================================================

class TestFsdp2Compile:
    """Test FSDP2 + torch.compile integration."""

    def test_config_has_fsdp2_compile_field(self):
        from soup_cli.config.schema import TrainingConfig

        cfg = TrainingConfig(use_fsdp2_compile=True)
        assert cfg.use_fsdp2_compile is True

    def test_config_default_false(self):
        from soup_cli.config.schema import TrainingConfig

        cfg = TrainingConfig()
        assert cfg.use_fsdp2_compile is False

    def test_validate_compile_requires_fsdp(self):
        from soup_cli.utils.fsdp import validate_fsdp2_compile_config

        errors = validate_fsdp2_compile_config(
            use_compile=True, fsdp_preset=None, backend="transformers", device="cuda"
        )
        assert any("FSDP" in e for e in errors)

    def test_validate_compile_requires_cuda(self):
        from soup_cli.utils.fsdp import validate_fsdp2_compile_config

        errors = validate_fsdp2_compile_config(
            use_compile=True, fsdp_preset="full_shard", backend="transformers", device="cpu"
        )
        assert any("CUDA" in e for e in errors)

    def test_validate_compile_requires_transformers_backend(self):
        from soup_cli.utils.fsdp import validate_fsdp2_compile_config

        errors = validate_fsdp2_compile_config(
            use_compile=True, fsdp_preset="full_shard", backend="unsloth", device="cuda"
        )
        assert any("unsloth" in e for e in errors)

    def test_validate_compile_ok(self):
        from soup_cli.utils.fsdp import validate_fsdp2_compile_config

        with patch("soup_cli.utils.fsdp.is_fsdp_available", return_value=True):
            errors = validate_fsdp2_compile_config(
                use_compile=True,
                fsdp_preset="full_shard",
                backend="transformers",
                device="cuda",
            )
            assert errors == []

    def test_validate_compile_disabled_noop(self):
        from soup_cli.utils.fsdp import validate_fsdp2_compile_config

        # When flag is off, no errors ever.
        errors = validate_fsdp2_compile_config(
            use_compile=False, fsdp_preset=None, backend="unsloth", device="cpu"
        )
        assert errors == []


# =============================================================================
# Part E: DeepSpeed-MII serve backend
# =============================================================================

class TestMiiBackend:
    """Test DeepSpeed-MII detection + runtime creation."""

    def test_is_mii_available_false_when_stubbed_to_none(self):
        from soup_cli.utils.mii import is_mii_available

        with patch.dict("sys.modules", {"mii": None}):
            assert is_mii_available() is False

    def test_is_mii_available_false_when_key_absent(self):
        """Cover the ImportError branch — no ``mii`` key in sys.modules at all."""
        from soup_cli.utils.mii import is_mii_available

        saved = sys.modules.pop("mii", "_SENTINEL")
        try:
            # Force the import path to fail by making the real module look gone.
            with patch.dict("sys.modules", {"mii": None}, clear=False):
                assert is_mii_available() is False
        finally:
            if saved != "_SENTINEL":
                sys.modules["mii"] = saved

    def test_is_mii_available_true_when_importable(self):
        from soup_cli.utils.mii import is_mii_available

        fake_mii = MagicMock()
        with patch.dict("sys.modules", {"mii": fake_mii}):
            assert is_mii_available() is True

    def test_create_mii_pipeline_mocked(self):
        from soup_cli.utils.mii import create_mii_pipeline

        fake_mii = MagicMock()
        fake_pipe = MagicMock()
        fake_mii.pipeline.return_value = fake_pipe
        with patch.dict("sys.modules", {"mii": fake_mii}):
            pipe = create_mii_pipeline(
                model_path="/models/llama", tensor_parallel=2, max_length=4096
            )
            assert pipe is fake_pipe
            fake_mii.pipeline.assert_called_once()
            kwargs = fake_mii.pipeline.call_args.kwargs
            assert kwargs.get("tensor_parallel") == 2

    def test_create_mii_raises_when_missing(self):
        from soup_cli.utils.mii import create_mii_pipeline

        with patch.dict("sys.modules", {"mii": None}):
            with pytest.raises(ImportError, match="deepspeed-mii"):
                create_mii_pipeline(model_path="/models/llama")

    def test_serve_help_shows_mii(self):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        result = runner.invoke(app, ["serve", "--help"])
        import re
        out = re.sub(r"\x1b\[[0-9;]*m", "", result.output)
        out = re.sub(r"\s+", " ", out)
        assert "mii" in out.lower()


# =============================================================================
# Part F: Pipeline parallelism config
# =============================================================================

class TestPipelineParallelConfig:
    """Test pipeline-parallel config fields + validation."""

    def test_default_parallelism_data(self):
        from soup_cli.config.schema import TrainingConfig

        cfg = TrainingConfig()
        assert cfg.parallelism == "data"
        assert cfg.pipeline_stages == 1

    def test_set_pipeline(self):
        from soup_cli.config.schema import TrainingConfig

        cfg = TrainingConfig(parallelism="pipeline", pipeline_stages=4)
        assert cfg.parallelism == "pipeline"
        assert cfg.pipeline_stages == 4

    def test_pipeline_stages_bounds_low(self):
        from soup_cli.config.schema import TrainingConfig

        with pytest.raises(ValueError):
            TrainingConfig(pipeline_stages=0)

    def test_pipeline_stages_bounds_high(self):
        from soup_cli.config.schema import TrainingConfig

        with pytest.raises(ValueError):
            TrainingConfig(pipeline_stages=64)

    def test_parallelism_invalid_literal(self):
        from soup_cli.config.schema import TrainingConfig

        with pytest.raises(ValueError):
            TrainingConfig(parallelism="hybrid")

    def test_validate_pipeline_requires_stages(self):
        from soup_cli.utils.pipeline import validate_pipeline_config

        errors = validate_pipeline_config(
            parallelism="pipeline", pipeline_stages=1, device="cuda", gpu_count=4,
        )
        assert any("pipeline_stages" in e for e in errors)

    def test_validate_pipeline_requires_cuda(self):
        from soup_cli.utils.pipeline import validate_pipeline_config

        errors = validate_pipeline_config(
            parallelism="pipeline", pipeline_stages=4, device="cpu", gpu_count=0,
        )
        assert any("CUDA" in e for e in errors)

    def test_validate_pipeline_requires_enough_gpus(self):
        from soup_cli.utils.pipeline import validate_pipeline_config

        errors = validate_pipeline_config(
            parallelism="pipeline", pipeline_stages=4, device="cuda", gpu_count=2,
        )
        assert any("GPUs" in e for e in errors)

    def test_validate_data_parallel_ok(self):
        from soup_cli.utils.pipeline import validate_pipeline_config

        errors = validate_pipeline_config(
            parallelism="data", pipeline_stages=1, device="cpu", gpu_count=0,
        )
        assert errors == []


# =============================================================================
# Part G: Multi-GPU recipes
# =============================================================================

class TestMultiGpuRecipes:
    """Test new multi-GPU recipes in catalog."""

    def test_llama3_70b_fsdp2_recipe_exists(self):
        from soup_cli.recipes.catalog import RECIPES

        assert "llama3-70b-fsdp2" in RECIPES

    def test_qwen3_32b_zeropp_recipe_exists(self):
        from soup_cli.recipes.catalog import RECIPES

        assert "qwen3-32b-zeropp" in RECIPES

    def test_deepseek_v3_pipeline_recipe_exists(self):
        from soup_cli.recipes.catalog import RECIPES

        assert "deepseek-v3-pipeline" in RECIPES

    def test_all_new_recipes_load_as_config(self):
        from soup_cli.config.loader import load_config_from_string
        from soup_cli.recipes.catalog import RECIPES

        for name in ("llama3-70b-fsdp2", "qwen3-32b-zeropp", "deepseek-v3-pipeline"):
            meta = RECIPES[name]
            cfg = load_config_from_string(meta.yaml_str)
            # Exact-match check — substring allowed false positives between
            # ``llama3-8b`` / ``llama3-70b`` recipes.
            assert cfg.base == meta.model

    def test_llama3_70b_fsdp2_has_fsdp2_compile(self):
        from soup_cli.recipes.catalog import RECIPES

        meta = RECIPES["llama3-70b-fsdp2"]
        doc = yaml.safe_load(meta.yaml_str)
        assert doc["training"].get("use_fsdp2_compile") is True

    def test_qwen3_32b_zeropp_tags(self):
        from soup_cli.recipes.catalog import RECIPES

        meta = RECIPES["qwen3-32b-zeropp"]
        # Recipe is multi-GPU focused
        assert "qwen3-32" in meta.model.lower()

    def test_deepseek_v3_pipeline_stages(self):
        from soup_cli.recipes.catalog import RECIPES

        meta = RECIPES["deepseek-v3-pipeline"]
        doc = yaml.safe_load(meta.yaml_str)
        assert doc["training"].get("parallelism") == "pipeline"
        assert doc["training"].get("pipeline_stages") >= 2


# =============================================================================
# Integration: config field → trainer wiring
# =============================================================================

class TestFsdp2CompileHelper:
    """Behavioral tests for apply_fsdp_training_kwargs (extracted helper).

    Tests the actual dict mutation that reaches ``TrainingArguments`` —
    no source grep, no brittle string matching.
    """

    def test_no_fsdp_leaves_kwargs_untouched(self):
        from soup_cli.utils.fsdp import apply_fsdp_training_kwargs

        kwargs = {"learning_rate": 2e-4}
        result = apply_fsdp_training_kwargs(
            kwargs, fsdp_config=None, use_fsdp2_compile=True
        )
        assert result is kwargs
        assert "torch_compile" not in result
        assert "fsdp" not in result

    def test_fsdp_without_compile_flag(self):
        from soup_cli.utils.fsdp import apply_fsdp_training_kwargs

        kwargs = {}
        apply_fsdp_training_kwargs(
            kwargs,
            fsdp_config={"fsdp": "full_shard auto_wrap", "fsdp_config": {}},
            use_fsdp2_compile=False,
        )
        assert kwargs["fsdp"] == "full_shard auto_wrap"
        assert "torch_compile" not in kwargs

    def test_fsdp_with_compile_flag_true(self):
        from soup_cli.utils.fsdp import apply_fsdp_training_kwargs

        kwargs = {}
        apply_fsdp_training_kwargs(
            kwargs,
            fsdp_config={"fsdp": "full_shard auto_wrap", "fsdp_config": {}},
            use_fsdp2_compile=True,
        )
        assert kwargs["torch_compile"] is True

    def test_unexpected_fsdp_keys_rejected(self):
        from soup_cli.utils.fsdp import apply_fsdp_training_kwargs

        with pytest.raises(ValueError, match="Unexpected FSDP config keys"):
            apply_fsdp_training_kwargs(
                {},
                fsdp_config={"fsdp": "x", "fsdp_config": {}, "shady": 1},
                use_fsdp2_compile=False,
            )


class TestFsdp2CompileValidatorDeepSpeed:
    """validate_fsdp2_compile_config rejects DeepSpeed + torch.compile."""

    def test_deepspeed_and_compile_rejected(self):
        from soup_cli.utils.fsdp import validate_fsdp2_compile_config

        errors = validate_fsdp2_compile_config(
            use_compile=True,
            fsdp_preset="full_shard",
            backend="transformers",
            device="cuda",
            deepspeed_config="/tmp/ds.json",
        )
        assert any("deepspeed" in err.lower() for err in errors)

    def test_no_deepspeed_no_error(self):
        from soup_cli.utils.fsdp import validate_fsdp2_compile_config

        with patch("soup_cli.utils.fsdp.is_fsdp_available", return_value=True):
            errors = validate_fsdp2_compile_config(
                use_compile=True,
                fsdp_preset="full_shard",
                backend="transformers",
                device="cuda",
                deepspeed_config=None,
            )
            assert errors == []


class TestNcclEnvApplication:
    """Verify train.py actually calls os.environ.setdefault in distributed mode
    and skips the call on single-GPU / CPU runs."""

    def test_setdefault_does_not_overwrite_user_override(self):
        from soup_cli.utils.topology import suggest_nccl_env

        env = suggest_nccl_env(gpu_count=4, interconnect="nvlink")
        user = {"NCCL_P2P_DISABLE": "1"}
        for key, val in env.items():
            user.setdefault(key, val)
        # User value preserved
        assert user["NCCL_P2P_DISABLE"] == "1"
        # At least one new key added (non-brittle: the exact key set may
        # evolve, we only need the non-overwrite invariant + "something new
        # came in" guarantee).
        new_keys = set(user) - {"NCCL_P2P_DISABLE"}
        assert new_keys, "suggest_nccl_env produced no new keys"

    def test_single_gpu_returns_empty_env(self):
        """Single-GPU path returns an empty dict (so setdefault is a no-op)."""
        from soup_cli.utils.topology import suggest_nccl_env

        assert suggest_nccl_env(gpu_count=1, interconnect="single") == {}


class TestTrainValidatorGating:
    """CLI-level tests proving the new validators actually block `soup train`
    when misconfigured (not silently continue)."""

    def test_use_fsdp2_compile_on_cpu_blocks_train(self, tmp_path):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        # Build a config that asks for use_fsdp2_compile but runs on CPU (no
        # --fsdp + no --gpus). Validators should fire before anything heavy
        # runs, so we don't need a real model here.
        data_file = tmp_path / "train.jsonl"
        data_file.write_text('{"prompt": "x", "completion": "y"}\n')
        cfg_file = tmp_path / "soup.yaml"
        cfg_file.write_text(
            "base: test-model\n"
            "task: sft\n"
            f"data:\n  train: {data_file.as_posix()}\n"
            "training:\n  use_fsdp2_compile: true\n"
            f"output: {(tmp_path / 'out').as_posix()}\n"
        )
        runner = CliRunner()
        result = runner.invoke(
            app, ["train", "--config", str(cfg_file), "--yes", "--dry-run"]
        )
        # Exit non-zero, with a clear message mentioning the flag.
        assert result.exit_code != 0, (result.output, repr(result.exception))
        assert "fsdp2_compile" in result.output.lower() or "fsdp" in result.output.lower()

    def test_pipeline_parallel_on_cpu_blocks_train(self, tmp_path):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        data_file = tmp_path / "train.jsonl"
        data_file.write_text('{"prompt": "x", "completion": "y"}\n')
        cfg_file = tmp_path / "soup.yaml"
        cfg_file.write_text(
            "base: test-model\n"
            "task: sft\n"
            f"data:\n  train: {data_file.as_posix()}\n"
            "training:\n  parallelism: pipeline\n  pipeline_stages: 4\n"
            f"output: {(tmp_path / 'out').as_posix()}\n"
        )
        runner = CliRunner()
        result = runner.invoke(
            app, ["train", "--config", str(cfg_file), "--yes", "--dry-run"]
        )
        assert result.exit_code != 0, (result.output, repr(result.exception))
        assert "pipeline" in result.output.lower() or "cuda" in result.output.lower()


class TestMasterAddrIsValidatedAsAnAddress:
    """`--master-addr` must be as strict as its own error message.

    The value is interpolated into the `accelerate launch` argv and exported
    as MASTER_ADDR on every node, and the rejection says "must be a hostname
    or IP address" -- so anything that is not one belongs on the error path.
    A character blacklist cannot do this: IPv6 literals are full of colons,
    while `head:29500` (the port glued onto the host) must still be refused.
    """

    @pytest.mark.parametrize(
        "bad",
        [
            "head:29500",       # port glued onto the hostname
            "$(whoami)",        # command substitution
            "`id`",             # backtick substitution
            "a|b",              # shell pipe
            "a;b",              # command separator
            "A" * 5000,         # no length ceiling (one oversized label)
            # 299 chars of individually LEGAL 2-char labels: only the
            # RFC 1035 total-length ceiling rejects this one.
            ".".join(["ab"] * 100),
            "host name",        # embedded space
            "-oProxyCommand",   # leading dash
            "-",                # bare dash
            "..",               # empty labels
            "node_01",          # underscore is not a hostname character
        ],
    )
    def test_rejects_values_that_are_not_addresses(self, bad: str) -> None:
        from soup_cli.utils.launcher import validate_multi_node_options

        with pytest.raises(ValueError, match="hostname or IP address"):
            validate_multi_node_options(2, 0, bad, 29500)

    @pytest.mark.parametrize(
        "good",
        [
            "10.0.0.5",
            "192.168.1.1",
            "::1",
            "fe80::1",
            "2001:db8::1",
            "headnode",
            "head.cluster.local",
            "node-01.eu-west.example.com",
            "head.cluster.local.",  # trailing root dot is legal
        ],
    )
    def test_accepts_real_addresses(self, good: str) -> None:
        from soup_cli.utils.launcher import validate_multi_node_options

        validate_multi_node_options(2, 0, good, 29500)

    def test_a_rejected_addr_never_reaches_argv(self) -> None:
        from soup_cli.utils.launcher import build_accelerate_argv

        with pytest.raises(ValueError, match="hostname or IP address"):
            build_accelerate_argv(
                2, ["soup", "train"], num_machines=2, main_process_ip="head:29500"
            )