Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

22.2 KB
Raw Blame
"""v0.35.0 Part D — Per-trainer × per-feature smoke-test matrix.

Without this file, Part A (#60) ships ~50 LOC of theoretical wiring across
8 trainer wrappers; with it, every trainer × every v0.28.0 feature combo is
exercised on every CI matrix job. Each test builds a minimal SoupConfig
with one feature enabled, mocks the model construction, calls the shared
helper, and asserts the right side-effect happened (or that a documented
no-op degrades gracefully).
"""

from __future__ import annotations

from types import SimpleNamespace
from unittest.mock import MagicMock, patch

import pytest

# All transformer-backend trainer tasks (v0.35.0 #60 expanded coverage)
ALL_TRAINER_TASKS = (
    "sft",
    "dpo",
    "pretrain",
    "grpo",
    "kto",
    "orpo",
    "simpo",
    "ipo",
    "ppo",
    "reward_model",
    "embedding",
)

# v0.28.0 speed/memory features
V028_FEATURES = (
    "use_cut_ce",
    "fp8",
    "kernel_auto_compose",
    "activation_offloading",
)


def _make_tcfg(feature: str) -> SimpleNamespace:
    """Build a minimal training-config SimpleNamespace with one feature on."""
    base = {
        "use_cut_ce": False,
        "quantization_aware": False,
        "kernel_auto_compose": False,
        "activation_offloading": None,
    }
    if feature == "use_cut_ce":
        base["use_cut_ce"] = True
    elif feature == "fp8":
        base["quantization_aware"] = "fp8"
    elif feature == "kernel_auto_compose":
        base["kernel_auto_compose"] = True
    elif feature == "activation_offloading":
        base["activation_offloading"] = "cpu"
    return SimpleNamespace(**base)


# ---------------------------------------------------------------------------
# supports_v028_features expansion
# ---------------------------------------------------------------------------


@pytest.mark.parametrize("task", ALL_TRAINER_TASKS)
def test_supports_v028_features_covers_every_trainer(task: str) -> None:
    """Each of the 11 transformer-backend trainers must report supported."""
    from soup_cli.utils.v028_features import supports_v028_features

    assert supports_v028_features(task) is True


def test_supports_v028_features_rejects_unknown_task() -> None:
    from soup_cli.utils.v028_features import supports_v028_features

    assert supports_v028_features("future_task_v999") is False


# ---------------------------------------------------------------------------
# apply_v028_speed_memory matrix — every trainer × every (apply-phase) feature
# ---------------------------------------------------------------------------


@pytest.mark.parametrize("task", ALL_TRAINER_TASKS)
@pytest.mark.parametrize("feature", ("use_cut_ce", "fp8", "kernel_auto_compose"))
def test_apply_v028_speed_memory_no_exception(task: str, feature: str, monkeypatch) -> None:
    """For every trainer × apply-phase feature, the helper must not raise.

    cut_ce / fp8 degrade silently if the underlying lib isn't installed (CI
    runs without torchao.float8 / cut_cross_entropy on most jobs); the
    helper returns a dict instead of crashing. kernel_auto_compose's picker
    raises when no candidates have finite times — also degrades silently.
    """
    # A card that CAN run FP8 (#835/#1044): since the hardware gate moved ahead
    # of the dependency probe, an explicit FP8 request on a machine without CUDA
    # -- every CI runner here -- is refused rather than degraded, which is the
    # ruling. The fp8 row is the OTHER half, INVERTED by the 2026-09-19 ruling:
    # a missing torchao used to degrade here and now stops the run, per trainer.
    import sys

    import torch

    from soup_cli.utils import v028_features as vf

    monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *_a, **_k: (9, 0))
    monkeypatch.setattr(sys, "platform", "linux")
    monkeypatch.setattr(torch.version, "cuda", "12.4")

    tcfg = _make_tcfg(feature)
    if feature == "fp8":
        from soup_cli.utils.fp8 import FP8DependencyMissingError

        # Missing whether or not this CI job installs torchao.
        monkeypatch.setattr("soup_cli.utils.fp8.is_fp8_available", lambda: False)
        with pytest.raises(FP8DependencyMissingError):
            vf.apply_v028_speed_memory(
                model=MagicMock(), tcfg=tcfg, base_model="meta-llama/Llama-3.2-1B",
                console=None,
            )
        return
    result = vf.apply_v028_speed_memory(
        model=MagicMock(),
        tcfg=tcfg,
        base_model="meta-llama/Llama-3.2-1B",
        console=None,
    )
    # Helper returns a dict of {feature: bool_applied} — never None / exception.
    assert isinstance(result, dict)
    assert "cut_ce" in result
    assert "fp8" in result
    assert "kernel_auto_compose" in result


@pytest.mark.parametrize("task", ALL_TRAINER_TASKS)
def test_apply_v028_speed_memory_invokes_cut_ce_patcher(task: str) -> None:
    """When use_cut_ce=True and the patcher succeeds, applied['cut_ce'] is True."""
    from soup_cli.utils import v028_features as vf

    tcfg = _make_tcfg("use_cut_ce")
    with patch("soup_cli.utils.cut_ce.apply_cut_ce", return_value=True):
        result = vf.apply_v028_speed_memory(
            model=MagicMock(), tcfg=tcfg,
            base_model="meta-llama/Llama-3.2-1B", console=None,
        )
    assert result["cut_ce"] is True


@pytest.mark.parametrize("task", ALL_TRAINER_TASKS)
def test_apply_v028_speed_memory_invokes_fp8(task: str) -> None:
    from soup_cli.utils import v028_features as vf

    tcfg = _make_tcfg("fp8")
    with patch("soup_cli.utils.fp8.apply_fp8_training", return_value=True):
        result = vf.apply_v028_speed_memory(
            model=MagicMock(), tcfg=tcfg,
            base_model="meta-llama/Llama-3.2-1B", console=None,
        )
    assert result["fp8"] is True


@pytest.mark.parametrize("task", ALL_TRAINER_TASKS)
def test_apply_v028_speed_memory_refuses_kernel_picker(task: str) -> None:
    """Bypassing validation cannot revive the misleading picker path."""
    from soup_cli.utils import v028_features as vf

    tcfg = _make_tcfg("kernel_auto_compose")
    result = vf.apply_v028_speed_memory(
        model=MagicMock(), tcfg=tcfg,
        base_model="meta-llama/Llama-3.2-1B", console=None,
        device="cuda", backend="transformers",
    )
    assert result["kernel_auto_compose"] is False


# ---------------------------------------------------------------------------
# activation_offloading_context — train()-phase wrapper
# ---------------------------------------------------------------------------


def test_activation_offloading_context_none_passes_through(tmp_path) -> None:
    from soup_cli.utils.v028_features import activation_offloading_context

    tcfg = SimpleNamespace(activation_offloading=None)
    with activation_offloading_context(tcfg, str(tmp_path)):
        pass


def test_activation_offloading_context_cpu_no_save_dir(tmp_path) -> None:
    from soup_cli.utils.v028_features import activation_offloading_context

    tcfg = SimpleNamespace(activation_offloading="cpu")
    with activation_offloading_context(tcfg, str(tmp_path)):
        pass


def test_activation_offloading_context_disk_rejects_outside_cwd() -> None:
    """Disk mode must refuse output dirs outside cwd — defence in depth."""
    import os

    from soup_cli.utils.v028_features import activation_offloading_context

    tcfg = SimpleNamespace(activation_offloading="disk")
    # Use the parent of cwd, which is guaranteed to be outside containment.
    parent = os.path.realpath(os.path.join(os.getcwd(), os.pardir))
    with pytest.raises(ValueError, match="under the current working directory"):
        with activation_offloading_context(tcfg, parent):
            pass


def test_activation_offloading_context_missing_attr_safe(tmp_path) -> None:
    """A tcfg-like object missing the attr should not raise."""
    from soup_cli.utils.v028_features import activation_offloading_context

    tcfg = SimpleNamespace()  # no activation_offloading attr at all
    with activation_offloading_context(tcfg, str(tmp_path)):
        pass


# ---------------------------------------------------------------------------
# Schema gate — v0.35.0 #60 lifts the gate for every transformer trainer
# ---------------------------------------------------------------------------


def _build_yaml_config(task: str, **training_extra) -> dict:
    """Helper: build a minimal config dict accepted by load_config_from_string."""
    # #1420: KTO refuses batch 1 (TRL KL term)
    default_bs = 2 if task in ("embedding", "kto") else 1
    body = {
        "base": "meta-llama/Llama-3.2-1B",
        "task": task,
        "training": {"epochs": 1, "lr": 1e-4, "batch_size": default_bs, **training_extra},
    }
    if task == "pretrain":
        body["data"] = {"train": "data.jsonl", "format": "plaintext"}
    elif task in ("dpo", "kto", "orpo", "simpo", "ipo", "grpo"):
        body["data"] = {"train": "data.jsonl", "format": "dpo"}
    elif task == "embedding":
        body["data"] = {"train": "data.jsonl", "format": "embedding"}
    elif task == "reward_model":
        body["data"] = {"train": "data.jsonl", "format": "dpo"}
    else:
        body["data"] = {"train": "data.jsonl", "format": "alpaca"}
    return body


@pytest.mark.parametrize(
    "task",
    ("grpo", "kto", "orpo", "simpo", "ipo", "ppo", "reward_model", "embedding"),
)
def test_schema_gate_lifted_for_use_cut_ce(task: str) -> None:
    """Each of the 8 newly-wired trainers must accept use_cut_ce at config-load."""
    import yaml

    from soup_cli.config.loader import load_config_from_string

    body = _build_yaml_config(task, use_cut_ce=True)
    cfg = load_config_from_string(yaml.safe_dump(body))
    assert cfg.task == task
    assert cfg.training.use_cut_ce is True


@pytest.mark.parametrize(
    "task",
    ("grpo", "kto", "orpo", "simpo", "ipo", "ppo", "reward_model", "embedding"),
)
def test_schema_gate_lifted_for_activation_offloading(task: str) -> None:
    """activation_offloading="cpu" must now be accepted on every trainer."""
    import yaml

    from soup_cli.config.loader import load_config_from_string

    body = _build_yaml_config(task, activation_offloading="cpu")
    cfg = load_config_from_string(yaml.safe_dump(body))
    assert cfg.training.activation_offloading == "cpu"


def test_mlx_backend_still_rejects_v028_features() -> None:
    """MLX backend has no equivalent kernels — gate must still fire with a
    schema-rejection ValueError (the loader wraps Pydantic's ValidationError
    in a ValueError carrying the same message)."""
    import yaml

    from soup_cli.config.loader import load_config_from_string

    body = {
        "base": "meta-llama/Llama-3.2-1B",
        "task": "sft",
        "backend": "mlx",
        "data": {"train": "data.jsonl", "format": "alpaca"},
        "training": {
            "epochs": 1, "lr": 1e-4, "batch_size": 1, "use_cut_ce": True,
        },
    }
    with pytest.raises(ValueError, match="mlx"):
        load_config_from_string(yaml.safe_dump(body))


def test_unknown_task_v028_features_rejected_with_distinct_message() -> None:
    """Code-review fix: the schema gate must NOT blame MLX for a non-MLX
    failure. An unknown task on the transformers backend should produce a
    "task=" message, not the MLX message."""
    from pydantic import ValidationError

    from soup_cli.config.schema import SoupConfig

    # Bypass the loader (which wraps ValueError) so we can inspect the
    # raw Pydantic error message directly.
    with pytest.raises(ValidationError) as exc_info:
        SoupConfig(
            base="m",
            task="sft",  # supported task, but...
            backend="mlx",  # ...MLX backend triggers the rejection
            data={"train": "./d.jsonl"},
            training={"use_cut_ce": True},
        )
    assert "mlx" in str(exc_info.value).lower()
    assert "Apple Silicon" in str(exc_info.value)


@pytest.mark.parametrize(
    "task",
    ("grpo", "kto", "orpo", "simpo", "ipo", "ppo", "reward_model", "embedding"),
)
def test_schema_rejects_kernel_auto_compose(task: str) -> None:
    """#801 — the non-functional selector is rejected on every trainer."""
    import yaml

    from soup_cli.config.loader import load_config_from_string

    body = _build_yaml_config(task, kernel_auto_compose=True)
    with pytest.raises(ValueError, match="use_flash_attn"):
        load_config_from_string(yaml.safe_dump(body))


@pytest.mark.parametrize(
    "task",
    ("grpo", "kto", "orpo", "simpo", "ipo", "ppo", "reward_model", "embedding"),
)
def test_schema_gate_lifted_for_fp8(task: str) -> None:
    """v0.35.0 #60 — quantization_aware="fp8" must now be accepted on every trainer."""
    import yaml

    from soup_cli.config.loader import load_config_from_string

    body = _build_yaml_config(task, quantization_aware="fp8")
    cfg = load_config_from_string(yaml.safe_dump(body))
    assert cfg.training.quantization_aware == "fp8"


# ---------------------------------------------------------------------------
# Per-trainer × per-feature integration smoke — verify the call site exists
# ---------------------------------------------------------------------------


@pytest.mark.parametrize(
    "module_path",
    (
        "soup_cli.trainer.sft",
        "soup_cli.trainer.dpo",
        "soup_cli.trainer.pretrain",
        "soup_cli.trainer.grpo",
        "soup_cli.trainer.kto",
        "soup_cli.trainer.orpo",
        "soup_cli.trainer.simpo",
        "soup_cli.trainer.ipo",
        "soup_cli.trainer.ppo",
        "soup_cli.trainer.reward_model",
        "soup_cli.trainer.embedding",
    ),
)
def test_trainer_module_calls_apply_v028_speed_memory(module_path: str) -> None:
    """Source-level proof that every trainer wires the helper.

    A regression that drops the call site silently re-introduces v0.28.0
    silent no-ops — caught here at parse time, not at training time.
    """
    import importlib
    import inspect

    mod = importlib.import_module(module_path)
    src = inspect.getsource(mod)
    # v0.33.0 #43 / #800 — all 11 transformer-backend trainers delegate
    # to apply_v028_speed_memory.
    assert "apply_v028_speed_memory" in src, (
        f"{module_path} does not call apply_v028_speed_memory — "
        "v0.28.0 features will silently no-op for this trainer."
    )


@pytest.mark.parametrize(
    "module_path",
    (
        "soup_cli.trainer.sft",
        "soup_cli.trainer.dpo",
        "soup_cli.trainer.pretrain",
        "soup_cli.trainer.grpo",
        "soup_cli.trainer.kto",
        "soup_cli.trainer.orpo",
        "soup_cli.trainer.simpo",
        "soup_cli.trainer.ipo",
        "soup_cli.trainer.ppo",
        "soup_cli.trainer.reward_model",
        "soup_cli.trainer.embedding",
    ),
)
def test_trainer_module_wraps_train_with_offloading_context(
    module_path: str,
) -> None:
    """Source-level proof that every trainer wraps trainer.train() with the
    activation-offloading context manager."""
    import importlib
    import inspect

    mod = importlib.import_module(module_path)
    src = inspect.getsource(mod)
    # SFT uses the inline offload_context (older pattern); the others use the
    # shared activation_offloading_context helper from v028_features.
    assert (
        "activation_offloading_context" in src
        or "offload_context" in src
    ), (
        f"{module_path} does not wrap trainer.train() with the activation-"
        "offloading context — disk/cpu offloading will silently no-op."
    )


class TestVllmCreateEngineQuantizationParam:
    def test_create_vllm_engine_accepts_quantization_kwarg(self) -> None:
        """Source-level proof that the new quantization kwarg is exposed."""
        import inspect

        from soup_cli.utils import vllm as vllm_mod

        sig = inspect.signature(vllm_mod.create_vllm_engine)
        assert "quantization" in sig.parameters
        # Default must be None so existing callers are unaffected.
        assert sig.parameters["quantization"].default is None

    def test_create_vllm_engine_rejects_invalid_quantization(self) -> None:
        """Behavioural proof that an unsupported quantization name raises ValueError.

        We patch out the heavy ``AsyncEngineArgs`` import so the validation
        check fires before any real engine machinery is loaded.
        """
        from soup_cli.utils import vllm as vllm_mod

        # Stub the vllm import so we exercise our own validation, not a
        # real engine-construction failure that would mask the assertion.
        fake_engine_args = MagicMock()
        fake_async_engine = MagicMock()
        with patch.dict(
            "sys.modules",
            {
                "vllm": MagicMock(
                    AsyncEngineArgs=fake_engine_args,
                    AsyncLLMEngine=fake_async_engine,
                ),
            },
        ):
            with pytest.raises(ValueError, match="quantization must be one of"):
                vllm_mod.create_vllm_engine(
                    model_path="x",
                    quantization="int8-fake",  # not in the allowlist
                )


# ---------------------------------------------------------------------------
# QAT regression — fp8 must not fall into the int8 prepare_model_for_qat path
# ---------------------------------------------------------------------------


# ---------------------------------------------------------------------------
# Part C — kernel benchmark warm-up loop
# ---------------------------------------------------------------------------


def test_benchmark_kernel_combos_cpu_returns_none_times() -> None:
    """CI runs CPU-only — benchmark must return None times (degrade gracefully)."""
    from soup_cli.utils.kernel_picker import benchmark_kernel_combos

    candidates = [
        {"name": "baseline", "use_liger": False, "use_flash_attn": False,
         "use_cut_ce": False},
        {"name": "liger", "use_liger": True, "use_flash_attn": False,
         "use_cut_ce": False},
    ]
    out = benchmark_kernel_combos(model=None, candidates=candidates, device="cpu")
    assert len(out) == 2
    for entry in out:
        assert entry["time_ms"] is None


def test_benchmark_kernel_combos_no_cuda_returns_none_times() -> None:
    """Even with device="cuda", if torch.cuda is unavailable, return None times."""
    from soup_cli.utils.kernel_picker import benchmark_kernel_combos

    candidates = [{"name": "baseline"}]
    # Mock torch.cuda.is_available returning False
    import torch  # noqa
    with patch("torch.cuda.is_available", return_value=False):
        out = benchmark_kernel_combos(
            model=MagicMock(), candidates=candidates, device="cuda",
        )
    assert out[0]["time_ms"] is None


def test_benchmark_kernel_combos_does_not_mutate_input() -> None:
    """Benchmark must return a NEW list — input candidates remain untouched."""
    from soup_cli.utils.kernel_picker import benchmark_kernel_combos

    candidates = [{"name": "baseline"}]
    out = benchmark_kernel_combos(
        model=None, candidates=candidates, device="cpu",
    )
    assert "time_ms" not in candidates[0]  # input untouched
    assert "time_ms" in out[0]              # output annotated


def test_benchmark_kernel_combos_clamps_seq_len() -> None:
    """seq_len > 512 must be clamped — defence against caller OOM."""
    from soup_cli.utils.kernel_picker import benchmark_kernel_combos

    # device="cpu" short-circuits without ever touching seq_len, so this just
    # confirms no exception escapes the bounds-clamp branch.
    out = benchmark_kernel_combos(
        model=None, candidates=[{"name": "baseline"}], device="cpu",
        seq_len=10**6,
    )
    assert out[0]["time_ms"] is None


def test_bench_and_pick_kernel_removed_from_v028_features() -> None:
    """Unreachable _bench_and_pick_kernel helper was removed under #976."""
    from soup_cli.utils import v028_features as vf

    assert not hasattr(vf, "_bench_and_pick_kernel")


def test_apply_v028_kernel_auto_compose_never_claims_application() -> None:
    """A caller bypassing config validation still cannot report a false pick."""
    from soup_cli.utils import v028_features as vf

    tcfg = SimpleNamespace(
        use_cut_ce=False, quantization_aware=False,
        kernel_auto_compose=True, activation_offloading=None,
    )
    result = vf.apply_v028_speed_memory(
        model=MagicMock(), tcfg=tcfg,
        base_model="meta-llama/Llama-3.2-1B", console=None,
        device="cuda", backend="transformers",
    )
    assert result["kernel_auto_compose"] is False


@pytest.mark.parametrize(
    "module_path",
    (
        "soup_cli.trainer.dpo",
        "soup_cli.trainer.pretrain",
        "soup_cli.trainer.grpo",
        "soup_cli.trainer.kto",
        "soup_cli.trainer.orpo",
        "soup_cli.trainer.simpo",
        "soup_cli.trainer.ipo",
        "soup_cli.trainer.ppo",
        "soup_cli.trainer.reward_model",
        "soup_cli.trainer.embedding",
    ),
)
def test_trainer_passes_device_and_backend_to_apply_helper(
    module_path: str,
) -> None:
    """Part C — every trainer must forward device + backend so the kernel
    benchmark loop knows whether it's safe to bench (CUDA + transformers)."""
    import importlib
    import inspect

    mod = importlib.import_module(module_path)
    src = inspect.getsource(mod)
    assert "device=self.device" in src, (
        f"{module_path} does not forward device= to apply_v028_speed_memory; "
        "kernel auto-compose benchmark cannot reliably skip on CPU."
    )
    assert "backend=cfg.backend" in src, (
        f"{module_path} does not forward backend= to apply_v028_speed_memory; "
        "unsloth/mlx backends will have their kernels mis-benchmarked."
    )


@pytest.mark.parametrize(
    "module_path",
    (
        "soup_cli.trainer.dpo",
        "soup_cli.trainer.pretrain",
        "soup_cli.trainer.grpo",
        "soup_cli.trainer.kto",
        "soup_cli.trainer.orpo",
        "soup_cli.trainer.simpo",
        "soup_cli.trainer.ipo",
        "soup_cli.trainer.ppo",
    ),
)
def test_trainer_qat_guard_excludes_fp8(module_path: str) -> None:
    """quantization_aware="fp8" must NOT trigger the legacy int8 QAT path."""
    import importlib
    import inspect

    mod = importlib.import_module(module_path)
    src = inspect.getsource(mod)
    # The correct pattern excludes fp8 from the int8 QAT branch.
    assert 'quantization_aware != "fp8"' in src, (
        f"{module_path} QAT guard does not exclude fp8; "
        'quantization_aware="fp8" would crash the legacy int8 QAT path.'
    )