Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

50.8 KB
Raw Blame
"""Tests for v0.28.0 — Training Speed & Memory features.

Covers:
- Part A: Cut Cross-Entropy (CCE)
- Part B: FP8 training (quantization_aware='fp8')
- Part C: Gradient checkpointing tiers (selective/medium/full/auto)
- Part D: Kernel auto-composition (utils/kernel_picker.py)
- Part E: Cross-document attention masking for sample packing
- Part F: Activation offloading to CPU/disk
"""

from __future__ import annotations

import os
import sys
from unittest.mock import MagicMock, patch

import pytest
from pydantic import ValidationError

from soup_cli.config.schema import SoupConfig, TrainingConfig

# _tiny_llama_dir lives in test_v07202 and is deliberately not duplicated here,
# same convention as test_v07300.py.
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

# ─── Part A: Cut Cross-Entropy (CCE) ───────────────────────────────────────


class TestCutCEConfig:
    """TrainingConfig.use_cut_ce boolean field."""

    def test_use_cut_ce_default_false(self):
        cfg = SoupConfig(base="test/model", data={"train": "./data.jsonl"})
        assert cfg.training.use_cut_ce is False

    def test_use_cut_ce_enabled(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"use_cut_ce": True},
        )
        assert cfg.training.use_cut_ce is True

    def test_use_cut_ce_type_bool_coerce(self):
        """Pydantic coerces bool-like strings; verify type is bool after."""
        cfg = TrainingConfig(use_cut_ce=True)
        assert cfg.use_cut_ce is True
        assert isinstance(cfg.use_cut_ce, bool)


class TestCutCEAvailability:
    """Cut Cross-Entropy availability + detection."""

    def test_check_cut_ce_available_returns_bool(self):
        from soup_cli.utils.cut_ce import check_cut_ce_available

        result = check_cut_ce_available()
        assert isinstance(result, bool)

    def test_check_cut_ce_not_installed(self):
        from soup_cli.utils.cut_ce import check_cut_ce_available

        # sys.modules[name]=None makes ``import name`` raise ImportError
        with patch.dict("sys.modules", {"cut_cross_entropy": None}):
            assert check_cut_ce_available() is False

    def test_get_cut_ce_version_not_installed(self):
        from soup_cli.utils.cut_ce import get_cut_ce_version

        # get_cut_ce_version() imports the package itself rather than going
        # through check_cut_ce_available(), so patching that predicate is a
        # no-op — the absence has to be simulated at the import, as above.
        with patch.dict("sys.modules", {"cut_cross_entropy": None}):
            assert get_cut_ce_version() is None


class TestCutCEApplication:
    """Applying Cut Cross-Entropy to a model."""

    def test_apply_cut_ce_not_installed(self):
        from soup_cli.utils.cut_ce import apply_cut_ce

        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=False
        ):
            result = apply_cut_ce("meta-llama/Llama-3.1-8B")
            assert result is False

    def test_apply_cut_ce_available_tries_patching(self):
        from soup_cli.utils.cut_ce import apply_cut_ce

        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=True
        ):
            # With cut_cross_entropy not actually installed, patch should return
            # False (can't import real module). Just verifying it doesn't crash.
            result = apply_cut_ce("meta-llama/Llama-3.1-8B")
            assert isinstance(result, bool)

    def test_apply_cut_ce_calls_llama_patch_for_llama_model(self):
        """Verify the llama detector routes to cce_patch('llama').

        ``_detect_model_type`` is forced to "" so this exercises the
        name-based fallback deterministically. Left unmocked, the real
        detector's live hub fetch runs inside ``patch.dict("sys.modules")``,
        which evicts ``tokenizers`` on exit; re-importing it crashes (PyO3
        extensions can't reinit), hidden in a full-suite run only because
        some other file imports it first."""
        from soup_cli.utils.cut_ce import apply_cut_ce

        fake_cce = MagicMock()
        fake_transformers = MagicMock(cce_patch=fake_cce)
        fake_module = MagicMock(transformers=fake_transformers)
        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=True
        ), patch(
            "soup_cli.utils.cut_ce._detect_model_type", return_value=""
        ), patch.dict(
            "sys.modules",
            {
                "cut_cross_entropy": fake_module,
                "cut_cross_entropy.transformers": fake_transformers,
            },
        ):
            assert apply_cut_ce("meta-llama/Llama-3.1-8B") is True
            fake_cce.assert_called_once_with("llama")

    def test_apply_cut_ce_deepseek_phi_does_not_use_phi(self):
        """Regression: org-prefix like 'deepseek-ai/...' must not trigger phi
        via the substring-fallback path. ``_detect_model_type`` is forced to ""
        here (offline) so this exercises the fallback deterministically:
        with real config resolution deepseek-coder is genuinely Llama-arch and
        the model_type path would (correctly) patch it, see
        TestCutCEDetectsArchitectureFromTheConfig below."""
        from soup_cli.utils.cut_ce import apply_cut_ce

        fake_cce = MagicMock()
        fake_transformers = MagicMock(cce_patch=fake_cce)
        fake_module = MagicMock(transformers=fake_transformers)
        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=True
        ), patch(
            "soup_cli.utils.cut_ce._detect_model_type", return_value=""
        ), patch.dict(
            "sys.modules",
            {
                "cut_cross_entropy": fake_module,
                "cut_cross_entropy.transformers": fake_transformers,
            },
        ):
            # Llama-distilled model name contains no "phi" substring
            # anymore thanks to the last-path-component detector.
            assert apply_cut_ce("deepseek-ai/deepseek-coder-7b-instruct") is False
            fake_cce.assert_not_called()

    def test_apply_cut_ce_plain_gemma_reports_false_instead_of_crashing(self):
        """Regression: cut_cross_entropy has no patcher for plain Gemma-1
        (model_type "gemma", only "gemma2" is supported). Pre-fix, a bare
        "gemma" fallback-detector entry dispatched cce_patch("gemma") and
        crashed with RuntimeError("Unknown model type gemma") for any
        Gemma-1 name that reached the name-based fallback. It must instead
        report False, the same advisory every other unsupported
        architecture gets."""
        from soup_cli.utils.cut_ce import apply_cut_ce

        def _patch(model_type, *a, **kw):
            if model_type not in ("llama", "phi3", "gemma2", "mistral", "qwen2"):
                raise RuntimeError(f"Unknown model type {model_type}")

        fake_cce = MagicMock(side_effect=_patch)
        fake_transformers = MagicMock(cce_patch=fake_cce)
        fake_module = MagicMock(transformers=fake_transformers)
        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=True
        ), patch(
            "soup_cli.utils.cut_ce._detect_model_type", return_value=""
        ), patch.dict(
            "sys.modules",
            {
                "cut_cross_entropy": fake_module,
                "cut_cross_entropy.transformers": fake_transformers,
            },
        ):
            assert apply_cut_ce("google/gemma-7b-it") is False
            fake_cce.assert_not_called()


class TestCutCEDetectsArchitectureFromTheConfig:
    """apply_cut_ce() matched on a SUBSTRING OF THE MODEL PATH and silently
    no-opped on every local checkpoint directory Soup itself produces
    (checkpoint-2000, my-finetune, ...): none of those names carry an
    architecture keyword. It also conflated Phi-2 with Phi-3 (both contain
    "phi", cut_cross_entropy has a patcher for neither's real model_type
    distinctly, "phi3" only). The architecture is in the config, exactly
    the fix liger.py already shipped for #78: ``AutoConfig.model_type`` is
    the same string ``cce_patch`` itself dispatches on."""

    def _fake_cce(
        self, monkeypatch, calls, supported=("llama", "phi3", "gemma2", "mistral", "qwen2")
    ):
        """Mirrors upstream's own dispatch (apple/ml-cross-entropy
        cut_cross_entropy/transformers/patch.py::PATCH_FNS): raises
        RuntimeError for a model_type it has no patcher for."""
        import sys
        import types

        def _patch(model_type_or_model, *args, **kwargs):
            calls.append(model_type_or_model)
            if model_type_or_model not in supported:
                raise RuntimeError(f"Unknown model type {model_type_or_model}")

        mod = types.ModuleType("cut_cross_entropy.transformers")
        mod.cce_patch = _patch
        parent = types.ModuleType("cut_cross_entropy")
        parent.transformers = mod
        monkeypatch.setitem(sys.modules, "cut_cross_entropy", parent)
        monkeypatch.setitem(sys.modules, "cut_cross_entropy.transformers", mod)
        return calls

    def test_a_local_checkpoint_dir_with_no_architecture_in_its_name_is_still_detected(
        self, tmp_path, monkeypatch
    ):
        from test_v07202 import _tiny_llama_dir

        from soup_cli.utils import cut_ce as cut_ce_mod

        weights, _, _ = _tiny_llama_dir(tmp_path)
        assert "llama" not in weights.lower(), (
            "the fixture path must NOT contain the architecture name, or this "
            "test passes for the substring matcher too and proves nothing"
        )
        calls = self._fake_cce(monkeypatch, [])
        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)

        assert cut_ce_mod.apply_cut_ce(weights) is True
        assert calls == ["llama"], calls

    def test_phi2_is_not_dispatched_to_the_phi3_patcher(self, tmp_path, monkeypatch):
        """CONTROL for the conflation half of the bug. Phi-2's real
        ``config.model_type`` is "phi", not "phi3": cut_cross_entropy has no
        Phi-2 patcher, so this must report False, not silently reuse phi3's."""
        import json

        from soup_cli.utils import cut_ce as cut_ce_mod

        directory = tmp_path / "step-2000"
        directory.mkdir()
        (directory / "config.json").write_text(json.dumps({"model_type": "phi"}))
        calls = self._fake_cce(monkeypatch, [])
        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)

        assert cut_ce_mod.apply_cut_ce(str(directory)) is False
        assert calls == ["phi"], calls

    def test_config_model_type_wins_over_a_conflicting_directory_name(
        self, tmp_path, monkeypatch
    ):
        """CONTROL pinning the config-over-name priority itself, not just its
        outcome on names that happen to agree. A ``soup merge``/``soup
        shrink`` output directory is commonly named after its *source*
        model, so a directory whose name says one architecture and whose
        config says another is a real shape, not a contrived one. Fails
        (routes to "phi3") if the config check is ever moved below the
        name-match fallback."""
        import json

        from soup_cli.utils import cut_ce as cut_ce_mod

        directory = tmp_path / "phi-3-mini-finetune"
        directory.mkdir()
        (directory / "config.json").write_text(json.dumps({"model_type": "llama"}))
        calls = self._fake_cce(monkeypatch, [])
        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)

        assert cut_ce_mod.apply_cut_ce(str(directory)) is True
        assert calls == ["llama"], calls

    def test_an_unsupported_architecture_still_reports_false(self, tmp_path, monkeypatch):
        """CONTROL: detection by config must not turn into "always True": an
        architecture cce_patch does not support (gpt2 has no CCE patcher) has
        to keep reporting False so the caller keeps printing its advisory."""
        import json

        from soup_cli.utils import cut_ce as cut_ce_mod

        directory = tmp_path / "gpt2-run"
        directory.mkdir()
        (directory / "config.json").write_text(json.dumps({"model_type": "gpt2"}))
        self._fake_cce(monkeypatch, [])
        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)

        assert cut_ce_mod.apply_cut_ce(str(directory)) is False

    def test_config_unavailable_falls_back_to_the_name_based_match(self, monkeypatch):
        """No local config.json and no network: model_type resolution fails
        quietly, and the last-path-component fallback still applies."""
        from soup_cli.utils import cut_ce as cut_ce_mod

        calls = self._fake_cce(monkeypatch, [])
        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)
        monkeypatch.setattr(cut_ce_mod, "_detect_model_type", lambda model_name: "")

        assert cut_ce_mod.apply_cut_ce("meta-llama/Llama-3.1-8B") is True
        assert calls == ["llama"], calls


class TestIssue456CrossPlatformPathComponentExtraction:
    r"""#456: cut_ce and liger must match architecture on the last component only.

    On Windows, naive rsplit("/") never splits on backslashes, so parent directory
    names (e.g. C:\experiments\phi-3-runs\step-2000) over-matched architecture
    keywords when config resolution was unavailable. Tests use explicit synthetic
    paths rather than tmp_path to prevent runner directory names from leaking.
    """

    def _fake_cce(
        self,
        monkeypatch,
        calls,
        supported=("llama", "phi3", "gemma2", "mistral", "qwen2"),
    ):
        import sys
        import types

        def _patch(model_type_or_model, *args, **kwargs):
            calls.append(model_type_or_model)
            if model_type_or_model not in supported:
                raise RuntimeError(f"Unknown model type {model_type_or_model}")

        mod = types.ModuleType("cut_cross_entropy.transformers")
        mod.cce_patch = _patch
        parent = types.ModuleType("cut_cross_entropy")
        parent.transformers = mod
        monkeypatch.setitem(sys.modules, "cut_cross_entropy", parent)
        monkeypatch.setitem(sys.modules, "cut_cross_entropy.transformers", mod)
        return calls

    def _fake_liger(self, monkeypatch, calls):
        import sys
        import types

        mod = types.ModuleType("liger_kernel.transformers")
        mod.apply_liger_kernel_to_llama = lambda: calls.append("llama")
        mod.apply_liger_kernel_to_mistral = lambda: calls.append("mistral")
        mod.apply_liger_kernel_to_gemma2 = lambda: calls.append("gemma2")
        mod.apply_liger_kernel_to_qwen2 = lambda: calls.append("qwen2")
        mod.apply_liger_kernel_to_phi3 = lambda: calls.append("phi3")
        parent = types.ModuleType("liger_kernel")
        parent.transformers = mod
        monkeypatch.setitem(sys.modules, "liger_kernel", parent)
        monkeypatch.setitem(sys.modules, "liger_kernel.transformers", mod)
        return calls

    def test_windows_parent_dir_with_architecture_keyword_does_not_overmatch_cut_ce(
        self, monkeypatch
    ):
        """CONTROL: A Windows path whose parent contains an architecture keyword
        must not match that keyword in cut_ce when config is unavailable (#456)."""
        from soup_cli.utils import cut_ce as cut_ce_mod

        calls = self._fake_cce(monkeypatch, [])
        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)
        monkeypatch.setattr(cut_ce_mod, "_detect_model_type", lambda _: "")

        assert (
            cut_ce_mod.apply_cut_ce(r"C:\experiments\phi-3-runs\step-2000") is False
        )
        assert (
            cut_ce_mod.apply_cut_ce(r"C:\models\llama-sweeps\checkpoint-100")
            is False
        )
        assert calls == []

    def test_posix_parent_dir_with_architecture_keyword_does_not_overmatch_cut_ce(
        self, monkeypatch
    ):
        """CONTROL: A POSIX path whose parent contains an architecture keyword
        must not match that keyword in cut_ce when config is unavailable (#456)."""
        from soup_cli.utils import cut_ce as cut_ce_mod

        calls = self._fake_cce(monkeypatch, [])
        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)
        monkeypatch.setattr(cut_ce_mod, "_detect_model_type", lambda _: "")

        assert (
            cut_ce_mod.apply_cut_ce("/tmp/phi-3-experiments/step-2000") is False
        )
        assert (
            cut_ce_mod.apply_cut_ce(
                "/var/checkpoints/llama-runs/checkpoint-100"
            )
            is False
        )
        assert calls == []

    def test_trailing_separators_preserve_last_component_in_cut_ce(
        self, monkeypatch
    ):
        """Trailing slashes and backslashes must not cause the last component to become empty."""
        from soup_cli.utils import cut_ce as cut_ce_mod

        calls = self._fake_cce(monkeypatch, [])
        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)
        monkeypatch.setattr(cut_ce_mod, "_detect_model_type", lambda _: "")

        assert (
            cut_ce_mod.apply_cut_ce("C:\\experiments\\runs\\llama-3.1-8b\\")
            is True
        )
        assert cut_ce_mod.apply_cut_ce("/tmp/models/phi-3-mini/") is True
        assert calls == ["llama", "phi3"]

    def test_windows_parent_dir_with_architecture_keyword_does_not_overmatch_liger(
        self, monkeypatch
    ):
        """CONTROL: A Windows path whose parent contains an architecture keyword
        must not match that keyword in liger when config is unavailable (#456)."""
        from soup_cli.utils import liger as liger_mod

        calls = self._fake_liger(monkeypatch, [])
        monkeypatch.setattr(liger_mod, "check_liger_available", lambda: True)
        monkeypatch.setattr(liger_mod, "_detect_model_type", lambda _: "")

        assert (
            liger_mod.apply_liger_kernel(
                r"C:\experiments\llama_experiments\step-2000"
            )
            is False
        )
        assert (
            liger_mod.apply_liger_kernel(r"C:\models\phi_runs\checkpoint-50")
            is False
        )
        assert calls == []

    def test_posix_parent_dir_does_not_overmatch_liger(self, monkeypatch):
        """CONTROL: A POSIX path whose parent contains an architecture keyword
        must not match that keyword in liger when config is unavailable (#456)."""
        from soup_cli.utils import liger as liger_mod

        calls = self._fake_liger(monkeypatch, [])
        monkeypatch.setattr(liger_mod, "check_liger_available", lambda: True)
        monkeypatch.setattr(liger_mod, "_detect_model_type", lambda _: "")

        assert (
            liger_mod.apply_liger_kernel("/home/llama_experiments/step-2000")
            is False
        )
        assert (
            liger_mod.apply_liger_kernel("/srv/qwen-sweeps/checkpoint-9")
            is False
        )
        assert calls == []

    def test_trailing_separators_and_last_component_match_liger(
        self, monkeypatch
    ):
        """Liger correctly patches when the last component itself matches."""
        from soup_cli.utils import liger as liger_mod

        calls = self._fake_liger(monkeypatch, [])
        monkeypatch.setattr(liger_mod, "check_liger_available", lambda: True)
        monkeypatch.setattr(liger_mod, "_detect_model_type", lambda _: "")

        assert (
            liger_mod.apply_liger_kernel("C:\\models\\llama-3.1-8b\\") is True
        )
        assert liger_mod.apply_liger_kernel("/tmp/runs/phi-3-mini/") is True
        assert calls == ["llama", "phi3"]

    def test_empty_and_root_paths_return_false(self, monkeypatch):
        """Degenerate boundary check: empty strings, None, and root slashes return False."""
        from soup_cli.utils import cut_ce as cut_ce_mod
        from soup_cli.utils import liger as liger_mod

        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)
        monkeypatch.setattr(cut_ce_mod, "_detect_model_type", lambda _: "")
        monkeypatch.setattr(liger_mod, "check_liger_available", lambda: True)
        monkeypatch.setattr(liger_mod, "_detect_model_type", lambda _: "")

        for bad_path in ("", "   ", None, "/", "\\", "///", "\\\\\\"):
            assert cut_ce_mod.apply_cut_ce(bad_path) is False
            assert liger_mod.apply_liger_kernel(bad_path) is False

    def test_repeated_separators_resolve_last_component(self, monkeypatch):
        """Repeated slashes or backslashes must resolve cleanly to the last component."""
        from soup_cli.utils import cut_ce as cut_ce_mod
        from soup_cli.utils import liger as liger_mod

        calls_cce = self._fake_cce(monkeypatch, [])
        calls_liger = self._fake_liger(monkeypatch, [])
        monkeypatch.setattr(cut_ce_mod, "check_cut_ce_available", lambda: True)
        monkeypatch.setattr(cut_ce_mod, "_detect_model_type", lambda _: "")
        monkeypatch.setattr(liger_mod, "check_liger_available", lambda: True)
        monkeypatch.setattr(liger_mod, "_detect_model_type", lambda _: "")

        assert (
            cut_ce_mod.apply_cut_ce("C:\\\\experiments\\\\runs\\\\llama-3.1-8b\\\\")
            is True
        )
        assert (
            liger_mod.apply_liger_kernel("///tmp///models///phi-3-mini///")
            is True
        )
        assert calls_cce == ["llama"]
        assert calls_liger == ["phi3"]


class TestCutCEUnifiedMessage:
    """The two call sites (trainer/sft.py, utils/v028_features.py) used to
    hand-write diverging copies of the "no matching architecture" advisory."""

    def test_sft_and_v028_features_reference_the_shared_constant(self):
        """Both call sites import ``NO_MATCHING_ARCHITECTURE_MESSAGE`` rather
        than hand-writing their own copy of the string, so it is checked by
        name here: a literal-text check would pass even if one side kept a
        hardcoded string that happened to match today and drifted tomorrow."""
        import inspect

        import soup_cli.trainer.sft as sft_mod
        import soup_cli.utils.v028_features as v028_mod

        assert "NO_MATCHING_ARCHITECTURE_MESSAGE" in inspect.getsource(sft_mod)
        assert "NO_MATCHING_ARCHITECTURE_MESSAGE" in inspect.getsource(v028_mod)


class TestCutCEValidation:
    """Cut Cross-Entropy config validation."""

    def test_validate_cut_ce_disabled_returns_empty(self):
        from soup_cli.utils.cut_ce import validate_cut_ce_config

        errors = validate_cut_ce_config(False, "transformers", "cuda")
        assert errors == []

    def test_validate_cut_ce_not_installed(self):
        from soup_cli.utils.cut_ce import validate_cut_ce_config

        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=False
        ):
            errors = validate_cut_ce_config(True, "transformers", "cuda")
            assert any("not installed" in err for err in errors)

    def test_validate_cut_ce_requires_cuda(self):
        from soup_cli.utils.cut_ce import validate_cut_ce_config

        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=True
        ):
            errors = validate_cut_ce_config(True, "transformers", "cpu")
            assert any("CUDA" in err for err in errors)

    def test_validate_cut_ce_unsloth_incompatible(self):
        from soup_cli.utils.cut_ce import validate_cut_ce_config

        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=True
        ):
            errors = validate_cut_ce_config(True, "unsloth", "cuda")
            assert any("unsloth" in err.lower() for err in errors)

    def test_validate_cut_ce_valid(self):
        from soup_cli.utils.cut_ce import validate_cut_ce_config

        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=True
        ):
            errors = validate_cut_ce_config(True, "transformers", "cuda")
            assert errors == []

    def test_validate_cut_ce_mlx_incompatible(self):
        from soup_cli.utils.cut_ce import validate_cut_ce_config

        with patch(
            "soup_cli.utils.cut_ce.check_cut_ce_available", return_value=True
        ):
            errors = validate_cut_ce_config(True, "mlx", "mps")
            assert any("mlx" in err.lower() for err in errors)


# ─── Part B: FP8 training ─────────────────────────────────────────────────


class TestFP8Config:
    """quantization_aware now accepts bool or literal 'fp8'."""

    def test_quantization_aware_default_false(self):
        cfg = SoupConfig(base="test/model", data={"train": "./data.jsonl"})
        assert cfg.training.quantization_aware is False

    def test_quantization_aware_bool_true_is_refused(self):
        """The bool form (int8 QAT) is refused at load since #1222."""
        with pytest.raises(ValidationError, match="#1222"):
            SoupConfig(
                base="test/model",
                data={"train": "./data.jsonl"},
                training={"quantization_aware": True},
            )

    def test_quantization_aware_fp8(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"quantization_aware": "fp8"},
        )
        assert cfg.training.quantization_aware == "fp8"

    def test_quantization_aware_invalid_string_rejected(self):
        """Only 'fp8' literal is accepted, other strings rejected."""
        with pytest.raises(ValidationError) as exc:
            SoupConfig(
                base="test/model",
                data={"train": "./data.jsonl"},
                training={"quantization_aware": "fp16"},
            )
        assert "fp8" in str(exc.value) or "quantization_aware" in str(exc.value)

    def test_quantization_aware_bool_still_works(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"quantization_aware": False},
        )
        assert cfg.training.quantization_aware is False


class TestFP8Availability:
    """FP8 training dependency checks."""

    def test_is_fp8_available_returns_bool(self):
        from soup_cli.utils.fp8 import is_fp8_available

        result = is_fp8_available()
        assert isinstance(result, bool)

    def test_fp8_requires_hopper_gpu_info(self):
        """is_fp8_gpu_supported should check GPU compute capability."""
        from soup_cli.utils.fp8 import is_fp8_gpu_supported

        # Shouldn't crash even without CUDA
        result = is_fp8_gpu_supported()
        assert isinstance(result, bool)

    def test_is_fp8_available_false_when_deps_missing(self):
        """Explicit false branch — both torchao.float8 and transformer_engine absent."""
        with patch.dict(
            "sys.modules",
            {"torchao.float8": None, "transformer_engine": None},
        ):
            from soup_cli.utils.fp8 import is_fp8_available

            assert is_fp8_available() is False

    def test_is_fp8_gpu_supported_pre_hopper_false(self):
        """Pre-Hopper (SM 8.x, e.g. A100) is not supported."""
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = True
        fake_torch.cuda.get_device_capability.return_value = (8, 0)
        with patch.dict("sys.modules", {"torch": fake_torch}):
            from soup_cli.utils.fp8 import is_fp8_gpu_supported

            assert is_fp8_gpu_supported() is False

    def test_is_fp8_gpu_supported_hopper_true(self):
        """Hopper (SM 9.x, H100) is supported."""
        fake_torch = MagicMock()
        fake_torch.cuda.is_available.return_value = True
        fake_torch.cuda.get_device_capability.return_value = (9, 0)
        with patch.dict("sys.modules", {"torch": fake_torch}):
            from soup_cli.utils.fp8 import is_fp8_gpu_supported

            assert is_fp8_gpu_supported() is True


# ─── Part C: Gradient checkpointing tiers ─────────────────────────────────


class TestGradientCheckpointingTiers:
    """gradient_checkpointing accepts bool or tier literal."""

    def test_default_false(self):
        cfg = SoupConfig(base="test/model", data={"train": "./data.jsonl"})
        assert cfg.training.gradient_checkpointing is False

    def test_bool_true(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"gradient_checkpointing": True},
        )
        assert cfg.training.gradient_checkpointing is True

    def test_tier_selective(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"gradient_checkpointing": "selective"},
        )
        assert cfg.training.gradient_checkpointing == "selective"

    def test_tier_medium(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"gradient_checkpointing": "medium"},
        )
        assert cfg.training.gradient_checkpointing == "medium"

    def test_tier_full(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"gradient_checkpointing": "full"},
        )
        assert cfg.training.gradient_checkpointing == "full"

    def test_tier_auto(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"gradient_checkpointing": "auto"},
        )
        assert cfg.training.gradient_checkpointing == "auto"

    def test_invalid_tier_rejected(self):
        with pytest.raises(ValidationError) as exc:
            SoupConfig(
                base="test/model",
                data={"train": "./data.jsonl"},
                training={"gradient_checkpointing": "partial"},
            )
        assert "gradient_checkpointing" in str(exc.value)


class TestGradientCheckpointingResolver:
    """Resolve config value + GPU info → kwargs for TrainingArguments."""

    def test_resolve_disabled_returns_empty(self):
        from soup_cli.utils.gradient_ckpt import resolve_gradient_checkpointing

        kwargs = resolve_gradient_checkpointing(False, gpu_memory_gb=80)
        assert kwargs == {}

    def test_resolve_bool_true_returns_full_ckpt(self):
        from soup_cli.utils.gradient_ckpt import resolve_gradient_checkpointing

        kwargs = resolve_gradient_checkpointing(True, gpu_memory_gb=80)
        assert kwargs["gradient_checkpointing"] is True

    def test_resolve_full_tier(self):
        from soup_cli.utils.gradient_ckpt import resolve_gradient_checkpointing

        kwargs = resolve_gradient_checkpointing("full", gpu_memory_gb=80)
        assert kwargs["gradient_checkpointing"] is True

    def test_resolve_selective_tier(self):
        from soup_cli.utils.gradient_ckpt import (
            resolve_gradient_checkpointing,
            resolve_granularity,
        )

        kwargs = resolve_gradient_checkpointing("selective", gpu_memory_gb=80)
        assert kwargs == {"gradient_checkpointing": False}
        # Granularity is exposed via a separate helper for the wrapper.
        assert resolve_granularity("selective", gpu_memory_gb=80) == "selective"

    def test_resolve_medium_tier(self):
        from soup_cli.utils.gradient_ckpt import resolve_gradient_checkpointing

        kwargs = resolve_gradient_checkpointing("medium", gpu_memory_gb=80)
        assert kwargs["gradient_checkpointing"] is True
        assert kwargs["gradient_checkpointing_kwargs"] == {
            "use_reentrant": False,
            "every_n_layers": 2,
        }

    def test_resolve_auto_low_memory_selects_full(self):
        from soup_cli.utils.gradient_ckpt import resolve_gradient_checkpointing

        kwargs = resolve_gradient_checkpointing("auto", gpu_memory_gb=16)
        # Low VRAM → full checkpointing
        assert kwargs["gradient_checkpointing"] is True

    def test_resolve_auto_80gb_selects_medium(self):
        from soup_cli.utils.gradient_ckpt import resolve_gradient_checkpointing

        # 80GB resolves to medium (the selective threshold is strictly > 80).
        kwargs = resolve_gradient_checkpointing("auto", gpu_memory_gb=80)
        assert kwargs["gradient_checkpointing"] is True
        assert kwargs["gradient_checkpointing_kwargs"]["every_n_layers"] == 2

    def test_resolve_auto_very_high_memory_selects_selective(self):
        from soup_cli.utils.gradient_ckpt import (
            resolve_granularity,
        )

        # 192GB (H200): selective (attention-only) tier, minimize slowdown
        assert resolve_granularity("auto", gpu_memory_gb=192) == "selective"

    def test_resolve_auto_medium_memory_selects_medium(self):
        from soup_cli.utils.gradient_ckpt import resolve_granularity

        # 40GB (A100 40GB): medium (every other block)
        assert resolve_granularity("auto", gpu_memory_gb=40) == "medium"

    def test_resolve_auto_no_gpu_info_full(self):
        from soup_cli.utils.gradient_ckpt import resolve_granularity

        assert resolve_granularity("auto", gpu_memory_gb=None) == "full"

    def test_describe_tier_off(self):
        from soup_cli.utils.gradient_ckpt import describe_tier

        assert describe_tier(False) == "off"

    def test_describe_tier_full(self):
        from soup_cli.utils.gradient_ckpt import describe_tier

        assert "full" in describe_tier(True)
        assert "full" in describe_tier("full")

    def test_describe_tier_auto(self):
        from soup_cli.utils.gradient_ckpt import describe_tier

        assert "auto" in describe_tier("auto")
        assert "auto" in describe_tier("auto", gpu_memory_gb=16)
        assert "selective" in describe_tier("auto", gpu_memory_gb=192)


# ─── Part D: Kernel auto-composition ─────────────────────────────────────


class TestKernelPickerConfig:
    """kernel_auto_compose config flag."""

    def test_kernel_auto_compose_default_false(self):
        cfg = SoupConfig(base="test/model", data={"train": "./data.jsonl"})
        assert cfg.training.kernel_auto_compose is False

    def test_kernel_auto_compose_enabled_is_rejected(self):
        with pytest.raises(ValidationError, match="does not apply the selected flags"):
            SoupConfig(
                base="test/model",
                data={"train": "./data.jsonl"},
                training={"kernel_auto_compose": True},
            )


class TestKernelPickerEnumerate:
    """Kernel picker enumerates available kernel combinations."""

    def test_enumerate_returns_list(self):
        from soup_cli.utils.kernel_picker import enumerate_kernel_combos

        combos = enumerate_kernel_combos(backend="transformers", device="cuda")
        assert isinstance(combos, list)

    def test_enumerate_baseline_always_present(self):
        from soup_cli.utils.kernel_picker import enumerate_kernel_combos

        combos = enumerate_kernel_combos(backend="transformers", device="cuda")
        # Baseline (no special kernels) must always be an option
        assert any(c.get("name") == "baseline" for c in combos)

    def test_enumerate_cpu_only_baseline(self):
        from soup_cli.utils.kernel_picker import enumerate_kernel_combos

        combos = enumerate_kernel_combos(backend="transformers", device="cpu")
        # On CPU, only baseline should be available
        assert len(combos) == 1
        assert combos[0]["name"] == "baseline"

    def test_enumerate_unsloth_skips_liger(self):
        from soup_cli.utils.kernel_picker import enumerate_kernel_combos

        combos = enumerate_kernel_combos(backend="unsloth", device="cuda")
        # Unsloth has its own fused kernels - no Liger combos
        for combo in combos:
            assert "liger" not in combo.get("name", "").lower()


class TestKernelPickerDecision:
    """Kernel picker decision logic (mocked benchmarks)."""

    def test_pick_best_returns_dict(self):
        from soup_cli.utils.kernel_picker import pick_best_kernel

        # With fake timing results, picks fastest
        candidates = [
            {"name": "baseline", "time_ms": 100.0},
            {"name": "liger", "time_ms": 70.0},
            {"name": "liger+flash", "time_ms": 50.0},
        ]
        best = pick_best_kernel(candidates)
        assert best["name"] == "liger+flash"

    def test_pick_best_baseline_if_only_one(self):
        from soup_cli.utils.kernel_picker import pick_best_kernel

        candidates = [{"name": "baseline", "time_ms": 100.0}]
        best = pick_best_kernel(candidates)
        assert best["name"] == "baseline"

    def test_pick_best_empty_raises(self):
        from soup_cli.utils.kernel_picker import pick_best_kernel

        with pytest.raises(ValueError, match="at least one"):
            pick_best_kernel([])

    def test_pick_best_tie_returns_first(self):
        """Ties broken by list order (first-wins) — preferred default first."""
        from soup_cli.utils.kernel_picker import pick_best_kernel

        candidates = [
            {"name": "baseline", "time_ms": 50.0},
            {"name": "liger", "time_ms": 50.0},
        ]
        best = pick_best_kernel(candidates)
        assert best["name"] == "baseline"

    def test_pick_best_all_missing_time_raises(self):
        """All-untimed candidates means benchmarking failed — must not promote silently."""
        from soup_cli.utils.kernel_picker import pick_best_kernel

        candidates = [
            {"name": "baseline"},
            {"name": "liger"},
            {"name": "flash", "time_ms": None},
        ]
        with pytest.raises(ValueError, match="finite time_ms"):
            pick_best_kernel(candidates)

    def test_pick_best_nan_time_treated_as_missing(self):
        from soup_cli.utils.kernel_picker import pick_best_kernel

        candidates = [
            {"name": "baseline", "time_ms": float("nan")},
            {"name": "liger", "time_ms": 50.0},
        ]
        best = pick_best_kernel(candidates)
        assert best["name"] == "liger"


# ─── Part E: Cross-document attention masking ────────────────────────────


class TestCrossDocAttnMaskConfig:
    """packing_cross_doc_attn_mask config."""

    def test_default_false(self):
        cfg = SoupConfig(base="test/model", data={"train": "./data.jsonl"})
        assert cfg.training.packing_cross_doc_attn_mask is False

    def test_enabled(self):
        with pytest.raises(ValidationError, match="bfd"):
            SoupConfig(
                base="test/model",
                data={"train": "./data.jsonl"},
                training={"packing_cross_doc_attn_mask": True, "packing": True},
            )

class TestCrossDocAttnMaskBuild:
    """Build cross-doc attention mask from document boundaries."""

    def test_build_mask_single_doc(self):
        from soup_cli.utils.cross_doc_attn import build_cross_doc_mask

        # Single doc spans the whole sequence — no masking needed
        boundaries = [0, 10]  # doc0 occupies positions 0..9
        mask = build_cross_doc_mask(boundaries, seq_length=10)
        # Shape (10, 10), lower-triangular within the doc
        assert mask.shape == (10, 10)

    def test_build_mask_two_docs(self):
        from soup_cli.utils.cross_doc_attn import build_cross_doc_mask

        # doc0: 0..4, doc1: 5..9
        boundaries = [0, 5, 10]
        mask = build_cross_doc_mask(boundaries, seq_length=10)
        # Position 5 (doc1 start) should NOT attend to position 0 (doc0)
        assert mask[5, 0] == 0
        # Position 5 attending to itself should be 1
        assert mask[5, 5] == 1
        # Position 0 attending to itself should be 1
        assert mask[0, 0] == 1
        # Position 1 attending to 0 should be 1 (same doc, causal)
        assert mask[1, 0] == 1
        # Position 0 attending to 1 should be 0 (causal — future)
        assert mask[0, 1] == 0

    def test_build_mask_boundaries_validation(self):
        from soup_cli.utils.cross_doc_attn import build_cross_doc_mask

        # Boundaries must start at 0 and end at seq_length
        with pytest.raises(ValueError):
            build_cross_doc_mask([1, 10], seq_length=10)

    def test_build_mask_boundaries_monotonic(self):
        from soup_cli.utils.cross_doc_attn import build_cross_doc_mask

        with pytest.raises(ValueError, match="increasing"):
            build_cross_doc_mask([0, 5, 3, 10], seq_length=10)

    def test_build_mask_empty_boundaries_raises(self):
        from soup_cli.utils.cross_doc_attn import build_cross_doc_mask

        with pytest.raises(ValueError, match="non-empty"):
            build_cross_doc_mask([], seq_length=10)

    def test_build_mask_wrong_end_raises(self):
        from soup_cli.utils.cross_doc_attn import build_cross_doc_mask

        with pytest.raises(ValueError, match="seq_length"):
            build_cross_doc_mask([0, 8], seq_length=10)

    def test_compute_doc_boundaries_empty(self):
        from soup_cli.utils.cross_doc_attn import compute_doc_boundaries

        with pytest.raises(ValueError):
            compute_doc_boundaries([])

    def test_compute_doc_boundaries_non_positive(self):
        from soup_cli.utils.cross_doc_attn import compute_doc_boundaries

        with pytest.raises(ValueError):
            compute_doc_boundaries([3, 0, 4])

    def test_compute_doc_boundaries_valid(self):
        from soup_cli.utils.cross_doc_attn import compute_doc_boundaries

        assert compute_doc_boundaries([3, 2, 4]) == [0, 3, 5, 9]


# ─── Part F: Activation offloading ────────────────────────────────────────


class TestActivationOffloadingConfig:
    """activation_offloading config."""

    def test_default_none(self):
        cfg = SoupConfig(base="test/model", data={"train": "./data.jsonl"})
        assert cfg.training.activation_offloading is None

    def test_cpu(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"activation_offloading": "cpu"},
        )
        assert cfg.training.activation_offloading == "cpu"

    def test_disk(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={"activation_offloading": "disk"},
        )
        assert cfg.training.activation_offloading == "disk"

    def test_invalid_target(self):
        with pytest.raises(ValidationError) as exc:
            SoupConfig(
                base="test/model",
                data={"train": "./data.jsonl"},
                training={"activation_offloading": "gpu"},
            )
        assert "activation_offloading" in str(exc.value)


class TestActivationOffloadingValidation:
    """Offloading config validation."""

    def test_validate_none_returns_empty(self):
        from soup_cli.utils.activation_offload import validate_offload_config

        errors = validate_offload_config(None, "transformers", "cuda")
        assert errors == []

    def test_validate_cpu_requires_cuda(self):
        from soup_cli.utils.activation_offload import validate_offload_config

        errors = validate_offload_config("cpu", "transformers", "cpu")
        assert any("CUDA" in err for err in errors)

    def test_validate_cpu_unsloth_incompatible(self):
        from soup_cli.utils.activation_offload import validate_offload_config

        errors = validate_offload_config("cpu", "unsloth", "cuda")
        assert any("unsloth" in err.lower() for err in errors)

    def test_validate_disk_valid(self):
        from soup_cli.utils.activation_offload import validate_offload_config

        errors = validate_offload_config(
            "disk", "transformers", "cuda", save_dir="./scratch"
        )
        assert errors == []

    def test_validate_disk_requires_save_dir(self):
        """Disk mode must reject calls without save_dir — fail-fast at validate()."""
        from soup_cli.utils.activation_offload import validate_offload_config

        errors = validate_offload_config(
            "disk", "transformers", "cuda", save_dir=None
        )
        assert any("save_dir" in err for err in errors)

    def test_validate_disk_on_cpu_rejected(self):
        from soup_cli.utils.activation_offload import validate_offload_config

        errors = validate_offload_config(
            "disk", "transformers", "cpu", save_dir="./scratch"
        )
        assert any("CUDA" in err for err in errors)

    def test_validate_mlx_rejected(self):
        from soup_cli.utils.activation_offload import validate_offload_config

        errors = validate_offload_config("cpu", "mlx", "cuda")
        assert any("mlx" in err.lower() for err in errors)


class TestActivationOffloadingHooks:
    """Install / uninstall hooks for offloading."""

    def test_context_manager_noop_when_none(self):
        from soup_cli.utils.activation_offload import offload_context

        # Should be a no-op when target is None
        with offload_context(None, save_dir=None):
            pass  # nothing to assert - just shouldn't crash

    def test_context_manager_cpu(self):
        from soup_cli.utils.activation_offload import offload_context

        # Should not crash even without torch
        with offload_context("cpu", save_dir=None):
            pass

    def test_offload_context_disk_requires_dir(self, tmp_path):
        from soup_cli.utils.activation_offload import offload_context

        # Disk mode should accept a save_dir
        with offload_context("disk", save_dir=str(tmp_path)):
            pass

    def test_offload_context_unknown_target_raises(self):
        """Defense-in-depth: unknown target raises even if torch is present."""
        from soup_cli.utils.activation_offload import offload_context

        # Only reaches the ValueError branch if torch imports successfully
        try:
            import torch  # noqa: F401
        except ImportError:
            pytest.skip("torch not installed; ValueError branch unreachable")

        with pytest.raises(ValueError, match="Unknown activation_offloading"):
            with offload_context("invalid", save_dir=None):
                pass

    def test_offload_context_disk_creates_save_dir(self, tmp_path):
        """Disk mode should create the scratch directory on context entry."""
        try:
            import torch  # noqa: F401
        except ImportError:
            pytest.skip("torch not installed; disk hooks unreachable")

        from soup_cli.utils.activation_offload import offload_context

        scratch = tmp_path / "offload_scratch"
        with offload_context("disk", save_dir=str(scratch)):
            assert scratch.exists()


# ─── Integration: multiple features composed ──────────────────────────────


class TestV028SFTOnlyValidator:
    """v0.28.0 features are wired only in SFTTrainerWrapper.

    The SoupConfig validator rejects non-SFT tasks when speed/memory flags
    are set — prevents silent no-ops and the known fp8 crash path in the
    legacy int8 QAT wrapper.
    """

    def test_use_cut_ce_now_accepted_on_dpo(self):
        # v0.33.0 #43 — DPO is now in the supported task set.
        cfg = SoupConfig(
            base="m",
            task="dpo",
            data={"train": "./d.jsonl", "format": "dpo"},
            training={"use_cut_ce": True},
        )
        assert cfg.training.use_cut_ce is True

    def test_fp8_now_accepted_on_grpo(self):
        # v0.35.0 #60 — every transformer-backend trainer now supported.
        cfg = SoupConfig(
            base="m",
            task="grpo",
            data={"train": "./d.jsonl"},
            training={"quantization_aware": "fp8"},
        )
        assert cfg.training.quantization_aware == "fp8"

    def test_activation_offloading_now_accepted_on_ppo(self):
        # v0.35.0 #60 — PPO accepts activation_offloading.
        cfg = SoupConfig(
            base="m",
            task="ppo",
            data={"train": "./d.jsonl"},
            training={"activation_offloading": "cpu"},
        )
        assert cfg.training.activation_offloading == "cpu"

    def test_kernel_auto_compose_rejected_on_kto(self):
        with pytest.raises(ValidationError, match="use_liger"):
            SoupConfig(
                base="m",
                task="kto",
                data={"train": "./d.jsonl", "format": "kto"},
                training={"kernel_auto_compose": True},
            )

    def test_v028_features_still_rejected_on_mlx_backend(self):
        # MLX backend has no equivalent kernels — gate must still fire.
        with pytest.raises(ValidationError) as exc:
            SoupConfig(
                base="m",
                task="sft",
                backend="mlx",
                data={"train": "./d.jsonl"},
                training={"use_cut_ce": True},
            )
        assert "mlx" in str(exc.value).lower()

    def test_sft_accepts_supported_features(self):
        """SFT still accepts the supported v0.28.0 flags."""
        cfg = SoupConfig(
            base="m",
            task="sft",
            data={"train": "./d.jsonl"},
            training={
                "use_cut_ce": True,
                "quantization_aware": "fp8",
                "activation_offloading": "cpu",
            },
        )
        assert cfg.training.use_cut_ce is True

    def test_non_sft_unaffected_when_flags_default(self):
        """DPO/GRPO with default v0.28.0 flags still validate."""
        cfg = SoupConfig(
            base="m",
            task="dpo",
            data={"train": "./d.jsonl", "format": "dpo"},
        )
        assert cfg.task == "dpo"

    def test_quantization_aware_bool_true_refused_on_dpo(self):
        """Int8 QAT (bool True) is refused on non-SFT tasks too (#1222)."""
        with pytest.raises(ValidationError, match="#1222"):
            SoupConfig(
                base="m",
                task="dpo",
                data={"train": "./d.jsonl", "format": "dpo"},
                training={"quantization_aware": True},
            )

    def test_gradient_checkpointing_tier_allowed_on_dpo(self):
        """Tier strings fall back to truthy (bool True) in non-SFT wrappers — no crash."""
        cfg = SoupConfig(
            base="m",
            task="dpo",
            data={"train": "./d.jsonl", "format": "dpo"},
            training={"gradient_checkpointing": "auto"},
        )
        assert cfg.training.gradient_checkpointing == "auto"


class TestV028Integration:
    """Multiple v0.28.0 features composed in one config."""

    def test_all_features_compose(self):
        cfg = SoupConfig(
            base="test/model",
            data={"train": "./data.jsonl"},
            training={
                "use_cut_ce": True,
                "quantization_aware": "fp8",
                "gradient_checkpointing": "auto",
                "packing": True,
                "activation_offloading": "cpu",
            },
        )
        tcfg = cfg.training
        assert tcfg.use_cut_ce is True
        assert tcfg.quantization_aware == "fp8"
        assert tcfg.gradient_checkpointing == "auto"
        assert tcfg.kernel_auto_compose is False
        assert tcfg.packing is True
        assert tcfg.activation_offloading == "cpu"