Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

30.7 KB
Raw Blame
"""#1223 — every trainer that receives a validation split evaluates it.

``data.val_split: 0.1`` moved 10% of the rows out of training and every task
wrapper passed them on as ``eval_dataset``, but none set ``eval_strategy``, whose
Hugging Face default is ``"no"``. ``Trainer.evaluate()`` never ran, so the rows
were used for nothing and #713's validation-loss sinks never got a value.

Driven for real on CPU, per task, because a per-task omission is the failure
mode: `setup()` is called on a tiny checkpoint, the schedule the trainer actually
received is asserted, and then the run trains, which is the only way to know the
evaluation pass works on that trainer. Two did not: the embedding trainer's
default ``prediction_step`` called ``model(**inputs)`` with its own batch keys,
and TRL's BCO evaluation fails before its first training step (which a schedule
never reaches).

The config and loader half is ``test_issue1223_eval_steps_config.py``.
"""

from __future__ import annotations

import json
import math
from pathlib import Path

import pytest

from tests import conftest
from tests.conftest import strip_ansi


def _requires_train_extra():
    for mod in ("torch", "transformers", "peft", "trl", "datasets"):
        pytest.importorskip(mod, reason=f"{mod} is only in the [train] extra")


@pytest.fixture(autouse=True)
def _no_wandb(monkeypatch):
    monkeypatch.setenv("WANDB_DISABLED", "true")


def _plain(text: str | None) -> str:
    """ANSI-stripped and whitespace-collapsed: Rich colours and wraps CLI output."""
    return " ".join(strip_ansi(text).split())


# --------------------------------------------------------------------------
# fixtures -- a real, tiny checkpoint on disk. Duplicated from
# test_issue353_seed_every_task.py deliberately, as the suites around it do, so
# this file stays runnable on its own.
# --------------------------------------------------------------------------
def _tiny_llama_dir(root: Path, vocab: int = 64, hidden: int = 64) -> str:
    import torch
    from safetensors.torch import save_file
    from tokenizers import Tokenizer, models, pre_tokenizers
    from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedTokenizerFast

    torch.manual_seed(7)
    config = LlamaConfig(
        vocab_size=vocab,
        hidden_size=hidden,
        intermediate_size=hidden * 2,
        num_hidden_layers=2,
        num_attention_heads=4,
        num_key_value_heads=2,
        tie_word_embeddings=True,
        max_position_embeddings=128,
    )
    model = LlamaForCausalLM(config).to(torch.float32).eval()
    weights = root / "model"
    weights.mkdir(parents=True, exist_ok=True)
    state = {k: v.contiguous() for k, v in model.state_dict().items()}
    state.pop("lm_head.weight", None)
    save_file(state, str(weights / "model.safetensors"))
    config.save_pretrained(str(weights))

    words = {"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3}
    for word in ("hello", "world", "hi", "good", "answer", "bad"):
        words[word] = len(words)
    tok = Tokenizer(models.WordLevel(vocab=words, unk_token="<unk>"))
    tok.pre_tokenizer = pre_tokenizers.Whitespace()
    PreTrainedTokenizerFast(
        tokenizer_object=tok,
        unk_token="<unk>",
        bos_token="<s>",
        eos_token="</s>",
        pad_token="<pad>",
    ).save_pretrained(str(weights))
    return str(weights)


def _chat_rows(n):
    return [
        {
            "messages": [
                {"role": "user", "content": "hi"},
                {"role": "assistant", "content": "good answer"},
            ]
        }
        for _ in range(n)
    ]


def _pref_rows(n):
    return [{"prompt": "hi", "chosen": " good answer", "rejected": " bad"} for _ in range(n)]


def _kto_rows(n):
    return [{"prompt": "hi", "completion": " good answer", "label": i % 2 == 0} for i in range(n)]


def _text_rows(n):
    return [{"text": "hello world good answer"} for _ in range(n)]


def _label_rows(n):
    return [{"text": "hello world", "label": i % 2} for i in range(n)]


def _embedding_rows(n):
    return [{"anchor": "hello world", "positive": "good answer"} for _ in range(n)]


def _prm_rows(n):
    return [
        {"prompt": "hi", "completions": ["good", "answer"], "labels": [1.0, 0.0]}
        for _ in range(n)
    ]


def _prompt_rows(n):
    return [{"prompt": "hi"} for _ in range(n)]


#: task -> (module, wrapper class, rows, extra `training:` keys). The issue's
#: list minus grpo, which is generation-based and has its own tests below.
_EVALUATING = {
    "sft": ("soup_cli.trainer.sft", "SFTTrainerWrapper", _chat_rows, {}),
    "pretrain": ("soup_cli.trainer.pretrain", "PretrainTrainerWrapper", _text_rows, {}),
    "dpo": ("soup_cli.trainer.dpo", "DPOTrainerWrapper", _pref_rows, {}),
    "ipo": ("soup_cli.trainer.ipo", "IPOTrainerWrapper", _pref_rows, {}),
    "kto": ("soup_cli.trainer.kto", "KTOTrainerWrapper", _kto_rows, {}),
    "orpo": ("soup_cli.trainer.orpo", "ORPOTrainerWrapper", _pref_rows, {}),
    "simpo": ("soup_cli.trainer.simpo", "SimPOTrainerWrapper", _pref_rows, {}),
    "bco": ("soup_cli.trainer.bco", "BCOTrainerWrapper", _pref_rows, {}),
    "distill": ("soup_cli.trainer.distill", "DistillTrainerWrapper", _chat_rows, {}),
    # batch 1: the tiny checkpoint has no pad id, and a sequence-classification
    # head refuses batches > 1 without one -- in training as in evaluation.
    "classifier": (
        "soup_cli.trainer.classifier",
        "ClassifierTrainerWrapper",
        _label_rows,
        {"num_labels": 2, "batch_size": 1},
    ),
    "embedding": ("soup_cli.trainer.embedding", "EmbeddingTrainerWrapper", _embedding_rows, {}),
    "reward_model": (
        "soup_cli.trainer.reward_model",
        "RewardModelTrainerWrapper",
        _pref_rows,
        {},
    ),
    "prm": ("soup_cli.trainer.prm", "PRMTrainerWrapper", _prm_rows, {}),
    "moe_lora_routing": (
        "soup_cli.trainer.mole_routing",
        "MoleRoutingTrainerWrapper",
        _text_rows,
        {"mole_task_adapters": ["./adapter_a", "./adapter_b"]},
    ),
    "asr": ("soup_cli.trainer.asr", "AsrTrainerWrapper", None, {}),
}

#: The wrappers the issue lists as receiving a validation split.
_ISSUE_TASKS = frozenset({
    "sft", "pretrain", "dpo", "ipo", "kto", "orpo", "simpo", "bco", "grpo",
    "distill", "classifier", "embedding", "reward_model", "prm",
    "moe_lora_routing", "asr",
})

#: These build their Trainer inside `train()`, so the schedule can only be
#: read once the run has happened.
_BUILDS_IN_TRAIN = frozenset({"prm", "moe_lora_routing"})

_TINY_WHISPER = "hf-internal-testing/tiny-random-WhisperForConditionalGeneration"
_TRAIN_ROWS = 8
_VAL_ROWS = 2


def _write_sine_wav(path: Path, seconds: float = 0.5, sr: int = 16000) -> None:
    import numpy as np
    import soundfile as sf

    t = np.linspace(0.0, seconds, int(sr * seconds), endpoint=False)
    sf.write(str(path), (0.1 * np.sin(2 * np.pi * 220.0 * t)).astype("float32"), sr)


def _asr_rows(root: Path, n: int, prefix: str) -> list[dict]:
    rows = []
    for i in range(n):
        clip = root / f"{prefix}{i}.wav"
        _write_sine_wav(clip)
        rows.append({"audio": str(clip), "text": "hello world"})
    return rows


def _tiny_whisper_dir(root: Path) -> str:
    """The hub's tiny-random Whisper, resized so it can actually train.

    As shipped it takes 60 mel frames, but the feature extractor pads every clip
    to 30 s (3000 frames), and its vocabulary stops below the ids the tokenizer
    gives Whisper's own special tokens, so a training step on it fails before
    any loss. Same config (d_model 16, 2 layers) with room for 1500 encoder
    positions and the tokenizer's full vocabulary, fresh random weights, the hub
    copy's processor.
    """
    import torch
    from transformers import WhisperConfig, WhisperForConditionalGeneration, WhisperProcessor

    processor = WhisperProcessor.from_pretrained(_TINY_WHISPER)
    config = WhisperConfig.from_pretrained(_TINY_WHISPER)
    config.max_source_positions = 1500
    config.vocab_size = len(processor.tokenizer)
    torch.manual_seed(7)
    target = root / "whisper"
    WhisperForConditionalGeneration(config).save_pretrained(str(target))
    processor.save_pretrained(str(target))
    return str(target)


def _write_task_adapters(root: Path, weights: str) -> None:
    """Two tiny LoRA adapters for the MoLE gate to route between."""
    from peft import LoraConfig, get_peft_model
    from transformers import AutoModelForCausalLM

    for name in ("adapter_a", "adapter_b"):
        base = AutoModelForCausalLM.from_pretrained(weights)
        peft_model = get_peft_model(
            base,
            LoraConfig(
                r=4, lora_alpha=8, target_modules=["q_proj", "v_proj"], task_type="CAUSAL_LM"
            ),
        )
        peft_model.save_pretrained(str(root / name))


def _cfg(task: str, base: str, out: Path, **training_over):
    import yaml

    from soup_cli.config.loader import load_config_from_string

    training = {
        "batch_size": 2,
        "gradient_accumulation_steps": 1,
        "quantization": "none",
        "epochs": 1,
        "logging_steps": 1,
        "save_steps": 1000,
        "lora": {"r": 4, "alpha": 8, "target_modules": ["q_proj", "v_proj"]},
    }
    if task in _EVALUATING:
        training.update(_EVALUATING[task][3])
    if task == "distill":
        training["teacher_model"] = base
    if task == "grpo":
        training.update({"num_generations": 2, "reward_fn": "format"})
    if task in {"asr", "prm"}:
        # prm fine-tunes every base parameter and refuses a lora block; asr's
        # LoRA is its own opt-in (asr_lora).
        training.pop("lora")
    training.update(training_over)
    data = {"train": "train.jsonl", "max_length": 64}
    if task in {"sft", "dpo", "ipo", "kto", "orpo", "simpo", "bco", "distill", "grpo"}:
        data["chat_template"] = "chatml"
    if task == "asr":
        data["format"] = "asr"
    return load_config_from_string(
        yaml.safe_dump(
            {
                "base": base,
                "task": task,
                "backend": "transformers",
                "modality": "text",
                "data": data,
                "training": training,
                "output": str(out),
            }
        )
    )


def _build(tmp_path, monkeypatch, task, *, n_train=_TRAIN_ROWS, n_val=_VAL_ROWS, **over):
    """A real wrapper over a real tiny checkpoint, `setup()` called with the
    train/val shape the loader hands over when `data.val_split` > 0."""
    import importlib

    monkeypatch.chdir(tmp_path)
    if task == "asr":
        pytest.importorskip("soundfile")
        try:  # the tiny Whisper config and processor come from the hub
            base = _tiny_whisper_dir(tmp_path)
        except OSError as exc:
            pytest.skip(f"tiny whisper unavailable: {exc}")
        dataset = {"train": _asr_rows(tmp_path, n_train, "t")}
        if n_val:
            dataset["val"] = _asr_rows(tmp_path, n_val, "v")
    else:
        base = _tiny_llama_dir(tmp_path)
        rows = _EVALUATING[task][2] if task in _EVALUATING else _prompt_rows
        dataset = {"train": rows(n_train)}
        if n_val:
            dataset["val"] = rows(n_val)
    if task == "moe_lora_routing":
        _write_task_adapters(tmp_path, base)
    if task in _EVALUATING:
        module, cls_name = _EVALUATING[task][:2]
    else:
        module, cls_name = "soup_cli.trainer.grpo", "GRPOTrainerWrapper"
    cfg = _cfg(task, base, tmp_path / "out", **over)
    wrapper = getattr(importlib.import_module(module), cls_name)(cfg, device="cpu")
    wrapper.setup(dataset)
    return wrapper


def _trainer_default_device_is_cpu() -> bool:
    """True when transformers' default device is the CPU these wrappers use.

    Every wrapper here is built with ``device="cpu"``, but transformers moves
    the model to its own default device: CUDA when a card is visible, MPS on
    Apple Silicon. A resident model moves whole and does not notice. A streamed
    model keeps its decoder weights where the wrapper streams them, so a
    streamed forward then mixes devices ("found at least two devices, mps:0 and
    cpu" on the macOS runners). ``test_v07200.py`` skips its CPU streaming steps
    on MPS for the same reason.

    CUDA is read through the suite's one probe, ``conftest.cuda_available``
    (#833 forbids a private one), looked up at call time so the condition test
    below can pin it.
    """
    import torch

    if conftest.cuda_available():
        return False
    backend = getattr(torch.backends, "mps", None)
    return not (backend is not None and backend.is_available())


def _hf_trainer(wrapper):
    """`embedding` composes HF's Trainer instead of subclassing it."""
    trainer = wrapper.trainer
    return getattr(trainer, "_trainer", trainer)


def _strategy(args) -> str:
    return getattr(args.eval_strategy, "value", args.eval_strategy)


def _eval_entries(trainer) -> list[dict]:
    return [e for e in trainer.state.log_history if "eval_loss" in e]


def _run(wrapper, task):
    """Train, and return the HF trainer that ran."""
    if task in _BUILDS_IN_TRAIN:
        wrapper.train()
    else:
        _hf_trainer(wrapper).train()
    return _hf_trainer(wrapper)


# ==========================================================================
# the acceptance matrix
# ==========================================================================
class TestEveryWrapperEvaluatesItsSplit:
    """Whenever the trainer receives a non-empty validation split, its schedule
    is not ``"no"``, the pass runs at the train batch, and it logs a finite
    ``eval_loss``. ``grpo`` is the generation-based exception, pinned below."""

    @pytest.mark.parametrize("task", sorted(_EVALUATING))
    def test_the_split_is_scheduled_and_evaluated(self, tmp_path, monkeypatch, task):
        _requires_train_extra()
        wrapper = _build(tmp_path, monkeypatch, task)
        if task not in _BUILDS_IN_TRAIN:
            trainer = _hf_trainer(wrapper)
            assert trainer.eval_dataset is not None and len(trainer.eval_dataset) > 0
            assert _strategy(trainer.args) == "epoch", (
                f"{task}: a non-empty validation split with eval_strategy="
                f"{_strategy(trainer.args)!r} is withheld and never evaluated"
            )
        trainer = _run(wrapper, task)
        args = trainer.args
        assert _strategy(args) == "epoch", task
        assert args.per_device_eval_batch_size == args.per_device_train_batch_size, (
            f"{task}: evaluation runs at batch {args.per_device_eval_batch_size}, "
            f"training at {args.per_device_train_batch_size}"
        )
        evals = _eval_entries(trainer)
        assert len(evals) == 1, f"{task}: one epoch should evaluate once, got {evals}"
        assert math.isfinite(evals[0]["eval_loss"]), (task, evals)

    def test_the_matrix_covers_the_issue_list(self):
        assert set(_EVALUATING) | {"grpo"} == _ISSUE_TASKS


class TestGrpoIsGenerationBased:
    """Documented as not evaluating by default, and the loader withholds nothing
    for it, so no row is spent on an evaluation that never runs."""

    def test_the_loader_withholds_nothing(self, tmp_path, monkeypatch):
        _requires_train_extra()
        from soup_cli.data.loader import load_dataset
        from soup_cli.trainer.grpo import GRPOTrainerWrapper
        from soup_cli.utils.eval_schedule import loader_data_config

        monkeypatch.chdir(tmp_path)
        weights = _tiny_llama_dir(tmp_path)
        (tmp_path / "train.jsonl").write_text(
            "\n".join(json.dumps(r) for r in _chat_rows(10)) + "\n", encoding="utf-8"
        )
        cfg = _cfg("grpo", weights, tmp_path / "out")
        assert cfg.data.val_split == pytest.approx(0.1)
        # Exactly what `soup train` hands the wrapper for this config.
        dataset = load_dataset(loader_data_config(cfg), preserve_source_columns=True)
        assert len(dataset["train"]) == 10
        assert not dataset.get("val")

        wrapper = GRPOTrainerWrapper(cfg, device="cpu")
        wrapper.setup(dataset)
        assert len(wrapper.trainer.train_dataset) == 10
        assert wrapper.trainer.eval_dataset is None
        assert _strategy(wrapper.trainer.args) == "no"

    def test_a_split_it_is_handed_anyway_is_not_evaluated_unasked(self, tmp_path, monkeypatch):
        """An HF-hub dataset's own validation split still arrives; it is not
        withheld from training, and grpo does not generate over it unasked."""
        _requires_train_extra()
        wrapper = _build(tmp_path, monkeypatch, "grpo")
        assert len(wrapper.trainer.eval_dataset) == _VAL_ROWS
        assert _strategy(wrapper.trainer.args) == "no"

    def test_eval_steps_opts_in_and_the_pass_runs(self, tmp_path, monkeypatch):
        _requires_train_extra()
        wrapper = _build(tmp_path, monkeypatch, "grpo", eval_steps=2)
        args = wrapper.trainer.args
        assert (_strategy(args), args.eval_steps) == ("steps", 2)
        assert args.per_device_eval_batch_size == args.per_device_train_batch_size
        args.max_steps = 2
        wrapper.trainer.train()
        evals = _eval_entries(wrapper.trainer)
        assert [e["step"] for e in evals] == [2]
        assert math.isfinite(evals[0]["eval_loss"])


class TestGrpoEvalBatchHoldsWholeGroups:
    """TRL evaluates ``num_generations`` completions per held-out prompt and
    refuses an eval batch that splits a group. Batch 3 with 2 generations
    trains (gradient accumulation 2 makes the generation batch 6) but raised at
    setup, on one process, as soon as ``eval_steps`` was set."""

    _NOTE = (
        "grpo evaluates at batch 2, not the train batch 3: TRL's evaluation needs "
        "whole groups of num_generations=2 completions."
    )

    def _build3(self, tmp_path, monkeypatch, **over):
        return _build(
            tmp_path, monkeypatch, "grpo", batch_size=3, gradient_accumulation_steps=2, **over
        )

    def test_the_eval_batch_rounds_down_to_whole_groups(self, tmp_path, monkeypatch, capsys):
        _requires_train_extra()
        wrapper = self._build3(tmp_path, monkeypatch, eval_steps=1)
        args = wrapper.trainer.args
        assert (args.per_device_train_batch_size, args.per_device_eval_batch_size) == (3, 2)
        assert self._NOTE in _plain(capsys.readouterr().out)
        args.max_steps = 1
        wrapper.trainer.train()
        assert [e["step"] for e in _eval_entries(wrapper.trainer)] == [1]

    def test_no_note_when_nothing_is_evaluated(self, tmp_path, monkeypatch, capsys):
        _requires_train_extra()
        self._build3(tmp_path, monkeypatch)
        assert "grpo evaluates at batch" not in _plain(capsys.readouterr().out)

    def test_no_note_when_the_train_batch_already_holds_whole_groups(
        self, tmp_path, monkeypatch, capsys
    ):
        _requires_train_extra()
        wrapper = _build(tmp_path, monkeypatch, "grpo", batch_size=4, eval_steps=1)
        assert wrapper.trainer.args.per_device_eval_batch_size == 4
        assert "grpo evaluates at batch" not in _plain(capsys.readouterr().out)


class TestLayerStreamingEvaluates:
    """``stream_layers: true`` is not special-cased: the streamed model runs the
    evaluation pass on CPU, through the streamed decoder layers, and the loss it
    reports is the resident model's (same checkpoint, same rows)."""

    def test_a_streamed_run_evaluates_its_split(self, tmp_path, monkeypatch):
        _requires_train_extra()
        if not _trainer_default_device_is_cpu():
            pytest.skip(
                "transformers places the model on cuda/mps here while the wrapper "
                "streams on cpu (see _trainer_default_device_is_cpu)"
            )
        monkeypatch.setenv("SOUP_LAYER_STREAM_CACHE_DIR", str(tmp_path / "cache"))
        (tmp_path / "streamed").mkdir()
        (tmp_path / "resident").mkdir()
        streamed = _build(
            tmp_path / "streamed",
            monkeypatch,
            "sft",
            n_train=4,
            batch_size=1,
            stream_layers=True,
            gradient_checkpointing=True,
        )
        assert _strategy(streamed.trainer.args) == "epoch"
        loads = streamed._stream_runtime.pool.loads
        streamed_loss = streamed.trainer.evaluate()["eval_loss"]
        assert streamed._stream_runtime.pool.loads > loads, "no layer was streamed"

        resident = _build(tmp_path / "resident", monkeypatch, "sft", n_train=4, batch_size=1)
        resident_loss = resident.trainer.evaluate()["eval_loss"]
        assert streamed_loss == pytest.approx(resident_loss, rel=1e-4)

        streamed.trainer.train()
        evals = _eval_entries(streamed.trainer)
        assert len(evals) == 1 and math.isfinite(evals[0]["eval_loss"]), evals


class TestTheStreamingSkipStaysNarrow:
    """The streaming test above runs only where transformers' default device is
    the CPU its wrapper streams on. Both edges are pinned: a condition that
    quietly widened would skip the test everywhere, and a skipped test is the
    same colour as a passing one."""

    @pytest.mark.parametrize(
        ("cuda", "mps", "runs"),
        [(False, False, True), (False, True, False), (True, False, False), (True, True, False)],
    )
    def test_the_condition(self, monkeypatch, cuda, mps, runs):
        torch = pytest.importorskip("torch")
        monkeypatch.setattr(conftest, "cuda_available", lambda: cuda)
        monkeypatch.setattr(torch.backends.mps, "is_available", lambda: mps)
        assert _trainer_default_device_is_cpu() is runs


class TestNoSplitNoEvaluation:
    """The control: with no validation rows nothing is scheduled, so a run with
    ``val_split: 0`` behaves exactly as it did."""

    @pytest.mark.parametrize("task", ["sft", "dpo"])
    def test_no_rows_no_schedule(self, tmp_path, monkeypatch, task):
        _requires_train_extra()
        wrapper = _build(tmp_path, monkeypatch, task, n_val=0)
        trainer = _hf_trainer(wrapper)
        assert trainer.eval_dataset is None
        assert _strategy(trainer.args) == "no"
        trainer.train()
        assert _eval_entries(trainer) == []


# ==========================================================================
# the evaluation batch is the resolved train batch
# ==========================================================================
class TestTheEvalBatchIsTheTrainBatch:
    """HF's default eval batch is 8. A card sized to train at 1 runs out of
    memory at 8, so the pass must run at whatever the train batch resolved to,
    including after ``batch_size: auto``."""

    @pytest.mark.parametrize("batch", [1, 3])
    def test_an_explicit_batch(self, tmp_path, monkeypatch, batch):
        _requires_train_extra()
        wrapper = _build(tmp_path, monkeypatch, "sft", batch_size=batch)
        assert wrapper.trainer.args.per_device_train_batch_size == batch
        assert wrapper.trainer.args.per_device_eval_batch_size == batch

    def test_auto_resolves_before_the_eval_batch_is_taken(self, tmp_path, monkeypatch):
        _requires_train_extra()
        import soup_cli.utils.batch_probe as batch_probe

        monkeypatch.setattr(batch_probe, "pick_batch_size", lambda **_: 3)
        wrapper = _build(tmp_path, monkeypatch, "sft", batch_size="auto")
        assert wrapper.trainer.args.per_device_train_batch_size == 3
        assert wrapper.trainer.args.per_device_eval_batch_size == 3

    def test_auto_on_a_preference_wrapper(self, tmp_path, monkeypatch):
        """DPO halves its estimate for the chosen/rejected pair; the eval batch
        follows the halved value, not the estimate and not HF's 8."""
        _requires_train_extra()
        import soup_cli.trainer.dpo as dpo

        monkeypatch.setattr(dpo, "estimate_batch_size", lambda **_: 6)
        wrapper = _build(tmp_path, monkeypatch, "dpo", batch_size="auto")
        assert wrapper.trainer.args.per_device_train_batch_size == 3
        assert wrapper.trainer.args.per_device_eval_batch_size == 3


# ==========================================================================
# the schedule variants
# ==========================================================================
class TestScheduleVariants:
    """16 train rows at batch 2 make 8 micro-batches per epoch. Steps counted
    are optimizer steps, i.e. after gradient accumulation."""

    @pytest.mark.parametrize(
        ("training", "expected_steps"),
        [
            pytest.param({"eval_steps": 3}, [3, 6, 8], id="eval_steps-below-total"),
            # Larger than the run: Hugging Face still evaluates at the last step.
            pytest.param({"eval_steps": 100}, [8], id="eval_steps-above-total"),
            pytest.param(
                {"eval_steps": 2, "gradient_accumulation_steps": 2}, [2, 4], id="grad-accum"
            ),
            pytest.param(
                {"epochs": 2, "gradient_accumulation_steps": 2}, [4, 8], id="epoch-ends"
            ),
            pytest.param(
                {"epochs": 2, "eval_steps": 3}, [3, 6, 9, 12, 15, 16], id="epochs-with-steps"
            ),
        ],
    )
    def test_evaluations_land_where_the_schedule_says(
        self, tmp_path, monkeypatch, training, expected_steps
    ):
        _requires_train_extra()
        wrapper = _build(tmp_path, monkeypatch, "sft", n_train=16, **training)
        wrapper.trainer.train()
        steps = [e["step"] for e in _eval_entries(wrapper.trainer)]
        assert steps == expected_steps


# ==========================================================================
# end to end: `soup train` records the validation loss
# ==========================================================================
class TestSoupTrainRecordsValLoss:
    """The issue's acceptance run, through the CLI: default ``val_split``, a real
    tiny SFT run on CPU, and every sink #713 added receiving a value."""

    _ROWS = 20

    def _run_cli(self, tmp_path, monkeypatch, *, val_split: str = ""):
        import transformers
        from typer.testing import CliRunner

        from soup_cli.cli import app

        monkeypatch.chdir(tmp_path)
        weights = _tiny_llama_dir(tmp_path)
        rows = [
            {
                "messages": [
                    {"role": "user", "content": f"hi {i}"},
                    {"role": "assistant", "content": "good answer"},
                ]
            }
            for i in range(self._ROWS)
        ]
        (tmp_path / "chat.jsonl").write_text(
            "\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8"
        )
        (tmp_path / "soup.yaml").write_text(
            f"base: {json.dumps(weights)}\n"
            "task: sft\n"
            "data:\n"
            "  train: chat.jsonl\n"
            "  format: chatml\n"
            "  chat_template: chatml\n"
            "  max_length: 64\n"
            f"{val_split}"
            "training:\n"
            "  epochs: 2\n"
            "  batch_size: 2\n"
            "  gradient_accumulation_steps: 1\n"
            "  logging_steps: 1\n"
            "  save_steps: 1000\n"
            "  quantization: none\n"
            "  lora:\n"
            "    r: 4\n"
            "    alpha: 8\n"
            "    target_modules: [q_proj, v_proj]\n"
            "output: ./out\n",
            encoding="utf-8",
        )
        seen = {"evaluate_calls": 0, "log_history": None}
        real_train = transformers.Trainer.train
        real_evaluate = transformers.Trainer.evaluate

        def train(self, *args, **kwargs):
            out = real_train(self, *args, **kwargs)
            seen["log_history"] = list(self.state.log_history)
            return out

        def evaluate(self, *args, **kwargs):
            seen["evaluate_calls"] += 1
            return real_evaluate(self, *args, **kwargs)

        monkeypatch.setattr(transformers.Trainer, "train", train)
        monkeypatch.setattr(transformers.Trainer, "evaluate", evaluate)
        result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--yes"])
        assert result.exit_code == 0, (result.output, repr(result.exception))

        from soup_cli.experiment.tracker import ExperimentTracker

        tracker = ExperimentTracker()
        runs = tracker.list_runs()
        assert len(runs) == 1, runs
        metrics = tracker.get_metrics(runs[0]["run_id"])
        return seen, metrics, _plain(result.output)

    def test_the_default_split_is_evaluated_and_recorded(self, tmp_path, monkeypatch):
        _requires_train_extra()
        seen, metrics, out = self._run_cli(tmp_path, monkeypatch)
        assert f"Loaded: {self._ROWS - 2} train samples" in out
        # Two epochs, evaluated at the end of each.
        assert seen["evaluate_calls"] == 2
        evals = [e for e in seen["log_history"] if "eval_loss" in e]
        assert len(evals) == 2
        assert all(math.isfinite(e["eval_loss"]) for e in evals)
        recorded = [row["val_loss"] for row in metrics if row["val_loss"] is not None]
        assert len(recorded) == 2
        assert recorded == pytest.approx([e["eval_loss"] for e in evals])
        assert "Val loss" in out

    def test_control_val_split_zero_evaluates_nothing_and_withholds_nothing(
        self, tmp_path, monkeypatch
    ):
        _requires_train_extra()
        seen, metrics, out = self._run_cli(tmp_path, monkeypatch, val_split="  val_split: 0\n")
        assert f"Loaded: {self._ROWS} train samples" in out
        assert seen["evaluate_calls"] == 0
        assert not [e for e in seen["log_history"] if "eval_loss" in e]
        assert metrics, "the run logged no metrics at all"
        assert all(row["val_loss"] is None for row in metrics)
        assert "Val loss" not in out


# ==========================================================================
# the guard that survives the next wrapper being added
# ==========================================================================
def _trainer_sources() -> dict[str, str]:
    import importlib.util

    spec = importlib.util.find_spec("soup_cli.trainer")
    return {
        path.stem: path.read_text(encoding="utf-8")
        for path in sorted(Path(spec.origin).parent.glob("*.py"))
    }


class TestEveryWrapperUsesTheHelper:
    """A wrapper that hands its trainer an ``eval_dataset`` must build its
    schedule with ``training_eval_kwargs``, or it withholds rows it never
    evaluates. Derived from the source, so a new wrapper is covered unasked."""

    @staticmethod
    def _passing_an_eval_dataset() -> list[str]:
        return [
            name
            for name, source in _trainer_sources().items()
            if "eval_dataset=" in source or '"eval_dataset":' in source
        ]

    def test_the_derivation_found_the_wrappers(self):
        """Non-vacuity: an empty derivation would pass the test below."""
        assert {"sft", "grpo", "prm", "asr", "embedding"} <= set(
            self._passing_an_eval_dataset()
        )

    def test_each_one_threads_the_helper(self):
        sources = _trainer_sources()
        missing = [
            name
            for name in self._passing_an_eval_dataset()
            if "training_eval_kwargs(" not in sources[name]
        ]
        assert not missing, (
            f"soup_cli.trainer.{missing} pass an eval_dataset without "
            "training_eval_kwargs(), so the split is withheld and never evaluated"
        )