Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

12.9 KB
Raw Blame
"""The canary for the blind spot that let a broken `trl` bound ship twice.

Six trainers — bco, dpo, ipo, kto, orpo, simpo — build a `trl` config inside
`setup()`. `trl` removed `max_prompt_length` from those configs in stages, so
`soup train --task orpo` failed at import for anyone who pip-installed fresh.
Nothing caught it, and the reason is exactly one sentence: **the `trl` imports
and the config construction live inside `setup()`, which no test had ever
called on those wrappers.** Constructing a wrapper does not touch `trl` at all
— `__init__` only stores the config — so a test that instantiates the wrapper
and mocks `setup` proves nothing about whether the package can train.

v0.72.4 closed that for four of the six through the layer-streaming suite
(`tests/test_v07204.py`), which drives the real `setup()`. It left `bco` and
`ipo` open: `tests/test_bco.py` and `tests/test_ipo.py` both patch `.setup`
out. This file closes it for all six on the ORDINARY (non-streaming) path,
which is what users actually run and what raising the cap (#326) has to keep
working.

What it is NOT: a training test. It asserts that the wrapper reaches a live
`trl` trainer with the arguments it means to pass. That is the whole failure
class — a `TypeError` on an unexpected keyword argument, or an `ImportError` on
a config class that moved out of the `trl` namespace — and it is cheap.
"""

import pytest


# --------------------------------------------------------------------------
# fixtures -- a real, tiny checkpoint on disk (mirrors tests/test_v07204.py;
# duplicated deliberately, as the four v0.72.x suites already do, so this file
# stays runnable on its own)
# --------------------------------------------------------------------------
def _tiny_llama_dir(tmp_path, vocab=64, hidden=64):
    import torch
    from safetensors.torch import save_file
    from transformers import LlamaConfig, LlamaForCausalLM

    torch.manual_seed(7)
    config = LlamaConfig(
        vocab_size=vocab,
        hidden_size=hidden,
        intermediate_size=hidden * 2,
        num_hidden_layers=2,
        num_attention_heads=4,
        num_key_value_heads=2,
        tie_word_embeddings=True,
        max_position_embeddings=128,
    )
    model = LlamaForCausalLM(config).to(torch.float32).eval()
    weights = tmp_path / "model"
    weights.mkdir(parents=True, exist_ok=True)
    state = {k: v.contiguous() for k, v in model.state_dict().items()}
    state.pop("lm_head.weight", None)
    save_file(state, str(weights / "model.safetensors"))
    config.save_pretrained(str(weights))
    return str(weights)


def _write_tiny_tokenizer(directory):
    from tokenizers import Tokenizer, models, pre_tokenizers
    from transformers import PreTrainedTokenizerFast

    vocab = {"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3}
    for word in ("hello", "world", "hi", "good", "answer", "bad"):
        vocab[word] = len(vocab)
    tok = Tokenizer(models.WordLevel(vocab=vocab, unk_token="<unk>"))
    tok.pre_tokenizer = pre_tokenizers.Whitespace()
    fast = PreTrainedTokenizerFast(
        tokenizer_object=tok,
        unk_token="<unk>",
        bos_token="<s>",
        eos_token="</s>",
        pad_token="<pad>",
    )
    fast.save_pretrained(str(directory))


def _pref_rows(n=8):
    return [{"prompt": "hi", "chosen": " good answer", "rejected": " bad"} for _ in range(n)]


def _long_pref_rows(n=8):
    prompt = " ".join(["hello"] * 161)
    return [{"prompt": prompt, "chosen": " good answer", "rejected": " bad"} for _ in range(n)]


def _kto_rows(n=8):
    return [{"prompt": "hi", "completion": " good answer", "label": i % 2 == 0} for i in range(n)]


#: KTO refuses `per_device_train_batch_size == 1` outright — its KL term is
#: degenerate at 1 — so the floor is per task, not global.
_MIN_BATCH = {"kto": 2}

_ROWS = {
    "bco": _pref_rows,
    "dpo": _pref_rows,
    "ipo": _pref_rows,
    "kto": _kto_rows,
    "orpo": _pref_rows,
    "simpo": _pref_rows,
}

#: task -> (wrapper import path, the trl config class its `setup()` builds).
#: The config class is named so a failure says *which* trl API moved, rather
#: than only that some import failed.
_WRAPPERS = {
    "bco": ("soup_cli.trainer.bco", "BCOTrainerWrapper", "BCOConfig"),
    "dpo": ("soup_cli.trainer.dpo", "DPOTrainerWrapper", "DPOConfig"),
    "ipo": ("soup_cli.trainer.ipo", "IPOTrainerWrapper", "DPOConfig"),
    "kto": ("soup_cli.trainer.kto", "KTOTrainerWrapper", "KTOConfig"),
    "orpo": ("soup_cli.trainer.orpo", "ORPOTrainerWrapper", "ORPOConfig"),
    "simpo": ("soup_cli.trainer.simpo", "SimPOTrainerWrapper", "CPOConfig"),
}

_ALL_SIX = tuple(_WRAPPERS)

#: The two this file exists for. Kept as its own name so the regression guard
#: below can assert they are covered without restating the list.
_PREVIOUSLY_UNCOVERED = ("bco", "ipo")


def _cfg(weights, out_dir, task):
    import yaml

    from soup_cli.config.loader import load_config_from_string

    return load_config_from_string(
        yaml.safe_dump(
            {
                "base": weights,
                "task": task,
                "backend": "transformers",
                "modality": "text",
                "data": {"train": "train.jsonl", "max_length": 64, "chat_template": "chatml"},
                "training": {
                    "batch_size": _MIN_BATCH.get(task, 1),
                    "quantization": "none",
                    "epochs": 1,
                    "logging_steps": 1,
                    "save_steps": 1000,
                    "lora": {"r": 4, "alpha": 8, "target_modules": ["q_proj", "v_proj"]},
                },
                "output": str(out_dir),
            }
        )
    )


def _build(tmp_path, monkeypatch, task):
    module, cls_name, _ = _WRAPPERS[task]
    import importlib

    weights = _tiny_llama_dir(tmp_path)
    _write_tiny_tokenizer(weights)
    monkeypatch.chdir(tmp_path)
    cfg = _cfg(weights, tmp_path / "out", task)
    wrapper = getattr(importlib.import_module(module), cls_name)(cfg, device="cpu")
    wrapper.setup({"train": _ROWS[task](8)})
    return wrapper


class TestEveryPreferenceTrainerReachesALiveTrlTrainer:
    """The load-bearing test. `setup()` is CALLED, not mocked."""

    @pytest.mark.parametrize("task", _ALL_SIX)
    def test_setup_builds_a_trainer(self, tmp_path, monkeypatch, task):
        wrapper = _build(tmp_path, monkeypatch, task)
        assert wrapper.trainer is not None, f"{task}: setup() left no trainer"
        assert wrapper.model is not None and wrapper.tokenizer is not None

    @pytest.mark.parametrize("task", _ALL_SIX)
    def test_the_config_carries_the_arguments_the_wrapper_passes(
        self, tmp_path, monkeypatch, task
    ):
        """`max_prompt_length` is the field that actually broke, so it is named
        here rather than left implicit in "setup() didn't raise". A trl release
        that keeps accepting the keyword but stops storing it would pass the
        test above and fail this one.

        #326 made the assertion conditional, because trl removed the field in
        stages (kto 0.27, bco/orpo/simpo 0.28, dpo/ipo 0.29) and the wrappers
        now pass it only to a config that still accepts it. The contract being
        pinned is therefore two-sided, and BOTH halves have teeth: where the
        field exists it must carry the value the wrapper computed, and where it
        does not it must be genuinely absent — not silently defaulted, which is
        how a wrapper that had quietly stopped passing it would look.
        """
        from soup_cli.trainer._trl_compat import config_accepts

        wrapper = _build(tmp_path, monkeypatch, task)
        args = wrapper.trainer.args
        if config_accepts(type(args), "max_prompt_length"):
            assert args.max_prompt_length == 32, (task, args.max_prompt_length)
        else:
            assert not hasattr(args, "max_prompt_length"), (
                f"{task}: installed trl does not accept `max_prompt_length` as a "
                f"keyword, yet the built config has the attribute — the capability "
                f"probe and the object disagree"
            )
        assert args.max_length == 64, (task, args.max_length)

    @pytest.mark.parametrize("task", ("dpo", "orpo", "simpo"))
    def test_removed_prompt_cap_is_enforced_on_the_effective_batch(
        self, tmp_path, monkeypatch, task
    ):
        """TRL 0.29 must not turn ``data.max_length`` into a cosmetic field.

        The assertion is on the tensors the model receives, not the config or
        an intermediate column: 0.29 accepted ``max_length=64`` while emitting
        164/165-token DPO/ORPO batches from this exact 161-token prompt, and
        SimPO did the same until #2112 gave it the cap.
        """
        from soup_cli.trainer._trl_compat import config_accepts

        module, cls_name, _ = _WRAPPERS[task]
        import importlib

        weights = _tiny_llama_dir(tmp_path)
        _write_tiny_tokenizer(weights)
        monkeypatch.chdir(tmp_path)
        cfg = _cfg(weights, tmp_path / "out", task)
        wrapper = getattr(importlib.import_module(module), cls_name)(cfg, device="cpu")
        wrapper.setup({"train": _long_pref_rows(8)})

        if config_accepts(type(wrapper.trainer.args), "max_prompt_length"):
            pytest.skip("installed TRL still enforces its own prompt cap")

        assert len(wrapper.trainer.train_dataset) == 8
        batch = next(iter(wrapper.trainer.get_train_dataloader()))
        sequence_keys = [key for key in batch if key.endswith("input_ids")]
        assert sequence_keys, batch.keys()
        assert all(batch[key].shape[-1] <= 64 for key in sequence_keys), {
            key: tuple(batch[key].shape) for key in sequence_keys
        }


class TestTheCanaryCoversWhatItClaims:
    """Guards the *coverage* property, not the code. The bug shipped because a
    blind spot was invisible; a shrinking parametrize list would make it
    invisible again."""

    def test_the_two_trainers_that_had_no_setup_test_are_covered(self):
        for task in _PREVIOUSLY_UNCOVERED:
            assert task in _ALL_SIX, task

    def test_every_wrapper_that_passes_max_prompt_length_is_listed(self):
        """Derived from the source, so a seventh trainer adopting the same trl
        argument joins this file automatically instead of silently reopening
        the gap.

        #326 moved the marker. The canary fired exactly as designed when the
        wrappers migrated — it asserted "did the path move?" and the path had
        moved: the literal `max_prompt_length=` was replaced by a call to
        `prompt_length_kwargs(...)`, which passes the keyword only to a trl that
        still accepts it. Both spellings are searched, so a trainer using either
        the old direct keyword or the new helper is covered.
        """
        import pathlib

        trainer_dir = pathlib.Path(__file__).resolve().parents[1] / "src" / "soup_cli" / "trainer"
        markers = ("max_prompt_length=", "prompt_length_kwargs(")
        users = {
            path.stem
            for path in trainer_dir.glob("*.py")
            if any(marker in path.read_text(encoding="utf-8") for marker in markers)
            and path.stem != "_trl_compat"  # the helper defines it, does not pass it
            # The native MLX loops build no trl config: their `max_prompt_length=`
            # goes to their own tokenizer (tests/test_mlx_dpo.py and
            # tests/test_mlx_orpo.py pin the cap).
            and path.stem not in ("mlx_dpo", "mlx_orpo")
        }
        assert users, "found no trainer passing a prompt-length cap — did the path move?"
        uncovered = sorted(users - set(_ALL_SIX))
        assert not uncovered, f"passes a prompt-length cap with no setup() test: {uncovered}"


class TestTheTrlBoundsAreConsistentWithTheCode:
    """The installed TRL must provide every symbol through Soup's real resolver."""

    def test_the_installed_trl_provides_every_symbol_the_trainers_import(self):
        """Exercise the same public-first, experimental-second imports as setup()."""
        from soup_cli.trainer._trl_compat import resolve_trl_symbol

        broken = {}
        symbols = {
            "DPOConfig": None,
            "DPOTrainer": None,
            "KTOConfig": None,
            "KTOTrainer": None,
            "ORPOConfig": "trl.experimental.orpo",
            "ORPOTrainer": "trl.experimental.orpo",
            "CPOConfig": "trl.experimental.cpo",
            "CPOTrainer": "trl.experimental.cpo",
            "BCOConfig": "trl.experimental.bco",
            "BCOTrainer": "trl.experimental.bco",
            "GRPOTrainer": None,
            "GRPOConfig": None,
        }
        for name, experimental_module in symbols.items():
            try:
                resolve_trl_symbol(name, experimental_module)
            except Exception as exc:  # noqa: BLE001 - reported, not swallowed
                broken[name] = f"{type(exc).__name__}: {exc}"
        assert not broken, (
            f"installed trl cannot supply {sorted(broken)} through Soup's "
            f"public/experimental resolver — the [train] bounds and the code "
            f"disagree: {broken}"
        )