Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

10.7 KB
Raw Blame
"""Tests for dynamic warmup resolution across trainer wrappers (#1431).

Verifies that:
1. All trainer wrappers pass warmup_ratio directly through as warmup_steps to
   HuggingFace TrainingArguments, so transformers dynamically resolves warmup
   over the trainer's actual optimizer steps (ceil(max_steps * warmup_ratio)).
2. GRPO warmup no longer shrinks by num_generations.
3. SFT with packing / multipack and pretrain with packing scale warmup to the
   real packed step count.
4. Unpacked SFT control matches the expected warmup within one step.
5. All 18 wrappers resolve warmup_steps as the configured fraction without multiplying
   warmup_ratio into a step count.
"""

from __future__ import annotations

import ast
import math
from pathlib import Path

import pytest
import torch
from tokenizers import AddedToken, Tokenizer, models, pre_tokenizers
from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedTokenizerFast

from soup_cli.config.loader import load_config_from_string
from soup_cli.utils.warmup import resolve_trainer_warmup_steps
from tests import test_trl_preference_config_contract as contract

WORDS = ["<pad>", "<s>", "</s>", "<unk>", "apple", "banana", "cherry", "date"]
TRAINER_DIR = Path(__file__).resolve().parent.parent / "src" / "soup_cli" / "trainer"
WARMUP_WRAPPERS = (
    "asr",
    "bco",
    "classifier",
    "distill",
    "dpo",
    "embedding",
    "grpo",
    "ipo",
    "kto",
    "mole_routing",
    "online_dpo",
    "orpo",
    "ppo",
    "pretrain",
    "prm",
    "reward_model",
    "sft",
    "simpo",
)


@pytest.fixture(scope="module")
def tiny_llama_model(tmp_path_factory: pytest.TempPathFactory) -> str:
    tmp = tmp_path_factory.mktemp("tiny_llama")
    raw = Tokenizer(models.BPE())
    raw.pre_tokenizer = pre_tokenizers.Whitespace()
    vocab = {w: i for i, w in enumerate(WORDS)}
    raw.add_tokens([AddedToken(w, special=True) for w in WORDS[:4]])
    raw.add_tokens([AddedToken(w) for w in WORDS[4:]])
    tok = PreTrainedTokenizerFast(
        tokenizer_object=raw,
        bos_token="<s>",
        eos_token="</s>",
        pad_token="<pad>",
        unk_token="<unk>",
    )
    tok.chat_template = (
        "{% for m in messages %}{{ '<|' + m['role'] + '|> ' }}"
        "{% if m['role'] == 'assistant' %}{% generation %}{{ m['content'] + ' </s>' }}"
        "{% endgeneration %}{% else %}{{ m['content'] + ' </s> ' }}{% endif %}{% endfor %}"
        "{% if add_generation_prompt %}{{ '<|assistant|> ' }}{% endif %}"
    )
    torch.manual_seed(0)
    LlamaForCausalLM(
        LlamaConfig(
            vocab_size=len(vocab),
            hidden_size=32,
            intermediate_size=64,
            num_hidden_layers=2,
            num_attention_heads=2,
            num_key_value_heads=2,
            max_position_embeddings=256,
            pad_token_id=0,
            bos_token_id=1,
            eos_token_id=2,
        )
    ).save_pretrained(str(tmp))
    tok.save_pretrained(str(tmp))
    return str(tmp).replace("\\", "/")


class TestWarmupHelper:
    def test_resolve_trainer_warmup_steps(self) -> None:
        assert resolve_trainer_warmup_steps(0.0) == 0.0
        assert resolve_trainer_warmup_steps(0.1) == 0.1
        assert resolve_trainer_warmup_steps(0.5) == 0.5
        assert resolve_trainer_warmup_steps(None) == 0.0

    def test_invalid_ratio_rejected(self) -> None:
        with pytest.raises(ValueError, match="warmup_ratio must be in"):
            resolve_trainer_warmup_steps(-0.01)
        with pytest.raises(ValueError, match="warmup_ratio must be in"):
            resolve_trainer_warmup_steps(1.0)
        with pytest.raises(ValueError, match="warmup_ratio must be in"):
            resolve_trainer_warmup_steps(1.5)


class TestGRPOWarmup:
    @pytest.mark.parametrize(
        ("prompts", "batch", "accum", "gens", "ratio", "epochs"),
        [
            (40, 4, 1, 4, 0.5, 1),
            (40, 8, 1, 8, 0.5, 1),
            (64, 4, 4, 4, 0.1, 1),
            (40, 4, 1, 4, 0.25, 2),
        ],
    )
    def test_grpo_warmup_matches_real_steps(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
        tiny_llama_model: str,
        prompts: int,
        batch: int,
        accum: int,
        gens: int,
        ratio: float,
        epochs: int,
    ) -> None:
        from soup_cli.trainer.grpo import GRPOTrainerWrapper

        monkeypatch.chdir(tmp_path)
        out_dir = str(tmp_path / "out-grpo").replace("\\", "/")
        cfg = load_config_from_string(
            f"base: {tiny_llama_model}\ntask: grpo\n"
            "data: {train: ./unused.jsonl, max_length: 64}\n"
            f"training:\n  epochs: {epochs}\n  batch_size: {batch}\n"
            f"  gradient_accumulation_steps: {accum}\n"
            f"  num_generations: {gens}\n  warmup_ratio: {ratio}\n  reward_fn: accuracy\n"
            "  quantization: none\n  lora: {r: 4, alpha: 8, target_modules: [q_proj, v_proj]}\n"
            f"output: {out_dir}\n"
        )
        row = {
            "messages": [
                {"role": "user", "content": "what is apple ?"},
                {"role": "assistant", "content": "apple"},
            ]
        }
        rows = [row] * prompts
        wrapper = GRPOTrainerWrapper(cfg, device="cpu")
        wrapper.setup({"train": rows})
        trainer = wrapper.trainer
        steps = (len(trainer.get_train_dataloader()) // accum) * epochs
        expected_warmup = math.ceil(steps * ratio)
        resolved_warmup = trainer.args.get_warmup_steps(steps)
        assert resolved_warmup == expected_warmup
        assert trainer.args.warmup_steps == ratio

        # Mutation check: the old row-count formula would have shrunk warmup by gens
        old_formula_steps = int(math.ceil(prompts / batch / accum) * epochs * ratio)
        if gens > 1:
            assert resolved_warmup > old_formula_steps


class TestSFTWarmup:
    @pytest.mark.parametrize("packing_mode", ["none", "packing", "multipack"])
    @pytest.mark.parametrize("epochs", [1, 2])
    def test_sft_warmup_scales_to_packed_optimizer_steps(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
        tiny_llama_model: str,
        packing_mode: str,
        epochs: int,
    ) -> None:
        from soup_cli.trainer.sft import SFTTrainerWrapper

        monkeypatch.chdir(tmp_path)
        out_dir = str(tmp_path / f"out-sft-{packing_mode}-{epochs}").replace("\\", "/")
        extra = ""
        if packing_mode == "packing":
            extra = "  packing: true\n"
        elif packing_mode == "multipack":
            extra = "  multipack: true\n"

        cfg = load_config_from_string(
            f"base: {tiny_llama_model}\ntask: sft\n"
            "data: {train: ./unused.jsonl, format: chatml, max_length: 64}\n"
            f"training:\n  epochs: {epochs}\n  batch_size: 1\n  gradient_accumulation_steps: 1\n"
            "  lr: 1.0e-3\n  warmup_ratio: 0.5\n"
            "  scheduler: cosine\n  quantization: none\n"
            f"{extra}  lora: {{r: 4, alpha: 8}}\noutput: {out_dir}\n"
        )
        rows = [
            {
                "messages": [
                    {"role": "user", "content": f"what is {w} ?"},
                    {"role": "assistant", "content": "ok"},
                ]
            }
            for w in WORDS[4:] * 5
        ]
        wrapper = SFTTrainerWrapper(cfg, device="cpu")
        wrapper.setup({"train": rows})
        trainer = wrapper.trainer
        steps = len(trainer.get_train_dataloader()) * epochs
        resolved_warmup = trainer.args.get_warmup_steps(steps)
        expected_warmup = math.ceil(steps * 0.5)
        assert resolved_warmup == expected_warmup
        assert trainer.args.warmup_steps == 0.5


class TestPretrainWarmup:
    def test_pretrain_warmup_ratio_preserved(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
        tiny_llama_model: str,
    ) -> None:
        from soup_cli.trainer.pretrain import PretrainTrainerWrapper

        monkeypatch.chdir(tmp_path)
        out_dir = str(tmp_path / "out-pretrain").replace("\\", "/")
        cfg = load_config_from_string(
            f"base: {tiny_llama_model}\ntask: pretrain\n"
            "data: {train: ./unused.jsonl, max_length: 64}\n"
            "training:\n  epochs: 1\n  batch_size: 2\n  gradient_accumulation_steps: 1\n"
            "  lr: 1.0e-3\n  warmup_ratio: 0.25\n"
            f"  quantization: none\n  lora: {{r: 4, alpha: 8}}\noutput: {out_dir}\n"
        )
        rows = [{"text": f"apple {w} banana"} for w in WORDS[4:] * 4]
        wrapper = PretrainTrainerWrapper(cfg, device="cpu")
        wrapper.setup({"train": rows})
        trainer = wrapper.trainer
        steps = len(trainer.get_train_dataloader())
        assert trainer.args.warmup_steps == 0.25
        assert trainer.args.get_warmup_steps(steps) == math.ceil(steps * 0.25)


@pytest.mark.parametrize("task", contract._ALL_SIX)
def test_live_preference_trainer_carries_the_warmup_fraction(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, task: str
) -> None:
    wrapper = contract._build(tmp_path, monkeypatch, task)
    assert wrapper.trainer.args.warmup_steps == pytest.approx(0.03), (
        task,
        wrapper.trainer.args.warmup_steps,
    )


@pytest.mark.parametrize("name", WARMUP_WRAPPERS)
def test_wrapper_source_passes_the_ratio_not_a_row_count_estimate(name: str) -> None:
    tree = ast.parse((TRAINER_DIR / f"{name}.py").read_text(encoding="utf-8"))
    calls = [
        node
        for node in ast.walk(tree)
        if isinstance(node, ast.Call)
        and getattr(node.func, "id", None) == "resolve_trainer_warmup_steps"
    ]
    assert len(calls) == 1, f"{name}: expected one resolve_trainer_warmup_steps call"
    assert ast.unparse(calls[0].args[0]) == "tcfg.warmup_ratio", name
    for node in ast.walk(tree):
        if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mult):
            assert "warmup_ratio" not in ast.unparse(node), (
                f"{name}: warmup_ratio is multiplied into a step count again"
            )
    has_arg = any(
        (
            isinstance(node, ast.keyword)
            and node.arg == "warmup_steps"
            and getattr(node.value, "id", None) == "warmup_steps"
        )
        or (
            isinstance(node, ast.Dict)
            and any(
                getattr(k, "value", None) == "warmup_steps"
                and getattr(v, "id", None) == "warmup_steps"
                for k, v in zip(node.keys, node.values)
            )
        )
        for node in ast.walk(tree)
    )
    assert has_arg, f"{name}: warmup_steps argument not passed in source"


def test_short_run_at_default_ratio_gets_one_warmup_step(tmp_path: Path) -> None:
    from transformers import TrainingArguments

    args = TrainingArguments(output_dir=str(tmp_path / "out"), warmup_steps=0.03)
    assert args.get_warmup_steps(10) == 1
    assert args.get_warmup_steps(33) == 1
    assert args.get_warmup_steps(34) == 2