Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

5.4 KB
Raw Blame
"""#1391 — ``task: ppo`` never completed a step on trl 0.29.

``setup()`` handed trl's ``PPOTrainer`` a dataset that still carried the
``prompt_text`` strings, and trl's ``DataCollatorWithPadding`` tried to turn
them into a tensor before the first rollout. A train set smaller than one
rollout batch hung instead, because trl's DataLoader drops the ragged batch and
repeats itself forever.

These tests drive the real trl trainer on CPU with a tiny on-disk Llama policy
and a tiny reward model. Nothing about the trainer is mocked.
"""

from __future__ import annotations

import pytest

for _mod in ("torch", "transformers", "peft", "trl", "datasets"):
    pytest.importorskip(_mod)


def _tiny_models(root):
    import torch
    from tokenizers import Tokenizer, models, pre_tokenizers
    from transformers import (
        LlamaConfig,
        LlamaForCausalLM,
        LlamaForSequenceClassification,
        PreTrainedTokenizerFast,
    )

    def config(**extra):
        return LlamaConfig(
            vocab_size=64, hidden_size=64, intermediate_size=128, num_hidden_layers=2,
            num_attention_heads=4, num_key_value_heads=2, tie_word_embeddings=True,
            max_position_embeddings=128, pad_token_id=3, bos_token_id=1, eos_token_id=2,
            **extra,
        )

    torch.manual_seed(7)
    names = ["<unk>", "<s>", "</s>", "<pad>", "say", "hi", "hello", "world"]
    tok = Tokenizer(models.WordLevel(vocab={w: i for i, w in enumerate(names)}, unk_token="<unk>"))
    tok.pre_tokenizer = pre_tokenizers.Whitespace()
    base, rm = root / "base", root / "rm"
    LlamaForCausalLM(config()).save_pretrained(base)
    PreTrainedTokenizerFast(
        tokenizer_object=tok, unk_token="<unk>", bos_token="<s>",
        eos_token="</s>", pad_token="<pad>",
    ).save_pretrained(base)
    LlamaForSequenceClassification(config(num_labels=1)).save_pretrained(rm)
    return base.as_posix(), rm.as_posix()


_PROMPTS = ("say hi", "say hello world", "hi", "hello")


def _wrapper(tmp_path, monkeypatch, grad_accum=1):
    from soup_cli.config.loader import load_config_from_string
    from soup_cli.trainer.ppo import PPOTrainerWrapper

    monkeypatch.setenv("WANDB_DISABLED", "true")
    monkeypatch.delenv("WORLD_SIZE", raising=False)
    monkeypatch.chdir(tmp_path)
    base, rm = _tiny_models(tmp_path)
    cfg = load_config_from_string(
        f"base: {base}\ntask: ppo\nbackend: transformers\n"
        "data:\n  train: train.jsonl\n  max_length: 64\n"
        # One rollout batch is batch_size * gradient_accumulation_steps rows, so
        # these 4 rows fill exactly one batch at batch_size 2, accumulation 1.
        f"training:\n  epochs: 1\n  batch_size: 2\n  gradient_accumulation_steps: {grad_accum}\n"
        f"  quantization: none\n  ppo_epochs: 1\n  reward_model: {rm}\n"
        "  lora:\n    r: 4\n    alpha: 8\n    target_modules: [q_proj, v_proj]\n"
        f"output: {(tmp_path / 'out').as_posix()}\n"
    )
    wrapper = PPOTrainerWrapper(cfg, device="cpu")
    wrapper.setup({"train": [{"messages": [{"role": "user", "content": p}]} for p in _PROMPTS]})
    return wrapper


def test_ppo_hands_trl_only_token_columns(tmp_path, monkeypatch):
    wrapper = _wrapper(tmp_path, monkeypatch)
    columns = set(wrapper.trainer.train_dataset.column_names)
    assert columns <= {"input_ids", "attention_mask"}, columns
    # the manual loop's copy keeps the prompt text
    assert "prompt_text" in wrapper._train_ds.column_names


def test_ppo_runs_a_real_step(tmp_path, monkeypatch):
    wrapper = _wrapper(tmp_path, monkeypatch)
    result = wrapper.train()
    assert result["total_steps"] >= 1, result
    assert (tmp_path / "out" / "adapter_config.json").is_file()
    assert any("loss/policy_avg" in e for e in wrapper.trainer.state.log_history)


def test_a_train_set_smaller_than_one_rollout_batch_is_refused(tmp_path, monkeypatch):
    # 4 rows, batch_size 2 x accumulation 4 = 8: trl would spin without a step
    with pytest.raises(ValueError) as exc:
        _wrapper(tmp_path, monkeypatch, grad_accum=4)
    msg = str(exc.value)
    assert "4 rows" in msg
    assert "batch_size=2" in msg
    assert "gradient_accumulation_steps=4" in msg


def test_the_rollout_batch_counts_every_process(monkeypatch):
    from soup_cli.trainer.ppo import _check_one_rollout_batch

    monkeypatch.setenv("WORLD_SIZE", "2")
    _check_one_rollout_batch(8, 2, 2)
    with pytest.raises(ValueError, match=r"x 2 processes = 8"):
        _check_one_rollout_batch(7, 2, 2)
    monkeypatch.setenv("WORLD_SIZE", "nonsense")
    _check_one_rollout_batch(4, 2, 2)


def test_the_refusal_comes_before_any_model_is_loaded(tmp_path, monkeypatch):
    """The refusal needs only the row count and two settings, so it must not
    wait for the reward model and the policy to load."""
    from soup_cli.trainer import ppo

    loaded = []
    real_reward = ppo.PPOTrainerWrapper._setup_reward
    real_policy = ppo.PPOTrainerWrapper._setup_transformers

    def reward(self, *args):
        loaded.append("reward model")
        return real_reward(self, *args)

    def policy(self, *args):
        loaded.append("policy")
        return real_policy(self, *args)

    monkeypatch.setattr(ppo.PPOTrainerWrapper, "_setup_reward", reward)
    monkeypatch.setattr(ppo.PPOTrainerWrapper, "_setup_transformers", policy)
    with pytest.raises(ValueError, match="4 rows"):
        _wrapper(tmp_path, monkeypatch, grad_accum=4)
    assert loaded == [], f"refused only after loading: {loaded}"