Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

15.1 KB
Raw Blame
"""Exercise #805's configuration values at real trainer/optimizer boundaries."""

from __future__ import annotations

import json
import math
from unittest.mock import patch

import pytest


@pytest.fixture
def tiny_model(tmp_path, monkeypatch):
    """An offline CPU model and tokenizer, including loadable task adapters."""
    import torch
    from tokenizers import Tokenizer
    from tokenizers.models import WordLevel
    from tokenizers.pre_tokenizers import Whitespace
    from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedTokenizerFast

    monkeypatch.chdir(tmp_path)
    previous_threads = torch.get_num_threads()
    torch.set_num_threads(1)
    monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
    monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False)
    # Transformers caches MPS availability across tests. Override the Trainer's
    # lookup too, so a prior MPS probe cannot move this CPU fixture's model.
    monkeypatch.setattr("transformers.training_args.is_torch_mps_available", lambda: False)
    torch.manual_seed(42)
    tokenizer = Tokenizer(WordLevel(
        {"<pad>": 0, "<unk>": 1, "<eos>": 2, "one": 3, "two": 4, "three": 5},
        unk_token="<unk>",
    ))
    tokenizer.pre_tokenizer = Whitespace()
    tokenizer = PreTrainedTokenizerFast(
        tokenizer_object=tokenizer, pad_token="<pad>", unk_token="<unk>", eos_token="<eos>"
    )
    model = LlamaForCausalLM(LlamaConfig(
        vocab_size=6, hidden_size=16, intermediate_size=32, num_hidden_layers=1,
        num_attention_heads=2, num_key_value_heads=2, max_position_embeddings=64,
        pad_token_id=0, eos_token_id=2,
    ))
    path = tmp_path / "tiny-model"
    model.save_pretrained(path)
    tokenizer.save_pretrained(path)
    yield path
    torch.set_num_threads(previous_threads)


def _config(tiny_model, task, **training):
    from soup_cli.config.schema import SoupConfig

    return SoupConfig(
        base=str(tiny_model), task=task, output="output-" + task,
        data={"train": "train.jsonl", "forget_set": "forget.jsonl", "max_length": 64},
        training={
            "epochs": 2, "batch_size": 2, "gradient_accumulation_steps": 2,
            "scheduler": "cosine", "warmup_ratio": 0.25, "weight_decay": 0.1,
            "max_grad_norm": 0.3, "optimizer": "adafactor", "lr": 0.001,
            "quantization": "none", "gradient_checkpointing": False,
            "logging_steps": 1, "save_steps": 100,
            "lora": {
                "r": 0 if task == "prm" else 2,
                "alpha": 4, "dropout": 0.0, "target_modules": ["q_proj"],
            },
            **training,
        },
    )


@pytest.mark.parametrize("kind", ["prm", "moe_lora_routing"])
def test_prm_and_mole_train_with_configured_optimizer(tiny_model, monkeypatch, kind):
    from peft import LoraConfig, get_peft_model
    from transformers import Adafactor, AutoModelForCausalLM

    from soup_cli.trainer import mole_routing, prm

    training = {"batch_size": "auto"}
    if kind == "moe_lora_routing":
        adapters = []
        for index in range(2):
            model = get_peft_model(
                AutoModelForCausalLM.from_pretrained(tiny_model),
                LoraConfig(r=2, target_modules=["q_proj"], task_type="CAUSAL_LM"),
            )
            path = tiny_model.parent / f"adapter-{index}"
            model.save_pretrained(path)
            adapters.append(str(path))
        training["mole_task_adapters"] = adapters
        module = mole_routing
        wrapper_class = mole_routing.MoleRoutingTrainerWrapper
        rows = [{"text": "one two three"}] * 8
    else:
        module = prm
        wrapper_class = prm.PRMTrainerWrapper
        rows = [{"prompt": "one", "completions": [" two", " three"], "labels": [0.0, 1.0]}] * 8
    monkeypatch.setattr(module, "estimate_batch_size", lambda **kwargs: 2)
    config = _config(tiny_model, kind, **training)
    wrapper = wrapper_class(config, device="cpu")
    wrapper.setup({"train": rows})
    result = wrapper.train()
    args = wrapper.trainer.args
    assert args.per_device_train_batch_size == 2
    assert args.warmup_steps == 0.25
    assert args.get_warmup_steps(result["total_steps"]) == 1
    assert args.weight_decay == 0.1
    assert args.max_grad_norm == 0.3
    assert args.optim.value == "adafactor"
    assert args.lr_scheduler_type.value == "cosine"
    assert not args.gradient_checkpointing
    optimizer = wrapper.trainer.optimizer
    optimizer = getattr(optimizer, "optimizer", optimizer)
    assert isinstance(optimizer, Adafactor)
    assert any(group["weight_decay"] == 0.1 for group in optimizer.param_groups)
    assert result["total_steps"] == 4


def test_ppo_constructs_real_config_with_nondefault_training_values(tiny_model):
    from trl.experimental.ppo import PPOConfig

    from soup_cli.trainer.ppo import PPOTrainerWrapper

    class ConfigCapture:
        def __init__(self, *, args, **kwargs):
            self.args = args

        def add_callback(self, callback):
            pass

    config = _config(tiny_model, "ppo", reward_fn="accuracy")
    wrapper = PPOTrainerWrapper(config, device="cpu")
    # Preserve actual model loading, dataset preparation and PPOConfig creation.
    # Only stop before PPO's separate rollout-training loop, which #805 does not alter.
    with (
        patch(
            "soup_cli.trainer.ppo._import_ppo_classes",
            return_value=(ConfigCapture, PPOConfig, True),
        ),
        patch.object(wrapper, "_get_or_create_reward_model", return_value=None),
        patch.object(wrapper, "_create_value_model", return_value=None),
    ):
        wrapper.setup({"train": [{"prompt": "one two", "answer": "three"}] * 8})
    args = wrapper.trainer.args
    assert isinstance(args, PPOConfig)
    assert args.warmup_steps == 0.25
    assert args.get_warmup_steps(4) == 1
    assert args.weight_decay == 0.1
    assert args.max_grad_norm == 0.3
    assert args.optim.value == "adafactor"
    assert args.lr_scheduler_type.value == "cosine"
    assert args.num_train_epochs == 2
    assert args.logging_steps == 1
    assert args.save_steps == 100
    assert args.save_total_limit == 3
    assert args.response_length == 64
    assert not args.gradient_checkpointing


def _unlearn_wrapper(tiny_model, **training):
    from soup_cli.trainer.unlearn import UnlearnTrainerWrapper

    (tiny_model.parent / "forget.jsonl").write_text(
        (json.dumps({"prompt": "one ", "completion": "two three"}) + "\n") * 5
    )
    training = {"epochs": 1, "unlearn_method": "simnpo", **training}
    config = _config(tiny_model, "unlearn", **training)
    wrapper = UnlearnTrainerWrapper(config, device="cpu")
    wrapper.setup()
    return wrapper


@pytest.mark.parametrize(("method", "model_loads"), [("simnpo", 1), ("npo", 2)])
def test_unlearn_default_auto_reaches_real_training(tiny_model, monkeypatch, method, model_loads):
    from soup_cli.utils import live_eval

    with patch.object(
        live_eval, "load_model_and_tokenizer", wraps=live_eval.load_model_and_tokenizer
    ) as load_model:
        wrapper = _unlearn_wrapper(tiny_model, batch_size="auto", unlearn_method=method)
    assert load_model.call_count == model_loads
    assert all("quantization" not in call.kwargs for call in load_model.call_args_list)
    assert wrapper._batch_size == 1
    assert wrapper.train()["total_steps"] == 5


def test_unlearn_auto_stays_one_on_a_large_card(tiny_model, monkeypatch):
    """``batch_size: auto`` means 1 on unlearn whatever the card reports.

    Without a GPU (CPU boxes, every CI runner) the memory estimator itself
    answers 1, so the test above cannot tell the ruling from a revert to
    ``estimate_batch_size``. Report an 80 GB card, check that the estimator
    then answers more than 1, and only then assert the ruling.
    """
    from soup_cli.trainer import unlearn
    from soup_cli.utils import gpu

    def large_card(*args, **kwargs):
        return {"memory_total": "80.0 GB", "memory_total_bytes": 80 * 1024**3, "gpu_count": 1}

    monkeypatch.setattr(gpu, "get_gpu_info", large_card)
    # A revert that imported the helper at module level would bind its own name.
    monkeypatch.setattr(unlearn, "get_gpu_info", large_card, raising=False)
    estimate = gpu.estimate_batch_size(
        model_params_b=gpu.model_size_from_name(str(tiny_model)),
        seq_length=64,
        gpu_memory_bytes=gpu.get_gpu_info()["memory_total_bytes"],
        quantization="none",
        lora_r=2,
    )
    assert estimate > 1, "the mocked card no longer separates the ruling from the estimator"

    wrapper = _unlearn_wrapper(tiny_model, batch_size="auto")
    assert wrapper._batch_size == 1
    # 5 forget rows, one epoch, accumulation 2: three updates, not one.
    assert wrapper._max_updates == 3


@pytest.mark.parametrize(("batch_size", "accumulation"), [(1, 1), (1, 4), (2, 4)])
def test_unlearn_reports_unscaled_loss_and_example_count(tiny_model, batch_size, accumulation):
    import torch
    from transformers import Adafactor

    from soup_cli.utils.live_eval import _tokenize_pair

    wrapper = _unlearn_wrapper(
        tiny_model, batch_size=batch_size, gradient_accumulation_steps=accumulation
    )
    expected = wrapper._step_preference(
        _tokenize_pair, "one ", "two three", "simnpo", "cpu"
    ).item()
    assert isinstance(wrapper._optimizer, Adafactor)
    assert wrapper._optimizer.param_groups[0]["weight_decay"] == 0.1
    updates = math.ceil(5 / (batch_size * accumulation))
    with patch("torch.nn.utils.clip_grad_norm_", wraps=torch.nn.utils.clip_grad_norm_) as clip:
        result = wrapper.train()
    assert result["initial_loss"] == pytest.approx(expected)
    assert result["total_steps"] == 5
    assert clip.call_count == updates
    assert all(call.args[1] == 0.3 for call in clip.call_args_list)
    assert wrapper._scheduler.last_epoch == updates
    warmup = int(updates * wrapper.config.training.warmup_ratio)
    probe_step = warmup + 0.25 * (updates - warmup)
    assert wrapper._scheduler.lr_lambdas[0](probe_step) == pytest.approx(
        0.5 * (1.0 + math.cos(math.pi * 0.25))
    )


def test_unlearn_uses_configured_sequence_length(tiny_model):
    from soup_cli.utils.live_eval import _tokenize_pair

    wrapper = _unlearn_wrapper(tiny_model)
    lengths = []

    def capture(*args, **kwargs):
        lengths.append(kwargs["max_length"])
        return _tokenize_pair(*args, **kwargs)

    wrapper._step_preference(capture, "one ", "two three " * 20, "simnpo", "cpu")
    assert lengths == [64]


def test_unlearn_partial_group_averages_actual_gradients(tiny_model):
    import torch

    wrapper = _unlearn_wrapper(tiny_model, batch_size=2, gradient_accumulation_steps=2)
    parameter = next(p for p in wrapper.model.parameters() if p.requires_grad)
    gradients = []
    real_clip = torch.nn.utils.clip_grad_norm_

    def capture_clip(parameters, max_norm):
        gradients.append(parameter.grad.detach().clone())
        return real_clip(parameters, max_norm)

    with (
        patch.object(wrapper, "_step_preference", side_effect=lambda *args: 2 * parameter.sum()),
        patch("torch.nn.utils.clip_grad_norm_", side_effect=capture_clip),
    ):
        result = wrapper.train()
    assert result["total_steps"] == 5
    assert len(gradients) == 2
    for gradient in gradients:
        torch.testing.assert_close(gradient, torch.full_like(gradient, 2.0))


def test_unlearn_cap_still_counts_examples(tiny_model, monkeypatch):
    from soup_cli.trainer import unlearn

    monkeypatch.setattr(unlearn, "_MAX_STEPS_CAP", 3)
    wrapper = _unlearn_wrapper(tiny_model, batch_size=2, gradient_accumulation_steps=4)
    assert wrapper.train()["total_steps"] == 3


def test_ppo_response_length_tracks_data_max_length(tiny_model):
    from trl.experimental.ppo import PPOConfig

    from soup_cli.trainer.ppo import PPOTrainerWrapper

    class ConfigCapture:
        def __init__(self, *, args, **kwargs):
            self.args = args

        def add_callback(self, callback):
            pass

    config = _config(tiny_model, "ppo", reward_fn="accuracy")
    config.data.max_length = 512
    wrapper = PPOTrainerWrapper(config, device="cpu")
    with (
        patch(
            "soup_cli.trainer.ppo._import_ppo_classes",
            return_value=(ConfigCapture, PPOConfig, True),
        ),
        patch.object(wrapper, "_get_or_create_reward_model", return_value=None),
        patch.object(wrapper, "_create_value_model", return_value=None),
    ):
        wrapper.setup({"train": [{"prompt": "one two", "answer": "three"}] * 8})
    args = wrapper.trainer.args
    assert args.response_length == 512
    assert args.response_length != 53


def test_ppo_forwards_deepspeed_config_and_guards(tiny_model):
    from trl.experimental.ppo import PPOConfig

    from soup_cli.trainer.ppo import PPOTrainerWrapper

    class ConfigCapture:
        def __init__(self, *, args, **kwargs):
            self.args = args

        def add_callback(self, callback):
            pass

        def create_optimizer(self):
            pass

    config = _config(tiny_model, "ppo", reward_fn="accuracy")

    # Without deepspeed:
    wrapper_plain = PPOTrainerWrapper(config, device="cpu")
    with (
        patch(
            "soup_cli.trainer.ppo._import_ppo_classes",
            return_value=(ConfigCapture, PPOConfig, True),
        ),
        patch.object(wrapper_plain, "_get_or_create_reward_model", return_value=None),
        patch.object(wrapper_plain, "_create_value_model", return_value=None),
    ):
        wrapper_plain.setup({"train": [{"prompt": "one two", "answer": "three"}] * 8})
    assert getattr(wrapper_plain.trainer, "_soup_empty_group_guard", False) is False
    assert wrapper_plain.trainer.args.deepspeed is None

    # With deepspeed:
    wrapper_ds = PPOTrainerWrapper(config, device="cpu", deepspeed_config="ds.json")
    with (
        patch("transformers.integrations.deepspeed.HfTrainerDeepSpeedConfig"),
        patch(
            "soup_cli.trainer.ppo._import_ppo_classes",
            return_value=(ConfigCapture, PPOConfig, True),
        ),
        patch.object(wrapper_ds, "_get_or_create_reward_model", return_value=None),
        patch.object(wrapper_ds, "_create_value_model", return_value=None),
    ):
        wrapper_ds.setup({"train": [{"prompt": "one two", "answer": "three"}] * 8})
    assert wrapper_ds.trainer.args.deepspeed == "ds.json"
    assert getattr(wrapper_ds.trainer, "_soup_empty_group_guard", False) is True


def test_ppo_refuses_deepspeed_when_unsupported(tmp_path):
    from soup_cli.config.schema import SoupConfig
    from soup_cli.trainer.ppo import PPOTrainerWrapper

    class StubPPOConfig:
        def __init__(self, output_dir=None, **kwargs):
            pass

    config = SoupConfig(
        base=str(tmp_path),
        task="ppo",
        output=str(tmp_path / "out"),
        data={"train": "dummy.jsonl", "max_length": 64},
        training={"epochs": 1, "reward_fn": "accuracy"},
    )
    wrapper = PPOTrainerWrapper(config, device="cpu", deepspeed_config="ds.json")
    with (
        patch(
            "soup_cli.trainer.ppo._import_ppo_classes",
            return_value=(object, StubPPOConfig, True),
        ),
        pytest.raises(
            ValueError, match="task: ppo does not support --deepspeed on this trl version"
        ),
    ):
        wrapper.setup({"train": []})