Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for #1247: allow grpo_beta: 0 for KL-free GRPO recipes."""
from __future__ import annotations
from pathlib import Path
import pytest
from pydantic import ValidationError
from soup_cli.config.loader import load_config_from_string
from soup_cli.config.schema import TrainingConfig
VARIANTS = ("standard", "gspo", "dapo", "dr_grpo", "bnpo", "two_sided", "rft")
class TestSchemaGrpoBetaZero:
"""Schema-level validation of grpo_beta: 0."""
@pytest.mark.parametrize("variant", VARIANTS)
def test_grpo_beta_zero_loads_for_all_variants(self, variant):
delta_line = " grpo_delta: 0.2\n" if variant in ("two_sided", "gspo") else ""
cfg_str = f"""
base: test-model
task: grpo
data:
train: dummy.jsonl
max_length: 64
training:
grpo_beta: 0
grpo_variant: {variant}
{delta_line}
"""
cfg = load_config_from_string(cfg_str)
assert cfg.training.grpo_beta == 0.0
def test_default_grpo_beta_unchanged(self):
cfg_str = """
base: test-model
task: grpo
data:
train: dummy.jsonl
max_length: 64
"""
cfg = load_config_from_string(cfg_str)
assert cfg.training.grpo_beta == 0.1
def test_grpo_beta_zero_float_loads(self):
tcfg = TrainingConfig(grpo_beta=0.0)
assert tcfg.grpo_beta == 0.0
def test_grpo_beta_negative_rejected(self):
with pytest.raises(ValidationError):
TrainingConfig(grpo_beta=-0.01)
def test_grpo_beta_bool_rejected(self):
with pytest.raises(ValidationError, match="grpo_beta must not be a boolean"):
TrainingConfig(grpo_beta=True)
with pytest.raises(ValidationError, match="grpo_beta must not be a boolean"):
TrainingConfig(grpo_beta=False)
def test_grpo_beta_nan_inf_rejected(self):
with pytest.raises(ValidationError, match="grpo_beta must be finite"):
TrainingConfig(grpo_beta=float("nan"))
with pytest.raises(ValidationError, match="grpo_beta must be finite"):
TrainingConfig(grpo_beta=float("inf"))
class TestRewardHackMitigationConflict:
"""grpo_beta: 0 is mutually exclusive with beta-adjusting mitigation."""
@pytest.mark.parametrize("mode", ["kl_control", "pid_lagrangian"])
def test_grpo_beta_zero_refused_with_beta_adjusting_mitigation(self, mode):
cfg_str = f"""
base: test-model
task: grpo
data:
train: dummy.jsonl
max_length: 64
training:
grpo_beta: 0
reward_hack_detector: info_rm
reward_hack_mitigation: {mode}
"""
with pytest.raises(ValueError) as excinfo:
load_config_from_string(cfg_str)
err = str(excinfo.value)
assert "grpo_beta" in err
assert "reward_hack_mitigation" in err
assert mode in err
def test_grpo_beta_zero_accepted_with_log_only_mitigation(self):
cfg_str = """
base: test-model
task: grpo
data:
train: dummy.jsonl
max_length: 64
training:
grpo_beta: 0
reward_hack_detector: info_rm
reward_hack_mitigation: log_only
"""
cfg = load_config_from_string(cfg_str)
assert cfg.training.grpo_beta == 0.0
assert cfg.training.reward_hack_mitigation == "log_only"
class TestKernelLossBitForBit:
"""Variant loss with beta=0 matches KL-free loss bit-for-bit without reference."""
@pytest.mark.parametrize("variant", ["gspo", "dapo", "dr_grpo", "bnpo", "two_sided", "rft"])
def test_variant_loss_matches_kl_free_bit_for_bit(self, variant):
torch = pytest.importorskip("torch")
from soup_cli.utils.grpo_variants import apply_variant_loss
gen = torch.Generator().manual_seed(1247)
dtype = torch.float64
logp_new = (-2.0 * torch.rand(3, 5, generator=gen, dtype=dtype)).requires_grad_(True)
logp_old = logp_new.detach() + 0.2 * torch.randn(3, 5, generator=gen, dtype=dtype)
advantages = torch.tensor([1.0, -0.5, 0.75], dtype=dtype)
mask = torch.tensor([[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [1, 1, 0, 0, 0]], dtype=dtype)
delta = 0.2 if variant in ("two_sided", "gspo") else None
# Call with beta=0.0 and NO reference_logp (simulates TRL skipping ref forward)
loss_no_ref = apply_variant_loss(
variant,
logp_new=logp_new,
logp_old=logp_old,
advantages=advantages,
beta=0.0,
delta=delta,
completion_mask=mask,
reference_logp=None,
)
# Call with dummy reference
dummy_ref = logp_new.detach() + 5.0
loss_with_ref = apply_variant_loss(
variant,
logp_new=logp_new,
logp_old=logp_old,
advantages=advantages,
beta=0.0,
delta=delta,
completion_mask=mask,
reference_logp=dummy_ref,
)
assert torch.equal(loss_no_ref, loss_with_ref), (
f"{variant}: loss with beta=0 changed when reference was provided"
)
class TestRealTrainerBetaZero:
"""Real GRPOTrainerWrapper on tiny CPU model skips reference forward when beta=0."""
@pytest.mark.parametrize("variant", ["standard", "dapo", "dr_grpo"])
def test_ref_forward_skipped_when_beta_zero(self, tmp_path, monkeypatch, variant):
for mod in ("torch", "transformers", "peft", "trl", "datasets"):
pytest.importorskip(mod)
import yaml
from soup_cli.trainer.grpo import GRPOTrainerWrapper
from tests.test_issue1232_grpo_variant_kl import (
_tiny_llama_dir,
_write_tiny_tokenizer,
)
weights = _tiny_llama_dir(tmp_path)
_write_tiny_tokenizer(weights)
monkeypatch.chdir(tmp_path)
cfg_dict = {
"base": weights,
"task": "grpo",
"backend": "transformers",
"modality": "text",
"data": {"train": "train.jsonl", "max_length": 64, "chat_template": "chatml"},
"training": {
"batch_size": 2,
"gradient_accumulation_steps": 1,
"num_generations": 2,
"reward_fn": "format",
"quantization": "none",
"epochs": 1,
"grpo_variant": variant,
"grpo_beta": 0,
"lora": {"r": 4, "alpha": 8, "target_modules": ["q_proj", "v_proj"]},
},
"output": str(tmp_path / "out"),
}
cfg = load_config_from_string(yaml.safe_dump(cfg_dict))
wrapper = GRPOTrainerWrapper(cfg, device="cpu")
wrapper.setup({"train": [{"prompt": "hi"} for _ in range(4)]})
trainer = wrapper.trainer
assert trainer.beta == 0.0
batch = next(iter(trainer.get_train_dataloader()))
inputs = trainer._prepare_inputs(batch)
# When beta=0, trl skips computing ref_per_token_logps
assert inputs.get("ref_per_token_logps") is None, (
"TRL should skip reference forward when beta is 0.0"
)
# Ensure loss computation succeeds without error
loss = trainer._compute_loss(trainer.model, inputs)
assert loss is not None
assert not loss.isnan()
class TestTheRealTrainerComputesTheKlFreeLoss:
"""Acceptance 2 on the real trainer: at grpo_beta 0 every variant computes the
KL-free loss itself (no #159 fallback to trl's stock loss), bit for bit."""
@pytest.mark.parametrize("variant", ["gspo", "dapo", "dr_grpo", "bnpo", "two_sided", "rft"])
def test_loss_and_gradient_equal_the_kl_free_loss(self, tmp_path, monkeypatch, variant):
for mod in ("torch", "transformers", "peft", "trl", "datasets"):
pytest.importorskip(mod)
import torch
from tests.test_issue1232_grpo_variant_kl import (
_build_real,
_loss_and_grad,
_pre_1232_loss,
)
trainer = _build_real(tmp_path, monkeypatch, variant, grpo_beta=0.0).trainer
torch.manual_seed(0)
inputs = trainer._prepare_inputs(next(iter(trainer.get_train_dataloader())))
assert inputs.get("ref_per_token_logps") is None
rows = inputs["advantages"].shape[0]
inputs["advantages"] = torch.tensor([1.0, -1.0] * (rows // 2))
trainer.model.train()
trainer.current_gradient_accumulation_steps = 1
loss, grad = _loss_and_grad(trainer, lambda: trainer._compute_loss(trainer.model, inputs))
assert not trainer._soup_fallback_warned, f"{variant}: fell back to trl's stock loss"
def kl_free():
input_ids = torch.cat([inputs["prompt_ids"], inputs["completion_ids"]], dim=1)
attention_mask = torch.cat([inputs["prompt_mask"], inputs["completion_mask"]], dim=1)
logp, _ = trainer._get_per_token_logps_and_entropies(
trainer.model,
input_ids,
attention_mask,
inputs["completion_ids"].size(1),
compute_entropy=False,
)
old = inputs.get("old_per_token_logps")
return _pre_1232_loss(
torch,
variant,
logp_new=logp,
logp_old=logp.detach() if old is None else old,
advantages=inputs["advantages"],
delta=getattr(trainer, "_soup_grpo_delta", None),
completion_mask=inputs["completion_mask"],
)
expected, expected_grad = _loss_and_grad(trainer, kl_free)
assert torch.equal(loss, expected), (variant, loss.item(), expected.item())
assert torch.equal(grad, expected_grad), f"{variant}: gradient is not the KL-free one"
class TestDocumentationMentionsKlFree:
def test_training_docs_say_how_to_run_a_kl_free_recipe(self):
docs = Path(__file__).parents[1] / "docs" / "training.md"
content = " ".join(docs.read_text(encoding="utf-8").split())
assert "`grpo_beta` must be greater than zero" not in content
assert "Set `grpo_beta: 0` for a KL-free run" in content
@pytest.mark.parametrize("spelling", ["inf", "Infinity", "1e400", ".inf", ".nan", "'nan'"])
def test_grpo_beta_non_finite_yaml_spellings_rejected(spelling):
with pytest.raises(ValueError, match="grpo_beta"):
load_config_from_string(
"base: test-model\ntask: grpo\ndata:\n train: dummy.jsonl\n"
f"training:\n grpo_beta: {spelling}\n"
)