Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#1223 — the validation split is evaluated, or it is not withheld.
``data.val_split: 0.1`` moved 10% of the rows out of training on every run, and
no trainer set ``eval_strategy``, whose Hugging Face default is ``"no"``. The
rows were withheld and never evaluated.
This file covers the config and loader half of the fix:
* ``training.eval_steps`` -- the new field, its bounds, and the footgun refusals
(a value that nothing would read);
* the shared helper that decides the evaluation kwargs, the per-run loader
config and the notice;
* the loader decision as ``soup train`` and ``soup sweep`` drive it: generation
tasks (``grpo``, ``ppo``, ``online_dpo``) do not withhold rows unless
``training.eval_steps`` asks for evaluation, and everything else is unchanged.
The trainer half (every wrapper schedules evaluation, a real run records
``val_loss``) is ``test_issue1223_evaluate_val_split.py``.
"""
from __future__ import annotations
import json
import re
from pathlib import Path
from unittest.mock import MagicMock
from unittest.mock import patch as mock_patch
import pytest
from tests.conftest import strip_ansi
_CHAT_ROWS = 20
def _plain(text: str | None) -> str:
"""ANSI-stripped and whitespace-collapsed: Rich colours and wraps CLI output."""
return " ".join(strip_ansi(text).split())
def _load(yaml_text: str):
from soup_cli.config.loader import load_config_from_string
return load_config_from_string(yaml_text)
def _yaml(
*,
task: str = "sft",
train: str = "chat.jsonl",
backend: str = "transformers",
data_extra: str = "",
training_extra: str = "",
) -> str:
extra = {
"grpo": " num_generations: 2\n reward_fn: format\n",
"ppo": " reward_fn: format\n",
"online_dpo": " online_dpo_judge: 'ollama://m'\n",
"unlearn": " unlearn_method: npo\n",
}.get(task, "")
data_block = f"data:\n train: {train}\n max_length: 64\n{data_extra}"
if task == "unlearn":
data_block += " forget_set: forget.jsonl\n"
return (
"base: some-org/tiny-model\n"
f"task: {task}\n"
f"backend: {backend}\n"
f"{data_block}"
"training:\n"
" epochs: 1\n"
" batch_size: 2\n"
" quantization: none\n"
f"{extra}"
f"{training_extra}"
"output: ./out\n"
)
# ==========================================================================
# training.eval_steps -- the field
# ==========================================================================
class TestEvalStepsField:
def test_unset_by_default(self):
cfg = _load(_yaml())
assert cfg.training.eval_steps is None
@pytest.mark.parametrize("value", [1, 7, 500])
def test_a_positive_int_is_accepted(self, value):
cfg = _load(_yaml(training_extra=f" eval_steps: {value}\n"))
assert cfg.training.eval_steps == value
@pytest.mark.parametrize("value", [0, -3])
def test_zero_and_negatives_are_refused(self, value):
with pytest.raises(ValueError, match="eval_steps") as info:
_load(_yaml(training_extra=f" eval_steps: {value}\n"))
assert "greater than or equal to 1" in str(info.value)
def test_a_yaml_bool_is_refused_rather_than_read_as_one(self):
with pytest.raises(ValueError, match="eval_steps must be an int, not a bool"):
_load(_yaml(training_extra=" eval_steps: true\n"))
class TestEvalStepsFootguns:
"""Every refusal names the setting, the reason, and the issue."""
def test_nothing_to_evaluate_on_a_local_file_with_val_split_zero(self):
with pytest.raises(ValueError) as info:
_load(_yaml(data_extra=" val_split: 0\n", training_extra=" eval_steps: 5\n"))
message = str(info.value)
assert "training.eval_steps" in message
assert "data.val_split is 0" in message
assert "#1223" in message
def test_nothing_to_evaluate_on_an_interleaved_list_of_local_files(self):
with pytest.raises(ValueError, match="data.val_split is 0"):
_load(
_yaml(
train="[a.jsonl, b.jsonl]",
data_extra=" val_split: 0\n interleave: concat\n",
training_extra=" eval_steps: 5\n",
)
)
@pytest.mark.parametrize(
"train", ["https://example.com/train.jsonl", "s3://bucket/train.jsonl"]
)
def test_nothing_to_evaluate_on_a_remote_uri_with_val_split_zero(self, train):
"""A remote URI is a file, not a hub dataset: it brings no validation
split of its own, so it is refused like a local file."""
with pytest.raises(ValueError, match="data.val_split is 0"):
_load(
_yaml(
train=train,
data_extra=" val_split: 0\n",
training_extra=" eval_steps: 5\n",
)
)
def test_a_hub_dataset_may_carry_its_own_validation_split(self):
"""The control. An HF-hub dataset's own ``validation`` split reaches the
trainer whatever ``val_split`` says, so refusing here would refuse a
config that evaluates. Whether the split exists is only known at load."""
cfg = _load(
_yaml(
train="some-org/some-dataset",
data_extra=" val_split: 0\n",
training_extra=" eval_steps: 5\n",
)
)
assert cfg.training.eval_steps == 5
def test_a_pre_tokenized_cache_may_carry_its_own_validation_split(self):
cfg = _load(
_yaml(
data_extra=(
" val_split: 0\n format: pre_tokenized\n tokenized_path: ./cache\n"
),
training_extra=" eval_steps: 5\n",
)
)
assert cfg.training.eval_steps == 5
def test_a_positive_val_split_is_enough(self):
cfg = _load(_yaml(data_extra=" val_split: 0.2\n", training_extra=" eval_steps: 5\n"))
assert cfg.training.eval_steps == 5
@pytest.mark.parametrize(
("task", "reason"),
[
("ppo", "TRL's PPO loop never calls evaluate()"),
("online_dpo", "online DPO's evaluate() crashes on prompt-only rows"),
("unlearn", "never evaluates a validation split"),
],
)
def test_tasks_without_an_evaluation_pass_refuse_it(self, task, reason):
with pytest.raises(ValueError) as info:
_load(_yaml(task=task, training_extra=" eval_steps: 5\n"))
message = str(info.value)
assert f"training.eval_steps is not supported for task='{task}'" in message
assert reason in message
assert "#1223" in message
@pytest.mark.parametrize("task", ["ppo", "online_dpo"])
def test_generation_based_evaluation_is_named_as_a_separate_feature(self, task):
with pytest.raises(ValueError) as info:
_load(_yaml(task=task, training_extra=" eval_steps: 5\n"))
assert f"a generation-based evaluation for {task} is a separate feature" in str(
info.value
)
def test_grpo_accepts_it_as_the_opt_in(self):
cfg = _load(_yaml(task="grpo", training_extra=" eval_steps: 5\n"))
assert cfg.training.eval_steps == 5
def test_mlx_sft_honours_it_since_1990(self):
"""Refused here until #1990, which counts it in optimizer updates on
MLX SFT (tests/test_mlx_eval_steps.py pins the schedule)."""
cfg = _load(_yaml(backend="mlx", training_extra=" eval_steps: 5\n"))
assert (cfg.backend, cfg.training.eval_steps) == ("mlx", 5)
def test_mlx_preference_tasks_still_refuse_it(self):
with pytest.raises(ValueError) as info:
_load(_yaml(task="dpo", backend="mlx", training_extra=" eval_steps: 5\n"))
message = str(info.value)
assert "training.eval_steps is not supported for task='dpo' on backend='mlx'" in message
assert "#1990" in message
def test_mlx_without_it_still_loads(self):
"""The control for the refusal above: MLX itself is untouched."""
cfg = _load(_yaml(backend="mlx"))
assert cfg.training.eval_steps is None
# ==========================================================================
# the helper
# ==========================================================================
class _Rows:
def __init__(self, n: int):
self.n = n
def __len__(self) -> int:
return self.n
class _Unsized:
"""An iterable dataset whose length is not known."""
def __iter__(self):
return iter(())
class TestTrainingEvalKwargs:
def test_a_non_empty_split_evaluates_every_epoch_at_the_train_batch(self):
from soup_cli.utils.eval_schedule import training_eval_kwargs
cfg = _load(_yaml())
assert training_eval_kwargs(cfg, _Rows(2), batch_size=3) == {
"per_device_eval_batch_size": 3,
"eval_strategy": "epoch",
}
def test_eval_steps_switches_to_a_step_schedule(self):
from soup_cli.utils.eval_schedule import training_eval_kwargs
cfg = _load(_yaml(training_extra=" eval_steps: 4\n"))
assert training_eval_kwargs(cfg, _Rows(2), batch_size=1) == {
"per_device_eval_batch_size": 1,
"eval_strategy": "steps",
"eval_steps": 4,
}
def test_a_generation_task_does_not_evaluate_unasked(self):
from soup_cli.utils.eval_schedule import training_eval_kwargs
cfg = _load(_yaml(task="grpo"))
kwargs = training_eval_kwargs(cfg, _Rows(2), batch_size=2)
assert kwargs["eval_strategy"] == "no"
assert kwargs["per_device_eval_batch_size"] == 2
def test_a_generation_task_evaluates_when_asked(self):
from soup_cli.utils.eval_schedule import training_eval_kwargs
cfg = _load(_yaml(task="grpo", training_extra=" eval_steps: 3\n"))
kwargs = training_eval_kwargs(cfg, _Rows(2), batch_size=2)
assert (kwargs["eval_strategy"], kwargs["eval_steps"]) == ("steps", 3)
@pytest.mark.parametrize("dataset", [None, _Rows(0)], ids=["none", "empty"])
def test_no_rows_means_no_evaluation(self, dataset):
"""HF raises on a step schedule with no eval_dataset, and divides by the
row count of an empty one; neither is a reason to fail a run nobody
asked to evaluate."""
from soup_cli.utils.eval_schedule import training_eval_kwargs
cfg = _load(_yaml())
kwargs = training_eval_kwargs(cfg, dataset, batch_size=2)
assert kwargs["eval_strategy"] == "no"
def test_an_unsized_dataset_counts_as_rows(self):
from soup_cli.utils.eval_schedule import training_eval_kwargs
cfg = _load(_yaml())
assert training_eval_kwargs(cfg, _Unsized(), batch_size=2)["eval_strategy"] == "epoch"
def test_eval_steps_with_no_rows_refuses_naming_the_empty_hub_split(self):
from soup_cli.utils.eval_schedule import training_eval_kwargs
cfg = _load(
_yaml(
train="some-org/some-dataset",
data_extra=" val_split: 0\n",
training_extra=" eval_steps: 5\n",
)
)
with pytest.raises(ValueError) as info:
training_eval_kwargs(cfg, None, batch_size=2)
message = str(info.value)
assert "training.eval_steps=5 asks for evaluation" in message
assert "carries no validation split of its own" in message
assert "#1223" in message
def test_eval_steps_with_every_row_dropped_says_so(self):
from soup_cli.utils.eval_schedule import training_eval_kwargs
cfg = _load(_yaml(training_extra=" eval_steps: 5\n"))
with pytest.raises(ValueError, match="every validation row was dropped"):
training_eval_kwargs(cfg, _Rows(0), batch_size=2)
class TestTheLoaderDecision:
@pytest.mark.parametrize("task", ["sft", "dpo", "pretrain", "distill", "classifier"])
def test_evaluating_tasks_keep_their_split(self, task):
from soup_cli.utils.eval_schedule import (
effective_val_split,
holds_out_validation,
loader_data_config,
)
training = " teacher_model: some-org/teacher\n" if task == "distill" else ""
training += " num_labels: 2\n" if task == "classifier" else ""
cfg = _load(_yaml(task=task, training_extra=training))
assert holds_out_validation(cfg) is True
assert loader_data_config(cfg) is cfg.data
assert effective_val_split(cfg) == pytest.approx(0.1)
@pytest.mark.parametrize("task", ["grpo", "ppo", "online_dpo"])
def test_generation_tasks_withhold_nothing_by_default(self, task):
from soup_cli.utils.eval_schedule import (
effective_val_split,
holds_out_validation,
loader_data_config,
)
cfg = _load(_yaml(task=task))
assert holds_out_validation(cfg) is False
data_cfg = loader_data_config(cfg)
assert data_cfg.val_split == 0.0
assert effective_val_split(cfg) == 0.0
# A copy, never the config itself: the recorded run config keeps the
# value the user wrote.
assert cfg.data.val_split == pytest.approx(0.1)
assert data_cfg.train == cfg.data.train
def test_grpo_with_eval_steps_holds_the_split_out(self):
from soup_cli.utils.eval_schedule import holds_out_validation, loader_data_config
cfg = _load(_yaml(task="grpo", training_extra=" eval_steps: 4\n"))
assert holds_out_validation(cfg) is True
assert loader_data_config(cfg) is cfg.data
def test_val_split_zero_holds_nothing_out(self):
from soup_cli.utils.eval_schedule import holds_out_validation, loader_data_config
cfg = _load(_yaml(data_extra=" val_split: 0\n"))
assert holds_out_validation(cfg) is False
assert loader_data_config(cfg) is cfg.data
def test_mlx_is_unchanged(self):
"""mlx-lm evaluates the split itself (#739), so it stays withheld there,
whatever the task. MLX admits only sft today, so the generation-task
case is built with ``model_copy`` (no re-validation): the branch has to
hold the day MLX ships one."""
from soup_cli.utils.eval_schedule import holds_out_validation, loader_data_config
assert holds_out_validation(_load(_yaml(backend="mlx"))) is True
mlx_grpo = _load(_yaml(task="grpo")).model_copy(update={"backend": "mlx"})
assert holds_out_validation(mlx_grpo) is True
assert loader_data_config(mlx_grpo) is mlx_grpo.data
class TestTheNotice:
def test_grpo_names_the_ignored_setting_and_the_way_out(self):
from soup_cli.utils.eval_schedule import validation_notice
notice = validation_notice(_load(_yaml(task="grpo")))
assert notice == (
"data.val_split is ignored for task=grpo because evaluation means "
"generation, so every row trains. Set training.eval_steps to evaluate "
"on a held-out split."
)
@pytest.mark.parametrize("task", ["ppo", "online_dpo"])
def test_tasks_that_refuse_eval_steps_are_not_told_to_set_it(self, task):
from soup_cli.utils.eval_schedule import validation_notice
notice = validation_notice(_load(_yaml(task=task)))
assert notice is not None
assert f"data.val_split is ignored for task={task}" in notice
assert "eval_steps" not in notice
def test_no_notice_when_nothing_is_ignored(self):
from soup_cli.utils.eval_schedule import validation_notice
assert validation_notice(_load(_yaml())) is None
assert validation_notice(_load(_yaml(task="grpo", data_extra=" val_split: 0\n"))) is None
assert (
validation_notice(_load(_yaml(task="grpo", training_extra=" eval_steps: 4\n")))
is None
)
def test_the_module_imports_without_torch(self):
"""``config/schema.py`` imports it, so it must stay import-light."""
import subprocess
import sys
code = (
"import sys;"
"sys.modules['torch'] = None;"
"sys.modules['transformers'] = None;"
"import soup_cli.utils.eval_schedule as s;"
"print(sorted(s.GENERATION_EVAL_TASKS))"
)
out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
assert out.returncode == 0, out.stderr
# A plain print, but routed through the ANSI-strip helper all the same:
# the #1068 suite guard reads any `.stdout` assertion as CLI output.
assert _plain(out.stdout) == "['grpo', 'online_dpo', 'ppo']"
# ==========================================================================
# the loader decision, as `soup train --dry-run` drives it
# ==========================================================================
def _write_chat_rows(directory: Path, n: int = _CHAT_ROWS) -> None:
rows = [
{
"messages": [
{"role": "user", "content": f"question {i}"},
{"role": "assistant", "content": f"answer {i}"},
]
}
for i in range(n)
]
(directory / "chat.jsonl").write_text(
"\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8"
)
def _dry_run(tmp_path, monkeypatch, yaml_text: str) -> str:
from typer.testing import CliRunner
import soup_cli.commands.train as train_cmd
from soup_cli.cli import app
# The row counts pinned here do not depend on the machine, but a visible card
# does: on a dev box with a GPU the hardware-fit gate refuses before the dry
# run prints anything. Answer the two probes the way a CPU runner does (the
# #1212 tests stub them the same way): unknown VRAM skips the gate.
monkeypatch.setattr(train_cmd, "detect_device", lambda backend=None: ("cpu", "CPU"))
monkeypatch.setattr(
train_cmd,
"get_gpu_info",
lambda backend=None: {"memory_total": "0 MB", "memory_total_bytes": 0, "gpu_count": 0},
)
monkeypatch.chdir(tmp_path)
_write_chat_rows(tmp_path)
(tmp_path / "soup.yaml").write_text(yaml_text, encoding="utf-8")
result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--dry-run", "--yes"])
assert result.exit_code == 0, (result.output, repr(result.exception))
return _plain(result.output)
class TestDryRunCounts:
@pytest.mark.parametrize("task", ["grpo", "ppo", "online_dpo"])
def test_generation_tasks_train_on_every_row(self, tmp_path, monkeypatch, task):
out = _dry_run(tmp_path, monkeypatch, _yaml(task=task, data_extra=" format: chatml\n"))
assert f"Data OK: {_CHAT_ROWS} train samples" in out
assert "Val:" not in out
assert f"data.val_split is ignored for task={task}" in out
def test_the_grpo_notice_offers_eval_steps(self, tmp_path, monkeypatch):
out = _dry_run(tmp_path, monkeypatch, _yaml(task="grpo", data_extra=" format: chatml\n"))
assert "Set training.eval_steps to evaluate on a held-out split." in out
def test_grpo_with_eval_steps_holds_the_split_out(self, tmp_path, monkeypatch):
out = _dry_run(
tmp_path,
monkeypatch,
_yaml(
task="grpo",
data_extra=" format: chatml\n",
training_extra=" eval_steps: 2\n",
),
)
assert f"Data OK: {_CHAT_ROWS - 2} train samples" in out
assert "Val: 2 samples" in out
assert "is ignored" not in out
def test_control_sft_still_holds_the_split_out(self, tmp_path, monkeypatch):
"""Nothing changes for a task that now evaluates its split."""
out = _dry_run(tmp_path, monkeypatch, _yaml(data_extra=" format: chatml\n"))
assert f"Data OK: {_CHAT_ROWS - 2} train samples" in out
assert "Val: 2 samples" in out
assert "is ignored" not in out
class TestSweepAndCostFollowTheSameRule:
"""`soup sweep` and `soup cost` reason about the same split `soup train` uses."""
@pytest.mark.parametrize(("task", "expected"), [("grpo", 0.0), ("sft", 0.1)])
def test_sweep_hands_the_loader_the_run_split(self, task, expected):
from soup_cli.commands.sweep import _run_single
cfg = _load(_yaml(task=task, data_extra=" format: chatml\n"))
wrapper = {
"grpo": "soup_cli.trainer.grpo.GRPOTrainerWrapper",
"sft": "soup_cli.trainer.sft.SFTTrainerWrapper",
}[task]
fake_result = {
"initial_loss": 1.0,
"final_loss": 0.5,
"total_steps": 1,
"duration_secs": 1.0,
"output_dir": "./out",
"duration": "0m",
}
with mock_patch(
"soup_cli.data.loader.load_dataset", return_value={"train": [{"x": 1}]}
) as mock_load, mock_patch(
"soup_cli.utils.gpu.detect_device", return_value=("cpu", "CPU")
), mock_patch(
"soup_cli.utils.gpu.get_gpu_info",
return_value={"memory_total": "0 MB", "memory_total_bytes": 0},
), mock_patch(
"soup_cli.experiment.tracker.ExperimentTracker"
) as mock_tracker_cls, mock_patch(
"soup_cli.monitoring.display.TrainingDisplay"
), mock_patch(f"{wrapper}.setup"), mock_patch(
f"{wrapper}.train", return_value=fake_result
):
mock_tracker = MagicMock()
mock_tracker.start_run.return_value = "run-1"
mock_tracker_cls.return_value = mock_tracker
_run_single(cfg, {}, "sweep_1", None)
data_cfg = mock_load.call_args.args[0]
assert data_cfg.val_split == pytest.approx(expected)
@pytest.mark.parametrize(("task", "rows"), [("grpo", 20), ("sft", 18)])
def test_cost_counts_the_rows_that_train(self, tmp_path, monkeypatch, task, rows):
from soup_cli.commands.cost import _get_dataset_size
monkeypatch.chdir(tmp_path)
_write_chat_rows(tmp_path)
cfg = _load(_yaml(task=task, data_extra=" format: chatml\n"))
assert _get_dataset_size(cfg) == (rows, False)
def test_the_plain_helper_collapses_what_rich_splits():
"""Guard for `_plain`: Rich wraps and colours per character on Linux CI."""
raw = "\x1b[1mData\x1b[0m OK:\n 20 train\x1b[36m samples\x1b[0m"
assert _plain(raw) == "Data OK: 20 train samples"
assert re.search(r"\x1b", _plain(raw)) is None