Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Evaluate the validation split that every task wrapper receives (#1223).
``data.val_split`` (default 0.1) moves that share of the rows out of training,
and the task wrappers hand the result to their trainer as ``eval_dataset``. No
wrapper set ``eval_strategy``, whose Hugging Face default is ``"no"``, so
``Trainer.evaluate()`` never ran. The rows were withheld and used for nothing,
and the validation-loss sinks #713 added (the metrics column,
``TrainEvent.val_loss``, the display row) never received a value on the
transformers backend.
The policy this module encodes:
* A trainer that receives a non-empty validation split evaluates it at the end
of every epoch, or every ``training.eval_steps`` optimizer steps when that is
set. Hugging Face also evaluates once more at the last step when that step
is not a multiple of ``eval_steps``, so a value larger than the run still
evaluates once.
* The evaluation batch is the resolved per-device TRAIN batch. Hugging Face's
default of 8 would run out of memory on a card sized to train at 1.
* Generation-based tasks (:data:`GENERATION_EVAL_TASKS`) do not evaluate by
default, because an evaluation there means generating completions. They do
not withhold rows either: ``data.val_split`` is ignored for them, with a
notice, unless ``training.eval_steps`` asks for evaluation.
* ``backend: mlx`` is untouched. mlx-lm runs its own validation loop over the
split (#739), so the loader keeps withholding it there.
Import-light on purpose: ``config/schema.py`` imports the task sets, so nothing
here may pull torch, transformers or the data stack.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, NamedTuple, Optional
if TYPE_CHECKING: # pragma: no cover - typing only
from soup_cli.config.schema import DataConfig, SoupConfig
#: Tasks where an evaluation pass means generating completions, so it costs as
#: much per row as training does. They evaluate only when asked to.
GENERATION_EVAL_TASKS: frozenset[str] = frozenset({"grpo", "ppo", "online_dpo"})
class ValidationMetric(NamedTuple):
"""Which number an evaluation produces for a task, and what it means (#1389).
``log_key`` is the entry in the trainer's ``on_log`` dict; ``field`` is the
name Soup records it under -- the tracker column, the ``TrainEvent`` field and
the display's attribute, which all spell it the same way; ``label`` is the
live panel's row; ``direction`` says how to read the series.
"""
log_key: str
field: str
label: str
direction: str
#: Every task's evaluation logs a held-out loss, lower is better ...
DEFAULT_VALIDATION_METRIC = ValidationMetric(
"eval_loss", "val_loss", "Val loss", "lower is better"
)
#: ... except the generation-scored ones. On ``grpo`` TRL's ``eval_loss`` is the
#: policy objective at importance ratio 1 with group-normalised advantages: it
#: sits near zero, is often negative and does not measure held-out quality. The
#: held-out reward, ``eval_reward``, is the run's validation number, and it is
#: recorded under its own name so nothing reads it as a loss (#1389). A task
#: that later gains a generation-based evaluation (``ppo``, ``online_dpo``, which
#: refuse ``eval_steps`` today) adds a row here, not a special case downstream.
VALIDATION_METRICS: dict[str, ValidationMetric] = {
"grpo": ValidationMetric("eval_reward", "val_reward", "Val reward", "higher is better"),
}
def validation_metric(task: Optional[str]) -> ValidationMetric:
"""The validation metric a task records; the loss for any task not in the table."""
if isinstance(task, str):
return VALIDATION_METRICS.get(task, DEFAULT_VALIDATION_METRIC)
return DEFAULT_VALIDATION_METRIC
#: Tasks that refuse ``training.eval_steps`` at config load, each with the
#: reason. The setting would be read by nothing on them (ppo, unlearn) or would
#: crash the run (online_dpo). A generation-based evaluation for ppo and
#: online_dpo -- scoring completions for held-out prompts -- is a separate
#: feature, not something this setting can switch on.
EVAL_STEPS_UNSUPPORTED_TASKS: dict[str, str] = {
"ppo": (
"TRL's PPO loop never calls evaluate(); a generation-based evaluation "
"for ppo is a separate feature"
),
"online_dpo": (
"online DPO's evaluate() crashes on prompt-only rows; a "
"generation-based evaluation for online_dpo is a separate feature"
),
"unlearn": (
"unlearning trains on data.forget_set and data.retain_set and never "
"evaluates a validation split"
),
}
def evaluates_by_default(task: str) -> bool:
"""Whether a run of ``task`` evaluates its validation split unasked."""
return task not in GENERATION_EVAL_TASKS
def holds_out_validation(cfg: SoupConfig) -> bool:
"""Whether the loader should carve ``data.val_split`` out of the training rows.
A split that nothing evaluates is not carved out: those rows train instead.
"""
if cfg.data.val_split <= 0:
return False
if getattr(cfg, "backend", None) == "mlx":
return True
return evaluates_by_default(cfg.task) or cfg.training.eval_steps is not None
def loader_data_config(cfg: SoupConfig) -> DataConfig:
"""The ``DataConfig`` to hand ``load_dataset`` for this run.
``cfg.data`` itself, unless the run would withhold rows that nothing
evaluates; then a copy with ``val_split`` at 0, so every row trains. A
dataset's own validation split (an HF-hub ``validation`` split) is not
withheld from training, so it still arrives either way.
"""
if cfg.data.val_split <= 0 or holds_out_validation(cfg):
return cfg.data
return cfg.data.model_copy(update={"val_split": 0.0})
def effective_val_split(cfg: SoupConfig) -> float:
"""The fraction of the rows this run actually withholds from training."""
return loader_data_config(cfg).val_split
def validation_notice(cfg: SoupConfig) -> Optional[str]:
"""One line explaining an ignored ``data.val_split``, or ``None``."""
if cfg.data.val_split <= 0 or holds_out_validation(cfg):
return None
notice = (
f"data.val_split is ignored for task={cfg.task} because evaluation "
"means generation, so every row trains."
)
if cfg.task not in EVAL_STEPS_UNSUPPORTED_TASKS:
notice += " Set training.eval_steps to evaluate on a held-out split."
return notice
def _has_rows(dataset: Any) -> bool:
if dataset is None:
return False
try:
return len(dataset) > 0
except TypeError: # an iterable dataset of unknown length
return True
def training_eval_kwargs(
cfg: SoupConfig, eval_dataset: Any, *, batch_size: int
) -> dict[str, Any]:
"""Evaluation kwargs for any ``TrainingArguments`` subclass.
``eval_dataset`` is what the wrapper is about to hand its trainer, after its
own row preparation, and ``batch_size`` the resolved per-device train batch
(never ``"auto"``). Raises ``ValueError`` when ``training.eval_steps`` asks
for an evaluation that has no rows to run on.
"""
eval_steps = getattr(cfg.training, "eval_steps", None)
kwargs: dict[str, Any] = {"per_device_eval_batch_size": int(batch_size)}
if not _has_rows(eval_dataset):
if eval_steps is not None:
if cfg.data.val_split > 0:
why = "every validation row was dropped while the dataset was prepared"
else:
why = (
"data.val_split is 0 and the dataset carries no validation "
"split of its own"
)
raise ValueError(
f"training.eval_steps={eval_steps} asks for evaluation, but this "
f"run has no validation rows: {why}. Set data.val_split above 0, "
"or remove training.eval_steps (#1223)."
)
kwargs["eval_strategy"] = "no"
return kwargs
if eval_steps is not None:
kwargs["eval_strategy"] = "steps"
kwargs["eval_steps"] = int(eval_steps)
elif evaluates_by_default(cfg.task):
kwargs["eval_strategy"] = "epoch"
else:
kwargs["eval_strategy"] = "no"
return kwargs