Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
fix(config): gate reasoning_effort and train_on_eot on the tasks that read them (#806) (#1587)
Dipak Chaudhari304e62e1d160
diff --git a/changelog.d/0.76.0/1587.fixed.md b/changelog.d/0.76.0/1587.fixed.mdnew file mode 100644index 00000000..962382ec--- /dev/null+++ b/changelog.d/0.76.0/1587.fixed.md@@ -0,0 +1,8 @@+- **`training.reasoning_effort` and `training.train_on_eot` now load only on the tasks that read them (#806 by @dchaudhari7177 in #1587).**+ Both are applied by the SFT formatter, which runs on `sft`, `tts` and+ `distill`. Before, the gate refused `tts`, where both flags take effect,+ and accepted `pretrain`, `classifier`, `reranker` and `cross_encoder`,+ where they loaded and changed nothing. Such a config now fails at load.+ On `pretrain`, `train_on_eot` changed only the `soup data preprocess`+ cache key, never the cached rows; a pretrain cache built with the flag+ has to be rebuilt without it.diff --git a/docs/training.md b/docs/training.mdindex 592673bf..00a2b8f4 100644--- a/docs/training.md+++ b/docs/training.md@@ -780,7 +780,7 @@ training:`reasoning_effort` injects `<|reasoning_effort|>high<|/reasoning_effort|>` intothe system turn (creating one if absent). `train_on_eot=True` makes the modellearn when to stop generating by training on the trailing EOS token instead of-masking it out. Both are gated to the SFT-family of tasks.+masking it out. Both load only on `task: sft`, `tts` and `distill`.## EBFT / GDPO Loss Variants@@ -2260,7 +2260,7 @@ the cross-entropy. See [EBFT / GDPO Loss Variants](#ebft--gdpo-loss-variants).## gpt-oss `reasoning_effort` + `train_on_eot` (v0.52.0)-`training.reasoning_effort: low | medium | high` injects a system-prefix token at training time for gpt-oss models; `training.train_on_eot: true` includes explicit EOT/EOS control tokens in the SFT loss (axolotl `train_on_eot`). Both are gated to the SFT-family task set (`sft` / `pretrain` / `distill` / `classifier` / `reranker` / `cross_encoder`) — setting them on DPO / GRPO / PPO / etc. fails at config load. Live formatter wiring in v0.52.1.+`training.reasoning_effort: low | medium | high` injects a system-prefix token at training time for gpt-oss models; `training.train_on_eot: true` includes explicit EOT/EOS control tokens in the SFT loss (axolotl `train_on_eot`). Both are applied by the SFT formatter, which runs on `sft`, `tts` and `distill`, so those are the tasks they load on. Setting either on any other task (`pretrain`, DPO, GRPO, PPO, `classifier`, `reranker`, `cross_encoder`) fails at config load. Live formatter wiring in v0.52.1.## MoLE — Per-Token Adapter Routing (`task='moe_lora_routing'`)diff --git a/src/soup_cli/config/schema.py b/src/soup_cli/config/schema.pyindex d4b74dc2..b171480a 100644--- a/src/soup_cli/config/schema.py+++ b/src/soup_cli/config/schema.py@@ -2150,7 +2150,7 @@ class TrainingConfig(BaseModel):default=None,description=("gpt-oss train-time reasoning effort level. Injected as a prompt "- "prefix by the SFT-family data formatter."+ "prefix by the SFT data formatter; task sft, tts or distill only."),)train_on_eot: bool = Field(@@ -2158,7 +2158,8 @@ class TrainingConfig(BaseModel):description=("Include explicit EOT / EOS control tokens in the SFT loss ""(axolotl ``train_on_eot``). Default False matches HF Trainer "- "convention. (v0.52.0)"+ "convention. Applied by the SFT data formatter; task sft, tts or "+ "distill only. (v0.52.0)"),)# ---- v0.53.0 Quant Menu II — UD GGUFs + KV cache + NVFP4 ---------------@@ -5072,6 +5073,16 @@ SFT_KERNEL_AWARE_TASKS: frozenset[str] = frozenset({"sft", "tts"})AMP_APPLYING_TASKS: frozenset[str] = frozenset({"sft", "tts"})+# #806: tasks whose trainers build rows with ``data.sft_format.build_format_row``+# (sft's text path, inherited by tts, and distill), the only reader of+# ``training.reasoning_effort``.+REASONING_EFFORT_AWARE_TASKS: frozenset[str] = frozenset({"sft", "tts", "distill"})+# #806: ``training.train_on_eot`` has the same reader. pretrain passes it to the+# pre-tokenized cache too, but a pretrain row is ``labels = input_ids`` whatever+# the mask mode, so there it changed only the cache key.+TRAIN_ON_EOT_AWARE_TASKS: frozenset[str] = REASONING_EFFORT_AWARE_TASKS++# #795: trainers that load the base unquantised and never read# ``training.quantization``._QUANTIZATION_UNHONOURED_TASKS = frozenset({@@ -6359,34 +6370,35 @@ class SoupConfig(BaseModel):@model_validator(mode="after")def _validate_reasoning_effort_task_gate(self) -> "SoupConfig":- """v0.52.0 Part G (code-review fix) — surface the silent-no-op- footgun when ``reasoning_effort`` / ``train_on_eot`` is set on a- task they cannot influence.-- Mirrors v0.50.0 ``_validate_grpo_stability_task_gate`` policy.- ``reasoning_effort`` only makes sense on SFT-family training- (sft / pretrain / distill / classifier-family) because the SFT- formatter injects it as a system-prefix token. The other- tasks (DPO / GRPO / KTO / ORPO / SimPO / IPO / BCO / preference /- PPO / reward_model / embedding / prm / tts) do not consume it.-- ``train_on_eot`` is an SFT loss-mask flag; setting it on- DPO/GRPO/etc. is a silent no-op.+ """Refuse ``reasoning_effort`` / ``train_on_eot`` on a task that never reads them.++ Both are applied by ``data.sft_format.build_format_row``, whose trainer+ callers are sft (its text path, inherited by tts's+ ``TTSTrainerWrapper.setup``) and distill (#806). pretrain, classifier,+ reranker and cross_encoder never call it, so the flags loaded there and+ changed nothing. pretrain also passes ``train_on_eot`` to the+ ``soup data preprocess`` cache key (#1054), but its rows are+ ``labels = input_ids`` either way, so the flag is refused there too."""tcfg = self.training- sft_family_tasks = {- "sft", "pretrain", "distill",- "classifier", "reranker", "cross_encoder",- }- if tcfg.reasoning_effort is not None and self.task not in sft_family_tasks:+ if (+ tcfg.reasoning_effort is not None+ and self.task not in REASONING_EFFORT_AWARE_TASKS+ ):raise ValueError(f"training.reasoning_effort={tcfg.reasoning_effort!r} requires "- f"task in {sorted(sft_family_tasks)}; got task={self.task!r}"+ f"task in {sorted(REASONING_EFFORT_AWARE_TASKS)}; got task={self.task!r}. "+ "Only those tasks build their rows with the SFT formatter that "+ "injects it, so elsewhere it would be ignored. Use task: sft, "+ "tts or distill, or remove training.reasoning_effort.")- if tcfg.train_on_eot and self.task not in sft_family_tasks:+ if tcfg.train_on_eot and self.task not in TRAIN_ON_EOT_AWARE_TASKS:raise ValueError(f"training.train_on_eot=true requires task in "- f"{sorted(sft_family_tasks)}; got task={self.task!r}"+ f"{sorted(TRAIN_ON_EOT_AWARE_TASKS)}; got task={self.task!r}. "+ "Only those tasks build their rows with the SFT formatter that "+ "applies the loss mask, so elsewhere it would be ignored. Use "+ "task: sft, tts or distill, or remove training.train_on_eot.")return selfdiff --git a/src/soup_cli/trainer/pretrain.py b/src/soup_cli/trainer/pretrain.pyindex 20c3d786..e3b9fcd6 100644--- a/src/soup_cli/trainer/pretrain.py+++ b/src/soup_cli/trainer/pretrain.py@@ -131,11 +131,9 @@ class PretrainTrainerWrapper:# via `soup data preprocess`. Skips the raw-text load entirely.from soup_cli.trainer.sft import _maybe_load_pretokenized- # #1054: pass ``tcfg`` -- ``preprocess_mask_mode`` reads- # ``training.train_on_eot`` from it, and ``task: pretrain`` is in the- # sft-family set the schema allows that flag on. Omitting it dropped the- # ``+eot`` suffix here but not in ``soup data preprocess``, so the cache- # was refused by a hash the re-run advised in the error reproduces.+ # #1054: pass ``tcfg`` so the cache key is derived the same way as in+ # ``soup data preprocess``. The schema refuses ``training.train_on_eot``+ # on ``task: pretrain`` (#806), so the mask mode here is the default.pretok = _maybe_load_pretokenized(cfg.data, cfg.base, console, getattr(cfg, "training", None),task=cfg.task,diff --git a/tests/test_issue1054_pretokenized_labels.py b/tests/test_issue1054_pretokenized_labels.pyindex 2b34d038..41d2e61c 100644--- a/tests/test_issue1054_pretokenized_labels.py+++ b/tests/test_issue1054_pretokenized_labels.py@@ -706,45 +706,36 @@ class TestCacheKeyCoversMaskMode:train_ds, _ = loadedassert len(train_ds) == 1- def test_train_on_eot_cache_loads_through_the_pretrain_caller(+ def test_pretrain_refuses_train_on_eot_before_writing_a_cache(self, tmp_path, monkeypatch):- """``task: pretrain`` + ``train_on_eot: true`` is a config the schema- allows (pretrain is in the sft-family set). ``trainer/pretrain.py`` must- derive the same mask mode ``soup data preprocess`` wrote, or the cache is- unloadable and re-running preprocess as the error advises regenerates the- very same rejected key.-- regression: must fail without the fix -- ``pretrain.py`` called the gate- without ``tcfg``, dropping the ``+eot`` suffix on its side only.+ """#806: a ``task: pretrain`` cache records ``labels = input_ids`` whatever+ the mask mode, so ``train_on_eot`` changed only the cache key there. The+ schema refuses it on pretrain, so ``soup data preprocess`` stops at load+ and no ``+eot`` pretrain cache is written."""- from rich.console import Console+ import re- from soup_cli.config.schema import DataConfig- from soup_cli.trainer.sft import _maybe_load_pretokenized+ from typer.testing import CliRunner- tok = _tokenizer()- cache_dir = _run_preprocess(- tmp_path,- monkeypatch,- tok,- task="pretrain",- rows=[{"text": "You are terse ."}],- data_extra="training:\n train_on_eot: true\n",- )- dcfg = DataConfig(- train="./d.jsonl",- format="pre_tokenized",- tokenized_path=str(cache_dir.relative_to(tmp_path)),- max_length=128,- )- tcfg = SimpleNamespace(train_on_eot=True)- # Exactly what trainer/pretrain.py now passes.- loaded = _maybe_load_pretokenized(dcfg, "x/y", Console(), tcfg, task="pretrain")- assert loaded is not None- train_ds, _ = loaded- assert len(train_ds) == 1+ from soup_cli.cli import app+ monkeypatch.chdir(tmp_path)+ (tmp_path / "soup.yaml").write_text(+ "base: x/y\ntask: pretrain\n"+ "data:\n train: ./d.jsonl\n format: chatml\n max_length: 128\n"+ "training:\n train_on_eot: true\n",+ encoding="utf-8",+ )+ (tmp_path / "d.jsonl").write_text("{}\n", encoding="utf-8")+ result = CliRunner().invoke(app, ["data", "preprocess", "soup.yaml", "--yes"])+ output = " ".join(re.sub(r"\x1b\[[0-9;]*m", "", result.output).split())+ assert result.exit_code == 1, (result.output, repr(result.exception))+ assert "train_on_eot=true requires task in" in output+ assert "got task='pretrain'" in output+ # Refused at load: preprocess never reached its own report or wrote a cache.+ assert "Loss mask:" not in output+ assert not (tmp_path / ".soup-tokenized").exists()class TestSampleCount:def test_pretokenized_count_comes_from_the_cache(self, tmp_path):diff --git a/tests/test_issue806_format_row_task_gates.py b/tests/test_issue806_format_row_task_gates.pynew file mode 100644index 00000000..d77d4542--- /dev/null+++ b/tests/test_issue806_format_row_task_gates.py@@ -0,0 +1,79 @@+"""Issue #806 (remainder): ``reasoning_effort`` / ``train_on_eot`` only where they are read.++Both are applied by ``data.sft_format.build_format_row``. Its trainer callers+are sft's text path (inherited by tts through ``TTSTrainerWrapper.setup``) and+distill; pretrain and the classifier family never call it, so the flags loaded+there and changed nothing, while tts, which does honour them, was refused.+pretrain also keys its pre-tokenized cache on ``train_on_eot`` (#1054), but its+rows are ``labels = input_ids`` either way, so it is refused there too.++Every refused config is first loaded without the flag, so a refusal can only+come from the flag itself.+"""++from __future__ import annotations++import pytest++from soup_cli.config.loader import load_config_from_string+from soup_cli.config.schema import REASONING_EFFORT_AWARE_TASKS, TRAIN_ON_EOT_AWARE_TASKS+from tests.test_issue1325_expand_layers_scope import _ALL_TASKS, _DATA, _ROOT, _TRAINING++_REASONING = " reasoning_effort: medium\n"+_EOT = " train_on_eot: true\n"+++def _yaml(task, flag=""):+ return (+ "base: meta-llama/Llama-3.1-8B\n"+ f"task: {task}\n"+ "backend: transformers\n"+ f"{_ROOT.get(task, '')}"+ "data:\n train: data.jsonl\n"+ + _DATA.get(task, "")+ + "training:\n quantization: none\n"+ + _TRAINING.get(task, "")+ + flag+ )+++def test_the_allowlists_match_the_consumers():+ assert REASONING_EFFORT_AWARE_TASKS == {"sft", "tts", "distill"}+ assert TRAIN_ON_EOT_AWARE_TASKS == {"sft", "tts", "distill"}++[email protected]("task", ["distill", "sft", "tts"])+def test_reasoning_effort_loads_where_the_formatter_runs(task):+ cfg = load_config_from_string(_yaml(task, _REASONING))+ assert cfg.training.reasoning_effort == "medium"++[email protected]("task", ["distill", "sft", "tts"])+def test_train_on_eot_loads_where_it_is_read(task):+ cfg = load_config_from_string(_yaml(task, _EOT))+ assert cfg.training.train_on_eot is True+++ "task", sorted(set(_ALL_TASKS) - REASONING_EFFORT_AWARE_TASKS)+)+def test_reasoning_effort_refused_everywhere_else(task):+ load_config_from_string(_yaml(task))+ match = rf"reasoning_effort='medium' requires task in .*task='{task}'"+ with pytest.raises(ValueError, match=match):+ load_config_from_string(_yaml(task, _REASONING))++[email protected]("task", sorted(set(_ALL_TASKS) - TRAIN_ON_EOT_AWARE_TASKS))+def test_train_on_eot_refused_everywhere_else(task):+ load_config_from_string(_yaml(task))+ match = rf"train_on_eot=true requires task in .*task='{task}'"+ with pytest.raises(ValueError, match=match):+ load_config_from_string(_yaml(task, _EOT))++[email protected]("task", ["pretrain", "classifier", "reranker", "cross_encoder"])+def test_reasoning_effort_no_longer_loads_on_the_tasks_that_ignored_it(task):+ """The four tasks the old allowlist admitted although none reads it."""+ with pytest.raises(ValueError, match="reasoning_effort"):+ load_config_from_string(_yaml(task, _REASONING))diff --git a/tests/test_v0520.py b/tests/test_v0520.pyindex c5b4c6f1..3c64ca98 100644--- a/tests/test_v0520.py+++ b/tests/test_v0520.py@@ -1054,9 +1054,9 @@ class TestTddReviewGaps:@pytest.mark.parametrize("task", ["grpo", "pretrain", "ppo", "embedding", "tts"])def test_reasoning_effort_task_gate_full_matrix(self, task):- # ``pretrain`` is in the SFT-family allowlist so it should accept.- # All other non-SFT-family tasks must reject.- sft_family = {"sft", "pretrain", "distill", "classifier", "reranker", "cross_encoder"}+ # #806: accepted only where the SFT formatter that injects it runs+ # (sft, tts through sft's setup, distill); pretrain never reads it.+ sft_family = {"sft", "tts", "distill"}# Need backend / modality / data to be valid; for tts we also need family.extra = ""modality = ""