Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(config): gate reasoning_effort and train_on_eot on the tasks that read them (#806) (#1587)

Dipak Chaudhari304e62e1d160
diff --git a/changelog.d/0.76.0/1587.fixed.md b/changelog.d/0.76.0/1587.fixed.md
new file mode 100644
index 00000000..962382ec
--- /dev/null
+++ b/changelog.d/0.76.0/1587.fixed.md
@@ -0,0 +1,8 @@
+- **`training.reasoning_effort` and `training.train_on_eot` now load only on the tasks that read them (#806 by @dchaudhari7177 in #1587).**
+ Both are applied by the SFT formatter, which runs on `sft`, `tts` and
+ `distill`. Before, the gate refused `tts`, where both flags take effect,
+ and accepted `pretrain`, `classifier`, `reranker` and `cross_encoder`,
+ where they loaded and changed nothing. Such a config now fails at load.
+ On `pretrain`, `train_on_eot` changed only the `soup data preprocess`
+ cache key, never the cached rows; a pretrain cache built with the flag
+ has to be rebuilt without it.
diff --git a/docs/training.md b/docs/training.md
index 592673bf..00a2b8f4 100644
--- a/docs/training.md
+++ b/docs/training.md
@@ -780,7 +780,7 @@ training:
`reasoning_effort` injects `<|reasoning_effort|>high<|/reasoning_effort|>` into
the system turn (creating one if absent). `train_on_eot=True` makes the model
learn when to stop generating by training on the trailing EOS token instead of
-masking it out. Both are gated to the SFT-family of tasks.
+masking it out. Both load only on `task: sft`, `tts` and `distill`.
## EBFT / GDPO Loss Variants
@@ -2260,7 +2260,7 @@ the cross-entropy. See [EBFT / GDPO Loss Variants](#ebft--gdpo-loss-variants).
## gpt-oss `reasoning_effort` + `train_on_eot` (v0.52.0)
-`training.reasoning_effort: low | medium | high` injects a system-prefix token at training time for gpt-oss models; `training.train_on_eot: true` includes explicit EOT/EOS control tokens in the SFT loss (axolotl `train_on_eot`). Both are gated to the SFT-family task set (`sft` / `pretrain` / `distill` / `classifier` / `reranker` / `cross_encoder`) — setting them on DPO / GRPO / PPO / etc. fails at config load. Live formatter wiring in v0.52.1.
+`training.reasoning_effort: low | medium | high` injects a system-prefix token at training time for gpt-oss models; `training.train_on_eot: true` includes explicit EOT/EOS control tokens in the SFT loss (axolotl `train_on_eot`). Both are applied by the SFT formatter, which runs on `sft`, `tts` and `distill`, so those are the tasks they load on. Setting either on any other task (`pretrain`, DPO, GRPO, PPO, `classifier`, `reranker`, `cross_encoder`) fails at config load. Live formatter wiring in v0.52.1.
## MoLE — Per-Token Adapter Routing (`task='moe_lora_routing'`)
diff --git a/src/soup_cli/config/schema.py b/src/soup_cli/config/schema.py
index d4b74dc2..b171480a 100644
--- a/src/soup_cli/config/schema.py
+++ b/src/soup_cli/config/schema.py
@@ -2150,7 +2150,7 @@ class TrainingConfig(BaseModel):
default=None,
description=(
"gpt-oss train-time reasoning effort level. Injected as a prompt "
- "prefix by the SFT-family data formatter."
+ "prefix by the SFT data formatter; task sft, tts or distill only."
),
)
train_on_eot: bool = Field(
@@ -2158,7 +2158,8 @@ class TrainingConfig(BaseModel):
description=(
"Include explicit EOT / EOS control tokens in the SFT loss "
"(axolotl ``train_on_eot``). Default False matches HF Trainer "
- "convention. (v0.52.0)"
+ "convention. Applied by the SFT data formatter; task sft, tts or "
+ "distill only. (v0.52.0)"
),
)
# ---- v0.53.0 Quant Menu II — UD GGUFs + KV cache + NVFP4 ---------------
@@ -5072,6 +5073,16 @@ SFT_KERNEL_AWARE_TASKS: frozenset[str] = frozenset({"sft", "tts"})
AMP_APPLYING_TASKS: frozenset[str] = frozenset({"sft", "tts"})
+# #806: tasks whose trainers build rows with ``data.sft_format.build_format_row``
+# (sft's text path, inherited by tts, and distill), the only reader of
+# ``training.reasoning_effort``.
+REASONING_EFFORT_AWARE_TASKS: frozenset[str] = frozenset({"sft", "tts", "distill"})
+# #806: ``training.train_on_eot`` has the same reader. pretrain passes it to the
+# pre-tokenized cache too, but a pretrain row is ``labels = input_ids`` whatever
+# the mask mode, so there it changed only the cache key.
+TRAIN_ON_EOT_AWARE_TASKS: frozenset[str] = REASONING_EFFORT_AWARE_TASKS
+
+
# #795: trainers that load the base unquantised and never read
# ``training.quantization``.
_QUANTIZATION_UNHONOURED_TASKS = frozenset({
@@ -6359,34 +6370,35 @@ class SoupConfig(BaseModel):
@model_validator(mode="after")
def _validate_reasoning_effort_task_gate(self) -> "SoupConfig":
- """v0.52.0 Part G (code-review fix) — surface the silent-no-op
- footgun when ``reasoning_effort`` / ``train_on_eot`` is set on a
- task they cannot influence.
-
- Mirrors v0.50.0 ``_validate_grpo_stability_task_gate`` policy.
- ``reasoning_effort`` only makes sense on SFT-family training
- (sft / pretrain / distill / classifier-family) because the SFT
- formatter injects it as a system-prefix token. The other
- tasks (DPO / GRPO / KTO / ORPO / SimPO / IPO / BCO / preference /
- PPO / reward_model / embedding / prm / tts) do not consume it.
-
- ``train_on_eot`` is an SFT loss-mask flag; setting it on
- DPO/GRPO/etc. is a silent no-op.
+ """Refuse ``reasoning_effort`` / ``train_on_eot`` on a task that never reads them.
+
+ Both are applied by ``data.sft_format.build_format_row``, whose trainer
+ callers are sft (its text path, inherited by tts's
+ ``TTSTrainerWrapper.setup``) and distill (#806). pretrain, classifier,
+ reranker and cross_encoder never call it, so the flags loaded there and
+ changed nothing. pretrain also passes ``train_on_eot`` to the
+ ``soup data preprocess`` cache key (#1054), but its rows are
+ ``labels = input_ids`` either way, so the flag is refused there too.
"""
tcfg = self.training
- sft_family_tasks = {
- "sft", "pretrain", "distill",
- "classifier", "reranker", "cross_encoder",
- }
- if tcfg.reasoning_effort is not None and self.task not in sft_family_tasks:
+ if (
+ tcfg.reasoning_effort is not None
+ and self.task not in REASONING_EFFORT_AWARE_TASKS
+ ):
raise ValueError(
f"training.reasoning_effort={tcfg.reasoning_effort!r} requires "
- f"task in {sorted(sft_family_tasks)}; got task={self.task!r}"
+ f"task in {sorted(REASONING_EFFORT_AWARE_TASKS)}; got task={self.task!r}. "
+ "Only those tasks build their rows with the SFT formatter that "
+ "injects it, so elsewhere it would be ignored. Use task: sft, "
+ "tts or distill, or remove training.reasoning_effort."
)
- if tcfg.train_on_eot and self.task not in sft_family_tasks:
+ if tcfg.train_on_eot and self.task not in TRAIN_ON_EOT_AWARE_TASKS:
raise ValueError(
f"training.train_on_eot=true requires task in "
- f"{sorted(sft_family_tasks)}; got task={self.task!r}"
+ f"{sorted(TRAIN_ON_EOT_AWARE_TASKS)}; got task={self.task!r}. "
+ "Only those tasks build their rows with the SFT formatter that "
+ "applies the loss mask, so elsewhere it would be ignored. Use "
+ "task: sft, tts or distill, or remove training.train_on_eot."
)
return self
diff --git a/src/soup_cli/trainer/pretrain.py b/src/soup_cli/trainer/pretrain.py
index 20c3d786..e3b9fcd6 100644
--- a/src/soup_cli/trainer/pretrain.py
+++ b/src/soup_cli/trainer/pretrain.py
@@ -131,11 +131,9 @@ class PretrainTrainerWrapper:
# via `soup data preprocess`. Skips the raw-text load entirely.
from soup_cli.trainer.sft import _maybe_load_pretokenized
- # #1054: pass ``tcfg`` -- ``preprocess_mask_mode`` reads
- # ``training.train_on_eot`` from it, and ``task: pretrain`` is in the
- # sft-family set the schema allows that flag on. Omitting it dropped the
- # ``+eot`` suffix here but not in ``soup data preprocess``, so the cache
- # was refused by a hash the re-run advised in the error reproduces.
+ # #1054: pass ``tcfg`` so the cache key is derived the same way as in
+ # ``soup data preprocess``. The schema refuses ``training.train_on_eot``
+ # on ``task: pretrain`` (#806), so the mask mode here is the default.
pretok = _maybe_load_pretokenized(
cfg.data, cfg.base, console, getattr(cfg, "training", None),
task=cfg.task,
diff --git a/tests/test_issue1054_pretokenized_labels.py b/tests/test_issue1054_pretokenized_labels.py
index 2b34d038..41d2e61c 100644
--- a/tests/test_issue1054_pretokenized_labels.py
+++ b/tests/test_issue1054_pretokenized_labels.py
@@ -706,45 +706,36 @@ class TestCacheKeyCoversMaskMode:
train_ds, _ = loaded
assert len(train_ds) == 1
- def test_train_on_eot_cache_loads_through_the_pretrain_caller(
+ def test_pretrain_refuses_train_on_eot_before_writing_a_cache(
self, tmp_path, monkeypatch
):
- """``task: pretrain`` + ``train_on_eot: true`` is a config the schema
- allows (pretrain is in the sft-family set). ``trainer/pretrain.py`` must
- derive the same mask mode ``soup data preprocess`` wrote, or the cache is
- unloadable and re-running preprocess as the error advises regenerates the
- very same rejected key.
-
- regression: must fail without the fix -- ``pretrain.py`` called the gate
- without ``tcfg``, dropping the ``+eot`` suffix on its side only.
+ """#806: a ``task: pretrain`` cache records ``labels = input_ids`` whatever
+ the mask mode, so ``train_on_eot`` changed only the cache key there. The
+ schema refuses it on pretrain, so ``soup data preprocess`` stops at load
+ and no ``+eot`` pretrain cache is written.
"""
- from rich.console import Console
+ import re
- from soup_cli.config.schema import DataConfig
- from soup_cli.trainer.sft import _maybe_load_pretokenized
+ from typer.testing import CliRunner
- tok = _tokenizer()
- cache_dir = _run_preprocess(
- tmp_path,
- monkeypatch,
- tok,
- task="pretrain",
- rows=[{"text": "You are terse ."}],
- data_extra="training:\n train_on_eot: true\n",
- )
- dcfg = DataConfig(
- train="./d.jsonl",
- format="pre_tokenized",
- tokenized_path=str(cache_dir.relative_to(tmp_path)),
- max_length=128,
- )
- tcfg = SimpleNamespace(train_on_eot=True)
- # Exactly what trainer/pretrain.py now passes.
- loaded = _maybe_load_pretokenized(dcfg, "x/y", Console(), tcfg, task="pretrain")
- assert loaded is not None
- train_ds, _ = loaded
- assert len(train_ds) == 1
+ from soup_cli.cli import app
+ monkeypatch.chdir(tmp_path)
+ (tmp_path / "soup.yaml").write_text(
+ "base: x/y\ntask: pretrain\n"
+ "data:\n train: ./d.jsonl\n format: chatml\n max_length: 128\n"
+ "training:\n train_on_eot: true\n",
+ encoding="utf-8",
+ )
+ (tmp_path / "d.jsonl").write_text("{}\n", encoding="utf-8")
+ result = CliRunner().invoke(app, ["data", "preprocess", "soup.yaml", "--yes"])
+ output = " ".join(re.sub(r"\x1b\[[0-9;]*m", "", result.output).split())
+ assert result.exit_code == 1, (result.output, repr(result.exception))
+ assert "train_on_eot=true requires task in" in output
+ assert "got task='pretrain'" in output
+ # Refused at load: preprocess never reached its own report or wrote a cache.
+ assert "Loss mask:" not in output
+ assert not (tmp_path / ".soup-tokenized").exists()
class TestSampleCount:
def test_pretokenized_count_comes_from_the_cache(self, tmp_path):
diff --git a/tests/test_issue806_format_row_task_gates.py b/tests/test_issue806_format_row_task_gates.py
new file mode 100644
index 00000000..d77d4542
--- /dev/null
+++ b/tests/test_issue806_format_row_task_gates.py
@@ -0,0 +1,79 @@
+"""Issue #806 (remainder): ``reasoning_effort`` / ``train_on_eot`` only where they are read.
+
+Both are applied by ``data.sft_format.build_format_row``. Its trainer callers
+are sft's text path (inherited by tts through ``TTSTrainerWrapper.setup``) and
+distill; pretrain and the classifier family never call it, so the flags loaded
+there and changed nothing, while tts, which does honour them, was refused.
+pretrain also keys its pre-tokenized cache on ``train_on_eot`` (#1054), but its
+rows are ``labels = input_ids`` either way, so it is refused there too.
+
+Every refused config is first loaded without the flag, so a refusal can only
+come from the flag itself.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from soup_cli.config.loader import load_config_from_string
+from soup_cli.config.schema import REASONING_EFFORT_AWARE_TASKS, TRAIN_ON_EOT_AWARE_TASKS
+from tests.test_issue1325_expand_layers_scope import _ALL_TASKS, _DATA, _ROOT, _TRAINING
+
+_REASONING = " reasoning_effort: medium\n"
+_EOT = " train_on_eot: true\n"
+
+
+def _yaml(task, flag=""):
+ return (
+ "base: meta-llama/Llama-3.1-8B\n"
+ f"task: {task}\n"
+ "backend: transformers\n"
+ f"{_ROOT.get(task, '')}"
+ "data:\n train: data.jsonl\n"
+ + _DATA.get(task, "")
+ + "training:\n quantization: none\n"
+ + _TRAINING.get(task, "")
+ + flag
+ )
+
+
+def test_the_allowlists_match_the_consumers():
+ assert REASONING_EFFORT_AWARE_TASKS == {"sft", "tts", "distill"}
+ assert TRAIN_ON_EOT_AWARE_TASKS == {"sft", "tts", "distill"}
+
+
[email protected]("task", ["distill", "sft", "tts"])
+def test_reasoning_effort_loads_where_the_formatter_runs(task):
+ cfg = load_config_from_string(_yaml(task, _REASONING))
+ assert cfg.training.reasoning_effort == "medium"
+
+
[email protected]("task", ["distill", "sft", "tts"])
+def test_train_on_eot_loads_where_it_is_read(task):
+ cfg = load_config_from_string(_yaml(task, _EOT))
+ assert cfg.training.train_on_eot is True
+
+
+ "task", sorted(set(_ALL_TASKS) - REASONING_EFFORT_AWARE_TASKS)
+)
+def test_reasoning_effort_refused_everywhere_else(task):
+ load_config_from_string(_yaml(task))
+ match = rf"reasoning_effort='medium' requires task in .*task='{task}'"
+ with pytest.raises(ValueError, match=match):
+ load_config_from_string(_yaml(task, _REASONING))
+
+
[email protected]("task", sorted(set(_ALL_TASKS) - TRAIN_ON_EOT_AWARE_TASKS))
+def test_train_on_eot_refused_everywhere_else(task):
+ load_config_from_string(_yaml(task))
+ match = rf"train_on_eot=true requires task in .*task='{task}'"
+ with pytest.raises(ValueError, match=match):
+ load_config_from_string(_yaml(task, _EOT))
+
+
[email protected]("task", ["pretrain", "classifier", "reranker", "cross_encoder"])
+def test_reasoning_effort_no_longer_loads_on_the_tasks_that_ignored_it(task):
+ """The four tasks the old allowlist admitted although none reads it."""
+ with pytest.raises(ValueError, match="reasoning_effort"):
+ load_config_from_string(_yaml(task, _REASONING))
diff --git a/tests/test_v0520.py b/tests/test_v0520.py
index c5b4c6f1..3c64ca98 100644
--- a/tests/test_v0520.py
+++ b/tests/test_v0520.py
@@ -1054,9 +1054,9 @@ class TestTddReviewGaps:
@pytest.mark.parametrize("task", ["grpo", "pretrain", "ppo", "embedding", "tts"])
def test_reasoning_effort_task_gate_full_matrix(self, task):
- # ``pretrain`` is in the SFT-family allowlist so it should accept.
- # All other non-SFT-family tasks must reject.
- sft_family = {"sft", "pretrain", "distill", "classifier", "reranker", "cross_encoder"}
+ # #806: accepted only where the SFT formatter that injects it runs
+ # (sft, tts through sft's setup, distill); pretrain never reads it.
+ sft_family = {"sft", "tts", "distill"}
# Need backend / modality / data to be valid; for tts we also need family.
extra = ""
modality = ""