Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#755 — which of your settings does your backend actually read?
Backend support cannot be *inferred*. Three implementations were measured and
rejected before this one (all recorded in the issue):
1. static reachability over the import graph — every trainer's closure is ~150
modules, so it detected **zero of five** independently-known MLX gaps;
2. static reads of the trainer module alone — finds those five, but reports
``train_on_responses_only`` and ``loraplus_lr_ratio`` as missing from the
transformers path, where they live in helper modules;
3. runtime tracing under ``--dry-run`` — ``commands/train.py:1255`` exits before
the trainer wrapper is constructed, so nothing is observed.
So the registry is **declared and reviewed**, and the guard's job is to stop it
drifting — which is the failure that produced #749, where ``max_grad_norm`` was
missing from ``mlx_sft.py``'s hand-maintained warning list while MLX dropped it.
No test here needs a GPU, MLX, the network, or a model download.
"""
from __future__ import annotations
import ast
import pathlib
import re
import textwrap
import pytest
import typer
from soup_cli.config.backend_support import REGISTRY as _REGISTRY
from soup_cli.config.schema import DataConfig, TrainingConfig
from tests._source_scan import memoized_scan
from tests.conftest import strip_ansi
MLX_SFT = pathlib.Path(__file__).resolve().parents[1] / (
"src/soup_cli/trainer/mlx_sft.py"
)
#: Every entry the registry declares as read-only-to-warn. Tests parametrise
#: over this so a new entry is covered without editing an assertion.
_WARNED_ENTRIES = [e for e in _REGISTRY[("sft", "mlx", "text")] if e.trainer_reads]
def _yaml_block(text: str) -> str:
"""Normalise a caller-supplied YAML block to exactly two spaces of indent.
Accepts the block at any consistent depth, so a call site's own
indentation is never load-bearing.
"""
return textwrap.indent(textwrap.dedent(text).strip("\n") + "\n", " ")
@pytest.fixture
def config_at(tmp_path):
"""Write a soup.yaml with the given task/backend and training block."""
def _make(task: str, backend: str, training: str = "", data: str = "") -> str:
train_file = tmp_path / "train.jsonl"
train_file.write_text(
'{"instruction": "a", "output": "b"}\n', encoding="utf-8"
)
# Built by concatenation rather than by interpolating a multi-line block
# into a dedented f-string. In an f-string only the FIRST line of a
# substitution inherits the template's indentation; later lines land
# wherever the caller left them, and ``dedent`` then takes its common
# prefix from the shallowest of those. A caller block whose second line
# was indented differently from its first therefore shredded the whole
# document into a YAML parser error, and the call sites that worked did
# so only because their continuation lines carried the template's own
# indentation literally. ``dedent`` then ``indent`` normalises whatever
# the caller passes, at any consistent depth.
body = textwrap.dedent(
f"""\
base: some-model
task: {task}
backend: {backend}
data:
train: {train_file}
format: alpaca
"""
)
if data:
body += _yaml_block(data)
body += "training:\n" + _yaml_block(training or "epochs: 1")
body += f"output: {tmp_path / 'out'}\n"
path = tmp_path / "soup.yaml"
path.write_text(body, encoding="utf-8")
return str(path)
return _make
# --------------------------------------------------------------------------
# the registry itself
# --------------------------------------------------------------------------
def _unknown_registry_fields(registry) -> list[str]:
"""Registry entries that do not resolve to a real field on the models."""
models = {"training": TrainingConfig, "data": DataConfig}
unknown = []
for entries in registry.values():
for entry in entries:
namespace, _, name = entry.field.partition(".")
if namespace not in models or name not in models[namespace].model_fields:
unknown.append(entry.field)
return unknown
def test_registry_names_only_fields_that_exist():
"""A renamed field must not be able to hide behind a stale entry."""
from soup_cli.config.backend_support import REGISTRY
unknown = _unknown_registry_fields(REGISTRY)
assert unknown == [], f"registry names fields that do not exist: {unknown}"
def test_an_entry_naming_a_field_that_no_longer_exists_is_caught():
"""Acceptance criterion 4 — the check must be able to fail.
Every entry is valid today, so observing the check green proves nothing:
deleting it entirely leaves the suite passing. This pins it against a
renamed field instead.
"""
from soup_cli.config.backend_support import IGNORED, SupportEntry
stale = {
("sft", "mlx"): (
SupportEntry("training.max_grad_norm", IGNORED, "real"),
SupportEntry("training.gone_in_v075", IGNORED, "renamed away"),
SupportEntry("nosuchsection.field", IGNORED, "bad namespace"),
)
}
assert _unknown_registry_fields(stale) == [
"training.gone_in_v075",
"nosuchsection.field",
]
def test_every_registry_entry_carries_a_status_and_a_reason():
from soup_cli.config.backend_support import REGISTRY, STATUSES
bad = [
entry.field
for entries in REGISTRY.values()
for entry in entries
if entry.status not in STATUSES or not entry.reason.strip()
]
assert bad == [], f"entries missing a valid status or a reason: {bad}"
def test_registry_covers_every_field_the_mlx_trainer_warns_about():
"""The drift that produced #749, pinned.
``mlx_sft.py`` names some unsupported settings in prose ("GaLore") and some
by dotted path. Only the dotted ones can be matched mechanically, and those
are the ones this asserts.
"""
from soup_cli.config.backend_support import unsupported_for
tree = ast.parse(MLX_SFT.read_text(encoding="utf-8"))
warned: set[str] = set()
for node in ast.walk(tree):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "append"
):
for arg in ast.walk(node):
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
warned.update(re.findall(r"\b(?:training|data)\.[a-z0-9_]+\b", arg.value))
assert warned, "found no dotted field names in mlx_sft.py's warning list"
known = {
f"{namespace}.{name}"
for namespace, model in (("training", TrainingConfig), ("data", DataConfig))
for name in model.model_fields
}
unknown = sorted(warned - known)
assert unknown == [], f"warning list names fields that do not exist: {unknown}"
registered = {e.field for e in unsupported_for("sft", "mlx")}
# These warnings are internal guards for schema-refused values. Their
# load-time refusal is pinned by test_issue1069 (monitoring flags) and
# test_mlx_use_mod_is_refused_at_load_and_has_no_doctor_row below (use_mod,
# #1752); doctor cannot list them.
schema_refused = {
"training.loss_watchdog", "training.loss_spike_recovery",
"training.grad_accum_auto_tune", "training.use_mod",
}
missing = sorted(warned - registered - schema_refused)
assert missing == [], (
f"mlx_sft.py warns about {missing} but the registry does not list them"
)
@pytest.mark.parametrize(
"field", ["training.use_fsdp2_compile", "training.bnb_4bit_use_double_quant"]
)
def test_warning_guard_preserves_digits_and_catches_missing_entries(tmp_path, monkeypatch, field):
from soup_cli.config import backend_support
from soup_cli.config.backend_support import IGNORED, SupportEntry
entry = SupportEntry(field, IGNORED, "unsupported on MLX")
assert _unknown_registry_fields({("sft", "mlx"): (entry,)}) == []
trainer = tmp_path / "mlx_sft.py"
trainer.write_text(f'ignored.append("{field} is unsupported")\n', encoding="utf-8")
monkeypatch.setitem(globals(), "MLX_SFT", trainer)
monkeypatch.setattr(backend_support, "unsupported_for", lambda task, backend: (entry,))
test_registry_covers_every_field_the_mlx_trainer_warns_about()
monkeypatch.setattr(backend_support, "unsupported_for", lambda task, backend: ())
with pytest.raises(AssertionError, match=re.escape(field)):
test_registry_covers_every_field_the_mlx_trainer_warns_about()
@pytest.mark.parametrize("field", ["training.use_fsdp2_compile2", "data.val_split2"])
def test_warning_guard_rejects_unknown_digit_suffixes(tmp_path, monkeypatch, field):
trainer = tmp_path / "mlx_sft.py"
trainer.write_text(f'ignored.append("{field} is unsupported")\n', encoding="utf-8")
monkeypatch.setitem(globals(), "MLX_SFT", trainer)
with pytest.raises(AssertionError, match="warning list names fields that do not exist"):
test_registry_covers_every_field_the_mlx_trainer_warns_about()
def test_an_unregistered_task_backend_pair_reports_nothing():
from soup_cli.config.backend_support import unsupported_for
assert unsupported_for("sft", "transformers") == ()
assert unsupported_for("no_such_task", "no_such_backend") == ()
# --------------------------------------------------------------------------
# check_config — only what the user set
# --------------------------------------------------------------------------
def test_a_setting_the_backend_ignores_is_reported(config_at):
"""Acceptance criterion 1.
This used to assert on ``training.max_grad_norm``. #750 wired it, so it is
honoured and no longer in the registry -- the guard below is what caught
that. So, later, was ``training.seed``, which MLX honours since #1850.
``training.use_galore`` is the live example now: MLX reads it only to say
it is ignored.
"""
from soup_cli.config.backend_support import check_config
from soup_cli.config.loader import load_config
cfg = load_config(config_at("sft", "mlx", " use_galore: true"))
reported = {e.field for e in check_config(cfg)}
assert "training.use_galore" in reported
def test_new_mlx_gaps_use_liger_and_neftune_alpha_are_reported(config_at):
"""#903: accepted-but-dropped fields must be reported, not all-cleared."""
from soup_cli.config.backend_support import check_config
from soup_cli.config.loader import load_config
cfg = load_config(
config_at("sft", "mlx", " use_liger: true\n neftune_alpha: 5")
)
reported = {e.field for e in check_config(cfg)}
assert {"training.use_liger", "training.neftune_alpha"} <= reported
def test_961_reachable_fields_reported_together(config_at):
"""#961 box 4: several new fields at once -> rows+count.
#961 declared four reachable fields. ``training.quantization_aware`` is no
longer one of them on MLX: ``true`` is refused at load on every backend
(#1222), and ``fp8`` and ``quest`` are refused on mlx. ``training.use_mod``
is refused at load too (#1752). The two that still load are reported
together, and adding either refused field stops the load instead.
"""
from soup_cli.config.backend_support import check_config
from soup_cli.config.loader import load_config
fields = " moe_lora: true\n use_fsdp2_compile: true"
cfg = load_config(config_at("sft", "mlx", fields))
rows = check_config(cfg)
assert {e.field for e in rows} == {
"training.moe_lora",
"training.use_fsdp2_compile",
}
assert len(rows) == 2
with pytest.raises(SystemExit):
load_config(config_at("sft", "mlx", fields + "\n quantization_aware: true"))
def test_mlx_use_mod_is_refused_at_load_and_has_no_doctor_row(config_at):
"""#1752 — ``use_mod: true`` on MLX SFT stops at config load, so doctor can
never see it and the registry lists no row for it (a ``REJECTED`` row would
be unreachable, #1330). ``mlx_sft.py`` keeps its own warning for the field."""
from soup_cli.config.backend_support import unsupported_for
from soup_cli.config.loader import load_config, load_config_from_string
assert "training.use_mod" not in {e.field for e in unsupported_for("sft", "mlx")}
path = config_at("sft", "mlx", " use_mod: true")
with pytest.raises(ValueError) as info:
load_config_from_string(pathlib.Path(path).read_text(encoding="utf-8"))
message = " ".join(str(info.value).split())
assert "training.use_mod is not applied by task='sft' with backend='mlx'" in message, message
assert message.endswith("Use backend: transformers, or set use_mod: false.")
with pytest.raises(SystemExit):
load_config(path)
def test_only_fields_the_user_actually_set_are_reported(config_at):
"""Not all 275 — and not the other MLX gaps the user never touched."""
from soup_cli.config.backend_support import check_config
from soup_cli.config.loader import load_config
cfg = load_config(config_at("sft", "mlx", " use_galore: true"))
reported = {e.field for e in check_config(cfg)}
assert reported == {"training.use_galore"}, (
f"reported fields the user never set: {sorted(reported)}"
)
def test_the_transformers_path_reports_nothing_for_the_same_config(config_at):
from soup_cli.config.backend_support import check_config
from soup_cli.config.loader import load_config
cfg = load_config(config_at("sft", "transformers", " use_galore: true"))
assert check_config(cfg) == []
def test_transformers_is_not_flagged_for_helper_owned_fields(config_at):
"""Acceptance criterion 5 — the false positives implementation 2 produced.
``sft.py`` reads neither ``train_on_responses_only`` (it lives in
``data/sft_format.py``) nor ``loraplus_lr_ratio`` (in the #738 helper), so a
per-module walker called both unsupported. They are supported.
"""
from soup_cli.config.backend_support import check_config
from soup_cli.config.loader import load_config
cfg = load_config(
config_at(
"sft",
"transformers",
" loraplus_lr_ratio: 16.0",
data="train_on_responses_only: true",
)
)
reported = {e.field for e in check_config(cfg)}
assert reported == set(), f"false positives on the transformers path: {reported}"
# --------------------------------------------------------------------------
# the doctor leg
# --------------------------------------------------------------------------
def test_doctor_config_names_the_ignored_field_and_its_reason(
config_at, capsys, monkeypatch
):
"""Rendered at a pinned width.
``doctor`` builds its Console at import, so the row layout follows whatever
terminal the developer happens to have. Asserting on wrapped output tested
the reviewer's terminal as much as the code — it passed on CI and on mine,
and failed at ``COLUMNS=35``. The repo already pins width explicitly
elsewhere (``CliRunner(env={"COLUMNS": "200"})``); this is the same idea for
a directly-called command.
"""
from rich.console import Console
import soup_cli.commands.doctor as doctor_module
from soup_cli.commands.doctor import doctor
monkeypatch.setattr(doctor_module, "console", Console(width=200))
path = config_at("sft", "mlx", " use_galore: true")
# Every typer parameter passed explicitly — see #752.
doctor(nccl=False, disk=False, config=path)
out = strip_ansi(capsys.readouterr().out)
assert "use_galore" in out
assert "no MLX implementation" in out, "the row must carry the reason, not just the name"
@pytest.mark.parametrize(
"path, why",
[
("does_not_exist.yaml", "missing"),
("broken.yaml", "unparseable"),
# load_config raises SystemExit(1) here -- a BaseException, so it walks
# past `except Exception` and used to exit 1, contradicting the 2 in
# docs/commands.md.
("invalid.yaml", "schema-invalid"),
],
)
def test_doctor_exits_non_zero_when_the_config_cannot_be_read(
path, why, tmp_path, monkeypatch
):
"""A leg CI can gate on must not report success on a config it never read."""
from soup_cli.commands.doctor import doctor
monkeypatch.chdir(tmp_path)
if why == "unparseable":
(tmp_path / path).write_text("base: [unclosed\n", encoding="utf-8")
elif why == "schema-invalid":
(tmp_path / path).write_text(
"base: m\ntask: sft\ndata: {train: x.jsonl}\n"
"training: {epochs: -5}\noutput: o\n",
encoding="utf-8",
)
with pytest.raises(typer.Exit) as excinfo:
doctor(nccl=False, disk=False, config=str(tmp_path / path))
# 2 specifically, not merely non-zero: docs/commands.md:218 documents it,
# and all three unreadable shapes must agree.
assert excinfo.value.exit_code == 2
def test_all_clear_states_what_was_checked(config_at, capsys, monkeypatch):
"""#903: the all-clear must name its evidence, not claim a universal."""
from rich.console import Console
import soup_cli.commands.doctor as doctor_module
from soup_cli.commands.doctor import doctor
from soup_cli.config.backend_support import unsupported_for
monkeypatch.setattr(doctor_module, "console", Console(width=200))
path = config_at("sft", "mlx")
doctor(nccl=False, disk=False, config=path)
out = strip_ansi(capsys.readouterr().out)
n = len(unsupported_for("sft", "mlx"))
assert f"None of the {n} setting(s) known to be unread" in out
@pytest.mark.parametrize("width", [35, 60, 200])
def test_the_setting_name_survives_at_any_pinned_width(
width, config_at, capsys, monkeypatch
):
"""A row that says a setting is ignored must say *which* setting.
A dotted field name is one unbreakable word, so Rich ellipsised it to
``training…`` on a narrow terminal — the report named no setting at all.
``overflow="fold"`` keeps it. Widths are pinned, never inherited.
"""
from rich.console import Console
import soup_cli.commands.doctor as doctor_module
from soup_cli.commands.doctor import doctor
monkeypatch.setattr(doctor_module, "console", Console(width=width))
doctor(nccl=False, disk=False, config=config_at("sft", "mlx", " use_galore: true"))
# Scope to the config section: the environment report above it has its own
# tables, and the dependency table legitimately ellipsises at 35 columns.
# Strip ANSI first. Rich reads FORCE_COLOR when the Console is built, so an
# explicitly-constructed console emits colour even though capsys is not a
# tty — and the escapes land *between* the folded halves of the name, so
# collapsing whitespace alone does not rescue the split. conftest's
# strip_ansi is the shared one; 38 files had grown their own copy.
out = strip_ansi(capsys.readouterr().out)
section = out[out.index("Config check"):]
# A folded name is split down the FIRST column across consecutive lines, so
# it has to be rebuilt column-wise. Concatenating the whole section instead
# interleaves the status and reason cells between the halves of the name --
# which is what my first version of this assertion did, and why it failed
# at width 35 while the output was perfectly readable.
first_column = "".join(
line.split("\u2502")[1].strip()
for line in section.splitlines()
if line.startswith("\u2502")
)
assert "training.use_galore" in first_column, (
f"name lost at width={width}; first column read as {first_column!r}"
)
assert "\u2026" not in section, f"a config row was ellipsised at width={width}"
def test_doctor_without_config_does_not_read_one(config_at, capsys):
"""The existing environment-only behaviour must be unchanged."""
from soup_cli.commands.doctor import doctor
doctor(nccl=False, disk=False, config=None)
out = capsys.readouterr().out
assert "Config check" not in out
# --------------------------------------------------------------------------
# the bidirectional guard: the registry cannot silently go stale
# --------------------------------------------------------------------------
def _fields_read_by(path: pathlib.Path) -> set[str]:
"""Attribute names and exact string constants in a module, docstrings out."""
return _names_read_in(path.read_text(encoding="utf-8"))
@memoized_scan
def _names_read_in(source: str) -> set[str]:
"""The scan behind ``_fields_read_by``, run once per distinct module text.
``_registry_drift`` asks for every module under ``src/`` once per registry pair,
and three tests run it on the real tree: twelve parses of the package for one
answer. Keyed on the text, so the synthetic trees the tests below write are each
scanned for what they contain. ``ast.parse`` and not the shared tree, because the
first loop edits what it walks.
"""
tree = ast.parse(source)
for node in ast.walk(tree):
if isinstance(
node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)
):
if (
node.body
and isinstance(node.body[0], ast.Expr)
and isinstance(node.body[0].value, ast.Constant)
and isinstance(node.body[0].value.value, str)
):
node.body = node.body[1:]
names: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Attribute):
names.add(node.attr)
elif isinstance(node, ast.Constant) and isinstance(node.value, str):
names.add(node.value)
return names
def _fields_read_outside(repo_root: pathlib.Path, modules) -> set[str]:
"""Names read by any module in the tree except the ones declared for a pair."""
src = repo_root / "src" / "soup_cli"
declared = {(repo_root / "src" / m).resolve() for m in modules}
names: set[str] = set()
for path in src.rglob("*.py"):
# schema.py declares every field, and config/staged_fields.py lists the
# ones nothing reads (#808): naming a field there is not reading it.
if path.resolve() in declared or path.name in ("schema.py", "staged_fields.py"):
continue
names |= _fields_read_by(path)
return names
def _registry_drift(repo_root: pathlib.Path) -> list[str]:
"""Entries whose ``trainer_reads`` no longer matches the real source.
Scans the **union** of every module declared for the pair. Grounding on the
trainer alone was a hole: #734 wired four schedule fields that live 7-12
times in ``mlx_optim.py`` and twice in ``mlx_sft.py``, and the guard caught
it only because the call site happened to name them.
"""
from soup_cli.config.backend_support import REGISTRY, TRAINER_MODULES
problems: list[str] = []
for pair, entries in REGISTRY.items():
modules = TRAINER_MODULES.get(pair)
if not modules:
problems.append(f"{pair} has entries but no trainer modules declared")
continue
read: dict[str, str] = {}
for module in modules:
path = repo_root / "src" / module
if not path.exists():
problems.append(f"{pair}: declared module {module} does not exist")
continue
for name in _fields_read_by(path):
read.setdefault(name, module)
# A claimed gap must be a gap *relative to something*. If no module in
# the tree reads the field at all, it is not "this backend ignores it"
# -- it is a field nothing consumes anywhere, which is #748's territory
# and #751's allowlist, not this table. Without this, an entry can
# assert an unfounded gap for any field and never be contradicted.
elsewhere = _fields_read_outside(repo_root, modules)
for entry in entries:
name = entry.field.split(".", 1)[1]
if not entry.trainer_reads and name not in elsewhere:
problems.append(
f"{entry.field}: declared a gap on {pair}, but no module "
f"outside {', '.join(modules)} reads it either — that is a "
f"globally unconsumed field (#748), not a per-backend gap"
)
if entry.trainer_reads and name not in read:
problems.append(
f"{entry.field}: declared read-to-warn, but none of "
f"{', '.join(modules)} mentions it"
)
if not entry.trainer_reads and name in read:
problems.append(
f"{entry.field}: declared unread, but {read[name]} now reads "
f"it — the field was wired; remove or reclassify the entry"
)
return problems
def test_the_registry_matches_what_the_backend_trainer_actually_reads():
repo_root = pathlib.Path(__file__).resolve().parents[1]
problems = _registry_drift(repo_root)
assert problems == [], "\n".join(problems)
def test_the_declared_modules_cover_every_helper_the_trainer_imports():
"""The map must not silently shrink back to trainer-only.
Derived from the trainer's own imports rather than hardcoded, so adding a
helper to ``mlx_sft.py`` without declaring it here fails, and dropping one
from the map fails too. Asserting a literal list would pass either way.
"""
from soup_cli.config.backend_support import TRAINER_MODULES
repo_root = pathlib.Path(__file__).resolve().parents[1]
missing: list[str] = []
for pair, modules in TRAINER_MODULES.items():
primary = repo_root / "src" / modules[0]
tree = ast.parse(primary.read_text(encoding="utf-8"))
imported: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module:
if node.module.startswith("soup_cli.trainer"):
imported.add(node.module)
elif isinstance(node, ast.Import):
for alias in node.names:
if alias.name.startswith("soup_cli.trainer"):
imported.add(alias.name)
declared = {m.removesuffix(".py").replace("/", ".") for m in modules}
for mod in sorted(imported):
if mod not in declared:
missing.append(f"{pair}: {modules[0]} imports {mod}, not declared")
assert missing == [], "\n".join(missing)
def test_an_unfounded_gap_claim_is_caught():
"""@MakazhanAlpamys's mutation on #756, which the first version survived.
He fabricated an entry declaring a globally unconsumed field ignored on
``(sft, mlx)`` and the suite stayed green: the drift check only compared
the entry against the declared modules, so an ``ignored`` claim about a
field those modules never mention could not be contradicted.
A gap is relative to something. If nothing anywhere reads the field, it is
not "this backend ignores it" -- it is a field no code consumes, which is
#748's subject and #751's allowlist.
"""
import soup_cli.config.backend_support as bs
repo_root = pathlib.Path(__file__).resolve().parents[1]
# #761 wired data.mask_history, which used to stand in here; the claim only
# has to be about a field nothing consumes, and training.lr_groups is one
# (see #748's KNOWN_UNCONSUMED).
fabricated = bs.SupportEntry(
"training.lr_groups", bs.IGNORED, "fabricated, unfounded claim"
)
real = bs.REGISTRY[("sft", "mlx", "text")]
try:
bs.REGISTRY[("sft", "mlx", "text")] = (fabricated,) + real
problems = _registry_drift(repo_root)
finally:
bs.REGISTRY[("sft", "mlx", "text")] = real
assert any(
"lr_groups" in p and "globally unconsumed" in p for p in problems
), problems
def test_a_gap_for_a_field_another_backend_reads_is_accepted():
"""Control for the above — the check must not fire on a real gap.
``training.use_galore`` is read by the transformers path and ignored by
MLX, so it is exactly the shape the registry exists to record.
"""
repo_root = pathlib.Path(__file__).resolve().parents[1]
assert _registry_drift(repo_root) == []
def test_a_declared_module_that_does_not_exist_is_caught(tmp_path, monkeypatch):
"""A renamed helper must not silently shrink the guard's scope."""
import soup_cli.config.backend_support as bs
monkeypatch.setitem(
bs.TRAINER_MODULES, ("sft", "mlx", "text"), ("soup_cli/trainer/gone.py",)
)
problems = _registry_drift(tmp_path)
assert any("does not exist" in p for p in problems), problems
def test_a_field_wired_only_inside_a_helper_is_caught(tmp_path, monkeypatch):
"""The hole this fix closes.
#734 wired the schedule fields mostly inside ``mlx_optim.py``. Had it left
nothing visible in ``mlx_sft.py``, a trainer-only scan would have passed.
"""
import soup_cli.config.backend_support as bs
trainer = tmp_path / "src" / "soup_cli" / "trainer"
trainer.mkdir(parents=True)
# the trainer never names the field ...
(trainer / "mlx_sft.py").write_text(
"def setup(cfg):\n return build(cfg)\n", encoding="utf-8"
)
# ... the helper it delegates to does the reading
(trainer / "mlx_optim.py").write_text(
"def build(cfg):\n return plan(warmup_ratio=cfg.training.warmup_ratio)\n",
encoding="utf-8",
)
monkeypatch.setitem(
bs.REGISTRY,
("sft", "mlx", "text"),
(bs.SupportEntry("training.warmup_ratio", bs.IGNORED, "stale on purpose"),),
)
monkeypatch.setitem(
bs.TRAINER_MODULES,
("sft", "mlx", "text"),
("soup_cli/trainer/mlx_sft.py", "soup_cli/trainer/mlx_optim.py"),
)
problems = _registry_drift(tmp_path)
assert any("warmup_ratio" in p and "mlx_optim.py" in p for p in problems), problems
def test_wiring_a_field_makes_its_stale_entry_fail(tmp_path, monkeypatch):
"""Acceptance criterion 1, the other direction — and this one really happened.
The registry used to carry ``training.max_grad_norm`` as unread on
``(sft, mlx)``. #750 wired it, this check went red naming it, and the entry
was removed. Every surviving entry is now ``trainer_reads=True``, so there
is no live entry left to prove the direction with — it is pinned against a
synthetic registry rather than quietly dropped, because a direction nothing
exercises is a direction that rots.
"""
import soup_cli.config.backend_support as bs
fake_src = tmp_path / "src" / "soup_cli" / "trainer"
fake_src.mkdir(parents=True)
(fake_src / "mlx_sft.py").write_text(
textwrap.dedent(
"""\
def build(tcfg):
# the shape #750 introduced
return clip(tcfg.max_grad_norm)
"""
),
encoding="utf-8",
)
monkeypatch.setitem(
bs.REGISTRY,
("sft", "mlx", "text"),
(
bs.SupportEntry(
"training.max_grad_norm",
bs.IGNORED,
"stale on purpose: the field is wired now",
),
),
)
monkeypatch.setitem(
bs.TRAINER_MODULES, ("sft", "mlx", "text"), ("soup_cli/trainer/mlx_sft.py",)
)
problems = _registry_drift(tmp_path)
assert any("max_grad_norm" in p and "now reads it" in p for p in problems), problems
@pytest.mark.parametrize(
"entry",
[e for e in _WARNED_ENTRIES],
ids=[e.field for e in _WARNED_ENTRIES],
)
def test_deleting_a_warning_makes_its_entry_fail(entry, tmp_path, monkeypatch):
"""The mirror case: an entry declared read-to-warn whose warning is gone.
Parametrised over every ``trainer_reads=True`` entry rather than pinning
``use_galore``, so this cannot pass by fixture coincidence and a new entry
is covered the moment it is added.
"""
import soup_cli.config.backend_support as bs
trainer = tmp_path / "src" / "soup_cli" / "trainer"
trainer.mkdir(parents=True)
(trainer / "mlx_sft.py").write_text(
"def build(tcfg):\n return None\n", encoding="utf-8"
)
monkeypatch.setitem(bs.REGISTRY, ("sft", "mlx", "text"), (entry,))
monkeypatch.setitem(
bs.TRAINER_MODULES, ("sft", "mlx", "text"), ("soup_cli/trainer/mlx_sft.py",)
)
problems = _registry_drift(tmp_path)
name = entry.field.split(".", 1)[1]
assert any(name in p and "declared read-to-warn" in p for p in problems), problems
class TestTheFixtureDoesNotDependOnCallerIndentation:
"""The `config_at` block must survive any consistent caller indentation.
The fixture used to interpolate a multi-line block into a dedented
f-string, where only the first line inherits the template's indent. A
caller whose second line was indented differently silently produced a
YAML parser error instead of the config under test -- so a test could
fail for a reason that had nothing to do with what it was asserting.
"""
BLOCKS = {
"two-space, the natural spelling": " use_liger: true\n neftune_alpha: 5",
"zero-indent": "use_liger: true\nneftune_alpha: 5",
"deep and consistent": " use_liger: true\n neftune_alpha: 5",
"single line": " use_liger: true",
"trailing newline": " use_liger: true\n neftune_alpha: 5\n",
}
@pytest.mark.parametrize("spelling", sorted(BLOCKS))
def test_every_consistent_indentation_yields_the_same_config(
self, config_at, spelling
):
import yaml
raw = pathlib.Path(
config_at("sft", "mlx", training=self.BLOCKS[spelling])
).read_text(encoding="utf-8")
parsed = yaml.safe_load(raw)
assert set(parsed) == {
"base",
"task",
"backend",
"data",
"training",
"output",
}, raw
expected = (
{"use_liger"}
if spelling == "single line"
else {"use_liger", "neftune_alpha"}
)
assert set(parsed["training"]) == expected, raw
assert parsed["training"]["use_liger"] is True, raw
def test_a_data_block_is_normalised_the_same_way(self, config_at):
import yaml
raw = pathlib.Path(
config_at("sft", "mlx", training="epochs: 1", data="max_length: 128")
).read_text(encoding="utf-8")
parsed = yaml.safe_load(raw)
assert parsed["data"]["max_length"] == 128, raw
assert parsed["data"]["format"] == "alpaca", raw
assert parsed["training"]["epochs"] == 1, raw