Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

34.1 KB
Raw Blame
"""#755 — which of your settings does your backend actually read?

Backend support cannot be *inferred*. Three implementations were measured and
rejected before this one (all recorded in the issue):

1. static reachability over the import graph — every trainer's closure is ~150
   modules, so it detected **zero of five** independently-known MLX gaps;
2. static reads of the trainer module alone — finds those five, but reports
   ``train_on_responses_only`` and ``loraplus_lr_ratio`` as missing from the
   transformers path, where they live in helper modules;
3. runtime tracing under ``--dry-run`` — ``commands/train.py:1255`` exits before
   the trainer wrapper is constructed, so nothing is observed.

So the registry is **declared and reviewed**, and the guard's job is to stop it
drifting — which is the failure that produced #749, where ``max_grad_norm`` was
missing from ``mlx_sft.py``'s hand-maintained warning list while MLX dropped it.

No test here needs a GPU, MLX, the network, or a model download.
"""

from __future__ import annotations

import ast
import pathlib
import re
import textwrap

import pytest
import typer

from soup_cli.config.backend_support import REGISTRY as _REGISTRY
from soup_cli.config.schema import DataConfig, TrainingConfig
from tests._source_scan import memoized_scan
from tests.conftest import strip_ansi

MLX_SFT = pathlib.Path(__file__).resolve().parents[1] / (
    "src/soup_cli/trainer/mlx_sft.py"
)

#: Every entry the registry declares as read-only-to-warn. Tests parametrise
#: over this so a new entry is covered without editing an assertion.
_WARNED_ENTRIES = [e for e in _REGISTRY[("sft", "mlx", "text")] if e.trainer_reads]


def _yaml_block(text: str) -> str:
    """Normalise a caller-supplied YAML block to exactly two spaces of indent.

    Accepts the block at any consistent depth, so a call site's own
    indentation is never load-bearing.
    """
    return textwrap.indent(textwrap.dedent(text).strip("\n") + "\n", "  ")


@pytest.fixture
def config_at(tmp_path):
    """Write a soup.yaml with the given task/backend and training block."""

    def _make(task: str, backend: str, training: str = "", data: str = "") -> str:
        train_file = tmp_path / "train.jsonl"
        train_file.write_text(
            '{"instruction": "a", "output": "b"}\n', encoding="utf-8"
        )
        # Built by concatenation rather than by interpolating a multi-line block
        # into a dedented f-string. In an f-string only the FIRST line of a
        # substitution inherits the template's indentation; later lines land
        # wherever the caller left them, and ``dedent`` then takes its common
        # prefix from the shallowest of those. A caller block whose second line
        # was indented differently from its first therefore shredded the whole
        # document into a YAML parser error, and the call sites that worked did
        # so only because their continuation lines carried the template's own
        # indentation literally. ``dedent`` then ``indent`` normalises whatever
        # the caller passes, at any consistent depth.
        body = textwrap.dedent(
            f"""\
            base: some-model
            task: {task}
            backend: {backend}
            data:
              train: {train_file}
              format: alpaca
            """
        )
        if data:
            body += _yaml_block(data)
        body += "training:\n" + _yaml_block(training or "epochs: 1")
        body += f"output: {tmp_path / 'out'}\n"
        path = tmp_path / "soup.yaml"
        path.write_text(body, encoding="utf-8")
        return str(path)

    return _make


# --------------------------------------------------------------------------
# the registry itself
# --------------------------------------------------------------------------

def _unknown_registry_fields(registry) -> list[str]:
    """Registry entries that do not resolve to a real field on the models."""
    models = {"training": TrainingConfig, "data": DataConfig}
    unknown = []
    for entries in registry.values():
        for entry in entries:
            namespace, _, name = entry.field.partition(".")
            if namespace not in models or name not in models[namespace].model_fields:
                unknown.append(entry.field)
    return unknown


def test_registry_names_only_fields_that_exist():
    """A renamed field must not be able to hide behind a stale entry."""
    from soup_cli.config.backend_support import REGISTRY

    unknown = _unknown_registry_fields(REGISTRY)
    assert unknown == [], f"registry names fields that do not exist: {unknown}"


def test_an_entry_naming_a_field_that_no_longer_exists_is_caught():
    """Acceptance criterion 4 — the check must be able to fail.

    Every entry is valid today, so observing the check green proves nothing:
    deleting it entirely leaves the suite passing. This pins it against a
    renamed field instead.
    """
    from soup_cli.config.backend_support import IGNORED, SupportEntry

    stale = {
        ("sft", "mlx"): (
            SupportEntry("training.max_grad_norm", IGNORED, "real"),
            SupportEntry("training.gone_in_v075", IGNORED, "renamed away"),
            SupportEntry("nosuchsection.field", IGNORED, "bad namespace"),
        )
    }
    assert _unknown_registry_fields(stale) == [
        "training.gone_in_v075",
        "nosuchsection.field",
    ]


def test_every_registry_entry_carries_a_status_and_a_reason():
    from soup_cli.config.backend_support import REGISTRY, STATUSES

    bad = [
        entry.field
        for entries in REGISTRY.values()
        for entry in entries
        if entry.status not in STATUSES or not entry.reason.strip()
    ]
    assert bad == [], f"entries missing a valid status or a reason: {bad}"


def test_registry_covers_every_field_the_mlx_trainer_warns_about():
    """The drift that produced #749, pinned.

    ``mlx_sft.py`` names some unsupported settings in prose ("GaLore") and some
    by dotted path. Only the dotted ones can be matched mechanically, and those
    are the ones this asserts.
    """
    from soup_cli.config.backend_support import unsupported_for

    tree = ast.parse(MLX_SFT.read_text(encoding="utf-8"))
    warned: set[str] = set()
    for node in ast.walk(tree):
        if (
            isinstance(node, ast.Call)
            and isinstance(node.func, ast.Attribute)
            and node.func.attr == "append"
        ):
            for arg in ast.walk(node):
                if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
                    warned.update(re.findall(r"\b(?:training|data)\.[a-z0-9_]+\b", arg.value))

    assert warned, "found no dotted field names in mlx_sft.py's warning list"
    known = {
        f"{namespace}.{name}"
        for namespace, model in (("training", TrainingConfig), ("data", DataConfig))
        for name in model.model_fields
    }
    unknown = sorted(warned - known)
    assert unknown == [], f"warning list names fields that do not exist: {unknown}"
    registered = {e.field for e in unsupported_for("sft", "mlx")}
    # These warnings are internal guards for schema-refused values. Their
    # load-time refusal is pinned by test_issue1069 (monitoring flags) and
    # test_mlx_use_mod_is_refused_at_load_and_has_no_doctor_row below (use_mod,
    # #1752); doctor cannot list them.
    schema_refused = {
        "training.loss_watchdog", "training.loss_spike_recovery",
        "training.grad_accum_auto_tune", "training.use_mod",
    }
    missing = sorted(warned - registered - schema_refused)
    assert missing == [], (
        f"mlx_sft.py warns about {missing} but the registry does not list them"
    )


@pytest.mark.parametrize(
    "field", ["training.use_fsdp2_compile", "training.bnb_4bit_use_double_quant"]
)
def test_warning_guard_preserves_digits_and_catches_missing_entries(tmp_path, monkeypatch, field):
    from soup_cli.config import backend_support
    from soup_cli.config.backend_support import IGNORED, SupportEntry

    entry = SupportEntry(field, IGNORED, "unsupported on MLX")
    assert _unknown_registry_fields({("sft", "mlx"): (entry,)}) == []
    trainer = tmp_path / "mlx_sft.py"
    trainer.write_text(f'ignored.append("{field} is unsupported")\n', encoding="utf-8")
    monkeypatch.setitem(globals(), "MLX_SFT", trainer)
    monkeypatch.setattr(backend_support, "unsupported_for", lambda task, backend: (entry,))
    test_registry_covers_every_field_the_mlx_trainer_warns_about()

    monkeypatch.setattr(backend_support, "unsupported_for", lambda task, backend: ())
    with pytest.raises(AssertionError, match=re.escape(field)):
        test_registry_covers_every_field_the_mlx_trainer_warns_about()


@pytest.mark.parametrize("field", ["training.use_fsdp2_compile2", "data.val_split2"])
def test_warning_guard_rejects_unknown_digit_suffixes(tmp_path, monkeypatch, field):
    trainer = tmp_path / "mlx_sft.py"
    trainer.write_text(f'ignored.append("{field} is unsupported")\n', encoding="utf-8")
    monkeypatch.setitem(globals(), "MLX_SFT", trainer)
    with pytest.raises(AssertionError, match="warning list names fields that do not exist"):
        test_registry_covers_every_field_the_mlx_trainer_warns_about()


def test_an_unregistered_task_backend_pair_reports_nothing():
    from soup_cli.config.backend_support import unsupported_for

    assert unsupported_for("sft", "transformers") == ()
    assert unsupported_for("no_such_task", "no_such_backend") == ()


# --------------------------------------------------------------------------
# check_config — only what the user set
# --------------------------------------------------------------------------

def test_a_setting_the_backend_ignores_is_reported(config_at):
    """Acceptance criterion 1.

    This used to assert on ``training.max_grad_norm``. #750 wired it, so it is
    honoured and no longer in the registry -- the guard below is what caught
    that. So, later, was ``training.seed``, which MLX honours since #1850.
    ``training.use_galore`` is the live example now: MLX reads it only to say
    it is ignored.
    """
    from soup_cli.config.backend_support import check_config
    from soup_cli.config.loader import load_config

    cfg = load_config(config_at("sft", "mlx", "  use_galore: true"))
    reported = {e.field for e in check_config(cfg)}
    assert "training.use_galore" in reported


def test_new_mlx_gaps_use_liger_and_neftune_alpha_are_reported(config_at):
    """#903: accepted-but-dropped fields must be reported, not all-cleared."""
    from soup_cli.config.backend_support import check_config
    from soup_cli.config.loader import load_config

    cfg = load_config(
        config_at("sft", "mlx", "  use_liger: true\n  neftune_alpha: 5")
    )
    reported = {e.field for e in check_config(cfg)}
    assert {"training.use_liger", "training.neftune_alpha"} <= reported


def test_961_reachable_fields_reported_together(config_at):
    """#961 box 4: several new fields at once -> rows+count.

    #961 declared four reachable fields. ``training.quantization_aware`` is no
    longer one of them on MLX: ``true`` is refused at load on every backend
    (#1222), and ``fp8`` and ``quest`` are refused on mlx. ``training.use_mod``
    is refused at load too (#1752). The two that still load are reported
    together, and adding either refused field stops the load instead.
    """
    from soup_cli.config.backend_support import check_config
    from soup_cli.config.loader import load_config

    fields = "  moe_lora: true\n  use_fsdp2_compile: true"
    cfg = load_config(config_at("sft", "mlx", fields))
    rows = check_config(cfg)
    assert {e.field for e in rows} == {
        "training.moe_lora",
        "training.use_fsdp2_compile",
    }
    assert len(rows) == 2

    with pytest.raises(SystemExit):
        load_config(config_at("sft", "mlx", fields + "\n  quantization_aware: true"))


def test_mlx_use_mod_is_refused_at_load_and_has_no_doctor_row(config_at):
    """#1752 — ``use_mod: true`` on MLX SFT stops at config load, so doctor can
    never see it and the registry lists no row for it (a ``REJECTED`` row would
    be unreachable, #1330). ``mlx_sft.py`` keeps its own warning for the field."""
    from soup_cli.config.backend_support import unsupported_for
    from soup_cli.config.loader import load_config, load_config_from_string

    assert "training.use_mod" not in {e.field for e in unsupported_for("sft", "mlx")}

    path = config_at("sft", "mlx", "  use_mod: true")
    with pytest.raises(ValueError) as info:
        load_config_from_string(pathlib.Path(path).read_text(encoding="utf-8"))
    message = " ".join(str(info.value).split())
    assert "training.use_mod is not applied by task='sft' with backend='mlx'" in message, message
    assert message.endswith("Use backend: transformers, or set use_mod: false.")
    with pytest.raises(SystemExit):
        load_config(path)


def test_only_fields_the_user_actually_set_are_reported(config_at):
    """Not all 275 — and not the other MLX gaps the user never touched."""
    from soup_cli.config.backend_support import check_config
    from soup_cli.config.loader import load_config

    cfg = load_config(config_at("sft", "mlx", "  use_galore: true"))
    reported = {e.field for e in check_config(cfg)}
    assert reported == {"training.use_galore"}, (
        f"reported fields the user never set: {sorted(reported)}"
    )


def test_the_transformers_path_reports_nothing_for_the_same_config(config_at):
    from soup_cli.config.backend_support import check_config
    from soup_cli.config.loader import load_config

    cfg = load_config(config_at("sft", "transformers", "  use_galore: true"))
    assert check_config(cfg) == []


def test_transformers_is_not_flagged_for_helper_owned_fields(config_at):
    """Acceptance criterion 5 — the false positives implementation 2 produced.

    ``sft.py`` reads neither ``train_on_responses_only`` (it lives in
    ``data/sft_format.py``) nor ``loraplus_lr_ratio`` (in the #738 helper), so a
    per-module walker called both unsupported. They are supported.
    """
    from soup_cli.config.backend_support import check_config
    from soup_cli.config.loader import load_config

    cfg = load_config(
        config_at(
            "sft",
            "transformers",
            "  loraplus_lr_ratio: 16.0",
            data="train_on_responses_only: true",
        )
    )
    reported = {e.field for e in check_config(cfg)}
    assert reported == set(), f"false positives on the transformers path: {reported}"


# --------------------------------------------------------------------------
# the doctor leg
# --------------------------------------------------------------------------

def test_doctor_config_names_the_ignored_field_and_its_reason(
    config_at, capsys, monkeypatch
):
    """Rendered at a pinned width.

    ``doctor`` builds its Console at import, so the row layout follows whatever
    terminal the developer happens to have. Asserting on wrapped output tested
    the reviewer's terminal as much as the code — it passed on CI and on mine,
    and failed at ``COLUMNS=35``. The repo already pins width explicitly
    elsewhere (``CliRunner(env={"COLUMNS": "200"})``); this is the same idea for
    a directly-called command.
    """
    from rich.console import Console

    import soup_cli.commands.doctor as doctor_module
    from soup_cli.commands.doctor import doctor

    monkeypatch.setattr(doctor_module, "console", Console(width=200))
    path = config_at("sft", "mlx", "  use_galore: true")
    # Every typer parameter passed explicitly — see #752.
    doctor(nccl=False, disk=False, config=path)

    out = strip_ansi(capsys.readouterr().out)
    assert "use_galore" in out
    assert "no MLX implementation" in out, "the row must carry the reason, not just the name"


@pytest.mark.parametrize(
    "path, why",
    [
        ("does_not_exist.yaml", "missing"),
        ("broken.yaml", "unparseable"),
        # load_config raises SystemExit(1) here -- a BaseException, so it walks
        # past `except Exception` and used to exit 1, contradicting the 2 in
        # docs/commands.md.
        ("invalid.yaml", "schema-invalid"),
    ],
)
def test_doctor_exits_non_zero_when_the_config_cannot_be_read(
    path, why, tmp_path, monkeypatch
):
    """A leg CI can gate on must not report success on a config it never read."""
    from soup_cli.commands.doctor import doctor

    monkeypatch.chdir(tmp_path)
    if why == "unparseable":
        (tmp_path / path).write_text("base: [unclosed\n", encoding="utf-8")
    elif why == "schema-invalid":
        (tmp_path / path).write_text(
            "base: m\ntask: sft\ndata: {train: x.jsonl}\n"
            "training: {epochs: -5}\noutput: o\n",
            encoding="utf-8",
        )

    with pytest.raises(typer.Exit) as excinfo:
        doctor(nccl=False, disk=False, config=str(tmp_path / path))
    # 2 specifically, not merely non-zero: docs/commands.md:218 documents it,
    # and all three unreadable shapes must agree.
    assert excinfo.value.exit_code == 2


def test_all_clear_states_what_was_checked(config_at, capsys, monkeypatch):
    """#903: the all-clear must name its evidence, not claim a universal."""
    from rich.console import Console

    import soup_cli.commands.doctor as doctor_module
    from soup_cli.commands.doctor import doctor
    from soup_cli.config.backend_support import unsupported_for

    monkeypatch.setattr(doctor_module, "console", Console(width=200))
    path = config_at("sft", "mlx")
    doctor(nccl=False, disk=False, config=path)

    out = strip_ansi(capsys.readouterr().out)
    n = len(unsupported_for("sft", "mlx"))
    assert f"None of the {n} setting(s) known to be unread" in out


@pytest.mark.parametrize("width", [35, 60, 200])
def test_the_setting_name_survives_at_any_pinned_width(
    width, config_at, capsys, monkeypatch
):
    """A row that says a setting is ignored must say *which* setting.

    A dotted field name is one unbreakable word, so Rich ellipsised it to
    ``training…`` on a narrow terminal — the report named no setting at all.
    ``overflow="fold"`` keeps it. Widths are pinned, never inherited.
    """
    from rich.console import Console

    import soup_cli.commands.doctor as doctor_module
    from soup_cli.commands.doctor import doctor

    monkeypatch.setattr(doctor_module, "console", Console(width=width))
    doctor(nccl=False, disk=False, config=config_at("sft", "mlx", "  use_galore: true"))

    # Scope to the config section: the environment report above it has its own
    # tables, and the dependency table legitimately ellipsises at 35 columns.
    # Strip ANSI first. Rich reads FORCE_COLOR when the Console is built, so an
    # explicitly-constructed console emits colour even though capsys is not a
    # tty — and the escapes land *between* the folded halves of the name, so
    # collapsing whitespace alone does not rescue the split. conftest's
    # strip_ansi is the shared one; 38 files had grown their own copy.
    out = strip_ansi(capsys.readouterr().out)
    section = out[out.index("Config check"):]

    # A folded name is split down the FIRST column across consecutive lines, so
    # it has to be rebuilt column-wise. Concatenating the whole section instead
    # interleaves the status and reason cells between the halves of the name --
    # which is what my first version of this assertion did, and why it failed
    # at width 35 while the output was perfectly readable.
    first_column = "".join(
        line.split("\u2502")[1].strip()
        for line in section.splitlines()
        if line.startswith("\u2502")
    )
    assert "training.use_galore" in first_column, (
        f"name lost at width={width}; first column read as {first_column!r}"
    )
    assert "\u2026" not in section, f"a config row was ellipsised at width={width}"


def test_doctor_without_config_does_not_read_one(config_at, capsys):
    """The existing environment-only behaviour must be unchanged."""
    from soup_cli.commands.doctor import doctor

    doctor(nccl=False, disk=False, config=None)
    out = capsys.readouterr().out
    assert "Config check" not in out


# --------------------------------------------------------------------------
# the bidirectional guard: the registry cannot silently go stale
# --------------------------------------------------------------------------

def _fields_read_by(path: pathlib.Path) -> set[str]:
    """Attribute names and exact string constants in a module, docstrings out."""
    return _names_read_in(path.read_text(encoding="utf-8"))


@memoized_scan
def _names_read_in(source: str) -> set[str]:
    """The scan behind ``_fields_read_by``, run once per distinct module text.

    ``_registry_drift`` asks for every module under ``src/`` once per registry pair,
    and three tests run it on the real tree: twelve parses of the package for one
    answer. Keyed on the text, so the synthetic trees the tests below write are each
    scanned for what they contain. ``ast.parse`` and not the shared tree, because the
    first loop edits what it walks.
    """
    tree = ast.parse(source)
    for node in ast.walk(tree):
        if isinstance(
            node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)
        ):
            if (
                node.body
                and isinstance(node.body[0], ast.Expr)
                and isinstance(node.body[0].value, ast.Constant)
                and isinstance(node.body[0].value.value, str)
            ):
                node.body = node.body[1:]
    names: set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Attribute):
            names.add(node.attr)
        elif isinstance(node, ast.Constant) and isinstance(node.value, str):
            names.add(node.value)
    return names


def _fields_read_outside(repo_root: pathlib.Path, modules) -> set[str]:
    """Names read by any module in the tree except the ones declared for a pair."""
    src = repo_root / "src" / "soup_cli"
    declared = {(repo_root / "src" / m).resolve() for m in modules}
    names: set[str] = set()
    for path in src.rglob("*.py"):
        # schema.py declares every field, and config/staged_fields.py lists the
        # ones nothing reads (#808): naming a field there is not reading it.
        if path.resolve() in declared or path.name in ("schema.py", "staged_fields.py"):
            continue
        names |= _fields_read_by(path)
    return names


def _registry_drift(repo_root: pathlib.Path) -> list[str]:
    """Entries whose ``trainer_reads`` no longer matches the real source.

    Scans the **union** of every module declared for the pair. Grounding on the
    trainer alone was a hole: #734 wired four schedule fields that live 7-12
    times in ``mlx_optim.py`` and twice in ``mlx_sft.py``, and the guard caught
    it only because the call site happened to name them.
    """
    from soup_cli.config.backend_support import REGISTRY, TRAINER_MODULES

    problems: list[str] = []
    for pair, entries in REGISTRY.items():
        modules = TRAINER_MODULES.get(pair)
        if not modules:
            problems.append(f"{pair} has entries but no trainer modules declared")
            continue
        read: dict[str, str] = {}
        for module in modules:
            path = repo_root / "src" / module
            if not path.exists():
                problems.append(f"{pair}: declared module {module} does not exist")
                continue
            for name in _fields_read_by(path):
                read.setdefault(name, module)
        # A claimed gap must be a gap *relative to something*. If no module in
        # the tree reads the field at all, it is not "this backend ignores it"
        # -- it is a field nothing consumes anywhere, which is #748's territory
        # and #751's allowlist, not this table. Without this, an entry can
        # assert an unfounded gap for any field and never be contradicted.
        elsewhere = _fields_read_outside(repo_root, modules)
        for entry in entries:
            name = entry.field.split(".", 1)[1]
            if not entry.trainer_reads and name not in elsewhere:
                problems.append(
                    f"{entry.field}: declared a gap on {pair}, but no module "
                    f"outside {', '.join(modules)} reads it either — that is a "
                    f"globally unconsumed field (#748), not a per-backend gap"
                )
            if entry.trainer_reads and name not in read:
                problems.append(
                    f"{entry.field}: declared read-to-warn, but none of "
                    f"{', '.join(modules)} mentions it"
                )
            if not entry.trainer_reads and name in read:
                problems.append(
                    f"{entry.field}: declared unread, but {read[name]} now reads "
                    f"it — the field was wired; remove or reclassify the entry"
                )
    return problems


def test_the_registry_matches_what_the_backend_trainer_actually_reads():
    repo_root = pathlib.Path(__file__).resolve().parents[1]
    problems = _registry_drift(repo_root)
    assert problems == [], "\n".join(problems)


def test_the_declared_modules_cover_every_helper_the_trainer_imports():
    """The map must not silently shrink back to trainer-only.

    Derived from the trainer's own imports rather than hardcoded, so adding a
    helper to ``mlx_sft.py`` without declaring it here fails, and dropping one
    from the map fails too. Asserting a literal list would pass either way.
    """
    from soup_cli.config.backend_support import TRAINER_MODULES

    repo_root = pathlib.Path(__file__).resolve().parents[1]
    missing: list[str] = []
    for pair, modules in TRAINER_MODULES.items():
        primary = repo_root / "src" / modules[0]
        tree = ast.parse(primary.read_text(encoding="utf-8"))
        imported: set[str] = set()
        for node in ast.walk(tree):
            if isinstance(node, ast.ImportFrom) and node.module:
                if node.module.startswith("soup_cli.trainer"):
                    imported.add(node.module)
            elif isinstance(node, ast.Import):
                for alias in node.names:
                    if alias.name.startswith("soup_cli.trainer"):
                        imported.add(alias.name)
        declared = {m.removesuffix(".py").replace("/", ".") for m in modules}
        for mod in sorted(imported):
            if mod not in declared:
                missing.append(f"{pair}: {modules[0]} imports {mod}, not declared")
    assert missing == [], "\n".join(missing)


def test_an_unfounded_gap_claim_is_caught():
    """@MakazhanAlpamys's mutation on #756, which the first version survived.

    He fabricated an entry declaring a globally unconsumed field ignored on
    ``(sft, mlx)`` and the suite stayed green: the drift check only compared
    the entry against the declared modules, so an ``ignored`` claim about a
    field those modules never mention could not be contradicted.

    A gap is relative to something. If nothing anywhere reads the field, it is
    not "this backend ignores it" -- it is a field no code consumes, which is
    #748's subject and #751's allowlist.
    """
    import soup_cli.config.backend_support as bs

    repo_root = pathlib.Path(__file__).resolve().parents[1]
    # #761 wired data.mask_history, which used to stand in here; the claim only
    # has to be about a field nothing consumes, and training.lr_groups is one
    # (see #748's KNOWN_UNCONSUMED).
    fabricated = bs.SupportEntry(
        "training.lr_groups", bs.IGNORED, "fabricated, unfounded claim"
    )
    real = bs.REGISTRY[("sft", "mlx", "text")]
    try:
        bs.REGISTRY[("sft", "mlx", "text")] = (fabricated,) + real
        problems = _registry_drift(repo_root)
    finally:
        bs.REGISTRY[("sft", "mlx", "text")] = real

    assert any(
        "lr_groups" in p and "globally unconsumed" in p for p in problems
    ), problems


def test_a_gap_for_a_field_another_backend_reads_is_accepted():
    """Control for the above — the check must not fire on a real gap.

    ``training.use_galore`` is read by the transformers path and ignored by
    MLX, so it is exactly the shape the registry exists to record.
    """
    repo_root = pathlib.Path(__file__).resolve().parents[1]
    assert _registry_drift(repo_root) == []


def test_a_declared_module_that_does_not_exist_is_caught(tmp_path, monkeypatch):
    """A renamed helper must not silently shrink the guard's scope."""
    import soup_cli.config.backend_support as bs

    monkeypatch.setitem(
        bs.TRAINER_MODULES, ("sft", "mlx", "text"), ("soup_cli/trainer/gone.py",)
    )
    problems = _registry_drift(tmp_path)
    assert any("does not exist" in p for p in problems), problems


def test_a_field_wired_only_inside_a_helper_is_caught(tmp_path, monkeypatch):
    """The hole this fix closes.

    #734 wired the schedule fields mostly inside ``mlx_optim.py``. Had it left
    nothing visible in ``mlx_sft.py``, a trainer-only scan would have passed.
    """
    import soup_cli.config.backend_support as bs

    trainer = tmp_path / "src" / "soup_cli" / "trainer"
    trainer.mkdir(parents=True)
    # the trainer never names the field ...
    (trainer / "mlx_sft.py").write_text(
        "def setup(cfg):\n    return build(cfg)\n", encoding="utf-8"
    )
    # ... the helper it delegates to does the reading
    (trainer / "mlx_optim.py").write_text(
        "def build(cfg):\n    return plan(warmup_ratio=cfg.training.warmup_ratio)\n",
        encoding="utf-8",
    )
    monkeypatch.setitem(
        bs.REGISTRY,
        ("sft", "mlx", "text"),
        (bs.SupportEntry("training.warmup_ratio", bs.IGNORED, "stale on purpose"),),
    )
    monkeypatch.setitem(
        bs.TRAINER_MODULES,
        ("sft", "mlx", "text"),
        ("soup_cli/trainer/mlx_sft.py", "soup_cli/trainer/mlx_optim.py"),
    )

    problems = _registry_drift(tmp_path)
    assert any("warmup_ratio" in p and "mlx_optim.py" in p for p in problems), problems


def test_wiring_a_field_makes_its_stale_entry_fail(tmp_path, monkeypatch):
    """Acceptance criterion 1, the other direction — and this one really happened.

    The registry used to carry ``training.max_grad_norm`` as unread on
    ``(sft, mlx)``. #750 wired it, this check went red naming it, and the entry
    was removed. Every surviving entry is now ``trainer_reads=True``, so there
    is no live entry left to prove the direction with — it is pinned against a
    synthetic registry rather than quietly dropped, because a direction nothing
    exercises is a direction that rots.
    """
    import soup_cli.config.backend_support as bs

    fake_src = tmp_path / "src" / "soup_cli" / "trainer"
    fake_src.mkdir(parents=True)
    (fake_src / "mlx_sft.py").write_text(
        textwrap.dedent(
            """\
            def build(tcfg):
                # the shape #750 introduced
                return clip(tcfg.max_grad_norm)
            """
        ),
        encoding="utf-8",
    )
    monkeypatch.setitem(
        bs.REGISTRY,
        ("sft", "mlx", "text"),
        (
            bs.SupportEntry(
                "training.max_grad_norm",
                bs.IGNORED,
                "stale on purpose: the field is wired now",
            ),
        ),
    )
    monkeypatch.setitem(
        bs.TRAINER_MODULES, ("sft", "mlx", "text"), ("soup_cli/trainer/mlx_sft.py",)
    )

    problems = _registry_drift(tmp_path)
    assert any("max_grad_norm" in p and "now reads it" in p for p in problems), problems


@pytest.mark.parametrize(
    "entry",
    [e for e in _WARNED_ENTRIES],
    ids=[e.field for e in _WARNED_ENTRIES],
)
def test_deleting_a_warning_makes_its_entry_fail(entry, tmp_path, monkeypatch):
    """The mirror case: an entry declared read-to-warn whose warning is gone.

    Parametrised over every ``trainer_reads=True`` entry rather than pinning
    ``use_galore``, so this cannot pass by fixture coincidence and a new entry
    is covered the moment it is added.
    """
    import soup_cli.config.backend_support as bs

    trainer = tmp_path / "src" / "soup_cli" / "trainer"
    trainer.mkdir(parents=True)
    (trainer / "mlx_sft.py").write_text(
        "def build(tcfg):\n    return None\n", encoding="utf-8"
    )
    monkeypatch.setitem(bs.REGISTRY, ("sft", "mlx", "text"), (entry,))
    monkeypatch.setitem(
        bs.TRAINER_MODULES, ("sft", "mlx", "text"), ("soup_cli/trainer/mlx_sft.py",)
    )

    problems = _registry_drift(tmp_path)
    name = entry.field.split(".", 1)[1]
    assert any(name in p and "declared read-to-warn" in p for p in problems), problems


class TestTheFixtureDoesNotDependOnCallerIndentation:
    """The `config_at` block must survive any consistent caller indentation.

    The fixture used to interpolate a multi-line block into a dedented
    f-string, where only the first line inherits the template's indent. A
    caller whose second line was indented differently silently produced a
    YAML parser error instead of the config under test -- so a test could
    fail for a reason that had nothing to do with what it was asserting.
    """

    BLOCKS = {
        "two-space, the natural spelling": "  use_liger: true\n  neftune_alpha: 5",
        "zero-indent": "use_liger: true\nneftune_alpha: 5",
        "deep and consistent": "        use_liger: true\n        neftune_alpha: 5",
        "single line": "  use_liger: true",
        "trailing newline": "  use_liger: true\n  neftune_alpha: 5\n",
    }

    @pytest.mark.parametrize("spelling", sorted(BLOCKS))
    def test_every_consistent_indentation_yields_the_same_config(
        self, config_at, spelling
    ):
        import yaml

        raw = pathlib.Path(
            config_at("sft", "mlx", training=self.BLOCKS[spelling])
        ).read_text(encoding="utf-8")
        parsed = yaml.safe_load(raw)

        assert set(parsed) == {
            "base",
            "task",
            "backend",
            "data",
            "training",
            "output",
        }, raw
        expected = (
            {"use_liger"}
            if spelling == "single line"
            else {"use_liger", "neftune_alpha"}
        )
        assert set(parsed["training"]) == expected, raw
        assert parsed["training"]["use_liger"] is True, raw

    def test_a_data_block_is_normalised_the_same_way(self, config_at):
        import yaml

        raw = pathlib.Path(
            config_at("sft", "mlx", training="epochs: 1", data="max_length: 128")
        ).read_text(encoding="utf-8")
        parsed = yaml.safe_load(raw)

        assert parsed["data"]["max_length"] == 128, raw
        assert parsed["data"]["format"] == "alpaca", raw
        assert parsed["training"]["epochs"] == 1, raw