Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#1116: can every shipped config actually attach a LoRA adapter?
A config can parse (#330), name a repo that resolves (#677), and still be
untrainable, because `target_modules: auto` resolves to nothing for an
architecture neither Soup nor peft maps. #1070 (every shipped MoE recipe),
#798's dropout refusal and #1074's `templates/moe.yaml` blocker were each found
by a person reading code, not by a check.
Two rules are load-bearing and are pinned hardest here, because breaking either
turns the guard into noise that gets muted:
1. **A failure to LOAD is not a failure to ATTACH.** Gated, remote-code and
unreadable configs are UNVERIFIED, never CANNOT_ATTACH, and never change the
exit code. On a box with no `HF_TOKEN` that is most of the gated bases in
the catalogue, so the rule is what keeps CI from going permanently red.
2. **Remote code is never executed**, so a repo needing it is unverifiable
rather than run.
The end-to-end tests build real `transformers` configs locally and inject them,
so the real resolution and the real `get_peft_model` run with no network.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from soup_cli.utils.attach_preflight import (
PREFLIGHT_LAYERS,
AttachCheck,
PreflightReport,
Verdict,
check_attach,
classify_load_failure,
count_adapted,
plan_adapter,
shrink_for_preflight,
)
transformers = pytest.importorskip("transformers")
pytest.importorskip("peft")
torch = pytest.importorskip("torch")
def _cfg(
base="org/m", task="sft", r=8, dropout=0.0, backend=None, targets="auto",
modality="text",
):
"""A stand-in for the loaded SoupConfig, carrying only what the check reads."""
return SimpleNamespace(
base=base,
task=task,
modality=modality,
backend=backend,
training=SimpleNamespace(
lora=SimpleNamespace(r=r, alpha=16, dropout=dropout, target_modules=targets)
),
)
def _real_cfg(r=8, dropout=0.0, targets="auto", moe_lora=False, target_parameters=None):
"""A config loaded through the real schema, for every test that reaches the
attach. The check now builds the adapter with the trainer's own
``build_lora_config``, which reads the whole LoRA block (``use_dora``,
``rank_pattern``, the #1037 variants ...), so a hand-built stand-in would be
testing a config no user can write."""
import json
from soup_cli.config.loader import load_config_from_string
return load_config_from_string(
"base: org/m\ntask: sft\n"
"data:\n train: ./x.jsonl\n format: alpaca\n"
f"training:\n moe_lora: {'true' if moe_lora else 'false'}\n"
f" lora:\n r: {r}\n alpha: 16\n dropout: {dropout}\n"
f" target_modules: {json.dumps(targets)}\n"
+ (
f" target_parameters: {json.dumps(target_parameters)}\n"
if target_parameters is not None else ""
)
)
class TestLoadFailuresAreNotAttachFailures:
"""Rule 1. Each of these is a config this machine could not answer for."""
@pytest.mark.parametrize(
"message, expected_fragment",
[
("You are trying to access a gated repo.", "gated"),
("contains custom code which must be executed", "trust_remote_code"),
(
"requires you to execute the configuration file... trust_remote_code",
"trust_remote_code",
),
("Unrecognized model in org/x. Should have a `model_type`", "cannot read"),
("org/x does not appear to have a file named config.json", "no config.json"),
("org/x is not a local folder and is not a valid model identifier", "#677"),
],
)
def test_each_known_loader_failure_is_named(self, message, expected_fragment):
assert expected_fragment in classify_load_failure(ValueError(message))
def test_an_unrecognised_failure_returns_empty_not_a_guess(self):
"""Empty is how the caller knows to print the exception instead. Filing an
unknown failure under the last branch is how a guard starts lying."""
assert classify_load_failure(ValueError("something entirely new")) == ""
@pytest.mark.parametrize(
"message",
[
"You are trying to access a gated repo.",
"contains custom code which must be executed",
"org/x is not a local folder and is not a valid model identifier",
],
)
def test_they_verdict_unverified_and_do_not_fail_the_run(self, message):
def _raise(_base):
raise OSError(message)
check = check_attach(
"r", _cfg(), load_hf_config=_raise, build_model=lambda *_: None
)
assert check.verdict is Verdict.UNVERIFIED, check
assert PreflightReport([check]).exit_code == 0, (
"an unverifiable config failed the run; on a box with no HF_TOKEN "
"that is 51 recipes, and the guard gets muted"
)
def test_an_unknown_loader_failure_still_does_not_claim_it_cannot_attach(self):
"""The conservative direction: we did not reach the attach, so we do not
get to say it fails. The message carries the exception verbatim."""
def _raise(_base):
raise RuntimeError("hub had a bad minute")
check = check_attach(
"r", _cfg(), load_hf_config=_raise, build_model=lambda *_: None
)
assert check.verdict is Verdict.UNVERIFIED
assert "hub had a bad minute" in check.detail
def test_a_recognised_build_failure_is_also_unverified(self):
"""The same rule one stage later, and the stage my first version of these
tests missed entirely: ``from_config`` raises "Unrecognized model" for a
config this transformers cannot build (MiniMax-M3 does exactly this).
Filing that as CANNOT_ATTACH blames the recipe for the library's age."""
def _boom(_config, _task):
raise ValueError("Unrecognized model in org/x. Should have a `model_type`")
check = check_attach(
"r", _cfg(), load_hf_config=lambda _b: SimpleNamespace(model_type="x"),
build_model=_boom,
)
assert check.verdict is Verdict.UNVERIFIED, check
assert check.stage == "build"
assert PreflightReport([check]).exit_code == 0
def test_an_unrecognised_build_failure_is_an_attach_failure(self):
"""The other side, so the rule is not "never fail". The config loaded and
the architecture is known; if it will not instantiate, that is real."""
def _boom(_config, _task):
raise TypeError("__init__() missing 1 required argument")
check = check_attach(
"r", _cfg(), load_hf_config=lambda _b: SimpleNamespace(model_type="x"),
build_model=_boom,
)
assert check.verdict is Verdict.CANNOT_ATTACH
assert check.stage == "build"
class TestNothingToAttach:
"""Neither of these is a failure, and reporting them as one puts permanent
red rows in a report meant to go green."""
def test_full_fine_tuning_has_no_adapter(self):
check = check_attach(
"r", _cfg(r=0), load_hf_config=_unreachable, build_model=_unreachable
)
assert check.verdict is Verdict.NO_ADAPTER
assert "lora.r" in check.detail
def test_the_mlx_backend_is_out_of_scope(self):
check = check_attach(
"r", _cfg(backend="mlx"),
load_hf_config=_unreachable, build_model=_unreachable,
)
assert check.verdict is Verdict.NO_ADAPTER
assert "mlx" in check.detail
def test_neither_touches_the_network(self):
"""Pinned by `_unreachable` above, which raises if called -- the check
must decide these before it asks the Hub anything."""
assert plan_adapter(_cfg(r=0))[0] is False
assert plan_adapter(_cfg(backend="mlx"))[0] is False
assert plan_adapter(_cfg())[0] is True
def _unreachable(*_args, **_kwargs):
raise AssertionError("the check reached the network for a config with no adapter")
class TestTheLoaderMatchesTheTrainer:
"""The class a config is built with has to be the one its trainer loads, or
the verdict is about a different model. SFT picks by MODALITY
(``trainer/sft.py``: causal LM for text, ``AutoModelForImageTextToText`` for
vision, ``AutoModel`` for audio); my first version keyed on the task and
built every vision recipe as a causal LM. It was caught by re-verifying the
multimodal rows of #1116 against the real loaders, not by these tests --
which is why these now exist."""
def test_vision_sft_uses_the_image_text_class(self):
from soup_cli.utils.attach_preflight import loader_for
assert loader_for(_cfg(modality="vision")) == ("AutoModelForImageTextToText",)
def test_audio_sft_uses_the_bare_model(self):
from soup_cli.utils.attach_preflight import loader_for
assert loader_for(_cfg(modality="audio")) == ("AutoModel",)
def test_text_sft_uses_a_causal_lm(self):
from soup_cli.utils.attach_preflight import loader_for
# Exact, not [0]: a fallback at [1] is the bug this pins (see
# test_a_text_config_the_causal_lm_class_refuses_is_not_attached).
assert loader_for(_cfg()) == ("AutoModelForCausalLM",)
def test_a_task_with_its_own_head_wins_over_modality(self):
"""An embedding run is ``AutoModel`` whatever the modality says --
``trainer/embedding.py`` never loads a causal LM."""
from soup_cli.utils.attach_preflight import loader_for
assert loader_for(_cfg(task="embedding")) == ("AutoModel",)
assert loader_for(_cfg(task="reward_model")) == (
"AutoModelForSequenceClassification",
)
class TestRemoteCodeIsNeverOffered:
"""The module promises remote code is never executed. The first version kept
that promise for the config and broke it for the model: ``from_config`` was
called without ``trust_remote_code``, and for a custom-code architecture
transformers does not refuse -- it PROMPTS on stdin, "Do you wish to run the
custom code? [y/N]". That hangs a terminal, corrupts ``--json``, and runs
remote code for anyone who types y. Kimi-K2.5 is the live case: its config
loads with ``trust_remote_code=False`` while its modeling code is remote.
Found by running the real command end to end, not by any test here."""
def test_every_build_passes_trust_remote_code_false(self, monkeypatch):
from soup_cli.utils import attach_preflight
seen = []
def _spy(_config, **kwargs):
seen.append(kwargs)
return SimpleNamespace()
# Patch the method on the real class: ``build_on_meta`` resolves the
# class through transformers' lazy module, which a module-level setattr
# does not reliably intercept.
monkeypatch.setattr(transformers.AutoModelForCausalLM, "from_config", _spy)
attach_preflight.build_on_meta(SimpleNamespace(), ("AutoModelForCausalLM",))
assert seen == [{"trust_remote_code": False}], (
"from_config was called without trust_remote_code=False; transformers "
"will prompt on stdin for a custom-code architecture"
)
def test_the_config_load_refuses_remote_code_too(self, monkeypatch):
seen = []
monkeypatch.setattr(
transformers.AutoConfig, "from_pretrained",
lambda base, **kwargs: seen.append(kwargs) or SimpleNamespace(),
)
from soup_cli.utils.attach_preflight import load_hf_config
load_hf_config("org/m")
assert seen == [{"trust_remote_code": False}]
class TestJsonIsParseable:
"""``--json`` exists for a program to read, so stdout must be the document
and nothing else."""
def _invoke(self, tmp_path, monkeypatch, *, force_colour):
import json as _json
from typer.testing import CliRunner
from soup_cli.cli import app
monkeypatch.setattr(
"soup_cli.utils.attach_preflight.check_attach",
lambda name, cfg, **_k: AttachCheck(
name, cfg.base, cfg.task, Verdict.ATTACHES, adapted=4
),
)
if force_colour:
# The module-level Console is built at import time, so FORCE_COLOR set
# here would never reach it and the assertion below would pass
# without testing anything -- which is exactly what my first version
# did (the print_json mutant survived). Pin a colour-forced console
# into the module instead; it resolves sys.stdout at write time, so
# CliRunner still captures it.
from rich.console import Console
import soup_cli.commands.recipes as recipes_cmd
monkeypatch.setattr(recipes_cmd, "console", Console(force_terminal=True))
good = tmp_path / "soup.yaml"
good.write_text(
"base: org/m\ntask: sft\ndata:\n train: ./x.jsonl\n format: alpaca\n"
"training:\n lora:\n r: 8\n",
encoding="utf-8",
)
result = CliRunner().invoke(app, ["recipes", "verify", "--config", str(good), "--json"])
return result, _json
def test_stdout_parses_as_json(self, tmp_path, monkeypatch):
result, json_mod = self._invoke(tmp_path, monkeypatch, force_colour=False)
rows = json_mod.loads(result.stdout)
assert rows[0]["verdict"] == "attaches"
def test_it_still_parses_when_colour_is_forced(self, tmp_path, monkeypatch):
"""Rich's ``print_json`` syntax-colours its output, and under
``FORCE_COLOR`` -- which many CI runners set -- the escape codes make the
document unparseable. Measured: ``json.loads`` raised JSONDecodeError on
``print_json`` output with a forced terminal."""
result, json_mod = self._invoke(tmp_path, monkeypatch, force_colour=True)
assert "\x1b[" not in result.stdout, "ANSI colour codes in --json output"
json_mod.loads(result.stdout)
class TestShrinking:
def test_sub_configs_are_cut_too(self):
"""A vision-language wrapper keeps the text tower's depth in
`text_config`; leaving it builds 61 layers whose names repeat after 2."""
config = SimpleNamespace(
num_hidden_layers=61,
text_config=SimpleNamespace(num_hidden_layers=61),
vision_config=SimpleNamespace(num_hidden_layers=27),
)
shrink_for_preflight(config)
assert config.num_hidden_layers == PREFLIGHT_LAYERS
assert config.text_config.num_hidden_layers == PREFLIGHT_LAYERS
assert config.vision_config.num_hidden_layers == PREFLIGHT_LAYERS
def test_a_model_already_smaller_is_left_alone(self):
"""Raising a 1-layer test model to 2 would change what is being checked."""
config = SimpleNamespace(num_hidden_layers=1)
shrink_for_preflight(config)
assert config.num_hidden_layers == 1
def test_a_config_without_layers_is_not_invented(self):
config = SimpleNamespace(model_type="x")
shrink_for_preflight(config)
assert not hasattr(config, "num_hidden_layers")
class TestTheRealAttach:
"""End to end with the real resolution and the real `get_peft_model`, on
real architectures built locally. No network, no weights."""
@staticmethod
def _local(config):
return lambda _base: config
@staticmethod
def _meta(config, classes):
from soup_cli.utils.attach_preflight import build_on_meta
return build_on_meta(config, classes)
def test_a_dense_llama_attaches(self):
from transformers import LlamaConfig
config = LlamaConfig(
vocab_size=64, hidden_size=16, intermediate_size=32,
num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=1,
)
check = check_attach(
"llama", _real_cfg(), load_hf_config=self._local(config), build_model=self._meta
)
assert check.verdict is Verdict.ATTACHES, check.detail
assert check.adapted == 4, "q_proj and v_proj on two layers"
assert check.expert_modules == 0
def test_a_phi3_shaped_model_cannot_attach(self):
"""The finding this check exists for, reduced to a test. phi-4 is
`model_type: phi3`; its attention is a fused `qkv_proj` + `o_proj`, so
there is no `q_proj`/`v_proj` for a default to hit, peft 0.20 has no
`phi3` key, and Soup's `auto` returns None. Three shipped recipes name
that base."""
from transformers import Phi3Config
config = Phi3Config(
vocab_size=64, hidden_size=16, intermediate_size=32,
num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=2,
)
check = check_attach(
"phi", _real_cfg(), load_hf_config=self._local(config), build_model=self._meta
)
assert check.verdict is Verdict.CANNOT_ATTACH, check
assert check.model_type == "phi3"
assert check.adapted == 0
def test_the_configs_own_lora_settings_reach_peft(self):
"""Substituting a tidy ``r=8, dropout=0.0`` would have hidden #798
completely: the dropout that made every MoE recipe unattachable was the
schema default those recipes inherited, and a check that normalises it
away cannot see the defect it exists to find.
Asserted on the kwargs rather than on a failure, because reaching peft's
``ParamWrapper`` (where a non-zero dropout actually raises) needs
``target_parameters``, which is #798/#1074 machinery and not this
module's business.
"""
from transformers import LlamaConfig
config = LlamaConfig(
vocab_size=64, hidden_size=16, intermediate_size=32,
num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=1,
)
seen = {}
def _record(model, peft_config):
seen["config"] = peft_config
return SimpleNamespace(named_modules=lambda: [("a.lora_A", None)])
check_attach(
"llama", _real_cfg(r=64, dropout=0.05),
load_hf_config=self._local(config), build_model=self._meta,
attach=_record,
)
# The config handed to peft IS the trainer's -- build_lora_config's output.
assert seen["config"].lora_dropout == 0.05, "the check normalised the dropout away"
assert seen["config"].r == 64, "the check substituted its own rank"
def test_an_attach_that_adapts_nothing_is_a_failure(self):
"""The defensive branch: peft normally raises when nothing matches, but if
it ever returns a model with no adapter, "it attached" is the wrong
answer -- that is a run that trains no LoRA parameters and looks fine,
which is the v0.73.3 shape this repo has been bitten by before."""
from transformers import LlamaConfig
config = LlamaConfig(
vocab_size=64, hidden_size=16, intermediate_size=32,
num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=1,
)
check = check_attach(
"llama", _real_cfg(),
load_hf_config=self._local(config), build_model=self._meta,
attach=lambda _m, _k: SimpleNamespace(named_modules=lambda: []),
)
assert check.verdict is Verdict.CANNOT_ATTACH, check
assert "no module carries an adapter" in check.detail
def test_an_explicit_target_that_matches_nothing_fails(self):
"""`auto` is not the only way to reach nothing -- a hand-written list
naming modules this architecture does not have reaches it too."""
from transformers import LlamaConfig
config = LlamaConfig(
vocab_size=64, hidden_size=16, intermediate_size=32,
num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=1,
)
check = check_attach(
"llama", _real_cfg(targets=["not_a_module_here"]),
load_hf_config=self._local(config), build_model=self._meta,
)
assert check.verdict is Verdict.CANNOT_ATTACH
class TestTheBreakdowns:
"""`adapted=8` says nothing about whether the right 8. These two columns are
the ones that caught a vision tower being adapted during a text fine-tune."""
def test_expert_and_vision_modules_are_counted_apart(self):
model = SimpleNamespace(
named_modules=lambda: [
("model.layers.0.self_attn.q_proj.lora_A", None),
("model.layers.0.mlp.experts.gate_up_proj.lora_A", None),
("vision_tower.layers.0.self_attn.q_proj.lora_A", None),
("model.layers.0.self_attn.q_proj.lora_B", None),
]
)
assert count_adapted(model) == (3, 1, 1)
def test_the_inner_adapter_module_is_not_counted_twice(self):
"""peft emits BOTH ``q_proj.lora_A`` (the ModuleDict) and
``q_proj.lora_A.<adapter>`` (the Linear inside it) from
``named_modules``. My first version matched either and reported 8 for a
model with 4 adapted modules -- caught by the real llama test below, not
by a fake."""
model = SimpleNamespace(
named_modules=lambda: [
("a.lora_A", None), ("a.lora_A.default", None),
("a.lora_B", None), ("a.lora_B.default", None),
]
)
assert count_adapted(model)[0] == 1
def test_a_non_default_adapter_name_is_still_counted(self):
"""The other half of why ``.lora_A`` is the right anchor: peft names the
inner Linear after the adapter, so matching ``lora_A.default`` alone
reports 0 for any run that named its adapter something else."""
model = SimpleNamespace(
named_modules=lambda: [("a.lora_A", None), ("a.lora_A.my_adapter", None)]
)
assert count_adapted(model)[0] == 1
class TestTheExitCode:
def test_only_cannot_attach_fails_the_run(self):
rows = [
AttachCheck("a", "b", "sft", Verdict.ATTACHES, adapted=4),
AttachCheck("b", "b", "sft", Verdict.UNVERIFIED, detail="gated"),
AttachCheck("c", "b", "sft", Verdict.NO_ADAPTER, detail="lora.r: 0"),
]
assert PreflightReport(rows).exit_code == 0
def test_one_failure_fails_the_run(self):
rows = [
AttachCheck("a", "b", "sft", Verdict.ATTACHES, adapted=4),
AttachCheck("c", "b", "sft", Verdict.CANNOT_ATTACH, detail="nothing"),
]
report = PreflightReport(rows)
# #1117 review: the gate/verdict contract, not "1 on any failure".
assert report.exit_code == 2
assert [c.name for c in report.failures] == ["c"]
class TestTheCommand:
"""The CLI is a thin wrapper, so this pins only what the wrapper decides:
which configs are in scope, and the exit code."""
def _run(self, tmp_path, monkeypatch, verdict):
from typer.testing import CliRunner
from soup_cli.cli import app
def _fake(name, cfg, **_kwargs):
return AttachCheck(name, cfg.base, cfg.task, verdict, detail="stubbed")
monkeypatch.setattr("soup_cli.utils.attach_preflight.check_attach", _fake)
path = tmp_path / "soup.yaml"
path.write_text(
"base: org/m\ntask: sft\ndata:\n train: ./x.jsonl\n format: alpaca\n"
"training:\n lora:\n r: 8\n",
encoding="utf-8",
)
return CliRunner().invoke(app, ["recipes", "verify", "--config", str(path)])
def test_a_config_that_cannot_attach_exits_2(self, tmp_path, monkeypatch):
"""Was 1. A finding is EXIT_GATE_FAILED under the gate/verdict contract,
so CI can tell it from a crash (1) or a bad --config (3)."""
result = self._run(tmp_path, monkeypatch, Verdict.CANNOT_ATTACH)
assert result.exit_code == 2, result.output
def test_an_unverified_config_exits_0(self, tmp_path, monkeypatch):
"""The rule again, through the command: CI must not go red because the
runner has no token."""
result = self._run(tmp_path, monkeypatch, Verdict.UNVERIFIED)
assert result.exit_code == 0, result.output
def test_a_missing_config_is_refused(self, tmp_path):
from typer.testing import CliRunner
from soup_cli.cli import app
result = CliRunner().invoke(
app, ["recipes", "verify", "--config", str(tmp_path / "nope.yaml")]
)
assert result.exit_code == 3, "an input error, not a finding or a crash"
from tests.conftest import strip_ansi
assert "not found" in strip_ansi(result.output).lower()
def test_an_unparseable_config_exits_3(self, tmp_path):
from typer.testing import CliRunner
from soup_cli.cli import app
from tests.conftest import strip_ansi
path = tmp_path / "bad.yaml"
path.write_text("base: org/m\ntask: sft\ndata:\n train: x.jsonl\n format: nope\n")
result = CliRunner().invoke(app, ["recipes", "verify", "--config", str(path)])
assert result.exit_code == 3, result.output
assert "does not parse" in strip_ansi(result.output)
def test_a_crash_exits_1(self, tmp_path, monkeypatch):
"""The fourth code: an unexpected error stays a runtime error."""
def _boom(*_a, **_k):
raise RuntimeError("boom")
monkeypatch.setattr("soup_cli.utils.attach_preflight.check_attach", _boom)
from typer.testing import CliRunner
from soup_cli.cli import app
path = tmp_path / "soup.yaml"
path.write_text(
"base: org/m\ntask: sft\ndata:\n train: ./x.jsonl\n format: alpaca\n"
"training:\n lora:\n r: 8\n",
encoding="utf-8",
)
result = CliRunner().invoke(app, ["recipes", "verify", "--config", str(path)])
assert result.exit_code == 1
# The 1 must come from the injected crash, not some other failure path.
assert isinstance(result.exception, RuntimeError), result.exception
class TestTheVerdictIsTheTrainers:
"""The check must answer for what the TRAINER does, not for a lookalike.
Two ways the first version answered for a lookalike, each now pinned."""
@staticmethod
def _qwen2_moe_config():
from transformers import Qwen2MoeConfig
return Qwen2MoeConfig(
vocab_size=64, hidden_size=16, intermediate_size=32,
moe_intermediate_size=16, shared_expert_intermediate_size=16,
num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=1,
num_experts=4, num_experts_per_tok=2,
)
def _check(self, cfg, hf_config, classes=None):
from soup_cli.utils.attach_preflight import build_on_meta
return check_attach(
"r", cfg, load_hf_config=lambda _b: hf_config,
build_model=lambda c, _cls: build_on_meta(c, classes or _cls),
)
def test_moe_lora_rescues_an_unmapped_moe_as_it_does_in_the_trainer(self):
"""``qwen2_moe`` is mapped by neither Soup nor peft, so ``auto`` resolves to
nothing -- and the trainer then replaces ``target_modules`` from the
``moe_lora`` scan. The first version skipped that step and called such a
recipe broken: measured on ``qwen3-30b-a3b-sft`` it said FAILS while the
trainer's path attached 12 modules."""
check = self._check(_real_cfg(moe_lora=True), self._qwen2_moe_config())
assert check.verdict is Verdict.ATTACHES, check.detail
assert check.adapted > 0
def test_without_moe_lora_the_same_model_cannot_attach(self):
"""The control: it is the ``moe_lora`` step that attached it above, not a
check that has stopped failing anything."""
check = self._check(_real_cfg(moe_lora=False), self._qwen2_moe_config())
assert check.verdict is Verdict.CANNOT_ATTACH, check
@staticmethod
def _qwen3_moe_config():
from transformers import Qwen3MoeConfig
return Qwen3MoeConfig(
vocab_size=64, hidden_size=16, intermediate_size=32,
moe_intermediate_size=16, num_hidden_layers=2, num_attention_heads=2,
num_key_value_heads=1, num_experts=4, num_experts_per_tok=2,
)
def test_explicit_target_parameters_reach_the_attach(self):
"""``qwen3_moe``'s experts are fused 3-D parameters that no
``target_modules`` entry can name; ``lora.target_parameters`` is how a
user reaches them, and the trainer hands them to ``build_lora_config``.
A check that dropped them would report such a config as unable to attach
-- or attach it without the experts it was written for."""
cfg = _real_cfg(target_parameters=["experts.gate_up_proj", "experts.down_proj"])
check = self._check(cfg, self._qwen3_moe_config())
assert check.verdict is Verdict.ATTACHES, check.detail
assert check.expert_modules > 0, check
def test_the_attach_sees_the_keys_the_trainer_hands_peft(self):
"""The maintainer's review ask on #1116: run against the object the trainer
hands ``get_peft_model``, INCLUDING the ``model.`` prefix peft matches on.
peft fullmatches a string target against the whole module key, and under
``AutoModelForImageTextToText`` -- what vision SFT loads -- the key is
``model.language_model....``. So a pattern anchored at ``language_model``
genuinely cannot attach, and a check that built the model at a different
level (``AutoModel``, which drops the ``model.`` wrapper) would report it
green. That exact regex shipped in #1102 and was caught at review."""
from transformers import CLIPVisionConfig, LlamaConfig, LlavaConfig
vl = LlavaConfig(
text_config=LlamaConfig(
vocab_size=64, hidden_size=16, intermediate_size=32,
num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=2,
pad_token_id=0,
),
vision_config=CLIPVisionConfig(
hidden_size=16, intermediate_size=32, num_hidden_layers=2,
num_attention_heads=2, image_size=32, patch_size=16,
),
image_token_index=63,
)
classes = ("AutoModelForImageTextToText",)
anchored = _real_cfg(targets=r"language_model\..*\.self_attn\.(q_proj|v_proj)")
prefixed = _real_cfg(targets=r".*language_model\..*\.self_attn\.(q_proj|v_proj)")
broken = self._check(anchored, vl, classes)
working = self._check(prefixed, vl, classes)
assert broken.verdict is Verdict.CANNOT_ATTACH, (
"an anchored regex attached -- the check is not seeing the model. prefix"
)
assert working.verdict is Verdict.ATTACHES, working.detail
assert working.vision_modules == 0, "and it stayed out of the vision tower"
def test_a_text_config_the_causal_lm_class_refuses_is_not_attached(self):
"""No trainer falls back from ``AutoModelForCausalLM`` to ``AutoModel``;
SFT's text path loads exactly one class. My first version did fall back,
so a config the causal-LM class refuses was built one level down and
reported as ATTACHING -- MiniMax-M3 with no ``modality`` read green, while
in SFT it cannot even load (#1145). A LLaVA reproduces the shape locally:
``AutoModelForCausalLM`` refuses it, ``AutoModel`` builds ``LlavaModel``."""
from transformers import CLIPVisionConfig, LlamaConfig, LlavaConfig
from soup_cli.utils.attach_preflight import build_on_meta, loader_for
vl = LlavaConfig(
text_config=LlamaConfig(
vocab_size=64, hidden_size=16, intermediate_size=32,
num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=2,
pad_token_id=0,
),
vision_config=CLIPVisionConfig(
hidden_size=16, intermediate_size=32, num_hidden_layers=2,
num_attention_heads=2, image_size=32, patch_size=16,
),
image_token_index=63,
)
text_cfg = _real_cfg() # no modality -> the text path
check = check_attach(
"vl-as-text", text_cfg, load_hf_config=lambda _b: vl,
build_model=lambda c, _cls: build_on_meta(c, loader_for(text_cfg)),
)
assert check.verdict is Verdict.CANNOT_ATTACH, (
f"reported {check.verdict.value} -- the check built it with a class "
"the trainer never uses"
)
assert check.stage == "build"
class TestReviewFollowUps:
"""The #1117 review's follow-ups, each pinned."""
def test_a_gated_message_that_also_says_not_a_local_folder_is_gated(self):
"""First match wins, so the order is the #677 lesson: a 401 folded into
"is not a local folder" text must still read as gated, not unresolvable."""
exc = OSError(
"org/m is not a local folder and is not a valid model identifier. "
"Cannot access gated repo for url https://huggingface.co/org/m."
)
assert "gated" in classify_load_failure(exc)
def test_catalogue_row_names_use_forward_slashes(self):
"""``--json`` keys must not depend on the OS: Windows printed
``templates\\audio.yaml``. Only the Windows cells can see a regression."""
from soup_cli.commands.recipes import _configs_to_verify
names = [name for name, _cfg in _configs_to_verify(None, True)]
nested = [n for n in names if "/" in n]
assert any(n.startswith("templates/") for n in nested), names[:5]
assert any(n.startswith("examples/") for n in nested), names[:5]
assert not [n for n in names if "\\" in n]
@staticmethod
def _cfg(modality):
from soup_cli.config.loader import load_config_from_string
fmt = {"text": "alpaca", "vision": "llava", "audio": "audio"}[modality]
return load_config_from_string(
f"base: org/m\ntask: sft\nmodality: {modality}\n"
f"data:\n train: ./x.jsonl\n format: {fmt}\n"
# No moe_lora: #1179 refuses it on SFT vision/audio at load, and the
# modality gate under test does not read it.
"training:\n"
" lora:\n r: 8\n alpha: 16\n dropout: 0.0\n"
' target_modules: ["q_proj"]\n'
)
@pytest.mark.parametrize(
("modality", "full_sequence"),
[("text", True), ("vision", False), ("audio", False)],
)
def test_sft_vision_and_audio_skip_the_moe_and_parameter_steps(
self, monkeypatch, modality, full_sequence
):
"""SFT's vision and audio branches call only ``resolve_lora_target_modules``
and ``build_lora_config``, so a vision or audio config must not be reported
through the MoE and ``target_parameters`` steps the trainer never runs there."""
import torch
import soup_cli.utils.moe as moe
import soup_cli.utils.peft_wiring as wiring
from soup_cli.utils.attach_preflight import trainer_lora_config
seen = []
real_moe, real_params = moe.resolve_moe_lora_targets, wiring.resolve_lora_target_parameters
def spy_moe(*a, **k):
seen.append("moe")
return real_moe(*a, **k)
def spy_params(*a, **k):
seen.append("params")
return real_params(*a, **k)
monkeypatch.setattr(moe, "resolve_moe_lora_targets", spy_moe)
monkeypatch.setattr(wiring, "resolve_lora_target_parameters", spy_params)
model = torch.nn.Sequential()
model.add_module("q_proj", torch.nn.Linear(4, 4))
_peft_config, targets = trainer_lora_config(model, self._cfg(modality))
assert targets == ["q_proj"]
assert (seen == ["params", "moe"]) is full_sequence, seen
assert (seen == []) is (not full_sequence), seen
class TestOnlySftReadsModality:
"""#1117 review, round 3: ``loader_for`` chose the class from ``modality`` for
every task, but only ``trainer/sft.py`` read it. A vision DPO config was
reported ATTACHES (8 modules, 4 of them in the vision tower) while DPO's own
``AutoModelForCausalLM`` refused the base outright. Since #1393 DPO reads
``modality`` too and is pinned in ``test_issue1393_dpo_vision_loader.py``; the
trainers here are the ones that still load one class whatever it says."""
@staticmethod
def _cfg(task, modality="text", **training):
import yaml
from soup_cli.config.loader import load_config_from_string
fmt = {
"text": "alpaca",
"vision": "dpo" if task == "dpo" else "llava",
"audio": "audio",
}[modality]
fmt = {"asr": "asr", "classifier": "auto", "reranker": "auto",
"cross_encoder": "auto"}.get(task, fmt)
raw = {
"base": "org/m", "task": task, "modality": modality,
"data": {"train": "./x.jsonl", "format": fmt},
"training": {"lora": {"r": 8, "dropout": 0.0}, **training},
}
return load_config_from_string(yaml.safe_dump(raw))
@pytest.mark.parametrize("task", ["grpo"])
def test_a_vision_config_on_a_text_trainer_is_a_causal_lm(self, task):
from soup_cli.utils.attach_preflight import loader_for
assert loader_for(self._cfg(task, "vision")) == ("AutoModelForCausalLM",)
def test_sft_still_chooses_by_modality(self):
"""The control: the split is SFT's, and it stays."""
from soup_cli.utils.attach_preflight import loader_for
assert loader_for(self._cfg("sft", "vision")) == ("AutoModelForImageTextToText",)
assert loader_for(self._cfg("sft", "audio")) == ("AutoModel",)
def test_asr_loads_whisper_seq2seq(self):
from soup_cli.utils.attach_preflight import loader_for
asr = self._cfg("asr", "text", asr_language="en")
assert loader_for(asr) == ("WhisperForConditionalGeneration",)
@pytest.mark.parametrize("task", ["classifier", "reranker", "cross_encoder"])
def test_the_classifier_family_loads_a_sequence_classifier(self, task):
"""All three: only ``classifier`` was asserted, so dropping the other two
entries survived (#1117 review, round 3)."""
from soup_cli.utils.attach_preflight import loader_for
clf = self._cfg(task, num_labels=2, classifier_lora=True)
assert loader_for(clf) == ("AutoModelForSequenceClassification",)
class TestTheTrainersOwnTargetPaths:
def test_a_vision_dpo_config_takes_the_full_sequence(self, monkeypatch):
"""Pins the ``task == "sft"`` half of the two-call gate: DPO has no
vision branch, so a vision DPO config runs the full MoE sequence."""
import torch
import soup_cli.utils.moe as moe
from soup_cli.utils.attach_preflight import trainer_lora_config
seen = []
real = moe.resolve_moe_lora_targets
def spy(*a, **k):
seen.append("moe")
return real(*a, **k)
monkeypatch.setattr(moe, "resolve_moe_lora_targets", spy)
model = torch.nn.Sequential()
model.add_module("q_proj", torch.nn.Linear(4, 4))
cfg = TestOnlySftReadsModality._cfg("dpo", "vision")
cfg.training.lora.target_modules = ["q_proj"]
trainer_lora_config(model, cfg)
assert seen == ["moe"]
def test_asr_auto_means_q_and_v_without_the_resolver(self, monkeypatch):
"""``trainer/asr.py`` never calls the resolver; ``auto`` is Whisper's q/v."""
import torch
import soup_cli.utils.peft_wiring as wiring
from soup_cli.utils.attach_preflight import trainer_lora_config
def _no(*_a, **_k):
raise AssertionError("asr must not call resolve_lora_target_modules")
monkeypatch.setattr(wiring, "resolve_lora_target_modules", _no)
cfg = TestOnlySftReadsModality._cfg("asr", "text", asr_language="en")
_peft, targets = trainer_lora_config(torch.nn.Linear(2, 2), cfg)
assert targets == ["q_proj", "v_proj"]
class TestTasksThatBuildNoAdapter:
@pytest.mark.parametrize("task", ["prm", "moe_lora_routing"])
def test_they_are_no_adapter_whatever_lora_r_says(self, task):
cfg = SimpleNamespace(
task=task, backend="transformers", training=SimpleNamespace(lora=SimpleNamespace(r=8))
)
wanted, why = plan_adapter(cfg)
assert not wanted and why
def test_the_classifier_family_needs_classifier_lora(self):
off = SimpleNamespace(task="reranker", backend="transformers", training=SimpleNamespace(
lora=SimpleNamespace(r=8), classifier_lora=False))
on = SimpleNamespace(task="reranker", backend="transformers", training=SimpleNamespace(
lora=SimpleNamespace(r=8), classifier_lora=True))
assert plan_adapter(off) == (
False, "classifier_lora is off --- that trainer full-fine-tunes"
)
assert plan_adapter(on) == (True, "")
class TestConcreteClassesBuild:
def test_whisper_seq2seq_builds_on_meta(self):
"""``asr`` loads ``WhisperForConditionalGeneration`` by name, which has no
``from_config``. Calling it anyway turned all three shipped Whisper recipes
from ATTACHES into CANNOT_ATTACH on a full catalogue run."""
from transformers import WhisperConfig
from soup_cli.utils.attach_preflight import build_on_meta
cfg = WhisperConfig(
vocab_size=64, d_model=16, encoder_layers=1, decoder_layers=1,
encoder_attention_heads=2, decoder_attention_heads=2,
encoder_ffn_dim=32, decoder_ffn_dim=32, max_source_positions=16,
max_target_positions=16, num_mel_bins=8,
pad_token_id=0, bos_token_id=1, eos_token_id=2, decoder_start_token_id=1,
)
model = build_on_meta(cfg, ("WhisperForConditionalGeneration",))
assert type(model).__name__ == "WhisperForConditionalGeneration"
assert next(model.parameters()).device.type == "meta"
class TestAsrNeedsItsAdapter:
def test_asr_without_asr_lora_is_no_adapter(self):
"""trainer/asr.py attaches only with asr_lora on (#1117 review, round 3):
without it that run full-fine-tunes, so the check has nothing to attach."""
off = SimpleNamespace(task="asr", backend="transformers", training=SimpleNamespace(
lora=SimpleNamespace(r=8), asr_lora=False))
on = SimpleNamespace(task="asr", backend="transformers", training=SimpleNamespace(
lora=SimpleNamespace(r=8), asr_lora=True))
assert plan_adapter(off) == (False, "asr_lora is off --- that trainer full-fine-tunes")
assert plan_adapter(on) == (True, "")
def _tiny_llama():
from transformers import LlamaConfig
return LlamaConfig(
vocab_size=64, hidden_size=16, intermediate_size=32, num_hidden_layers=2,
num_attention_heads=2, num_key_value_heads=2, pad_token_id=0,
)
@pytest.fixture
def plain_consoles(monkeypatch):
"""Both of the command's consoles with no colour system: Rich then emits no
escape of its own, so any ESC byte in the output came from the config."""
from rich.console import Console
import soup_cli.commands.recipes as recipes_cmd
monkeypatch.setattr(recipes_cmd, "console", Console(color_system=None, width=400))
monkeypatch.setattr(
recipes_cmd, "err_console", Console(stderr=True, color_system=None, width=400)
)
@pytest.mark.usefixtures("plain_consoles")
class TestTheReportIsSafeForATerminal:
"""#1117 review, round 3: peft quotes a string target_modules verbatim, and the
"Cannot attach" table printed it as Rich markup with control bytes intact."""
def _verify(self, tmp_path, monkeypatch, targets):
import json as _json
from typer.testing import CliRunner
from soup_cli.cli import app
monkeypatch.setattr(
"soup_cli.utils.attach_preflight.load_hf_config", lambda _base: _tiny_llama()
)
path = tmp_path / "soup.yaml"
path.write_text(
"base: org/m\ntask: sft\ndata:\n train: ./x.jsonl\n format: alpaca\n"
f"training:\n lora:\n r: 8\n target_modules: {_json.dumps(targets)}\n",
encoding="utf-8",
)
return CliRunner().invoke(app, ["recipes", "verify", "--config", str(path)])
def test_markup_in_the_config_is_printed_literally(self, tmp_path, monkeypatch):
"""'[/]' used to raise MarkupError after the verdict, turning a finding
into a crash (exit 1)."""
result = self._verify(tmp_path, monkeypatch, ".*[/]nothing")
assert result.exit_code == 2, (result.output, result.exception)
assert "[/]nothing" in result.output
def test_a_character_class_survives(self, tmp_path, monkeypatch):
"""'[qkv]' was swallowed as a markup tag."""
result = self._verify(tmp_path, monkeypatch, r".*\.[qkv]_nothing")
assert result.exit_code == 2
assert "[qkv]_nothing" in result.output
def test_control_bytes_do_not_reach_the_terminal(self, tmp_path, monkeypatch):
"""'\\x1bc' is a full terminal reset."""
result = self._verify(tmp_path, monkeypatch, "x\x1bcEVIL_nothing")
assert result.exit_code == 2
assert "\x1b" not in result.output
assert "EVIL_nothing" in result.output
@pytest.mark.usefixtures("plain_consoles")
class TestConfigNamesAreSafeToo:
def test_a_missing_config_path_prints_literally(self):
from typer.testing import CliRunner
from soup_cli.cli import app
# Bare, not joined onto tmp_path: Windows pathlib splits at the "/" and
# the literal "[/]x" never reaches the output (#1117 review).
result = CliRunner().invoke(
app, ["recipes", "verify", "--config", "[/]x\x1b[2J.yaml"]
)
assert result.exit_code == 3
assert "[/]x" in result.output and "\x1b" not in result.output
def test_a_vision_tower_name_prints_literally(self, tmp_path, monkeypatch):
"""The "adapted a vision tower" line lists config names, which come from
the file name in --config mode."""
from transformers import CLIPVisionConfig, LlamaConfig, LlavaConfig
from typer.testing import CliRunner
from soup_cli.cli import app
vl = LlavaConfig(
text_config=LlamaConfig(
vocab_size=64, hidden_size=16, intermediate_size=32,
num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=2,
pad_token_id=0,
),
vision_config=CLIPVisionConfig(
hidden_size=16, intermediate_size=32, num_hidden_layers=2,
num_attention_heads=2, image_size=32, patch_size=16,
),
image_token_index=63,
)
monkeypatch.setattr("soup_cli.utils.attach_preflight.load_hf_config", lambda _b: vl)
path = tmp_path / "[bold]v.yaml"
path.write_text(
"base: org/vl\ntask: sft\nmodality: vision\n"
"data:\n train: ./x.jsonl\n format: llava\n"
'training:\n lora:\n r: 8\n target_modules: ".*(q_proj|v_proj)"\n',
encoding="utf-8",
)
result = CliRunner().invoke(app, ["recipes", "verify", "--config", str(path)])
assert result.exit_code == 0, (result.output, result.exception)
assert "adapted a vision tower: [bold]v.yaml" in " ".join(result.output.split())
def test_a_parse_error_prints_literally(self, tmp_path, monkeypatch):
"""The parse-error line carries the file name and pydantic's text, which
quotes the key: both are user-controlled."""
from typer.testing import CliRunner
from soup_cli.cli import app
# "[bold]", not "[/]": a "/" in the name is a separator on Windows.
path = tmp_path / "[bold]p.yaml"
path.write_text(
'base: org/m\ntask: sft\ndata:\n train: ./x.jsonl\n format: alpaca\n"[/]x": 1\n',
encoding="utf-8",
)
# Bare name: on Windows the full path puts a "\\" before "[bold]", Rich reads
# "\\[" as an escaped bracket, and a raw print would pass (#1117 review).
monkeypatch.chdir(tmp_path)
result = CliRunner().invoke(app, ["recipes", "verify", "--config", path.name])
assert result.exit_code == 3, (result.output, result.exception)
out = " ".join(result.output.split())
assert "Config does not parse" in out and "[/]x" in out and "[bold]p.yaml" in out
class TestAMissingLibraryIsAnEnvironmentError:
@pytest.mark.parametrize("lib", ["torch", "transformers", "peft"])
@pytest.mark.parametrize(
"error",
[
ModuleNotFoundError("No module named 'x'"),
# Installed but broken: find_spec finds it, the import raises. A torch
# DLL that will not load on Windows is an OSError (#1117 review).
OSError("DLL load failed"),
],
ids=["absent", "broken"],
)
def test_it_exits_1_naming_the_extra(self, tmp_path, monkeypatch, lib, error):
"""Without these the per-stage catch-alls turned a missing library into a
verdict: unverified with exit 0, or cannot_attach with exit 2."""
import importlib
from typer.testing import CliRunner
from soup_cli.cli import app
from tests.conftest import strip_ansi
real = importlib.import_module
def fake(name, *a, **k):
if name == lib:
raise error
return real(name, *a, **k)
monkeypatch.setattr(importlib, "import_module", fake)
path = tmp_path / "soup.yaml"
path.write_text(
"base: org/m\ntask: sft\ndata:\n train: ./x.jsonl\n format: alpaca\n",
encoding="utf-8",
)
result = CliRunner().invoke(app, ["recipes", "verify", "--config", str(path)])
assert result.exit_code == 1
out = " ".join(strip_ansi(result.output).split())
assert f"needs {lib}" in out and 'pip install "soup-cli[train]"' in out
assert type(error).__name__ in out
class TestTheAdvisoryGoesToStderr:
def test_json_stdout_parses_with_a_partial_coverage_advisory(self, tmp_path, monkeypatch):
"""#1164's resolver prints a partial-coverage advisory for
``granitemoehybrid``. It must reach the user on stderr and leave --json's
stdout a document."""
import json as _json
from typer.testing import CliRunner
from soup_cli.cli import app
from tests.test_issue1122_glm4_granite_moe_targets import _granite_config
monkeypatch.setattr(
"soup_cli.utils.attach_preflight.load_hf_config",
lambda _base: _granite_config(num_hidden_layers=10, attention_every=5),
)
path = tmp_path / "soup.yaml"
path.write_text(
"base: org/granite\ntask: sft\ndata:\n train: ./x.jsonl\n format: alpaca\n"
"training:\n lora:\n r: 8\n target_modules: auto\n",
encoding="utf-8",
)
# Ask this runner: Typer may use a vendored implementation whose API
# is independent of the installed standalone Click version.
import inspect
supports_separate_stderr = "mix_stderr" in inspect.signature(CliRunner).parameters
kwargs = {"mix_stderr": False} if supports_separate_stderr else {}
runner = CliRunner(**kwargs)
result = runner.invoke(app, ["recipes", "verify", "--config", str(path), "--json"])
rows = _json.loads(result.stdout)
assert rows[0]["model_type"] == "granitemoehybrid"
assert rows[0]["verdict"] == "attaches", rows[0]["detail"]
assert "Partial LoRA coverage" in result.stderr
assert "Partial LoRA coverage" not in result.stdout
class TestNothingToVerify:
def test_an_empty_scope_exits_3(self, monkeypatch):
from typer.testing import CliRunner
import soup_cli.recipes.catalog as catalog
from soup_cli.cli import app
monkeypatch.setattr(catalog, "RECIPES", {})
result = CliRunner().invoke(app, ["recipes", "verify", "--no-templates"])
assert result.exit_code == 3, result.output
class TestTheSkeletonKeepsAnExpertLayer:
"""A local CodeRabbit review: shrinking every config to two layers kept only
dense layers on DeepSeek-V3 and GLM-5.1 (``first_k_dense_replace: 3``), so their
``moe_lora`` recipes were reported ATTACHES with zero experts built -- the
thing those recipes exist for was never checked."""
@pytest.mark.parametrize(
("fields", "depth"),
[
({"first_k_dense_replace": 3, "moe_layer_freq": 1}, 4),
({"first_k_dense_replace": 1, "moe_layer_freq": 1}, 2),
# Frequency 2 skips layer 1: only this case tells the moe_layer_freq
# rule from its absence (#1117 review).
({"first_k_dense_replace": 1, "moe_layer_freq": 2}, 3),
({"decoder_sparse_step": 2}, 2),
# A step whose first sparse layer lies past the 2-layer minimum: the
# step-2 case alone could not tell the rule from its absence.
({"decoder_sparse_step": 4}, 4),
({"decoder_sparse_step": 1, "mlp_only_layers": [0, 1, 2]}, 4),
({"interleave_moe_layer_step": 4}, 4),
({}, 2),
],
)
def test_the_depth_reaches_the_first_sparse_layer(self, fields, depth):
config = SimpleNamespace(num_hidden_layers=61, **fields)
assert shrink_for_preflight(config).num_hidden_layers == depth
def test_a_shallow_model_is_not_deepened(self):
config = SimpleNamespace(num_hidden_layers=2, first_k_dense_replace=3, moe_layer_freq=1)
assert shrink_for_preflight(config).num_hidden_layers == 2
def test_a_deepseek_v3_moe_lora_config_reaches_its_experts(self):
from transformers import DeepseekV3Config
from soup_cli.utils.attach_preflight import build_on_meta
hf = DeepseekV3Config(
vocab_size=64, hidden_size=32, intermediate_size=32, moe_intermediate_size=16,
num_hidden_layers=61, num_attention_heads=2, num_key_value_heads=2,
n_routed_experts=4, num_experts_per_tok=2, n_group=1, topk_group=1,
first_k_dense_replace=3, moe_layer_freq=1, q_lora_rank=16, kv_lora_rank=16,
qk_rope_head_dim=8, qk_nope_head_dim=8, v_head_dim=8, pad_token_id=0,
)
check = check_attach(
"r", _real_cfg(moe_lora=True), load_hf_config=lambda _b: hf,
build_model=lambda c, cls: build_on_meta(c, cls),
)
assert check.verdict is Verdict.ATTACHES, check.detail
assert check.expert_modules > 0, "the skeleton built no expert layer"
class TestTheSkeletonKeepsEveryLayerKind:
"""#1117 review: on a hybrid decoder the first two layers can hold no
attention layer, so a config the trainer attaches read cannot_attach."""
def test_a_hybrid_skeleton_keeps_an_attention_layer(self):
from soup_cli.utils.attach_preflight import build_on_meta
from tests.test_issue1122_glm4_granite_moe_targets import _granite_config
hf = _granite_config(num_hidden_layers=10, attention_every=5) # attention at 2 and 7
check = check_attach(
"g", _real_cfg(), load_hf_config=lambda _b: hf, build_model=build_on_meta
)
assert check.verdict is Verdict.ATTACHES, check.detail
assert check.adapted == 4 # q/k/v/o of the first attention layer
@pytest.mark.parametrize(
("kinds", "depth"),
[
(["mamba"] * 5 + ["attention"] + ["mamba"] * 10, 6),
(["linear_attention"] * 3 + ["full_attention"] * 13, 4),
(["full_attention"] * 16, 2),
],
)
def test_the_depth_reaches_the_first_layer_of_each_kind(self, kinds, depth):
config = SimpleNamespace(num_hidden_layers=len(kinds), layer_types=kinds)
assert shrink_for_preflight(config).num_hidden_layers == depth
class TestTheLoadersWarningsLeaveJsonStdoutAlone:
def test_json_stdout_parses_when_the_loader_warns(self, tmp_path, monkeypatch):
"""The config loader prints a deprecated value (#759) or a staged field (#808)
as a warning on stdout. Under --json that line came before the document."""
import inspect
import json as _json
from transformers import LlamaConfig
from typer.testing import CliRunner
from soup_cli.cli import app
from tests.conftest import strip_ansi
tiny = LlamaConfig(
vocab_size=64, hidden_size=16, intermediate_size=32, num_hidden_layers=2,
num_attention_heads=2, num_key_value_heads=2,
)
monkeypatch.setattr("soup_cli.utils.attach_preflight.load_hf_config", lambda _b: tiny)
path = tmp_path / "soup.yaml"
path.write_text(
"base: org/m\ntask: sft\ndata:\n train: ./x.jsonl\n format: alpaca\n"
# soup autopilot wrote this line into every config it generated (#759).
" remove_unused_columns: true\n"
"training:\n lora:\n r: 8\n target_modules: [q_proj, v_proj]\n",
encoding="utf-8",
)
supports_separate_stderr = "mix_stderr" in inspect.signature(CliRunner).parameters
kwargs = {"mix_stderr": False} if supports_separate_stderr else {}
runner = CliRunner(**kwargs)
result = runner.invoke(app, ["recipes", "verify", "--config", str(path), "--json"])
assert result.exit_code == 0, (result.output, result.exception)
rows = _json.loads(result.stdout)
assert rows[0]["verdict"] == "attaches", rows[0]["detail"]
assert "remove_unused_columns" in " ".join(strip_ansi(result.stderr).split())