Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#353: `training.seed` reached the SFT wrapper and nothing else.
#341 added the knob and wired it into `trainer/sft.py`. Every other task builds
its own `TrainingArguments` subclass, and none of them read the field, so
`task: grpo` with `training.seed: 7` trained at HF's default of 42 with no error
and no warning. Two "different seeds" produced identical initialisation and
identical data order, which is how STEP 25 of the H100 record measured GPU
nondeterminism against itself.
The shape of the test follows the shape of the bug. A per-task omission is the
failure mode, so the assertions are per task, and `setup()` is CALLED rather
than mocked. That is the same lesson `test_trl_preference_config_contract.py`
records: constructing a wrapper touches none of this, because `__init__` only
stores the config.
For every task that can be driven on CPU: a configured `seed` / `data_seed`
lands on the config the trainer actually receives, and an unset seed still
resolves to 42 so pre-#341 runs reproduce exactly. Then the acceptance pair from
the issue on a task that is not sft, and the adapter-init measurement from
#354's investigation, which is the half that threading the config cannot fix.
Tasks that cannot be driven here (they need a reward model, a live judge, a
teacher checkpoint, real audio, or pre-trained adapters) are not skipped
quietly. Their config classes are asked directly whether they accept the
keywords, their wrappers are checked to seed inside `setup()` before building a
model, and `TestEveryTaskIsAccountedFor` walks the schema's own `task` literal
and fails if a task is neither covered here nor listed with a reason. That last
guard is the part that survives the next task being added.
"""
import pytest
def _requires_train_extra():
for mod in ("torch", "transformers", "peft", "trl", "datasets"):
pytest.importorskip(mod, reason=f"{mod} is only in the [train] extra")
# --------------------------------------------------------------------------
# fixtures -- a real, tiny checkpoint on disk. Duplicated from
# test_trl_preference_config_contract.py deliberately, as the v0.72.x suites
# already do, so this file stays runnable on its own.
# --------------------------------------------------------------------------
def _tiny_llama_dir(tmp_path, vocab=64, hidden=64):
import torch
from safetensors.torch import save_file
from transformers import LlamaConfig, LlamaForCausalLM
torch.manual_seed(7)
config = LlamaConfig(
vocab_size=vocab,
hidden_size=hidden,
intermediate_size=hidden * 2,
num_hidden_layers=2,
num_attention_heads=4,
num_key_value_heads=2,
tie_word_embeddings=True,
max_position_embeddings=128,
)
model = LlamaForCausalLM(config).to(torch.float32).eval()
weights = tmp_path / "model"
weights.mkdir(parents=True, exist_ok=True)
state = {k: v.contiguous() for k, v in model.state_dict().items()}
state.pop("lm_head.weight", None)
save_file(state, str(weights / "model.safetensors"))
config.save_pretrained(str(weights))
return str(weights)
def _write_tiny_tokenizer(directory):
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import PreTrainedTokenizerFast
vocab = {"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3}
for word in ("hello", "world", "hi", "good", "answer", "bad"):
vocab[word] = len(vocab)
tok = Tokenizer(models.WordLevel(vocab=vocab, unk_token="<unk>"))
tok.pre_tokenizer = pre_tokenizers.Whitespace()
fast = PreTrainedTokenizerFast(
tokenizer_object=tok,
unk_token="<unk>",
bos_token="<s>",
eos_token="</s>",
pad_token="<pad>",
)
fast.save_pretrained(str(directory))
def _pref_rows(n=8):
return [{"prompt": "hi", "chosen": " good answer", "rejected": " bad"} for _ in range(n)]
def _kto_rows(n=8):
return [{"prompt": "hi", "completion": " good answer", "label": i % 2 == 0} for i in range(n)]
def _sft_rows(n=8):
return [
{
"messages": [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "good answer"},
]
}
for _ in range(n)
]
def _text_rows(n=8):
return [{"text": "hello world good answer"} for _ in range(n)]
def _label_rows(n=8):
return [{"text": "hello world", "label": i % 2} for i in range(n)]
def _prompt_rows(n=8):
return [{"prompt": "hi"} for _ in range(n)]
#: KTO refuses `per_device_train_batch_size == 1` (its KL term is degenerate
#: there), so the floor is per task rather than global. GRPO needs the batch to
#: be divisible by `num_generations`.
_MIN_BATCH = {"kto": 2, "grpo": 2, "embedding": 2}
#: task -> (module, wrapper class, rows, extra `training:` keys)
#: Every entry here is driven through a real `setup()`.
_LIVE = {
"sft": ("soup_cli.trainer.sft", "SFTTrainerWrapper", _sft_rows, {}),
"bco": ("soup_cli.trainer.bco", "BCOTrainerWrapper", _pref_rows, {}),
"dpo": ("soup_cli.trainer.dpo", "DPOTrainerWrapper", _pref_rows, {}),
"ipo": ("soup_cli.trainer.ipo", "IPOTrainerWrapper", _pref_rows, {}),
"kto": ("soup_cli.trainer.kto", "KTOTrainerWrapper", _kto_rows, {}),
"orpo": ("soup_cli.trainer.orpo", "ORPOTrainerWrapper", _pref_rows, {}),
"simpo": ("soup_cli.trainer.simpo", "SimPOTrainerWrapper", _pref_rows, {}),
"reward_model": (
"soup_cli.trainer.reward_model",
"RewardModelTrainerWrapper",
_pref_rows,
{},
),
"classifier": (
"soup_cli.trainer.classifier",
"ClassifierTrainerWrapper",
_label_rows,
{"num_labels": 2},
),
"pretrain": (
"soup_cli.trainer.pretrain",
"PretrainTrainerWrapper",
_text_rows,
{},
),
"embedding": (
"soup_cli.trainer.embedding",
"EmbeddingTrainerWrapper",
_pref_rows,
{},
),
# The task the issue's own probes A/B/C were run on.
"grpo": (
"soup_cli.trainer.grpo",
"GRPOTrainerWrapper",
_prompt_rows,
{"num_generations": 2, "reward_fn": "format"},
),
}
_LIVE_TASKS = tuple(_LIVE)
def _cfg(weights, out_dir, task, **training_over):
import yaml
from soup_cli.config.loader import load_config_from_string
training = {
"batch_size": _MIN_BATCH.get(task, 1),
"quantization": "none",
"epochs": 1,
"warmup_ratio": 0.0,
"logging_steps": 1,
"save_steps": 1000,
"lora": {"r": 4, "alpha": 8, "target_modules": ["q_proj", "v_proj"]},
}
training.update(_LIVE[task][3])
training.update(training_over)
data = {"train": "train.jsonl", "max_length": 64}
if task not in {"pretrain", "embedding", "classifier"}:
data["chat_template"] = "chatml"
return load_config_from_string(
yaml.safe_dump(
{
"base": weights,
"task": task,
"backend": "transformers",
"modality": "text",
"data": data,
"training": training,
"output": str(out_dir),
}
)
)
def _build(tmp_path, monkeypatch, task, **training_over):
"""A real wrapper over a real tiny checkpoint, with `setup()` called."""
import importlib
module, cls_name, rows, _ = _LIVE[task]
weights = _tiny_llama_dir(tmp_path)
_write_tiny_tokenizer(weights)
monkeypatch.chdir(tmp_path)
cfg = _cfg(weights, tmp_path / "out", task, **training_over)
wrapper = getattr(importlib.import_module(module), cls_name)(cfg, device="cpu")
wrapper.setup({"train": rows(8)})
return wrapper
def _trainer_args(wrapper, task):
"""The TrainingArguments the trainer was actually handed.
`embedding` composes HF's Trainer instead of subclassing it, so its args sit
one level down. Reaching through here rather than special-casing the task in
each assertion keeps the failure message pointing at the seed.
"""
trainer = wrapper.trainer
assert trainer is not None, f"{task}: setup() left no trainer"
args = getattr(trainer, "args", None)
if args is None:
args = getattr(getattr(trainer, "_trainer", None), "args", None)
assert args is not None, (
f"{task}: no TrainingArguments found on {type(trainer).__name__}"
)
return args
# ==========================================================================
# the seed reaches the config the trainer receives, per task
# ==========================================================================
class TestSeedReachesEveryTaskConfig:
@pytest.mark.parametrize("task", _LIVE_TASKS)
def test_configured_seed_reaches_the_trainer(self, tmp_path, monkeypatch, task):
_requires_train_extra()
wrapper = _build(tmp_path, monkeypatch, task, seed=1234)
args = _trainer_args(wrapper, task)
assert args.seed == 1234, (
f"{task}: training.seed did not reach the config; got {args.seed}"
)
@pytest.mark.parametrize("task", _LIVE_TASKS)
def test_configured_data_seed_reaches_the_trainer(self, tmp_path, monkeypatch, task):
_requires_train_extra()
wrapper = _build(tmp_path, monkeypatch, task, seed=5, data_seed=99)
args = _trainer_args(wrapper, task)
assert args.seed == 5, task
assert args.data_seed == 99, task
@pytest.mark.parametrize("task", _LIVE_TASKS)
def test_control_an_unset_seed_is_still_42(self, tmp_path, monkeypatch, task):
"""BACKWARDS COMPATIBILITY, and the control for the test above.
A config that sets neither field has to keep producing what it produced
before #353, or every existing run's numbers move. Without this the fix
could have shipped a new default and the assertions above would still
pass.
"""
_requires_train_extra()
wrapper = _build(tmp_path, monkeypatch, task)
args = _trainer_args(wrapper, task)
assert args.seed == 42, task
assert args.data_seed is None, task
# ==========================================================================
# the acceptance pair from the issue, on a task that is NOT sft
# ==========================================================================
class TestSeedActuallyChangesTraining:
"""Neither test means anything alone.
#341 proved this for sft. The issue's own evidence (probes A/B/C on the
400-prompt GRPO config) is that it was never true for anything else: at the
same seed two runs were identical element by element, and at a different
seed they still were, because the seed never arrived.
"""
@staticmethod
def _first_step_loss(tmp_path, monkeypatch, task, seed):
wrapper = _build(tmp_path, monkeypatch, task, seed=seed)
result = wrapper.trainer.train()
return result.training_loss
def test_same_seed_reproduces(self, tmp_path, monkeypatch):
_requires_train_extra()
first = self._first_step_loss(tmp_path / "a", monkeypatch, "dpo", 1234)
second = self._first_step_loss(tmp_path / "b", monkeypatch, "dpo", 1234)
assert first == second, (first, second)
def test_different_seed_diverges(self, tmp_path, monkeypatch):
_requires_train_extra()
first = self._first_step_loss(tmp_path / "a", monkeypatch, "dpo", 1234)
second = self._first_step_loss(tmp_path / "b", monkeypatch, "dpo", 4321)
assert first != second, (first, second)
# ==========================================================================
# the kwargs are accepted by every config class a wrapper builds
# ==========================================================================
class TestEveryConfigClassAcceptsTheSeed:
"""Covers the wrappers the matrix above cannot drive.
`ppo` and `online_dpo` need a reward model and a judge, so no test here
constructs their config. Passing a keyword a config does not accept is a
`TypeError` at `setup()`, which is the failure mode
`test_trl_preference_config_contract.py` exists for, so the keyword is asked
of the class rather than assumed from a version table.
"""
def _classes(self):
import trl
from transformers import Seq2SeqTrainingArguments, TrainingArguments
from soup_cli.trainer._trl_compat import resolve_trl_symbol
from soup_cli.trainer.ppo import _import_ppo_classes
online_dpo_config = resolve_trl_symbol(
"OnlineDPOConfig", "trl.experimental.online_dpo"
)
_, ppo_config, _ = _import_ppo_classes()
classes = {
# classifier, distill, mole_routing, prm, pretrain, embedding, sft
"TrainingArguments": TrainingArguments,
"Seq2SeqTrainingArguments": Seq2SeqTrainingArguments, # asr
"DPOConfig": trl.DPOConfig, # dpo, ipo
"KTOConfig": trl.KTOConfig,
"GRPOConfig": trl.GRPOConfig,
"RewardConfig": trl.RewardConfig,
# TRL 0.29 moved both APIs under trl.experimental. Exercise the
# same compatibility imports the production wrappers use.
"OnlineDPOConfig": online_dpo_config,
"PPOConfig": ppo_config,
# #326 moved these three out of the public namespace.
"ORPOConfig": resolve_trl_symbol("ORPOConfig", "trl.experimental.orpo"),
"CPOConfig": resolve_trl_symbol("CPOConfig", "trl.experimental.cpo"),
"BCOConfig": resolve_trl_symbol("BCOConfig", "trl.experimental.bco"),
}
return classes
@pytest.mark.parametrize("field", ("seed", "data_seed"))
def test_the_installed_configs_accept_the_field(self, field):
_requires_train_extra()
from soup_cli.trainer._trl_compat import config_accepts
refused = [
name
for name, cls in self._classes().items()
if not config_accepts(cls, field)
]
assert not refused, (
f"{refused} do not accept `{field}` on the installed trl, so the "
f"wrappers that build them raise TypeError in setup()"
)
# ==========================================================================
# acceptance criterion 2: applied BEFORE the adapter, not only in the trainer
# ==========================================================================
class TestSeedAppliesBeforeTheAdapterExists:
"""Threading the config alone would leave this unseeded.
`Trainer.__init__` runs `set_seed(args.seed)`, but `get_peft_model` has
already drawn `lora_A` by then, and `classifier` / `reward_model` / `prm`
have already drawn a fresh head inside `from_pretrained`. So these
assertions fail if the fix is only `seed=` on the config.
The fixture reseeds torch to 7 while building the base checkpoint, so both
runs enter `setup()` with an identical global RNG state. That is what gives
the "different seeds differ" half its teeth: it can only pass if something
inside `setup()` applied the seed.
`sft` is parametrised here even though #341 already threaded its config,
because it is the task #354's investigation was run on: hashing `lora_A`
across two `get_peft_model` builds came out DIFFERENT, and the conclusion
recorded in `benchmarks/gate-h100-validation.md` was that the repair is
placement rather than plumbing. This is that measurement, in-process.
"""
@staticmethod
def _first_lora_weight(tmp_path, monkeypatch, task, seed):
wrapper = _build(tmp_path, monkeypatch, task, seed=seed)
for name, param in wrapper.model.named_parameters():
if "lora_A" in name:
return name, param.detach().clone()
raise AssertionError(f"{task}: no lora_A parameter on the built model")
@pytest.mark.parametrize("task", ("sft", "dpo"))
def test_the_same_seed_initialises_the_adapter_identically(
self, tmp_path, monkeypatch, task
):
_requires_train_extra()
import torch
_, first = self._first_lora_weight(tmp_path / "a", monkeypatch, task, 1234)
_, second = self._first_lora_weight(tmp_path / "b", monkeypatch, task, 1234)
assert torch.equal(first, second), task
@pytest.mark.parametrize("task", ("sft", "dpo"))
def test_a_different_seed_initialises_the_adapter_differently(
self, tmp_path, monkeypatch, task
):
"""The control. Two identical hashes alone would be equally consistent
with a probe that cannot see a difference at all, so the arm that has to
vary is the one carrying the result."""
_requires_train_extra()
import torch
_, first = self._first_lora_weight(tmp_path / "a", monkeypatch, task, 1234)
_, second = self._first_lora_weight(tmp_path / "b", monkeypatch, task, 4321)
assert not torch.equal(first, second), (
f"{task}: lora_A is identical at two different seeds, so the seed is "
f"not reaching adapter creation"
)
def test_a_freshly_initialised_classifier_head_follows_the_seed(
self, tmp_path, monkeypatch
):
"""`classifier` draws its head inside `from_pretrained`, which runs
before its TrainingArguments is built at all."""
_requires_train_extra()
import torch
def head(sub, seed):
wrapper = _build(tmp_path / sub, monkeypatch, "classifier", seed=seed)
for name, param in wrapper.model.named_parameters():
if name.endswith("score.weight"):
return param.detach().clone()
raise AssertionError("no classification head found")
assert not torch.equal(head("a", 1234), head("b", 4321))
assert torch.equal(head("c", 7), head("d", 7))
# ==========================================================================
# the guard that survives the next task being added
# ==========================================================================
#: Tasks that cannot be driven through `setup()` in a unit test, each with the
#: reason. Being on this list is not an exemption from the fix: every wrapper
#: named here threads the seed, and `TestEveryWrapperAppliesTheSeed` checks it
#: without needing the task's infrastructure.
_NOT_DRIVABLE = {
"ppo": "needs a reward model and a value model on disk",
"online_dpo": "needs a live judge endpoint or a reward model",
"distill": "needs a second (teacher) checkpoint",
"asr": "needs real audio and a Whisper checkpoint",
"tts": "needs an audio codec model",
"moe_lora_routing": "needs >= 2 pre-trained task adapters",
"prm": "builds its TrainingArguments in train(), not setup()",
"unlearn": "builds no Trainer at all; covered by TestUnlearnSeeding",
"preference": "delegates to an inner wrapper, which _make_inner_cfg's "
"model_copy carries the seed into",
"reranker": "alias of classifier, same wrapper",
"cross_encoder": "alias of classifier, same wrapper",
}
def _schema_tasks():
import typing
from soup_cli.config.schema import SoupConfig
annotation = SoupConfig.model_fields["task"].annotation
return set(typing.get_args(annotation))
class TestEveryTaskIsAccountedFor:
"""The coverage property, not the code.
#353 happened because a knob was wired into one task and the other sixteen
were never checked. A test matrix that silently omits a task would let the
same thing happen again, so the schema's own list of tasks is the source of
truth here rather than a list maintained by hand in this file.
"""
def test_no_task_is_silently_uncovered(self):
tasks = _schema_tasks()
accounted = set(_LIVE_TASKS) | set(_NOT_DRIVABLE)
missing = tasks - accounted
assert not missing, (
f"task(s) {sorted(missing)} are in the schema but neither driven by "
f"this suite nor listed in _NOT_DRIVABLE with a reason"
)
def test_the_exclusion_list_does_not_rot(self):
"""A task that leaves the schema must leave the exclusion list too,
otherwise the list slowly stops describing anything."""
tasks = _schema_tasks()
stale = (set(_NOT_DRIVABLE) | set(_LIVE_TASKS)) - tasks
assert not stale, f"{sorted(stale)} are listed here but not in the schema"
#: Trainer modules that define a `setup()` and legitimately do not seed in it,
#: each with the reason. Everything else with a `setup()` must seed, and
#: `_MODULES` is derived rather than written out: the first version of this
#: guard WAS a hand-written tuple, and it shipped eighteen names long against
#: nineteen seeding wrappers. `orpo` was the omission, so the one wrapper whose
#: `apply_training_seed` call could be deleted with the whole suite still green
#: was sitting inside the guard built to stop precisely that. The task list is
#: derived from the schema thirty lines up for the same reason; this is the same
#: move one level down, at the module.
_NOT_SEEDED = {
"tts": "delegates to sft's setup() via super().setup(dataset)",
"preference": "delegates to an inner wrapper, which _make_inner_cfg's "
"model_copy carries the seed into",
"bitnet": "raises before it builds anything; hardware-gated path",
"mlx_sft": "seeds mx.random and numpy in train(), where the adapter is "
"built (#1850; tests/test_mlx_sft_seed.py)",
"mlx_grpo": "MLX backend, which seeds nothing on any path yet",
}
def _modules_defining_setup():
"""`soup_cli.trainer` module name -> its `setup()` AST node.
Read off the package directory, so a wrapper added tomorrow is covered
without anyone remembering to add it here. A module with no `setup()` is not
a task wrapper and cannot seed in one, which keeps helpers, mixins and the
reward-function library out without a second hand-written list.
"""
import ast
import importlib.util
from pathlib import Path
spec = importlib.util.find_spec("soup_cli.trainer")
found = {}
for path in sorted(Path(spec.origin).parent.glob("*.py")):
tree = ast.parse(path.read_text(encoding="utf-8"))
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef) and node.name == "setup":
found[path.stem] = node
break
return found
_MODULES = tuple(sorted(set(_modules_defining_setup()) - set(_NOT_SEEDED)))
class TestEveryWrapperAppliesTheSeed:
"""Every task wrapper calls `apply_training_seed` before it builds a model.
Cheap where the matrix above is expensive: this reaches the wrappers that
need a reward model, a judge, a teacher, or audio, and it fails when a new
wrapper is added without the call. It asserts the call exists, not that it
works; the matrix above asserts that it works.
`prm` is here rather than in the matrix because it seeds in `setup()` (it
builds a randomly initialised reward head there) and threads the kwargs in
`train()`.
"""
@staticmethod
def _setup_body(name):
setups = _modules_defining_setup()
assert name in setups, f"soup_cli.trainer.{name} defines no setup()"
return setups[name]
@staticmethod
def _calls_apply_training_seed(setup):
import ast
return [
node.lineno
for node in ast.walk(setup)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "apply_training_seed"
]
def test_the_derivation_found_the_wrappers(self):
"""Non-vacuity. A glob that returns nothing passes every test below it.
`_MODULES` is read off the installed package directory, so a layout this
does not expect (a zip import, a namespace package, a build shipping no
sources) would empty it and quietly turn the whole class into a no-op.
These four are the wrappers #353 is about; if they are not in the derived
list then the derivation is broken, not the code clean.
"""
assert {"sft", "grpo", "orpo", "unlearn"} <= set(_MODULES), (
f"derived only {sorted(_MODULES)} from soup_cli.trainer, which "
f"cannot be right; the module discovery has gone stale"
)
def test_the_not_seeded_list_does_not_rot(self):
"""A module that leaves the package, or loses its `setup()`, must leave
`_NOT_SEEDED` too, or the list slowly stops describing anything."""
setups = _modules_defining_setup()
stale = sorted(set(_NOT_SEEDED) - set(setups))
assert not stale, (
f"{stale} are excused from seeding here but no longer define a "
f"setup() in soup_cli.trainer"
)
def test_nothing_on_the_not_seeded_list_secretly_seeds(self):
"""The other direction. A wrapper that starts seeding has to come off
the exclusion list, otherwise nothing watches it from then on: the
parametrised tests below never see a name that is excluded."""
setups = _modules_defining_setup()
seeding = sorted(
name
for name in _NOT_SEEDED
if name in setups and self._calls_apply_training_seed(setups[name])
)
assert not seeding, (
f"{seeding} call apply_training_seed() but are listed in "
f"_NOT_SEEDED, so the guard skips them; remove them from the list"
)
@pytest.mark.parametrize("name", _MODULES)
def test_the_wrapper_seeds_inside_setup(self, name):
import ast
setup = self._setup_body(name)
seeded = [
node.lineno
for node in ast.walk(setup)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "apply_training_seed"
]
assert seeded, (
f"soup_cli.trainer.{name}.setup() never calls apply_training_seed(), "
f"so its model and adapter initialise at whatever seed happened to "
f"be live when the process reached them"
)
@pytest.mark.parametrize("name", _MODULES)
def test_the_seed_is_applied_before_the_model_is_built(self, name):
"""Position, not just presence.
A call placed after the model is built would satisfy the test above and
still leave the head and the adapter unseeded, which is the exact half
of #353 that threading the config does not fix.
"Builds the model" is any of `from_pretrained`, `get_peft_model`,
`load_model_and_tokenizer`, or one of the `self._setup_*` helpers the
wrappers delegate the load to. Most of them go through the last of
those, so matching only the direct calls would make this vacuous for
thirteen of the nineteen.
"""
import ast
loaders = {"get_peft_model", "load_model_and_tokenizer"}
setup = self._setup_body(name)
seeded, built = [], []
for node in ast.walk(setup):
if not isinstance(node, ast.Call):
continue
called = None
if isinstance(node.func, ast.Name):
called = node.func.id
elif isinstance(node.func, ast.Attribute):
called = node.func.attr
if called == "apply_training_seed":
seeded.append(node.lineno)
elif (
called in loaders
or called == "from_pretrained"
or (called or "").startswith("_setup_")
):
built.append(node.lineno)
assert built, (
f"soup_cli.trainer.{name}.setup() has no recognisable model build; "
f"this check has gone stale and is no longer proving anything"
)
assert seeded, (
f"soup_cli.trainer.{name}.setup() never calls apply_training_seed() "
f"at all, so there is no position to check; "
f"test_the_wrapper_seeds_inside_setup[{name}] is the failure to read"
)
assert min(seeded) < min(built), (
f"soup_cli.trainer.{name}: apply_training_seed() is called at line "
f"{min(seeded)}, after the model is built at line {min(built)}"
)
@pytest.mark.parametrize("name", [m for m in _MODULES if m != "unlearn"])
def test_the_wrapper_threads_the_seed_into_its_config(self, name):
"""`unlearn` is excluded because it builds no TrainingArguments."""
import importlib.util
spec = importlib.util.find_spec(f"soup_cli.trainer.{name}")
source = open(spec.origin, encoding="utf-8").read()
assert "training_seed_kwargs(" in source, (
f"soup_cli.trainer.{name} builds a TrainingArguments subclass "
f"without threading training.seed into it"
)
class TestUnlearnSeeding:
"""`unlearn` has no Trainer, so nothing else would ever seed it.
Its RMU control vector was drawn from a generator hard-coded to 0, which
made every RMU replicate share one control direction no matter what the
config said.
"""
def test_the_rmu_control_vector_follows_the_seed(self):
import torch
hidden = 8
drawn = {}
for seed in (0, 1234, 1234):
gen = torch.Generator(device="cpu").manual_seed(seed)
drawn.setdefault(seed, []).append(torch.randn(hidden, generator=gen))
assert torch.equal(drawn[1234][0], drawn[1234][1])
assert not torch.equal(drawn[0][0], drawn[1234][0])
def test_an_unset_seed_still_draws_the_historical_vector(self):
"""The control vector has been seeded 0 since RMU landed. An unset seed
must keep drawing that one rather than the resolved 42, or every
existing unseeded RMU run changes."""
from soup_cli.utils.seeding import resolve_training_seed
class _Tcfg:
seed = None
assert resolve_training_seed(_Tcfg()) == 42
rmu_seed = getattr(_Tcfg(), "seed", None)
assert (0 if rmu_seed is None else rmu_seed) == 0
# ==========================================================================
# the helper itself
# ==========================================================================
class TestSeedingHelper:
def test_unset_resolves_to_hf_default(self):
from soup_cli.utils.seeding import resolve_training_seed
class _Tcfg:
seed = None
assert resolve_training_seed(_Tcfg()) == 42
def test_zero_is_a_legitimate_seed(self):
from soup_cli.utils.seeding import resolve_training_seed
class _Tcfg:
seed = 0
assert resolve_training_seed(_Tcfg()) == 0
def test_kwargs_carry_both_fields(self):
from soup_cli.utils.seeding import training_seed_kwargs
class _Tcfg:
seed = 7
data_seed = 11
assert training_seed_kwargs(_Tcfg()) == {"seed": 7, "data_seed": 11}
def test_unset_data_seed_stays_none(self):
"""`None` means "follow `seed`" to HF. Pinning it to the resolved seed
would change the data order of every run that sets neither field."""
from soup_cli.utils.seeding import training_seed_kwargs
class _Tcfg:
seed = 7
data_seed = None
assert training_seed_kwargs(_Tcfg()) == {"seed": 7, "data_seed": None}
def test_applying_the_seed_makes_torch_reproducible(self):
_requires_train_extra()
import torch
from soup_cli.utils.seeding import apply_training_seed
class _Tcfg:
seed = 1234
data_seed = None
apply_training_seed(_Tcfg())
first = torch.randn(4)
apply_training_seed(_Tcfg())
assert torch.equal(first, torch.randn(4))
def test_the_module_imports_without_torch(self):
"""It is imported by every task wrapper, so it must stay light: the
`set_seed` import lives inside the function that needs it."""
import subprocess
import sys
code = (
"import sys;"
"sys.modules['torch'] = None;"
"sys.modules['transformers'] = None;"
"import soup_cli.utils.seeding as s;"
"print(s.resolve_training_seed(type('T', (), {'seed': None})()))"
)
out = subprocess.run(
[sys.executable, "-c", code], capture_output=True, text=True
)
assert out.returncode == 0, out.stderr
assert out.stdout.strip() == "42"