Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
fix(mlx): seed MLX SFT so a run reproduces (#1850) (#1855)
dryn85-maxb0a20e07c99b
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 986b0058..ac2efe11 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -512,7 +512,7 @@ jobs:# loss, gradient and training-loop tests ran on no job at all.# tests/test_ci_mlx_smoke_files.py fails when a tests/test_mlx_*.py that# needs the real library is neither listed here nor exempted there.- run: pytest -o addopts= -q tests/test_rewind_mlx.py tests/test_mlx_dora_native.py tests/test_mlx_dpo.py tests/test_mlx_orpo.py tests/test_mlx_eval_steps.py tests/test_mlx_simpo.py+ run: pytest -o addopts= -q tests/test_rewind_mlx.py tests/test_mlx_dora_native.py tests/test_mlx_dpo.py tests/test_mlx_orpo.py tests/test_mlx_eval_steps.py tests/test_mlx_simpo.py tests/test_mlx_sft_seed.pytorchao-contract:needs: plan# Not on a pull request, approval run included: it is not a required check.diff --git a/changelog.d/0.76.0/1855.fixed.md b/changelog.d/0.76.0/1855.fixed.mdnew file mode 100644index 00000000..48528f00--- /dev/null+++ b/changelog.d/0.76.0/1855.fixed.md@@ -0,0 +1,8 @@+- **MLX SFT runs are reproducible: `training.seed` and `training.data_seed` are applied+ (#1850 by @dryn85-max in #1855).** mlx-lm's batch shuffle draws from numpy's global RNG, and+ LoRA init and dropout draw from `mx.random`. Soup seeded neither, so every run of one+ `soup.yaml` trained on a different batch order, even with the seed left at the 42 the panel+ prints. `mx.random` is now seeded before the adapter is built and numpy right before training,+ and the `MLX backend ignores: training.seed` warning is gone. Runs started before this change+ cannot be reproduced. Try: run `soup train --config soup.yaml` twice with `backend: mlx` and+ compare the losses.diff --git a/docs/training.md b/docs/training.mdindex 9047551a..887616a8 100644--- a/docs/training.md+++ b/docs/training.md@@ -221,11 +221,13 @@ pick `seed` up for their samplers as well.Through v0.73.0 this reached the **SFT** trainer only, so `training.seed: 7` ona DPO or GRPO run was accepted and silently trained at 42.-The one path that still ignores both fields is the **MLX backend**-(`backend: mlx`), whose trainers seed nothing at all — MLX has its own RNG-(`mx.random`). Setting either field there now prints a warning naming it-(`MLX backend ignores: training.seed ...`) rather than accepting it in silence,-so an MLX run cannot look seeded while it is not.+On the **MLX backend** (`backend: mlx`), SFT seeds MLX's own RNG+(`mx.random`, which draws LoRA `A` and dropout) with `training.seed` before the+adapter is built, and numpy's global RNG, which mlx-lm's batch shuffle draws+from, with `training.data_seed` (or `training.seed` when that is unset) right+before training starts (#1850). Before #1850 it seeded neither and warned+`MLX backend ignores: training.seed`, so every MLX run trained on a different+batch order.**Not a determinism guarantee.** A fixed seed makes the *software* RNGreproducible. It does not make CUDA kernels bit-reproducible — non-deterministicdiff --git a/src/soup_cli/config/backend_support.py b/src/soup_cli/config/backend_support.pyindex 18bc8b7f..00196638 100644--- a/src/soup_cli/config/backend_support.py+++ b/src/soup_cli/config/backend_support.py@@ -57,11 +57,6 @@ def _default_active(value: Any) -> bool:return bool(value)-def _seed_active(value: Any) -> bool:- """0 is a valid random seed, so any non-None value is active."""- return value is not None--@dataclass(frozen=True)class SupportEntry:"""One declared gap: a field, what happens to it, and why."""@@ -89,20 +84,6 @@ _MLX_SFT: tuple[SupportEntry, ...] = (# max_grad_norm, warmup_ratio, weight_decay, optimizer, scheduler -- were# wired by #734 and #750 and are now honoured, so they are gone rather than# reclassified. The guard is what noticed; see the PR body.- SupportEntry(- "training.seed",- IGNORED,- "MLX seeds through mx.random, not this field",- trainer_reads=True,- active=_seed_active,- ),- SupportEntry(- "training.data_seed",- IGNORED,- "MLX seeds through mx.random, not this field",- trainer_reads=True,- active=_seed_active,- ),SupportEntry("training.use_galore",IGNORED,@@ -213,12 +194,9 @@ _TRANSFORMERS_AUDIO_SFT: tuple[SupportEntry, ...] = (_UNSLOTH_SFT: tuple[SupportEntry, ...] = ()-#: Fields MLX SFT warns about that have no meaning for DPO, or that MLX DPO-#: honours: the seeds (its own loop seeds ``mx.random`` and the batch order)-#: and SFT's response-masking switches (DPO supervises completions by design).+#: Fields MLX SFT warns about that have no meaning for DPO: SFT's+#: response-masking switches (DPO supervises completions by design)._NOT_DPO_GAPS = frozenset({- "training.seed",- "training.data_seed","data.train_on_messages_with_train_field","data.mask_history","data.train_on_prompt",@@ -323,7 +301,7 @@ def check_config(cfg: "SoupConfig") -> list[SupportEntry]:at their schema default, which is the difference between a useful pre-flightcheck and a wall of 275 rows. ``entry.active`` then drops a field written inits off position (e.g. ``false``). A written schema default counts as- unwritten, including a default of ``true``; ``seed: 0`` still counts as set.+ unwritten, including a default of ``true``."""backend = getattr(cfg, "backend", DEFAULT_BACKEND)modality = getattr(cfg, "modality", DEFAULT_MODALITY)diff --git a/src/soup_cli/trainer/mlx_sft.py b/src/soup_cli/trainer/mlx_sft.pyindex 837fbccc..0ff978fe 100644--- a/src/soup_cli/trainer/mlx_sft.py+++ b/src/soup_cli/trainer/mlx_sft.py@@ -456,14 +456,6 @@ class MLXSFTTrainerWrapper:"training.use_fsdp2_compile (torch.compile on FSDP2 requires ""CUDA and the transformers backend)")- # #353's fourth criterion. #381 threaded training.seed through every- # transformers task wrapper; MLX has its own RNG (mx.random) and reads- # neither field, so a seeded MLX run is silently unseeded. `is not None`- # rather than truthiness: 0 is a real seed.- if tcfg.seed is not None:- unsupported.append("training.seed (MLX seeds through mx.random)")- if tcfg.data_seed is not None:- unsupported.append("training.data_seed (MLX seeds through mx.random)")if isinstance(tcfg.gradient_checkpointing, str):unsupported.append(f"gradient_checkpointing tier {tcfg.gradient_checkpointing!r} "@@ -615,9 +607,10 @@ class MLXSFTTrainerWrapper:not a resume of training state: the step count and data positionboth restart from zero regardless of how far the checkpoint got.Say so rather than implying a full resume. Replaying the dataset- from the saved iteration is a separate, harder claim — it needs a- reproducible iteration order tied to training.seed/data_seed, which- the MLX path does not thread yet (#353) — and is out of scope here.+ from the saved iteration is a separate, harder claim. The order is+ reproducible from training.seed/data_seed since #1850, but mlx-lm+ exposes no way to start its batch iterator part-way, so it is out of+ scope here.``strict=False`` means a checkpoint saved under a different``lora.r`` or ``target_modules`` drops every tensor in silence —@@ -686,6 +679,23 @@ class MLXSFTTrainerWrapper:"MLX backend: setup(dataset) must be called before train()")+ # #1850: mlx-lm's `train()` shuffles through numpy's global RNG+ # (`iterate_batches` is called without a seed) and LoRA `A` init and+ # dropout draw from `mx.random`. The `mlx_lm.lora` CLI seeds both before+ # training; calling `train()` directly seeded neither, so every run+ # trained on a different batch order. `mx.random` is seeded here, before+ # the adapter exists; numpy right before `train()` below, so nothing in+ # between can move the batch order.+ import mlx.core as mx++ from soup_cli.utils.seeding import resolve_training_seed++ seed = resolve_training_seed(cfg.training)+ data_seed = (+ seed if cfg.training.data_seed is None else int(cfg.training.data_seed)+ )+ mx.random.seed(seed)+self._apply_lora(self.model)if resume_from_checkpoint is not None:@@ -1164,6 +1174,9 @@ class MLXSFTTrainerWrapper:train_dataset = rewind.wrap(train_dataset)train_hooks = {**train_hooks, "loss": rewind.loss}+ import numpy as np++ np.random.seed(data_seed)t0 = time.time()completed_iterations = itersstopped = Falsediff --git a/tests/test_issue1156_multimodal_mask_history_doctor.py b/tests/test_issue1156_multimodal_mask_history_doctor.pyindex a7bed053..7cff0b73 100644--- a/tests/test_issue1156_multimodal_mask_history_doctor.py+++ b/tests/test_issue1156_multimodal_mask_history_doctor.py@@ -216,7 +216,7 @@ def test_mlx_reports_unread_flags_regardless_of_modality(backend: mlxmodality: visiontraining:- seed: 42+ use_galore: truedata:train: ./data.jsonl""")@@ -224,7 +224,7 @@ def test_mlx_reports_unread_flags_regardless_of_modality(doctor(nccl=False, disk=False, config=cfg_path)out = strip_ansi(capsys.readouterr().out)assert "Config check - task=sft backend=mlx modality=vision" in out- assert "training.seed" in out+ assert "training.use_galore" in outassert ("1 setting(s) switched on here are not read on backend=mlx (modality=vision)"in outdiff --git a/tests/test_issue1330_backend_support_active_values.py b/tests/test_issue1330_backend_support_active_values.pyindex fcd29f86..0e16d8dc 100644--- a/tests/test_issue1330_backend_support_active_values.py+++ b/tests/test_issue1330_backend_support_active_values.py@@ -1,8 +1,7 @@"""Tests for Issue #1330: soup doctor --config reports settings by spelling, not value.Ensures that settings written in their inactive state (e.g. false) do not produce-false positive 'ignored' or 'rejected' rows, and that active values like seed: 0-are properly detected.+false positive 'ignored' or 'rejected' rows."""from __future__ import annotations@@ -101,8 +100,11 @@ def test_a_full_dump_reports_what_the_minimal_config_reports(key, tmp_path):assert [e.field for e in check_config(full)] == [e.field for e in check_config(minimal)]-def test_seed_zero_on_mlx_is_still_reported():- """AC3: training.seed: 0 and data_seed: 0 on MLX are still reported (0 is an active seed)."""+def test_seed_zero_on_mlx_is_not_reported_now_that_mlx_seeds():+ """AC3 was "seed: 0 on MLX is still reported (0 is an active seed)". Since+ #1850 MLX SFT honours both seeds, so neither value is a gap any more. That+ 0 is a real seed rather than "unset" is now pinned where it matters, in the+ run itself: tests/test_mlx_sft_seed.py::test_seed_zero_is_a_seed."""yaml_text = textwrap.dedent("""\base: mlx-community/Qwen2.5-0.5B-Instruct-4bit@@ -120,8 +122,8 @@ def test_seed_zero_on_mlx_is_still_reported():)cfg = load_config_from_string(yaml_text)reported_fields = {g.field for g in check_config(cfg)}- assert "training.seed" in reported_fields- assert "training.data_seed" in reported_fields+ assert "training.seed" not in reported_fields+ assert "training.data_seed" not in reported_fieldsdef test_schema_refused_settings_have_no_doctor_rows():diff --git a/tests/test_issue353_mlx_seed_warning.py b/tests/test_issue353_mlx_seed_warning.pyindex a30e648e..fe450a42 100644--- a/tests/test_issue353_mlx_seed_warning.py+++ b/tests/test_issue353_mlx_seed_warning.py@@ -1,20 +1,15 @@-"""#353's fourth acceptance criterion, for the one backend that was unreachable-when the issue was written.--#353 asks for "parse-time rejection (or a loud warning) when ``training.seed``-is set on a task that ignores it". #381 threaded the seed through all eighteen-transformers task wrappers, and named the MLX backend as the one path that-reads neither ``training.seed`` nor ``training.data_seed``: ``mlx_sft`` /-``mlx_dpo`` / ``mlx_grpo`` reference no seed and no ``mx.random`` on any path.--That gap only became reachable *between* the issue being filed and #381-landing: ``backend: mlx`` was never dispatched at all until #362 (#363), so a-user could not previously get an MLX run to ignore anything. Now they can, and-silence is the failure mode #353 exists to remove — a seeded MLX run that is-not seeded looks exactly like a seeded one until two replicates disagree.--Seeding MLX itself (``mx.random.seed``) is separate work with its own RNG and-is deliberately NOT what this pins. What is pinned is that the run says so.+"""#353's fourth acceptance criterion on MLX, and what replaced it.++#353 asked for "parse-time rejection (or a loud warning) when ``training.seed``+is set on a task that ignores it". MLX SFT read neither ``training.seed`` nor+``training.data_seed``, so it warned. #1850 seeds it for real (``mx.random``+before the adapter is built, numpy's global RNG before mlx-lm's ``train()``+shuffles), and that test said to remove the warning in the same commit. The+behaviour is pinned in tests/test_mlx_sft_seed.py (real mlx) and+tests/test_mlx_sft_seed_wiring.py (everywhere). What stays here needs no mlx:+a set seed / data_seed is no longer announced as ignored, and the+controls: the unsupported-feature list still fires, and says nothing about+seeds."""from io import StringIO@@ -50,31 +45,14 @@ def _warnings_for(monkeypatch, **training):return buffer.getvalue()-class TestTheSeedIsAnnouncedAsIgnored:- def test_a_set_seed_is_named(self, monkeypatch):- out = _warnings_for(monkeypatch, seed=7)- assert "seed" in out- # The user set `training.seed`; the message has to be greppable as that,- # not as some paraphrase they cannot search for.- assert "training.seed" in out+class TestTheSeedIsNotAnnouncedAsIgnored:+ @pytest.mark.parametrize("field", ["seed", "data_seed"])+ def test_no_ignored_warning(self, monkeypatch, field):+ """MLX SFT honours both fields since #1850, so a run that sets one must+ not be told it is ignored."""+ out = _warnings_for(monkeypatch, **{field: 11})+ assert f"training.{field}" not in out- def test_a_set_data_seed_is_named(self, monkeypatch):- out = _warnings_for(monkeypatch, data_seed=3)- assert "training.data_seed" in out-- def test_seed_zero_still_warns(self, monkeypatch):- """0 is a real seed, not "unset". A truthiness check here would drop the- one value a user picks precisely because it is conventional."""- out = _warnings_for(monkeypatch, seed=0)- assert "training.seed" in out-- def test_both_are_named_together(self, monkeypatch):- out = _warnings_for(monkeypatch, seed=7, data_seed=3)- assert "training.seed" in out- assert "training.data_seed" in out---class TestItStaysQuietOtherwise:def test_control_an_unseeded_run_says_nothing_about_seeds(self, monkeypatch):"""CONTROL. A warning that fires on every MLX run is noise, and noise ishow the next real one gets ignored."""@@ -82,38 +60,16 @@ class TestItStaysQuietOtherwise:assert "seed" not in out.lower()def test_control_the_existing_unsupported_warnings_still_fire(self, monkeypatch):- """CONTROL. The seed lines are appended to the same list that already- reports GaLore and friends, so this pins that the list was extended- rather than replaced."""+ """CONTROL. Removing the seed lines (#1850) must not have taken the rest+ of the list with them: GaLore and friends are still reported."""out = _warnings_for(monkeypatch, use_galore=True)assert "GaLore" in outassert "seed" not in out.lower()-def test_the_mlx_wrappers_still_do_not_read_the_seed(monkeypatch):- """The warning is only honest while it is true. If a future change seeds MLX- for real, this fails and whoever did it removes the warning in the same- commit instead of leaving a run lying about itself."""- import inspect-- from soup_cli.trainer import mlx_grpo, mlx_sft-- # mlx_dpo is not here: it runs its own loop and seeds `mx.random` and the- # batch order (tests/test_mlx_dpo.py pins the reproducibility), so it never- # carried the warning.- for module in (mlx_sft, mlx_grpo):- source = inspect.getsource(module)- # `_check_unsupported` legitimately mentions the field it is warning- # about, so the sources are checked for an actual RNG call instead.- assert "mx.random.seed" not in source, (- f"{module.__name__} now seeds MLX; drop the 'ignored' warning in "- f"mlx_sft._check_unsupported and delete this test"- )--@pytest.mark.parametrize("field", ["seed", "data_seed"])def test_the_schema_still_accepts_the_field_on_mlx(field):- """A warning, not a rejection: #353 allows either, and a config that is- valid on transformers should not become unloadable by switching backend."""+ """Accepted, and since #1850 honoured: a config that is valid on+ transformers stays loadable when the backend switches to MLX."""cfg = _mlx_cfg(**{field: 11})assert getattr(cfg.training, field) == 11diff --git a/tests/test_issue353_seed_every_task.py b/tests/test_issue353_seed_every_task.pyindex 20812f5e..c030821f 100644--- a/tests/test_issue353_seed_every_task.py+++ b/tests/test_issue353_seed_every_task.py@@ -513,7 +513,8 @@ _NOT_SEEDED = {"preference": "delegates to an inner wrapper, which _make_inner_cfg's ""model_copy carries the seed into","bitnet": "raises before it builds anything; hardware-gated path",- "mlx_sft": "MLX backend, which seeds nothing on any path yet",+ "mlx_sft": "seeds mx.random and numpy in train(), where the adapter is "+ "built (#1850; tests/test_mlx_sft_seed.py)","mlx_grpo": "MLX backend, which seeds nothing on any path yet",}diff --git a/tests/test_issue634_mlx_resume.py b/tests/test_issue634_mlx_resume.pyindex 9631804d..19e12bdf 100644--- a/tests/test_issue634_mlx_resume.py+++ b/tests/test_issue634_mlx_resume.py@@ -391,6 +391,8 @@ def _install_fake_mlx(monkeypatch):"""mlx = types.ModuleType("mlx")mlx_core = types.ModuleType("mlx.core")+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.+ mlx_core.random = types.SimpleNamespace(seed=lambda _seed: None)mlx_optimizers = types.ModuleType("mlx.optimizers")mlx_optimizers.AdamW = lambda **kwargs: object()# #686 wires a real LR schedule, so the MLX backend now reaches these threediff --git a/tests/test_issue683_mlx_response_masking.py b/tests/test_issue683_mlx_response_masking.pyindex 022715c3..7d75e269 100644--- a/tests/test_issue683_mlx_response_masking.py+++ b/tests/test_issue683_mlx_response_masking.py@@ -413,6 +413,8 @@ def _install_numpy_mlx(monkeypatch):them."""core = types.ModuleType("mlx.core")+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.+ core.random = types.SimpleNamespace(seed=lambda _seed: None)core.array = np.arraycore.maximum = np.maximumcore.float32 = np.float32diff --git a/tests/test_issue684_mlx_grad_accumulation.py b/tests/test_issue684_mlx_grad_accumulation.pyindex b654c947..9632a5e4 100644--- a/tests/test_issue684_mlx_grad_accumulation.py+++ b/tests/test_issue684_mlx_grad_accumulation.py@@ -31,6 +31,8 @@ def _install_fake_mlx(monkeypatch):"""mlx = types.ModuleType("mlx")mlx_core = types.ModuleType("mlx.core")+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.+ mlx_core.random = types.SimpleNamespace(seed=lambda _seed: None)mlx_optimizers = types.ModuleType("mlx.optimizers")mlx_optimizers.AdamW = lambda **kwargs: object()# #686 wires a real LR schedule, so the MLX backend now reaches these threediff --git a/tests/test_issue685_mlx_grad_checkpoint.py b/tests/test_issue685_mlx_grad_checkpoint.pyindex 7a8746b0..8a55baf0 100644--- a/tests/test_issue685_mlx_grad_checkpoint.py+++ b/tests/test_issue685_mlx_grad_checkpoint.py@@ -51,6 +51,8 @@ def _install_fake_mlx(monkeypatch):"""mlx = types.ModuleType("mlx")mlx_core = types.ModuleType("mlx.core")+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.+ mlx_core.random = types.SimpleNamespace(seed=lambda _seed: None)mlx_optimizers = types.ModuleType("mlx.optimizers")mlx_optimizers.AdamW = lambda **kwargs: object()# #686 builds a real LR schedule, so the MLX path reaches these threediff --git a/tests/test_issue755_backend_support_registry.py b/tests/test_issue755_backend_support_registry.pyindex 2d0da945..113bb435 100644--- a/tests/test_issue755_backend_support_registry.py+++ b/tests/test_issue755_backend_support_registry.py@@ -238,15 +238,16 @@ def test_a_setting_the_backend_ignores_is_reported(config_at):This used to assert on ``training.max_grad_norm``. #750 wired it, so it ishonoured and no longer in the registry -- the guard below is what caught- that. ``training.seed`` is the live example now: MLX seeds through- ``mx.random`` and reads the field only to say so.+ that. So, later, was ``training.seed``, which MLX honours since #1850.+ ``training.use_galore`` is the live example now: MLX reads it only to say+ it is ignored."""from soup_cli.config.backend_support import check_configfrom soup_cli.config.loader import load_config- cfg = load_config(config_at("sft", "mlx", " seed: 42"))+ cfg = load_config(config_at("sft", "mlx", " use_galore: true"))reported = {e.field for e in check_config(cfg)}- assert "training.seed" in reported+ assert "training.use_galore" in reporteddef test_new_mlx_gaps_use_liger_and_neftune_alpha_are_reported(config_at):@@ -310,9 +311,9 @@ def test_only_fields_the_user_actually_set_are_reported(config_at):from soup_cli.config.backend_support import check_configfrom soup_cli.config.loader import load_config- cfg = load_config(config_at("sft", "mlx", " seed: 42"))+ cfg = load_config(config_at("sft", "mlx", " use_galore: true"))reported = {e.field for e in check_config(cfg)}- assert reported == {"training.seed"}, (+ assert reported == {"training.use_galore"}, (f"reported fields the user never set: {sorted(reported)}")@@ -321,7 +322,7 @@ def test_the_transformers_path_reports_nothing_for_the_same_config(config_at):from soup_cli.config.backend_support import check_configfrom soup_cli.config.loader import load_config- cfg = load_config(config_at("sft", "transformers", " seed: 42"))+ cfg = load_config(config_at("sft", "transformers", " use_galore: true"))assert check_config(cfg) == []@@ -369,13 +370,13 @@ def test_doctor_config_names_the_ignored_field_and_its_reason(from soup_cli.commands.doctor import doctormonkeypatch.setattr(doctor_module, "console", Console(width=200))- path = config_at("sft", "mlx", " seed: 42")+ path = config_at("sft", "mlx", " use_galore: true")# Every typer parameter passed explicitly — see #752.doctor(nccl=False, disk=False, config=path)out = strip_ansi(capsys.readouterr().out)- assert "seed" in out- assert "mx.random" in out, "the row must carry the reason, not just the name"+ assert "use_galore" in out+ assert "no MLX implementation" in out, "the row must carry the reason, not just the name"@pytest.mark.parametrize(@@ -445,7 +446,7 @@ def test_the_setting_name_survives_at_any_pinned_width(from soup_cli.commands.doctor import doctormonkeypatch.setattr(doctor_module, "console", Console(width=width))- doctor(nccl=False, disk=False, config=config_at("sft", "mlx", " seed: 42"))+ doctor(nccl=False, disk=False, config=config_at("sft", "mlx", " use_galore: true"))# Scope to the config section: the environment report above it has its own# tables, and the dependency table legitimately ellipsises at 35 columns.@@ -467,7 +468,7 @@ def test_the_setting_name_survives_at_any_pinned_width(for line in section.splitlines()if line.startswith("\u2502"))- assert "training.seed" in first_column, (+ assert "training.use_galore" in first_column, (f"name lost at width={width}; first column read as {first_column!r}")assert "\u2026" not in section, f"a config row was ellipsised at width={width}"@@ -659,8 +660,8 @@ def test_an_unfounded_gap_claim_is_caught():def test_a_gap_for_a_field_another_backend_reads_is_accepted():"""Control for the above — the check must not fire on a real gap.- ``training.seed`` is read by the transformers path and ignored by MLX, so- it is exactly the shape the registry exists to record.+ ``training.use_galore`` is read by the transformers path and ignored by+ MLX, so it is exactly the shape the registry exists to record."""repo_root = pathlib.Path(__file__).resolve().parents[1]assert _registry_drift(repo_root) == []diff --git a/tests/test_mlx_backend.py b/tests/test_mlx_backend.pyindex 52fa89dd..ad0caa6c 100644--- a/tests/test_mlx_backend.py+++ b/tests/test_mlx_backend.py@@ -38,6 +38,8 @@ class TestMLXDetection:fake_mlx = types.ModuleType("mlx")fake_mlx.__version__ = "0.20.0"fake_core = types.ModuleType("mlx.core")+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.+ fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)fake_core.metal = types.SimpleNamespace(is_available=lambda: True)fake_mlx.core = fake_coremonkeypatch.setitem(sys.modules, "mlx", fake_mlx)@@ -70,6 +72,8 @@ class TestMLXDetection:fake_mlx = types.ModuleType("mlx") # no __version__ on the top-level pkgfake_core = types.ModuleType("mlx.core")+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.+ fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)fake_core.__version__ = "0.32.2"fake_mlx.core = fake_coremonkeypatch.setitem(sys.modules, "mlx", fake_mlx)@@ -91,6 +95,8 @@ class TestMLXDetection:fake_mlx = types.ModuleType("mlx")fake_core = types.ModuleType("mlx.core")+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.+ fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)fake_core.metal = types.SimpleNamespace(is_available=lambda: True)fake_mlx.core = fake_coremonkeypatch.setitem(sys.modules, "mlx", fake_mlx)@@ -370,6 +376,8 @@ class TestMLXDoctor:fake_mlx = types.ModuleType("mlx")fake_mlx.__spec__ = ModuleSpec("mlx", loader=None)fake_core = types.ModuleType("mlx.core")+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.+ fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)fake_core.__spec__ = ModuleSpec("mlx.core", loader=None)fake_core.__version__ = "0.32.2"fake_core.metal = types.SimpleNamespace(is_available=lambda: True)diff --git a/tests/test_mlx_sft_seed.py b/tests/test_mlx_sft_seed.pynew file mode 100644index 00000000..4f1873e3--- /dev/null+++ b/tests/test_mlx_sft_seed.py@@ -0,0 +1,214 @@+"""#1850 — ``training.seed`` reaches MLX SFT.++``mlx_lm.tuner.trainer.train()`` calls ``iterate_batches`` without a seed, so+the shuffle draws from numpy's global RNG, and LoRA ``A`` init plus dropout+draw from ``mx.random``. The ``mlx_lm.lora`` CLI seeds both before training;+Soup called ``train()`` directly and seeded neither, so two runs of one+``soup.yaml`` trained on different batches in a different order.++These run the real wrapper on a real (tiny, random) mlx-lm model and replace+only upstream's ``train()``, which records what the RNGs produce at the moment+training would start, and the LoRA ``A`` matrices it would train from.+"""++from __future__ import annotations++import sys++import pytest++pytest.importorskip("mlx.core")+pytest.importorskip("mlx_lm")+++def _tiny_llama():+ from mlx_lm.models import llama++ return llama.Model(+ llama.ModelArgs(+ model_type="llama",+ hidden_size=32,+ num_hidden_layers=2,+ intermediate_size=64,+ num_attention_heads=4,+ num_key_value_heads=2,+ rms_norm_eps=1e-5,+ vocab_size=64,+ tie_word_embeddings=True,+ )+ )+++def _start_of_training(tmp_path, monkeypatch, **training) -> dict:+ """What a run sees when upstream's ``train()`` is entered."""+ import mlx.core as mx+ import mlx_lm.tuner.trainer # noqa: F401 — registered before it is patched+ import numpy as np+ from mlx.utils import tree_flatten++ from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig+ from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper++ seen: dict = {}++ def _recording_train(*, model, **_kwargs):+ # The shuffle mlx-lm's iterate_batches would draw, and the next+ # mx.random draw (dropout masks come from the same generator).+ seen["batch_order"] = np.random.permutation(1000).tolist()+ seen["mx_draw"] = mx.random.uniform(shape=(8,)).tolist()+ seen["lora_a"] = [+ v.tolist() for k, v in tree_flatten(model.trainable_parameters()) if "lora_a" in k+ ]++ monkeypatch.setattr(sys.modules["mlx_lm.tuner.trainer"], "train", _recording_train)++ training.setdefault("epochs", 1)+ training.setdefault("batch_size", 1)+ cfg = SoupConfig(+ base="mlx-community/tiny",+ task="sft",+ backend="mlx",+ data=DataConfig(train="./data/train.jsonl", format="plaintext"),+ training=TrainingConfig(**training),+ output=str(tmp_path),+ )+ wrapper = MLXSFTTrainerWrapper(cfg)+ wrapper.model = _tiny_llama()+ wrapper.tokenizer = object()+ wrapper._dataset = {"train": [{"text": "hi"}] * 8, "val": []}+ wrapper.train()+ return seen+++def _scramble_global_rngs():+ """Whatever ran before training must not decide what training sees."""+ import mlx.core as mx+ import numpy as np++ np.random.seed(None)+ mx.random.seed(int(np.random.randint(0, 2**31 - 1)))+++class TestTheSeedReachesTraining:+ def test_same_seed_same_batch_order_dropout_and_lora_init(self, tmp_path, monkeypatch):+ _scramble_global_rngs()+ first = _start_of_training(tmp_path / "a", monkeypatch, seed=7)+ _scramble_global_rngs()+ second = _start_of_training(tmp_path / "b", monkeypatch, seed=7)++ assert first["batch_order"] == second["batch_order"]+ assert first["mx_draw"] == second["mx_draw"]+ assert first["lora_a"] and first["lora_a"] == second["lora_a"]++ def test_different_seeds_differ(self, tmp_path, monkeypatch):+ first = _start_of_training(tmp_path / "a", monkeypatch, seed=1)+ second = _start_of_training(tmp_path / "b", monkeypatch, seed=2)++ assert first["batch_order"] != second["batch_order"]+ assert first["lora_a"] != second["lora_a"]++ def test_an_unset_seed_uses_the_schema_default(self, tmp_path, monkeypatch):+ """The panel prints ``Seed: 42 (unset default)``; an unset run must be+ the run that line describes, not an unseeded one."""+ _scramble_global_rngs()+ unset = _start_of_training(tmp_path / "a", monkeypatch)+ _scramble_global_rngs()+ explicit = _start_of_training(tmp_path / "b", monkeypatch, seed=42)++ assert unset["batch_order"] == explicit["batch_order"]+ assert unset["lora_a"] == explicit["lora_a"]++ def test_seed_zero_is_a_seed(self, tmp_path, monkeypatch):+ """0 is a valid seed, not "unset": it must not fall back to 42."""+ zero_a = _start_of_training(tmp_path / "a", monkeypatch, seed=0)+ zero_b = _start_of_training(tmp_path / "b", monkeypatch, seed=0)+ default = _start_of_training(tmp_path / "c", monkeypatch)++ assert zero_a["batch_order"] == zero_b["batch_order"]+ assert zero_a["batch_order"] != default["batch_order"]++ def test_data_seed_zero_is_a_seed(self, tmp_path, monkeypatch):+ """``data_seed: 0`` is read with ``is None``, not truthiness: it orders+ the batches like seed 0 does, not like the ``seed`` it sits next to."""+ zero = _start_of_training(tmp_path / "a", monkeypatch, seed=7, data_seed=0)+ seed_zero = _start_of_training(tmp_path / "b", monkeypatch, seed=0)+ seed_seven = _start_of_training(tmp_path / "c", monkeypatch, seed=7)++ assert zero["batch_order"] == seed_zero["batch_order"]+ assert zero["batch_order"] != seed_seven["batch_order"]+ assert zero["lora_a"] == seed_seven["lora_a"]++ def test_data_seed_owns_the_batch_order(self, tmp_path, monkeypatch):+ """As on the transformers path: ``data_seed`` decides the data order,+ ``seed`` everything else."""+ a = _start_of_training(tmp_path / "a", monkeypatch, seed=1, data_seed=9)+ b = _start_of_training(tmp_path / "b", monkeypatch, seed=2, data_seed=9)++ assert a["batch_order"] == b["batch_order"]+ assert a["lora_a"] != b["lora_a"]+++# --------------------------------------------------------------------------+# End to end: mlx-lm's real train() loop, nothing replaced+# --------------------------------------------------------------------------+++class _CharTokenizer:+ """Just what mlx-lm's text dataset reads: ``encode`` and ``eos_token_id``.+ Ids stay inside the tiny model's 64-token vocabulary."""++ eos_token_id = 2++ def encode(self, text):+ return [1] + [3 + ord(c) % 60 for c in text]+++def _train_for_real(tmp_path, **training):+ import mlx.core as mx++ from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig+ from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper++ training.setdefault("epochs", 2)+ training.setdefault("batch_size", 2)+ training.setdefault("lr", 1e-2)+ cfg = SoupConfig(+ base="mlx-community/tiny",+ task="sft",+ backend="mlx",+ data=DataConfig(train="./data/train.jsonl", format="plaintext"),+ training=TrainingConfig(logging_steps=1, gradient_accumulation_steps=1, **training),+ output=str(tmp_path),+ )+ wrapper = MLXSFTTrainerWrapper(cfg)+ # A checkpoint on disk is the same model every run; the tiny stand-in is+ # built from mx.random, so pin its weights. Then scramble every RNG: what+ # training sees must come from training.seed alone.+ mx.random.seed(1234)+ wrapper.model = _tiny_llama()+ _scramble_global_rngs()+ wrapper.tokenizer = _CharTokenizer()+ texts = [f"sample number {i} " + "abc" * i for i in range(12)]+ wrapper._dataset = {"train": [{"text": t} for t in texts], "val": []}+ result = wrapper.train()+ weights = mx.load(str(tmp_path / "adapters.safetensors"))+ return result, {k: v.tolist() for k, v in weights.items()}+++class TestSameSeedSameLosses:+ """Mirrors #1846's MLX DPO check on the SFT path: two runs at one seed+ report the same losses and save the same adapter; another seed does not."""++ def test_same_seed_reproduces_losses_and_adapter(self, tmp_path):+ first, first_weights = _train_for_real(tmp_path / "a", seed=5)+ second, second_weights = _train_for_real(tmp_path / "b", seed=5)++ assert first["initial_loss"] == second["initial_loss"]+ assert first["final_loss"] == second["final_loss"]+ assert first_weights and first_weights == second_weights++ def test_another_seed_trains_differently(self, tmp_path):+ first, first_weights = _train_for_real(tmp_path / "a", seed=5)+ second, second_weights = _train_for_real(tmp_path / "b", seed=6)++ assert first_weights != second_weightsdiff --git a/tests/test_mlx_sft_seed_wiring.py b/tests/test_mlx_sft_seed_wiring.pynew file mode 100644index 00000000..7cf4aa89--- /dev/null+++ b/tests/test_mlx_sft_seed_wiring.py@@ -0,0 +1,69 @@+"""#1850 — the MLX SFT seeding wiring, on every platform.++tests/test_mlx_sft_seed.py checks the behaviour on a real mlx-lm model, so it+needs Apple Silicon and is skipped everywhere else. This file runs the real+``MLXSFTTrainerWrapper.train()`` body against the fake ``mlx`` / ``mlx_lm``+modules from tests/test_issue634_mlx_resume.py and records the calls that+matter: ``mx.random.seed``, the LoRA conversion, ``np.random.seed`` and+mlx-lm's ``train()``. It pins which value reaches each RNG (``0`` is a seed,+``data_seed`` owns the batch order, unset is 42) and the order: ``mx.random``+before the adapter is built, numpy after it and immediately before ``train()``.++No ``importorskip``: the fakes replace whatever mlx is (or is not) installed.+"""++from __future__ import annotations++import sys+import types++import numpy as np+import pytest++from tests.test_issue634_mlx_resume import _FakeMlxModel, _install_fake_mlx, _mlx_wrapper+++def _run(tmp_path, monkeypatch, **training):+ _install_fake_mlx(monkeypatch)+ events = []+ sys.modules["mlx.core"].random = types.SimpleNamespace(+ seed=lambda value: events.append(("mx", value))+ )+ sys.modules["mlx_lm.tuner.utils"].linear_to_lora_layers = (+ lambda model, num_layers, config: events.append(("lora", None))+ )+ original = np.random.seed+ monkeypatch.setattr(+ np.random,+ "seed",+ lambda value=None: (events.append(("np", value)), original(value)),+ )+ sys.modules["mlx_lm.tuner.trainer"].train = lambda **_kw: events.append(("train", None))++ wrapper = _mlx_wrapper(tmp_path, epochs=1, lr=1e-4, batch_size=1, **training)+ wrapper.model, wrapper.tokenizer = _FakeMlxModel(), object()+ wrapper._dataset = {"train": [{"text": "hi"}], "val": []}+ wrapper.train()+ return events+++ "training, mx_seed, np_seed",+ [+ ({}, 42, 42),+ ({"seed": 7}, 7, 7),+ ({"seed": 0}, 0, 0),+ ({"seed": 7, "data_seed": 9}, 7, 9),+ ({"seed": 7, "data_seed": 0}, 7, 0),+ ],+)+def test_seeds_reach_mlx_and_numpy_in_order(tmp_path, monkeypatch, training, mx_seed, np_seed):+ events = _run(tmp_path, monkeypatch, **training)++ assert [e for e in events if e[0] == "mx"] == [("mx", mx_seed)]+ assert [e for e in events if e[0] == "np"] == [("np", np_seed)]+ names = [e[0] for e in events]+ assert names.count("lora") == 1 and names.count("train") == 1+ assert (+ names.index("mx") < names.index("lora") < names.index("np") == names.index("train") - 1+ )diff --git a/tests/test_review_fixes_v07133.py b/tests/test_review_fixes_v07133.pyindex b572c855..c400a910 100644--- a/tests/test_review_fixes_v07133.py+++ b/tests/test_review_fixes_v07133.py@@ -410,8 +410,13 @@ class TestMlxOptimizer:lambda step: scheds[-1](step))mlx_root = types.ModuleType("mlx")+ mlx_root.__path__ = [] # a package, so `import mlx.core` resolves below+ # #1850: train() seeds mx.random before attaching LoRA.+ core_mod = types.ModuleType("mlx.core")+ core_mod.random = types.SimpleNamespace(seed=lambda _seed: None)monkeypatch.setitem(sys.modules, "mlx", mlx_root)+ monkeypatch.setitem(sys.modules, "mlx.core", core_mod)monkeypatch.setitem(sys.modules, "mlx.optimizers", opt_mod)monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm_mod)monkeypatch.setitem(sys.modules, "mlx_lm.tuner", tuner_mod)