Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(mlx): seed MLX SFT so a run reproduces (#1850) (#1855)

dryn85-maxb0a20e07c99b
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 986b0058..ac2efe11 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -512,7 +512,7 @@ jobs:
# loss, gradient and training-loop tests ran on no job at all.
# tests/test_ci_mlx_smoke_files.py fails when a tests/test_mlx_*.py that
# needs the real library is neither listed here nor exempted there.
- run: pytest -o addopts= -q tests/test_rewind_mlx.py tests/test_mlx_dora_native.py tests/test_mlx_dpo.py tests/test_mlx_orpo.py tests/test_mlx_eval_steps.py tests/test_mlx_simpo.py
+ run: pytest -o addopts= -q tests/test_rewind_mlx.py tests/test_mlx_dora_native.py tests/test_mlx_dpo.py tests/test_mlx_orpo.py tests/test_mlx_eval_steps.py tests/test_mlx_simpo.py tests/test_mlx_sft_seed.py
torchao-contract:
needs: plan
# Not on a pull request, approval run included: it is not a required check.
diff --git a/changelog.d/0.76.0/1855.fixed.md b/changelog.d/0.76.0/1855.fixed.md
new file mode 100644
index 00000000..48528f00
--- /dev/null
+++ b/changelog.d/0.76.0/1855.fixed.md
@@ -0,0 +1,8 @@
+- **MLX SFT runs are reproducible: `training.seed` and `training.data_seed` are applied
+ (#1850 by @dryn85-max in #1855).** mlx-lm's batch shuffle draws from numpy's global RNG, and
+ LoRA init and dropout draw from `mx.random`. Soup seeded neither, so every run of one
+ `soup.yaml` trained on a different batch order, even with the seed left at the 42 the panel
+ prints. `mx.random` is now seeded before the adapter is built and numpy right before training,
+ and the `MLX backend ignores: training.seed` warning is gone. Runs started before this change
+ cannot be reproduced. Try: run `soup train --config soup.yaml` twice with `backend: mlx` and
+ compare the losses.
diff --git a/docs/training.md b/docs/training.md
index 9047551a..887616a8 100644
--- a/docs/training.md
+++ b/docs/training.md
@@ -221,11 +221,13 @@ pick `seed` up for their samplers as well.
Through v0.73.0 this reached the **SFT** trainer only, so `training.seed: 7` on
a DPO or GRPO run was accepted and silently trained at 42.
-The one path that still ignores both fields is the **MLX backend**
-(`backend: mlx`), whose trainers seed nothing at all — MLX has its own RNG
-(`mx.random`). Setting either field there now prints a warning naming it
-(`MLX backend ignores: training.seed ...`) rather than accepting it in silence,
-so an MLX run cannot look seeded while it is not.
+On the **MLX backend** (`backend: mlx`), SFT seeds MLX's own RNG
+(`mx.random`, which draws LoRA `A` and dropout) with `training.seed` before the
+adapter is built, and numpy's global RNG, which mlx-lm's batch shuffle draws
+from, with `training.data_seed` (or `training.seed` when that is unset) right
+before training starts (#1850). Before #1850 it seeded neither and warned
+`MLX backend ignores: training.seed`, so every MLX run trained on a different
+batch order.
**Not a determinism guarantee.** A fixed seed makes the *software* RNG
reproducible. It does not make CUDA kernels bit-reproducible — non-deterministic
diff --git a/src/soup_cli/config/backend_support.py b/src/soup_cli/config/backend_support.py
index 18bc8b7f..00196638 100644
--- a/src/soup_cli/config/backend_support.py
+++ b/src/soup_cli/config/backend_support.py
@@ -57,11 +57,6 @@ def _default_active(value: Any) -> bool:
return bool(value)
-def _seed_active(value: Any) -> bool:
- """0 is a valid random seed, so any non-None value is active."""
- return value is not None
-
-
@dataclass(frozen=True)
class SupportEntry:
"""One declared gap: a field, what happens to it, and why."""
@@ -89,20 +84,6 @@ _MLX_SFT: tuple[SupportEntry, ...] = (
# max_grad_norm, warmup_ratio, weight_decay, optimizer, scheduler -- were
# wired by #734 and #750 and are now honoured, so they are gone rather than
# reclassified. The guard is what noticed; see the PR body.
- SupportEntry(
- "training.seed",
- IGNORED,
- "MLX seeds through mx.random, not this field",
- trainer_reads=True,
- active=_seed_active,
- ),
- SupportEntry(
- "training.data_seed",
- IGNORED,
- "MLX seeds through mx.random, not this field",
- trainer_reads=True,
- active=_seed_active,
- ),
SupportEntry(
"training.use_galore",
IGNORED,
@@ -213,12 +194,9 @@ _TRANSFORMERS_AUDIO_SFT: tuple[SupportEntry, ...] = (
_UNSLOTH_SFT: tuple[SupportEntry, ...] = ()
-#: Fields MLX SFT warns about that have no meaning for DPO, or that MLX DPO
-#: honours: the seeds (its own loop seeds ``mx.random`` and the batch order)
-#: and SFT's response-masking switches (DPO supervises completions by design).
+#: Fields MLX SFT warns about that have no meaning for DPO: SFT's
+#: response-masking switches (DPO supervises completions by design).
_NOT_DPO_GAPS = frozenset({
- "training.seed",
- "training.data_seed",
"data.train_on_messages_with_train_field",
"data.mask_history",
"data.train_on_prompt",
@@ -323,7 +301,7 @@ def check_config(cfg: "SoupConfig") -> list[SupportEntry]:
at their schema default, which is the difference between a useful pre-flight
check and a wall of 275 rows. ``entry.active`` then drops a field written in
its off position (e.g. ``false``). A written schema default counts as
- unwritten, including a default of ``true``; ``seed: 0`` still counts as set.
+ unwritten, including a default of ``true``.
"""
backend = getattr(cfg, "backend", DEFAULT_BACKEND)
modality = getattr(cfg, "modality", DEFAULT_MODALITY)
diff --git a/src/soup_cli/trainer/mlx_sft.py b/src/soup_cli/trainer/mlx_sft.py
index 837fbccc..0ff978fe 100644
--- a/src/soup_cli/trainer/mlx_sft.py
+++ b/src/soup_cli/trainer/mlx_sft.py
@@ -456,14 +456,6 @@ class MLXSFTTrainerWrapper:
"training.use_fsdp2_compile (torch.compile on FSDP2 requires "
"CUDA and the transformers backend)"
)
- # #353's fourth criterion. #381 threaded training.seed through every
- # transformers task wrapper; MLX has its own RNG (mx.random) and reads
- # neither field, so a seeded MLX run is silently unseeded. `is not None`
- # rather than truthiness: 0 is a real seed.
- if tcfg.seed is not None:
- unsupported.append("training.seed (MLX seeds through mx.random)")
- if tcfg.data_seed is not None:
- unsupported.append("training.data_seed (MLX seeds through mx.random)")
if isinstance(tcfg.gradient_checkpointing, str):
unsupported.append(
f"gradient_checkpointing tier {tcfg.gradient_checkpointing!r} "
@@ -615,9 +607,10 @@ class MLXSFTTrainerWrapper:
not a resume of training state: the step count and data position
both restart from zero regardless of how far the checkpoint got.
Say so rather than implying a full resume. Replaying the dataset
- from the saved iteration is a separate, harder claim — it needs a
- reproducible iteration order tied to training.seed/data_seed, which
- the MLX path does not thread yet (#353) — and is out of scope here.
+ from the saved iteration is a separate, harder claim. The order is
+ reproducible from training.seed/data_seed since #1850, but mlx-lm
+ exposes no way to start its batch iterator part-way, so it is out of
+ scope here.
``strict=False`` means a checkpoint saved under a different
``lora.r`` or ``target_modules`` drops every tensor in silence —
@@ -686,6 +679,23 @@ class MLXSFTTrainerWrapper:
"MLX backend: setup(dataset) must be called before train()"
)
+ # #1850: mlx-lm's `train()` shuffles through numpy's global RNG
+ # (`iterate_batches` is called without a seed) and LoRA `A` init and
+ # dropout draw from `mx.random`. The `mlx_lm.lora` CLI seeds both before
+ # training; calling `train()` directly seeded neither, so every run
+ # trained on a different batch order. `mx.random` is seeded here, before
+ # the adapter exists; numpy right before `train()` below, so nothing in
+ # between can move the batch order.
+ import mlx.core as mx
+
+ from soup_cli.utils.seeding import resolve_training_seed
+
+ seed = resolve_training_seed(cfg.training)
+ data_seed = (
+ seed if cfg.training.data_seed is None else int(cfg.training.data_seed)
+ )
+ mx.random.seed(seed)
+
self._apply_lora(self.model)
if resume_from_checkpoint is not None:
@@ -1164,6 +1174,9 @@ class MLXSFTTrainerWrapper:
train_dataset = rewind.wrap(train_dataset)
train_hooks = {**train_hooks, "loss": rewind.loss}
+ import numpy as np
+
+ np.random.seed(data_seed)
t0 = time.time()
completed_iterations = iters
stopped = False
diff --git a/tests/test_issue1156_multimodal_mask_history_doctor.py b/tests/test_issue1156_multimodal_mask_history_doctor.py
index a7bed053..7cff0b73 100644
--- a/tests/test_issue1156_multimodal_mask_history_doctor.py
+++ b/tests/test_issue1156_multimodal_mask_history_doctor.py
@@ -216,7 +216,7 @@ def test_mlx_reports_unread_flags_regardless_of_modality(
backend: mlx
modality: vision
training:
- seed: 42
+ use_galore: true
data:
train: ./data.jsonl
""")
@@ -224,7 +224,7 @@ def test_mlx_reports_unread_flags_regardless_of_modality(
doctor(nccl=False, disk=False, config=cfg_path)
out = strip_ansi(capsys.readouterr().out)
assert "Config check - task=sft backend=mlx modality=vision" in out
- assert "training.seed" in out
+ assert "training.use_galore" in out
assert (
"1 setting(s) switched on here are not read on backend=mlx (modality=vision)"
in out
diff --git a/tests/test_issue1330_backend_support_active_values.py b/tests/test_issue1330_backend_support_active_values.py
index fcd29f86..0e16d8dc 100644
--- a/tests/test_issue1330_backend_support_active_values.py
+++ b/tests/test_issue1330_backend_support_active_values.py
@@ -1,8 +1,7 @@
"""Tests for Issue #1330: soup doctor --config reports settings by spelling, not value.
Ensures that settings written in their inactive state (e.g. false) do not produce
-false positive 'ignored' or 'rejected' rows, and that active values like seed: 0
-are properly detected.
+false positive 'ignored' or 'rejected' rows.
"""
from __future__ import annotations
@@ -101,8 +100,11 @@ def test_a_full_dump_reports_what_the_minimal_config_reports(key, tmp_path):
assert [e.field for e in check_config(full)] == [e.field for e in check_config(minimal)]
-def test_seed_zero_on_mlx_is_still_reported():
- """AC3: training.seed: 0 and data_seed: 0 on MLX are still reported (0 is an active seed)."""
+def test_seed_zero_on_mlx_is_not_reported_now_that_mlx_seeds():
+ """AC3 was "seed: 0 on MLX is still reported (0 is an active seed)". Since
+ #1850 MLX SFT honours both seeds, so neither value is a gap any more. That
+ 0 is a real seed rather than "unset" is now pinned where it matters, in the
+ run itself: tests/test_mlx_sft_seed.py::test_seed_zero_is_a_seed."""
yaml_text = textwrap.dedent(
"""\
base: mlx-community/Qwen2.5-0.5B-Instruct-4bit
@@ -120,8 +122,8 @@ def test_seed_zero_on_mlx_is_still_reported():
)
cfg = load_config_from_string(yaml_text)
reported_fields = {g.field for g in check_config(cfg)}
- assert "training.seed" in reported_fields
- assert "training.data_seed" in reported_fields
+ assert "training.seed" not in reported_fields
+ assert "training.data_seed" not in reported_fields
def test_schema_refused_settings_have_no_doctor_rows():
diff --git a/tests/test_issue353_mlx_seed_warning.py b/tests/test_issue353_mlx_seed_warning.py
index a30e648e..fe450a42 100644
--- a/tests/test_issue353_mlx_seed_warning.py
+++ b/tests/test_issue353_mlx_seed_warning.py
@@ -1,20 +1,15 @@
-"""#353's fourth acceptance criterion, for the one backend that was unreachable
-when the issue was written.
-
-#353 asks for "parse-time rejection (or a loud warning) when ``training.seed``
-is set on a task that ignores it". #381 threaded the seed through all eighteen
-transformers task wrappers, and named the MLX backend as the one path that
-reads neither ``training.seed`` nor ``training.data_seed``: ``mlx_sft`` /
-``mlx_dpo`` / ``mlx_grpo`` reference no seed and no ``mx.random`` on any path.
-
-That gap only became reachable *between* the issue being filed and #381
-landing: ``backend: mlx`` was never dispatched at all until #362 (#363), so a
-user could not previously get an MLX run to ignore anything. Now they can, and
-silence is the failure mode #353 exists to remove — a seeded MLX run that is
-not seeded looks exactly like a seeded one until two replicates disagree.
-
-Seeding MLX itself (``mx.random.seed``) is separate work with its own RNG and
-is deliberately NOT what this pins. What is pinned is that the run says so.
+"""#353's fourth acceptance criterion on MLX, and what replaced it.
+
+#353 asked for "parse-time rejection (or a loud warning) when ``training.seed``
+is set on a task that ignores it". MLX SFT read neither ``training.seed`` nor
+``training.data_seed``, so it warned. #1850 seeds it for real (``mx.random``
+before the adapter is built, numpy's global RNG before mlx-lm's ``train()``
+shuffles), and that test said to remove the warning in the same commit. The
+behaviour is pinned in tests/test_mlx_sft_seed.py (real mlx) and
+tests/test_mlx_sft_seed_wiring.py (everywhere). What stays here needs no mlx:
+a set seed / data_seed is no longer announced as ignored, and the
+controls: the unsupported-feature list still fires, and says nothing about
+seeds.
"""
from io import StringIO
@@ -50,31 +45,14 @@ def _warnings_for(monkeypatch, **training):
return buffer.getvalue()
-class TestTheSeedIsAnnouncedAsIgnored:
- def test_a_set_seed_is_named(self, monkeypatch):
- out = _warnings_for(monkeypatch, seed=7)
- assert "seed" in out
- # The user set `training.seed`; the message has to be greppable as that,
- # not as some paraphrase they cannot search for.
- assert "training.seed" in out
+class TestTheSeedIsNotAnnouncedAsIgnored:
+ @pytest.mark.parametrize("field", ["seed", "data_seed"])
+ def test_no_ignored_warning(self, monkeypatch, field):
+ """MLX SFT honours both fields since #1850, so a run that sets one must
+ not be told it is ignored."""
+ out = _warnings_for(monkeypatch, **{field: 11})
+ assert f"training.{field}" not in out
- def test_a_set_data_seed_is_named(self, monkeypatch):
- out = _warnings_for(monkeypatch, data_seed=3)
- assert "training.data_seed" in out
-
- def test_seed_zero_still_warns(self, monkeypatch):
- """0 is a real seed, not "unset". A truthiness check here would drop the
- one value a user picks precisely because it is conventional."""
- out = _warnings_for(monkeypatch, seed=0)
- assert "training.seed" in out
-
- def test_both_are_named_together(self, monkeypatch):
- out = _warnings_for(monkeypatch, seed=7, data_seed=3)
- assert "training.seed" in out
- assert "training.data_seed" in out
-
-
-class TestItStaysQuietOtherwise:
def test_control_an_unseeded_run_says_nothing_about_seeds(self, monkeypatch):
"""CONTROL. A warning that fires on every MLX run is noise, and noise is
how the next real one gets ignored."""
@@ -82,38 +60,16 @@ class TestItStaysQuietOtherwise:
assert "seed" not in out.lower()
def test_control_the_existing_unsupported_warnings_still_fire(self, monkeypatch):
- """CONTROL. The seed lines are appended to the same list that already
- reports GaLore and friends, so this pins that the list was extended
- rather than replaced."""
+ """CONTROL. Removing the seed lines (#1850) must not have taken the rest
+ of the list with them: GaLore and friends are still reported."""
out = _warnings_for(monkeypatch, use_galore=True)
assert "GaLore" in out
assert "seed" not in out.lower()
-def test_the_mlx_wrappers_still_do_not_read_the_seed(monkeypatch):
- """The warning is only honest while it is true. If a future change seeds MLX
- for real, this fails and whoever did it removes the warning in the same
- commit instead of leaving a run lying about itself."""
- import inspect
-
- from soup_cli.trainer import mlx_grpo, mlx_sft
-
- # mlx_dpo is not here: it runs its own loop and seeds `mx.random` and the
- # batch order (tests/test_mlx_dpo.py pins the reproducibility), so it never
- # carried the warning.
- for module in (mlx_sft, mlx_grpo):
- source = inspect.getsource(module)
- # `_check_unsupported` legitimately mentions the field it is warning
- # about, so the sources are checked for an actual RNG call instead.
- assert "mx.random.seed" not in source, (
- f"{module.__name__} now seeds MLX; drop the 'ignored' warning in "
- f"mlx_sft._check_unsupported and delete this test"
- )
-
-
@pytest.mark.parametrize("field", ["seed", "data_seed"])
def test_the_schema_still_accepts_the_field_on_mlx(field):
- """A warning, not a rejection: #353 allows either, and a config that is
- valid on transformers should not become unloadable by switching backend."""
+ """Accepted, and since #1850 honoured: a config that is valid on
+ transformers stays loadable when the backend switches to MLX."""
cfg = _mlx_cfg(**{field: 11})
assert getattr(cfg.training, field) == 11
diff --git a/tests/test_issue353_seed_every_task.py b/tests/test_issue353_seed_every_task.py
index 20812f5e..c030821f 100644
--- a/tests/test_issue353_seed_every_task.py
+++ b/tests/test_issue353_seed_every_task.py
@@ -513,7 +513,8 @@ _NOT_SEEDED = {
"preference": "delegates to an inner wrapper, which _make_inner_cfg's "
"model_copy carries the seed into",
"bitnet": "raises before it builds anything; hardware-gated path",
- "mlx_sft": "MLX backend, which seeds nothing on any path yet",
+ "mlx_sft": "seeds mx.random and numpy in train(), where the adapter is "
+ "built (#1850; tests/test_mlx_sft_seed.py)",
"mlx_grpo": "MLX backend, which seeds nothing on any path yet",
}
diff --git a/tests/test_issue634_mlx_resume.py b/tests/test_issue634_mlx_resume.py
index 9631804d..19e12bdf 100644
--- a/tests/test_issue634_mlx_resume.py
+++ b/tests/test_issue634_mlx_resume.py
@@ -391,6 +391,8 @@ def _install_fake_mlx(monkeypatch):
"""
mlx = types.ModuleType("mlx")
mlx_core = types.ModuleType("mlx.core")
+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
+ mlx_core.random = types.SimpleNamespace(seed=lambda _seed: None)
mlx_optimizers = types.ModuleType("mlx.optimizers")
mlx_optimizers.AdamW = lambda **kwargs: object()
# #686 wires a real LR schedule, so the MLX backend now reaches these three
diff --git a/tests/test_issue683_mlx_response_masking.py b/tests/test_issue683_mlx_response_masking.py
index 022715c3..7d75e269 100644
--- a/tests/test_issue683_mlx_response_masking.py
+++ b/tests/test_issue683_mlx_response_masking.py
@@ -413,6 +413,8 @@ def _install_numpy_mlx(monkeypatch):
them.
"""
core = types.ModuleType("mlx.core")
+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
+ core.random = types.SimpleNamespace(seed=lambda _seed: None)
core.array = np.array
core.maximum = np.maximum
core.float32 = np.float32
diff --git a/tests/test_issue684_mlx_grad_accumulation.py b/tests/test_issue684_mlx_grad_accumulation.py
index b654c947..9632a5e4 100644
--- a/tests/test_issue684_mlx_grad_accumulation.py
+++ b/tests/test_issue684_mlx_grad_accumulation.py
@@ -31,6 +31,8 @@ def _install_fake_mlx(monkeypatch):
"""
mlx = types.ModuleType("mlx")
mlx_core = types.ModuleType("mlx.core")
+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
+ mlx_core.random = types.SimpleNamespace(seed=lambda _seed: None)
mlx_optimizers = types.ModuleType("mlx.optimizers")
mlx_optimizers.AdamW = lambda **kwargs: object()
# #686 wires a real LR schedule, so the MLX backend now reaches these three
diff --git a/tests/test_issue685_mlx_grad_checkpoint.py b/tests/test_issue685_mlx_grad_checkpoint.py
index 7a8746b0..8a55baf0 100644
--- a/tests/test_issue685_mlx_grad_checkpoint.py
+++ b/tests/test_issue685_mlx_grad_checkpoint.py
@@ -51,6 +51,8 @@ def _install_fake_mlx(monkeypatch):
"""
mlx = types.ModuleType("mlx")
mlx_core = types.ModuleType("mlx.core")
+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
+ mlx_core.random = types.SimpleNamespace(seed=lambda _seed: None)
mlx_optimizers = types.ModuleType("mlx.optimizers")
mlx_optimizers.AdamW = lambda **kwargs: object()
# #686 builds a real LR schedule, so the MLX path reaches these three
diff --git a/tests/test_issue755_backend_support_registry.py b/tests/test_issue755_backend_support_registry.py
index 2d0da945..113bb435 100644
--- a/tests/test_issue755_backend_support_registry.py
+++ b/tests/test_issue755_backend_support_registry.py
@@ -238,15 +238,16 @@ def test_a_setting_the_backend_ignores_is_reported(config_at):
This used to assert on ``training.max_grad_norm``. #750 wired it, so it is
honoured and no longer in the registry -- the guard below is what caught
- that. ``training.seed`` is the live example now: MLX seeds through
- ``mx.random`` and reads the field only to say so.
+ that. So, later, was ``training.seed``, which MLX honours since #1850.
+ ``training.use_galore`` is the live example now: MLX reads it only to say
+ it is ignored.
"""
from soup_cli.config.backend_support import check_config
from soup_cli.config.loader import load_config
- cfg = load_config(config_at("sft", "mlx", " seed: 42"))
+ cfg = load_config(config_at("sft", "mlx", " use_galore: true"))
reported = {e.field for e in check_config(cfg)}
- assert "training.seed" in reported
+ assert "training.use_galore" in reported
def test_new_mlx_gaps_use_liger_and_neftune_alpha_are_reported(config_at):
@@ -310,9 +311,9 @@ def test_only_fields_the_user_actually_set_are_reported(config_at):
from soup_cli.config.backend_support import check_config
from soup_cli.config.loader import load_config
- cfg = load_config(config_at("sft", "mlx", " seed: 42"))
+ cfg = load_config(config_at("sft", "mlx", " use_galore: true"))
reported = {e.field for e in check_config(cfg)}
- assert reported == {"training.seed"}, (
+ assert reported == {"training.use_galore"}, (
f"reported fields the user never set: {sorted(reported)}"
)
@@ -321,7 +322,7 @@ def test_the_transformers_path_reports_nothing_for_the_same_config(config_at):
from soup_cli.config.backend_support import check_config
from soup_cli.config.loader import load_config
- cfg = load_config(config_at("sft", "transformers", " seed: 42"))
+ cfg = load_config(config_at("sft", "transformers", " use_galore: true"))
assert check_config(cfg) == []
@@ -369,13 +370,13 @@ def test_doctor_config_names_the_ignored_field_and_its_reason(
from soup_cli.commands.doctor import doctor
monkeypatch.setattr(doctor_module, "console", Console(width=200))
- path = config_at("sft", "mlx", " seed: 42")
+ path = config_at("sft", "mlx", " use_galore: true")
# Every typer parameter passed explicitly — see #752.
doctor(nccl=False, disk=False, config=path)
out = strip_ansi(capsys.readouterr().out)
- assert "seed" in out
- assert "mx.random" in out, "the row must carry the reason, not just the name"
+ assert "use_galore" in out
+ assert "no MLX implementation" in out, "the row must carry the reason, not just the name"
@pytest.mark.parametrize(
@@ -445,7 +446,7 @@ def test_the_setting_name_survives_at_any_pinned_width(
from soup_cli.commands.doctor import doctor
monkeypatch.setattr(doctor_module, "console", Console(width=width))
- doctor(nccl=False, disk=False, config=config_at("sft", "mlx", " seed: 42"))
+ doctor(nccl=False, disk=False, config=config_at("sft", "mlx", " use_galore: true"))
# Scope to the config section: the environment report above it has its own
# tables, and the dependency table legitimately ellipsises at 35 columns.
@@ -467,7 +468,7 @@ def test_the_setting_name_survives_at_any_pinned_width(
for line in section.splitlines()
if line.startswith("\u2502")
)
- assert "training.seed" in first_column, (
+ assert "training.use_galore" in first_column, (
f"name lost at width={width}; first column read as {first_column!r}"
)
assert "\u2026" not in section, f"a config row was ellipsised at width={width}"
@@ -659,8 +660,8 @@ def test_an_unfounded_gap_claim_is_caught():
def test_a_gap_for_a_field_another_backend_reads_is_accepted():
"""Control for the above — the check must not fire on a real gap.
- ``training.seed`` is read by the transformers path and ignored by MLX, so
- it is exactly the shape the registry exists to record.
+ ``training.use_galore`` is read by the transformers path and ignored by
+ MLX, so it is exactly the shape the registry exists to record.
"""
repo_root = pathlib.Path(__file__).resolve().parents[1]
assert _registry_drift(repo_root) == []
diff --git a/tests/test_mlx_backend.py b/tests/test_mlx_backend.py
index 52fa89dd..ad0caa6c 100644
--- a/tests/test_mlx_backend.py
+++ b/tests/test_mlx_backend.py
@@ -38,6 +38,8 @@ class TestMLXDetection:
fake_mlx = types.ModuleType("mlx")
fake_mlx.__version__ = "0.20.0"
fake_core = types.ModuleType("mlx.core")
+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
+ fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
fake_core.metal = types.SimpleNamespace(is_available=lambda: True)
fake_mlx.core = fake_core
monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
@@ -70,6 +72,8 @@ class TestMLXDetection:
fake_mlx = types.ModuleType("mlx") # no __version__ on the top-level pkg
fake_core = types.ModuleType("mlx.core")
+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
+ fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
fake_core.__version__ = "0.32.2"
fake_mlx.core = fake_core
monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
@@ -91,6 +95,8 @@ class TestMLXDetection:
fake_mlx = types.ModuleType("mlx")
fake_core = types.ModuleType("mlx.core")
+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
+ fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
fake_core.metal = types.SimpleNamespace(is_available=lambda: True)
fake_mlx.core = fake_core
monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
@@ -370,6 +376,8 @@ class TestMLXDoctor:
fake_mlx = types.ModuleType("mlx")
fake_mlx.__spec__ = ModuleSpec("mlx", loader=None)
fake_core = types.ModuleType("mlx.core")
+ # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
+ fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
fake_core.__spec__ = ModuleSpec("mlx.core", loader=None)
fake_core.__version__ = "0.32.2"
fake_core.metal = types.SimpleNamespace(is_available=lambda: True)
diff --git a/tests/test_mlx_sft_seed.py b/tests/test_mlx_sft_seed.py
new file mode 100644
index 00000000..4f1873e3
--- /dev/null
+++ b/tests/test_mlx_sft_seed.py
@@ -0,0 +1,214 @@
+"""#1850 — ``training.seed`` reaches MLX SFT.
+
+``mlx_lm.tuner.trainer.train()`` calls ``iterate_batches`` without a seed, so
+the shuffle draws from numpy's global RNG, and LoRA ``A`` init plus dropout
+draw from ``mx.random``. The ``mlx_lm.lora`` CLI seeds both before training;
+Soup called ``train()`` directly and seeded neither, so two runs of one
+``soup.yaml`` trained on different batches in a different order.
+
+These run the real wrapper on a real (tiny, random) mlx-lm model and replace
+only upstream's ``train()``, which records what the RNGs produce at the moment
+training would start, and the LoRA ``A`` matrices it would train from.
+"""
+
+from __future__ import annotations
+
+import sys
+
+import pytest
+
+pytest.importorskip("mlx.core")
+pytest.importorskip("mlx_lm")
+
+
+def _tiny_llama():
+ from mlx_lm.models import llama
+
+ return llama.Model(
+ llama.ModelArgs(
+ model_type="llama",
+ hidden_size=32,
+ num_hidden_layers=2,
+ intermediate_size=64,
+ num_attention_heads=4,
+ num_key_value_heads=2,
+ rms_norm_eps=1e-5,
+ vocab_size=64,
+ tie_word_embeddings=True,
+ )
+ )
+
+
+def _start_of_training(tmp_path, monkeypatch, **training) -> dict:
+ """What a run sees when upstream's ``train()`` is entered."""
+ import mlx.core as mx
+ import mlx_lm.tuner.trainer # noqa: F401 — registered before it is patched
+ import numpy as np
+ from mlx.utils import tree_flatten
+
+ from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig
+ from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper
+
+ seen: dict = {}
+
+ def _recording_train(*, model, **_kwargs):
+ # The shuffle mlx-lm's iterate_batches would draw, and the next
+ # mx.random draw (dropout masks come from the same generator).
+ seen["batch_order"] = np.random.permutation(1000).tolist()
+ seen["mx_draw"] = mx.random.uniform(shape=(8,)).tolist()
+ seen["lora_a"] = [
+ v.tolist() for k, v in tree_flatten(model.trainable_parameters()) if "lora_a" in k
+ ]
+
+ monkeypatch.setattr(sys.modules["mlx_lm.tuner.trainer"], "train", _recording_train)
+
+ training.setdefault("epochs", 1)
+ training.setdefault("batch_size", 1)
+ cfg = SoupConfig(
+ base="mlx-community/tiny",
+ task="sft",
+ backend="mlx",
+ data=DataConfig(train="./data/train.jsonl", format="plaintext"),
+ training=TrainingConfig(**training),
+ output=str(tmp_path),
+ )
+ wrapper = MLXSFTTrainerWrapper(cfg)
+ wrapper.model = _tiny_llama()
+ wrapper.tokenizer = object()
+ wrapper._dataset = {"train": [{"text": "hi"}] * 8, "val": []}
+ wrapper.train()
+ return seen
+
+
+def _scramble_global_rngs():
+ """Whatever ran before training must not decide what training sees."""
+ import mlx.core as mx
+ import numpy as np
+
+ np.random.seed(None)
+ mx.random.seed(int(np.random.randint(0, 2**31 - 1)))
+
+
+class TestTheSeedReachesTraining:
+ def test_same_seed_same_batch_order_dropout_and_lora_init(self, tmp_path, monkeypatch):
+ _scramble_global_rngs()
+ first = _start_of_training(tmp_path / "a", monkeypatch, seed=7)
+ _scramble_global_rngs()
+ second = _start_of_training(tmp_path / "b", monkeypatch, seed=7)
+
+ assert first["batch_order"] == second["batch_order"]
+ assert first["mx_draw"] == second["mx_draw"]
+ assert first["lora_a"] and first["lora_a"] == second["lora_a"]
+
+ def test_different_seeds_differ(self, tmp_path, monkeypatch):
+ first = _start_of_training(tmp_path / "a", monkeypatch, seed=1)
+ second = _start_of_training(tmp_path / "b", monkeypatch, seed=2)
+
+ assert first["batch_order"] != second["batch_order"]
+ assert first["lora_a"] != second["lora_a"]
+
+ def test_an_unset_seed_uses_the_schema_default(self, tmp_path, monkeypatch):
+ """The panel prints ``Seed: 42 (unset default)``; an unset run must be
+ the run that line describes, not an unseeded one."""
+ _scramble_global_rngs()
+ unset = _start_of_training(tmp_path / "a", monkeypatch)
+ _scramble_global_rngs()
+ explicit = _start_of_training(tmp_path / "b", monkeypatch, seed=42)
+
+ assert unset["batch_order"] == explicit["batch_order"]
+ assert unset["lora_a"] == explicit["lora_a"]
+
+ def test_seed_zero_is_a_seed(self, tmp_path, monkeypatch):
+ """0 is a valid seed, not "unset": it must not fall back to 42."""
+ zero_a = _start_of_training(tmp_path / "a", monkeypatch, seed=0)
+ zero_b = _start_of_training(tmp_path / "b", monkeypatch, seed=0)
+ default = _start_of_training(tmp_path / "c", monkeypatch)
+
+ assert zero_a["batch_order"] == zero_b["batch_order"]
+ assert zero_a["batch_order"] != default["batch_order"]
+
+ def test_data_seed_zero_is_a_seed(self, tmp_path, monkeypatch):
+ """``data_seed: 0`` is read with ``is None``, not truthiness: it orders
+ the batches like seed 0 does, not like the ``seed`` it sits next to."""
+ zero = _start_of_training(tmp_path / "a", monkeypatch, seed=7, data_seed=0)
+ seed_zero = _start_of_training(tmp_path / "b", monkeypatch, seed=0)
+ seed_seven = _start_of_training(tmp_path / "c", monkeypatch, seed=7)
+
+ assert zero["batch_order"] == seed_zero["batch_order"]
+ assert zero["batch_order"] != seed_seven["batch_order"]
+ assert zero["lora_a"] == seed_seven["lora_a"]
+
+ def test_data_seed_owns_the_batch_order(self, tmp_path, monkeypatch):
+ """As on the transformers path: ``data_seed`` decides the data order,
+ ``seed`` everything else."""
+ a = _start_of_training(tmp_path / "a", monkeypatch, seed=1, data_seed=9)
+ b = _start_of_training(tmp_path / "b", monkeypatch, seed=2, data_seed=9)
+
+ assert a["batch_order"] == b["batch_order"]
+ assert a["lora_a"] != b["lora_a"]
+
+
+# --------------------------------------------------------------------------
+# End to end: mlx-lm's real train() loop, nothing replaced
+# --------------------------------------------------------------------------
+
+
+class _CharTokenizer:
+ """Just what mlx-lm's text dataset reads: ``encode`` and ``eos_token_id``.
+ Ids stay inside the tiny model's 64-token vocabulary."""
+
+ eos_token_id = 2
+
+ def encode(self, text):
+ return [1] + [3 + ord(c) % 60 for c in text]
+
+
+def _train_for_real(tmp_path, **training):
+ import mlx.core as mx
+
+ from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig
+ from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper
+
+ training.setdefault("epochs", 2)
+ training.setdefault("batch_size", 2)
+ training.setdefault("lr", 1e-2)
+ cfg = SoupConfig(
+ base="mlx-community/tiny",
+ task="sft",
+ backend="mlx",
+ data=DataConfig(train="./data/train.jsonl", format="plaintext"),
+ training=TrainingConfig(logging_steps=1, gradient_accumulation_steps=1, **training),
+ output=str(tmp_path),
+ )
+ wrapper = MLXSFTTrainerWrapper(cfg)
+ # A checkpoint on disk is the same model every run; the tiny stand-in is
+ # built from mx.random, so pin its weights. Then scramble every RNG: what
+ # training sees must come from training.seed alone.
+ mx.random.seed(1234)
+ wrapper.model = _tiny_llama()
+ _scramble_global_rngs()
+ wrapper.tokenizer = _CharTokenizer()
+ texts = [f"sample number {i} " + "abc" * i for i in range(12)]
+ wrapper._dataset = {"train": [{"text": t} for t in texts], "val": []}
+ result = wrapper.train()
+ weights = mx.load(str(tmp_path / "adapters.safetensors"))
+ return result, {k: v.tolist() for k, v in weights.items()}
+
+
+class TestSameSeedSameLosses:
+ """Mirrors #1846's MLX DPO check on the SFT path: two runs at one seed
+ report the same losses and save the same adapter; another seed does not."""
+
+ def test_same_seed_reproduces_losses_and_adapter(self, tmp_path):
+ first, first_weights = _train_for_real(tmp_path / "a", seed=5)
+ second, second_weights = _train_for_real(tmp_path / "b", seed=5)
+
+ assert first["initial_loss"] == second["initial_loss"]
+ assert first["final_loss"] == second["final_loss"]
+ assert first_weights and first_weights == second_weights
+
+ def test_another_seed_trains_differently(self, tmp_path):
+ first, first_weights = _train_for_real(tmp_path / "a", seed=5)
+ second, second_weights = _train_for_real(tmp_path / "b", seed=6)
+
+ assert first_weights != second_weights
diff --git a/tests/test_mlx_sft_seed_wiring.py b/tests/test_mlx_sft_seed_wiring.py
new file mode 100644
index 00000000..7cf4aa89
--- /dev/null
+++ b/tests/test_mlx_sft_seed_wiring.py
@@ -0,0 +1,69 @@
+"""#1850 — the MLX SFT seeding wiring, on every platform.
+
+tests/test_mlx_sft_seed.py checks the behaviour on a real mlx-lm model, so it
+needs Apple Silicon and is skipped everywhere else. This file runs the real
+``MLXSFTTrainerWrapper.train()`` body against the fake ``mlx`` / ``mlx_lm``
+modules from tests/test_issue634_mlx_resume.py and records the calls that
+matter: ``mx.random.seed``, the LoRA conversion, ``np.random.seed`` and
+mlx-lm's ``train()``. It pins which value reaches each RNG (``0`` is a seed,
+``data_seed`` owns the batch order, unset is 42) and the order: ``mx.random``
+before the adapter is built, numpy after it and immediately before ``train()``.
+
+No ``importorskip``: the fakes replace whatever mlx is (or is not) installed.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import numpy as np
+import pytest
+
+from tests.test_issue634_mlx_resume import _FakeMlxModel, _install_fake_mlx, _mlx_wrapper
+
+
+def _run(tmp_path, monkeypatch, **training):
+ _install_fake_mlx(monkeypatch)
+ events = []
+ sys.modules["mlx.core"].random = types.SimpleNamespace(
+ seed=lambda value: events.append(("mx", value))
+ )
+ sys.modules["mlx_lm.tuner.utils"].linear_to_lora_layers = (
+ lambda model, num_layers, config: events.append(("lora", None))
+ )
+ original = np.random.seed
+ monkeypatch.setattr(
+ np.random,
+ "seed",
+ lambda value=None: (events.append(("np", value)), original(value)),
+ )
+ sys.modules["mlx_lm.tuner.trainer"].train = lambda **_kw: events.append(("train", None))
+
+ wrapper = _mlx_wrapper(tmp_path, epochs=1, lr=1e-4, batch_size=1, **training)
+ wrapper.model, wrapper.tokenizer = _FakeMlxModel(), object()
+ wrapper._dataset = {"train": [{"text": "hi"}], "val": []}
+ wrapper.train()
+ return events
+
+
+ "training, mx_seed, np_seed",
+ [
+ ({}, 42, 42),
+ ({"seed": 7}, 7, 7),
+ ({"seed": 0}, 0, 0),
+ ({"seed": 7, "data_seed": 9}, 7, 9),
+ ({"seed": 7, "data_seed": 0}, 7, 0),
+ ],
+)
+def test_seeds_reach_mlx_and_numpy_in_order(tmp_path, monkeypatch, training, mx_seed, np_seed):
+ events = _run(tmp_path, monkeypatch, **training)
+
+ assert [e for e in events if e[0] == "mx"] == [("mx", mx_seed)]
+ assert [e for e in events if e[0] == "np"] == [("np", np_seed)]
+ names = [e[0] for e in events]
+ assert names.count("lora") == 1 and names.count("train") == 1
+ assert (
+ names.index("mx") < names.index("lora") < names.index("np") == names.index("train") - 1
+ )
diff --git a/tests/test_review_fixes_v07133.py b/tests/test_review_fixes_v07133.py
index b572c855..c400a910 100644
--- a/tests/test_review_fixes_v07133.py
+++ b/tests/test_review_fixes_v07133.py
@@ -410,8 +410,13 @@ class TestMlxOptimizer:
lambda step: scheds[-1](step)
)
mlx_root = types.ModuleType("mlx")
+ mlx_root.__path__ = [] # a package, so `import mlx.core` resolves below
+ # #1850: train() seeds mx.random before attaching LoRA.
+ core_mod = types.ModuleType("mlx.core")
+ core_mod.random = types.SimpleNamespace(seed=lambda _seed: None)
monkeypatch.setitem(sys.modules, "mlx", mlx_root)
+ monkeypatch.setitem(sys.modules, "mlx.core", core_mod)
monkeypatch.setitem(sys.modules, "mlx.optimizers", opt_mod)
monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm_mod)
monkeypatch.setitem(sys.modules, "mlx_lm.tuner", tuner_mod)