Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

19.8 KB
Raw Blame
"""#1990 — ``training.eval_steps`` on MLX SFT, in completed optimizer updates.

mlx-lm evaluates *before* the step at iteration ``it``, so under accumulation a
cadence of ``steps_per_eval`` misses the update that completes at ``it``, its
"final" pass precedes the last update, and it reads at most 25 whole batches,
dropping a tail. An explicit ``eval_steps`` is therefore run by Soup:

- after each completed update ``N, 2N, ...`` and after the last update, once;
- over every validation row, the tail included, token-weighted;
- without moving parameters, optimizer state, training mode or the seeded
  training order.

The unset field keeps today's mlx-lm cadence. Real MLX on a tiny random Llama;
no downloads. The schedule, the refusals and the trigger in ``train()`` (through
the shared fake-MLX harness) also run where MLX is not installed.
"""

from __future__ import annotations

import sys
from io import StringIO

import numpy as np
import pytest
from rich.console import Console

from tests.conftest import strip_ansi


@pytest.fixture
def native():
    """Real MLX and mlx-lm; only the classes that use this fixture skip without them."""
    pytest.importorskip("mlx.core")
    pytest.importorskip("mlx_lm")


class _IntTokenizer:
    """Text: one id per character. Chat: role and word markers, all ids < 200."""

    eos_token_id = 2

    def encode(self, text, add_special_tokens=True):
        return ([1] if add_special_tokens else []) + [3 + ord(c) % 150 for c in text]

    def apply_chat_template(self, messages, tools=None, add_generation_prompt=False,
                            return_dict=False, **_):
        out = []
        for m in messages:
            out += [160 if m["role"] == "user" else 161]
            out += [3 + ord(c) % 150 for c in m["content"]] + [162]
        if add_generation_prompt:
            out.append(161)
        return out


def _tiny(hidden=32, quantize=False):
    import mlx.core as mx
    import mlx.nn as nn
    from mlx_lm.models import llama

    mx.random.seed(0)
    model = llama.Model(llama.ModelArgs(
        model_type="llama", hidden_size=hidden, num_hidden_layers=2,
        intermediate_size=2 * hidden, num_attention_heads=4, num_key_value_heads=2,
        rms_norm_eps=1e-5, vocab_size=200, tie_word_embeddings=True,
    ))
    if quantize:
        nn.quantize(model, group_size=32, bits=4)
    return model


def _text_rows(n, prefix="row"):
    return [{"text": f"{prefix} {i} " + "ab" * (i % 4 + 1)} for i in range(n)]


def _wrapper(tmp_path, *, train=8, val=4, chat=False, model=None, **training):
    from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig
    from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper

    training.setdefault("epochs", 1)
    training.setdefault("batch_size", 2)
    training.setdefault("lr", 1e-3)
    training.setdefault("gradient_accumulation_steps", 1)
    cfg = SoupConfig(
        base="mlx-community/tiny", task="sft", backend="mlx",
        data=DataConfig(train="./d.jsonl", format="chatml" if chat else "plaintext",
                        max_length=256),
        training=TrainingConfig(logging_steps=1, **training),
        output=str(tmp_path),
    )
    wrapper = MLXSFTTrainerWrapper(cfg)
    wrapper.model = model if model is not None else _tiny()
    wrapper.tokenizer = _IntTokenizer()
    if chat:
        rows = [{"messages": [{"role": "user", "content": f"q{i}"},
                              {"role": "assistant", "content": "ans" * (i % 3 + 1)}]}
                for i in range(train + val)]
        wrapper._dataset = {"train": rows[:train], "val": rows[train:]}
    else:
        wrapper._dataset = {"train": _text_rows(train), "val": _text_rows(val, "val")}
    return wrapper


class TestTheSchedule:
    @pytest.mark.parametrize("total, every, expected", [
        (4, 1, [1, 2, 3, 4]),
        (4, 2, [2, 4]),
        (4, 3, [3, 4]),
        (4, 10, [4]),
        (1, 1, [1]),
    ])
    def test_updates(self, total, every, expected):
        from soup_cli.trainer.mlx_sft import scheduled_eval_updates

        assert scheduled_eval_updates(total, every) == expected


@pytest.mark.usefixtures("native")
class TestEvaluationsHappenAfterCompletedUpdates:
    @pytest.mark.parametrize("epochs, accum, every, expected", [
        (1, 1, 1, [1, 2, 3, 4]),   # every update
        (2, 1, 3, [3, 6, 8]),      # N does not divide the 8-update budget
        (2, 2, 2, [2, 4]),         # accumulation: 8 iterations, 4 updates
        (2, 2, 3, [3, 4]),         # accumulation, non-dividing
        (1, 1, 10, [4]),           # N past the budget: once, at the end
        (1, 4, 1, [1]),            # a run of one update
    ])
    def test_exact_update_indices(self, tmp_path, epochs, accum, every, expected):
        wrapper = _wrapper(tmp_path, epochs=epochs, gradient_accumulation_steps=accum,
                           eval_steps=every)
        result = wrapper.train()
        log = wrapper._eval_log
        assert [e["update"] for e in log] == expected
        # The optimizer had taken exactly that many steps when each ran:
        # completed updates, not micro-batches and not callback calls.
        assert [e["optimizer_step"] for e in log] == expected
        assert result["eval_updates"] == expected
        assert result["val_loss"] == pytest.approx(log[-1]["val_loss"])


@pytest.mark.usefixtures("native")
class TestTheUnsetFieldKeepsMlxLmsCadence:
    def _seen(self, tmp_path, monkeypatch, **training):
        seen = {}
        monkeypatch.setattr(sys.modules["mlx_lm.tuner.trainer"], "train",
                            lambda **kw: seen.update(kw))
        _wrapper(tmp_path, **training).train()
        return seen

    def test_unset_passes_the_split_to_mlx_lm_at_a_quarter_of_the_run(self, tmp_path, monkeypatch):
        import mlx_lm.tuner.trainer  # noqa: F401 — registered before it is patched

        seen = self._seen(tmp_path, monkeypatch, epochs=2)
        assert seen["val_dataset"] is not None
        assert seen["args"].steps_per_eval == max(1, seen["args"].iters // 4)

    def test_explicit_takes_the_split_away_from_mlx_lm(self, tmp_path, monkeypatch):
        import mlx_lm.tuner.trainer  # noqa: F401

        seen = self._seen(tmp_path, monkeypatch, epochs=2, eval_steps=2)
        assert seen["val_dataset"] is None
        assert seen["args"].steps_per_eval > seen["args"].iters


class TestRefusals:
    @pytest.mark.parametrize("value, message", [
        (0, "greater than or equal to 1"),
        (-1, "greater than or equal to 1"),
        (True, "must be an int, not a bool"),
    ])
    def test_invalid_values(self, value, message):
        from pydantic import ValidationError

        from soup_cli.config.schema import TrainingConfig

        with pytest.raises(ValidationError, match=message):
            TrainingConfig(eval_steps=value)

    def test_no_validation_rows_is_refused_before_the_model_loads(self, monkeypatch):
        from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig
        from soup_cli.trainer import mlx_sft

        def _must_not_load(*_a, **_k):
            raise AssertionError("loaded")

        monkeypatch.setattr("soup_cli.utils.mlx.load_mlx_model", _must_not_load)
        cfg = SoupConfig(
            base="m", task="sft", backend="mlx",
            data=DataConfig(train="./d.jsonl", format="plaintext"),
            training=TrainingConfig(eval_steps=2, batch_size=1), output="./o",
        )
        wrapper = mlx_sft.MLXSFTTrainerWrapper(cfg)
        monkeypatch.setattr(wrapper, "_require_mlx", lambda: None)
        with pytest.raises(ValueError, match="validation split is\\s+empty"):
            wrapper.setup({"train": _text_rows(4), "val": []})


def _one_ahead(stream):
    pending = next(stream)
    for following in stream:
        yield pending
        pending = following


class TestTheTriggerWithoutMlx:
    """``train()``'s real body through the shared fake-MLX harness, with a
    stand-in for mlx-lm's loop of the same shape: ``zip(range(1, iters + 1),
    iterate_batches(..., loop=True))``, the optimizer stepping only when
    ``it % accum == 0``. The evaluator is a recorder of the optimizer steps
    taken when it ran, so the schedule is checked on every CI cell."""

    def _run(self, tmp_path, monkeypatch, prefetch=False, **training):
        from soup_cli.trainer import mlx_sft
        from tests.test_issue634_mlx_resume import _FakeMlxModel, _install_fake_mlx

        _install_fake_mlx(monkeypatch)
        loop = {"steps": 0}

        class _Step:
            def item(self):
                return loop["steps"]

        class _AdamW:
            def __init__(self, **_):
                self.state = {"step": _Step()}

        monkeypatch.setattr(sys.modules["mlx.optimizers"], "AdamW", _AdamW)

        def iterate_batches(*, dataset, batch_size, loop=False, **_):
            while True:
                for start in range(0, len(dataset) - batch_size + 1, batch_size):
                    yield dataset[start:start + batch_size]
                if not loop:
                    return

        def train(*, optimizer, train_dataset, val_dataset, args, iterate_batches=iterate_batches,
                  **_):
            loop.update(val_dataset=val_dataset, args=args)
            stream = iterate_batches(dataset=train_dataset, batch_size=args.batch_size,
                                     max_seq_length=args.max_seq_length, loop=True)
            if prefetch:  # a loop that reads one batch ahead of its step
                stream = _one_ahead(stream)
            for it, _batch in zip(range(1, args.iters + 1), stream):
                if it % args.grad_accumulation_steps == 0:
                    loop["steps"] += 1

        trainer = sys.modules["mlx_lm.tuner.trainer"]
        monkeypatch.setattr(trainer, "train", train)
        monkeypatch.setattr(trainer, "iterate_batches", iterate_batches, raising=False)
        monkeypatch.setattr(trainer, "default_loss", object(), raising=False)
        seen = []

        def recorder(_model, dataset, **_):
            seen.append((loop["steps"], len(dataset)))
            return 1.0

        monkeypatch.setattr(mlx_sft, "evaluate_all_rows", recorder)
        loop["printed"] = StringIO()
        monkeypatch.setattr(mlx_sft, "console", Console(file=loop["printed"], width=400))
        monkeypatch.chdir(tmp_path)
        loop["wrapper"] = wrapper = _wrapper(tmp_path, model=_FakeMlxModel(), **training)
        result = wrapper.train()
        return result, seen, loop

    @pytest.mark.parametrize("epochs, accum, every, expected", [
        (1, 1, 1, [1, 2, 3, 4]),
        (2, 1, 3, [3, 6, 8]),
        (2, 2, 2, [2, 4]),
        (2, 2, 3, [3, 4]),
        (1, 1, 10, [4]),
        (1, 4, 1, [1]),
    ])
    def test_evaluations_follow_completed_updates(self, tmp_path, monkeypatch,
                                                  epochs, accum, every, expected):
        result, seen, loop = self._run(tmp_path, monkeypatch, epochs=epochs,
                                    gradient_accumulation_steps=accum, eval_steps=every)
        # The optimizer had taken exactly `update` steps when each evaluation
        # ran, over the whole split; the last one follows train()'s return.
        assert seen == [(update, 4) for update in expected]
        assert result["eval_updates"] == expected
        assert [e["optimizer_step"] for e in loop["wrapper"]._eval_log] == expected
        # In step, the drift warning stays silent.
        assert "fetches batches ahead" not in loop["printed"].getvalue()

    def test_a_loop_that_prefetches_is_reported_once(self, tmp_path, monkeypatch):
        """The trigger relies on mlx-lm fetching batch `it` after step
        `it - 1`; a release that reads ahead must not move it in silence."""
        _, seen, loop = self._run(tmp_path, monkeypatch, prefetch=True, eval_steps=1)
        assert [step for step, _ in seen][:3] == [0, 1, 2]
        text = " ".join(strip_ansi(loop["printed"].getvalue()).split())
        assert text.count("fetches batches ahead of the step") == 1
        assert "evaluation for update 1 ran after 0 optimizer step(s)" in text

    def test_the_split_is_taken_away_from_mlx_lm(self, tmp_path, monkeypatch):
        _, _, loop = self._run(tmp_path, monkeypatch, epochs=2, eval_steps=2)
        assert loop["val_dataset"] is None
        assert loop["args"].steps_per_eval > loop["args"].iters

    def test_unset_keeps_mlx_lms_cadence_and_runs_no_evaluation_of_its_own(
        self, tmp_path, monkeypatch,
    ):
        result, seen, loop = self._run(tmp_path, monkeypatch, epochs=2)
        assert seen == []
        assert loop["val_dataset"] is not None
        assert loop["args"].steps_per_eval == max(1, loop["args"].iters // 4)
        assert "eval_updates" not in result


# --------------------------------------------------------------------------
# The evaluator: every row, token-weighted, no side effects
# --------------------------------------------------------------------------


def _np_ce(model, ids, mask=None):
    """(sum of target NLL, number of targets) for one row, independently.

    Text rows follow mlx-lm 0.32's ``default_loss``, the loss MLX SFT trains
    on: its mask is ``steps <= length``, so it also scores the first padding
    token (id 0) after the row. The evaluator reuses the training loss, so its
    number is comparable with the training loss; the oracle states that rule
    rather than inheriting it. Masked chat rows score exactly their mask.
    """
    import mlx.core as mx

    ids = list(ids) + ([0] if mask is None else [])
    logits = np.array(model(mx.array([ids[:-1]]))[0].astype(mx.float32), dtype=np.float64)
    logits -= logits.max(-1, keepdims=True)
    logp = logits - np.log(np.exp(logits).sum(-1, keepdims=True))
    targets = np.array(ids[1:])
    nll = -logp[np.arange(len(targets)), targets]
    keep = np.ones(len(targets), bool) if mask is None else np.array(mask[1:], bool)
    return float(nll[keep].sum()), int(keep.sum())


def _datasets(wrapper, chat):
    from mlx_lm.tuner.datasets import CacheDataset, create_dataset
    from mlx_lm.tuner.trainer import TrainingArgs, default_loss, iterate_batches

    rows = wrapper._dataset["val"]
    if chat:
        from soup_cli.trainer.mlx_masking import (
            MaskedChatDataset,
            masked_iterate_batches,
            masked_loss,
        )

        ds = CacheDataset(MaskedChatDataset(rows, wrapper.tokenizer, chat_key="messages"))
        return ds, masked_loss, masked_iterate_batches
    return (CacheDataset(create_dataset(rows, wrapper.tokenizer, TrainingArgs())),
            default_loss, iterate_batches)


@pytest.mark.usefixtures("native")
class TestEveryRowCounts:
    @pytest.mark.parametrize("chat", [False, True])
    def test_loss_equals_a_hand_oracle_over_every_target_tail_included(self, tmp_path, chat):
        from soup_cli.trainer.mlx_sft import evaluate_all_rows

        wrapper = _wrapper(tmp_path, val=5, chat=chat)
        ds, loss, iterate = _datasets(wrapper, chat)
        value = evaluate_all_rows(wrapper.model, ds, batch_size=2, max_seq_length=256,
                                  loss_fn=loss, iterate_fn=iterate)
        total, count = 0.0, 0
        for i in range(len(ds)):
            item = ds[i]
            ids, mask = (item if chat else (item[0] if isinstance(item, tuple) else item, None))
            row_sum, row_n = _np_ce(wrapper.model, list(ids), mask)
            total, count = total + row_sum, count + row_n
        assert value == pytest.approx(total / count, rel=1e-4)

        # The fifth row is a tail smaller than one batch: leaving it out changes
        # the number, so dropping it is detectable.
        head, head_n = 0.0, 0
        for i in range(4):
            item = ds[i]
            ids, mask = (item if chat else (item[0] if isinstance(item, tuple) else item, None))
            s, n = _np_ce(wrapper.model, list(ids), mask)
            head, head_n = head + s, head_n + n
        assert abs(value - head / head_n) > 1e-4

    def test_evaluation_has_no_side_effects(self, tmp_path):
        import mlx.core as mx
        from mlx.utils import tree_flatten

        from soup_cli.trainer.mlx_sft import evaluate_all_rows

        wrapper = _wrapper(tmp_path, val=5)
        ds, loss, iterate = _datasets(wrapper, False)
        wrapper.model.train()
        before = {k: np.array(v) for k, v in tree_flatten(wrapper.model.parameters())}
        np.random.seed(123)
        rng_before = np.random.get_state()[1].copy()
        mx.random.seed(9)
        evaluate_all_rows(wrapper.model, ds, batch_size=2, max_seq_length=256,
                          loss_fn=loss, iterate_fn=iterate)
        after_draw = mx.random.uniform(shape=(4,)).tolist()
        mx.random.seed(9)
        assert after_draw == mx.random.uniform(shape=(4,)).tolist()
        assert (np.random.get_state()[1] == rng_before).all()
        assert wrapper.model.training
        after = {k: np.array(v) for k, v in tree_flatten(wrapper.model.parameters())}
        for name in before:
            np.testing.assert_array_equal(before[name], after[name], err_msg=name)

    def test_a_batching_helper_that_draws_randomness_still_leaves_numpy_untouched(
        self, tmp_path
    ):
        """mlx-lm's batch helpers shuffle through numpy's global RNG; whatever a
        helper draws during an evaluation is given back."""
        from soup_cli.trainer.mlx_sft import evaluate_all_rows

        wrapper = _wrapper(tmp_path, val=5)
        ds, loss, iterate = _datasets(wrapper, False)

        def drawing(**kwargs):
            np.random.random(17)
            yield from iterate(**kwargs)

        np.random.seed(5)
        state = np.random.get_state()[1].copy()
        evaluate_all_rows(wrapper.model, ds, batch_size=2, max_seq_length=256,
                          loss_fn=loss, iterate_fn=drawing)
        assert (np.random.get_state()[1] == state).all()

    def test_a_split_smaller_than_one_batch_is_evaluated_not_skipped(self, tmp_path, capsys):
        """#1857 skips such a split when the cadence is mlx-lm's; with an
        explicit eval_steps every row is evaluated, so the row stays."""
        wrapper = _wrapper(tmp_path, val=1, eval_steps=2)
        result = wrapper.train()
        assert result["eval_updates"] == [2, 4]
        # Rich colours the "1" on a colour terminal; compare the plain text.
        assert "validation split has 1 row" not in strip_ansi(capsys.readouterr().out)

    def test_scheduled_evaluations_do_not_change_what_trains(self, tmp_path):
        """Evaluating after every update and only at the end train the same
        adapter: the evaluations take no step, no state and no batch."""
        import mlx.core as mx
        from mlx.utils import tree_flatten

        def adapter(path, every):
            wrapper = _wrapper(path, epochs=2, eval_steps=every)
            # Seeded the way #1855 seeds MLX SFT (numpy's global RNG draws the
            # batch order, mx.random the LoRA init), so the two runs are
            # comparable on main with or without it.
            np.random.seed(7)
            mx.random.seed(7)
            wrapper.train()
            return {k: np.array(v) for k, v in
                    tree_flatten(wrapper.model.trainable_parameters())}

        often, once = adapter(tmp_path / "a", 1), adapter(tmp_path / "b", 1000)
        assert often
        for name in often:
            np.testing.assert_array_equal(often[name], once[name], err_msg=name)


@pytest.mark.usefixtures("native")
class TestNativeVariants:
    @pytest.mark.parametrize("quantize", [False, True], ids=["dense", "4bit"])
    def test_runs_and_records_its_evaluations(self, tmp_path, quantize):
        import json

        model = _tiny(hidden=64, quantize=quantize)
        wrapper = _wrapper(tmp_path, epochs=2, gradient_accumulation_steps=2, eval_steps=1,
                           model=model)
        result = wrapper.train()
        assert result["eval_updates"] == [1, 2, 3, 4]
        assert all(np.isfinite(e["val_loss"]) for e in wrapper._eval_log)
        meta = json.loads((tmp_path / "adapter_config.json").read_text())
        assert (meta["eval_steps"], meta["eval_unit"]) == (1, "optimizer updates")