Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

40.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Batched multi-LoRA worker on a tiny random Qwen3 (CPU, PyTorch fallback of the kernels)."""

from __future__ import annotations

import copy
import json
import math
import os
import queue
import types
from pathlib import Path

import sys

import pytest

# The repo root (pytest pythonpath) shadows PYTHONPATH.
if "unsloth" not in sys.modules:
    for _entry in os.environ.get("PYTHONPATH", "").split(os.pathsep)[::-1]:
        if _entry and os.path.isfile(os.path.join(_entry, "unsloth", "kernels", "multi_lora.py")):
            sys.path.insert(0, _entry)

try:
    import torch
    from unsloth.kernels import multi_lora as ml
except Exception as exc:  # noqa: BLE001 - no multi-LoRA kernels or no importable unsloth here
    pytest.skip(f"unsloth.kernels.multi_lora unavailable: {exc}", allow_module_level = True)

from datasets import Dataset
from safetensors.torch import load_file
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import PreTrainedTokenizerFast, ProcessorMixin, Qwen3Config, Qwen3ForCausalLM

from core.training import multi_lora_trainer as mlt

WORDS = [f"w{i}" for i in range(56)]


def _tokenizer():
    vocab = {"[PAD]": 0, "[EOS]": 1, "[U]": 2, "[B]": 3}
    vocab.update({w: i + 4 for i, w in enumerate(WORDS)})
    tok = Tokenizer(models.WordLevel(vocab = vocab, unk_token = "[PAD]"))
    tok.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
    return PreTrainedTokenizerFast(tokenizer_object = tok, pad_token = "[PAD]", eos_token = "[EOS]")


def _base():
    torch.manual_seed(0)
    config = Qwen3Config(
        vocab_size = 64,
        hidden_size = 32,
        intermediate_size = 64,
        num_hidden_layers = 2,
        num_attention_heads = 4,
        num_key_value_heads = 2,
        head_dim = 8,
        max_position_embeddings = 128,
        use_cache = False,
        tie_word_embeddings = False,
    )
    return Qwen3ForCausalLM(config).float()


def _qwen3_5(moe: bool):
    """Tiny Qwen3.5 text model: one Gated DeltaNet (linear attention) layer, one full attention layer; ``moe``
    swaps the MLPs for fused experts plus a shared expert, as in Qwen3.6-35B-A3B."""
    import transformers

    torch.manual_seed(0)
    common = dict(
        vocab_size = 64,
        hidden_size = 32,
        num_hidden_layers = 2,
        num_attention_heads = 4,
        num_key_value_heads = 2,
        head_dim = 8,
        max_position_embeddings = 128,
        use_cache = False,
        tie_word_embeddings = False,
        layer_types = ["linear_attention", "full_attention"],
        linear_num_key_heads = 2,
        linear_num_value_heads = 4,
        linear_key_head_dim = 8,
        linear_value_head_dim = 8,
        linear_conv_kernel_dim = 4,
    )
    if moe:
        config = transformers.Qwen3_5MoeTextConfig(
            **common,
            num_experts = 4,
            num_experts_per_tok = 2,
            moe_intermediate_size = 16,
            shared_expert_intermediate_size = 16,
        )
        return transformers.Qwen3_5MoeForCausalLM(config).float()
    config = transformers.Qwen3_5TextConfig(**common, intermediate_size = 64)
    return transformers.Qwen3_5ForCausalLM(config).float()


def _kind_targets(kind):
    # Lone runs train fused experts for the default targets; a bare "all-linear" string reaches no experts there.
    return {"target_modules": []} if kind == "moe" else {}


def _kind_base(kind):
    if kind == "dense":
        return _base()
    pytest.importorskip("transformers", minversion = "5.2")
    return _qwen3_5(moe = kind == "moe")


def _texts(n, seed):
    g = torch.Generator().manual_seed(seed)
    out = []
    for _ in range(n):
        k = int(torch.randint(4, 12, (1,), generator = g))
        out.append(
            " ".join(WORDS[int(i)] for i in torch.randint(0, 56, (k,), generator = g)) + " [EOS]"
        )
    return out


MEMBERS = [
    dict(run_id = "job_a", rows = 6, batch_size = 2, num_epochs = 1, lr = 1e-2, r = 4, alpha = 8, seed = 11),
    dict(run_id = "job_b", rows = 12, batch_size = 2, num_epochs = 2, lr = 5e-3, r = 8, alpha = 16, seed = 22),
    dict(
        run_id = "job_c",
        rows = 9,
        batch_size = 3,
        num_epochs = 1,
        lr = 2e-2,
        r = 2,
        alpha = 4,
        seed = 33,
        max_steps = 4,
    ),
]
EXPECTED_STEPS = {"job_a": 2, "job_b": 6, "job_c": 4}


def _member_config(m):
    return {
        "model_name": "tiny/qwen3",
        "local_datasets": [m["run_id"]],
        "format_type": "",
        "learning_rate": m["lr"],
        "num_epochs": m["num_epochs"],
        "batch_size": m["batch_size"],
        "lora_r": m["r"],
        "lora_alpha": m["alpha"],
        "random_seed": m["seed"],
        "max_steps": m.get("max_steps", 0),
        "optim": "adamw_torch",
        "warmup_steps": m.get("warmup_steps", 0),
        "weight_decay": m.get("weight_decay", 0.0),
        "gradient_accumulation_steps": 2,
        "packing": m.get("packing", False),
    }


class _Events:
    def __init__(self, on_event = None):
        self.items = []
        self.on_event = on_event

    def put(self, event):
        self.items.append(event)
        if self.on_event is not None:
            self.on_event(event)


def _run(
    monkeypatch,
    members,
    on_event = None,
    trainer_attrs = None,
    datasets = None,
    model_attrs = None,
    base = None,
    wrap_tokenizer = None,
    **extra,
):
    base = _base() if base is None else base
    for name, value in (model_attrs or {}).items():
        setattr(base, name, value)
    pristine = copy.deepcopy(base)
    tokenizer = _tokenizer()
    trainer = types.SimpleNamespace(
        model = None,
        tokenizer = None,
        should_stop = False,
        save_on_stop = True,
        load_in_4bit = False,
        is_cpt = False,
        **(trainer_attrs or {}),
    )
    data = {
        m["run_id"]: {
            "dataset": Dataset.from_dict({"text": _texts(m["rows"], m["seed"])}),
            "final_format": "",
        }
        for m in members
    }
    data.update(datasets or {})
    inboxes = []

    def load_base(trainer, config, hf_token, event_queue):
        trainer.model = base
        trainer.tokenizer = tokenizer if wrap_tokenizer is None else wrap_tokenizer(tokenizer)
        return False

    def stop_reader(stop_queue, inbox, trainer):
        inboxes.append(inbox)

    monkeypatch.setattr(mlt, "_new_trainer", lambda: trainer)
    monkeypatch.setattr(mlt, "_detect_model", lambda trainer, config, hf_token: None)
    monkeypatch.setattr(mlt, "_format_dataset", lambda t, cfg, tok: data[cfg["local_datasets"][0]])
    monkeypatch.setattr(mlt, "_load_base", load_base)
    monkeypatch.setattr(mlt, "_start_stop_reader", stop_reader)
    configs = [_member_config(m) for m in members]
    config = {
        **configs[0],
        "max_seq_length": 64,
        "gradient_checkpointing": "unsloth",
        "target_modules": ["all-linear"],
        "batch_members": [{"run_id": m["run_id"], "config": c} for m, c in zip(members, configs)],
        **extra,
    }
    events = _Events(None if on_event is None else lambda e: on_event(e, inboxes[0]))
    mlt.run_multi_lora_training(events, queue.Queue(), config)
    return events.items, base, pristine


def _member_events(events, run_id):
    return [e for e in events if e.get("member_run_id") == run_id]


def _adapter_logits(model, slot, input_ids):
    ml.set_multi_lora_batch(
        model, torch.full((input_ids.shape[0],), slot), seq_len = input_ids.shape[1]
    )
    with torch.no_grad():
        return model(input_ids = input_ids, use_cache = False).logits


@pytest.fixture(scope = "module")
def batched_run(tmp_path_factory):
    monkeypatch = pytest.MonkeyPatch()
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path_factory.mktemp("studio_home")))
    try:
        yield _run(monkeypatch, MEMBERS)
    finally:
        monkeypatch.undo()


def test_each_member_reports_and_exports_on_its_own(batched_run):
    events, model, pristine = batched_run
    assert [e["type"] for e in events if "member_run_id" not in e][-1] == "complete"
    dirs = set()
    for m in MEMBERS:
        mine = _member_events(events, m["run_id"])
        kinds = [e["type"] for e in mine]
        assert kinds[0] == "output_dir" and kinds[-1] == "complete"
        assert set(kinds[1:-1]) == {"progress"}
        steps = [e["step"] for e in mine if e["type"] == "progress"]
        assert steps == list(range(EXPECTED_STEPS[m["run_id"]] + 1))
        tokens = [e["num_tokens"] for e in mine if e["type"] == "progress"]
        assert tokens == sorted(tokens) and tokens[-1] > 0
        assert all(
            e["total_steps"] == EXPECTED_STEPS[m["run_id"]] for e in mine if e["type"] == "progress"
        )
        out = mine[0]["output_dir"]
        assert mine[-1]["output_dir"] == out
        dirs.add(out)
        config = json.load(open(os.path.join(out, "adapter_config.json")))
        assert config["r"] == m["r"] and config["lora_alpha"] == m["alpha"]
        assert config["unsloth_training_method"] == "lora"
        assert os.path.exists(os.path.join(out, "tokenizer.json"))
    assert len(dirs) == len(MEMBERS)


def test_each_step_reports_the_rate_it_trained_with(batched_run):
    events, _model, _pristine = batched_run
    for m in MEMBERS:
        total = EXPECTED_STEPS[m["run_id"]]
        reported = [
            e["learning_rate"]
            for e in _member_events(events, m["run_id"])
            if e["type"] == "progress" and e["step"] > 0
        ]
        # Linear decay without warmup, read before each step: lr, then lr * (1 - 1/total), ...
        assert reported == pytest.approx([m["lr"] * (1 - k / total) for k in range(total)])


def test_exported_adapter_reloads_with_peft_and_matches(batched_run):
    from peft import PeftModel

    events, model, pristine = batched_run
    model.eval()
    input_ids = torch.randint(4, 60, (2, 10), generator = torch.Generator().manual_seed(5))
    for slot, m in enumerate(MEMBERS):
        out = _member_events(events, m["run_id"])[0]["output_dir"]
        state = load_file(os.path.join(out, "adapter_model.safetensors"))
        assert any(v.abs().sum() > 0 for k, v in state.items() if "lora_B" in k)
        peft_model = PeftModel.from_pretrained(copy.deepcopy(pristine), out).eval()
        with torch.no_grad():
            expected = peft_model(input_ids = input_ids).logits
        torch.testing.assert_close(
            _adapter_logits(model, slot, input_ids), expected, atol = 1e-5, rtol = 1e-5
        )


def test_finished_member_gets_no_more_updates(batched_run):
    events, model, pristine = batched_run
    out = _member_events(events, "job_a")[0]["output_dir"]
    exported = load_file(os.path.join(out, "adapter_model.safetensors"))
    current = ml.adapter_state_dict(model, 0)
    assert exported.keys() == current.keys()
    for key in exported:
        torch.testing.assert_close(current[key], exported[key], atol = 0, rtol = 0)


def test_stop_member_stops_only_that_member(monkeypatch):
    def on_event(event, inbox):
        if (
            event.get("member_run_id") == "job_b"
            and event["type"] == "progress"
            and event["step"] == 2
        ):
            inbox.put({"type": "stop_member", "run_id": "job_b", "save": True})

    events, model, pristine = _run(monkeypatch, MEMBERS, on_event)
    b = _member_events(events, "job_b")
    assert b[-1]["type"] == "stopped" and b[-1]["saved"] is True
    assert max(e["step"] for e in b if e["type"] == "progress") == 2
    assert os.path.exists(os.path.join(b[-1]["output_dir"], "adapter_model.safetensors"))
    for run_id in ("job_a", "job_c"):
        mine = _member_events(events, run_id)
        assert mine[-1]["type"] == "complete"
        assert max(e["step"] for e in mine if e["type"] == "progress") == EXPECTED_STEPS[run_id]
    assert events[-1]["type"] == "complete" and "member_run_id" not in events[-1]


def test_seed_init_is_independent_of_batch_order():
    first, second = _base(), _base()
    ml.inject_multi_lora(first, ["q_proj", "down_proj"], 2, r = [4, 8], lora_alpha = [8, 16])
    ml.inject_multi_lora(second, ["q_proj", "down_proj"], 2, r = [8, 4], lora_alpha = [16, 8])
    mlt._seed_adapter(first, 0, 11)
    mlt._seed_adapter(first, 1, 22)
    mlt._seed_adapter(second, 0, 22)
    mlt._seed_adapter(second, 1, 11)
    a, b = ml.adapter_state_dict(first, 0), ml.adapter_state_dict(second, 1)
    for key in a:
        torch.testing.assert_close(a[key], b[key], atol = 0, rtol = 0)
    assert all(v.abs().sum() == 0 for k, v in a.items() if "lora_B" in k)


def test_completion_masking_masks_the_prompt():
    tokenizer = _tokenizer()
    tokenizer._unsloth_input_part = "[U]"
    tokenizer._unsloth_output_part = "[B]"
    info = {
        "dataset": Dataset.from_dict({"text": ["[U] w1 w2 w3 [B] w4 w5 [EOS]"] * 3}),
        "final_format": "",
    }
    tok, rows = mlt._tokenize_member(
        info, tokenizer, {"train_on_completions": True}, "tiny/qwen3", 64
    )
    assert tok is tokenizer
    ids, labels = rows[0]
    response = ids.index(tokenizer.convert_tokens_to_ids("[B]")) + 1
    assert all(t == -100 for t in labels[:response])
    assert labels[response:] == ids[response:]


def test_completion_drops_from_truncation_name_the_context_length():
    tokenizer = _tokenizer()
    tokenizer._unsloth_input_part = "[U]"
    tokenizer._unsloth_output_part = "[B]"
    long_prompt = "[U] " + " ".join(WORDS[:20]) + " [B] w4 w5 [EOS]"
    info = {
        "dataset": Dataset.from_dict({"text": [long_prompt] * 3 + ["[U] w1 [B] w2 [EOS]"]}),
        "final_format": "",
    }
    with pytest.raises(ValueError) as exc:
        mlt._tokenize_member(info, tokenizer, {"train_on_completions": True}, "tiny/qwen3", 16)
    message = str(exc.value)
    assert "3/4 samples (75.0%) were dropped" in message
    assert "3 of them reach the context length (16 tokens)" in message
    assert "response template was not found in the formatted samples" not in message


def test_completion_drops_without_a_template_keep_the_template_advice():
    tokenizer = _tokenizer()
    tokenizer._unsloth_input_part = "[U]"
    tokenizer._unsloth_output_part = "[B]"
    info = {"dataset": Dataset.from_dict({"text": ["[U] w1 w2 [EOS]"] * 3}), "final_format": ""}
    with pytest.raises(ValueError) as exc:
        mlt._tokenize_member(info, tokenizer, {"train_on_completions": True}, "tiny/qwen3", 64)
    assert "response template was not found" in str(exc.value)
    assert "context length" not in str(exc.value)


def test_batch_trains_through_sft_trainer_and_saves_checkpoints(monkeypatch, tmp_path):
    from trl import SFTTrainer

    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    built = []
    original = mlt._build_sft_trainer

    def spy(*args, **kwargs):
        built.append(original(*args, **kwargs))
        return built[-1]

    monkeypatch.setattr(mlt, "_build_sft_trainer", spy)
    events, model, pristine = _run(monkeypatch, MEMBERS[:2], save_steps = 2)
    assert isinstance(built[0], SFTTrainer) and built[0].state.global_step == 6
    found = {}
    for m in MEMBERS[:2]:
        out = _member_events(events, m["run_id"])[0]["output_dir"]
        found[m["run_id"]] = sorted(p.name for p in Path(out).glob("checkpoint-*"))
        last = os.path.join(out, found[m["run_id"]][-1], "adapter_model.safetensors")
        final = load_file(os.path.join(out, "adapter_model.safetensors"))
        for key, value in load_file(last).items():
            torch.testing.assert_close(value, final[key], atol = 0, rtol = 0)
    assert found == {
        "job_a": ["checkpoint-2"],
        "job_b": ["checkpoint-2", "checkpoint-4", "checkpoint-6"],
    }


def test_gradients_are_clipped_per_member_every_step(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    seen = []
    real = ml.clip_grad_norm_per_adapter_

    def spy(model, max_norm):
        norms = real(model, max_norm)
        seen.append((max_norm, norms.clone()))
        return norms

    monkeypatch.setattr(ml, "clip_grad_norm_per_adapter_", spy)
    events, _model, _pristine = _run(monkeypatch, MEMBERS[:2], max_grad_norm = 0.01)
    assert len(seen) == EXPECTED_STEPS["job_b"]
    assert all(max_norm == 0.01 for max_norm, _ in seen)
    # Each member reports its own pre-clip norm, not one shared global norm.
    a = [
        e["grad_norm"]
        for e in _member_events(events, "job_a")
        if e["type"] == "progress" and e["step"]
    ]
    b = [
        e["grad_norm"]
        for e in _member_events(events, "job_b")
        if e["type"] == "progress" and e["step"]
    ]
    assert a == pytest.approx([float(n[0]) for _, n in seen[:2]])
    assert b == pytest.approx([float(n[1]) for _, n in seen])


def test_each_member_keeps_its_own_weight_decay_and_warmup(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    members = [
        {**MEMBERS[0], "weight_decay": 0.1, "warmup_steps": 1},
        {**MEMBERS[1], "weight_decay": 0.0, "warmup_steps": 0},
    ]
    built = []
    original = mlt._build_sft_trainer
    monkeypatch.setattr(
        mlt, "_build_sft_trainer", lambda *a, **k: built.append(original(*a, **k)) or built[-1]
    )
    events, _model, _pristine = _run(monkeypatch, members)
    assert [g["weight_decay"] for g in built[0].optimizer.param_groups] == [0.1, 0.0]
    for m in members:
        total = EXPECTED_STEPS[m["run_id"]]
        reported = [
            e["learning_rate"]
            for e in _member_events(events, m["run_id"])
            if e["type"] == "progress" and e["step"]
        ]
        warmup = m["warmup_steps"]
        # transformers' linear schedule: ramp over the member's own warmup, then decay over its own length.
        expected = [
            m["lr"] * (k / warmup if k < warmup else (total - k) / (total - warmup))
            for k in range(total)
        ]
        assert reported == pytest.approx(expected)


def test_targets_follow_the_single_job_rules():
    model = _base()
    every_linear = mlt._resolve_targets(model, ["all-linear"])
    assert (
        "q_proj" in every_linear and "down_proj" in every_linear and "lm_head" not in every_linear
    )
    # Explicit names beside "all-linear" win, as UnslothTrainer.prepare_model_for_training does.
    assert mlt._resolve_targets(model, ["all-linear", "q_proj"]) == ["q_proj"]
    assert mlt._resolve_targets(model, None) == mlt._DEFAULT_TARGETS


def test_a_text_diffusion_model_never_trains_batched(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    events, _model, _pristine = _run(
        monkeypatch, MEMBERS[:2], trainer_attrs = {"diffusion_lm_profile": "dream"}
    )
    assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]
    assert "text-diffusion" in events[-1]["error"]


def test_an_existing_lora_adapter_never_trains_batched(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    events, _model, _pristine = _run(
        monkeypatch, MEMBERS[:2], model_attrs = {"peft_config": {"default": object()}}
    )
    assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]
    assert "existing LoRA adapter" in events[-1]["error"]


def test_a_moe_model_never_trains_batched(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    config = copy.deepcopy(_base().config)
    config.num_experts = 8
    events, _model, _pristine = _run(monkeypatch, MEMBERS[:2], model_attrs = {"config": config})
    assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]
    assert "MoE" in events[-1]["error"]


def test_pretokenized_assistant_masks_become_labels():
    ids = [[5, 6, 7, 8], [9, 10, 11]]
    info = {
        "dataset": Dataset.from_dict(
            {"input_ids": ids, "assistant_masks": [[0, 0, 1, 1], [0, 1, 1]]}
        ),
        "final_format": "pretokenized",
    }
    _tok, rows = mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64)
    assert [labels for _, labels in rows] == [[-100, -100, 7, 8], [-100, 10, 11]]


def test_a_pretokenized_mask_of_the_wrong_length_is_refused():
    info = {
        "dataset": Dataset.from_dict({"input_ids": [[5, 6, 7, 8]], "assistant_masks": [[0, 1]]}),
        "final_format": "pretokenized",
    }
    with pytest.raises(ValueError, match = "assistant_masks"):
        mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64)


def test_a_pretokenized_token_outside_the_vocabulary_is_refused():
    info = {
        "dataset": Dataset.from_dict({"input_ids": [[5, 6, 900]]}),
        "final_format": "pretokenized",
    }
    with pytest.raises(ValueError, match = "outside this model's vocabulary"):
        mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64, vocab_size = 64)


def test_one_members_bad_dataset_fails_only_that_member(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    bad = {
        "dataset": Dataset.from_dict({"input_ids": [[5, 6, 900]] * 4}),
        "final_format": "pretokenized",
    }
    events, _model, _pristine = _run(monkeypatch, MEMBERS[:2], datasets = {"job_a": bad})
    a, b = _member_events(events, "job_a"), _member_events(events, "job_b")
    assert [e["type"] for e in a] == ["error"] and "vocabulary" in a[0]["error"]
    assert b[-1]["type"] == "complete"
    assert events[-1]["type"] == "complete" and "member_run_id" not in events[-1]


@pytest.mark.parametrize("collator_masks_labels", [True, False])
def test_pretokenized_labels_and_assistant_masks(monkeypatch, collator_masks_labels):
    import utils.datasets.pretokenized as pretokenized

    monkeypatch.setattr(pretokenized, "_collator_masks_user_labels", lambda: collator_masks_labels)
    info = {
        "dataset": Dataset.from_dict(
            {
                "input_ids": [[5, 6, 7, 8]],
                "labels": [[5, 6, 7, 8]],
                "assistant_masks": [[0, 0, 1, 1]],
            }
        ),
        "final_format": "pretokenized",
    }
    _tok, rows = mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64)
    # The single-job collator masks user labels only before TRL 1.7.
    assert rows[0][1] == ([-100, -100, 7, 8] if collator_masks_labels else [5, 6, 7, 8])


def test_batched_member_trains_as_it_would_alone(batched_run, monkeypatch):
    events, model, pristine = batched_run
    batched = load_file(
        os.path.join(_member_events(events, "job_b")[0]["output_dir"], "adapter_model.safetensors")
    )
    alone_events, _, _ = _run(monkeypatch, [MEMBERS[1]])
    alone = load_file(
        os.path.join(
            _member_events(alone_events, "job_b")[0]["output_dir"], "adapter_model.safetensors"
        )
    )
    batched_loss = [e["loss"] for e in _member_events(events, "job_b") if e["type"] == "progress"]
    alone_loss = [
        e["loss"] for e in _member_events(alone_events, "job_b") if e["type"] == "progress"
    ]
    assert batched_loss[1:] == pytest.approx(alone_loss[1:], rel = 1e-4)
    for key in alone:
        torch.testing.assert_close(batched[key], alone[key], atol = 1e-5, rtol = 1e-4)


@pytest.mark.parametrize("kind", ["dense", "hybrid", "moe"])
def test_a_neighbour_never_moves_a_members_training(monkeypatch, tmp_path, kind):
    """Job B batched beside A, then beside a different A (data, rate, rank, seed): B's losses and adapter match."""
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    other = dict(MEMBERS[0], rows = 10, lr = 3e-2, r = 2, alpha = 2, seed = 99)
    runs = []
    for neighbour in (MEMBERS[0], other):
        events, _, _ = _run(
            monkeypatch, [neighbour, MEMBERS[1]], base = _kind_base(kind), **_kind_targets(kind)
        )
        mine = _member_events(events, "job_b")
        losses = [e["loss"] for e in mine if e["type"] == "progress"]
        runs.append(
            (losses, load_file(os.path.join(mine[0]["output_dir"], "adapter_model.safetensors")))
        )
    (loss_1, adapter_1), (loss_2, adapter_2) = runs
    assert len(loss_1) == EXPECTED_STEPS["job_b"] + 1
    assert loss_2 == pytest.approx(loss_1, rel = 1e-5)
    assert adapter_1.keys() == adapter_2.keys()
    for key in adapter_1:
        torch.testing.assert_close(adapter_2[key], adapter_1[key], atol = 1e-6, rtol = 1e-5)


@pytest.mark.parametrize("kind", ["hybrid", "moe"])
def test_a_hybrid_or_moe_member_trains_as_it_would_alone(monkeypatch, tmp_path, kind):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    batched, _, _ = _run(monkeypatch, MEMBERS[:2], base = _kind_base(kind), **_kind_targets(kind))
    alone, _, _ = _run(monkeypatch, [MEMBERS[1]], base = _kind_base(kind), **_kind_targets(kind))
    out = {}
    for name, events in (("batched", batched), ("alone", alone)):
        mine = _member_events(events, "job_b")
        assert mine[-1]["type"] == "complete"
        out[name] = (
            [e["loss"] for e in mine if e["type"] == "progress"],
            load_file(os.path.join(mine[0]["output_dir"], "adapter_model.safetensors")),
            json.load(open(os.path.join(mine[0]["output_dir"], "adapter_config.json"))),
        )
    assert out["batched"][0][1:] == pytest.approx(out["alone"][0][1:], rel = 1e-4)
    for key, value in out["alone"][1].items():
        torch.testing.assert_close(out["batched"][1][key], value, atol = 1e-5, rtol = 1e-4)
    # Every adapter trained, dense and expert alike (injecting experts must not freeze the dense ones).
    trained = {
        k: v.abs().max() > 0 for k, v in out["batched"][1].items() if k.endswith("lora_B.weight")
    }
    assert trained and all(trained.values()), [k for k, ok in trained.items() if not ok][:3]
    if kind == "moe":
        config = out["batched"][2]
        assert sorted(config["target_parameters"]) == [
            "mlp.experts.down_proj",
            "mlp.experts.gate_up_proj",
        ]
        experts = [
            k for k in out["batched"][1] if ".mlp.experts." in k and k.endswith("lora_B.weight")
        ]
        assert len(experts) == 4 and all(out["batched"][1][k].abs().max() > 0 for k in experts)
    else:
        # Gated DeltaNet projections train exactly when all-linear asks for them, as alone.
        assert any(".linear_attn.in_proj_qkv." in k for k in out["batched"][1])


def test_an_allowlisted_moe_with_a_router_aux_loss_trains_alone(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    pytest.importorskip("transformers", minversion = "5.2")
    base = _qwen3_5(moe = True)
    base.config.output_router_logits = True
    events, _, _ = _run(monkeypatch, MEMBERS[:2], base = base, target_modules = [])
    assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]
    assert "auxiliary loss" in events[-1]["error"]


def test_a_state_space_model_trains_alone(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    base = _base()
    base.model.layers[0].mixer = type("MambaMixer", (torch.nn.Module,), {})()
    events, _, _ = _run(monkeypatch, MEMBERS[:2], base = base)
    assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]
    assert "state-space" in events[-1]["error"]


PACKED = [
    dict(
        run_id = "job_a",
        rows = 12,
        batch_size = 2,
        num_epochs = 2,
        lr = 1e-2,
        r = 4,
        alpha = 8,
        seed = 11,
        packing = True,
    ),
    dict(
        run_id = "job_b",
        rows = 16,
        batch_size = 1,
        num_epochs = 1,
        lr = 5e-3,
        r = 8,
        alpha = 16,
        seed = 22,
        packing = True,
    ),
    dict(run_id = "job_c", rows = 7, batch_size = 2, num_epochs = 1, lr = 2e-2, r = 2, alpha = 4, seed = 33),
]


def _final(events, run_id):
    mine = _member_events(events, run_id)
    losses = [e["loss"] for e in mine if e["type"] == "progress" and e["step"] > 0]
    state = load_file(os.path.join(mine[0]["output_dir"], "adapter_model.safetensors"))
    return losses, state


def _spy_packed(monkeypatch):
    seen = []
    real = mlt._collate_packed

    def spy(rows, slots, lengths):
        batch = real(rows, slots, lengths)
        seen.append(batch)
        return batch

    monkeypatch.setattr(mlt, "_collate_packed", spy)
    return seen


def test_packing_members_never_share_a_document_or_see_a_neighbour(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    seen = _spy_packed(monkeypatch)
    events, _, _ = _run(monkeypatch, PACKED, max_seq_length = 24)
    assert seen
    for batch in seen:
        ids, pos, lengths = (
            batch["adapter_ids"][0],
            batch["position_ids"][0],
            batch["packed_seq_lengths"],
        )
        starts = (pos == 0).nonzero().flatten()
        assert starts.tolist() == (torch.cumsum(lengths, 0) - lengths).tolist()
        # One adapter per document, and the label opening each document is never trained.
        for start, n in zip(starts.tolist(), lengths.tolist()):
            assert ids[start : start + n].unique().numel() == 1
        assert (batch["labels"][0, starts[1:]] == -100).all()
    rows_a = math.ceil(12 / 2)  # unpacked micro-batches per epoch
    steps_a = max(e["step"] for e in _member_events(events, "job_a") if e["type"] == "progress")
    assert steps_a < 2 * math.ceil(rows_a / 2)
    neighbour = [
        dict(PACKED[0]),
        dict(PACKED[1], rows = 9, seed = 99, lr = 3e-2),
        dict(PACKED[2], packing = True),
    ]
    other, _, _ = _run(monkeypatch, neighbour, max_seq_length = 24)
    alone, _, _ = _run(monkeypatch, [PACKED[0]], max_seq_length = 24)
    losses, state = _final(events, "job_a")
    for run in (other, alone):
        run_losses, run_state = _final(run, "job_a")
        assert run_losses == pytest.approx(losses, rel = 1e-4)
        # Float noise from the shared row's size only; a leaked neighbour moves the losses ~1e-3.
        for key in state:
            torch.testing.assert_close(run_state[key], state[key], atol = 1e-4, rtol = 1e-3)


def test_an_unpacked_member_beside_packed_ones_trains_as_it_would_alone(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    mixed, _, _ = _run(monkeypatch, PACKED, max_seq_length = 24)
    alone, _, _ = _run(monkeypatch, [PACKED[2]], max_seq_length = 24)
    losses, state = _final(mixed, "job_c")
    alone_losses, alone_state = _final(alone, "job_c")
    assert losses == pytest.approx(alone_losses, rel = 1e-4)
    for key in state:
        torch.testing.assert_close(state[key], alone_state[key], atol = 1e-5, rtol = 1e-4)


def test_packing_is_ignored_where_the_single_job_ignores_it(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    monkeypatch.setattr(mlt, "_packs", lambda model: False)
    seen = _spy_packed(monkeypatch)
    events, _, _ = _run(monkeypatch, PACKED[:1], max_seq_length = 24)
    assert seen == []
    steps = max(e["step"] for e in _member_events(events, "job_a") if e["type"] == "progress")
    assert steps == 2 * math.ceil(math.ceil(12 / 2) / 2)


@pytest.mark.parametrize("varlen", [True, False])
def test_a_hybrid_packs_only_with_the_varlen_kernels(monkeypatch, varlen):
    transformers = pytest.importorskip("transformers", minversion = "5.2")
    from unsloth import trainer as unsloth_trainer

    config = transformers.Qwen3_5TextConfig(
        vocab_size = 64,
        hidden_size = 32,
        intermediate_size = 64,
        num_hidden_layers = 2,
        num_attention_heads = 4,
        num_key_value_heads = 2,
        head_dim = 8,
        layer_types = ["linear_attention", "full_attention"],
        linear_num_key_heads = 2,
        linear_num_value_heads = 4,
        linear_key_head_dim = 8,
        linear_value_head_dim = 8,
    )
    model = transformers.Qwen3_5ForCausalLM(config)
    calls = []

    def patch(m):
        calls.append(m)
        return varlen

    monkeypatch.setattr(unsloth_trainer, "patch_hybrid_linear_attention_varlen", patch)
    assert mlt._packs(model) is varlen and calls == [model]
    assert mlt._packs(_base()) is True


def test_pack_rows_keeps_each_documents_labels_with_its_tokens():
    rows = [([i] * n, [-100] + [i] * (n - 1)) for i, n in enumerate([5, 9, 3, 7, 4], start = 1)]
    packed, lengths = mlt._pack_rows(rows, 12)
    assert all(len(ids) <= 12 and sum(n) == len(ids) for (ids, _), n in zip(packed, lengths))
    docs = sorted(
        (ids[s : s + n], lab[s : s + n])
        for (ids, lab), ns in zip(packed, lengths)
        for s, n in zip([sum(ns[:k]) for k in range(len(ns))], ns)
    )
    assert docs == sorted(rows)


def _force_member_loss_scale(monkeypatch, poison = None):
    """fp16's per-member loss scale on this CPU run; ``poison`` = (slot, call) overflows that member there."""
    monkeypatch.setattr(mlt, "_fp16_loss_scale", lambda fp16, n: mlt._MemberLossScale(n))
    if poison is None:
        return
    real, calls = mlt._MemberLossScale.unscale_, []

    def unscale_(self, slot, params):
        if slot == poison[0]:
            calls.append(slot)
            if len(calls) == poison[1]:
                for p in params:
                    if p.grad is not None:
                        p.grad.fill_(float("inf"))
        return real(self, slot, params)

    monkeypatch.setattr(mlt._MemberLossScale, "unscale_", unscale_)


def test_member_loss_scale_trains_as_unscaled(batched_run, monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    events, _, _ = batched_run
    _force_member_loss_scale(monkeypatch)
    scaled, _, _ = _run(monkeypatch, MEMBERS)
    for m in MEMBERS:
        losses, state = _final(events, m["run_id"])
        scaled_losses, scaled_state = _final(scaled, m["run_id"])
        assert scaled_losses == pytest.approx(losses, rel = 1e-5)
        for key in state:
            torch.testing.assert_close(scaled_state[key], state[key], atol = 1e-6, rtol = 1e-5)


def test_an_fp16_overflow_skips_only_that_members_step(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    _force_member_loss_scale(monkeypatch)
    clean, _, _ = _run(monkeypatch, MEMBERS)
    _force_member_loss_scale(monkeypatch, poison = (0, 1))
    hit, _, _ = _run(monkeypatch, MEMBERS)
    for run_id in ("job_b", "job_c"):
        losses, state = _final(clean, run_id)
        hit_losses, hit_state = _final(hit, run_id)
        assert hit_losses == losses
        for key in state:
            torch.testing.assert_close(hit_state[key], state[key], atol = 0, rtol = 0)
    a = [e for e in _member_events(hit, "job_a") if e["type"] == "progress" and e["step"] > 0]
    # Its first step was skipped: no gradient norm, and the next step trains at the rate it would have used.
    assert a[0]["grad_norm"] is None and a[1]["grad_norm"] is not None
    assert a[1]["learning_rate"] == a[0]["learning_rate"]
    hit_a, clean_a = _final(hit, "job_a")[1], _final(clean, "job_a")[1]
    assert any(not torch.equal(hit_a[k], clean_a[k]) for k in clean_a)


def test_member_loss_scale_follows_grad_scaler_rules():
    scale = mlt._MemberLossScale(2, init_scale = 8.0, growth_interval = 2)
    p = torch.nn.Parameter(torch.zeros(3))
    for _ in range(2):
        p.grad = torch.full((3,), 16.0)
        assert scale.unscale_(1, [p]) and torch.equal(p.grad, torch.full((3,), 2.0))
    assert scale.scale == [8.0, 16.0]
    p.grad = torch.tensor([1.0, float("nan"), 0.0])
    assert not scale.unscale_(0, [p]) and p.grad is None
    assert scale.scale == [4.0, 16.0] and scale.good_steps == [0, 2 % 2]


def test_pack_rows_on_trl_without_packing_strategies(monkeypatch):
    import trl.data_utils

    real = trl.data_utils.pack_dataset

    def legacy(
        dataset,
        seq_length,
        map_kwargs = None,
    ):
        return real(dataset, seq_length, strategy = "wrapped")

    monkeypatch.setattr(trl.data_utils, "pack_dataset", legacy)
    rows = [([i] * n, [-100] + [i] * (n - 1)) for i, n in enumerate([5, 9, 3, 7, 4], start = 1)]
    packed, lengths = mlt._pack_rows(rows, 12)
    # Documents wrap across rows; each row is one sequence, as TRL's wrapped packing trains it.
    assert [ids for ids, _ in packed] == [
        [t for ids, _ in rows for t in ids][i : i + 12] for i in range(0, 28, 12)
    ]
    assert [labels for _, labels in packed] == [
        [t for _, lab in rows for t in lab][i : i + 12] for i in range(0, 28, 12)
    ]
    assert lengths == [[12], [12], [4]]


class _Processor(ProcessorMixin):
    """A vision model's processor: called with text, it reads it as images."""

    def __init__(self, tokenizer):
        self.tokenizer = tokenizer

    def __call__(
        self,
        images = None,
        text = None,
        **kwargs,
    ):
        raise ValueError("Incorrect image source.")

    def save_pretrained(self, path, **kwargs):
        self.tokenizer.save_pretrained(path)
        open(os.path.join(path, "preprocessor_config.json"), "w").write("{}")


def test_a_vision_models_processor_tokenizes_text_as_the_single_job_does(
    batched_run, monkeypatch, tmp_path
):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    events, _, _ = _run(monkeypatch, MEMBERS, wrap_tokenizer = _Processor)
    plain, _, _ = batched_run
    for m in MEMBERS:
        mine = _member_events(events, m["run_id"])
        assert mine[-1]["type"] == "complete", mine[-1]
        assert os.path.exists(os.path.join(mine[-1]["output_dir"], "preprocessor_config.json"))
        losses = [e["loss"] for e in mine if e["type"] == "progress"]
        assert losses == pytest.approx(
            [e["loss"] for e in _member_events(plain, m["run_id"]) if e["type"] == "progress"],
            rel = 1e-5,
        )


def test_pack_rows_packs_each_worker_shard_as_the_single_job_does():
    from datasets import Dataset
    from trl.data_utils import pack_dataset

    rows = [([i % 50 + 4] * 8, [i % 50 + 4] * 8) for i in range(16)]
    single = pack_dataset(
        Dataset.from_dict({"input_ids": [r[0] for r in rows], "labels": [r[1] for r in rows]}),
        2048,
        strategy = "bfd",
        map_kwargs = {"num_proc": 4},
    )
    packed, lengths = mlt._pack_rows(rows, 2048, num_proc = 4)
    assert len(packed) == len(single) == 4
    assert [ids for ids, _ in packed] == [list(x) for x in single["input_ids"]]
    assert lengths == [list(x) for x in single["seq_lengths"]]


def test_packing_is_refused_where_packed_documents_see_each_other(monkeypatch, tmp_path):
    import transformers.masking_utils as masking_utils

    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    assert mlt._packed_forward_isolated(_base().eval())
    # Attention that ignores the document boundaries, as Qwen3.5's full attention did on transformers 5.3.
    monkeypatch.setattr(masking_utils, "find_packed_sequence_indices", lambda position_ids: None)
    assert not mlt._packed_forward_isolated(_base().eval())
    seen = _spy_packed(monkeypatch)
    events, _, _ = _run(monkeypatch, PACKED[:2], max_seq_length = 24)
    assert seen == []
    assert all(_member_events(events, m["run_id"])[-1]["type"] == "complete" for m in PACKED[:2])


@pytest.mark.parametrize("target", ["embed_tokens", "lm_head", "model.embed_tokens"])
def test_an_embedding_target_fails_instead_of_training_something_else(
    monkeypatch, tmp_path, target
):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    events, model, pristine = _run(monkeypatch, MEMBERS[:2], target_modules = ["q_proj", target])
    for m in MEMBERS[:2]:
        mine = _member_events(events, m["run_id"])
        assert mine[-1]["type"] == "error" and target in mine[-1]["error"], mine[-1]
        assert not any(e.get("output_dir") for e in mine)
    assert type(model.lm_head) is type(pristine.lm_head)
    assert [e["type"] for e in events if "member_run_id" not in e][-1] == "error"


def test_an_expert_only_moe_target_trains_batched(monkeypatch, tmp_path):
    monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
    events, _, _ = _run(
        monkeypatch, MEMBERS[:2], base = _kind_base("moe"), target_modules = ["gate_up_proj"]
    )
    mine = _member_events(events, "job_b")
    assert mine[-1]["type"] == "complete", mine[-1]
    adapter = load_file(os.path.join(mine[0]["output_dir"], "adapter_model.safetensors"))
    assert adapter and all(".mlp.experts." in k for k in adapter), sorted(adapter)[:3]
    assert all(v.abs().max() > 0 for k, v in adapter.items() if k.endswith("lora_B.weight"))


def test_routing_check_runs_on_a_kernel_without_expert_support(monkeypatch):
    import sys
    import types

    import unsloth.kernels

    multi_lora_trainer = mlt
    from unsloth.kernels import multi_lora

    model = torch.nn.Sequential(torch.nn.Linear(4, 4))
    multi_lora.inject_multi_lora(model, ["0"], 2, 2, 2)
    # An Unsloth release that predates the expert kernel: only the dense layer class exists.
    old = types.SimpleNamespace(MultiLoraLinear = multi_lora.MultiLoraLinear)
    monkeypatch.setattr(unsloth.kernels, "multi_lora", old)
    monkeypatch.setitem(sys.modules, "unsloth.kernels.multi_lora", old)
    assert [name for name, _ in multi_lora_trainer._routed_modules(model)] == ["0"]