Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

67.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""_preflight_first_batch rejects an empty/non-integer first batch (the base-model
empty-chat-template crash) before train(). The real methods are bound onto a light
fake self so the production logic runs against controlled batches."""

import contextlib
import importlib
import json
import os
import queue
import subprocess
import sys
import threading
import types
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock

import pytest


def _shared_setup_1(monkeypatch):
    from hub.utils import dataset_cache

    _patch_dataset_formatting(monkeypatch)
    trainer = _dataset_loader_self()
    return dataset_cache, trainer


torch = pytest.importorskip("torch")


_STUBBED: list[str] = []


def _stub_if_missing(name, attrs):
    """Stubs unsloth and trl when absent, since the backend CI job lacks them; real installs are kept."""
    if name in sys.modules:
        return
    try:
        importlib.import_module(name)
        return
    except Exception:
        pass
    _STUBBED.append(name)
    mod = types.ModuleType(name)
    mod.__spec__ = None
    for attr in attrs:
        setattr(mod, attr, MagicMock())
    sys.modules[name] = mod
    parent, _, child = name.rpartition(".")
    if parent and parent in sys.modules:
        setattr(sys.modules[parent], child, mod)


_STUB_SPECS = (
    ("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported")),
    ("unsloth.chat_templates", ("get_chat_template",)),
    ("trl", ("SFTTrainer", "SFTConfig")),
)


@contextlib.contextmanager
def _stubbed():
    """Scope the stubs to the trainer import: left in sys.modules, shared-policy dataset tests skip."""
    for name, attrs in _STUB_SPECS:
        _stub_if_missing(name, attrs)
    try:
        yield
    finally:
        while _STUBBED:
            sys.modules.pop(_STUBBED.pop(), None)


with _stubbed():
    from core.training.trainer import UnslothTrainer  # noqa: E402

_preflight = UnslothTrainer._preflight_first_batch
_renders_empty = UnslothTrainer._chat_template_renders_empty
_auto_detect_eval = UnslothTrainer._auto_detect_eval_split_from_hf
_resolve_eval_split = UnslothTrainer._resolve_eval_split_from_dataset
_load_and_format_dataset = UnslothTrainer.load_and_format_dataset


class _FakeInnerTrainer:
    def __init__(
        self,
        *,
        batch = None,
        dataloader_error = None,
        train_dataset = None,
    ):
        self._batch = batch
        self._dataloader_error = dataloader_error
        self.train_dataset = train_dataset

    def get_train_dataloader(self):
        if self._dataloader_error is not None:
            raise self._dataloader_error
        return [self._batch]


def _fake_self(
    *,
    inner,
    model_name = "org/Some-Model",
    tokenizer = None,
):
    s = SimpleNamespace(trainer = inner, model_name = model_name, tokenizer = tokenizer)
    # Bind real methods so self._chat_template_renders_empty() resolves.
    s._preflight_first_batch = _preflight.__get__(s)
    s._chat_template_renders_empty = _renders_empty.__get__(s)
    return s


class _EmptyTemplateTokenizer:
    def apply_chat_template(
        self,
        messages,
        tokenize = False,
        add_generation_prompt = False,
    ):
        return ""


class _RealTemplateTokenizer:
    def apply_chat_template(
        self,
        messages,
        tokenize = False,
        add_generation_prompt = False,
    ):
        return "<|im_start|>user\nhi<|im_end|>"


class _SizedDataset:
    def __init__(
        self,
        size,
        splits = (),
    ):
        self.size = size
        self.info = SimpleNamespace(splits = {name: object() for name in splits})
        self.shuffle_seeds = []

    def __len__(self):
        return self.size

    def select(self, indices):
        selected = _SizedDataset(len(indices), tuple(self.info.splits))
        selected.shuffle_seeds = list(self.shuffle_seeds)
        return selected

    def shuffle(self, seed = None):
        shuffled = _SizedDataset(self.size, tuple(self.info.splits))
        shuffled.shuffle_seeds = [*self.shuffle_seeds, seed]
        return shuffled


class _SplittableDataset(_SizedDataset):
    def __init__(
        self,
        size,
        calls = None,
    ):
        super().__init__(size)
        self.calls = [] if calls is None else calls

    def train_test_split(self, *, test_size, seed):
        self.calls.append((test_size, seed))
        return {
            "train": _SizedDataset(self.size - test_size),
            "test": _SizedDataset(test_size),
        }


def _dataset_loader_self():
    trainer = SimpleNamespace(
        should_stop = False,
        _audio_type = None,
        is_audio_vlm = False,
        is_vlm = False,
        model_name = "org/model",
        tokenizer = None,
        _update_progress = lambda **kwargs: None,
        _resolve_eval_split_from_dataset = lambda dataset: None,
    )
    trainer._auto_detect_eval_split_from_hf = _auto_detect_eval.__get__(trainer)
    trainer.load_and_format_dataset = _load_and_format_dataset.__get__(trainer)
    return trainer


def _patch_dataset_formatting(monkeypatch):
    monkeypatch.setattr(
        "core.training.trainer.format_and_template_dataset",
        lambda dataset, **kwargs: {
            "dataset": dataset,
            "detected_format": "test",
            "success": True,
        },
    )


def test_cached_auto_eval_uses_supplied_splits_and_loader(monkeypatch):
    remote_calls: list[object] = []
    local_calls: list[str] = []
    expected = _SizedDataset(20)

    def fail_remote(*args, **kwargs):
        remote_calls.append((args, kwargs))
        raise AssertionError("remote dataset access is not allowed")

    monkeypatch.setattr("core.training.trainer.load_dataset", fail_remote)
    monkeypatch.setattr(sys.modules["datasets"], "get_dataset_split_names", fail_remote)

    result = _auto_detect_eval(
        SimpleNamespace(),
        "org/dataset",
        None,
        available_splits = ["train", "validation"],
        split_loader = lambda split: local_calls.append(split) or expected,
        excluded_split = "train",
    )

    assert result is expected
    assert local_calls == ["validation"]
    assert remote_calls == []


def test_cached_auto_eval_with_no_splits_stays_local(monkeypatch):
    remote_calls: list[object] = []

    def fail_remote(*args, **kwargs):
        remote_calls.append((args, kwargs))
        raise AssertionError("remote dataset access is not allowed")

    monkeypatch.setattr("core.training.trainer.load_dataset", fail_remote)
    monkeypatch.setattr(sys.modules["datasets"], "get_dataset_split_names", fail_remote)

    result = _auto_detect_eval(
        SimpleNamespace(),
        "org/dataset",
        None,
        available_splits = [],
        split_loader = fail_remote,
        excluded_split = "train",
    )

    assert result is None
    assert remote_calls == []


def test_cached_auto_eval_excludes_training_split():
    local_calls: list[str] = []

    result = _auto_detect_eval(
        SimpleNamespace(),
        "org/dataset",
        None,
        available_splits = ["train", "validation"],
        split_loader = lambda split: local_calls.append(split) or _SizedDataset(20),
        excluded_split = "validation",
    )

    assert result is None
    assert local_calls == []


def test_cached_auto_eval_excludes_every_split_in_training_instruction():
    local_calls: list[str] = []

    result = _auto_detect_eval(
        SimpleNamespace(),
        "org/dataset",
        None,
        available_splits = ["train", "validation", "test"],
        split_loader = lambda split: local_calls.append(split) or _SizedDataset(20),
        excluded_split = ("train", "validation"),
    )

    assert isinstance(result, _SizedDataset)
    assert local_calls == ["test"]


def test_cached_auto_eval_propagates_loader_failure_for_exact_resume():
    def fail_load(_split):
        raise FileNotFoundError("validation")

    with pytest.raises(FileNotFoundError, match = "validation"):
        _auto_detect_eval(
            SimpleNamespace(),
            "org/dataset",
            None,
            available_splits = ["train", "validation"],
            split_loader = fail_load,
            excluded_split = "train",
            strict_split_loading = True,
        )


def test_auto_eval_probe_failure_records_a_durable_warning(monkeypatch):
    warnings: list[str] = []

    def fail_probe(**_kwargs):
        raise OSError("metadata unavailable")

    monkeypatch.setattr(sys.modules["datasets"], "get_dataset_split_names", fail_probe)

    result = _auto_detect_eval(
        SimpleNamespace(_record_warning = warnings.append),
        "org/dataset",
        None,
    )

    assert result is None
    assert len(warnings) == 1
    assert "held-out split" in warnings[0]
    assert "metadata unavailable" in warnings[0]


@pytest.mark.parametrize(
    "value, expected",
    [
        (0, False),
        (-1, False),
        (False, False),
        (True, False),
        (None, False),
        ("", False),
        ("not-a-number", False),
        (float("nan"), False),
        (float("inf"), False),
        (0.1, True),
        ("2", True),
    ],
)
def test_evaluation_enabled_accepts_only_finite_positive_intervals(value, expected):
    from core.training.eval_dataset import evaluation_enabled
    assert evaluation_enabled(value) is expected


@pytest.mark.parametrize(
    "rows, expected_eval_rows",
    [
        (0, None),
        (31, None),
        (32, 16),
        (1_000, 50),
        (10_000, 128),
    ],
)
def test_shared_eval_split_is_bounded_and_deterministic(rows, expected_eval_rows):
    from core.training.eval_dataset import split_dataset_for_evaluation

    dataset = _SplittableDataset(rows)
    result = split_dataset_for_evaluation(dataset)

    if expected_eval_rows is None:
        assert result is None
        assert dataset.calls == []
        return

    train, evaluation = result
    assert len(train) == rows - expected_eval_rows
    assert len(evaluation) == expected_eval_rows
    assert dataset.calls == [(expected_eval_rows, 3407)]


def test_shared_eval_split_carves_a_vision_list_like_train_test_split():
    from datasets import Dataset

    from core.training.eval_dataset import split_dataset_for_evaluation

    rows = [{"messages": [{"role": "user", "content": f"row {i}"}]} for i in range(40)]
    train, evaluation = split_dataset_for_evaluation(rows)

    assert isinstance(train, list) and isinstance(evaluation, list)
    assert len(train) == 24 and len(evaluation) == 16
    assert sorted(train + evaluation, key = rows.index) == rows

    expected = Dataset.from_dict({"i": list(range(40))}).train_test_split(test_size = 16, seed = 3407)
    assert [rows.index(r) for r in train] == expected["train"]["i"]
    assert [rows.index(r) for r in evaluation] == expected["test"]["i"]


def test_torch_eval_split_warns_when_dataset_is_too_small():
    warnings: list[str] = []
    owner = SimpleNamespace(_record_warning = warnings.append)

    result = _resolve_eval_split(owner, _SplittableDataset(31))

    assert result is None
    assert len(warnings) == 1
    assert "only 31 rows" in warnings[0]
    assert "at least 32" in warnings[0]


def test_cached_train_auto_eval_stays_on_pinned_dataset(monkeypatch):
    dataset_cache, trainer = _shared_setup_1(monkeypatch)
    cache_calls: list[str] = []
    train = _SizedDataset(40, ("train", "validation"))
    validation = _SizedDataset(20, ("train", "validation"))

    def load_cached(
        repo_id,
        local_path,
        *,
        subset,
        split,
        token = None,
    ):
        cache_calls.append(split)
        return validation if split == "validation" else train

    def fail_remote(*args, **kwargs):
        raise AssertionError("remote dataset access is not allowed")

    monkeypatch.setattr(dataset_cache, "load_cached_hf_dataset", load_cached)
    monkeypatch.setattr("core.training.trainer.load_dataset", fail_remote)
    monkeypatch.setattr(sys.modules["datasets"], "get_dataset_split_names", fail_remote)

    result = trainer.load_and_format_dataset(
        "org/dataset",
        eval_steps = 1,
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
    )

    assert result is not None
    assert result[0]["dataset"] is train
    assert result[1] is validation
    assert cache_calls == ["train", "validation"]


def test_bounded_cached_train_forwards_only_required_row_count(monkeypatch):
    dataset_cache, trainer = _shared_setup_1(monkeypatch)
    cache_calls: list[tuple[str, int | None]] = []
    validation = _SizedDataset(20, ("train", "validation"))

    def load_cached(
        repo_id,
        local_path,
        *,
        subset,
        split,
        token = None,
        row_limit = None,
    ):
        cache_calls.append((split, row_limit))
        if split == "validation":
            return validation
        return _SizedDataset(row_limit or 100, ("train", "validation"))

    monkeypatch.setattr(dataset_cache, "load_cached_hf_dataset", load_cached)
    monkeypatch.setattr(
        "core.training.trainer.load_dataset",
        lambda *args, **kwargs: pytest.fail("remote dataset access is not allowed"),
    )

    result = trainer.load_and_format_dataset(
        "org/dataset",
        eval_steps = 1,
        dataset_slice_start = 8,
        dataset_slice_end = 32,
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
    )

    assert result is not None
    assert len(result[0]["dataset"]) == 25
    assert result[1] is validation
    assert cache_calls == [("train", 33), ("validation", None)]


def _cached_only_loader(
    monkeypatch,
    train,
    validation = None,
):
    """A trainer whose dataset comes from cache, with remote access fatal."""
    from hub.utils import dataset_cache

    _patch_dataset_formatting(monkeypatch)

    # row_limit arrives on the explicit-slice path, which fetches end + 1 rows.
    def load_cached(
        repo_id,
        local_path,
        *,
        subset,
        split,
        token = None,
        row_limit = None,
    ):
        if split == "validation":
            return validation
        return _SizedDataset(row_limit) if row_limit else train

    def fail_remote(*args, **kwargs):
        raise AssertionError("remote dataset access is not allowed")

    monkeypatch.setattr(dataset_cache, "load_cached_hf_dataset", load_cached)
    monkeypatch.setattr("core.training.trainer.load_dataset", fail_remote)
    monkeypatch.setattr(sys.modules["datasets"], "get_dataset_split_names", fail_remote)
    return _dataset_loader_self()


def test_max_steps_dataset_rows_bounds_the_run():
    from core.training.dataset_bounds import (
        MAX_STEPS_ROW_SLACK,
        MIN_MAX_STEPS_ROWS,
        max_steps_dataset_rows,
    )

    assert max_steps_dataset_rows(0, 2, 4) is None
    assert max_steps_dataset_rows(None, 2, 4) is None

    assert max_steps_dataset_rows(2000, 8, 16) == 2000 * 8 * 16 * MAX_STEPS_ROW_SLACK
    assert max_steps_dataset_rows(30, 2, 4) == MIN_MAX_STEPS_ROWS
    assert max_steps_dataset_rows(1, 1, 1) == MIN_MAX_STEPS_ROWS


def test_max_steps_bound_subsets_before_formatting(monkeypatch):
    # 30 steps must not tokenize a corpus of 500k rows.
    trainer = _cached_only_loader(monkeypatch, _SizedDataset(500_000))

    result = trainer.load_and_format_dataset(
        "org/dataset",
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
        max_train_rows = 1024,
        max_train_rows_seed = 99,
    )

    assert result is not None
    bounded = result[0]["dataset"]
    assert len(bounded) == 1024
    # Shuffled: the head of a corpus ordered by source is not a sample of it.
    assert bounded.shuffle_seeds == [99]
    assert trainer._kept_row_fraction == 1024 / 500_000


def test_max_steps_bound_leaves_a_small_dataset_alone(monkeypatch):
    train = _SizedDataset(40)
    trainer = _cached_only_loader(monkeypatch, train)
    trainer._kept_row_fraction = 0.5

    result = trainer.load_and_format_dataset(
        "org/dataset",
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
        max_train_rows = 1024,
    )

    assert result is not None
    assert result[0]["dataset"] is train
    assert trainer._kept_row_fraction == 1.0


def test_max_steps_bound_defers_to_an_explicit_slice(monkeypatch):
    trainer = _cached_only_loader(monkeypatch, _SizedDataset(500_000))

    result = trainer.load_and_format_dataset(
        "org/dataset",
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
        dataset_slice_start = 8,
        dataset_slice_end = 32,
        max_train_rows = 1024,
    )

    assert result is not None
    sliced = result[0]["dataset"]
    # The user named the rows; the bound must not resample them.
    assert len(sliced) == 25
    assert sliced.shuffle_seeds == []


@pytest.mark.parametrize("slice_start, slice_end", [(50, 60), (50, None), (10, None)])
def test_explicit_slice_start_past_the_last_row_is_refused(monkeypatch, slice_start, slice_end):
    from hub.utils import dataset_cache

    trainer = _cached_only_loader(monkeypatch, _SizedDataset(10))
    monkeypatch.setattr(
        dataset_cache,
        "load_cached_hf_dataset",
        lambda *args, **kwargs: _SizedDataset(10),
    )
    updates = []
    trainer._update_progress = lambda **kwargs: updates.append(kwargs)

    result = trainer.load_and_format_dataset(
        "org/dataset",
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
        dataset_slice_start = slice_start,
        dataset_slice_end = slice_end,
    )

    assert result is None
    errors = [u["error"] for u in updates if "error" in u]
    assert errors == [
        f"Train Split Start {slice_start} is past the last row of this dataset (10 rows)."
    ]


def test_explicit_slice_end_past_the_last_row_still_clamps(monkeypatch):
    from hub.utils import dataset_cache

    trainer = _cached_only_loader(monkeypatch, _SizedDataset(10))
    monkeypatch.setattr(
        dataset_cache,
        "load_cached_hf_dataset",
        lambda *args, **kwargs: _SizedDataset(10),
    )

    result = trainer.load_and_format_dataset(
        "org/dataset",
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
        dataset_slice_start = 6,
        dataset_slice_end = 60,
    )

    assert result is not None
    assert len(result[0]["dataset"]) == 4


def test_max_steps_bound_defers_to_a_split_instruction(monkeypatch):
    train = _SizedDataset(500_000)
    trainer = _cached_only_loader(monkeypatch, train)

    result = trainer.load_and_format_dataset(
        "org/dataset",
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
        train_split = "train[1000:200000]",
        max_train_rows = 1024,
    )

    assert result is not None
    assert result[0]["dataset"] is train


def test_max_steps_bound_is_off_without_it(monkeypatch):
    train = _SizedDataset(500_000)
    trainer = _cached_only_loader(monkeypatch, train)

    result = trainer.load_and_format_dataset(
        "org/dataset",
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
    )

    assert result is not None
    assert result[0]["dataset"] is train


def test_max_steps_dataset_rows_survives_unusable_numbers():
    from core.training.dataset_bounds import MIN_MAX_STEPS_ROWS, max_steps_dataset_rows

    # The DB and direct callers hand over Nones and strings; this must never raise.
    assert max_steps_dataset_rows(30, None, None) == MIN_MAX_STEPS_ROWS
    assert max_steps_dataset_rows(30, "2", "4") == MIN_MAX_STEPS_ROWS
    assert max_steps_dataset_rows("30", 2, 4) == MIN_MAX_STEPS_ROWS
    assert max_steps_dataset_rows(-5, 2, 4) is None
    assert max_steps_dataset_rows("not a number", 2, 4) is None
    assert max_steps_dataset_rows(10**9, 2, 4) == 10**9 * 8 * 4


def _single_process_launch(monkeypatch):
    """Clear every launcher variable, so a bound reads as Unsloth's own launch."""
    from core.training.dataset_bounds import WORLD_SIZE_ENV_FILES, WORLD_SIZE_ENV_VARS
    for name in WORLD_SIZE_ENV_VARS + WORLD_SIZE_ENV_FILES:
        monkeypatch.delenv(name, raising = False)


def test_max_steps_dataset_rows_scales_with_world_size(monkeypatch):
    from core.training.dataset_bounds import (
        MAX_STEPS_ROW_SLACK,
        MIN_MAX_STEPS_ROWS,
        max_steps_dataset_rows,
    )

    _single_process_launch(monkeypatch)

    assert max_steps_dataset_rows(2000, 8, 16, world_size = 1) == max_steps_dataset_rows(2000, 8, 16)
    assert max_steps_dataset_rows(2000, 8, 16) == 2000 * 8 * 16 * MAX_STEPS_ROW_SLACK
    assert max_steps_dataset_rows(30, 2, 4, world_size = 1) == MIN_MAX_STEPS_ROWS

    # Every replica draws its own batch per step, so the subset grows with them.
    for world_size in (2, 4, 8):
        assert (
            max_steps_dataset_rows(2000, 8, 16, world_size = world_size)
            == 2000 * 8 * 16 * world_size * MAX_STEPS_ROW_SLACK
        )
        # The slack is what stops a run recycling rows, so it must survive the scaling.
        rows = max_steps_dataset_rows(60, 2, 4, world_size = world_size)
        assert rows >= 60 * 2 * 4 * world_size * MAX_STEPS_ROW_SLACK

    assert max_steps_dataset_rows(0, 2, 4, world_size = 8) is None


def test_max_steps_dataset_rows_survives_an_unusable_world_size(monkeypatch):
    from core.training.dataset_bounds import MIN_MAX_STEPS_ROWS, max_steps_dataset_rows

    _single_process_launch(monkeypatch)

    baseline = max_steps_dataset_rows(2000, 8, 16)
    for world_size in (None, 0, -4, "", "auto", "not a number", float("inf"), object()):
        assert max_steps_dataset_rows(2000, 8, 16, world_size = world_size) == baseline
    assert max_steps_dataset_rows(30, 2, 4, world_size = None) == MIN_MAX_STEPS_ROWS

    assert max_steps_dataset_rows(2000, 8, 16, world_size = "4") == baseline * 4


def test_world_size_comes_from_the_launcher_env(monkeypatch):
    from core.training.dataset_bounds import (
        MAX_STEPS_ROW_SLACK,
        max_steps_dataset_rows,
        world_size_from_env,
    )

    _single_process_launch(monkeypatch)
    assert world_size_from_env() == 1

    # torchrun and accelerate set WORLD_SIZE; mlx.launch and mpirun set their own.
    for name, size in (("WORLD_SIZE", 4), ("MLX_WORLD_SIZE", 2), ("OMPI_COMM_WORLD_SIZE", 8)):
        _single_process_launch(monkeypatch)
        monkeypatch.setenv(name, str(size))
        assert world_size_from_env() == size
        assert max_steps_dataset_rows(2000, 8, 16) == 2000 * 8 * 16 * size * MAX_STEPS_ROW_SLACK

    # A multi-node torchrun sets both, and the global count is the one that sizes rows.
    _single_process_launch(monkeypatch)
    monkeypatch.setenv("WORLD_SIZE", "16")
    monkeypatch.setenv("LOCAL_WORLD_SIZE", "8")
    assert world_size_from_env() == 16

    for junk in ("", "auto", "0", "-2", "3.5"):
        _single_process_launch(monkeypatch)
        monkeypatch.setenv("WORLD_SIZE", junk)
        assert world_size_from_env() == 1

    # An explicit count outranks the env, which cannot see DataParallel's devices.
    _single_process_launch(monkeypatch)
    monkeypatch.setenv("WORLD_SIZE", "2")
    assert max_steps_dataset_rows(2000, 8, 16, world_size = 8) == 2000 * 8 * 16 * 8 * (
        MAX_STEPS_ROW_SLACK
    )
    assert world_size_from_env({"WORLD_SIZE": "4"}) == 4
    assert world_size_from_env({}) == 1


def test_world_size_comes_from_an_mlx_launch_hostfile(tmp_path, monkeypatch):
    """mlx.launch on Apple exports ranks as a hostfile path, so numeric variables alone undercount."""
    from core.training.dataset_bounds import (
        MAX_STEPS_ROW_SLACK,
        max_steps_dataset_rows,
        world_size_from_env,
        world_size_from_rank_files,
    )

    _single_process_launch(monkeypatch)
    assert world_size_from_rank_files() == 1

    # The ring backend writes one "ip:port" list per rank; mlx.launch --hostfile ring-4.
    ring = tmp_path / "ring.json"
    ring.write_text(json.dumps([[f"10.0.0.{rank}:5000"] for rank in range(4)]), encoding = "utf-8")
    _single_process_launch(monkeypatch)
    monkeypatch.setenv("MLX_RANK", "0")
    monkeypatch.setenv("MLX_HOSTFILE", str(ring))
    assert world_size_from_env() == 4
    assert max_steps_dataset_rows(60, 2, 4) == 60 * 2 * 4 * 4 * MAX_STEPS_ROW_SLACK

    # The jaccl backend writes the RDMA matrix, one row per rank.
    rdma = tmp_path / "rdma.json"
    rdma.write_text(
        json.dumps([[None if a == b else "mlx5_0" for b in range(3)] for a in range(3)]),
        encoding = "utf-8",
    )
    _single_process_launch(monkeypatch)
    monkeypatch.setenv("MLX_IBV_DEVICES", str(rdma))
    assert world_size_from_env() == 3

    # A single host is not a distributed launch: mlx.launch writes an empty hostfile.
    empty = tmp_path / "empty.json"
    empty.write_text("", encoding = "utf-8")
    _single_process_launch(monkeypatch)
    monkeypatch.setenv("MLX_HOSTFILE", str(empty))
    assert world_size_from_env() == 1

    # Inline payloads too, as unsloth_cli/_inference.py's _json_rank_count_from_env reads them.
    for inline, expected in (
        (json.dumps([[f"10.0.0.{rank}:5000"] for rank in range(6)]), 6),
        (json.dumps({"hosts": ["a", "b", "c"]}), 3),
        ("  " + json.dumps([["a"], ["b"]]), 2),
        (json.dumps([]), 1),
    ):
        _single_process_launch(monkeypatch)
        monkeypatch.setenv("MLX_HOSTFILE", inline)
        assert world_size_from_env() == expected

    bad_json = tmp_path / "bad.json"
    bad_json.write_text('[["10.0.0.1:5000"],', encoding = "utf-8")
    not_a_list = tmp_path / "object.json"
    not_a_list.write_text(json.dumps({"hosts": 4}), encoding = "utf-8")
    a_directory = tmp_path / "adir"
    a_directory.mkdir()
    for value in (
        str(tmp_path / "missing.json"),
        str(bad_json),
        str(not_a_list),
        str(a_directory),
        '[["10.0.0.1:5000"],',
        '{"hosts": 4}',
        "[",
        "{",
        "",
        "   ",
    ):
        _single_process_launch(monkeypatch)
        monkeypatch.setenv("MLX_HOSTFILE", value)
        assert world_size_from_env() == 1

    # A fifo would block open() forever, so only regular files are read at all.
    fifo = tmp_path / "fifo"
    try:
        os.mkfifo(fifo)
    except (AttributeError, NotImplementedError, OSError):
        pass  # Windows has no fifos, which is the point.
    else:
        assert world_size_from_rank_files({"MLX_HOSTFILE": str(fifo)}) == 1

    assert world_size_from_rank_files({"MLX_HOSTFILE": str(ring)}) == 4
    assert world_size_from_env({"MLX_HOSTFILE": str(ring), "WORLD_SIZE": "8"}) == 8
    assert world_size_from_env({"MLX_HOSTFILE": str(ring), "WORLD_SIZE": "2"}) == 4
    assert world_size_from_rank_files({"MLX_HOSTFILE": None}) == 1
    assert world_size_from_rank_files({"MLX_HOSTFILE": 17}) == 1
    assert world_size_from_rank_files({}) == 1


def test_a_rank_file_read_is_capped_in_bytes_not_characters(tmp_path):
    """Rank files are read as bytes, since a text-mode read(n) counts characters, not bytes."""
    from core.training.dataset_bounds import (
        MAX_WORLD_SIZE_FILE_BYTES,
        world_size_from_rank_files,
    )

    # Under the cap in characters, over it in bytes; binary truncates to one process.
    wide = tmp_path / "wide.json"
    filler = "\U0001f600" * (MAX_WORLD_SIZE_FILE_BYTES // 3)  # 4 bytes per character
    hosts = [filler] + [f"10.0.0.{rank}:5000" for rank in range(7)]
    # ensure_ascii would make the two readings agree.
    wide.write_text(json.dumps(hosts, ensure_ascii = False), encoding = "utf-8")
    assert len(wide.read_text(encoding = "utf-8")) < MAX_WORLD_SIZE_FILE_BYTES
    assert wide.stat().st_size > MAX_WORLD_SIZE_FILE_BYTES
    assert world_size_from_rank_files({"MLX_HOSTFILE": str(wide)}) == 1

    invalid = tmp_path / "invalid.bin"
    invalid.write_bytes(b'["\xff\xfe10.0.0.1:5000"]')
    assert world_size_from_rank_files({"MLX_HOSTFILE": str(invalid)}) == 1


def test_the_launcher_env_report_names_the_variable_that_claimed_the_ranks():
    """The launcher report names the variable that claimed the ranks and stays bounded in size."""
    from core.training.dataset_bounds import world_size_env_report

    assert world_size_env_report({}) == "no launcher variable set"
    assert world_size_env_report({"WORLD_SIZE": ""}) == "no launcher variable set"
    assert world_size_env_report({"NOT_A_LAUNCHER": "8"}) == "no launcher variable set"

    report = world_size_env_report({"OMPI_COMM_WORLD_SIZE": "8", "WORLD_SIZE": "2"})
    assert "OMPI_COMM_WORLD_SIZE=8" in report and "WORLD_SIZE=2" in report

    long_payload = world_size_env_report({"MLX_HOSTFILE": json.dumps(["h"] * 500)})
    assert len(long_payload) < 200

    class _Hostile:
        def get(self, name):
            raise RuntimeError("environment lookup exploded")

    assert world_size_env_report(_Hostile()) == "no launcher variable set"


def test_effective_packing_decides_the_opt_out():
    from core.training.dataset_bounds import effective_packing, max_train_rows_for_config

    text = {"max_steps": 30, "batch_size": 2, "gradient_accumulation_steps": 4}

    # Packing spans unknown rows per sample, so text runs opt out.
    assert effective_packing({**text, "packing": True}) is True
    assert max_train_rows_for_config({**text, "packing": True}) is None

    assert effective_packing({**text, "packing": True}, branch_never_packs = True) is False
    assert max_train_rows_for_config({**text, "packing": True}, branch_never_packs = True) == 1024

    # Dataset flags are client-supplied and match on column NAME, so they prove nothing.
    assert effective_packing({**text, "packing": True, "is_dataset_image": True}) is True
    assert effective_packing({**text, "packing": True, "is_dataset_audio": True}) is True
    assert max_train_rows_for_config({**text, "packing": True, "is_dataset_audio": True}) is None

    assert max_train_rows_for_config({"max_steps": 0, "packing": False}) is None

    # The caller decides the branch: vision is gated on `not raw_text_mode`, audio is not.
    for raw in ({"training_type": "Continued Pretraining"}, {"format_type": "raw"}):
        assert effective_packing({**text, **raw, "packing": True}, branch_never_packs = True) is False
        assert effective_packing({**text, **raw, "packing": True}) is True
        assert effective_packing({**text, **raw}, branch_never_packs = True) is False
        assert max_train_rows_for_config({**text, **raw}, branch_never_packs = True) == 1024


def test_bound_dataset_rows_edges():
    from core.training.dataset_bounds import bound_dataset_rows

    class _Streaming:
        """No __len__, like an IterableDataset."""

        def shuffle(self, seed = None):
            raise AssertionError("a streaming dataset must not be shuffled eagerly")

    exact = _SizedDataset(1024)
    assert bound_dataset_rows(exact, 1024, 3407) is exact
    assert len(bound_dataset_rows(_SizedDataset(1025), 1024, 3407)) == 1024

    # A non-positive bound would otherwise select an empty dataset.
    untouched = _SizedDataset(500_000)
    assert bound_dataset_rows(untouched, 0, 3407) is untouched
    assert bound_dataset_rows(untouched, -5, 3407) is untouched
    assert bound_dataset_rows(untouched, None, 3407) is untouched

    streaming = _Streaming()
    assert bound_dataset_rows(streaming, 1024, 3407) is streaming

    assert len(bound_dataset_rows(_SizedDataset(500_000), 1024, None)) == 1024


def test_bound_dataset_rows_keeps_seed_zero():
    from datasets import Dataset

    from core.training.dataset_bounds import bound_dataset_rows

    source = Dataset.from_dict({"row": list(range(5000))})

    # 0 is a legitimate seed and must not collapse onto the default.
    assert (
        bound_dataset_rows(source, 1024, 0)["row"] != bound_dataset_rows(source, 1024, 3407)["row"]
    )
    assert bound_dataset_rows(source, 1024, 0)["row"] == bound_dataset_rows(source, 1024, 0)["row"]


def test_bound_dataset_rows_survives_a_hostile_seed():
    from datasets import Dataset

    from core.training.dataset_bounds import bound_dataset_rows

    source = Dataset.from_dict({"row": list(range(3000))})

    # numpy rejects negative seeds and json allows Infinity; neither may fail a run.
    for seed in (-1, -3407, float("inf"), float("nan"), "3407", None, "seed"):
        assert len(bound_dataset_rows(source, 1024, seed)) == 1024


def test_max_steps_dataset_rows_survives_infinity():
    from core.training.dataset_bounds import MIN_MAX_STEPS_ROWS, max_steps_dataset_rows

    infinity = float("inf")
    assert max_steps_dataset_rows(infinity, 2, 4) is None
    assert max_steps_dataset_rows(30, infinity, 4) == MIN_MAX_STEPS_ROWS
    assert max_steps_dataset_rows(30, 2, infinity) == MIN_MAX_STEPS_ROWS


def test_bound_dataset_rows_leaves_a_dataset_dict_alone():
    from datasets import Dataset, DatasetDict

    from core.training.dataset_bounds import bound_dataset_rows

    # len() on a DatasetDict is the split count, and it has no select().
    splits = DatasetDict(
        {
            "train": Dataset.from_dict({"row": list(range(5000))}),
            "test": Dataset.from_dict({"row": list(range(100))}),
        }
    )
    assert bound_dataset_rows(splits, 1024, 3407) is splits


def test_row_bound_marker_round_trips_through_a_resume(tmp_path):
    from core.training.dataset_bounds import record_row_bound, row_bound_for_resume

    run_dir = tmp_path / "run"
    run_dir.mkdir()
    checkpoint = run_dir / "checkpoint-30"
    checkpoint.mkdir()

    assert row_bound_for_resume(None, 4096, 3407) == (4096, 3407)

    record_row_bound(str(run_dir), 4096, 3407)
    # Resuming reads back the original bound, so edits do not move the rows.
    assert row_bound_for_resume(str(checkpoint), 40960, 99) == (4096, 3407)
    assert row_bound_for_resume(str(run_dir), 40960, 99) == (4096, 3407)

    unbounded = tmp_path / "unbounded"
    unbounded.mkdir()
    record_row_bound(str(unbounded), None, 3407)
    assert row_bound_for_resume(str(unbounded / "checkpoint-5"), 1024, 3407) == (None, 3407)


def test_row_bound_marker_survives_a_run_directory_named_like_a_checkpoint(tmp_path):
    from core.training.dataset_bounds import record_row_bound, row_bound_for_resume

    # A run dir merely prefixed "checkpoint-" is not a checkpoint.
    run_dir = tmp_path / "checkpoint-model__project-x"
    (run_dir / "checkpoint-30").mkdir(parents = True)
    record_row_bound(str(run_dir), 4096, 3407)

    assert (run_dir / "unsloth_row_bound.json").exists()
    assert not (tmp_path / "unsloth_row_bound.json").exists()
    assert row_bound_for_resume(str(run_dir / "checkpoint-30"), 40960, 99) == (4096, 3407)


def test_row_bound_marker_is_replaced_atomically(tmp_path):
    import os

    from core.training.dataset_bounds import record_row_bound, row_bound_for_resume

    # Truncating in place then failing would leave an empty marker.
    run_dir = tmp_path / "run"
    run_dir.mkdir()
    record_row_bound(str(run_dir), 4096, 3407)

    real_replace = os.replace

    def _fail_replace(src, dst):
        raise OSError(28, "No space left on device")

    os.replace = _fail_replace
    try:
        record_row_bound(str(run_dir), 8192, 99)
    finally:
        os.replace = real_replace

    assert row_bound_for_resume(str(run_dir), 40960, 99) == (4096, 3407)
    assert [p.name for p in run_dir.iterdir()] == ["unsloth_row_bound.json"]


def test_run_dir_for_a_bare_relative_checkpoint(tmp_path, monkeypatch):
    from core.training.dataset_bounds import (
        record_row_bound,
        row_bound_for_resume,
        run_dir_for_checkpoint,
    )

    # "checkpoint-30" has an empty head; its run dir is the cwd.
    assert run_dir_for_checkpoint("checkpoint-30") == os.curdir
    assert run_dir_for_checkpoint("run/checkpoint-30") == "run"
    assert run_dir_for_checkpoint("checkpoint-model") == "checkpoint-model"

    run_dir = tmp_path / "run"
    (run_dir / "checkpoint-30").mkdir(parents = True)
    record_row_bound(str(run_dir), 4096, 3407)
    monkeypatch.chdir(run_dir)
    assert row_bound_for_resume("checkpoint-30", 40960, 99) == (4096, 3407)


def test_record_row_bound_reports_whether_it_wrote():
    from core.training.dataset_bounds import record_row_bound

    # The dataset is already bounded by now, so the caller only logs a failure.
    assert record_row_bound(None, 1024, 3407) is False
    assert record_row_bound("/definitely/not/a/directory/here", 1024, 3407) is False


def test_row_bound_is_dropped_for_a_checkpoint_that_predates_it(tmp_path):
    from core.training.dataset_bounds import record_row_bound, row_bound_for_resume

    # A pre-marker checkpoint trained on the whole corpus; resume is by batch index, so no bound.
    legacy = tmp_path / "legacy"
    (legacy / "checkpoint-30").mkdir(parents = True)
    assert row_bound_for_resume(str(legacy / "checkpoint-30"), 1024, 3407) == (None, 3407)

    (legacy / "checkpoint-30" / "trainer_state.json").write_text(
        json.dumps({"global_step": 15, "epoch": 120 / 1500, "train_batch_size": 2})
    )
    assert row_bound_for_resume(str(legacy / "checkpoint-30"), 1024, 3407) == (None, 3407)

    for name, body in (("empty", "{}"), ("broken", "not json"), ("null", "null")):
        run_dir = tmp_path / name
        run_dir.mkdir()
        (run_dir / "unsloth_row_bound.json").write_text(body)
        assert row_bound_for_resume(str(run_dir), 1024, 3407) == (None, 3407)

    record_row_bound(str(tmp_path / "does" / "not" / "exist"), 1024, 3407)
    record_row_bound(None, 1024, 3407)


def test_bound_dataset_rows_is_deterministic_and_seed_sensitive():
    from datasets import Dataset

    from core.training.dataset_bounds import bound_dataset_rows

    source = Dataset.from_dict({"row": list(range(5000)), "text": [f"t{i}" for i in range(5000)]})

    first = bound_dataset_rows(source, 1024, 3407)["row"]
    second = bound_dataset_rows(source, 1024, 3407)["row"]
    other = bound_dataset_rows(source, 1024, 99)["row"]

    assert len(first) == 1024
    assert first == second
    assert first != other
    # The head of a corpus ordered by source is not a sample of it.
    assert first != list(range(1024))
    assert bound_dataset_rows(source, 1024, 3407).column_names == ["row", "text"]


def test_bound_leaves_enough_rows_after_the_eval_carve():
    from datasets import Dataset

    from core.training.dataset_bounds import bound_dataset_rows, max_train_rows_for_config
    from core.training.eval_dataset import split_dataset_for_evaluation

    config = {"max_steps": 30, "batch_size": 2, "gradient_accumulation_steps": 4}
    rows = max_train_rows_for_config(config)
    source = Dataset.from_dict({"text": [f"t{i}" for i in range(500_000)]})

    bounded = bound_dataset_rows(source, rows, 3407)
    train, _eval = split_dataset_for_evaluation(bounded)

    # Run must still reach max_steps without re-reading rows; MAX_STEPS_ROW_SLACK covers the carve.
    needed = config["max_steps"] * config["batch_size"] * config["gradient_accumulation_steps"]
    assert len(train) >= needed


def test_bound_leaves_enough_rows_for_every_rank_after_the_eval_carve(monkeypatch):
    from datasets import Dataset

    from core.training.dataset_bounds import bound_dataset_rows, max_train_rows_for_config
    from core.training.eval_dataset import split_dataset_for_evaluation

    _single_process_launch(monkeypatch)

    config = {"max_steps": 60, "batch_size": 2, "gradient_accumulation_steps": 4}
    source = Dataset.from_dict({"text": [f"t{i}" for i in range(500_000)]})

    for world_size in (1, 2, 4, 8):
        rows = max_train_rows_for_config(config, world_size = world_size)
        bounded = bound_dataset_rows(source, rows, 3407)
        train, _eval = split_dataset_for_evaluation(bounded)

        # Each rank draws its own batch, so rows needed scale with the ranks.
        needed = (
            config["max_steps"]
            * config["batch_size"]
            * config["gradient_accumulation_steps"]
            * world_size
        )
        assert len(train) >= needed

    assert max_train_rows_for_config({**config, "packing": True}, world_size = 8) is None


def test_row_bound_marker_round_trips_a_world_size_scaled_bound(tmp_path, monkeypatch):
    from core.training.dataset_bounds import (
        max_train_rows_for_config,
        record_row_bound,
        row_bound_for_resume,
    )

    _single_process_launch(monkeypatch)

    run_dir = tmp_path / "run"
    run_dir.mkdir()
    checkpoint = run_dir / "checkpoint-60"
    checkpoint.mkdir()

    config = {"max_steps": 60, "batch_size": 2, "gradient_accumulation_steps": 4}
    rows = max_train_rows_for_config(config, world_size = 4)
    assert rows == 60 * 2 * 4 * 4 * 4
    assert record_row_bound(str(run_dir), rows, 3407) is True

    # The marker records rows, so a resume on a different machine keeps them.
    assert row_bound_for_resume(str(checkpoint), max_train_rows_for_config(config), 99) == (
        rows,
        3407,
    )

    legacy_dir = tmp_path / "legacy"
    legacy_dir.mkdir()
    (legacy_dir / "checkpoint-60").mkdir()
    legacy_rows = 60 * 2 * 4 * 4
    record_row_bound(str(legacy_dir), legacy_rows, 3407)
    assert row_bound_for_resume(str(legacy_dir / "checkpoint-60"), rows, 3407) == (
        legacy_rows,
        3407,
    )

    unmarked = tmp_path / "unmarked"
    (unmarked / "checkpoint-60").mkdir(parents = True)
    assert row_bound_for_resume(str(unmarked / "checkpoint-60"), rows, 3407) == (None, 3407)


def test_both_loaders_size_the_bound_for_the_world():
    """Both loaders must pass the world size into the row bound; checked from source, not run."""
    import ast
    from pathlib import Path

    worker_src = (Path(__file__).resolve().parents[1] / "core/training/worker.py").read_text(
        encoding = "utf-8"
    )
    tree = ast.parse(worker_src)
    calls = {}
    for node in ast.walk(tree):
        if not isinstance(node, ast.FunctionDef):
            continue
        calls[node.name] = {
            sub.func.id
            for sub in ast.walk(node)
            if isinstance(sub, ast.Call) and isinstance(sub.func, ast.Name)
        }

    for loader in ("run_training_process", "_run_mlx_training"):
        assert "_data_parallel_world_size" in calls[loader]
    assert worker_src.count("world_size = _data_parallel_world_size()") == 2
    # World size belongs to this launch; never read it from a config shipped across a spawn.
    assert 'config.get("world_size")' not in worker_src


def test_data_parallel_world_size_counts_ranks_and_devices(monkeypatch):
    import types

    from core.training import worker as training_worker

    _single_process_launch(monkeypatch)

    # No torch in sys.modules is the MLX host: the env is all there is.
    monkeypatch.setitem(sys.modules, "torch", None)
    assert training_worker._data_parallel_world_size() == 1
    monkeypatch.setenv("MLX_WORLD_SIZE", "4")
    assert training_worker._data_parallel_world_size() == 4

    def _torch(devices, world_size = None):
        distributed = types.SimpleNamespace(
            is_available = lambda: world_size is not None,
            is_initialized = lambda: world_size is not None,
            get_world_size = lambda: world_size,
        )
        return types.SimpleNamespace(
            cuda = types.SimpleNamespace(device_count = lambda: devices),
            distributed = distributed,
        )

    _single_process_launch(monkeypatch)
    monkeypatch.setitem(sys.modules, "torch", _torch(1))
    assert training_worker._data_parallel_world_size() == 1

    # non-distributed multi-GPU runs get DataParallel, scaling the batch by visible devices.
    monkeypatch.setitem(sys.modules, "torch", _torch(4))
    assert training_worker._data_parallel_world_size() == 4

    # A torchrun rank pins n_gpu to 1: take the larger, never the product.
    monkeypatch.setenv("WORLD_SIZE", "8")
    monkeypatch.setitem(sys.modules, "torch", _torch(8, world_size = 8))
    assert training_worker._data_parallel_world_size() == 8

    _single_process_launch(monkeypatch)
    monkeypatch.setitem(sys.modules, "torch", _torch(0))
    assert training_worker._data_parallel_world_size() == 1

    def _raises():
        raise RuntimeError("no CUDA driver")

    broken = types.SimpleNamespace(
        cuda = types.SimpleNamespace(device_count = _raises),
        distributed = types.SimpleNamespace(is_available = _raises, is_initialized = _raises),
    )
    monkeypatch.setitem(sys.modules, "torch", broken)
    assert training_worker._data_parallel_world_size() == 1


def test_both_loaders_apply_the_row_bound():
    """Both loaders must still call the row-bound helpers; checked from source, as CI cannot run them."""
    import ast
    from pathlib import Path

    worker_src = (Path(__file__).resolve().parents[1] / "core/training/worker.py").read_text(
        encoding = "utf-8"
    )
    tree = ast.parse(worker_src)
    calls = {}
    for node in ast.walk(tree):
        if not isinstance(node, ast.FunctionDef):
            continue
        names = {
            sub.func.id
            for sub in ast.walk(node)
            if isinstance(sub, ast.Call) and isinstance(sub.func, ast.Name)
        }
        calls[node.name] = names

    assert "max_train_rows_for_config" in calls["run_training_process"]
    assert "max_train_rows = max_train_rows" in worker_src
    assert "bound_dataset_rows" in calls["_slice"]
    assert "max_train_rows_for_config" in calls["_run_mlx_training"]
    # Both must resume on the recorded bound, or a resume trains on unseen rows.
    for loader in ("run_training_process", "_run_mlx_training"):
        assert "row_bound_for_resume" in calls[loader]
        assert "record_row_bound" in calls[loader]

    # Both pass the detected branch, never a default: client dataset flags cannot stand in.
    assert worker_src.count("branch_never_packs = ") >= 2
    assert worker_src.index("_pre_detect_training_model(\n") < worker_src.index(
        "branch_never_packs = bool("
    )


def test_mlx_adapter_keeps_one_source_of_truth_for_the_bound():
    from core.training.training import _build_training_worker_config

    config = _build_training_worker_config(
        {"model_name": "org/model", "max_steps": 30, "batch_size": 2}
    )
    # The normalized config is a whitelist; a forwarded bound would be dropped.
    assert "max_train_rows" not in config
    assert "max_train_rows_seed" not in config
    assert config["max_steps"] == 30
    assert config["batch_size"] == 2
    assert config["gradient_accumulation_steps"] == 4
    assert config["random_seed"] == 3407


def test_remote_train_fallback_keeps_auto_eval_remote(monkeypatch):
    dataset_cache, trainer = _shared_setup_1(monkeypatch)
    cache_calls: list[str] = []
    remote_calls: list[tuple[str, str | None]] = []
    train = _SizedDataset(40, ("train", "validation"))
    validation = _SizedDataset(20, ("train", "validation"))

    def load_cached(
        repo_id,
        local_path,
        *,
        subset,
        split,
        token = None,
    ):
        cache_calls.append(split)
        raise FileNotFoundError(split)

    def load_remote(*, path, split, **kwargs):
        remote_calls.append((split, kwargs.get("revision")))
        return validation if split == "validation" else train

    monkeypatch.setattr(dataset_cache, "load_cached_hf_dataset", load_cached)
    monkeypatch.setattr("core.training.trainer.load_dataset", load_remote)
    monkeypatch.setattr(
        sys.modules["datasets"],
        "get_dataset_split_names",
        lambda **kwargs: ["train", "validation"],
    )

    result = trainer.load_and_format_dataset(
        "org/dataset",
        eval_steps = 1,
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
        dataset_revision = "dataset-commit",
    )

    assert result is not None
    assert result[0]["dataset"] is train
    assert result[1] is validation
    assert cache_calls == ["train"]
    assert remote_calls == [("train", "dataset-commit"), ("validation", "dataset-commit")]


def test_first_remote_train_load_records_exact_dataset_snapshot(monkeypatch, tmp_path):
    from hub.utils import hf_cache_state

    _patch_dataset_formatting(monkeypatch)
    trainer = _dataset_loader_self()
    snapshot = tmp_path / "datasets--org--dataset" / "snapshots" / "dataset-commit"
    snapshot.mkdir(parents = True)
    (snapshot / "train.parquet").write_bytes(b"dataset")
    train = _SizedDataset(40)
    train.info.download_checksums = {
        "hf://datasets/org/dataset@dataset-commit/train.parquet": {
            "num_bytes": 7,
            "checksum": None,
        }
    }

    monkeypatch.setattr(hf_cache_state, "hf_cache_roots", lambda **kw: [tmp_path])
    monkeypatch.setattr(
        "core.training.trainer.load_dataset",
        lambda **_kwargs: train,
    )

    result = trainer.load_and_format_dataset("org/dataset")

    assert result is not None
    assert trainer.dataset_snapshot_path == str(snapshot.resolve())
    assert trainer.dataset_loaded_from_exact_snapshot is True


def test_manual_eager_slice_attests_original_hub_stream(monkeypatch, tmp_path):
    from hub.utils import hf_cache_state

    _patch_dataset_formatting(monkeypatch)
    trainer = _dataset_loader_self()
    snapshot = tmp_path / "datasets--org--dataset" / "snapshots" / "dataset-commit"
    snapshot.mkdir(parents = True)
    (snapshot / "train.parquet").write_bytes(b"dataset")
    stream = SimpleNamespace(
        info = SimpleNamespace(
            download_checksums = {
                "hf://datasets/org/dataset@dataset-commit/train.parquet": {
                    "num_bytes": 7,
                    "checksum": None,
                }
            }
        ),
        take = lambda _count: [{"text": "example"}],
    )

    monkeypatch.setattr(hf_cache_state, "hf_cache_roots", lambda **kw: [tmp_path])
    monkeypatch.setattr(
        "core.training.trainer.load_dataset",
        lambda **kwargs: (
            stream if kwargs.get("streaming") else pytest.fail("eager download should not run")
        ),
    )

    result = trainer.load_and_format_dataset(
        "org/dataset",
        dataset_slice_end = 0,
    )

    assert result is not None
    assert trainer.dataset_snapshot_path == str(snapshot.resolve())
    assert trainer.dataset_loaded_from_exact_snapshot is True


@pytest.mark.parametrize(
    "cached_eval_error",
    [
        FileNotFoundError("validation"),
        ValueError("Unknown split \"validation\". Should be one of ['train']."),
    ],
)
def test_cached_explicit_eval_failure_reloads_remote_pair(monkeypatch, cached_eval_error):
    dataset_cache, trainer = _shared_setup_1(monkeypatch)
    cache_calls: list[str] = []
    remote_calls: list[str] = []
    cached_train = _SizedDataset(40, ("train", "validation"))
    remote_train = _SizedDataset(50, ("train", "validation"))
    remote_validation = _SizedDataset(20, ("train", "validation"))

    def load_cached(
        repo_id,
        local_path,
        *,
        subset,
        split,
        token = None,
    ):
        cache_calls.append(split)
        if split == "validation":
            raise cached_eval_error
        return cached_train

    def load_remote(*, path, split, **kwargs):
        remote_calls.append(split)
        return remote_validation if split == "validation" else remote_train

    monkeypatch.setattr(dataset_cache, "load_cached_hf_dataset", load_cached)
    monkeypatch.setattr("core.training.trainer.load_dataset", load_remote)

    result = trainer.load_and_format_dataset(
        "org/dataset",
        eval_split = "validation",
        eval_steps = 1,
        dataset_local_files_only = True,
        dataset_local_path = "/cache/snapshot",
    )

    assert result is not None
    assert result[0]["dataset"] is remote_train
    assert result[1] is remote_validation
    assert cache_calls == ["train", "validation"]
    assert remote_calls == ["train", "validation"]


class TestPreflightFirstBatch(unittest.TestCase):
    def test_float_input_ids_with_empty_template_suggests_instruct(self):
        ds = [{"messages": [{"role": "user", "content": [{"type": "text", "text": "x"}]}]}]
        inner = _FakeInnerTrainer(
            batch = {"input_ids": torch.zeros((1, 0), dtype = torch.float32)},
            train_dataset = ds,
        )
        s = _fake_self(
            inner = inner, model_name = "Qwen/Qwen2-VL-7B", tokenizer = _EmptyTemplateTokenizer()
        )
        msg = s._preflight_first_batch()
        self.assertIsNotNone(msg)
        self.assertIn("chat template", msg)
        self.assertIn("Qwen/Qwen2-VL-7B-Instruct", msg)
        self.assertIn("base (pretrained) model", msg)

    def test_no_instruct_hint_when_model_already_instruct(self):
        ds = [{"messages": [{"role": "user", "content": [{"type": "text", "text": "x"}]}]}]
        inner = _FakeInnerTrainer(
            batch = {"input_ids": torch.zeros((1, 0), dtype = torch.float32)},
            train_dataset = ds,
        )
        s = _fake_self(
            inner = inner, model_name = "org/Foo-Instruct", tokenizer = _EmptyTemplateTokenizer()
        )
        msg = s._preflight_first_batch()
        self.assertIsNotNone(msg)
        self.assertNotIn("such as", msg)  # no Instruct suggestion for an Instruct model
        self.assertIn("instruction-tuned variant", msg)

    def test_empty_int_input_ids_generic_message(self):
        inner = _FakeInnerTrainer(
            batch = {"input_ids": torch.zeros((1, 0), dtype = torch.long)},
            train_dataset = [{"text": "already tokenized path"}],
        )
        s = _fake_self(inner = inner, tokenizer = _RealTemplateTokenizer())
        msg = s._preflight_first_batch()
        self.assertIsNotNone(msg)
        self.assertIn("invalid token IDs", msg)
        self.assertNotIn("chat template", msg)

    def test_valid_batch_returns_none(self):
        inner = _FakeInnerTrainer(
            batch = {"input_ids": torch.randint(0, 1000, (2, 34), dtype = torch.long)},
        )
        s = _fake_self(inner = inner)
        self.assertIsNone(s._preflight_first_batch())

    def test_dataloader_error_is_surfaced(self):
        inner = _FakeInnerTrainer(dataloader_error = RuntimeError("boom"))
        s = _fake_self(inner = inner, model_name = "org/M")
        msg = s._preflight_first_batch()
        self.assertIsNotNone(msg)
        self.assertIn("failed to build the first training batch", msg)
        self.assertIn("org/M", msg)

    def test_missing_input_ids_does_not_false_positive(self):
        inner = _FakeInnerTrainer(batch = {"pixel_values": torch.zeros((1, 3))})
        s = _fake_self(inner = inner)
        self.assertIsNone(s._preflight_first_batch())


class TestChatTemplateRendersEmpty(unittest.TestCase):
    def _self(self, *, train_dataset, tokenizer):
        inner = _FakeInnerTrainer(train_dataset = train_dataset)
        return _fake_self(inner = inner, tokenizer = tokenizer)

    def test_empty_render_detected(self):
        ds = [{"messages": [{"role": "user", "content": [{"type": "text", "text": "x"}]}]}]
        s = self._self(train_dataset = ds, tokenizer = _EmptyTemplateTokenizer())
        self.assertTrue(s._chat_template_renders_empty())

    def test_nonempty_render_not_flagged(self):
        ds = [{"messages": [{"role": "user", "content": [{"type": "text", "text": "x"}]}]}]
        s = self._self(train_dataset = ds, tokenizer = _RealTemplateTokenizer())
        self.assertFalse(s._chat_template_renders_empty())

    def test_no_messages_key_not_flagged(self):
        s = self._self(train_dataset = [{"text": "raw"}], tokenizer = _EmptyTemplateTokenizer())
        self.assertFalse(s._chat_template_renders_empty())


def _clear_trainer_module(package: str):
    sys.modules.pop(f"{package}.trainer", None)
    pkg = sys.modules.get(package)
    if pkg is not None and hasattr(pkg, "trainer"):
        delattr(pkg, "trainer")


def _set_training_platform(monkeypatch, package: str, backend: str):
    training_mod = importlib.import_module(f"{package}.training")
    from utils.hardware import hardware as hw

    monkeypatch.setattr(hw, "DEVICE", None)
    monkeypatch.setattr(
        training_mod.platform,
        "system",
        lambda: "Darwin" if backend == "mlx" else "Linux",
    )
    monkeypatch.setattr(
        training_mod.platform,
        "machine",
        lambda: "arm64" if backend == "mlx" else "x86_64",
    )


def _load_trainer_module(
    monkeypatch,
    backend: str,
    package: str = "core.training",
):
    _set_training_platform(monkeypatch, package, backend)
    _clear_trainer_module(package)
    with _stubbed():
        if package in sys.modules:
            importlib.reload(sys.modules[package])
        trainer_mod = importlib.import_module(f"{package}.trainer")
        training_mod = importlib.import_module(f"{package}.training")
    monkeypatch.setattr(
        training_mod._MLXTrainerAdapter,
        "_activate_transformers_for_model",
        lambda self, model_name, hf_token: None,
    )
    return trainer_mod


class _ExitedProc:
    def join(self, timeout = None):
        return None

    def is_alive(self):
        return False


class _TerminableProc:
    def __init__(self):
        self.terminated = False
        self._done = threading.Event()

    def join(self, timeout = None):
        self._done.wait(timeout = timeout or 5)

    def is_alive(self):
        return not self.terminated

    def terminate(self):
        self.terminated = True
        self._done.set()


def test_unsloth_trainer_dispatches_for_mlx_and_torch(monkeypatch):
    trainer_mod = _load_trainer_module(monkeypatch, "mlx")

    mlx_trainer = trainer_mod.UnslothTrainer()

    assert type(mlx_trainer).__module__ == "core.training.training"
    assert mlx_trainer.get_training_progress().status_message == "Ready to train"

    trainer_mod = _load_trainer_module(monkeypatch, "torch")

    assert trainer_mod.UnslothTrainer().__class__ is trainer_mod.UnslothTrainer


def test_cli_mlx_trainer_activates_before_importing_trainer():
    repo_root = Path(__file__).resolve().parents[3]
    script = """
import json
import sys
import unsloth_cli.commands.train as train_cmd
from studio.backend.core.training import training as training_mod
from utils.hardware import hardware as hw

training_mod.platform.system = lambda: "Darwin"
training_mod.platform.machine = lambda: "arm64"
hw.DEVICE = None
events = []

def fake_activate(model_name, hf_token):
    events.append({
        "model_name": model_name,
        "trainer_loaded": "studio.backend.core.training.trainer" in sys.modules,
    })

train_cmd._activate_mlx_transformers = fake_activate
trainer = train_cmd._create_cli_trainer("mlx-community/Qwen3-0.6B-4bit", None)
print(json.dumps({
    "trainer_module": type(trainer).__module__,
    "events": events,
}))
"""
    env = os.environ.copy()
    env["PYTHONPATH"] = os.pathsep.join(
        [str(repo_root), str(repo_root / "studio" / "backend"), env.get("PYTHONPATH", "")]
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        cwd = repo_root,
        env = env,
        text = True,
        stdout = subprocess.PIPE,
        stderr = subprocess.PIPE,
        check = True,
    )
    payload = json.loads(result.stdout)

    assert payload["trainer_module"] == "studio.backend.core.training.training"
    assert payload["events"] == [
        {"model_name": "mlx-community/Qwen3-0.6B-4bit", "trainer_loaded": False}
    ]


def test_mlx_adapter_builds_config_and_reports_completion(tmp_path, monkeypatch):
    trainer_mod = _load_trainer_module(monkeypatch, "mlx")
    captured = {}

    def fake_run_worker(config, event_queue, stop_queue):
        captured["config"] = config
        event_queue.put({"type": "progress", "step": 1, "total_steps": 1, "loss": 0.25})
        event_queue.put(
            {"type": "complete", "status_message": "done", "output_dir": config["output_dir"]}
        )

    trainer = trainer_mod.UnslothTrainer()
    monkeypatch.setattr(trainer, "_run_mlx_worker", fake_run_worker)

    assert trainer.load_model("mlx-community/Qwen3-0.6B-4bit", max_seq_length = 1024)
    assert trainer.prepare_model_for_training(use_lora = False)
    dataset, eval_dataset = trainer.load_and_format_dataset("org/dataset")
    output_dir = tmp_path / "mlx-out"

    assert trainer.start_training(
        dataset = dataset,
        eval_dataset = eval_dataset,
        output_dir = output_dir,
        project_name = "Sales Assistant",
        max_steps = 1,
        learning_rate = 3e-4,
    )
    trainer.training_thread.join(timeout = 5)

    progress = trainer.get_training_progress()
    config = captured["config"]
    assert progress.is_completed
    assert progress.output_dir == str(output_dir.resolve())
    progress.status_message = "mutated"
    assert trainer.get_training_progress().status_message == "done"
    assert config["model_name"] == "mlx-community/Qwen3-0.6B-4bit"
    assert config["project_name"] == "Sales Assistant"
    assert config["hf_dataset"] == "org/dataset"
    assert config["training_type"] == "Full Finetuning"
    assert config["load_in_4bit"] is False
    assert config["max_seq_length"] == 1024
    assert config["learning_rate"] == 3e-4
    assert config["output_dir"] == str(output_dir.resolve())
    assert config["allow_external_output_dir"] is True


def test_mlx_worker_helpers_cover_cli_paths(tmp_path, monkeypatch):
    _load_trainer_module(monkeypatch, "mlx")
    from core.training.worker import (
        _resolve_mlx_local_dataset_files,
        _resolve_mlx_output_dir,
    )

    dataset = tmp_path / "train.jsonl"
    dataset.write_text('{"text":"hello"}\n', encoding = "utf-8")
    monkeypatch.chdir(tmp_path)

    assert _resolve_mlx_local_dataset_files(["train.jsonl"]) == [str(dataset)]
    assert _resolve_mlx_output_dir(
        {"output_dir": "cli-out", "allow_external_output_dir": True},
        "mlx-community/Qwen3-0.6B-4bit",
    ) == str((tmp_path / "cli-out").resolve())


def test_run_mlx_training_process_applies_side_effects_before_hardware_detection(monkeypatch):
    _load_trainer_module(monkeypatch, "mlx")
    from core.training import worker
    from utils.hardware import hardware as hw

    order = []

    def fake_activate(model_name, hf_token):
        order.append(("activate", model_name, hf_token))

    def fake_validate(config, event_queue):
        order.append("validate")
        assert os.environ["HF_HUB_DISABLE_XET"] == "1"
        assert os.environ["HF_HUB_ENABLE_HF_TRANSFER"] == "0"
        return True

    def fake_detect_hardware():
        order.append("detect")
        hw.DEVICE = hw.DeviceType.CPU
        return hw.DEVICE

    monkeypatch.delenv("HF_HUB_DISABLE_XET", raising = False)
    monkeypatch.delenv("HF_HUB_ENABLE_HF_TRANSFER", raising = False)
    monkeypatch.setattr(worker, "_validate_training_worker_config", fake_validate)
    monkeypatch.setattr(worker, "_activate_transformers_version_or_warn", fake_activate)
    monkeypatch.setattr(hw, "detect_hardware", fake_detect_hardware)

    event_queue = queue.Queue()
    worker.run_mlx_training_process(
        event_queue = event_queue,
        stop_queue = queue.Queue(),
        config = {"model_name": "mlx-community/Gemma-4-12B", "disable_xet": True},
    )

    event = event_queue.get_nowait()
    assert order == ["validate", ("activate", "mlx-community/Gemma-4-12B", None), "detect"]
    assert os.environ["HF_HUB_DISABLE_XET"] == "1"
    assert os.environ["HF_HUB_ENABLE_HF_TRANSFER"] == "0"
    assert "MLX training requires Apple Silicon" in event["error"]


def test_run_mlx_training_process_gates_a_decision_model_before_its_download(monkeypatch):
    _load_trainer_module(monkeypatch, "mlx")
    from core.training import worker
    from utils.hardware import hardware as hw

    monkeypatch.setattr(worker, "_validate_training_worker_config", lambda *args: True)
    monkeypatch.setattr(worker, "_activate_transformers_version_or_warn", lambda *args: None)
    monkeypatch.setattr(hw, "detect_hardware", lambda: None)
    monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.MLX)
    blocked = lambda config, target, token: {"error": f"blocked {target}"}
    monkeypatch.setattr(worker, "_model_load_security_error", blocked)
    monkeypatch.setattr(
        worker, "_download_decision_checkpoint", lambda *args: pytest.fail("downloaded")
    )
    events = queue.Queue()
    config = {"model_name": "org/clef", "is_decision": True}
    worker.run_mlx_training_process(event_queue = events, stop_queue = queue.Queue(), config = config)
    assert events.get_nowait()["error"] == "blocked org/clef"


def test_run_mlx_training_process_rejects_untrainable_format_before_side_effects(monkeypatch):
    _load_trainer_module(monkeypatch, "mlx")
    from core.training import worker
    from utils.hardware import hardware as hw

    monkeypatch.setattr(
        worker,
        "_activate_transformers_version_or_warn",
        lambda *_args: pytest.fail("activation must not run"),
    )
    monkeypatch.setattr(
        hw,
        "detect_hardware",
        lambda: pytest.fail("hardware detection must not run"),
    )
    event_queue = queue.Queue()

    worker.run_mlx_training_process(
        event_queue = event_queue,
        stop_queue = queue.Queue(),
        config = {"model_name": "org/model", "model_format": "gguf"},
    )

    assert "GGUF" in event_queue.get_nowait()["error"]


def test_run_mlx_training_process_rejects_invalid_exact_pin_before_side_effects(
    tmp_path, monkeypatch
):
    _load_trainer_module(monkeypatch, "mlx")
    from core.training import worker
    from utils.hardware import hardware as hw

    monkeypatch.setattr(
        worker,
        "_activate_transformers_version_or_warn",
        lambda *_args: pytest.fail("activation must not run"),
    )
    monkeypatch.setattr(
        hw,
        "detect_hardware",
        lambda: pytest.fail("hardware detection must not run"),
    )
    event_queue = queue.Queue()

    worker.run_mlx_training_process(
        event_queue = event_queue,
        stop_queue = queue.Queue(),
        config = {
            "model_name": "org/model",
            "model_snapshot_path": str(tmp_path / "missing"),
            "load_in_4bit": False,
            "require_exact_model_resource": True,
        },
    )

    assert "exact model snapshot" in event_queue.get_nowait()["error"]


def test_run_mlx_training_process_skips_duplicate_config_validation(monkeypatch):
    _load_trainer_module(monkeypatch, "mlx")
    from core.training import worker
    from utils.hardware import hardware as hw

    monkeypatch.setattr(
        worker,
        "_validate_training_worker_config",
        lambda *_args: pytest.fail("prevalidated config must not be checked twice"),
    )

    def fake_detect_hardware():
        hw.DEVICE = hw.DeviceType.CPU
        return hw.DEVICE

    monkeypatch.setattr(hw, "detect_hardware", fake_detect_hardware)
    event_queue = queue.Queue()

    worker.run_mlx_training_process(
        event_queue = event_queue,
        stop_queue = queue.Queue(),
        config = {"model_name": "org/model"},
        transformers_activated = True,
        config_prevalidated = True,
    )

    assert "MLX training requires Apple Silicon" in event_queue.get_nowait()["error"]


def test_mlx_worker_callsites_select_config_validation_policy(monkeypatch):
    import inspect

    _load_trainer_module(monkeypatch, "mlx")
    from core.training import training, worker

    outer_source = inspect.getsource(worker.run_training_process)
    adapter_source = inspect.getsource(training._MLXTrainerAdapter._run_mlx_worker)

    assert outer_source.count("_validate_training_worker_config(config, event_queue)") == 1
    assert "config_prevalidated = True" in outer_source
    assert "config_prevalidated" not in adapter_source


if __name__ == "__main__":
    unittest.main()


def test_a_cached_spark_snapshot_root_still_gets_the_llm_subfolder(tmp_path):
    """An offline or cache-pinned snapshot root has an LLM/ child, which the previous check
    read as "already at the tokenizer" and sent AutoTokenizer at the root, which has none."""
    from core.training.trainer import _spark_tts_tokenizer_kwargs

    snapshot = tmp_path / "snapshots" / "abc123"
    (snapshot / "LLM").mkdir(parents = True)

    assert _spark_tts_tokenizer_kwargs("bicodec", str(snapshot)) == {"subfolder": "LLM"}
    assert _spark_tts_tokenizer_kwargs("bicodec", "unsloth/Spark-TTS-0.5B") == {"subfolder": "LLM"}
    assert _spark_tts_tokenizer_kwargs("bicodec", str(snapshot / "LLM")) == {}
    local = tmp_path / "my-ft"
    local.mkdir()
    (local / "tokenizer_config.json").write_text("{}", encoding = "utf-8")
    assert _spark_tts_tokenizer_kwargs("bicodec", str(local)) == {}
    assert _spark_tts_tokenizer_kwargs("snac", str(snapshot)) == {}