Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

14.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Callbacks distinguish loop startup from measured training; the parent preserves non-empty statuses."""

from __future__ import annotations

import importlib
import queue as _queue
import sys
import types
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch

import pytest

_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

# trainer.py imports unsloth and trl at module level; stub missing ones only for the import.
_STUBS = {
    "unsloth": ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"),
    "unsloth.chat_templates": ("get_chat_template",),
    "trl": ("SFTTrainer", "SFTConfig"),
}
_STUBBED: list[str] = []
_TRAINER_PRE_IMPORTED = "core.training.trainer" in sys.modules


def _stub_if_missing(name, attrs):
    """Stub ``name`` unless the real package is installed."""
    if name in sys.modules:
        return
    try:
        importlib.import_module(name)
        return
    except Exception:
        pass
    _STUBBED.append(name)
    module = types.ModuleType(name)
    # A spec-less module reads as "no namespace shadow" to ensure_real_packages.
    module.__spec__ = None
    for attr in attrs:
        setattr(module, attr, MagicMock())
    sys.modules[name] = module
    parent, _, child = name.rpartition(".")
    if parent and parent in sys.modules:
        setattr(sys.modules[parent], child, module)


if not _TRAINER_PRE_IMPORTED:
    for _name, _attrs in _STUBS.items():
        _stub_if_missing(_name, _attrs)

from core.training.trainer import UnslothTrainer  # noqa: E402
from core.training.training import TrainingBackend, _MLXTrainerAdapter  # noqa: E402
from core.training.worker import (  # noqa: E402
    _create_embedding_progress_callback,
    _create_trainer_progress_callback,
)

if not _TRAINER_PRE_IMPORTED:
    for _name in _STUBBED:
        sys.modules.pop(_name, None)
    # Drop the stub-bound module so later tests re-import against the real packages.
    sys.modules.pop("core.training.trainer", None)
    sys.modules.pop("core.training", None)

ACTIVE = "Training in progress..."
STARTING = "Starting training loop..."


class _FakeQueue:
    """Stands in for the mp.Queue the worker sends events on."""

    def __init__(self):
        self.events: list[dict] = []

    def put(self, event, *args, **kwargs):
        self.events.append(event)


def _state():
    return SimpleNamespace(global_step = 0, epoch = 0.0, num_input_tokens_seen = 0)


def _drive(
    callback,
    steps = 3,
    control = None,
    on_step = None,
):
    """Run the HuggingFace callback lifecycle the way Trainer.train() does."""
    state = _state()
    control = control if control is not None else SimpleNamespace(should_training_stop = False)
    callback.on_train_begin(None, state, control)
    for step in range(1, steps + 1):
        state.global_step = step
        state.epoch = round(0.5 * step, 2)
        state.num_input_tokens_seen = 128 * step
        callback.on_log(None, state, control, logs = {"loss": 1.0 / step, "learning_rate": 1e-4})
        callback.on_step_end(None, state, control)
        if on_step is not None:
            on_step(step)
    # Once at the end: HuggingFace calls on_epoch_end per epoch, not per step.
    callback.on_epoch_end(None, state, control)
    return state, control


def _make_owner():
    # __new__ dispatches to the MLX adapter on Apple hardware.
    owner = object.__new__(UnslothTrainer)
    UnslothTrainer.__init__(owner)
    owner._update_progress(is_training = True, total_steps = 4, status_message = "Starting training...")
    return owner


def test_train_begin_reports_loop_entry_before_any_loss():
    owner = _make_owner()
    callback = owner._create_progress_callback()

    callback.on_train_begin(None, _state(), SimpleNamespace())

    assert owner.training_progress.status_message == STARTING


def test_logging_reports_an_empty_status_so_the_active_one_is_sent_once():
    owner = _make_owner()
    reported: list[str] = []
    owner.add_progress_callback(lambda progress: reported.append(progress.status_message))

    _drive(owner._create_progress_callback(), steps = 3)

    assert owner.training_progress.status_message == ""
    assert [status for status in reported if status] == [STARTING, ACTIVE]
    assert owner.training_progress.step == 3
    assert owner.training_progress.loss == pytest.approx(1 / 3)
    assert owner.training_progress.num_tokens == 384


def test_unbounded_run_reports_the_trainer_epoch():
    owner = _make_owner()

    _drive(owner._create_progress_callback(), steps = 3)

    assert owner.training_progress.epoch == 1.5


def test_bounded_run_reports_epochs_over_the_whole_dataset():
    owner = _make_owner()
    owner._kept_row_fraction = 0.1
    reported: list[float] = []
    owner.add_progress_callback(lambda progress: reported.append(progress.epoch))

    _drive(owner._create_progress_callback(), steps = 3)

    assert [epoch for epoch in reported if epoch][:3] == [0.05, 0.1, 0.15]
    assert owner.training_progress.epoch == pytest.approx(0.15)


def test_parent_status_advances_over_the_whole_chain():
    owner = _make_owner()
    backend = TrainingBackend()
    event_queue = _FakeQueue()
    owner.add_progress_callback(_create_trainer_progress_callback(event_queue))
    event_queue.put({"type": "status", "message": "Starting training...", "ts": 0.0})

    _drive(owner._create_progress_callback(), steps = 3)
    for event in event_queue.events:
        backend._handle_event(event)

    assert backend._progress.status_message == ACTIVE
    assert backend._progress.step == 3
    assert backend._progress.is_training is True


def test_training_warning_is_emitted_once_and_survives_later_status_updates():
    owner = _make_owner()
    backend = TrainingBackend()
    event_queue = _FakeQueue()
    owner.add_progress_callback(_create_trainer_progress_callback(event_queue))

    owner._record_warning("Evaluation fell back to a held-out training split.")
    owner._record_warning("Evaluation fell back to a held-out training split.")
    owner._update_progress(status_message = ACTIVE)
    for event in event_queue.events:
        backend._handle_event(event)

    warning_events = [event for event in event_queue.events if event["type"] == "warning"]
    assert [event["message"] for event in warning_events] == [
        "Evaluation fell back to a held-out training split."
    ]
    assert backend._progress.warnings == ["Evaluation fell back to a held-out training split."]
    assert backend._progress.status_message == ACTIVE


def test_mlx_adapter_deduplicates_warning_events():
    adapter = _MLXTrainerAdapter()

    adapter._handle_event({"type": "warning", "message": "Evaluation was disabled."})
    adapter._handle_event({"type": "warning", "message": "Evaluation was disabled."})
    adapter._handle_event({"type": "warning", "message": "  "})

    assert adapter.training_progress.warnings == ["Evaluation was disabled."]


@pytest.mark.parametrize(
    "stop_status",
    [
        "Stopping training and saving checkpoint...",
        "Cancelling training...",
    ],
)
def test_stop_status_is_never_replaced_by_the_active_one(stop_status):
    owner = _make_owner()
    backend = TrainingBackend()
    event_queue = _FakeQueue()
    owner.add_progress_callback(_create_trainer_progress_callback(event_queue))
    callback = owner._create_progress_callback()

    def _stop_after_first_step(step):
        if step == 1:
            owner.should_stop = True
            owner._update_progress(status_message = stop_status)

    _, control = _drive(callback, steps = 2, on_step = _stop_after_first_step)
    # A resumed run re-enters on_train_begin; an already requested stop must survive.
    callback.on_train_begin(None, _state(), SimpleNamespace())
    for event in event_queue.events:
        backend._handle_event(event)

    assert [e["message"] for e in event_queue.events if e["type"] == "status"] == [
        STARTING,
        ACTIVE,
        stop_status,
    ]
    assert backend._progress.status_message == stop_status
    assert control.should_training_stop is True


def _drive_resumed(callback, start_step, step):
    state = SimpleNamespace(global_step = start_step, epoch = 0.9, num_input_tokens_seen = 0)
    control = SimpleNamespace(should_training_stop = False)
    with patch("time.time", return_value = 700):
        callback.on_train_begin(None, state, control)
    state.global_step = step
    with patch("time.time", return_value = 760):
        callback.on_log(None, state, control, logs = {"loss": 0.5, "learning_rate": 1e-4})


def test_resumed_run_eta_uses_the_steps_done_in_this_session():
    owner = _make_owner()
    owner._update_progress(total_steps = 1000)
    backend = TrainingBackend()
    event_queue = _FakeQueue()
    owner.add_progress_callback(_create_trainer_progress_callback(event_queue))
    owner.training_start_time = 100

    _drive_resumed(owner._create_progress_callback(), start_step = 900, step = 910)
    for event in event_queue.events:
        backend._handle_event(event)

    assert owner.training_progress.eta_seconds == pytest.approx(540, rel = 0.02)
    assert backend._progress.eta_seconds == pytest.approx(540, rel = 0.02)
    assert backend._progress.session_start_step == 900
    assert backend._progress.elapsed_seconds == 60


def test_resumed_run_reports_no_eta_before_its_first_step():
    owner = _make_owner()
    owner._update_progress(total_steps = 1000)
    owner.training_start_time = 100

    _drive_resumed(owner._create_progress_callback(), start_step = 900, step = 900)

    assert owner.training_progress.eta_seconds is None


def test_fresh_run_eta_is_unchanged():
    owner = _make_owner()
    owner._update_progress(total_steps = 1000)
    owner.training_start_time = 700

    _drive_resumed(owner._create_progress_callback(), start_step = 0, step = 100)

    assert owner.training_progress.eta_seconds == pytest.approx(540, rel = 0.02)


def test_mlx_adapter_keeps_the_session_start_step():
    adapter = _MLXTrainerAdapter()

    adapter._handle_event({"type": "progress", "step": 910, "session_start_step": 900})
    adapter._handle_event({"type": "progress", "step": 920, "eval_loss": 0.4})

    assert adapter.training_progress.session_start_step == 900


def _make_embedding_callback(event_queue, should_stop = lambda: False):
    return _create_embedding_progress_callback(
        event_queue,
        total_steps = 4,
        training_start_time = 0.0,
        should_stop = should_stop,
    )


def test_embedding_parent_status_advances_over_the_whole_chain():
    event_queue = _FakeQueue()
    backend = TrainingBackend()
    event_queue.put({"type": "status", "message": "Starting embedding training...", "ts": 0.0})

    _drive(_make_embedding_callback(event_queue), steps = 3)
    for event in event_queue.events:
        backend._handle_event(event)

    assert [e["message"] for e in event_queue.events if e["type"] == "status"] == [
        "Starting embedding training...",
        STARTING,
    ]
    assert backend._progress.status_message == ACTIVE
    assert backend._progress.step == 3
    assert backend._progress.loss == pytest.approx(1 / 3)
    assert backend._progress.total_steps == 4


def test_embedding_train_begin_reports_nothing_once_a_stop_was_requested():
    event_queue = _FakeQueue()
    control = SimpleNamespace(should_training_stop = False)

    _drive(
        _make_embedding_callback(event_queue, should_stop = lambda: True), steps = 1, control = control
    )

    assert [e for e in event_queue.events if e["type"] == "status"] == []
    assert control.should_training_stop is True


def test_embedding_callback_survives_a_real_queue():
    # The worker's queue is an mp.Queue; nothing put on it may be unpicklable.
    import pickle

    event_queue = _queue.Queue()
    _drive(_make_embedding_callback(event_queue), steps = 1)

    events = [event_queue.get_nowait() for _ in range(event_queue.qsize())]
    assert [e["type"] for e in events] == ["status", "progress"]
    assert pickle.loads(pickle.dumps(events)) == events


def test_embedding_resumed_run_eta_uses_the_steps_done_in_this_session():
    event_queue = _FakeQueue()
    backend = TrainingBackend()
    callback = _create_embedding_progress_callback(
        event_queue,
        total_steps = 1000,
        training_start_time = 100,
        should_stop = lambda: False,
    )

    _drive_resumed(callback, start_step = 900, step = 910)
    for event in event_queue.events:
        backend._handle_event(event)

    assert backend._progress.eta_seconds == pytest.approx(540, rel = 0.02)
    assert backend._progress.session_start_step == 900
    assert backend._progress.elapsed_seconds == 60


@pytest.mark.parametrize("kind", ["sft", "decision_embedding"])
def test_first_step_reports_accumulation_then_waits_for_the_first_loss(kind):
    event_queue = _FakeQueue()
    if kind == "sft":
        owner = _make_owner()
        owner.add_progress_callback(_create_trainer_progress_callback(event_queue))
        callback = owner._create_progress_callback()
    else:
        callback = _make_embedding_callback(event_queue)
    args = SimpleNamespace(gradient_accumulation_steps = 4)
    state, control = _state(), SimpleNamespace(should_training_stop = False)
    callback.on_train_begin(args, state, control)
    callback.on_step_begin(args, state, control)
    for _ in range(3):
        callback.on_substep_end(args, state, control)
    assert not [e for e in event_queue.events if e.get("loss") is not None]
    state.global_step = 1
    callback.on_step_end(args, state, control)
    statuses = [e["message"] for e in event_queue.events if e["type"] == "status"]
    assert statuses[0] == STARTING
    assert [f"batch {i} of up to 4" in statuses[i] for i in range(1, 5)] == [True] * 4
    assert "Waiting for the first loss measurement" in statuses[-1]
    callback.on_log(args, state, control, logs = {"loss": 0.0})
    progress = [e for e in event_queue.events if e["type"] == "progress"][-1]
    assert progress["loss"] == 0.0
    assert progress["status_message"] == ACTIVE