Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

31.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

from __future__ import annotations

import contextlib
import json
import math
import os
import random
import re
import sys
import threading
import time
from contextvars import copy_context
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable

from loggers import get_logger

logger = get_logger(__name__)

EVAL_MAX = 2000
MIN_REPORTED_ITEMS = 50
# The from-LM recipe's head rate (scripts/train_decision_from_lm.py --head-lr).
FRESH_HEAD_LEARNING_RATE = 3e-4
STRUCT_COLUMNS_WARNING = (
    "The state, questions or gold columns are stored as objects rather than JSON strings, so "
    "missing fields come back as nulls and choice options can change order. Store them as JSON "
    "strings to train on what the Decision API sees."
)


class _Stopped(Exception):
    pass


_abandoned_downloads: list[threading.Thread] = []


def end_abandoned_download(event_queue: Any) -> None:
    # Shutdown joins the hub executor threads, so the worker would keep downloading and refuse the next run.
    # Only a worker process owns its exit; the in-process MLX adapter passes a plain queue from a thread.
    owns_process = (
        hasattr(event_queue, "join_thread")
        and threading.current_thread() is threading.main_thread()
    )
    if not owns_process or not any(thread.is_alive() for thread in _abandoned_downloads):
        return
    with contextlib.suppress(Exception):
        event_queue.close()
        event_queue.join_thread()
    for stream in (sys.stdout, sys.stderr):
        with contextlib.suppress(Exception):
            stream.flush()
    with contextlib.suppress(Exception):
        from utils.worker_stderr import stop_worker_stderr_mirror
        stop_worker_stderr_mirror()
    os._exit(0)


def _no_decisions_message(reason: str | None) -> str:
    detail = f" ({reason})" if reason else ""
    return f"No usable decisions in the dataset{detail}. Each row needs state, questions and gold."


def _decision_count(items: list) -> int:
    # A Clef item holds every question of its row, as split_holdout counts them.
    return sum(len(item.get("labels", (None,))) for item in items)


def _studio_validate(name: str, question) -> None:
    from fastapi import HTTPException
    from pydantic import ValidationError

    from routes.systemone import QuestionIn, _validate

    # The Decision API's own checks, so every trained question is one it will serve.
    try:
        _validate(name, QuestionIn.model_validate(question))
    except ValidationError:
        raise ValueError("is not a valid question") from None
    except HTTPException as exc:
        raise ValueError(exc.detail["message"]) from None


def _without_nulls(value):
    if isinstance(value, dict):
        return {k: _without_nulls(v) for k, v in value.items() if v is not None}
    if isinstance(value, list):
        return [_without_nulls(v) for v in value]
    return value


def _has_struct(feature) -> bool:
    if isinstance(feature, dict):
        return True
    inner = getattr(feature, "feature", None)
    return inner is not None and _has_struct(inner)


def _from_arrow(rows, warn: Callable[[str], None]) -> list[dict]:
    features = getattr(rows, "features", None) or {}
    if any(
        _has_struct(features[key])
        for key in ("state", "questions", "gold", "answers")
        if key in features
    ):
        warn(STRUCT_COLUMNS_WARNING)
    # Arrow pads each row with every row's keys as None; drop those, but keep "" descriptions.
    return [
        {
            **row,
            **{
                key: _without_nulls(row[key])
                for key in ("questions", "gold", "answers")
                if isinstance(row.get(key), dict)
            },
        }
        for row in rows
    ]


def _local_files(paths: list[str]) -> list[Path]:
    from utils.paths import dataset_files_in_dir, datasets_root

    files: list[Path] = []
    for entry in paths:
        path = Path(entry if os.path.isabs(entry) else os.path.join(str(datasets_root()), entry))
        files.extend(dataset_files_in_dir(path) if path.is_dir() else [path])
    return files


def _bad_json(path: Path, line: int, exc: json.JSONDecodeError) -> ValueError:
    name = re.sub(r"^[0-9a-f]{32}_", "", path.name)
    return ValueError(f"Invalid JSON at {name}:{line}: {exc.msg}")


def _read_local_rows(paths: list[str], load_dataset, warn: Callable[[str], None]) -> list[dict]:
    from utils.datasets.cells import csv_as_text_kwargs

    files = _local_files(paths)
    if not files:
        raise ValueError("No local dataset files found")
    rows: list[dict] = []
    for path in files:
        suffix = path.suffix.lower()
        if suffix in (".json", ".jsonl"):
            # Read directly: Arrow would merge every question's criteria keys into one struct.
            text = path.read_text(encoding = "utf-8-sig")
            if not text.strip():
                continue
            try:
                data = json.loads(text)
                rows.extend(data if isinstance(data, list) else [data])
            except json.JSONDecodeError as exc:
                # "Extra data" means the first line parsed, so this is JSONL.
                if exc.msg != "Extra data":
                    raise _bad_json(path, exc.lineno, exc) from None
                # LF only: U+2028, U+2029 and U+0085 inside a string are data, not a row end.
                for number, line in enumerate(text.split("\n"), 1):
                    if not line.strip():
                        continue
                    try:
                        rows.append(json.loads(line))
                    except json.JSONDecodeError as bad_line:
                        raise _bad_json(path, number, bad_line) from None
        elif suffix in (".csv", ".parquet"):
            dataset = load_dataset(
                suffix[1:],
                data_files = [str(path)],
                split = "train",
                **csv_as_text_kwargs([str(path)]),
            )
            rows.extend(_from_arrow(dataset, warn))
        else:
            raise ValueError(f"Unsupported local dataset format: {path.name}")
    return [row for row in rows if isinstance(row, dict)]


def _hold_audio(rows: list[dict]) -> None:
    # The audio files an S3 download rewrote rows to name are deleted with it, before training reads them.
    for row in rows:
        audio = row.get("audio")
        if audio is None:
            continue
        clips = audio if isinstance(audio, list) else [audio]
        held = [_held_clip(clip) for clip in clips]
        row["audio"] = held if isinstance(audio, list) else held[0]


def _held_clip(clip):
    # The rewrite leaves a plain path string or a {"path"} dict naming the download.
    if isinstance(clip, str) and os.path.isabs(clip) and os.path.isfile(clip):
        return {"bytes": Path(clip).read_bytes(), "path": clip}
    if isinstance(clip, dict) and clip.get("bytes") is None and clip.get("path"):
        if os.path.isfile(clip["path"]):
            clip["bytes"] = Path(clip["path"]).read_bytes()
    return clip


def _eval_sample(rows, seed: int):
    # Every row holds at least one decision, so EVAL_MAX rows can fill the eval set.
    if len(rows) <= EVAL_MAX:
        return rows
    return [rows[i] for i in sorted(random.Random(seed).sample(range(len(rows)), EVAL_MAX))]


def _load_rows(
    config: dict,
    should_stop: Callable[[], bool],
    status: Callable[[str], None],
    warn: Callable[[str], None],
) -> tuple[list, list | None]:
    from core.training.eval_dataset import evaluation_enabled
    from core.training.s3_dataset import S3DownloadCancelled, prepare_s3_dataset_download
    from core.training.worker import _load_embedding_hf_dataset
    from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset

    def check_stop():
        if should_stop():
            raise _Stopped("Training cancelled")

    check_stop()
    evaluate = evaluation_enabled(config.get("eval_steps"))
    seed = config["random_seed"]
    eval_rows = None
    # Local files, then S3, then the Hub: the order the route's cached-dataset check relies on.
    if config.get("local_datasets"):
        rows = _read_local_rows(config["local_datasets"], load_dataset, warn)
    elif config.get("s3_config"):
        status("Downloading dataset from S3...")
        try:
            download = prepare_s3_dataset_download(config["s3_config"], cancel_callback = should_stop)
        except S3DownloadCancelled:
            raise _Stopped("Training cancelled") from None
        try:
            rows = _read_local_rows(download.files, load_dataset, warn)
            _hold_audio(_sliced(rows, config))
        finally:
            download.cleanup()
    elif config.get("hf_dataset"):
        rows = _from_arrow(_load_embedding_hf_dataset(config, load_dataset, status), warn)
        check_stop()
        eval_split = config.get("eval_split") if evaluate else None
        if eval_split and eval_split == (config.get("train_split") or "train"):
            warn(
                "The evaluation split is the training split, so part of it is held out "
                "of training for evaluation instead."
            )
        elif eval_split:
            # Loaded like the train split, from the same cached snapshot, without the row slice.
            held = {
                **config,
                "train_split": eval_split,
                "dataset_slice_start": None,
                "dataset_slice_end": None,
            }
            eval_rows = _from_arrow(
                _eval_sample(_load_embedding_hf_dataset(held, load_dataset, status), seed), warn
            )
    else:
        raise ValueError("No dataset specified for decision training.")
    check_stop()
    if (
        evaluate
        and config.get("local_eval_datasets")
        and config.get("local_datasets")
        and {p.resolve() for p in _local_files(config["local_eval_datasets"])}
        & {p.resolve() for p in _local_files(config["local_datasets"])}
    ):
        # Scoring or calibrating on training rows would report the fit, not held-out accuracy.
        warn(
            "The evaluation files include training files, so part of the training data is "
            "held out for evaluation instead."
        )
    elif evaluate and config.get("local_eval_datasets"):
        eval_rows = _eval_sample(
            _read_local_rows(config["local_eval_datasets"], load_dataset, warn), seed
        )

    check_stop()
    return _sliced(rows, config), eval_rows


class _DecisionRows:
    """Reads raw rows on a thread, in this job's context, while the model loads."""

    def __init__(
        self,
        config,
        should_stop,
        status,
        warn,
        check = None,
    ):
        self._config = config
        self._should_stop = should_stop
        self._status = status
        self._warn = warn
        self._check = check
        self._cancelled = threading.Event()
        self._done = threading.Event()
        self._lock = threading.Lock()
        self._waiting = False
        self._last_status = None
        self._result = None
        self._error = None

    def _stopped(self):
        return self._cancelled.is_set() or self._should_stop()

    def _report_status(self, message):
        with self._lock:
            if not self._cancelled.is_set():
                self._last_status = message
                # Held until result(): the model-loading status stays on screen meanwhile.
                if self._waiting:
                    self._status(message)

    def _report_warning(self, message):
        with self._lock:
            if not self._cancelled.is_set():
                self._warn(message)

    def _load(self):
        try:
            if self._stopped():
                raise _Stopped("Training cancelled")
            result = _load_rows(
                self._config, self._stopped, self._report_status, self._report_warning
            )
            if self._check is not None:
                self._check(result[0])
            self._result = result
        except BaseException as exc:
            self._error = exc
        finally:
            self._done.set()

    def __enter__(self):
        from utils.account_context import account_thread

        context = copy_context()
        self._thread = account_thread(target = context.run, args = (self._load,), daemon = True)
        self._thread.start()
        return self

    def result(self, check_stop):
        with self._lock:
            self._waiting = True
            if not self._done.is_set() and self._last_status is not None:
                self._status(self._last_status)
        while not self._done.wait(0.1):
            check_stop()
        check_stop()
        if self._error is not None:
            raise self._error
        return self._result

    def alongside(self, work, check_stop):
        from utils.account_context import account_thread

        failure = []

        def run():
            try:
                work()
            except BaseException as exc:
                failure.append(exc)

        context = copy_context()
        thread = account_thread(target = context.run, args = (run,), daemon = True)
        thread.start()
        try:
            while thread.is_alive():
                thread.join(0.1)
                check_stop()
                if self._done.is_set() and self._error is not None:
                    raise self._error
        except BaseException:
            _abandoned_downloads.append(thread)
            raise
        if failure:
            raise failure[0]

    def __exit__(self, *_exc):
        with self._lock:
            self._cancelled.set()
        # Hub reads cannot be interrupted, so never wait on one; S3 polls the cancel flag itself.
        self._thread.join(timeout = 1)


class _ProgressRows(list):
    """Rows that report how far build_dataset has read them, as the overlay's counted progress."""

    INTERVAL = 1.0

    def __init__(
        self, rows, label: str, status: Callable[[str], None], check_stop: Callable[[], None]
    ):
        super().__init__(rows)
        self._label, self._status, self._check_stop = label, status, check_stop

    def _report(self, done: int) -> None:
        total = len(self)
        self._status(f"{self._label} {done * 100 // total}% ({done:,}/{total:,})")

    def __iter__(self):
        if not self:
            return
        self._report(0)
        last = time.monotonic()
        for done, row in enumerate(super().__iter__()):
            self._check_stop()
            now = time.monotonic()
            if now - last >= self.INTERVAL:
                self._report(done)
                last = now
            yield row
        self._report(len(self))


def _sliced(rows: list, config: dict) -> list:
    start, end = config.get("dataset_slice_start"), config.get("dataset_slice_end")
    if start is None and end is None:
        return rows
    return rows[start or 0 : (len(rows) if end is None else end + 1)]


def _is_mlx() -> bool:
    from utils.hardware import hardware
    return hardware.DEVICE == hardware.DeviceType.MLX


def _zoo_trains_d1() -> bool:
    try:
        from unsloth_zoo.mlx import decision
    except Exception:
        return False
    return hasattr(decision, "load_language_model_as_d1")


def _lm_decision_head(model_name: str, token: str | None) -> str:
    # An unsloth-zoo without the d1 trainer keeps the Clef head these models trained into before.
    if not _is_mlx() or not _zoo_trains_d1():
        return "clef"
    try:
        folder = Path(model_name).expanduser()
        if folder.is_dir():
            config = folder / "config.json"
        else:
            from huggingface_hub import hf_hub_download
            config = hf_hub_download(model_name, "config.json", token = token)
        model_type = json.loads(Path(config).read_text(encoding = "utf-8")).get("model_type")
    except (OSError, ValueError, AttributeError):
        return "clef"
    return "d1" if model_type in ("lfm3_vl", "lfm3_omni") else "clef"


def _peak_memory_gb() -> float | None:
    import torch

    if torch.cuda.is_available():
        return torch.cuda.max_memory_allocated() / 1e9
    if _is_mlx():
        import mlx.core as mx
        return mx.get_peak_memory() / 1e9
    return None


def _mlx_tracking_callback(report_to: list):
    from transformers import TrainerCallback

    # transformers' reporting callbacks are added by its own Trainer, which MLX does not run.
    class Tracking(TrainerCallback):
        writer = None

        def on_train_begin(self, args, state, control, **kwargs):
            if "tensorboard" in report_to:
                try:
                    from tensorboardX import SummaryWriter
                except ImportError:
                    from torch.utils.tensorboard import SummaryWriter
                self.writer = SummaryWriter(log_dir = os.environ["TENSORBOARD_LOGGING_DIR"])

        def on_log(
            self,
            args,
            state,
            control,
            logs = None,
            **kwargs,
        ):
            scalars = {
                f"eval/{name[5:]}" if name.startswith("eval_") else f"train/{name}": value
                for name, value in (logs or {}).items()
                if isinstance(value, (int, float))
            }
            for name, value in scalars.items() if self.writer else ():
                self.writer.add_scalar(name, value, state.global_step)
            if "wandb" in report_to:
                import wandb
                wandb.log(scalars, step = state.global_step)

        def on_train_end(self, args, state, control, **kwargs):
            if self.writer:
                self.writer.close()

    return Tracking()


def run_decision_training(event_queue: Any, stop_queue: Any, config: dict) -> None:
    import torch

    from utils.paths import is_local_path, resolve_output_dir
    from utils.training_runs import build_default_output_dir_name

    model_name, subfolder = config["model_name"], config.get("model_subfolder")
    output_dir = str(
        resolve_output_dir(
            config.get("output_dir")
            or build_default_output_dir_name(
                model_name
                if is_local_path(model_name) or not subfolder
                else f"{model_name}-{subfolder}",
                config.get("project_name"),
            )
        )
    )
    try:
        _run(event_queue, stop_queue, config, output_dir)
    except _Stopped as stopped:
        event_queue.put(
            {
                "type": "complete",
                "output_dir": None,
                "status_message": str(stopped),
                "ts": time.time(),
            }
        )
    except torch.cuda.OutOfMemoryError:
        event_queue.put(
            {
                "type": "error",
                "error": (
                    "Out of GPU memory while training the decision model. Lower the batch size "
                    "and raise gradient accumulation to keep the same effective batch."
                ),
                "stack": "",
                "ts": time.time(),
            }
        )
    finally:
        # The trainer creates the folder at init; a run that saved nothing leaves none behind.
        with contextlib.suppress(OSError):
            os.rmdir(output_dir)


def _run(event_queue: Any, stop_queue: Any, config: dict, output_dir: str) -> None:
    from core.import_guards import ensure_real_packages

    ensure_real_packages("unsloth_zoo", "unsloth")
    from unsloth import DecisionTrainer, FastDecisionModel, is_bfloat16_supported

    import torch
    from huggingface_hub.errors import LocalEntryNotFoundError
    from transformers import TrainingArguments

    from core.systemone import laya_runtime
    from core.systemone.catalog import Checkpoint
    from core.training.eval_dataset import evaluation_enabled
    from core.training.trainer import (
        _drop_hf_stdout_callbacks,
        _hf_stdout_progress_disabled,
        normalize_gradient_checkpointing,
    )
    from core.training.worker import (
        _create_embedding_progress_callback,
        _download_decision_checkpoint,
        _emit_output_dir,
        _send_status,
        _start_worker_stop_poller,
        _worker_hf_token,
    )

    def send(kind: str, **payload) -> None:
        event_queue.put({"type": kind, **payload, "ts": time.time()})

    def status(message: str) -> None:
        _send_status(event_queue, message)

    def warn(message: str) -> None:
        send("warning", message = message)

    stop = {"requested": False, "save": True}

    def on_stop(save: bool) -> None:
        stop["requested"], stop["save"] = True, save
        logger.info("Decision training: stop signal received (save=%s)", save)

    def check_stop() -> None:
        if stop["requested"]:
            raise _Stopped("Training stopped" if stop["save"] else "Training cancelled")

    def validate(name: str, question) -> None:
        # build_dataset calls this for every decision, so a stop does not wait for the whole dataset.
        check_stop()
        _studio_validate(name, question)

    _start_worker_stop_poller(stop_queue, on_stop)

    model_name = config["model_name"]
    subfolder = config.get("model_subfolder") or None
    llm = config.get("decision_layout") == "llm"
    clef = llm or config.get("decision_layout") == "clef"
    hf_token = _worker_hf_token(config)
    if hf_token:
        os.environ["HF_TOKEN"] = hf_token
    seed = config["random_seed"]
    use_lora = config["training_type"] == "LoRA/QLoRA"
    gradient_checkpointing = normalize_gradient_checkpointing(config["gradient_checkpointing"])

    def check_rows(rows: list) -> None:
        problem = FastDecisionModel.rows_problem(rows, clef, validate)
        if problem is not None:
            raise ValueError(_no_decisions_message(problem))

    with _DecisionRows(config, lambda: stop["requested"], status, warn, check_rows) as pending_rows:
        check_stop()
        if config.get("local_datasets"):
            pending_rows.result(check_stop)
        pending_rows.alongside(
            lambda: _download_decision_checkpoint(event_queue, config), check_stop
        )
        check_stop()
        status("Loading decision model...")
        if llm:
            head = config.get("decision_head") or _lm_decision_head(model_name, hf_token)
            model, tokenizer = FastDecisionModel.from_pretrained(
                model_name,
                decision_head = head,
                max_seq_length = config.get("max_seq_length") or None,
                load_in_4bit = use_lora and bool(config.get("load_in_4bit")) and head != "d1",
                full_finetuning = not use_lora,
                token = hf_token or None,
                use_gradient_checkpointing = gradient_checkpointing,
                random_state = seed,
            )
        else:
            try:
                root = laya_runtime._checkpoint_dir(
                    Checkpoint("base", model_name, subfolder, "", layout = "clef" if clef else "laya")
                )
            except LocalEntryNotFoundError as exc:
                send("error", error = f"Could not download {model_name}: {exc}", stack = "")
                return
            except FileNotFoundError as exc:
                kind = "Clef" if clef else "Laya"
                send("error", error = f"Not a {kind} decision checkpoint: {exc}", stack = "")
                return
            model, tokenizer = FastDecisionModel.from_pretrained(
                str(root),
                subfolder = subfolder,
                full_finetuning = not use_lora,
                use_gradient_checkpointing = gradient_checkpointing,
                # Clef trains through Unsloth's Qwen3.5 loader, in 4-bit for QLoRA; Laya is always 16-bit.
                **(
                    {
                        "load_in_4bit": use_lora and bool(config.get("load_in_4bit")),
                        "max_seq_length": config.get("max_seq_length") or None,
                    }
                    if clef
                    else {}
                ),
            )
        if use_lora:
            model = FastDecisionModel.get_peft_model(
                model,
                r = config["lora_r"],
                lora_alpha = config["lora_alpha"],
                lora_dropout = config["lora_dropout"],
                use_gradient_checkpointing = gradient_checkpointing,
                random_state = seed,
                use_rslora = config["use_rslora"],
            )
        check_stop()

        status("Loading dataset...")
        rows, eval_rows = pending_rows.result(check_stop)
    check_stop()
    # "rows" files the step under the dataset in the overlay, with its counted progress bar.
    status("Preparing decision rows...")
    items, report = FastDecisionModel.build_dataset(
        _ProgressRows(rows, "Preparing decision rows", status, check_stop),
        tokenizer,
        model,
        validate,
    )
    eval_items = []
    if eval_rows is not None:
        eval_items, eval_report = FastDecisionModel.build_dataset(
            _ProgressRows(eval_rows, "Preparing evaluation rows", status, check_stop),
            tokenizer,
            model,
            validate,
        )
        report = {
            "total": report["total"] + eval_report["total"],
            "skipped": report["skipped"] + eval_report["skipped"],
            "reason": report["reason"] or eval_report["reason"],
        }
    if not items:
        send("error", error = _no_decisions_message(report["reason"]), stack = "")
        return
    if report["skipped"]:
        warn(f"Skipped {report['skipped']:,} of {report['total']:,} decisions: {report['reason']}.")
    if not eval_items:
        items, eval_items = FastDecisionModel.split_holdout(items, seed)
        if eval_items:
            status(
                f"Holding out {_decision_count(eval_items):,} decisions to calibrate confidence..."
            )
    elif len(eval_items) > EVAL_MAX:
        eval_items = random.Random(seed).sample(eval_items, EVAL_MAX)

    batch_size, accumulation = config["batch_size"], config["gradient_accumulation_steps"]
    max_steps, epochs = config["max_steps"] or 0, max(1, config["num_epochs"])
    steps_per_epoch = max(1, math.ceil(math.ceil(len(items) / batch_size) / accumulation))
    total_steps = max_steps if max_steps > 0 else steps_per_epoch * epochs
    bf16 = is_bfloat16_supported()
    arguments = {
        "output_dir": output_dir,
        "per_device_train_batch_size": batch_size,
        # A Clef record is every question of a row over a 9B-27B backbone, so it evaluates at the train batch.
        "per_device_eval_batch_size": batch_size if clef else 16,
        "gradient_accumulation_steps": accumulation,
        "learning_rate": float(config["learning_rate"]),
        "weight_decay": config["weight_decay"],
        "lr_scheduler_type": config["lr_scheduler_type"],
        "optim": config["optim"],
        "seed": seed,
        "bf16": bf16,
        "fp16": torch.cuda.is_available() and not bf16,
        "logging_steps": 1,
        "report_to": "none",
        "disable_tqdm": _hf_stdout_progress_disabled(),
        "warmup_steps": config["warmup_steps"]
        or round((config["warmup_ratio"] or 0.0) * total_steps),
        "save_strategy": "no",
    }
    if max_steps > 0:
        arguments["max_steps"] = max_steps
    else:
        arguments["num_train_epochs"] = epochs
    if config["max_grad_norm"] is not None:
        arguments["max_grad_norm"] = config["max_grad_norm"]
    if eval_items:
        if evaluation_enabled(config["eval_steps"]):
            arguments.update(eval_strategy = "steps", eval_steps = float(config["eval_steps"]))
        else:
            arguments["eval_strategy"] = "epoch"
        send("eval_configured")
    report_to = []
    if config.get("enable_tensorboard"):
        from utils.paths import resolve_tensorboard_dir
        report_to.append("tensorboard")
        # transformers 5 ignores logging_dir; its TensorBoard callback reads this instead.
        os.environ["TENSORBOARD_LOGGING_DIR"] = str(
            resolve_tensorboard_dir(config.get("tensorboard_dir"))
        )
    if config["enable_wandb"]:
        try:
            import wandb

            if config.get("wandb_token"):
                os.environ["WANDB_API_KEY"] = config["wandb_token"]
            wandb.init(project = config.get("wandb_project") or "unsloth-training")
            report_to.append("wandb")
        except Exception as exc:
            warn(f"Weights & Biases logging is off: {exc}")
    # On MLX the run's own tracking callback reports instead, into the directory and W&B run opened above.
    arguments["report_to"] = "none" if _is_mlx() or not report_to else report_to

    base_metrics = None
    if eval_items:
        status(
            f"Evaluating the base model on {_decision_count(eval_items):,} held-out decisions..."
        )
        base_metrics = FastDecisionModel.evaluate(
            model, tokenizer, eval_items, batch_size = config["batch_size"]
        )
        logger.info("Base held-out metrics: %s", base_metrics)
    check_stop()

    _emit_output_dir(event_queue, output_dir)
    trainer = DecisionTrainer(
        model = model,
        args = TrainingArguments(**arguments),
        train_dataset = items,
        eval_dataset = eval_items or None,
        processing_class = tokenizer,
        # A fresh head starts from random weights, so it learns faster than a trained Clef head.
        head_learning_rate = FRESH_HEAD_LEARNING_RATE if llm else None,
        callbacks = [
            _create_embedding_progress_callback(
                event_queue,
                total_steps = total_steps,
                training_start_time = time.time(),
                should_stop = lambda: stop["requested"],
            )
        ]
        + ([_mlx_tracking_callback(report_to)] if report_to and _is_mlx() else []),
    )
    _drop_hf_stdout_callbacks(trainer)
    start = time.time()
    trainer.train()
    if stop["requested"] and not stop["save"]:
        raise _Stopped("Training cancelled")

    tuned_metrics = None
    if eval_items:
        status("Calibrating confidence...")
        tuned_metrics = FastDecisionModel.calibrate(
            model, tokenizer, eval_items, batch_size = config["batch_size"]
        )
        logger.info("Fine-tuned held-out metrics: %s", tuned_metrics)
    # A cancel during calibration still saves nothing; a stop with save keeps the model.
    if stop["requested"] and not stop["save"]:
        raise _Stopped("Training cancelled")

    status("Saving model...")
    peak_gb = _peak_memory_gb()
    model.decision_config["training"] = {
        "base": model_name,
        "subfolder": subfolder,
        "decision_head": f"{head} (new)" if llm else None,
        "method": ("qlora" if clef and config.get("load_in_4bit") else "lora")
        if use_lora
        else "full",
        "objective": "soft_cross_entropy",
        "dataset": [Path(path).name for path in config.get("local_datasets") or []]
        or config.get("hf_dataset")
        or None,
        "steps": trainer.state.global_step,
        "epochs": round(trainer.state.epoch or 0, 2),
        "heldout_decisions": _decision_count(eval_items),
        "heldout_accuracy_base": base_metrics and round(base_metrics["accuracy"], 4),
        "heldout_accuracy": tuned_metrics and round(tuned_metrics["accuracy"], 4),
        "heldout_ece_base": base_metrics and round(base_metrics["ece"], 4),
        "heldout_ece": tuned_metrics and round(tuned_metrics["ece"], 4),
        "seconds": round(time.time() - start, 1),
        "peak_memory_gb": peak_gb and round(peak_gb, 2),
        "date": datetime.now(timezone.utc).isoformat(timespec = "seconds"),
    }
    model.save_pretrained_merged(output_dir, tokenizer)
    logger.info("Decision model saved to %s: %s", output_dir, model.decision_config["training"])

    message = "Decision training completed"
    if base_metrics and tuned_metrics and _decision_count(eval_items) >= MIN_REPORTED_ITEMS:
        message = (
            f"Held-out accuracy {base_metrics['accuracy']:.2f} -> {tuned_metrics['accuracy']:.2f}, "
            f"calibration error {tuned_metrics['ece']:.2f}"
        )
    send("complete", output_dir = output_dir, status_message = message)