Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
from __future__ import annotations
import contextlib
import json
import math
import os
import random
import re
import sys
import threading
import time
from contextvars import copy_context
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable
from loggers import get_logger
logger = get_logger(__name__)
EVAL_MAX = 2000
MIN_REPORTED_ITEMS = 50
# The from-LM recipe's head rate (scripts/train_decision_from_lm.py --head-lr).
FRESH_HEAD_LEARNING_RATE = 3e-4
STRUCT_COLUMNS_WARNING = (
"The state, questions or gold columns are stored as objects rather than JSON strings, so "
"missing fields come back as nulls and choice options can change order. Store them as JSON "
"strings to train on what the Decision API sees."
)
class _Stopped(Exception):
pass
_abandoned_downloads: list[threading.Thread] = []
def end_abandoned_download(event_queue: Any) -> None:
# Shutdown joins the hub executor threads, so the worker would keep downloading and refuse the next run.
# Only a worker process owns its exit; the in-process MLX adapter passes a plain queue from a thread.
owns_process = (
hasattr(event_queue, "join_thread")
and threading.current_thread() is threading.main_thread()
)
if not owns_process or not any(thread.is_alive() for thread in _abandoned_downloads):
return
with contextlib.suppress(Exception):
event_queue.close()
event_queue.join_thread()
for stream in (sys.stdout, sys.stderr):
with contextlib.suppress(Exception):
stream.flush()
with contextlib.suppress(Exception):
from utils.worker_stderr import stop_worker_stderr_mirror
stop_worker_stderr_mirror()
os._exit(0)
def _no_decisions_message(reason: str | None) -> str:
detail = f" ({reason})" if reason else ""
return f"No usable decisions in the dataset{detail}. Each row needs state, questions and gold."
def _decision_count(items: list) -> int:
# A Clef item holds every question of its row, as split_holdout counts them.
return sum(len(item.get("labels", (None,))) for item in items)
def _studio_validate(name: str, question) -> None:
from fastapi import HTTPException
from pydantic import ValidationError
from routes.systemone import QuestionIn, _validate
# The Decision API's own checks, so every trained question is one it will serve.
try:
_validate(name, QuestionIn.model_validate(question))
except ValidationError:
raise ValueError("is not a valid question") from None
except HTTPException as exc:
raise ValueError(exc.detail["message"]) from None
def _without_nulls(value):
if isinstance(value, dict):
return {k: _without_nulls(v) for k, v in value.items() if v is not None}
if isinstance(value, list):
return [_without_nulls(v) for v in value]
return value
def _has_struct(feature) -> bool:
if isinstance(feature, dict):
return True
inner = getattr(feature, "feature", None)
return inner is not None and _has_struct(inner)
def _from_arrow(rows, warn: Callable[[str], None]) -> list[dict]:
features = getattr(rows, "features", None) or {}
if any(
_has_struct(features[key])
for key in ("state", "questions", "gold", "answers")
if key in features
):
warn(STRUCT_COLUMNS_WARNING)
# Arrow pads each row with every row's keys as None; drop those, but keep "" descriptions.
return [
{
**row,
**{
key: _without_nulls(row[key])
for key in ("questions", "gold", "answers")
if isinstance(row.get(key), dict)
},
}
for row in rows
]
def _local_files(paths: list[str]) -> list[Path]:
from utils.paths import dataset_files_in_dir, datasets_root
files: list[Path] = []
for entry in paths:
path = Path(entry if os.path.isabs(entry) else os.path.join(str(datasets_root()), entry))
files.extend(dataset_files_in_dir(path) if path.is_dir() else [path])
return files
def _bad_json(path: Path, line: int, exc: json.JSONDecodeError) -> ValueError:
name = re.sub(r"^[0-9a-f]{32}_", "", path.name)
return ValueError(f"Invalid JSON at {name}:{line}: {exc.msg}")
def _read_local_rows(paths: list[str], load_dataset, warn: Callable[[str], None]) -> list[dict]:
from utils.datasets.cells import csv_as_text_kwargs
files = _local_files(paths)
if not files:
raise ValueError("No local dataset files found")
rows: list[dict] = []
for path in files:
suffix = path.suffix.lower()
if suffix in (".json", ".jsonl"):
# Read directly: Arrow would merge every question's criteria keys into one struct.
text = path.read_text(encoding = "utf-8-sig")
if not text.strip():
continue
try:
data = json.loads(text)
rows.extend(data if isinstance(data, list) else [data])
except json.JSONDecodeError as exc:
# "Extra data" means the first line parsed, so this is JSONL.
if exc.msg != "Extra data":
raise _bad_json(path, exc.lineno, exc) from None
# LF only: U+2028, U+2029 and U+0085 inside a string are data, not a row end.
for number, line in enumerate(text.split("\n"), 1):
if not line.strip():
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError as bad_line:
raise _bad_json(path, number, bad_line) from None
elif suffix in (".csv", ".parquet"):
dataset = load_dataset(
suffix[1:],
data_files = [str(path)],
split = "train",
**csv_as_text_kwargs([str(path)]),
)
rows.extend(_from_arrow(dataset, warn))
else:
raise ValueError(f"Unsupported local dataset format: {path.name}")
return [row for row in rows if isinstance(row, dict)]
def _hold_audio(rows: list[dict]) -> None:
# The audio files an S3 download rewrote rows to name are deleted with it, before training reads them.
for row in rows:
audio = row.get("audio")
if audio is None:
continue
clips = audio if isinstance(audio, list) else [audio]
held = [_held_clip(clip) for clip in clips]
row["audio"] = held if isinstance(audio, list) else held[0]
def _held_clip(clip):
# The rewrite leaves a plain path string or a {"path"} dict naming the download.
if isinstance(clip, str) and os.path.isabs(clip) and os.path.isfile(clip):
return {"bytes": Path(clip).read_bytes(), "path": clip}
if isinstance(clip, dict) and clip.get("bytes") is None and clip.get("path"):
if os.path.isfile(clip["path"]):
clip["bytes"] = Path(clip["path"]).read_bytes()
return clip
def _eval_sample(rows, seed: int):
# Every row holds at least one decision, so EVAL_MAX rows can fill the eval set.
if len(rows) <= EVAL_MAX:
return rows
return [rows[i] for i in sorted(random.Random(seed).sample(range(len(rows)), EVAL_MAX))]
def _load_rows(
config: dict,
should_stop: Callable[[], bool],
status: Callable[[str], None],
warn: Callable[[str], None],
) -> tuple[list, list | None]:
from core.training.eval_dataset import evaluation_enabled
from core.training.s3_dataset import S3DownloadCancelled, prepare_s3_dataset_download
from core.training.worker import _load_embedding_hf_dataset
from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset
def check_stop():
if should_stop():
raise _Stopped("Training cancelled")
check_stop()
evaluate = evaluation_enabled(config.get("eval_steps"))
seed = config["random_seed"]
eval_rows = None
# Local files, then S3, then the Hub: the order the route's cached-dataset check relies on.
if config.get("local_datasets"):
rows = _read_local_rows(config["local_datasets"], load_dataset, warn)
elif config.get("s3_config"):
status("Downloading dataset from S3...")
try:
download = prepare_s3_dataset_download(config["s3_config"], cancel_callback = should_stop)
except S3DownloadCancelled:
raise _Stopped("Training cancelled") from None
try:
rows = _read_local_rows(download.files, load_dataset, warn)
_hold_audio(_sliced(rows, config))
finally:
download.cleanup()
elif config.get("hf_dataset"):
rows = _from_arrow(_load_embedding_hf_dataset(config, load_dataset, status), warn)
check_stop()
eval_split = config.get("eval_split") if evaluate else None
if eval_split and eval_split == (config.get("train_split") or "train"):
warn(
"The evaluation split is the training split, so part of it is held out "
"of training for evaluation instead."
)
elif eval_split:
# Loaded like the train split, from the same cached snapshot, without the row slice.
held = {
**config,
"train_split": eval_split,
"dataset_slice_start": None,
"dataset_slice_end": None,
}
eval_rows = _from_arrow(
_eval_sample(_load_embedding_hf_dataset(held, load_dataset, status), seed), warn
)
else:
raise ValueError("No dataset specified for decision training.")
check_stop()
if (
evaluate
and config.get("local_eval_datasets")
and config.get("local_datasets")
and {p.resolve() for p in _local_files(config["local_eval_datasets"])}
& {p.resolve() for p in _local_files(config["local_datasets"])}
):
# Scoring or calibrating on training rows would report the fit, not held-out accuracy.
warn(
"The evaluation files include training files, so part of the training data is "
"held out for evaluation instead."
)
elif evaluate and config.get("local_eval_datasets"):
eval_rows = _eval_sample(
_read_local_rows(config["local_eval_datasets"], load_dataset, warn), seed
)
check_stop()
return _sliced(rows, config), eval_rows
class _DecisionRows:
"""Reads raw rows on a thread, in this job's context, while the model loads."""
def __init__(
self,
config,
should_stop,
status,
warn,
check = None,
):
self._config = config
self._should_stop = should_stop
self._status = status
self._warn = warn
self._check = check
self._cancelled = threading.Event()
self._done = threading.Event()
self._lock = threading.Lock()
self._waiting = False
self._last_status = None
self._result = None
self._error = None
def _stopped(self):
return self._cancelled.is_set() or self._should_stop()
def _report_status(self, message):
with self._lock:
if not self._cancelled.is_set():
self._last_status = message
# Held until result(): the model-loading status stays on screen meanwhile.
if self._waiting:
self._status(message)
def _report_warning(self, message):
with self._lock:
if not self._cancelled.is_set():
self._warn(message)
def _load(self):
try:
if self._stopped():
raise _Stopped("Training cancelled")
result = _load_rows(
self._config, self._stopped, self._report_status, self._report_warning
)
if self._check is not None:
self._check(result[0])
self._result = result
except BaseException as exc:
self._error = exc
finally:
self._done.set()
def __enter__(self):
from utils.account_context import account_thread
context = copy_context()
self._thread = account_thread(target = context.run, args = (self._load,), daemon = True)
self._thread.start()
return self
def result(self, check_stop):
with self._lock:
self._waiting = True
if not self._done.is_set() and self._last_status is not None:
self._status(self._last_status)
while not self._done.wait(0.1):
check_stop()
check_stop()
if self._error is not None:
raise self._error
return self._result
def alongside(self, work, check_stop):
from utils.account_context import account_thread
failure = []
def run():
try:
work()
except BaseException as exc:
failure.append(exc)
context = copy_context()
thread = account_thread(target = context.run, args = (run,), daemon = True)
thread.start()
try:
while thread.is_alive():
thread.join(0.1)
check_stop()
if self._done.is_set() and self._error is not None:
raise self._error
except BaseException:
_abandoned_downloads.append(thread)
raise
if failure:
raise failure[0]
def __exit__(self, *_exc):
with self._lock:
self._cancelled.set()
# Hub reads cannot be interrupted, so never wait on one; S3 polls the cancel flag itself.
self._thread.join(timeout = 1)
class _ProgressRows(list):
"""Rows that report how far build_dataset has read them, as the overlay's counted progress."""
INTERVAL = 1.0
def __init__(
self, rows, label: str, status: Callable[[str], None], check_stop: Callable[[], None]
):
super().__init__(rows)
self._label, self._status, self._check_stop = label, status, check_stop
def _report(self, done: int) -> None:
total = len(self)
self._status(f"{self._label} {done * 100 // total}% ({done:,}/{total:,})")
def __iter__(self):
if not self:
return
self._report(0)
last = time.monotonic()
for done, row in enumerate(super().__iter__()):
self._check_stop()
now = time.monotonic()
if now - last >= self.INTERVAL:
self._report(done)
last = now
yield row
self._report(len(self))
def _sliced(rows: list, config: dict) -> list:
start, end = config.get("dataset_slice_start"), config.get("dataset_slice_end")
if start is None and end is None:
return rows
return rows[start or 0 : (len(rows) if end is None else end + 1)]
def _is_mlx() -> bool:
from utils.hardware import hardware
return hardware.DEVICE == hardware.DeviceType.MLX
def _zoo_trains_d1() -> bool:
try:
from unsloth_zoo.mlx import decision
except Exception:
return False
return hasattr(decision, "load_language_model_as_d1")
def _lm_decision_head(model_name: str, token: str | None) -> str:
# An unsloth-zoo without the d1 trainer keeps the Clef head these models trained into before.
if not _is_mlx() or not _zoo_trains_d1():
return "clef"
try:
folder = Path(model_name).expanduser()
if folder.is_dir():
config = folder / "config.json"
else:
from huggingface_hub import hf_hub_download
config = hf_hub_download(model_name, "config.json", token = token)
model_type = json.loads(Path(config).read_text(encoding = "utf-8")).get("model_type")
except (OSError, ValueError, AttributeError):
return "clef"
return "d1" if model_type in ("lfm3_vl", "lfm3_omni") else "clef"
def _peak_memory_gb() -> float | None:
import torch
if torch.cuda.is_available():
return torch.cuda.max_memory_allocated() / 1e9
if _is_mlx():
import mlx.core as mx
return mx.get_peak_memory() / 1e9
return None
def _mlx_tracking_callback(report_to: list):
from transformers import TrainerCallback
# transformers' reporting callbacks are added by its own Trainer, which MLX does not run.
class Tracking(TrainerCallback):
writer = None
def on_train_begin(self, args, state, control, **kwargs):
if "tensorboard" in report_to:
try:
from tensorboardX import SummaryWriter
except ImportError:
from torch.utils.tensorboard import SummaryWriter
self.writer = SummaryWriter(log_dir = os.environ["TENSORBOARD_LOGGING_DIR"])
def on_log(
self,
args,
state,
control,
logs = None,
**kwargs,
):
scalars = {
f"eval/{name[5:]}" if name.startswith("eval_") else f"train/{name}": value
for name, value in (logs or {}).items()
if isinstance(value, (int, float))
}
for name, value in scalars.items() if self.writer else ():
self.writer.add_scalar(name, value, state.global_step)
if "wandb" in report_to:
import wandb
wandb.log(scalars, step = state.global_step)
def on_train_end(self, args, state, control, **kwargs):
if self.writer:
self.writer.close()
return Tracking()
def run_decision_training(event_queue: Any, stop_queue: Any, config: dict) -> None:
import torch
from utils.paths import is_local_path, resolve_output_dir
from utils.training_runs import build_default_output_dir_name
model_name, subfolder = config["model_name"], config.get("model_subfolder")
output_dir = str(
resolve_output_dir(
config.get("output_dir")
or build_default_output_dir_name(
model_name
if is_local_path(model_name) or not subfolder
else f"{model_name}-{subfolder}",
config.get("project_name"),
)
)
)
try:
_run(event_queue, stop_queue, config, output_dir)
except _Stopped as stopped:
event_queue.put(
{
"type": "complete",
"output_dir": None,
"status_message": str(stopped),
"ts": time.time(),
}
)
except torch.cuda.OutOfMemoryError:
event_queue.put(
{
"type": "error",
"error": (
"Out of GPU memory while training the decision model. Lower the batch size "
"and raise gradient accumulation to keep the same effective batch."
),
"stack": "",
"ts": time.time(),
}
)
finally:
# The trainer creates the folder at init; a run that saved nothing leaves none behind.
with contextlib.suppress(OSError):
os.rmdir(output_dir)
def _run(event_queue: Any, stop_queue: Any, config: dict, output_dir: str) -> None:
from core.import_guards import ensure_real_packages
ensure_real_packages("unsloth_zoo", "unsloth")
from unsloth import DecisionTrainer, FastDecisionModel, is_bfloat16_supported
import torch
from huggingface_hub.errors import LocalEntryNotFoundError
from transformers import TrainingArguments
from core.systemone import laya_runtime
from core.systemone.catalog import Checkpoint
from core.training.eval_dataset import evaluation_enabled
from core.training.trainer import (
_drop_hf_stdout_callbacks,
_hf_stdout_progress_disabled,
normalize_gradient_checkpointing,
)
from core.training.worker import (
_create_embedding_progress_callback,
_download_decision_checkpoint,
_emit_output_dir,
_send_status,
_start_worker_stop_poller,
_worker_hf_token,
)
def send(kind: str, **payload) -> None:
event_queue.put({"type": kind, **payload, "ts": time.time()})
def status(message: str) -> None:
_send_status(event_queue, message)
def warn(message: str) -> None:
send("warning", message = message)
stop = {"requested": False, "save": True}
def on_stop(save: bool) -> None:
stop["requested"], stop["save"] = True, save
logger.info("Decision training: stop signal received (save=%s)", save)
def check_stop() -> None:
if stop["requested"]:
raise _Stopped("Training stopped" if stop["save"] else "Training cancelled")
def validate(name: str, question) -> None:
# build_dataset calls this for every decision, so a stop does not wait for the whole dataset.
check_stop()
_studio_validate(name, question)
_start_worker_stop_poller(stop_queue, on_stop)
model_name = config["model_name"]
subfolder = config.get("model_subfolder") or None
llm = config.get("decision_layout") == "llm"
clef = llm or config.get("decision_layout") == "clef"
hf_token = _worker_hf_token(config)
if hf_token:
os.environ["HF_TOKEN"] = hf_token
seed = config["random_seed"]
use_lora = config["training_type"] == "LoRA/QLoRA"
gradient_checkpointing = normalize_gradient_checkpointing(config["gradient_checkpointing"])
def check_rows(rows: list) -> None:
problem = FastDecisionModel.rows_problem(rows, clef, validate)
if problem is not None:
raise ValueError(_no_decisions_message(problem))
with _DecisionRows(config, lambda: stop["requested"], status, warn, check_rows) as pending_rows:
check_stop()
if config.get("local_datasets"):
pending_rows.result(check_stop)
pending_rows.alongside(
lambda: _download_decision_checkpoint(event_queue, config), check_stop
)
check_stop()
status("Loading decision model...")
if llm:
head = config.get("decision_head") or _lm_decision_head(model_name, hf_token)
model, tokenizer = FastDecisionModel.from_pretrained(
model_name,
decision_head = head,
max_seq_length = config.get("max_seq_length") or None,
load_in_4bit = use_lora and bool(config.get("load_in_4bit")) and head != "d1",
full_finetuning = not use_lora,
token = hf_token or None,
use_gradient_checkpointing = gradient_checkpointing,
random_state = seed,
)
else:
try:
root = laya_runtime._checkpoint_dir(
Checkpoint("base", model_name, subfolder, "", layout = "clef" if clef else "laya")
)
except LocalEntryNotFoundError as exc:
send("error", error = f"Could not download {model_name}: {exc}", stack = "")
return
except FileNotFoundError as exc:
kind = "Clef" if clef else "Laya"
send("error", error = f"Not a {kind} decision checkpoint: {exc}", stack = "")
return
model, tokenizer = FastDecisionModel.from_pretrained(
str(root),
subfolder = subfolder,
full_finetuning = not use_lora,
use_gradient_checkpointing = gradient_checkpointing,
# Clef trains through Unsloth's Qwen3.5 loader, in 4-bit for QLoRA; Laya is always 16-bit.
**(
{
"load_in_4bit": use_lora and bool(config.get("load_in_4bit")),
"max_seq_length": config.get("max_seq_length") or None,
}
if clef
else {}
),
)
if use_lora:
model = FastDecisionModel.get_peft_model(
model,
r = config["lora_r"],
lora_alpha = config["lora_alpha"],
lora_dropout = config["lora_dropout"],
use_gradient_checkpointing = gradient_checkpointing,
random_state = seed,
use_rslora = config["use_rslora"],
)
check_stop()
status("Loading dataset...")
rows, eval_rows = pending_rows.result(check_stop)
check_stop()
# "rows" files the step under the dataset in the overlay, with its counted progress bar.
status("Preparing decision rows...")
items, report = FastDecisionModel.build_dataset(
_ProgressRows(rows, "Preparing decision rows", status, check_stop),
tokenizer,
model,
validate,
)
eval_items = []
if eval_rows is not None:
eval_items, eval_report = FastDecisionModel.build_dataset(
_ProgressRows(eval_rows, "Preparing evaluation rows", status, check_stop),
tokenizer,
model,
validate,
)
report = {
"total": report["total"] + eval_report["total"],
"skipped": report["skipped"] + eval_report["skipped"],
"reason": report["reason"] or eval_report["reason"],
}
if not items:
send("error", error = _no_decisions_message(report["reason"]), stack = "")
return
if report["skipped"]:
warn(f"Skipped {report['skipped']:,} of {report['total']:,} decisions: {report['reason']}.")
if not eval_items:
items, eval_items = FastDecisionModel.split_holdout(items, seed)
if eval_items:
status(
f"Holding out {_decision_count(eval_items):,} decisions to calibrate confidence..."
)
elif len(eval_items) > EVAL_MAX:
eval_items = random.Random(seed).sample(eval_items, EVAL_MAX)
batch_size, accumulation = config["batch_size"], config["gradient_accumulation_steps"]
max_steps, epochs = config["max_steps"] or 0, max(1, config["num_epochs"])
steps_per_epoch = max(1, math.ceil(math.ceil(len(items) / batch_size) / accumulation))
total_steps = max_steps if max_steps > 0 else steps_per_epoch * epochs
bf16 = is_bfloat16_supported()
arguments = {
"output_dir": output_dir,
"per_device_train_batch_size": batch_size,
# A Clef record is every question of a row over a 9B-27B backbone, so it evaluates at the train batch.
"per_device_eval_batch_size": batch_size if clef else 16,
"gradient_accumulation_steps": accumulation,
"learning_rate": float(config["learning_rate"]),
"weight_decay": config["weight_decay"],
"lr_scheduler_type": config["lr_scheduler_type"],
"optim": config["optim"],
"seed": seed,
"bf16": bf16,
"fp16": torch.cuda.is_available() and not bf16,
"logging_steps": 1,
"report_to": "none",
"disable_tqdm": _hf_stdout_progress_disabled(),
"warmup_steps": config["warmup_steps"]
or round((config["warmup_ratio"] or 0.0) * total_steps),
"save_strategy": "no",
}
if max_steps > 0:
arguments["max_steps"] = max_steps
else:
arguments["num_train_epochs"] = epochs
if config["max_grad_norm"] is not None:
arguments["max_grad_norm"] = config["max_grad_norm"]
if eval_items:
if evaluation_enabled(config["eval_steps"]):
arguments.update(eval_strategy = "steps", eval_steps = float(config["eval_steps"]))
else:
arguments["eval_strategy"] = "epoch"
send("eval_configured")
report_to = []
if config.get("enable_tensorboard"):
from utils.paths import resolve_tensorboard_dir
report_to.append("tensorboard")
# transformers 5 ignores logging_dir; its TensorBoard callback reads this instead.
os.environ["TENSORBOARD_LOGGING_DIR"] = str(
resolve_tensorboard_dir(config.get("tensorboard_dir"))
)
if config["enable_wandb"]:
try:
import wandb
if config.get("wandb_token"):
os.environ["WANDB_API_KEY"] = config["wandb_token"]
wandb.init(project = config.get("wandb_project") or "unsloth-training")
report_to.append("wandb")
except Exception as exc:
warn(f"Weights & Biases logging is off: {exc}")
# On MLX the run's own tracking callback reports instead, into the directory and W&B run opened above.
arguments["report_to"] = "none" if _is_mlx() or not report_to else report_to
base_metrics = None
if eval_items:
status(
f"Evaluating the base model on {_decision_count(eval_items):,} held-out decisions..."
)
base_metrics = FastDecisionModel.evaluate(
model, tokenizer, eval_items, batch_size = config["batch_size"]
)
logger.info("Base held-out metrics: %s", base_metrics)
check_stop()
_emit_output_dir(event_queue, output_dir)
trainer = DecisionTrainer(
model = model,
args = TrainingArguments(**arguments),
train_dataset = items,
eval_dataset = eval_items or None,
processing_class = tokenizer,
# A fresh head starts from random weights, so it learns faster than a trained Clef head.
head_learning_rate = FRESH_HEAD_LEARNING_RATE if llm else None,
callbacks = [
_create_embedding_progress_callback(
event_queue,
total_steps = total_steps,
training_start_time = time.time(),
should_stop = lambda: stop["requested"],
)
]
+ ([_mlx_tracking_callback(report_to)] if report_to and _is_mlx() else []),
)
_drop_hf_stdout_callbacks(trainer)
start = time.time()
trainer.train()
if stop["requested"] and not stop["save"]:
raise _Stopped("Training cancelled")
tuned_metrics = None
if eval_items:
status("Calibrating confidence...")
tuned_metrics = FastDecisionModel.calibrate(
model, tokenizer, eval_items, batch_size = config["batch_size"]
)
logger.info("Fine-tuned held-out metrics: %s", tuned_metrics)
# A cancel during calibration still saves nothing; a stop with save keeps the model.
if stop["requested"] and not stop["save"]:
raise _Stopped("Training cancelled")
status("Saving model...")
peak_gb = _peak_memory_gb()
model.decision_config["training"] = {
"base": model_name,
"subfolder": subfolder,
"decision_head": f"{head} (new)" if llm else None,
"method": ("qlora" if clef and config.get("load_in_4bit") else "lora")
if use_lora
else "full",
"objective": "soft_cross_entropy",
"dataset": [Path(path).name for path in config.get("local_datasets") or []]
or config.get("hf_dataset")
or None,
"steps": trainer.state.global_step,
"epochs": round(trainer.state.epoch or 0, 2),
"heldout_decisions": _decision_count(eval_items),
"heldout_accuracy_base": base_metrics and round(base_metrics["accuracy"], 4),
"heldout_accuracy": tuned_metrics and round(tuned_metrics["accuracy"], 4),
"heldout_ece_base": base_metrics and round(base_metrics["ece"], 4),
"heldout_ece": tuned_metrics and round(tuned_metrics["ece"], 4),
"seconds": round(time.time() - start, 1),
"peak_memory_gb": peak_gb and round(peak_gb, 2),
"date": datetime.now(timezone.utc).isoformat(timespec = "seconds"),
}
model.save_pretrained_merged(output_dir, tokenizer)
logger.info("Decision model saved to %s: %s", output_dir, model.decision_config["training"])
message = "Decision training completed"
if base_metrics and tuned_metrics and _decision_count(eval_items) >= MIN_REPORTED_ITEMS:
message = (
f"Held-out accuracy {base_metrics['accuracy']:.2f} -> {tuned_metrics['accuracy']:.2f}, "
f"calibration error {tuned_metrics['ece']:.2f}"
)
send("complete", output_dir = output_dir, status_message = message)