Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Diffusion LoRA job service: runs the trainer subprocess, kept apart from the LLM TrainingBackend."""
from __future__ import annotations
from core.inference.inference_gate import pauses_background_embedding
from core.training.account_jobs import (
account_is_retired,
init_job_owner,
job_busy,
job_control,
job_pump,
job_read,
owned_job,
validate_job_paths,
worker_alive,
)
from utils.account_context import account_thread
from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memory
import contextlib
import json
import math
import multiprocessing as mp
import re
import threading
import time
import uuid
from pathlib import Path
from typing import Any, Callable, Optional
from utils.paths.path_utils import drop_appledouble_metadata
from hub.services import training_downloads
# Spawn (not fork): a fresh interpreter so parent CUDA/torch state never leaks into the trainer.
_CTX = mp.get_context("spawn")
# Terminal event types after which the pump stops.
_TERMINAL = ("complete", "error")
def _finite_or_none(value: Any) -> Optional[float]:
"""NaN and Infinity are invalid in strict JSON, so non-finite values become None here."""
if value is None:
return None
try:
f = float(value)
except (TypeError, ValueError):
return None
return f if math.isfinite(f) else None
class _DownloadPhaseQueue:
"""Publish the real child phase before its status message reaches the parent."""
def __init__(self, queue, phase):
self.queue = queue
self.phase = phase
def put(self, event):
kind = event.get("type")
if kind == "model_load_started" or kind in (
"model_load_completed",
"preparing",
"progress",
"complete",
"error",
):
flag, lock = self.phase
with lock:
if kind == "model_load_started":
flag.set()
else:
flag.clear()
self.queue.put(event)
def _run_diffusion_child(
*,
event_queue: Any,
stop_queue: Any,
config: dict,
download_phase = None,
) -> None:
# Fresh spawned interpreter: re-apply network injections inside the secret scrub and before the trainer imports diffusers.
from utils.native_tls import activate_native_tls
from utils.happy_eyeballs import activate_happy_eyeballs
activate_native_tls()
activate_happy_eyeballs()
# Restore only Unsloth's own quiet default before Hub caches it at import.
try:
from loggers.config import keep_progress_bars_countable, quiet_third_party_progress_bars
quiet_third_party_progress_bars()
keep_progress_bars_countable()
except Exception: # noqa: BLE001 -- progress setup must never stop training
pass
from .diffusion_lora_trainer import run_diffusion_training_process
run_diffusion_training_process(
event_queue = _DownloadPhaseQueue(event_queue, download_phase)
if download_phase
else event_queue,
stop_queue = stop_queue,
config = config,
)
def _default_target(
*,
event_queue: Any,
stop_queue: Any,
config: dict,
unsloth_stderr_mirror_path: Optional[str] = None,
download_phase = None,
) -> None:
# First in the child, before torch: bind to parent death and scrub the native path secret; token policy precedes the returning account branch.
if unsloth_stderr_mirror_path:
# Before the token setup below, so its failure is captured too.
try:
from utils.worker_stderr import install_worker_stderr_mirror
install_worker_stderr_mirror(unsloth_stderr_mirror_path)
except Exception:
pass
if not config.get("allow_ambient", True):
# Before any huggingface_hub import: the child env inherits the parent's token.
import os
from hub.utils.hf_tokens import apply_token_to_child_env, hf_token_arg, is_anonymous
hf_token = hf_token_arg(config.get("hf_token"), allow_ambient_token = False)
apply_token_to_child_env(os.environ, hf_token)
if is_anonymous(hf_token):
os.environ["HF_TOKEN_PATH"] = os.devnull
account = config.pop("_job_account", None)
if account is not None:
from core.training.account_jobs import run_account_child
from utils.native_path_leases import run_without_native_path_secret
run_without_native_path_secret(
run_account_child,
account = account,
job_module = "core.training.diffusion_training_service",
job_target = "_run_diffusion_child",
event_queue = event_queue,
stop_queue = stop_queue,
config = config,
**({"download_phase": download_phase} if download_phase is not None else {}),
)
return
from utils.native_path_leases import run_without_native_path_secret
run_without_native_path_secret(
_run_diffusion_child,
event_queue = event_queue,
stop_queue = stop_queue,
config = config,
**({"download_phase": download_phase} if download_phase is not None else {}),
)
# Cap on retained metric points; past it the arrays are decimated so a long run stays bounded.
_METRIC_CAP = 4000
class TrainingActiveError(RuntimeError):
"""A dataset mutation was refused because a run owns the dataset (route: 409)."""
class DatasetMutationInFlight(RuntimeError):
"""A start was refused because a dataset mutation is open (route: 409)."""
def _llm_training_active() -> bool:
"""Whether the LLM trainer holds the GPU. Best-effort: an import/health failure must not
block a diffusion start, exactly as the route's own reciprocal check fails open."""
try:
from core.training import get_training_backend
return bool(get_training_backend().is_training_active())
except Exception: # noqa: BLE001
return False
# One JSON file per terminal run, not the LLM sqlite, so diffusion runs stay off the LLM Runs page.
def _runs_dir() -> Path:
from utils.paths.storage_roots import tensorboard_root
d = tensorboard_root() / "diffusion"
d.mkdir(parents = True, exist_ok = True)
return d
def _resume_fields(
output_dir: Optional[str],
*,
status: Optional[str] = None,
started_at: Optional[float] = None,
ended_at: Optional[float] = None,
total_steps: Optional[int] = None,
write_error: Optional[str] = None,
source_checkpoint: Optional[str] = None,
source_created_at: Optional[float] = None,
) -> dict[str, Any]:
"""Derived from disk; write_error is sticky and blocks resume; started_at and ended_at fence off bundles from other runs. Never raises."""
try:
from core.training.diffusion_checkpoint import describe_resume_state
state = describe_resume_state(
output_dir,
status = status,
started_at = started_at,
ended_at = ended_at,
total_steps = total_steps,
source_checkpoint = source_checkpoint,
source_created_at = source_created_at,
)
except Exception: # noqa: BLE001 -- history must never break on a checkpoint scan
state = {}
if write_error:
return {
"can_resume": False,
"checkpoint_step": state.get("checkpoint_step"),
"checkpoint_path": None,
"resume_blocked_reason": write_error,
}
can_resume = bool(state.get("can_resume"))
return {
"can_resume": can_resume,
"checkpoint_step": state.get("checkpoint_step"),
# The EXACT bundle this run would continue, not the newest in a folder two runs share.
"checkpoint_path": state.get("checkpoint_path") if can_resume else None,
"resume_blocked_reason": state.get("resume_blocked_reason"),
}
def _refresh_resume_state(rec: dict) -> dict:
"""Backfills output_dir for old records, else can_resume would enable a resume the client refuses."""
config = rec.get("config")
output_dir = rec.get("output_dir") or (
config.get("output_dir") if isinstance(config, dict) else None
)
if output_dir and not rec.get("output_dir"):
rec["output_dir"] = str(output_dir)
rec.update(
_resume_fields(
output_dir,
status = rec.get("status"),
started_at = rec.get("started_at"),
ended_at = rec.get("ended_at"),
# Without total_steps, the manifest's older target would block Resume.
total_steps = _resolved_total_steps(rec, config if isinstance(config, dict) else {}),
write_error = rec.get("checkpoint_write_error"),
# Source bundle this run resumed from, so a run that died before its first save can still offer it.
source_checkpoint = (
config.get("resume_from_checkpoint") if isinstance(config, dict) else None
),
# Identity of that source bundle; a slot another run has rewritten is not offered as lineage.
source_created_at = rec.get("resumed_source_created_at"),
)
)
return rec
def list_diffusion_runs(limit: int = 20) -> list[dict]:
"""Summaries of persisted diffusion runs, newest first. The heavy per-run payload
(metric logs, config) stays in the file; fetch it via ``get_diffusion_run``."""
try:
# Sliced by limit below, so a companion per run would halve the history window.
files = drop_appledouble_metadata(
sorted(_runs_dir().glob("*.json"), key = lambda p: p.stat().st_mtime, reverse = True)
)
except Exception: # noqa: BLE001 -- unreadable dir -> no history
return []
out: list[dict] = []
for p in files[: max(0, int(limit))]:
try:
rec = json.loads(p.read_text(encoding = "utf-8"))
except Exception: # noqa: BLE001 -- a corrupt record never breaks the listing
continue
# Skip wrong-shape records: one bad file must not break DiffusionTrainingRunSummary(**r) or panel.
if not isinstance(rec, dict):
continue
if not (isinstance(rec.get("job_id"), str) and isinstance(rec.get("status"), str)):
continue
# Resume state read from disk first; per record, since one nonnumeric total_steps broke the endpoint.
try:
_refresh_resume_state(rec)
_restate_live_job(rec)
except Exception: # noqa: BLE001 -- a record we cannot refresh is still a record
continue
rec.pop("metric_history", None)
rec.pop("config", None)
rec.pop("samples", None)
out.append(rec)
return out
def get_diffusion_run(job_id: str) -> Optional[dict]:
"""The full persisted record for one run (summary + config + metric logs)."""
# Reject non-uuid4-hex ids so a crafted id cannot traverse out of the dir.
if not re.fullmatch(r"[0-9a-f]{32}", str(job_id or "")):
return None
p = _runs_dir() / f"{job_id}.json"
try:
rec = json.loads(p.read_text(encoding = "utf-8"))
except Exception: # noqa: BLE001 -- missing/corrupt record
return None
if not isinstance(rec, dict):
return rec
try:
_refresh_resume_state(rec)
_restate_live_job(rec)
except Exception: # noqa: BLE001 -- the stored record beats a 500
return rec
return rec
def _restate_live_job(rec: dict) -> dict:
"""A live job reports running, not the interim interrupted state, so Resume is not offered."""
global _service
service = _service
if service is None:
return rec
try:
live = service.status()
except Exception: # noqa: BLE001 -- a status we cannot read leaves the record as written
return rec
if not isinstance(live, dict) or not live.get("job_id"):
return rec
if rec.get("job_id") == live.get("job_id") and live.get("status") == "running":
rec["status"] = "running"
rec["message"] = live.get("message") or ""
# _UNRESUMABLE_STATUS rejects a running job because its output directory is being written right now,
# so a Resume there can only race the live run and 409.
rec["can_resume"] = False
rec["checkpoint_path"] = None
return rec
def _idle_state() -> dict[str, Any]:
return {
"active": False,
"job_id": None,
"status": "idle",
"message": "",
"step": 0,
"total_steps": 0,
"loss": None,
"avg_loss": None,
"learning_rate": None,
"grad_norm": None,
# The last per-modality losses of a joint run; None on families that train one modality.
"video_loss": None,
"audio_loss": None,
"num_images": None,
"in_model_load": False,
"preparation": None,
"model_load_stage": None,
"model_load_progress": None,
"download_progress": None,
"output_dir": None,
"lora_path": None,
# The optional second (EMA-averaged) adapter, when ema_decay was enabled.
"ema_path": None,
"catalog_path": None,
"family": None,
"base_model": None,
"samples_per_second": None,
"peak_memory_gb": None,
# Newest checkpoint-<N> path and step; why it failed to save (Resume tooltip); resumed-from step.
"checkpoint_path": None,
"checkpoint_step": None,
"resume_blocked_reason": None,
"resumed_from_step": None,
# A path is not identity: another run can overwrite checkpoint-<N>, so the fallback lineage is wrong.
"resumed_source_created_at": None,
"started_at": None,
"updated_at": None,
# Bounded, paired history arrays for the live loss chart (see _append_metric).
"metric_steps": [],
"metric_loss": [],
"metric_lr": [],
"metric_grad_norm": [],
# Null for other families and H3 steps with only a combined loss; series stay aligned.
"metric_video_loss": [],
"metric_audio_loss": [],
"samples": [],
}
# Past this the oldest half after the step-0 baseline is thinned, like the metrics.
_SAMPLES_CAP = 400
def _append_samples(
state: dict[str, Any],
step: Any,
images: Any,
output_dir: Optional[str] = None,
) -> None:
"""Fold one ``sample`` event; thinned rounds' images are deleted so every file on disk stays listed."""
from core.training.diffusion_samples import SAMPLE_PATH_RE, delete_sample_files
try:
istep = int(step)
except (TypeError, ValueError):
return
entries = state.setdefault("samples", [])
for img in images if isinstance(images, list) else []:
if not isinstance(img, dict) or not SAMPLE_PATH_RE.match(str(img.get("path") or "")):
continue
entries.append(
{
"step": istep,
"path": str(img["path"]),
"prompt": str(img.get("prompt") or ""),
"seed": img.get("seed") if isinstance(img.get("seed"), int) else None,
}
)
if len(entries) > _SAMPLES_CAP:
steps = sorted({e["step"] for e in entries})
keep = set(steps[:1] + steps[1:-1][::2] + steps[-1:])
state["samples"] = [e for e in entries if e["step"] in keep]
delete_sample_files(output_dir, [e["path"] for e in entries if e["step"] not in keep])
# Values paired index-for-index with metric_steps; one list so no series is forgotten in decimation.
_METRIC_SERIES: tuple[str, ...] = (
"metric_loss",
"metric_lr",
"metric_grad_norm",
"metric_video_loss",
"metric_audio_loss",
)
def _append_metric(
state: dict[str, Any],
step: Any,
loss: Any,
lr: Any,
grad_norm: Any = None,
video_loss: Any = None,
audio_loss: Any = None,
) -> None:
"""Only finite, positive-step points are kept; at the cap, every other point is dropped."""
try:
istep = int(step)
except (TypeError, ValueError):
return
if istep <= 0 or loss is None:
return
floss = _finite_or_none(loss)
if floss is None: # non-numeric or non-finite: skip, keep the curve JSON-safe
return
# Non-finite is nulled rather than dropped, to stay index-aligned.
values = {
"metric_loss": floss,
"metric_lr": _finite_or_none(lr),
"metric_grad_norm": _finite_or_none(grad_norm),
"metric_video_loss": _finite_or_none(video_loss),
"metric_audio_loss": _finite_or_none(audio_loss),
}
steps = state["metric_steps"]
if len(steps) >= _METRIC_CAP:
state["metric_steps"] = steps[::2]
for key in _METRIC_SERIES:
state[key] = (state.get(key) or [])[::2]
steps = state["metric_steps"]
steps.append(istep)
for key in _METRIC_SERIES:
# setdefault: older records may lack newer series.
state.setdefault(key, []).append(values[key])
def _resolved_total_steps(state: dict[str, Any], cfg: dict[str, Any]) -> int:
"""In epoch mode train_steps is a stale request default, so return 0 and let the manifest answer."""
live = int(state.get("total_steps") or 0)
if live:
return live
if int(cfg.get("num_epochs") or 0) > 0:
return 0
return int(cfg.get("train_steps") or 0)
class DiffusionTrainingService:
"""One diffusion LoRA training job at a time, spawned as a subprocess."""
def __init__(
self,
*,
ctx: Any = None,
target: Optional[Callable[..., None]] = None,
) -> None:
init_job_owner(
self,
lambda: worker_alive(self, pump = "_pump") or self.is_active(),
lambda: self.stop(save = False),
self._clear_account_result,
)
self._ctx = ctx if ctx is not None else _CTX
self._target = target if target is not None else _default_target
self._stderr_capture = None
self._lock = threading.Lock()
# Set by reserve() while a start is in flight (before the route frees GPU models) so the load guards
# refuse a concurrent load. Cleared by unreserve().
self._reserved = False
# Two-sided under _lock: starts and dataset mutations exclude each other.
self._dataset_mutations = 0
# A child killed after a no-save stop never reports it; remember it here.
self._discard_requested = False
# Bundle paths this job wrote; a discard the child cannot finish removes only those, not older ones.
self._own_checkpoints: list[str] = []
# The child already cleaned up; a parent pathname cleanup could delete another run's bundle.
self._child_cleared_own = False
# Set by a checkpoint_saved event, cleared once the pump has written the record.
self._persist_interim = False
# Only the first stop reaches the child, so only the first may set the parent's disposition.
self._stop_signalled = False
self._download_key: Optional[str] = None
self._download_repo: Optional[str] = None
self._download_phase = None
# GPU load admissions in flight; same two-sided rule as dataset mutations.
self._gpu_admissions = 0
self._proc: Any = None
self._stop_queue: Any = None
self._pump: Optional[threading.Thread] = None
self._state: dict[str, Any] = _idle_state()
# The active job's start config, scrubbed of secrets, kept for the run record.
self._config: dict[str, Any] = {}
# ── lifecycle ────────────────────────────────────────────────────────────
def _clear_account_result(self):
self._state = _idle_state()
self._config = {}
def is_active(self) -> bool:
with self._lock:
if self._reserved:
return True
return self._proc is not None and self._proc.is_alive()
@owned_job()
def reserve(self) -> None:
"""Mark a diffusion-training start as in flight so the image/video load guards (which read
is_active) refuse a concurrent load BEFORE the route frees resident GPU models. Without this
the training becomes active only at start(), after the free, so an overlapping load passes
its guard, acquires the GPU, and both workloads allocate VRAM.
Compare-and-set: raise if a start is already reserved or a job is already running, so a
second overlapping /diffusion/start is rejected (409) BEFORE it frees GPU residents, instead
of both requests tearing down residents and racing to start(). Paired with unreserve() in a
finally by the reserving caller, so a failed start never leaves training active.
"""
with self._lock:
if self._reserved or (self._proc is not None and self._proc.is_alive()):
raise RuntimeError("A diffusion training job is already running.")
if self._dataset_mutations:
raise DatasetMutationInFlight(
"The training images are being changed right now. Wait for that to finish, "
"then start the run."
)
if self._gpu_admissions:
# Refuse: reserving now would free residents this load has not registered.
raise RuntimeError(
"A model is being loaded onto the GPU right now. Wait for that to finish, "
"then start the run."
)
# Checked under the same lock as the LLM trainer so one of the two always raises.
if _llm_training_active():
raise RuntimeError(
"An LLM training job is already running. "
"Stop it before starting diffusion (Images) training."
)
self._reserved = True
@job_control
def unreserve(self) -> None:
"""Clear the reservation set by reserve(). Only touches the reservation flag, never
_proc, so a live job stays active on success and a failed start is fully rolled back."""
with self._lock:
self._reserved = False
@contextlib.contextmanager
def dataset_mutation(self):
"""Registers under reserve()'s lock, closing the gap between an is_active() check and the
mutation."""
with self._lock:
if self._reserved or (self._proc is not None and self._proc.is_alive()):
raise TrainingActiveError(
"Training images cannot be changed while diffusion training is active. "
"Stop the run before uploading, importing, editing captions, or deleting images."
)
self._dataset_mutations += 1
try:
yield
finally:
with self._lock:
self._dataset_mutations = max(0, self._dataset_mutations - 1)
@contextlib.contextmanager
def gpu_load_admission(self):
"""Held only until the load registers, because a start preempts an in-flight load from then on."""
with self._lock:
if self._reserved or (self._proc is not None and self._proc.is_alive()):
raise TrainingActiveError(
"Diffusion training is running, so the GPU is in use. Stop the run before "
"loading a model."
)
self._gpu_admissions += 1
try:
yield
finally:
with self._lock:
self._gpu_admissions = max(0, self._gpu_admissions - 1)
@_invalidates_gpu_memory("diffusion training start")
@owned_job()
@pauses_background_embedding
def start(self, config: dict) -> str:
"""Validate ``config``, spawn the trainer, and start pumping its events.
Raises ValueError for an unusable config (before any spawn) and RuntimeError if a
job is already running. Returns the new job id."""
validate_job_paths(config)
# Validate before spawning; keep the normalised config, whose resolved family keys recipe overrides.
from .diffusion_lora_trainer import _config_from_dict
from .diffusion_train_common import train_recipe_overrides
normalized_cfg = _config_from_dict(config).normalized()
# Join outside the lock: the pump's final writes take it.
with self._lock:
if self._proc is not None and self._proc.is_alive():
raise RuntimeError("A diffusion training job is already running.")
pump = self._pump
if pump is not None and pump.is_alive():
pump.join(timeout = 5.0)
with self._lock:
# Re-check: another start() may have won the race while we joined.
if self._proc is not None and self._proc.is_alive():
raise RuntimeError("A diffusion training job is already running.")
job_id = uuid.uuid4().hex
self._discard_requested = False
self._own_checkpoints = []
self._child_cleared_own = False
self._stop_signalled = False
event_queue = self._ctx.Queue()
self._stop_queue = self._ctx.Queue()
if self.job_account is not None:
config = {**config, "_job_account": self.job_account}
child_kwargs = {
"event_queue": event_queue,
"stop_queue": self._stop_queue,
"config": config,
}
# Test targets take only the three job kwargs.
if self._target is _default_target:
self._download_phase = (self._ctx.Event(), self._ctx.Lock())
self._download_phase[0].set()
child_kwargs["download_phase"] = self._download_phase
self._open_worker_stderr_capture()
if self._stderr_capture is not None:
from utils.native_path_leases import STDERR_MIRROR_KWARG
child_kwargs[STDERR_MIRROR_KWARG] = self._stderr_capture.path
self._proc = self._ctx.Process(
target = self._target,
kwargs = child_kwargs,
daemon = True,
)
from .diffusion_train_common import training_fetch_repo
self._download_repo = (
training_fetch_repo(normalized_cfg)
or config.get("base_model")
or config.get("model_name")
)
self._download_key = training_downloads.claim(self, job_id, self._download_repo)
# Keep the lease secret out of the child's env, as other orchestrators do.
from utils.native_path_leases import native_path_secret_removed_for_child_start
try:
with native_path_secret_removed_for_child_start():
self._proc.start()
except BaseException:
training_downloads.settle(self._download_key)
raise
try:
from utils.process_lifetime import adopt_pid
adopt_pid(self._proc.pid)
except Exception: # noqa: BLE001 -- lifetime binding is best-effort
pass
now = time.time()
self._state = _idle_state()
self._state.update(
active = True,
job_id = job_id,
status = "running",
in_model_load = True,
message = "Starting diffusion LoRA training...",
base_model = config.get("base_model") or config.get("model_name"),
started_at = now,
updated_at = now,
)
# After the reset: records the source identity for a resume that dies during load.
self._seed_source_identity(config)
# Record the recipe the family loop actually runs.
self._config = {
k: v for k, v in dict(config).items() if k not in {"hf_token", "_job_account"}
}
# A resume's unset bucketing stays unset; the trainer adopts the bundle's.
if normalized_cfg.bucketing is not None:
self._config["bucketing"] = normalized_cfg.bucketing
self._config.update(train_recipe_overrides(normalized_cfg))
self._pump = account_thread(
target = self._pump_loop, args = (event_queue, self._proc), daemon = True
)
self._pump.start()
return job_id
@_invalidates_gpu_memory("diffusion training stop")
@job_control
def stop(
self,
save: bool = True,
*,
job_id: Optional[str] = None,
download_only: bool = False,
download_generation: Optional[int] = None,
) -> bool:
"""Request a clean stop: the trainer finishes the current step, then either saves
a partial adapter (``save=True``, the default) or discards the run (``save=False``,
matching the LLM trainer's cancel). Returns True if a stop was signalled, False if
nothing was running."""
with (
self._lock,
self._download_phase[1] if self._download_phase else contextlib.nullcontext(),
):
# Check under the signalling lock: never cancel a different run or its training.
if job_id is not None and self._state.get("job_id") != job_id:
return False
if download_generation is not None:
from hub.utils.download_registry import get_models_registry
if (
not self._download_key
or get_models_registry().current_generation(self._download_key)
!= download_generation
):
return False
if download_only and (
job_id is None
or not self._state.get("in_model_load")
or self._state.get("preparation") is not None
or self._state.get("status") != "running"
or (self._download_phase is not None and not self._download_phase[0].is_set())
):
return False
if self._proc is None or not self._proc.is_alive() or self._stop_queue is None:
return False
loading = self._state.get("in_model_load") and self._state.get("preparation") is None
if self._download_phase is not None:
loading = loading and self._download_phase[0].is_set()
if loading and hasattr(self._proc, "terminate"):
# Native Xet ignores the cooperative queue; no optimizer runs yet, so terminate the worker.
self._proc.terminate()
self._proc.join(timeout = 2.0)
if self._proc.is_alive() and hasattr(self._proc, "kill"):
self._proc.kill()
self._proc.join(timeout = 2.0)
if self._proc.is_alive():
raise RuntimeError(
"The training download worker has not stopped yet. Try stopping it again."
)
self._stop_signalled = True
self._discard_requested = not save
self._state.update(
active = False,
status = "stopped",
in_model_load = False,
preparation = None,
model_load_stage = None,
model_load_progress = None,
download_progress = None,
message = "Stopped while loading model files; training had not started.",
updated_at = time.time(),
)
training_downloads.settle(self._download_key, "cancelled")
return True
if self._stop_signalled:
# Child consumes only the FIRST signal; a later stop-without-saving must not set a parent discard, which would delete checkpoints the child kept.
return True
try:
# Bare True = older wire format; the dict form carries the no-save cancel flag.
self._stop_queue.put(True if save else {"save": False})
except Exception: # noqa: BLE001
return False
self._stop_signalled = True
if not save:
# Kept so a child dying before a discarded completion still blocks the resume.
self._discard_requested = True
self._state["message"] = (
"Stop requested; finishing the current step and saving a partial adapter..."
if save
else "Cancel requested; finishing the current step (no adapter will be saved)..."
)
self._state["updated_at"] = time.time()
return True
def stop_for_shutdown(self, timeout: float) -> bool:
from utils.account_context import run_as
with self._lock:
proc = self._proc
pump = self._pump
account = self._result_account
def settled():
# The pump writes the run record after the child exits, so wait for it too.
return (proc is None or not proc.is_alive()) and (pump is None or not pump.is_alive())
if settled():
return True
if proc is not None and proc.is_alive():
# The signal path runs as the owner, which job_control refuses for a managed account's run.
run_as(account, self.stop, save = True)
deadline = time.monotonic() + max(0.0, timeout)
while time.monotonic() < deadline:
if settled():
return True
time.sleep(0.25)
return False
@job_read(
lambda self: {
**_idle_state(),
"status": "busy" if job_busy(self) else "idle",
"message": "Busy" if job_busy(self) else "",
}
)
def status(self) -> dict[str, Any]:
with self._lock:
snap = dict(self._state)
# Keep ``active`` honest even if the process died between events.
snap["active"] = self._proc is not None and self._proc.is_alive()
return snap
def sample_source(self, job_id: str) -> Optional[tuple[Optional[str], list]]:
"""(output_dir, samples) for the live job ``job_id``, or None when it is not the live job or
belongs to another account (the run record answers for a finished one)."""
from core.training.account_jobs import job_is_foreign
if job_is_foreign(self):
return None
with self._lock:
if not job_id or self._state.get("job_id") != job_id:
return None
out = self._state.get("output_dir") or self._config.get("output_dir")
return (str(out) if out else None), list(self._state.get("samples") or [])
def _open_worker_stderr_capture(self) -> None:
previous = self._stderr_capture
self._stderr_capture = None
if previous is not None:
try:
previous.close()
except Exception:
pass
try:
from utils.worker_stderr import WorkerStderrCapture
self._stderr_capture = WorkerStderrCapture(prefix = "unsloth-diffusion-worker-")
except Exception:
self._stderr_capture = None
def _unexpected_exit_message(self, proc: Any) -> str:
from utils.worker_stderr import unexpected_exit_message
text = ""
capture = self._stderr_capture
if capture is not None:
try:
text = capture.text()
except Exception:
text = ""
return unexpected_exit_message(
getattr(proc, "pid", None),
getattr(proc, "exitcode", None),
text,
)
# ── event pump ───────────────────────────────────────────────────────────
@job_pump
def _pump_loop(self, event_queue: Any, proc: Any) -> None:
while True:
try:
ev = event_queue.get(timeout = 1.0)
except Exception: # noqa: BLE001 -- Empty (timeout) or a closed queue
if not proc.is_alive():
drained = False
while True:
try:
self._apply_event(event_queue.get_nowait(), proc = proc)
drained = True
except Exception: # noqa: BLE001
break
with self._lock:
if self._proc is not proc:
return # superseded by a newer job; don't touch its state
if self._state.get("status") not in ("completed", "stopped", "error"):
self._state.update(
active = False,
status = "error",
# A worker killed mid-load / mid-preparation sends no terminal event to clear these.
in_model_load = False,
preparation = None,
model_load_stage = None,
model_load_progress = None,
download_progress = None,
message = self._unexpected_exit_message(proc),
updated_at = time.time(),
)
discarding = self._discard_requested
training_downloads.settle(
self._download_key,
"cancelled"
if self._state.get("status") == "stopped"
else "error"
if self._state.get("status") == "error"
else "idle",
self._state.get("message"),
)
if discarding:
self._apply_discard_intent()
_ = drained
self._persist_run_record()
return
continue
self._apply_event(ev, proc = proc)
if self._persist_interim:
self._persist_interim = False
self._persist_run_record(interim = True)
if ev.get("type") in _TERMINAL:
# An exception raised on the current step is a terminal error and returns here rather than through
# the dead-process branch, so the discard has to be applied on this path too.
with self._lock:
discarding = self._discard_requested and self._proc is proc
child_cleared = self._child_cleared_own
if discarding:
self._apply_discard_intent(delete = not child_cleared)
self._persist_run_record()
return
def _seed_source_identity(self, config: dict[str, Any]) -> None:
"""Record which bundle the route accepted, before the child can report it itself."""
source = config.get("resume_from_checkpoint")
if not source:
return
try:
from core.training.diffusion_checkpoint import read_checkpoint
manifest = read_checkpoint(Path(str(source)).expanduser())
except Exception: # noqa: BLE001 -- an unreadable bundle simply has no identity here
manifest = None
if isinstance(manifest, dict):
self._state["resumed_source_created_at"] = manifest.get("created_at")
def _apply_discard_intent(self, *, delete: bool = True) -> None:
"""Keep bundles when the child already ran cleanup: its restore put back a predecessor's bundle."""
if account_is_retired():
delete = False
with self._lock:
own = list(self._own_checkpoints) if delete else []
samples = [e.get("path") for e in self._state.get("samples") or []] if delete else []
sample_root = self._state.get("output_dir") or self._config.get("output_dir")
self._state["resume_blocked_reason"] = (
"This run was stopped without saving, so it was discarded."
)
self._state["checkpoint_path"] = None
self._state["checkpoint_step"] = None
# Their files are deleted, so listing them would serve 404s.
self._state["samples"] = []
self._own_checkpoints = []
if samples:
try:
from core.training.diffusion_samples import discard_sample_paths
discard_sample_paths(sample_root, samples)
except Exception: # noqa: BLE001 -- cleanup must never break the terminal transition
pass
if not own:
return
try:
from core.training.diffusion_checkpoint import discard_named_checkpoints
discard_named_checkpoints(own)
except Exception: # noqa: BLE001 -- cleanup must never break the terminal transition
pass
def _persist_run_record(self, *, interim: bool = False) -> None:
"""Best-effort and never fatal. Interim mode records the run as interrupted so a checkpoint survives if Unsloth itself dies."""
if account_is_retired():
return
try:
with self._lock:
s = dict(self._state)
cfg = dict(self._config)
if not s.get("job_id"):
return
if interim:
s["status"] = "error"
s["message"] = "Unsloth exited while this run was training."
elif s.get("status") not in ("completed", "stopped", "error"):
return
adapter = s.get("output_dir") or cfg.get("output_dir")
record = {
"job_id": s.get("job_id"),
"status": s.get("status"),
"message": s.get("message") or "",
"family": s.get("family") or cfg.get("model_family"),
"base_model": s.get("base_model") or cfg.get("base_model"),
"adapter": Path(str(adapter)).name if adapter else None,
"instance_prompt": cfg.get("instance_prompt"),
"step": s.get("step") or 0,
"total_steps": s.get("total_steps") or 0,
"loss": s.get("loss"),
"avg_loss": s.get("avg_loss"),
"learning_rate": s.get("learning_rate"),
"grad_norm": s.get("grad_norm"),
"video_loss": s.get("video_loss"),
"audio_loss": s.get("audio_loss"),
"samples_per_second": s.get("samples_per_second"),
"peak_memory_gb": s.get("peak_memory_gb"),
"num_images": s.get("num_images"),
"started_at": s.get("started_at"),
"ended_at": s.get("updated_at"),
"lora_path": s.get("lora_path"),
"ema_path": s.get("ema_path"),
"catalog_path": s.get("catalog_path"),
"saved": bool(s.get("lora_path")),
# output_dir is recorded for Resume; can_resume and checkpoint_step re-derive from disk each time.
"output_dir": str(adapter) if adapter else None,
"resumed_from_job_id": cfg.get("resumed_from_job_id") or None,
"resumed_from_step": s.get("resumed_from_step"),
"resumed_source_created_at": s.get("resumed_source_created_at"),
"checkpoint_write_error": s.get("resume_blocked_reason") or None,
**_resume_fields(
str(adapter) if adapter else None,
status = s.get("status"),
started_at = s.get("started_at"),
# Use the resolved total; epoch mode leaves train_steps at the default 500.
total_steps = _resolved_total_steps(s, cfg),
write_error = s.get("resume_blocked_reason"),
source_checkpoint = cfg.get("resume_from_checkpoint"),
source_created_at = s.get("resumed_source_created_at"),
),
"config": cfg,
"metric_history": {
"steps": s.get("metric_steps") or [],
"loss": s.get("metric_loss") or [],
"lr": s.get("metric_lr") or [],
"grad_norm": s.get("metric_grad_norm") or [],
"video_loss": s.get("metric_video_loss") or [],
"audio_loss": s.get("metric_audio_loss") or [],
},
"samples": s.get("samples") or [],
}
path = _runs_dir() / f"{s['job_id']}.json"
path.write_text(json.dumps(record), encoding = "utf-8")
except Exception: # noqa: BLE001 -- persisting history must never break the run
pass
def _apply_event(
self,
ev: dict[str, Any],
proc: Any = None,
) -> None:
"""Events from a superseded process are dropped so a stale pump cannot touch state."""
etype = ev.get("type")
with self._lock:
if proc is not None and self._proc is not proc:
return
if self._stop_signalled and self._state.get("status") == "stopped":
return
s = self._state
s["updated_at"] = time.time()
if etype == "model_load_started":
s.update(
in_model_load = True,
preparation = None,
model_load_stage = None,
model_load_progress = None,
download_progress = None,
status = "running",
message = "Loading base model...",
)
s["model_load_stage"] = (
ev.get("stage")
if ev.get("stage") in ("conditioning", "transformer", "quantizing")
else None
)
if ev.get("num_images") is not None:
s["num_images"] = ev.get("num_images")
elif etype == "model_load_progress":
done, total = ev.get("done"), ev.get("total")
if (
s.get("in_model_load")
and s.get("preparation") is None
and type(done) is int
and type(total) is int
and 0 <= done <= total
and total > 0
and ev.get("unit") in ("tensors", "components", "shards")
):
s["model_load_progress"] = {
"done": done,
"total": total,
"unit": ev["unit"],
"label": str(ev.get("label", "Model weights"))[:200],
}
elif etype == "model_download_progress":
if s.get("in_model_load") and s.get("preparation") is None:
progress = ev.get("download_progress")
progress = progress if isinstance(progress, dict) else {}
done = _finite_or_none(progress.get("bytes_done"))
total = _finite_or_none(progress.get("bytes_total"))
s["download_progress"] = (
{
"bytes_done": int(min(done, total)),
"bytes_total": int(total),
"label": str(progress.get("label", "Model files"))[:200],
"scope": "active_files",
}
if done is not None and done >= 0 and total is not None and total > 0
else None
)
elif etype == "model_load_completed":
s.update(
in_model_load = False,
preparation = None,
model_load_stage = None,
model_load_progress = None,
download_progress = None,
message = "Training...",
)
elif etype == "preparing":
s["model_load_stage"] = None
s["model_load_progress"] = None
# Long precompute such as the VAE latent cache, surfaced so the UI does not look stalled.
done, total = ev.get("done"), ev.get("total")
stage = str(ev.get("stage", "prepare")).replace("_", " ")
s.update(
status = "running",
in_model_load = True,
preparation = {"stage": ev.get("stage"), "done": done, "total": total},
download_progress = None,
message = (
f"Preparing ({stage} {done}/{total})..."
if done is not None and total is not None
else f"Preparing ({stage})..."
),
)
elif etype == "warning":
# Non-fatal trainer notes; keep training state, surface the text.
s["message"] = str(ev.get("message", "warning"))
elif etype == "resumed":
# This is the step resumed FROM, not a checkpoint this run wrote, so it does not touch checkpoint_step.
s.update(
resumed_from_step = ev.get("step"),
resumed_source_created_at = ev.get("source_created_at"),
message = f"Resuming from step {ev.get('step')}...",
)
# Seed live counters from the resume event, else they read 0/0 until the first post-resume progress.
if ev.get("step") is not None:
s["step"] = int(ev["step"])
if ev.get("total_steps") is not None:
s["total_steps"] = int(ev["total_steps"])
elif etype == "checkpoint_saved":
# Folded as each bundle lands, not only at the end, so a run that later crashes is still reported as
# resumable; a good write also clears an earlier failure.
s.update(
checkpoint_path = ev.get("checkpoint_path"),
checkpoint_step = ev.get("step"),
resume_blocked_reason = None,
)
written = ev.get("checkpoint_path")
if written and written not in self._own_checkpoints:
self._own_checkpoints.append(str(written))
# Persist now: only terminal events write the run JSON.
self._persist_interim = True
elif etype == "checkpoint_failed":
# Sticky: resuming from an older bundle would silently lose steps.
s["resume_blocked_reason"] = str(ev.get("message", "checkpoint write failed"))
# Persisted: in memory only, an exit left the record advertising an older checkpoint as resumable.
self._persist_interim = True
elif etype == "progress":
# Null any non-finite float so the JSON stays strict-parseable; a missing key keeps the last value.
loss = _finite_or_none(ev["loss"]) if "loss" in ev else s["loss"]
avg_loss = _finite_or_none(ev["avg_loss"]) if "avg_loss" in ev else s["avg_loss"]
learning_rate = (
_finite_or_none(ev["learning_rate"])
if "learning_rate" in ev
else s["learning_rate"]
)
grad_norm = (
_finite_or_none(ev["grad_norm"]) if "grad_norm" in ev else s["grad_norm"]
)
# Per-modality losses can reveal degradation the combined loss hides (H3).
video_loss = (
_finite_or_none(ev["video_loss"]) if "video_loss" in ev else s["video_loss"]
)
audio_loss = (
_finite_or_none(ev["audio_loss"]) if "audio_loss" in ev else s["audio_loss"]
)
s.update(
status = "running",
step = ev.get("step", s["step"]),
preparation = None,
model_load_stage = None,
model_load_progress = None,
download_progress = None,
in_model_load = False,
total_steps = ev.get("total_steps", s["total_steps"]),
loss = loss,
avg_loss = avg_loss,
learning_rate = learning_rate,
grad_norm = grad_norm,
video_loss = video_loss,
audio_loss = audio_loss,
message = "Training...",
)
# Fold optional perf fields so the UI shows throughput + peak VRAM.
if ev.get("samples_per_second") is not None:
s["samples_per_second"] = ev.get("samples_per_second")
if ev.get("peak_memory_gb") is not None:
s["peak_memory_gb"] = ev.get("peak_memory_gb")
# Retain a bounded per-step history for the live charts.
_append_metric(
s,
ev.get("step"),
ev.get("loss"),
ev.get("learning_rate"),
ev.get("grad_norm"),
ev.get("video_loss"),
ev.get("audio_loss"),
)
elif etype == "sample":
_append_samples(
s,
ev.get("step"),
ev.get("images"),
s.get("output_dir") or self._config.get("output_dir"),
)
elif etype == "complete":
# Reset in_model_load: a stop during load skips model_load_completed and would leave it stale.
s.update(
active = False,
in_model_load = False,
status = "stopped" if ev.get("stopped") else "completed",
preparation = None,
model_load_stage = None,
model_load_progress = None,
download_progress = None,
output_dir = ev.get("output_dir"),
lora_path = ev.get("lora_path"),
ema_path = ev.get("ema_path"),
message = (
"Stopped (partial adapter saved)."
if ev.get("lora_path")
else "Stopped (no adapter saved)."
)
if ev.get("stopped")
else "Training complete.",
)
if ev.get("catalog_path") is not None:
s["catalog_path"] = ev.get("catalog_path")
if ev.get("family") is not None:
s["family"] = ev.get("family")
if ev.get("base_model") is not None:
s["base_model"] = ev.get("base_model")
# Only the lineage is re-confirmed here, for a pump that missed the earlier "resumed" event.
if ev.get("resumed_from_step") is not None:
s["resumed_from_step"] = ev.get("resumed_from_step")
if ev.get("discarded"):
# The child already cleaned up; never repeat by pathname, since a slot this run overwrote now holds a bundle from another run.
self._child_cleared_own = True
s["resume_blocked_reason"] = (
"This run was stopped without saving, so it was discarded."
)
s["samples"] = []
elif etype == "error":
# Reset in_model_load too: an error during model loading has no model_load_completed.
s.update(
active = False,
in_model_load = False,
status = "error",
preparation = None,
model_load_stage = None,
model_load_progress = None,
download_progress = None,
message = str(ev.get("message", "error")),
)
if etype == "model_load_started":
try:
self._download_key = training_downloads.claim(
self,
s.get("job_id"),
ev.get("fetch_repo")
or self._download_repo
or s.get("base_model")
or self._config.get("base_model"),
)
except RuntimeError as exc:
s.update(status = "error", message = str(exc), in_model_load = False)
if self._proc is not None and hasattr(self._proc, "terminate"):
self._proc.terminate()
elif etype in ("preparing", "model_load_completed", "progress", "complete", "error"):
training_downloads.settle(
self._download_key,
"error"
if etype == "error"
else "cancelled"
if s.get("status") == "stopped"
else "idle",
s.get("message") if etype == "error" else None,
)
_service: Optional[DiffusionTrainingService] = None
_service_lock = threading.Lock()
def get_diffusion_training_service() -> DiffusionTrainingService:
"""Process-wide singleton used by the route layer."""
global _service
with _service_lock:
if _service is None:
_service = DiffusionTrainingService()
return _service