Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: cache checkpoint losses per unchanged trainer_state and scan checkpoints off the event loop (#16320)
Daniel Hanfa19404e6c4f
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.pyindex c41bafdd976..ebb52794eaf 100644--- a/studio/backend/routes/models.py+++ b/studio/backend/routes/models.py@@ -5624,7 +5624,10 @@ async def list_checkpoints(outputs_dir = account_access.private_directory(outputs_dir, "outputs")try:resolved_outputs_dir = str(resolve_output_dir(outputs_dir))- raw_models = scan_checkpoints(outputs_dir = resolved_outputs_dir, include_decision = True)+ # Reads every checkpoint's trainer_state.json: keep it off the event loop.+ raw_models = await asyncio.to_thread(+ scan_checkpoints, outputs_dir = resolved_outputs_dir, include_decision = True+ )models = [ModelCheckpoints(diff --git a/studio/backend/routes/preview.py b/studio/backend/routes/preview.pyindex e8527c6e8e9..70c55b1a626 100644--- a/studio/backend/routes/preview.py+++ b/studio/backend/routes/preview.py@@ -211,7 +211,7 @@ async def list_previews(base = str(request.base_url)sharing_on = get_preview_sharing_enabled() and not no_credentialpreviews = []- for target in list_preview_targets():+ for target in await asyncio.to_thread(list_preview_targets):ref = quote(target["ref"], safe = "/")token = sign_preview_ref(target["ref"]) if sharing_on else Nonepreviews.append(diff --git a/studio/backend/tests/test_checkpoints_scan.py b/studio/backend/tests/test_checkpoints_scan.pyindex bb7156271de..13059a28240 100644--- a/studio/backend/tests/test_checkpoints_scan.py+++ b/studio/backend/tests/test_checkpoints_scan.py@@ -2,11 +2,15 @@# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import json+import osimport sqlite3import sys+import timeimport types as _typesfrom pathlib import Path+import pytest+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)if _BACKEND_DIR not in sys.path:sys.path.insert(0, _BACKEND_DIR)@@ -483,3 +487,69 @@ def test_scan_checkpoints_tied_exact_match_follows_sqlite_order(tmp_path, monkeymodels = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))assert models[0][2]["base_model"] == want+++def _write_loss_state(path: Path, loss: str) -> None:+ path.write_text('{"log_history": [{"loss": ' + loss + ', "step": 10}]}')+++def _counting_json_load(monkeypatch) -> list:+ calls = []+ real_load = checkpoints_module.json.load++ def load(f, *args, **kwargs):+ calls.append(f.name)+ return real_load(f, *args, **kwargs)++ monkeypatch.setattr(checkpoints_module.json, "load", load)+ return calls+++def _skip_settle_window(monkeypatch) -> None:+ real_time_ns = time.time_ns+ monkeypatch.setattr(time, "time_ns", lambda: real_time_ns() + 3600 * 10**9)+++def test_checkpoint_loss_cache_skips_reparse_of_unchanged_file(tmp_path, monkeypatch):+ _skip_settle_window(monkeypatch)+ checkpoint_dir = tmp_path / "checkpoint-10"+ checkpoint_dir.mkdir()+ _write_loss_state(checkpoint_dir / "trainer_state.json", "1.25")+ calls = _counting_json_load(monkeypatch)++ assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 1.25+ assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 1.25+ assert len(calls) == 1+++ sys.platform == "win32",+ reason = "st_ctime is the creation time on Windows, so it does not move on a rewrite",+)+def test_checkpoint_loss_cache_sees_same_size_rewrite_with_restored_mtime(tmp_path, monkeypatch):+ _skip_settle_window(monkeypatch)+ checkpoint_dir = tmp_path / "checkpoint-10"+ checkpoint_dir.mkdir()+ state = checkpoint_dir / "trainer_state.json"+ _write_loss_state(state, "1.25")+ assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 1.25++ before = state.stat()+ with open(state, "r+") as f:+ f.write('{"log_history": [{"loss": 2.75, "step": 10}]}')+ os.utime(state, ns = (before.st_atime_ns, before.st_mtime_ns))+ assert state.stat().st_size == before.st_size++ assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 2.75+++def test_checkpoint_loss_cache_ignores_recently_written_files(tmp_path, monkeypatch):+ checkpoint_dir = tmp_path / "checkpoint-10"+ checkpoint_dir.mkdir()+ _write_loss_state(checkpoint_dir / "trainer_state.json", "1.25")+ calls = _counting_json_load(monkeypatch)++ checkpoints_module._read_checkpoint_loss(checkpoint_dir)+ checkpoints_module._read_checkpoint_loss(checkpoint_dir)++ assert len(calls) == 2diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.pyindex add1f19a898..8f325eac68d 100644--- a/studio/backend/utils/models/checkpoints.py+++ b/studio/backend/utils/models/checkpoints.py@@ -8,7 +8,10 @@ from __future__ import annotationsimport jsonimport osimport re+import threading+import timeimport structlog+from collections import OrderedDictfrom loggers import get_loggerfrom pathlib import Pathfrom typing import Dict, List, Optional, Tuple@@ -138,19 +141,63 @@ def _lookup_base_model(checkpoint_dir: Path, history: _History) -> Optional[str]return None+# trainer_state.json reaches MBs on long runs. Files changed within the settle window are not+# cached: a coarse filesystem clock can give a same-size rewrite identical timestamps.+_LOSS_CACHE: "OrderedDict[str, tuple[tuple, Optional[float]]]" = OrderedDict()+_LOSS_CACHE_MAX = 4096+_LOSS_CACHE_SETTLE_NS = 2_000_000_000+_LOSS_CACHE_LOCK = threading.Lock()+++def _stat_key(st: os.stat_result) -> tuple:+ return (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns)+++def _cached_loss(cache_id: str, key: tuple):+ with _LOSS_CACHE_LOCK:+ hit = _LOSS_CACHE.get(cache_id)+ if hit is None or hit[0] != key:+ return False, None+ _LOSS_CACHE.move_to_end(cache_id)+ return True, hit[1]+++def _store_loss(cache_id: str, key: tuple, loss) -> None:+ if not (loss is None or type(loss) in (int, float)):+ return+ if time.time_ns() - max(key[3], key[4]) <= _LOSS_CACHE_SETTLE_NS:+ return+ with _LOSS_CACHE_LOCK:+ _LOSS_CACHE[cache_id] = (key, loss)+ _LOSS_CACHE.move_to_end(cache_id)+ while len(_LOSS_CACHE) > _LOSS_CACHE_MAX:+ _LOSS_CACHE.popitem(last = False)++def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:"""Read the latest training loss from trainer_state.json's log_history, or None."""trainer_state = checkpoint_path / "trainer_state.json"if not own_entry(trainer_state):return None+ cache_id = str(trainer_state)try:with open(trainer_state, encoding = "utf-8-sig") as f:+ key = _stat_key(os.fstat(f.fileno()))+ hit, loss = _cached_loss(cache_id, key)+ if hit:+ return lossstate = json.load(f)- # Not just the last entry: on a step that also evaluates, the Trainer logs the- # eval metrics after the training loss and before it saves the checkpoint.- for entry in reversed(state.get("log_history") or []):- if isinstance(entry, dict) and "loss" in entry:- return entry["loss"]+ loss = None+ # Not just the last entry: on a step that also evaluates, the Trainer logs the+ # eval metrics after the training loss and before it saves the checkpoint.+ for entry in reversed(state.get("log_history") or []):+ if isinstance(entry, dict) and "loss" in entry:+ loss = entry["loss"]+ break+ # A write during the read changes the stat, so that read is not cached.+ if _stat_key(os.fstat(f.fileno())) == key:+ _store_loss(cache_id, key, loss)+ return lossexcept Exception as e:logger.debug(f"Could not read loss from {trainer_state}: {e}")return None