Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: cache checkpoint losses per unchanged trainer_state and scan checkpoints off the event loop (#16320)

Daniel Hanfa19404e6c4f
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index c41bafdd976..ebb52794eaf 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -5624,7 +5624,10 @@ async def list_checkpoints(
outputs_dir = account_access.private_directory(outputs_dir, "outputs")
try:
resolved_outputs_dir = str(resolve_output_dir(outputs_dir))
- raw_models = scan_checkpoints(outputs_dir = resolved_outputs_dir, include_decision = True)
+ # Reads every checkpoint's trainer_state.json: keep it off the event loop.
+ raw_models = await asyncio.to_thread(
+ scan_checkpoints, outputs_dir = resolved_outputs_dir, include_decision = True
+ )
models = [
ModelCheckpoints(
diff --git a/studio/backend/routes/preview.py b/studio/backend/routes/preview.py
index e8527c6e8e9..70c55b1a626 100644
--- a/studio/backend/routes/preview.py
+++ b/studio/backend/routes/preview.py
@@ -211,7 +211,7 @@ async def list_previews(
base = str(request.base_url)
sharing_on = get_preview_sharing_enabled() and not no_credential
previews = []
- for target in list_preview_targets():
+ for target in await asyncio.to_thread(list_preview_targets):
ref = quote(target["ref"], safe = "/")
token = sign_preview_ref(target["ref"]) if sharing_on else None
previews.append(
diff --git a/studio/backend/tests/test_checkpoints_scan.py b/studio/backend/tests/test_checkpoints_scan.py
index bb7156271de..13059a28240 100644
--- a/studio/backend/tests/test_checkpoints_scan.py
+++ b/studio/backend/tests/test_checkpoints_scan.py
@@ -2,11 +2,15 @@
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import json
+import os
import sqlite3
import sys
+import time
import types as _types
from pathlib import Path
+import pytest
+
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
@@ -483,3 +487,69 @@ def test_scan_checkpoints_tied_exact_match_follows_sqlite_order(tmp_path, monkey
models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))
assert models[0][2]["base_model"] == want
+
+
+def _write_loss_state(path: Path, loss: str) -> None:
+ path.write_text('{"log_history": [{"loss": ' + loss + ', "step": 10}]}')
+
+
+def _counting_json_load(monkeypatch) -> list:
+ calls = []
+ real_load = checkpoints_module.json.load
+
+ def load(f, *args, **kwargs):
+ calls.append(f.name)
+ return real_load(f, *args, **kwargs)
+
+ monkeypatch.setattr(checkpoints_module.json, "load", load)
+ return calls
+
+
+def _skip_settle_window(monkeypatch) -> None:
+ real_time_ns = time.time_ns
+ monkeypatch.setattr(time, "time_ns", lambda: real_time_ns() + 3600 * 10**9)
+
+
+def test_checkpoint_loss_cache_skips_reparse_of_unchanged_file(tmp_path, monkeypatch):
+ _skip_settle_window(monkeypatch)
+ checkpoint_dir = tmp_path / "checkpoint-10"
+ checkpoint_dir.mkdir()
+ _write_loss_state(checkpoint_dir / "trainer_state.json", "1.25")
+ calls = _counting_json_load(monkeypatch)
+
+ assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 1.25
+ assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 1.25
+ assert len(calls) == 1
+
+
+ sys.platform == "win32",
+ reason = "st_ctime is the creation time on Windows, so it does not move on a rewrite",
+)
+def test_checkpoint_loss_cache_sees_same_size_rewrite_with_restored_mtime(tmp_path, monkeypatch):
+ _skip_settle_window(monkeypatch)
+ checkpoint_dir = tmp_path / "checkpoint-10"
+ checkpoint_dir.mkdir()
+ state = checkpoint_dir / "trainer_state.json"
+ _write_loss_state(state, "1.25")
+ assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 1.25
+
+ before = state.stat()
+ with open(state, "r+") as f:
+ f.write('{"log_history": [{"loss": 2.75, "step": 10}]}')
+ os.utime(state, ns = (before.st_atime_ns, before.st_mtime_ns))
+ assert state.stat().st_size == before.st_size
+
+ assert checkpoints_module._read_checkpoint_loss(checkpoint_dir) == 2.75
+
+
+def test_checkpoint_loss_cache_ignores_recently_written_files(tmp_path, monkeypatch):
+ checkpoint_dir = tmp_path / "checkpoint-10"
+ checkpoint_dir.mkdir()
+ _write_loss_state(checkpoint_dir / "trainer_state.json", "1.25")
+ calls = _counting_json_load(monkeypatch)
+
+ checkpoints_module._read_checkpoint_loss(checkpoint_dir)
+ checkpoints_module._read_checkpoint_loss(checkpoint_dir)
+
+ assert len(calls) == 2
diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.py
index add1f19a898..8f325eac68d 100644
--- a/studio/backend/utils/models/checkpoints.py
+++ b/studio/backend/utils/models/checkpoints.py
@@ -8,7 +8,10 @@ from __future__ import annotations
import json
import os
import re
+import threading
+import time
import structlog
+from collections import OrderedDict
from loggers import get_logger
from pathlib import Path
from typing import Dict, List, Optional, Tuple
@@ -138,19 +141,63 @@ def _lookup_base_model(checkpoint_dir: Path, history: _History) -> Optional[str]
return None
+# trainer_state.json reaches MBs on long runs. Files changed within the settle window are not
+# cached: a coarse filesystem clock can give a same-size rewrite identical timestamps.
+_LOSS_CACHE: "OrderedDict[str, tuple[tuple, Optional[float]]]" = OrderedDict()
+_LOSS_CACHE_MAX = 4096
+_LOSS_CACHE_SETTLE_NS = 2_000_000_000
+_LOSS_CACHE_LOCK = threading.Lock()
+
+
+def _stat_key(st: os.stat_result) -> tuple:
+ return (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns)
+
+
+def _cached_loss(cache_id: str, key: tuple):
+ with _LOSS_CACHE_LOCK:
+ hit = _LOSS_CACHE.get(cache_id)
+ if hit is None or hit[0] != key:
+ return False, None
+ _LOSS_CACHE.move_to_end(cache_id)
+ return True, hit[1]
+
+
+def _store_loss(cache_id: str, key: tuple, loss) -> None:
+ if not (loss is None or type(loss) in (int, float)):
+ return
+ if time.time_ns() - max(key[3], key[4]) <= _LOSS_CACHE_SETTLE_NS:
+ return
+ with _LOSS_CACHE_LOCK:
+ _LOSS_CACHE[cache_id] = (key, loss)
+ _LOSS_CACHE.move_to_end(cache_id)
+ while len(_LOSS_CACHE) > _LOSS_CACHE_MAX:
+ _LOSS_CACHE.popitem(last = False)
+
+
def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:
"""Read the latest training loss from trainer_state.json's log_history, or None."""
trainer_state = checkpoint_path / "trainer_state.json"
if not own_entry(trainer_state):
return None
+ cache_id = str(trainer_state)
try:
with open(trainer_state, encoding = "utf-8-sig") as f:
+ key = _stat_key(os.fstat(f.fileno()))
+ hit, loss = _cached_loss(cache_id, key)
+ if hit:
+ return loss
state = json.load(f)
- # Not just the last entry: on a step that also evaluates, the Trainer logs the
- # eval metrics after the training loss and before it saves the checkpoint.
- for entry in reversed(state.get("log_history") or []):
- if isinstance(entry, dict) and "loss" in entry:
- return entry["loss"]
+ loss = None
+ # Not just the last entry: on a step that also evaluates, the Trainer logs the
+ # eval metrics after the training loss and before it saves the checkpoint.
+ for entry in reversed(state.get("log_history") or []):
+ if isinstance(entry, dict) and "loss" in entry:
+ loss = entry["loss"]
+ break
+ # A write during the read changes the stat, so that read is not cached.
+ if _stat_key(os.fstat(f.fileno())) == key:
+ _store_loss(cache_id, key, loss)
+ return loss
except Exception as e:
logger.debug(f"Could not read loss from {trainer_state}: {e}")
return None