Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: show a checkpoint's training loss when that step also evaluated (#13879)

Mohammad Hijjawi299e905a1313
diff --git a/studio/backend/tests/test_checkpoints_scan.py b/studio/backend/tests/test_checkpoints_scan.py
index 0742fa93258..331d05340ad 100644
--- a/studio/backend/tests/test_checkpoints_scan.py
+++ b/studio/backend/tests/test_checkpoints_scan.py
@@ -367,3 +367,27 @@ def test_scan_checkpoints_keeps_root_adapter_first(tmp_path, monkeypatch):
},
)
]
+
+
+def test_scan_checkpoints_reads_training_loss_past_an_eval_entry(tmp_path, monkeypatch):
+ outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)
+ run_dir = outputs_dir / "run_eval"
+ checkpoint_dir = run_dir / "checkpoint-50"
+ checkpoint_dir.mkdir(parents = True)
+ (checkpoint_dir / "adapter_config.json").write_text("{}")
+ # A step that both evaluates and saves: the Trainer logs the eval metrics last.
+ (checkpoint_dir / "trainer_state.json").write_text(
+ json.dumps(
+ {
+ "log_history": [
+ {"epoch": 0.25, "loss": 0.5, "step": 25},
+ {"epoch": 0.5, "grad_norm": 1.5, "loss": 0.25, "step": 50},
+ {"epoch": 0.5, "eval_loss": 0.75, "eval_runtime": 2.0, "step": 50},
+ ]
+ }
+ )
+ )
+
+ models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))
+
+ assert models[0][1] == [("checkpoint-50", str(checkpoint_dir), 0.25)]
diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.py
index 2b64be9d8b0..99db8c49a0a 100644
--- a/studio/backend/utils/models/checkpoints.py
+++ b/studio/backend/utils/models/checkpoints.py
@@ -129,16 +129,18 @@ def _infer_base_model_from_history(checkpoint_dir: Path) -> Optional[str]:
def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:
- """Read loss from the last log_history entry of trainer_state.json, or None."""
+ """Read the latest training loss from trainer_state.json's log_history, or None."""
trainer_state = checkpoint_path / "trainer_state.json"
if not own_entry(trainer_state):
return None
try:
with open(trainer_state, encoding = "utf-8-sig") as f:
state = json.load(f)
- log_history = state.get("log_history", [])
- if log_history:
- return log_history[-1].get("loss")
+ # Not just the last entry: on a step that also evaluates, the Trainer logs the
+ # eval metrics after the training loss and before it saves the checkpoint.
+ for entry in reversed(state.get("log_history") or []):
+ if isinstance(entry, dict) and "loss" in entry:
+ return entry["loss"]
except Exception as e:
logger.debug(f"Could not read loss from {trainer_state}: {e}")
return None