Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: show a checkpoint's training loss when that step also evaluated (#13879)
Mohammad Hijjawi299e905a1313
diff --git a/studio/backend/tests/test_checkpoints_scan.py b/studio/backend/tests/test_checkpoints_scan.pyindex 0742fa93258..331d05340ad 100644--- a/studio/backend/tests/test_checkpoints_scan.py+++ b/studio/backend/tests/test_checkpoints_scan.py@@ -367,3 +367,27 @@ def test_scan_checkpoints_keeps_root_adapter_first(tmp_path, monkeypatch):},)]+++def test_scan_checkpoints_reads_training_loss_past_an_eval_entry(tmp_path, monkeypatch):+ outputs_dir = _make_outputs_dir(tmp_path, monkeypatch)+ run_dir = outputs_dir / "run_eval"+ checkpoint_dir = run_dir / "checkpoint-50"+ checkpoint_dir.mkdir(parents = True)+ (checkpoint_dir / "adapter_config.json").write_text("{}")+ # A step that both evaluates and saves: the Trainer logs the eval metrics last.+ (checkpoint_dir / "trainer_state.json").write_text(+ json.dumps(+ {+ "log_history": [+ {"epoch": 0.25, "loss": 0.5, "step": 25},+ {"epoch": 0.5, "grad_norm": 1.5, "loss": 0.25, "step": 50},+ {"epoch": 0.5, "eval_loss": 0.75, "eval_runtime": 2.0, "step": 50},+ ]+ }+ )+ )++ models = checkpoints_module.scan_checkpoints(outputs_dir = str(outputs_dir))++ assert models[0][1] == [("checkpoint-50", str(checkpoint_dir), 0.25)]diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.pyindex 2b64be9d8b0..99db8c49a0a 100644--- a/studio/backend/utils/models/checkpoints.py+++ b/studio/backend/utils/models/checkpoints.py@@ -129,16 +129,18 @@ def _infer_base_model_from_history(checkpoint_dir: Path) -> Optional[str]:def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:- """Read loss from the last log_history entry of trainer_state.json, or None."""+ """Read the latest training loss from trainer_state.json's log_history, or None."""trainer_state = checkpoint_path / "trainer_state.json"if not own_entry(trainer_state):return Nonetry:with open(trainer_state, encoding = "utf-8-sig") as f:state = json.load(f)- log_history = state.get("log_history", [])- if log_history:- return log_history[-1].get("loss")+ # Not just the last entry: on a step that also evaluates, the Trainer logs the+ # eval metrics after the training loss and before it saves the checkpoint.+ for entry in reversed(state.get("log_history") or []):+ if isinstance(entry, dict) and "loss" in entry:+ return entry["loss"]except Exception as e:logger.debug(f"Could not read loss from {trainer_state}: {e}")return None