Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix resumed training eta in studio (#11297)
Nilay10d3dac1da9d
diff --git a/studio/backend/core/training/resume.py b/studio/backend/core/training/resume.pyindex 681b398d332..ba6caf51ad9 100644--- a/studio/backend/core/training/resume.py+++ b/studio/backend/core/training/resume.py@@ -94,6 +94,16 @@ def _valid_state_file(path: Path, require_tensor: bool = True) -> bool:return False+def session_eta_seconds(+ elapsed_seconds: Optional[float], step: int, start_step: int, total_steps: int+) -> Optional[float]:+ steps_done = step - start_step+ steps_remaining = total_steps - step+ if elapsed_seconds is None or steps_done <= 0 or steps_remaining <= 0:+ return None+ return (elapsed_seconds / steps_done) * steps_remaining++def _checkpoint_state(path: Path) -> Optional[int]:try:state = json.loads((path / "trainer_state.json").read_text(encoding = "utf-8"))diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex c1b6232b462..97e5f0d6fdb 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -88,6 +88,7 @@ from utils.paths import ()from trl import SFTTrainer, SFTConfig+from .resume import session_eta_secondsfrom .training import (TrainingProgress,apply_save_strategy,@@ -358,6 +359,7 @@ class UnslothTrainer:self.dataset_snapshot_path = Noneself.training_start_time: Optional[float] = None+ self.session_start_step: int = 0self.batch_size: Optional[int] = Noneself.max_seq_length: Optional[int] = Noneself.gradient_accumulation_steps: Optional[int] = None@@ -563,6 +565,9 @@ class UnslothTrainer:_eval_last_report = 0.0def on_train_begin(self, args, state, control, **kwargs):+ trainer_ref.session_start_step = state.global_step+ if state.global_step > 0:+ trainer_ref.training_start_time = time.time()# on_log reports an empty status, else the UI stays on "Starting training...".if trainer_ref.should_stop:return@@ -642,13 +647,12 @@ class UnslothTrainer:if trainer_ref.training_start_time is not None:elapsed_seconds = time.time() - trainer_ref.training_start_time- eta_seconds = None- if elapsed_seconds is not None and current_step > 0:- total_steps = trainer_ref.training_progress.total_steps- if total_steps > 0:- steps_remaining = total_steps - current_step- if steps_remaining > 0:- eta_seconds = (elapsed_seconds / current_step) * steps_remaining+ eta_seconds = session_eta_seconds(+ elapsed_seconds,+ current_step,+ trainer_ref.session_start_step,+ trainer_ref.training_progress.total_steps,+ )num_tokens = getattr(state, "num_input_tokens_seen", None)@@ -659,6 +663,7 @@ class UnslothTrainer:learning_rate = logs.get("learning_rate", None),elapsed_seconds = elapsed_seconds,eta_seconds = eta_seconds,+ session_start_step = trainer_ref.session_start_step,grad_norm = grad_norm,num_tokens = num_tokens,eval_loss = logs.get("eval_loss", None),diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex c19419c056f..15cba413722 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -653,6 +653,7 @@ class TrainingProgress:status_message: str = "Ready to train"elapsed_seconds: Optional[float] = Noneeta_seconds: Optional[float] = None+ session_start_step: int = 0grad_norm: Optional[float] = Nonenum_tokens: Optional[int] = Noneeval_loss: Optional[float] = None@@ -1051,6 +1052,10 @@ class _MLXTrainerAdapter:self.training_progress.elapsed_seconds,),eta_seconds = event.get("eta_seconds", self.training_progress.eta_seconds),+ session_start_step = event.get(+ "session_start_step",+ self.training_progress.session_start_step,+ ),grad_norm = event.get("grad_norm", self.training_progress.grad_norm),num_tokens = event.get("num_tokens", self.training_progress.num_tokens),eval_loss = event.get("eval_loss", self.training_progress.eval_loss),@@ -3064,6 +3069,9 @@ class TrainingBackend:self._progress.total_steps = event.get("total_steps", self._progress.total_steps)self._progress.elapsed_seconds = event.get("elapsed_seconds")self._progress.eta_seconds = event.get("eta_seconds")+ self._progress.session_start_step = event.get(+ "session_start_step", self._progress.session_start_step+ )self._progress.grad_norm = event.get("grad_norm")self._progress.num_tokens = event.get("num_tokens")self._progress.eval_loss = event.get("eval_loss")diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 6140934b0f2..2e4405c9ea1 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -60,6 +60,7 @@ from core.training.dataset_bounds import (row_bound_for_resume,world_size_from_env,)+from core.training.resume import _checkpoint_state, session_eta_secondsfrom utils.training_runs import build_default_output_dir_namefrom utils.wheel_utils import (direct_wheel_url,@@ -3079,6 +3080,10 @@ def _run_mlx_training(event_queue, stop_queue, config):_send("status", status_message = f"Training {model_name}...")+ start_step = 0+ if resume_from_checkpoint:+ start_step = _checkpoint_state(Path(resume_from_checkpoint)) or 0+def _on_step(step,total,@@ -3090,7 +3095,7 @@ def _run_mlx_training(event_queue, stop_queue, config):num_tokens,grad_norm = None,):- eta = (elapsed / step * (total - step)) if step > 0 else 0+ eta = session_eta_seconds(elapsed, step, start_step, total) or 0_send("progress",step = step,@@ -3099,7 +3104,8 @@ def _run_mlx_training(event_queue, stop_queue, config):learning_rate = lr,total_steps = total,elapsed_seconds = elapsed,- eta_seconds = max(0, eta),+ eta_seconds = eta,+ session_start_step = start_step,grad_norm = grad_norm,num_tokens = num_tokens,eval_loss = None,@@ -4715,6 +4721,7 @@ def _create_trainer_progress_callback(event_queue: Any) -> Callable[[TrainingPro"total_steps": progress.total_steps,"elapsed_seconds": progress.elapsed_seconds,"eta_seconds": progress.eta_seconds,+ "session_start_step": progress.session_start_step,"grad_norm": progress.grad_norm,"num_tokens": progress.num_tokens,"eval_loss": progress.eval_loss,@@ -4744,7 +4751,13 @@ def _create_embedding_progress_callback(from transformers import TrainerCallbackclass _EmbeddingProgressCallback(TrainerCallback):+ _start_step = 0+ _training_start_time = training_start_time+def on_train_begin(self, args, state, control, **kwargs):+ self._start_step = state.global_step+ if state.global_step > 0:+ self._training_start_time = time.time()# Progress events carry an empty status, else the parent keeps showing "Starting...".if should_stop():return@@ -4785,12 +4798,8 @@ def _create_embedding_progress_callback()current_step = state.global_step- elapsed = time.time() - training_start_time- eta = None- if current_step > 0 and total_steps > 0:- remaining = total_steps - current_step- if remaining > 0:- eta = (elapsed / current_step) * remaining+ elapsed = time.time() - self._training_start_time+ eta = session_eta_seconds(elapsed, current_step, self._start_step, total_steps)event_queue.put({@@ -4802,6 +4811,7 @@ def _create_embedding_progress_callback("total_steps": total_steps,"elapsed_seconds": elapsed,"eta_seconds": eta,+ "session_start_step": self._start_step,"grad_norm": logs.get("grad_norm"),"num_tokens": getattr(state, "num_input_tokens_seen", None),"eval_loss": logs.get("eval_loss"),diff --git a/studio/backend/models/training.py b/studio/backend/models/training.pyindex d2a1dc0b7da..e6c3e8202d1 100644--- a/studio/backend/models/training.py+++ b/studio/backend/models/training.py@@ -721,6 +721,9 @@ class TrainingProgress(BaseModel):None, description = "Time elapsed since training started")eta_seconds: Optional[float] = Field(None, description = "Estimated time remaining")+ session_start_step: Optional[int] = Field(+ None, description = "Step this session started from (non-zero on a resumed run)"+ )grad_norm: Optional[float] = Field(None, description = "L2 norm of gradients, computed before gradient clipping")diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.pyindex 9223cf654fb..6dcb39e493b 100644--- a/studio/backend/routes/training.py+++ b/studio/backend/routes/training.py@@ -2457,6 +2457,7 @@ async def stream_training_progress(elapsed_seconds = getattr(progress, "elapsed_seconds", None) if progress else Noneeta_seconds = getattr(progress, "eta_seconds", None) if progress else None+ session_start_step = getattr(progress, "session_start_step", None) if progress else Nonegrad_norm = grad_norm_overrideif grad_norm is None and progress:grad_norm = getattr(progress, "grad_norm", None)@@ -2475,6 +2476,7 @@ async def stream_training_progress(epoch = epoch,elapsed_seconds = elapsed_seconds,eta_seconds = eta_seconds,+ session_start_step = session_start_step,grad_norm = grad_norm,num_tokens = num_tokens,eval_loss = eval_loss,diff --git a/studio/backend/tests/test_final_loss_not_average.py b/studio/backend/tests/test_final_loss_not_average.pyindex a669664eaba..4cf24a000df 100644--- a/studio/backend/tests/test_final_loss_not_average.py+++ b/studio/backend/tests/test_final_loss_not_average.py@@ -112,6 +112,7 @@ def test_the_terminal_summary_still_reports_elapsed_time():learning_rate = 0.0elapsed_seconds = 412.5eta_seconds = None+ session_start_step = 0grad_norm = Nonenum_tokens = 12345status_message = ""@@ -148,6 +149,7 @@ def test_a_lossless_mid_run_record_is_still_dropped():learning_rate = 0.0elapsed_seconds = 40.0eta_seconds = None+ session_start_step = 0grad_norm = Nonenum_tokens = 1status_message = ""@@ -183,6 +185,7 @@ def test_an_early_stopped_run_still_reports_its_duration():learning_rate = 0.0elapsed_seconds = 91.0eta_seconds = None+ session_start_step = 0grad_norm = Nonenum_tokens = 5status_message = ""diff --git a/studio/backend/tests/test_training_progress_callback.py b/studio/backend/tests/test_training_progress_callback.pyindex 55a32fc8b87..9b4e271b6e7 100644--- a/studio/backend/tests/test_training_progress_callback.py+++ b/studio/backend/tests/test_training_progress_callback.py@@ -18,7 +18,7 @@ import sysimport typesfrom pathlib import Pathfrom types import SimpleNamespace-from unittest.mock import MagicMock+from unittest.mock import MagicMock, patchimport pytest@@ -234,6 +234,63 @@ def test_stop_status_is_never_replaced_by_the_active_one(stop_status):assert control.should_training_stop is True+def _drive_resumed(callback, start_step, step):+ state = SimpleNamespace(global_step = start_step, epoch = 0.9, num_input_tokens_seen = 0)+ control = SimpleNamespace(should_training_stop = False)+ with patch("time.time", return_value = 700):+ callback.on_train_begin(None, state, control)+ state.global_step = step+ with patch("time.time", return_value = 760):+ callback.on_log(None, state, control, logs = {"loss": 0.5, "learning_rate": 1e-4})+++def test_resumed_run_eta_uses_the_steps_done_in_this_session():+ owner = _make_owner()+ owner._update_progress(total_steps = 1000)+ backend = TrainingBackend()+ event_queue = _FakeQueue()+ owner.add_progress_callback(_create_trainer_progress_callback(event_queue))+ owner.training_start_time = 100++ _drive_resumed(owner._create_progress_callback(), start_step = 900, step = 910)+ for event in event_queue.events:+ backend._handle_event(event)++ assert owner.training_progress.eta_seconds == pytest.approx(540, rel = 0.02)+ assert backend._progress.eta_seconds == pytest.approx(540, rel = 0.02)+ assert backend._progress.session_start_step == 900+ assert backend._progress.elapsed_seconds == 60+++def test_resumed_run_reports_no_eta_before_its_first_step():+ owner = _make_owner()+ owner._update_progress(total_steps = 1000)+ owner.training_start_time = 100++ _drive_resumed(owner._create_progress_callback(), start_step = 900, step = 900)++ assert owner.training_progress.eta_seconds is None+++def test_fresh_run_eta_is_unchanged():+ owner = _make_owner()+ owner._update_progress(total_steps = 1000)+ owner.training_start_time = 700++ _drive_resumed(owner._create_progress_callback(), start_step = 0, step = 100)++ assert owner.training_progress.eta_seconds == pytest.approx(540, rel = 0.02)+++def test_mlx_adapter_keeps_the_session_start_step():+ adapter = _MLXTrainerAdapter()++ adapter._handle_event({"type": "progress", "step": 910, "session_start_step": 900})+ adapter._handle_event({"type": "progress", "step": 920, "eval_loss": 0.4})++ assert adapter.training_progress.session_start_step == 900++# ---------------------------------------------------------------------------# Embedding path: worker._create_embedding_progress_callback# ---------------------------------------------------------------------------@@ -290,3 +347,22 @@ def test_embedding_callback_survives_a_real_queue():events = [event_queue.get_nowait() for _ in range(event_queue.qsize())]assert [e["type"] for e in events] == ["status", "progress"]assert pickle.loads(pickle.dumps(events)) == events+++def test_embedding_resumed_run_eta_uses_the_steps_done_in_this_session():+ event_queue = _FakeQueue()+ backend = TrainingBackend()+ callback = _create_embedding_progress_callback(+ event_queue,+ total_steps = 1000,+ training_start_time = 100,+ should_stop = lambda: False,+ )++ _drive_resumed(callback, start_step = 900, step = 910)+ for event in event_queue.events:+ backend._handle_event(event)++ assert backend._progress.eta_seconds == pytest.approx(540, rel = 0.02)+ assert backend._progress.session_start_step == 900+ assert backend._progress.elapsed_seconds == 60diff --git a/studio/backend/tests/test_training_progress_stream_nan.py b/studio/backend/tests/test_training_progress_stream_nan.pyindex 5cd84bbca5b..35b97801ad2 100644--- a/studio/backend/tests/test_training_progress_stream_nan.py+++ b/studio/backend/tests/test_training_progress_stream_nan.py@@ -151,3 +151,20 @@ def test_stream_uses_finite_history_when_progress_in_sync(monkeypatch):payloads = _progress_payloads(_collect_events(response))finite = [p for p in payloads if p.get("step") == 2]assert finite and finite[0]["loss"] == 1.5+++def test_stream_carries_the_session_start_step(monkeypatch):+ backend = _FakeBackend(active_polls = 2)+ monkeypatch.setattr(rt, "get_training_backend", lambda: backend)++ response = asyncio.run(rt.stream_training_progress(_FakeRequest(), current_subject = "tester"))+ assert {p["session_start_step"] for p in _progress_payloads(_collect_events(response))} == {+ None+ }++ backend = _FakeBackend(active_polls = 2)+ backend.trainer.training_progress.session_start_step = 3+ monkeypatch.setattr(rt, "get_training_backend", lambda: backend)++ response = asyncio.run(rt.stream_training_progress(_FakeRequest(), current_subject = "tester"))+ assert {p["session_start_step"] for p in _progress_payloads(_collect_events(response))} == {3}diff --git a/studio/frontend/src/features/studio/live-training-view.tsx b/studio/frontend/src/features/studio/live-training-view.tsxindex 1365ef3092b..76fdb7984fe 100644--- a/studio/frontend/src/features/studio/live-training-view.tsx+++ b/studio/frontend/src/features/studio/live-training-view.tsx@@ -54,6 +54,7 @@ export function LiveTrainingView(): ReactElement {progressPercent: state.progressPercent,elapsedSeconds: state.elapsedSeconds,etaSeconds: state.etaSeconds,+ sessionStartStep: state.sessionStartStep,evalEnabled: state.evalEnabled,outputDir: state.outputDir,isTrainingRunning: state.isTrainingRunning,@@ -144,6 +145,7 @@ export function LiveTrainingView(): ReactElement {progressPercent: runtime.progressPercent,elapsedSeconds: runtime.elapsedSeconds,etaSeconds: runtime.etaSeconds,+ sessionStartStep: runtime.sessionStartStep,evalEnabled: runtime.evalEnabled,message: runtime.message,error: runtime.error,diff --git a/studio/frontend/src/features/studio/sections/progress-section-lib.ts b/studio/frontend/src/features/studio/sections/progress-section-lib.tsindex 2ab2c8ee5ef..492e3ef64ce 100644--- a/studio/frontend/src/features/studio/sections/progress-section-lib.ts+++ b/studio/frontend/src/features/studio/sections/progress-section-lib.ts@@ -38,6 +38,32 @@ export function formatDuration(seconds: number | null | undefined): string {return `${s}s`;}+export function sessionStepsPerSecond(+ currentStep: number,+ sessionStartStep: number,+ elapsedSeconds: number | null,+): number | null {+ const sessionSteps = currentStep - sessionStartStep;+ if (elapsedSeconds == null || elapsedSeconds <= 0 || sessionSteps < 0) {+ return null;+ }+ return sessionSteps / elapsedSeconds;+}++export function sessionEtaSeconds(+ currentStep: number,+ sessionStartStep: number,+ totalSteps: number,+ elapsedSeconds: number | null,+): number | null {+ const sessionSteps = currentStep - sessionStartStep;+ const remaining = totalSteps - currentStep;+ if (elapsedSeconds == null || sessionSteps <= 0 || remaining < 0) {+ return null;+ }+ return Math.round((elapsedSeconds * remaining) / sessionSteps);+}+export function formatNumber(value: number | null | undefined, digits: number): string {if (value == null || !Number.isFinite(value)) return "--";return value.toFixed(digits);diff --git a/studio/frontend/src/features/studio/sections/progress-section.tsx b/studio/frontend/src/features/studio/sections/progress-section.tsxindex 6134e4b9db8..c4f4138f399 100644--- a/studio/frontend/src/features/studio/sections/progress-section.tsx+++ b/studio/frontend/src/features/studio/sections/progress-section.tsx@@ -53,6 +53,8 @@ import {formatDuration,formatNumber,phaseColors,+ sessionEtaSeconds,+ sessionStepsPerSecond,} from "./progress-section-lib";import type { RunConfigOverride } from "./run-config-override";@@ -131,14 +133,25 @@ export function ProgressSection({: Math.round(data.progressPercent);const elapsed = data.elapsedSeconds;+ const sessionStartStep = data.sessionStartStep ?? 0;const derivedEta =- elapsed != null && pct > 0- ? Math.round((elapsed * (100 - pct)) / Math.max(pct, 1))- : null;+ sessionStartStep > 0+ ? sessionEtaSeconds(+ data.currentStep,+ sessionStartStep,+ data.totalSteps,+ elapsed,+ )+ : elapsed != null && pct > 0+ ? Math.round((elapsed * (100 - pct)) / Math.max(pct, 1))+ : null;const eta = data.etaSeconds ?? derivedEta;- const stepsPerSecond =- elapsed != null && elapsed > 0 ? data.currentStep / elapsed : null;+ const stepsPerSecond = sessionStepsPerSecond(+ data.currentStep,+ sessionStartStep,+ elapsed,+ );const showHalfwayHint =data.phase === "training" && pct >= 50 && pct < 100;const showCompletedHint = data.phase === "completed";diff --git a/studio/frontend/src/features/training/lib/training-sse-stream.ts b/studio/frontend/src/features/training/lib/training-sse-stream.tsindex 72d5f460b4a..f4a26efbe1b 100644--- a/studio/frontend/src/features/training/lib/training-sse-stream.ts+++ b/studio/frontend/src/features/training/lib/training-sse-stream.ts@@ -29,6 +29,7 @@ const NULLABLE_NUMERIC_FIELDS = ["epoch","elapsed_seconds","eta_seconds",+ "session_start_step","grad_norm","num_tokens","eval_loss",@@ -88,6 +89,9 @@ function parseTrainingProgressPayload(epoch: normalizeNullableFiniteNumber(payload.epoch),elapsed_seconds: normalizeNullableFiniteNumber(payload.elapsed_seconds),eta_seconds: normalizeNullableFiniteNumber(payload.eta_seconds),+ session_start_step: normalizeNullableFiniteNumber(+ payload.session_start_step,+ ),grad_norm: normalizeNullableFiniteNumber(payload.grad_norm),num_tokens: normalizeNullableFiniteNumber(payload.num_tokens),eval_loss: normalizeNullableFiniteNumber(payload.eval_loss),diff --git a/studio/frontend/src/features/training/stores/training-runtime-store.ts b/studio/frontend/src/features/training/stores/training-runtime-store.tsindex 7106f0bec95..33f38bc6cb9 100644--- a/studio/frontend/src/features/training/stores/training-runtime-store.ts+++ b/studio/frontend/src/features/training/stores/training-runtime-store.ts@@ -79,6 +79,7 @@ const initialState: TrainingRuntimeState = {progressPercent: 0,elapsedSeconds: null,etaSeconds: null,+ sessionStartStep: 0,currentGradNorm: null,currentNumTokens: null,outputDir: null,@@ -321,6 +322,7 @@ export const useTrainingRuntimeStore = create<TrainingRuntimeStore>()(progressPercent: 0,elapsedSeconds: null,etaSeconds: null,+ sessionStartStep: 0,currentGradNorm: null,currentNumTokens: null,outputDir: null,@@ -400,6 +402,7 @@ export const useTrainingRuntimeStore = create<TrainingRuntimeStore>()(progressPercent: 0,elapsedSeconds: null,etaSeconds: null,+ sessionStartStep: 0,currentGradNorm: null,currentNumTokens: null,outputDir: null,@@ -579,6 +582,7 @@ export const useTrainingRuntimeStore = create<TrainingRuntimeStore>()(: state.currentEpoch,elapsedSeconds: payload.elapsed_seconds,etaSeconds: payload.eta_seconds,+ sessionStartStep: payload.session_start_step ?? state.sessionStartStep,currentGradNorm,currentNumTokens: payload.num_tokens,firstStepReceived: state.firstStepReceived || step > 0,diff --git a/studio/frontend/src/features/training/types/runtime.ts b/studio/frontend/src/features/training/types/runtime.tsindex 8b122aa3a9f..9e96f7ffd9a 100644--- a/studio/frontend/src/features/training/types/runtime.ts+++ b/studio/frontend/src/features/training/types/runtime.ts@@ -67,6 +67,7 @@ export interface TrainingProgressPayload {epoch: number | null;elapsed_seconds: number | null;eta_seconds: number | null;+ session_start_step?: number | null;grad_norm: number | null;num_tokens: number | null;eval_loss: number | null;@@ -107,6 +108,7 @@ export interface TrainingRuntimeState {progressPercent: number;elapsedSeconds: number | null;etaSeconds: number | null;+ sessionStartStep: number;currentGradNorm: number | null;currentNumTokens: number | null;outputDir: string | null;@@ -169,6 +171,7 @@ export interface TrainingViewData {progressPercent: number;elapsedSeconds: number | null;etaSeconds: number | null;+ sessionStartStep?: number;evalEnabled: boolean;message: string;error: string | null;diff --git a/studio/frontend/tests/training-resumed-session-rate.test.ts b/studio/frontend/tests/training-resumed-session-rate.test.tsnew file mode 100644index 00000000000..bf644f063b8--- /dev/null+++ b/studio/frontend/tests/training-resumed-session-rate.test.ts@@ -0,0 +1,74 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import {+ sessionEtaSeconds,+ sessionStepsPerSecond,+} from "../src/features/studio/sections/progress-section-lib.ts";+import { consumeTrainingProgressStream } from "../src/features/training/lib/training-sse-stream.ts";+import { useTrainingRuntimeStore } from "../src/features/training/stores/training-runtime-store.ts";++function payload(step: number, extra: Record<string, unknown> = {}) {+ return {+ job_id: "job-resumed",+ step,+ total_steps: 1000,+ loss: 0.5,+ learning_rate: 0.0001,+ progress_percent: step / 10,+ epoch: 0.9,+ elapsed_seconds: 60,+ eta_seconds: 540,+ grad_norm: null,+ num_tokens: null,+ eval_loss: null,+ ...extra,+ };+}++test("a resumed run's throughput counts only the steps done in this session", () => {+ assert.equal(sessionStepsPerSecond(910, 900, 60), 10 / 60);+ assert.equal(sessionStepsPerSecond(100, 0, 60), 100 / 60);+ assert.equal(sessionStepsPerSecond(900, 900, 60), 0);+ assert.equal(sessionStepsPerSecond(0, 900, 60), null);+ assert.equal(sessionStepsPerSecond(910, 900, null), null);+ assert.equal(sessionStepsPerSecond(910, 900, 0), null);+});++test("the fallback ETA uses the same session window", () => {+ assert.equal(sessionEtaSeconds(910, 900, 1000, 60), 540);+ assert.equal(sessionEtaSeconds(900, 900, 1000, 60), null);+ assert.equal(sessionEtaSeconds(910, 900, 1000, null), null);+});++test("the session start step travels from the SSE frame into the store", async () => {+ const frames = [payload(910, { session_start_step: 900 }), payload(920)]+ .map((p) => `event: progress\nid: ${p.step}\ndata: ${JSON.stringify(p)}\n\n`)+ .join("");+ const body = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(new TextEncoder().encode(frames));+ controller.close();+ },+ });+ useTrainingRuntimeStore.getState().resetRuntime();+ useTrainingRuntimeStore.getState().setStartPending("job-resumed", "Starting");+ assert.equal(useTrainingRuntimeStore.getState().sessionStartStep, 0);++ await consumeTrainingProgressStream({+ body,+ signal: new AbortController().signal,+ onEvent: ({ payload: parsed, id }) =>+ useTrainingRuntimeStore.getState().applyProgress(parsed, id ?? undefined),+ });++ const state = useTrainingRuntimeStore.getState();+ assert.equal(state.currentStep, 920);+ assert.equal(state.sessionStartStep, 900);++ useTrainingRuntimeStore.getState().setStartPending("job-next", "Starting");+ assert.equal(useTrainingRuntimeStore.getState().sessionStartStep, 0);+});