Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio recipes: keep a finished run's pump off the next run (#13845)

Daniel Han2a7c55f37122
diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.py
index 93cb03eda60..a214ce75073 100644
--- a/studio/backend/core/data_recipe/jobs/manager.py
+++ b/studio/backend/core/data_recipe/jobs/manager.py
@@ -532,12 +532,13 @@ class JobManager:
Guarded so no single event can end the loop; it is the sole writer of the
snapshot the UI polls, so its death would freeze status/SSE.
"""
+ # Bound once: a later start() swaps in the next run, which has its own pump. Re-reading the
+ # snapshot here let this pump finalize that run as "process exited" and revoke its key.
+ snap = self._snapshot()
+ if snap is None:
+ return
+ job, proc, mp_q = snap
while True:
- snap = self._snapshot()
- if snap is None:
- return
- job, proc, mp_q = snap
-
try:
event = self._read_queue_with_timeout(mp_q, timeout_sec = 0.25)
except Exception:
@@ -561,30 +562,29 @@ class JobManager:
retired_job: Job | None = None
with self._lock:
- if self._job and self._job.status in {
+ if job.status in {
"pending",
"active",
"cancelling",
}:
- if self._job.status == "cancelling":
- self._job.status = "cancelled"
+ if job.status == "cancelling":
+ job.status = "cancelled"
else:
- self._job.status = "error"
- self._job.error = self._job.error or "process exited"
- self._job.finished_at = time.time()
+ job.status = "error"
+ job.error = job.error or "process exited"
+ job.finished_at = time.time()
event_type = (
- EVENT_JOB_CANCELLED
- if self._job.status == "cancelled"
- else EVENT_JOB_ERROR
- )
- self._emit(
- {
- "type": event_type,
- "ts": time.time(),
- "job_id": self._job.job_id,
- }
+ EVENT_JOB_CANCELLED if job.status == "cancelled" else EVENT_JOB_ERROR
)
- retired_job = self._job
+ if self._job is job:
+ self._emit(
+ {
+ "type": event_type,
+ "ts": time.time(),
+ "job_id": job.job_id,
+ }
+ )
+ retired_job = job
if retired_job is not None:
self._retire_workflow_key(retired_job)
except Exception:
diff --git a/studio/backend/tests/test_data_recipe_pump_resilience.py b/studio/backend/tests/test_data_recipe_pump_resilience.py
index e702be78110..e90e6f8856c 100644
--- a/studio/backend/tests/test_data_recipe_pump_resilience.py
+++ b/studio/backend/tests/test_data_recipe_pump_resilience.py
@@ -150,3 +150,48 @@ def test_pump_finalizes_when_read_keeps_raising_on_dead_worker(monkeypatch):
assert not pump.is_alive(), "pump must finalize a dead worker even when reads keep raising"
assert m._job.status == "error"
assert retired and retired[0] is m._job
+
+
+def test_finished_pump_leaves_the_next_run_alone(monkeypatch):
+ # Run 1's worker exits and run 2 starts before run 1's pump notices: run 1's pump must
+ # finalize run 1 only, not fail run 2 and revoke the key run 2's worker is about to use.
+ m = _manager_with_active_job()
+ first = m._job
+ first.status = "completed"
+ first.internal_api_key_id = 1
+ m._proc._alive = False
+ emitted: list = []
+ monkeypatch.setattr(m, "_emit", lambda e: emitted.append(e))
+ revoked: list = []
+
+ class _Storage:
+ @staticmethod
+ def revoke_internal_api_key(key_id):
+ revoked.append(key_id)
+
+ import auth
+
+ monkeypatch.setattr(auth, "storage", _Storage, raising = False)
+ second = Job(job_id = "job-next")
+ second.status = "pending"
+ second.internal_api_key_id = 2
+ second_q = _ScriptedQueue([])
+
+ class _NextRunStartsDuringRead:
+ def get(self, timeout = None):
+ with m._lock: # what start() does for run 2
+ m._job, m._proc, m._mp_q = second, _FakeProc(alive = True), second_q
+ raise queue.Empty
+
+ def get_nowait(self):
+ raise queue.Empty
+
+ m._mp_q = _NextRunStartsDuringRead()
+ pump = threading.Thread(target = m._pump_loop, daemon = True)
+ pump.start()
+ pump.join(timeout = 5)
+
+ assert not pump.is_alive(), "run 1's pump must stop instead of serving run 2's queue"
+ assert second.status == "pending" and second.error is None
+ assert second.internal_api_key_id == 2 and 2 not in revoked
+ assert emitted == []