Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio recipes: keep a finished run's pump off the next run (#13845)
Daniel Han2a7c55f37122
diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.pyindex 93cb03eda60..a214ce75073 100644--- a/studio/backend/core/data_recipe/jobs/manager.py+++ b/studio/backend/core/data_recipe/jobs/manager.py@@ -532,12 +532,13 @@ class JobManager:Guarded so no single event can end the loop; it is the sole writer of thesnapshot the UI polls, so its death would freeze status/SSE."""+ # Bound once: a later start() swaps in the next run, which has its own pump. Re-reading the+ # snapshot here let this pump finalize that run as "process exited" and revoke its key.+ snap = self._snapshot()+ if snap is None:+ return+ job, proc, mp_q = snapwhile True:- snap = self._snapshot()- if snap is None:- return- job, proc, mp_q = snap-try:event = self._read_queue_with_timeout(mp_q, timeout_sec = 0.25)except Exception:@@ -561,30 +562,29 @@ class JobManager:retired_job: Job | None = Nonewith self._lock:- if self._job and self._job.status in {+ if job.status in {"pending","active","cancelling",}:- if self._job.status == "cancelling":- self._job.status = "cancelled"+ if job.status == "cancelling":+ job.status = "cancelled"else:- self._job.status = "error"- self._job.error = self._job.error or "process exited"- self._job.finished_at = time.time()+ job.status = "error"+ job.error = job.error or "process exited"+ job.finished_at = time.time()event_type = (- EVENT_JOB_CANCELLED- if self._job.status == "cancelled"- else EVENT_JOB_ERROR- )- self._emit(- {- "type": event_type,- "ts": time.time(),- "job_id": self._job.job_id,- }+ EVENT_JOB_CANCELLED if job.status == "cancelled" else EVENT_JOB_ERROR)- retired_job = self._job+ if self._job is job:+ self._emit(+ {+ "type": event_type,+ "ts": time.time(),+ "job_id": job.job_id,+ }+ )+ retired_job = jobif retired_job is not None:self._retire_workflow_key(retired_job)except Exception:diff --git a/studio/backend/tests/test_data_recipe_pump_resilience.py b/studio/backend/tests/test_data_recipe_pump_resilience.pyindex e702be78110..e90e6f8856c 100644--- a/studio/backend/tests/test_data_recipe_pump_resilience.py+++ b/studio/backend/tests/test_data_recipe_pump_resilience.py@@ -150,3 +150,48 @@ def test_pump_finalizes_when_read_keeps_raising_on_dead_worker(monkeypatch):assert not pump.is_alive(), "pump must finalize a dead worker even when reads keep raising"assert m._job.status == "error"assert retired and retired[0] is m._job+++def test_finished_pump_leaves_the_next_run_alone(monkeypatch):+ # Run 1's worker exits and run 2 starts before run 1's pump notices: run 1's pump must+ # finalize run 1 only, not fail run 2 and revoke the key run 2's worker is about to use.+ m = _manager_with_active_job()+ first = m._job+ first.status = "completed"+ first.internal_api_key_id = 1+ m._proc._alive = False+ emitted: list = []+ monkeypatch.setattr(m, "_emit", lambda e: emitted.append(e))+ revoked: list = []++ class _Storage:+ @staticmethod+ def revoke_internal_api_key(key_id):+ revoked.append(key_id)++ import auth++ monkeypatch.setattr(auth, "storage", _Storage, raising = False)+ second = Job(job_id = "job-next")+ second.status = "pending"+ second.internal_api_key_id = 2+ second_q = _ScriptedQueue([])++ class _NextRunStartsDuringRead:+ def get(self, timeout = None):+ with m._lock: # what start() does for run 2+ m._job, m._proc, m._mp_q = second, _FakeProc(alive = True), second_q+ raise queue.Empty++ def get_nowait(self):+ raise queue.Empty++ m._mp_q = _NextRunStartsDuringRead()+ pump = threading.Thread(target = m._pump_loop, daemon = True)+ pump.start()+ pump.join(timeout = 5)++ assert not pump.is_alive(), "run 1's pump must stop instead of serving run 2's queue"+ assert second.status == "pending" and second.error is None+ assert second.internal_api_key_id == 2 and 2 not in revoked+ assert emitted == []