Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep reading durable chat output while its events are being saved (#14135)
Daniel Hanffd6c3033dd3
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.pyindex 9cf2493ccd7..26730039935 100644--- a/studio/backend/core/inference/chat_generation_runs.py+++ b/studio/backend/core/inference/chat_generation_runs.py@@ -26,9 +26,10 @@ from state import active_generationsfrom storage import chat_generation_runs_db as dblogger = get_logger(__name__)-_EVENT_BATCH_SIZE = 16# Followers only see appended chunks, so this is the chat's text frame rate: at most one commit per display frame._EVENT_FLUSH_SECONDS = 1 / 60+# Reading pauses past this many events behind a running commit, so a wedged database cannot buffer a generation.+_MAX_PENDING_EVENTS = 4096_SHUTDOWN_GRACE_SECONDS = 10.0# Second budget, after task.cancel(). Shorter than the grace period: by this point the run is already being abandoned,# and the only question is whether shutdown returns.@@ -712,7 +713,12 @@ class ChatGenerationSupervisor:activity.reserve()pending: list[tuple[str, dict[str, Any], int]] = []iterator: AsyncIterator[Any] | None = None- last_flush = time.monotonic()+ # Never await a commit inline: the GGUF tool loop only advances as fast as this producer pulls.+ writing: list[tuple[str, dict[str, Any], int]] = []+ write_task: asyncio.Task | None = None+ held_cancel: asyncio.CancelledError | None = None+ # When the last commit STARTED: a commit slower than a frame must not leave each later chunk its own commit.+ last_flush = -math.inffinish_reason: str | None = Noneerror: str | None = None# Sent on its own chunk, since the finish chunk still reads `stop`.@@ -720,6 +726,33 @@ class ChatGenerationSupervisor:saw_done = Falseworker_token: str | None = Nonenext_raw_task: asyncio.Task | None = None++ def start_write() -> None:+ nonlocal writing, pending, write_task, last_flush+ writing, pending = pending, []+ last_flush = time.monotonic()+ write_task = asyncio.create_task(+ asyncio.to_thread(db.append_events, run_id, worker_token, writing)+ )++ async def settle_write() -> BaseException | None:+ """Wait out the in-flight commit and return its failure; a failed batch goes back in front of pending.+ Never cancelled: finish_run must not overtake the thread, so a cancel is held until the run settles."""+ nonlocal writing, pending, write_task, held_cancel+ if write_task is None:+ return None+ while not write_task.done():+ try:+ await asyncio.wait({write_task})+ except asyncio.CancelledError as exc:+ held_cancel = exc+ task, write_task = write_task, None+ failure = asyncio.CancelledError() if task.cancelled() else task.exception()+ if failure is not None:+ pending = writing + pending+ writing = []+ return failure+try:worker_run = await asyncio.to_thread(db.get_worker_run, run_id)if worker_run is None:@@ -793,32 +826,33 @@ class ChatGenerationSupervisor:last_keepalive = time.monotonic()next_raw_task = asyncio.create_task(iterator.__anext__())while True:- timeout = (- max(0.0, _EVENT_FLUSH_SECONDS - (time.monotonic() - last_flush))- if pending- else None- )+ timeout = None+ if pending and write_task is None:+ since_flush = time.monotonic() - last_flush+ if since_flush >= _EVENT_FLUSH_SECONDS or len(pending) >= _MAX_PENDING_EVENTS:+ start_write()+ else:+ timeout = _EVENT_FLUSH_SECONDS - since_flush# Wake to renew the lease across a first-token wait the backend is still inside its own deadline for.renew_interval = _renew_interval_seconds()timeout = renew_interval if timeout is None else min(timeout, renew_interval)- ready, _waiting = await asyncio.wait({next_raw_task}, timeout = timeout)+ waiting = {next_raw_task} if len(pending) < _MAX_PENDING_EVENTS else set()+ if write_task is not None:+ waiting.add(write_task)+ ready, _waiting = await asyncio.wait(+ waiting, timeout = timeout, return_when = asyncio.FIRST_COMPLETED+ )if (first_token_wait.active()and time.monotonic() - last_keepalive >= renew_interval):await self._try_touch_progress(run_id)last_keepalive = time.monotonic()- if not ready:- if not pending:- continue- await asyncio.to_thread(- db.append_events,- run_id,- worker_token,- pending,- )- pending = []- last_flush = time.monotonic()+ if write_task is not None and write_task.done():+ failure = await settle_write()+ if failure is not None:+ raise failure+ if next_raw_task not in ready:continuetry:raw = next_raw_task.result()@@ -855,22 +889,14 @@ class ChatGenerationSupervisor:finish_reason = _chunk_finish_reason(chunk) or finish_reasonerror = _chunk_error(chunk) or errorquote_cut = quote_cut or chunk.get("quote_cut") is True- now = time.monotonic()- if (- len(pending) >= _EVENT_BATCH_SIZE- or now - last_flush >= _EVENT_FLUSH_SECONDS- ):- await asyncio.to_thread(- db.append_events,- run_id,- worker_token,- pending,- )- pending = []- last_flush = nowif saw_done:break+ failure = await settle_write()+ if held_cancel is not None:+ raise held_cancel+ if failure is not None:+ raise failurecurrent = await asyncio.to_thread(db.get_run, run_id)if current is None:return@@ -904,6 +930,7 @@ class ChatGenerationSupervisor:)pending = []except asyncio.CancelledError:+ await settle_write()if worker_token is not None:shutting_down = run_id in self._shutdown_runscancelled = cancel_event.is_set() and not shutting_down@@ -919,6 +946,7 @@ class ChatGenerationSupervisor:pending = []raiseexcept Exception as exc:+ await settle_write()if worker_token is not None:shutting_down = run_id in self._shutdown_runscancelled = cancel_event.is_set() and not shutting_down@@ -940,6 +968,8 @@ class ChatGenerationSupervisor:pending_events = pending,)pending = []+ if held_cancel is not None:+ raise held_cancelfinally:if next_raw_task is not None:if not next_raw_task.done():diff --git a/studio/backend/tests/test_chat_generation_supervisor.py b/studio/backend/tests/test_chat_generation_supervisor.pyindex c8bffef5699..f12a330df1a 100644--- a/studio/backend/tests/test_chat_generation_supervisor.py+++ b/studio/backend/tests/test_chat_generation_supervisor.py@@ -555,12 +555,17 @@ async def test_native_gguf_prefill_progress_holds_the_lease(async def body():comments = 0+ chunks = 0async for raw in inner:text = raw.decode() if isinstance(raw, bytes) else str(raw)+ chunks += text.count("data: {")if text.startswith(":") and entered.is_set():comments += 1sampled["comments"].add(text.strip())if comments == 2:+ # Commits run beside the stream now, so let the chunks read so far land before the clock+ # jumps; a slow runner would otherwise stamp them as progress made after it.+ await _await_chunk_payloads("run-1", chunks, 10.0)now["ms"] += 21 * 60_000elif comments == 5 and "reaped" not in sampled:sampled["reaped"] = runs_db.reconcile_runs(stale_after_ms = 1_200_000)@@ -767,6 +772,166 @@ async def test_a_lone_chunk_is_appended_without_waiting_for_another(durable_run,await task+def _numbered_chunks(count):+ return [{"choices": [{"delta": {"content": f"t{i} "}}]} for i in range(count)]+++def _held_append(monkeypatch):+ """Patch append_events so every commit blocks until `gate` is set. Returns (entered, gate)."""+ entered = threading.Event()+ gate = threading.Event()+ original = runs_db.append_events++ def held(run_id, worker_token, events):+ entered.set()+ assert gate.wait(10)+ return original(run_id, worker_token, events)++ monkeypatch.setattr(runs_db, "append_events", held)+ return entered, gate+++async def test_a_slow_commit_does_not_stop_the_stream_being_read(durable_run, monkeypatch):+ chunks = _numbered_chunks(50)+ drained = asyncio.Event()+ release = asyncio.Event()++ async def body():+ yield f"data: {json.dumps(chunks[0])}\n\n"+ # The rest is only offered once the first commit is under way and held.+ assert await asyncio.to_thread(entered.wait, 2)+ for chunk in chunks[1:]:+ yield f"data: {json.dumps(chunk)}\n\n"+ drained.set()+ await release.wait()+ yield "data: [DONE]\n\n"++ async def fake(*_args, **_kwargs):+ return SimpleNamespace(status_code = 200, body_iterator = body())++ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)+ entered, gate = _held_append(monkeypatch)+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))+ task = asyncio.create_task(supervisor._produce("run-1"))+ try:+ await asyncio.wait_for(drained.wait(), timeout = 2)+ assert not gate.is_set()+ finally:+ gate.set()+ release.set()+ await task+ stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]+ assert stored == chunks+ assert runs_db.get_run("run-1", "alice")["status"] == "completed"+++async def test_slow_commits_batch_a_burst_instead_of_one_chunk_each(durable_run, monkeypatch):+ chunks = _numbered_chunks(400)+ calls = []+ original = runs_db.append_events++ def slow(run_id, worker_token, events):+ calls.append(len(events))+ time.sleep(0.04)+ return original(run_id, worker_token, events)++ async def body():+ for chunk in chunks:+ yield f"data: {json.dumps(chunk)}\n\n"+ yield "data: [DONE]\n\n"++ async def fake(*_args, **_kwargs):+ return SimpleNamespace(status_code = 200, body_iterator = body())++ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)+ monkeypatch.setattr(runs_db, "append_events", slow)+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))+ await supervisor._produce("run-1")+ stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]+ assert stored == chunks+ assert len(calls) <= 5, calls+ assert runs_db.get_run("run-1", "alice")["status"] == "completed"+++async def test_a_failed_commit_keeps_its_events_and_fails_the_run(durable_run, monkeypatch):+ chunks = _numbered_chunks(3)++ def broken(*_args, **_kwargs):+ raise RuntimeError("database is locked")++ async def body():+ for chunk in chunks:+ yield f"data: {json.dumps(chunk)}\n\n"+ yield "data: [DONE]\n\n"++ async def fake(*_args, **_kwargs):+ return SimpleNamespace(status_code = 200, body_iterator = body())++ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)+ monkeypatch.setattr(runs_db, "append_events", broken)+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))+ await supervisor._produce("run-1")+ run = runs_db.get_run("run-1", "alice")+ assert (run["status"], run["finishReason"], run["error"]) == (+ "failed",+ "error",+ "database is locked",+ )+ stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]+ assert stored and stored == chunks[: len(stored)]++[email protected]("cancels", [1, 2], ids = ["cancel", "cancel-again-while-settling"])+async def test_cancel_during_a_commit_waits_for_it_before_settling(+ durable_run, monkeypatch, cancels+):+ chunks = _numbered_chunks(20)+ leaked = []+ drained = asyncio.Event()++ async def body():+ for chunk in chunks:+ yield f"data: {json.dumps(chunk)}\n\n"+ drained.set()+ await asyncio.Event().wait()++ async def fake(*_args, **_kwargs):+ return SimpleNamespace(status_code = 200, body_iterator = body())++ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)+ entered, gate = _held_append(monkeypatch)+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))+ loop = asyncio.get_running_loop()+ previous_handler = loop.get_exception_handler()+ loop.set_exception_handler(lambda _loop, context: leaked.append(context))+ cancel_event = threading.Event()+ task = asyncio.create_task(supervisor._produce("run-1", cancel_event))+ try:+ assert await asyncio.to_thread(entered.wait, 2)+ await asyncio.wait_for(drained.wait(), timeout = 2)+ cancel_event.set()+ task.cancel()+ # A second cancel (shutdown's forced one) lands while settlement waits on the commit; the run must still settle.+ for _ in range(cancels - 1):+ loop.call_later(0.05, task.cancel)+ # finish_run must not overtake the commit still holding its batch.+ loop.call_later(0.1, gate.set)+ with pytest.raises(asyncio.CancelledError):+ await task+ await asyncio.sleep(0)+ finally:+ gate.set()+ loop.set_exception_handler(previous_handler)+ assert leaked == []+ assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"+ stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]+ assert stored == chunks++@pytest.mark.asyncioasync def test_model_lifecycle_cancel_reaches_same_registered_event(durable_run, monkeypatch):registered = asyncio.Event()