Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep reading durable chat output while its events are being saved (#14135)

Daniel Hanffd6c3033dd3
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.py
index 9cf2493ccd7..26730039935 100644
--- a/studio/backend/core/inference/chat_generation_runs.py
+++ b/studio/backend/core/inference/chat_generation_runs.py
@@ -26,9 +26,10 @@ from state import active_generations
from storage import chat_generation_runs_db as db
logger = get_logger(__name__)
-_EVENT_BATCH_SIZE = 16
# Followers only see appended chunks, so this is the chat's text frame rate: at most one commit per display frame.
_EVENT_FLUSH_SECONDS = 1 / 60
+# Reading pauses past this many events behind a running commit, so a wedged database cannot buffer a generation.
+_MAX_PENDING_EVENTS = 4096
_SHUTDOWN_GRACE_SECONDS = 10.0
# Second budget, after task.cancel(). Shorter than the grace period: by this point the run is already being abandoned,
# and the only question is whether shutdown returns.
@@ -712,7 +713,12 @@ class ChatGenerationSupervisor:
activity.reserve()
pending: list[tuple[str, dict[str, Any], int]] = []
iterator: AsyncIterator[Any] | None = None
- last_flush = time.monotonic()
+ # Never await a commit inline: the GGUF tool loop only advances as fast as this producer pulls.
+ writing: list[tuple[str, dict[str, Any], int]] = []
+ write_task: asyncio.Task | None = None
+ held_cancel: asyncio.CancelledError | None = None
+ # When the last commit STARTED: a commit slower than a frame must not leave each later chunk its own commit.
+ last_flush = -math.inf
finish_reason: str | None = None
error: str | None = None
# Sent on its own chunk, since the finish chunk still reads `stop`.
@@ -720,6 +726,33 @@ class ChatGenerationSupervisor:
saw_done = False
worker_token: str | None = None
next_raw_task: asyncio.Task | None = None
+
+ def start_write() -> None:
+ nonlocal writing, pending, write_task, last_flush
+ writing, pending = pending, []
+ last_flush = time.monotonic()
+ write_task = asyncio.create_task(
+ asyncio.to_thread(db.append_events, run_id, worker_token, writing)
+ )
+
+ async def settle_write() -> BaseException | None:
+ """Wait out the in-flight commit and return its failure; a failed batch goes back in front of pending.
+ Never cancelled: finish_run must not overtake the thread, so a cancel is held until the run settles."""
+ nonlocal writing, pending, write_task, held_cancel
+ if write_task is None:
+ return None
+ while not write_task.done():
+ try:
+ await asyncio.wait({write_task})
+ except asyncio.CancelledError as exc:
+ held_cancel = exc
+ task, write_task = write_task, None
+ failure = asyncio.CancelledError() if task.cancelled() else task.exception()
+ if failure is not None:
+ pending = writing + pending
+ writing = []
+ return failure
+
try:
worker_run = await asyncio.to_thread(db.get_worker_run, run_id)
if worker_run is None:
@@ -793,32 +826,33 @@ class ChatGenerationSupervisor:
last_keepalive = time.monotonic()
next_raw_task = asyncio.create_task(iterator.__anext__())
while True:
- timeout = (
- max(0.0, _EVENT_FLUSH_SECONDS - (time.monotonic() - last_flush))
- if pending
- else None
- )
+ timeout = None
+ if pending and write_task is None:
+ since_flush = time.monotonic() - last_flush
+ if since_flush >= _EVENT_FLUSH_SECONDS or len(pending) >= _MAX_PENDING_EVENTS:
+ start_write()
+ else:
+ timeout = _EVENT_FLUSH_SECONDS - since_flush
# Wake to renew the lease across a first-token wait the backend is still inside its own deadline for.
renew_interval = _renew_interval_seconds()
timeout = renew_interval if timeout is None else min(timeout, renew_interval)
- ready, _waiting = await asyncio.wait({next_raw_task}, timeout = timeout)
+ waiting = {next_raw_task} if len(pending) < _MAX_PENDING_EVENTS else set()
+ if write_task is not None:
+ waiting.add(write_task)
+ ready, _waiting = await asyncio.wait(
+ waiting, timeout = timeout, return_when = asyncio.FIRST_COMPLETED
+ )
if (
first_token_wait.active()
and time.monotonic() - last_keepalive >= renew_interval
):
await self._try_touch_progress(run_id)
last_keepalive = time.monotonic()
- if not ready:
- if not pending:
- continue
- await asyncio.to_thread(
- db.append_events,
- run_id,
- worker_token,
- pending,
- )
- pending = []
- last_flush = time.monotonic()
+ if write_task is not None and write_task.done():
+ failure = await settle_write()
+ if failure is not None:
+ raise failure
+ if next_raw_task not in ready:
continue
try:
raw = next_raw_task.result()
@@ -855,22 +889,14 @@ class ChatGenerationSupervisor:
finish_reason = _chunk_finish_reason(chunk) or finish_reason
error = _chunk_error(chunk) or error
quote_cut = quote_cut or chunk.get("quote_cut") is True
- now = time.monotonic()
- if (
- len(pending) >= _EVENT_BATCH_SIZE
- or now - last_flush >= _EVENT_FLUSH_SECONDS
- ):
- await asyncio.to_thread(
- db.append_events,
- run_id,
- worker_token,
- pending,
- )
- pending = []
- last_flush = now
if saw_done:
break
+ failure = await settle_write()
+ if held_cancel is not None:
+ raise held_cancel
+ if failure is not None:
+ raise failure
current = await asyncio.to_thread(db.get_run, run_id)
if current is None:
return
@@ -904,6 +930,7 @@ class ChatGenerationSupervisor:
)
pending = []
except asyncio.CancelledError:
+ await settle_write()
if worker_token is not None:
shutting_down = run_id in self._shutdown_runs
cancelled = cancel_event.is_set() and not shutting_down
@@ -919,6 +946,7 @@ class ChatGenerationSupervisor:
pending = []
raise
except Exception as exc:
+ await settle_write()
if worker_token is not None:
shutting_down = run_id in self._shutdown_runs
cancelled = cancel_event.is_set() and not shutting_down
@@ -940,6 +968,8 @@ class ChatGenerationSupervisor:
pending_events = pending,
)
pending = []
+ if held_cancel is not None:
+ raise held_cancel
finally:
if next_raw_task is not None:
if not next_raw_task.done():
diff --git a/studio/backend/tests/test_chat_generation_supervisor.py b/studio/backend/tests/test_chat_generation_supervisor.py
index c8bffef5699..f12a330df1a 100644
--- a/studio/backend/tests/test_chat_generation_supervisor.py
+++ b/studio/backend/tests/test_chat_generation_supervisor.py
@@ -555,12 +555,17 @@ async def test_native_gguf_prefill_progress_holds_the_lease(
async def body():
comments = 0
+ chunks = 0
async for raw in inner:
text = raw.decode() if isinstance(raw, bytes) else str(raw)
+ chunks += text.count("data: {")
if text.startswith(":") and entered.is_set():
comments += 1
sampled["comments"].add(text.strip())
if comments == 2:
+ # Commits run beside the stream now, so let the chunks read so far land before the clock
+ # jumps; a slow runner would otherwise stamp them as progress made after it.
+ await _await_chunk_payloads("run-1", chunks, 10.0)
now["ms"] += 21 * 60_000
elif comments == 5 and "reaped" not in sampled:
sampled["reaped"] = runs_db.reconcile_runs(stale_after_ms = 1_200_000)
@@ -767,6 +772,166 @@ async def test_a_lone_chunk_is_appended_without_waiting_for_another(durable_run,
await task
+def _numbered_chunks(count):
+ return [{"choices": [{"delta": {"content": f"t{i} "}}]} for i in range(count)]
+
+
+def _held_append(monkeypatch):
+ """Patch append_events so every commit blocks until `gate` is set. Returns (entered, gate)."""
+ entered = threading.Event()
+ gate = threading.Event()
+ original = runs_db.append_events
+
+ def held(run_id, worker_token, events):
+ entered.set()
+ assert gate.wait(10)
+ return original(run_id, worker_token, events)
+
+ monkeypatch.setattr(runs_db, "append_events", held)
+ return entered, gate
+
+
+async def test_a_slow_commit_does_not_stop_the_stream_being_read(durable_run, monkeypatch):
+ chunks = _numbered_chunks(50)
+ drained = asyncio.Event()
+ release = asyncio.Event()
+
+ async def body():
+ yield f"data: {json.dumps(chunks[0])}\n\n"
+ # The rest is only offered once the first commit is under way and held.
+ assert await asyncio.to_thread(entered.wait, 2)
+ for chunk in chunks[1:]:
+ yield f"data: {json.dumps(chunk)}\n\n"
+ drained.set()
+ await release.wait()
+ yield "data: [DONE]\n\n"
+
+ async def fake(*_args, **_kwargs):
+ return SimpleNamespace(status_code = 200, body_iterator = body())
+
+ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
+ entered, gate = _held_append(monkeypatch)
+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
+ task = asyncio.create_task(supervisor._produce("run-1"))
+ try:
+ await asyncio.wait_for(drained.wait(), timeout = 2)
+ assert not gate.is_set()
+ finally:
+ gate.set()
+ release.set()
+ await task
+ stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
+ assert stored == chunks
+ assert runs_db.get_run("run-1", "alice")["status"] == "completed"
+
+
+async def test_slow_commits_batch_a_burst_instead_of_one_chunk_each(durable_run, monkeypatch):
+ chunks = _numbered_chunks(400)
+ calls = []
+ original = runs_db.append_events
+
+ def slow(run_id, worker_token, events):
+ calls.append(len(events))
+ time.sleep(0.04)
+ return original(run_id, worker_token, events)
+
+ async def body():
+ for chunk in chunks:
+ yield f"data: {json.dumps(chunk)}\n\n"
+ yield "data: [DONE]\n\n"
+
+ async def fake(*_args, **_kwargs):
+ return SimpleNamespace(status_code = 200, body_iterator = body())
+
+ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
+ monkeypatch.setattr(runs_db, "append_events", slow)
+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
+ await supervisor._produce("run-1")
+ stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
+ assert stored == chunks
+ assert len(calls) <= 5, calls
+ assert runs_db.get_run("run-1", "alice")["status"] == "completed"
+
+
+async def test_a_failed_commit_keeps_its_events_and_fails_the_run(durable_run, monkeypatch):
+ chunks = _numbered_chunks(3)
+
+ def broken(*_args, **_kwargs):
+ raise RuntimeError("database is locked")
+
+ async def body():
+ for chunk in chunks:
+ yield f"data: {json.dumps(chunk)}\n\n"
+ yield "data: [DONE]\n\n"
+
+ async def fake(*_args, **_kwargs):
+ return SimpleNamespace(status_code = 200, body_iterator = body())
+
+ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
+ monkeypatch.setattr(runs_db, "append_events", broken)
+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
+ await supervisor._produce("run-1")
+ run = runs_db.get_run("run-1", "alice")
+ assert (run["status"], run["finishReason"], run["error"]) == (
+ "failed",
+ "error",
+ "database is locked",
+ )
+ stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
+ assert stored and stored == chunks[: len(stored)]
+
+
[email protected]("cancels", [1, 2], ids = ["cancel", "cancel-again-while-settling"])
+async def test_cancel_during_a_commit_waits_for_it_before_settling(
+ durable_run, monkeypatch, cancels
+):
+ chunks = _numbered_chunks(20)
+ leaked = []
+ drained = asyncio.Event()
+
+ async def body():
+ for chunk in chunks:
+ yield f"data: {json.dumps(chunk)}\n\n"
+ drained.set()
+ await asyncio.Event().wait()
+
+ async def fake(*_args, **_kwargs):
+ return SimpleNamespace(status_code = 200, body_iterator = body())
+
+ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
+ entered, gate = _held_append(monkeypatch)
+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
+ loop = asyncio.get_running_loop()
+ previous_handler = loop.get_exception_handler()
+ loop.set_exception_handler(lambda _loop, context: leaked.append(context))
+ cancel_event = threading.Event()
+ task = asyncio.create_task(supervisor._produce("run-1", cancel_event))
+ try:
+ assert await asyncio.to_thread(entered.wait, 2)
+ await asyncio.wait_for(drained.wait(), timeout = 2)
+ cancel_event.set()
+ task.cancel()
+ # A second cancel (shutdown's forced one) lands while settlement waits on the commit; the run must still settle.
+ for _ in range(cancels - 1):
+ loop.call_later(0.05, task.cancel)
+ # finish_run must not overtake the commit still holding its batch.
+ loop.call_later(0.1, gate.set)
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ await asyncio.sleep(0)
+ finally:
+ gate.set()
+ loop.set_exception_handler(previous_handler)
+ assert leaked == []
+ assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"
+ stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
+ assert stored == chunks
+
+
@pytest.mark.asyncio
async def test_model_lifecycle_cancel_reaches_same_registered_event(durable_run, monkeypatch):
registered = asyncio.Event()