Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stream durable chat runs at display frame rate (#11900)
Michael Hana8d998f8d6b4
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.pyindex 5d6bdae9b29..6b136a3600f 100644--- a/studio/backend/core/inference/chat_generation_runs.py+++ b/studio/backend/core/inference/chat_generation_runs.py@@ -26,9 +26,8 @@ from storage import chat_generation_runs_db as dblogger = get_logger(__name__)_EVENT_BATCH_SIZE = 16-_EVENT_BATCH_MIN_SIZE = 2-_EVENT_BATCH_SECONDS = 0.1-_EVENT_SINGLE_FLUSH_SECONDS = 1.0+# Followers only see appended chunks, so this is the chat's text frame rate: at most one commit per display frame.+_EVENT_FLUSH_SECONDS = 1 / 60_SHUTDOWN_GRACE_SECONDS = 10.0# Second budget, after task.cancel(). Shorter than the grace period: by this point the run is already being abandoned,# and the only question is whether shutdown returns.@@ -771,15 +770,7 @@ class ChatGenerationSupervisor:next_raw_task = asyncio.create_task(iterator.__anext__())while True:timeout = (- max(- 0.0,- (- _EVENT_BATCH_SECONDS- if len(pending) >= _EVENT_BATCH_MIN_SIZE- else _EVENT_SINGLE_FLUSH_SECONDS- )- - (time.monotonic() - last_flush),- )+ max(0.0, _EVENT_FLUSH_SECONDS - (time.monotonic() - last_flush))if pendingelse None)@@ -828,9 +819,9 @@ class ChatGenerationSupervisor:finish_reason = _chunk_finish_reason(chunk) or finish_reasonerror = _chunk_error(chunk) or errornow = time.monotonic()- if len(pending) >= _EVENT_BATCH_SIZE or (- len(pending) >= _EVENT_BATCH_MIN_SIZE- and now - last_flush >= _EVENT_BATCH_SECONDS+ if (+ len(pending) >= _EVENT_BATCH_SIZE+ or now - last_flush >= _EVENT_FLUSH_SECONDS):await asyncio.to_thread(db.append_events,diff --git a/studio/backend/tests/test_chat_generation_supervisor.py b/studio/backend/tests/test_chat_generation_supervisor.pyindex c81daac2f7e..f8dcde4e1d5 100644--- a/studio/backend/tests/test_chat_generation_supervisor.py+++ b/studio/backend/tests/test_chat_generation_supervisor.py@@ -14,8 +14,7 @@ from starlette.requests import Requestfrom core.inference import llama_keepwarmfrom core.inference.chat_generation_runs import (- _EVENT_BATCH_SECONDS,- _EVENT_SINGLE_FLUSH_SECONDS,+ _EVENT_FLUSH_SECONDS,ChatGenerationSupervisor,)from models.inference import ChatCompletionRequest@@ -285,8 +284,7 @@ async def test_a_prefill_reporting_only_progress_renews_the_lease(durable_run, masync def body():for processed in (1024, 8192, 65536):yield f"data: {json.dumps(_progress(processed))}\n\n"- # Polled, not slept: the idle flush is on a 0.1s timer- # (_EVENT_BATCH_SECONDS) and a sleep sized against it flakes under load.+ # Polled, not slept: a sleep sized against the flush timer flakes under load._deadline = time.monotonic() + 10.0while time.monotonic() < _deadline:sampled["events"] = [@@ -435,19 +433,41 @@ async def test_event_batch_flushes_while_upstream_is_idle(durable_run, monkeypatmonkeypatch.setattr(inference, "produce_openai_chat_completions", fake)supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))task = asyncio.create_task(supervisor._produce("run-1"))- # Poll rather than sleep a fixed span. The flush costs the batch timer plus a- # thread hop and a SQLite write, which measures ~0.11s on an idle machine, so- # the old bare sleep(0.2) left under 2x headroom and lost the race on a loaded- # runner. The budget is still bounded well below _EVENT_SINGLE_FLUSH_SECONDS,- # so a regression that drops these two events onto the single-event timer, or- # never flushes them at all, still fails here rather than passing slowly.- deadline = (_EVENT_BATCH_SECONDS + _EVENT_SINGLE_FLUSH_SECONDS) / 2- stored = await _await_chunk_payloads("run-1", len(chunks), deadline)+ # Polled, not slept: a fixed sleep races the flush on a loaded runner.+ stored = await _await_chunk_payloads("run-1", len(chunks), 0.5)assert stored == chunksrelease.set()await task+def test_event_flush_interval_is_one_display_frame():+ # This interval is the chat's text frame rate; 0.1s made streaming stutter (#11778).+ assert _EVENT_FLUSH_SECONDS <= 1 / 60+++async def test_a_lone_chunk_is_appended_without_waiting_for_another(durable_run, monkeypatch):+ # A lone chunk used to wait up to 1s for a second one before it was appended.+ release = asyncio.Event()+ chunk = {"choices": [{"delta": {"content": "Hello"}}]}++ async def body():+ yield f"data: {json.dumps(chunk)}\n\n"+ await release.wait()+ yield "data: [DONE]\n\n"++ async def fake(*_args, **_kwargs):+ return SimpleNamespace(status_code = 200, body_iterator = body())++ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))+ task = asyncio.create_task(supervisor._produce("run-1"))+ stored = await _await_chunk_payloads("run-1", 1, 0.5)+ assert stored == [chunk]+ release.set()+ await task++@pytest.mark.asyncioasync def test_model_lifecycle_cancel_reaches_same_registered_event(durable_run, monkeypatch):registered = asyncio.Event()