Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop generating when the client goes away (#10096)
Nilayc9f98dbd5e3f
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 59fdcf6e7c4..f32eced8e68 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -5818,17 +5818,22 @@ def _monitor_anthropic_json_response(response,monitor_id: Optional[str],context_length = None,+ cancel_event = None,) -> None:if not monitor_id:return+ # A cancelled non-streaming run still returns a normal 200 body built from the+ # partial output, so the body alone cannot tell the two apart. The streaming+ # sibling reads cancel_event for the same reason.+ status = "cancelled" if cancel_event is not None and cancel_event.is_set() else "completed"body = getattr(response, "body", b"")try:data = json.loads(body.decode("utf-8") if isinstance(body, bytes) else body)except Exception:- api_monitor.finish(monitor_id)+ api_monitor.finish(monitor_id, status)returnif not isinstance(data, dict):- api_monitor.finish(monitor_id)+ api_monitor.finish(monitor_id, status)returntext = _monitor_anthropic_content_blocks(data.get("content"))if text:@@ -5836,7 +5841,7 @@ def _monitor_anthropic_json_response(if data.get("stop_reason"):api_monitor.set_perf(monitor_id, stop_reason = str(data["stop_reason"]))_monitor_anthropic_usage(monitor_id, data.get("usage"), context_length)- api_monitor.finish(monitor_id)+ api_monitor.finish(monitor_id, status)def _monitor_anthropic_response(@@ -5849,7 +5854,7 @@ def _monitor_anthropic_response(return responsebody_iterator = getattr(response, "body_iterator", None)if body_iterator is None:- _monitor_anthropic_json_response(response, monitor_id, context_length)+ _monitor_anthropic_json_response(response, monitor_id, context_length, cancel_event)return responseasync def _monitored_body():@@ -28724,6 +28729,7 @@ async def anthropic_messages()return await _admitted_anthropic(_anthropic_tool_non_streaming(+ request,_run_tool_gen,message_id,model_name,@@ -28776,6 +28782,7 @@ async def anthropic_messages()return await _admitted_anthropic(_anthropic_plain_non_streaming(+ request,_run_plain_gen,message_id,model_name,@@ -29340,6 +29347,7 @@ def _anthropic_tool_response_from_events(async def _anthropic_tool_non_streaming(+ request,run_gen,message_id,model_name,@@ -29362,12 +29370,21 @@ async def _anthropic_tool_non_streaming(think_provenance = think_provenance,)- return await _run_blocking_generation(- _drain_and_build,- cancel_event,- name = "anthropic-tool-nonstream",- daemon = True,- )+ disconnect_watcher = asyncio.create_task(_await_disconnect_then_cancel(request, cancel_event))+ try:+ response = await _run_blocking_generation(+ _drain_and_build,+ cancel_event,+ name = "anthropic-tool-nonstream",+ daemon = True,+ )+ finally:+ await _stop_local_disconnect_cancel_watcher(disconnect_watcher)+ # The 200 is built from partial output, so unmarked the keepwarm middleware reads it+ # as a clean completion and takes the resident model away from a preview.+ if cancel_event is not None:+ _mark_cancelled_json_response_failed(request, cancel_event)+ return responsedef _anthropic_plain_response_from_events(@@ -29417,6 +29434,7 @@ def _anthropic_plain_response_from_events(async def _anthropic_plain_non_streaming(+ request,run_gen,message_id,model_name,@@ -29435,11 +29453,20 @@ async def _anthropic_plain_non_streaming(think_provenance = think_provenance,)- return await _run_blocking_generation(- _drain_and_build,- cancel_event,- name = "anthropic-plain-nonstream",- )+ disconnect_watcher = asyncio.create_task(_await_disconnect_then_cancel(request, cancel_event))+ try:+ response = await _run_blocking_generation(+ _drain_and_build,+ cancel_event,+ name = "anthropic-plain-nonstream",+ )+ finally:+ await _stop_local_disconnect_cancel_watcher(disconnect_watcher)+ # The 200 is built from partial output, so unmarked the keepwarm middleware reads it+ # as a clean completion and takes the resident model away from a preview.+ if cancel_event is not None:+ _mark_cancelled_json_response_failed(request, cancel_event)+ return response# =====================================================================diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.pyindex 3647a6c77af..ce48304d7f7 100644--- a/studio/backend/tests/test_anthropic_messages.py+++ b/studio/backend/tests/test_anthropic_messages.py@@ -1458,6 +1458,13 @@ class TestAnthropicStreamEmitter:# =====================================================================+def _connected_request(disconnected = False):+ async def _is_disconnected():+ return disconnected++ return SimpleNamespace(is_disconnected = _is_disconnected)++class TestAnthropicToolNonStreaming:@pytest.mark.parametrize(("helper", "event"),@@ -1480,7 +1487,7 @@ class TestAnthropicToolNonStreaming:yield eventasync def _run():- task = asyncio.create_task(helper(_run_gen, "msg_1", "m"))+ task = asyncio.create_task(helper(_connected_request(), _run_gen, "msg_1", "m"))await asyncio.sleep(0)heartbeat_ticks = 0while not task.done():@@ -1512,7 +1519,9 @@ class TestAnthropicToolNonStreaming:yield {"type": "content", "text": "ok"}async def _run():- task = asyncio.create_task(_anthropic_tool_non_streaming(_run_gen, "msg_1", "m"))+ task = asyncio.create_task(+ _anthropic_tool_non_streaming(_connected_request(), _run_gen, "msg_1", "m")+ )await asyncio.sleep(0)heartbeat_ticks = 0while not task.done():@@ -1551,7 +1560,9 @@ class TestAnthropicToolNonStreaming:yield eventasync def _cancel_generation():- task = asyncio.create_task(helper(_run_gen, "msg_1", "m", cancel_event = cancel_event))+ task = asyncio.create_task(+ helper(_connected_request(), _run_gen, "msg_1", "m", cancel_event = cancel_event)+ )assert await asyncio.to_thread(generator_started.wait, 1.0)task.cancel()with pytest.raises(asyncio.CancelledError):@@ -1561,6 +1572,43 @@ class TestAnthropicToolNonStreaming:asyncio.run(_cancel_generation())+ @pytest.mark.parametrize(+ ("helper", "event"),+ [+ pytest.param(+ _anthropic_tool_non_streaming,+ {"type": "content", "text": "ok"},+ id = "tools",+ ),+ pytest.param(_anthropic_plain_non_streaming, "ok", id = "plain"),+ ],+ )+ def test_client_disconnect_cancels_generation(self, helper, event):+ generator_started = threading.Event()+ cancel_event = threading.Event()+ emitted = 0++ def _run_gen():+ nonlocal emitted+ generator_started.set()+ for _ in range(400):+ if cancel_event.wait(0.005):+ return+ emitted += 1+ yield event++ async def _drive():+ request = _connected_request(disconnected = True)+ response = await helper(request, _run_gen, "msg_1", "m", cancel_event = cancel_event)+ assert await asyncio.to_thread(generator_started.wait, 1.0)+ return response++ response = asyncio.run(_drive())++ assert cancel_event.is_set()+ assert emitted < 400+ assert response.status_code == 200+def test_duplicate_tool_start_replaces_provisional_tool_block(self):def _run_gen():yield {@@ -1582,7 +1630,9 @@ class TestAnthropicToolNonStreaming:"result": "Rendered HTML canvas.",}- response = asyncio.run(_anthropic_tool_non_streaming(_run_gen, "msg_1", "m"))+ response = asyncio.run(+ _anthropic_tool_non_streaming(_connected_request(), _run_gen, "msg_1", "m")+ )body = json.loads(response.body)tool_blocks = [block for block in body["content"] if block["type"] == "tool_use"]@@ -1603,7 +1653,9 @@ class TestAnthropicToolNonStreaming:tools = [{"type": "function", "function": {"name": "web_search", "parameters": {}}}]response = asyncio.run(- _anthropic_tool_non_streaming(_run_gen, "msg_1", "m", openai_tools = tools)+ _anthropic_tool_non_streaming(+ _connected_request(), _run_gen, "msg_1", "m", openai_tools = tools+ ))body = json.loads(response.body)text = "".join(b["text"] for b in body["content"] if b["type"] == "text")@@ -2576,6 +2628,68 @@ class TestAnthropicMessagesToolRouting:assert entry["context_length"] == 2048assert monitor.active_count() == 0+ @pytest.mark.parametrize("with_tools", [False, True])+ def test_non_streaming_disconnect_stops_generation_and_records_it_cancelled(+ self, monkeypatch, with_tools+ ):+ # The route already 499s a client that is gone before admission, so the gap is a+ # client that leaves once a slot was granted and tokens are already flowing.+ import routes.inference as inf_mod++ total = 200+ emitted = []+ started = threading.Event()++ class _LeavingRequest:+ state = SimpleNamespace()+ url = SimpleNamespace(path = "/v1/messages")+ method = "POST"++ async def is_disconnected(self):+ return started.is_set()++ def _emit(kwargs, event):+ cancel_event = kwargs["cancel_event"]+ for _ in range(total):+ if cancel_event.wait(0.005):+ return+ emitted.append(1)+ started.set()+ yield event++ def _gen_plain(**kwargs):+ text = ""+ for _ in _emit(kwargs, None):+ text += "x"+ yield text++ def _gen_tools(**kwargs):+ yield from _emit(kwargs, {"type": "content", "text": "x"})++ _mock_backend(+ monkeypatch,+ supports_tool_passthrough = False,+ generate_chat_completion = _gen_plain,+ generate_chat_completion_with_tools = _gen_tools,+ )+ monitor = ApiMonitor(max_entries = 3)+ monkeypatch.setattr(inf_mod, "api_monitor", monitor)+ fields = {"tools": [{"name": "x", "input_schema": {"type": "object"}}]} if with_tools else {}++ response = _drive(+ anthropic_messages(+ _basic_payload(**fields), request = _LeavingRequest(), current_subject = "t"+ )+ )++ assert response.status_code == 200+ assert len(emitted) < total, "generation ran to completion after the client left"+ [entry] = monitor.snapshot()+ assert entry["status"] == "cancelled"+ # A cancelled run has no natural stop reason; end_turn would read as a full answer.+ assert entry["stop_reason"] is None+ assert monitor.active_count() == 0+@pytest.mark.parametrize("stream", [False, True])@pytest.mark.parametrize("with_tools", [False, True])def test_reasoning_only_output_is_not_duplicated(self, monkeypatch, stream, with_tools):diff --git a/studio/backend/tests/test_preview_routes.py b/studio/backend/tests/test_preview_routes.pyindex 5d85d3d830d..9942dd2ee67 100644--- a/studio/backend/tests/test_preview_routes.py+++ b/studio/backend/tests/test_preview_routes.py@@ -773,6 +773,47 @@ def test_cancelled_json_response_does_not_claim_slot(slot_state):_reset_keepwarm_counters()+def test_cancelled_anthropic_non_streaming_does_not_claim_slot(slot_state):+ """/v1/messages answers 200 from partial output after a disconnect, same as the twin."""+ _reset_keepwarm_counters()+ inference._set_preview_resident("/outputs/run/ckpt")++ total = 200+ emitted = []++ async def _app(scope, receive, send):+ cancel_event = threading.Event()+ started = threading.Event()++ class _LeavingRequest:+ def __init__(self):+ self.scope = scope++ async def is_disconnected(self):+ return started.is_set()++ def _run_gen():+ for _ in range(total):+ if cancel_event.wait(0.005):+ return+ emitted.append(1)+ started.set()+ yield "x"++ response = await inference._anthropic_plain_non_streaming(+ _LeavingRequest(), _run_gen, "msg_1", "m", cancel_event = cancel_event+ )+ assert response.status_code == 200+ assert cancel_event.is_set()+ await send({"type": "http.response.start", "status": 200, "headers": []})+ await send({"type": "http.response.body", "body": b"{}", "more_body": False})++ _run_middleware(_app, "/v1/messages")+ assert len(emitted) < total+ assert inference._is_preview_resident("/outputs/run/ckpt")+ _reset_keepwarm_counters()++def test_queued_preview_does_not_deadlock_studio_switch():from core.inference import llama_keepwarm as kwasync def _run():