Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop generating when the client disconnects on a non-streaming request (#11961)
Nilayb3f8ffe12f84
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex c0e0a2b886b..7ffcb492a9f 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -27347,13 +27347,19 @@ async def produce_openai_chat_completions(# context COPY, so only a slot opened first reaches `_friendly_error`.context_refusal.open_slot()drain_task = asyncio.create_task(asyncio.to_thread(_drain_gguf_tool_loop))- (- full_text,- completion_usage,- completion_finish,- completion_timings,- completion_context_truncation,- ) = await asyncio.shield(drain_task)+ disconnect_watcher = asyncio.create_task(+ _await_disconnect_then_cancel(request, cancel_event)+ )+ try:+ (+ full_text,+ completion_usage,+ completion_finish,+ completion_timings,+ completion_context_truncation,+ ) = await asyncio.shield(drain_task)+ finally:+ await _stop_local_disconnect_cancel_watcher(disconnect_watcher)reasoning_text, visible_text = _extract_responses_reasoning(full_text,parse_think_markers = _responses_should_parse_think_markers(@@ -28070,17 +28076,23 @@ async def produce_openai_chat_completions(# See the tool-loop drain: the slot has to exist before the copies.context_refusal.open_slot()drain_task = asyncio.create_task(asyncio.to_thread(_drain_gguf_choices))- (- _n,- _choices,- _monitor_replies,- _prompt_tokens,- _sum_completion,- _prompt_details,- _last_timings,- _last_finish,- _context_truncation,- ) = await asyncio.shield(drain_task)+ disconnect_watcher = asyncio.create_task(+ _await_disconnect_then_cancel(request, cancel_event)+ )+ try:+ (+ _n,+ _choices,+ _monitor_replies,+ _prompt_tokens,+ _sum_completion,+ _prompt_details,+ _last_timings,+ _last_finish,+ _context_truncation,+ ) = await asyncio.shield(drain_task)+ finally:+ await _stop_local_disconnect_cancel_watcher(disconnect_watcher)_mark_cancelled_json_response_failed(request, cancel_event)@@ -28119,7 +28131,9 @@ async def produce_openai_chat_completions(else None),)- api_monitor.finish(monitor_id)+ api_monitor.finish(+ monitor_id, "cancelled" if cancel_event.is_set() else "completed"+ )return _model_json_response_with_context_truncation(response, _context_truncation)except asyncio.CancelledError:@@ -32875,7 +32889,9 @@ async def _responses_non_streaming(),stop_reason = (choices[0].get("finish_reason") if choices else None),)- api_monitor.finish(monitor_id)+ # The inner chat route stops generating on disconnect and returns the partial reply.+ client_left = await _embeddings_client_gone(request)+ api_monitor.finish(monitor_id, "cancelled" if client_left else "completed")return _model_json_response(response)except asyncio.CancelledError:api_monitor.finish(monitor_id, "cancelled")diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex 37045f7cd21..1cacbb6daff 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -6381,6 +6381,79 @@ class TestGgufVisionToolRouting:asyncio.run(_run())+ @pytest.mark.parametrize(+ "payload_kwargs",+ [+ pytest.param({}, id = "plain"),+ pytest.param({"n": 3}, id = "plain-n"),+ pytest.param({"enable_tools": True}, id = "tools"),+ ],+ )+ def test_non_streaming_gguf_disconnect_stops_generation(self, monkeypatch, payload_kwargs):+ import routes.inference as inf_mod++ total = 200+ emitted = []+ generations = []+ started = threading.Event()++ class _LeavingRequest(self._Request):+ async def is_disconnected(self):+ return started.is_set()++ def _emit(kwargs, event):+ generations.append("plain" if event is None else "tools")+ cancel_event = kwargs["cancel_event"]+ for _ in range(total):+ if cancel_event.wait(0.005):+ return+ emitted.append(1)+ started.set()+ yield event++ def _generate(**kwargs):+ text = ""+ for _ in _emit(kwargs, None):+ text += "x"+ yield text++ def _tools(**kwargs):+ yield from _emit(kwargs, {"type": "content", "text": "x"})++ backend = SimpleNamespace(+ is_loaded = True,+ is_vision = False,+ supports_tools = True,+ _is_audio = False,+ model_identifier = "test-gguf",+ context_length = 4096,+ base_url = "http://llama.disconnect.test",+ effective_parallel_slots = 1,+ generate_chat_completion = _generate,+ generate_chat_completion_with_tools = _tools,+ )+ monitor = ApiMonitor(max_entries = 3)+ monkeypatch.setattr(inf_mod, "api_monitor", monitor)+ monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend)+ monkeypatch.setattr(inf_mod, "_select_request_tools", fake_select_tools)++ payload = ChatCompletionRequest(+ model = "default",+ messages = [{"role": "user", "content": "hi"}],+ **payload_kwargs,+ )+ response = self._drive(+ openai_chat_completions(payload, request = _LeavingRequest(), current_subject = "test")+ )++ assert response.status_code == 200+ assert len(emitted) < total, "generation ran to completion after the client left"+ assert generations == ["tools" if payload_kwargs.get("enable_tools") else "plain"]+ [entry] = monitor.snapshot()+ assert entry["status"] == "cancelled"+ assert monitor.active_count() == 0+ assert get_llama_admission_queue("http://llama.disconnect.test").snapshot().active == 0+def _drive_standard_gguf(self, monkeypatch, date_line: str) -> list[dict]:"""Run one non-tool GGUF completion with the current-date setting pinned."""import routes.inference as inf_moddiff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.pyindex 38f478e17fe..db68d3ccb2e 100644--- a/studio/backend/tests/test_responses_tool_passthrough.py+++ b/studio/backend/tests/test_responses_tool_passthrough.py@@ -1706,6 +1706,37 @@ class TestResponsesNonStreamingAdapter:assert entry["completion_tokens"] == 3assert request.state.skip_api_monitor is False+ def test_monitor_records_client_disconnect_as_cancelled(self, monkeypatch):+ import routes.inference as inf_mod++ async def fake_chat_completions(+ chat_req,+ request,+ current_subject = None,+ ):+ return JSONResponse(+ content = {+ "model": "test-model",+ "choices": [{"message": {"content": "par"}, "finish_reason": "stop"}],+ "usage": {"prompt_tokens": 2, "completion_tokens": 1},+ }+ )++ async def is_disconnected():+ return True++ monitor = ApiMonitor(max_entries = 3)+ monkeypatch.setattr(inf_mod, "api_monitor", monitor)+ monkeypatch.setattr(inf_mod, "openai_chat_completions", fake_chat_completions)+ messages, request = _shared_setup_4()+ request.is_disconnected = is_disconnected++ asyncio.run(_responses_non_streaming(ResponsesRequest(input = "hi"), messages, request))++ [entry] = monitor.snapshot()+ assert entry["status"] == "cancelled"+ assert monitor.active_count() == 0+@staticmethoddef _run_in_process_completion(monkeypatch,