Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop generating when the client disconnects on a non-streaming request (#11961)

Nilayb3f8ffe12f84
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index c0e0a2b886b..7ffcb492a9f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -27347,13 +27347,19 @@ async def produce_openai_chat_completions(
# context COPY, so only a slot opened first reaches `_friendly_error`.
context_refusal.open_slot()
drain_task = asyncio.create_task(asyncio.to_thread(_drain_gguf_tool_loop))
- (
- full_text,
- completion_usage,
- completion_finish,
- completion_timings,
- completion_context_truncation,
- ) = await asyncio.shield(drain_task)
+ disconnect_watcher = asyncio.create_task(
+ _await_disconnect_then_cancel(request, cancel_event)
+ )
+ try:
+ (
+ full_text,
+ completion_usage,
+ completion_finish,
+ completion_timings,
+ completion_context_truncation,
+ ) = await asyncio.shield(drain_task)
+ finally:
+ await _stop_local_disconnect_cancel_watcher(disconnect_watcher)
reasoning_text, visible_text = _extract_responses_reasoning(
full_text,
parse_think_markers = _responses_should_parse_think_markers(
@@ -28070,17 +28076,23 @@ async def produce_openai_chat_completions(
# See the tool-loop drain: the slot has to exist before the copies.
context_refusal.open_slot()
drain_task = asyncio.create_task(asyncio.to_thread(_drain_gguf_choices))
- (
- _n,
- _choices,
- _monitor_replies,
- _prompt_tokens,
- _sum_completion,
- _prompt_details,
- _last_timings,
- _last_finish,
- _context_truncation,
- ) = await asyncio.shield(drain_task)
+ disconnect_watcher = asyncio.create_task(
+ _await_disconnect_then_cancel(request, cancel_event)
+ )
+ try:
+ (
+ _n,
+ _choices,
+ _monitor_replies,
+ _prompt_tokens,
+ _sum_completion,
+ _prompt_details,
+ _last_timings,
+ _last_finish,
+ _context_truncation,
+ ) = await asyncio.shield(drain_task)
+ finally:
+ await _stop_local_disconnect_cancel_watcher(disconnect_watcher)
_mark_cancelled_json_response_failed(request, cancel_event)
@@ -28119,7 +28131,9 @@ async def produce_openai_chat_completions(
else None
),
)
- api_monitor.finish(monitor_id)
+ api_monitor.finish(
+ monitor_id, "cancelled" if cancel_event.is_set() else "completed"
+ )
return _model_json_response_with_context_truncation(response, _context_truncation)
except asyncio.CancelledError:
@@ -32875,7 +32889,9 @@ async def _responses_non_streaming(
),
stop_reason = (choices[0].get("finish_reason") if choices else None),
)
- api_monitor.finish(monitor_id)
+ # The inner chat route stops generating on disconnect and returns the partial reply.
+ client_left = await _embeddings_client_gone(request)
+ api_monitor.finish(monitor_id, "cancelled" if client_left else "completed")
return _model_json_response(response)
except asyncio.CancelledError:
api_monitor.finish(monitor_id, "cancelled")
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index 37045f7cd21..1cacbb6daff 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -6381,6 +6381,79 @@ class TestGgufVisionToolRouting:
asyncio.run(_run())
+ @pytest.mark.parametrize(
+ "payload_kwargs",
+ [
+ pytest.param({}, id = "plain"),
+ pytest.param({"n": 3}, id = "plain-n"),
+ pytest.param({"enable_tools": True}, id = "tools"),
+ ],
+ )
+ def test_non_streaming_gguf_disconnect_stops_generation(self, monkeypatch, payload_kwargs):
+ import routes.inference as inf_mod
+
+ total = 200
+ emitted = []
+ generations = []
+ started = threading.Event()
+
+ class _LeavingRequest(self._Request):
+ async def is_disconnected(self):
+ return started.is_set()
+
+ def _emit(kwargs, event):
+ generations.append("plain" if event is None else "tools")
+ cancel_event = kwargs["cancel_event"]
+ for _ in range(total):
+ if cancel_event.wait(0.005):
+ return
+ emitted.append(1)
+ started.set()
+ yield event
+
+ def _generate(**kwargs):
+ text = ""
+ for _ in _emit(kwargs, None):
+ text += "x"
+ yield text
+
+ def _tools(**kwargs):
+ yield from _emit(kwargs, {"type": "content", "text": "x"})
+
+ backend = SimpleNamespace(
+ is_loaded = True,
+ is_vision = False,
+ supports_tools = True,
+ _is_audio = False,
+ model_identifier = "test-gguf",
+ context_length = 4096,
+ base_url = "http://llama.disconnect.test",
+ effective_parallel_slots = 1,
+ generate_chat_completion = _generate,
+ generate_chat_completion_with_tools = _tools,
+ )
+ monitor = ApiMonitor(max_entries = 3)
+ monkeypatch.setattr(inf_mod, "api_monitor", monitor)
+ monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend)
+ monkeypatch.setattr(inf_mod, "_select_request_tools", fake_select_tools)
+
+ payload = ChatCompletionRequest(
+ model = "default",
+ messages = [{"role": "user", "content": "hi"}],
+ **payload_kwargs,
+ )
+ response = self._drive(
+ openai_chat_completions(payload, request = _LeavingRequest(), current_subject = "test")
+ )
+
+ assert response.status_code == 200
+ assert len(emitted) < total, "generation ran to completion after the client left"
+ assert generations == ["tools" if payload_kwargs.get("enable_tools") else "plain"]
+ [entry] = monitor.snapshot()
+ assert entry["status"] == "cancelled"
+ assert monitor.active_count() == 0
+ assert get_llama_admission_queue("http://llama.disconnect.test").snapshot().active == 0
+
def _drive_standard_gguf(self, monkeypatch, date_line: str) -> list[dict]:
"""Run one non-tool GGUF completion with the current-date setting pinned."""
import routes.inference as inf_mod
diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.py
index 38f478e17fe..db68d3ccb2e 100644
--- a/studio/backend/tests/test_responses_tool_passthrough.py
+++ b/studio/backend/tests/test_responses_tool_passthrough.py
@@ -1706,6 +1706,37 @@ class TestResponsesNonStreamingAdapter:
assert entry["completion_tokens"] == 3
assert request.state.skip_api_monitor is False
+ def test_monitor_records_client_disconnect_as_cancelled(self, monkeypatch):
+ import routes.inference as inf_mod
+
+ async def fake_chat_completions(
+ chat_req,
+ request,
+ current_subject = None,
+ ):
+ return JSONResponse(
+ content = {
+ "model": "test-model",
+ "choices": [{"message": {"content": "par"}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 2, "completion_tokens": 1},
+ }
+ )
+
+ async def is_disconnected():
+ return True
+
+ monitor = ApiMonitor(max_entries = 3)
+ monkeypatch.setattr(inf_mod, "api_monitor", monitor)
+ monkeypatch.setattr(inf_mod, "openai_chat_completions", fake_chat_completions)
+ messages, request = _shared_setup_4()
+ request.is_disconnected = is_disconnected
+
+ asyncio.run(_responses_non_streaming(ResponsesRequest(input = "hi"), messages, request))
+
+ [entry] = monitor.snapshot()
+ assert entry["status"] == "cancelled"
+ assert monitor.active_count() == 0
+
@staticmethod
def _run_in_process_completion(
monkeypatch,