Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop generating when the client goes away (#10096)

Nilayc9f98dbd5e3f
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 59fdcf6e7c4..f32eced8e68 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -5818,17 +5818,22 @@ def _monitor_anthropic_json_response(
response,
monitor_id: Optional[str],
context_length = None,
+ cancel_event = None,
) -> None:
if not monitor_id:
return
+ # A cancelled non-streaming run still returns a normal 200 body built from the
+ # partial output, so the body alone cannot tell the two apart. The streaming
+ # sibling reads cancel_event for the same reason.
+ status = "cancelled" if cancel_event is not None and cancel_event.is_set() else "completed"
body = getattr(response, "body", b"")
try:
data = json.loads(body.decode("utf-8") if isinstance(body, bytes) else body)
except Exception:
- api_monitor.finish(monitor_id)
+ api_monitor.finish(monitor_id, status)
return
if not isinstance(data, dict):
- api_monitor.finish(monitor_id)
+ api_monitor.finish(monitor_id, status)
return
text = _monitor_anthropic_content_blocks(data.get("content"))
if text:
@@ -5836,7 +5841,7 @@ def _monitor_anthropic_json_response(
if data.get("stop_reason"):
api_monitor.set_perf(monitor_id, stop_reason = str(data["stop_reason"]))
_monitor_anthropic_usage(monitor_id, data.get("usage"), context_length)
- api_monitor.finish(monitor_id)
+ api_monitor.finish(monitor_id, status)
def _monitor_anthropic_response(
@@ -5849,7 +5854,7 @@ def _monitor_anthropic_response(
return response
body_iterator = getattr(response, "body_iterator", None)
if body_iterator is None:
- _monitor_anthropic_json_response(response, monitor_id, context_length)
+ _monitor_anthropic_json_response(response, monitor_id, context_length, cancel_event)
return response
async def _monitored_body():
@@ -28724,6 +28729,7 @@ async def anthropic_messages(
)
return await _admitted_anthropic(
_anthropic_tool_non_streaming(
+ request,
_run_tool_gen,
message_id,
model_name,
@@ -28776,6 +28782,7 @@ async def anthropic_messages(
)
return await _admitted_anthropic(
_anthropic_plain_non_streaming(
+ request,
_run_plain_gen,
message_id,
model_name,
@@ -29340,6 +29347,7 @@ def _anthropic_tool_response_from_events(
async def _anthropic_tool_non_streaming(
+ request,
run_gen,
message_id,
model_name,
@@ -29362,12 +29370,21 @@ async def _anthropic_tool_non_streaming(
think_provenance = think_provenance,
)
- return await _run_blocking_generation(
- _drain_and_build,
- cancel_event,
- name = "anthropic-tool-nonstream",
- daemon = True,
- )
+ disconnect_watcher = asyncio.create_task(_await_disconnect_then_cancel(request, cancel_event))
+ try:
+ response = await _run_blocking_generation(
+ _drain_and_build,
+ cancel_event,
+ name = "anthropic-tool-nonstream",
+ daemon = True,
+ )
+ finally:
+ await _stop_local_disconnect_cancel_watcher(disconnect_watcher)
+ # The 200 is built from partial output, so unmarked the keepwarm middleware reads it
+ # as a clean completion and takes the resident model away from a preview.
+ if cancel_event is not None:
+ _mark_cancelled_json_response_failed(request, cancel_event)
+ return response
def _anthropic_plain_response_from_events(
@@ -29417,6 +29434,7 @@ def _anthropic_plain_response_from_events(
async def _anthropic_plain_non_streaming(
+ request,
run_gen,
message_id,
model_name,
@@ -29435,11 +29453,20 @@ async def _anthropic_plain_non_streaming(
think_provenance = think_provenance,
)
- return await _run_blocking_generation(
- _drain_and_build,
- cancel_event,
- name = "anthropic-plain-nonstream",
- )
+ disconnect_watcher = asyncio.create_task(_await_disconnect_then_cancel(request, cancel_event))
+ try:
+ response = await _run_blocking_generation(
+ _drain_and_build,
+ cancel_event,
+ name = "anthropic-plain-nonstream",
+ )
+ finally:
+ await _stop_local_disconnect_cancel_watcher(disconnect_watcher)
+ # The 200 is built from partial output, so unmarked the keepwarm middleware reads it
+ # as a clean completion and takes the resident model away from a preview.
+ if cancel_event is not None:
+ _mark_cancelled_json_response_failed(request, cancel_event)
+ return response
# =====================================================================
diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py
index 3647a6c77af..ce48304d7f7 100644
--- a/studio/backend/tests/test_anthropic_messages.py
+++ b/studio/backend/tests/test_anthropic_messages.py
@@ -1458,6 +1458,13 @@ class TestAnthropicStreamEmitter:
# =====================================================================
+def _connected_request(disconnected = False):
+ async def _is_disconnected():
+ return disconnected
+
+ return SimpleNamespace(is_disconnected = _is_disconnected)
+
+
class TestAnthropicToolNonStreaming:
@pytest.mark.parametrize(
("helper", "event"),
@@ -1480,7 +1487,7 @@ class TestAnthropicToolNonStreaming:
yield event
async def _run():
- task = asyncio.create_task(helper(_run_gen, "msg_1", "m"))
+ task = asyncio.create_task(helper(_connected_request(), _run_gen, "msg_1", "m"))
await asyncio.sleep(0)
heartbeat_ticks = 0
while not task.done():
@@ -1512,7 +1519,9 @@ class TestAnthropicToolNonStreaming:
yield {"type": "content", "text": "ok"}
async def _run():
- task = asyncio.create_task(_anthropic_tool_non_streaming(_run_gen, "msg_1", "m"))
+ task = asyncio.create_task(
+ _anthropic_tool_non_streaming(_connected_request(), _run_gen, "msg_1", "m")
+ )
await asyncio.sleep(0)
heartbeat_ticks = 0
while not task.done():
@@ -1551,7 +1560,9 @@ class TestAnthropicToolNonStreaming:
yield event
async def _cancel_generation():
- task = asyncio.create_task(helper(_run_gen, "msg_1", "m", cancel_event = cancel_event))
+ task = asyncio.create_task(
+ helper(_connected_request(), _run_gen, "msg_1", "m", cancel_event = cancel_event)
+ )
assert await asyncio.to_thread(generator_started.wait, 1.0)
task.cancel()
with pytest.raises(asyncio.CancelledError):
@@ -1561,6 +1572,43 @@ class TestAnthropicToolNonStreaming:
asyncio.run(_cancel_generation())
+ @pytest.mark.parametrize(
+ ("helper", "event"),
+ [
+ pytest.param(
+ _anthropic_tool_non_streaming,
+ {"type": "content", "text": "ok"},
+ id = "tools",
+ ),
+ pytest.param(_anthropic_plain_non_streaming, "ok", id = "plain"),
+ ],
+ )
+ def test_client_disconnect_cancels_generation(self, helper, event):
+ generator_started = threading.Event()
+ cancel_event = threading.Event()
+ emitted = 0
+
+ def _run_gen():
+ nonlocal emitted
+ generator_started.set()
+ for _ in range(400):
+ if cancel_event.wait(0.005):
+ return
+ emitted += 1
+ yield event
+
+ async def _drive():
+ request = _connected_request(disconnected = True)
+ response = await helper(request, _run_gen, "msg_1", "m", cancel_event = cancel_event)
+ assert await asyncio.to_thread(generator_started.wait, 1.0)
+ return response
+
+ response = asyncio.run(_drive())
+
+ assert cancel_event.is_set()
+ assert emitted < 400
+ assert response.status_code == 200
+
def test_duplicate_tool_start_replaces_provisional_tool_block(self):
def _run_gen():
yield {
@@ -1582,7 +1630,9 @@ class TestAnthropicToolNonStreaming:
"result": "Rendered HTML canvas.",
}
- response = asyncio.run(_anthropic_tool_non_streaming(_run_gen, "msg_1", "m"))
+ response = asyncio.run(
+ _anthropic_tool_non_streaming(_connected_request(), _run_gen, "msg_1", "m")
+ )
body = json.loads(response.body)
tool_blocks = [block for block in body["content"] if block["type"] == "tool_use"]
@@ -1603,7 +1653,9 @@ class TestAnthropicToolNonStreaming:
tools = [{"type": "function", "function": {"name": "web_search", "parameters": {}}}]
response = asyncio.run(
- _anthropic_tool_non_streaming(_run_gen, "msg_1", "m", openai_tools = tools)
+ _anthropic_tool_non_streaming(
+ _connected_request(), _run_gen, "msg_1", "m", openai_tools = tools
+ )
)
body = json.loads(response.body)
text = "".join(b["text"] for b in body["content"] if b["type"] == "text")
@@ -2576,6 +2628,68 @@ class TestAnthropicMessagesToolRouting:
assert entry["context_length"] == 2048
assert monitor.active_count() == 0
+ @pytest.mark.parametrize("with_tools", [False, True])
+ def test_non_streaming_disconnect_stops_generation_and_records_it_cancelled(
+ self, monkeypatch, with_tools
+ ):
+ # The route already 499s a client that is gone before admission, so the gap is a
+ # client that leaves once a slot was granted and tokens are already flowing.
+ import routes.inference as inf_mod
+
+ total = 200
+ emitted = []
+ started = threading.Event()
+
+ class _LeavingRequest:
+ state = SimpleNamespace()
+ url = SimpleNamespace(path = "/v1/messages")
+ method = "POST"
+
+ async def is_disconnected(self):
+ return started.is_set()
+
+ def _emit(kwargs, event):
+ cancel_event = kwargs["cancel_event"]
+ for _ in range(total):
+ if cancel_event.wait(0.005):
+ return
+ emitted.append(1)
+ started.set()
+ yield event
+
+ def _gen_plain(**kwargs):
+ text = ""
+ for _ in _emit(kwargs, None):
+ text += "x"
+ yield text
+
+ def _gen_tools(**kwargs):
+ yield from _emit(kwargs, {"type": "content", "text": "x"})
+
+ _mock_backend(
+ monkeypatch,
+ supports_tool_passthrough = False,
+ generate_chat_completion = _gen_plain,
+ generate_chat_completion_with_tools = _gen_tools,
+ )
+ monitor = ApiMonitor(max_entries = 3)
+ monkeypatch.setattr(inf_mod, "api_monitor", monitor)
+ fields = {"tools": [{"name": "x", "input_schema": {"type": "object"}}]} if with_tools else {}
+
+ response = _drive(
+ anthropic_messages(
+ _basic_payload(**fields), request = _LeavingRequest(), current_subject = "t"
+ )
+ )
+
+ assert response.status_code == 200
+ assert len(emitted) < total, "generation ran to completion after the client left"
+ [entry] = monitor.snapshot()
+ assert entry["status"] == "cancelled"
+ # A cancelled run has no natural stop reason; end_turn would read as a full answer.
+ assert entry["stop_reason"] is None
+ assert monitor.active_count() == 0
+
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("with_tools", [False, True])
def test_reasoning_only_output_is_not_duplicated(self, monkeypatch, stream, with_tools):
diff --git a/studio/backend/tests/test_preview_routes.py b/studio/backend/tests/test_preview_routes.py
index 5d85d3d830d..9942dd2ee67 100644
--- a/studio/backend/tests/test_preview_routes.py
+++ b/studio/backend/tests/test_preview_routes.py
@@ -773,6 +773,47 @@ def test_cancelled_json_response_does_not_claim_slot(slot_state):
_reset_keepwarm_counters()
+def test_cancelled_anthropic_non_streaming_does_not_claim_slot(slot_state):
+ """/v1/messages answers 200 from partial output after a disconnect, same as the twin."""
+ _reset_keepwarm_counters()
+ inference._set_preview_resident("/outputs/run/ckpt")
+
+ total = 200
+ emitted = []
+
+ async def _app(scope, receive, send):
+ cancel_event = threading.Event()
+ started = threading.Event()
+
+ class _LeavingRequest:
+ def __init__(self):
+ self.scope = scope
+
+ async def is_disconnected(self):
+ return started.is_set()
+
+ def _run_gen():
+ for _ in range(total):
+ if cancel_event.wait(0.005):
+ return
+ emitted.append(1)
+ started.set()
+ yield "x"
+
+ response = await inference._anthropic_plain_non_streaming(
+ _LeavingRequest(), _run_gen, "msg_1", "m", cancel_event = cancel_event
+ )
+ assert response.status_code == 200
+ assert cancel_event.is_set()
+ await send({"type": "http.response.start", "status": 200, "headers": []})
+ await send({"type": "http.response.body", "body": b"{}", "more_body": False})
+
+ _run_middleware(_app, "/v1/messages")
+ assert len(emitted) < total
+ assert inference._is_preview_resident("/outputs/run/ckpt")
+ _reset_keepwarm_counters()
+
+
def test_queued_preview_does_not_deadlock_studio_switch():
from core.inference import llama_keepwarm as kw
async def _run():