Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-8986): stop SSE disconnect from killing the copilot agent (#5560)
Andrew Neilson2fa21f8799a3
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 152bc24b1..c1ccc8333 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -281,7 +281,6 @@ async def run_copilot_agent(from skyvern.cli.mcp_tools import mcp as skyvern_mcpfrom skyvern.forge.sdk.copilot.enforcement import (- CopilotClientDisconnectedError,CopilotTotalTimeoutError,run_with_enforcement,)@@ -392,8 +391,8 @@ async def run_copilot_agent(chat_request,organization_id,)- except (CopilotClientDisconnectedError, asyncio.CancelledError):- LOG.info("Copilot client disconnected")+ except asyncio.CancelledError:+ LOG.info("Copilot run cancelled")return AgentResult(user_response="Request cancelled.",updated_workflow=ctx.last_workflow,diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex a0a50084e..9b23e099f 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -246,10 +246,6 @@ def _is_progress_narration(user_response: Any) -> bool:return any(pattern.search(user_response) for pattern in _PROGRESS_NARRATION_PATTERNS)-class CopilotClientDisconnectedError(Exception):- """Raised when the client disconnects during agent execution."""--class CopilotTotalTimeoutError(Exception):"""Raised when the copilot agent exceeds the total allowed runtime."""@@ -853,9 +849,10 @@ async def run_with_enforcement(pending_recovery_nudge: str | None = Nonewhile True:- if await stream.is_disconnected():- raise CopilotClientDisconnectedError()-+ # Client disconnect is no longer treated as a stop signal. The+ # SSE stream silently drops events once the browser is gone, but+ # the agent keeps running so the reply can be persisted to the+ # chat history on the server side (see SKY-8986).elapsed = time.monotonic() - start_timeif elapsed > TOTAL_TIMEOUT_SECONDS:raise CopilotTotalTimeoutError()@@ -924,9 +921,6 @@ async def run_with_enforcement()raise- if await stream.is_disconnected():- raise CopilotClientDisconnectedError()-# Inject pending screenshots as a follow-up user message because OpenAI# rejects images in tool messages.screenshot_msg = _consume_pending_screenshots(ctx)diff --git a/skyvern/forge/sdk/copilot/exceptions.py b/skyvern/forge/sdk/copilot/exceptions.pydeleted file mode 100644index 470ddcea7..000000000--- a/skyvern/forge/sdk/copilot/exceptions.py+++ /dev/null@@ -1,7 +0,0 @@-from __future__ import annotations--from skyvern.exceptions import SkyvernException---class CopilotClientDisconnectedError(SkyvernException):- """Raised when the SSE client disconnects during a streaming copilot run."""diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.pyindex 14d030b37..efc23d759 100644--- a/skyvern/forge/sdk/copilot/streaming_adapter.py+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py@@ -11,7 +11,6 @@ import structlog# Reuse the HTTP-logging redactor so SSE tool inputs and request-body logs# share one exact-match sensitive-key policy.from skyvern.forge.request_logging import redact_sensitive_fields-from skyvern.forge.sdk.copilot.exceptions import CopilotClientDisconnectedErrorfrom skyvern.forge.sdk.copilot.output_utils import summarize_tool_resultfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotStreamMessageType,@@ -51,13 +50,15 @@ async def stream_to_sse(``post_update_nudge_count``, ``navigate_called``, and``observation_after_navigate``.- A true client disconnect -- detected by ``stream.is_disconnected()`` or a- failed ``stream.send()`` -- cancels the agent run and raises- ``CopilotClientDisconnectedError``. A plain ``asyncio.CancelledError``- from some other source (task-group cancel, upstream timeout, parent- abort) is allowed to propagate unchanged so asyncio's cancellation- machinery runs normally; callers that want the two to share a UX path- should catch both at the call site.+ A client disconnect does NOT cancel the agent run: we continue to iterate+ ``result.stream_events()`` so the agent completes whatever work it is+ in the middle of and the caller can persist the reply to the DB. Events+ sent through ``stream.send`` after disconnect are silently dropped by the+ stream, so the queue cannot grow unbounded.++ Real asyncio cancellation (server shutdown, parent task cancelled for+ reasons unrelated to a dropped client) is re-raised unchanged so+ asyncio's cancellation machinery still runs normally."""from agents.stream_events import RunItemStreamEvent@@ -69,39 +70,40 @@ async def stream_to_sse(try:async for event in result.stream_events():- if await stream.is_disconnected():- result.cancel()- raise CopilotClientDisconnectedError()if not isinstance(event, RunItemStreamEvent):continue+ # Skip emission work (serialization, redaction) once the client+ # is gone, but keep draining the SDK stream so the agent can+ # finish. stream.send below would drop the payload anyway.+ client_gone = await stream.is_disconnected()+if event.name == "tool_called":raw = event.item.raw_itemcall_id = _get_raw_field(raw, "call_id") or _get_raw_field(raw, "id") or ""tool_name = _get_raw_field(raw, "name") or "unknown"call_id_to_name[call_id] = tool_name- raw_args = _get_raw_field(raw, "arguments")- tool_input: dict[str, Any] = {}- if isinstance(raw_args, str):- try:- tool_input = json.loads(raw_args)- except (json.JSONDecodeError, TypeError):- tool_input = {"raw": raw_args}- elif isinstance(raw_args, dict):- tool_input = raw_args-- if not await stream.send(- WorkflowCopilotToolCallUpdate(- type=WorkflowCopilotStreamMessageType.TOOL_CALL,- tool_name=tool_name,- tool_input=_sanitize_input(tool_input),- iteration=iteration,- tool_call_id=call_id,+ if not client_gone:+ raw_args = _get_raw_field(raw, "arguments")+ tool_input: dict[str, Any] = {}+ if isinstance(raw_args, str):+ try:+ tool_input = json.loads(raw_args)+ except (json.JSONDecodeError, TypeError):+ tool_input = {"raw": raw_args}+ elif isinstance(raw_args, dict):+ tool_input = raw_args++ await stream.send(+ WorkflowCopilotToolCallUpdate(+ type=WorkflowCopilotStreamMessageType.TOOL_CALL,+ tool_name=tool_name,+ tool_input=_sanitize_input(tool_input),+ iteration=iteration,+ tool_call_id=call_id,+ ))- ):- result.cancel()- raise CopilotClientDisconnectedError()elif event.name == "tool_output":raw = event.item.raw_item@@ -110,31 +112,27 @@ async def stream_to_sse(output = getattr(event.item, "output", None)parsed = parse_tool_output(output)- summary = summarize_tool_result(tool_name, parsed)- success = parsed.get("ok", True)-- if not await stream.send(- WorkflowCopilotToolResultUpdate(- type=WorkflowCopilotStreamMessageType.TOOL_RESULT,- tool_name=tool_name,- success=success,- summary=summary,- iteration=iteration,- tool_call_id=call_id,++ if not client_gone:+ summary = summarize_tool_result(tool_name, parsed)+ success = parsed.get("ok", True)+ await stream.send(+ WorkflowCopilotToolResultUpdate(+ type=WorkflowCopilotStreamMessageType.TOOL_RESULT,+ tool_name=tool_name,+ success=success,+ summary=summary,+ iteration=iteration,+ tool_call_id=call_id,+ ))- ):- result.cancel()- raise CopilotClientDisconnectedError()_update_enforcement_from_tool(ctx, tool_name, parsed)iteration += 1except asyncio.CancelledError:- # Don't relabel generic cancellation as a client disconnect -- the- # inline is_disconnected / send-failure branches above already raise- # CopilotClientDisconnectedError when there is real evidence of a- # dropped client. Preserve cancellation semantics by re-raising so- # asyncio's task machinery sees the cancel and any upstream- # except Exception does NOT swallow it.+ # Real cancellation (server shutdown, upstream abort). Propagate so+ # asyncio's task machinery sees the cancel; also cancel the SDK+ # run to free provider resources.result.cancel()raisediff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 59fe8202d..fafa04f19 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -826,9 +826,12 @@ async def _run_blocks_and_collect_debug(final_status = run.statusbreak- if await ctx.stream.is_disconnected():- await _cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id)- return {"ok": False, "error": "Client disconnected during block execution."}+ # Deliberately do NOT short-circuit on client disconnect here.+ # The agent loop is allowed to run to completion after the SSE+ # stream is gone (see SKY-8986) so its reply can be persisted;+ # aborting an in-flight block execution would leave the+ # workflow run in a limbo state and the agent would have no+ # debug output to summarize in the final chat message.run = await app.DATABASE.workflow_runs.get_workflow_run(workflow_run_id=workflow_run.workflow_run_id,diff --git a/skyvern/forge/sdk/routes/event_source_stream.py b/skyvern/forge/sdk/routes/event_source_stream.pyindex f8bff123f..5ae42584f 100644--- a/skyvern/forge/sdk/routes/event_source_stream.py+++ b/skyvern/forge/sdk/routes/event_source_stream.py@@ -1,12 +1,22 @@import asynciofrom typing import Any, Awaitable, Callable, Protocol+import structlogfrom fastapi import Requestfrom pydantic import BaseModelfrom sse_starlette import EventSourceResponse, JSONServerSentEvent, ServerSentEvent+LOG = structlog.get_logger()+DEFAULT_KEEPALIVE_INTERVAL_SECONDS = 10+# Strong references to handler tasks that outlive their SSE response. When a+# client disconnects mid-stream, we let the handler keep running so any+# in-flight work (agent loops, DB persistence) finishes cleanly. Without a+# strong reference, the event loop could garbage-collect the task while it+# is still running.+_BACKGROUND_HANDLER_TASKS: set["asyncio.Task[None]"] = set()+class EventSourceStream(Protocol):"""Protocol for Server-Sent Events (SSE) streams."""@@ -16,7 +26,11 @@ class EventSourceStream(Protocol):Send data as an SSE event.Returns:- True if the event was queued successfully, False if disconnected or closed.+ True if the event was accepted (queued for delivery or dropped+ because the client is gone). False only if the stream has been+ explicitly closed. Callers should treat a False return as a+ terminal state and stop emitting; they should NOT treat a+ client disconnect as a reason to abort in-flight work."""...@@ -33,8 +47,12 @@ class FastAPIEventSourceStream:"""FastAPI implementation of EventSourceStream.- This class provides a cleaner interface for sending SSE updates from async functions- instead of using yield-based generators directly.+ Sending is decoupled from client presence. When the client disconnects,+ the handler task keeps running (so agent loops and DB persistence+ complete) and subsequent send() calls silently drop their payload+ instead of growing the queue. The generator stops yielding and+ sse-starlette closes the HTTP response; the handler runs out in the+ background.Usage:@app.post("/stream")@@ -51,22 +69,52 @@ class FastAPIEventSourceStream:self._request = requestself._queue: asyncio.Queue[Any] = asyncio.Queue()self._closed = False+ # Latches once the client goes away so repeated is_disconnected()+ # calls don't hit the ASGI receive channel after the response has+ # been torn down.+ self._client_gone = Falseasync def send(self, data: Any) -> bool:- """- Send data as an SSE event. Accepts Pydantic models or dicts.+ """Send data as an SSE event. Accepts Pydantic models or dicts.- Returns:- True if the event was queued successfully, False if disconnected or closed.+ When the client is still connected, the event is queued for+ delivery. When the client has disconnected, the event is dropped+ silently so the handler can continue to completion without+ growing memory. Returns False only if the stream has been+ explicitly closed."""- if self._closed or await self.is_disconnected():+ if self._closed:return False+ if await self.is_disconnected():+ return Trueawait self._queue.put(data)return Trueasync def is_disconnected(self) -> bool:- """Check if the client has disconnected."""- return await self._request.is_disconnected()+ """Check if the client has disconnected.++ Caches the first positive result so callers made after the ASGI+ response has been torn down don't try to pull from a receive+ channel that may no longer be live.+ """+ if self._client_gone:+ return True+ try:+ disconnected = await self._request.is_disconnected()+ except Exception as exc:+ # Starlette's is_disconnected can raise various errors once+ # the ASGI receive channel is gone (RuntimeError on closed+ # queues, anyio/asyncio cancellation oddities, etc.). Treat+ # any such failure as "client gone" -- we're polling to+ # decide whether to skip emitting events, and if we can't+ # tell we'd rather stop emitting than spin. Log so a+ # genuinely new failure mode shows up in telemetry instead+ # of hiding behind the disconnect path.+ LOG.debug("is_disconnected raised; treating as disconnect", error=str(exc))+ disconnected = True+ if disconnected:+ self._client_gone = True+ return disconnectedasync def close(self) -> None:"""Signal that the stream is complete."""@@ -111,19 +159,27 @@ class FastAPIEventSourceStream:An EventSourceResponse that can be returned from a FastAPI endpoint"""stream = cls(request)+ task = asyncio.create_task(cls._run_handler(stream, handler))+ # Hold a strong reference so the event loop can't GC the task if the+ # generator is torn down first (SSE client disconnect). The set is+ # bounded in practice by the handler's own timeout — every copilot+ # handler must eventually return (see TOTAL_TIMEOUT_SECONDS in+ # skyvern/forge/sdk/copilot/enforcement.py) which removes the task+ # from the set via the done_callback below. If you wire a new+ # EventSource endpoint through here, give its handler a similar+ # hard cap or this set becomes an unbounded leak.+ _BACKGROUND_HANDLER_TASKS.add(task)+ task.add_done_callback(_BACKGROUND_HANDLER_TASKS.discard)async def event_generator() -> Any:- task = asyncio.create_task(cls._run_handler(stream, handler))- try:- async for event in stream._generate():- yield event- finally:- if not task.done():- task.cancel()- try:- await task- except asyncio.CancelledError:- pass+ async for event in stream._generate():+ yield event+ # Intentionally do NOT cancel the handler task here. SSE+ # disconnect must not kill in-flight work (the copilot agent+ # often needs tens of seconds to finish and then persist its+ # reply to the chat history). The handler keeps running in+ # the background; stream.send() silently drops events once+ # the client is gone so the queue cannot grow unbounded.def ping_message_factory() -> ServerSentEvent:return ServerSentEvent(comment="keep-alive")@@ -142,5 +198,11 @@ class FastAPIEventSourceStream:"""Run the handler and ensure the stream is closed when done."""try:await handler(stream)+ except asyncio.CancelledError:+ # Process/server shutdown — propagate. Client disconnect alone+ # no longer cancels this task (see create()).+ raise+ except Exception:+ LOG.exception("SSE handler failed")finally:await stream.close()diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 657ca2b5b..5739d8495 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -795,12 +795,9 @@ async def _new_copilot_chat_post())- if await stream.is_disconnected():- LOG.info(- "Workflow copilot v2 chat request is disconnected before agent loop",- workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,- )- return+ # No early exit on disconnect (SKY-8986): the agent runs to+ # completion even after the SSE stream drops so its reply is+ # persisted to the chat history and visible after reconnect.original_workflow = await app.DATABASE.workflows.get_workflow_by_permanent_id(workflow_permanent_id=chat_request.workflow_permanent_id,@@ -838,15 +835,11 @@ async def _new_copilot_chat_post(updated_workflow = agent_result.updated_workflowupdated_global_llm_context = agent_result.global_llm_context- if await stream.is_disconnected():- LOG.info(- "Workflow copilot v2 chat request is disconnected after agent loop",- workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,- )- if _should_restore_persisted_workflow(chat.auto_accept, agent_result):- await _restore_workflow_definition(original_workflow, organization.organization_id)- return-+ # Persist rollback / proposed-workflow state and the chat+ # messages regardless of whether the SSE client is still+ # connected: the user needs to see the reply on reconnect.+ # SKY-8986: client disconnect used to short-circuit this block+ # and leave the chat history without the AI response.if chat.auto_accept is not True:if _should_restore_persisted_workflow(chat.auto_accept, agent_result):await _restore_workflow_definition(original_workflow, organization.organization_id)@@ -1017,13 +1010,9 @@ async def workflow_copilot_chat_post())- if await stream.is_disconnected():- LOG.info(- "Workflow copilot chat request is disconnected before LLM call",- workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,- )- return-+ # SKY-8986: do not short-circuit on client disconnect. The LLM+ # call and the DB persistence below must complete so the reply+ # is in the chat history when the user reconnects.user_response, updated_workflow, updated_global_llm_context = await copilot_call_llm(stream,organization.organization_id,@@ -1033,13 +1022,6 @@ async def workflow_copilot_chat_post(debug_run_info_text,)- if await stream.is_disconnected():- LOG.info(- "Workflow copilot chat request is disconnected after LLM call",- workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,- )- return-if updated_workflow and chat.auto_accept is not True:await app.DATABASE.workflow_params.update_workflow_copilot_chat(organization_id=chat.organization_id,diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.pyindex 8fe347bcd..101368c02 100644--- a/tests/unit/test_copilot_streaming_adapter.py+++ b/tests/unit/test_copilot_streaming_adapter.py@@ -7,7 +7,6 @@ from unittest.mock import AsyncMock, MagicMockimport pytest-from skyvern.forge.sdk.copilot.exceptions import CopilotClientDisconnectedErrorfrom skyvern.forge.sdk.copilot.streaming_adapter import _sanitize_input, stream_to_sse@@ -85,38 +84,50 @@ async def _stream_events_from(*events: Any) -> Any:@pytest.mark.asyncio-async def test_stream_to_sse_raises_when_send_reports_disconnect() -> None:+async def test_stream_to_sse_keeps_running_after_client_disconnect() -> None:+ """SKY-8986 regression: a dropped SSE client must NOT cancel the agent run.++ The handler task outlives the SSE response so the agent's reply can be+ persisted to the chat history. stream_to_sse keeps draining the SDK's+ event stream; emissions turn into no-ops when is_disconnected() returns+ True, but result.cancel() is never called and no exception escapes.+ """from agents.items import RunItemfrom agents.stream_events import RunItemStreamEvent- raw = {"call_id": "c1", "name": "click", "arguments": "{}"}- item = MagicMock(spec=RunItem)- item.raw_item = raw- event = RunItemStreamEvent(name="tool_called", item=item)+ raw_call = {"call_id": "c1", "name": "click", "arguments": "{}"}+ call_item = MagicMock(spec=RunItem)+ call_item.raw_item = raw_call+ tool_call = RunItemStreamEvent(name="tool_called", item=call_item)++ raw_output = {"call_id": "c1"}+ output_item = MagicMock(spec=RunItem)+ output_item.raw_item = raw_output+ output_item.output = None+ tool_output = RunItemStreamEvent(name="tool_output", item=output_item)result = MagicMock()- result.stream_events = lambda: _stream_events_from(event)+ result.stream_events = lambda: _stream_events_from(tool_call, tool_output)result.cancel = MagicMock()stream = MagicMock()- stream.is_disconnected = AsyncMock(return_value=False)- stream.send = AsyncMock(return_value=False)+ stream.is_disconnected = AsyncMock(return_value=True)+ stream.send = AsyncMock(return_value=True)ctx = SimpleNamespace()- with pytest.raises(CopilotClientDisconnectedError):- await stream_to_sse(result, stream, ctx)+ await stream_to_sse(result, stream, ctx)- result.cancel.assert_called_once()+ result.cancel.assert_not_called()+ stream.send.assert_not_called()@pytest.mark.asyncioasync def test_stream_to_sse_propagates_cancelled_error() -> None:- """A generic asyncio.CancelledError must not be relabeled as a client- disconnect. Relabeling would silence cancellation (CopilotClientDisconnectedError- is a plain Exception, CancelledError is BaseException) and break the event- loop's cancellation machinery for non-disconnect cancels such as task-group- cancel, upstream timeout, or parent abort.+ """A generic asyncio.CancelledError must propagate up from stream_to_sse so+ the event loop's cancellation machinery still works for task-group cancel,+ upstream timeout, or parent abort. The adapter must not catch it and turn+ it into a normal return."""async def _raises_cancelled() -> Any:diff --git a/tests/unit/test_event_source_stream_disconnect.py b/tests/unit/test_event_source_stream_disconnect.pynew file mode 100644index 000000000..852b9d302--- /dev/null+++ b/tests/unit/test_event_source_stream_disconnect.py@@ -0,0 +1,151 @@+"""Regression tests for SKY-8986: SSE disconnect must not kill the handler.++The SSE stream is a view of work the backend is doing on the client's behalf+(e.g., the workflow copilot agent). Closing the browser tab or losing the TCP+connection mid-stream used to cancel the handler task, which in turn+cancelled the agent run and lost the unpersisted chat reply. The fix in+SKY-8986 decouples the handler from the SSE response lifecycle: the handler+runs to completion even after the client goes away, and subsequent send()+calls drop their payload silently instead of backing up the in-memory queue.+"""++from __future__ import annotations++import asyncio+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.forge.sdk.routes.event_source_stream import (+ _BACKGROUND_HANDLER_TASKS,+ FastAPIEventSourceStream,+)+++def _make_request(is_disconnected_values: list[bool]) -> MagicMock:+ """Build a fake Starlette Request whose is_disconnected() replays a script."""+ request = MagicMock()+ request.is_disconnected = AsyncMock(side_effect=is_disconnected_values)+ return request+++async def test_send_drops_events_silently_after_disconnect() -> None:+ """Once the client is gone, send() returns True but does not queue events.++ If send() queued instead, a long-running agent would grow the queue+ unbounded (no one is reading it) and leak memory until the process+ was restarted.+ """+ request = _make_request([False, True, True, True])+ stream = FastAPIEventSourceStream(request)++ first = await stream.send({"n": 1})+ second = await stream.send({"n": 2})+ third = await stream.send({"n": 3})++ assert first is True+ assert second is True+ assert third is True+ # Only the first send (when connected) queued anything.+ assert stream._queue.qsize() == 1+++async def test_is_disconnected_latches_after_first_positive() -> None:+ """Avoid hammering the ASGI receive channel after the response is torn down.++ The underlying Request.is_disconnected reads from the receive channel,+ which may not be live after the ASGI task group has exited. Once we+ observe disconnect once, cache it so later calls don't hit the channel.+ """+ request = MagicMock()+ # If we didn't cache, the second call would raise.+ request.is_disconnected = AsyncMock(side_effect=[True, RuntimeError("channel closed")])+ stream = FastAPIEventSourceStream(request)++ assert await stream.is_disconnected() is True+ assert await stream.is_disconnected() is True+ # Only one underlying call thanks to caching.+ assert request.is_disconnected.await_count == 1+++async def test_is_disconnected_treats_exception_as_disconnect() -> None:+ """If checking the ASGI receive channel fails, assume the client is gone.++ This protects handlers that keep running after the response has been+ torn down: they still call is_disconnected periodically and must not+ crash on a stale receive channel.+ """+ request = MagicMock()+ request.is_disconnected = AsyncMock(side_effect=RuntimeError("closed"))+ stream = FastAPIEventSourceStream(request)++ assert await stream.is_disconnected() is True+++async def test_handler_runs_to_completion_after_sse_generator_exits() -> None:+ """SKY-8986 regression: handler must NOT be cancelled on client disconnect.++ Simulates a client that disconnects immediately (generator receives no+ events before exiting). The handler should keep running in the+ background and finish its work. This is the bug fix: the previous+ implementation cancelled the handler task in the generator's finally+ block, killing an in-flight copilot agent.+ """+ request = MagicMock()+ request.is_disconnected = AsyncMock(return_value=True)++ handler_finished = asyncio.Event()++ async def handler(stream: Any) -> None:+ # Simulate agent work that takes some time and runs past the+ # moment the SSE generator decides the client is gone.+ await asyncio.sleep(0.01)+ await stream.send({"progress": "halfway"})+ await asyncio.sleep(0.01)+ await stream.send({"progress": "done"})+ handler_finished.set()++ response = FastAPIEventSourceStream.create(request, handler)++ # The EventSourceResponse body iterator should close immediately since+ # the client is already disconnected, but the handler keeps running in+ # the background. We only care that the handler eventually finishes.+ async for _ in response.body_iterator: # drain (should be empty)+ pass++ await asyncio.wait_for(handler_finished.wait(), timeout=2.0)+ # One task can remain in the registry briefly while its done callback+ # fires; yield once so the callback runs.+ await asyncio.sleep(0)+ # The registry is cleaned up once the task is fully done.+ assert not any(not t.done() for t in _BACKGROUND_HANDLER_TASKS)+++async def test_handler_exception_does_not_break_other_streams() -> None:+ """An error inside a handler after disconnect must not crash the process.++ The handler runs as a background task after disconnect; without the+ catch-and-log inside _run_handler an unhandled exception would surface+ only as an asyncio warning at GC time.+ """+ request = MagicMock()+ request.is_disconnected = AsyncMock(return_value=True)++ async def handler(stream: Any) -> None:+ raise RuntimeError("boom")++ response = FastAPIEventSourceStream.create(request, handler)+ async for _ in response.body_iterator:+ pass++ # Allow the handler task to complete and its done-callback to fire.+ await asyncio.sleep(0.05)+ # No exception propagated out of the ASGI response iteration.+ # Background tasks are cleaned from the registry on completion.+ assert not any(not t.done() for t in _BACKGROUND_HANDLER_TASKS)diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex b8ab73d20..a1204a7af 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -15,6 +15,7 @@ real database -- all DB / LLM / agent surfaces are patched.from __future__ import annotations+from datetime import datetime, timezonefrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock@@ -144,7 +145,7 @@ def _setup_new_copilot_mocks(get_workflow_copilot_chat_messages=AsyncMock(return_value=[]),update_workflow_copilot_chat=AsyncMock(),create_workflow_copilot_chat_message=AsyncMock(- return_value=SimpleNamespace(created_at=SimpleNamespace(isoformat=lambda: "2026-04-14T00:00:00Z"))+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 14, tzinfo=timezone.utc))),)app.DATABASE.workflows = SimpleNamespace(