Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-9305): hard-cancel for workflow copilot v2 (1/2 backend) (#5701)
Andrew Neilson3d7429c6635b
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 908a4f849..d70febfd0 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -160,7 +160,12 @@ def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:return None, None-def _build_exit_result(ctx: CopilotContext, user_response: str, global_llm_context: str | None) -> AgentResult:+def _build_exit_result(+ ctx: CopilotContext,+ user_response: str,+ global_llm_context: str | None,+ cancelled: bool = False,+) -> AgentResult:"""AgentResult for agent-loop exits that don't go through ``_translate_to_agent_result``."""verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)return AgentResult(@@ -170,6 +175,7 @@ def _build_exit_result(ctx: CopilotContext, user_response: str, global_llm_conteworkflow_yaml=verified_yaml,workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,+ cancelled=cancelled,)@@ -479,8 +485,10 @@ async def run_copilot_agent(organization_id,)except asyncio.CancelledError:- LOG.info("Copilot run cancelled")- return _build_exit_result(ctx, "Request cancelled.", global_llm_context)+ # Re-raising would leave the route with ``agent_result is None``+ # and skip its ``workflow_was_persisted`` rollback decision.+ LOG.info("Copilot run cancelled by user")+ return _build_exit_result(ctx, "Cancelled by user.", global_llm_context, cancelled=True)except MaxTurnsExceeded:return _build_exit_result(ctx,diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex fcd14d479..7d5a65786 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -122,6 +122,11 @@ class AgentResult:# "0 tokens" so eval cost grading can flag missing telemetry instead of# silently passing as cheap.total_tokens: int | None = None+ # Set when the agent absorbed an asyncio cancellation initiated by an+ # explicit user Stop. Lets the route route to a cancel-specific+ # persistence path (rollback + ``Cancelled by user.`` chat row) without+ # losing ``workflow_was_persisted`` the way a re-raise would.+ cancelled: bool = False@dataclassdiff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 9465a4ab9..d7cd742d6 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -1,8 +1,9 @@import asyncio+import contextlibimport timefrom contextlib import contextmanagerfrom dataclasses import dataclass-from datetime import datetime, timezone+from datetime import datetime, timedelta, timezonefrom pathlib import Pathfrom typing import Any, Iterator@@ -21,6 +22,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactTypefrom skyvern.forge.sdk.copilot.agent import run_copilot_agentfrom skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_write+from skyvern.forge.sdk.copilot.context import AgentResultfrom skyvern.forge.sdk.copilot.output_utils import truncate_outputfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type@@ -29,6 +31,7 @@ from skyvern.forge.sdk.routes.routers import base_routerfrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotApplyProposedWorkflowRequest,+ WorkflowCopilotCancelRequest,WorkflowCopilotChatHistoryMessage,WorkflowCopilotChatHistoryResponse,WorkflowCopilotChatMessage,@@ -101,6 +104,50 @@ class BlockRunInfo:output: str | None+COPILOT_CANCEL_TTL = timedelta(minutes=5)+# Polling cadence for the cancel-watcher sidecar. Worst-case latency from a+# user's Stop click to ``handler_task.cancel()`` is one cadence period plus+# the Redis round-trip — well under the 5-minute scenario this feature+# exists to fix, and far below any client-side timeout budget.+COPILOT_CANCEL_POLL_SECONDS = 1.5+++def _copilot_cancel_key(organization_id: str, cancel_token: str) -> str:+ return f"copilot_cancel:{organization_id}:{cancel_token}"+++async def _watch_for_cancel(+ cache: Any,+ organization_id: str,+ cancel_token: str,+ handler_task: asyncio.Task,+ observed: list[bool],+) -> None:+ """Cancel ``handler_task`` when the matching Redis flag flips truthy.++ Sets ``observed[0] = True`` before issuing the cancel so the handler's+ ``except CancelledError`` block can tell a user cancel apart from+ server shutdown — only the user path writes a ``Cancelled by user.`` row.+ """+ key = _copilot_cancel_key(organization_id, cancel_token)+ while not handler_task.done():+ await asyncio.sleep(COPILOT_CANCEL_POLL_SECONDS)+ try:+ flag = await cache.get(key)+ except Exception:+ LOG.debug("Copilot cancel-watcher get failed; will retry", exc_info=True)+ continue+ if flag:+ LOG.info(+ "Copilot cancel signal observed; cancelling handler task",+ cancel_token=cancel_token,+ organization_id=organization_id,+ )+ observed[0] = True+ handler_task.cancel()+ return++async def _ensure_terminal_frame(stream: EventSourceStream, already_emitted: bool) -> None:"""Emit a fallback ERROR frame if the turn hasn't sent a terminal one.@@ -138,6 +185,137 @@ def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: oreturn auto_accept is not True+async def _persist_cancel_turn(+ stream: EventSourceStream,+ chat: Any,+ organization_id: str,+ original_workflow: Workflow | None,+ user_message: str,+ agent_result: AgentResult | None,+) -> None:+ """Persist a cancelled turn and emit a terminal SSE error frame.++ Pass the agent's ``AgentResult`` for cancels during the agent run so+ rollback uses the same ``workflow_was_persisted`` source of truth as+ the success path; pass ``None`` for pre-agent cancels.+ """+ if agent_result is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):+ await asyncio.shield(_restore_workflow_definition(original_workflow, organization_id))+ await asyncio.shield(+ app.DATABASE.workflow_params.create_workflow_copilot_chat_message(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ sender=WorkflowCopilotChatSender.USER,+ content=user_message,+ )+ )+ await asyncio.shield(+ app.DATABASE.workflow_params.create_workflow_copilot_chat_message(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ sender=WorkflowCopilotChatSender.AI,+ content="Cancelled by user.",+ )+ )+ # Best-effort: client may have already aborted, in which case stream.send+ # silently drops. The optimistic FE bubble already shows the cancellation.+ with contextlib.suppress(Exception):+ await stream.send(+ WorkflowCopilotStreamErrorUpdate(+ type=WorkflowCopilotStreamMessageType.ERROR,+ error="Cancelled by user.",+ )+ )+++async def _finalise_normal_turn(+ stream: EventSourceStream,+ chat: Any,+ organization_id: str,+ original_workflow: Workflow | None,+ chat_request: WorkflowCopilotChatRequest,+ agent_result: AgentResult,+) -> None:+ """Atomic post-agent finalisation: rollback, proposal, chat rows, RESPONSE.++ Wrapped by the caller in ``asyncio.shield`` so a late user cancel cannot+ interrupt these writes mid-way and leave chat history with a partial turn+ (e.g. proposed_workflow updated but no AI message persisted).+ """+ user_response = agent_result.user_response+ updated_workflow = agent_result.updated_workflow+ updated_global_llm_context = agent_result.global_llm_context++ # Persist rollback / proposed-workflow state and the chat+ # messages regardless of whether the SSE client is still+ # connected: the user needs to see the reply on reconnect.+ # SKY-8986: client disconnect used to short-circuit this block+ # and leave the chat history without the AI response.+ #+ # SKY-9143: restore runs outside the auto_accept wrapper so+ # auto-accept turns that ended without a viable proposal still+ # roll back a mid-turn _update_workflow write. The Accept/Reject+ # panel state below stays gated on auto_accept — the frontend+ # applies proposals via applyWorkflowUpdate when auto-accept is+ # on.+ restored = _should_restore_persisted_workflow(chat.auto_accept, agent_result)+ if restored:+ await _restore_workflow_definition(original_workflow, organization_id)++ if chat.auto_accept is not True:+ if updated_workflow:+ proposed_data = updated_workflow.model_dump(mode="json")+ if agent_result.workflow_yaml:+ proposed_data["_copilot_yaml"] = agent_result.workflow_yaml+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ proposed_workflow=proposed_data,+ )+ elif (+ restored or getattr(agent_result, "clear_proposed_workflow", False)+ ) and chat.proposed_workflow is not None:+ # Null any previously-persisted proposed_workflow so a+ # page reload does not resurrect a stale Accept/Reject+ # card next to an assistant message that just explained+ # why no verified proposal is available. Covers:+ # * feasibility-gate fast-path clarifications, and+ # * SKY-9143 strict-gate turns where a mid-turn draft was+ # rolled back (``restored=True``).+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ proposed_workflow=None,+ )++ await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ sender=WorkflowCopilotChatSender.USER,+ content=chat_request.message,+ )++ assistant_message = await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ sender=WorkflowCopilotChatSender.AI,+ content=user_response,+ global_llm_context=updated_global_llm_context,+ )++ await stream.send(+ WorkflowCopilotStreamResponseUpdate(+ type=WorkflowCopilotStreamMessageType.RESPONSE,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ message=user_response,+ updated_workflow=updated_workflow.model_dump(mode="json") if updated_workflow else None,+ response_time=assistant_message.created_at,+ total_tokens=getattr(agent_result, "total_tokens", None),+ response_type=getattr(agent_result, "response_type", "REPLY"),+ )+ )++async def _restore_workflow_definition(original_workflow: Workflow | None, organization_id: str) -> None:"""Roll the workflow back to ``original_workflow``.@@ -790,8 +968,15 @@ async def _new_copilot_chat_post(original_workflow: Workflow | None = Nonechat = None- agent_result: Any = None+ agent_result: AgentResult | None = Noneterminal_frame_emitted = False+ cancel_watcher: asyncio.Task[None] | None = None+ # Single-element list used as a closure flag (mutable bool by reference).+ # The watcher sets [0] = True before issuing handler_task.cancel() so the+ # except CancelledError block can distinguish a user-driven cancel from+ # operational cancels (server shutdown / deploy drain) and only persist+ # a "Cancelled by user." chat row in the user case.+ user_cancel_observed: list[bool] = [False]try:await stream.send(@@ -881,6 +1066,24 @@ async def _new_copilot_chat_post(api_key = request.headers.get("x-api-key")security_rules = app.AGENT_FUNCTION.get_copilot_security_rules()+ # Spawn the cancel watcher only after the chat row exists; cancels+ # that land during pre-agent setup are not user-cancellable+ # (setup is short and the watcher needs a chat row to attach+ # any cancellation messages to).+ cache = getattr(app, "CACHE", None)+ if chat_request.cancel_token and cache is not None:+ handler_task = asyncio.current_task()+ if handler_task is not None:+ cancel_watcher = asyncio.create_task(+ _watch_for_cancel(+ cache,+ organization.organization_id,+ chat_request.cancel_token,+ handler_task,+ user_cancel_observed,+ )+ )+with bind_copilot_session_id(chat.workflow_copilot_chat_id):agent_result = await run_copilot_agent(stream=stream,@@ -894,79 +1097,38 @@ async def _new_copilot_chat_post(security_rules=security_rules,)- user_response = agent_result.user_response- updated_workflow = agent_result.updated_workflow- updated_global_llm_context = agent_result.global_llm_context-- # Persist rollback / proposed-workflow state and the chat- # messages regardless of whether the SSE client is still- # connected: the user needs to see the reply on reconnect.- # SKY-8986: client disconnect used to short-circuit this block- # and leave the chat history without the AI response.- #- # SKY-9143: restore runs outside the auto_accept wrapper so- # auto-accept turns that ended without a viable proposal still- # roll back a mid-turn _update_workflow write. The Accept/Reject- # panel state below stays gated on auto_accept — the frontend- # applies proposals via applyWorkflowUpdate when auto-accept is- # on.- restored = _should_restore_persisted_workflow(chat.auto_accept, agent_result)- if restored:- await _restore_workflow_definition(original_workflow, organization.organization_id)-- if chat.auto_accept is not True:- if updated_workflow:- proposed_data = updated_workflow.model_dump(mode="json")- if agent_result.workflow_yaml:- proposed_data["_copilot_yaml"] = agent_result.workflow_yaml- await app.DATABASE.workflow_params.update_workflow_copilot_chat(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- proposed_workflow=proposed_data,- )- elif (- restored or getattr(agent_result, "clear_proposed_workflow", False)- ) and chat.proposed_workflow is not None:- # Null any previously-persisted proposed_workflow so a- # page reload does not resurrect a stale Accept/Reject- # card next to an assistant message that just explained- # why no verified proposal is available. Covers:- # * feasibility-gate fast-path clarifications, and- # * SKY-9143 strict-gate turns where a mid-turn draft was- # rolled back (``restored=True``).- await app.DATABASE.workflow_params.update_workflow_copilot_chat(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- proposed_workflow=None,- )-- await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- sender=WorkflowCopilotChatSender.USER,- content=chat_request.message,- )-- assistant_message = await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- sender=WorkflowCopilotChatSender.AI,- content=user_response,- global_llm_context=updated_global_llm_context,- )+ if getattr(agent_result, "cancelled", False):+ # The agent absorbed the CancelledError and returned a result+ # carrying ``workflow_was_persisted`` so rollback proceeds normally.+ await _persist_cancel_turn(+ stream=stream,+ chat=chat,+ organization_id=organization.organization_id,+ original_workflow=original_workflow,+ user_message=chat_request.message,+ agent_result=agent_result,+ )+ terminal_frame_emitted = True+ LOG.info(+ "Workflow copilot v2 cancelled by user",+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,+ )+ return- terminal_frame_emitted = True- await stream.send(- WorkflowCopilotStreamResponseUpdate(- type=WorkflowCopilotStreamMessageType.RESPONSE,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- message=user_response,- updated_workflow=updated_workflow.model_dump(mode="json") if updated_workflow else None,- response_time=assistant_message.created_at,- total_tokens=getattr(agent_result, "total_tokens", None),- response_type=getattr(agent_result, "response_type", "REPLY"),+ # Atomic finalisation — a late cancel that fires here cannot tear+ # the success-path writes apart mid-way (no half-written turn,+ # no duplicate user/AI rows).+ await asyncio.shield(+ _finalise_normal_turn(+ stream=stream,+ chat=chat,+ organization_id=organization.organization_id,+ original_workflow=original_workflow,+ chat_request=chat_request,+ agent_result=agent_result,))+ terminal_frame_emitted = Trueexcept HTTPException as exc:if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):await _restore_workflow_definition(original_workflow, organization.organization_id)@@ -996,10 +1158,37 @@ async def _new_copilot_chat_post(except asyncio.CancelledError:if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):await asyncio.shield(_restore_workflow_definition(original_workflow, organization.organization_id))- LOG.info(- "Client disconnected during workflow copilot v2",- workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,- )+ if user_cancel_observed[0] and chat is not None and agent_result is None:+ # User cancel landed before the agent started running, so+ # the agent_result.cancelled branch above couldn't run.+ # _persist_cancel_turn skips rollback when agent_result is None.+ await asyncio.shield(+ _persist_cancel_turn(+ stream=stream,+ chat=chat,+ organization_id=organization.organization_id,+ original_workflow=None,+ user_message=chat_request.message,+ agent_result=None,+ )+ )+ terminal_frame_emitted = True+ LOG.info(+ "Workflow copilot v2 cancelled by user during pre-agent setup",+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,+ )+ else:+ # Operational cancel (worker shutdown, deploy drain) or a+ # cancel that arrived after _finalise_normal_turn started+ # its shielded write. Don't manufacture a "Cancelled by+ # user." chat row — chat history should not record an+ # operational cancel as user intent.+ LOG.info(+ "Workflow copilot v2 task cancelled (operational or post-finalisation)",+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,+ user_cancel_observed=user_cancel_observed[0],+ )+ raiseexcept Exception as exc:if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):await _restore_workflow_definition(original_workflow, organization.organization_id)@@ -1017,6 +1206,10 @@ async def _new_copilot_chat_post())finally:+ if cancel_watcher is not None and not cancel_watcher.done():+ cancel_watcher.cancel()+ with contextlib.suppress(asyncio.CancelledError, Exception):+ await cancel_watcherawait _ensure_terminal_frame(stream, terminal_frame_emitted)return FastAPIEventSourceStream.create(request, stream_handler)@@ -1231,6 +1424,36 @@ async def workflow_copilot_chat_history()+@base_router.post("/workflow/copilot/cancel", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT)+async def workflow_copilot_cancel(+ cancel_request: WorkflowCopilotCancelRequest,+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> None:+ """Hard-cancel an in-progress workflow copilot v2 turn.++ Sets a per-token Redis flag the SSE handler's cancel-watcher polls; the+ watcher cancels the handler task, propagating ``CancelledError`` into+ whichever ``await`` is currently parked (LLM chunk, browser action, DB+ write). Returns 503 when ``app.CACHE`` is absent — the FE Stop button+ still aborts client-side, but the backend can't signal the running handler.+ """+ cache = getattr(app, "CACHE", None)+ if cache is None:+ LOG.warning(+ "Workflow copilot cancel attempted without cache",+ organization_id=organization.organization_id,+ )+ raise HTTPException(+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,+ detail="Cancel not supported in this environment",+ )+ await cache.set(+ _copilot_cancel_key(organization.organization_id, cancel_request.cancel_token),+ "1",+ ex=COPILOT_CANCEL_TTL,+ )++@base_router.post("/workflow/copilot/clear-proposed-workflow", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT)diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex cc48593eb..4c7c5cd94 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -42,6 +42,17 @@ class WorkflowCopilotChatRequest(BaseModel):workflow_run_id: str | None = Field(None, description="The workflow run ID to use for the context")message: str = Field(..., description="The message that user sends")workflow_yaml: str = Field(..., description="Current workflow YAML including unsaved changes")+ cancel_token: str | None = Field(+ None,+ description=(+ "Client-generated UUID. POST it to /workflow/copilot/cancel to hard-cancel this turn. "+ "Optional; legacy clients omit it and cancel becomes a no-op for those requests."+ ),+ )+++class WorkflowCopilotCancelRequest(BaseModel):+ cancel_token: str = Field(..., description="The cancel_token sent on the original /chat-post request")class WorkflowCopilotClearProposedWorkflowRequest(BaseModel):diff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.pynew file mode 100644index 000000000..afb761d12--- /dev/null+++ b/tests/unit/test_workflow_copilot_cancel.py@@ -0,0 +1,443 @@+"""Tests for the workflow-copilot v2 hard-cancel feature (SKY-9305).++Covers:++- ``_watch_for_cancel`` cancels its handler exactly when the cache flag flips+ truthy, sets the ``observed`` closure flag before the cancel, and exits+ cleanly when the handler completes on its own.+- ``_build_exit_result(cancelled=True)`` produces an ``AgentResult`` whose+ ``cancelled`` flag is True and whose ``workflow_was_persisted`` mirrors+ ``CopilotContext.workflow_persisted`` (so the route's rollback decision has+ the same source of truth on cancel as it does on success).+- The route's success-path branch on ``agent_result.cancelled`` runs the+ cancel-specific persistence (rollback + user msg + ``Cancelled by user.``+ AI msg + ERROR frame) and skips the normal RESPONSE finalisation.+- ``/workflow/copilot/cancel`` returns 503 when the Redis cache is absent and+ 204 + the expected key/TTL when it is present.+- An operational cancel (``task.cancel()`` without ``user_cancel_observed[0]``+ set) does NOT persist a ``Cancelled by user.`` chat row.+"""++from __future__ import annotations++import asyncio+from datetime import datetime, timedelta, timezone+from types import SimpleNamespace+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest+from fastapi import HTTPException, status++from skyvern.config import settings+from skyvern.forge import app+from skyvern.forge.sdk.copilot.agent import _build_exit_result+from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext+from skyvern.forge.sdk.routes.workflow_copilot import (+ COPILOT_CANCEL_TTL,+ _copilot_cancel_key,+ _watch_for_cancel,+ workflow_copilot_cancel,+ workflow_copilot_chat_post,+)+from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotCancelRequest,+ WorkflowCopilotChatRequest,+ WorkflowCopilotStreamMessageType,+)+++class _FakeCache:+ """Minimal in-memory double of the ``get`` / ``set`` surface of app.CACHE."""++ def __init__(self) -> None:+ self.store: dict[str, Any] = {}+ self.set_calls: list[tuple[str, Any, Any]] = []++ async def get(self, key: str) -> Any:+ return self.store.get(key)++ async def set(self, key: str, value: Any, ex: Any = None) -> None:+ self.store[key] = value+ self.set_calls.append((key, value, ex))+++def _make_chat_request(cancel_token: str | None = "tok_abc") -> WorkflowCopilotChatRequest:+ return WorkflowCopilotChatRequest(+ workflow_permanent_id="wpid-1",+ workflow_id="wf-1",+ workflow_copilot_chat_id="chat-1",+ workflow_run_id=None,+ message="please update",+ workflow_yaml="title: Example",+ cancel_token=cancel_token,+ )+++# ---------------------------------------------------------------------------+# _watch_for_cancel+# ---------------------------------------------------------------------------+++async def test_watch_for_cancel_signals_and_sets_observed(monkeypatch: pytest.MonkeyPatch) -> None:+ """Flag flips truthy -> handler_task.cancel() is issued once and observed[0] is True."""+ # Speed the watcher up so the test doesn't wait for the production cadence.+ monkeypatch.setattr("skyvern.forge.sdk.routes.workflow_copilot.COPILOT_CANCEL_POLL_SECONDS", 0.01)+ cache = _FakeCache()+ handler_task = asyncio.create_task(asyncio.sleep(60))+ observed: list[bool] = [False]+ watcher = asyncio.create_task(_watch_for_cancel(cache, "org-1", "tok_abc", handler_task, observed))++ await asyncio.sleep(0.05)+ cache.store[_copilot_cancel_key("org-1", "tok_abc")] = "1"++ with pytest.raises(asyncio.CancelledError):+ await handler_task++ await watcher # Watcher exits after issuing the cancel.++ assert observed[0] is True+++async def test_watch_for_cancel_exits_when_handler_finishes_first(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Handler returns before flag flips -> watcher exits cleanly without raising."""+ monkeypatch.setattr("skyvern.forge.sdk.routes.workflow_copilot.COPILOT_CANCEL_POLL_SECONDS", 0.01)+ cache = _FakeCache()+ handler_task = asyncio.create_task(asyncio.sleep(0.02))+ observed: list[bool] = [False]+ watcher = asyncio.create_task(_watch_for_cancel(cache, "org-1", "tok_abc", handler_task, observed))++ await handler_task+ await asyncio.wait_for(watcher, timeout=1.0)++ assert observed[0] is False+++# ---------------------------------------------------------------------------+# _build_exit_result(cancelled=...)+# ---------------------------------------------------------------------------+++def test_build_exit_result_cancelled_round_trips_workflow_persisted() -> None:+ ctx = CopilotContext(+ organization_id="org-1",+ workflow_id="wf-1",+ workflow_permanent_id="wpid-1",+ workflow_yaml="title: Example",+ browser_session_id=None,+ stream=MagicMock(),+ api_key=None,+ user_message="please update",+ workflow_copilot_chat_id="chat-1",+ )+ ctx.workflow_persisted = True++ result = _build_exit_result(ctx, "Cancelled by user.", None, cancelled=True)++ assert isinstance(result, AgentResult)+ assert result.cancelled is True+ assert result.workflow_was_persisted is True+ assert result.user_response == "Cancelled by user."+++def test_build_exit_result_default_cancelled_false() -> None:+ ctx = CopilotContext(+ organization_id="org-1",+ workflow_id="wf-1",+ workflow_permanent_id="wpid-1",+ workflow_yaml="",+ browser_session_id=None,+ stream=MagicMock(),+ api_key=None,+ user_message="msg",+ workflow_copilot_chat_id="chat-1",+ )+ result = _build_exit_result(ctx, "Done.", None)+ assert result.cancelled is False+++# ---------------------------------------------------------------------------+# Route cancel branch (agent_result.cancelled=True)+# ---------------------------------------------------------------------------+++def _install_fake_create(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]:+ captured: dict[str, object] = {}+ sentinel = object()++ def fake_create(request: object, handler: object, ping_interval: int = 10) -> object:+ del request, ping_interval+ captured["handler"] = handler+ return sentinel++ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot.FastAPIEventSourceStream.create",+ fake_create,+ )+ captured["sentinel"] = sentinel+ return captured+++def _setup_route_mocks(+ monkeypatch: pytest.MonkeyPatch,+ chat: SimpleNamespace,+ original_workflow: SimpleNamespace,+ agent_result: SimpleNamespace,+) -> tuple[AsyncMock, SimpleNamespace]:+ async def fake_llm_handler(*args: object, **kwargs: object) -> None:+ del args, kwargs+ return None++ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot.get_llm_handler_for_prompt_type",+ fake_llm_handler,+ )++ restore_mock = AsyncMock()+ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot._restore_workflow_definition",+ restore_mock,+ )++ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot.run_copilot_agent",+ AsyncMock(return_value=agent_result),+ )++ workflow_params = SimpleNamespace(+ get_workflow_copilot_chat_by_id=AsyncMock(return_value=chat),+ get_workflow_copilot_chat_messages=AsyncMock(return_value=[]),+ update_workflow_copilot_chat=AsyncMock(),+ create_workflow_copilot_chat_message=AsyncMock(+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))+ ),+ )+ app.DATABASE.workflow_params = workflow_params+ app.DATABASE.workflows = SimpleNamespace(+ get_workflow_by_permanent_id=AsyncMock(return_value=original_workflow),+ )+ app.DATABASE.observer = SimpleNamespace(+ get_workflow_run_blocks=AsyncMock(return_value=[]),+ )+ app.AGENT_FUNCTION.get_copilot_security_rules = MagicMock(return_value="")++ return restore_mock, workflow_params+++async def test_route_cancel_branch_persists_user_and_cancelled_messages(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """``agent_result.cancelled=True`` -> rollback + USER msg + ``Cancelled by user.`` AI msg + ERROR frame."""+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)+ captured = _install_fake_create(monkeypatch)++ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow=None,+ auto_accept=False,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description="Original description",+ workflow_definition=None,+ )+ agent_result = SimpleNamespace(+ user_response="Cancelled by user.",+ updated_workflow=None,+ global_llm_context=None,+ workflow_yaml=None,+ workflow_was_persisted=True,+ clear_proposed_workflow=False,+ cancelled=True,+ total_tokens=None,+ response_type="REPLY",+ )+ restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)++ request = MagicMock()+ request.headers = {"x-api-key": "sk-test"}+ organization = SimpleNamespace(organization_id="org-1")++ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)+ assert response is captured["sentinel"]++ sent_payloads: list[Any] = []+ stream = MagicMock()++ async def _send(payload: Any) -> bool:+ sent_payloads.append(payload)+ return True++ stream.send = _send+ stream.is_disconnected = AsyncMock(return_value=False)++ handler = captured["handler"]+ assert callable(handler)+ await handler(stream)++ # Rollback fired (workflow_was_persisted=True, auto_accept=False, no proposal).+ restore_mock.assert_awaited_once()++ # Two chat-message inserts: user msg + Cancelled by user. AI msg.+ insert_calls = workflow_params.create_workflow_copilot_chat_message.await_args_list+ senders = [c.kwargs.get("sender") for c in insert_calls]+ contents = [c.kwargs.get("content") for c in insert_calls]+ assert senders.count("user") == 1+ assert senders.count("ai") == 1+ assert "please update" in contents+ assert "Cancelled by user." in contents++ # Normal RESPONSE finalisation did NOT run: no proposed_workflow update,+ # no RESPONSE frame.+ workflow_params.update_workflow_copilot_chat.assert_not_awaited()+ response_types = [getattr(p, "type", None) for p in sent_payloads]+ assert WorkflowCopilotStreamMessageType.RESPONSE not in response_types+ # An ERROR frame carrying "Cancelled by user." was emitted.+ error_frames = [p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.ERROR]+ assert any(getattr(f, "error", "") == "Cancelled by user." for f in error_frames)+++# ---------------------------------------------------------------------------+# /workflow/copilot/cancel endpoint+# ---------------------------------------------------------------------------+#+# ``app`` is an ``AppHolder`` proxy that forwards ``__setattr__`` / ``__getattr__``+# to a wrapped ``ForgeApp`` instance, but does not implement ``__delattr__``.+# That means ``monkeypatch.setattr(app, "CACHE", ...)`` works on assignment but+# its teardown ``delattr`` raises. Manipulate the underlying instance instead so+# monkeypatch's teardown lands on a normal attribute.+++async def test_cancel_endpoint_503_when_cache_missing(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_cancel(+ WorkflowCopilotCancelRequest(cancel_token="tok_abc"),+ organization=organization,+ )+ assert excinfo.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE+++async def test_cancel_endpoint_204_writes_redis_flag(monkeypatch: pytest.MonkeyPatch) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ result = await workflow_copilot_cancel(+ WorkflowCopilotCancelRequest(cancel_token="tok_abc"),+ organization=organization,+ )+ assert result is None # 204 No Content++ expected_key = _copilot_cancel_key("org-1", "tok_abc")+ assert cache.store[expected_key] == "1"+ assert len(cache.set_calls) == 1+ key, value, ex = cache.set_calls[0]+ assert (key, value) == (expected_key, "1")+ assert isinstance(ex, timedelta)+ assert ex == COPILOT_CANCEL_TTL+++# ---------------------------------------------------------------------------+# Operational cancel disambiguation+# ---------------------------------------------------------------------------+++async def test_operational_cancel_does_not_persist_cancelled_message(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """task.cancel() without user_cancel_observed[0] -> no 'Cancelled by user.' chat row.++ The route should treat such a cancel as operational (deploy drain / SIGINT)+ and re-raise without manufacturing a user-cancel chat row.+ """+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)+ captured = _install_fake_create(monkeypatch)++ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow=None,+ auto_accept=False,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description=None,+ workflow_definition=None,+ )+ # Agent raises CancelledError synchronously (simulates operational cancel+ # propagating into the await before user_cancel_observed gets set).+ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot.run_copilot_agent",+ AsyncMock(side_effect=asyncio.CancelledError()),+ )++ async def fake_llm_handler(*args: object, **kwargs: object) -> None:+ del args, kwargs+ return None++ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot.get_llm_handler_for_prompt_type",+ fake_llm_handler,+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot._restore_workflow_definition",+ AsyncMock(),+ )++ workflow_params = SimpleNamespace(+ get_workflow_copilot_chat_by_id=AsyncMock(return_value=chat),+ get_workflow_copilot_chat_messages=AsyncMock(return_value=[]),+ update_workflow_copilot_chat=AsyncMock(),+ create_workflow_copilot_chat_message=AsyncMock(+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))+ ),+ )+ app.DATABASE.workflow_params = workflow_params+ app.DATABASE.workflows = SimpleNamespace(+ get_workflow_by_permanent_id=AsyncMock(return_value=original_workflow),+ )+ app.DATABASE.observer = SimpleNamespace(+ get_workflow_run_blocks=AsyncMock(return_value=[]),+ )+ app.AGENT_FUNCTION.get_copilot_security_rules = MagicMock(return_value="")++ # Make sure no cache is configured so the watcher never spawns and+ # user_cancel_observed[0] stays False.+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)++ request = MagicMock()+ request.headers = {"x-api-key": "sk-test"}+ organization = SimpleNamespace(organization_id="org-1")++ response = await workflow_copilot_chat_post(request, _make_chat_request(cancel_token=None), organization)+ assert response is captured["sentinel"]++ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ stream.is_disconnected = AsyncMock(return_value=False)++ handler = captured["handler"]+ assert callable(handler)+ with pytest.raises(asyncio.CancelledError):+ await handler(stream)++ insert_calls = workflow_params.create_workflow_copilot_chat_message.await_args_list+ contents = [c.kwargs.get("content") for c in insert_calls]+ # No "Cancelled by user." row was written.+ assert "Cancelled by user." not in contents