Skip to content

Automate browser based workflows with AI

feat(SKY-9305): hard-cancel for workflow copilot v2 (1/2 backend) (#5701)

Andrew Neilson3d7429c6635b
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 908a4f849..d70febfd0 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -160,7 +160,12 @@ def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:
return None, None
-def _build_exit_result(ctx: CopilotContext, user_response: str, global_llm_context: str | None) -> AgentResult:
+def _build_exit_result(
+ ctx: CopilotContext,
+ user_response: str,
+ global_llm_context: str | None,
+ cancelled: bool = False,
+) -> AgentResult:
"""AgentResult for agent-loop exits that don't go through ``_translate_to_agent_result``."""
verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)
return AgentResult(
@@ -170,6 +175,7 @@ def _build_exit_result(ctx: CopilotContext, user_response: str, global_llm_conte
workflow_yaml=verified_yaml,
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
+ cancelled=cancelled,
)
@@ -479,8 +485,10 @@ async def run_copilot_agent(
organization_id,
)
except asyncio.CancelledError:
- LOG.info("Copilot run cancelled")
- return _build_exit_result(ctx, "Request cancelled.", global_llm_context)
+ # Re-raising would leave the route with ``agent_result is None``
+ # and skip its ``workflow_was_persisted`` rollback decision.
+ LOG.info("Copilot run cancelled by user")
+ return _build_exit_result(ctx, "Cancelled by user.", global_llm_context, cancelled=True)
except MaxTurnsExceeded:
return _build_exit_result(
ctx,
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index fcd14d479..7d5a65786 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -122,6 +122,11 @@ class AgentResult:
# "0 tokens" so eval cost grading can flag missing telemetry instead of
# silently passing as cheap.
total_tokens: int | None = None
+ # Set when the agent absorbed an asyncio cancellation initiated by an
+ # explicit user Stop. Lets the route route to a cancel-specific
+ # persistence path (rollback + ``Cancelled by user.`` chat row) without
+ # losing ``workflow_was_persisted`` the way a re-raise would.
+ cancelled: bool = False
@dataclass
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 9465a4ab9..d7cd742d6 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -1,8 +1,9 @@
import asyncio
+import contextlib
import time
from contextlib import contextmanager
from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Iterator
@@ -21,6 +22,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
from skyvern.forge.sdk.copilot.agent import run_copilot_agent
from skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_write
+from skyvern.forge.sdk.copilot.context import AgentResult
from skyvern.forge.sdk.copilot.output_utils import truncate_output
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type
@@ -29,6 +31,7 @@ from skyvern.forge.sdk.routes.routers import base_router
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotApplyProposedWorkflowRequest,
+ WorkflowCopilotCancelRequest,
WorkflowCopilotChatHistoryMessage,
WorkflowCopilotChatHistoryResponse,
WorkflowCopilotChatMessage,
@@ -101,6 +104,50 @@ class BlockRunInfo:
output: str | None
+COPILOT_CANCEL_TTL = timedelta(minutes=5)
+# Polling cadence for the cancel-watcher sidecar. Worst-case latency from a
+# user's Stop click to ``handler_task.cancel()`` is one cadence period plus
+# the Redis round-trip — well under the 5-minute scenario this feature
+# exists to fix, and far below any client-side timeout budget.
+COPILOT_CANCEL_POLL_SECONDS = 1.5
+
+
+def _copilot_cancel_key(organization_id: str, cancel_token: str) -> str:
+ return f"copilot_cancel:{organization_id}:{cancel_token}"
+
+
+async def _watch_for_cancel(
+ cache: Any,
+ organization_id: str,
+ cancel_token: str,
+ handler_task: asyncio.Task,
+ observed: list[bool],
+) -> None:
+ """Cancel ``handler_task`` when the matching Redis flag flips truthy.
+
+ Sets ``observed[0] = True`` before issuing the cancel so the handler's
+ ``except CancelledError`` block can tell a user cancel apart from
+ server shutdown — only the user path writes a ``Cancelled by user.`` row.
+ """
+ key = _copilot_cancel_key(organization_id, cancel_token)
+ while not handler_task.done():
+ await asyncio.sleep(COPILOT_CANCEL_POLL_SECONDS)
+ try:
+ flag = await cache.get(key)
+ except Exception:
+ LOG.debug("Copilot cancel-watcher get failed; will retry", exc_info=True)
+ continue
+ if flag:
+ LOG.info(
+ "Copilot cancel signal observed; cancelling handler task",
+ cancel_token=cancel_token,
+ organization_id=organization_id,
+ )
+ observed[0] = True
+ handler_task.cancel()
+ return
+
+
async def _ensure_terminal_frame(stream: EventSourceStream, already_emitted: bool) -> None:
"""Emit a fallback ERROR frame if the turn hasn't sent a terminal one.
@@ -138,6 +185,137 @@ def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: o
return auto_accept is not True
+async def _persist_cancel_turn(
+ stream: EventSourceStream,
+ chat: Any,
+ organization_id: str,
+ original_workflow: Workflow | None,
+ user_message: str,
+ agent_result: AgentResult | None,
+) -> None:
+ """Persist a cancelled turn and emit a terminal SSE error frame.
+
+ Pass the agent's ``AgentResult`` for cancels during the agent run so
+ rollback uses the same ``workflow_was_persisted`` source of truth as
+ the success path; pass ``None`` for pre-agent cancels.
+ """
+ if agent_result is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):
+ await asyncio.shield(_restore_workflow_definition(original_workflow, organization_id))
+ await asyncio.shield(
+ app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ sender=WorkflowCopilotChatSender.USER,
+ content=user_message,
+ )
+ )
+ await asyncio.shield(
+ app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ sender=WorkflowCopilotChatSender.AI,
+ content="Cancelled by user.",
+ )
+ )
+ # Best-effort: client may have already aborted, in which case stream.send
+ # silently drops. The optimistic FE bubble already shows the cancellation.
+ with contextlib.suppress(Exception):
+ await stream.send(
+ WorkflowCopilotStreamErrorUpdate(
+ type=WorkflowCopilotStreamMessageType.ERROR,
+ error="Cancelled by user.",
+ )
+ )
+
+
+async def _finalise_normal_turn(
+ stream: EventSourceStream,
+ chat: Any,
+ organization_id: str,
+ original_workflow: Workflow | None,
+ chat_request: WorkflowCopilotChatRequest,
+ agent_result: AgentResult,
+) -> None:
+ """Atomic post-agent finalisation: rollback, proposal, chat rows, RESPONSE.
+
+ Wrapped by the caller in ``asyncio.shield`` so a late user cancel cannot
+ interrupt these writes mid-way and leave chat history with a partial turn
+ (e.g. proposed_workflow updated but no AI message persisted).
+ """
+ user_response = agent_result.user_response
+ updated_workflow = agent_result.updated_workflow
+ updated_global_llm_context = agent_result.global_llm_context
+
+ # Persist rollback / proposed-workflow state and the chat
+ # messages regardless of whether the SSE client is still
+ # connected: the user needs to see the reply on reconnect.
+ # SKY-8986: client disconnect used to short-circuit this block
+ # and leave the chat history without the AI response.
+ #
+ # SKY-9143: restore runs outside the auto_accept wrapper so
+ # auto-accept turns that ended without a viable proposal still
+ # roll back a mid-turn _update_workflow write. The Accept/Reject
+ # panel state below stays gated on auto_accept — the frontend
+ # applies proposals via applyWorkflowUpdate when auto-accept is
+ # on.
+ restored = _should_restore_persisted_workflow(chat.auto_accept, agent_result)
+ if restored:
+ await _restore_workflow_definition(original_workflow, organization_id)
+
+ if chat.auto_accept is not True:
+ if updated_workflow:
+ proposed_data = updated_workflow.model_dump(mode="json")
+ if agent_result.workflow_yaml:
+ proposed_data["_copilot_yaml"] = agent_result.workflow_yaml
+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ proposed_workflow=proposed_data,
+ )
+ elif (
+ restored or getattr(agent_result, "clear_proposed_workflow", False)
+ ) and chat.proposed_workflow is not None:
+ # Null any previously-persisted proposed_workflow so a
+ # page reload does not resurrect a stale Accept/Reject
+ # card next to an assistant message that just explained
+ # why no verified proposal is available. Covers:
+ # * feasibility-gate fast-path clarifications, and
+ # * SKY-9143 strict-gate turns where a mid-turn draft was
+ # rolled back (``restored=True``).
+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ proposed_workflow=None,
+ )
+
+ await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ sender=WorkflowCopilotChatSender.USER,
+ content=chat_request.message,
+ )
+
+ assistant_message = await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ sender=WorkflowCopilotChatSender.AI,
+ content=user_response,
+ global_llm_context=updated_global_llm_context,
+ )
+
+ await stream.send(
+ WorkflowCopilotStreamResponseUpdate(
+ type=WorkflowCopilotStreamMessageType.RESPONSE,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ message=user_response,
+ updated_workflow=updated_workflow.model_dump(mode="json") if updated_workflow else None,
+ response_time=assistant_message.created_at,
+ total_tokens=getattr(agent_result, "total_tokens", None),
+ response_type=getattr(agent_result, "response_type", "REPLY"),
+ )
+ )
+
+
async def _restore_workflow_definition(original_workflow: Workflow | None, organization_id: str) -> None:
"""Roll the workflow back to ``original_workflow``.
@@ -790,8 +968,15 @@ async def _new_copilot_chat_post(
original_workflow: Workflow | None = None
chat = None
- agent_result: Any = None
+ agent_result: AgentResult | None = None
terminal_frame_emitted = False
+ cancel_watcher: asyncio.Task[None] | None = None
+ # Single-element list used as a closure flag (mutable bool by reference).
+ # The watcher sets [0] = True before issuing handler_task.cancel() so the
+ # except CancelledError block can distinguish a user-driven cancel from
+ # operational cancels (server shutdown / deploy drain) and only persist
+ # a "Cancelled by user." chat row in the user case.
+ user_cancel_observed: list[bool] = [False]
try:
await stream.send(
@@ -881,6 +1066,24 @@ async def _new_copilot_chat_post(
api_key = request.headers.get("x-api-key")
security_rules = app.AGENT_FUNCTION.get_copilot_security_rules()
+ # Spawn the cancel watcher only after the chat row exists; cancels
+ # that land during pre-agent setup are not user-cancellable
+ # (setup is short and the watcher needs a chat row to attach
+ # any cancellation messages to).
+ cache = getattr(app, "CACHE", None)
+ if chat_request.cancel_token and cache is not None:
+ handler_task = asyncio.current_task()
+ if handler_task is not None:
+ cancel_watcher = asyncio.create_task(
+ _watch_for_cancel(
+ cache,
+ organization.organization_id,
+ chat_request.cancel_token,
+ handler_task,
+ user_cancel_observed,
+ )
+ )
+
with bind_copilot_session_id(chat.workflow_copilot_chat_id):
agent_result = await run_copilot_agent(
stream=stream,
@@ -894,79 +1097,38 @@ async def _new_copilot_chat_post(
security_rules=security_rules,
)
- user_response = agent_result.user_response
- updated_workflow = agent_result.updated_workflow
- updated_global_llm_context = agent_result.global_llm_context
-
- # Persist rollback / proposed-workflow state and the chat
- # messages regardless of whether the SSE client is still
- # connected: the user needs to see the reply on reconnect.
- # SKY-8986: client disconnect used to short-circuit this block
- # and leave the chat history without the AI response.
- #
- # SKY-9143: restore runs outside the auto_accept wrapper so
- # auto-accept turns that ended without a viable proposal still
- # roll back a mid-turn _update_workflow write. The Accept/Reject
- # panel state below stays gated on auto_accept — the frontend
- # applies proposals via applyWorkflowUpdate when auto-accept is
- # on.
- restored = _should_restore_persisted_workflow(chat.auto_accept, agent_result)
- if restored:
- await _restore_workflow_definition(original_workflow, organization.organization_id)
-
- if chat.auto_accept is not True:
- if updated_workflow:
- proposed_data = updated_workflow.model_dump(mode="json")
- if agent_result.workflow_yaml:
- proposed_data["_copilot_yaml"] = agent_result.workflow_yaml
- await app.DATABASE.workflow_params.update_workflow_copilot_chat(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- proposed_workflow=proposed_data,
- )
- elif (
- restored or getattr(agent_result, "clear_proposed_workflow", False)
- ) and chat.proposed_workflow is not None:
- # Null any previously-persisted proposed_workflow so a
- # page reload does not resurrect a stale Accept/Reject
- # card next to an assistant message that just explained
- # why no verified proposal is available. Covers:
- # * feasibility-gate fast-path clarifications, and
- # * SKY-9143 strict-gate turns where a mid-turn draft was
- # rolled back (``restored=True``).
- await app.DATABASE.workflow_params.update_workflow_copilot_chat(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- proposed_workflow=None,
- )
-
- await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- sender=WorkflowCopilotChatSender.USER,
- content=chat_request.message,
- )
-
- assistant_message = await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- sender=WorkflowCopilotChatSender.AI,
- content=user_response,
- global_llm_context=updated_global_llm_context,
- )
+ if getattr(agent_result, "cancelled", False):
+ # The agent absorbed the CancelledError and returned a result
+ # carrying ``workflow_was_persisted`` so rollback proceeds normally.
+ await _persist_cancel_turn(
+ stream=stream,
+ chat=chat,
+ organization_id=organization.organization_id,
+ original_workflow=original_workflow,
+ user_message=chat_request.message,
+ agent_result=agent_result,
+ )
+ terminal_frame_emitted = True
+ LOG.info(
+ "Workflow copilot v2 cancelled by user",
+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
+ )
+ return
- terminal_frame_emitted = True
- await stream.send(
- WorkflowCopilotStreamResponseUpdate(
- type=WorkflowCopilotStreamMessageType.RESPONSE,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- message=user_response,
- updated_workflow=updated_workflow.model_dump(mode="json") if updated_workflow else None,
- response_time=assistant_message.created_at,
- total_tokens=getattr(agent_result, "total_tokens", None),
- response_type=getattr(agent_result, "response_type", "REPLY"),
+ # Atomic finalisation — a late cancel that fires here cannot tear
+ # the success-path writes apart mid-way (no half-written turn,
+ # no duplicate user/AI rows).
+ await asyncio.shield(
+ _finalise_normal_turn(
+ stream=stream,
+ chat=chat,
+ organization_id=organization.organization_id,
+ original_workflow=original_workflow,
+ chat_request=chat_request,
+ agent_result=agent_result,
)
)
+ terminal_frame_emitted = True
except HTTPException as exc:
if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):
await _restore_workflow_definition(original_workflow, organization.organization_id)
@@ -996,10 +1158,37 @@ async def _new_copilot_chat_post(
except asyncio.CancelledError:
if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):
await asyncio.shield(_restore_workflow_definition(original_workflow, organization.organization_id))
- LOG.info(
- "Client disconnected during workflow copilot v2",
- workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
- )
+ if user_cancel_observed[0] and chat is not None and agent_result is None:
+ # User cancel landed before the agent started running, so
+ # the agent_result.cancelled branch above couldn't run.
+ # _persist_cancel_turn skips rollback when agent_result is None.
+ await asyncio.shield(
+ _persist_cancel_turn(
+ stream=stream,
+ chat=chat,
+ organization_id=organization.organization_id,
+ original_workflow=None,
+ user_message=chat_request.message,
+ agent_result=None,
+ )
+ )
+ terminal_frame_emitted = True
+ LOG.info(
+ "Workflow copilot v2 cancelled by user during pre-agent setup",
+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
+ )
+ else:
+ # Operational cancel (worker shutdown, deploy drain) or a
+ # cancel that arrived after _finalise_normal_turn started
+ # its shielded write. Don't manufacture a "Cancelled by
+ # user." chat row — chat history should not record an
+ # operational cancel as user intent.
+ LOG.info(
+ "Workflow copilot v2 task cancelled (operational or post-finalisation)",
+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
+ user_cancel_observed=user_cancel_observed[0],
+ )
+ raise
except Exception as exc:
if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):
await _restore_workflow_definition(original_workflow, organization.organization_id)
@@ -1017,6 +1206,10 @@ async def _new_copilot_chat_post(
)
)
finally:
+ if cancel_watcher is not None and not cancel_watcher.done():
+ cancel_watcher.cancel()
+ with contextlib.suppress(asyncio.CancelledError, Exception):
+ await cancel_watcher
await _ensure_terminal_frame(stream, terminal_frame_emitted)
return FastAPIEventSourceStream.create(request, stream_handler)
@@ -1231,6 +1424,36 @@ async def workflow_copilot_chat_history(
)
+@base_router.post("/workflow/copilot/cancel", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT)
+async def workflow_copilot_cancel(
+ cancel_request: WorkflowCopilotCancelRequest,
+ organization: Organization = Depends(org_auth_service.get_current_org),
+) -> None:
+ """Hard-cancel an in-progress workflow copilot v2 turn.
+
+ Sets a per-token Redis flag the SSE handler's cancel-watcher polls; the
+ watcher cancels the handler task, propagating ``CancelledError`` into
+ whichever ``await`` is currently parked (LLM chunk, browser action, DB
+ write). Returns 503 when ``app.CACHE`` is absent — the FE Stop button
+ still aborts client-side, but the backend can't signal the running handler.
+ """
+ cache = getattr(app, "CACHE", None)
+ if cache is None:
+ LOG.warning(
+ "Workflow copilot cancel attempted without cache",
+ organization_id=organization.organization_id,
+ )
+ raise HTTPException(
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+ detail="Cancel not supported in this environment",
+ )
+ await cache.set(
+ _copilot_cancel_key(organization.organization_id, cancel_request.cancel_token),
+ "1",
+ ex=COPILOT_CANCEL_TTL,
+ )
+
+
@base_router.post(
"/workflow/copilot/clear-proposed-workflow", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT
)
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index cc48593eb..4c7c5cd94 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -42,6 +42,17 @@ class WorkflowCopilotChatRequest(BaseModel):
workflow_run_id: str | None = Field(None, description="The workflow run ID to use for the context")
message: str = Field(..., description="The message that user sends")
workflow_yaml: str = Field(..., description="Current workflow YAML including unsaved changes")
+ cancel_token: str | None = Field(
+ None,
+ description=(
+ "Client-generated UUID. POST it to /workflow/copilot/cancel to hard-cancel this turn. "
+ "Optional; legacy clients omit it and cancel becomes a no-op for those requests."
+ ),
+ )
+
+
+class WorkflowCopilotCancelRequest(BaseModel):
+ cancel_token: str = Field(..., description="The cancel_token sent on the original /chat-post request")
class WorkflowCopilotClearProposedWorkflowRequest(BaseModel):
diff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.py
new file mode 100644
index 000000000..afb761d12
--- /dev/null
+++ b/tests/unit/test_workflow_copilot_cancel.py
@@ -0,0 +1,443 @@
+"""Tests for the workflow-copilot v2 hard-cancel feature (SKY-9305).
+
+Covers:
+
+- ``_watch_for_cancel`` cancels its handler exactly when the cache flag flips
+ truthy, sets the ``observed`` closure flag before the cancel, and exits
+ cleanly when the handler completes on its own.
+- ``_build_exit_result(cancelled=True)`` produces an ``AgentResult`` whose
+ ``cancelled`` flag is True and whose ``workflow_was_persisted`` mirrors
+ ``CopilotContext.workflow_persisted`` (so the route's rollback decision has
+ the same source of truth on cancel as it does on success).
+- The route's success-path branch on ``agent_result.cancelled`` runs the
+ cancel-specific persistence (rollback + user msg + ``Cancelled by user.``
+ AI msg + ERROR frame) and skips the normal RESPONSE finalisation.
+- ``/workflow/copilot/cancel`` returns 503 when the Redis cache is absent and
+ 204 + the expected key/TTL when it is present.
+- An operational cancel (``task.cancel()`` without ``user_cancel_observed[0]``
+ set) does NOT persist a ``Cancelled by user.`` chat row.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from fastapi import HTTPException, status
+
+from skyvern.config import settings
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.agent import _build_exit_result
+from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext
+from skyvern.forge.sdk.routes.workflow_copilot import (
+ COPILOT_CANCEL_TTL,
+ _copilot_cancel_key,
+ _watch_for_cancel,
+ workflow_copilot_cancel,
+ workflow_copilot_chat_post,
+)
+from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotCancelRequest,
+ WorkflowCopilotChatRequest,
+ WorkflowCopilotStreamMessageType,
+)
+
+
+class _FakeCache:
+ """Minimal in-memory double of the ``get`` / ``set`` surface of app.CACHE."""
+
+ def __init__(self) -> None:
+ self.store: dict[str, Any] = {}
+ self.set_calls: list[tuple[str, Any, Any]] = []
+
+ async def get(self, key: str) -> Any:
+ return self.store.get(key)
+
+ async def set(self, key: str, value: Any, ex: Any = None) -> None:
+ self.store[key] = value
+ self.set_calls.append((key, value, ex))
+
+
+def _make_chat_request(cancel_token: str | None = "tok_abc") -> WorkflowCopilotChatRequest:
+ return WorkflowCopilotChatRequest(
+ workflow_permanent_id="wpid-1",
+ workflow_id="wf-1",
+ workflow_copilot_chat_id="chat-1",
+ workflow_run_id=None,
+ message="please update",
+ workflow_yaml="title: Example",
+ cancel_token=cancel_token,
+ )
+
+
+# ---------------------------------------------------------------------------
+# _watch_for_cancel
+# ---------------------------------------------------------------------------
+
+
+async def test_watch_for_cancel_signals_and_sets_observed(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Flag flips truthy -> handler_task.cancel() is issued once and observed[0] is True."""
+ # Speed the watcher up so the test doesn't wait for the production cadence.
+ monkeypatch.setattr("skyvern.forge.sdk.routes.workflow_copilot.COPILOT_CANCEL_POLL_SECONDS", 0.01)
+ cache = _FakeCache()
+ handler_task = asyncio.create_task(asyncio.sleep(60))
+ observed: list[bool] = [False]
+ watcher = asyncio.create_task(_watch_for_cancel(cache, "org-1", "tok_abc", handler_task, observed))
+
+ await asyncio.sleep(0.05)
+ cache.store[_copilot_cancel_key("org-1", "tok_abc")] = "1"
+
+ with pytest.raises(asyncio.CancelledError):
+ await handler_task
+
+ await watcher # Watcher exits after issuing the cancel.
+
+ assert observed[0] is True
+
+
+async def test_watch_for_cancel_exits_when_handler_finishes_first(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Handler returns before flag flips -> watcher exits cleanly without raising."""
+ monkeypatch.setattr("skyvern.forge.sdk.routes.workflow_copilot.COPILOT_CANCEL_POLL_SECONDS", 0.01)
+ cache = _FakeCache()
+ handler_task = asyncio.create_task(asyncio.sleep(0.02))
+ observed: list[bool] = [False]
+ watcher = asyncio.create_task(_watch_for_cancel(cache, "org-1", "tok_abc", handler_task, observed))
+
+ await handler_task
+ await asyncio.wait_for(watcher, timeout=1.0)
+
+ assert observed[0] is False
+
+
+# ---------------------------------------------------------------------------
+# _build_exit_result(cancelled=...)
+# ---------------------------------------------------------------------------
+
+
+def test_build_exit_result_cancelled_round_trips_workflow_persisted() -> None:
+ ctx = CopilotContext(
+ organization_id="org-1",
+ workflow_id="wf-1",
+ workflow_permanent_id="wpid-1",
+ workflow_yaml="title: Example",
+ browser_session_id=None,
+ stream=MagicMock(),
+ api_key=None,
+ user_message="please update",
+ workflow_copilot_chat_id="chat-1",
+ )
+ ctx.workflow_persisted = True
+
+ result = _build_exit_result(ctx, "Cancelled by user.", None, cancelled=True)
+
+ assert isinstance(result, AgentResult)
+ assert result.cancelled is True
+ assert result.workflow_was_persisted is True
+ assert result.user_response == "Cancelled by user."
+
+
+def test_build_exit_result_default_cancelled_false() -> None:
+ ctx = CopilotContext(
+ organization_id="org-1",
+ workflow_id="wf-1",
+ workflow_permanent_id="wpid-1",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=MagicMock(),
+ api_key=None,
+ user_message="msg",
+ workflow_copilot_chat_id="chat-1",
+ )
+ result = _build_exit_result(ctx, "Done.", None)
+ assert result.cancelled is False
+
+
+# ---------------------------------------------------------------------------
+# Route cancel branch (agent_result.cancelled=True)
+# ---------------------------------------------------------------------------
+
+
+def _install_fake_create(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]:
+ captured: dict[str, object] = {}
+ sentinel = object()
+
+ def fake_create(request: object, handler: object, ping_interval: int = 10) -> object:
+ del request, ping_interval
+ captured["handler"] = handler
+ return sentinel
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot.FastAPIEventSourceStream.create",
+ fake_create,
+ )
+ captured["sentinel"] = sentinel
+ return captured
+
+
+def _setup_route_mocks(
+ monkeypatch: pytest.MonkeyPatch,
+ chat: SimpleNamespace,
+ original_workflow: SimpleNamespace,
+ agent_result: SimpleNamespace,
+) -> tuple[AsyncMock, SimpleNamespace]:
+ async def fake_llm_handler(*args: object, **kwargs: object) -> None:
+ del args, kwargs
+ return None
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot.get_llm_handler_for_prompt_type",
+ fake_llm_handler,
+ )
+
+ restore_mock = AsyncMock()
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot._restore_workflow_definition",
+ restore_mock,
+ )
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot.run_copilot_agent",
+ AsyncMock(return_value=agent_result),
+ )
+
+ workflow_params = SimpleNamespace(
+ get_workflow_copilot_chat_by_id=AsyncMock(return_value=chat),
+ get_workflow_copilot_chat_messages=AsyncMock(return_value=[]),
+ update_workflow_copilot_chat=AsyncMock(),
+ create_workflow_copilot_chat_message=AsyncMock(
+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))
+ ),
+ )
+ app.DATABASE.workflow_params = workflow_params
+ app.DATABASE.workflows = SimpleNamespace(
+ get_workflow_by_permanent_id=AsyncMock(return_value=original_workflow),
+ )
+ app.DATABASE.observer = SimpleNamespace(
+ get_workflow_run_blocks=AsyncMock(return_value=[]),
+ )
+ app.AGENT_FUNCTION.get_copilot_security_rules = MagicMock(return_value="")
+
+ return restore_mock, workflow_params
+
+
+async def test_route_cancel_branch_persists_user_and_cancelled_messages(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """``agent_result.cancelled=True`` -> rollback + USER msg + ``Cancelled by user.`` AI msg + ERROR frame."""
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+ captured = _install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=None,
+ auto_accept=False,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ agent_result = SimpleNamespace(
+ user_response="Cancelled by user.",
+ updated_workflow=None,
+ global_llm_context=None,
+ workflow_yaml=None,
+ workflow_was_persisted=True,
+ clear_proposed_workflow=False,
+ cancelled=True,
+ total_tokens=None,
+ response_type="REPLY",
+ )
+ restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)
+
+ request = MagicMock()
+ request.headers = {"x-api-key": "sk-test"}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+ assert response is captured["sentinel"]
+
+ sent_payloads: list[Any] = []
+ stream = MagicMock()
+
+ async def _send(payload: Any) -> bool:
+ sent_payloads.append(payload)
+ return True
+
+ stream.send = _send
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(stream)
+
+ # Rollback fired (workflow_was_persisted=True, auto_accept=False, no proposal).
+ restore_mock.assert_awaited_once()
+
+ # Two chat-message inserts: user msg + Cancelled by user. AI msg.
+ insert_calls = workflow_params.create_workflow_copilot_chat_message.await_args_list
+ senders = [c.kwargs.get("sender") for c in insert_calls]
+ contents = [c.kwargs.get("content") for c in insert_calls]
+ assert senders.count("user") == 1
+ assert senders.count("ai") == 1
+ assert "please update" in contents
+ assert "Cancelled by user." in contents
+
+ # Normal RESPONSE finalisation did NOT run: no proposed_workflow update,
+ # no RESPONSE frame.
+ workflow_params.update_workflow_copilot_chat.assert_not_awaited()
+ response_types = [getattr(p, "type", None) for p in sent_payloads]
+ assert WorkflowCopilotStreamMessageType.RESPONSE not in response_types
+ # An ERROR frame carrying "Cancelled by user." was emitted.
+ error_frames = [p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.ERROR]
+ assert any(getattr(f, "error", "") == "Cancelled by user." for f in error_frames)
+
+
+# ---------------------------------------------------------------------------
+# /workflow/copilot/cancel endpoint
+# ---------------------------------------------------------------------------
+#
+# ``app`` is an ``AppHolder`` proxy that forwards ``__setattr__`` / ``__getattr__``
+# to a wrapped ``ForgeApp`` instance, but does not implement ``__delattr__``.
+# That means ``monkeypatch.setattr(app, "CACHE", ...)`` works on assignment but
+# its teardown ``delattr`` raises. Manipulate the underlying instance instead so
+# monkeypatch's teardown lands on a normal attribute.
+
+
+async def test_cancel_endpoint_503_when_cache_missing(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_cancel(
+ WorkflowCopilotCancelRequest(cancel_token="tok_abc"),
+ organization=organization,
+ )
+ assert excinfo.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
+
+
+async def test_cancel_endpoint_204_writes_redis_flag(monkeypatch: pytest.MonkeyPatch) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ result = await workflow_copilot_cancel(
+ WorkflowCopilotCancelRequest(cancel_token="tok_abc"),
+ organization=organization,
+ )
+ assert result is None # 204 No Content
+
+ expected_key = _copilot_cancel_key("org-1", "tok_abc")
+ assert cache.store[expected_key] == "1"
+ assert len(cache.set_calls) == 1
+ key, value, ex = cache.set_calls[0]
+ assert (key, value) == (expected_key, "1")
+ assert isinstance(ex, timedelta)
+ assert ex == COPILOT_CANCEL_TTL
+
+
+# ---------------------------------------------------------------------------
+# Operational cancel disambiguation
+# ---------------------------------------------------------------------------
+
+
+async def test_operational_cancel_does_not_persist_cancelled_message(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """task.cancel() without user_cancel_observed[0] -> no 'Cancelled by user.' chat row.
+
+ The route should treat such a cancel as operational (deploy drain / SIGINT)
+ and re-raise without manufacturing a user-cancel chat row.
+ """
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+ captured = _install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=None,
+ auto_accept=False,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description=None,
+ workflow_definition=None,
+ )
+ # Agent raises CancelledError synchronously (simulates operational cancel
+ # propagating into the await before user_cancel_observed gets set).
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot.run_copilot_agent",
+ AsyncMock(side_effect=asyncio.CancelledError()),
+ )
+
+ async def fake_llm_handler(*args: object, **kwargs: object) -> None:
+ del args, kwargs
+ return None
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot.get_llm_handler_for_prompt_type",
+ fake_llm_handler,
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot._restore_workflow_definition",
+ AsyncMock(),
+ )
+
+ workflow_params = SimpleNamespace(
+ get_workflow_copilot_chat_by_id=AsyncMock(return_value=chat),
+ get_workflow_copilot_chat_messages=AsyncMock(return_value=[]),
+ update_workflow_copilot_chat=AsyncMock(),
+ create_workflow_copilot_chat_message=AsyncMock(
+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))
+ ),
+ )
+ app.DATABASE.workflow_params = workflow_params
+ app.DATABASE.workflows = SimpleNamespace(
+ get_workflow_by_permanent_id=AsyncMock(return_value=original_workflow),
+ )
+ app.DATABASE.observer = SimpleNamespace(
+ get_workflow_run_blocks=AsyncMock(return_value=[]),
+ )
+ app.AGENT_FUNCTION.get_copilot_security_rules = MagicMock(return_value="")
+
+ # Make sure no cache is configured so the watcher never spawns and
+ # user_cancel_observed[0] stays False.
+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)
+
+ request = MagicMock()
+ request.headers = {"x-api-key": "sk-test"}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(cancel_token=None), organization)
+ assert response is captured["sentinel"]
+
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ handler = captured["handler"]
+ assert callable(handler)
+ with pytest.raises(asyncio.CancelledError):
+ await handler(stream)
+
+ insert_calls = workflow_params.create_workflow_copilot_chat_message.await_args_list
+ contents = [c.kwargs.get("content") for c in insert_calls]
+ # No "Cancelled by user." row was written.
+ assert "Cancelled by user." not in contents