Skip to content

Automate browser based workflows with AI

Keep a Copilot turn in the chat when the server restarts mid-turn (SKY-13205) (#7911)

Andrew Neilsone3e1f933c013
diff --git a/alembic/versions/2026_07_31_1634-92ebe14532b9_add_pending_turns_to_workflow_copilot_chats.py b/alembic/versions/2026_07_31_1634-92ebe14532b9_add_pending_turns_to_workflow_copilot_chats.py
new file mode 100644
index 000000000..29f1612c9
--- /dev/null
+++ b/alembic/versions/2026_07_31_1634-92ebe14532b9_add_pending_turns_to_workflow_copilot_chats.py
@@ -0,0 +1,27 @@
+"""add pending_turns to workflow_copilot_chats
+
+Revision ID: 92ebe14532b9
+Revises: 9d85c51a8468
+Create Date: 2026-07-31T16:34:24.245309+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "92ebe14532b9"
+down_revision: Union[str, None] = "9d85c51a8468"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.add_column("workflow_copilot_chats", sa.Column("pending_turns", sa.JSON(), nullable=True))
+
+
+def downgrade() -> None:
+ op.drop_column("workflow_copilot_chats", "pending_turns")
diff --git a/skyvern/cli/core/mcp_http_auth.py b/skyvern/cli/core/mcp_http_auth.py
index a5e9ca4a2..da6486814 100644
--- a/skyvern/cli/core/mcp_http_auth.py
+++ b/skyvern/cli/core/mcp_http_auth.py
@@ -15,6 +15,8 @@ from urllib.parse import urlsplit, urlunsplit
import structlog
from fastapi import HTTPException
+from mcp.server.auth.middleware.bearer_auth import AuthenticatedUser
+from mcp.server.auth.provider import AccessToken
from starlette.requests import Request
from starlette.responses import JSONResponse
from starlette.types import ASGIApp, Receive, Scope, Send
@@ -28,6 +30,7 @@ from skyvern.forge.sdk.services.org_auth_service import resolve_org_from_api_key
from .api_key_hash import hash_api_key_for_cache
from .client import reset_api_key_override, set_api_key_override
+from .session_manager import request_session_scope
LOG = structlog.get_logger(__name__)
API_KEY_HEADER = "x-api-key"
@@ -604,6 +607,32 @@ class MCPAPIKeyMiddleware:
def __init__(self, app: ASGIApp) -> None:
self.app = app
+ async def _forward_authenticated_request(
+ self,
+ scope: Scope,
+ receive: Receive,
+ send: Send,
+ *,
+ organization_id: str,
+ api_key: str,
+ ) -> None:
+ scope.setdefault("state", {})
+ scope["state"]["organization_id"] = organization_id
+ scope["user"] = AuthenticatedUser(
+ AccessToken(
+ token="mcp-http-session-owner",
+ client_id=organization_id,
+ subject=hash_api_key_for_cache(api_key),
+ scopes=[],
+ )
+ )
+ token = set_api_key_override(api_key)
+ try:
+ with request_session_scope(organization_id):
+ await self.app(scope, receive, send)
+ finally:
+ reset_api_key_override(token)
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
@@ -657,13 +686,13 @@ class MCPAPIKeyMiddleware:
try:
resolution = await validate_mcp_oauth_token(bearer_token)
- scope.setdefault("state", {})
- scope["state"]["organization_id"] = resolution.validation.organization_id
- token = set_api_key_override(resolution.api_key)
- try:
- await self.app(scope, receive, send)
- finally:
- reset_api_key_override(token)
+ await self._forward_authenticated_request(
+ scope,
+ receive,
+ send,
+ organization_id=resolution.validation.organization_id,
+ api_key=resolution.api_key,
+ )
return
except HTTPException as e:
if e.status_code == 503:
@@ -682,8 +711,6 @@ class MCPAPIKeyMiddleware:
# Fall back: treat Bearer value as a raw API key
try:
validation = await validate_mcp_api_key(bearer_token)
- scope.setdefault("state", {})
- scope["state"]["organization_id"] = validation.organization_id
except HTTPException as e:
if e.status_code in {401, 403}:
if oauth_service_error is not None:
@@ -705,18 +732,18 @@ class MCPAPIKeyMiddleware:
await response(scope, receive, send)
return
- token = set_api_key_override(bearer_token)
- try:
- await self.app(scope, receive, send)
- finally:
- reset_api_key_override(token)
+ await self._forward_authenticated_request(
+ scope,
+ receive,
+ send,
+ organization_id=validation.organization_id,
+ api_key=bearer_token,
+ )
async def _handle_api_key(self, scope: Scope, receive: Receive, send: Send, api_key: str) -> None:
"""Validate x-api-key header and forward the request (original flow)."""
try:
validation = await validate_mcp_api_key(api_key)
- scope.setdefault("state", {})
- scope["state"]["organization_id"] = validation.organization_id
except HTTPException as e:
if e.status_code in {401, 403}:
response = _unauthorized_response(_INVALID_API_KEY_MESSAGE, include_oauth_challenge=False)
@@ -733,8 +760,10 @@ class MCPAPIKeyMiddleware:
await response(scope, receive, send)
return
- token = set_api_key_override(api_key)
- try:
- await self.app(scope, receive, send)
- finally:
- reset_api_key_override(token)
+ await self._forward_authenticated_request(
+ scope,
+ receive,
+ send,
+ organization_id=validation.organization_id,
+ api_key=api_key,
+ )
diff --git a/skyvern/cli/core/session_manager.py b/skyvern/cli/core/session_manager.py
index b437d5cad..f6abc0ac6 100644
--- a/skyvern/cli/core/session_manager.py
+++ b/skyvern/cli/core/session_manager.py
@@ -6,10 +6,10 @@ import secrets
import time
import weakref
from collections import deque
-from contextlib import asynccontextmanager
+from contextlib import asynccontextmanager, contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, AsyncIterator
+from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator
import structlog
@@ -36,6 +36,7 @@ class SessionState:
browser: SkyvernBrowser | None = None
context: BrowserContext | None = None
api_key_hash: str | None = None
+ organization_id: str | None = None
console_messages: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=1000))
network_requests: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=1000))
dialog_events: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=1000))
@@ -73,14 +74,18 @@ class SessionState:
_current_session: ContextVar[SessionState | None] = ContextVar("mcp_session", default=None)
+_current_organization_id: ContextVar[str | None] = ContextVar("mcp_session_organization_id", default=None)
_global_session: SessionState | None = None
+_organization_sessions: dict[str, SessionState] = {}
_stateless_http_mode = False
_stdio_local_file_access_enabled = False
-# Process-wide registry for copilot browser sessions. Keyed by browser_session_id.
+# Process-wide registry for copilot browser sessions. Keys always carry the
+# owning organization so authenticated HTTP requests cannot address another
+# tenant's entry even when the browser session ID is known.
# This bypasses ContextVar propagation issues when FastMCP runs tool handlers
# in a separate task whose context snapshot predates scoped_session().
-_copilot_sessions: dict[str, SessionState] = {}
+_copilot_sessions: dict[tuple[str, str], SessionState] = {}
# Ref snapshots and their invalidation generations, FIFO-capped like _body_store.
# Capacity eviction can drop a live session's refs early; that fails safe (unknown
# ref → the caller re-observes), which is why the tool contract doesn't mention it.
@@ -88,8 +93,8 @@ _copilot_sessions: dict[str, SessionState] = {}
# touch, so an evicted entry re-materializes with a NEVER-seen token — a stale
# observe holding an old token can't collide with it (no ABA reset to a default).
_SESSION_REF_STORE_MAX = 64
-_session_ref_maps: dict[tuple[str, str, str | None], dict[str, Any]] = {}
-_session_ref_generations: dict[tuple[str, str, str | None], int] = {}
+_session_ref_maps: dict[tuple[str | None, str, str, str | None], dict[str, Any]] = {}
+_session_ref_generations: dict[tuple[str | None, str, str, str | None], int] = {}
_session_ref_generation_counter = itertools.count(1)
# Identity tokens instead of id(): a GC'd page's id() can be reused by a new
# page, making a stale snapshot look valid. Tokens are never reissued.
@@ -119,7 +124,7 @@ def page_ref_key(page: SkyvernBrowserPage) -> tuple[int, int | None, str, str |
)
-def _generation_for(key: tuple[str, str, str | None]) -> int:
+def _generation_for(key: tuple[str | None, str, str, str | None]) -> int:
generation = _session_ref_generations.get(key)
if generation is None:
generation = next(_session_ref_generation_counter)
@@ -134,18 +139,19 @@ def _session_ref_key(
*,
session_id: str | None = None,
cdp_url: str | None = None,
-) -> tuple[str, str, str | None] | None:
+) -> tuple[str | None, str, str, str | None] | None:
context = state.context
resolved_session_id = session_id or (context.session_id if context else None)
resolved_cdp_url = cdp_url or (context.cdp_url if context else None)
# Prefer the hash stored at resolve_browser time — recomputing runs a
# deliberately slow PBKDF2 per call, and this sits on the per-ref hot path.
api_key_hash = state.api_key_hash or _api_key_hash(get_active_api_key())
+ organization_id = _current_organization_id.get() or state.organization_id
if resolved_session_id:
- return ("cloud_session", resolved_session_id, api_key_hash)
+ return (organization_id, "cloud_session", resolved_session_id, api_key_hash)
if resolved_cdp_url:
- return ("cdp", resolved_cdp_url, api_key_hash)
+ return (organization_id, "cdp", resolved_cdp_url, api_key_hash)
return None
@@ -233,24 +239,33 @@ def clear_session_ref_map(
state._observed_refs_generation = next(_session_ref_generation_counter)
-def register_copilot_session(session_id: str, state: SessionState) -> None:
+def register_copilot_session(session_id: str, state: SessionState, *, organization_id: str) -> None:
"""Register a pre-configured browser session for cross-task lookup.
The registry is process-local and in-memory: entries do not survive a
process restart and are not shared across uvicorn workers. Callers that
need cross-process continuity must reconnect via the cloud session API.
"""
- _copilot_sessions[session_id] = state
+ if not organization_id:
+ raise ValueError("organization_id is required for a copilot browser session")
+ if state.organization_id not in {None, organization_id}:
+ raise ValueError("session state belongs to a different organization")
+ state.organization_id = organization_id
+ _copilot_sessions[(organization_id, session_id)] = state
-def unregister_copilot_session(session_id: str) -> None:
+def unregister_copilot_session(session_id: str, *, organization_id: str) -> None:
"""Remove a copilot browser session from the process-local registry."""
- _copilot_sessions.pop(session_id, None)
+ _copilot_sessions.pop((organization_id, session_id), None)
def active_copilot_session_ids() -> set[str]:
"""Browser-session IDs currently bound to an active copilot turn."""
- return set(_copilot_sessions)
+ return {session_id for _, session_id in _copilot_sessions}
+
+
+def _registered_copilot_session(session_id: str, *, organization_id: str) -> SessionState | None:
+ return _copilot_sessions.get((organization_id, session_id))
def _explicit_cloud_session_can_access_localhost() -> bool:
@@ -260,7 +275,17 @@ def _explicit_cloud_session_can_access_localhost() -> bool:
def get_current_session() -> SessionState:
global _global_session
+ organization_id = _current_organization_id.get()
state = _current_session.get()
+
+ if organization_id is not None and not _stateless_http_mode:
+ state = _organization_sessions.get(organization_id)
+ if state is None:
+ state = SessionState(organization_id=organization_id)
+ _organization_sessions[organization_id] = state
+ _current_session.set(state)
+ return state
+
if state is not None:
return state
@@ -280,11 +305,30 @@ def get_current_session() -> SessionState:
def set_current_session(state: SessionState) -> None:
global _global_session
- if not _stateless_http_mode:
+
+ organization_id = _current_organization_id.get()
+ if organization_id is not None and not _stateless_http_mode:
+ state.organization_id = organization_id
+ _organization_sessions[organization_id] = state
+ elif not _stateless_http_mode:
_global_session = state
_current_session.set(state)
+@contextmanager
+def request_session_scope(organization_id: str) -> Iterator[None]:
+ if not organization_id:
+ raise ValueError("organization_id is required for an authenticated MCP request")
+
+ organization_token = _current_organization_id.set(organization_id)
+ session_token = _current_session.set(None)
+ try:
+ yield
+ finally:
+ _current_session.reset(session_token)
+ _current_organization_id.reset(organization_token)
+
+
@asynccontextmanager
async def scoped_session(state: SessionState) -> AsyncIterator[None]:
"""Temporarily push a SessionState into ContextVar scope.
@@ -396,12 +440,17 @@ async def resolve_browser(
# Check copilot session registry (cross-task fallback when ContextVar
# does not propagate through FastMCP in-process transport).
- registered = _copilot_sessions.get(session_id) if session_id else None
+ organization_id = _current_organization_id.get()
+ registered = (
+ _registered_copilot_session(session_id, organization_id=organization_id)
+ if session_id and organization_id
+ else None
+ )
if registered is not None and registered.browser is not None and registered.context is not None:
if _hashes_equal(registered.api_key_hash, active_api_key_hash) or not has_api_key_override():
# FastMCP in-process tool tasks may miss the parent ContextVar.
# Explicit request overrides still win; otherwise use the temporary Copilot registry.
- _current_session.set(registered)
+ set_current_session(registered)
return registered.browser, registered.context
browser: SkyvernBrowser | None = None
@@ -452,13 +501,16 @@ async def resolve_browser(
raise BrowserNotAvailableError()
-async def close_current_session() -> None:
- """Close the active browser session (if any) and clear local session state."""
+async def _close_session_state(current: SessionState, *, close_via_active_client: bool) -> None:
from .session_ops import do_session_close
- current = get_current_session()
try:
- if current.context and current.context.mode == "cloud_session" and current.context.session_id:
+ if (
+ close_via_active_client
+ and current.context
+ and current.context.mode == "cloud_session"
+ and current.context.session_id
+ ):
try:
skyvern = get_skyvern()
await do_session_close(skyvern, current.context.session_id)
@@ -483,10 +535,53 @@ async def close_current_session() -> None:
clear_session_ref_map()
if current.context and current.context.session_id:
delete_session_trajectories(current.context.session_id)
- unregister_copilot_session(current.context.session_id)
+ organization_id = _current_organization_id.get() or current.organization_id
+ if organization_id is not None:
+ unregister_copilot_session(current.context.session_id, organization_id=organization_id)
+
+
+async def close_current_session() -> None:
+ """Close the active browser session (if any) and clear local session state."""
+ current = get_current_session()
+ try:
+ await _close_session_state(current, close_via_active_client=True)
+ finally:
set_current_session(SessionState())
+async def close_all_sessions() -> None:
+ errors: list[tuple[str | None, BaseException]] = []
+ for organization_id in list(_organization_sessions):
+ try:
+ with request_session_scope(organization_id):
+ current = get_current_session()
+ try:
+ # Preserve the session ID so browser.close() uses the browser's owning client for remote cleanup.
+ await _close_session_state(current, close_via_active_client=False)
+ finally:
+ set_current_session(SessionState())
+ except BaseException as exc:
+ errors.append((organization_id, exc))
+
+ _organization_sessions.clear()
+ try:
+ await close_current_session()
+ except BaseException as exc:
+ errors.append((None, exc))
+ finally:
+ _current_session.set(None)
+ _current_organization_id.set(None)
+
+ if errors:
+ for failed_organization_id, cleanup_error in errors[1:]:
+ LOG.warning(
+ "Additional session cleanup failed",
+ organization_id=failed_organization_id,
+ exc_info=(type(cleanup_error), cleanup_error, cleanup_error.__traceback__),
+ )
+ raise errors[0][1]
+
+
async def get_page(
session_id: str | None = None,
cdp_url: str | None = None,
diff --git a/skyvern/cli/run_commands.py b/skyvern/cli/run_commands.py
index 20c5121d6..4a601dd92 100644
--- a/skyvern/cli/run_commands.py
+++ b/skyvern/cli/run_commands.py
@@ -65,13 +65,13 @@ async def _cleanup_mcp_resources() -> None:
from skyvern.cli.core.action_log import shutdown_action_log_worker # noqa: PLC0415
from skyvern.cli.core.client import close_skyvern # noqa: PLC0415
from skyvern.cli.core.mcp_http_auth import close_auth_db # noqa: PLC0415
- from skyvern.cli.core.session_manager import close_current_session # noqa: PLC0415
+ from skyvern.cli.core.session_manager import close_all_sessions # noqa: PLC0415
try:
await shutdown_action_log_worker()
finally:
try:
- await close_current_session()
+ await close_all_sessions()
finally:
try:
await close_skyvern()
diff --git a/skyvern/forge/sdk/copilot/canonical_ownership.py b/skyvern/forge/sdk/copilot/canonical_ownership.py
new file mode 100644
index 000000000..7dc1135c3
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/canonical_ownership.py
@@ -0,0 +1,24 @@
+"""Ownership of a copilot turn's canonical workflow write.
+
+An interrupted turn may only roll back a canonical write it still owns. Ownership is
+recorded, not inferred: the writer stamps the fingerprint of what it wrote onto the turn's
+pending marker, and reconcile rolls back only while canonical still carries that content.
+A later write by anyone — another chat, a manual edit, a subsequent turn — changes the
+fingerprint and forfeits the rollback.
+"""
+
+import hashlib
+import json
+from typing import Any
+
+# A write always moves these, so they carry no information about who wrote it.
+VOLATILE_WORKFLOW_FIELDS = frozenset({"workflow_id", "version", "created_at", "modified_at"})
+
+
+def workflow_content(workflow: dict[str, Any]) -> dict[str, Any]:
+ return {key: value for key, value in workflow.items() if key not in VOLATILE_WORKFLOW_FIELDS}
+
+
+def workflow_content_fingerprint(workflow: dict[str, Any]) -> str:
+ canonical = json.dumps(workflow_content(workflow), sort_keys=True, default=str)
+ return hashlib.sha256(canonical.encode()).hexdigest()
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 25387bcad..43ca7e634 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -25,6 +25,7 @@ from mcp.types import (
)
from playwright.async_api import Browser, BrowserContext
+from skyvern.cli.core.session_manager import request_session_scope
from skyvern.forge import app
from skyvern.forge.agent_functions import CopilotCandidateNetworkHop
from skyvern.forge.sdk.copilot.blocker_signal import (
@@ -308,7 +309,8 @@ class SkyvernOverlayMCPServer(MCPServer):
stack = AsyncExitStack()
await stack.__aenter__()
client = Client(self._transport)
- await stack.enter_async_context(client)
+ with request_session_scope(self._context_provider().organization_id):
+ await stack.enter_async_context(client)
self._client = client
self._exit_stack = stack
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 187540fa8..de5fff6b2 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -733,13 +733,14 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:
browser=skyvern_browser,
context=mcp_ctx,
api_key_hash=hash_api_key_for_cache(active_key) if active_key else None,
+ organization_id=ctx.organization_id,
)
if working_page is not None:
# Seed the tab pin from the already-probed page (mirrors what skyvern_tab_switch
# sets interactively) so self-heal tools land on the adopted tab instead of the
# new SkyvernBrowser's pages[-1] fallback.
state._active_page = working_page
- register_copilot_session(browser_session_id, state)
+ register_copilot_session(browser_session_id, state, organization_id=ctx.organization_id)
if is_self_heal_session_id(browser_session_id):
LOG.info(
"registered self-heal browser session",
@@ -756,7 +757,7 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:
)
else:
ctx.sdk_action_workflow_run_ids_by_browser_session.pop(sdk_action_workflow_run_cache_key, None)
- unregister_copilot_session(browser_session_id)
+ unregister_copilot_session(browser_session_id, organization_id=ctx.organization_id)
if is_self_heal_session_id(browser_session_id):
LOG.info("unregistered self-heal browser session", session_id=browser_session_id)
finally:
diff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.py
index bb7dc0217..9a49278f4 100644
--- a/skyvern/forge/sdk/copilot/terminal_envelope.py
+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py
@@ -16,6 +16,12 @@ _FINAL_RUN_VERDICTS = frozenset({"demonstrated", "not_demonstrated", "not_evalua
_REVIEW_PROPOSAL_DISPOSITIONS = frozenset({"review_untested", "review_tested"})
_SHADOW_REASON_TRAILING_PUNCTUATION = ".,;:!?"
+INTERRUPTED_TERMINAL_REASON = "interrupted"
+INTERRUPTED_TERMINAL_MESSAGE = (
+ "This turn was interrupted before it could finish — the server stopped part-way through it. "
+ "Send your message again to retry."
+)
+
class TerminalOutcomeEnvelope(BaseModel):
next_state: TerminalNextState
@@ -108,6 +114,16 @@ def finalize_applied_state(
)
+def interrupted_terminal_envelope() -> TerminalOutcomeEnvelope:
+ """Envelope for a turn whose process died mid-run — stopped, but never user-cancelled."""
+ return TerminalOutcomeEnvelope(
+ next_state="stopped",
+ verified=False,
+ response_kind="stopped",
+ halt_kind=INTERRUPTED_TERMINAL_REASON,
+ )
+
+
def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: str, cancelled: bool) -> tuple[str, bool]:
# Diagnose/refuse answers share next_state="stopped" but their text IS the
# deliverable — only stopped-kind turns get the honest-stop replacement.
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index a5b01b8a6..b066d1bd8 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import ast
+import contextlib
import copy
import hashlib
import io
@@ -64,6 +65,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
recorded_outcome_from_author_time_reject,
recorded_outcome_from_authoring_repair_context,
)
+from skyvern.forge.sdk.copilot.canonical_ownership import workflow_content_fingerprint
from skyvern.forge.sdk.copilot.code_block_preflight import (
SANDBOX_UNRESOLVED_NAME_REASON_CODE,
author_time_code_block_diagnostics,
@@ -10155,6 +10157,31 @@ def _enrich_scout_trajectory_input_correspondences(workflow_yaml: str, ctx: Agen
interaction.pop("input_correspondences", None)
+async def _record_canonical_write_ownership(ctx: CopilotContext, workflow: Workflow) -> None:
+ """Stamp the turn's marker with what it just left canonical as.
+
+ Best-effort: a missed stamp only costs this turn its rollback claim, while raising here
+ would fail a write that already succeeded. Canonical is re-read rather than fingerprinted
+ from the in-memory object so the value matches what reconcile computes.
+ """
+ with contextlib.suppress(Exception):
+ workflow_permanent_id = workflow.workflow_permanent_id
+ if not (ctx.workflow_copilot_chat_id and ctx.turn_id and workflow_permanent_id and ctx.organization_id):
+ return
+ persisted = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ )
+ if persisted is None:
+ return
+ await app.DATABASE.workflow_params.record_pending_copilot_turn_canonical_write(
+ organization_id=ctx.organization_id,
+ workflow_copilot_chat_id=ctx.workflow_copilot_chat_id,
+ turn_id=ctx.turn_id,
+ fingerprint=workflow_content_fingerprint(persisted.model_dump(mode="json")),
+ )
+
+
async def _update_workflow(
params: dict[str, Any],
ctx: AgentContext,
@@ -10537,6 +10564,10 @@ async def _update_workflow(
edited_by="copilot",
)
ctx.canonical_was_persisted_due_to_param_change = True
+ # isinstance narrows the declared ``AgentContext`` to the marker-aware
+ # ``CopilotContext`` for mypy, matching the narrative-emit seam below.
+ if isinstance(ctx, CopilotContext):
+ await _record_canonical_write_ownership(ctx, workflow)
ctx.staged_workflow_yaml = workflow_yaml
ctx.staged_workflow = workflow
ctx.has_staged_proposal = True
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 5be42f463..6f29f823d 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -1719,6 +1719,7 @@ class WorkflowCopilotChatModel(Base):
workflow_permanent_id = Column(String, nullable=False, index=True)
proposed_workflow = Column(JSON, nullable=True)
auto_accept = Column(Boolean, nullable=True, default=False)
+ pending_turns = Column(JSON, nullable=True)
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
modified_at = Column(
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index 2fb96f466..9cd79f4c6 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -8,6 +8,7 @@ from typing import Any
import structlog
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.orm import defer
from skyvern.config import settings
from skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_json, criterion_authority_projection
@@ -50,6 +51,7 @@ from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
from skyvern.forge.sdk.schemas.task_generations import TaskGeneration
from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
from skyvern.forge.sdk.schemas.workflow_copilot import (
+ CopilotPendingTurn,
NonAdoptableCriteriaSet,
WorkflowCopilotChat,
WorkflowCopilotChatMessage,
@@ -77,6 +79,8 @@ from skyvern.webeye.actions.actions import Action
LOG = structlog.get_logger()
+PENDING_TURN_RETENTION = timedelta(days=30)
+
def _floor_rekeyed_association_is_coherent(item: Mapping[str, object]) -> bool:
marker_present = "requested_output_floor_rekeyed" in item
@@ -171,6 +175,32 @@ def _decode_completion_criteria_set(
)
+def _parse_pending_turn_timestamp(value: object) -> datetime | None:
+ if not isinstance(value, str):
+ return None
+ try:
+ parsed = datetime.fromisoformat(value)
+ except ValueError:
+ return None
+ return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
+
+
+def _prune_pending_turns(pending_turns: object) -> dict[str, Any]:
+ """Drop markers too old to still be worth reconciling, so the column cannot grow without bound."""
+ if not isinstance(pending_turns, dict):
+ return {}
+ cutoff = datetime.now(timezone.utc) - PENDING_TURN_RETENTION
+ kept: dict[str, Any] = {}
+ for turn_id, entry in pending_turns.items():
+ if not isinstance(entry, dict):
+ continue
+ started_at = _parse_pending_turn_timestamp(entry.get("started_at"))
+ if started_at is not None and started_at < cutoff:
+ continue
+ kept[turn_id] = entry
+ return kept
+
+
class WorkflowParametersRepository(BaseRepository):
"""Database operations for workflow parameters, copilot chat, task generation, actions, and runs."""
@@ -666,6 +696,168 @@ class WorkflowParametersRepository(BaseRepository):
await session.refresh(new_message)
return convert_to_workflow_copilot_chat_message(new_message, self.debug_enabled)
+ @db_operation("start_copilot_turn")
+ async def start_copilot_turn(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ pending_turn: CopilotPendingTurn,
+ user_message: str,
+ audio_artifact_id: str | None = None,
+ ) -> WorkflowCopilotChatMessage:
+ """Write the turn's user row and its pending marker in one transaction.
+
+ Two separate commits would leave a crash window in which the user
+ message exists with no marker to reconcile it.
+ """
+ async with self.Session() as session:
+ new_message = WorkflowCopilotChatMessageModel(
+ workflow_copilot_chat_id=workflow_copilot_chat_id,
+ organization_id=organization_id,
+ sender=WorkflowCopilotChatSender.USER,
+ content=user_message,
+ audio_artifact_id=audio_artifact_id,
+ )
+ session.add(new_message)
+ await session.flush()
+ chat = (
+ await session.scalars(
+ select(WorkflowCopilotChatModel)
+ .where(WorkflowCopilotChatModel.organization_id == organization_id)
+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)
+ .with_for_update()
+ )
+ ).first()
+ if chat is None:
+ raise NotFoundError(f"workflow copilot chat {workflow_copilot_chat_id}")
+ pending = _prune_pending_turns(chat.pending_turns)
+ pending[pending_turn.turn_id] = pending_turn.model_copy(
+ update={"user_message_id": new_message.workflow_copilot_chat_message_id}
+ ).model_dump(mode="json")
+ chat.pending_turns = pending
+ await session.commit()
+ await session.refresh(new_message)
+ return convert_to_workflow_copilot_chat_message(new_message, self.debug_enabled)
+
+ @db_operation("claim_pending_copilot_turn")
+ async def claim_pending_copilot_turn(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ claim_before: datetime,
+ ) -> bool:
+ """Stamp ``recovering_at`` on one pending turn, returning whether this caller won it.
+
+ A recovery that itself crashes leaves the stamp behind; a later reader
+ reclaims it once the stamp predates ``claim_before``.
+ """
+ async with self.Session() as session:
+ chat = (
+ await session.scalars(
+ select(WorkflowCopilotChatModel)
+ .where(WorkflowCopilotChatModel.organization_id == organization_id)
+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)
+ .with_for_update()
+ )
+ ).first()
+ if chat is None:
+ return False
+ pending = dict(chat.pending_turns or {})
+ entry = pending.get(turn_id)
+ if not isinstance(entry, dict):
+ return False
+ recovering_at = _parse_pending_turn_timestamp(entry.get("recovering_at"))
+ if recovering_at is not None and recovering_at > claim_before:
+ return False
+ pending[turn_id] = {**entry, "recovering_at": datetime.now(timezone.utc).isoformat()}
+ chat.pending_turns = pending
+ await session.commit()
+ return True
+
+ @db_operation("clear_pending_copilot_turn")
+ async def clear_pending_copilot_turn(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ ) -> None:
+ async with self.Session() as session:
+ chat = (
+ await session.scalars(
+ select(WorkflowCopilotChatModel)
+ .where(WorkflowCopilotChatModel.organization_id == organization_id)
+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)
+ .with_for_update()
+ )
+ ).first()
+ if chat is None:
+ return
+ pending = dict(chat.pending_turns or {})
+ if pending.pop(turn_id, None) is None:
+ return
+ chat.pending_turns = pending
+ await session.commit()
+
+ @db_operation("record_pending_copilot_turn_canonical_write")
+ async def record_pending_copilot_turn_canonical_write(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ fingerprint: str,
+ ) -> None:
+ """Stamp what this turn left canonical as, so reconcile can tell its own write from anyone else's."""
+ async with self.Session() as session:
+ chat = (
+ await session.scalars(
+ select(WorkflowCopilotChatModel)
+ .where(WorkflowCopilotChatModel.organization_id == organization_id)
+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)
+ .with_for_update()
+ )
+ ).first()
+ if chat is None:
+ return
+ pending = dict(chat.pending_turns or {})
+ entry = pending.get(turn_id)
+ if not isinstance(entry, dict):
+ return
+ pending[turn_id] = {**entry, "canonical_write_fingerprint": fingerprint}
+ chat.pending_turns = pending
+ await session.commit()
+
+ @db_operation("replace_workflow_copilot_chat_message")
+ async def replace_workflow_copilot_chat_message(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_message_id: str,
+ content: str,
+ global_llm_context: str | None,
+ turn_outcome: TurnOutcome | None,
+ narrative_payload: TurnNarrativePayload | dict[str, Any] | None,
+ ) -> WorkflowCopilotChatMessage | None:
+ async with self.Session() as session:
+ message = (
+ await session.scalars(
+ select(WorkflowCopilotChatMessageModel)
+ .where(WorkflowCopilotChatMessageModel.organization_id == organization_id)
+ .where(
+ WorkflowCopilotChatMessageModel.workflow_copilot_chat_message_id
+ == workflow_copilot_chat_message_id
+ )
+ )
+ ).first()
+ if message is None:
+ return None
+ message.content = content
+ message.global_llm_context = global_llm_context
+ message.turn_outcome = turn_outcome.model_dump(mode="json") if turn_outcome else None
+ message.narrative_payload = narrative_payload
+ await session.commit()
+ await session.refresh(message)
+ return WorkflowCopilotChatMessage.model_validate(message)
+
@db_operation("get_workflow_copilot_chat_messages")
async def get_workflow_copilot_chat_messages(
self,
@@ -748,6 +940,7 @@ class WorkflowParametersRepository(BaseRepository):
)
query = (
select(WorkflowCopilotChatModel, first_message.c.title)
+ .options(defer(WorkflowCopilotChatModel.pending_turns))
.join(first_message, first_message.c.chat_id == WorkflowCopilotChatModel.workflow_copilot_chat_id)
.where(WorkflowCopilotChatModel.organization_id == organization_id)
)
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index cf7c467d2..e3b7a2af1 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
from skyvern.forge.sdk.copilot.agent import run_copilot_agent
from skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_write, resolve_copilot_created_by_stamp
+from skyvern.forge.sdk.copilot.canonical_ownership import workflow_content_fingerprint
from skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_steps
from skyvern.forge.sdk.copilot.completion_criteria_store import (
CRITERIA_SET_STATUS_ACTIVE,
@@ -49,6 +50,7 @@ from skyvern.forge.sdk.copilot.credential_pause import (
resolve_credential_pause,
)
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
+from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
from skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handler
from skyvern.forge.sdk.copilot.output_utils import truncate_output
from skyvern.forge.sdk.copilot.recoverable_failure import (
@@ -59,8 +61,11 @@ from skyvern.forge.sdk.copilot.recoverable_failure import (
)
from skyvern.forge.sdk.copilot.request_policy import is_defer_authoring_durable_fill_criterion
from skyvern.forge.sdk.copilot.terminal_envelope import (
+ INTERRUPTED_TERMINAL_MESSAGE,
+ INTERRUPTED_TERMINAL_REASON,
TerminalOutcomeEnvelope,
finalize_applied_state,
+ interrupted_terminal_envelope,
reason_in_reply_shadow,
render_terminal_message,
)
@@ -78,6 +83,7 @@ from skyvern.forge.sdk.routes.routers import base_router
from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.workflow_copilot import (
+ CopilotPendingTurn,
NonAdoptableCriteriaSet,
WorkflowCopilotApplyProposedWorkflowRequest,
WorkflowCopilotAudioUploadResponse,
@@ -108,11 +114,18 @@ from skyvern.schemas.workflows import (
WorkflowDefinitionYAML,
)
from skyvern.utils.prompt_truncation import truncate_page_html_for_summary
+from skyvern.utils.secret_headers import merge_masked_headers
from skyvern.utils.strings import escape_code_fences
from skyvern.utils.yaml_loader import safe_load_no_dates
WORKFLOW_KNOWLEDGE_BASE_PATH = Path("skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt")
CHAT_HISTORY_CONTEXT_MESSAGES = 10
+
+# Wall clock, unlike TOTAL_TIMEOUT_SECONDS: that budget excludes time parked in a
+# credential pause, so a legitimately live turn can outlast it in real time.
+RECONCILE_ABANDON_AFTER_SECONDS = (
+ TOTAL_TIMEOUT_SECONDS + settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS + 120
+)
WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS = 3_000_000
ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES = {
"audio/mp4",
@@ -152,6 +165,7 @@ def bind_copilot_session_id(chat_id: str | None) -> Iterator[None]:
COPILOT_CODE_MODE_OPT_OUT_EVENT = "copilot_code_mode_opt_out"
COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON = "copilot_recoverable_failure"
+USER_CANCELLED_TERMINAL_REASON = "user_cancelled"
def _effective_copilot_composer_mode(
@@ -757,9 +771,131 @@ async def _persist_proposed_workflow_state(
await _clear_proposed_workflow(chat)
+def _as_utc(value: datetime) -> datetime:
+ return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
+
+
+async def _assistant_row_for_turn(chat: WorkflowCopilotChat, turn_id: str) -> WorkflowCopilotChatMessage | None:
+ # Reads the whole chat to find one turn's row, on every finalize. Fine at current per-chat message
+ # counts; if histories grow, index turn_outcome->>'copilot_turn_id' and query for it directly.
+ messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ )
+ return next(
+ (
+ message
+ for message in messages
+ if message.sender == WorkflowCopilotChatSender.AI
+ and message.turn_outcome is not None
+ and message.turn_outcome.copilot_turn_id == turn_id
+ ),
+ None,
+ )
+
+
+async def _assistant_row_exists_for_turn(chat: WorkflowCopilotChat, turn_id: str) -> bool:
+ return await _assistant_row_for_turn(chat, turn_id) is not None
+
+
+def _is_interrupted_recovery_row(message: WorkflowCopilotChatMessage) -> bool:
+ return message.turn_outcome is not None and message.turn_outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON
+
+
+async def _clear_pending_turn(chat: WorkflowCopilotChat, turn_id: str) -> None:
+ # A failed clear is self-healing: the reconcile pass sees the turn already
+ # answered, skips recovery and retries the clear.
+ with contextlib.suppress(Exception):
+ await app.DATABASE.workflow_params.clear_pending_copilot_turn(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ turn_id=turn_id,
+ )
+
+
+async def _persist_turn_messages(
+ *,
+ chat: WorkflowCopilotChat,
+ turn_id: str | None,
+ user_message: str,
+ audio_artifact_id: str | None,
+ user_row_already_persisted: bool,
+ assistant_content: str,
+ global_llm_context: str | None,
+ turn_outcome: TurnOutcome | None,
+ narrative_payload: TurnNarrativePayload | None,
+) -> WorkflowCopilotChatMessage | None:
+ """The only writer of a copilot turn's chat rows; idempotent per ``turn_id``.
+
+ Returns None when the assistant row for this turn already exists, and always
+ drops the turn's pending marker last so a crash mid-write stays recoverable.
+ """
+ if turn_id is not None and turn_outcome is not None and not turn_outcome.copilot_turn_id:
+ turn_outcome = turn_outcome.model_copy(update={"copilot_turn_id": turn_id})
+
+ if not user_row_already_persisted:
+ await asyncio.shield(
+ app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ sender=WorkflowCopilotChatSender.USER,
+ content=user_message,
+ audio_artifact_id=audio_artifact_id,
+ )
+ )
+
+ assistant_message: WorkflowCopilotChatMessage | None = None
+ existing = await _assistant_row_for_turn(chat, turn_id) if turn_id is not None else None
+ superseding_recovery = (
+ existing is not None
+ and _is_interrupted_recovery_row(existing)
+ and turn_outcome is not None
+ and turn_outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON
+ )
+ if existing is not None and superseding_recovery:
+ # Recovery reached this turn first and wrote an interrupted row. The turn then finished,
+ # so its real reply is the truth and replaces that row rather than being dropped.
+ LOG.info(
+ "Copilot turn finished after being recovered; replacing the interrupted row with its reply",
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ turn_id=turn_id,
+ )
+ assistant_message = await asyncio.shield(
+ app.DATABASE.workflow_params.replace_workflow_copilot_chat_message(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_message_id=existing.workflow_copilot_chat_message_id,
+ content=assistant_content,
+ global_llm_context=global_llm_context,
+ turn_outcome=turn_outcome,
+ narrative_payload=narrative_payload,
+ )
+ )
+ elif existing is not None:
+ LOG.info(
+ "Copilot turn already has an assistant row; skipping duplicate persist",
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ turn_id=turn_id,
+ )
+ else:
+ assistant_message = await asyncio.shield(
+ app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ sender=WorkflowCopilotChatSender.AI,
+ content=assistant_content,
+ global_llm_context=global_llm_context,
+ turn_outcome=turn_outcome,
+ narrative_payload=narrative_payload,
+ )
+ )
+
+ if turn_id is not None:
+ await _clear_pending_turn(chat, turn_id)
+ return assistant_message
+
+
async def _persist_cancel_turn(
stream: EventSourceStream,
- chat: Any,
+ chat: WorkflowCopilotChat,
organization_id: str,
original_workflow: Workflow | None,
user_message: str,
@@ -768,6 +904,7 @@ async def _persist_cancel_turn(
turn_id: str | None = None,
keep_pending_proposal: bool = False,
prior_global_llm_context: str | None = None,
+ user_row_already_persisted: bool = False,
) -> None:
"""Persist a cancelled turn and emit a terminal SSE response frame.
@@ -785,7 +922,12 @@ async def _persist_cancel_turn(
total_tokens = None
response_type = "REPLY"
output_policy_diagnostics = None
- turn_outcome = None
+ turn_outcome = TurnOutcome(
+ response_kind=ResponseKind.RECOVER,
+ reason_code=USER_CANCELLED_TERMINAL_REASON,
+ terminal_reason=USER_CANCELLED_TERMINAL_REASON,
+ copilot_turn_id=turn_id,
+ )
response_turn_id = turn_id
narrative_summary = None
narrative_payload = None
@@ -860,26 +1002,18 @@ async def _persist_cancel_turn(
terminal_envelope=terminal_envelope,
)
- await asyncio.shield(
- app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- sender=WorkflowCopilotChatSender.USER,
- content=user_message,
- audio_artifact_id=audio_artifact_id,
- )
- )
- assistant_message = await asyncio.shield(
- app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- sender=WorkflowCopilotChatSender.AI,
- content=user_response,
- global_llm_context=updated_global_llm_context,
- turn_outcome=turn_outcome,
- narrative_payload=narrative_payload,
- )
+ assistant_message = await _persist_turn_messages(
+ chat=chat,
+ turn_id=turn_id,
+ user_message=user_message,
+ audio_artifact_id=audio_artifact_id,
+ user_row_already_persisted=user_row_already_persisted,
+ assistant_content=user_response,
+ global_llm_context=updated_global_llm_context,
+ turn_outcome=turn_outcome,
+ narrative_payload=narrative_payload,
)
+ response_time = assistant_message.created_at if assistant_message else datetime.now(timezone.utc)
try:
await asyncio.shield(
stream.send(
@@ -888,7 +1022,7 @@ async def _persist_cancel_turn(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
message=user_response,
updated_workflow=updated_workflow.model_dump(mode="json") if updated_workflow else None,
- response_time=assistant_message.created_at,
+ response_time=response_time,
total_tokens=total_tokens,
response_type=response_type,
proposal_disposition=proposal_disposition,
@@ -917,6 +1051,8 @@ async def _finalise_normal_turn(
original_workflow: Workflow | None,
chat_request: WorkflowCopilotChatRequest,
agent_result: AgentResult,
+ turn_id: str | None = None,
+ user_row_already_persisted: bool = False,
) -> None:
"""Atomic post-agent finalisation: rollback, proposal, chat rows, RESPONSE.
@@ -1027,19 +1163,13 @@ async def _finalise_normal_turn(
terminal_envelope=terminal_envelope,
)
- await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- sender=WorkflowCopilotChatSender.USER,
- content=chat_request.message,
+ assistant_message = await _persist_turn_messages(
+ chat=chat,
+ turn_id=turn_id,
+ user_message=chat_request.message,
audio_artifact_id=chat_request.audio_artifact_id,
- )
-
- assistant_message = await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- sender=WorkflowCopilotChatSender.AI,
- content=user_response,
+ user_row_already_persisted=user_row_already_persisted,
+ assistant_content=user_response,
global_llm_context=updated_global_llm_context,
turn_outcome=agent_result.turn_outcome,
narrative_payload=narrative_payload,
@@ -1051,7 +1181,7 @@ async def _finalise_normal_turn(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
message=user_response,
updated_workflow=updated_workflow.model_dump(mode="json") if updated_workflow else None,
- response_time=assistant_message.created_at,
+ response_time=assistant_message.created_at if assistant_message else datetime.now(timezone.utc),
total_tokens=agent_result.total_tokens,
response_type=agent_result.response_type,
proposal_disposition=proposal_disposition,
@@ -1662,6 +1792,7 @@ async def _new_copilot_chat_post(
original_workflow: Workflow | None = None
chat = None
+ turn_started = False
# Snapshot before any write this turn — lets a recovery handler tell
# "this turn's own write already superseded it" from "the write itself
# is what failed", which agent_result.updated_workflow alone can't.
@@ -1761,6 +1892,8 @@ async def _new_copilot_chat_post(
original_workflow=original_workflow,
chat_request=chat_request,
agent_result=recovered_result,
+ turn_id=turn_id,
+ user_row_already_persisted=turn_started,
)
)
terminal_frame_emitted = True
@@ -1959,6 +2092,21 @@ async def _new_copilot_chat_post(
stored_completion_criteria = await _load_completion_criteria_snapshot(chat)
+ await app.DATABASE.workflow_params.start_copilot_turn(
+ organization_id=organization.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ pending_turn=CopilotPendingTurn(
+ turn_id=turn_id,
+ started_at=datetime.now(timezone.utc),
+ pre_turn_workflow=original_workflow.model_dump(mode="json"),
+ pre_turn_proposed_workflow=proposed_workflow_at_turn_start,
+ keep_pending_proposal=chat_request.keep_pending_proposal,
+ ),
+ user_message=chat_request.message,
+ audio_artifact_id=chat_request.audio_artifact_id,
+ )
+ turn_started = True
+
with bind_copilot_session_id(chat.workflow_copilot_chat_id):
agent_result = await run_copilot_agent(
stream=stream,
@@ -1998,6 +2146,7 @@ async def _new_copilot_chat_post(
audio_artifact_id=chat_request.audio_artifact_id,
turn_id=turn_id,
keep_pending_proposal=chat_request.keep_pending_proposal,
+ user_row_already_persisted=turn_started,
)
terminal_frame_emitted = True
capture_code_mode_opt_out_after_persist()
@@ -2018,6 +2167,8 @@ async def _new_copilot_chat_post(
original_workflow=original_workflow,
chat_request=chat_request,
agent_result=agent_result,
+ turn_id=turn_id,
+ user_row_already_persisted=turn_started,
)
)
terminal_frame_emitted = True
@@ -2075,6 +2226,7 @@ async def _new_copilot_chat_post(
turn_id=turn_id,
keep_pending_proposal=chat_request.keep_pending_proposal,
prior_global_llm_context=global_llm_context,
+ user_row_already_persisted=turn_started,
)
)
terminal_frame_emitted = True
@@ -2473,6 +2625,123 @@ async def list_workflow_copilot_chats(
)
+async def _restore_canonical_after_interrupted_turn(
+ chat: WorkflowCopilotChat,
+ organization_id: str,
+ entry: CopilotPendingTurn,
+) -> None:
+ """Undo an interrupted turn's canonical write, stashing the displaced draft as the proposal.
+
+ Only fires while canonical still carries the fingerprint this turn recorded when it wrote.
+ Anyone writing after it — another chat, a manual edit, a later turn — forfeits the rollback.
+ """
+ if entry.pre_turn_workflow is None or entry.canonical_write_fingerprint is None:
+ return
+ current = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=chat.workflow_permanent_id,
+ organization_id=organization_id,
+ )
+ if current is None:
+ return
+ if workflow_content_fingerprint(current.model_dump(mode="json")) != entry.canonical_write_fingerprint:
+ return
+
+ if not (entry.keep_pending_proposal and entry.pre_turn_proposed_workflow is not None):
+ stashed_draft = current.model_dump(mode="json")
+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(
+ organization_id=organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ proposed_workflow=stashed_draft,
+ )
+ chat.proposed_workflow = stashed_draft
+
+ pre_turn_workflow = Workflow.model_validate(entry.pre_turn_workflow)
+ # The snapshot was taken with model_dump, which masks cdp_connect_headers, so restoring it
+ # verbatim would overwrite the live browser credentials with the mask string.
+ pre_turn_workflow.cdp_connect_headers = merge_masked_headers(
+ pre_turn_workflow.cdp_connect_headers, current.cdp_connect_headers
+ )
+ await _restore_workflow_definition(pre_turn_workflow, organization_id)
+
+
+async def _recover_interrupted_copilot_turn(
+ chat: WorkflowCopilotChat,
+ organization_id: str,
+ entry: CopilotPendingTurn,
+) -> None:
+ if await _assistant_row_exists_for_turn(chat, entry.turn_id):
+ await _clear_pending_turn(chat, entry.turn_id)
+ return
+
+ try:
+ await _restore_canonical_after_interrupted_turn(chat, organization_id, entry)
+ except Exception:
+ # Answering the turn here would strand canonical on the half-written draft
+ # while history claims it was recovered; leave the marker for a later read.
+ LOG.warning(
+ "Canonical restore failed while recovering an interrupted copilot turn",
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ turn_id=entry.turn_id,
+ exc_info=True,
+ )
+ return
+
+ turn_outcome = TurnOutcome(
+ response_kind=ResponseKind.RECOVER,
+ reason_code=INTERRUPTED_TERMINAL_REASON,
+ terminal_reason=INTERRUPTED_TERMINAL_REASON,
+ copilot_turn_id=entry.turn_id,
+ )
+ narrative_payload = _with_terminal_narrative_metadata(
+ _make_error_narrative_payload(entry.turn_id, None, INTERRUPTED_TERMINAL_MESSAGE),
+ cancelled=False,
+ proposal_disposition=_proposal_disposition(None),
+ terminal_envelope=interrupted_terminal_envelope().model_dump(mode="json"),
+ )
+ await _persist_turn_messages(
+ chat=chat,
+ turn_id=entry.turn_id,
+ user_message="",
+ audio_artifact_id=None,
+ user_row_already_persisted=True,
+ assistant_content=INTERRUPTED_TERMINAL_MESSAGE,
+ global_llm_context=None,
+ turn_outcome=turn_outcome,
+ narrative_payload=narrative_payload,
+ )
+ LOG.info(
+ "Recovered an interrupted copilot turn on read",
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ turn_id=entry.turn_id,
+ )
+
+
+async def _reconcile_interrupted_copilot_turns(chat: WorkflowCopilotChat, organization_id: str) -> None:
+ if not chat.pending_turns:
+ return
+ abandoned_before = datetime.now(timezone.utc) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS)
+ for entry in sorted(chat.pending_turns.values(), key=lambda item: _as_utc(item.started_at)):
+ if _as_utc(entry.started_at) > abandoned_before:
+ continue
+ claimed = await app.DATABASE.workflow_params.claim_pending_copilot_turn(
+ organization_id=organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ turn_id=entry.turn_id,
+ claim_before=abandoned_before,
+ )
+ if not claimed:
+ continue
+ try:
+ await _recover_interrupted_copilot_turn(chat, organization_id, entry)
+ except Exception:
+ LOG.warning(
+ "Failed to recover an interrupted copilot turn",
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ turn_id=entry.turn_id,
+ exc_info=True,
+ )
+
+
@base_router.get("/workflow/copilot/chat-history", include_in_schema=False)
async def workflow_copilot_chat_history(
workflow_permanent_id: str | None = None,
@@ -2495,6 +2764,7 @@ async def workflow_copilot_chat_history(
detail="workflow_permanent_id or workflow_copilot_chat_id is required",
)
if chat:
+ await _reconcile_interrupted_copilot_turns(chat, organization.organization_id)
chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(
chat.workflow_copilot_chat_id
)
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index b76aaf040..ebb50515a 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -3,13 +3,34 @@ from datetime import datetime
from enum import StrEnum
from typing import Any, Literal
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
from skyvern.forge.sdk.copilot.context import ProposalDisposition, ResponseType, TurnNarrativePayload
from skyvern.forge.sdk.copilot.run_outcome import RunOutcomeReasonCode, RunOutcomeVerdict
from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
+class CopilotPendingTurn(BaseModel):
+ """Durable write-ahead marker for one in-flight copilot turn.
+
+ Written in the same transaction as the turn's user message so a hard kill
+ leaves enough state to reconcile the turn and roll canonical back.
+ """
+
+ model_config = ConfigDict(extra="ignore")
+
+ turn_id: str
+ started_at: datetime
+ pre_turn_workflow: dict[str, Any] | None = None
+ pre_turn_proposed_workflow: dict[str, Any] | None = None
+ keep_pending_proposal: bool = False
+ user_message_id: str | None = None
+ recovering_at: datetime | None = None
+ # Fingerprint of the canonical workflow as this turn last left it. None means the turn
+ # never wrote canonical, so it owns no write to roll back.
+ canonical_write_fingerprint: str | None = None
+
+
class WorkflowCopilotChat(BaseModel):
model_config = ConfigDict(from_attributes=True)
@@ -18,6 +39,15 @@ class WorkflowCopilotChat(BaseModel):
workflow_permanent_id: str = Field(..., description="Workflow permanent ID for the chat")
proposed_workflow: dict | None = Field(None, description="Latest workflow proposed by the copilot")
auto_accept: bool | None = Field(False, description="Whether copilot auto-accepts workflow updates")
+ pending_turns: dict[str, CopilotPendingTurn] = Field(
+ default_factory=dict, description="In-flight turns keyed by turn id"
+ )
+
+ @field_validator("pending_turns", mode="before")
+ @classmethod
+ def _default_pending_turns(cls, value: dict[str, Any] | None) -> dict[str, Any]:
+ return value or {}
+
created_at: datetime = Field(..., description="When the chat was created")
modified_at: datetime = Field(..., description="When the chat was last modified")
diff --git a/tests/unit/copilot/test_turn_origin_seam.py b/tests/unit/copilot/test_turn_origin_seam.py
index 290916342..dd85d3d5e 100644
--- a/tests/unit/copilot/test_turn_origin_seam.py
+++ b/tests/unit/copilot/test_turn_origin_seam.py
@@ -112,7 +112,10 @@ async def test_synthetic_session_unregisters_when_context_body_raises(
async with mcp_browser_context(ctx):
raise RuntimeError("boom")
- unregister_mock.assert_called_once_with(make_self_heal_session_id("wr_777"))
+ unregister_mock.assert_called_once_with(
+ make_self_heal_session_id("wr_777"),
+ organization_id=ctx.organization_id,
+ )
@pytest.mark.asyncio
@@ -155,8 +158,9 @@ async def test_self_heal_injected_state_assigns_synthetic_session_and_mcp_regist
with capture_logs() as logs:
async with mcp_browser_context(ctx):
assert register_mock.call_args.args[0] == expected_session_id
+ assert register_mock.call_args.kwargs == {"organization_id": ctx.organization_id}
- unregister_mock.assert_called_once_with(expected_session_id)
+ unregister_mock.assert_called_once_with(expected_session_id, organization_id=ctx.organization_id)
manager.get_browser_state.assert_not_awaited()
registered = [entry for entry in logs if entry["event"] == "registered self-heal browser session"]
unregistered = [entry for entry in logs if entry["event"] == "unregistered self-heal browser session"]
diff --git a/tests/unit/copilot_route_test_support.py b/tests/unit/copilot_route_test_support.py
index 86cc01914..20ea15b8e 100644
--- a/tests/unit/copilot_route_test_support.py
+++ b/tests/unit/copilot_route_test_support.py
@@ -56,6 +56,10 @@ def setup_new_copilot_mocks(
agent_result.narrative_summary = None
if not hasattr(agent_result, "narrative_payload"):
agent_result.narrative_payload = None
+ if not hasattr(original_workflow, "modified_at"):
+ original_workflow.modified_at = datetime(2026, 4, 14, tzinfo=timezone.utc)
+ if not hasattr(original_workflow, "model_dump"):
+ original_workflow.model_dump = MagicMock(return_value={"workflow_id": original_workflow.workflow_id})
async def fake_llm_handler(*args: object, **kwargs: object) -> None:
del args, kwargs
@@ -84,6 +88,11 @@ def setup_new_copilot_mocks(
create_workflow_copilot_chat_message=AsyncMock(
return_value=SimpleNamespace(created_at=datetime(2026, 4, 14, tzinfo=timezone.utc))
),
+ start_copilot_turn=AsyncMock(
+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 14, tzinfo=timezone.utc))
+ ),
+ claim_pending_copilot_turn=AsyncMock(return_value=True),
+ clear_pending_copilot_turn=AsyncMock(),
)
app.DATABASE.workflow_params = workflow_params
app.DATABASE.workflows = SimpleNamespace(
diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.py
index a2d115c9e..79a395517 100644
--- a/tests/unit/test_copilot_session_injection.py
+++ b/tests/unit/test_copilot_session_injection.py
@@ -191,13 +191,14 @@ class TestMcpBrowserContextBridge:
# that resolve_browser(session_id=...) relies on.
args = register_mock.call_args.args
assert args[0] == ctx.browser_session_id
+ assert register_mock.call_args.kwargs == {"organization_id": ctx.organization_id}
registered_state = args[1]
assert isinstance(registered_state, SessionState)
assert registered_state.context.session_id == ctx.browser_session_id
assert register_mock.call_count == 1
assert unregister_mock.call_count == 1
- unregister_mock.assert_called_with(ctx.browser_session_id)
+ unregister_mock.assert_called_with(ctx.browser_session_id, organization_id=ctx.organization_id)
# Override installed then reset.
assert [c[0] for c in override_calls] == ["set", "reset"]
diff --git a/tests/unit/test_mcp_http_auth.py b/tests/unit/test_mcp_http_auth.py
index d1d789720..fd2f9ee49 100644
--- a/tests/unit/test_mcp_http_auth.py
+++ b/tests/unit/test_mcp_http_auth.py
@@ -10,14 +10,17 @@ from unittest.mock import AsyncMock, Mock
import httpx
import pytest
from fastapi import HTTPException
+from fastmcp import FastMCP
from starlette.applications import Starlette
from starlette.middleware import Middleware
from starlette.requests import Request
from starlette.responses import JSONResponse
from starlette.routing import Route
+from starlette.testclient import TestClient
from skyvern.cli.core import client as client_mod
-from skyvern.cli.core import mcp_http_auth
+from skyvern.cli.core import mcp_http_auth, session_manager
+from skyvern.cli.core.result import BrowserContext
_TEST_BASE_URL = "http://testserver"
@@ -25,6 +28,12 @@ _TEST_BASE_URL = "http://testserver"
@pytest.fixture(autouse=True)
def _reset_auth_context() -> None:
client_mod._api_key_override.set(None)
+ session_manager._current_session.set(None)
+ session_manager._global_session = None
+ session_manager._copilot_sessions.clear()
+ session_manager._organization_sessions.clear()
+ session_manager._current_organization_id.set(None)
+ session_manager.set_stateless_http_mode(False)
mcp_http_auth._auth_db = None
mcp_http_auth._api_key_validation_cache.clear()
mcp_http_auth._API_KEY_CACHE_TTL_SECONDS = 30.0
@@ -264,6 +273,152 @@ async def test_mcp_http_auth_sets_request_scoped_api_key(monkeypatch: pytest.Mon
assert client_mod.get_active_api_key() != "sk_live_abc"
+async def test_stateful_http_session_is_scoped_to_authenticated_organization_with_exact_session_id(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ session_id = "pbs_owned_session"
+ first_browser = SimpleNamespace(owner="first")
+ denied_connection = AsyncMock(side_effect=PermissionError("session is not owned by this organization"))
+ monkeypatch.setattr(
+ session_manager,
+ "get_skyvern",
+ lambda: SimpleNamespace(connect_to_cloud_browser_session=denied_connection),
+ )
+
+ async def session_endpoint(request: Request) -> JSONResponse:
+ payload = await request.json()
+ if payload["action"] == "seed":
+ state = session_manager.SessionState(
+ browser=first_browser,
+ context=BrowserContext(mode="cloud_session", session_id=session_id),
+ api_key_hash=session_manager.active_api_key_hash(),
+ )
+ session_manager.set_current_session(state)
+ session_manager.register_copilot_session(session_id, state, organization_id="org_first")
+ return JSONResponse({"owner": first_browser.owner})
+
+ try:
+ browser, _ = await session_manager.resolve_browser(session_id=payload["session_id"])
+ except PermissionError:
+ return JSONResponse({"error": "forbidden"}, status_code=403)
+ return JSONResponse({"owner": browser.owner})
+
+ app = Starlette(
+ routes=[Route("/mcp", endpoint=session_endpoint, methods=["POST"])],
+ middleware=[Middleware(mcp_http_auth.MCPAPIKeyMiddleware)],
+ )
+ shared_api_key = "sk_shared_test_key"
+ validate_oauth_token = AsyncMock(
+ side_effect=[
+ SimpleNamespace(validation=_build_validation("org_first"), api_key=shared_api_key),
+ SimpleNamespace(validation=_build_validation("org_second"), api_key=shared_api_key),
+ ]
+ )
+ monkeypatch.setattr(mcp_http_auth, "validate_mcp_oauth_token", validate_oauth_token)
+
+ first_response = await _request(
+ app,
+ "POST",
+ "/mcp",
+ headers={"authorization": f"Bearer {_jwtish_token(payload={'sub': 'first'})}"},
+ json={"action": "seed"},
+ )
+ second_response = await _request(
+ app,
+ "POST",
+ "/mcp",
+ headers={"authorization": f"Bearer {_jwtish_token(payload={'sub': 'second'})}"},
+ json={"action": "resolve", "session_id": session_id},
+ )
+
+ assert first_response.status_code == 200
+ assert second_response.status_code == 403
+ assert denied_connection.await_count == 1
+ assert session_manager._current_session.get() is None
+
+
+def test_stateful_fastmcp_transport_rejects_exact_session_id_from_another_organization(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ server = FastMCP("session-owner-test")
+
+ @server.tool
+ async def session_owner() -> str:
+ return session_manager._current_organization_id.get() or "missing"
+
+ validations = {
+ "sk_org_first": _build_validation("org_first"),
+ "sk_org_first_rotated": _build_validation("org_first"),
+ "sk_org_second": _build_validation("org_second"),
+ }
+
+ async def validate_api_key(api_key: str) -> mcp_http_auth.MCPAPIKeyValidation:
+ return validations[api_key]
+
+ monkeypatch.setattr(mcp_http_auth, "validate_mcp_api_key", validate_api_key)
+ app = server.http_app(
+ path="/",
+ middleware=[Middleware(mcp_http_auth.MCPAPIKeyMiddleware)],
+ stateless_http=False,
+ json_response=True,
+ )
+ initialize_request = {
+ "jsonrpc": "2.0",
+ "id": 1,
+ "method": "initialize",
+ "params": {
+ "protocolVersion": "2025-06-18",
+ "capabilities": {},
+ "clientInfo": {"name": "test", "version": "1"},
+ },
+ }
+ tool_request = {
+ "jsonrpc": "2.0",
+ "id": 2,
+ "method": "tools/call",
+ "params": {"name": "session_owner", "arguments": {}},
+ }
+ headers = {
+ "accept": "application/json, text/event-stream",
+ "content-type": "application/json",
+ }
+
+ with TestClient(app) as client:
+ initialize_response = client.post(
+ "/",
+ headers={**headers, "x-api-key": "sk_org_first"},
+ json=initialize_request,
+ )
+ session_id = initialize_response.headers["mcp-session-id"]
+ rotated_credential_response = client.post(
+ "/",
+ headers={
+ **headers,
+ "x-api-key": "sk_org_first_rotated",
+ "mcp-session-id": session_id,
+ },
+ json=tool_request,
+ )
+ hijack_response = client.post(
+ "/",
+ headers={
+ **headers,
+ "x-api-key": "sk_org_second",
+ "mcp-session-id": session_id,
+ },
+ json=tool_request,
+ )
+
+ assert initialize_response.status_code == 200
+ assert rotated_credential_response.status_code == 404
+ assert rotated_credential_response.json()["error"]["message"] == "Session not found"
+ assert hijack_response.status_code == 404
+ assert hijack_response.json()["error"]["message"] == "Session not found"
+ assert session_manager._current_session.get() is None
+ assert session_manager._current_organization_id.get() is None
+
+
@pytest.mark.asyncio
async def test_validate_mcp_api_key_uses_ttl_cache(monkeypatch: pytest.MonkeyPatch) -> None:
calls = 0
diff --git a/tests/unit/test_mcp_session_lifecycle.py b/tests/unit/test_mcp_session_lifecycle.py
index 830cbb04c..c3b9f6141 100644
--- a/tests/unit/test_mcp_session_lifecycle.py
+++ b/tests/unit/test_mcp_session_lifecycle.py
@@ -31,8 +31,12 @@ def _reset_singletons() -> None:
client_mod._api_key_clients.clear()
session_manager._current_session.set(None)
+ session_manager._current_organization_id.set(None)
session_manager._global_session = None
+ session_manager._organization_sessions.clear()
session_manager._copilot_sessions.clear()
+ session_manager._session_ref_maps.clear()
+ session_manager._session_ref_generations.clear()
session_manager.set_stateless_http_mode(False)
@@ -375,24 +379,34 @@ async def test_resolve_browser_classifies_explicit_cloud_session_localhost_reach
@pytest.mark.asyncio
-async def test_resolve_browser_does_not_reuse_registered_copilot_session_for_different_api_key_override(
+ ("organization_id", "api_key"),
+ [(None, "sk_copilot_org"), ("org_copilot", "sk_other_org")],
+)
+async def test_resolve_browser_requires_matching_organization_and_credential_for_registered_copilot_session(
monkeypatch: pytest.MonkeyPatch,
+ organization_id: str | None,
+ api_key: str,
) -> None:
registered_state = session_manager.SessionState(
browser=MagicMock(),
context=BrowserContext(mode="cloud_session", session_id="pbs_copilot"),
api_key_hash=session_manager._api_key_hash("sk_copilot_org"),
)
- session_manager.register_copilot_session("pbs_copilot", registered_state)
+ session_manager.register_copilot_session("pbs_copilot", registered_state, organization_id="org_copilot")
fallback_browser = MagicMock()
fake_skyvern = MagicMock()
fake_skyvern.connect_to_cloud_browser_session = AsyncMock(return_value=fallback_browser)
monkeypatch.setattr(session_manager, "get_skyvern", lambda: fake_skyvern)
- token = client_mod.set_api_key_override("sk_other_org")
+ token = client_mod.set_api_key_override(api_key)
try:
- browser, ctx = await session_manager.resolve_browser(session_id="pbs_copilot")
+ if organization_id:
+ with session_manager.request_session_scope(organization_id):
+ browser, ctx = await session_manager.resolve_browser(session_id="pbs_copilot")
+ else:
+ browser, ctx = await session_manager.resolve_browser(session_id="pbs_copilot")
finally:
client_mod.reset_api_key_override(token)
@@ -828,6 +842,78 @@ async def test_close_current_session_still_closes_browser_when_api_fails(monkeyp
assert session_manager.get_current_session().browser is None
+async def test_close_all_sessions_closes_each_organization_cache(monkeypatch: pytest.MonkeyPatch) -> None:
+ first_browser = MagicMock()
+ first_browser.close = AsyncMock()
+ second_error = RuntimeError("second organization cleanup failed")
+ second_browser = MagicMock()
+ second_browser.close = AsyncMock(side_effect=second_error)
+ third_error = RuntimeError("third organization cleanup failed")
+ third_browser = MagicMock()
+ third_browser.close = AsyncMock(side_effect=third_error)
+
+ with session_manager.request_session_scope("org_first"):
+ session_manager.set_current_session(
+ session_manager.SessionState(
+ browser=first_browser,
+ context=BrowserContext(mode="local"),
+ )
+ )
+ with session_manager.request_session_scope("org_second"):
+ session_manager.set_current_session(
+ session_manager.SessionState(
+ browser=second_browser,
+ context=BrowserContext(mode="local"),
+ )
+ )
+ with session_manager.request_session_scope("org_third"):
+ session_manager.set_current_session(
+ session_manager.SessionState(browser=third_browser, context=BrowserContext(mode="local"))
+ )
+
+ warning = MagicMock()
+ monkeypatch.setattr(session_manager.LOG, "warning", warning)
+ with pytest.raises(RuntimeError, match=str(second_error)):
+ await session_manager.close_all_sessions()
+
+ first_browser.close.assert_awaited_once()
+ second_browser.close.assert_awaited_once()
+ third_browser.close.assert_awaited_once()
+ assert warning.call_args.kwargs["organization_id"] == "org_third"
+ assert warning.call_args.kwargs["exc_info"][1] is third_error
+ assert session_manager._organization_sessions == {}
+ assert session_manager._current_session.get() is None
+ assert session_manager._current_organization_id.get() is None
+
+
+def test_session_ref_lookup_is_scoped_to_authenticated_organization() -> None:
+ session_id = "pbs_shared_identifier"
+ page_key = (1, None, "https://example.com", None)
+ element = {"tag": "button"}
+
+ with session_manager.request_session_scope("org_first"):
+ session_manager.set_current_session(
+ session_manager.SessionState(
+ context=BrowserContext(mode="cloud_session", session_id=session_id),
+ )
+ )
+ assert session_manager.replace_session_ref_map(
+ {"e0": element},
+ session_id=session_id,
+ page_key=page_key,
+ )
+ assert session_manager.get_session_ref("e0", session_id=session_id, page_key=page_key) == element
+
+ with session_manager.request_session_scope("org_second"):
+ session_manager.set_current_session(
+ session_manager.SessionState(
+ context=BrowserContext(mode="cloud_session", session_id=session_id),
+ )
+ )
+ assert session_manager.get_session_ref("e0", session_id=session_id, page_key=page_key) is None
+
+
# ---------------------------------------------------------------------------
# Tests for stateless HTTP mode session creation
# ---------------------------------------------------------------------------
diff --git a/tests/unit/test_run_commands_cleanup.py b/tests/unit/test_run_commands_cleanup.py
index 1290a1e47..07ea77efc 100644
--- a/tests/unit/test_run_commands_cleanup.py
+++ b/tests/unit/test_run_commands_cleanup.py
@@ -24,18 +24,18 @@ def _reset_cleanup_state() -> None:
async def test_cleanup_mcp_resources_closes_auth_db(monkeypatch: pytest.MonkeyPatch) -> None:
order: list[str] = []
shutdown_action_log_worker = AsyncMock(side_effect=lambda: order.append("action_log"))
- close_current_session = AsyncMock(side_effect=lambda: order.append("session"))
+ close_all_sessions = AsyncMock(side_effect=lambda: order.append("session"))
close_skyvern = AsyncMock(side_effect=lambda: order.append("client"))
close_auth_db = AsyncMock(side_effect=lambda: order.append("auth"))
monkeypatch.setattr("skyvern.cli.core.action_log.shutdown_action_log_worker", shutdown_action_log_worker)
- monkeypatch.setattr("skyvern.cli.core.session_manager.close_current_session", close_current_session)
+ monkeypatch.setattr("skyvern.cli.core.session_manager.close_all_sessions", close_all_sessions)
monkeypatch.setattr("skyvern.cli.core.client.close_skyvern", close_skyvern)
monkeypatch.setattr("skyvern.cli.core.mcp_http_auth.close_auth_db", close_auth_db)
await run_commands._cleanup_mcp_resources()
- close_current_session.assert_awaited_once()
+ close_all_sessions.assert_awaited_once()
close_skyvern.assert_awaited_once()
close_auth_db.assert_awaited_once()
assert order == ["action_log", "session", "client", "auth"]
@@ -43,20 +43,20 @@ async def test_cleanup_mcp_resources_closes_auth_db(monkeypatch: pytest.MonkeyPa
@pytest.mark.asyncio
async def test_cleanup_mcp_resources_closes_auth_db_on_skyvern_close_error(monkeypatch: pytest.MonkeyPatch) -> None:
- close_current_session = AsyncMock()
+ close_all_sessions = AsyncMock()
close_auth_db = AsyncMock()
async def _failing_close_skyvern() -> None:
raise RuntimeError("close failed")
- monkeypatch.setattr("skyvern.cli.core.session_manager.close_current_session", close_current_session)
+ monkeypatch.setattr("skyvern.cli.core.session_manager.close_all_sessions", close_all_sessions)
monkeypatch.setattr("skyvern.cli.core.client.close_skyvern", _failing_close_skyvern)
monkeypatch.setattr("skyvern.cli.core.mcp_http_auth.close_auth_db", close_auth_db)
with pytest.raises(RuntimeError, match="close failed"):
await run_commands._cleanup_mcp_resources()
- close_current_session.assert_awaited_once()
+ close_all_sessions.assert_awaited_once()
close_auth_db.assert_awaited_once()
diff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.py
index 2cd573a04..21cc8256e 100644
--- a/tests/unit/test_workflow_copilot_cancel.py
+++ b/tests/unit/test_workflow_copilot_cancel.py
@@ -35,6 +35,8 @@ from skyvern.forge.sdk.copilot.agent import _build_exit_result
from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext, StructuredContext
from skyvern.forge.sdk.routes.workflow_copilot import (
COPILOT_CANCEL_TTL,
+ USER_CANCELLED_TERMINAL_REASON,
+ _assistant_row_exists_for_turn,
_copilot_cancel_key,
_persist_cancel_turn,
_persist_proposed_workflow_state,
@@ -45,6 +47,7 @@ from skyvern.forge.sdk.routes.workflow_copilot import (
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotCancelRequest,
WorkflowCopilotChatRequest,
+ WorkflowCopilotChatSender,
WorkflowCopilotStreamMessageType,
)
from tests.unit.copilot_route_test_support import install_fake_create, setup_new_copilot_mocks
@@ -260,9 +263,10 @@ async def _drive_cancel_route(
handler = captured["handler"]
assert callable(handler)
await handler(stream)
- # Cancel turn always emits exactly two chat rows: the user prompt and the
- # AI cancellation reply. Guard against a future regression that double-inserts.
- assert workflow_params.create_workflow_copilot_chat_message.await_count == 2
+ # The user prompt is written ahead of the agent by start_copilot_turn; the
+ # cancel path adds only the AI reply. Guard against a double-insert of either.
+ assert workflow_params.start_copilot_turn.await_count == 1
+ assert workflow_params.create_workflow_copilot_chat_message.await_count == 1
return restore_mock, workflow_params, sent_payloads
@@ -291,13 +295,13 @@ async def test_route_cancel_branch_persists_user_and_cancelled_messages(
restore_mock.assert_awaited_once()
- # Two chat-message inserts: user msg + Cancelled by user. AI msg.
insert_calls = workflow_params.create_workflow_copilot_chat_message.await_args_list
senders = [c.kwargs.get("sender") for c in insert_calls]
contents = [c.kwargs.get("content") for c in insert_calls]
- assert senders.count("user") == 1
+ assert senders.count("user") == 0
assert senders.count("ai") == 1
- assert "please update" in contents
+ assert workflow_params.start_copilot_turn.await_args.kwargs["user_message"] == "please update"
+
assert "Cancelled by user." in contents
# No proposed_workflow update when the cancelled result has no WIP.
@@ -391,7 +395,7 @@ async def test_route_cancel_wip_persists_proposal_and_response_frame(
insert_calls = workflow_params.create_workflow_copilot_chat_message.await_args_list
contents = [c.kwargs.get("content") for c in insert_calls]
- assert "please update" in contents
+ assert workflow_params.start_copilot_turn.await_args.kwargs["user_message"] == "please update"
assert user_response in contents
response_frames = [
@@ -942,6 +946,8 @@ async def test_operational_cancel_does_not_persist_cancelled_message(
title="Original",
description=None,
workflow_definition=None,
+ modified_at=datetime(2026, 4, 27, tzinfo=timezone.utc),
+ model_dump=MagicMock(return_value={"workflow_id": "wf-canonical"}),
)
# Agent raises CancelledError synchronously (simulates operational cancel
# propagating into the await before user_cancel_observed gets set).
@@ -970,6 +976,10 @@ async def test_operational_cancel_does_not_persist_cancelled_message(
create_workflow_copilot_chat_message=AsyncMock(
return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))
),
+ start_copilot_turn=AsyncMock(
+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))
+ ),
+ clear_pending_copilot_turn=AsyncMock(),
)
app.DATABASE.workflow_params = workflow_params
app.DATABASE.workflows = SimpleNamespace(
@@ -1004,3 +1014,56 @@ async def test_operational_cancel_does_not_persist_cancelled_message(
contents = [c.kwargs.get("content") for c in insert_calls]
# No "Cancelled by user." row was written.
assert "Cancelled by user." not in contents
+
+
+async def test_pre_agent_cancel_row_is_visible_to_the_turn_idempotence_check() -> None:
+ """Without a turn-stamped outcome, reconcile would append an interrupted row beside the cancel."""
+ chat = SimpleNamespace(
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ proposed_workflow=None,
+ auto_accept=False,
+ )
+ persisted: list[SimpleNamespace] = []
+
+ async def capture_message(**kwargs: Any) -> SimpleNamespace:
+ row = SimpleNamespace(
+ sender=kwargs["sender"],
+ content=kwargs["content"],
+ turn_outcome=kwargs.get("turn_outcome"),
+ created_at=datetime(2026, 4, 27, tzinfo=timezone.utc),
+ )
+ persisted.append(row)
+ return row
+
+ workflow_params = SimpleNamespace(
+ update_workflow_copilot_chat=AsyncMock(),
+ create_workflow_copilot_chat_message=capture_message,
+ clear_pending_copilot_turn=AsyncMock(),
+ get_workflow_copilot_chat_messages=AsyncMock(return_value=persisted),
+ )
+ app.DATABASE.workflow_params = workflow_params
+
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+
+ await _persist_cancel_turn(
+ stream=stream,
+ chat=chat,
+ organization_id="org-1",
+ original_workflow=None,
+ user_message="keep going",
+ agent_result=None,
+ turn_id="turn-a",
+ user_row_already_persisted=True,
+ )
+
+ assistant_rows = [row for row in persisted if row.sender == WorkflowCopilotChatSender.AI]
+ assert len(assistant_rows) == 1
+ outcome = assistant_rows[0].turn_outcome
+ assert outcome is not None
+ assert outcome.copilot_turn_id == "turn-a"
+ assert outcome.terminal_reason == USER_CANCELLED_TERMINAL_REASON
+ assert outcome.terminal_reason != "interrupted"
+ assert await _assistant_row_exists_for_turn(chat, "turn-a") is True
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 6cb55df91..4c5e04b7c 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -15,7 +15,8 @@ real database -- all DB / LLM / agent surfaces are patched.
from __future__ import annotations
-from datetime import datetime, timezone
+import contextlib
+from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
@@ -30,23 +31,36 @@ from skyvern.forge.agent_functions import AgentFunction
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot import agent as agent_module
-from skyvern.forge.sdk.copilot.context import AgentResult
-from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope
+from skyvern.forge.sdk.copilot.canonical_ownership import workflow_content_fingerprint
+from skyvern.forge.sdk.copilot.context import AgentResult, TurnNarrativePayload
+from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
+from skyvern.forge.sdk.copilot.terminal_envelope import (
+ INTERRUPTED_TERMINAL_MESSAGE,
+ INTERRUPTED_TERMINAL_REASON,
+ assemble_terminal_envelope,
+)
from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route
from skyvern.forge.sdk.routes.workflow_copilot import (
COPILOT_V2_FLAG_KEY,
+ RECONCILE_ABANDON_AFTER_SECONDS,
+ _persist_turn_messages,
_validate_copilot_audio_artifact_id,
convert_to_history_messages,
workflow_copilot_chat_audio,
+ workflow_copilot_chat_history,
workflow_copilot_chat_post,
)
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
from skyvern.forge.sdk.schemas.workflow_copilot import (
+ CopilotPendingTurn,
+ WorkflowCopilotChat,
WorkflowCopilotChatMessage,
WorkflowCopilotChatRequest,
WorkflowCopilotChatSender,
WorkflowCopilotStreamErrorUpdate,
WorkflowCopilotStreamResponseUpdate,
)
+from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition
from tests.unit.copilot_route_test_support import install_fake_create, setup_new_copilot_mocks
@@ -1305,8 +1319,8 @@ async def test_flag_on_route_error_after_chat_persists_recoverable_reply(
call.kwargs.get("content")
for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list
]
- assert "Please update it" in contents
- assistant_contents = [content for content in contents if isinstance(content, str) and content != "Please update it"]
+ assert app.DATABASE.workflow_params.start_copilot_turn.await_args.kwargs["user_message"] == "Please update it"
+ assistant_contents = [content for content in contents if isinstance(content, str)]
assert len(assistant_contents) == 1
assert expected_summary in assistant_contents[0]
assert "The workflow was not modified" in assistant_contents[0]
@@ -2774,3 +2788,708 @@ async def test_persist_state_auto_applicable_without_staged_commit_still_protect
)
app.DATABASE.workflow_params.update_workflow_copilot_chat.assert_not_awaited()
+
+
+_NOW = datetime(2026, 7, 29, 12, 0, tzinfo=timezone.utc)
+
+
+def _make_copilot_workflow(title: str, modified_at: datetime) -> Workflow:
+ return Workflow(
+ workflow_id="wf-1",
+ organization_id="org-1",
+ title=title,
+ workflow_permanent_id="wpid-1",
+ version=1,
+ is_saved_task=False,
+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),
+ created_at=_NOW - timedelta(days=1),
+ modified_at=modified_at,
+ )
+
+
+def _fingerprint_of(workflow: Workflow) -> str:
+ return workflow_content_fingerprint(workflow.model_dump(mode="json"))
+
+
+def _make_pending_turn(turn_id: str, age_seconds: float, **overrides: Any) -> CopilotPendingTurn:
+ return CopilotPendingTurn(
+ turn_id=turn_id,
+ started_at=datetime.now(timezone.utc) - timedelta(seconds=age_seconds),
+ **overrides,
+ )
+
+
+def _make_persisted_chat(
+ pending: list[CopilotPendingTurn], proposed_workflow: dict | None = None
+) -> WorkflowCopilotChat:
+ return WorkflowCopilotChat(
+ workflow_copilot_chat_id="chat-1",
+ organization_id="org-1",
+ workflow_permanent_id="wpid-1",
+ proposed_workflow=proposed_workflow,
+ auto_accept=False,
+ pending_turns={entry.turn_id: entry for entry in pending},
+ created_at=_NOW,
+ modified_at=_NOW,
+ )
+
+
+class _FakeCopilotChatStore:
+ """In-memory double of the chat/message repo surface reconcile-on-read touches."""
+
+ def __init__(self, chat: WorkflowCopilotChat) -> None:
+ self.chat = chat
+ self.messages: list[WorkflowCopilotChatMessage] = []
+ self.claim_calls: list[str] = []
+
+ def add_message(
+ self,
+ sender: WorkflowCopilotChatSender,
+ content: str,
+ turn_outcome: TurnOutcome | None = None,
+ ) -> WorkflowCopilotChatMessage:
+ message = WorkflowCopilotChatMessage(
+ workflow_copilot_chat_message_id=f"wccm-{len(self.messages)}",
+ workflow_copilot_chat_id=self.chat.workflow_copilot_chat_id,
+ sender=sender,
+ content=content,
+ turn_outcome=turn_outcome,
+ created_at=_NOW,
+ modified_at=_NOW,
+ )
+ self.messages.append(message)
+ return message
+
+ @property
+ def assistant_messages(self) -> list[WorkflowCopilotChatMessage]:
+ return [m for m in self.messages if m.sender == WorkflowCopilotChatSender.AI]
+
+ @property
+ def user_messages(self) -> list[WorkflowCopilotChatMessage]:
+ return [m for m in self.messages if m.sender == WorkflowCopilotChatSender.USER]
+
+ async def get_workflow_copilot_chat_by_id(
+ self, organization_id: str, workflow_copilot_chat_id: str
+ ) -> WorkflowCopilotChat:
+ return self.chat
+
+ async def get_workflow_copilot_chat_messages(
+ self, workflow_copilot_chat_id: str
+ ) -> list[WorkflowCopilotChatMessage]:
+ return list(self.messages)
+
+ async def create_workflow_copilot_chat_message(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ sender: WorkflowCopilotChatSender,
+ content: str,
+ audio_artifact_id: str | None = None,
+ global_llm_context: str | None = None,
+ turn_outcome: TurnOutcome | None = None,
+ narrative_payload: Any = None,
+ ) -> WorkflowCopilotChatMessage:
+ return self.add_message(sender, content, turn_outcome)
+
+ async def replace_workflow_copilot_chat_message(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_message_id: str,
+ content: str,
+ global_llm_context: str | None,
+ turn_outcome: TurnOutcome | None,
+ narrative_payload: TurnNarrativePayload | None = None,
+ ) -> WorkflowCopilotChatMessage | None:
+ for index, message in enumerate(self.messages):
+ if message.workflow_copilot_chat_message_id == workflow_copilot_chat_message_id:
+ replaced = message.model_copy(update={"content": content, "turn_outcome": turn_outcome})
+ self.messages[index] = replaced
+ return replaced
+ return None
+
+ async def record_pending_copilot_turn_canonical_write(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ fingerprint: str,
+ ) -> None:
+ entry = self.chat.pending_turns.get(turn_id)
+ if entry is not None:
+ entry.canonical_write_fingerprint = fingerprint
+
+ async def claim_pending_copilot_turn(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ claim_before: datetime,
+ ) -> bool:
+ self.claim_calls.append(turn_id)
+ entry = self.chat.pending_turns.get(turn_id)
+ if entry is None:
+ return False
+ if entry.recovering_at is not None and entry.recovering_at > claim_before:
+ return False
+ entry.recovering_at = datetime.now(timezone.utc)
+ return True
+
+ async def clear_pending_copilot_turn(
+ self, organization_id: str, workflow_copilot_chat_id: str, turn_id: str
+ ) -> None:
+ self.chat.pending_turns.pop(turn_id, None)
+
+ async def update_workflow_copilot_chat(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ proposed_workflow: dict | None = None,
+ auto_accept: bool | None = None,
+ ) -> WorkflowCopilotChat:
+ self.chat.proposed_workflow = proposed_workflow
+ return self.chat
+
+
+def _install_reconcile_store(
+ monkeypatch: pytest.MonkeyPatch,
+ chat: WorkflowCopilotChat,
+ canonical: Workflow | None = None,
+) -> tuple[_FakeCopilotChatStore, AsyncMock]:
+ store = _FakeCopilotChatStore(chat)
+ app.DATABASE.workflow_params = store
+ app.DATABASE.workflows = SimpleNamespace(get_workflow_by_permanent_id=AsyncMock(return_value=canonical))
+ restore_mock = AsyncMock()
+ monkeypatch.setattr(workflow_copilot_route, "_restore_workflow_definition", restore_mock)
+ return store, restore_mock
+
+
+async def _load_history(chat_id: str = "chat-1") -> Any:
+ return await workflow_copilot_chat_history(
+ workflow_copilot_chat_id=chat_id,
+ organization=SimpleNamespace(organization_id="org-1"),
+ )
+
+
+async def test_chat_history_marks_abandoned_turn_interrupted_not_cancelled(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A turn whose process died leaves the user row plus an honest interrupted outcome."""
+ chat = _make_persisted_chat([_make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS + 60)])
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ response = await _load_history()
+
+ assert [m.content for m in store.user_messages] == ["build me a scraper"]
+ assert len(store.assistant_messages) == 1
+ outcome = store.assistant_messages[0].turn_outcome
+ assert outcome is not None
+ assert outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON
+ assert outcome.terminal_reason != "cancel"
+ assert outcome.response_kind == ResponseKind.RECOVER
+ assert outcome.copilot_turn_id == "turn-a"
+ assert chat.pending_turns == {}
+ assert response.chat_history[-1].turn_outcome is not None
+ restore_mock.assert_not_awaited()
+
+
+async def test_chat_history_leaves_a_young_pending_turn_alone(monkeypatch: pytest.MonkeyPatch) -> None:
+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 30)])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ await _load_history()
+
+ assert store.assistant_messages == []
+ assert "turn-a" in chat.pending_turns
+
+
+async def test_chat_history_shows_the_user_message_while_the_turn_is_still_running(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Reloading mid-turn — a tab refresh — shows the user's message without inventing an outcome."""
+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 30)])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ response = await _load_history()
+
+ assert [m.content for m in response.chat_history] == ["build me a scraper"]
+ assert all(m.turn_outcome is None for m in response.chat_history)
+ assert "turn-a" in chat.pending_turns
+
+
+async def test_chat_history_leaves_a_turn_inside_its_enforcement_budget_alone(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Past the 900s agent budget but still live in wall clock thanks to a credential pause."""
+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 1000)])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ await _load_history()
+
+ assert store.assistant_messages == []
+ assert "turn-a" in chat.pending_turns
+
+
+async def test_reconcile_restores_real_cdp_headers_not_the_masked_snapshot(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The pre-turn snapshot is a JSON dump, which masks cdp_connect_headers; restoring it verbatim would destroy them."""
+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))
+ pre_turn.cdp_connect_headers = {"x-api-key": "real-secret"}
+ mid_turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1500))
+ mid_turn_draft.cdp_connect_headers = {"x-api-key": "real-secret"}
+ chat = _make_persisted_chat(
+ [
+ _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 60,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ canonical_write_fingerprint=_fingerprint_of(mid_turn_draft),
+ )
+ ]
+ )
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=mid_turn_draft)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ await _load_history()
+
+ restore_mock.assert_awaited_once()
+ assert restore_mock.await_args.args[0].cdp_connect_headers == {"x-api-key": "real-secret"}
+
+
+async def test_reconcile_does_not_stash_a_canonical_the_dying_turn_already_restored(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A finalizer that restored canonical just before the kill leaves no displaced draft to stash."""
+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))
+ turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1600))
+ already_restored = _make_copilot_workflow("Before the turn", datetime.now(timezone.utc) - timedelta(seconds=1500))
+ chat = _make_persisted_chat(
+ [
+ _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 60,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ canonical_write_fingerprint=_fingerprint_of(turn_draft),
+ )
+ ]
+ )
+ store, _ = _install_reconcile_store(monkeypatch, chat, canonical=already_restored)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ response = await _load_history()
+
+ assert response.proposed_workflow is None
+
+
+async def test_reconcile_restores_canonical_and_stashes_the_mid_turn_draft(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))
+ mid_turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1500))
+ chat = _make_persisted_chat(
+ [
+ _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 60,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ canonical_write_fingerprint=_fingerprint_of(mid_turn_draft),
+ )
+ ]
+ )
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=mid_turn_draft)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ response = await _load_history()
+
+ restore_mock.assert_awaited_once()
+ restored_workflow = restore_mock.await_args.args[0]
+ assert restored_workflow.title == "Before the turn"
+ assert restored_workflow.workflow_definition == pre_turn.workflow_definition
+ assert response.proposed_workflow is not None
+ assert response.proposed_workflow["title"] == "Half-built draft"
+
+
+async def test_reconcile_keeps_an_unowned_canonical_change_rather_than_undoing_it(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """An auto-accept commit records no ownership; undoing it would discard work the user opted to apply."""
+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))
+ committed = _make_copilot_workflow("Auto-accepted build", datetime.now(timezone.utc) - timedelta(seconds=1500))
+ chat = _make_persisted_chat(
+ [
+ _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 60,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ )
+ ]
+ )
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=committed)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ response = await _load_history()
+
+ restore_mock.assert_not_awaited()
+ assert response.proposed_workflow is None
+ assert len(store.assistant_messages) == 1
+
+
+async def test_reconcile_leaves_canonical_alone_when_the_turn_never_wrote(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ pre_turn = _make_copilot_workflow("Untouched", _NOW - timedelta(hours=2))
+ chat = _make_persisted_chat(
+ [
+ _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 60,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ )
+ ]
+ )
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=pre_turn)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ response = await _load_history()
+
+ restore_mock.assert_not_awaited()
+ assert response.proposed_workflow is None
+ assert len(store.assistant_messages) == 1
+
+
+async def test_reconcile_leaves_canonical_alone_when_someone_else_wrote_after_the_turn(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A hand edit landing after the turn's own write forfeits the turn's rollback claim."""
+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(days=2))
+ turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1600))
+ later_edit = _make_copilot_workflow("Edited by hand afterwards", datetime.now(timezone.utc))
+ chat = _make_persisted_chat(
+ [
+ _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS * 3,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ canonical_write_fingerprint=_fingerprint_of(turn_draft),
+ )
+ ]
+ )
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=later_edit)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ response = await _load_history()
+
+ restore_mock.assert_not_awaited()
+ assert response.proposed_workflow is None
+
+
+async def test_reconcile_keeps_an_earlier_proposal_when_keep_pending_was_requested(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))
+ mid_turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1500))
+ earlier_proposal = {"workflow_id": "wf-1", "title": "Earlier proposal"}
+ chat = _make_persisted_chat(
+ [
+ _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 60,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ canonical_write_fingerprint=_fingerprint_of(mid_turn_draft),
+ pre_turn_proposed_workflow=earlier_proposal,
+ keep_pending_proposal=True,
+ )
+ ],
+ proposed_workflow=earlier_proposal,
+ )
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=mid_turn_draft)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ response = await _load_history()
+
+ restore_mock.assert_awaited_once()
+ assert response.proposed_workflow == earlier_proposal
+
+
+async def test_reconcile_completes_after_a_crash_mid_recovery(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A recovery that died after claiming the turn is reclaimed and still yields one reply."""
+ entry = _make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS * 3)
+ entry.recovering_at = datetime.now(timezone.utc) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS * 2)
+ chat = _make_persisted_chat([entry])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ await _load_history()
+
+ assert len(store.assistant_messages) == 1
+ assert chat.pending_turns == {}
+
+
+async def test_reconcile_skips_a_turn_another_reader_is_already_recovering(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ entry = _make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS + 60)
+ entry.recovering_at = datetime.now(timezone.utc)
+ chat = _make_persisted_chat([entry])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ await _load_history()
+
+ assert store.assistant_messages == []
+ assert "turn-a" in chat.pending_turns
+
+
+async def test_reconcile_handles_two_overlapping_abandoned_turns(monkeypatch: pytest.MonkeyPatch) -> None:
+ chat = _make_persisted_chat(
+ [
+ _make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS + 120),
+ _make_pending_turn("turn-b", RECONCILE_ABANDON_AFTER_SECONDS + 60),
+ ]
+ )
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "first ask")
+ store.add_message(WorkflowCopilotChatSender.USER, "second ask")
+
+ await _load_history()
+
+ recovered_turn_ids = [m.turn_outcome.copilot_turn_id for m in store.assistant_messages if m.turn_outcome]
+ assert sorted(recovered_turn_ids) == ["turn-a", "turn-b"]
+ assert chat.pending_turns == {}
+
+
+async def test_reconcile_drops_the_marker_of_a_turn_that_already_replied(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A finalised turn whose marker clear failed must not be rolled back or answered twice."""
+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))
+ committed = _make_copilot_workflow("Accepted build", datetime.now(timezone.utc) - timedelta(seconds=1500))
+ chat = _make_persisted_chat(
+ [
+ _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 60,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ )
+ ]
+ )
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=committed)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+ store.add_message(
+ WorkflowCopilotChatSender.AI,
+ "Done.",
+ TurnOutcome(response_kind=ResponseKind.BUILD, copilot_turn_id="turn-a"),
+ )
+
+ response = await _load_history()
+
+ assert len(store.assistant_messages) == 1
+ assert chat.pending_turns == {}
+ restore_mock.assert_not_awaited()
+ assert response.proposed_workflow is None
+
+
+async def test_persist_turn_messages_is_idempotent_for_one_turn(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A second finalizer pass for the same turn adds no duplicate user or assistant row."""
+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 10)])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+
+ for _ in range(2):
+ await _persist_turn_messages(
+ chat=chat,
+ turn_id="turn-a",
+ user_message="build me a scraper",
+ audio_artifact_id=None,
+ user_row_already_persisted=True,
+ assistant_content="Here is your workflow.",
+ global_llm_context=None,
+ turn_outcome=TurnOutcome(response_kind=ResponseKind.BUILD),
+ narrative_payload=None,
+ )
+
+ assert len(store.user_messages) == 1
+ assert len(store.assistant_messages) == 1
+ assert store.assistant_messages[0].turn_outcome is not None
+ assert store.assistant_messages[0].turn_outcome.copilot_turn_id == "turn-a"
+
+
+async def test_a_finished_turn_replaces_the_interrupted_row_rather_than_dropping_its_reply(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Recovery can reach a slow-but-live turn first; when that turn finishes, its reply is the truth."""
+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 10)])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+ store.add_message(
+ WorkflowCopilotChatSender.AI,
+ INTERRUPTED_TERMINAL_MESSAGE,
+ TurnOutcome(
+ response_kind=ResponseKind.RECOVER,
+ reason_code=INTERRUPTED_TERMINAL_REASON,
+ terminal_reason=INTERRUPTED_TERMINAL_REASON,
+ copilot_turn_id="turn-a",
+ ),
+ )
+
+ await _persist_turn_messages(
+ chat=chat,
+ turn_id="turn-a",
+ user_message="build me a scraper",
+ audio_artifact_id=None,
+ user_row_already_persisted=True,
+ assistant_content="Here is your workflow.",
+ global_llm_context=None,
+ turn_outcome=TurnOutcome(response_kind=ResponseKind.BUILD),
+ narrative_payload=None,
+ )
+
+ assert len(store.assistant_messages) == 1
+ assert store.assistant_messages[0].content == "Here is your workflow."
+ outcome = store.assistant_messages[0].turn_outcome
+ assert outcome is not None
+ assert outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON
+
+
+def test_reconcile_threshold_outlasts_the_turn_enforcement_ceiling() -> None:
+ assert RECONCILE_ABANDON_AFTER_SECONDS > TOTAL_TIMEOUT_SECONDS
+
+
+async def test_marker_survives_a_finalizer_that_raises_with_no_assistant_row(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """No assistant row was written, so the turn stays recoverable instead of being orphaned."""
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+ captured = install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=None,
+ auto_accept=False,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ agent_result = SimpleNamespace(
+ user_response="Here is your workflow.",
+ updated_workflow=None,
+ global_llm_context=None,
+ workflow_yaml=None,
+ workflow_was_persisted=False,
+ clear_proposed_workflow=False,
+ unvalidated=False,
+ turn_outcome=None,
+ )
+ setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ monkeypatch.setattr(
+ workflow_copilot_route,
+ "_persist_turn_messages",
+ AsyncMock(side_effect=RuntimeError("database down")),
+ )
+
+ request = MagicMock()
+ request.headers = {"x-api-key": "sk-test-key"}
+
+ await workflow_copilot_chat_post(request, _make_chat_request(), SimpleNamespace(organization_id="org-1"))
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ stream.is_disconnected = AsyncMock(return_value=False)
+ handler = captured["handler"]
+ assert callable(handler)
+ with contextlib.suppress(RuntimeError):
+ await handler(stream)
+
+ assert app.DATABASE.workflow_params.start_copilot_turn.await_count == 1
+ app.DATABASE.workflow_params.clear_pending_copilot_turn.assert_not_awaited()
+
+
+async def test_reconcile_writes_no_outcome_when_the_canonical_restore_fails(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A transient restore failure must not persist a 'recovered' turn over a stranded canonical."""
+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))
+ mid_turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1500))
+ entry = _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 60,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ canonical_write_fingerprint=_fingerprint_of(mid_turn_draft),
+ )
+ chat = _make_persisted_chat([entry])
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=mid_turn_draft)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+ restore_mock.side_effect = RuntimeError("restore failed")
+
+ await _load_history()
+
+ assert store.assistant_messages == []
+ assert "turn-a" in chat.pending_turns
+
+ restore_mock.side_effect = None
+ entry.recovering_at = None
+
+ await _load_history()
+
+ assert len(store.assistant_messages) == 1
+ assert store.assistant_messages[0].turn_outcome is not None
+ assert store.assistant_messages[0].turn_outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON
+ assert chat.pending_turns == {}
+
+
+async def test_reconcile_does_not_roll_back_a_later_turns_canonical_write(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """An older abandoned turn must not attribute — and undo — a newer turn's canonical edit."""
+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))
+ later_write = _make_copilot_workflow(
+ "Written by the later turn", datetime.now(timezone.utc) - timedelta(seconds=1350)
+ )
+ older_draft = _make_copilot_workflow("Older turn draft", datetime.now(timezone.utc) - timedelta(seconds=1500))
+ older = _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 180,
+ pre_turn_workflow=pre_turn.model_dump(mode="json"),
+ canonical_write_fingerprint=_fingerprint_of(older_draft),
+ )
+ newer = _make_pending_turn("turn-b", RECONCILE_ABANDON_AFTER_SECONDS + 80)
+ chat = _make_persisted_chat([older, newer])
+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=later_write)
+ store.add_message(WorkflowCopilotChatSender.USER, "first ask")
+ store.add_message(WorkflowCopilotChatSender.USER, "second ask")
+
+ response = await _load_history()
+
+ restore_mock.assert_not_awaited()
+ recovered = sorted(m.turn_outcome.copilot_turn_id for m in store.assistant_messages if m.turn_outcome)
+ assert recovered == ["turn-a", "turn-b"]
+ assert chat.pending_turns == {}
+ assert response.proposed_workflow is None