Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Keep a Copilot turn in the chat when the server restarts mid-turn (SKY-13205) (#7911)
Andrew Neilsone3e1f933c013
diff --git a/alembic/versions/2026_07_31_1634-92ebe14532b9_add_pending_turns_to_workflow_copilot_chats.py b/alembic/versions/2026_07_31_1634-92ebe14532b9_add_pending_turns_to_workflow_copilot_chats.pynew file mode 100644index 000000000..29f1612c9--- /dev/null+++ b/alembic/versions/2026_07_31_1634-92ebe14532b9_add_pending_turns_to_workflow_copilot_chats.py@@ -0,0 +1,27 @@+"""add pending_turns to workflow_copilot_chats++Revision ID: 92ebe14532b9+Revises: 9d85c51a8468+Create Date: 2026-07-31T16:34:24.245309+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "92ebe14532b9"+down_revision: Union[str, None] = "9d85c51a8468"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ op.add_column("workflow_copilot_chats", sa.Column("pending_turns", sa.JSON(), nullable=True))+++def downgrade() -> None:+ op.drop_column("workflow_copilot_chats", "pending_turns")diff --git a/skyvern/cli/core/mcp_http_auth.py b/skyvern/cli/core/mcp_http_auth.pyindex a5e9ca4a2..da6486814 100644--- a/skyvern/cli/core/mcp_http_auth.py+++ b/skyvern/cli/core/mcp_http_auth.py@@ -15,6 +15,8 @@ from urllib.parse import urlsplit, urlunsplitimport structlogfrom fastapi import HTTPException+from mcp.server.auth.middleware.bearer_auth import AuthenticatedUser+from mcp.server.auth.provider import AccessTokenfrom starlette.requests import Requestfrom starlette.responses import JSONResponsefrom starlette.types import ASGIApp, Receive, Scope, Send@@ -28,6 +30,7 @@ from skyvern.forge.sdk.services.org_auth_service import resolve_org_from_api_keyfrom .api_key_hash import hash_api_key_for_cachefrom .client import reset_api_key_override, set_api_key_override+from .session_manager import request_session_scopeLOG = structlog.get_logger(__name__)API_KEY_HEADER = "x-api-key"@@ -604,6 +607,32 @@ class MCPAPIKeyMiddleware:def __init__(self, app: ASGIApp) -> None:self.app = app+ async def _forward_authenticated_request(+ self,+ scope: Scope,+ receive: Receive,+ send: Send,+ *,+ organization_id: str,+ api_key: str,+ ) -> None:+ scope.setdefault("state", {})+ scope["state"]["organization_id"] = organization_id+ scope["user"] = AuthenticatedUser(+ AccessToken(+ token="mcp-http-session-owner",+ client_id=organization_id,+ subject=hash_api_key_for_cache(api_key),+ scopes=[],+ )+ )+ token = set_api_key_override(api_key)+ try:+ with request_session_scope(organization_id):+ await self.app(scope, receive, send)+ finally:+ reset_api_key_override(token)+async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:if scope["type"] != "http":await self.app(scope, receive, send)@@ -657,13 +686,13 @@ class MCPAPIKeyMiddleware:try:resolution = await validate_mcp_oauth_token(bearer_token)- scope.setdefault("state", {})- scope["state"]["organization_id"] = resolution.validation.organization_id- token = set_api_key_override(resolution.api_key)- try:- await self.app(scope, receive, send)- finally:- reset_api_key_override(token)+ await self._forward_authenticated_request(+ scope,+ receive,+ send,+ organization_id=resolution.validation.organization_id,+ api_key=resolution.api_key,+ )returnexcept HTTPException as e:if e.status_code == 503:@@ -682,8 +711,6 @@ class MCPAPIKeyMiddleware:# Fall back: treat Bearer value as a raw API keytry:validation = await validate_mcp_api_key(bearer_token)- scope.setdefault("state", {})- scope["state"]["organization_id"] = validation.organization_idexcept HTTPException as e:if e.status_code in {401, 403}:if oauth_service_error is not None:@@ -705,18 +732,18 @@ class MCPAPIKeyMiddleware:await response(scope, receive, send)return- token = set_api_key_override(bearer_token)- try:- await self.app(scope, receive, send)- finally:- reset_api_key_override(token)+ await self._forward_authenticated_request(+ scope,+ receive,+ send,+ organization_id=validation.organization_id,+ api_key=bearer_token,+ )async def _handle_api_key(self, scope: Scope, receive: Receive, send: Send, api_key: str) -> None:"""Validate x-api-key header and forward the request (original flow)."""try:validation = await validate_mcp_api_key(api_key)- scope.setdefault("state", {})- scope["state"]["organization_id"] = validation.organization_idexcept HTTPException as e:if e.status_code in {401, 403}:response = _unauthorized_response(_INVALID_API_KEY_MESSAGE, include_oauth_challenge=False)@@ -733,8 +760,10 @@ class MCPAPIKeyMiddleware:await response(scope, receive, send)return- token = set_api_key_override(api_key)- try:- await self.app(scope, receive, send)- finally:- reset_api_key_override(token)+ await self._forward_authenticated_request(+ scope,+ receive,+ send,+ organization_id=validation.organization_id,+ api_key=api_key,+ )diff --git a/skyvern/cli/core/session_manager.py b/skyvern/cli/core/session_manager.pyindex b437d5cad..f6abc0ac6 100644--- a/skyvern/cli/core/session_manager.py+++ b/skyvern/cli/core/session_manager.py@@ -6,10 +6,10 @@ import secretsimport timeimport weakreffrom collections import deque-from contextlib import asynccontextmanager+from contextlib import asynccontextmanager, contextmanagerfrom contextvars import ContextVarfrom dataclasses import dataclass, field-from typing import TYPE_CHECKING, Any, AsyncIterator+from typing import TYPE_CHECKING, Any, AsyncIterator, Iteratorimport structlog@@ -36,6 +36,7 @@ class SessionState:browser: SkyvernBrowser | None = Nonecontext: BrowserContext | None = Noneapi_key_hash: str | None = None+ organization_id: str | None = Noneconsole_messages: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=1000))network_requests: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=1000))dialog_events: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=1000))@@ -73,14 +74,18 @@ class SessionState:_current_session: ContextVar[SessionState | None] = ContextVar("mcp_session", default=None)+_current_organization_id: ContextVar[str | None] = ContextVar("mcp_session_organization_id", default=None)_global_session: SessionState | None = None+_organization_sessions: dict[str, SessionState] = {}_stateless_http_mode = False_stdio_local_file_access_enabled = False-# Process-wide registry for copilot browser sessions. Keyed by browser_session_id.+# Process-wide registry for copilot browser sessions. Keys always carry the+# owning organization so authenticated HTTP requests cannot address another+# tenant's entry even when the browser session ID is known.# This bypasses ContextVar propagation issues when FastMCP runs tool handlers# in a separate task whose context snapshot predates scoped_session().-_copilot_sessions: dict[str, SessionState] = {}+_copilot_sessions: dict[tuple[str, str], SessionState] = {}# Ref snapshots and their invalidation generations, FIFO-capped like _body_store.# Capacity eviction can drop a live session's refs early; that fails safe (unknown# ref → the caller re-observes), which is why the tool contract doesn't mention it.@@ -88,8 +93,8 @@ _copilot_sessions: dict[str, SessionState] = {}# touch, so an evicted entry re-materializes with a NEVER-seen token — a stale# observe holding an old token can't collide with it (no ABA reset to a default)._SESSION_REF_STORE_MAX = 64-_session_ref_maps: dict[tuple[str, str, str | None], dict[str, Any]] = {}-_session_ref_generations: dict[tuple[str, str, str | None], int] = {}+_session_ref_maps: dict[tuple[str | None, str, str, str | None], dict[str, Any]] = {}+_session_ref_generations: dict[tuple[str | None, str, str, str | None], int] = {}_session_ref_generation_counter = itertools.count(1)# Identity tokens instead of id(): a GC'd page's id() can be reused by a new# page, making a stale snapshot look valid. Tokens are never reissued.@@ -119,7 +124,7 @@ def page_ref_key(page: SkyvernBrowserPage) -> tuple[int, int | None, str, str |)-def _generation_for(key: tuple[str, str, str | None]) -> int:+def _generation_for(key: tuple[str | None, str, str, str | None]) -> int:generation = _session_ref_generations.get(key)if generation is None:generation = next(_session_ref_generation_counter)@@ -134,18 +139,19 @@ def _session_ref_key(*,session_id: str | None = None,cdp_url: str | None = None,-) -> tuple[str, str, str | None] | None:+) -> tuple[str | None, str, str, str | None] | None:context = state.contextresolved_session_id = session_id or (context.session_id if context else None)resolved_cdp_url = cdp_url or (context.cdp_url if context else None)# Prefer the hash stored at resolve_browser time — recomputing runs a# deliberately slow PBKDF2 per call, and this sits on the per-ref hot path.api_key_hash = state.api_key_hash or _api_key_hash(get_active_api_key())+ organization_id = _current_organization_id.get() or state.organization_idif resolved_session_id:- return ("cloud_session", resolved_session_id, api_key_hash)+ return (organization_id, "cloud_session", resolved_session_id, api_key_hash)if resolved_cdp_url:- return ("cdp", resolved_cdp_url, api_key_hash)+ return (organization_id, "cdp", resolved_cdp_url, api_key_hash)return None@@ -233,24 +239,33 @@ def clear_session_ref_map(state._observed_refs_generation = next(_session_ref_generation_counter)-def register_copilot_session(session_id: str, state: SessionState) -> None:+def register_copilot_session(session_id: str, state: SessionState, *, organization_id: str) -> None:"""Register a pre-configured browser session for cross-task lookup.The registry is process-local and in-memory: entries do not survive aprocess restart and are not shared across uvicorn workers. Callers thatneed cross-process continuity must reconnect via the cloud session API."""- _copilot_sessions[session_id] = state+ if not organization_id:+ raise ValueError("organization_id is required for a copilot browser session")+ if state.organization_id not in {None, organization_id}:+ raise ValueError("session state belongs to a different organization")+ state.organization_id = organization_id+ _copilot_sessions[(organization_id, session_id)] = state-def unregister_copilot_session(session_id: str) -> None:+def unregister_copilot_session(session_id: str, *, organization_id: str) -> None:"""Remove a copilot browser session from the process-local registry."""- _copilot_sessions.pop(session_id, None)+ _copilot_sessions.pop((organization_id, session_id), None)def active_copilot_session_ids() -> set[str]:"""Browser-session IDs currently bound to an active copilot turn."""- return set(_copilot_sessions)+ return {session_id for _, session_id in _copilot_sessions}+++def _registered_copilot_session(session_id: str, *, organization_id: str) -> SessionState | None:+ return _copilot_sessions.get((organization_id, session_id))def _explicit_cloud_session_can_access_localhost() -> bool:@@ -260,7 +275,17 @@ def _explicit_cloud_session_can_access_localhost() -> bool:def get_current_session() -> SessionState:global _global_session+ organization_id = _current_organization_id.get()state = _current_session.get()++ if organization_id is not None and not _stateless_http_mode:+ state = _organization_sessions.get(organization_id)+ if state is None:+ state = SessionState(organization_id=organization_id)+ _organization_sessions[organization_id] = state+ _current_session.set(state)+ return state+if state is not None:return state@@ -280,11 +305,30 @@ def get_current_session() -> SessionState:def set_current_session(state: SessionState) -> None:global _global_session- if not _stateless_http_mode:++ organization_id = _current_organization_id.get()+ if organization_id is not None and not _stateless_http_mode:+ state.organization_id = organization_id+ _organization_sessions[organization_id] = state+ elif not _stateless_http_mode:_global_session = state_current_session.set(state)+@contextmanager+def request_session_scope(organization_id: str) -> Iterator[None]:+ if not organization_id:+ raise ValueError("organization_id is required for an authenticated MCP request")++ organization_token = _current_organization_id.set(organization_id)+ session_token = _current_session.set(None)+ try:+ yield+ finally:+ _current_session.reset(session_token)+ _current_organization_id.reset(organization_token)++@asynccontextmanagerasync def scoped_session(state: SessionState) -> AsyncIterator[None]:"""Temporarily push a SessionState into ContextVar scope.@@ -396,12 +440,17 @@ async def resolve_browser(# Check copilot session registry (cross-task fallback when ContextVar# does not propagate through FastMCP in-process transport).- registered = _copilot_sessions.get(session_id) if session_id else None+ organization_id = _current_organization_id.get()+ registered = (+ _registered_copilot_session(session_id, organization_id=organization_id)+ if session_id and organization_id+ else None+ )if registered is not None and registered.browser is not None and registered.context is not None:if _hashes_equal(registered.api_key_hash, active_api_key_hash) or not has_api_key_override():# FastMCP in-process tool tasks may miss the parent ContextVar.# Explicit request overrides still win; otherwise use the temporary Copilot registry.- _current_session.set(registered)+ set_current_session(registered)return registered.browser, registered.contextbrowser: SkyvernBrowser | None = None@@ -452,13 +501,16 @@ async def resolve_browser(raise BrowserNotAvailableError()-async def close_current_session() -> None:- """Close the active browser session (if any) and clear local session state."""+async def _close_session_state(current: SessionState, *, close_via_active_client: bool) -> None:from .session_ops import do_session_close- current = get_current_session()try:- if current.context and current.context.mode == "cloud_session" and current.context.session_id:+ if (+ close_via_active_client+ and current.context+ and current.context.mode == "cloud_session"+ and current.context.session_id+ ):try:skyvern = get_skyvern()await do_session_close(skyvern, current.context.session_id)@@ -483,10 +535,53 @@ async def close_current_session() -> None:clear_session_ref_map()if current.context and current.context.session_id:delete_session_trajectories(current.context.session_id)- unregister_copilot_session(current.context.session_id)+ organization_id = _current_organization_id.get() or current.organization_id+ if organization_id is not None:+ unregister_copilot_session(current.context.session_id, organization_id=organization_id)+++async def close_current_session() -> None:+ """Close the active browser session (if any) and clear local session state."""+ current = get_current_session()+ try:+ await _close_session_state(current, close_via_active_client=True)+ finally:set_current_session(SessionState())+async def close_all_sessions() -> None:+ errors: list[tuple[str | None, BaseException]] = []+ for organization_id in list(_organization_sessions):+ try:+ with request_session_scope(organization_id):+ current = get_current_session()+ try:+ # Preserve the session ID so browser.close() uses the browser's owning client for remote cleanup.+ await _close_session_state(current, close_via_active_client=False)+ finally:+ set_current_session(SessionState())+ except BaseException as exc:+ errors.append((organization_id, exc))++ _organization_sessions.clear()+ try:+ await close_current_session()+ except BaseException as exc:+ errors.append((None, exc))+ finally:+ _current_session.set(None)+ _current_organization_id.set(None)++ if errors:+ for failed_organization_id, cleanup_error in errors[1:]:+ LOG.warning(+ "Additional session cleanup failed",+ organization_id=failed_organization_id,+ exc_info=(type(cleanup_error), cleanup_error, cleanup_error.__traceback__),+ )+ raise errors[0][1]++async def get_page(session_id: str | None = None,cdp_url: str | None = None,diff --git a/skyvern/cli/run_commands.py b/skyvern/cli/run_commands.pyindex 20c5121d6..4a601dd92 100644--- a/skyvern/cli/run_commands.py+++ b/skyvern/cli/run_commands.py@@ -65,13 +65,13 @@ async def _cleanup_mcp_resources() -> None:from skyvern.cli.core.action_log import shutdown_action_log_worker # noqa: PLC0415from skyvern.cli.core.client import close_skyvern # noqa: PLC0415from skyvern.cli.core.mcp_http_auth import close_auth_db # noqa: PLC0415- from skyvern.cli.core.session_manager import close_current_session # noqa: PLC0415+ from skyvern.cli.core.session_manager import close_all_sessions # noqa: PLC0415try:await shutdown_action_log_worker()finally:try:- await close_current_session()+ await close_all_sessions()finally:try:await close_skyvern()diff --git a/skyvern/forge/sdk/copilot/canonical_ownership.py b/skyvern/forge/sdk/copilot/canonical_ownership.pynew file mode 100644index 000000000..7dc1135c3--- /dev/null+++ b/skyvern/forge/sdk/copilot/canonical_ownership.py@@ -0,0 +1,24 @@+"""Ownership of a copilot turn's canonical workflow write.++An interrupted turn may only roll back a canonical write it still owns. Ownership is+recorded, not inferred: the writer stamps the fingerprint of what it wrote onto the turn's+pending marker, and reconcile rolls back only while canonical still carries that content.+A later write by anyone — another chat, a manual edit, a subsequent turn — changes the+fingerprint and forfeits the rollback.+"""++import hashlib+import json+from typing import Any++# A write always moves these, so they carry no information about who wrote it.+VOLATILE_WORKFLOW_FIELDS = frozenset({"workflow_id", "version", "created_at", "modified_at"})+++def workflow_content(workflow: dict[str, Any]) -> dict[str, Any]:+ return {key: value for key, value in workflow.items() if key not in VOLATILE_WORKFLOW_FIELDS}+++def workflow_content_fingerprint(workflow: dict[str, Any]) -> str:+ canonical = json.dumps(workflow_content(workflow), sort_keys=True, default=str)+ return hashlib.sha256(canonical.encode()).hexdigest()diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex 25387bcad..43ca7e634 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -25,6 +25,7 @@ from mcp.types import ()from playwright.async_api import Browser, BrowserContext+from skyvern.cli.core.session_manager import request_session_scopefrom skyvern.forge import appfrom skyvern.forge.agent_functions import CopilotCandidateNetworkHopfrom skyvern.forge.sdk.copilot.blocker_signal import (@@ -308,7 +309,8 @@ class SkyvernOverlayMCPServer(MCPServer):stack = AsyncExitStack()await stack.__aenter__()client = Client(self._transport)- await stack.enter_async_context(client)+ with request_session_scope(self._context_provider().organization_id):+ await stack.enter_async_context(client)self._client = clientself._exit_stack = stackdiff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 187540fa8..de5fff6b2 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -733,13 +733,14 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:browser=skyvern_browser,context=mcp_ctx,api_key_hash=hash_api_key_for_cache(active_key) if active_key else None,+ organization_id=ctx.organization_id,)if working_page is not None:# Seed the tab pin from the already-probed page (mirrors what skyvern_tab_switch# sets interactively) so self-heal tools land on the adopted tab instead of the# new SkyvernBrowser's pages[-1] fallback.state._active_page = working_page- register_copilot_session(browser_session_id, state)+ register_copilot_session(browser_session_id, state, organization_id=ctx.organization_id)if is_self_heal_session_id(browser_session_id):LOG.info("registered self-heal browser session",@@ -756,7 +757,7 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:)else:ctx.sdk_action_workflow_run_ids_by_browser_session.pop(sdk_action_workflow_run_cache_key, None)- unregister_copilot_session(browser_session_id)+ unregister_copilot_session(browser_session_id, organization_id=ctx.organization_id)if is_self_heal_session_id(browser_session_id):LOG.info("unregistered self-heal browser session", session_id=browser_session_id)finally:diff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.pyindex bb7dc0217..9a49278f4 100644--- a/skyvern/forge/sdk/copilot/terminal_envelope.py+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py@@ -16,6 +16,12 @@ _FINAL_RUN_VERDICTS = frozenset({"demonstrated", "not_demonstrated", "not_evalua_REVIEW_PROPOSAL_DISPOSITIONS = frozenset({"review_untested", "review_tested"})_SHADOW_REASON_TRAILING_PUNCTUATION = ".,;:!?"+INTERRUPTED_TERMINAL_REASON = "interrupted"+INTERRUPTED_TERMINAL_MESSAGE = (+ "This turn was interrupted before it could finish — the server stopped part-way through it. "+ "Send your message again to retry."+)+class TerminalOutcomeEnvelope(BaseModel):next_state: TerminalNextState@@ -108,6 +114,16 @@ def finalize_applied_state()+def interrupted_terminal_envelope() -> TerminalOutcomeEnvelope:+ """Envelope for a turn whose process died mid-run — stopped, but never user-cancelled."""+ return TerminalOutcomeEnvelope(+ next_state="stopped",+ verified=False,+ response_kind="stopped",+ halt_kind=INTERRUPTED_TERMINAL_REASON,+ )++def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: str, cancelled: bool) -> tuple[str, bool]:# Diagnose/refuse answers share next_state="stopped" but their text IS the# deliverable — only stopped-kind turns get the honest-stop replacement.diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex a5b01b8a6..b066d1bd8 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -1,6 +1,7 @@from __future__ import annotationsimport ast+import contextlibimport copyimport hashlibimport io@@ -64,6 +65,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (recorded_outcome_from_author_time_reject,recorded_outcome_from_authoring_repair_context,)+from skyvern.forge.sdk.copilot.canonical_ownership import workflow_content_fingerprintfrom skyvern.forge.sdk.copilot.code_block_preflight import (SANDBOX_UNRESOLVED_NAME_REASON_CODE,author_time_code_block_diagnostics,@@ -10155,6 +10157,31 @@ def _enrich_scout_trajectory_input_correspondences(workflow_yaml: str, ctx: Ageninteraction.pop("input_correspondences", None)+async def _record_canonical_write_ownership(ctx: CopilotContext, workflow: Workflow) -> None:+ """Stamp the turn's marker with what it just left canonical as.++ Best-effort: a missed stamp only costs this turn its rollback claim, while raising here+ would fail a write that already succeeded. Canonical is re-read rather than fingerprinted+ from the in-memory object so the value matches what reconcile computes.+ """+ with contextlib.suppress(Exception):+ workflow_permanent_id = workflow.workflow_permanent_id+ if not (ctx.workflow_copilot_chat_id and ctx.turn_id and workflow_permanent_id and ctx.organization_id):+ return+ persisted = await app.DATABASE.workflows.get_workflow_by_permanent_id(+ workflow_permanent_id=workflow_permanent_id,+ organization_id=ctx.organization_id,+ )+ if persisted is None:+ return+ await app.DATABASE.workflow_params.record_pending_copilot_turn_canonical_write(+ organization_id=ctx.organization_id,+ workflow_copilot_chat_id=ctx.workflow_copilot_chat_id,+ turn_id=ctx.turn_id,+ fingerprint=workflow_content_fingerprint(persisted.model_dump(mode="json")),+ )++async def _update_workflow(params: dict[str, Any],ctx: AgentContext,@@ -10537,6 +10564,10 @@ async def _update_workflow(edited_by="copilot",)ctx.canonical_was_persisted_due_to_param_change = True+ # isinstance narrows the declared ``AgentContext`` to the marker-aware+ # ``CopilotContext`` for mypy, matching the narrative-emit seam below.+ if isinstance(ctx, CopilotContext):+ await _record_canonical_write_ownership(ctx, workflow)ctx.staged_workflow_yaml = workflow_yamlctx.staged_workflow = workflowctx.has_staged_proposal = Truediff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 5be42f463..6f29f823d 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -1719,6 +1719,7 @@ class WorkflowCopilotChatModel(Base):workflow_permanent_id = Column(String, nullable=False, index=True)proposed_workflow = Column(JSON, nullable=True)auto_accept = Column(Boolean, nullable=True, default=False)+ pending_turns = Column(JSON, nullable=True)created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)modified_at = Column(diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.pyindex 2fb96f466..9cd79f4c6 100644--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py@@ -8,6 +8,7 @@ from typing import Anyimport structlogfrom sqlalchemy import selectfrom sqlalchemy.dialects.postgresql import insert as pg_insert+from sqlalchemy.orm import deferfrom skyvern.config import settingsfrom skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_json, criterion_authority_projection@@ -50,6 +51,7 @@ from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcomefrom skyvern.forge.sdk.schemas.task_generations import TaskGenerationfrom skyvern.forge.sdk.schemas.tasks import Task, TaskStatusfrom skyvern.forge.sdk.schemas.workflow_copilot import (+ CopilotPendingTurn,NonAdoptableCriteriaSet,WorkflowCopilotChat,WorkflowCopilotChatMessage,@@ -77,6 +79,8 @@ from skyvern.webeye.actions.actions import ActionLOG = structlog.get_logger()+PENDING_TURN_RETENTION = timedelta(days=30)+def _floor_rekeyed_association_is_coherent(item: Mapping[str, object]) -> bool:marker_present = "requested_output_floor_rekeyed" in item@@ -171,6 +175,32 @@ def _decode_completion_criteria_set()+def _parse_pending_turn_timestamp(value: object) -> datetime | None:+ if not isinstance(value, str):+ return None+ try:+ parsed = datetime.fromisoformat(value)+ except ValueError:+ return None+ return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)+++def _prune_pending_turns(pending_turns: object) -> dict[str, Any]:+ """Drop markers too old to still be worth reconciling, so the column cannot grow without bound."""+ if not isinstance(pending_turns, dict):+ return {}+ cutoff = datetime.now(timezone.utc) - PENDING_TURN_RETENTION+ kept: dict[str, Any] = {}+ for turn_id, entry in pending_turns.items():+ if not isinstance(entry, dict):+ continue+ started_at = _parse_pending_turn_timestamp(entry.get("started_at"))+ if started_at is not None and started_at < cutoff:+ continue+ kept[turn_id] = entry+ return kept++class WorkflowParametersRepository(BaseRepository):"""Database operations for workflow parameters, copilot chat, task generation, actions, and runs."""@@ -666,6 +696,168 @@ class WorkflowParametersRepository(BaseRepository):await session.refresh(new_message)return convert_to_workflow_copilot_chat_message(new_message, self.debug_enabled)+ @db_operation("start_copilot_turn")+ async def start_copilot_turn(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ pending_turn: CopilotPendingTurn,+ user_message: str,+ audio_artifact_id: str | None = None,+ ) -> WorkflowCopilotChatMessage:+ """Write the turn's user row and its pending marker in one transaction.++ Two separate commits would leave a crash window in which the user+ message exists with no marker to reconcile it.+ """+ async with self.Session() as session:+ new_message = WorkflowCopilotChatMessageModel(+ workflow_copilot_chat_id=workflow_copilot_chat_id,+ organization_id=organization_id,+ sender=WorkflowCopilotChatSender.USER,+ content=user_message,+ audio_artifact_id=audio_artifact_id,+ )+ session.add(new_message)+ await session.flush()+ chat = (+ await session.scalars(+ select(WorkflowCopilotChatModel)+ .where(WorkflowCopilotChatModel.organization_id == organization_id)+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)+ .with_for_update()+ )+ ).first()+ if chat is None:+ raise NotFoundError(f"workflow copilot chat {workflow_copilot_chat_id}")+ pending = _prune_pending_turns(chat.pending_turns)+ pending[pending_turn.turn_id] = pending_turn.model_copy(+ update={"user_message_id": new_message.workflow_copilot_chat_message_id}+ ).model_dump(mode="json")+ chat.pending_turns = pending+ await session.commit()+ await session.refresh(new_message)+ return convert_to_workflow_copilot_chat_message(new_message, self.debug_enabled)++ @db_operation("claim_pending_copilot_turn")+ async def claim_pending_copilot_turn(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ claim_before: datetime,+ ) -> bool:+ """Stamp ``recovering_at`` on one pending turn, returning whether this caller won it.++ A recovery that itself crashes leaves the stamp behind; a later reader+ reclaims it once the stamp predates ``claim_before``.+ """+ async with self.Session() as session:+ chat = (+ await session.scalars(+ select(WorkflowCopilotChatModel)+ .where(WorkflowCopilotChatModel.organization_id == organization_id)+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)+ .with_for_update()+ )+ ).first()+ if chat is None:+ return False+ pending = dict(chat.pending_turns or {})+ entry = pending.get(turn_id)+ if not isinstance(entry, dict):+ return False+ recovering_at = _parse_pending_turn_timestamp(entry.get("recovering_at"))+ if recovering_at is not None and recovering_at > claim_before:+ return False+ pending[turn_id] = {**entry, "recovering_at": datetime.now(timezone.utc).isoformat()}+ chat.pending_turns = pending+ await session.commit()+ return True++ @db_operation("clear_pending_copilot_turn")+ async def clear_pending_copilot_turn(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ ) -> None:+ async with self.Session() as session:+ chat = (+ await session.scalars(+ select(WorkflowCopilotChatModel)+ .where(WorkflowCopilotChatModel.organization_id == organization_id)+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)+ .with_for_update()+ )+ ).first()+ if chat is None:+ return+ pending = dict(chat.pending_turns or {})+ if pending.pop(turn_id, None) is None:+ return+ chat.pending_turns = pending+ await session.commit()++ @db_operation("record_pending_copilot_turn_canonical_write")+ async def record_pending_copilot_turn_canonical_write(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ fingerprint: str,+ ) -> None:+ """Stamp what this turn left canonical as, so reconcile can tell its own write from anyone else's."""+ async with self.Session() as session:+ chat = (+ await session.scalars(+ select(WorkflowCopilotChatModel)+ .where(WorkflowCopilotChatModel.organization_id == organization_id)+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)+ .with_for_update()+ )+ ).first()+ if chat is None:+ return+ pending = dict(chat.pending_turns or {})+ entry = pending.get(turn_id)+ if not isinstance(entry, dict):+ return+ pending[turn_id] = {**entry, "canonical_write_fingerprint": fingerprint}+ chat.pending_turns = pending+ await session.commit()++ @db_operation("replace_workflow_copilot_chat_message")+ async def replace_workflow_copilot_chat_message(+ self,+ organization_id: str,+ workflow_copilot_chat_message_id: str,+ content: str,+ global_llm_context: str | None,+ turn_outcome: TurnOutcome | None,+ narrative_payload: TurnNarrativePayload | dict[str, Any] | None,+ ) -> WorkflowCopilotChatMessage | None:+ async with self.Session() as session:+ message = (+ await session.scalars(+ select(WorkflowCopilotChatMessageModel)+ .where(WorkflowCopilotChatMessageModel.organization_id == organization_id)+ .where(+ WorkflowCopilotChatMessageModel.workflow_copilot_chat_message_id+ == workflow_copilot_chat_message_id+ )+ )+ ).first()+ if message is None:+ return None+ message.content = content+ message.global_llm_context = global_llm_context+ message.turn_outcome = turn_outcome.model_dump(mode="json") if turn_outcome else None+ message.narrative_payload = narrative_payload+ await session.commit()+ await session.refresh(message)+ return WorkflowCopilotChatMessage.model_validate(message)+@db_operation("get_workflow_copilot_chat_messages")async def get_workflow_copilot_chat_messages(self,@@ -748,6 +940,7 @@ class WorkflowParametersRepository(BaseRepository):)query = (select(WorkflowCopilotChatModel, first_message.c.title)+ .options(defer(WorkflowCopilotChatModel.pending_turns)).join(first_message, first_message.c.chat_id == WorkflowCopilotChatModel.workflow_copilot_chat_id).where(WorkflowCopilotChatModel.organization_id == organization_id))diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex cf7c467d2..e3b7a2af1 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -25,6 +25,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityTypefrom skyvern.forge.sdk.copilot.agent import run_copilot_agentfrom skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_write, resolve_copilot_created_by_stamp+from skyvern.forge.sdk.copilot.canonical_ownership import workflow_content_fingerprintfrom skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_stepsfrom skyvern.forge.sdk.copilot.completion_criteria_store import (CRITERIA_SET_STATUS_ACTIVE,@@ -49,6 +50,7 @@ from skyvern.forge.sdk.copilot.credential_pause import (resolve_credential_pause,)from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure+from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDSfrom skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handlerfrom skyvern.forge.sdk.copilot.output_utils import truncate_outputfrom skyvern.forge.sdk.copilot.recoverable_failure import (@@ -59,8 +61,11 @@ from skyvern.forge.sdk.copilot.recoverable_failure import ()from skyvern.forge.sdk.copilot.request_policy import is_defer_authoring_durable_fill_criterionfrom skyvern.forge.sdk.copilot.terminal_envelope import (+ INTERRUPTED_TERMINAL_MESSAGE,+ INTERRUPTED_TERMINAL_REASON,TerminalOutcomeEnvelope,finalize_applied_state,+ interrupted_terminal_envelope,reason_in_reply_shadow,render_terminal_message,)@@ -78,6 +83,7 @@ from skyvern.forge.sdk.routes.routers import base_routerfrom skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcomefrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.workflow_copilot import (+ CopilotPendingTurn,NonAdoptableCriteriaSet,WorkflowCopilotApplyProposedWorkflowRequest,WorkflowCopilotAudioUploadResponse,@@ -108,11 +114,18 @@ from skyvern.schemas.workflows import (WorkflowDefinitionYAML,)from skyvern.utils.prompt_truncation import truncate_page_html_for_summary+from skyvern.utils.secret_headers import merge_masked_headersfrom skyvern.utils.strings import escape_code_fencesfrom skyvern.utils.yaml_loader import safe_load_no_datesWORKFLOW_KNOWLEDGE_BASE_PATH = Path("skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt")CHAT_HISTORY_CONTEXT_MESSAGES = 10++# Wall clock, unlike TOTAL_TIMEOUT_SECONDS: that budget excludes time parked in a+# credential pause, so a legitimately live turn can outlast it in real time.+RECONCILE_ABANDON_AFTER_SECONDS = (+ TOTAL_TIMEOUT_SECONDS + settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS + 120+)WORKFLOW_COPILOT_DEBUG_HTML_MAX_CHARS = 3_000_000ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES = {"audio/mp4",@@ -152,6 +165,7 @@ def bind_copilot_session_id(chat_id: str | None) -> Iterator[None]:COPILOT_CODE_MODE_OPT_OUT_EVENT = "copilot_code_mode_opt_out"COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON = "copilot_recoverable_failure"+USER_CANCELLED_TERMINAL_REASON = "user_cancelled"def _effective_copilot_composer_mode(@@ -757,9 +771,131 @@ async def _persist_proposed_workflow_state(await _clear_proposed_workflow(chat)+def _as_utc(value: datetime) -> datetime:+ return value if value.tzinfo else value.replace(tzinfo=timezone.utc)+++async def _assistant_row_for_turn(chat: WorkflowCopilotChat, turn_id: str) -> WorkflowCopilotChatMessage | None:+ # Reads the whole chat to find one turn's row, on every finalize. Fine at current per-chat message+ # counts; if histories grow, index turn_outcome->>'copilot_turn_id' and query for it directly.+ messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ )+ return next(+ (+ message+ for message in messages+ if message.sender == WorkflowCopilotChatSender.AI+ and message.turn_outcome is not None+ and message.turn_outcome.copilot_turn_id == turn_id+ ),+ None,+ )+++async def _assistant_row_exists_for_turn(chat: WorkflowCopilotChat, turn_id: str) -> bool:+ return await _assistant_row_for_turn(chat, turn_id) is not None+++def _is_interrupted_recovery_row(message: WorkflowCopilotChatMessage) -> bool:+ return message.turn_outcome is not None and message.turn_outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON+++async def _clear_pending_turn(chat: WorkflowCopilotChat, turn_id: str) -> None:+ # A failed clear is self-healing: the reconcile pass sees the turn already+ # answered, skips recovery and retries the clear.+ with contextlib.suppress(Exception):+ await app.DATABASE.workflow_params.clear_pending_copilot_turn(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ turn_id=turn_id,+ )+++async def _persist_turn_messages(+ *,+ chat: WorkflowCopilotChat,+ turn_id: str | None,+ user_message: str,+ audio_artifact_id: str | None,+ user_row_already_persisted: bool,+ assistant_content: str,+ global_llm_context: str | None,+ turn_outcome: TurnOutcome | None,+ narrative_payload: TurnNarrativePayload | None,+) -> WorkflowCopilotChatMessage | None:+ """The only writer of a copilot turn's chat rows; idempotent per ``turn_id``.++ Returns None when the assistant row for this turn already exists, and always+ drops the turn's pending marker last so a crash mid-write stays recoverable.+ """+ if turn_id is not None and turn_outcome is not None and not turn_outcome.copilot_turn_id:+ turn_outcome = turn_outcome.model_copy(update={"copilot_turn_id": turn_id})++ if not user_row_already_persisted:+ await asyncio.shield(+ app.DATABASE.workflow_params.create_workflow_copilot_chat_message(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ sender=WorkflowCopilotChatSender.USER,+ content=user_message,+ audio_artifact_id=audio_artifact_id,+ )+ )++ assistant_message: WorkflowCopilotChatMessage | None = None+ existing = await _assistant_row_for_turn(chat, turn_id) if turn_id is not None else None+ superseding_recovery = (+ existing is not None+ and _is_interrupted_recovery_row(existing)+ and turn_outcome is not None+ and turn_outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON+ )+ if existing is not None and superseding_recovery:+ # Recovery reached this turn first and wrote an interrupted row. The turn then finished,+ # so its real reply is the truth and replaces that row rather than being dropped.+ LOG.info(+ "Copilot turn finished after being recovered; replacing the interrupted row with its reply",+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ turn_id=turn_id,+ )+ assistant_message = await asyncio.shield(+ app.DATABASE.workflow_params.replace_workflow_copilot_chat_message(+ organization_id=chat.organization_id,+ workflow_copilot_chat_message_id=existing.workflow_copilot_chat_message_id,+ content=assistant_content,+ global_llm_context=global_llm_context,+ turn_outcome=turn_outcome,+ narrative_payload=narrative_payload,+ )+ )+ elif existing is not None:+ LOG.info(+ "Copilot turn already has an assistant row; skipping duplicate persist",+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ turn_id=turn_id,+ )+ else:+ assistant_message = await asyncio.shield(+ app.DATABASE.workflow_params.create_workflow_copilot_chat_message(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ sender=WorkflowCopilotChatSender.AI,+ content=assistant_content,+ global_llm_context=global_llm_context,+ turn_outcome=turn_outcome,+ narrative_payload=narrative_payload,+ )+ )++ if turn_id is not None:+ await _clear_pending_turn(chat, turn_id)+ return assistant_message++async def _persist_cancel_turn(stream: EventSourceStream,- chat: Any,+ chat: WorkflowCopilotChat,organization_id: str,original_workflow: Workflow | None,user_message: str,@@ -768,6 +904,7 @@ async def _persist_cancel_turn(turn_id: str | None = None,keep_pending_proposal: bool = False,prior_global_llm_context: str | None = None,+ user_row_already_persisted: bool = False,) -> None:"""Persist a cancelled turn and emit a terminal SSE response frame.@@ -785,7 +922,12 @@ async def _persist_cancel_turn(total_tokens = Noneresponse_type = "REPLY"output_policy_diagnostics = None- turn_outcome = None+ turn_outcome = TurnOutcome(+ response_kind=ResponseKind.RECOVER,+ reason_code=USER_CANCELLED_TERMINAL_REASON,+ terminal_reason=USER_CANCELLED_TERMINAL_REASON,+ copilot_turn_id=turn_id,+ )response_turn_id = turn_idnarrative_summary = Nonenarrative_payload = None@@ -860,26 +1002,18 @@ async def _persist_cancel_turn(terminal_envelope=terminal_envelope,)- await asyncio.shield(- app.DATABASE.workflow_params.create_workflow_copilot_chat_message(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- sender=WorkflowCopilotChatSender.USER,- content=user_message,- audio_artifact_id=audio_artifact_id,- )- )- assistant_message = await asyncio.shield(- app.DATABASE.workflow_params.create_workflow_copilot_chat_message(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- sender=WorkflowCopilotChatSender.AI,- content=user_response,- global_llm_context=updated_global_llm_context,- turn_outcome=turn_outcome,- narrative_payload=narrative_payload,- )+ assistant_message = await _persist_turn_messages(+ chat=chat,+ turn_id=turn_id,+ user_message=user_message,+ audio_artifact_id=audio_artifact_id,+ user_row_already_persisted=user_row_already_persisted,+ assistant_content=user_response,+ global_llm_context=updated_global_llm_context,+ turn_outcome=turn_outcome,+ narrative_payload=narrative_payload,)+ response_time = assistant_message.created_at if assistant_message else datetime.now(timezone.utc)try:await asyncio.shield(stream.send(@@ -888,7 +1022,7 @@ async def _persist_cancel_turn(workflow_copilot_chat_id=chat.workflow_copilot_chat_id,message=user_response,updated_workflow=updated_workflow.model_dump(mode="json") if updated_workflow else None,- response_time=assistant_message.created_at,+ response_time=response_time,total_tokens=total_tokens,response_type=response_type,proposal_disposition=proposal_disposition,@@ -917,6 +1051,8 @@ async def _finalise_normal_turn(original_workflow: Workflow | None,chat_request: WorkflowCopilotChatRequest,agent_result: AgentResult,+ turn_id: str | None = None,+ user_row_already_persisted: bool = False,) -> None:"""Atomic post-agent finalisation: rollback, proposal, chat rows, RESPONSE.@@ -1027,19 +1163,13 @@ async def _finalise_normal_turn(terminal_envelope=terminal_envelope,)- await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- sender=WorkflowCopilotChatSender.USER,- content=chat_request.message,+ assistant_message = await _persist_turn_messages(+ chat=chat,+ turn_id=turn_id,+ user_message=chat_request.message,audio_artifact_id=chat_request.audio_artifact_id,- )-- assistant_message = await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- sender=WorkflowCopilotChatSender.AI,- content=user_response,+ user_row_already_persisted=user_row_already_persisted,+ assistant_content=user_response,global_llm_context=updated_global_llm_context,turn_outcome=agent_result.turn_outcome,narrative_payload=narrative_payload,@@ -1051,7 +1181,7 @@ async def _finalise_normal_turn(workflow_copilot_chat_id=chat.workflow_copilot_chat_id,message=user_response,updated_workflow=updated_workflow.model_dump(mode="json") if updated_workflow else None,- response_time=assistant_message.created_at,+ response_time=assistant_message.created_at if assistant_message else datetime.now(timezone.utc),total_tokens=agent_result.total_tokens,response_type=agent_result.response_type,proposal_disposition=proposal_disposition,@@ -1662,6 +1792,7 @@ async def _new_copilot_chat_post(original_workflow: Workflow | None = Nonechat = None+ turn_started = False# Snapshot before any write this turn — lets a recovery handler tell# "this turn's own write already superseded it" from "the write itself# is what failed", which agent_result.updated_workflow alone can't.@@ -1761,6 +1892,8 @@ async def _new_copilot_chat_post(original_workflow=original_workflow,chat_request=chat_request,agent_result=recovered_result,+ turn_id=turn_id,+ user_row_already_persisted=turn_started,))terminal_frame_emitted = True@@ -1959,6 +2092,21 @@ async def _new_copilot_chat_post(stored_completion_criteria = await _load_completion_criteria_snapshot(chat)+ await app.DATABASE.workflow_params.start_copilot_turn(+ organization_id=organization.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ pending_turn=CopilotPendingTurn(+ turn_id=turn_id,+ started_at=datetime.now(timezone.utc),+ pre_turn_workflow=original_workflow.model_dump(mode="json"),+ pre_turn_proposed_workflow=proposed_workflow_at_turn_start,+ keep_pending_proposal=chat_request.keep_pending_proposal,+ ),+ user_message=chat_request.message,+ audio_artifact_id=chat_request.audio_artifact_id,+ )+ turn_started = True+with bind_copilot_session_id(chat.workflow_copilot_chat_id):agent_result = await run_copilot_agent(stream=stream,@@ -1998,6 +2146,7 @@ async def _new_copilot_chat_post(audio_artifact_id=chat_request.audio_artifact_id,turn_id=turn_id,keep_pending_proposal=chat_request.keep_pending_proposal,+ user_row_already_persisted=turn_started,)terminal_frame_emitted = Truecapture_code_mode_opt_out_after_persist()@@ -2018,6 +2167,8 @@ async def _new_copilot_chat_post(original_workflow=original_workflow,chat_request=chat_request,agent_result=agent_result,+ turn_id=turn_id,+ user_row_already_persisted=turn_started,))terminal_frame_emitted = True@@ -2075,6 +2226,7 @@ async def _new_copilot_chat_post(turn_id=turn_id,keep_pending_proposal=chat_request.keep_pending_proposal,prior_global_llm_context=global_llm_context,+ user_row_already_persisted=turn_started,))terminal_frame_emitted = True@@ -2473,6 +2625,123 @@ async def list_workflow_copilot_chats()+async def _restore_canonical_after_interrupted_turn(+ chat: WorkflowCopilotChat,+ organization_id: str,+ entry: CopilotPendingTurn,+) -> None:+ """Undo an interrupted turn's canonical write, stashing the displaced draft as the proposal.++ Only fires while canonical still carries the fingerprint this turn recorded when it wrote.+ Anyone writing after it — another chat, a manual edit, a later turn — forfeits the rollback.+ """+ if entry.pre_turn_workflow is None or entry.canonical_write_fingerprint is None:+ return+ current = await app.DATABASE.workflows.get_workflow_by_permanent_id(+ workflow_permanent_id=chat.workflow_permanent_id,+ organization_id=organization_id,+ )+ if current is None:+ return+ if workflow_content_fingerprint(current.model_dump(mode="json")) != entry.canonical_write_fingerprint:+ return++ if not (entry.keep_pending_proposal and entry.pre_turn_proposed_workflow is not None):+ stashed_draft = current.model_dump(mode="json")+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(+ organization_id=organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ proposed_workflow=stashed_draft,+ )+ chat.proposed_workflow = stashed_draft++ pre_turn_workflow = Workflow.model_validate(entry.pre_turn_workflow)+ # The snapshot was taken with model_dump, which masks cdp_connect_headers, so restoring it+ # verbatim would overwrite the live browser credentials with the mask string.+ pre_turn_workflow.cdp_connect_headers = merge_masked_headers(+ pre_turn_workflow.cdp_connect_headers, current.cdp_connect_headers+ )+ await _restore_workflow_definition(pre_turn_workflow, organization_id)+++async def _recover_interrupted_copilot_turn(+ chat: WorkflowCopilotChat,+ organization_id: str,+ entry: CopilotPendingTurn,+) -> None:+ if await _assistant_row_exists_for_turn(chat, entry.turn_id):+ await _clear_pending_turn(chat, entry.turn_id)+ return++ try:+ await _restore_canonical_after_interrupted_turn(chat, organization_id, entry)+ except Exception:+ # Answering the turn here would strand canonical on the half-written draft+ # while history claims it was recovered; leave the marker for a later read.+ LOG.warning(+ "Canonical restore failed while recovering an interrupted copilot turn",+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ turn_id=entry.turn_id,+ exc_info=True,+ )+ return++ turn_outcome = TurnOutcome(+ response_kind=ResponseKind.RECOVER,+ reason_code=INTERRUPTED_TERMINAL_REASON,+ terminal_reason=INTERRUPTED_TERMINAL_REASON,+ copilot_turn_id=entry.turn_id,+ )+ narrative_payload = _with_terminal_narrative_metadata(+ _make_error_narrative_payload(entry.turn_id, None, INTERRUPTED_TERMINAL_MESSAGE),+ cancelled=False,+ proposal_disposition=_proposal_disposition(None),+ terminal_envelope=interrupted_terminal_envelope().model_dump(mode="json"),+ )+ await _persist_turn_messages(+ chat=chat,+ turn_id=entry.turn_id,+ user_message="",+ audio_artifact_id=None,+ user_row_already_persisted=True,+ assistant_content=INTERRUPTED_TERMINAL_MESSAGE,+ global_llm_context=None,+ turn_outcome=turn_outcome,+ narrative_payload=narrative_payload,+ )+ LOG.info(+ "Recovered an interrupted copilot turn on read",+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ turn_id=entry.turn_id,+ )+++async def _reconcile_interrupted_copilot_turns(chat: WorkflowCopilotChat, organization_id: str) -> None:+ if not chat.pending_turns:+ return+ abandoned_before = datetime.now(timezone.utc) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS)+ for entry in sorted(chat.pending_turns.values(), key=lambda item: _as_utc(item.started_at)):+ if _as_utc(entry.started_at) > abandoned_before:+ continue+ claimed = await app.DATABASE.workflow_params.claim_pending_copilot_turn(+ organization_id=organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ turn_id=entry.turn_id,+ claim_before=abandoned_before,+ )+ if not claimed:+ continue+ try:+ await _recover_interrupted_copilot_turn(chat, organization_id, entry)+ except Exception:+ LOG.warning(+ "Failed to recover an interrupted copilot turn",+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ turn_id=entry.turn_id,+ exc_info=True,+ )++@base_router.get("/workflow/copilot/chat-history", include_in_schema=False)async def workflow_copilot_chat_history(workflow_permanent_id: str | None = None,@@ -2495,6 +2764,7 @@ async def workflow_copilot_chat_history(detail="workflow_permanent_id or workflow_copilot_chat_id is required",)if chat:+ await _reconcile_interrupted_copilot_turns(chat, organization.organization_id)chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(chat.workflow_copilot_chat_id)diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex b76aaf040..ebb50515a 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -3,13 +3,34 @@ from datetime import datetimefrom enum import StrEnumfrom typing import Any, Literal-from pydantic import BaseModel, ConfigDict, Field+from pydantic import BaseModel, ConfigDict, Field, field_validatorfrom skyvern.forge.sdk.copilot.context import ProposalDisposition, ResponseType, TurnNarrativePayloadfrom skyvern.forge.sdk.copilot.run_outcome import RunOutcomeReasonCode, RunOutcomeVerdictfrom skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome+class CopilotPendingTurn(BaseModel):+ """Durable write-ahead marker for one in-flight copilot turn.++ Written in the same transaction as the turn's user message so a hard kill+ leaves enough state to reconcile the turn and roll canonical back.+ """++ model_config = ConfigDict(extra="ignore")++ turn_id: str+ started_at: datetime+ pre_turn_workflow: dict[str, Any] | None = None+ pre_turn_proposed_workflow: dict[str, Any] | None = None+ keep_pending_proposal: bool = False+ user_message_id: str | None = None+ recovering_at: datetime | None = None+ # Fingerprint of the canonical workflow as this turn last left it. None means the turn+ # never wrote canonical, so it owns no write to roll back.+ canonical_write_fingerprint: str | None = None++class WorkflowCopilotChat(BaseModel):model_config = ConfigDict(from_attributes=True)@@ -18,6 +39,15 @@ class WorkflowCopilotChat(BaseModel):workflow_permanent_id: str = Field(..., description="Workflow permanent ID for the chat")proposed_workflow: dict | None = Field(None, description="Latest workflow proposed by the copilot")auto_accept: bool | None = Field(False, description="Whether copilot auto-accepts workflow updates")+ pending_turns: dict[str, CopilotPendingTurn] = Field(+ default_factory=dict, description="In-flight turns keyed by turn id"+ )++ @field_validator("pending_turns", mode="before")+ @classmethod+ def _default_pending_turns(cls, value: dict[str, Any] | None) -> dict[str, Any]:+ return value or {}+created_at: datetime = Field(..., description="When the chat was created")modified_at: datetime = Field(..., description="When the chat was last modified")diff --git a/tests/unit/copilot/test_turn_origin_seam.py b/tests/unit/copilot/test_turn_origin_seam.pyindex 290916342..dd85d3d5e 100644--- a/tests/unit/copilot/test_turn_origin_seam.py+++ b/tests/unit/copilot/test_turn_origin_seam.py@@ -112,7 +112,10 @@ async def test_synthetic_session_unregisters_when_context_body_raises(async with mcp_browser_context(ctx):raise RuntimeError("boom")- unregister_mock.assert_called_once_with(make_self_heal_session_id("wr_777"))+ unregister_mock.assert_called_once_with(+ make_self_heal_session_id("wr_777"),+ organization_id=ctx.organization_id,+ )@pytest.mark.asyncio@@ -155,8 +158,9 @@ async def test_self_heal_injected_state_assigns_synthetic_session_and_mcp_registwith capture_logs() as logs:async with mcp_browser_context(ctx):assert register_mock.call_args.args[0] == expected_session_id+ assert register_mock.call_args.kwargs == {"organization_id": ctx.organization_id}- unregister_mock.assert_called_once_with(expected_session_id)+ unregister_mock.assert_called_once_with(expected_session_id, organization_id=ctx.organization_id)manager.get_browser_state.assert_not_awaited()registered = [entry for entry in logs if entry["event"] == "registered self-heal browser session"]unregistered = [entry for entry in logs if entry["event"] == "unregistered self-heal browser session"]diff --git a/tests/unit/copilot_route_test_support.py b/tests/unit/copilot_route_test_support.pyindex 86cc01914..20ea15b8e 100644--- a/tests/unit/copilot_route_test_support.py+++ b/tests/unit/copilot_route_test_support.py@@ -56,6 +56,10 @@ def setup_new_copilot_mocks(agent_result.narrative_summary = Noneif not hasattr(agent_result, "narrative_payload"):agent_result.narrative_payload = None+ if not hasattr(original_workflow, "modified_at"):+ original_workflow.modified_at = datetime(2026, 4, 14, tzinfo=timezone.utc)+ if not hasattr(original_workflow, "model_dump"):+ original_workflow.model_dump = MagicMock(return_value={"workflow_id": original_workflow.workflow_id})async def fake_llm_handler(*args: object, **kwargs: object) -> None:del args, kwargs@@ -84,6 +88,11 @@ def setup_new_copilot_mocks(create_workflow_copilot_chat_message=AsyncMock(return_value=SimpleNamespace(created_at=datetime(2026, 4, 14, tzinfo=timezone.utc))),+ start_copilot_turn=AsyncMock(+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 14, tzinfo=timezone.utc))+ ),+ claim_pending_copilot_turn=AsyncMock(return_value=True),+ clear_pending_copilot_turn=AsyncMock(),)app.DATABASE.workflow_params = workflow_paramsapp.DATABASE.workflows = SimpleNamespace(diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.pyindex a2d115c9e..79a395517 100644--- a/tests/unit/test_copilot_session_injection.py+++ b/tests/unit/test_copilot_session_injection.py@@ -191,13 +191,14 @@ class TestMcpBrowserContextBridge:# that resolve_browser(session_id=...) relies on.args = register_mock.call_args.argsassert args[0] == ctx.browser_session_id+ assert register_mock.call_args.kwargs == {"organization_id": ctx.organization_id}registered_state = args[1]assert isinstance(registered_state, SessionState)assert registered_state.context.session_id == ctx.browser_session_idassert register_mock.call_count == 1assert unregister_mock.call_count == 1- unregister_mock.assert_called_with(ctx.browser_session_id)+ unregister_mock.assert_called_with(ctx.browser_session_id, organization_id=ctx.organization_id)# Override installed then reset.assert [c[0] for c in override_calls] == ["set", "reset"]diff --git a/tests/unit/test_mcp_http_auth.py b/tests/unit/test_mcp_http_auth.pyindex d1d789720..fd2f9ee49 100644--- a/tests/unit/test_mcp_http_auth.py+++ b/tests/unit/test_mcp_http_auth.py@@ -10,14 +10,17 @@ from unittest.mock import AsyncMock, Mockimport httpximport pytestfrom fastapi import HTTPException+from fastmcp import FastMCPfrom starlette.applications import Starlettefrom starlette.middleware import Middlewarefrom starlette.requests import Requestfrom starlette.responses import JSONResponsefrom starlette.routing import Route+from starlette.testclient import TestClientfrom skyvern.cli.core import client as client_mod-from skyvern.cli.core import mcp_http_auth+from skyvern.cli.core import mcp_http_auth, session_manager+from skyvern.cli.core.result import BrowserContext_TEST_BASE_URL = "http://testserver"@@ -25,6 +28,12 @@ _TEST_BASE_URL = "http://testserver"@pytest.fixture(autouse=True)def _reset_auth_context() -> None:client_mod._api_key_override.set(None)+ session_manager._current_session.set(None)+ session_manager._global_session = None+ session_manager._copilot_sessions.clear()+ session_manager._organization_sessions.clear()+ session_manager._current_organization_id.set(None)+ session_manager.set_stateless_http_mode(False)mcp_http_auth._auth_db = Nonemcp_http_auth._api_key_validation_cache.clear()mcp_http_auth._API_KEY_CACHE_TTL_SECONDS = 30.0@@ -264,6 +273,152 @@ async def test_mcp_http_auth_sets_request_scoped_api_key(monkeypatch: pytest.Monassert client_mod.get_active_api_key() != "sk_live_abc"+async def test_stateful_http_session_is_scoped_to_authenticated_organization_with_exact_session_id(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ session_id = "pbs_owned_session"+ first_browser = SimpleNamespace(owner="first")+ denied_connection = AsyncMock(side_effect=PermissionError("session is not owned by this organization"))+ monkeypatch.setattr(+ session_manager,+ "get_skyvern",+ lambda: SimpleNamespace(connect_to_cloud_browser_session=denied_connection),+ )++ async def session_endpoint(request: Request) -> JSONResponse:+ payload = await request.json()+ if payload["action"] == "seed":+ state = session_manager.SessionState(+ browser=first_browser,+ context=BrowserContext(mode="cloud_session", session_id=session_id),+ api_key_hash=session_manager.active_api_key_hash(),+ )+ session_manager.set_current_session(state)+ session_manager.register_copilot_session(session_id, state, organization_id="org_first")+ return JSONResponse({"owner": first_browser.owner})++ try:+ browser, _ = await session_manager.resolve_browser(session_id=payload["session_id"])+ except PermissionError:+ return JSONResponse({"error": "forbidden"}, status_code=403)+ return JSONResponse({"owner": browser.owner})++ app = Starlette(+ routes=[Route("/mcp", endpoint=session_endpoint, methods=["POST"])],+ middleware=[Middleware(mcp_http_auth.MCPAPIKeyMiddleware)],+ )+ shared_api_key = "sk_shared_test_key"+ validate_oauth_token = AsyncMock(+ side_effect=[+ SimpleNamespace(validation=_build_validation("org_first"), api_key=shared_api_key),+ SimpleNamespace(validation=_build_validation("org_second"), api_key=shared_api_key),+ ]+ )+ monkeypatch.setattr(mcp_http_auth, "validate_mcp_oauth_token", validate_oauth_token)++ first_response = await _request(+ app,+ "POST",+ "/mcp",+ headers={"authorization": f"Bearer {_jwtish_token(payload={'sub': 'first'})}"},+ json={"action": "seed"},+ )+ second_response = await _request(+ app,+ "POST",+ "/mcp",+ headers={"authorization": f"Bearer {_jwtish_token(payload={'sub': 'second'})}"},+ json={"action": "resolve", "session_id": session_id},+ )++ assert first_response.status_code == 200+ assert second_response.status_code == 403+ assert denied_connection.await_count == 1+ assert session_manager._current_session.get() is None+++def test_stateful_fastmcp_transport_rejects_exact_session_id_from_another_organization(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ server = FastMCP("session-owner-test")++ @server.tool+ async def session_owner() -> str:+ return session_manager._current_organization_id.get() or "missing"++ validations = {+ "sk_org_first": _build_validation("org_first"),+ "sk_org_first_rotated": _build_validation("org_first"),+ "sk_org_second": _build_validation("org_second"),+ }++ async def validate_api_key(api_key: str) -> mcp_http_auth.MCPAPIKeyValidation:+ return validations[api_key]++ monkeypatch.setattr(mcp_http_auth, "validate_mcp_api_key", validate_api_key)+ app = server.http_app(+ path="/",+ middleware=[Middleware(mcp_http_auth.MCPAPIKeyMiddleware)],+ stateless_http=False,+ json_response=True,+ )+ initialize_request = {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "initialize",+ "params": {+ "protocolVersion": "2025-06-18",+ "capabilities": {},+ "clientInfo": {"name": "test", "version": "1"},+ },+ }+ tool_request = {+ "jsonrpc": "2.0",+ "id": 2,+ "method": "tools/call",+ "params": {"name": "session_owner", "arguments": {}},+ }+ headers = {+ "accept": "application/json, text/event-stream",+ "content-type": "application/json",+ }++ with TestClient(app) as client:+ initialize_response = client.post(+ "/",+ headers={**headers, "x-api-key": "sk_org_first"},+ json=initialize_request,+ )+ session_id = initialize_response.headers["mcp-session-id"]+ rotated_credential_response = client.post(+ "/",+ headers={+ **headers,+ "x-api-key": "sk_org_first_rotated",+ "mcp-session-id": session_id,+ },+ json=tool_request,+ )+ hijack_response = client.post(+ "/",+ headers={+ **headers,+ "x-api-key": "sk_org_second",+ "mcp-session-id": session_id,+ },+ json=tool_request,+ )++ assert initialize_response.status_code == 200+ assert rotated_credential_response.status_code == 404+ assert rotated_credential_response.json()["error"]["message"] == "Session not found"+ assert hijack_response.status_code == 404+ assert hijack_response.json()["error"]["message"] == "Session not found"+ assert session_manager._current_session.get() is None+ assert session_manager._current_organization_id.get() is None++@pytest.mark.asyncioasync def test_validate_mcp_api_key_uses_ttl_cache(monkeypatch: pytest.MonkeyPatch) -> None:calls = 0diff --git a/tests/unit/test_mcp_session_lifecycle.py b/tests/unit/test_mcp_session_lifecycle.pyindex 830cbb04c..c3b9f6141 100644--- a/tests/unit/test_mcp_session_lifecycle.py+++ b/tests/unit/test_mcp_session_lifecycle.py@@ -31,8 +31,12 @@ def _reset_singletons() -> None:client_mod._api_key_clients.clear()session_manager._current_session.set(None)+ session_manager._current_organization_id.set(None)session_manager._global_session = None+ session_manager._organization_sessions.clear()session_manager._copilot_sessions.clear()+ session_manager._session_ref_maps.clear()+ session_manager._session_ref_generations.clear()session_manager.set_stateless_http_mode(False)@@ -375,24 +379,34 @@ async def test_resolve_browser_classifies_explicit_cloud_session_localhost_reach@pytest.mark.asyncio-async def test_resolve_browser_does_not_reuse_registered_copilot_session_for_different_api_key_override(+ ("organization_id", "api_key"),+ [(None, "sk_copilot_org"), ("org_copilot", "sk_other_org")],+)+async def test_resolve_browser_requires_matching_organization_and_credential_for_registered_copilot_session(monkeypatch: pytest.MonkeyPatch,+ organization_id: str | None,+ api_key: str,) -> None:registered_state = session_manager.SessionState(browser=MagicMock(),context=BrowserContext(mode="cloud_session", session_id="pbs_copilot"),api_key_hash=session_manager._api_key_hash("sk_copilot_org"),)- session_manager.register_copilot_session("pbs_copilot", registered_state)+ session_manager.register_copilot_session("pbs_copilot", registered_state, organization_id="org_copilot")fallback_browser = MagicMock()fake_skyvern = MagicMock()fake_skyvern.connect_to_cloud_browser_session = AsyncMock(return_value=fallback_browser)monkeypatch.setattr(session_manager, "get_skyvern", lambda: fake_skyvern)- token = client_mod.set_api_key_override("sk_other_org")+ token = client_mod.set_api_key_override(api_key)try:- browser, ctx = await session_manager.resolve_browser(session_id="pbs_copilot")+ if organization_id:+ with session_manager.request_session_scope(organization_id):+ browser, ctx = await session_manager.resolve_browser(session_id="pbs_copilot")+ else:+ browser, ctx = await session_manager.resolve_browser(session_id="pbs_copilot")finally:client_mod.reset_api_key_override(token)@@ -828,6 +842,78 @@ async def test_close_current_session_still_closes_browser_when_api_fails(monkeypassert session_manager.get_current_session().browser is None+async def test_close_all_sessions_closes_each_organization_cache(monkeypatch: pytest.MonkeyPatch) -> None:+ first_browser = MagicMock()+ first_browser.close = AsyncMock()+ second_error = RuntimeError("second organization cleanup failed")+ second_browser = MagicMock()+ second_browser.close = AsyncMock(side_effect=second_error)+ third_error = RuntimeError("third organization cleanup failed")+ third_browser = MagicMock()+ third_browser.close = AsyncMock(side_effect=third_error)++ with session_manager.request_session_scope("org_first"):+ session_manager.set_current_session(+ session_manager.SessionState(+ browser=first_browser,+ context=BrowserContext(mode="local"),+ )+ )+ with session_manager.request_session_scope("org_second"):+ session_manager.set_current_session(+ session_manager.SessionState(+ browser=second_browser,+ context=BrowserContext(mode="local"),+ )+ )+ with session_manager.request_session_scope("org_third"):+ session_manager.set_current_session(+ session_manager.SessionState(browser=third_browser, context=BrowserContext(mode="local"))+ )++ warning = MagicMock()+ monkeypatch.setattr(session_manager.LOG, "warning", warning)+ with pytest.raises(RuntimeError, match=str(second_error)):+ await session_manager.close_all_sessions()++ first_browser.close.assert_awaited_once()+ second_browser.close.assert_awaited_once()+ third_browser.close.assert_awaited_once()+ assert warning.call_args.kwargs["organization_id"] == "org_third"+ assert warning.call_args.kwargs["exc_info"][1] is third_error+ assert session_manager._organization_sessions == {}+ assert session_manager._current_session.get() is None+ assert session_manager._current_organization_id.get() is None+++def test_session_ref_lookup_is_scoped_to_authenticated_organization() -> None:+ session_id = "pbs_shared_identifier"+ page_key = (1, None, "https://example.com", None)+ element = {"tag": "button"}++ with session_manager.request_session_scope("org_first"):+ session_manager.set_current_session(+ session_manager.SessionState(+ context=BrowserContext(mode="cloud_session", session_id=session_id),+ )+ )+ assert session_manager.replace_session_ref_map(+ {"e0": element},+ session_id=session_id,+ page_key=page_key,+ )+ assert session_manager.get_session_ref("e0", session_id=session_id, page_key=page_key) == element++ with session_manager.request_session_scope("org_second"):+ session_manager.set_current_session(+ session_manager.SessionState(+ context=BrowserContext(mode="cloud_session", session_id=session_id),+ )+ )+ assert session_manager.get_session_ref("e0", session_id=session_id, page_key=page_key) is None++# ---------------------------------------------------------------------------# Tests for stateless HTTP mode session creation# ---------------------------------------------------------------------------diff --git a/tests/unit/test_run_commands_cleanup.py b/tests/unit/test_run_commands_cleanup.pyindex 1290a1e47..07ea77efc 100644--- a/tests/unit/test_run_commands_cleanup.py+++ b/tests/unit/test_run_commands_cleanup.py@@ -24,18 +24,18 @@ def _reset_cleanup_state() -> None:async def test_cleanup_mcp_resources_closes_auth_db(monkeypatch: pytest.MonkeyPatch) -> None:order: list[str] = []shutdown_action_log_worker = AsyncMock(side_effect=lambda: order.append("action_log"))- close_current_session = AsyncMock(side_effect=lambda: order.append("session"))+ close_all_sessions = AsyncMock(side_effect=lambda: order.append("session"))close_skyvern = AsyncMock(side_effect=lambda: order.append("client"))close_auth_db = AsyncMock(side_effect=lambda: order.append("auth"))monkeypatch.setattr("skyvern.cli.core.action_log.shutdown_action_log_worker", shutdown_action_log_worker)- monkeypatch.setattr("skyvern.cli.core.session_manager.close_current_session", close_current_session)+ monkeypatch.setattr("skyvern.cli.core.session_manager.close_all_sessions", close_all_sessions)monkeypatch.setattr("skyvern.cli.core.client.close_skyvern", close_skyvern)monkeypatch.setattr("skyvern.cli.core.mcp_http_auth.close_auth_db", close_auth_db)await run_commands._cleanup_mcp_resources()- close_current_session.assert_awaited_once()+ close_all_sessions.assert_awaited_once()close_skyvern.assert_awaited_once()close_auth_db.assert_awaited_once()assert order == ["action_log", "session", "client", "auth"]@@ -43,20 +43,20 @@ async def test_cleanup_mcp_resources_closes_auth_db(monkeypatch: pytest.MonkeyPa@pytest.mark.asyncioasync def test_cleanup_mcp_resources_closes_auth_db_on_skyvern_close_error(monkeypatch: pytest.MonkeyPatch) -> None:- close_current_session = AsyncMock()+ close_all_sessions = AsyncMock()close_auth_db = AsyncMock()async def _failing_close_skyvern() -> None:raise RuntimeError("close failed")- monkeypatch.setattr("skyvern.cli.core.session_manager.close_current_session", close_current_session)+ monkeypatch.setattr("skyvern.cli.core.session_manager.close_all_sessions", close_all_sessions)monkeypatch.setattr("skyvern.cli.core.client.close_skyvern", _failing_close_skyvern)monkeypatch.setattr("skyvern.cli.core.mcp_http_auth.close_auth_db", close_auth_db)with pytest.raises(RuntimeError, match="close failed"):await run_commands._cleanup_mcp_resources()- close_current_session.assert_awaited_once()+ close_all_sessions.assert_awaited_once()close_auth_db.assert_awaited_once()diff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.pyindex 2cd573a04..21cc8256e 100644--- a/tests/unit/test_workflow_copilot_cancel.py+++ b/tests/unit/test_workflow_copilot_cancel.py@@ -35,6 +35,8 @@ from skyvern.forge.sdk.copilot.agent import _build_exit_resultfrom skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext, StructuredContextfrom skyvern.forge.sdk.routes.workflow_copilot import (COPILOT_CANCEL_TTL,+ USER_CANCELLED_TERMINAL_REASON,+ _assistant_row_exists_for_turn,_copilot_cancel_key,_persist_cancel_turn,_persist_proposed_workflow_state,@@ -45,6 +47,7 @@ from skyvern.forge.sdk.routes.workflow_copilot import (from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotCancelRequest,WorkflowCopilotChatRequest,+ WorkflowCopilotChatSender,WorkflowCopilotStreamMessageType,)from tests.unit.copilot_route_test_support import install_fake_create, setup_new_copilot_mocks@@ -260,9 +263,10 @@ async def _drive_cancel_route(handler = captured["handler"]assert callable(handler)await handler(stream)- # Cancel turn always emits exactly two chat rows: the user prompt and the- # AI cancellation reply. Guard against a future regression that double-inserts.- assert workflow_params.create_workflow_copilot_chat_message.await_count == 2+ # The user prompt is written ahead of the agent by start_copilot_turn; the+ # cancel path adds only the AI reply. Guard against a double-insert of either.+ assert workflow_params.start_copilot_turn.await_count == 1+ assert workflow_params.create_workflow_copilot_chat_message.await_count == 1return restore_mock, workflow_params, sent_payloads@@ -291,13 +295,13 @@ async def test_route_cancel_branch_persists_user_and_cancelled_messages(restore_mock.assert_awaited_once()- # Two chat-message inserts: user msg + Cancelled by user. AI msg.insert_calls = workflow_params.create_workflow_copilot_chat_message.await_args_listsenders = [c.kwargs.get("sender") for c in insert_calls]contents = [c.kwargs.get("content") for c in insert_calls]- assert senders.count("user") == 1+ assert senders.count("user") == 0assert senders.count("ai") == 1- assert "please update" in contents+ assert workflow_params.start_copilot_turn.await_args.kwargs["user_message"] == "please update"+assert "Cancelled by user." in contents# No proposed_workflow update when the cancelled result has no WIP.@@ -391,7 +395,7 @@ async def test_route_cancel_wip_persists_proposal_and_response_frame(insert_calls = workflow_params.create_workflow_copilot_chat_message.await_args_listcontents = [c.kwargs.get("content") for c in insert_calls]- assert "please update" in contents+ assert workflow_params.start_copilot_turn.await_args.kwargs["user_message"] == "please update"assert user_response in contentsresponse_frames = [@@ -942,6 +946,8 @@ async def test_operational_cancel_does_not_persist_cancelled_message(title="Original",description=None,workflow_definition=None,+ modified_at=datetime(2026, 4, 27, tzinfo=timezone.utc),+ model_dump=MagicMock(return_value={"workflow_id": "wf-canonical"}),)# Agent raises CancelledError synchronously (simulates operational cancel# propagating into the await before user_cancel_observed gets set).@@ -970,6 +976,10 @@ async def test_operational_cancel_does_not_persist_cancelled_message(create_workflow_copilot_chat_message=AsyncMock(return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))),+ start_copilot_turn=AsyncMock(+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))+ ),+ clear_pending_copilot_turn=AsyncMock(),)app.DATABASE.workflow_params = workflow_paramsapp.DATABASE.workflows = SimpleNamespace(@@ -1004,3 +1014,56 @@ async def test_operational_cancel_does_not_persist_cancelled_message(contents = [c.kwargs.get("content") for c in insert_calls]# No "Cancelled by user." row was written.assert "Cancelled by user." not in contents+++async def test_pre_agent_cancel_row_is_visible_to_the_turn_idempotence_check() -> None:+ """Without a turn-stamped outcome, reconcile would append an interrupted row beside the cancel."""+ chat = SimpleNamespace(+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ proposed_workflow=None,+ auto_accept=False,+ )+ persisted: list[SimpleNamespace] = []++ async def capture_message(**kwargs: Any) -> SimpleNamespace:+ row = SimpleNamespace(+ sender=kwargs["sender"],+ content=kwargs["content"],+ turn_outcome=kwargs.get("turn_outcome"),+ created_at=datetime(2026, 4, 27, tzinfo=timezone.utc),+ )+ persisted.append(row)+ return row++ workflow_params = SimpleNamespace(+ update_workflow_copilot_chat=AsyncMock(),+ create_workflow_copilot_chat_message=capture_message,+ clear_pending_copilot_turn=AsyncMock(),+ get_workflow_copilot_chat_messages=AsyncMock(return_value=persisted),+ )+ app.DATABASE.workflow_params = workflow_params++ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)++ await _persist_cancel_turn(+ stream=stream,+ chat=chat,+ organization_id="org-1",+ original_workflow=None,+ user_message="keep going",+ agent_result=None,+ turn_id="turn-a",+ user_row_already_persisted=True,+ )++ assistant_rows = [row for row in persisted if row.sender == WorkflowCopilotChatSender.AI]+ assert len(assistant_rows) == 1+ outcome = assistant_rows[0].turn_outcome+ assert outcome is not None+ assert outcome.copilot_turn_id == "turn-a"+ assert outcome.terminal_reason == USER_CANCELLED_TERMINAL_REASON+ assert outcome.terminal_reason != "interrupted"+ assert await _assistant_row_exists_for_turn(chat, "turn-a") is Truediff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 6cb55df91..4c5e04b7c 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -15,7 +15,8 @@ real database -- all DB / LLM / agent surfaces are patched.from __future__ import annotations-from datetime import datetime, timezone+import contextlib+from datetime import datetime, timedelta, timezonefrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock@@ -30,23 +31,36 @@ from skyvern.forge.agent_functions import AgentFunctionfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot import agent as agent_module-from skyvern.forge.sdk.copilot.context import AgentResult-from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope+from skyvern.forge.sdk.copilot.canonical_ownership import workflow_content_fingerprint+from skyvern.forge.sdk.copilot.context import AgentResult, TurnNarrativePayload+from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS+from skyvern.forge.sdk.copilot.terminal_envelope import (+ INTERRUPTED_TERMINAL_MESSAGE,+ INTERRUPTED_TERMINAL_REASON,+ assemble_terminal_envelope,+)from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_routefrom skyvern.forge.sdk.routes.workflow_copilot import (COPILOT_V2_FLAG_KEY,+ RECONCILE_ABANDON_AFTER_SECONDS,+ _persist_turn_messages,_validate_copilot_audio_artifact_id,convert_to_history_messages,workflow_copilot_chat_audio,+ workflow_copilot_chat_history,workflow_copilot_chat_post,)+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcomefrom skyvern.forge.sdk.schemas.workflow_copilot import (+ CopilotPendingTurn,+ WorkflowCopilotChat,WorkflowCopilotChatMessage,WorkflowCopilotChatRequest,WorkflowCopilotChatSender,WorkflowCopilotStreamErrorUpdate,WorkflowCopilotStreamResponseUpdate,)+from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinitionfrom tests.unit.copilot_route_test_support import install_fake_create, setup_new_copilot_mocks@@ -1305,8 +1319,8 @@ async def test_flag_on_route_error_after_chat_persists_recoverable_reply(call.kwargs.get("content")for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list]- assert "Please update it" in contents- assistant_contents = [content for content in contents if isinstance(content, str) and content != "Please update it"]+ assert app.DATABASE.workflow_params.start_copilot_turn.await_args.kwargs["user_message"] == "Please update it"+ assistant_contents = [content for content in contents if isinstance(content, str)]assert len(assistant_contents) == 1assert expected_summary in assistant_contents[0]assert "The workflow was not modified" in assistant_contents[0]@@ -2774,3 +2788,708 @@ async def test_persist_state_auto_applicable_without_staged_commit_still_protect)app.DATABASE.workflow_params.update_workflow_copilot_chat.assert_not_awaited()+++_NOW = datetime(2026, 7, 29, 12, 0, tzinfo=timezone.utc)+++def _make_copilot_workflow(title: str, modified_at: datetime) -> Workflow:+ return Workflow(+ workflow_id="wf-1",+ organization_id="org-1",+ title=title,+ workflow_permanent_id="wpid-1",+ version=1,+ is_saved_task=False,+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),+ created_at=_NOW - timedelta(days=1),+ modified_at=modified_at,+ )+++def _fingerprint_of(workflow: Workflow) -> str:+ return workflow_content_fingerprint(workflow.model_dump(mode="json"))+++def _make_pending_turn(turn_id: str, age_seconds: float, **overrides: Any) -> CopilotPendingTurn:+ return CopilotPendingTurn(+ turn_id=turn_id,+ started_at=datetime.now(timezone.utc) - timedelta(seconds=age_seconds),+ **overrides,+ )+++def _make_persisted_chat(+ pending: list[CopilotPendingTurn], proposed_workflow: dict | None = None+) -> WorkflowCopilotChat:+ return WorkflowCopilotChat(+ workflow_copilot_chat_id="chat-1",+ organization_id="org-1",+ workflow_permanent_id="wpid-1",+ proposed_workflow=proposed_workflow,+ auto_accept=False,+ pending_turns={entry.turn_id: entry for entry in pending},+ created_at=_NOW,+ modified_at=_NOW,+ )+++class _FakeCopilotChatStore:+ """In-memory double of the chat/message repo surface reconcile-on-read touches."""++ def __init__(self, chat: WorkflowCopilotChat) -> None:+ self.chat = chat+ self.messages: list[WorkflowCopilotChatMessage] = []+ self.claim_calls: list[str] = []++ def add_message(+ self,+ sender: WorkflowCopilotChatSender,+ content: str,+ turn_outcome: TurnOutcome | None = None,+ ) -> WorkflowCopilotChatMessage:+ message = WorkflowCopilotChatMessage(+ workflow_copilot_chat_message_id=f"wccm-{len(self.messages)}",+ workflow_copilot_chat_id=self.chat.workflow_copilot_chat_id,+ sender=sender,+ content=content,+ turn_outcome=turn_outcome,+ created_at=_NOW,+ modified_at=_NOW,+ )+ self.messages.append(message)+ return message++ @property+ def assistant_messages(self) -> list[WorkflowCopilotChatMessage]:+ return [m for m in self.messages if m.sender == WorkflowCopilotChatSender.AI]++ @property+ def user_messages(self) -> list[WorkflowCopilotChatMessage]:+ return [m for m in self.messages if m.sender == WorkflowCopilotChatSender.USER]++ async def get_workflow_copilot_chat_by_id(+ self, organization_id: str, workflow_copilot_chat_id: str+ ) -> WorkflowCopilotChat:+ return self.chat++ async def get_workflow_copilot_chat_messages(+ self, workflow_copilot_chat_id: str+ ) -> list[WorkflowCopilotChatMessage]:+ return list(self.messages)++ async def create_workflow_copilot_chat_message(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ sender: WorkflowCopilotChatSender,+ content: str,+ audio_artifact_id: str | None = None,+ global_llm_context: str | None = None,+ turn_outcome: TurnOutcome | None = None,+ narrative_payload: Any = None,+ ) -> WorkflowCopilotChatMessage:+ return self.add_message(sender, content, turn_outcome)++ async def replace_workflow_copilot_chat_message(+ self,+ organization_id: str,+ workflow_copilot_chat_message_id: str,+ content: str,+ global_llm_context: str | None,+ turn_outcome: TurnOutcome | None,+ narrative_payload: TurnNarrativePayload | None = None,+ ) -> WorkflowCopilotChatMessage | None:+ for index, message in enumerate(self.messages):+ if message.workflow_copilot_chat_message_id == workflow_copilot_chat_message_id:+ replaced = message.model_copy(update={"content": content, "turn_outcome": turn_outcome})+ self.messages[index] = replaced+ return replaced+ return None++ async def record_pending_copilot_turn_canonical_write(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ fingerprint: str,+ ) -> None:+ entry = self.chat.pending_turns.get(turn_id)+ if entry is not None:+ entry.canonical_write_fingerprint = fingerprint++ async def claim_pending_copilot_turn(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ claim_before: datetime,+ ) -> bool:+ self.claim_calls.append(turn_id)+ entry = self.chat.pending_turns.get(turn_id)+ if entry is None:+ return False+ if entry.recovering_at is not None and entry.recovering_at > claim_before:+ return False+ entry.recovering_at = datetime.now(timezone.utc)+ return True++ async def clear_pending_copilot_turn(+ self, organization_id: str, workflow_copilot_chat_id: str, turn_id: str+ ) -> None:+ self.chat.pending_turns.pop(turn_id, None)++ async def update_workflow_copilot_chat(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ proposed_workflow: dict | None = None,+ auto_accept: bool | None = None,+ ) -> WorkflowCopilotChat:+ self.chat.proposed_workflow = proposed_workflow+ return self.chat+++def _install_reconcile_store(+ monkeypatch: pytest.MonkeyPatch,+ chat: WorkflowCopilotChat,+ canonical: Workflow | None = None,+) -> tuple[_FakeCopilotChatStore, AsyncMock]:+ store = _FakeCopilotChatStore(chat)+ app.DATABASE.workflow_params = store+ app.DATABASE.workflows = SimpleNamespace(get_workflow_by_permanent_id=AsyncMock(return_value=canonical))+ restore_mock = AsyncMock()+ monkeypatch.setattr(workflow_copilot_route, "_restore_workflow_definition", restore_mock)+ return store, restore_mock+++async def _load_history(chat_id: str = "chat-1") -> Any:+ return await workflow_copilot_chat_history(+ workflow_copilot_chat_id=chat_id,+ organization=SimpleNamespace(organization_id="org-1"),+ )+++async def test_chat_history_marks_abandoned_turn_interrupted_not_cancelled(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A turn whose process died leaves the user row plus an honest interrupted outcome."""+ chat = _make_persisted_chat([_make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS + 60)])+ store, restore_mock = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ response = await _load_history()++ assert [m.content for m in store.user_messages] == ["build me a scraper"]+ assert len(store.assistant_messages) == 1+ outcome = store.assistant_messages[0].turn_outcome+ assert outcome is not None+ assert outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON+ assert outcome.terminal_reason != "cancel"+ assert outcome.response_kind == ResponseKind.RECOVER+ assert outcome.copilot_turn_id == "turn-a"+ assert chat.pending_turns == {}+ assert response.chat_history[-1].turn_outcome is not None+ restore_mock.assert_not_awaited()+++async def test_chat_history_leaves_a_young_pending_turn_alone(monkeypatch: pytest.MonkeyPatch) -> None:+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 30)])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ await _load_history()++ assert store.assistant_messages == []+ assert "turn-a" in chat.pending_turns+++async def test_chat_history_shows_the_user_message_while_the_turn_is_still_running(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Reloading mid-turn — a tab refresh — shows the user's message without inventing an outcome."""+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 30)])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ response = await _load_history()++ assert [m.content for m in response.chat_history] == ["build me a scraper"]+ assert all(m.turn_outcome is None for m in response.chat_history)+ assert "turn-a" in chat.pending_turns+++async def test_chat_history_leaves_a_turn_inside_its_enforcement_budget_alone(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Past the 900s agent budget but still live in wall clock thanks to a credential pause."""+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 1000)])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ await _load_history()++ assert store.assistant_messages == []+ assert "turn-a" in chat.pending_turns+++async def test_reconcile_restores_real_cdp_headers_not_the_masked_snapshot(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The pre-turn snapshot is a JSON dump, which masks cdp_connect_headers; restoring it verbatim would destroy them."""+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))+ pre_turn.cdp_connect_headers = {"x-api-key": "real-secret"}+ mid_turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1500))+ mid_turn_draft.cdp_connect_headers = {"x-api-key": "real-secret"}+ chat = _make_persisted_chat(+ [+ _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 60,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ canonical_write_fingerprint=_fingerprint_of(mid_turn_draft),+ )+ ]+ )+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=mid_turn_draft)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ await _load_history()++ restore_mock.assert_awaited_once()+ assert restore_mock.await_args.args[0].cdp_connect_headers == {"x-api-key": "real-secret"}+++async def test_reconcile_does_not_stash_a_canonical_the_dying_turn_already_restored(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A finalizer that restored canonical just before the kill leaves no displaced draft to stash."""+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))+ turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1600))+ already_restored = _make_copilot_workflow("Before the turn", datetime.now(timezone.utc) - timedelta(seconds=1500))+ chat = _make_persisted_chat(+ [+ _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 60,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ canonical_write_fingerprint=_fingerprint_of(turn_draft),+ )+ ]+ )+ store, _ = _install_reconcile_store(monkeypatch, chat, canonical=already_restored)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ response = await _load_history()++ assert response.proposed_workflow is None+++async def test_reconcile_restores_canonical_and_stashes_the_mid_turn_draft(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))+ mid_turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1500))+ chat = _make_persisted_chat(+ [+ _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 60,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ canonical_write_fingerprint=_fingerprint_of(mid_turn_draft),+ )+ ]+ )+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=mid_turn_draft)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ response = await _load_history()++ restore_mock.assert_awaited_once()+ restored_workflow = restore_mock.await_args.args[0]+ assert restored_workflow.title == "Before the turn"+ assert restored_workflow.workflow_definition == pre_turn.workflow_definition+ assert response.proposed_workflow is not None+ assert response.proposed_workflow["title"] == "Half-built draft"+++async def test_reconcile_keeps_an_unowned_canonical_change_rather_than_undoing_it(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """An auto-accept commit records no ownership; undoing it would discard work the user opted to apply."""+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))+ committed = _make_copilot_workflow("Auto-accepted build", datetime.now(timezone.utc) - timedelta(seconds=1500))+ chat = _make_persisted_chat(+ [+ _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 60,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ )+ ]+ )+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=committed)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ response = await _load_history()++ restore_mock.assert_not_awaited()+ assert response.proposed_workflow is None+ assert len(store.assistant_messages) == 1+++async def test_reconcile_leaves_canonical_alone_when_the_turn_never_wrote(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ pre_turn = _make_copilot_workflow("Untouched", _NOW - timedelta(hours=2))+ chat = _make_persisted_chat(+ [+ _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 60,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ )+ ]+ )+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=pre_turn)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ response = await _load_history()++ restore_mock.assert_not_awaited()+ assert response.proposed_workflow is None+ assert len(store.assistant_messages) == 1+++async def test_reconcile_leaves_canonical_alone_when_someone_else_wrote_after_the_turn(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A hand edit landing after the turn's own write forfeits the turn's rollback claim."""+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(days=2))+ turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1600))+ later_edit = _make_copilot_workflow("Edited by hand afterwards", datetime.now(timezone.utc))+ chat = _make_persisted_chat(+ [+ _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS * 3,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ canonical_write_fingerprint=_fingerprint_of(turn_draft),+ )+ ]+ )+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=later_edit)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ response = await _load_history()++ restore_mock.assert_not_awaited()+ assert response.proposed_workflow is None+++async def test_reconcile_keeps_an_earlier_proposal_when_keep_pending_was_requested(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))+ mid_turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1500))+ earlier_proposal = {"workflow_id": "wf-1", "title": "Earlier proposal"}+ chat = _make_persisted_chat(+ [+ _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 60,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ canonical_write_fingerprint=_fingerprint_of(mid_turn_draft),+ pre_turn_proposed_workflow=earlier_proposal,+ keep_pending_proposal=True,+ )+ ],+ proposed_workflow=earlier_proposal,+ )+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=mid_turn_draft)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ response = await _load_history()++ restore_mock.assert_awaited_once()+ assert response.proposed_workflow == earlier_proposal+++async def test_reconcile_completes_after_a_crash_mid_recovery(monkeypatch: pytest.MonkeyPatch) -> None:+ """A recovery that died after claiming the turn is reclaimed and still yields one reply."""+ entry = _make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS * 3)+ entry.recovering_at = datetime.now(timezone.utc) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS * 2)+ chat = _make_persisted_chat([entry])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ await _load_history()++ assert len(store.assistant_messages) == 1+ assert chat.pending_turns == {}+++async def test_reconcile_skips_a_turn_another_reader_is_already_recovering(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ entry = _make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS + 60)+ entry.recovering_at = datetime.now(timezone.utc)+ chat = _make_persisted_chat([entry])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ await _load_history()++ assert store.assistant_messages == []+ assert "turn-a" in chat.pending_turns+++async def test_reconcile_handles_two_overlapping_abandoned_turns(monkeypatch: pytest.MonkeyPatch) -> None:+ chat = _make_persisted_chat(+ [+ _make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS + 120),+ _make_pending_turn("turn-b", RECONCILE_ABANDON_AFTER_SECONDS + 60),+ ]+ )+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "first ask")+ store.add_message(WorkflowCopilotChatSender.USER, "second ask")++ await _load_history()++ recovered_turn_ids = [m.turn_outcome.copilot_turn_id for m in store.assistant_messages if m.turn_outcome]+ assert sorted(recovered_turn_ids) == ["turn-a", "turn-b"]+ assert chat.pending_turns == {}+++async def test_reconcile_drops_the_marker_of_a_turn_that_already_replied(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A finalised turn whose marker clear failed must not be rolled back or answered twice."""+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))+ committed = _make_copilot_workflow("Accepted build", datetime.now(timezone.utc) - timedelta(seconds=1500))+ chat = _make_persisted_chat(+ [+ _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 60,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ )+ ]+ )+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=committed)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")+ store.add_message(+ WorkflowCopilotChatSender.AI,+ "Done.",+ TurnOutcome(response_kind=ResponseKind.BUILD, copilot_turn_id="turn-a"),+ )++ response = await _load_history()++ assert len(store.assistant_messages) == 1+ assert chat.pending_turns == {}+ restore_mock.assert_not_awaited()+ assert response.proposed_workflow is None+++async def test_persist_turn_messages_is_idempotent_for_one_turn(monkeypatch: pytest.MonkeyPatch) -> None:+ """A second finalizer pass for the same turn adds no duplicate user or assistant row."""+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 10)])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")++ for _ in range(2):+ await _persist_turn_messages(+ chat=chat,+ turn_id="turn-a",+ user_message="build me a scraper",+ audio_artifact_id=None,+ user_row_already_persisted=True,+ assistant_content="Here is your workflow.",+ global_llm_context=None,+ turn_outcome=TurnOutcome(response_kind=ResponseKind.BUILD),+ narrative_payload=None,+ )++ assert len(store.user_messages) == 1+ assert len(store.assistant_messages) == 1+ assert store.assistant_messages[0].turn_outcome is not None+ assert store.assistant_messages[0].turn_outcome.copilot_turn_id == "turn-a"+++async def test_a_finished_turn_replaces_the_interrupted_row_rather_than_dropping_its_reply(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Recovery can reach a slow-but-live turn first; when that turn finishes, its reply is the truth."""+ chat = _make_persisted_chat([_make_pending_turn("turn-a", 10)])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")+ store.add_message(+ WorkflowCopilotChatSender.AI,+ INTERRUPTED_TERMINAL_MESSAGE,+ TurnOutcome(+ response_kind=ResponseKind.RECOVER,+ reason_code=INTERRUPTED_TERMINAL_REASON,+ terminal_reason=INTERRUPTED_TERMINAL_REASON,+ copilot_turn_id="turn-a",+ ),+ )++ await _persist_turn_messages(+ chat=chat,+ turn_id="turn-a",+ user_message="build me a scraper",+ audio_artifact_id=None,+ user_row_already_persisted=True,+ assistant_content="Here is your workflow.",+ global_llm_context=None,+ turn_outcome=TurnOutcome(response_kind=ResponseKind.BUILD),+ narrative_payload=None,+ )++ assert len(store.assistant_messages) == 1+ assert store.assistant_messages[0].content == "Here is your workflow."+ outcome = store.assistant_messages[0].turn_outcome+ assert outcome is not None+ assert outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON+++def test_reconcile_threshold_outlasts_the_turn_enforcement_ceiling() -> None:+ assert RECONCILE_ABANDON_AFTER_SECONDS > TOTAL_TIMEOUT_SECONDS+++async def test_marker_survives_a_finalizer_that_raises_with_no_assistant_row(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """No assistant row was written, so the turn stays recoverable instead of being orphaned."""+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)+ captured = install_fake_create(monkeypatch)++ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow=None,+ auto_accept=False,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description="Original description",+ workflow_definition=None,+ )+ agent_result = SimpleNamespace(+ user_response="Here is your workflow.",+ updated_workflow=None,+ global_llm_context=None,+ workflow_yaml=None,+ workflow_was_persisted=False,+ clear_proposed_workflow=False,+ unvalidated=False,+ turn_outcome=None,+ )+ setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)+ monkeypatch.setattr(+ workflow_copilot_route,+ "_persist_turn_messages",+ AsyncMock(side_effect=RuntimeError("database down")),+ )++ request = MagicMock()+ request.headers = {"x-api-key": "sk-test-key"}++ await workflow_copilot_chat_post(request, _make_chat_request(), SimpleNamespace(organization_id="org-1"))+ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ stream.is_disconnected = AsyncMock(return_value=False)+ handler = captured["handler"]+ assert callable(handler)+ with contextlib.suppress(RuntimeError):+ await handler(stream)++ assert app.DATABASE.workflow_params.start_copilot_turn.await_count == 1+ app.DATABASE.workflow_params.clear_pending_copilot_turn.assert_not_awaited()+++async def test_reconcile_writes_no_outcome_when_the_canonical_restore_fails(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A transient restore failure must not persist a 'recovered' turn over a stranded canonical."""+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))+ mid_turn_draft = _make_copilot_workflow("Half-built draft", datetime.now(timezone.utc) - timedelta(seconds=1500))+ entry = _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 60,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ canonical_write_fingerprint=_fingerprint_of(mid_turn_draft),+ )+ chat = _make_persisted_chat([entry])+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=mid_turn_draft)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")+ restore_mock.side_effect = RuntimeError("restore failed")++ await _load_history()++ assert store.assistant_messages == []+ assert "turn-a" in chat.pending_turns++ restore_mock.side_effect = None+ entry.recovering_at = None++ await _load_history()++ assert len(store.assistant_messages) == 1+ assert store.assistant_messages[0].turn_outcome is not None+ assert store.assistant_messages[0].turn_outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON+ assert chat.pending_turns == {}+++async def test_reconcile_does_not_roll_back_a_later_turns_canonical_write(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """An older abandoned turn must not attribute — and undo — a newer turn's canonical edit."""+ pre_turn = _make_copilot_workflow("Before the turn", _NOW - timedelta(hours=2))+ later_write = _make_copilot_workflow(+ "Written by the later turn", datetime.now(timezone.utc) - timedelta(seconds=1350)+ )+ older_draft = _make_copilot_workflow("Older turn draft", datetime.now(timezone.utc) - timedelta(seconds=1500))+ older = _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 180,+ pre_turn_workflow=pre_turn.model_dump(mode="json"),+ canonical_write_fingerprint=_fingerprint_of(older_draft),+ )+ newer = _make_pending_turn("turn-b", RECONCILE_ABANDON_AFTER_SECONDS + 80)+ chat = _make_persisted_chat([older, newer])+ store, restore_mock = _install_reconcile_store(monkeypatch, chat, canonical=later_write)+ store.add_message(WorkflowCopilotChatSender.USER, "first ask")+ store.add_message(WorkflowCopilotChatSender.USER, "second ask")++ response = await _load_history()++ restore_mock.assert_not_awaited()+ recovered = sorted(m.turn_outcome.copilot_turn_id for m in store.assistant_messages if m.turn_outcome)+ assert recovered == ["turn-a", "turn-b"]+ assert chat.pending_turns == {}+ assert response.proposed_workflow is None