Skip to content

Automate browser based workflows with AI

Log an archived S3 object download at warning without a traceback (#8884)

Shuchang Zhenge22ecd624510
diff --git a/skyvern/cli/core/session_manager.py b/skyvern/cli/core/session_manager.py
index 14616ef21..517ce39e0 100644
--- a/skyvern/cli/core/session_manager.py
+++ b/skyvern/cli/core/session_manager.py
@@ -10,7 +10,7 @@ from collections import deque
from contextlib import asynccontextmanager, contextmanager, suppress
from contextvars import ContextVar
from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, AsyncIterator, Callable, Iterator
+from typing import TYPE_CHECKING, Any, AsyncIterator, Callable, Iterator, TypedDict
import httpx
import structlog
@@ -31,7 +31,7 @@ LOG = structlog.get_logger(__name__)
_BODY_SEMAPHORE_LIMIT = 5 # concurrent CDP body downloads (worst case: 5 * 10s timeout = 50s backlog)
if TYPE_CHECKING:
- from playwright.async_api import Frame, Page
+ from playwright.async_api import Frame, Page, Request
from skyvern.browser_extension.runtime import BrowserExtensionRuntime
from skyvern.library.skyvern_browser import SkyvernBrowser
@@ -56,6 +56,55 @@ class ObserveV2State:
self.refs = {}
+class RequestFailure(TypedDict):
+ url: str
+ method: str
+ resource_type: str
+ failure_text: str
+ page_url: str
+ tab_id: str
+ timestamp: float
+
+
+_MAX_REQUEST_FAILURES_PER_TAB = 20
+
+
+def newest_per_failure_text_first(entries: list[RequestFailure], limit: int) -> list[RequestFailure]:
+ """Up to ``limit`` entries in time order: the newest of each failure text first, then the newest of the rest,
+ so a burst of one kind of failure cannot displace a different one."""
+ seen: set[str] = set()
+ firsts: list[RequestFailure] = []
+ rest: list[RequestFailure] = []
+ for entry in reversed(entries):
+ (rest if entry["failure_text"] in seen else firsts).append(entry)
+ seen.add(entry["failure_text"])
+ kept = {id(entry) for entry in (firsts + rest)[:limit]}
+ return [entry for entry in entries if id(entry) in kept]
+
+
+@dataclass
+class RequestFailureLog:
+ # Keyed by tab so a noisy background tab cannot evict the working tab's failures.
+ entries: dict[str, list[RequestFailure]] = field(default_factory=dict)
+ # Several SessionStates can hook the same page, so each failed request is recorded once.
+ seen: weakref.WeakSet[Request] = field(default_factory=weakref.WeakSet)
+
+ def append(self, entry: RequestFailure) -> None:
+ tab = self.entries.setdefault(entry["tab_id"], [])
+ tab.append(entry)
+ if len(tab) > _MAX_REQUEST_FAILURES_PER_TAB:
+ tab[:] = newest_per_failure_text_first(tab, _MAX_REQUEST_FAILURES_PER_TAB)
+
+ def take_tab(self, tab_id: str) -> list[RequestFailure]:
+ return self.entries.pop(tab_id, [])
+
+ def restore(self, taken: list[RequestFailure]) -> None:
+ if taken:
+ tab_id = taken[0]["tab_id"]
+ merged = [*taken, *self.entries.get(tab_id, [])]
+ self.entries[tab_id] = newest_per_failure_text_first(merged, _MAX_REQUEST_FAILURES_PER_TAB)
+
+
@dataclass
class SessionState:
browser: SkyvernBrowser | None = None
@@ -66,6 +115,7 @@ class SessionState:
network_requests: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=1000))
dialog_events: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=1000))
page_errors: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=1000))
+ request_failures: RequestFailureLog = field(default_factory=RequestFailureLog)
tracing_active: bool = False
har_enabled: bool = False
_har_entries: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=5000))
diff --git a/skyvern/cli/mcp_tools/inspection.py b/skyvern/cli/mcp_tools/inspection.py
index a3468c6fb..2b77b4b33 100644
--- a/skyvern/cli/mcp_tools/inspection.py
+++ b/skyvern/cli/mcp_tools/inspection.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio
import time
from datetime import datetime, timedelta, timezone
-from typing import Annotated, Any, Literal
+from typing import TYPE_CHECKING, Annotated, Any, Literal
from urllib.parse import parse_qs, parse_qsl, urlencode, urlparse, urlsplit, urlunsplit
import structlog
@@ -31,6 +31,11 @@ from ._element_state import DEFAULT_ACTION_TIMEOUT_MS, action_deadline_error
from ._read_retry import retry_read_on_page_change
from ._session import BrowserNotAvailableError, get_current_session, get_page, no_browser_error
+if TYPE_CHECKING:
+ from playwright.async_api import Page, Request
+
+ from skyvern.cli.core.session_manager import RequestFailure
+
# Query param keys whose values are redacted from captured URLs.
_SECRET_QUERY_PARAMS = frozenset(
{
@@ -228,6 +233,9 @@ async def _capture_body(response: Any, request_id: int, state: Any) -> None:
state._body_store[request_id] = safe_body
+_RECORDED_REQUEST_FAILURE_TYPES = frozenset({"document", "xhr", "fetch"})
+
+
def _make_page_handlers(state: Any, raw_page: Any) -> dict[str, Any]:
"""Create console/network/dialog/pageerror handlers bound to a specific page."""
from skyvern.cli.core.session_manager import is_stateless_http_mode
@@ -378,6 +386,38 @@ def _make_page_handlers(state: Any, raw_page: Any) -> dict[str, Any]:
except BaseException:
pass
+ def _on_requestfailed(request: Request) -> None:
+ try:
+ if request.resource_type not in _RECORDED_REQUEST_FAILURE_TYPES or raw_page.is_closed():
+ return
+ log = state.request_failures
+ if request in log.seen:
+ return
+ log.seen.add(request)
+ failure: RequestFailure = {
+ "url": _redact_url(request.url),
+ "method": request.method,
+ "resource_type": request.resource_type,
+ "failure_text": request.failure or "",
+ "page_url": _redact_url(raw_page.url),
+ "tab_id": str(id(raw_page)),
+ "timestamp": time.time(),
+ }
+ entry = _redact_inspection_value(state, failure)
+ if type(entry) is dict:
+ # tab_id is an internal key the model never sees; redacting it would file the entry under no tab.
+ entry["tab_id"] = failure["tab_id"]
+ log.append(entry)
+ except BaseException:
+ pass
+
+ def _on_close(_page: Page) -> None:
+ # Python reuses a closed page's id() for a later page, which would inherit these entries.
+ try:
+ state.request_failures.take_tab(str(id(raw_page)))
+ except BaseException:
+ pass
+
def _body_capture_done(task: asyncio.Task[None], request_id: int, url: str) -> None:
if task.cancelled():
return
@@ -443,7 +483,14 @@ def _make_page_handlers(state: Any, raw_page: Any) -> dict[str, Any]:
except BaseException:
pass
- return {"console": _on_console, "response": _on_response, "dialog": _on_dialog, "pageerror": _on_pageerror}
+ return {
+ "console": _on_console,
+ "response": _on_response,
+ "requestfailed": _on_requestfailed,
+ "close": _on_close,
+ "dialog": _on_dialog,
+ "pageerror": _on_pageerror,
+ }
def _register_hooks_on_page(state: Any, raw_page: Any) -> None:
@@ -455,6 +502,8 @@ def _register_hooks_on_page(state: Any, raw_page: Any) -> None:
handlers = _make_page_handlers(state, raw_page)
raw_page.on("console", handlers["console"])
raw_page.on("response", handlers["response"])
+ raw_page.on("requestfailed", handlers["requestfailed"])
+ raw_page.on("close", handlers["close"])
raw_page.on("dialog", handlers["dialog"])
raw_page.on("pageerror", handlers["pageerror"])
state._hooked_page_ids.add(page_id)
diff --git a/skyvern/forge/sdk/api/aws.py b/skyvern/forge/sdk/api/aws.py
index f850db360..775d33837 100644
--- a/skyvern/forge/sdk/api/aws.py
+++ b/skyvern/forge/sdk/api/aws.py
@@ -49,6 +49,8 @@ def _recording_content_type(uri_or_path: str) -> str | None:
_S3_OPERATION_RETRIES = 2
# get_object on a missing key raises NoSuchKey; head-style paths use 404/NotFound.
S3_NOT_FOUND_ERROR_CODES = frozenset({"NoSuchKey", "NotFound", "404"})
+# get_object on an object in an archive tier (GLACIER, DEEP_ARCHIVE) that has not been restored.
+S3_ARCHIVED_OBJECT_ERROR_CODES = frozenset({"InvalidObjectState"})
# Expired AWS credentials surface under either exact code depending on the service/path.
_EXPIRED_TOKEN_ERROR_CODES = frozenset({"ExpiredTokenException", "ExpiredToken"})
# Aborting an upload id that no longer exists (e.g. an ambiguous Complete actually landed) returns this; it
@@ -470,6 +472,13 @@ class AsyncAWSClient:
isinstance(error, ClientError) and error.response.get("Error", {}).get("Code") in S3_NOT_FOUND_ERROR_CODES
)
+ def _is_archived_object_error(self, error: Exception) -> bool:
+ """Check if an exception means the object exists but sits in an archive tier and cannot be read."""
+ return (
+ isinstance(error, ClientError)
+ and error.response.get("Error", {}).get("Code") in S3_ARCHIVED_OBJECT_ERROR_CODES
+ )
+
def _error_code(self, error: Exception) -> str:
if isinstance(error, ClientError):
return error.response.get("Error", {}).get("Code", error.__class__.__name__)
@@ -1338,6 +1347,11 @@ class AsyncAWSClient:
if log_exception:
LOG.info("S3 object not found", uri=uri)
return None
+ # An archived object stays unreadable until restored, so a traceback per request adds nothing.
+ if self._is_archived_object_error(e):
+ if log_exception:
+ LOG.warning("S3 object is archived and not restored", uri=uri)
+ return None
if log_exception:
LOG.exception("S3 download failed", uri=uri)
return None
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 910ac12bb..c96de11d6 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -252,7 +252,7 @@ from skyvern.forge.sdk.copilot.tools.run_execution import (
from skyvern.forge.sdk.copilot.tools.scouting import (
_release_scout_challenge_listeners,
hydrate_prior_carried_trajectory,
- read_page_state,
+ read_page_state_on_page,
)
from skyvern.forge.sdk.copilot.tools.workflow_update import (
_candidate_proposal_data,
@@ -2136,6 +2136,7 @@ def _make_agent_result(
result.resolved_model = ctx.resolved_model
result.proposal_owner_turn_id = ctx.proposal_owner_turn_id
result.proposal_revision = ctx.proposal_revision
+ result.proposal_adopted = ctx.proposal_adopted
result.proposal_workflow_run_id = ctx.proposal_workflow_run_id
result.clear_persisted_completion_contract = ctx.clear_persisted_completion_contract
result.work_plan = ctx.work_plan if ctx.work_plan is None else list(ctx.work_plan)
@@ -3801,6 +3802,7 @@ async def _translate_to_agent_result(
LOG.info(
"copilot_inline_replace_candidate_publication_conflict",
workflow_permanent_id=ctx.workflow_permanent_id,
+ reason=publication_conflict.reason,
error=str(publication_conflict),
)
user_response = _with_inline_reject_note(
@@ -4230,7 +4232,7 @@ async def _run_agent_loop_with_surface(
dispatch_allowlist_enforced(ctx.tool_surface_identity)
or ctx.turn_origin == TurnOrigin.code_block_ai_fallback
),
- page_state_reader=read_page_state,
+ page_state_reader=read_page_state_on_page,
)
ctx.discovery_mcp_server = mcp_server
agent = Agent(
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 838ca5749..6d41b0d00 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -1189,6 +1189,8 @@ class AgentResult:
has_staged_proposal: bool = False
proposal_owner_turn_id: str | None = None
proposal_revision: int | None = None
+ # The turn took over another turn's stored candidate and authored nothing since, so it has no turn-end write.
+ proposal_adopted: bool = False
proposal_workflow_run_id: str | None = None
code_artifact_metadata: dict[str, dict[str, Any]] | None = None
executed_block_fingerprints: dict[str, set[str]] = field(default_factory=dict)
@@ -1477,6 +1479,11 @@ class CopilotContext(AgentContext):
proposal_mutation_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False)
proposal_owner_turn_id: str | None = None
proposal_revision: int | None = None
+ # A token this turn adopted but did not author never authorizes a turn-end write.
+ proposal_adopted: bool = False
+ proposal_adoptions: int = 0
+ # Adoptions the latest model response had seen; its tool calls were written before any later one.
+ proposal_adoptions_seen_by_model: int = 0
proposal_canonical_fingerprint: str | None = None
proposal_workflow_run_id: str | None = None
# A restored candidate whose request private settings differ from its own; its stored bytes cannot vouch
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 689f67d00..3f7591ecf 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -111,6 +111,10 @@ from skyvern.forge.sdk.copilot.turn_halt import (
raise_if_turn_halt,
stash_turn_halt_from_blocker_signal,
)
+from skyvern.forge.sdk.copilot.unrecoverable_tool_error import (
+ PAGE_FACT_KEYS,
+ PAGE_REQUEST_FAILURES_KEY,
+)
from skyvern.forge.sdk.copilot.unrecoverable_tool_error import (
CopilotUnrecoverableToolError as CopilotUnrecoverableToolError,
)
@@ -817,10 +821,10 @@ def _summarize_tool_output(output: str) -> str:
if not isinstance(parsed, dict):
return _truncated_output_fallback(output)
- # page_state rides on every browser result; it must not be what pushes a short result into compaction.
- if isinstance(parsed.get("page_state"), dict):
- without_page_state = {key: value for key, value in parsed.items() if key != "page_state"}
- if len(json.dumps(without_page_state)) <= _TOOL_OUTPUT_SUMMARIZE_THRESHOLD:
+ # Page facts ride on browser results; they must not be what pushes a short result into compaction.
+ if parsed.keys() & PAGE_FACT_KEYS:
+ without_page_facts = {key: value for key, value in parsed.items() if key not in PAGE_FACT_KEYS}
+ if len(json.dumps(without_page_facts)) <= _TOOL_OUTPUT_SUMMARIZE_THRESHOLD:
return output
# An ask_user result is the user's own answer; reduced to {"ok": true}, the model re-asks what
@@ -846,6 +850,9 @@ def _summarize_tool_output(output: str) -> str:
synopsis["error"] = str(parsed["error"])[:200]
if isinstance(parsed.get("page_state"), dict):
synopsis["page_state"] = _summarize_page_state(parsed["page_state"])
+ request_failures = parsed.get(PAGE_REQUEST_FAILURES_KEY)
+ if isinstance(request_failures, list) and request_failures:
+ synopsis[PAGE_REQUEST_FAILURES_KEY] = request_failures
data = parsed.get("data")
# Session continuation can compact the same output again. Re-bound the
diff --git a/skyvern/forge/sdk/copilot/hooks.py b/skyvern/forge/sdk/copilot/hooks.py
index 187a18549..5ac2af55c 100644
--- a/skyvern/forge/sdk/copilot/hooks.py
+++ b/skyvern/forge/sdk/copilot/hooks.py
@@ -120,6 +120,7 @@ class CopilotRunHooks(RunHooksBase):
async def on_llm_end(self, context: RunContextWrapper, agent: Agent, response: ModelResponse) -> None:
self._ctx.model_call_in_flight = False
self._ctx.last_model_response = response
+ self._ctx.proposal_adoptions_seen_by_model = self._ctx.proposal_adoptions
if self._ctx.check_model_work_deadline is not None:
self._ctx.check_model_work_deadline()
try:
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 4beb8fc1b..ae5dcf987 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -5,14 +5,18 @@ from __future__ import annotations
import asyncio
import json
import logging
+import posixpath
import re
import time
+import unicodedata
from collections.abc import AsyncIterator, Awaitable, Callable
from contextlib import AsyncExitStack, asynccontextmanager
+from contextvars import ContextVar
from copy import deepcopy
from dataclasses import dataclass, field, replace
from datetime import timedelta
-from typing import TYPE_CHECKING, Any, Literal, Protocol, cast
+from typing import TYPE_CHECKING, Any, Literal, NotRequired, Protocol, TypedDict, cast
+from urllib.parse import urlsplit, urlunsplit
import structlog
from agents.agent import AgentBase
@@ -29,11 +33,16 @@ from mcp.types import (
ListPromptsResult,
TextContent,
)
-from playwright.async_api import Browser, BrowserContext
+from playwright.async_api import Browser, BrowserContext, Page
from skyvern.cli.core.client import reset_api_key_override, set_api_key_override
from skyvern.cli.core.result import ErrorCode
-from skyvern.cli.core.session_manager import request_session_scope
+from skyvern.cli.core.session_manager import (
+ RequestFailure,
+ RequestFailureLog,
+ newest_per_failure_text_first,
+ request_session_scope,
+)
from skyvern.cli.mcp_tools.response import MCP_MAX_RESPONSE_CHARS
from skyvern.forge import app
from skyvern.forge.sdk.copilot.blocker_signal import (
@@ -45,7 +54,13 @@ from skyvern.forge.sdk.copilot.context import USER_FACING_REASON_PARAM
from skyvern.forge.sdk.copilot.enforcement import requested_output_paths_for_derivation
from skyvern.forge.sdk.copilot.hooks import _copilot_log_fields
from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
-from skyvern.forge.sdk.copilot.output_utils import mark_mcp_result_untrusted_for_llm, sanitize_tool_result_for_llm
+from skyvern.forge.sdk.copilot.output_utils import (
+ _RECENT_TOOL_OUTPUT_CHAR_CAP,
+ mark_mcp_result_untrusted_for_llm,
+ sanitize_tool_result_for_llm,
+ screened_recorded_url,
+)
+from skyvern.forge.sdk.copilot.page_identity import safe_page_origin
from skyvern.forge.sdk.copilot.pending_operation import pending_operation
from skyvern.forge.sdk.copilot.runtime import (
_BROWSER_BOOT_WAIT_SECONDS,
@@ -60,6 +75,7 @@ from skyvern.forge.sdk.copilot.runtime import (
browser_session_turn,
close_browser_session_quietly,
current_call_browser_session_override,
+ effective_browser_session_id,
ensure_browser_session,
mcp_browser_context,
mcp_to_copilot,
@@ -73,11 +89,15 @@ from skyvern.forge.sdk.copilot.screenshot_utils import (
enqueue_screenshot_from_result,
)
from skyvern.forge.sdk.copilot.secret_scrub import (
+ encoded_secret_variants,
is_registered_scrub_value,
matching_origin_run_redaction_parameters,
+ registered_scrub_values,
scrub_secrets_from_structure,
+ scrub_secrets_from_text,
)
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
+from skyvern.forge.sdk.copilot.unrecoverable_tool_error import PAGE_REQUEST_FAILURES_KEY
from skyvern.forge.sdk.workflow.models.block import parameter_strings
from skyvern.utils.contained_effects import contained_effect
from skyvern.webeye.browser_retirement import BrowserRetirementReason
@@ -115,10 +135,14 @@ class PageStateReader(Protocol):
result: dict[str, Any],
binding: BrowserSessionBinding,
probe: bool,
- ) -> PageState: ...
+ ) -> tuple[PageState, Page | None]: ...
_SHARED_BROWSER_OUTCOME_TOOLS = frozenset({"skyvern_evaluate", "skyvern_screenshot"})
+_REPORTED_REQUEST_METHODS = frozenset({"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"})
+_MAX_REPORTED_REQUEST_FAILURES = 5
+_MAX_REPORTED_FAILURE_TEXT_CHARS = 200
+_MAX_REPORTED_URL_CHARS = 300
_BrowserCallErrorKind = Literal["tool", "protocol"]
_BrowserSessionLossDisposition = Literal["reestablished", "generation_retired", "failed"]
@@ -192,6 +216,207 @@ def _browser_error_code(raw_result: dict[str, Any]) -> str | None:
return code if isinstance(code, str) and code else None
+class ReportedRequestFailure(TypedDict):
+ method: str
+ resource_type: str
+ failure_text: str
+ url: NotRequired[str]
+ page_url: NotRequired[str]
+
+
+def _reported_url(url: str) -> str | None:
+ # The query goes first because _redact_url's masks would make the screen withhold the whole URL.
+ try:
+ path_only = urlunsplit(urlsplit(url)._replace(query="", fragment=""))
+ except ValueError:
+ return None
+ screened = screened_recorded_url(path_only)[0]
+ return screened if screened is not None and len(screened) <= _MAX_REPORTED_URL_CHARS else None
+
+
+def _reported_request_url(url: str, page_url: str) -> str | None:
+ # A cross-origin path can carry a token the model has no use for, so only its origin is shown.
+ if (screened := _reported_url(url)) is None or (origin := safe_page_origin(screened)) is None:
+ return None
+ return (urlsplit(screened).path or "/") if origin == safe_page_origin(page_url) else origin.rstrip("/")
+
+
+def _screened_request_failure(entry: RequestFailure) -> ReportedRequestFailure:
+ # The method and failure text are page-influenced strings, so both are bounded before the model sees them.
+ method = entry["method"].upper()
+ reported = ReportedRequestFailure(
+ method=method if method in _REPORTED_REQUEST_METHODS else "OTHER",
+ resource_type=entry["resource_type"],
+ failure_text=entry["failure_text"][:_MAX_REPORTED_FAILURE_TEXT_CHARS],
+ )
+ if (url := _reported_request_url(entry["url"], entry["page_url"])) is not None:
+ reported["url"] = url
+ if (page_url := _reported_url(entry["page_url"])) is not None:
+ reported["page_url"] = page_url
+ return reported
+
+
+def reported_request_failures(entries: list[RequestFailure]) -> list[ReportedRequestFailure]:
+ return [
+ _screened_request_failure(entry)
+ for entry in newest_per_failure_text_first(entries, _MAX_REPORTED_REQUEST_FAILURES)
+ ]
+
+
+# Chromium serializes URLs and net error names as ASCII with quotes percent-encoded, so JSON escaping adds nothing.
+_LARGEST_REPORTED_REQUEST_FAILURE_CHARS = len(
+ json.dumps(
+ ReportedRequestFailure(
+ method="OPTIONS",
+ resource_type="document",
+ failure_text="f" * _MAX_REPORTED_FAILURE_TEXT_CHARS,
+ url="u" * _MAX_REPORTED_URL_CHARS,
+ page_url="u" * _MAX_REPORTED_URL_CHARS,
+ )
+ )
+)
+
+
+def _refit_scrubbed_failure(entry: dict[str, str]) -> dict[str, str] | None:
+ # Scrubbing can lengthen a field after its bound was checked, so a URL that no longer fits is left out.
+ fitted = {
+ key: value
+ for key, value in entry.items()
+ if key not in ("url", "page_url") or len(value) <= _MAX_REPORTED_URL_CHARS
+ }
+ return fitted if len(json.dumps(fitted)) <= _LARGEST_REPORTED_REQUEST_FAILURE_CHARS else None
+
+
+def _secret_url_markers(ctx: AgentContext) -> set[str]:
+ """Lowercased forms a registered value can take in a URL Chromium recorded: the value, every form of the part
+ before its first ``?`` or ``#``, its compatibility mapping in a host, and what dot-segment removal leaves of it."""
+ values = registered_scrub_values(ctx)
+ heads = {head for value in values if (head := value.partition("?")[0].partition("#")[0]) and head != value}
+ markers = {value.lower() for value in values} | {
+ form.lower() for head in heads for form in encoded_secret_variants(head)
+ }
+ for text in (*values, *heads):
+ if not text.isascii():
+ markers.add(unicodedata.normalize("NFKC", text).casefold())
+ if "/" in text and "." in text:
+ markers.add(posixpath.normpath(f"/{text}").lstrip("/").lower())
+ markers.discard("")
+ return markers
+
+
+def _with_decoded_host(url: str) -> str:
+ # Chromium punycode-encodes a whole non-ASCII host label, so a secret inside one is matched only once decoded.
+ try:
+ host = urlsplit(url).hostname
+ except ValueError:
+ return url
+ if not host or "xn--" not in host:
+ return url
+ labels = []
+ for label in host.split("."):
+ if label.startswith("xn--"):
+ try:
+ label = label[4:].encode("ascii").decode("punycode")
+ except UnicodeError:
+ pass
+ labels.append(label)
+ return url.replace(host, ".".join(labels), 1)
+
+
+def _scrubbed_url(ctx: AgentContext, url: str, secret_markers: set[str]) -> str:
+ # Scrubbed whole, before the report drops the query and fragment: a secret can span the path and the query.
+ scrubbed = scrub_secrets_from_text(ctx, url)
+ # What the exact scrub cannot match: a host Chromium lowercased or punycode-encoded, or a secret cut at
+ # its ? or # (a masked query value, or a fragment the browser never sends).
+ lowered = _with_decoded_host(scrubbed).lower()
+ return "" if any(marker in lowered for marker in secret_markers) else scrubbed
+
+
+def _with_scrubbed_urls(ctx: AgentContext, entry: RequestFailure, secret_markers: set[str]) -> RequestFailure:
+ scrubbed = entry.copy()
+ scrubbed["url"] = _scrubbed_url(ctx, entry["url"], secret_markers)
+ scrubbed["page_url"] = _scrubbed_url(ctx, entry["page_url"], secret_markers)
+ return scrubbed
+
+
+@dataclass
+class HeldRequestFailures:
+ taken: list[tuple[RequestFailureLog, Page, list[RequestFailure]]] = field(default_factory=list)
+
+ def take(self, ctx: AgentContext, page: Page | None) -> list[ReportedRequestFailure]:
+ session_id = effective_browser_session_id(ctx)
+ log = ctx.request_failure_logs_by_browser_session.get(session_id) if session_id else None
+ if log is None or page is None:
+ return []
+ taken = log.take_tab(str(id(page)))
+ if not taken:
+ return []
+ self.taken.append((log, page, taken))
+ secret_markers = _secret_url_markers(ctx)
+ return reported_request_failures([_with_scrubbed_urls(ctx, entry, secret_markers) for entry in taken])
+
+ def restore(self) -> None:
+ for log, page, taken in reversed(self.taken):
+ # A closed tab's id() can be reused by a later page, which would inherit these entries.
+ if not page.is_closed():
+ log.restore(taken)
+ self.taken.clear()
+
+ def release(self) -> None:
+ self.taken.clear()
+
+
+# Set while a model-facing post-hook runs, so a hook that stamps its own page_state drains into the call's hold.
+POST_HOOK_HELD_REQUEST_FAILURES: ContextVar[HeldRequestFailures | None] = ContextVar(
+ "post_hook_held_request_failures", default=None
+)
+
+
+def _json_chars(value: dict[str, Any]) -> int:
+ return len(json.dumps(value, default=str))
+
+
+def stamp_page_facts(
+ ctx: AgentContext,
+ held: HeldRequestFailures | None,
+ result: dict[str, Any],
+ page_state: PageState,
+ page: Page | None,
+) -> dict[str, Any]:
+ """``result`` led by ``page_state`` and the failed requests of the tab it was read from, taken into ``held``.
+ A page_state read from no tab, or a call with no ``held``, drains nothing."""
+ stamped: dict[str, Any] = {"page_state": page_state, **result}
+ if held is None or not (failures := held.take(ctx, page)):
+ return stamped
+ reported = scrub_model_facing_tool_result(ctx, {PAGE_REQUEST_FAILURES_KEY: failures}).get(PAGE_REQUEST_FAILURES_KEY)
+ fitted = [f for entry in reported if (f := _refit_scrubbed_failure(entry))] if isinstance(reported, list) else []
+ with_failures = {"page_state": page_state, PAGE_REQUEST_FAILURES_KEY: fitted, **result}
+ # Failures never push a result past the recent-output cap, where its tail would be cut off.
+ if fitted and (
+ _json_chars(with_failures) <= _RECENT_TOOL_OUTPUT_CHAR_CAP
+ or _json_chars(stamped) > _RECENT_TOOL_OUTPUT_CHAR_CAP
+ ):
+ return with_failures
+ held.restore()
+ return stamped
+
+
+def _scrub_stamped_tool_result(
+ ctx: AgentContext, held: HeldRequestFailures, result: Any, *, tool_name: str
+) -> dict[str, Any]:
+ # Failures must not be what withholds a result that passes without them, so they go back instead.
+ if (
+ isinstance(result, dict)
+ and PAGE_REQUEST_FAILURES_KEY in result
+ and _scrub_model_facing_tool_result(ctx, result) == "redaction_budget"
+ ):
+ del result[PAGE_REQUEST_FAILURES_KEY]
+ scrubbed = scrub_model_facing_tool_result(ctx, result, tool_name=tool_name)
+ if PAGE_REQUEST_FAILURES_KEY not in scrubbed:
+ held.restore()
+ return scrubbed
+
+
def _screenshot_reference(raw_tool_name: str, raw_result: dict[str, Any]) -> tuple[bool, str | None]:
if raw_tool_name != "skyvern_screenshot":
return False, None
@@ -1677,13 +1902,19 @@ class SkyvernOverlayMCPServer(MCPServer):
if binding is None:
binding = resolve_browser_session_binding(copilot_ctx, arguments or {})
+ held_failures = HeldRequestFailures()
+
async def _stamp_page_state(result: dict[str, Any], *, probe: bool = True) -> None:
reader = self._page_state_reader
if reader is None or not result or not overlay.requires_browser or "page_state" in result:
return
+ # Failures an abandoned result took go back, so this one reports the tab it reads.
+ held_failures.restore()
# First, so a head-truncated copy of a long result still carries it.
- page_state = await reader(copilot_ctx, tool_name=tool_name, result=result, binding=binding, probe=probe)
- stamped = {"page_state": page_state, **result}
+ page_state, page = await reader(
+ copilot_ctx, tool_name=tool_name, result=result, binding=binding, probe=probe
+ )
+ stamped = stamp_page_facts(copilot_ctx, held_failures, result, page_state, page)
result.clear()
result.update(stamped)
@@ -1946,6 +2177,7 @@ class SkyvernOverlayMCPServer(MCPServer):
await evidence_stack.enter_async_context(browser_evidence_commit_lock(copilot_ctx))
base_copilot_result = deepcopy(copilot_result)
ctx_snapshot = _snapshot_post_hook_context(copilot_ctx)
+ held_token = POST_HOOK_HELD_REQUEST_FAILURES.set(held_failures)
try:
copilot_result = await overlay.post_hook(copilot_result, raw_mcp, copilot_ctx)
except asyncio.CancelledError:
@@ -1970,10 +2202,12 @@ class SkyvernOverlayMCPServer(MCPServer):
}
else:
copilot_result = base_copilot_result
+ finally:
+ POST_HOOK_HELD_REQUEST_FAILURES.reset(held_token)
if isinstance(copilot_result, dict):
await _stamp_page_state(copilot_result)
- copilot_result = scrub_model_facing_tool_result(copilot_ctx, copilot_result, tool_name=mcp_name)
+ copilot_result = _scrub_stamped_tool_result(copilot_ctx, held_failures, copilot_result, tool_name=mcp_name)
if browser_outcome is not None and is_redaction_withheld(copilot_result):
browser_outcome = browser_outcome.as_redaction_withheld()
@@ -2030,24 +2264,25 @@ class SkyvernOverlayMCPServer(MCPServer):
try:
async with browser_scope:
pre_hook_result = await _run_pre_hook()
- if pre_hook_result is not None:
- return pre_hook_result
- phases.enter("dispatch")
- dispatch_started = True
- try:
- raw_result = await asyncio.wait_for(
- self._client.call_tool(mcp_name, mcp_args, raise_on_error=False),
- timeout=overlay.timeout,
+ # A refusal is returned only after the scope exits, which can still fail and
+ # abandon it; its failures stay held until then.
+ if pre_hook_result is None:
+ phases.enter("dispatch")
+ dispatch_started = True
+ try:
+ raw_result = await asyncio.wait_for(
+ self._client.call_tool(mcp_name, mcp_args, raise_on_error=False),
+ timeout=overlay.timeout,
+ )
+ except BaseException:
+ phases.unwind()
+ raise
+ projected_result, timing_raw_mcp, failed, commit_evidence = await _project_result(
+ raw_result,
+ defer_evidence=True,
+ evidence_lock_held=True,
)
- except BaseException:
- phases.unwind()
- raise
- projected_result, timing_raw_mcp, failed, commit_evidence = await _project_result(
- raw_result,
- defer_evidence=True,
- evidence_lock_held=True,
- )
- phases.enter("context_exit")
+ phases.enter("context_exit")
except (asyncio.CancelledError, CopilotBrowserGenerationRetired):
_restore_post_hook_context(copilot_ctx, lease_context_snapshot)
raise
@@ -2055,6 +2290,9 @@ class SkyvernOverlayMCPServer(MCPServer):
# The click post hook is skipped on session loss, timeout, and errors, and
# the viewport bytes the pre hook held for it must not outlive this call.
copilot_ctx.pending_scout_click_pre_frame = None
+ if pre_hook_result is not None:
+ held_failures.release()
+ return pre_hook_result
commit_evidence()
phases.settle()
_log_mcp_timing(
@@ -2066,6 +2304,7 @@ class SkyvernOverlayMCPServer(MCPServer):
"model",
"error" if failed else "ok",
)
+ held_failures.release()
return projected_result
else:
phases.enter("dispatch")
@@ -2217,7 +2456,10 @@ class SkyvernOverlayMCPServer(MCPServer):
err = _scrub_tool_exception(copilot_ctx, tool_name, exc)
await _stamp_page_state(err)
record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, err)
+ held_failures.release()
return _copilot_to_call_tool_result(err, tool_name)
+ finally:
+ held_failures.restore()
async def call_internal_tool(
self,
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 020615b1b..ccabc6380 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -26,6 +26,7 @@ from skyvern.cli.core.client import (
)
from skyvern.cli.core.result import BrowserContext as MCPBrowserContext
from skyvern.cli.core.session_manager import (
+ RequestFailureLog,
SessionState,
register_copilot_session,
scoped_session,
@@ -477,6 +478,9 @@ class AgentContext:
sdk_action_workflow_run_ids_by_browser_session: dict[SdkActionWorkflowRunCacheKey, str] = field(
default_factory=dict
)
+ # Shared by this turn's per-dispatch SessionStates, whose page listeners keep writing between calls. Those listeners
+ # keep a log alive past the turn, but only this turn reads it.
+ request_failure_logs_by_browser_session: dict[str, RequestFailureLog] = field(default_factory=dict)
browser_session_recovery_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
browser_session_recovery_owner: asyncio.Task[Any] | None = field(default=None, init=False, repr=False)
browser_session_recovery_depth: int = field(default=0, init=False, repr=False)
@@ -1531,6 +1535,9 @@ async def _mcp_browser_context_impl(
context=mcp_ctx,
api_key_hash=hash_api_key_for_cache(active_key) if active_key else None,
organization_id=ctx.organization_id,
+ request_failures=ctx.request_failure_logs_by_browser_session.setdefault(
+ browser_session_id, RequestFailureLog()
+ ),
)
if working_page is not None:
# The browser state's pin outlives this per-call SessionState, so every tool call
diff --git a/skyvern/forge/sdk/copilot/secret_scrub.py b/skyvern/forge/sdk/copilot/secret_scrub.py
index 85dd57468..9f7b66244 100644
--- a/skyvern/forge/sdk/copilot/secret_scrub.py
+++ b/skyvern/forge/sdk/copilot/secret_scrub.py
@@ -76,10 +76,14 @@ def _owned_scrub_values(context: object) -> set[str]:
# encodeURIComponent leaves these literal where Python's quote does not.
_ENCODE_URI_COMPONENT_SAFE = "-_.!~*'()"
+# Chromium leaves these literal when it serializes a URL path, so a value in a path segment keeps them.
+_CHROMIUM_URL_PATH_SAFE = "!$%&'()*+,/:;=@[]"
+# Before serializing a path, Chromium turns a backslash into a slash and drops tabs and newlines.
+_CHROMIUM_URL_PATH_REWRITES = str.maketrans({"\\": "/", "\t": None, "\n": None, "\r": None})
def encoded_secret_variants(value: str) -> list[str]:
- """The forms a page can echo a value in: a URL query, escaped markup, a JSON or Python literal."""
+ """The forms a page can echo a value in: a URL query or path, escaped markup, a JSON or Python literal."""
try:
forms = [
value,
@@ -88,6 +92,7 @@ def encoded_secret_variants(value: str) -> list[str]:
quote(value),
quote(value, safe=""),
quote(value, safe=_ENCODE_URI_COMPONENT_SAFE),
+ quote(value.translate(_CHROMIUM_URL_PATH_REWRITES), safe=_CHROMIUM_URL_PATH_SAFE),
quote_plus(value, safe=""),
json.dumps(value, ensure_ascii=True)[1:-1],
repr(value)[1:-1],
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 63d74ab50..5f7031e52 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -58,6 +58,7 @@ from skyvern.forge.sdk.copilot.credential_pause import (
)
from skyvern.forge.sdk.copilot.enforcement import requested_output_paths_for_derivation
from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
+from skyvern.forge.sdk.copilot.mcp_adapter import HeldRequestFailures, stamp_page_facts
from skyvern.forge.sdk.copilot.output_extraction_plan import (
requested_output_designation_capability,
value_designation_probe_expression,
@@ -319,7 +320,7 @@ from .scouting import _record_scouted_interaction as _record_scouted_interaction
from .scouting import _register_scout_interaction_observation as _register_scout_interaction_observation
from .scouting import _resolve_scout_role_name as _resolve_scout_role_name
from .scouting import _role_name_from_selector as _role_name_from_selector
-from .scouting import read_page_state as read_page_state
+from .scouting import read_page_state_on_page as read_page_state_on_page
from .web_search import _search_web_impl as _search_web_impl
from .workflow_update import BlockObservationRef as BlockObservationRef
from .workflow_update import CodeArtifactMetadata as CodeArtifactMetadata
@@ -2414,14 +2415,15 @@ def _with_page_state(tool: FunctionTool) -> FunctionTool:
# An empty result is the fail-closed answer, and a stamp would turn it into a successful call.
if not isinstance(result, dict) or not result or "page_state" in result:
return output
- page_state = await read_page_state(
+ page_state, page = await read_page_state_on_page(
copilot_ctx,
tool_name=tool.name,
result=result,
binding=binding,
custody_lock=browser_session_turn(copilot_ctx, session_id=binding.session_id_for(copilot_ctx)),
)
- return json.dumps({"page_state": page_state, **result})
+ stamped = stamp_page_facts(copilot_ctx, HeldRequestFailures(), result, page_state, page)
+ return json.dumps(stamped)
return dataclasses.replace(tool, on_invoke_tool=invoke_with_page_state)
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index 7e168ed68..ce31abd66 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -13,6 +13,7 @@ from playwright.async_api import Page
from typing_extensions import TypedDict
from skyvern.cli.core.js_dispatch import outer_cap_seconds
+from skyvern.cli.core.session_manager import ensure_browser_hooks, get_current_session
from skyvern.forge import app
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
@@ -1091,6 +1092,9 @@ async def on_working_page(
page = await live_working_page(ctx)
if page is None:
return {"ok": False, "error": no_page_error}
+ # An MCP call attaches this turn's inspection listeners in get_page, which a native action skips.
+ if (browser := get_current_session().browser) is not None:
+ ensure_browser_hooks(browser)
return await act(page)
except (CopilotBrowserGenerationRetired, CopilotBrowserSessionUnavailable) as exc:
disposition = await _browser_session_error_disposition(ctx, exc, tool_name=tool_name, call_path="model")
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index f35a33336..b571ab7f4 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -40,9 +40,11 @@ from skyvern.forge.sdk.copilot.enforcement import (
from skyvern.forge.sdk.copilot.mcp_adapter import (
BROWSER_TARGET_PARAM,
BROWSER_TARGET_PARAM_NAME,
+ POST_HOOK_HELD_REQUEST_FAILURES,
PreHook,
SchemaOverlay,
scrub_model_facing_tool_result,
+ stamp_page_facts,
)
from skyvern.forge.sdk.copilot.output_extraction_plan import (
requested_output_designation_capability,
@@ -158,9 +160,10 @@ from .scouting import (
_shed_scout_page_summary_section,
_start_scout_challenge_settle,
_take_viewport_frame,
- read_page_state,
+ read_page_state_on_page,
record_signed_out_page_observation,
)
+from .workflow_update import proposal_is_displayable
LOG = structlog.get_logger()
@@ -1061,10 +1064,10 @@ async def _navigate_post_hook(
try:
navigated = await _navigate_post_hook_body(result, raw, ctx)
# The adapter refuses an unavailable binding before dispatch, and a post-hook runs only after one.
- page_state = await read_page_state(
+ page_state, page = await read_page_state_on_page(
ctx, tool_name="navigate_browser", result=navigated, binding=None, settle=True
)
- return {"page_state": page_state, **navigated}
+ return stamp_page_facts(ctx, POST_HOOK_HELD_REQUEST_FAILURES.get(), navigated, page_state, page)
finally:
ctx.pending_scout_challenge_frames = []
ctx.pending_scout_challenge_prior_frames = []
@@ -2173,7 +2176,11 @@ async def _chat_has_pending_proposal(ctx: AgentContext) -> bool:
organization_id=ctx.organization_id,
workflow_copilot_chat_id=chat_id,
)
- return chat is not None and isinstance(chat.proposed_workflow, dict)
+ return chat is not None and await proposal_is_displayable(
+ chat.proposed_workflow,
+ chat.workflow_permanent_id,
+ ctx.organization_id,
+ )
async def _create_workflow_schedule_pre_hook(
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 2d2a09726..523e0b304 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -234,6 +234,7 @@ from skyvern.forge.sdk.copilot.workflow_yaml import (
)
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.db.enums import BrowserSeedSource
+from skyvern.forge.sdk.db.exceptions import CopilotProposalConflictError, CopilotProposalConflictReason
from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
from skyvern.forge.sdk.schemas.copilot_turn_outcome import DeliveredOutputFile
from skyvern.forge.sdk.schemas.credentials import CredentialVaultType
@@ -341,7 +342,14 @@ from .guardrails import (
_placeholder_for_parameter_type,
)
from .scouting import _mark_post_run_page_observed, _redact_codeblock_value
-from .workflow_update import strip_copilot_yaml_headers
+from .workflow_update import (
+ ProposalAdoptionOutcome,
+ adopt_stored_workflow_candidate,
+ prior_draft_settings_match_stored_candidate,
+ proposal_conflict_result,
+ strip_copilot_yaml_headers,
+ unseen_adoption,
+)
LOG = structlog.get_logger()
@@ -363,7 +371,7 @@ RUN_BLOCKS_POLL_INTERVAL_SECONDS = 5.0
COPILOT_SANDBOX_UNAVAILABLE_ERROR = "Sandboxed worker is unavailable; execution was not started."
COPILOT_SNAPSHOT_PREPARATION_ERROR = "Unable to prepare the Copilot test-run snapshot; execution was not started."
-COPILOT_PROPOSAL_CHANGED_ERROR = "The pending Copilot proposal changed before the test started; reload and try again."
+_PROPOSAL_BIND_REFUSED_KEY = "proposal_bind_refused"
COPILOT_PROPOSAL_SETTINGS_CHANGED_ERROR = (
"This request's workflow settings differ from the pending Copilot proposal's; "
"re-save the proposal with the current settings, then test again."
@@ -1983,6 +1991,8 @@ class CopilotExecutionSnapshot:
workflow_parameters: tuple[WorkflowParameter, ...]
output_parameters: tuple[OutputParameter, ...]
workflow_yaml: str | None = None
+ # The stored candidate bytes a prior-draft run was parsed from; only the proposal bind compares them.
+ proposal_bind_yaml: str | None = None
@dataclass
@@ -2131,7 +2141,15 @@ async def _select_execution_snapshot(
if allow_prior_draft and not _workflow_has_blocks(workflow):
prior_draft = await _workflow_from_prior_draft(ctx, list(requested_labels))
if prior_draft is not None:
- return _declared_execution_snapshot(prior_draft)
+ snapshot = _declared_execution_snapshot(prior_draft)
+ settings_match = await prior_draft_settings_match_stored_candidate(ctx)
+ if settings_match:
+ snapshot = replace(snapshot, proposal_bind_yaml=ctx.prior_copilot_workflow_yaml)
+ elif (
+ settings_match is False and ctx.proposal_owner_turn_id is not None and ctx.proposal_revision is not None
+ ):
+ ctx.settings_diverged_proposal_token = (ctx.proposal_owner_turn_id, ctx.proposal_revision)
+ return snapshot
if workflow is None:
return None
@@ -3754,6 +3772,14 @@ async def _run_blocks_and_collect_debug(
reused_origin_output_labels=reused_origin_output_labels,
origin_workflow_run_id=ctx.repair_origin_outputs_run_id if reused_origin_output_labels else None,
)
+ # Same check as an edit's, synchronous with the token capture above: the labels and parameters were chosen
+ # against the draft an unseen adoption replaced.
+ if (adoption := unseen_adoption(ctx)) is not None:
+ LOG.info("copilot candidate run refused", reason="token_changed", adoption=adoption)
+ return {
+ **proposal_conflict_result(ctx, lost="The test was not run.", reason="token_changed", adoption=adoption),
+ "data": {"run_dispatched": False, _PROPOSAL_BIND_REFUSED_KEY: True},
+ }
if ctx.settings_diverged_proposal_token == (execution.proposal_owner_turn_id, execution.proposal_revision):
LOG.info(
"copilot candidate run refused",
@@ -4451,37 +4477,78 @@ async def _run_blocks_and_collect_debug(
and execution.proposal_owner_turn_id is not None
and execution.proposal_revision is not None
):
- try:
- async with ctx.proposal_mutation_lock:
+ bind_conflict: tuple[CopilotProposalConflictReason, ProposalAdoptionOutcome] | None = None
+ bind_unexpected = False
+ async with ctx.proposal_mutation_lock:
+ try:
await app.DATABASE.workflow_params.bind_workflow_copilot_candidate_run(
organization_id=ctx.organization_id,
workflow_copilot_chat_id=ctx.workflow_copilot_chat_id,
expected_owner_turn_id=execution.proposal_owner_turn_id,
expected_revision=execution.proposal_revision,
- expected_workflow_yaml=execution.workflow_yaml,
+ expected_workflow_yaml=execution.snapshot.proposal_bind_yaml or execution.workflow_yaml,
+ workflow_run_id=workflow_run.workflow_run_id,
+ )
+ except CopilotProposalConflictError as conflict:
+ # A token or source this call no longer shares with ctx belongs to a candidate the turn replaced.
+ ctx_holds_call_token = (ctx.proposal_owner_turn_id, ctx.proposal_revision) == (
+ execution.proposal_owner_turn_id,
+ execution.proposal_revision,
+ ) and execution.source_is_current(ctx)
+ bind_conflict = (
+ conflict.reason,
+ await adopt_stored_workflow_candidate(ctx) if ctx_holds_call_token else "kept",
+ )
+ except Exception: # noqa: BLE001 - any custody failure must stop dispatch.
+ bind_unexpected = True
+ LOG.warning(
+ "copilot candidate run bind failed",
workflow_run_id=workflow_run.workflow_run_id,
+ reason="unexpected_error",
+ exc_info=True,
)
+ else:
if (
ctx.proposal_owner_turn_id == execution.proposal_owner_turn_id
and ctx.proposal_revision == execution.proposal_revision
and execution.source_is_current(ctx)
):
ctx.proposal_workflow_run_id = workflow_run.workflow_run_id
- except Exception: # noqa: BLE001 - any custody failure must stop dispatch.
- await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ if bind_conflict is not None or bind_unexpected:
+ if bind_conflict is None:
+ cancel_reason = "The Copilot test run could not be bound to its proposal."
+ elif bind_conflict[0] == "being_accepted":
+ cancel_reason = "The Copilot proposal was being accepted, so its test run was not started."
+ else:
+ cancel_reason = "The Copilot proposal changed before its test run started."
+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final(
workflow_run_id=workflow_run.workflow_run_id,
- failure_reason="The Copilot proposal changed before its test run started.",
+ failure_reason=cancel_reason,
)
if dispatch_draft_workflow_id is not None:
await _delete_dispatch_draft(dispatch_draft_workflow_id, ctx.organization_id)
dispatch_draft_workflow_id = None
+ if bind_conflict is None:
+ return {
+ "ok": False,
+ "error": (
+ "The test run could not be bound to the pending Copilot proposal, so it was not started."
+ ),
+ "data": {"run_dispatched": False, _PROPOSAL_BIND_REFUSED_KEY: True},
+ }
+ reason, adoption = bind_conflict
LOG.info(
"copilot candidate run bind lost",
workflow_run_id=workflow_run.workflow_run_id,
proposal_owner_turn_id=execution.proposal_owner_turn_id,
proposal_revision=execution.proposal_revision,
+ reason=reason,
+ adoption=adoption,
)
- return {"ok": False, "error": COPILOT_PROPOSAL_CHANGED_ERROR}
+ return {
+ **proposal_conflict_result(ctx, lost="The test was not run.", reason=reason, adoption=adoption),
+ "data": {"run_dispatched": False, _PROPOSAL_BIND_REFUSED_KEY: True},
+ }
ctx.dispatched_run_ids_this_turn.add(workflow_run.workflow_run_id)
# A seeded producer's value belongs to the run that produced it, never to this one.
seeded_only_labels = frozenset(block_outputs_to_seed) - frozenset(labels_that_may_execute)
@@ -6860,21 +6927,25 @@ async def _capture_delivered_output_files(copilot_ctx: CopilotContext, result: d
)
-def _is_budget_run_denial(result: Mapping[str, object]) -> bool:
+def _is_undispatched_run(result: Mapping[str, object]) -> bool:
data = result.get("data")
- return isinstance(data, dict) and data.get("budget_expired") is True and data.get("run_dispatched") is False
+ return isinstance(data, dict) and data.get("run_dispatched") is False
-def _is_refused_new_exit(result: Mapping[str, object]) -> bool:
+def _skips_build_test_finalize(result: Mapping[str, object]) -> bool:
data = result.get("data")
- return isinstance(data, dict) and "new_exit" in data and data.get("run_dispatched") is False
+ return (
+ isinstance(data, dict)
+ and data.get("run_dispatched") is False
+ and (data.get("budget_expired") is True or data.get(_PROPOSAL_BIND_REFUSED_KEY) is True)
+ )
async def _verify_and_record_run_blocks_result(
copilot_ctx: Any, result: dict[str, Any], _handler_start: float
) -> RecordedBuildTestOutcome | None:
"""Record and emit the run fact once; no authoring judge may rewrite it."""
- if _is_budget_run_denial(result) or _is_refused_new_exit(result):
+ if _is_undispatched_run(result):
# A denied dispatch is not a new failed run; retain the last actual run's evidence.
return None
if result.get(_INTERNAL_RUN_OUTCOME_RECORDED_KEY) is True:
@@ -7579,7 +7650,7 @@ def finalize_build_test_result(
recorded_outcome: RecordedBuildTestOutcome | None = None,
) -> dict[str, Any]:
"""Finalize diagnosis state and attach the one shared factual packet."""
- if _is_budget_run_denial(result):
+ if _skips_build_test_finalize(result):
return result
# Tool seams sometimes deliberately suppress collection and return no recorded outcome.
# Treat any non-record as absent rather than allowing a test double or failed observer to
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 13570c6e3..9f1ae8568 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -2278,7 +2278,7 @@ def page_state_probe_allowed(
)
-async def read_page_state(
+async def read_page_state_on_page(
ctx: AgentContext,
*,
tool_name: str,
@@ -2287,12 +2287,11 @@ async def read_page_state(
probe: bool = True,
settle: bool = False,
custody_lock: AbstractAsyncContextManager[None] | None = None,
-) -> PageState:
- """What the call's browser shows now, scrubbed for the model, or the unread state without touching a browser
- the call may not probe. A vendor frame that could not be measured makes the read fail rather than report no
- challenge."""
+) -> tuple[PageState, Page | None]:
+ """What the call's browser shows now, scrubbed for the model, and the page read, or the unread state and None
+ without touching a browser the call may not probe. An unmeasured vendor frame fails the read, not "no challenge"."""
if not (probe and page_state_probe_allowed(ctx, result, binding=binding)):
- return unread_page_state(ctx)
+ return unread_page_state(ctx), None
deadline = _PAGE_STATE_READ_TIMEOUT_SECONDS
if settle:
deadline += settings.COPILOT_SCOUT_ACT_OBSERVE_RECAPTURE_DELAY_SECONDS
@@ -2319,7 +2318,7 @@ async def read_page_state(
}
scrubbed = scrub_model_facing_tool_result(ctx, {"page_state": state}).get("page_state")
if isinstance(scrubbed, dict):
- return cast(PageState, scrubbed)
+ return cast(PageState, scrubbed), reading.page
failure = "scrubbed_away"
LOG.info(
"copilot_page_state_read_failed",
@@ -2328,13 +2327,14 @@ async def read_page_state(
reason=failure,
duration_ms=round((time.monotonic() - started) * 1000),
)
- return unread_page_state(ctx)
+ return unread_page_state(ctx), None
class _LivePageReading(NamedTuple):
url: str
title: str
vendor: str | None
+ page: Page
_PageStateReadFailure = Literal["no_page", "vendor_frame_unmeasured", "unreadable_location", "navigated_during_read"]
@@ -2363,7 +2363,7 @@ async def _read_live_page_state(ctx: AgentContext, *, settle: bool) -> _LivePage
# A navigation between the vendor probe and the title read would pair one document's vendor with another's.
if url != url_before:
return "navigated_during_read"
- return _LivePageReading(url, title, vendor)
+ return _LivePageReading(url, title, vendor, page)
def _child_frames(page: Page) -> list[Frame]:
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index c5ca7daea..9cc6e7c76 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -10,6 +10,7 @@ import re
import textwrap
from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass, replace
+from datetime import datetime, timezone
from enum import StrEnum
from typing import Annotated, Any, Literal, NamedTuple, cast
@@ -82,7 +83,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
format_output_policy_tool_error,
output_policy_verdict_to_trace_data,
)
-from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIX
+from skyvern.forge.sdk.copilot.output_utils import _BUILD_TEST_WORKFLOW_MAX_CHARS, INTERNAL_VALIDATION_FAILURE_PREFIX
from skyvern.forge.sdk.copilot.reached_download_target import (
REGISTERED_DOWNLOAD_OUTPUT_KEYS,
code_uses_download_claim,
@@ -106,6 +107,7 @@ from skyvern.forge.sdk.copilot.schema_incompatibility import (
render_schema_incompatibility_agent_steer,
render_schema_incompatibility_user_reason,
)
+from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.secret_scrub import registered_scrub_values, scrub_secrets_from_text
from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_end
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -134,13 +136,14 @@ from skyvern.forge.sdk.copilot.workflow_yaml import (
strip_private_workflow_settings,
submitted_private_workflow_settings,
with_workflow_yaml_title,
+ workflow_to_copilot_yaml,
)
-from skyvern.forge.sdk.db.exceptions import CopilotProposalConflictError
+from skyvern.forge.sdk.db.exceptions import CopilotProposalConflictError, CopilotProposalConflictReason
from skyvern.forge.sdk.schemas.workflow_copilot import COPILOT_PRIVATE_SETTINGS_KEY, copilot_proposal_metadata
from skyvern.forge.sdk.services import google_oauth_service
from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, InsecureCodeDetected
from skyvern.forge.sdk.workflow.models.block import CodeBlock
-from skyvern.forge.sdk.workflow.models.workflow import Workflow
+from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition
from skyvern.forge.sdk.workflow.runtime_completion import contract_from_code_artifact_metadata
from skyvern.schemas.proxy_location import ProxyLocationInput, runtime_proxy_location
from skyvern.schemas.workflows import (
@@ -4072,6 +4075,19 @@ def proposal_workflow_fingerprint_matches(workflow: Workflow, fingerprint: str)
)
+async def proposal_is_displayable(proposal: object, workflow_permanent_id: str, organization_id: str) -> bool:
+ if not isinstance(proposal, dict):
+ return False
+ metadata = copilot_proposal_metadata(proposal)
+ if metadata is None:
+ return True
+ canonical = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=workflow_permanent_id,
+ organization_id=organization_id,
+ )
+ return canonical is not None and proposal_workflow_fingerprint_matches(canonical, metadata.canonical_fingerprint)
+
+
def private_workflow_settings_from_proposal(proposal: dict[str, Any]) -> dict[str, Any]:
private_settings = proposal.get(COPILOT_PRIVATE_SETTINGS_KEY)
proposal_yaml = proposal.get("_copilot_yaml")
@@ -4090,11 +4106,12 @@ def private_workflow_settings_from_proposal(proposal: dict[str, Any]) -> dict[st
return submitted
-async def _validated_pending_workflow_proposal(
- ctx: CopilotContext,
- *,
- required_workflow_run_id: str | None = None,
-) -> tuple[dict[str, Any], str] | None:
+ProposalAdoptionOutcome = Literal[
+ "adopted", "cleared", "kept", "metadata_invalid", "claimed", "canonical_fingerprint_mismatch", "read_failed"
+]
+
+
+async def _stored_candidate_proposal(ctx: CopilotContext) -> dict[str, Any] | None:
if not ctx.workflow_copilot_chat_id:
return None
chat = await app.DATABASE.workflow_params.get_workflow_copilot_chat_by_id(
@@ -4104,7 +4121,16 @@ async def _validated_pending_workflow_proposal(
if chat is None or chat.workflow_permanent_id != ctx.workflow_permanent_id:
return None
proposal = chat.proposed_workflow
- if not isinstance(proposal, dict):
+ return proposal if isinstance(proposal, dict) else None
+
+
+async def _validated_pending_workflow_proposal(
+ ctx: CopilotContext,
+ *,
+ required_workflow_run_id: str | None = None,
+) -> tuple[dict[str, Any], str] | None:
+ proposal = await _stored_candidate_proposal(ctx)
+ if proposal is None:
return None
metadata = copilot_proposal_metadata(proposal)
if metadata is not None:
@@ -4113,6 +4139,7 @@ async def _validated_pending_workflow_proposal(
# so every early return below this point keeps the token.
ctx.proposal_owner_turn_id = metadata.owner_turn_id
ctx.proposal_revision = metadata.revision
+ ctx.proposal_adopted = False
ctx.proposal_canonical_fingerprint = metadata.canonical_fingerprint
if required_workflow_run_id is not None and (
metadata is None or metadata.workflow_run_id != required_workflow_run_id
@@ -4201,51 +4228,9 @@ def _effective_private_settings(*sources: Mapping[str, object]) -> dict[str, obj
}
-async def restore_pending_workflow_proposal(
- ctx: CopilotContext,
- *,
- required_workflow_run_id: str | None = None,
-) -> None:
- """Restore only the server's pending proposal, optionally constrained to its exact run."""
- ctx.workflow_yaml = strip_copilot_yaml_headers(ctx.workflow_yaml)
- if ctx.workflow_yaml is None:
- ctx.workflow_yaml = strip_copilot_yaml_headers(ctx.persisted_workflow_yaml)
- if ctx.staged_workflow is not None:
- return
- pending = await _validated_pending_workflow_proposal(ctx, required_workflow_run_id=required_workflow_run_id)
- proposal = pending[0] if pending is not None else {}
- proposal_settings = private_workflow_settings_from_proposal(proposal)
- # Keep unresolved masks in the authored snapshot for Accept to resolve against the saved workflow.
- proposal_settings = merge_private_workflow_settings(proposal_settings, inherited_settings=proposal_settings)
- request_settings = merge_private_workflow_settings(
- ctx.private_workflow_settings, inherited_settings=ctx.private_workflow_settings
- )
- ctx.authored_private_workflow_settings = submitted_private_workflow_settings(proposal_settings, request_settings)
- for name in ("extra_http_headers", "cdp_connect_headers"):
- headers = request_settings.get(name)
- prior_headers = proposal_settings.get(name)
- if isinstance(headers, dict) and isinstance(prior_headers, dict):
- ctx.authored_private_workflow_settings[name] = merge_masked_headers(headers, {**headers, **prior_headers})
- saved = None
- if pending is not None or any(
- isinstance(request_settings.get(name), dict) and SECRET_HEADER_MASK in request_settings[name].values()
- for name in ("extra_http_headers", "cdp_connect_headers")
- ):
- saved = await app.DATABASE.workflows.get_workflow_by_permanent_id(
- workflow_permanent_id=ctx.workflow_permanent_id, organization_id=ctx.organization_id
- )
- inherited_settings = inherited_header_settings(saved) if saved is not None else None
- private_settings = merge_private_workflow_settings(
- proposal_settings, request_settings, inherited_settings=inherited_settings
- )
- ctx.private_workflow_settings = private_settings
- if pending is None:
- return
- _, sanitized_yaml = pending
- metadata = copilot_proposal_metadata(proposal)
- stored_document = safe_load_no_dates(sanitized_yaml)
- restored_document = {**stored_document, **private_settings}
- restored_yaml = dump_workflow_yaml(restored_document) if private_settings else sanitized_yaml
+async def _parse_stored_candidate(
+ ctx: CopilotContext, proposal: dict[str, Any], restored_yaml: str, saved: Workflow | None
+) -> Workflow:
if "workflow_definition" in proposal:
workflow = Workflow.model_validate(proposal)
# A proposal froze its title in an earlier turn, so a rename saved between the two outranks
@@ -4287,15 +4272,93 @@ async def restore_pending_workflow_proposal(
settings_fallback_yaml=ctx.persisted_workflow_yaml,
prefer_live_title=True,
)
- # Serialization masks CDP headers, so the saved side takes its real header maps from the row itself.
- saved_settings = (
- {**submitted_private_workflow_settings(saved.model_dump(mode="json")), **inherited_header_settings(saved)}
- if saved is not None
- else {}
+ return workflow
+
+
+class _CandidatePrivateSettings(NamedTuple):
+ proposal: dict[str, Any]
+ authored: dict[str, Any]
+ effective: dict[str, Any]
+
+ def overlay(self, stored_yaml: str) -> str:
+ if not self.effective:
+ return stored_yaml
+ return dump_workflow_yaml({**safe_load_no_dates(stored_yaml), **self.effective})
+
+ def own_and_turn_effective(
+ self, turn_settings: dict[str, Any], stored_document: Mapping[str, object], saved: Workflow | None
+ ) -> tuple[dict[str, object], dict[str, object]]:
+ # Serialization masks CDP headers, so the saved side takes its real header maps from the row itself.
+ saved_settings = (
+ {**submitted_private_workflow_settings(saved.model_dump(mode="json")), **inherited_header_settings(saved)}
+ if saved is not None
+ else {}
+ )
+ own = merge_private_workflow_settings(
+ self.proposal, inherited_settings=inherited_header_settings(saved) if saved is not None else None
+ )
+ return (
+ _effective_private_settings(own, stored_document, saved_settings),
+ _effective_private_settings(turn_settings, stored_document, saved_settings),
+ )
+
+
+def _candidate_private_settings(
+ proposal: dict[str, Any], request_settings: dict[str, Any], saved: Workflow | None
+) -> _CandidatePrivateSettings:
+ proposal_settings = private_workflow_settings_from_proposal(proposal)
+ # Keep unresolved masks in the authored snapshot for Accept to resolve against the saved workflow.
+ proposal_settings = merge_private_workflow_settings(proposal_settings, inherited_settings=proposal_settings)
+ authored = submitted_private_workflow_settings(proposal_settings, request_settings)
+ for name in ("extra_http_headers", "cdp_connect_headers"):
+ headers = request_settings.get(name)
+ prior_headers = proposal_settings.get(name)
+ if isinstance(headers, dict) and isinstance(prior_headers, dict):
+ authored[name] = merge_masked_headers(headers, {**headers, **prior_headers})
+ effective = merge_private_workflow_settings(
+ proposal_settings,
+ request_settings,
+ inherited_settings=inherited_header_settings(saved) if saved is not None else None,
)
- own_settings = merge_private_workflow_settings(proposal_settings, inherited_settings=inherited_settings)
- own_effective = _effective_private_settings(own_settings, stored_document, saved_settings)
- effective = _effective_private_settings(private_settings, stored_document, saved_settings)
+ return _CandidatePrivateSettings(proposal_settings, authored, effective)
+
+
+async def restore_pending_workflow_proposal(
+ ctx: CopilotContext,
+ *,
+ required_workflow_run_id: str | None = None,
+) -> None:
+ """Restore only the server's pending proposal, optionally constrained to its exact run."""
+ ctx.workflow_yaml = strip_copilot_yaml_headers(ctx.workflow_yaml)
+ if ctx.workflow_yaml is None:
+ ctx.workflow_yaml = strip_copilot_yaml_headers(ctx.persisted_workflow_yaml)
+ if ctx.staged_workflow is not None:
+ return
+ pending = await _validated_pending_workflow_proposal(ctx, required_workflow_run_id=required_workflow_run_id)
+ proposal = pending[0] if pending is not None else {}
+ request_settings = merge_private_workflow_settings(
+ ctx.private_workflow_settings, inherited_settings=ctx.private_workflow_settings
+ )
+ saved = None
+ if pending is not None or any(
+ isinstance(request_settings.get(name), dict) and SECRET_HEADER_MASK in request_settings[name].values()
+ for name in ("extra_http_headers", "cdp_connect_headers")
+ ):
+ saved = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=ctx.workflow_permanent_id, organization_id=ctx.organization_id
+ )
+ settings = _candidate_private_settings(proposal, request_settings, saved)
+ proposal_settings, private_settings = settings.proposal, settings.effective
+ ctx.authored_private_workflow_settings = settings.authored
+ ctx.private_workflow_settings = private_settings
+ if pending is None:
+ return
+ _, sanitized_yaml = pending
+ metadata = copilot_proposal_metadata(proposal)
+ stored_document = safe_load_no_dates(sanitized_yaml)
+ restored_yaml = settings.overlay(sanitized_yaml)
+ workflow = await _parse_stored_candidate(ctx, proposal, restored_yaml, saved)
+ own_effective, effective = settings.own_and_turn_effective(private_settings, stored_document, saved)
if effective == own_effective:
# The turn-end publish compares settings as written, so a request value that only echoes the proposal's
# effective value is recorded as the proposal wrote it.
@@ -4329,6 +4392,182 @@ async def restore_pending_workflow_proposal(
ctx.proposal_workflow_run_id = metadata.workflow_run_id
+async def prior_draft_settings_match_stored_candidate(ctx: CopilotContext) -> bool | None:
+ """A prior draft parses under this turn's settings, so the stored bytes vouch for it only when those are its own.
+ None means no stored candidate built on the current saved workflow could be compared."""
+ try:
+ proposal = await _stored_candidate_proposal(ctx)
+ stored_yaml = strip_copilot_yaml_headers(proposal.get("_copilot_yaml")) if proposal is not None else None
+ if proposal is None or not isinstance(stored_yaml, str) or not stored_yaml:
+ return None
+ saved = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=ctx.workflow_permanent_id, organization_id=ctx.organization_id
+ )
+ metadata = copilot_proposal_metadata(proposal)
+ if metadata is not None and (
+ saved is None or not proposal_workflow_fingerprint_matches(saved, metadata.canonical_fingerprint)
+ ):
+ return None
+ own, turn = _candidate_private_settings(proposal, {}, saved).own_and_turn_effective(
+ ctx.private_workflow_settings, safe_load_no_dates(stored_yaml), saved
+ )
+ except Exception:
+ LOG.warning("copilot_prior_draft_settings_check_failed", exc_info=True)
+ return None
+ return own == turn
+
+
+async def adopt_stored_workflow_candidate(ctx: CopilotContext) -> ProposalAdoptionOutcome:
+ """Take the stored candidate's token and bytes together; the caller holds ``ctx.proposal_mutation_lock``."""
+ workflow: Workflow | None = None
+ workflow_yaml = ""
+ code_artifact_metadata: dict[str, dict[str, Any]] = {}
+ associations: dict[str, str] = {}
+ settings: _CandidatePrivateSettings | None = None
+ canonical: Workflow | None = None
+ try:
+ proposal = await _stored_candidate_proposal(ctx)
+ metadata = copilot_proposal_metadata(proposal)
+ if metadata is not None and metadata.claim_is_live(datetime.now(timezone.utc)):
+ return "claimed"
+ if proposal is not None and metadata is None:
+ return "metadata_invalid"
+ if metadata is not None and metadata.owner_turn_id == ctx.turn_id:
+ return "kept"
+ canonical = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=ctx.workflow_permanent_id, organization_id=ctx.organization_id
+ )
+ outcome: ProposalAdoptionOutcome = "cleared"
+ stored_yaml = proposal.get("_copilot_yaml") if proposal is not None else None
+ if proposal is not None and metadata is not None and isinstance(stored_yaml, str) and stored_yaml:
+ if canonical is None or not proposal_workflow_fingerprint_matches(
+ canonical, metadata.canonical_fingerprint
+ ):
+ outcome = "canonical_fingerprint_mismatch"
+ else:
+ # The candidate's own settings replace this turn's, as if the turn had published it.
+ settings = _candidate_private_settings(proposal, {}, canonical)
+ workflow = await _parse_stored_candidate(ctx, proposal, settings.overlay(stored_yaml), canonical)
+ workflow_yaml = stored_yaml
+ stored_metadata = proposal.get("_copilot_code_artifact_metadata")
+ code_artifact_metadata = copy.deepcopy(stored_metadata) if isinstance(stored_metadata, dict) else {}
+ associations = runner_code_block_associations(workflow_yaml)
+ outcome = "adopted"
+ except Exception:
+ LOG.warning(
+ "copilot_stored_candidate_adoption_read_failed",
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ exc_info=True,
+ )
+ return "read_failed"
+
+ ctx.proposal_adopted = True
+ ctx.proposal_adoptions += 1
+ # The stored candidate does not record whether its author meant to delete the saved contract; keep it.
+ ctx.clear_persisted_completion_contract = False
+ ctx.proposal_owner_turn_id = metadata.owner_turn_id if metadata is not None else None
+ ctx.proposal_revision = metadata.revision if metadata is not None else None
+ ctx.proposal_canonical_fingerprint = metadata.canonical_fingerprint if metadata is not None else None
+ ctx.proposal_workflow_run_id = metadata.workflow_run_id if workflow is not None and metadata is not None else None
+ # This turn's own draft is gone, so no turn-end exit may present or re-publish it. The adopted candidate
+ # becomes this turn's result only once this turn edits it.
+ ctx.has_staged_proposal = False
+ ctx.last_workflow = None
+ ctx.last_workflow_yaml = None
+ ctx.last_good_workflow = None
+ ctx.last_good_workflow_yaml = None
+ ctx.last_failed_workflow_yaml = None
+ if settings is not None:
+ ctx.authored_private_workflow_settings = settings.authored
+ ctx.private_workflow_settings = settings.effective
+ ctx.settings_diverged_proposal_token = None
+ ctx.workflow_yaml = workflow_yaml
+ elif canonical is not None:
+ # With no candidate left the saved row is the only source; omitted settings make edits inherit its own.
+ ctx.workflow_id = canonical.workflow_id
+ ctx.workflow_yaml = workflow_to_copilot_yaml(canonical)
+ ctx.authored_private_workflow_settings = {}
+ ctx.private_workflow_settings = {}
+ else:
+ ctx.workflow_yaml = ctx.persisted_workflow_yaml or ""
+ ctx.staged_workflow = workflow
+ ctx.staged_workflow_yaml = workflow_yaml if workflow is not None else None
+ ctx.prior_copilot_workflow_yaml = workflow_yaml if workflow is not None else None
+ ctx.code_artifact_metadata = code_artifact_metadata
+ ctx.workflow_verification_evidence.code_artifact_metadata = code_artifact_metadata
+ ctx.runner_code_block_associations_by_label = associations
+ ctx.last_full_workflow_test_ok = False
+ # Credit and receipts were earned under the replaced candidate's settings, which block fingerprints omit.
+ ctx.executed_block_fingerprints = {}
+ _invalidate_test_evidence_on_edit(ctx, None, workflow.workflow_definition if workflow is not None else None)
+ return outcome
+
+
+def unseen_adoption(ctx: CopilotContext) -> ProposalAdoptionOutcome | None:
+ """An adoption the latest model response had not seen: its calls were written against the replaced draft."""
+ if ctx.proposal_adoptions == ctx.proposal_adoptions_seen_by_model:
+ return None
+ return "adopted" if ctx.staged_workflow is not None else "cleared"
+
+
+def _unseen_adoption_edit_refusal(ctx: CopilotContext) -> dict[str, Any] | None:
+ adoption = unseen_adoption(ctx)
+ if adoption is None:
+ return None
+ LOG.info(
+ "copilot_candidate_publication_conflict",
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ reason="token_changed",
+ adoption=adoption,
+ )
+ return proposal_conflict_result(
+ ctx, lost="This edit was not saved, so nothing was written.", reason="token_changed", adoption=adoption
+ )
+
+
+def proposal_conflict_result(
+ ctx: CopilotContext,
+ *,
+ lost: str,
+ reason: CopilotProposalConflictReason,
+ adoption: ProposalAdoptionOutcome,
+) -> dict[str, Any]:
+ result: dict[str, Any] = {"ok": False, "conflict_reason": reason, "adoption": adoption}
+ if adoption == "adopted":
+ result["error"] = f"{lost} Another change replaced the pending proposal, and this turn now works on it."
+ adopted_yaml = redact_raw_secrets_for_prompt(ctx.staged_workflow_yaml or "")
+ if len(adopted_yaml) > _BUILD_TEST_WORKFLOW_MAX_CHARS:
+ result["adopted_workflow_yaml"] = adopted_yaml[: _BUILD_TEST_WORKFLOW_MAX_CHARS - 3] + "..."
+ result["next_step"] = (
+ "adopted_workflow_yaml is the workflow this turn now works on, "
+ f"shortened at {_BUILD_TEST_WORKFLOW_MAX_CHARS} characters. "
+ "Change it with edit_block, or test it as it is."
+ )
+ else:
+ result["adopted_workflow_yaml"] = adopted_yaml
+ result["next_step"] = (
+ "adopted_workflow_yaml is the workflow this turn now works on. "
+ "Apply your change to it, or test it as it is."
+ )
+ elif adoption in ("cleared", "canonical_fingerprint_mismatch"):
+ result["error"] = (
+ f"{lost} The pending proposal was removed or no longer matches the saved workflow, "
+ "so this turn's draft was dropped."
+ )
+ result["next_step"] = "This turn now works from the saved workflow; read it again before editing."
+ elif adoption == "kept":
+ result["error"] = f"{lost} The proposal changed while this step was starting."
+ result["next_step"] = "This turn's current draft is unchanged; edit or test it again."
+ elif adoption == "claimed":
+ result["error"] = f"{lost} The pending proposal is being accepted."
+ result["next_step"] = "Wait for the accept to finish before editing or testing."
+ elif adoption == "metadata_invalid":
+ result["error"] = f"{lost} The pending proposal's record is unreadable, so this turn cannot replace it."
+ else:
+ result["error"] = f"{lost} The current proposal could not be read."
+ return result
+
+
def _candidate_proposal_data(workflow: Workflow, workflow_yaml: str, ctx: CopilotContext) -> dict[str, Any]:
proposal = dict(workflow.model_dump(mode="json"))
proposal["_copilot_yaml"] = workflow_yaml
@@ -4419,6 +4658,7 @@ async def publish_workflow_candidate(
raise RuntimeError("Candidate publication returned no ownership token")
ctx.proposal_owner_turn_id = metadata.owner_turn_id
ctx.proposal_revision = metadata.revision
+ ctx.proposal_adopted = False
ctx.proposal_canonical_fingerprint = metadata.canonical_fingerprint
ctx.authored_private_workflow_settings = proposal.get(COPILOT_PRIVATE_SETTINGS_KEY, {})
ctx.private_workflow_settings = merge_private_workflow_settings(
@@ -4461,6 +4701,9 @@ async def _update_workflow(
authority_error = _authority_tool_error(ctx, "update_workflow")
if authority_error is not None:
return _tool_error(authority_error)
+ # Checked before the submission touches ctx; no await separates this from the metadata merge below.
+ if isinstance(ctx, CopilotContext) and (unseen_refusal := _unseen_adoption_edit_refusal(ctx)) is not None:
+ return unseen_refusal
workflow_yaml = params["workflow_yaml"]
submitted_workflow_yaml = workflow_yaml
@@ -4484,6 +4727,7 @@ async def _update_workflow(
prior_workflow_yaml = ctx.workflow_yaml
existing_metadata = ctx.code_artifact_metadata
previous_metadata_contract = contract_from_code_artifact_metadata(existing_metadata)
+ clear_persisted_completion_contract: bool | None = None
code_safety_errors = _code_block_safety_errors(workflow_yaml, prior_workflow_yaml)
code_safety_rejection_facts: list[CodeSafetyRejectionFact] = []
if code_safety_errors:
@@ -4635,8 +4879,6 @@ async def _update_workflow(
accepted_metadata_contract,
previous_contract=previous_metadata_contract,
)
- if isinstance(ctx, CopilotContext):
- ctx.clear_persisted_completion_contract = clear_persisted_completion_contract
params["workflow_yaml"] = workflow_yaml
submitted_definition = parse_workflow_yaml(workflow_yaml)
prior_definition = parse_workflow_yaml(prior_yaml) if prior_yaml else None
@@ -4696,6 +4938,9 @@ async def _update_workflow(
# The durable write and the staged fields it backs move together, so a parallel tool
# call cannot read a staged draft the store has not accepted.
async with ctx.proposal_mutation_lock:
+ # Publishing under an adopted token the model has not seen would overwrite that candidate.
+ if (unseen_refusal := _unseen_adoption_edit_refusal(ctx)) is not None:
+ return unseen_refusal
# A sibling write may have published since entry; the dropped/retyped facts
# must name the draft this write replaces.
prior_yaml = _latest_draft_yaml(ctx)
@@ -4704,24 +4949,30 @@ async def _update_workflow(
ctx, workflow=workflow, workflow_yaml=workflow_yaml
)
except CopilotProposalConflictError as conflict:
+ stale_owner_turn_id, stale_revision = ctx.proposal_owner_turn_id, ctx.proposal_revision
+ adoption = await adopt_stored_workflow_candidate(ctx)
LOG.info(
"copilot_candidate_publication_conflict",
workflow_permanent_id=ctx.workflow_permanent_id,
- proposal_owner_turn_id=ctx.proposal_owner_turn_id,
- proposal_revision=ctx.proposal_revision,
+ proposal_owner_turn_id=stale_owner_turn_id,
+ proposal_revision=stale_revision,
+ reason=conflict.reason,
+ adoption=adoption,
error=str(conflict),
)
- return {
- "ok": False,
- "error": (
- "The pending proposal changed while this edit was being saved, so nothing was written. "
- "Read the current workflow again before re-submitting this edit."
- ),
- }
+ return proposal_conflict_result(
+ ctx,
+ lost="This edit was not saved, so nothing was written.",
+ reason=conflict.reason,
+ adoption=adoption,
+ )
ctx.staged_workflow_yaml = workflow_yaml
ctx.staged_workflow = workflow
ctx.has_staged_proposal = True
ctx.workflow_yaml = workflow_yaml
+ # A refused edit must not leave its contract deletion for a later accept of other content.
+ if clear_persisted_completion_contract is not None:
+ ctx.clear_persisted_completion_contract = clear_persisted_completion_contract
# Published under the lock so a waiting sibling reads this draft as its baseline;
# the wrapper re-assigns the same pair after the tool returns.
ctx.last_workflow = workflow
@@ -4934,21 +5185,27 @@ def _record_workflow_proxy_location_span(workflow_yaml: str, workflow: Workflow)
def _record_workflow_update_result(
- copilot_ctx: Any, result: dict[str, Any], prior_definition: object | None = None
+ copilot_ctx: Any, result: dict[str, Any], prior_definition: WorkflowDefinition | None = None
) -> None:
if not (result.get("ok") and "_workflow" in result):
return
- wf = result["_workflow"]
+ wf: Workflow = result["_workflow"]
copilot_ctx.last_workflow = wf
copilot_ctx.last_workflow_yaml = copilot_ctx.workflow_yaml or None
- copilot_ctx.effective_workflow_proxy_location = runtime_proxy_location(getattr(wf, "proxy_location", None))
+ copilot_ctx.effective_workflow_proxy_location = runtime_proxy_location(wf.proxy_location)
data = result.get("data")
if isinstance(data, dict):
block_count = data.get("block_count")
if isinstance(block_count, int):
copilot_ctx.last_update_block_count = block_count
copilot_ctx.update_workflow_called = True
+ _invalidate_test_evidence_on_edit(copilot_ctx, prior_definition, wf.workflow_definition)
+
+
+def _invalidate_test_evidence_on_edit(
+ copilot_ctx: CopilotContext, prior_definition: WorkflowDefinition | None, new_definition: WorkflowDefinition | None
+) -> None:
copilot_ctx.test_after_update_done = False
copilot_ctx.last_test_ok = None
clear_active_run_evidence_on_workflow_edit(copilot_ctx)
@@ -4957,4 +5214,4 @@ def _record_workflow_update_result(
# CopilotNonRetriableNavError with the old run's error, telling the user
# to "verify the URL" for a URL they just corrected in the new draft.
copilot_ctx.last_test_non_retriable_nav_error = None
- _invalidate_verified_state_on_edit(copilot_ctx, prior_definition, getattr(wf, "workflow_definition", None))
+ _invalidate_verified_state_on_edit(copilot_ctx, prior_definition, new_definition)
diff --git a/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py b/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py
index 0eb7a2d48..b8e016abd 100644
--- a/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py
+++ b/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py
@@ -76,9 +76,13 @@ def _result_text_values(value: Any) -> list[str]:
return []
-def _result_text_without_page_state(output: dict[str, Any]) -> str:
- # The top-level page_state carries the page's own title, and a page can title itself anything.
- return " ".join(_result_text_values({key: item for key, item in output.items() if key != "page_state"}))
+PAGE_REQUEST_FAILURES_KEY = "page_request_failures"
+PAGE_FACT_KEYS = frozenset({"page_state", PAGE_REQUEST_FAILURES_KEY})
+
+
+def _result_text_without_page_facts(output: dict[str, Any]) -> str:
+ # These top-level keys carry the page's own title and request URLs, and a page can name them anything.
+ return " ".join(_result_text_values({key: item for key, item in output.items() if key not in PAGE_FACT_KEYS}))
def _unrecoverable_tool_error_reason(output: dict[str, Any]) -> str:
@@ -87,7 +91,7 @@ def _unrecoverable_tool_error_reason(output: dict[str, Any]) -> str:
data = output.get("data")
raw_reason = data.get("failure_reason") if isinstance(data, dict) else None
if not isinstance(raw_reason, str) or not raw_reason.strip():
- raw_reason = _result_text_without_page_state(output)
+ raw_reason = _result_text_without_page_facts(output)
reason = " ".join(str(raw_reason or "Browser session was no longer reachable.").split())
reason = redact_browser_session_references(reason)
return reason[:240].rstrip()
@@ -100,7 +104,7 @@ def _is_unrecoverable_browser_session_error(tool_name: str, output: dict[str, An
# neither "not found" nor a status, and the same dead browser must count whichever tool met it.
if output.get("error_code") == BROWSER_SESSION_UNAVAILABLE_ERROR_CODE:
return True
- lowered = _result_text_without_page_state(output).lower()
+ lowered = _result_text_without_page_facts(output).lower()
if "no browser context" in lowered:
return True
has_session_signal = "browser session" in lowered or "browser context" in lowered
diff --git a/skyvern/forge/sdk/db/exceptions.py b/skyvern/forge/sdk/db/exceptions.py
index 6bebe0654..a569d59aa 100644
--- a/skyvern/forge/sdk/db/exceptions.py
+++ b/skyvern/forge/sdk/db/exceptions.py
@@ -1,3 +1,5 @@
+from typing import Literal
+
try:
import psycopg
except ImportError: # psycopg ships with the server extra only
@@ -36,9 +38,25 @@ class DuplicateCopilotTurnError(Exception):
self.turn_id = turn_id
+CopilotProposalConflictReason = Literal[
+ "token_changed",
+ "candidate_missing",
+ "metadata_invalid",
+ "being_accepted",
+ "bytes_changed",
+ "content_changed",
+ "claim_changed",
+ "disposition_changed",
+]
+
+
class CopilotProposalConflictError(Exception):
"""The candidate changed after the caller read its owner/revision token."""
+ def __init__(self, message: str, *, reason: CopilotProposalConflictReason) -> None:
+ super().__init__(message)
+ self.reason = reason
+
class ScheduleLimitExceededError(Exception):
"""Raised when attempting to create a schedule that would exceed the org-wide tier limit."""
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index df410edc4..81ba476cb 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -825,7 +825,9 @@ class WorkflowParametersRepository(BaseRepository):
if copilot_proposal_metadata(chat.proposed_workflow) is not None and (
copilot_proposal_metadata(proposed_workflow) is None
):
- raise CopilotProposalConflictError("Copilot proposal is owned by a newer candidate")
+ raise CopilotProposalConflictError(
+ "Copilot proposal is owned by a newer candidate", reason="token_changed"
+ )
chat.proposed_workflow = proposed_workflow
if auto_accept is not None:
chat.auto_accept = auto_accept
@@ -858,12 +860,12 @@ class WorkflowParametersRepository(BaseRepository):
current = copilot_proposal_metadata(proposal)
if current is None:
if isinstance(proposal, Mapping) and COPILOT_PROPOSAL_METADATA_KEY in proposal:
- raise CopilotProposalConflictError("Copilot proposal metadata is invalid")
+ raise CopilotProposalConflictError("Copilot proposal metadata is invalid", reason="metadata_invalid")
if expected_owner_turn_id is not None or expected_revision is not None:
- raise CopilotProposalConflictError("Copilot proposal changed")
+ raise CopilotProposalConflictError("Copilot proposal changed", reason="candidate_missing")
return None
if current.owner_turn_id != expected_owner_turn_id or current.revision != expected_revision:
- raise CopilotProposalConflictError("Copilot proposal changed")
+ raise CopilotProposalConflictError("Copilot proposal changed", reason="token_changed")
return current
@db_operation("publish_workflow_copilot_candidate", expected_errors=(CopilotProposalConflictError,))
@@ -886,7 +888,7 @@ class WorkflowParametersRepository(BaseRepository):
if current is not None and current.claim_is_live(datetime.now(timezone.utc)):
# The accept between its claim and its canonical write is committing bytes it has
# already read; replacing them underneath it would make it write a stale workflow.
- raise CopilotProposalConflictError("Copilot proposal is being accepted")
+ raise CopilotProposalConflictError("Copilot proposal is being accepted", reason="being_accepted")
revision = current.revision + 1 if current is not None and current.owner_turn_id == owner_turn_id else 1
metadata = CopilotProposalMetadata(
owner_turn_id=owner_turn_id,
@@ -918,17 +920,17 @@ class WorkflowParametersRepository(BaseRepository):
chat = await self._locked_copilot_chat(session, organization_id, workflow_copilot_chat_id)
current = self._require_candidate_token(chat.proposed_workflow, expected_owner_turn_id, expected_revision)
if current is None or not isinstance(chat.proposed_workflow, dict):
- raise CopilotProposalConflictError("Copilot proposal changed")
+ raise CopilotProposalConflictError("Copilot proposal changed", reason="candidate_missing")
if current.claim_is_live(datetime.now(timezone.utc)):
- raise CopilotProposalConflictError("Copilot proposal is being accepted")
+ raise CopilotProposalConflictError("Copilot proposal is being accepted", reason="being_accepted")
if chat.proposed_workflow.get("_copilot_yaml") != proposal.get("_copilot_yaml"):
- raise CopilotProposalConflictError("Copilot proposal bytes changed")
+ raise CopilotProposalConflictError("Copilot proposal bytes changed", reason="bytes_changed")
current_workflow = {
key: value for key, value in chat.proposed_workflow.items() if not key.startswith("_copilot_")
}
proposed_workflow = {key: value for key, value in proposal.items() if not key.startswith("_copilot_")}
if current_workflow != proposed_workflow:
- raise CopilotProposalConflictError("Copilot proposal content changed")
+ raise CopilotProposalConflictError("Copilot proposal content changed", reason="content_changed")
metadata = current.model_copy(update={"disposition": disposition})
chat.proposed_workflow = {
**proposal,
@@ -953,11 +955,11 @@ class WorkflowParametersRepository(BaseRepository):
chat = await self._locked_copilot_chat(session, organization_id, workflow_copilot_chat_id)
current = self._require_candidate_token(chat.proposed_workflow, expected_owner_turn_id, expected_revision)
if current is None or not isinstance(chat.proposed_workflow, dict):
- raise CopilotProposalConflictError("Copilot proposal changed")
+ raise CopilotProposalConflictError("Copilot proposal changed", reason="candidate_missing")
if current.claim_is_live(datetime.now(timezone.utc)):
- raise CopilotProposalConflictError("Copilot proposal is being accepted")
+ raise CopilotProposalConflictError("Copilot proposal is being accepted", reason="being_accepted")
if chat.proposed_workflow.get("_copilot_yaml") != expected_workflow_yaml:
- raise CopilotProposalConflictError("Copilot proposal bytes changed")
+ raise CopilotProposalConflictError("Copilot proposal bytes changed", reason="bytes_changed")
# A new run is not yet proof. Turn end re-records the disposition from its result, but a turn
# that exits without one must not leave a stale Tested beside this run.
proven = current.disposition in ("review_tested", "auto_applicable")
@@ -987,12 +989,10 @@ class WorkflowParametersRepository(BaseRepository):
async with self.Session() as session:
chat = await self._locked_copilot_chat(session, organization_id, workflow_copilot_chat_id)
current = self._require_candidate_token(chat.proposed_workflow, expected_owner_turn_id, expected_revision)
- if (
- current is None
- or not isinstance(chat.proposed_workflow, dict)
- or current.claim_is_live(datetime.now(timezone.utc))
- ):
- raise CopilotProposalConflictError("Copilot proposal changed")
+ if current is None or not isinstance(chat.proposed_workflow, dict):
+ raise CopilotProposalConflictError("Copilot proposal changed", reason="candidate_missing")
+ if current.claim_is_live(datetime.now(timezone.utc)):
+ raise CopilotProposalConflictError("Copilot proposal is being accepted", reason="being_accepted")
metadata = current.model_copy(update={"disposition": "accepting", "claimed_at": datetime.now(timezone.utc)})
chat.proposed_workflow = {
**chat.proposed_workflow,
@@ -1015,7 +1015,7 @@ class WorkflowParametersRepository(BaseRepository):
async with self.Session() as session:
chat = await self._locked_copilot_chat(session, organization_id, workflow_copilot_chat_id)
if chat.proposed_workflow != expected_proposal:
- raise CopilotProposalConflictError("Copilot proposal changed")
+ raise CopilotProposalConflictError("Copilot proposal changed", reason="content_changed")
# The proposal records no base, so this is the only version Accept can hold a later save against.
# Same filter as WorkflowsRepository.get_workflow_by_permanent_id, the read expected_version uses.
base_version = await session.scalar(
@@ -1069,7 +1069,7 @@ class WorkflowParametersRepository(BaseRepository):
or current.disposition != "accepting"
or current.claimed_at != expected_claimed_at
):
- raise CopilotProposalConflictError("Copilot proposal claim changed")
+ raise CopilotProposalConflictError("Copilot proposal claim changed", reason="claim_changed")
if restore_untokened:
chat.proposed_workflow = {
key: value for key, value in chat.proposed_workflow.items() if key != COPILOT_PROPOSAL_METADATA_KEY
@@ -1106,10 +1106,12 @@ class WorkflowParametersRepository(BaseRepository):
# A claim the caller named must still be the one it took; see release above.
or (expected_claimed_at is not None and current.claimed_at != expected_claimed_at)
):
- raise CopilotProposalConflictError("Copilot proposal disposition changed")
+ raise CopilotProposalConflictError("Copilot proposal disposition changed", reason="disposition_changed")
if record_acceptance:
if current is None:
- raise CopilotProposalConflictError("Copilot proposal has no owner to record")
+ raise CopilotProposalConflictError(
+ "Copilot proposal has no owner to record", reason="candidate_missing"
+ )
accepted_turn_ids = list(chat.accepted_turn_ids or [])
if current.owner_turn_id not in accepted_turn_ids:
accepted_turn_ids.append(current.owner_turn_id)
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index cb0f17be7..e87fc1c34 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -102,6 +102,7 @@ from skyvern.forge.sdk.copilot.tools.credential_deletion import credential_delet
from skyvern.forge.sdk.copilot.tools.workflow_update import (
_validated_pending_workflow_proposal,
private_workflow_settings_from_proposal,
+ proposal_is_displayable,
proposal_workflow_fingerprint_matches,
strip_copilot_yaml_headers,
)
@@ -1095,7 +1096,7 @@ async def _persist_proposed_workflow_state(
chat: Any, agent_result: AgentResult, restored: bool, keep_pending_proposal: bool = False
) -> None:
updated_workflow = agent_result.updated_workflow
- if not await _proposal_ownership_is_current(chat, agent_result):
+ if agent_result.proposal_adopted or not await _proposal_ownership_is_current(chat, agent_result):
_discard_superseded_proposal(agent_result)
return
auto_accept_effective = _effective_auto_accept(chat.auto_accept, agent_result)
@@ -1601,6 +1602,7 @@ async def _persist_cancel_turn(
agent_result.updated_workflow is None
and chat.proposed_workflow is not None
and not effective_keep_pending_proposal
+ and not agent_result.proposal_adopted
):
await asyncio.shield(_clear_proposed_workflow(chat))
else:
@@ -2591,6 +2593,9 @@ async def _new_copilot_chat_post(
# the row holds now, so its clear would take a candidate another turn published.
recovered_result.proposal_owner_turn_id = agent_result.proposal_owner_turn_id
recovered_result.proposal_revision = agent_result.proposal_revision
+ recovered_result.proposal_adopted = agent_result.proposal_adopted
+ elif turn_ctx_sink and turn_ctx_sink[0].proposal_adopted:
+ recovered_result.proposal_adopted = True
LOG.error(
translated_log_message,
organization_id=organization.organization_id,
@@ -3908,22 +3913,15 @@ async def _history_proposal_state(
if not isinstance(chat.proposed_workflow, dict):
return None, None, None, claim_expires_in
metadata = copilot_proposal_metadata(chat.proposed_workflow)
- if metadata is None:
- # Safe to show unfingerprinted: Accept gives a tokenless proposal a token before it writes.
- return chat.proposed_workflow, None, None, claim_expires_in
- # A live claim is reported, not hidden. A server that dies mid-accept leaves the claim behind
- # for the length of the lease, and withholding the card for that long is the invisible-work
- # problem this store exists to remove; Accept meanwhile answers 409 and the client refetches.
- canonical = await app.DATABASE.workflows.get_workflow_by_permanent_id(
- workflow_permanent_id=chat.workflow_permanent_id,
- organization_id=organization_id,
- )
- if canonical is None or not proposal_workflow_fingerprint_matches(canonical, metadata.canonical_fingerprint):
+ if not await proposal_is_displayable(chat.proposed_workflow, chat.workflow_permanent_id, organization_id):
# Canonical moved, so the proposal is no longer displayable - but a concurrent Accept may
# still be writing under this claim, and that is exactly when a client must not save over
# it. Hiding the proposal here MUST NOT hide the claim; reporting no claim while one is
# live is not ambiguity, it is a client-trusted lie.
return None, None, None, claim_expires_in
+ if metadata is None:
+ # Safe to show unfingerprinted: Accept gives a tokenless proposal a token before it writes.
+ return chat.proposed_workflow, None, None, claim_expires_in
run_facts = await _proposal_run_facts(metadata, chat.workflow_permanent_id, organization_id)
return chat.proposed_workflow, metadata, run_facts, claim_expires_in
diff --git a/tests/unit/_mcp_browser_fakes.py b/tests/unit/_mcp_browser_fakes.py
index 997642508..8893b2a0a 100644
--- a/tests/unit/_mcp_browser_fakes.py
+++ b/tests/unit/_mcp_browser_fakes.py
@@ -4,20 +4,23 @@ import asyncio
from collections import deque
from collections.abc import Awaitable, Callable
from types import SimpleNamespace
-from typing import Any
+from typing import TYPE_CHECKING, Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from skyvern.cli.core import browser_ops
from skyvern.cli.core.result import BrowserContext
-from skyvern.cli.core.session_manager import SessionState
+from skyvern.cli.core.session_manager import RequestFailure, RequestFailureLog, SessionState
from skyvern.cli.mcp_tools import browser as mcp_browser
from skyvern.cli.mcp_tools import tabs as mcp_tabs
from skyvern.core.script_generations.skyvern_page import SkyvernPage
from skyvern.exceptions import StaleFrameSelectionError
from skyvern.webeye import action_deadline
+if TYPE_CHECKING:
+ from skyvern.forge.sdk.copilot.runtime import AgentContext
+
HANGING_PAGE_URL = "https://fixture.test/metrics"
HANGING_BOUND_MS = 200
HANGING_HEADROOM_MS = 50
@@ -57,6 +60,38 @@ def make_session_state(**overrides: Any) -> SessionState:
return state
+def make_request_failure(
+ page: object,
+ path: str = "/report",
+ *,
+ resource_type: str = "xhr",
+ method: str = "POST",
+ failure_text: str = "net::ERR_EMPTY_RESPONSE",
+ url: str | None = None,
+ page_url: str = "https://app.test/",
+) -> RequestFailure:
+ return RequestFailure(
+ url=url or f"https://app.test{path}",
+ method=method,
+ resource_type=resource_type,
+ failure_text=failure_text,
+ page_url=page_url,
+ tab_id=str(id(page)),
+ timestamp=0.0,
+ )
+
+
+def seed_request_failures(ctx: AgentContext, session_id: str, *entries: RequestFailure) -> RequestFailureLog:
+ log = ctx.request_failure_logs_by_browser_session.setdefault(session_id, RequestFailureLog())
+ for entry in entries:
+ log.append(entry)
+ return log
+
+
+def logged_request_failures(log: RequestFailureLog) -> list[RequestFailure]:
+ return [entry for tab in log.entries.values() for entry in tab]
+
+
def make_page(raw: MagicMock | None = None) -> SimpleNamespace:
if raw is None:
raw = MagicMock()
diff --git a/tests/unit/_socket_closing_fixture.py b/tests/unit/_socket_closing_fixture.py
new file mode 100644
index 000000000..c1db36767
--- /dev/null
+++ b/tests/unit/_socket_closing_fixture.py
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from aiohttp import web
+from playwright.async_api import Page, async_playwright
+
+BODY_SENTINEL = "BODY-SENTINEL-7f3a"
+_FIXTURE_PAGE = f"""<html><body>
+<img src="/broken.png">
+<button id="view" onclick="view()">View</button>
+<button id="fast" onclick="fetch('/ok')">Fast</button>
+<button id="slow" onclick="fetch('/slow')">Slow</button>
+<script>
+function view() {{
+ const xhr = new XMLHttpRequest();
+ xhr.open("POST", "/report?token=request-secret&sessionid=sess-secret#frag");
+ xhr.send("{BODY_SENTINEL}");
+}}
+</script>
+</body></html>"""
+
+
+async def _fixture_index(_request: web.Request) -> web.Response:
+ return web.Response(text=_FIXTURE_PAGE, content_type="text/html")
+
+
+async def _fixture_ok(request: web.Request) -> web.Response:
+ if request.path == "/slow":
+ await asyncio.sleep(20)
+ return web.Response(text="ok")
+
+
+async def _fixture_close_socket(request: web.Request) -> web.Response:
+ await request.read()
+ assert request.transport is not None
+ request.transport.close()
+ return web.Response()
+
+
+@asynccontextmanager
+async def socket_closing_fixture_page() -> AsyncIterator[tuple[Page, str]]:
+ app = web.Application()
+ app.router.add_get("/", _fixture_index)
+ app.router.add_get("/ok", _fixture_ok)
+ app.router.add_get("/slow", _fixture_ok)
+ app.router.add_get("/broken.png", _fixture_close_socket)
+ app.router.add_post("/report", _fixture_close_socket)
+ # aiohttp keeps a disconnected client's handler running by default, so aborted /slow fetches would hold cleanup.
+ runner = web.AppRunner(app, handler_cancellation=True)
+ await runner.setup()
+ site = web.TCPSite(runner, "127.0.0.1", 0)
+ await site.start()
+ try:
+ async with async_playwright() as playwright:
+ browser = await playwright.chromium.launch(headless=True)
+ try:
+ page = await browser.new_page()
+ yield page, f"http://127.0.0.1:{runner.addresses[0][1]}"
+ finally:
+ await browser.close()
+ finally:
+ await runner.cleanup()
diff --git a/tests/unit/forge/sdk/copilot/test_browser_code_tool.py b/tests/unit/forge/sdk/copilot/test_browser_code_tool.py
index d13c48326..6f29b6f66 100644
--- a/tests/unit/forge/sdk/copilot/test_browser_code_tool.py
+++ b/tests/unit/forge/sdk/copilot/test_browser_code_tool.py
@@ -51,6 +51,7 @@ from skyvern.forge.sdk.copilot.tools import (
from skyvern.forge.sdk.copilot.workflow_yaml import stored_block_code
from skyvern.webeye.browser_errors import BrowserCdpConnectionError
from skyvern.webeye.persistent_sessions_manager import BrowserRetirementReason
+from tests.unit._mcp_browser_fakes import logged_request_failures, make_request_failure, seed_request_failures
from tests.unit.conftest import make_copilot_context
from tests.unit.copilot_test_helpers import (
AUTO_RENDER_TURNSTILE_HTML,
@@ -1852,14 +1853,20 @@ async def test_a_last_run_browser_code_call_reads_the_run_browser(monkeypatch: p
@pytest.mark.asyncio
@pytest.mark.parametrize("code", [pytest.param(" ", id="refused"), pytest.param("1", id="ran-with-source-reference")])
-async def test_a_browser_code_result_the_scrub_cannot_redact_is_withheld(
- monkeypatch: pytest.MonkeyPatch, code: str
[email protected]("failures_redactable", [True, False])
+async def test_a_browser_code_result_the_scrub_cannot_redact_is_withheld_and_carries_only_redactable_failures(
+ monkeypatch: pytest.MonkeyPatch, code: str, failures_redactable: bool
) -> None:
_patch_cell_runtime(monkeypatch)
ctx = make_copilot_context()
ctx.browser_session_id = "pbs_scrub_emptied"
register_secret_scrub_value(ctx, "ok")
- patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(_titled_page("Orders")))
+ if not failures_redactable:
+ register_secret_scrub_value(ctx, "resource_type")
+ working_page = _titled_page("Orders")
+ patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(working_page))
+ failure = make_request_failure(working_page)
+ log = seed_request_failures(ctx, "pbs_scrub_emptied", failure)
try:
result = await _invoke_advertised_tool(ctx, {"code": code})
finally:
@@ -1867,4 +1874,6 @@ async def test_a_browser_code_result_the_scrub_cannot_redact_is_withheld(
assert mcp_adapter.is_redaction_withheld(result) and result["ok"] is False
assert result["data"]["redaction_withheld"]["reason"] == "redaction_unavailable"
- assert set(result) == {"page_state", "ok", "error", "data"}
+ reported = ["page_request_failures"] if failures_redactable else []
+ assert list(result) == ["page_state", *reported, "ok", "error", "data"]
+ assert logged_request_failures(log) == ([] if failures_redactable else [failure])
diff --git a/tests/unit/test_aws_client_ttl.py b/tests/unit/test_aws_client_ttl.py
index 464d6b601..819dbb486 100644
--- a/tests/unit/test_aws_client_ttl.py
+++ b/tests/unit/test_aws_client_ttl.py
@@ -55,6 +55,19 @@ async def test_download_file_missing_key_returns_none_without_traceback() -> Non
mock_log.exception.assert_not_called()
+async def test_download_file_archived_object_returns_none_with_a_warning_and_no_traceback() -> None:
+ client = aws.AsyncAWSClient()
+ with (
+ patch.object(client, "_s3_with_retry", AsyncMock(side_effect=_client_error("InvalidObjectState"))),
+ patch.object(aws, "LOG") as mock_log,
+ ):
+ result = await client.download_file("s3://bucket/archived.png")
+ assert result is None
+ mock_log.exception.assert_not_called()
+ mock_log.warning.assert_called_once()
+
+
@pytest.mark.asyncio
async def test_download_file_real_error_still_logs_exception() -> None:
client = aws.AsyncAWSClient()
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 2af0ec7d9..acfac57ab 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -4,7 +4,7 @@ import asyncio
import copy
import hashlib
import json
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Coroutine
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
@@ -13,6 +13,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import ValidationError
+from sqlalchemy.ext.asyncio import AsyncEngine
+from structlog.testing import capture_logs
from skyvern.forge import app as forge_app
from skyvern.forge.failure_classifier import with_sign_in_form_visible
@@ -23,8 +25,10 @@ from skyvern.forge.sdk.copilot.agent import (
_RECORDED_PAGE_TEXT_SECURITY_BOUNDARY,
_build_dynamic_system_prompt,
_build_user_context,
+ _build_wip_exit_result,
_prior_run_debug_text,
_recorded_build_test_outcome_prompt,
+ _review_projection_for,
)
from skyvern.forge.sdk.copilot.build_test_connect_failure import build_test_connect_failure_sentence
from skyvern.forge.sdk.copilot.build_test_outcome import (
@@ -64,9 +68,11 @@ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerifica
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.enforcement import _summarize_tool_output
+from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
from skyvern.forge.sdk.copilot.output_utils import (
_BUILD_TEST_PACKET_MAX_CHARS,
_INTERNAL_RUN_OUTCOME_RECORDED_KEY,
+ BUILD_TEST_PACKET_KEY,
_compact_packet_for_aggregate_limit,
_compacted_newest_labelled,
labelled_url_screen_reasons,
@@ -74,6 +80,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
project_direct_test_handoff_packet_for_llm,
sanitize_tool_result_for_llm,
)
+from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
build_test_page_state_from_evidence,
@@ -81,9 +88,10 @@ from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
)
from skyvern.forge.sdk.copilot.secret_scrub import clear_session_scrub_values, register_secret_scrub_value
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
+from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.tools.composition_capture import store_post_run_page_evidence
from skyvern.forge.sdk.copilot.tools.run_execution import (
- COPILOT_PROPOSAL_CHANGED_ERROR,
+ COPILOT_PROPOSAL_SETTINGS_CHANGED_ERROR,
_authored_literal_locator_selectors,
_carry_prior_attempt_change_identity_into_result,
_failed_block_code,
@@ -100,9 +108,22 @@ from skyvern.forge.sdk.copilot.tools.run_execution import (
project_run_results_page,
with_loop_observations,
)
-from skyvern.forge.sdk.copilot.workflow_yaml import runner_code_block_associations
+from skyvern.forge.sdk.copilot.tools.workflow_update import (
+ _update_workflow,
+ publish_workflow_candidate,
+ restore_pending_workflow_proposal,
+)
+from skyvern.forge.sdk.copilot.workflow_yaml import runner_code_block_associations, workflow_to_copilot_yaml
+from skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDB
from skyvern.forge.sdk.db.exceptions import CopilotProposalConflictError
+from skyvern.forge.sdk.db.repositories.workflow_parameters import WorkflowParametersRepository
+from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
+from skyvern.forge.sdk.schemas.workflow_copilot import (
+ COPILOT_PRIVATE_SETTINGS_KEY,
+ CopilotProposalMetadata,
+ copilot_proposal_metadata,
+)
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import CodeBlock, ForLoopBlock
@@ -115,6 +136,8 @@ from skyvern.forge.sdk.workflow.models.parameter import (
)
from skyvern.forge.sdk.workflow.models.workflow import (
COPILOT_TEST_WORKFLOW_CREATOR,
+ Workflow,
+ WorkflowDefinition,
WorkflowRunStatus,
workflow_definition_sha256,
)
@@ -5499,8 +5522,11 @@ async def test_the_repaired_block_executes_and_the_run_owns_the_outputs_it_was_a
@pytest.mark.asyncio
[email protected](("token_moved", "source_moved"), [(True, True), (False, True), (True, False)])
async def test_candidate_run_bind_uses_the_call_owned_source_and_token(
monkeypatch: pytest.MonkeyPatch,
+ token_moved: bool,
+ source_moved: bool,
) -> None:
ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")
ctx.workflow_copilot_chat_id = "chat-1"
@@ -5511,12 +5537,15 @@ async def test_candidate_run_bind_uses_the_call_owned_source_and_token(
async def bind_candidate_run(**kwargs: object) -> None:
captured.update(kwargs)
- raise CopilotProposalConflictError("candidate changed")
+ raise CopilotProposalConflictError("candidate changed", reason="token_changed")
async def prepare_workflow(**_kwargs: object) -> SimpleNamespace:
- ctx.proposal_owner_turn_id = "turn-newer"
- ctx.proposal_revision = 1
- ctx.staged_workflow = ctx.staged_workflow.model_copy(update={"title": "Newer candidate"})
+ if token_moved:
+ ctx.proposal_owner_turn_id = "turn-newer"
+ ctx.proposal_revision = 1
+ if source_moved:
+ ctx.staged_workflow = ctx.staged_workflow.model_copy(update={"title": "Newer candidate"})
+ captured["ctx_at_bind"] = (ctx.proposal_owner_turn_id, ctx.proposal_revision, ctx.staged_workflow)
return SimpleNamespace(
workflow_run_id="wr_source",
workflow_id="w_source",
@@ -5526,18 +5555,17 @@ async def test_candidate_run_bind_uses_the_call_owned_source_and_token(
forge_app.DATABASE.workflow_params.bind_workflow_copilot_candidate_run = bind_candidate_run
forge_app.DATABASE.workflows.soft_delete_workflow_by_id = AsyncMock()
- forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final = AsyncMock()
+ forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final = AsyncMock()
monkeypatch.setattr(workflow_service_module, "prepare_workflow", prepare_workflow)
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
- assert result == {
- "ok": False,
- "error": COPILOT_PROPOSAL_CHANGED_ERROR,
- }
+ assert (result["ok"], result["conflict_reason"], result["adoption"]) == (False, "token_changed", "kept")
+ assert result["data"] == {"run_dispatched": False, "proposal_bind_refused": True}
assert captured["expected_owner_turn_id"] == "turn-source"
assert captured["expected_revision"] == 3
assert captured["expected_workflow_yaml"] == HANDBACK_WORKFLOW_YAML
+ assert (ctx.proposal_owner_turn_id, ctx.proposal_revision, ctx.staged_workflow) == captured["ctx_at_bind"]
@pytest.mark.asyncio
@@ -5568,6 +5596,446 @@ async def test_canonical_run_does_not_bind_a_stale_proposal_token(
assert harness["executor_cancelled"] is False
+_NEWER_CANDIDATE_YAML = HANDBACK_WORKFLOW_YAML.replace("title: extraction example", "title: newer example")
+_STORED_CODE_ARTIFACT_METADATA = {"extract_heading": {"goal_value_paths": ["heading"]}}
+_OTHER_TURN_PRIVATE_SETTINGS = {
+ "totp_identifier": "[email protected]",
+ "webhook_callback_url": "https://hooks.test/done",
+}
+
+
+async def _install_real_candidate_cas(
+ monkeypatch: pytest.MonkeyPatch,
+ sqlite_engine: AsyncEngine,
+ *,
+ blockless_saved_workflow: bool = False,
+) -> tuple[dict[str, Any], WorkflowParametersRepository, str]:
+ harness = await install_run_blocks_harness(
+ monkeypatch,
+ workflow_yaml=HANDBACK_WORKFLOW_YAML,
+ polled_status="completed",
+ terminal_blocks=[terminal_extraction_block("completed")],
+ )
+ saved = harness["workflow"]
+ if blockless_saved_workflow:
+ saved = saved.model_copy(update={"workflow_definition": WorkflowDefinition(parameters=[], blocks=[])})
+ forge_app.DATABASE.workflows.get_workflow = AsyncMock(return_value=saved)
+ forge_app.DATABASE.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=saved)
+ forge_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=saved)
+ repo = WorkflowParametersRepository(BaseAlchemyDB(sqlite_engine).Session)
+ forge_app.DATABASE.workflow_params = repo
+ forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final = AsyncMock()
+ chat = await repo.create_workflow_copilot_chat(organization_id="org-1", workflow_permanent_id="wfp-1")
+ turn_one = make_copilot_ctx(workflow_copilot_chat_id=chat.workflow_copilot_chat_id, turn_id="turn-one")
+ await publish_workflow_candidate(turn_one, workflow=harness["workflow"], workflow_yaml=HANDBACK_WORKFLOW_YAML)
+ return harness, repo, chat.workflow_copilot_chat_id
+
+
+async def _restored_turn(chat_id: str, turn_id: str, **overrides: object) -> CopilotContext:
+ ctx = make_copilot_ctx(
+ browser_session_id="pbs_chat", workflow_copilot_chat_id=chat_id, turn_id=turn_id, **overrides
+ )
+ await restore_pending_workflow_proposal(ctx)
+ return ctx
+
+
+async def _run_extract_heading(ctx: CopilotContext) -> dict[str, Any]:
+ ctx.frontier_resume_session_id = "pbs_run"
+ return await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+
+async def _publish_from_another_turn(chat_id: str, code_artifact_metadata: dict[str, Any] | None = None) -> None:
+ turn_b = await _restored_turn(chat_id, "turn-b")
+ turn_b.code_artifact_metadata = code_artifact_metadata
+ turn_b.authored_private_workflow_settings = dict(_OTHER_TURN_PRIVATE_SETTINGS)
+ assert turn_b.staged_workflow is not None
+ newer = turn_b.staged_workflow.model_copy(update={"title": "newer example"})
+ await publish_workflow_candidate(turn_b, workflow=newer, workflow_yaml=_NEWER_CANDIDATE_YAML)
+
+
+async def _stored_candidate(
+ repo: WorkflowParametersRepository, chat_id: str
+) -> tuple[dict[str, Any], CopilotProposalMetadata]:
+ chat = await repo.get_workflow_copilot_chat_by_id(organization_id="org-1", workflow_copilot_chat_id=chat_id)
+ assert chat is not None and isinstance(chat.proposed_workflow, dict)
+ metadata = copilot_proposal_metadata(chat.proposed_workflow)
+ assert metadata is not None
+ return chat.proposed_workflow, metadata
+
+
+async def _next_model_response(ctx: CopilotContext) -> None:
+ await CopilotRunHooks(ctx).on_llm_end(MagicMock(), MagicMock(), MagicMock(output=[]))
+
+
+def _bind_lost_logs(logs: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ return [entry for entry in logs if entry["event"] == "copilot candidate run bind lost"]
+
+
[email protected]("case", ["as_stored", "other_settings", "cleared", "saved_changed"])
+async def test_candidate_run_bind_after_a_rejected_restore_dispatches_only_the_stored_prior_draft(
+ monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, case: str
+) -> None:
+ _harness, repo, chat_id = await _install_real_candidate_cas(
+ monkeypatch, sqlite_engine, blockless_saved_workflow=True
+ )
+ stored, _ = await _stored_candidate(repo, chat_id)
+ if case == "saved_changed":
+ changed = forge_app.DATABASE.workflows.get_workflow_by_permanent_id.return_value.model_copy(
+ update={"description": "edited after the proposal"}
+ )
+ forge_app.DATABASE.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=changed)
+ ctx = await _restored_turn(
+ chat_id,
+ "turn-two",
+ workflow_yaml=HANDBACK_WORKFLOW_YAML.replace("extract_heading", "user_edited_block"),
+ prior_copilot_workflow_yaml=stored["_copilot_yaml"],
+ private_workflow_settings={"webhook_callback_url": "https://hooks.test/request"}
+ if case == "other_settings"
+ else {},
+ )
+ assert (ctx.staged_workflow, ctx.proposal_owner_turn_id, ctx.proposal_revision) == (None, "turn-one", 1)
+ if case == "cleared":
+ await repo.clear_workflow_copilot_candidate(
+ "org-1", chat_id, expected_owner_turn_id="turn-one", expected_revision=1
+ )
+ settings_before = copy.deepcopy(ctx.private_workflow_settings)
+
+ with capture_logs() as logs:
+ first = await _run_extract_heading(ctx)
+
+ assert first["ok"] is (case == "as_stored"), first
+ assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == (1 if case == "as_stored" else 0)
+ if case == "saved_changed":
+ assert (first["conflict_reason"], first["adoption"]) == ("bytes_changed", "canonical_fingerprint_mismatch")
+ elif case == "cleared":
+ assert (first["conflict_reason"], first["adoption"]) == ("candidate_missing", "cleared")
+ assert [entry["reason"] for entry in _bind_lost_logs(logs)] == ["candidate_missing"]
+ await _next_model_response(ctx)
+ assert (await _run_extract_heading(ctx))["ok"] is False
+ assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 0
+ elif case == "other_settings":
+ second = await _run_extract_heading(ctx)
+ assert [first, second] == [{"ok": False, "error": COPILOT_PROPOSAL_SETTINGS_CHANGED_ERROR}] * 2
+ workflow_service_module.prepare_workflow.assert_not_awaited()
+ assert ctx.private_workflow_settings == settings_before
+ assert (ctx.proposal_owner_turn_id, ctx.proposal_revision, ctx.staged_workflow) == ("turn-one", 1, None)
+ assert (await _stored_candidate(repo, chat_id))[1].workflow_run_id is None
+ else:
+ assert (await _stored_candidate(repo, chat_id))[1].workflow_run_id == "wr_paused"
+
+
+async def test_candidate_run_bind_and_publish_stay_refused_while_an_accept_claim_is_live(
+ monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine
+) -> None:
+ _harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
+ ctx = await _restored_turn(chat_id, "turn-a")
+ passed = await _run_extract_heading(ctx)
+ assert passed["ok"] is True, passed
+ await repo.claim_workflow_copilot_candidate(
+ "org-1", chat_id, expected_owner_turn_id="turn-one", expected_revision=1
+ )
+ claimed_before, _ = await _stored_candidate(repo, chat_id)
+ staged_before, outcome_before, trace_before = ctx.staged_workflow, ctx.last_run_outcome, list(ctx.run_outcome_trace)
+ ctx.last_test_ok, ctx.frontier_resume_session_id = True, "pbs_run"
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args: None)
+ monkeypatch.setattr(run_execution_module, "_authority_tool_error", lambda *_args: None)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ tools_module, "_plan_frontier", lambda *_args: (["extract_heading"], {}, "extract_heading", "initial")
+ )
+ workflow_service_module.prepare_workflow.return_value = SimpleNamespace(
+ workflow_run_id="wr_refused",
+ workflow_id="w_source",
+ browser_session_id="pbs_run",
+ sequential_credential_id=None,
+ )
+ tool_context = SimpleNamespace(context=ctx, tool_name="run_blocks_and_collect_debug")
+ arguments = json.dumps({"block_labels": ["extract_heading"], "parameters": {}})
+ edit = HANDBACK_WORKFLOW_YAML.replace("Extract the page heading.", "Extract the page title.")
+
+ with capture_logs() as logs:
+ runs = [json.loads(await tools_module.run_blocks_tool.on_invoke_tool(tool_context, arguments))]
+ runs.append(await _run_extract_heading(ctx))
+ edits = [await _update_workflow({"workflow_yaml": edit}, ctx, allow_missing_credentials=True) for _ in "ab"]
+
+ assert [(result["ok"], result["conflict_reason"], result["adoption"]) for result in runs + edits] == [
+ (False, "being_accepted", "claimed")
+ ] * 4
+ assert BUILD_TEST_PACKET_KEY not in runs[0]["data"]
+ assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 1
+ assert [(entry["reason"], entry["adoption"]) for entry in _bind_lost_logs(logs)] == [
+ ("being_accepted", "claimed")
+ ] * 2
+ assert (ctx.proposal_owner_turn_id, ctx.proposal_revision, ctx.staged_workflow) == ("turn-one", 1, staged_before)
+ assert (ctx.last_test_ok, ctx.last_run_outcome, ctx.run_outcome_trace) == (True, outcome_before, trace_before)
+ assert (await _stored_candidate(repo, chat_id))[0] == claimed_before
+
+
+ ("reason", "event"),
+ [("bytes_changed", "copilot candidate run bind lost"), ("unexpected_error", "copilot candidate run bind failed")],
+)
+async def test_candidate_run_bind_refusal_logs_its_reason_and_dispatches_nothing(
+ monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, reason: str, event: str
+) -> None:
+ _harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
+ ctx = await _restored_turn(chat_id, "turn-a")
+ if reason == "bytes_changed":
+ ctx.staged_workflow_yaml = HANDBACK_WORKFLOW_YAML.replace("https://example.com", "https://example.org")
+ else:
+ monkeypatch.setattr(repo, "bind_workflow_copilot_candidate_run", AsyncMock(side_effect=RuntimeError("down")))
+
+ with capture_logs() as logs:
+ result = await _run_extract_heading(ctx)
+
+ assert (result["ok"], result.get("conflict_reason")) == (False, reason if reason == "bytes_changed" else None)
+ assert result["data"] == {"run_dispatched": False, "proposal_bind_refused": True}
+ assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 0
+ bind_logs = [entry for entry in logs if entry["event"].startswith("copilot candidate run bind")]
+ assert [(entry["event"], entry["reason"]) for entry in bind_logs] == [(event, reason)]
+ assert (await _stored_candidate(repo, chat_id))[1].workflow_run_id is None
+
+
+_ACCEPTED_WEBHOOK = "https://hooks.example.test/accepted"
+
+
+async def _accept_concurrently(harness: dict[str, Any], repo: WorkflowParametersRepository, chat_id: str) -> Workflow:
+ accepted = harness["workflow"].model_copy(
+ update={"workflow_id": "w_accepted", "webhook_callback_url": _ACCEPTED_WEBHOOK}
+ )
+ versions = {harness["workflow"].workflow_id: harness["workflow"], accepted.workflow_id: accepted}
+ forge_app.DATABASE.workflows.get_workflow = AsyncMock(
+ side_effect=lambda workflow_id, organization_id: versions.get(workflow_id)
+ )
+ forge_app.DATABASE.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=accepted)
+ forge_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=accepted)
+ await repo.clear_workflow_copilot_candidate(
+ "org-1", chat_id, expected_owner_turn_id="turn-one", expected_revision=1
+ )
+ return accepted
+
+
[email protected]("path", ["test", "edit"])
[email protected]("concurrent", ["publish", "accept"])
+async def test_stale_proposal_token_conflict_recovers_so_the_next_call_succeeds(
+ monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, path: str, concurrent: str
+) -> None:
+ harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
+ parameters = harness["workflow"].workflow_definition.parameters
+ outputs = [parameter for parameter in parameters if isinstance(parameter, OutputParameter)]
+ monkeypatch.setattr(repo, "get_workflow_output_parameters", AsyncMock(return_value=outputs))
+ ctx = await _restored_turn(
+ chat_id,
+ "turn-a",
+ workflow_id=harness["workflow"].workflow_id,
+ private_workflow_settings={"webhook_callback_url": "https://hooks.example.test/stale"}
+ if path == "edit"
+ else {},
+ )
+ ctx.executed_block_fingerprints = workflow_block_fingerprints(HANDBACK_WORKFLOW_YAML)
+ ctx.verified_prefix_labels = ["extract_heading"]
+ ctx.workflow_verification_evidence.full_workflow_verified = True
+ if concurrent == "publish":
+ await _publish_from_another_turn(chat_id, code_artifact_metadata=_STORED_CODE_ARTIFACT_METADATA)
+ expected = ("token_changed", "adopted")
+ else:
+ accepted = await _accept_concurrently(harness, repo, chat_id)
+ expected = ("candidate_missing", "cleared")
+ edit = HANDBACK_WORKFLOW_YAML.replace("Extract the page heading.", "Extract the page title.")
+
+ async def test_or_edit() -> dict[str, Any]:
+ if path == "test":
+ return await _run_extract_heading(ctx)
+ return await _update_workflow({"workflow_yaml": edit}, ctx, allow_missing_credentials=True)
+
+ with capture_logs() as logs:
+ first = await test_or_edit()
+
+ assert (first["ok"], first["conflict_reason"], first["adoption"]) == (False, *expected)
+ conflict_events = {"copilot candidate run bind lost", "copilot_candidate_publication_conflict"}
+ assert [(entry["reason"], entry["adoption"]) for entry in logs if entry["event"] in conflict_events] == [expected]
+ assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 0
+ if concurrent == "publish":
+ assert first["adopted_workflow_yaml"] == _NEWER_CANDIDATE_YAML
+ assert (ctx.proposal_owner_turn_id, ctx.proposal_revision) == ("turn-b", 1)
+ assert ctx.code_artifact_metadata == _STORED_CODE_ARTIFACT_METADATA
+ assert (ctx.verified_prefix_labels, ctx.workflow_verification_evidence.full_workflow_verified) == ([], False)
+ review = _review_projection_for(ctx, ctx.staged_workflow_yaml)
+ assert review is not None and [block["neverTested"] for block in review["blocks"]] == [True]
+ else:
+ assert ctx.workflow_yaml == workflow_to_copilot_yaml(accepted)
+
+ await _next_model_response(ctx)
+ second = await test_or_edit()
+
+ assert second["ok"] is True, second
+ if path == "test":
+ assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 1
+ if concurrent == "accept":
+ resolved_workflow_id = workflow_service_module.prepare_workflow.await_args.kwargs["resolved_workflow_id"]
+ assert resolved_workflow_id == accepted.workflow_id
+ return
+ create_draft = forge_app.WORKFLOW_SERVICE.create_copilot_dispatch_draft_version
+ dispatched = create_draft.await_args.kwargs["runtime_workflow"]
+ assert [dispatched.totp_identifier, dispatched.webhook_callback_url] == [*_OTHER_TURN_PRIVATE_SETTINGS.values()]
+ return
+ stored, metadata = await _stored_candidate(repo, chat_id)
+ assert (metadata.owner_turn_id, metadata.revision) == ("turn-a", 1)
+ assert "Extract the page title." in stored["_copilot_yaml"]
+ if concurrent == "publish":
+ assert stored[COPILOT_PRIVATE_SETTINGS_KEY] == _OTHER_TURN_PRIVATE_SETTINGS
+ else:
+ assert ctx.staged_workflow is not None and ctx.staged_workflow.webhook_callback_url == _ACCEPTED_WEBHOOK
+ assert stored.get("webhook_callback_url") is None
+
+
+ ("first_call", "second_call", "overlap"),
+ [
+ ("edit", "edit", True),
+ ("test", "edit", True),
+ ("edit", "edit", False),
+ ("edit", "test", False),
+ ("test", "test", False),
+ ],
+)
+async def test_stale_proposal_token_adoption_refuses_a_call_written_before_it(
+ monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, first_call: str, second_call: str, overlap: bool
+) -> None:
+ harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
+ parameters = harness["workflow"].workflow_definition.parameters
+ outputs = [parameter for parameter in parameters if isinstance(parameter, OutputParameter)]
+ monkeypatch.setattr(repo, "get_workflow_output_parameters", AsyncMock(return_value=outputs))
+ ctx = await _restored_turn(chat_id, "turn-a", workflow_id=harness["workflow"].workflow_id)
+ await _publish_from_another_turn(chat_id)
+ stored_before, _ = await _stored_candidate(repo, chat_id)
+ edit = HANDBACK_WORKFLOW_YAML.replace("Extract the page heading.", "Extract the page title.")
+ stale_metadata = {"extract_heading": {"goal_value_paths": ["stale"]}}
+
+ def call(kind: str, metadata: object = None) -> Coroutine[Any, Any, dict[str, Any]]:
+ if kind == "test":
+ return _run_extract_heading(ctx)
+ params = {"workflow_yaml": edit, "code_artifact_metadata": metadata}
+ return _update_workflow(params, ctx, allow_missing_credentials=True)
+
+ if overlap:
+ results = await asyncio.gather(call(first_call), call(second_call, stale_metadata))
+ else:
+ # The second call of the same model response starts after the first adopted, as when a wrapper waits on
+ # a lock or the credential gate before reaching the write.
+ results = [await call(first_call)]
+ submission_before = (ctx.raw_code_artifact_metadata, ctx.code_artifact_metadata)
+ results.append(await call(second_call, stale_metadata))
+ assert (ctx.raw_code_artifact_metadata, ctx.code_artifact_metadata) == submission_before
+
+ assert [(result["ok"], result["conflict_reason"], result["adoption"]) for result in results] == [
+ (False, "token_changed", "adopted")
+ ] * 2
+ assert (await _stored_candidate(repo, chat_id))[0] == stored_before
+ assert forge_app.WORKFLOW_SERVICE.execute_workflow.call_count == 0
+
+
[email protected]("dropped_by", ["in_flight_edit", "earlier_edit"])
+async def test_stale_proposal_token_adoption_keeps_the_saved_completion_contract(
+ monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, dropped_by: str
+) -> None:
+ harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
+ parameters = harness["workflow"].workflow_definition.parameters
+ outputs = [parameter for parameter in parameters if isinstance(parameter, OutputParameter)]
+ monkeypatch.setattr(repo, "get_workflow_output_parameters", AsyncMock(return_value=outputs))
+ ctx = await _restored_turn(chat_id, "turn-a", workflow_id=harness["workflow"].workflow_id)
+ ctx.code_artifact_metadata = {
+ "extract_heading": {"completion_criteria": [{"deliverable_kind": "generated_file", "id": "report"}]}
+ }
+ contract_dropping_edit = HANDBACK_WORKFLOW_YAML.replace(
+ " parameters: []\n",
+ " parameters: []\n completion_contract:\n schema_version: 1\n criteria:\n"
+ " - id: report\n kind: generated_file\n min_count: 1\n",
+ )
+ if dropped_by == "earlier_edit":
+ dropped = await _update_workflow({"workflow_yaml": contract_dropping_edit}, ctx, allow_missing_credentials=True)
+ assert dropped["ok"] is True and ctx.clear_persisted_completion_contract is True
+ await _publish_from_another_turn(chat_id)
+ adopting = await _run_extract_heading(ctx)
+ else:
+ await _publish_from_another_turn(chat_id)
+ sibling_adopted = asyncio.Event()
+ real_canonicalize = workflow_update_module.canonicalize_named_google_sheet_bindings
+
+ async def canonicalize_after_the_sibling_adopts(workflow_yaml: str, call_ctx: object) -> object:
+ # Holds the contract-dropping edit past its entry check until its sibling has adopted.
+ if "completion_contract" in workflow_yaml:
+ await sibling_adopted.wait()
+ return await real_canonicalize(workflow_yaml, call_ctx)
+
+ monkeypatch.setattr(
+ workflow_update_module, "canonicalize_named_google_sheet_bindings", canonicalize_after_the_sibling_adopts
+ )
+
+ async def sibling_edit() -> dict[str, Any]:
+ try:
+ return await _update_workflow(
+ {"workflow_yaml": HANDBACK_WORKFLOW_YAML}, ctx, allow_missing_credentials=True
+ )
+ finally:
+ sibling_adopted.set()
+
+ adopting, in_flight = await asyncio.gather(
+ sibling_edit(),
+ _update_workflow({"workflow_yaml": contract_dropping_edit}, ctx, allow_missing_credentials=True),
+ )
+ assert (in_flight["ok"], in_flight["adoption"]) == (False, "adopted")
+ assert (adopting["ok"], adopting["adoption"]) == (False, "adopted")
+
+ await _next_model_response(ctx)
+ edit = HANDBACK_WORKFLOW_YAML.replace("Extract the page heading.", "Extract the page title.")
+ assert (await _update_workflow({"workflow_yaml": edit}, ctx, allow_missing_credentials=True))["ok"] is True
+
+ assert (ctx.has_staged_proposal, ctx.clear_persisted_completion_contract) == (True, False)
+
+
[email protected]("exit_path", ["auto_accept", "cancel"])
+async def test_stale_proposal_token_adoption_leaves_the_stored_candidate_untouched_at_turn_end(
+ monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine, exit_path: str
+) -> None:
+ _harness, repo, chat_id = await _install_real_candidate_cas(monkeypatch, sqlite_engine)
+ turn_start_chat = await repo.update_workflow_copilot_chat(
+ organization_id="org-1", workflow_copilot_chat_id=chat_id, auto_accept=exit_path == "auto_accept"
+ )
+ assert turn_start_chat is not None
+ ctx = await _restored_turn(chat_id, "turn-a")
+ ctx.staged_workflow_yaml = HANDBACK_WORKFLOW_YAML.replace("https://example.com", "https://example.org")
+ conflict = await _run_extract_heading(ctx)
+ assert (conflict["conflict_reason"], conflict["adoption"]) == ("bytes_changed", "adopted")
+ await _next_model_response(ctx)
+ assert (await _run_extract_heading(ctx))["ok"] is True
+ stored_before, _ = await _stored_candidate(repo, chat_id)
+
+ if exit_path == "cancel":
+ await workflow_copilot_route._persist_cancel_turn(
+ stream=MagicMock(send=AsyncMock(return_value=True)),
+ chat=turn_start_chat,
+ organization_id="org-1",
+ original_workflow=None,
+ user_message="stop",
+ agent_result=_build_wip_exit_result(ctx, None, cancelled=True),
+ turn_id="turn-a",
+ )
+ else:
+ await workflow_copilot_route._persist_proposed_workflow_state(
+ turn_start_chat, _build_wip_exit_result(ctx, None), restored=False
+ )
+
+ assert (await _stored_candidate(repo, chat_id))[0] == stored_before
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("withheld_yaml", [False, True])
async def test_completed_run_is_recorded_before_the_browser_enrichment_await(
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 5a4bbdce0..de0322176 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -1098,7 +1098,7 @@ def test_an_evicted_browser_result_keeps_a_bounded_page_state(
assert json.loads(pruned[0]["output"])["page_state"] == expected
-def test_page_state_alone_does_not_push_a_short_result_into_compaction() -> None:
+def test_page_facts_alone_do_not_push_a_short_result_into_compaction() -> None:
solved = {
"ok": True,
"outcome": "solved",
@@ -1113,7 +1113,8 @@ def test_page_state_alone_does_not_push_a_short_result_into_compaction() -> None
"title": "Northfield Parts - Catalog",
"challenge_vendor": None,
}
- old_output = json.dumps({"page_state": page_state, **solved})
+ failures = [{"method": "POST", "resource_type": "xhr", "failure_text": "net::ERR_EMPTY_RESPONSE", "url": "/r"}]
+ old_output = json.dumps({"page_state": page_state, "page_request_failures": failures, **solved})
assert len(json.dumps(solved)) <= 300 < len(old_output)
items = [_fco("c_old", old_output), _reasoning()] + [
_fco(f"c{i}", '{"ok":true}') for i in range(KEEP_RECENT_TOOL_OUTPUTS)
diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.py
index 741be3b10..458ebc1ef 100644
--- a/tests/unit/test_copilot_frontier.py
+++ b/tests/unit/test_copilot_frontier.py
@@ -155,6 +155,7 @@ class _FakeDefinition:
class _FakeWorkflow:
def __init__(self, definition: _FakeDefinition) -> None:
self.workflow_definition = definition
+ self.proxy_location = None
def model_copy(self, *, deep: bool = False) -> _FakeWorkflow:
return copy.deepcopy(self) if deep else _FakeWorkflow(self.workflow_definition)
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index 526fcf1f8..b5ea117b8 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -18,6 +18,7 @@ import yaml
from structlog.testing import capture_logs
from skyvern.config import settings
+from skyvern.forge import app
from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.build_test_connect_failure import SUPERSEDED_BY_NEWER_TEST_REASON
from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_code_block
@@ -58,6 +59,7 @@ from skyvern.forge.sdk.copilot.tools.scouting import (
_summary_disclosure_control,
_summary_entry,
)
+from skyvern.forge.sdk.copilot.tools.workflow_update import proposal_workflow_fingerprint
from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, TurnHaltKind
from skyvern.webeye.persistent_sessions_manager import BrowserOperation, BrowserRetirement
from skyvern.webeye.utils import challenge_signature as challenge_signature_module
@@ -4115,3 +4117,41 @@ class TestSensitiveOriginActionContinuation:
assert not set(recorded) & set(scouting_module._RETAINED_LOCATOR_IDENTITY_FIELDS)
assert recorded["source_url"] == f"http://pathfold.test/app?code={REDACTED_SECRET_PLACEHOLDER}"
assert CONTINUATION_RUN_OTP not in json.dumps(ctx.scout_trajectory)
+
+
+async def test_schedule_pre_hook_only_blocks_chat_history_visible_proposals(monkeypatch: pytest.MonkeyPatch) -> None:
+ canonical = SimpleNamespace(model_dump=lambda **_: {"title": "Saved workflow"})
+ current_fingerprint = proposal_workflow_fingerprint(canonical)
+ monkeypatch.setattr(mcp_hooks, "_schedule_parameters_credential_pre_hook", AsyncMock(return_value=None))
+ ctx = make_copilot_ctx(workflow_copilot_chat_id="chat-test")
+
+ for fingerprint, disposition, expected_pending in (
+ ("", "accepting", False),
+ ("stale", "review_untested", False),
+ (None, None, True),
+ (current_fingerprint, "accepting", True),
+ ):
+ proposal: dict[str, Any] = {"_copilot_yaml": "title: Draft\n"}
+ if fingerprint is not None:
+ proposal["_copilot_proposal"] = {
+ "owner_turn_id": "turn-test",
+ "revision": 1,
+ "canonical_fingerprint": fingerprint,
+ "disposition": disposition,
+ }
+ chat = SimpleNamespace(proposed_workflow=proposal, workflow_permanent_id="wfp-test")
+ monkeypatch.setattr(
+ app.DATABASE.workflow_params,
+ "get_workflow_copilot_chat_by_id",
+ AsyncMock(return_value=chat),
+ )
+ monkeypatch.setattr(
+ app.DATABASE.workflows,
+ "get_workflow_by_permanent_id",
+ AsyncMock(return_value=canonical),
+ )
+
+ result = await mcp_hooks._create_workflow_schedule_pre_hook({}, ctx)
+
+ assert (result is not None) is expected_pending
diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.py
index 7d06c2b96..0d0185bed 100644
--- a/tests/unit/test_copilot_mcp_adapter.py
+++ b/tests/unit/test_copilot_mcp_adapter.py
@@ -4,7 +4,7 @@ import copy
import io
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
-from contextlib import asynccontextmanager
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
from dataclasses import replace
from functools import partial
from types import SimpleNamespace
@@ -21,16 +21,18 @@ from fastmcp import FastMCP
from mcp import Tool as MCPTool
from mcp.types import CallToolResult
from PIL import Image
-from playwright.async_api import Route
+from playwright.async_api import Page, Request, Route
from structlog.testing import capture_logs
from skyvern.cli.core import client as client_module
from skyvern.cli.core.client import get_active_api_key
from skyvern.cli.core.guards import VALID_WAIT_UNTIL
from skyvern.cli.core.result import ErrorCode, make_error
+from skyvern.cli.core.session_manager import RequestFailureLog, get_current_session
from skyvern.cli.mcp_tools import mcp
from skyvern.cli.mcp_tools._element_state import action_deadline_error, element_state_error
from skyvern.cli.mcp_tools.blocks import skyvern_block_validate, skyvern_workflow_knowledge
+from skyvern.cli.mcp_tools.inspection import ensure_hooks_on_all_pages
from skyvern.cli.mcp_tools.workflow import skyvern_workflow_get
from skyvern.forge.sdk.cache.base import NoopLock
from skyvern.forge.sdk.cache.local import LocalCache
@@ -62,7 +64,7 @@ from skyvern.forge.sdk.copilot.mcp_adapter import (
resolve_browser_session_binding,
)
from skyvern.forge.sdk.copilot.model_input_capture import serialize_tool_surface
-from skyvern.forge.sdk.copilot.output_utils import MCP_RESULT_PROVENANCE_KEY
+from skyvern.forge.sdk.copilot.output_utils import _RECENT_TOOL_OUTPUT_CHAR_CAP, MCP_RESULT_PROVENANCE_KEY
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
@@ -81,7 +83,7 @@ from skyvern.forge.sdk.copilot.secret_scrub import (
)
from skyvern.forge.sdk.copilot.tools import NATIVE_TOOLS, _with_action_reason, mcp_hooks, run_execution
from skyvern.forge.sdk.copilot.tools import scouting as scouting_module
-from skyvern.forge.sdk.copilot.tools._shared import _composition_get_structured_evidence_result
+from skyvern.forge.sdk.copilot.tools._shared import _composition_get_structured_evidence_result, on_working_page
from skyvern.forge.sdk.copilot.tools.banned_blocks import _block_authoring_violations
from skyvern.forge.sdk.copilot.tools.mcp_hooks import (
_FOR_LOOP_EXAMPLE,
@@ -98,6 +100,13 @@ from skyvern.webeye.persistent_sessions_manager import (
BrowserRetirement,
BrowserRetirementReason,
)
+from tests.unit._mcp_browser_fakes import (
+ logged_request_failures,
+ make_request_failure,
+ make_session_state,
+ seed_request_failures,
+)
+from tests.unit._socket_closing_fixture import BODY_SENTINEL, socket_closing_fixture_page
from tests.unit.copilot_test_helpers import (
AUTO_RENDER_TURNSTILE_HTML,
CATALOG_PAGE_HTML,
@@ -801,25 +810,21 @@ async def test_retirement_rollback_preserves_concurrent_sibling_evidence(
assert ctx.flow_evidence == [{"source": "sibling"}]
-async def test_browser_pre_hook_internal_read_reuses_the_outer_generation(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- ctx = make_copilot_ctx(browser_session_id="pbs_nested_hook")
- ctx.api_key = "test-api-key"
+def _attach_mocked_browser_session(
+ monkeypatch: pytest.MonkeyPatch, working_page: Page | None
+) -> tuple[MagicMock, list[str]]:
browser = MagicMock()
browser.is_connected.return_value = True
browser_context = MagicMock()
browser_context.browser = browser
browser_context._impl_obj = SimpleNamespace(_close_was_called=False, _closed=False)
browser_state = MagicMock(browser_context=browser_context)
- browser_state.get_working_page = AsyncMock(return_value=None)
- operation_entries = 0
+ browser_state.get_working_page = AsyncMock(return_value=working_page)
+ operation_entries: list[str] = []
@asynccontextmanager
async def _operation(_session_id: str, resolved_state: Any) -> AsyncIterator[BrowserOperation]:
- nonlocal operation_entries
- operation_entries += 1
+ operation_entries.append(_session_id)
yield BrowserOperation(resolved_state, BrowserRetirement())
manager = MagicMock()
@@ -831,15 +836,25 @@ async def test_browser_pre_hook_internal_read_reuses_the_outer_generation(
monkeypatch.setattr(copilot_runtime, "get_active_api_key", MagicMock(return_value="test-api-key"))
monkeypatch.setattr(copilot_runtime, "set_api_key_override", MagicMock(return_value=object()))
monkeypatch.setattr(copilot_runtime, "reset_api_key_override", MagicMock())
- register = MagicMock()
- unregister = MagicMock()
- monkeypatch.setattr(copilot_runtime, "register_copilot_session", register)
- monkeypatch.setattr(copilot_runtime, "unregister_copilot_session", unregister)
async def _prepared(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None, None]:
return None, None, None
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _prepared)
+ return manager, operation_entries
+
+
+async def test_browser_pre_hook_internal_read_reuses_the_outer_generation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_nested_hook")
+ ctx.api_key = "test-api-key"
+ manager, operation_entries = _attach_mocked_browser_session(monkeypatch, None)
+ register = MagicMock()
+ unregister = MagicMock()
+ monkeypatch.setattr(copilot_runtime, "register_copilot_session", register)
+ monkeypatch.setattr(copilot_runtime, "unregister_copilot_session", unregister)
server: SkyvernOverlayMCPServer
async def _pre_hook(_args: dict[str, Any], _ctx: AgentContext) -> None:
@@ -856,7 +871,7 @@ async def test_browser_pre_hook_internal_read_reuses_the_outer_generation(
result = await server.call_tool("evaluate", {"expression": "action()"})
assert result.isError is False
- assert operation_entries == 1
+ assert len(operation_entries) == 1
manager.get_browser_state.assert_awaited_once()
register.assert_called_once()
unregister.assert_called_once()
@@ -3242,7 +3257,7 @@ def _browser_evaluate_server(
ctx,
payload or {"ok": True},
overlay or SchemaOverlay(requires_browser=True),
- page_state_reader=scouting_module.read_page_state,
+ page_state_reader=scouting_module.read_page_state_on_page,
)
@@ -3356,7 +3371,7 @@ class TestPageStateOnBrowserResults:
_FakeClient({"ok": False, "error": "No element matches #go"}),
"click",
_build_skyvern_mcp_overlays()["click"],
- page_state_reader=scouting_module.read_page_state,
+ page_state_reader=scouting_module.read_page_state_on_page,
)
result = await server.call_tool("click", {"selector": "#go"})
@@ -3902,3 +3917,429 @@ async def test_a_failed_browser_action_reports_typed_facts_instead_of_the_server
)
assert surfaced.get("element_state") == element_state
assert server_error["hint"] not in result.content[0].text
+
+
+async def _rebuilding_post_hook(
+ copilot_result: dict[str, Any], _raw_mcp: dict[str, Any], _ctx: AgentContext
+) -> dict[str, Any]:
+ return {"ok": copilot_result["ok"], "data": copilot_result["data"]}
+
+
+def _tab(title: str) -> SimpleNamespace:
+ return SimpleNamespace(
+ url=f"https://app.test/{title.lower()}", frames=[], title=AsyncMock(return_value=title), is_closed=lambda: False
+ )
+
+
+def _scope_exiting_with(exc: BaseException) -> Callable[..., AbstractAsyncContextManager[None]]:
+ @asynccontextmanager
+ async def scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
+ yield
+ raise exc
+
+ return scope
+
+
+async def _evaluate_result(server: SkyvernOverlayMCPServer) -> dict[str, Any]:
+ result = json.loads((await server.call_tool("evaluate", {"expression": "read()"})).content[0].text)
+ if "page_request_failures" in result:
+ keys = list(result)
+ assert keys.index("page_request_failures") == keys.index("page_state") + 1
+ return result
+
+
+def _evaluate_browser_server(
+ ctx: AgentContext,
+ raw_result: dict[str, Any] | Exception,
+ on_call: Callable[[], None] | None = None,
+ overlay: SchemaOverlay | None = None,
+) -> SkyvernOverlayMCPServer:
+ overlay, reader = (
+ overlay or SchemaOverlay(requires_browser=True, timeout=1),
+ scouting_module.read_page_state_on_page,
+ )
+ return _make_server(
+ ctx, raw_result, overlay, alias_map={"evaluate": "skyvern_evaluate"}, on_call=on_call, page_state_reader=reader
+ )
+
+
+def _working_tab_with_failures(
+ monkeypatch: pytest.MonkeyPatch, ctx: AgentContext, *paths: str, closed: bool = False
+) -> RequestFailureLog:
+ working_tab = _tab("Orders")
+ working_tab.is_closed = lambda: closed
+ patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(working_tab))
+ return seed_request_failures(ctx, "pbs_failures", *[make_request_failure(working_tab, path) for path in paths])
+
+
+class TestPageRequestFailures:
+ @pytest.fixture(autouse=True)
+ def _fresh_scrub_values(self) -> Iterator[None]:
+ clear_session_scrub_values("pbs_failures")
+ yield
+ clear_session_scrub_values("pbs_failures")
+
+ @skip_no_browser
+ @pytest.mark.asyncio
+ async def test_a_socket_closed_xhr_between_calls_surfaces_once_on_the_next_browser_result(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures", api_key="test-api-key")
+ async with socket_closing_fixture_page() as (page, origin):
+ await page.goto(f"{origin}/")
+ _attach_mocked_browser_session(monkeypatch, page)
+
+ def _tool_body() -> None:
+ ensure_hooks_on_all_pages(get_current_session(), [page])
+
+ server = _make_server(
+ ctx,
+ {"ok": True, "data": {"result": "done"}},
+ SchemaOverlay(requires_browser=True, post_hook=_rebuilding_post_hook),
+ on_call=_tool_body,
+ page_state_reader=scouting_module.read_page_state_on_page,
+ )
+
+ earlier = [(await _evaluate_result(server)).get("page_request_failures") for _ in range(2)]
+ async with page.expect_event("requestfailed"):
+ await page.click("#view")
+ aborts_seen = asyncio.Event()
+ aborted = 0
+
+ def _count_abort(request: Request) -> None:
+ nonlocal aborted
+ aborted += 1
+ if aborted == 21:
+ aborts_seen.set()
+
+ page.on("requestfailed", _count_abort)
+ await page.evaluate(
+ """async () => {
+ const controllers = Array.from({length: 21}, () => new AbortController());
+ const pending = controllers.map((c) => fetch('/slow', {signal: c.signal}).catch(() => null));
+ await new Promise((resolve) => setTimeout(resolve, 300));
+ controllers.forEach((c) => c.abort());
+ await Promise.all(pending);
+ }"""
+ )
+ await asyncio.wait_for(aborts_seen.wait(), timeout=10)
+ after_failure = await _evaluate_result(server)
+ next_call = await _evaluate_result(server)
+
+ assert earlier == [None, None]
+ reported = {
+ "method": "POST",
+ "resource_type": "xhr",
+ "failure_text": "net::ERR_EMPTY_RESPONSE",
+ "url": "/report",
+ }
+ # 21 aborts overflow both caps; the one different failure keeps its slot at each.
+ assert after_failure["page_request_failures"][0] == {**reported, "page_url": f"{origin}/"}
+ assert [(f["failure_text"], f.get("url")) for f in after_failure["page_request_failures"][1:]] == [
+ ("net::ERR_ABORTED", "/slow")
+ ] * 4
+ assert BODY_SENTINEL not in json.dumps(after_failure)
+ assert "page_request_failures" not in next_call
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ async def test_model_results_drain_only_the_tab_they_read_and_internal_calls_drain_nothing(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ working_tab, other_tab = _tab("Orders"), _tab("Reports")
+ browser = FakeTabbedBrowserState(working_tab, other_tab)
+ patch_browser_tabs(monkeypatch, browser)
+ working_failures = [make_request_failure(working_tab, f"/{n}") for n in range(7)]
+ noisy_failures = [make_request_failure(other_tab, f"/bg{n}", resource_type="fetch") for n in range(120)]
+ log = seed_request_failures(ctx, "pbs_failures", *working_failures, *noisy_failures)
+ recorded = logged_request_failures(log)
+ browser_server = _evaluate_browser_server(ctx, {"ok": True})
+
+ internal = await browser_server.call_internal_tool("skyvern_evaluate", {"expression": "probe()"})
+ assert internal["ok"] is True and "page_request_failures" not in internal
+ assert logged_request_failures(log) == recorded
+
+ refusing = SchemaOverlay(requires_browser=True, pre_hook=_refuse)
+ refused = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}, overlay=refusing))
+ assert refused["ok"] is False and refused["page_state"]["title"] == "Orders"
+ assert [f["url"] for f in refused["page_request_failures"]] == [f"/{n}" for n in range(2, 7)]
+ assert logged_request_failures(log) == noisy_failures[-20:]
+
+ browser.active = other_tab
+ after_switch = (await _evaluate_result(browser_server))["page_request_failures"]
+ assert [(f["url"], f["resource_type"]) for f in after_switch] == [(f"/bg{n}", "fetch") for n in range(115, 120)]
+ assert logged_request_failures(log) == []
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("raw_result", "scope_exit_error", "reads_the_popup"),
+ [
+ pytest.param(TimeoutError(), None, False, id="timed-out-without-a-read"),
+ pytest.param(RuntimeError("client failed"), None, True, id="client-error"),
+ pytest.param({"ok": True}, RuntimeError("scope exit failed"), True, id="failed-after-projection"),
+ ],
+ )
+ async def test_an_error_result_reports_only_the_failures_of_the_tab_its_page_state_read(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ raw_result: dict[str, Any] | Exception,
+ scope_exit_error: Exception | None,
+ reads_the_popup: bool,
+ ) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ register_secret_scrub_value(ctx, "acct-secret-991")
+ opener, popup = _tab("Opener"), _tab("Popup")
+ browser = FakeTabbedBrowserState(opener)
+ patch_browser_tabs(monkeypatch, browser)
+ if scope_exit_error is not None:
+ monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope_exiting_with(scope_exit_error))
+ opener_failure = make_request_failure(opener, "/opener")
+ log = seed_request_failures(ctx, "pbs_failures", opener_failure)
+ popup_failure = make_request_failure(popup, "/accounts/acct-secret-991/report")
+
+ def _open_popup() -> None:
+ browser.tabs.append(popup)
+ browser.active = popup
+ log.append(popup_failure)
+
+ result = await _evaluate_result(_evaluate_browser_server(ctx, raw_result, on_call=_open_popup))
+
+ assert result["ok"] is False
+ assert "acct-secret-991" not in json.dumps(result)
+ assert result["page_state"].get("title") == ("Popup" if reads_the_popup else None)
+ assert len(result.get("page_request_failures", [])) == int(reads_the_popup)
+ assert logged_request_failures(log) == [opener_failure] + ([] if reads_the_popup else [popup_failure])
+
+ @pytest.mark.asyncio
+ async def test_a_native_action_that_is_the_first_browser_call_records_its_own_failures(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures", api_key="test-api-key")
+ page = MagicMock(url="https://app.test/upload")
+ page.is_closed.return_value = False
+ _attach_mocked_browser_session(monkeypatch, page)
+ monkeypatch.setattr(
+ copilot_runtime,
+ "SkyvernBrowser",
+ MagicMock(return_value=MagicMock(_browser_context=MagicMock(pages=[page]))),
+ )
+
+ async def upload(acted_on: Page) -> dict[str, Any]:
+ listeners = [call.args[1] for call in page.on.call_args_list if call.args[0] == "requestfailed"]
+ for listener in listeners:
+ listener(
+ MagicMock(
+ url="https://app.test/upload/save",
+ method="POST",
+ resource_type="xhr",
+ failure="net::ERR_EMPTY_RESPONSE",
+ )
+ )
+ return {"ok": True}
+
+ result = await on_working_page(ctx, tool_name="upload_attached_file", no_page_error="no page", act=upload)
+
+ assert result == {"ok": True}
+ log = ctx.request_failure_logs_by_browser_session["pbs_failures"]
+ assert [(entry["failure_text"], entry["url"]) for entry in logged_request_failures(log)] == [
+ ("net::ERR_EMPTY_RESPONSE", "https://app.test/upload/save")
+ ]
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ async def test_failures_a_result_cannot_show_go_back_to_the_log(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ log = _working_tab_with_failures(monkeypatch, ctx, "/0")
+ register_secret_scrub_value(ctx, "ok")
+
+ result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}))
+
+ assert "page_request_failures" not in result
+ assert len(logged_request_failures(log)) == 1
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("secret", "reported_path"),
+ [
+ ("p@ss word", "/users/p@ss%20word/verify"),
+ ("foo\\bar", "/users/foo/bar/verify"),
+ ("abc?def", "/users/abc?def/verify"),
+ ],
+ ids=["encoded-space", "backslash-as-slash", "spans-path-and-query"],
+ )
+ async def test_a_browser_a_secret_run_used_still_reports_its_failures_with_the_secret_scrubbed(
+ self, monkeypatch: pytest.MonkeyPatch, secret: str, reported_path: str
+ ) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ ctx.secret_run_browser_session_ids.add("pbs_failures")
+ log = _working_tab_with_failures(monkeypatch, ctx, reported_path)
+ register_secret_scrub_value(ctx, secret)
+
+ result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}))
+
+ assert [failure["url"] for failure in result["page_request_failures"]] == ["/users/[REDACTED_SECRET]/verify"]
+ assert logged_request_failures(log) == []
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("secret", "request_url"),
+ [
+ ("abc?token=def", "https://app.test/users/abc?token=def"),
+ ("Tr0ub4dor&3#x", "https://app.test/users/Tr0ub4dor&3"),
+ ("ABC?token=def", "https://abc/?token=def"),
+ ("TenantKey9", "https://tenantkey9/report"),
+ ("Straße9", "https://xn--strae9-eta/report"),
+ ("Straße9", "https://xn--tenant-strae9-ldb.example/report"),
+ ("PASS9", "https://pass9.example/report"),
+ ("foo/../bar9", "https://app.test/users/bar9/verify"),
+ ],
+ ids=[
+ "query-value-masked-at-record",
+ "fragment-never-sent",
+ "host-lowercased-cut",
+ "host-lowercased-whole",
+ "host-punycode",
+ "host-punycode-inside-a-label",
+ "host-compatibility-mapped",
+ "path-dot-segments-removed",
+ ],
+ )
+ async def test_a_url_holding_a_secret_cut_at_its_query_or_fragment_is_withheld(
+ self, monkeypatch: pytest.MonkeyPatch, secret: str, request_url: str
+ ) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ register_secret_scrub_value(ctx, secret)
+ tab = _tab("Users")
+ tab.on = MagicMock()
+ patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(tab))
+ log = ctx.request_failure_logs_by_browser_session.setdefault("pbs_failures", RequestFailureLog())
+ ensure_hooks_on_all_pages(make_session_state(request_failures=log), [tab])
+ record = next(call.args[1] for call in tab.on.call_args_list if call.args[0] == "requestfailed")
+ record(MagicMock(url=request_url, method="POST", resource_type="xhr", failure="net::ERR_EMPTY_RESPONSE"))
+
+ result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}))
+
+ assert result["page_request_failures"] == [
+ {
+ "method": "POST",
+ "resource_type": "xhr",
+ "failure_text": "net::ERR_EMPTY_RESPONSE",
+ "page_url": "https://app.test/users",
+ }
+ ]
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("tab_closed", "pre_hook", "kept"),
+ [(False, None, 1), (True, None, 0), (False, _refuse, 1)],
+ ids=["open-tab", "closed-tab", "refused-before-dispatch"],
+ )
+ async def test_a_cancelled_call_puts_its_failures_back_unless_their_tab_closed(
+ self, monkeypatch: pytest.MonkeyPatch, tab_closed: bool, pre_hook: mcp_adapter.PreHook | None, kept: int
+ ) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ log = _working_tab_with_failures(monkeypatch, ctx, "/0", closed=tab_closed)
+ monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope_exiting_with(asyncio.CancelledError()))
+ overlay = SchemaOverlay(requires_browser=True, timeout=1, pre_hook=pre_hook)
+
+ with pytest.raises(asyncio.CancelledError):
+ await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}, overlay=overlay))
+
+ assert len(logged_request_failures(log)) == kept
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ async def test_a_navigate_result_drains_the_page_it_read(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ monkeypatch.setattr(mcp_hooks, "_bind_login_credential_for_observed_url", AsyncMock())
+ monkeypatch.setattr(mcp_hooks, "_capture_post_interaction_screenshot", AsyncMock(return_value=False))
+ log = _working_tab_with_failures(monkeypatch, ctx, "/0")
+ navigate = SchemaOverlay(requires_browser=True, timeout=1, post_hook=mcp_hooks._navigate_post_hook)
+
+ result = await _evaluate_result(
+ _evaluate_browser_server(ctx, {"ok": True, "data": {"url": "https://app.test/orders"}}, overlay=navigate)
+ )
+
+ assert result["page_state"]["title"] == "Orders"
+ assert [f["url"] for f in result["page_request_failures"]] == ["/0"]
+ assert logged_request_failures(log) == []
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ async def test_failures_past_the_redaction_budget_go_back(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ ctx.codeblock_redaction_parameters = {"token": "unused-value"}
+
+ def redact_within_budget(value: Any, _parameters: dict[str, Any], **_budget: object) -> Any:
+ return value if len(json.dumps(value)) <= 1_500 else None
+
+ monkeypatch.setattr(mcp_adapter.app.AGENT_FUNCTION, "redact_codeblock_parameter_values", redact_within_budget)
+ log = _working_tab_with_failures(monkeypatch, ctx, *[f"/{n}" for n in range(5)])
+
+ result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True, "data": {"result": "x" * 1_000}}))
+
+ assert result["ok"] is True and result["data"] == {"result": "x" * 1_000}
+ assert "page_request_failures" not in result
+ assert len(logged_request_failures(log)) == 5
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("result_chars", "reported"),
+ [(_RECENT_TOOL_OUTPUT_CHAR_CAP - 500, False), (_RECENT_TOOL_OUTPUT_CHAR_CAP, True)],
+ ids=["fits-without-them", "past-the-cap-anyway"],
+ )
+ async def test_failures_never_push_a_result_past_the_recent_output_cap(
+ self, monkeypatch: pytest.MonkeyPatch, result_chars: int, reported: bool
+ ) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ log = _working_tab_with_failures(monkeypatch, ctx, *[f"/{n}" for n in range(5)])
+
+ result = await _evaluate_result(
+ _evaluate_browser_server(ctx, {"ok": True, "data": {"result": "x" * result_chars}})
+ )
+
+ assert ("page_request_failures" in result) is reported
+ assert len(logged_request_failures(log)) == (0 if reported else 5)
+
+ @pytest.mark.usefixtures("_stub_browser_session")
+ @pytest.mark.asyncio
+ async def test_a_url_that_scrubbing_grows_past_the_bound_is_left_out(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_failures")
+ register_secret_scrub_value(ctx, "7")
+ _working_tab_with_failures(monkeypatch, ctx, "/" + "7" * 280)
+
+ result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}))
+
+ (reported,) = result["page_request_failures"]
+ assert reported == {
+ "method": "POST",
+ "resource_type": "xhr",
+ "failure_text": "net::ERR_EMPTY_RESPONSE",
+ "page_url": "https://app.test/",
+ }
+
+ def test_the_model_sees_bounded_fields_and_only_the_host_of_a_cross_origin_request(self) -> None:
+ too_long = f"https://app.test/{'p' * 400}"
+ reported = mcp_adapter.reported_request_failures(
+ [
+ make_request_failure(
+ object(), url="https://app.test/api/r?id=7#top", page_url="https://app.test/h?s=1"
+ ),
+ make_request_failure(object(), url="https://api.other.test/v1/accounts/123", method="X" * 9_999),
+ make_request_failure(object(), url=too_long, failure_text="F" * 9_999),
+ make_request_failure(object(), url="http://[redacted]/r", page_url=too_long),
+ ]
+ )
+
+ assert [(r.get("url"), r.get("page_url"), r["method"], len(r["failure_text"])) for r in reported] == [
+ ("/api/r", "https://app.test/h", "POST", 23),
+ ("https://api.other.test", "https://app.test/", "OTHER", 23),
+ (None, "https://app.test/", "POST", 200),
+ (None, None, "POST", 23),
+ ]
diff --git a/tests/unit/test_copilot_non_retriable_nav.py b/tests/unit/test_copilot_non_retriable_nav.py
index 5508e7cff..46c566891 100644
--- a/tests/unit/test_copilot_non_retriable_nav.py
+++ b/tests/unit/test_copilot_non_retriable_nav.py
@@ -411,7 +411,7 @@ def test_workflow_update_clears_non_retriable_flag() -> None:
{
"ok": True,
"data": {"block_count": 2},
- "_workflow": SimpleNamespace(workflow_id="wf_new"),
+ "_workflow": SimpleNamespace(workflow_id="wf_new", proxy_location=None, workflow_definition=None),
},
)
assert ctx.last_test_non_retriable_nav_error is None
@@ -443,7 +443,9 @@ def test_workflow_update_records_runtime_proxy_default(
{
"ok": True,
"data": {"block_count": 1},
- "_workflow": SimpleNamespace(workflow_id="wf_new", proxy_location=workflow_proxy_location),
+ "_workflow": SimpleNamespace(
+ workflow_id="wf_new", proxy_location=workflow_proxy_location, workflow_definition=None
+ ),
},
)
diff --git a/tests/unit/test_copilot_runtime.py b/tests/unit/test_copilot_runtime.py
index a1d1cb8ef..3ca53fa4c 100644
--- a/tests/unit/test_copilot_runtime.py
+++ b/tests/unit/test_copilot_runtime.py
@@ -915,6 +915,7 @@ def test_a_page_titled_like_a_lost_session_does_not_read_as_session_loss() -> No
"ok": False,
"error": "evaluate failed: element not visible",
"page_state": {"read": "ok", "url": None, "title": "Browser session not found", "challenge_vendor": None},
+ "page_request_failures": [{"method": "GET", "url": "/no browser context", "failure_text": "net::ERR_FAILED"}],
}
assert not _is_unrecoverable_browser_session_error("evaluate", output)
diff --git a/tests/unit/test_copilot_secret_scrub.py b/tests/unit/test_copilot_secret_scrub.py
index 6afd45f70..9ad2337cb 100644
--- a/tests/unit/test_copilot_secret_scrub.py
+++ b/tests/unit/test_copilot_secret_scrub.py
@@ -498,12 +498,31 @@ class TestPersistenceSeam:
assert form not in scrubbed
assert REDACTED_SECRET_PLACEHOLDER in scrubbed
- def test_browser_style_url_encoding_is_a_registered_form(self) -> None:
- """encodeURIComponent leaves `!*'()` literal where Python's quote does not."""
+ @pytest.mark.parametrize(
+ ("password", "browser_encoded"),
+ [
+ ("Sp1r!t&Level<2026>", "Sp1r!t%26Level%3C2026%3E"),
+ ("p@ss word", "p@ss%20word"),
+ ('a"b:c', "a%22b:c"),
+ ("Tr0ub4dor&3 x", "Tr0ub4dor&3%20x"),
+ ("50%off now", "50%off%20now"),
+ ("foo\\bar x", "foo/bar%20x"),
+ ("tab\tline\nbreak", "tablinebreak"),
+ ],
+ ids=[
+ "encodeURIComponent-query",
+ "chromium-path-space",
+ "chromium-path-quote",
+ "chromium-path-ampersand",
+ "chromium-path-percent",
+ "chromium-path-backslash",
+ "chromium-path-tab-newline",
+ ],
+ )
+ def test_browser_style_url_encoding_is_a_registered_form(self, password: str, browser_encoded: str) -> None:
+ """encodeURIComponent leaves `!*'()` literal in a query, and Chromium leaves `@&:` literal in a path."""
ctx = _agent_ctx()
- password = "Sp1r!t&Level<2026>"
register_secret_scrub_value(ctx, password)
- browser_encoded = "Sp1r!t%26Level%3C2026%3E"
assert browser_encoded in registered_scrub_values(ctx)
assert scrub_secrets_from_text(ctx, f"?next={browser_encoded}") == f"?next={REDACTED_SECRET_PLACEHOLDER}"
diff --git a/tests/unit/test_mcp_inspection.py b/tests/unit/test_mcp_inspection.py
index 616487033..2c821270e 100644
--- a/tests/unit/test_mcp_inspection.py
+++ b/tests/unit/test_mcp_inspection.py
@@ -1,11 +1,12 @@
from __future__ import annotations
+import json
from unittest.mock import MagicMock
import pytest
from skyvern.cli.core.result import BrowserContext
-from skyvern.cli.core.session_manager import SessionState
+from skyvern.cli.core.session_manager import RequestFailureLog, SessionState
from skyvern.cli.mcp_tools.inspection import (
_redact_url,
_register_hooks_on_page,
@@ -13,8 +14,11 @@ from skyvern.cli.mcp_tools.inspection import (
skyvern_handle_dialog,
skyvern_network_requests,
)
+from tests.unit._mcp_browser_fakes import logged_request_failures
from tests.unit._mcp_browser_fakes import make_page as _make_page
from tests.unit._mcp_browser_fakes import make_session_state as _make_state
+from tests.unit._socket_closing_fixture import BODY_SENTINEL, socket_closing_fixture_page
+from tests.unit.copilot_test_helpers import skip_no_browser
def _patch(monkeypatch: pytest.MonkeyPatch, state: SessionState) -> None:
@@ -43,13 +47,14 @@ def _network_entry(url: str = "https://a.com", method: str = "GET", status: int
class TestEnsureHooks:
- def test_registers_four_listeners(self) -> None:
+ def test_registers_six_listeners(self) -> None:
state = _make_state()
raw = MagicMock()
raw.on = MagicMock()
_register_hooks_on_page(state, raw)
- assert raw.on.call_count == 4
- assert {c.args[0] for c in raw.on.call_args_list} == {"console", "response", "dialog", "pageerror"}
+ assert raw.on.call_count == 6
+ events = {c.args[0] for c in raw.on.call_args_list}
+ assert events == {"console", "response", "requestfailed", "close", "dialog", "pageerror"}
def test_idempotent(self) -> None:
state = _make_state()
@@ -57,7 +62,7 @@ class TestEnsureHooks:
raw.on = MagicMock()
_register_hooks_on_page(state, raw)
_register_hooks_on_page(state, raw)
- assert raw.on.call_count == 4
+ assert raw.on.call_count == 6
def test_keeps_hooks_on_both_pages(self) -> None:
"""Multi-page: hooks are registered on ALL pages, not removed on switch."""
@@ -71,8 +76,56 @@ class TestEnsureHooks:
_register_hooks_on_page(state, raw2)
# Both pages should have hooks registered — no removal
assert raw1.remove_listener.call_count == 0
- assert raw1.on.call_count == 4
- assert raw2.on.call_count == 4
+ assert raw1.on.call_count == 6
+ assert raw2.on.call_count == 6
+
+
+class TestRequestFailed:
+ @skip_no_browser
+ @pytest.mark.asyncio
+ async def test_real_chromium_records_a_socket_closed_xhr_once_ignores_successes_and_forgets_a_closed_tab(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ log = RequestFailureLog()
+ states = [_make_state(request_failures=log) for _ in range(3)]
+ async with socket_closing_fixture_page() as (page, origin):
+ tab_id = str(id(page))
+ for state in states:
+ state._codeblock_redactor = lambda value: json.loads(json.dumps(value).replace(tab_id, "[REDACTED]"))
+ _register_hooks_on_page(state, page)
+ await page.goto(f"{origin}/?token=page-secret")
+
+ async with page.expect_response(f"{origin}/ok"):
+ await page.click("#fast")
+ async with page.expect_response(f"{origin}/slow", timeout=30_000):
+ await page.click("#slow")
+ assert log.entries == {}
+
+ async with page.expect_event("requestfailed"):
+ await page.click("#view")
+ recorded = logged_request_failures(log)
+ await page.close()
+ assert log.entries == {}
+
+ (entry,) = recorded
+ assert set(entry) == {"url", "method", "resource_type", "failure_text", "page_url", "tab_id", "timestamp"}
+ assert entry["method"] == "POST" and entry["resource_type"] == "xhr"
+ assert entry["failure_text"] == "net::ERR_EMPTY_RESPONSE"
+ assert entry["url"].startswith(f"{origin}/report?")
+ assert "token=REDACTED" in entry["url"] and "request-secret" not in entry["url"]
+ assert entry["page_url"] == f"{origin}/?token=REDACTED"
+ assert entry["tab_id"] == tab_id
+ assert BODY_SENTINEL not in json.dumps(entry)
+
+ _patch(monkeypatch, states[0])
+ network = await skyvern_network_requests(verbosity="full")
+ requests = network["data"]["requests"]
+ assert sorted((r["method"], r["url"], r["status"]) for r in requests) == [
+ ("GET", f"{origin}/?token=REDACTED", 200),
+ ("GET", f"{origin}/ok", 200),
+ ("GET", f"{origin}/slow", 200),
+ ]
+ assert BODY_SENTINEL not in json.dumps(network)
# --- Console messages ---
diff --git a/tests/unit/test_mcp_tab_tools.py b/tests/unit/test_mcp_tab_tools.py
index da3753bd7..11d3d362c 100644
--- a/tests/unit/test_mcp_tab_tools.py
+++ b/tests/unit/test_mcp_tab_tools.py
@@ -993,9 +993,9 @@ class TestMultiPageInspectionHooks:
# Both pages should have hooks
assert id(page_a) in state._hooked_page_ids
assert id(page_b) in state._hooked_page_ids
- # 4 events per page: console, response, dialog, pageerror
- assert page_a.on.call_count == 4
- assert page_b.on.call_count == 4
+ # 6 events per page: console, response, requestfailed, close, dialog, pageerror
+ assert page_a.on.call_count == 6
+ assert page_b.on.call_count == 6
def test_hooks_idempotent(self) -> None:
from skyvern.cli.mcp_tools.inspection import ensure_hooks_on_all_pages
@@ -1010,7 +1010,7 @@ class TestMultiPageInspectionHooks:
ensure_hooks_on_all_pages(state, [page_a])
# Should only register once
- assert page_a.on.call_count == 4
+ assert page_a.on.call_count == 6
def test_stale_pages_pruned(self) -> None:
from skyvern.cli.mcp_tools.inspection import ensure_hooks_on_all_pages
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index a0895ed01..ad02b6249 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -890,7 +890,7 @@ async def test_a_candidate_claimed_mid_turn_does_not_cost_the_user_the_reply(
_, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
workflow_params.get_workflow_copilot_chat_by_id = AsyncMock(return_value=SimpleNamespace(proposed_workflow=stored))
workflow_params.enrich_workflow_copilot_candidate = AsyncMock(
- side_effect=CopilotProposalConflictError("Copilot proposal is being accepted")
+ side_effect=CopilotProposalConflictError("Copilot proposal is being accepted", reason="being_accepted")
)
stream = MagicMock(send=AsyncMock(return_value=True))
@@ -3229,7 +3229,7 @@ def _serve_live_chat_row(workflow_params: SimpleNamespace, chat: SimpleNamespace
) -> None:
token = (chat_row["proposed_workflow"] or {}).get(COPILOT_PROPOSAL_METADATA_KEY, {})
if (token.get("owner_turn_id"), token.get("revision")) != (expected_owner_turn_id, expected_revision):
- raise CopilotProposalConflictError("Copilot proposal changed")
+ raise CopilotProposalConflictError("Copilot proposal changed", reason="token_changed")
chat_row["proposed_workflow"] = None
workflow_params.get_workflow_copilot_chat_by_id = AsyncMock(side_effect=get_chat)
@@ -3339,6 +3339,58 @@ async def test_route_error_recovery_keeps_the_committed_workflow_and_other_turns
assert frames[-1].workflow_applied is not agent_fails
[email protected]("failure", ["agent-raises", "assistant-row-write-fails"])
+async def test_route_error_recovery_leaves_a_candidate_the_turn_adopted(
+ monkeypatch: pytest.MonkeyPatch,
+ api_key_request: MagicMock,
+ copilot_stream: MagicMock,
+ organization: SimpleNamespace,
+ failure: str,
+) -> None:
+ captured = install_fake_create(monkeypatch)
+ chat_row: dict[str, Any] = {"proposed_workflow": _candidate_owned_by("turn-newer", unvalidated=True)}
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=chat_row["proposed_workflow"],
+ auto_accept=True,
+ )
+ unused_result = AgentResult(user_response="", updated_workflow=None, global_llm_context=None)
+ _, workflow_params = setup_new_copilot_mocks(
+ monkeypatch, chat, _make_copilot_workflow("Saved workflow", _NOW), unused_result
+ )
+ _serve_live_chat_row(workflow_params, chat, chat_row)
+
+ adopted_result = AgentResult(user_response="done", updated_workflow=None, global_llm_context=None)
+ adopted_result.proposal_owner_turn_id, adopted_result.proposal_revision = "turn-newer", 1
+ adopted_result.proposal_adopted = True
+
+ async def run_agent(*args: object, ctx_sink: list[Any], **kwargs: object) -> AgentResult:
+ if failure == "assistant-row-write-fails":
+ return adopted_result
+ ctx_sink.append(SimpleNamespace(proposal_adopted=True, executed_block_fingerprints={}))
+ raise RuntimeError("agent run failed")
+
+ monkeypatch.setattr(workflow_copilot_route, "run_copilot_agent", run_agent)
+ first_assistant_write_failure = [RuntimeError("assistant row write failed")] if failure != "agent-raises" else []
+
+ async def create_message(**kwargs: Any) -> SimpleNamespace:
+ if kwargs["sender"] == WorkflowCopilotChatSender.AI and first_assistant_write_failure:
+ raise first_assistant_write_failure.pop()
+ return SimpleNamespace(created_at=_NOW)
+
+ workflow_params.create_workflow_copilot_chat_message = AsyncMock(side_effect=create_message)
+
+ await workflow_copilot_chat_post(api_key_request, _make_chat_request(), organization)
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(copilot_stream)
+
+ assert chat_row["proposed_workflow"] == _candidate_owned_by("turn-newer", unvalidated=True)
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
("newer_candidate_lands", "applied"),
@@ -4153,6 +4205,7 @@ async def test_persist_state_keeps_verified_review_tested_proposal(monkeypatch:
has_staged_proposal=False,
proposal_owner_turn_id=None,
proposal_revision=None,
+ proposal_adopted=False,
proposal_workflow_run_id=None,
)
@@ -4179,6 +4232,7 @@ def _make_bypassed_proposal_agent_result(**overrides: object) -> SimpleNamespace
"private_workflow_settings": {},
"proposal_owner_turn_id": None,
"proposal_revision": None,
+ "proposal_adopted": False,
"proposal_workflow_run_id": None,
}
fields.update(overrides)