Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Restore a local dev path for copilot code-block test runs (#7856)
Andrew Neilsonaab2a29327cd
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex 4105efa22..059ffc007 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -4247,6 +4247,19 @@}}},+ "202": {+ "description": "Raw OTP content accepted for later parsing",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RawTOTPCodeAccepted"+ }+ }+ }+ },+ "402": {+ "description": "Insufficient credits to parse OTP content"+ },"422": {"description": "Validation Error","content": {@@ -10328,7 +10341,7 @@"type": "number","title": "Timeout","description": "Timeout in milliseconds",- "default": 10000+ "default": 5000}},"type": "object",@@ -11093,18 +11106,6 @@},"CreateBrowserSessionRequest": {"properties": {- "url": {- "anyOf": [- {- "type": "string"- },- {- "type": "null"- }- ],- "title": "Url",- "description": "Optional URL to open when the standalone browser session starts."- },"timeout": {"anyOf": [{@@ -17655,7 +17656,7 @@"type": "number","title": "Timeout","description": "Timeout in milliseconds",- "default": 10000+ "default": 5000}},"type": "object",@@ -20424,6 +20425,25 @@],"title": "ProxyLocation"},+ "RawTOTPCodeAccepted": {+ "properties": {+ "totp_code_id": {+ "type": "string",+ "title": "Totp Code Id"+ },+ "status": {+ "type": "string",+ "const": "raw_pending",+ "title": "Status",+ "default": "raw_pending"+ }+ },+ "type": "object",+ "required": [+ "totp_code_id"+ ],+ "title": "RawTOTPCodeAccepted"+ },"RetryRunWebhookRequest": {"properties": {"webhook_url": {@@ -21119,7 +21139,7 @@"type": "number","title": "Timeout","description": "Timeout in milliseconds",- "default": 10000+ "default": 5000}},"type": "object",@@ -24910,7 +24930,7 @@"type": "number","title": "Timeout","description": "Timeout in milliseconds",- "default": 10000+ "default": 5000}},"type": "object",diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.jsonindex c064e2eb4..515c267f6 100644--- a/fern/openapi/skyvern_openapi.json+++ b/fern/openapi/skyvern_openapi.json@@ -10698,18 +10698,6 @@},"CreateBrowserSessionRequest": {"properties": {- "url": {- "anyOf": [- {- "type": "string"- },- {- "type": "null"- }- ],- "title": "Url",- "description": "Optional URL to open when the standalone browser session starts."- },"timeout": {"anyOf": [{diff --git a/pyproject.toml b/pyproject.tomlindex d263086f2..2158107cd 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -289,7 +289,7 @@ constraint-dependencies = ["joserfc>=1.6.8","jupyter-server>=2.20.0","mako>=1.3.12",- "mistune>=3.3.0",+ "mistune>=3.2.1","msgpack>=1.2.1","pyasn1>=0.6.4","starlette>=1.3.1",diff --git a/skyvern-frontend/src/routes/streaming/useCdpInput.test.ts b/skyvern-frontend/src/routes/streaming/useCdpInput.test.tsnew file mode 100644index 000000000..6f2c4583c--- /dev/null+++ b/skyvern-frontend/src/routes/streaming/useCdpInput.test.ts@@ -0,0 +1,131 @@+import { act, renderHook } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import { useCdpInput } from "./useCdpInput";++const mocks = vi.hoisted(() => ({+ credentialGetter: vi.fn(async () => null),+ getCredentialParam: vi.fn(async () => "token=Bearer+test-token"),+}));++vi.mock("@/hooks/useCredentialGetter", () => ({+ useCredentialGetter: () => mocks.credentialGetter,+}));++vi.mock("@/store/useClientIdStore", () => ({+ useClientIdStore: (selector: (state: { clientId: string }) => unknown) =>+ selector({ clientId: "client-test" }),+}));++vi.mock("@/util/env", () => ({+ getCredentialParam: mocks.getCredentialParam,+}));++type SocketListener = (event: Event) => void;++class MockWebSocket {+ static readonly OPEN = 1;+ static instances: MockWebSocket[] = [];++ readonly readyState = MockWebSocket.OPEN;+ readonly send = vi.fn();+ readonly close = vi.fn();++ private listeners: Record<string, SocketListener[]> = {};++ constructor() {+ MockWebSocket.instances.push(this);+ }++ addEventListener(type: string, listener: SocketListener) {+ this.listeners[type] = [...(this.listeners[type] ?? []), listener];+ }++ emitClose(code: number) {+ const event = new CloseEvent("close", { code });+ for (const listener of this.listeners.close ?? []) {+ listener(event);+ }+ }+}++async function renderInputHook() {+ renderHook(() =>+ useCdpInput({+ inputWsUrl: "wss://input.test",+ interactive: true,+ viewportWidth: 1280,+ viewportHeight: 720,+ }),+ );++ await act(async () => {+ await Promise.resolve();+ });+}++function closeLatestSocket(code: number) {+ const socket = MockWebSocket.instances[MockWebSocket.instances.length - 1];+ if (!socket) throw new Error("No WebSocket was constructed");++ act(() => {+ socket.emitClose(code);+ });+}++async function advanceReconnectDelay() {+ await act(async () => {+ await vi.advanceTimersByTimeAsync(2000);+ });+}++describe("useCdpInput reconnects", () => {+ beforeEach(() => {+ vi.useFakeTimers();+ vi.clearAllMocks();+ MockWebSocket.instances = [];+ vi.stubGlobal("WebSocket", MockWebSocket);+ });++ afterEach(() => {+ vi.useRealTimers();+ vi.unstubAllGlobals();+ });++ it("constructs a new socket after a 4411 close", async () => {+ await renderInputHook();+ expect(MockWebSocket.instances).toHaveLength(1);++ closeLatestSocket(4411);+ await advanceReconnectDelay();++ expect(MockWebSocket.instances).toHaveLength(2);+ });++ it("does not reconnect after a normal close", async () => {+ await renderInputHook();+ expect(MockWebSocket.instances).toHaveLength(1);++ closeLatestSocket(1000);+ await advanceReconnectDelay();++ expect(MockWebSocket.instances).toHaveLength(1);+ });++ // Uses 4410, not 4411: the cap only bites when the server never reaches "ready", which resets the+ // counter. 4410 is the setup-time close, so repeated failures without "ready" is its real shape.+ it("stops reconnecting after five attempts that never reach ready", async () => {+ await renderInputHook();++ for (let attempt = 0; attempt < 5; attempt += 1) {+ closeLatestSocket(4410);+ await advanceReconnectDelay();+ }+ expect(MockWebSocket.instances).toHaveLength(6);++ closeLatestSocket(4410);+ await advanceReconnectDelay();++ expect(MockWebSocket.instances).toHaveLength(6);+ });+});diff --git a/skyvern-frontend/src/routes/streaming/useCdpInput.ts b/skyvern-frontend/src/routes/streaming/useCdpInput.tsindex 72f7119d7..42f39abca 100644--- a/skyvern-frontend/src/routes/streaming/useCdpInput.ts+++ b/skyvern-frontend/src/routes/streaming/useCdpInput.ts@@ -9,7 +9,9 @@ import {mapMouseCoordinates,} from "./cdpInputUtils";-const RECONNECTABLE_CODES = new Set([1006, 1011, 4408, 4410]);+// 4411 is emitted mid-session (page resolution or dispatch failed), so unlike a setup-time close it+// leaves a channel the user still believes is live; without a reconnect, input is dead until reload.+const RECONNECTABLE_CODES = new Set([1006, 1011, 4408, 4410, 4411]);interface UseCdpInputOptions {inputWsUrl: string | null;diff --git a/skyvern-frontend/src/routes/workflows/studio/BrowserPaneHeader.tsx b/skyvern-frontend/src/routes/workflows/studio/BrowserPaneHeader.tsxindex 5a55a24f0..a2eae7c3f 100644--- a/skyvern-frontend/src/routes/workflows/studio/BrowserPaneHeader.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/BrowserPaneHeader.tsx@@ -31,7 +31,10 @@ import { useRecordingStore } from "@/store/useRecordingStore";import { useStudioBrowserStore } from "@/store/useStudioBrowserStore";import { cn } from "@/util/utils";-import { PANE_HEADER_ICON_BUTTON_CLASS } from "./constants";+import {+ PANE_HEADER_ICON_BUTTON_CLASS,+ PANE_HEADER_ICON_BUTTON_DESTRUCTIVE_CLASS,+} from "./constants";import { ControlTooltip } from "./ControlTooltip";import { PaneHeaderDivider } from "./PaneHeaderDivider";import { useBrowserPaneView } from "./useBrowserPaneView";@@ -253,7 +256,7 @@ export function BrowserPaneActions() {disabled={!workflowPermanentId || !browserSessionId || debugHidden}- className={PANE_HEADER_ICON_BUTTON_CLASS}+ className={PANE_HEADER_ICON_BUTTON_DESTRUCTIVE_CLASS}><PowerIcon className="h-3.5 w-3.5" /></button>diff --git a/skyvern-frontend/src/routes/workflows/studio/BrowserTab.test.tsx b/skyvern-frontend/src/routes/workflows/studio/BrowserTab.test.tsxindex 70c92bd74..759122892 100644--- a/skyvern-frontend/src/routes/workflows/studio/BrowserTab.test.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/BrowserTab.test.tsx@@ -490,6 +490,23 @@ describe("BrowserTab pills and selection sync", () => {}});+ it("styles only Turn off browser as destructive, not its siblings", () => {+ mocks.debugSession = { browser_session_id: "pbs_test" };+ renderBrowserPane(STUDIO_PATH);++ expect(screen.getByLabelText("Turn off browser").className).toContain(+ "badge-destructive",+ );+ for (const name of [+ "Reconnect browser stream",+ "Open browser in new tab",+ ]) {+ expect(screen.getByLabelText(name).className).not.toContain(+ "badge-destructive",+ );+ }+ });+it("marks the resolved view's pill as pressed", () => {seedRun({ status: Status.Completed, recordingUrl: "https://r.test/1.mp4" });renderBrowserPane(`${STUDIO_PATH}&wr=wr_1`);diff --git a/skyvern-frontend/src/routes/workflows/studio/constants.ts b/skyvern-frontend/src/routes/workflows/studio/constants.tsindex 5f7510bd1..510737669 100644--- a/skyvern-frontend/src/routes/workflows/studio/constants.ts+++ b/skyvern-frontend/src/routes/workflows/studio/constants.ts@@ -10,3 +10,10 @@ export const studioPanelId = (tab: string) => `studio-panel-${tab}`;// (cloud_docs/frontend/studio-button-system.md).export const PANE_HEADER_ICON_BUTTON_CLASS ="inline-flex h-7 w-7 shrink-0 items-center justify-center rounded-md border border-border text-muted-foreground hover:bg-accent hover:text-accent-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-40";++// Same icon-square footprint as PANE_HEADER_ICON_BUTTON_CLASS, reserved for+// the one control that actually ends the live browser session (documented+// exception in studio-button-system.md, SKY-12247) — do not reuse for+// Reconnect/Open-in-new-tab/pane-close, which don't carry that consequence.+export const PANE_HEADER_ICON_BUTTON_DESTRUCTIVE_CLASS =+ "inline-flex h-7 w-7 shrink-0 items-center justify-center rounded-md border border-badge-destructive bg-badge-destructive text-foreground hover:bg-badge-destructive/80 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-40";diff --git a/skyvern/cli/core/session_manager.py b/skyvern/cli/core/session_manager.pyindex 255442786..b437d5cad 100644--- a/skyvern/cli/core/session_manager.py+++ b/skyvern/cli/core/session_manager.py@@ -75,6 +75,7 @@ class SessionState:_current_session: ContextVar[SessionState | None] = ContextVar("mcp_session", default=None)_global_session: SessionState | None = None_stateless_http_mode = False+_stdio_local_file_access_enabled = False# Process-wide registry for copilot browser sessions. Keyed by browser_session_id.# This bypasses ContextVar propagation issues when FastMCP runs tool handlers@@ -307,6 +308,15 @@ def is_stateless_http_mode() -> bool:return _stateless_http_mode+def set_stdio_local_file_access_enabled(enabled: bool) -> None:+ global _stdio_local_file_access_enabled+ _stdio_local_file_access_enabled = enabled+++def is_stdio_local_file_access_enabled() -> bool:+ return _stdio_local_file_access_enabled++def _api_key_hash(api_key: str | None) -> str | None:if not api_key:return Nonediff --git a/skyvern/cli/mcp_tools/_session.py b/skyvern/cli/mcp_tools/_session.pyindex 143306e81..ffb7102f2 100644--- a/skyvern/cli/mcp_tools/_session.py+++ b/skyvern/cli/mcp_tools/_session.py@@ -17,6 +17,7 @@ from skyvern.cli.core.session_manager import (get_current_session,get_page,get_session_ref,+ is_stdio_local_file_access_enabled,no_browser_error,page_ref_key,replace_session_ref_map,@@ -37,6 +38,7 @@ __all__ = ["get_page","get_session_ref","get_skyvern",+ "is_stdio_local_file_access_enabled","no_browser_error","page_ref_key","replace_session_ref_map",diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.pyindex cf46aaeab..2c020649c 100644--- a/skyvern/cli/mcp_tools/browser.py+++ b/skyvern/cli/mcp_tools/browser.py@@ -3,13 +3,17 @@ from __future__ import annotationsimport asyncioimport base64import json+import mimetypesimport osimport re+import statfrom datetime import datetime, timezone+from pathlib import Pathfrom typing import Annotated, Any, Callable, Literalfrom uuid import uuid4import structlog+from playwright.async_api import FilePayloadfrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrorfrom pydantic import Field@@ -54,8 +58,11 @@ from skyvern.cli.core.guards import ()from skyvern.cli.core.session_manager import is_stateless_http_modefrom skyvern.cli.core.trajectory_store import append_trajectory_entry+from skyvern.config import settingsfrom skyvern.core.script_generations.skyvern_page import SkyvernPage+from skyvern.forge.sdk.api.files import resolve_run_download_idfrom skyvern.forge.sdk.copilot.typed_value_policy import typed_text_looks_secret+from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.schemas.action_log import ActionLogOutcome, project_action_eventfrom skyvern.schemas.run_blocks import CredentialType@@ -88,6 +95,7 @@ from ._session import (get_current_session,get_page,get_session_ref,+ is_stdio_local_file_access_enabled,no_browser_error,page_ref_key,replace_session_ref_map,@@ -101,6 +109,71 @@ _AWAIT_RE = re.compile(r"\bawait\b")_SINGLE_LINE_COMMENT_RE = re.compile(r"//[^\n]*")_ERROR_MESSAGE_MAX_CHARS = 500_ERROR_BODY_MESSAGE_KEYS = ("detail", "error", "message")+_LOCAL_UPLOAD_MAX_FILES = 20+_LOCAL_UPLOAD_MAX_BYTES = 50 * 1024 * 1024+_LOCAL_UPLOAD_SUPPORTS_DIR_FD = os.open in getattr(os, "supports_dir_fd", set())+++def _read_run_owned_upload(candidate_path: str, run_id: str, max_bytes: int) -> FilePayload:+ nofollow_flag = getattr(os, "O_NOFOLLOW", None)+ directory_flag = getattr(os, "O_DIRECTORY", None)+ nonblocking_flag = getattr(os, "O_NONBLOCK", None)+ if nofollow_flag is None or directory_flag is None or nonblocking_flag is None or not _LOCAL_UPLOAD_SUPPORTS_DIR_FD:+ raise PermissionError("Secure local file access is not supported on this platform")++ allowed_root = (Path(settings.DOWNLOAD_PATH) / run_id).resolve(strict=True)+ resolved_candidate = Path(candidate_path).resolve(strict=True)+ try:+ relative_path = resolved_candidate.relative_to(allowed_root)+ except ValueError as exc:+ raise PermissionError("Local file is outside the run upload directory") from exc+ if not relative_path.parts:+ raise PermissionError("Local upload path must identify a file")++ close_on_exec = getattr(os, "O_CLOEXEC", 0)+ directory_open_flags = os.O_RDONLY | close_on_exec | nofollow_flag | directory_flag+ file_open_flags = os.O_RDONLY | close_on_exec | nofollow_flag | nonblocking_flag+ directory_fds: list[int] = []+ file_fd: int | None = None+ try:+ directory_fds.append(os.open(allowed_root, directory_open_flags))+ for component in relative_path.parts[:-1]:+ directory_fds.append(os.open(component, directory_open_flags, dir_fd=directory_fds[-1]))+ file_fd = os.open(relative_path.parts[-1], file_open_flags, dir_fd=directory_fds[-1])+ file_stat = os.fstat(file_fd)+ if not stat.S_ISREG(file_stat.st_mode):+ raise PermissionError("Local upload path must identify a regular file")+ if file_stat.st_size > max_bytes:+ raise ValueError("Local upload file exceeds the size limit")++ with os.fdopen(file_fd, "rb") as upload_file:+ file_fd = None+ content = upload_file.read(max_bytes + 1)+ if len(content) > max_bytes:+ raise ValueError("Local upload file exceeds the size limit")+ finally:+ if file_fd is not None:+ os.close(file_fd)+ for directory_fd in reversed(directory_fds):+ os.close(directory_fd)++ mime_type = mimetypes.guess_type(resolved_candidate.name)[0] or "application/octet-stream"+ return FilePayload(name=resolved_candidate.name, mimeType=mime_type, buffer=content)+++def _read_run_owned_uploads(file_paths: list[str], run_id: str | None) -> list[FilePayload]:+ if run_id is None:+ raise PermissionError("Local file upload requires an allowed download scope")+ if len(file_paths) > _LOCAL_UPLOAD_MAX_FILES:+ raise ValueError("Too many local upload files")++ payloads: list[FilePayload] = []+ remaining_bytes = _LOCAL_UPLOAD_MAX_BYTES+ for file_path in file_paths:+ payload = _read_run_owned_upload(file_path, run_id, remaining_bytes)+ payloads.append(payload)+ remaining_bytes -= len(payload["buffer"])+ return payloadsdef _trajectory_source_url(page: Any) -> str | None:@@ -747,7 +820,11 @@ async def skyvern_file_upload(file_paths: Annotated[list[str],Field(- description="List of file paths or URLs to upload. URLs are downloaded automatically. Max 50MB per file."+ description=(+ "List of file paths or URLs to upload. Local files must be in the active run's download directory, "+ "or the local server download directory for stdio, and require a selector, with at most 20 files "+ "and 50MB total. URLs are downloaded automatically."+ )),],selector: Annotated[@@ -763,7 +840,7 @@ async def skyvern_file_upload(intent: Annotated[str | None, Field(description=AI_FALLBACK_DESCRIPTION)] = None,) -> dict[str, Any]:"""Upload files to a file input element. Accepts local paths or URLs (auto-downloaded).- Supports AI intent, CSS/XPath selector, or both to find the input.+ Local paths must be in an allowed download scope and use a CSS/XPath selector."""if not file_paths:return make_result(@@ -826,11 +903,41 @@ async def skyvern_file_upload(),)+ if has_local and not selector:+ return make_result(+ "skyvern_file_upload",+ ok=False,+ error=make_error(+ ErrorCode.INVALID_INPUT,+ "Local file upload requires a selector",+ "Provide selector='input[type=file]' for local file uploads",+ ),+ )+try:page, ctx = await get_page(session_id=session_id, cdp_url=cdp_url)except BrowserNotAvailableError:return make_result("skyvern_file_upload", ok=False, error=no_browser_error())+ local_uploads: list[FilePayload] | None = None+ if has_local:+ run_id = resolve_run_download_id(skyvern_context.current())+ if run_id is None and is_stdio_local_file_access_enabled():+ run_id = str(None)+ try:+ local_uploads = await asyncio.to_thread(_read_run_owned_uploads, file_paths, run_id)+ except (OSError, RuntimeError, ValueError):+ return make_result(+ "skyvern_file_upload",+ ok=False,+ browser_context=ctx,+ error=make_error(+ ErrorCode.INVALID_INPUT,+ "Local file access denied",+ "Use a regular file from an allowed download directory",+ ),+ )+action_result = _action_result_factory(ctx=ctx, page=page, selector=selector)with Timer() as timer:@@ -849,20 +956,11 @@ async def skyvern_file_upload(else:assert selector is not Noneawait page.upload_file(selector=selector, files=fp, timeout=action_timeout)- elif ai_mode is not None and len(file_paths) == 1:- # Single local file + intent: use SDK for AI element resolution- await page.upload_file(- selector=selector, # type: ignore[arg-type]- files=file_paths[0],- prompt=intent,- ai=ai_mode,- timeout=action_timeout,- )else:- # Local files + selector: set directly via Playwrightassert selector is not None+ assert local_uploads is not Nonelocator = page.page.locator(selector).first- await locator.set_input_files(file_paths, timeout=action_timeout)+ await locator.set_input_files(local_uploads, timeout=action_timeout)timer.mark("sdk")except PlaywrightTimeoutError as e:diff --git a/skyvern/cli/run_commands.py b/skyvern/cli/run_commands.pyindex 99fe3bbba..20c5121d6 100644--- a/skyvern/cli/run_commands.py+++ b/skyvern/cli/run_commands.py@@ -622,7 +622,10 @@ def run_mcp(except Exception:LOG.warning("local_browser_profile_startup_sweep_failed", exc_info=True)from skyvern.cli.core.mcp_http_auth import MCPAPIKeyMiddleware # noqa: PLC0415- from skyvern.cli.core.session_manager import set_stateless_http_mode # noqa: PLC0415+ from skyvern.cli.core.session_manager import ( # noqa: PLC0415+ set_stateless_http_mode,+ set_stdio_local_file_access_enabled,+ )from skyvern.cli.mcp_tools import mcp # noqa: PLC0415from skyvern.cli.mcp_tools.telemetry import configure_mcp_telemetry_runtime # noqa: PLC0415@@ -632,6 +635,7 @@ def run_mcp(# EOF dispatches the SIGINT cleanup handler; finally covers normal returns, with atexit as the last backstop.atexit.register(_cleanup_mcp_resources_sync)set_stateless_http_mode(stateless_http_enabled)+ set_stdio_local_file_access_enabled(transport == "stdio")set_concise_responses(not verbose)eof_watcher_stop: threading.Event | None = Noneshutdown_complete: threading.Event | None = None@@ -669,6 +673,7 @@ def run_mcp(eof_watcher_stop.set()try:set_stateless_http_mode(False)+ set_stdio_local_file_access_enabled(False)set_concise_responses(False)finally:for handled_signal, original_handler in original_signal_handlers.items():diff --git a/skyvern/client/client.py b/skyvern/client/client.pyindex c6ccfd591..db102f1d0 100644--- a/skyvern/client/client.py+++ b/skyvern/client/client.py@@ -2135,7 +2135,6 @@ class Skyvern:def create_browser_session(self,*,- url: typing.Optional[str] = OMIT,timeout: typing.Optional[int] = OMIT,proxy_location: typing.Optional[CreateBrowserSessionRequestProxyLocation] = OMIT,proxy_session_id: typing.Optional[str] = OMIT,@@ -2150,9 +2149,6 @@ class Skyvern:Parameters----------- url : typing.Optional[str]- Optional URL to open when the standalone browser session starts.-timeout : typing.Optional[int]Timeout in minutes for the session. Timeout is applied after the session is started. Must be between 5 and 1440. Defaults to 60.@@ -2226,7 +2222,6 @@ class Skyvern:client.create_browser_session()"""_response = self._raw_client.create_browser_session(- url=url,timeout=timeout,proxy_location=proxy_location,proxy_session_id=proxy_session_id,@@ -6290,7 +6285,6 @@ class AsyncSkyvern:async def create_browser_session(self,*,- url: typing.Optional[str] = OMIT,timeout: typing.Optional[int] = OMIT,proxy_location: typing.Optional[CreateBrowserSessionRequestProxyLocation] = OMIT,proxy_session_id: typing.Optional[str] = OMIT,@@ -6305,9 +6299,6 @@ class AsyncSkyvern:Parameters----------- url : typing.Optional[str]- Optional URL to open when the standalone browser session starts.-timeout : typing.Optional[int]Timeout in minutes for the session. Timeout is applied after the session is started. Must be between 5 and 1440. Defaults to 60.@@ -6389,7 +6380,6 @@ class AsyncSkyvern:asyncio.run(main())"""_response = await self._raw_client.create_browser_session(- url=url,timeout=timeout,proxy_location=proxy_location,proxy_session_id=proxy_session_id,diff --git a/skyvern/client/raw_client.py b/skyvern/client/raw_client.pyindex d008055ed..66b8141a3 100644--- a/skyvern/client/raw_client.py+++ b/skyvern/client/raw_client.py@@ -3193,7 +3193,6 @@ class RawSkyvern:def create_browser_session(self,*,- url: typing.Optional[str] = OMIT,timeout: typing.Optional[int] = OMIT,proxy_location: typing.Optional[CreateBrowserSessionRequestProxyLocation] = OMIT,proxy_session_id: typing.Optional[str] = OMIT,@@ -3208,9 +3207,6 @@ class RawSkyvern:Parameters----------- url : typing.Optional[str]- Optional URL to open when the standalone browser session starts.-timeout : typing.Optional[int]Timeout in minutes for the session. Timeout is applied after the session is started. Must be between 5 and 1440. Defaults to 60.@@ -3278,7 +3274,6 @@ class RawSkyvern:"v1/browser_sessions",method="POST",json={- "url": url,"timeout": timeout,"proxy_location": convert_and_respect_annotation_metadata(object_=proxy_location, annotation=CreateBrowserSessionRequestProxyLocation, direction="write"@@ -8798,7 +8793,6 @@ class AsyncRawSkyvern:async def create_browser_session(self,*,- url: typing.Optional[str] = OMIT,timeout: typing.Optional[int] = OMIT,proxy_location: typing.Optional[CreateBrowserSessionRequestProxyLocation] = OMIT,proxy_session_id: typing.Optional[str] = OMIT,@@ -8813,9 +8807,6 @@ class AsyncRawSkyvern:Parameters----------- url : typing.Optional[str]- Optional URL to open when the standalone browser session starts.-timeout : typing.Optional[int]Timeout in minutes for the session. Timeout is applied after the session is started. Must be between 5 and 1440. Defaults to 60.@@ -8883,7 +8874,6 @@ class AsyncRawSkyvern:"v1/browser_sessions",method="POST",json={- "url": url,"timeout": timeout,"proxy_location": convert_and_respect_annotation_metadata(object_=proxy_location, annotation=CreateBrowserSessionRequestProxyLocation, direction="write"diff --git a/skyvern/config.py b/skyvern/config.pyindex fb794c8aa..50e9a682a 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -225,6 +225,10 @@ class Settings(BaseSettings):# Kill switch for the live codegen-progress SSE frame (drafted block labels while an authoring# tool call streams). Off restores exact pre-change behavior; old frontends drop the frame either way.WORKFLOW_COPILOT_CODEGEN_PROGRESS_ENABLED: bool = True+ # Local-development escape hatch: run a copilot block test run in the API process when sandbox+ # dispatch is unavailable, instead of failing closed. Grants nothing on its own -- see+ # AgentFunction.allow_copilot_inline_code_execution for the conditions it is ANDed with.+ COPILOT_ALLOW_INLINE_CODE_EXECUTION: bool = False# Default code_only for MCP block/workflow tools. Off = permissive.MCP_CODE_ONLY_MODE: bool = False# Default for the bounded code-block self-heal; off by default.diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 14c847511..acf3dbf56 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -81,6 +81,25 @@ GMAIL_OTP_MAX_RESULTS = 5GMAIL_OTP_SEARCH_INTERVAL_SECONDS = 30_LLM_CALL_TIMEOUT_SECONDS = 30 # 30s++# Injected by kubelet and the ECS agent respectively.+_PACKAGED_DEPLOYMENT_ENV_MARKERS = (+ "KUBERNETES_SERVICE_HOST",+ "ECS_CONTAINER_METADATA_URI_V4",+ "ECS_CONTAINER_METADATA_URI",+)+# Docker writes the first, Podman the second. Neither is created by containerd or nerdctl, so this+# is a best-effort courtesy check and never an isolation boundary -- ENV is the boundary.+_PACKAGED_DEPLOYMENT_MARKER_FILES = ("/.dockerenv", "/run/.containerenv")+++def running_in_packaged_deployment() -> bool:+ """Whether this process looks like a packaged deployment rather than a developer checkout."""+ if any(os.environ.get(marker) for marker in _PACKAGED_DEPLOYMENT_ENV_MARKERS):+ return True+ return any(os.path.exists(path) for path in _PACKAGED_DEPLOYMENT_MARKER_FILES)++USELESS_SHAPE_ATTRIBUTE = [SKYVERN_ID_ATTR, "id", "aria-describedby"]SVG_SHAPE_CONVERTION_ATTEMPTS = 3CSS_SHAPE_CONVERTION_ATTEMPTS = 1@@ -964,13 +983,22 @@ class AgentFunction:organization_id: str,workflow_permanent_id: str,) -> bool:- """Base no-op (copilot runs its block test inline); overridden per deployment."""+ """Base no-op; callers fail closed when worker dispatch is unavailable."""return Falsedef resolve_copilot_dispatch_trigger_type(self) -> WorkflowRunTriggerType | None:"""Base no-op (no dispatch routing hint); overridden per deployment."""return None+ def allow_copilot_inline_code_execution(self) -> bool:+ """Whether a copilot block test run may execute in the API process when sandbox dispatch is+ unavailable. Developer checkouts only."""+ # ENV alone cannot carry this: self-hosted Docker and Kubernetes are documented with+ # ENV=local, so a packaged deployment would look identical to a laptop.+ if settings.is_cloud_environment() or running_in_packaged_deployment():+ return False+ return settings.COPILOT_ALLOW_INLINE_CODE_EXECUTION+async def is_workflow_tagging_enabled(self, organization_id: str) -> bool:"""OSS always-on; cloud overrides to gate per-org for staged rollout."""return True@@ -1521,7 +1549,7 @@ class AgentFunction:if "@" not in totp_identifier:return None- from skyvern.services.otp_service import parse_otp_login+ from skyvern.services.otp_service import InsufficientCreditsForOTPParse, parse_otp_loginlookup_context = context or GmailOTPVerificationContext()now = datetime.now(timezone.utc)@@ -1589,6 +1617,11 @@ class AgentFunction:continuetry:otp_value = await parse_otp_login(candidate.content, organization_id)+ except InsufficientCreditsForOTPParse:+ # Credit eligibility is organization-wide, so stop this scan.+ # Remember the candidate so later polls do not retry the same material.+ lookup_context.remember_message_id(candidate.message_id)+ return Noneexcept Exception:LOG.warning("Failed to parse Gmail OTP candidate",diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.pyindex 7be398454..4f53e46ed 100644--- a/skyvern/forge/failure_classifier.py+++ b/skyvern/forge/failure_classifier.py@@ -6,6 +6,16 @@ import re# timeout finalizes a run. Word-anchored so "inactivity timeout" (a distinct page-level reason)# is NOT caught and correctly stays PAGE_LOAD_TIMEOUT._INFRA_TIMEOUT_RE = re.compile(r"\b(?:activity|heartbeat) timeout\b")+# Playwright names the operation that timed out. A locator operation waiting for an element to+# reach a state is not a page-load failure, and telling the user to confirm the URL for one sends+# both them and copilot repair after the wrong thing. Truncated messages keep only the call-log+# tail ("waiting for locator(...)"), so the bare forms match too.+_ELEMENT_OPERATION_RE = re.compile(+ r"\b(?:locator\.[a-z_]+\b|locator\(|wait_for_selector\b|waiting for selector\b|get_by_[a-z_]+)"+)+# Selector payloads name page structure, not failure semantics: `locator('#password')` is the+# element waited for, not a rejected login. Auth keyword scans run on text with these excised.+_SELECTOR_PAYLOAD_RE = re.compile(r"(?:locator|get_by_[a-z_]+|wait_for_selector)\([^)]*\)|selector ['\"][^'\"]*['\"]")def classify_from_failure_reason(@@ -28,9 +38,9 @@ def classify_from_failure_reason(where the absence of a classification may simply mean the termination wasuser-guided / expected.- Categories (16):+ Categories (17):ANTI_BOT_DETECTION, PROXY_ERROR, BROWSER_ERROR, NAVIGATION_FAILURE,- PAGE_LOAD_TIMEOUT, AUTH_FAILURE, LLM_ERROR, CREDENTIAL_ERROR,+ PAGE_LOAD_TIMEOUT, ELEMENT_STATE_TIMEOUT, AUTH_FAILURE, LLM_ERROR, CREDENTIAL_ERROR,DATA_EXTRACTION_FAILURE, ELEMENT_NOT_FOUND, WRONG_PAGE_STATE,MAX_STEPS_EXCEEDED, LLM_REASONING_ERROR, INFRASTRUCTURE_ERROR,PARAMETER_BINDING_ERROR, UNKNOWN@@ -39,13 +49,14 @@ def classify_from_failure_reason(return Nonereason = (failure_reason or "").lower()+ auth_scan_reason = _SELECTOR_PAYLOAD_RE.sub(" ", reason)exc_name = type(exception).__name__ if exception else (exception_name or "")categories: list[dict] = []# Bot detection / CAPTCHA — use specific phrases to avoid false positives_auth_context_keywords = ["login", "auth", "password", "permission", "credential"]- _has_auth_context = any(kw in reason for kw in _auth_context_keywords)+ _has_auth_context = any(kw in auth_scan_reason for kw in _auth_context_keywords)_antibot_keywords = ["captcha","cloudflare",@@ -130,8 +141,22 @@ def classify_from_failure_reason(})+ _is_timeout = "Timeout" in exc_name or "timeout" in reason+ _is_element_state_timeout = _is_timeout and bool(_ELEMENT_OPERATION_RE.search(reason))++ # Element-state timeout — a locator operation that never reached the state it waited for.+ if _is_element_state_timeout:+ categories.append(+ {+ "category": "ELEMENT_STATE_TIMEOUT",+ "confidence_float": 0.85,+ "reason_code": "locator_wait_for_timeout",+ "reasoning": "Locator operation timed out waiting for element state",+ }+ )+# Page load timeout- if ("Timeout" in exc_name or "timeout" in reason) and not _is_infra_timeout:+ if _is_timeout and not _is_infra_timeout and not _is_element_state_timeout:categories.append({"category": "PAGE_LOAD_TIMEOUT",@@ -140,9 +165,10 @@ def classify_from_failure_reason(})- # Auth failure — also catches "access denied" when auth context is present- if any(kw in reason for kw in ["login fail", "authentication fail", "auth fail", "mfa", "password"]) or (- "access denied" in reason and _has_auth_context+ # Auth failure — also catches "access denied" when auth context is present. Selector payloads+ # are stripped above, so a genuine auth message keeps its signal even alongside a locator timeout.+ if any(kw in auth_scan_reason for kw in ["login fail", "authentication fail", "auth fail", "mfa", "password"]) or (+ "access denied" in auth_scan_reason and _has_auth_context):categories.append({diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2index 196240f2a..5f4f85bc1 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2@@ -2,11 +2,11 @@ You are an expert Skyvern Workflow assistant with access to tools that let you bCRITICAL ROLE BOUNDARY:You are a workflow builder, NOT a chatbot or search engine. Your ONLY job is to create Skyvern workflows that accomplish what the user asks.-- NEVER directly answer the user's question or provide information you found during testing.+- NEVER answer from what you saw while testing INSTEAD of building the workflow. A reply that reports a value without a workflow that produces it is a failed turn, however correct the value is.- ALWAYS respond in terms of the workflow you built: "I created a workflow that extracts the top 3 posts from the site."-- NEVER include scraped data, extracted content, or live website information in your response.-- If the test run produced extracted data, mention that the extraction succeeded and describe the data shape (e.g., "3 items with rank, title, URL, and points"), but do NOT include the actual values.-- The user will run the workflow themselves to get results. Your job is to build and verify the workflow, not to relay results.+- Once a test run has produced the requested output, DO report that value in your reply, alongside the workflow: "I created a workflow that extracts the error count; the latest run returned 778." The run is what earns the claim, and the user asked for the value.+- Before a run has produced the requested output, do NOT include scraped data, extracted content, or live website information — describe the data shape instead (e.g., "3 items with rank, title, URL, and points").+- The workflow is the deliverable and the user will run it themselves for fresh results; reporting what the test run returned is reporting on the workflow, not replacing it.- The "no chatbot" rule above scopes to questions whose answer is *data from an external site or a test-run extraction* — those still need a workflow. Questions whose answer is *Skyvern documentation* (what a block type does, how the copilot works, what a Skyvern concept means, or comparisons between Skyvern concepts) get answered inline: respond with `ASK_QUESTION` and put the explanation in `user_response`; do NOT build or run a workflow to launder the answer through `text_prompt`. Disambiguators: (1) "Can Skyvern do X on site Y?" is a request to build X — proceed with the build, not the docs lookup. (2) If the user explicitly asks for a fix, improvement, prompt change, or alternative for a runtime symptom on their current workflow, that is a debugging/edit request — inspect the current workflow/run evidence, then fix the workflow with `update_and_run_blocks` when the correction is clear. (3) Docs-inline does NOT apply to a runtime symptom that names a specific workflow entity (a block label, workflow parameter key, or {% raw %}`{{ name.output... }}`{% endraw %} Jinja reference) until the current workflow YAML or debugger evidence confirms that entity is present — use the ASK-vs-EDIT routing below to decide whether to build/edit, ask, or answer docs inline only after ruling out a pending workflow build/edit.{% if security_rules %}@@ -72,7 +72,7 @@ CRITICAL RULES WHEN USING DIRECT BROWSER TOOLS:- If you call navigate_browser, you MUST call at least one observation tool (evaluate, get_browser_screenshot, click, type_text, scroll, select_option, press_key, or console_messages) before responding.- NEVER answer questions about page content, link counts, link targets, or page structure from memory or training data. Use evaluate or get_browser_screenshot.- Use evaluate for reading/inspection; do NOT use evaluate to click or type. Use click and type_text for interactions.-- Direct browser `evaluate` is for scouting only. Persisted workflow `code` blocks must not use `page.evaluate`, `page.evaluate_handle`, `page.request`, or `page.context`; use `page.locator(...)`, `page.get_by_role(...)`, and locator DOM-reading methods such as `inner_text`, `text_content`, `get_attribute`, `count`, and `is_visible`.+- Prefer locator reads in persisted `code` blocks: `page.locator(...)`, `page.get_by_role(...)`, and methods such as `inner_text`, `text_content`, `get_attribute`, `count`, `is_visible`. Locators auto-wait; `page.evaluate` does not, so a raw evaluate that worked while you were looking at the page can run too early on replay. `page.evaluate` is permitted when you are replaying an expression you already proved on this page and no locator read expresses it; `page.evaluate_handle`, `page.request`, and `page.context` remain unavailable.- CSS SELECTORS: interactions target elements by CSS selector, which is instant and deterministic. The selectors reported by page inspection are verified to match exactly one element, so prefer copying one of those over composing your own. When a shared class matches many elements (e.g. one Add-to-Cart per result row), scope the selector to the specific item (its container, a unique attribute, or `:nth-of-type`). If a selector does not resolve or the action fails, observe the page again — with `evaluate`, `inspect_page_for_composition`, or `get_browser_screenshot`, whichever answers the question — and derive a better selector. jQuery pseudo-selectors (:contains(), :eq(), :first, :visible, :has()) are NOT valid — use standard selectors like 'button.classname', 'a[href*="keyword"]', '#id', 'tr:nth-of-type(2) td a'.- SCROLL: Use scroll to reveal content below the fold. Provide direction (up/down/left/right) and optionally amount in pixels, a selector for the scrollable container, or an intent to scroll a specific element into view.- SELECT_OPTION: Use select_option for <select> dropdowns. Provide the value and a selector to target the element. For free-text inputs, use type_text instead.diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 17bbcf05e..5a5970c70 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -6,18 +6,14 @@ Uses the OpenAI Agents SDK with LiteLLM for multi-provider LLM support.from __future__ import annotationsimport asyncio-import base64-import binasciiimport contextlibimport json-import mathimport osimport reimport uuidfrom collections.abc import Callable, Iterator, Mapping, Sequencefrom dataclasses import dataclassfrom datetime import datetime, timezone-from io import BytesIOfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Protocol, runtime_checkable@@ -33,12 +29,10 @@ if TYPE_CHECKING:import structlogimport yamlfrom litellm.exceptions import NotFoundError as LiteLLMNotFoundError-from PIL import Image, UnidentifiedImageErrorfrom pydantic import ValidationErrorfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine-from skyvern.forge.request_logging import redact_sensitive_fieldsfrom skyvern.forge.sdk.copilot import llm_configfrom skyvern.forge.sdk.copilot.blocker_signal import (CopilotToolBlockerSignal,@@ -93,7 +87,6 @@ from skyvern.forge.sdk.copilot.context import (AgentResult,CodeAuthoringRepairContext,CopilotContext,- DeliveredUnverifiedPublicOutputs,NarrativeActivityEntry,NarrativeBlock,NarrativeDraft,@@ -118,9 +111,6 @@ from skyvern.forge.sdk.copilot.credential_literal_rebind import (from skyvern.forge.sdk.copilot.credential_pause import preflight_credential_pausefrom skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failurefrom skyvern.forge.sdk.copilot.enforcement import (- BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,- SCOUTED_SPINE_TURN_HALT_USER_REASON,- _get_scouted_spine_missing_steps_for_halt,artifact_health_blocked,log_scouted_spine_unresolved_at_turn_halt,outcome_fully_verified,@@ -146,7 +136,6 @@ from skyvern.forge.sdk.copilot.output_contracts import (from skyvern.forge.sdk.copilot.output_policy import (ACTUATION_OBLIGATION_STEER_REASON_CODE,ACTUATION_OBLIGATION_UNMET_REASON_CODE,- UNVALIDATED_DISCLOSURE_PHRASES,WORKFLOW_PRESENT_SENTINEL,ActuationObligationEvaluation,ActuationObligationStatus,@@ -192,8 +181,7 @@ from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, run_outcomfrom skyvern.forge.sdk.copilot.runtime import (_browser_context_is_attachable,)-from skyvern.forge.sdk.copilot.secret_redaction import SECRET_KEYWORD_ASSIGNMENT_PATTERN-from skyvern.forge.sdk.copilot.secret_scrub import registered_scrub_values, scrub_secrets_from_text+from skyvern.forge.sdk.copilot.secret_scrub import registered_scrub_valuesfrom skyvern.forge.sdk.copilot.streaming_adapter import (emit_turn_start,emit_workflow_draft,@@ -209,7 +197,6 @@ from skyvern.forge.sdk.copilot.turn_halt import (_INVOLUNTARY_BLOCKER_REASON_CODES,CopilotTurnHalt,TurnHalt,- TurnHaltKind,expire_output_contract_ladder_at_turn_end,raise_if_turn_halt,turn_halt_to_trace_data,@@ -1657,13 +1644,21 @@ def _partial_verification_response(ctx: CopilotContext) -> str | None:)-def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> str:+def _rewrite_failed_test_response(+ user_response: str, ctx: CopilotContext, *, agent_admits_incomplete: bool = True+) -> str:has_keepable_draft = ctx.last_workflow is not None and bool(ctx.last_workflow_yaml)keep_draft_affordance = " Keep the draft to iterate on, or discard." if has_keepable_draft else ""block_count = ctx.last_update_block_count if isinstance(ctx.last_update_block_count, int) else Nonepositive_block_count = block_count if block_count is not None and block_count > 0 else Noneif outcome_fully_verified(ctx) and has_keepable_draft:+ # Only the model's own admission of failure gets rewritten, which is this function's whole+ # job: a verification landing after the model gave up would otherwise reach the user as a+ # false negative. When the model does not admit failure its reply is its report of a run it+ # saw, values included, and a template would replace that with an unasked-for block count.+ if not agent_admits_incomplete:+ return user_responseif positive_block_count is not None:block_word = "block" if positive_block_count == 1 else "blocks"return (@@ -1736,15 +1731,6 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> streturn user_response-def _shape_ask_question_response(user_response: str, ctx: CopilotContext) -> str:- from skyvern.forge.sdk.copilot.enforcement import build_probable_site_block_user_question-- site_block_question = build_probable_site_block_user_question(ctx)- if site_block_question is not None:- return site_block_question- return user_response--def _completion_contract_not_violated(ctx: CopilotContext) -> bool:if artifact_health_blocked(ctx):return False@@ -1776,32 +1762,6 @@ def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:return None, None-_BUILT_UNVERIFIED_COMPLETED_REPLY = (- "I built the workflow and the test run completed, but the goal outcome was not independently verified. "- "The workflow is available on the canvas for review."-)---def _should_use_built_unverified_completed_reply(- ctx: CopilotContext,- *,- response_type: str,- updated_workflow: Any,- validated: bool,- blocker_active: bool,-) -> bool:- return (- response_type == "REPLY"- and updated_workflow is not None- and validated- and not blocker_active- and ctx.last_test_ok is True- and ctx.last_full_workflow_test_ok is True- and not ctx.last_test_suspicious_success- and not verified_goal_claim_authorized(ctx)- )--def _terminal_envelope_run_outcomes(ctx: CopilotContext) -> list[RecordedRunOutcome]:raw = ctx.terminal_envelope_run_outcomesif isinstance(raw, Sequence) and not isinstance(raw, (str, bytes, bytearray)):@@ -1883,7 +1843,6 @@ def _make_agent_result(*,global_llm_context: str | None = None,turn_outcome: TurnOutcome | None = None,- _delivered_unverified_snapshot: _DeliveredUnverifiedSnapshot | None = None,**kwargs: Any,) -> AgentResult:"""Sole ``AgentResult`` constructor in this module.@@ -1903,7 +1862,6 @@ def _make_agent_result(if ctx is not Noneelse global_llm_context)- kwargs.pop("delivered_unverified_observed_outputs", None)narrative_payload = kwargs.get("narrative_payload")if ctx is not None and narrative_payload is None:raise ValueError("_make_agent_result requires narrative_payload when ctx is provided")@@ -1938,13 +1896,6 @@ def _make_agent_result(adjudication = _build_outcome_adjudication_payload(ctx)if adjudication is not None:payload_updates["outcomeAdjudication"] = adjudication- if (- ctx is not None- and ctx.delivered_unverified_terminal is True- and isinstance(_delivered_unverified_snapshot, _DeliveredUnverifiedSnapshot)- and _delivered_unverified_snapshot- ):- payload_updates["deliveredUnverifiedObservedOutputs"] = _delivered_unverified_snapshotif payload_updates or len(payload_base) != len(narrative_payload):kwargs["narrative_payload"] = {**payload_base, **payload_updates}terminal_envelope: dict[str, Any] | None = None@@ -2109,7 +2060,6 @@ def _build_exit_result(cancelled: bool = False,terminal_reason: str | None = None,proposal_disposition: ProposalDisposition = "auto_applicable",- _delivered_unverified_snapshot: _DeliveredUnverifiedSnapshot | None = None,) -> AgentResult:"""AgentResult for agent-loop exits that don't go through ``_translate_to_agent_result``."""verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)@@ -2163,7 +2113,6 @@ def _build_exit_result(ctx,_make_agent_result(ctx,- _delivered_unverified_snapshot=_delivered_unverified_snapshot,user_response=final_text,updated_workflow=verified_workflow,global_llm_context=global_llm_context,@@ -2197,7 +2146,6 @@ async def _build_goal_satisfied_exit_result(terminal_reason: str = "verified_goal_satisfied",exit_site: str = "verified_goal_satisfied",flush_goal_satisfied: bool = True,- name_missing_spine_steps: bool = False,) -> AgentResult:# Bypass one extra LLM turn after a full workflow test already satisfies# the diagnosis contract.@@ -2207,14 +2155,12 @@ async def _build_goal_satisfied_exit_result(except Exception as flush_err:LOG.warning("copilot_goal_satisfied_tool_result_flush_failed", error=str(flush_err))verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)- clean_test = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True- if clean_test and verified_goal_claim_authorized(ctx):+ # A run that produced the value is deliverable whether or not the chain was re-run from cold;+ # re-running a proven prefix to earn the claim costs a login and teaches the turn nothing.+ clean_test = ctx.last_test_ok is True+ if clean_test:+ # A clean test IS the evidence; a judge's reading of the same run does not gate saying so.user_response = _verified_workflow_success_reply(ctx)- elif clean_test:- user_response = (- "I built the workflow and the test run completed, but the goal outcome was not "- "independently verified. Review the draft to confirm it does what you need."- )elif ctx.last_test_ok is False:user_response = "I reached the requested outcome, but the workflow test did not finish successfully."else:@@ -2222,10 +2168,6 @@ async def _build_goal_satisfied_exit_result("I reached the requested outcome, but the workflow has not been tested end-to-end. ""Review the draft before using it.")- if name_missing_spine_steps and log_scouted_spine_unresolved_at_turn_halt(ctx):- user_response = _with_scouted_spine_missing_steps(- ctx, user_response, _get_scouted_spine_missing_steps_for_halt(ctx), held_signal_unrelated=True- )final_text, outcome = apply_repeated_reply_guard(final_text=user_response,attempted_kind=derive_response_kind(ctx.turn_intent),@@ -2307,79 +2249,20 @@ async def _build_goal_satisfied_exit_result()-async def _build_built_unverified_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:- return await _build_goal_satisfied_exit_result(- ctx,- global_llm_context,- terminal_reason=BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,- exit_site=BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,- flush_goal_satisfied=False,- name_missing_spine_steps=True,- )---_SCOUTED_SPINE_HALT_REPLY_KINDS = frozenset({TurnHaltKind.LOOP_DETECTED})-_SCOUTED_SPINE_MISSING_STEPS_PREFIX = "This draft is still missing steps you demonstrated:"---def _with_scouted_spine_missing_steps(- ctx: CopilotContext,- user_response: str,- missing_steps: str | None,- *,- held_signal_unrelated: bool = False,-) -> str:- """Name the demonstrated-but-missing steps on any give-up offer that carries a staged proposal, and- anchor them in the held blocker signal so the blocker-override finalizer cannot re-render them away.- When the held signal is an unrelated blocker that owns the turn (output policy, timeout, credential),- the steps are surfaced in the reply only; its reason and claimant stay intact so the turn's single- owner keeps rendering its own reply. Idempotent so a pre-finalizer append plus the finalizer re-render- never doubles the note."""- if not missing_steps or not ctx.has_staged_proposal:- return user_response- if _SCOUTED_SPINE_MISSING_STEPS_PREFIX in user_response:- return user_response- reframed = f"{user_response}\n\n{_SCOUTED_SPINE_MISSING_STEPS_PREFIX} {missing_steps}"- if not held_signal_unrelated and isinstance(ctx.blocker_signal, CopilotToolBlockerSignal):- ctx.blocker_signal = ctx.blocker_signal.model_copy(update={"user_facing_reason": reframed})- ctx.blocker_signal_claimant = None- return reframed--def _build_turn_halt_exit_result(ctx: CopilotContext,global_llm_context: str | None,halt: TurnHalt,) -> AgentResult:- under_build_open = log_scouted_spine_unresolved_at_turn_halt(ctx)- missing_steps = _get_scouted_spine_missing_steps_for_halt(ctx) if under_build_open else None- if halt.kind == TurnHaltKind.DELIVERED_UNVERIFIED:- return _build_wip_exit_result(- ctx,- global_llm_context,- default_reply=_BUILT_UNVERIFIED_COMPLETED_REPLY,- unvalidated_reply=_BUILT_UNVERIFIED_COMPLETED_REPLY,- tested_reply=_BUILT_UNVERIFIED_COMPLETED_REPLY,- terminal_reason=f"turn_halt:{halt.kind.value}",- missing_spine_steps=missing_steps,- )+ log_scouted_spine_unresolved_at_turn_halt(ctx)signal = halt.blocker_signalif isinstance(signal, CopilotToolBlockerSignal) and signal.blocker_kind == "loop_detected":refresh_held_loop_blocker_evidence(ctx)signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else signal- scouted_spine_owns = under_build_open and halt.kind in _SCOUTED_SPINE_HALT_REPLY_KINDS- if scouted_spine_owns:- user_response = SCOUTED_SPINE_TURN_HALT_USER_REASON- if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal):- ctx.blocker_signal = ctx.blocker_signal.model_copy(update={"user_facing_reason": user_response})- ctx.blocker_signal_claimant = None- elif isinstance(signal, CopilotToolBlockerSignal):+ if isinstance(signal, CopilotToolBlockerSignal):user_response = signal.user_facing_reasonelse:user_response = "I could not continue this turn safely. Tell me what to change and I'll try again."- user_response = _with_scouted_spine_missing_steps(- ctx, user_response, missing_steps, held_signal_unrelated=not scouted_spine_owns- )return _build_exit_result(ctx,user_response,@@ -2468,10 +2351,6 @@ _BLOCK_YAML_IN_REPLY_REWRITE_NO_PROPOSAL = (_BLOCK_YAML_IN_REPLY_REWRITE_WITH_PROPOSAL = "I made the change you described to the workflow."_PROPOSAL_ACCEPT_UI_ACTION_RE = re.compile(r"\b(?:accept|always\s+accept)\b", re.IGNORECASE)_PROPOSAL_REJECT_UI_ACTION_RE = re.compile(r"\b(?:reject|discard)\b", re.IGNORECASE)-_UNVALIDATED_PROPOSAL_AFFORDANCE = (- "I have a draft workflow proposal. Use Review to inspect it, Accept to save it, or Reject to discard it. "- "It has not been tested or verified end-to-end."-)_VERIFIED_CLASSIFICATION_CONTEXT_KEYS = ("visible_page_path_label","safest_reachable_next_step",@@ -2485,360 +2364,6 @@ _VERIFIED_CLASSIFICATION_GATE_KEYS = ()_VERIFIED_CLASSIFICATION_GATE_PHRASES = ("Sign in or register to continue",)_VERIFIED_TERMINAL_VALUE_MAX_CHARS = 180-_DELIVERED_UNVERIFIED_SUMMARY_MAX_CHARS = 720-_DELIVERED_UNVERIFIED_OMISSION_KEY = "$skyvernOmitted"-_DELIVERED_UNVERIFIED_METADATA_KEY = "$skyvernOutput"-_DELIVERED_UNVERIFIED_MAX_DEPTH = 16-_DELIVERED_UNVERIFIED_MAX_NODES = 64-_DELIVERED_UNVERIFIED_MAX_STRING_CHARS = 512-_DELIVERED_UNVERIFIED_MAX_SERIALIZED_BYTES = 65_536-_DELIVERED_UNVERIFIED_SERIALIZED_METADATA_RESERVE_BYTES = 2_048-_DELIVERED_UNVERIFIED_MAX_IMAGE_ENCODED_BYTES = 65_536-_DELIVERED_UNVERIFIED_MAX_IMAGE_DECODED_BYTES = 49_152-_DELIVERED_UNVERIFIED_MAX_IMAGE_PIXELS = 1_000_000-_REDACTED_INTERNAL_OUTPUT = "[REDACTED_INTERNAL]"-_UNSUPPORTED_DELIVERED_OUTPUT = object()-_INVALID_DELIVERED_IMAGE = object()-_PNG_START = b"\x89PNG\r\n\x1a\n"-_PNG_END = b"\x00\x00\x00\x00IEND\xaeB\x60\x82"-_JPEG_START = b"\xff\xd8"-_JPEG_END = b"\xff\xd9"-_CAMEL_CASE_ACRONYM_BOUNDARY_RE = re.compile(r"(?<=[A-Z])(?=[A-Z][a-z])")-_CAMEL_CASE_KEY_BOUNDARY_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")-_DELIMITED_KEY_BOUNDARY_RE = re.compile(r"[-\s]+")---def _delivered_unverified_image_payload(value: str) -> tuple[bool, str]:- lowered = value[:32].lower()- for prefix in ("data:image/png;base64,", "data:image/jpeg;base64,", "data:image/jpg;base64,"):- if lowered.startswith(prefix):- return True, value[len(prefix) :]- if lowered.startswith("data:image/"):- return True, ""- try:- decoded_prefix = base64.b64decode(value[:16], validate=True)- except (binascii.Error, ValueError):- return False, value- return decoded_prefix.startswith((_PNG_START, _JPEG_START)), value---def _delivered_unverified_decoded_size(encoded: str) -> int | None:- if not encoded or len(encoded) % 4 != 0:- return None- padding = len(encoded) - len(encoded.rstrip("="))- if padding > 2:- return None- return (len(encoded) // 4) * 3 - padding---def _canonical_delivered_unverified_image(value: str) -> tuple[bool, str | None]:- image_input, encoded = _delivered_unverified_image_payload(value)- if not image_input:- return False, None- encoded_size = len(encoded.encode("ascii", errors="ignore"))- decoded_size = _delivered_unverified_decoded_size(encoded)- if (- encoded_size != len(encoded)- or encoded_size > _DELIVERED_UNVERIFIED_MAX_IMAGE_ENCODED_BYTES- or decoded_size is None- or decoded_size > _DELIVERED_UNVERIFIED_MAX_IMAGE_DECODED_BYTES- ):- return True, None- try:- decoded = base64.b64decode(encoded, validate=True)- except (binascii.Error, ValueError):- return True, None- if len(decoded) > _DELIVERED_UNVERIFIED_MAX_IMAGE_DECODED_BYTES:- return True, None- if not decoded.startswith((_PNG_START, _JPEG_START)):- return True, None- try:- with Image.open(BytesIO(decoded)) as image:- width, height = image.size- if width <= 0 or height <= 0 or width * height > _DELIVERED_UNVERIFIED_MAX_IMAGE_PIXELS:- return True, None- if image.format == "PNG":- if not decoded.endswith(_PNG_END):- return True, None- canonical_image = image.convert("RGBA")- elif image.format == "JPEG":- if not decoded.endswith(_JPEG_END):- return True, None- canonical_image = image.convert("RGB")- else:- return True, None- image.load()- pixels_only = Image.frombytes(canonical_image.mode, canonical_image.size, canonical_image.tobytes())- canonical_bytes = BytesIO()- pixels_only.save(canonical_bytes, format=image.format)- except (binascii.Error, Image.DecompressionBombError, OSError, SyntaxError, UnidentifiedImageError, ValueError):- return True, None- return True, base64.b64encode(canonical_bytes.getvalue()).decode()---def _sanitize_delivered_unverified_scalar(ctx: CopilotContext, value: Any) -> Any:- if value is None or isinstance(value, bool | int):- return value- if isinstance(value, float):- return value if math.isfinite(value) else _UNSUPPORTED_DELIVERED_OUTPUT- if isinstance(value, str):- image_input, canonical_image = _canonical_delivered_unverified_image(value)- if image_input:- return canonical_image if canonical_image is not None else _INVALID_DELIVERED_IMAGE- registered_scrubbed = scrub_secrets_from_text(ctx, value)- redacted = redact_sensitive_fields(redact_raw_secrets_for_prompt(registered_scrubbed))- return _REDACTED_INTERNAL_OUTPUT if contains_internal_machinery_leak(redacted) else redacted- return _UNSUPPORTED_DELIVERED_OUTPUT---def _delivered_unverified_sensitive_key(key: str) -> bool:- original = key.strip()- normalized = _CAMEL_CASE_ACRONYM_BOUNDARY_RE.sub("_", original)- normalized = _CAMEL_CASE_KEY_BOUNDARY_RE.sub("_", normalized)- normalized = _DELIMITED_KEY_BOUNDARY_RE.sub("_", normalized).lower()- for candidate in dict.fromkeys((original, normalized)):- if SECRET_KEYWORD_ASSIGNMENT_PATTERN.fullmatch(candidate) or SECRET_KEYWORD_ASSIGNMENT_PATTERN.fullmatch(- f"{candidate}=value"- ):- return True- redacted = redact_sensitive_fields({candidate: None})- if isinstance(redacted, dict) and redacted.get(candidate) == "****":- return True- return False---def _disambiguate_captured_output_key(key: str) -> str:- if key in {_DELIVERED_UNVERIFIED_OMISSION_KEY, _DELIVERED_UNVERIFIED_METADATA_KEY}:- return f"{key} [captured]"- return key---def _unique_sanitized_output_key(target: dict[str, Any], key: str) -> str:- if key not in target:- return key- collision_index = 2- while f"{key} [{collision_index}]" in target:- collision_index += 1- return f"{key} [{collision_index}]"---def _delivered_unverified_omission(reason: str) -> dict[str, dict[str, str | int]]:- return {_DELIVERED_UNVERIFIED_OMISSION_KEY: {"reason": reason, "count": 1}}---@dataclass-class _DeliveredUnverifiedAdmissionBudget:- nodes: int = 0- depth_omitted: int = 0- node_omitted: int = 0- string_omitted: int = 0- serialized_bytes_omitted: int = 0- non_finite_omitted: int = 0- cycle_omitted: int = 0- unsupported_omitted: int = 0- image_omitted: int = 0-- def omission_counts(self) -> dict[str, int]:- counts = {- "depth": self.depth_omitted,- "node": self.node_omitted,- "string": self.string_omitted,- "serializedBytes": self.serialized_bytes_omitted,- "nonFinite": self.non_finite_omitted,- "cycle": self.cycle_omitted,- "unsupported": self.unsupported_omitted,- "image": self.image_omitted,- }- return {reason: count for reason, count in counts.items() if count}---def _bounded_delivered_unverified_scalar(- ctx: CopilotContext,- value: Any,- budget: _DeliveredUnverifiedAdmissionBudget,-) -> Any:- scalar = _sanitize_delivered_unverified_scalar(ctx, value)- if scalar is _INVALID_DELIVERED_IMAGE:- budget.image_omitted += 1- return _delivered_unverified_omission("invalid image")- return scalar---def _delivered_unverified_admission_priority(value: Any) -> int:- """Admit bounded scalar outcomes before containers and long prose."""- if value is None or isinstance(value, bool | int):- return 0- if isinstance(value, float):- return 0 if math.isfinite(value) else 3- if isinstance(value, str):- return 0 if len(value) <= _VERIFIED_TERMINAL_VALUE_MAX_CHARS else 2- if isinstance(value, dict | list):- return 1- return 3---def _delivered_unverified_fits_serialized_budget(root: dict[str, Any] | list[Any]) -> bool:- # Count the conservative ASCII-escaped wire form so alternate JSON- # serializers cannot expand admitted Unicode beyond the public cap.- serialized_size = len(json.dumps(root, separators=(",", ":")).encode())- return serialized_size <= (- _DELIVERED_UNVERIFIED_MAX_SERIALIZED_BYTES - _DELIVERED_UNVERIFIED_SERIALIZED_METADATA_RESERVE_BYTES- )---def _admit_delivered_unverified_mapping_value(- root: dict[str, Any] | list[Any],- target: dict[str, Any],- key: str,- value: Any,- budget: _DeliveredUnverifiedAdmissionBudget,-) -> bool:- target[key] = value- if _delivered_unverified_fits_serialized_budget(root):- return True- del target[key]- budget.serialized_bytes_omitted += 1- return False---def _admit_delivered_unverified_list_value(- root: dict[str, Any] | list[Any],- target: list[Any],- value: Any,- budget: _DeliveredUnverifiedAdmissionBudget,-) -> bool:- target.append(value)- if _delivered_unverified_fits_serialized_budget(root):- return True- target.pop()- budget.serialized_bytes_omitted += 1- return False---def _sanitize_delivered_unverified_value(ctx: CopilotContext, value: Any) -> Any:- budget = _DeliveredUnverifiedAdmissionBudget()- scalar = _bounded_delivered_unverified_scalar(ctx, value, budget)- if scalar is not _UNSUPPORTED_DELIVERED_OUTPUT:- return scalar- if not isinstance(value, dict | list):- return _UNSUPPORTED_DELIVERED_OUTPUT-- root: dict[str, Any] | list[Any] = {} if isinstance(value, dict) else []- pending: list[tuple[dict[Any, Any] | list[Any], dict[str, Any] | list[Any], frozenset[int], int]] = [- (value, root, frozenset({id(value)}), 0)- ]- while pending:- source, target, ancestors, depth = pending.pop(0)- if isinstance(source, dict) and isinstance(target, dict):- ordered_items = sorted(source.items(), key=lambda entry: _delivered_unverified_admission_priority(entry[1]))- for index, (key, item) in enumerate(ordered_items):- if budget.nodes >= _DELIVERED_UNVERIFIED_MAX_NODES:- budget.node_omitted += len(source) - index- break- budget.nodes += 1- if not isinstance(key, str):- budget.unsupported_omitted += 1- continue- if len(key) > _DELIVERED_UNVERIFIED_MAX_STRING_CHARS:- budget.string_omitted += 1- continue- source_key_sensitive = _delivered_unverified_sensitive_key(key)- sanitized_key = _bounded_delivered_unverified_scalar(ctx, key, budget)- if not isinstance(sanitized_key, str) or sanitized_key == _REDACTED_INTERNAL_OUTPUT:- budget.unsupported_omitted += 1- continue- sanitized_key = _disambiguate_captured_output_key(sanitized_key)- sanitized_key = _unique_sanitized_output_key(target, sanitized_key)- if source_key_sensitive or _delivered_unverified_sensitive_key(sanitized_key):- _admit_delivered_unverified_mapping_value(root, target, sanitized_key, "****", budget)- continue- sanitized_item = _bounded_delivered_unverified_scalar(ctx, item, budget)- if sanitized_item is not _UNSUPPORTED_DELIVERED_OUTPUT:- _admit_delivered_unverified_mapping_value(root, target, sanitized_key, sanitized_item, budget)- continue- if isinstance(item, float) and not math.isfinite(item):- budget.non_finite_omitted += 1- _admit_delivered_unverified_mapping_value(- root, target, sanitized_key, _delivered_unverified_omission("non-finite number"), budget- )- elif isinstance(item, dict | list) and id(item) in ancestors:- budget.cycle_omitted += 1- _admit_delivered_unverified_mapping_value(- root, target, sanitized_key, _delivered_unverified_omission("cycle"), budget- )- elif isinstance(item, dict | list):- if depth >= _DELIVERED_UNVERIFIED_MAX_DEPTH:- budget.depth_omitted += 1- _admit_delivered_unverified_mapping_value(- root, target, sanitized_key, _delivered_unverified_omission("depth budget"), budget- )- continue- child: dict[str, Any] | list[Any] = {} if isinstance(item, dict) else []- if _admit_delivered_unverified_mapping_value(root, target, sanitized_key, child, budget):- pending.append((item, child, ancestors | {id(item)}, depth + 1))- else:- budget.unsupported_omitted += 1- _admit_delivered_unverified_mapping_value(- root, target, sanitized_key, _delivered_unverified_omission("unsupported value"), budget- )- elif isinstance(source, list) and isinstance(target, list):- for index, item in enumerate(source):- if budget.nodes >= _DELIVERED_UNVERIFIED_MAX_NODES:- budget.node_omitted += len(source) - index- break- budget.nodes += 1- sanitized_item = _bounded_delivered_unverified_scalar(ctx, item, budget)- if sanitized_item is not _UNSUPPORTED_DELIVERED_OUTPUT:- _admit_delivered_unverified_list_value(root, target, sanitized_item, budget)- continue- if isinstance(item, float) and not math.isfinite(item):- budget.non_finite_omitted += 1- _admit_delivered_unverified_list_value(- root, target, _delivered_unverified_omission("non-finite number"), budget- )- elif isinstance(item, dict | list) and id(item) in ancestors:- budget.cycle_omitted += 1- _admit_delivered_unverified_list_value(- root, target, _delivered_unverified_omission("cycle"), budget- )- elif isinstance(item, dict | list):- if depth >= _DELIVERED_UNVERIFIED_MAX_DEPTH:- budget.depth_omitted += 1- _admit_delivered_unverified_list_value(- root, target, _delivered_unverified_omission("depth budget"), budget- )- continue- child = {} if isinstance(item, dict) else []- if _admit_delivered_unverified_list_value(root, target, child, budget):- pending.append((item, child, ancestors | {id(item)}, depth + 1))- else:- budget.unsupported_omitted += 1- _admit_delivered_unverified_list_value(- root, target, _delivered_unverified_omission("unsupported value"), budget- )- if isinstance(root, dict):- root = {_DELIVERED_UNVERIFIED_METADATA_KEY: {"omitted": budget.omission_counts()}, **root}- return root---class _DeliveredUnverifiedSnapshot(dict[str, Any]):- """Sanitized, admission-bounded output shared by terminal prose and payload."""---def _delivered_unverified_observed_outputs(ctx: CopilotContext) -> _DeliveredUnverifiedSnapshot:- observed_outputs = ctx.delivered_unverified_observed_outputs- producer_halt = ctx.turn_halt- producer_admitted = (- producer_halt is not None- and producer_halt.kind == TurnHaltKind.DELIVERED_UNVERIFIED- and ctx.delivered_unverified_terminal is True- )- if not isinstance(observed_outputs, DeliveredUnverifiedPublicOutputs) and not producer_admitted:- return _DeliveredUnverifiedSnapshot()- if not observed_outputs:- return _DeliveredUnverifiedSnapshot()- sanitized = _sanitize_delivered_unverified_value(ctx, observed_outputs)- return _DeliveredUnverifiedSnapshot(sanitized) if isinstance(sanitized, dict) else _DeliveredUnverifiedSnapshot()def _terminal_summary_scalar(value: Any) -> str | None:@@ -2856,106 +2381,6 @@ def _terminal_summary_scalar(value: Any) -> str | None:return cleaned-def _terminal_output_root_path(key: str) -> str:- if key.strip() == key and key and not any(char in key for char in ('"', "[", "]", "\r", "\n", "\t")):- return key- return f"$[{json.dumps(key, ensure_ascii=False)}]"---def _terminal_output_child_path(path: str, segment: str | int) -> str:- if isinstance(segment, int):- return f"{path}[{segment}]"- return f"{path}[{json.dumps(segment, ensure_ascii=False)}]"---def _flatten_terminal_output(value: Any, path: str) -> list[tuple[int, str, str]]:- entries: list[tuple[int, str, str]] = []- pending: list[tuple[Any, str]] = [(value, path)]- while pending:- current, current_path = pending.pop()- if isinstance(current, dict):- if not current:- entries.append((3, current_path, "{}"))- continue- children = [- (item, _terminal_output_child_path(current_path, key))- for key, item in current.items()- if isinstance(key, str)- ]- pending.extend(reversed(children))- continue- if isinstance(current, list):- if not current:- entries.append((3, current_path, "[]"))- continue- pending.extend(- (item, _terminal_output_child_path(current_path, index))- for index, item in reversed(list(enumerate(current)))- )- continue- rendered: str | None- if isinstance(current, str) and not current.strip():- rendered = '""'- else:- rendered = "null" if current is None else _terminal_summary_scalar(current)- if rendered is None:- continue- if current is None or isinstance(current, bool | int | float):- rank = 0- elif isinstance(current, str) and len(" ".join(current.strip().split())) <= _VERIFIED_TERMINAL_VALUE_MAX_CHARS:- rank = 1- else:- rank = 2- entries.append((rank, current_path, rendered))- return entries---def _delivered_unverified_summary(observed_outputs: dict[str, Any]) -> str | None:- entries: list[tuple[int, str, str]] = []- for key, value in observed_outputs.items():- if key == _DELIVERED_UNVERIFIED_METADATA_KEY:- continue- entries.extend(_flatten_terminal_output(value, _terminal_output_root_path(key)))- entries.sort(key=lambda entry: (entry[0], entry[1]))- if not entries:- return None-- parts: list[str] = []- for index, (_, path, rendered) in enumerate(entries):- part = f"{path}: {rendered}"- candidate_parts = [*parts, part]- remaining = len(entries) - index - 1- suffix = f"; {remaining} more fields available in structured output" if remaining else ""- if len("; ".join(candidate_parts)) + len(suffix) <= _DELIVERED_UNVERIFIED_SUMMARY_MAX_CHARS:- parts.append(part)- else:- break-- omitted = len(entries) - len(parts)- if omitted:- suffix = f"{omitted} more field{'s' if omitted != 1 else ''} available in structured output"- return f"{'; '.join(parts)}; {suffix}" if parts else suffix- return "; ".join(parts)---def _delivered_unverified_reply(- ctx: CopilotContext,- observed_outputs: _DeliveredUnverifiedSnapshot,-) -> str | None:- if ctx.delivered_unverified_terminal is not True:- return None- summary = _delivered_unverified_summary(observed_outputs)- if summary:- return (- "I built and ran the workflow. The latest run returned "- f"{summary}. That value was not independently verified, so review the draft before using it."- )- return (- "I built and ran the workflow, and the latest run returned the requested output. "- "That output was not independently verified, so review the draft before using it."- )--def _verified_output_value(ctx: CopilotContext, output_key: str | None) -> Any:if not output_key:return None@@ -3042,12 +2467,6 @@ def _verified_workflow_success_reply(ctx: CopilotContext) -> str:return f"{_VERIFIED_WORKFLOW_SUCCESS_REPLY} Verified result: {summary}."-@dataclass(frozen=True)-class _TypedRunOutcomeReply:- user_response: str- demonstrated: bool--def _safe_run_outcome_display_reason(recorded: RecordedRunOutcome) -> str | None:reason = run_outcome_display_reason(recorded.display_reason)if reason is None or contains_internal_machinery_leak(reason):@@ -3059,42 +2478,6 @@ def _safe_run_outcome_display_reason(recorded: RecordedRunOutcome) -> str | Nonereturn reason.rstrip(".")-def _render_typed_run_outcome_reply(- ctx: CopilotContext,- *,- response_type: ResponseType,- has_verified_workflow: bool,- blocker_active: bool,-) -> _TypedRunOutcomeReply | None:- if blocker_active or response_type != "REPLY":- return None- recorded = ctx.last_run_outcome- if not isinstance(recorded, RecordedRunOutcome):- return None- if recorded.verdict == "demonstrated":- if not has_verified_workflow or not verified_goal_claim_authorized(ctx):- return None- return _TypedRunOutcomeReply(- user_response=f"{_verified_workflow_success_reply(ctx)} The latest run demonstrated the requested outcome.",- demonstrated=True,- )-- reason = _safe_run_outcome_display_reason(recorded)- if recorded.verdict == "not_demonstrated":- user_response = (- "I built and ran the workflow, but the latest run did not demonstrate the requested outcome. "- "Review the draft before using it."- )- else:- user_response = (- "I built and ran the workflow, but the latest run could not verify the requested outcome. "- "Review the draft before using it."- )- if reason is not None:- user_response = f"{user_response} Reason: {reason}."- return _TypedRunOutcomeReply(user_response=user_response, demonstrated=False)--# Pre-validated safe string the finalization shim falls back to when the# rendered blocker reply somehow trips OutputPolicy. Asserted clean at module# load time so a future OutputPolicy regression doesn't silently land here.@@ -3215,9 +2598,6 @@ def _finalize_result_with_blocker_override(return preservedrendered_reply, rendered_resp_type = _render_blocker_reply(local_signal, exit_site=exit_site)- rendered_reply = _with_scouted_spine_missing_steps(- ctx, rendered_reply, _get_scouted_spine_missing_steps_for_halt(ctx), held_signal_unrelated=True- )rendered_kind = (CopilotOutputKind.CLARIFICATION_REQUEST@@ -3232,8 +2612,6 @@ def _finalize_result_with_blocker_override(if preserved_workflow is not None:preserved_workflow_yaml = result.workflow_yaml or result.staged_workflow_yaml or ctx.staged_workflow_yamlpreserved_proposal = preserve_draft and preserved_workflow is not None- if preserved_proposal:- rendered_reply = _ensure_unvalidated_proposal_affordance(rendered_reply)rendered_verdict = evaluate_output_policy(request_policy=ctx.request_policy,response_type=rendered_resp_type,@@ -3562,19 +2940,6 @@ def _recorded_failure_reply(return f"I couldn't finish the Copilot turn: {reason}.{status_sentence}"-def _ensure_unvalidated_proposal_affordance(user_response: str) -> str:- lower = user_response.lower()- has_ui_affordance = bool(- _PROPOSAL_ACCEPT_UI_ACTION_RE.search(user_response) and _PROPOSAL_REJECT_UI_ACTION_RE.search(user_response)- )- has_unvalidated_disclosure = any(phrase in lower for phrase in UNVALIDATED_DISCLOSURE_PHRASES)- if has_ui_affordance and has_unvalidated_disclosure:- return user_response- if user_response.strip():- return f"{user_response}\n\n{_UNVALIDATED_PROPOSAL_AFFORDANCE}"- return _UNVALIDATED_PROPOSAL_AFFORDANCE--def _build_wip_exit_result(ctx: CopilotContext,global_llm_context: str | None,@@ -3584,22 +2949,14 @@ def _build_wip_exit_result(tested_reply: str,cancelled: bool = False,terminal_reason: str | None = None,- missing_spine_steps: str | None = None,) -> AgentResult:"""Selected non-success exits surface the most recent successfully parsed workflow."""- if missing_spine_steps is None:- missing_spine_steps = _get_scouted_spine_missing_steps_for_halt(ctx)internal_tool_instruction_failure = _recorded_failure_is_internal_tool_instruction(ctx)halted_mid_progress = _halted_mid_progress(ctx, internal_tool_instruction_failure)recorded_failure_reply = _recorded_failure_reply(ctx, cancelled=cancelled, internal_tool_instruction_failure=internal_tool_instruction_failure)effective_terminal = terminal_reason or ("cancel" if cancelled else None)- delivered_unverified_snapshot = (- _delivered_unverified_observed_outputs(ctx)- if ctx.delivered_unverified_terminal is True- else _DeliveredUnverifiedSnapshot()- )def _guard(text: str) -> tuple[str, TurnOutcome]:if contains_internal_machinery_leak(text):@@ -3608,7 +2965,6 @@ def _build_wip_exit_result(terminal_reason=effective_terminal,)text = _observed_facts_halt_reply(ctx)- text = _with_scouted_spine_missing_steps(ctx, text, missing_spine_steps, held_signal_unrelated=True)return apply_repeated_reply_guard(final_text=text,attempted_kind=ResponseKind.CLARIFY,@@ -3624,7 +2980,6 @@ def _build_wip_exit_result(ctx,_make_agent_result(ctx,- _delivered_unverified_snapshot=delivered_unverified_snapshot,user_response=final_text,updated_workflow=verified_workflow,global_llm_context=global_llm_context,@@ -3657,7 +3012,6 @@ def _build_wip_exit_result(and ctx.last_good_workflow_yamland ctx.last_workflow is not ctx.last_good_workflowand not ctx.last_test_suspicious_success- and ctx.delivered_unverified_terminal is not True):reply = _last_good_failure_reply(ctx, tested_reply) if recorded_failure_reply else tested_replyfinal_text, outcome = _guard(reply)@@ -3665,7 +3019,6 @@ def _build_wip_exit_result(ctx,_make_agent_result(ctx,- _delivered_unverified_snapshot=delivered_unverified_snapshot,user_response=final_text,updated_workflow=ctx.last_good_workflow,global_llm_context=global_llm_context,@@ -3698,14 +3051,8 @@ def _build_wip_exit_result():full_test_ok = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is Trueunvalidated = not full_test_ok- delivered_reply = _delivered_unverified_reply(ctx, delivered_unverified_snapshot)- if delivered_reply is not None:- reply = delivered_reply- unvalidated = True- elif unvalidated and recorded_failure_reply:+ if unvalidated and recorded_failure_reply:reply = recorded_failure_reply- if halted_mid_progress:- reply = _ensure_unvalidated_proposal_affordance(reply)else:reply = unvalidated_reply if unvalidated else tested_replyfinal_text, outcome = _guard(reply)@@ -3716,7 +3063,6 @@ def _build_wip_exit_result(ctx,_make_agent_result(ctx,- _delivered_unverified_snapshot=delivered_unverified_snapshot,user_response=final_text,updated_workflow=ctx.last_workflow,global_llm_context=global_llm_context,@@ -3741,17 +3087,13 @@ def _build_wip_exit_result(),exit_site="wip_last_workflow",)- fallback_reply = recorded_failure_reply or default_reply- if ctx.delivered_unverified_terminal is True:- fallback_reply = _delivered_unverified_reply(ctx, delivered_unverified_snapshot) or fallback_replyreturn _build_exit_result(ctx,- fallback_reply,+ recorded_failure_reply or default_reply,global_llm_context,cancelled=cancelled,terminal_reason=effective_terminal,- proposal_disposition="review_untested" if ctx.delivered_unverified_terminal is True else "auto_applicable",- _delivered_unverified_snapshot=delivered_unverified_snapshot,+ proposal_disposition="auto_applicable",)@@ -3939,13 +3281,12 @@ def _inline_replace_workflow_credential_verdict(steered_reason_codes=[reason.value for reason in steered_reasons],)if inline_rebind.residual_selectors and author_time_verdict.allowed:- # This path bypasses the update_workflow guardrail, and the output-policy scan does not catch a bare- # `page.fill(sel, ...)`, so an inline secret the rebind could not neutralize would slip through here.+ # Rebinding and persistence scrubbing own a residual raw fill, the same as on the+ # update_workflow path; refusing here would only refuse the user their own credential.LOG.info(- "copilot inline REPLACE_WORKFLOW credential residual raw fill fail-closed",+ "copilot inline REPLACE_WORKFLOW credential residual raw fill observed",residual_raw_credential_fill_selectors=list(inline_rebind.residual_selectors),)- author_time_verdict.add(OutputPolicyReason.RAW_SECRET_LEAK)if not author_time_verdict.allowed:_record_output_policy_guardrail_churn(ctx, "replace_workflow_inline", workflow_yaml, author_time_verdict)return workflow_yaml, raw_verdict, author_time_verdict@@ -4202,33 +3543,28 @@ async def _translate_to_agent_result(# ``blocker_active`` short-circuits the salvage/failure rewrites — the# renderer owns the final reply, so reshaping the agent's prose first# would be wasted work the finalization shim discards.- if not blocker_active:- if resp_type == "ASK_QUESTION":- user_response = _shape_ask_question_response(str(user_response), ctx)- elif not salvaged_reply:- user_response = _rewrite_failed_test_response(str(user_response), ctx)+ if not blocker_active and resp_type != "ASK_QUESTION" and not salvaged_reply:+ user_response = _rewrite_failed_test_response(+ str(user_response), ctx, agent_admits_incomplete=_is_explicit_false(action_data.get("goal_reached"))+ )verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)- verified_terminal_ready = (- verified_workflow is not None and verified_goal_claim_authorized(ctx) and not blocker_active- )- if verified_terminal_ready:- resp_type = "REPLY"- user_response = _verified_workflow_success_reply(ctx)- agent_admits_incomplete = False- else:- # Default-true preserves backwards-compat with stale prompts and missing fields.- agent_admits_incomplete = _is_explicit_false(- action_data.get("goal_reached")- ) and not verified_goal_claim_authorized(ctx)- typed_outcome_reply = _render_typed_run_outcome_reply(- ctx,- response_type=resp_type,- has_verified_workflow=verified_workflow is not None,- blocker_active=blocker_active,- )- if typed_outcome_reply is not None:- user_response = typed_outcome_reply.user_response- agent_admits_incomplete = not typed_outcome_reply.demonstrated+ # The model's reply is the claim; a template must not overwrite it, and the model's own+ # goal_reached admission is never vetoed by a promoted claim tier.+ agent_admits_incomplete = _is_explicit_false(action_data.get("goal_reached"))+ recorded = ctx.last_run_outcome+ if (+ resp_type == "REPLY"+ and not blocker_active+ and isinstance(recorded, RecordedRunOutcome)+ and recorded.verdict != "demonstrated"+ ):+ # The record renders beside the claim rather than replacing it: a run that did not+ # demonstrate the outcome caps what the reply may promote, exactly like a red test run+ # sitting under a green summary.+ agent_admits_incomplete = True+ reason = _safe_run_outcome_display_reason(recorded)+ if reason and reason.lower() not in str(user_response).lower():+ user_response = f"{str(user_response).rstrip()} Note: the latest run did not demonstrate the requested outcome — {reason}."last_workflow = Nonelast_workflow_yaml = None@@ -4257,14 +3593,6 @@ async def _translate_to_agent_result(_log_output_policy_parity(ctx, has_workflow_proposal=last_workflow is not None, workflow_attempted=workflow_attempted)- if _should_use_built_unverified_completed_reply(- ctx,- response_type=resp_type,- updated_workflow=last_workflow,- validated=not unvalidated,- blocker_active=blocker_active,- ):- user_response = _BUILT_UNVERIFIED_COMPLETED_REPLYoutput_kind = derive_output_kind(response_type=resp_type,request_policy=ctx.request_policy,@@ -4332,10 +3660,6 @@ async def _translate_to_agent_result(if OutputPolicyReason.MISSING_PROPOSAL_STATE in output_policy_verdict.reason_codes:soft_rewrite_reasons.append(OutputPolicyReason.MISSING_PROPOSAL_STATE)output_policy_verdict.remove(OutputPolicyReason.MISSING_PROPOSAL_STATE)- if OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE in output_policy_verdict.reason_codes:- user_response = _ensure_unvalidated_proposal_affordance(str(user_response))- soft_rewrite_reasons.append(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)- output_policy_verdict.remove(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)final_output_kind = (_blocked_final_output_kind(output_policy_verdict)if not output_policy_verdict.allowed@@ -4802,7 +4126,9 @@ def _evaluate_copilot_final_output_policy(surface_untested_draft = _should_surface_untested_draft_despite_question(ctx, response_type)policy_response_type = "REPLY" if surface_untested_draft else response_typeif surface_untested_draft:- policy_user_response = _rewrite_failed_test_response(policy_user_response, ctx)+ policy_user_response = _rewrite_failed_test_response(+ policy_user_response, ctx, agent_admits_incomplete=_is_explicit_false(action_data.get("goal_reached"))+ )updated_workflow_for_kind = (ctx.last_workflow if ctx.last_workflow is not None else WORKFLOW_PRESENT_SENTINEL if workflow_yaml else None)@@ -5142,9 +4468,6 @@ def _build_output_policy_blocked_result(composed_from_recorded_evidence = Trueif preserved_workflow is not None and add_saved_draft_copy:user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"- user_response = _with_scouted_spine_missing_steps(- ctx, user_response, _get_scouted_spine_missing_steps_for_halt(ctx), held_signal_unrelated=True- )has_non_actuation_hard_block = any(reasonnot in {@@ -5623,8 +4946,6 @@ async def _run_copilot_turn_impl()from skyvern.forge.sdk.copilot.enforcement import (- CopilotBuiltUnverified,- CopilotGoalSatisfied,CopilotNonRetriableNavError,CopilotTotalTimeoutError,CopilotUnrecoverableToolError,@@ -5819,20 +5140,6 @@ async def _run_copilot_turn_impl(prior_workflow_yaml=chat_request.workflow_yaml,output_policy_diagnostics=_output_policy_diagnostics_from_guardrail_exception(exc),)- except CopilotGoalSatisfied:- LOG.info(- "Copilot run stopped after verified goal satisfaction",- workflow_permanent_id=chat_request.workflow_permanent_id,- workflow_run_id=ctx.last_successful_run_blocks_workflow_run_id,- )- return await _build_goal_satisfied_exit_result(ctx, global_llm_context)- except CopilotBuiltUnverified:- LOG.info(- "Copilot run stopped after built-unverified repair-inert outcome",- workflow_permanent_id=chat_request.workflow_permanent_id,- workflow_run_id=ctx.last_successful_run_blocks_workflow_run_id,- )- return await _build_built_unverified_exit_result(ctx, global_llm_context)except CopilotTurnHalt as exc:LOG.info("Copilot run stopped after typed turn halt",diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex 44b80798f..27e838bb3 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -11,7 +11,6 @@ from typing import TYPE_CHECKING, Any, Literal, Protocolimport structlogfrom pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODEfrom skyvern.forge.sdk.copilot.output_contracts import (OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,@@ -78,7 +77,6 @@ _RUN_ID_LEAK_RE = re.compile(r"\b(?:wr|pbs)_[a-z0-9_]+", re.IGNORECASE)_INTERNAL_GUARD_TOKENS: tuple[str, ...] = ("per_tool_budget","per-tool-call budget",- "active_run_terminal_evidence","block-running tool","block running tool",)@@ -264,9 +262,6 @@ class _TerminalEvidenceResetCtx(Protocol):last_outcome_gate_reason: str | Nonelast_outcome_gate_workflow_run_id: str | Nonelast_test_anti_bot: str | None- delivered_unverified_terminal: bool- delivered_unverified_workflow_run_id: str | None- delivered_unverified_observed_outputs: dict[str, Any]completion_verification_result: Any | Noneoutcome_verification_trace_snapshot: dict[str, Any]@@ -301,9 +296,6 @@ def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) ->ctx.last_outcome_gate_reason = Nonectx.last_outcome_gate_workflow_run_id = Nonectx.last_test_anti_bot = None- ctx.delivered_unverified_terminal = False- ctx.delivered_unverified_workflow_run_id = None- ctx.delivered_unverified_observed_outputs = {}ctx.completion_verification_result = Nonectx.outcome_verification_trace_snapshot = {}@@ -328,7 +320,6 @@ def loop_blocker_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> LoopBlockerEvidence_LOOP_PROGRESS_TOOL_SUCCESS_REASON_CODES = frozenset({"loop_detected_credential_or_parameter_misconfig",- "loop_detected_repeated_failed_step","loop_detected_generic",})@@ -361,12 +352,10 @@ _OUTPUT_CONTRACT_TERMINAL_REASON_CODES = frozenset(# halt kind over a later non-terminal trip (e.g. the code-authoring churn backstop).GENUINELY_TERMINAL_BLOCKER_REASON_CODES: frozenset[str] = frozenset({- ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,"tool_error_run_output_terminal_blocker","tool_error_post_budget_challenge_blocker","tool_error_challenge_gated_submit_disabled",- "probable_site_block_stop",OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,"advisory_dispatch_stalled",@@ -390,11 +379,6 @@ def _should_stash_over_existing(and existing.blocker_kind == "loop_detected"):return True- if (- incoming.internal_reason_code == ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE- and existing.internal_reason_code in _ACTIVE_TERMINAL_REPLACEABLE_REASON_CODES- ):- return Trueif (incoming.internal_reason_code == "tool_error_post_budget_challenge_blocker"and existing.internal_reason_code in _TERMINAL_CHALLENGE_REPLACEABLE_REASON_CODES@@ -485,19 +469,7 @@ _LOOP_CREDENTIAL_TEMPLATE = (CREDENTIAL_SCOUT_VERIFY_REPLY = ("I need to verify the saved-credential login in the browser before I can save or run this code.")-# Mirrors OutputPolicyReason.RAW_SECRET_LEAK.value; kept local because output_policy imports this-# module, so importing back would be circular.-RAW_SECRET_LEAK_REASON_CODE = "raw_secret_leak"-RAW_SECRET_EMBED_REFUSAL_REPLY = (- "I couldn't save this because the login code kept embedding the credential's secret value directly "- "instead of referencing your saved credential. Your draft is preserved — tell me to reference the "- "saved credential and I'll try again."-)_LOOP_BRANCH_COPY: dict[str, tuple[str, str]] = {- "loop_detected_repeated_failed_step": (- "I retried without making progress.",- "Tell me what to change and I'll try a different approach.",- ),"loop_detected_consecutive_same_tool": ("I'm stuck retrying the same step.","Tell me what to change and I'll try a different approach.",@@ -644,9 +616,6 @@ def compose_loop_blocker_user_facing_reason(if internal_reason_code == "loop_detected_credential_or_parameter_misconfig":return _LOOP_CREDENTIAL_TEMPLATE, draft_tierif internal_reason_code == "credential_priority_authoring_churn":- reject_reason_codes = evidence.output_policy_reject_reason_codes if evidence is not None else None- if reject_reason_codes == frozenset({RAW_SECRET_LEAK_REASON_CODE}):- return RAW_SECRET_EMBED_REFUSAL_REPLY, draft_tierreturn CREDENTIAL_SCOUT_VERIFY_REPLY, draft_tierframing, ask = _LOOP_BRANCH_COPY.get(internal_reason_code or "", _LOOP_BRANCH_COPY["loop_detected_generic"])result_steer = evidence.latest_evaluate_result_composition_steer if evidence is not None else None@@ -676,9 +645,6 @@ def build_loop_blocker_signal(if "with CREDENTIAL_ERROR" in loop_message or "with PARAMETER_BINDING_ERROR" in loop_message:internal = "loop_detected_credential_or_parameter_misconfig"recovery_hint: RecoveryHint = "ask_user_clarifying"- elif "has already failed" in loop_message:- internal = "loop_detected_repeated_failed_step"- recovery_hint = "report_blocker_to_user"elif "has been called" in loop_message:internal = "loop_detected_consecutive_same_tool"recovery_hint = "report_blocker_to_user"diff --git a/skyvern/forge/sdk/copilot/code_block_preflight.py b/skyvern/forge/sdk/copilot/code_block_preflight.pyindex 3c386a8d5..9f23b7be5 100644--- a/skyvern/forge/sdk/copilot/code_block_preflight.py+++ b/skyvern/forge/sdk/copilot/code_block_preflight.py@@ -48,6 +48,7 @@ class CodeBlockSandboxNameDiagnostic:unresolved_names: tuple[str, ...]class_names: tuple[str, ...]parameter_keys: tuple[str, ...]+ block_bound_names: tuple[str, ...]allowed_global_names: tuple[str, ...]allowed_helper_surface: dict[str, tuple[str, ...]]@@ -461,6 +462,45 @@ def sandbox_unresolved_name_diagnostics(return [CodeBlockPreflightDiagnostic(code=repair_diagnostic.code, message=repair_diagnostic.message)]+def sandbox_absent_name_diagnostics(+ code: str,+ *,+ parameter_keys: Iterable[str] = (),+) -> list[CodeBlockPreflightDiagnostic]:+ """Find names the sandbox can never supply, ignoring definite-assignment gaps.++ A name the block binds on some path resolves whenever that path runs, so a run+ is the oracle for it; only a name bound nowhere can never resolve.+ """++ diagnostic = sandbox_unresolved_name_repair_diagnostic(code, parameter_keys=parameter_keys)+ if diagnostic is None:+ return []+ bound = set(diagnostic.block_bound_names)+ absent = [name for name in diagnostic.unresolved_names if name not in bound]+ if not absent and not diagnostic.class_names:+ return []++ detail_parts: list[str] = []+ if absent:+ detail_parts.append(f"unresolved names: {', '.join(f'`{name}`' for name in absent)}")+ if diagnostic.class_names:+ detail_parts.append(+ "class definitions unavailable in the code sandbox: "+ + ", ".join(f"`{name}`" for name in diagnostic.class_names)+ )+ return [+ CodeBlockPreflightDiagnostic(+ code=diagnostic.code,+ message=(+ f"Code block references names that are unavailable in the runtime code sandbox "+ f"({'; '.join(detail_parts)}). The sandbox provides `page`, declared code-block parameter "+ "keys, and its explicit safe helper namespace; `Exception` is the only available exception type."+ ),+ )+ ]++def sandbox_unresolved_name_repair_diagnostic(code: str,*,@@ -472,7 +512,8 @@ def sandbox_unresolved_name_repair_diagnostic(return Noneparameter_key_list = sorted(key for key in dict.fromkeys(parameter_keys) if _valid_python_identifier(key))- unresolved_names, class_names = _SandboxNameAnalyzer(parameter_keys=parameter_key_list).analyze(tree.body)+ analyzer = _SandboxNameAnalyzer(parameter_keys=parameter_key_list)+ unresolved_names, class_names = analyzer.analyze(tree.body)if not unresolved_names and not class_names:return None@@ -497,6 +538,7 @@ def sandbox_unresolved_name_repair_diagnostic(unresolved_names=tuple(names),class_names=tuple(rejected_classes),parameter_keys=tuple(parameter_key_list),+ block_bound_names=tuple(sorted(analyzer.block_bound_names)),allowed_global_names=tuple(sandbox_allowed_global_names()),allowed_helper_surface={helper: tuple(attributes) for helper, attributes in sandbox_allowed_helper_surface().items()@@ -692,10 +734,12 @@ class _SandboxNameAnalyzer:self.outer_names = set(outer_names)self.unresolved_names: set[str] = set()self.class_names: set[str] = set()+ self.block_bound_names: set[str] = set()def analyze(self, statements: list[ast.stmt]) -> tuple[set[str], set[str]]:local_names = self._local_names(statements)function_names = self._function_names(statements)+ self.block_bound_names = set(local_names) | set(function_names)self._statements(statements,set(self.parameter_names),diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.pyindex a0e522862..6eb82b8ad 100644--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py@@ -346,6 +346,11 @@ _STRUCTURAL_DISMISSAL_SELECTOR_PATTERN = re.compile(re.I,)+# Ceiling for a wait the scout proved must succeed (entry target, replayed read, extraction+# container). A wait returns the moment its condition holds, so a fast page pays nothing; only a+# genuinely absent state pays the full budget. Distinct from the deliberate 1s speculative probes.+_REQUIRED_STATE_TIMEOUT_MS = 120_000+_SYNTHESIZED_BLOCK_LABEL = "scout_synthesized_browser_steps"# Names the code-block executor reserves in its exec() namespace (block.py build_safe_vars@@ -2090,16 +2095,32 @@ def synthesize_code_block(reached_download_target: ReachedDownloadTarget | None = None,parameter_binding_snapshot: AuthoringParameterBindingSnapshot | None = None,file_match_transform: SameMonthFileMatchTransform | None = None,+ emit_read_return: bool = True,) -> SynthesizedCodeBlock | None:"""Deterministically synthesize a code block from a scout trajectory, or None if empty."""if not trajectory:return None+ # A named path keeps only its latest read: a re-read of the same requested value is a+ # refinement, including one that corrects a stale selector. The anonymous path is shared by every+ # read of an unnamed request, so there a different expression is an unrelated probe.+ latest_read_by_identity: dict[tuple[str, str], int] = {}+ for i, step in enumerate(trajectory):+ if str(step.get("tool_name") or "") == "read_value":+ path = str(step.get("read_output_path") or "")+ expression = str(step.get("read_expression") or "") if path == "output.scouted_read" else ""+ latest_read_by_identity[(path, expression)] = i+ if latest_read_by_identity:+ keep = set(latest_read_by_identity.values())+ trajectory = [+ step for i, step in enumerate(trajectory) if str(step.get("tool_name") or "") != "read_value" or i in keep+ ]lines: list[str] = []notes: list[str] = []parameters: list[dict[str, str]] = []diagnostics = SynthesisDiagnostics()steps: list[dict[str, Any]] = []+ read_bindings: list[tuple[str, str]] = []used_param_keys: set[str] = set()typed_param_keys: dict[tuple[str, str, str, str], str] = {}credential_param_keys: dict[str, str] = {}@@ -2433,9 +2454,13 @@ def synthesize_code_block(line_start=recovery_line_start,lane="entry_recovery",)- lines.append(f'{_INDENT * recovery_indent}await {_ENTRY_TARGET_VAR}.wait_for(state="visible")')+ lines.append(+ f'{_INDENT * recovery_indent}await {_ENTRY_TARGET_VAR}.wait_for(state="visible", timeout={_REQUIRED_STATE_TIMEOUT_MS})'+ )elif not login_only_presence_guard_active:- lines.append(f'{_INDENT * post_goto_indent}await {_ENTRY_TARGET_VAR}.wait_for(state="visible")')+ lines.append(+ f'{_INDENT * post_goto_indent}await {_ENTRY_TARGET_VAR}.wait_for(state="visible", timeout={_REQUIRED_STATE_TIMEOUT_MS})'+ )else:lines.append(f"{_INDENT}await page.goto("@@ -2559,6 +2584,28 @@ def synthesize_code_block(emitted += 1continue+ if tool_name == "read_value":+ expression = str(interaction.get("read_expression") or "").strip()+ output_path = str(interaction.get("read_output_path") or "").strip()+ if not expression or not output_path.startswith("output."):+ diagnostics.dropped_interactions.append(+ {"trajectory_index": trajectory_index, "tool_name": tool_name, "reason_code": "missing_read"}+ )+ continue+ line_start = len(lines) + 1+ variable = f"_read_value_{len(read_bindings)}"+ lines.append(f"{action_indent}{variable} = await page.evaluate({expression!r})")+ poll_rounds = _REQUIRED_STATE_TIMEOUT_MS // 1000+ lines.append(f"{action_indent}for _ in range({poll_rounds}):")+ lines.append(f'{action_indent}{_INDENT}if {variable} is not None and {variable} != "":')+ lines.append(f"{action_indent}{_INDENT * 2}break")+ lines.append(f"{action_indent}{_INDENT}await page.wait_for_timeout(1000)")+ lines.append(f"{action_indent}{_INDENT}{variable} = await page.evaluate({expression!r})")+ read_bindings.append((output_path, variable))+ append_step(f"Read {output_path.removeprefix('output.')}", "read", line_start)+ emitted += 1+ continue+if tool_name == "wait":try:duration_ms = int(interaction.get("duration_ms") or 0)@@ -2845,6 +2892,17 @@ def synthesize_code_block(lines.append(f"{_INDENT * 2}del {scout_var}")lines.append(f"{_INDENT}except Exception:")lines.append(f"{_INDENT * 2}pass")+ if read_bindings and emit_read_return:+ return_root = _ExtractionReturnNode()+ # Distinct reads that share a path (only the anonymous path can) each keep their value under+ # a suffixed key: choosing one would silently discard evidence.+ seen_paths: dict[str, int] = {}+ for output_path, variable in read_bindings:+ occurrence = seen_paths.get(output_path, 0)+ seen_paths[output_path] = occurrence + 1+ keyed_path = output_path if occurrence == 0 else f"{output_path}_{occurrence + 1}"+ _set_return_expression(return_root, output_path_segments(keyed_path.removeprefix("output.")), variable)+ lines.append(f"{_INDENT}return {_return_node_expression(return_root)}")if steps:steps[-1]["line_end"] = len(lines)@@ -3247,25 +3305,26 @@ def _key_value_scalar_read_statements(binding: LiveReadBinding, variable: str, *target = f"{container}.nth({binding.selector_index})"children = f'{target}.locator(":scope > *")'statements = [+ f'await {container}.first.wait_for(state="visible", timeout={_REQUIRED_STATE_TIMEOUT_MS})',f"if await {container}.count() != {binding.selector_count}:",- f'{_INDENT}raise ValueError("Observed scalar selector cardinality changed")',+ f'{_INDENT}raise Exception("Observed scalar selector cardinality changed")',f"if not await {target}.is_visible():",- f'{_INDENT}raise ValueError("Observed scalar relation is no longer visible")',+ f'{_INDENT}raise Exception("Observed scalar relation is no longer visible")',f"if await {children}.count() != {binding.child_count}:",- f'{_INDENT}raise ValueError("Observed scalar direct-child shape changed")',+ f'{_INDENT}raise Exception("Observed scalar direct-child shape changed")',f"if not await {children}.nth(0).is_visible():",- f'{_INDENT}raise ValueError("Observed scalar label is no longer visible")',+ f'{_INDENT}raise Exception("Observed scalar label is no longer visible")',f"if (await {children}.nth(0).inner_text()).strip() != {json.dumps(binding.relation_label)}:",- f'{_INDENT}raise ValueError("Observed scalar label changed")',+ f'{_INDENT}raise Exception("Observed scalar label changed")',f"if not await {children}.nth({binding.child_index}).is_visible():",- f'{_INDENT}raise ValueError("Observed scalar value is no longer visible")',+ f'{_INDENT}raise Exception("Observed scalar value is no longer visible")',f"{variable} = (await {children}.nth({binding.child_index}).inner_text()).strip()",]if guard_empty:statements.extend([f"if not {variable}:",- f'{_INDENT}raise ValueError("Observed scalar value is empty")',+ f'{_INDENT}raise Exception("Observed scalar value is empty")',])return statements@@ -3294,20 +3353,20 @@ def _table_group_read_lines(rows = f"page.locator({json.dumps(row_selector)})"headers = f'{table}.nth({exemplar.selector_index}).locator(":scope > thead > tr > th")'lines.append(f"if await {table}.count() != {exemplar.selector_count}:")- lines.append(f'{_INDENT}raise ValueError("Observed table identity changed")')+ lines.append(f'{_INDENT}raise Exception("Observed table identity changed")')lines.append(f"if not await {table}.nth({exemplar.selector_index}).is_visible():")- lines.append(f'{_INDENT}raise ValueError("Observed table is no longer visible")')+ lines.append(f'{_INDENT}raise Exception("Observed table is no longer visible")')lines.append(f'if await {selected_table}.locator(":scope table").count() != 0:')- lines.append(f'{_INDENT}raise ValueError("Observed table gained a nested table")')+ lines.append(f'{_INDENT}raise Exception("Observed table gained a nested table")')lines.append(f'if await {table}.nth({exemplar.selector_index}).locator("[colspan], [rowspan]").count() != 0:')- lines.append(f'{_INDENT}raise ValueError("Observed table gained spanning cells")')+ lines.append(f'{_INDENT}raise Exception("Observed table gained spanning cells")')lines.append(f"if await {headers}.count() != {len(exemplar.headers)}:")- lines.append(f'{_INDENT}raise ValueError("Observed table header cardinality changed")')+ lines.append(f'{_INDENT}raise Exception("Observed table header cardinality changed")')for header_index, header_text in enumerate(exemplar.headers):lines.append(f"if (await {headers}.nth({header_index}).inner_text()).strip() != {json.dumps(header_text)}:")- lines.append(f'{_INDENT}raise ValueError("Observed table header identity changed")')+ lines.append(f'{_INDENT}raise Exception("Observed table header identity changed")')lines.append(f"if await {rows}.count() != {row_count}:")- lines.append(f'{_INDENT}raise ValueError("Observed table row count changed")')+ lines.append(f'{_INDENT}raise Exception("Observed table row count changed")')row_expressions: list[str] = []if not assemble_as_literal:lines.append(f"{records_variable} = []")@@ -3315,23 +3374,23 @@ def _table_group_read_lines(row = f"{rows}.nth({row_index})"cells = f'{row}.locator(":scope > th, :scope > td")'lines.append(f"if not await {row}.is_visible():")- lines.append(f'{_INDENT}raise ValueError("Observed table row is no longer visible")')+ lines.append(f'{_INDENT}raise Exception("Observed table row is no longer visible")')lines.append(f"if await {cells}.count() != {exemplar.row_cell_counts[row_index]}:")- lines.append(f'{_INDENT}raise ValueError("Observed table direct-cell cardinality changed")')+ lines.append(f'{_INDENT}raise Exception("Observed table direct-cell cardinality changed")')lines.append(f'if await {row}.locator(":scope > th").count() != 0:')- lines.append(f'{_INDENT}raise ValueError("Observed table row gained a row header")')+ lines.append(f'{_INDENT}raise Exception("Observed table row gained a row header")')lines.append(f'if " ".join((await {row}.inner_text()).split()) != {json.dumps(exemplar.row_identities[row_index])}:')- lines.append(f'{_INDENT}raise ValueError("Observed table row identity changed")')+ lines.append(f'{_INDENT}raise Exception("Observed table row identity changed")')for binding_index, binding in enumerate(sorted(bindings, key=lambda item: item.output_path)):value_variable = f"{cell_variable_base}_{group_index}_{row_index}_{binding_index}"lines.append(f"if not await {cells}.nth({binding.column_index}).is_visible():")- lines.append(f'{_INDENT}raise ValueError("Observed table cell is no longer visible")')+ lines.append(f'{_INDENT}raise Exception("Observed table cell is no longer visible")')lines.append(f"{value_variable} = (await {cells}.nth({binding.column_index}).inner_text()).strip()")if guard_empty:lines.append(f"if not {value_variable}:")- lines.append(f'{_INDENT}raise ValueError("Observed table cell value is empty")')+ lines.append(f'{_INDENT}raise Exception("Observed table cell value is empty")')_set_return_expression(record_root, output_path_segments(binding.output_path)[len(prefix) :], value_variable)@@ -3629,12 +3688,15 @@ def synthesize_code_block_with_extraction() -> SynthesizedCodeBlock | None:if not _trajectory_contains_reveal(trajectory, extraction_plan):return None+ # The suffix carries the terminal return; a base read-return here would sit above the appended+ # suffix and make it unreachable.interaction = synthesize_code_block(trajectory,strict_selectors=strict_selectors,reached_download_target=reached_download_target,parameter_binding_snapshot=parameter_binding_snapshot,file_match_transform=file_match_transform,+ emit_read_return=False,)suffix = synthesize_extraction_suffix(extraction_plan)if interaction is None or suffix is None:diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 8f77a8801..6715e8e32 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -375,11 +375,6 @@ class CompletionVerificationResult:}-@dataclass(frozen=True)-class DeliveredUnverifiedTerminalState:- observed_verdicts: tuple[CriterionVerdict, ...]--@dataclass(frozen=True)class RunEvidenceSnapshot:workflow_run_id: str | None = None@@ -547,7 +542,6 @@ _MISSING_VERDICT_EVIDENCE = "judge did not return a verdict for this criterion"_INCOMPLETE_VALIDATION_CLASSIFICATION_CONTRACT = "incomplete typed classification contract"_INCOMPLETE_VALIDATION_CLASSIFICATION_ABSTENTION_REASON = "validation_classification_incomplete_contract"_MISSING_REGISTERED_DOWNLOAD_EVIDENCE = "run output did not include a non-empty registered browser download"-_DELIVERED_UNVERIFIED_OUTPUT_SOURCES = frozenset({"runtime_output", "registered_output_parameter"})def registered_download_completion_criterion() -> CompletionCriterion:@@ -2519,20 +2513,6 @@ def _has_independent_satisfied_requested_output_corroborator()-def _judgment_tier_satisfied_corroborator(- verdict_by_id: dict[str, CriterionVerdict],- criterion_id: str,-) -> CriterionVerdict | None:- for verdict in verdict_by_id.values():- if (- verdict.satisfied- and _is_requested_output_corroborator_id(verdict.criterion_id, criterion_id)- and verdict.evidence_source in _FLOOR_REKEYED_DELIVERABLE_CREDIT_SOURCES- ):- return verdict- return None--def _is_requested_output_corroborator_id(candidate_id: str, criterion_id: str) -> bool:prefix = f"{criterion_id}__requested_output_corroborator"if candidate_id == prefix:@@ -2797,134 +2777,6 @@ def only_degraded_blocking(result: CompletionVerificationResult) -> bool:return blocking <= degraded-def _delivered_unverified_observed_verdict(verdict: CriterionVerdict) -> bool:- if not verdict.evidence_ref or not verdict.output_path:- return False- normalized_path = verdict.output_path.removeprefix("output.").strip()- if normalized_path.split(".")[-1] == "evidence_text":- return False- if verdict.evidence_source not in _DELIVERED_UNVERIFIED_OUTPUT_SOURCES:- return False- if verdict.self_emitted_judgment_not_independent:- return False- if verdict.requested_output_evidence_source == "independent_run_evidence":- return False- if verdict.grounding_mode in {"shape", "judgment_boolean"}:- return False- if verdict.satisfied:- return (- verdict.reason_code == "evidence_confirms"- and verdict.grounding_mode == "exact_value"- and verdict.has_exact_value- )- return _is_structural_requested_output_abstention(verdict) and not verdict.has_exact_value---def degraded_contract_delivered_unverified_terminal_state(- result: CompletionVerificationResult | None,- *,- run_ok: bool,- workflow_run_id: str | None,- latest_workflow_run_id: str | None,- artifact_health_blocked: bool,- terminal_blocked: bool,-) -> DeliveredUnverifiedTerminalState | None:- if (- result is None- or result.status != "evaluated"- or result.is_fully_satisfied()- or not run_ok- or not workflow_run_id- or workflow_run_id != latest_workflow_run_id- or artifact_health_blocked- or terminal_blocked- ):- return None- degraded = degraded_lane_criterion_ids(result)- if not degraded:- return None- verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}- decision = plain_outcome_no_evidence_abstention_decision(result, verdict_by_id)- plain_outcome_abstained_ids = set(decision.criterion_ids) if decision else set()- observed: list[CriterionVerdict] = []- blocking: set[str] = set()- for criterion_id in result.criterion_ids:- verdict = verdict_by_id.get(criterion_id)- if verdict is not None and _delivered_unverified_observed_verdict(verdict):- observed.append(verdict)- continue- if verdict is not None and verdict.satisfied:- continue- if verdict is not None and result.is_structural_contingent_abstention(verdict):- continue- if criterion_id in plain_outcome_abstained_ids:- continue- blocking.add(criterion_id)- if not observed or not blocking or not blocking <= degraded:- return None- return DeliveredUnverifiedTerminalState(observed_verdicts=tuple(observed))---def zero_requested_output_criteria_credit(- result: CompletionVerificationResult | None,- *,- has_meaningful_registered_output: bool,-) -> bool:- """A satisfied run-plane verdict crediting completion when no requested-output- criterion formed to grade the delivered payload is an unverified deliverable,- not a verified success. Keying on meaningful produced content (not mere payload- presence) lets a reach-state goal whose only registered output is an empty- task-output envelope (e.g. login-only) keep crediting normally."""- return (- result is not None- and result.status == "evaluated"- and result.requested_output_criteria_count == 0- and has_meaningful_registered_output- and result.is_fully_satisfied()- )---@dataclass(frozen=True)-class FloorRekeyedDeliverableCredit:- criterion_ids: tuple[str, ...]- evidence_sources: tuple[str, ...]- evidence_refs: tuple[str, ...]- output_paths: tuple[str, ...]---def floor_rekeyed_deliverable_credit(- result: CompletionVerificationResult | None,-) -> FloorRekeyedDeliverableCredit | None:- """Credit a floor-rekeyed presence-only deliverable on a fully satisfied run when every marked- criterion carries its own satisfied requested-output corroborator whose derived evidence_source is- judgment-tier. Registered-output and self-emitted corroboration earn no credit and keep hedging."""- if result is None or result.status != "evaluated" or not result.is_fully_satisfied():- return None- marked_ids = list(dict.fromkeys(result.floor_rekeyed_criterion_ids))- if not marked_ids:- return None- verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}- evidence_sources: list[str] = []- evidence_refs: list[str] = []- output_paths: list[str] = []- for criterion_id in marked_ids:- marked = verdict_by_id.get(criterion_id)- if marked is None:- return None- corroborator = _judgment_tier_satisfied_corroborator(verdict_by_id, criterion_id)- if corroborator is None or corroborator.evidence_source is None:- return None- evidence_sources.append(corroborator.evidence_source)- evidence_refs.append(corroborator.evidence_ref or "")- output_paths.append(result.floor_rekeyed_output_path_by_criterion_id.get(criterion_id, ""))- return FloorRekeyedDeliverableCredit(- criterion_ids=tuple(marked_ids),- evidence_sources=tuple(evidence_sources),- evidence_refs=tuple(evidence_refs),- output_paths=tuple(output_paths),- )--def gradeable_completion_criteria(criteria: Iterable[CompletionCriterion]) -> list[CompletionCriterion]:return [criterion@@ -3061,75 +2913,6 @@ def carry_floor_rekeyed_path_backing(return replace(result, floor_rekeyed_backed_by_criterion_id=merged)-@dataclass(frozen=True)-class FloorRekeyedEmissionWithhold:- criterion_ids: tuple[str, ...]- unbacked_output_paths: tuple[str, ...]- backed_output_paths: tuple[str, ...]---def floor_rekeyed_effective_marked_ids(result: CompletionVerificationResult) -> list[str]:- """Floor-rekeyed markers that required an emission this run: the marked set minus the- structurally-unfired contingent (abstained) markers, which needed no emission to fire."""- abstained = result.abstained_criterion_ids()- return [criterion_id for criterion_id in result.floor_rekeyed_criterion_ids if criterion_id not in abstained]---def floor_rekeyed_emission_withhold(- result: CompletionVerificationResult | None,-) -> FloorRekeyedEmissionWithhold | None:- """A fully satisfied run whose floor-rekeyed emission markers lack a meaningful runtime value at- their original path delivered no emission that corroboration can substitute for. Keyed on the typed- marker id-set (never payload emptiness) so never-minted reach-state and structurally-unfired- contingent markers are untouched, and a missing backing entry reads as unbacked to fail closed."""- if result is None or result.status != "evaluated" or not result.is_fully_satisfied():- return None- effective_ids = floor_rekeyed_effective_marked_ids(result)- if not effective_ids:- return None- unbacked_ids: list[str] = []- unbacked_paths: list[str] = []- backed_paths: list[str] = []- for criterion_id in effective_ids:- path = result.floor_rekeyed_output_path_by_criterion_id.get(criterion_id, "")- if result.floor_rekeyed_backed_by_criterion_id.get(criterion_id, False):- backed_paths.append(path)- else:- unbacked_ids.append(criterion_id)- unbacked_paths.append(path)- if not unbacked_ids:- return None- return FloorRekeyedEmissionWithhold(- criterion_ids=tuple(unbacked_ids),- unbacked_output_paths=tuple(unbacked_paths),- backed_output_paths=tuple(backed_paths),- )---def floor_rekeyed_emission_lane_fields(result: CompletionVerificationResult | None) -> dict[str, Any] | None:- """Structured fingerprint fields for the floor-rekeyed emission lane, emitted whenever an- evaluated result carries markers — on satisfied and unsatisfied results alike, independent of- outcome."""- if result is None or result.status != "evaluated" or not result.floor_rekeyed_criterion_ids:- return None- effective_ids = floor_rekeyed_effective_marked_ids(result)- backed_ids = [cid for cid in effective_ids if result.floor_rekeyed_backed_by_criterion_id.get(cid, False)]- unbacked_ids = [cid for cid in effective_ids if cid not in backed_ids]- withhold = floor_rekeyed_emission_withhold(result)- return {- "criterion_ids": list(result.floor_rekeyed_criterion_ids),- "effective_criterion_ids": list(effective_ids),- "abstained_excluded_criterion_ids": sorted(set(result.floor_rekeyed_criterion_ids) - set(effective_ids)),- "backed_criterion_ids": backed_ids,- "unbacked_criterion_ids": unbacked_ids,- "backed_output_paths": [result.floor_rekeyed_output_path_by_criterion_id.get(cid, "") for cid in backed_ids],- "unbacked_output_paths": [- result.floor_rekeyed_output_path_by_criterion_id.get(cid, "") for cid in unbacked_ids- ],- "engaged": withhold is not None,- }--async def evaluate_completion_criteria(criteria: list[CompletionCriterion],snapshot: RunEvidenceSnapshot,diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.pyindex 1bccf3f2d..9184c3ae1 100644--- a/skyvern/forge/sdk/copilot/config.py+++ b/skyvern/forge/sdk/copilot/config.py@@ -267,25 +267,6 @@ PRE_DISCOVERY_URL_QUESTION_NUDGE = ("runs and cannot resolve a site.")-PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX = (- "STOP — the target site has failed to scrape on every attempt across "- "multiple workflow shapes. Every run navigated successfully but the "- 'scraper could not read the page ("failed to load the website" / '- '"page may have navigated unexpectedly"). This pattern indicates the '- "site is either blocking automated access, genuinely unresponsive in "- "this environment, or rendering content Skyvern cannot read reliably.\n"- "Do NOT retry with another workflow variation. Do NOT call "- "update_and_run_blocks or run_blocks_and_collect_debug again.\n"- "Reply to the user now: state that the site could not be loaded after "- "multiple attempts, quote the last failure_reason verbatim, keep the "- "message concise, and ask "-)--POST_PROBABLE_SITE_BLOCK_STOP_NUDGE = (- PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX- + "whether to try a different URL, configure a proxy, or provide an alternate entry point."-)-POST_ANTI_BOT_FAILED_TEST_NUDGE = ("STOP — your last test run failed due to an anti-bot/WAF block ""(Access Denied, CAPTCHA, human-verification, or similar challenge evidence).\n"@@ -342,8 +323,6 @@ DEFAULT_ENFORCEMENT_NUDGES: dict[str, str] = {"post_no_workflow_delivery": POST_NO_WORKFLOW_DELIVERY_NUDGE,"post_discovery_entrypoint_url_question": POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE,"pre_discovery_url_question": PRE_DISCOVERY_URL_QUESTION_NUDGE,- "post_probable_site_block_stop_prefix": PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX,- "post_probable_site_block_stop": POST_PROBABLE_SITE_BLOCK_STOP_NUDGE,"post_anti_bot_failed_test": POST_ANTI_BOT_FAILED_TEST_NUDGE,"post_format": POST_FORMAT_NUDGE,}diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex c8c7e395c..328000ebe 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -47,15 +47,6 @@ def parsed_ask_refs(value: object) -> list[str]:return [ref for ref in value if isinstance(ref, str) and ref]-class DeliveredUnverifiedPublicOutputs(dict[str, Any]):- """Run-output values explicitly selected for terminal presentation.-- The values remain dynamically shaped JSON until the presentation sanitizer- validates them. The concrete marker prevents arbitrary result-factory- callers from minting the public structured-output surface.- """--class NarrativeDraft(TypedDict):blockCount: intblockLabels: list[str]@@ -112,8 +103,6 @@ class TurnNarrativePayload(TypedDict):verifiedSuccess: NotRequired[bool]# Verdict-state summary from the turn's latest evaluated adjudication.outcomeAdjudication: NotRequired[NarrativeOutcomeAdjudication]- # Sanitized JSON boundary for reviewing outputs that were delivered but not independently verified.- deliveredUnverifiedObservedOutputs: NotRequired[dict[str, Any]]# {"reason": <credential_prompt_reason() token>}, set when this turn surfaces a credential need.credentialPrompt: NotRequired[dict[str, str]]# {"outcome": "connected"|"skipped"|"timeout", "credentialId": ...}, set when a mid-build@@ -857,17 +846,9 @@ class CopilotContext(AgentContext):# per-run pointer resets (``last_run_outcome = None``); cleared only by the# workflow-edit evidence reset, which invalidates pre-edit run evidence.terminal_envelope_run_outcomes: list[RecordedRunOutcome] = field(default_factory=list)- delivered_unverified_terminal: bool = False- delivered_unverified_workflow_run_id: str | None = None- delivered_unverified_observed_outputs: dict[str, Any] = field(default_factory=DeliveredUnverifiedPublicOutputs)# Consecutive failed runs where navigation completed but the scraper# could not read the page (generic "failed to load the website" template).# Resets on any non-matching run outcome. Streak crosses workflow-shape- # changes deliberately — the frontier fingerprint resets each time the- # copilot rewrites the workflow, but the underlying site-block pattern is- # shape-independent.- probable_site_block_streak_count: int = 0- probable_site_block_stop_nudge_count: int = 0per_tool_budget_nudge_count: int = 0effective_workflow_proxy_location: Any | None = None# Labels of navigation blocks that were canceled/failed inside adiff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex c36c581f4..0efa21013 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -16,8 +16,6 @@ from skyvern.forge.sdk.copilot.completion_verification import (from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controlsfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.failure_tracking import (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,- ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,RepairRootCauseIdentity,compute_repair_root_cause_signature,)@@ -53,9 +51,7 @@ class DiagnosisFailureType(StrEnum):TERMINAL_CHALLENGE_BLOCKER = "terminal_challenge_blocker"MISSING_CREDENTIAL_OR_INIT = "missing_credential_or_init"REPAIRABLE_BLOCK_FAILURE = "repairable_block_failure"- ACTIVE_RUN_TERMINAL_EVIDENCE = "active_run_terminal_evidence"UNRECOVERABLE_TOOL_ERROR = "unrecoverable_tool_error"- DELIVERED_UNVERIFIED = "delivered_unverified"UNKNOWN = "unknown"@@ -176,18 +172,11 @@ def build_diagnosis_repair_contract(outcome_verified = outcome_fully_verified(ctx)completion_verification = getattr(ctx, "completion_verification_result", None)completion_verification_failed = _completion_verification_failed(completion_verification)- delivered_unverified = bool(- run_ok- and getattr(ctx, "delivered_unverified_terminal", False) is True- and workflow_run_id- and workflow_run_id == getattr(ctx, "delivered_unverified_workflow_run_id", None)- )failure_type = _failure_type(run_ok,suspicious,outcome_verified,completion_verification_failed,- delivered_unverified,failed_blocks,categories,terminal_challenge_categories,@@ -243,13 +232,6 @@ def build_diagnosis_repair_contract(decision_summary = ("Stop retrying because structured evidence shows the workflow path is blocked by a site challenge.")- elif failure_type == DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCE:- decision_summary = (- "Stop the current retry loop: the active run reached the requested browser state, "- "but the reusable workflow is not verified end-to-end."- )- elif failure_type == DiagnosisFailureType.DELIVERED_UNVERIFIED:- decision_summary = "No repair selected; the latest run returned requested output but it was not verified."if (next_action == RepairNextAction.NO_CHANGEand user_goal_satisfied is True@@ -258,9 +240,7 @@ def build_diagnosis_repair_contract(completion_check = "Current run already satisfies the goal."else:completion_check = {- RepairNextAction.NO_CHANGE: "No repair selected; completion is delivered but not independently verified."- if failure_type == DiagnosisFailureType.DELIVERED_UNVERIFIED- else "No repair selected; completion remains unverified.",+ RepairNextAction.NO_CHANGE: "No repair selected; completion remains unverified.",RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",RepairNextAction.STOP: "Do not rerun unchanged; user-visible blocker must be resolved first.",}.get(@@ -479,7 +459,6 @@ def _failure_type(suspicious: bool,outcome_verified: bool,completion_verification_failed: bool,- delivered_unverified: bool,failed_blocks: list[str],categories: list[str],terminal_challenge_categories: list[str],@@ -508,13 +487,6 @@ def _failure_type(or ("404" in error_text and "browser session" in error_text)):return DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR- if (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY in categories- or data.get("active_run_terminal_evidence_detected") is True- ):- if outcome_verified and _active_run_terminal_evidence_reason_code(data):- return DiagnosisFailureType.NO_FAILURE- return DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCEif (category_set & _PRE_RUN_CREDENTIAL_FAILURE_CATEGORIESor "organization not found" in error_text@@ -530,8 +502,6 @@ def _failure_type(return DiagnosisFailureType.NO_FAILUREif failed_blocks:return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE- if delivered_unverified:- return DiagnosisFailureType.DELIVERED_UNVERIFIEDif category_set & _REPAIRABLE_RUNTIME_CATEGORIES:return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILUREif completion_verification_failed:@@ -543,22 +513,16 @@ def _failure_type(return DiagnosisFailureType.FAILED_RUN if result.get("ok") is False else DiagnosisFailureType.UNKNOWN-def _active_run_terminal_evidence_reason_code(data: dict[str, Any]) -> bool:- return _safe_str(data.get("active_run_terminal_evidence_reason_code")) == ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE--def _next_action(failure_type: DiagnosisFailureType,ctx: CopilotContext,data: dict[str, Any],repair_context: CodeAuthoringRepairContext | None,) -> RepairNextAction:- if failure_type in {DiagnosisFailureType.NO_FAILURE, DiagnosisFailureType.DELIVERED_UNVERIFIED}:+ if failure_type == DiagnosisFailureType.NO_FAILURE:return RepairNextAction.NO_CHANGEif failure_type == DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR:return RepairNextAction.STOP- if failure_type == DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCE:- return RepairNextAction.STOPif failure_type == DiagnosisFailureType.TERMINAL_CHALLENGE_BLOCKER:return RepairNextAction.STOPif ctx.last_test_non_retriable_nav_error:@@ -645,12 +609,6 @@ def _verification_satisfaction() -> tuple[bool | None, bool | None]:if failure_type == DiagnosisFailureType.TERMINAL_CHALLENGE_BLOCKER:return False, False- if failure_type == DiagnosisFailureType.DELIVERED_UNVERIFIED:- return True, False- if isinstance(data, dict) and data.get("active_run_terminal_evidence_detected") is True:- trace = data.get("active_run_terminal_completion_verification")- fully_satisfied = isinstance(trace, dict) and trace.get("fully_satisfied") is True- return fully_satisfied, fully_satisfiedif failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT:return False, Falseif outcome_fully_verified(ctx):@@ -709,7 +667,6 @@ def _prior_repair_attempts(ctx: Any) -> dict[str, int]:keys = ("repeated_failure_streak_count","failed_test_nudge_count",- "probable_site_block_streak_count","per_tool_budget_nudge_count","repeated_action_fingerprint_streak_count",)diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 6e9c29924..60b163a90 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -19,21 +19,14 @@ from agents.run import Runnerfrom skyvern.forge.sdk.copilot import config as copilot_config_defaultsfrom skyvern.forge.sdk.copilot import streaming_adapterfrom skyvern.forge.sdk.copilot.blocker_signal import (- RAW_SECRET_LEAK_REASON_CODE,SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,CopilotToolBlockerSignal,clear_tool_blocker_signals_for_reason_codes,- compose_loop_blocker_user_facing_reason,- loop_blocker_evidence_from_ctx,stash_blocker_signal,)from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_FAILURE_STREAK_ESCAPE_THRESHOLD, DISCOVERY_PERMITTED_PHASESfrom skyvern.forge.sdk.copilot.build_test_outcome import (- PostRunPagePathFailure,- RecordedBuildTestOutcome,latest_recorded_build_test_outcome_repeated,- record_build_test_outcome,- recorded_outcome_from_authoring_repair_context,)from skyvern.forge.sdk.copilot.challenge_evidence import composition_challenge_carrierfrom skyvern.forge.sdk.copilot.code_block_synthesis import (@@ -47,9 +40,6 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (is_generic_entry_opener_click,is_optional_dismissal_only_trajectory,missing_rung_text,- obligation_finding_reason_code,- obligation_finding_selector,- render_missing_rung_call_sources,render_obligation_findings,render_synthesized_offer_text,spine_partition_findings,@@ -76,7 +66,6 @@ from skyvern.forge.sdk.copilot.config import (POST_PARAMETER_BINDING_WARN_NUDGE,POST_PER_TOOL_BUDGET_NUDGE,POST_PER_TOOL_BUDGET_STOP_NUDGE,- POST_PROBABLE_SITE_BLOCK_STOP_NUDGE,POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE,POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE,POST_SUSPICIOUS_SUCCESS_NUDGE,@@ -90,7 +79,6 @@ from skyvern.forge.sdk.copilot.config import ()from skyvern.forge.sdk.copilot.context import (AskSubject,- CodeAuthoringRepairContext,coerce_ask_subject,parsed_ask_refs,)@@ -125,6 +113,7 @@ from skyvern.forge.sdk.copilot.request_policy import (requested_output_path_for_field,schema_output_path_aliases_from_criteria,)+from skyvern.forge.sdk.copilot.request_slots import is_canonical_request_slot_pathfrom skyvern.forge.sdk.copilot.result_evidence import (COVERAGE_TOKEN_RE,ScoutObservationContract,@@ -141,7 +130,6 @@ from skyvern.forge.sdk.copilot.run_outcome import ()from skyvern.forge.sdk.copilot.runtime import (AgentContext,- PostRunPagePathInteractionWindow,)from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntryfrom skyvern.forge.sdk.copilot.terminal_predicates import (@@ -157,14 +145,8 @@ from skyvern.forge.sdk.copilot.turn_halt import ()from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentModefrom skyvern.forge.sdk.copilot.turn_ownership import (- ClaimOutcome,TurnClaimant,- claim_and_stash_blocker_signal,claim_turn,- claim_would_succeed,- claimant_outranks,- current_turn_owner,- emit_blocker_signal_payload,)from skyvern.forge.sdk.copilot.unrecoverable_tool_error import (CopilotUnrecoverableToolError as CopilotUnrecoverableToolError,@@ -205,28 +187,10 @@ REPEATED_FRONTIER_STREAK_STOP_AT = 3# Stop after this many consecutive runs where navigation succeeded but the# scraper could not read the page. Aligned with MAX_FAILED_TEST_NUDGES so the# copilot gets one generic retry nudge, then stops on the second occurrence.-PROBABLE_SITE_BLOCK_STREAK_STOP_AT = 2-# Caps how many times the stop nudge can re-fire — without this, the streak-# stays latched while no new test runs reset it and every subsequent turn-# re-injects the same nudge until MAX_ITERATIONS. Independent of-# PROBABLE_SITE_BLOCK_STREAK_STOP_AT (both default to 2 but tune different-# axes: streak depth vs nudge count).-MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES = 2# Caps how many times the per-tool-budget split nudge can fire. After two# trips the agent should already be at single-block granularity; further# trips fall through to the repeated-frontier escalation path.MAX_PER_TOOL_BUDGET_NUDGES = 2-# Code-authoring guardrail churn: distinct-reject convergence backstop. An-# accepted persist resets the counter, so this many unaccepted rejections is-# pathological and leaves three free repair attempts before the halt fires —-# far inside both the 900s budget and the SDK max-turns cap.-MAX_CODE_AUTHORING_GUARDRAIL_REJECTS = 4-# Credential-priority rejects defer to the credential-scout message until this-# higher bound, which must stay above MAX_CODE_AUTHORING_GUARDRAIL_REJECTS so the-# non-credential backstop and the low-count credential deferral are untouched.-MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS = 8-# Far under the SDK max-turns cap so the halt arrives first.-MAX_NO_PROGRESS_INTERACTION_ATTEMPTS = 4_NO_PROGRESS_INTERACTION_REASON_CODES = frozenset({"loop_detected_no_forward_progress_interaction"})MIN_BLOCKS_FOR_AUTO_COMPLETE = 10TOTAL_TIMEOUT_SECONDS = 900@@ -234,10 +198,6 @@ TOTAL_TIMEOUT_SECONDS = 900# never yields ``wait_for(timeout=0)`` (which raises immediately). Kept as a# constant so tests can shrink it instead of paying a full second per deadline.MIN_DEADLINE_REMAINING_SECONDS = 1.0-# Belt-and-braces cap alongside the elapsed-time budget. Per-nudge caps-# already prevent individual branches from looping; this stops a brand-new-# enforcement rule that forgets its own counter from spinning within 900s.-MAX_ITERATIONS = 50SCREENSHOT_SENTINEL = "[copilot:screenshot] "NUDGE_SENTINEL = "[copilot:nudge] "SCREENSHOT_PLACEHOLDER = SCREENSHOT_SENTINEL + "[prior screenshot removed to save context]"@@ -335,139 +295,13 @@ def _effective_proxy_label(ctx: Any) -> str | None:return _normalized_proxy_label(getattr(workflow, "proxy_location", None))-def _probable_site_block_proxy_options(ctx: Any, *, include_whether: bool = True) -> str:- proxy_label = _effective_proxy_label(ctx)- if proxy_label is None:- options = "try a different URL, configure a proxy, or provide an alternate entry point."- return f"whether to {options}" if include_whether else options- if proxy_label == "RESIDENTIAL":- options = (- "try a different proxy location (for example US-CA or US-NY), use a different "- "residential/ISP option if supported, or provide an alternate entry point."- )- return f"whether to {options}" if include_whether else options- options = (- f"try a different proxy/location than {proxy_label}, use a different residential/ISP option if supported, "- "or provide an alternate entry point."- )- return f"whether to {options}" if include_whether else options---def _probable_site_block_stop_nudge(ctx: Any, config: CopilotConfig | None = None) -> str:- return _nudge(config, "post_probable_site_block_stop_prefix") + _probable_site_block_proxy_options(ctx)---def _probable_site_block_stop_agent_text(ctx: Any, config: CopilotConfig | None = None) -> str:- return (- f"{_probable_site_block_stop_nudge(ctx, config)}\n"- f"Latest internal failure reason: {_single_line_failure_reason(ctx)}"- )---def _single_line_failure_reason(ctx: Any) -> str:- reason = getattr(ctx, "last_test_failure_reason", None)- if not isinstance(reason, str) or not reason.strip():- return "Skyvern failed to load the website."- return " ".join(reason.split())---def build_probable_site_block_user_question(ctx: Any) -> str | None:- """Return a concise user-facing blocker question after the site-block stop nudge."""- if _get_int(ctx, "probable_site_block_stop_nudge_count") <= 0:- return None-- failure_reason = _single_line_failure_reason(ctx)- options = _probable_site_block_proxy_options(ctx, include_whether=False)- return (- "The site could not be loaded after repeated attempts. "- f'The latest failure_reason was: "{failure_reason}". '- "Repeating the same IP/workflow shape is unlikely to help, so I should stop this path.\n\n"- f"Would you like me to {options}"- )---def _probable_site_block_stop_signal(ctx: Any, config: CopilotConfig | None = None) -> CopilotToolBlockerSignal:- user_facing = build_probable_site_block_user_question(ctx)- if user_facing is None:- user_facing = (- "The site could not be loaded after repeated attempts. Tell me whether to try a different URL, "- "configure a proxy, or use an alternate entry point."- )- return CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text=_probable_site_block_stop_agent_text(ctx, config),- user_facing_reason=user_facing,- recovery_hint="ask_user_clarifying",- cleared_by_tools=frozenset(),- preserves_workflow_draft=True,- renders_final_reply=True,- internal_reason_code="probable_site_block_stop",- blocked_tool="update_and_run_blocks",- )--def _repair_loop_state(ctx: Any) -> RepairLoopState | None:contract = getattr(ctx, "latest_diagnosis_repair_contract", None)state = getattr(contract, "repair_loop_state", None)return state if isinstance(state, RepairLoopState) else None-def code_authoring_churn_stop_signal(ctx: Any) -> CopilotToolBlockerSignal:- count = _get_int(ctx, "code_authoring_guardrail_reject_count")- evidence = loop_blocker_evidence_from_ctx(ctx)- user_facing, _tiers = compose_loop_blocker_user_facing_reason(- "code_authoring_guardrail_churn", evidence, blocked_tool="update_workflow"- )- agent_steering = (- f"The generated code has been rejected {count} times without an accepted save; "- "stop rewriting it and report the recorded blocker from the preserved draft."- )- return CopilotToolBlockerSignal(- blocker_kind="loop_detected",- agent_steering_text=agent_steering,- user_facing_reason=user_facing,- recovery_hint="report_blocker_to_user",- cleared_by_tools=frozenset(),- preserves_workflow_draft=evidence.has_draft,- renders_final_reply=True,- internal_reason_code="code_authoring_guardrail_churn",- blocked_tool="update_workflow",- )---def credential_priority_authoring_churn_stop_signal(ctx: Any) -> CopilotToolBlockerSignal:- count = _get_int(ctx, "code_authoring_guardrail_reject_count")- evidence = loop_blocker_evidence_from_ctx(ctx)- user_facing, _tiers = compose_loop_blocker_user_facing_reason(- "credential_priority_authoring_churn", evidence, blocked_tool="update_workflow"- )- # Older context snapshots may not carry fields added after the snapshot was created.- reject_reason_codes = getattr(ctx, "last_output_policy_reject_reason_codes", None)- if reject_reason_codes == frozenset({RAW_SECRET_LEAK_REASON_CODE}):- agent_steering = (- "The generated login code embedded the credential's secret value and was refused; "- "stop rewriting it and report the recorded blocker from the preserved draft."- )- else:- agent_steering = (- f"The credential-scout gate has rejected the generated code {count} times without an accepted save; "- "stop rewriting it and report the recorded blocker from the preserved draft."- )- return CopilotToolBlockerSignal(- blocker_kind="loop_detected",- agent_steering_text=agent_steering,- user_facing_reason=user_facing,- recovery_hint="report_blocker_to_user",- cleared_by_tools=frozenset(),- preserves_workflow_draft=evidence.has_draft,- renders_final_reply=True,- internal_reason_code="credential_priority_authoring_churn",- blocked_tool="update_workflow",- )--_CHURN_REASON_CODES = frozenset({"code_authoring_guardrail_churn", "credential_priority_authoring_churn"})-_SCOUTED_SPINE_CHECKPOINT_BLOCK_LABEL = "persisted_draft"def _scouted_spine_open_obligation(ctx: AgentContext) -> list[ObligationFinding]:@@ -510,13 +344,6 @@ def _log_scouted_spine_unresolved(findings: list[ObligationFinding], *, site: st)-SCOUTED_SPINE_TURN_HALT_USER_REASON = (- "I couldn't get past the same problem after several attempts. The saved draft is still missing "- "steps I demonstrated while scouting, and my rewrites kept leaving them out. "- "Tell me what to change and I'll try a different approach."-)--def _get_scouted_spine_missing_steps_for_halt(ctx: AgentContext) -> str | None:"""Missing-steps text for any give-up offer, covering every open obligation family(uncovered rungs, unforgiven drops, unrecorded indices, truncation), not uncovered rungs alone."""@@ -543,72 +370,12 @@ def log_scouted_spine_unresolved_at_turn_halt(ctx: AgentContext) -> bool:return True-def _scouted_spine_turn_end_nudge(ctx: AgentContext) -> str | None:- try:- findings = _scouted_spine_open_obligation(ctx)- except Exception:- LOG.warning("copilot_scouted_spine_turn_end_check_failed", exc_info=True)- return None- if not findings:- return None- if ctx.scouted_spine_checkpoint_fired:- _log_scouted_spine_unresolved(findings, site="turn_end")- return None- ctx.scouted_spine_checkpoint_fired = True- first = findings[0]- reason_code = obligation_finding_reason_code(first)- repair_context = CodeAuthoringRepairContext(- block_label=_SCOUTED_SPINE_CHECKPOINT_BLOCK_LABEL,- reason_code=reason_code,- selector=obligation_finding_selector(first),- )- ctx.last_code_authoring_repair_context = repair_context- record_build_test_outcome(ctx, recorded_outcome_from_authoring_repair_context(repair_context))- _record_code_authoring_guardrail_reject(ctx)- uncovered = uncovered_rung_records(findings)- missing_text = _scouted_spine_missing_text(findings)- LOG.info(- "copilot_scouted_spine_under_build",- block_label=_SCOUTED_SPINE_CHECKPOINT_BLOCK_LABEL,- site="turn_end",- reason_code=reason_code,- missing_rung_count=len(uncovered),- missing_rungs=missing_text,- )- nudge = (- f"The persisted draft under-builds the scouted spine ({reason_code}): "- f"missing rung(s): {missing_text}. Resubmit the code block through update_workflow so every scouted "- "rung is replayed — reuse the synthesized code block verbatim."- )- artifact = render_missing_rung_call_sources(uncovered)- if artifact:- nudge += "\n" + artifact- return nudge--def _code_authoring_reject_count_resets(repeated_outcome: bool | None, frontier_unchanged: bool) -> bool:# A frontier-unchanged reject is churn even when sibling edits move the whole-signature key each# turn (which reads as a non-repeat); it must accumulate toward the churn stop, not reset.return repeated_outcome is False and not frontier_unchanged-def code_authoring_churn_stop_would_claim(ctx: AgentContext, *, frontier_unchanged: bool = False) -> bool:- """Read-only predicate: would recording one more plain (non-credential) guardrail reject reach- the churn ceiling AND win the turn? Mirrors _record_code_authoring_guardrail_reject's threshold,- terminal-defer, and claim precedence without mutating ctx."""- repeated_outcome = latest_recorded_build_test_outcome_repeated(ctx)- base = (- 0- if _code_authoring_reject_count_resets(repeated_outcome, frontier_unchanged)- else ctx.code_authoring_guardrail_reject_count- )- if base + 1 < MAX_CODE_AUTHORING_GUARDRAIL_REJECTS:- return False- if blocker_signal_is_genuinely_terminal(ctx.blocker_signal):- return False- return claim_would_succeed(ctx, TurnClaimant.CODE_AUTHORING_CHURN)--def _record_code_authoring_guardrail_reject(ctx: AgentContext,*,@@ -630,55 +397,6 @@ def _record_code_authoring_guardrail_reject(reject_count=ctx.code_authoring_guardrail_reject_count,credential_priority=defer_churn_stop,)- if defer_churn_stop:- if ctx.code_authoring_guardrail_reject_count < MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS:- return- signal: CopilotToolBlockerSignal = credential_priority_authoring_churn_stop_signal(ctx)- elif ctx.code_authoring_guardrail_reject_count < MAX_CODE_AUTHORING_GUARDRAIL_REJECTS:- return- else:- signal = code_authoring_churn_stop_signal(ctx)- # A genuinely-terminal held blocker keeps both the rendered reply and the- # halt kind, so the churn stop defers to it rather than overriding.- if blocker_signal_is_genuinely_terminal(ctx.blocker_signal):- return- claimant = TurnClaimant.CREDENTIAL_PRIORITY_CHURN if defer_churn_stop else TurnClaimant.CODE_AUTHORING_CHURN- if claim_and_stash_blocker_signal(ctx, claimant, signal, force_stash=True) is None:- return- try:- unresolved_obligation = _scouted_spine_open_obligation(ctx)- except Exception:- LOG.warning("copilot_scouted_spine_churn_stop_check_failed", exc_info=True)- unresolved_obligation = []- if unresolved_obligation:- _log_scouted_spine_unresolved(unresolved_obligation, site="churn_stop")---def no_forward_progress_interaction_stop_signal(ctx: Any) -> CopilotToolBlockerSignal:- count = _get_int(ctx, "consecutive_no_progress_interaction_count")- evidence = loop_blocker_evidence_from_ctx(ctx)- user_facing, _tiers = compose_loop_blocker_user_facing_reason(- "loop_detected_no_forward_progress_interaction", evidence, blocked_tool="click"- )- agent_steering = (- f"Clicking has not advanced the page across {count} attempts; "- "stop trying new selectors and report the recorded blocker from the preserved draft."- )- return CopilotToolBlockerSignal(- blocker_kind="loop_detected",- agent_steering_text=agent_steering,- user_facing_reason=user_facing,- recovery_hint="report_blocker_to_user",- cleared_by_tools=frozenset(),- preserves_workflow_draft=evidence.has_draft,- renders_final_reply=True,- internal_reason_code="loop_detected_no_forward_progress_interaction",- blocked_tool="click",- )---def _needs_no_progress_interaction_halt(ctx: Any) -> bool:- return _get_int(ctx, "consecutive_no_progress_interaction_count") >= MAX_NO_PROGRESS_INTERACTION_ATTEMPTSdef reset_no_progress_interaction_count(ctx: Any) -> None:@@ -693,27 +411,6 @@ def register_no_progress_interaction_click(ctx: Any, *, outcome: str) -> None:count = _get_int(ctx, "consecutive_no_progress_interaction_count") + 1ctx.consecutive_no_progress_interaction_count = countLOG.info("copilot_no_progress_interaction_click", outcome=outcome, count=count)- if count < MAX_NO_PROGRESS_INTERACTION_ATTEMPTS:- return- if blocker_signal_is_genuinely_terminal(ctx.blocker_signal):- return- signal = no_forward_progress_interaction_stop_signal(ctx)- claim_and_stash_blocker_signal(ctx, TurnClaimant.LOOP_DETECTED, signal, force_stash=True)---def _needs_code_authoring_churn_halt(ctx: Any) -> bool:- count = _get_int(ctx, "code_authoring_guardrail_reject_count")- if getattr(ctx, "last_code_authoring_reject_was_credential_priority", False) is True:- return count >= MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS- return count >= MAX_CODE_AUTHORING_GUARDRAIL_REJECTS---def _churn_signal_if_halting(ctx: Any) -> CopilotToolBlockerSignal | None:- if not _needs_code_authoring_churn_halt(ctx):- return None- if getattr(ctx, "last_code_authoring_reject_was_credential_priority", False) is True:- return credential_priority_authoring_churn_stop_signal(ctx)- return code_authoring_churn_stop_signal(ctx)def _typed_terminal_challenge_outcome(ctx: Any) -> RecordedRunOutcome | None:@@ -888,17 +585,6 @@ class CopilotTotalTimeoutError(Exception):"""Raised when the copilot agent exceeds the total allowed runtime."""-class CopilotGoalSatisfied(Exception):- """Raised when a tool proves the workflow already satisfies the turn."""---class CopilotBuiltUnverified(Exception):- """Raised when clean tests should stop repair without claiming goal success."""---BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON = "built_unverified_repair_inert"--def latest_diagnosis_contract_satisfies_goal(ctx: CopilotContext) -> bool:contract = ctx.latest_diagnosis_repair_contractif contract is None:@@ -1189,9 +875,6 @@ def _present_completion_contract_ask_retry(ctx: CopilotContext, parsed: dict[strdef _typed_ask_subject_auto_answer(ctx: CopilotContext, ask_subject: AskSubject, parsed: dict[str, Any]) -> str | None:if ask_subject != "output_schema":return None- refs = parsed_ask_refs(parsed.get("refs"))- if not refs:- return None# Reads the raw policy criteria rather than the turn-active set the other requested-output# consumers use: floor-rekey annotations are baked in at request-policy build time, and what# the model may cite as refs is rendered from this same set in `prompt_summary`.@@ -1200,18 +883,41 @@ def _typed_ask_subject_auto_answer(ctx: CopilotContext, ask_subject: AskSubject,return Nonecriteria = policy.graded_completion_criteria()requested = requested_output_paths(criteria) | floor_rekeyed_requested_output_paths(criteria)- if not requested or not set(refs) <= requested:+ if not requested:+ return None+ refs = parsed_ask_refs(parsed.get("refs"))+ if refs and set(refs) <= requested:+ resolved = sorted(set(refs))+ LOG.info(+ "copilot_ask_subject_auto_answered",+ subject=ask_subject,+ resolved_refs=resolved,+ )+ return (+ "The outputs you asked to confirm are already pinned by this request's completion contract. "+ f"Requested output paths: {', '.join(resolved)}. Author and test the workflow to produce them, "+ "choosing a reasonable representation for each, instead of asking the user to re-confirm."+ )+ # A request that named no output key is pinned to anonymous slots, so no name the model could+ # propose is citable and the coverage check above can never be satisfied. The contract still+ # settles that an output is owed; only what to call it is open, and that belongs to the author.+ if any(not is_canonical_request_slot_path(path) for path in requested):return None- resolved = sorted(set(refs))LOG.info("copilot_ask_subject_auto_answered",subject=ask_subject,- resolved_refs=resolved,+ resolved_refs=[],+ proposed_refs=sorted(set(refs)),+ anonymous_slot_count=len(requested),)+ owed = len(requested)+ subject_phrase = "one output is owed" if owed == 1 else f"{owed} outputs are owed"+ naming_phrase = "what to call it" if owed == 1 else "what to call them"return (- "The outputs you asked to confirm are already pinned by this request's completion contract. "- f"Requested output paths: {', '.join(resolved)}. Author and test the workflow to produce them, "- "choosing a reasonable representation for each, instead of asking the user to re-confirm."+ f"This request's completion contract already settles that {subject_phrase}; the only open "+ f"question is {naming_phrase}, and that is yours to choose. Author and test the workflow to "+ "produce every value the user asked for, under reasonable names and representations, instead "+ "of asking the user to confirm field names.")@@ -1539,14 +1245,6 @@ def _needs_per_tool_budget_nudge(ctx: Any) -> bool:return _get_int(ctx, "per_tool_budget_nudge_count") < MAX_PER_TOOL_BUDGET_NUDGES-def _needs_probable_site_block_stop_nudge(ctx: Any) -> bool:- """Return True when the site-block-wall streak has reached the stop level- AND the per-streak nudge cap has not been exhausted."""- if _get_int(ctx, "probable_site_block_streak_count") < PROBABLE_SITE_BLOCK_STREAK_STOP_AT:- return False- return _get_int(ctx, "probable_site_block_stop_nudge_count") < MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES--def _get_int(ctx: Any, name: str, default: int = 0) -> int:value = getattr(ctx, name, default)return value if isinstance(value, int) else default@@ -1561,11 +1259,6 @@ def _repeated_frontier_failure_nudge(ctx: Any, config: CopilotConfig | None = No# repeated-frontier nudge smuggle different retry advice past the gate.if getattr(ctx, "last_test_non_retriable_nav_error", None):return None- # Defer to the probable-site-block stop path once the wall has been- # confirmed across ≥ PROBABLE_SITE_BLOCK_STREAK_STOP_AT shape-independent- # attempts — at that point "try yet another shape" is empirically wrong.- if _get_int(ctx, "probable_site_block_streak_count") >= PROBABLE_SITE_BLOCK_STREAK_STOP_AT:- return Nonestreak = _get_int(ctx, "repeated_failure_streak_count")emitted = _get_int(ctx, "repeated_failure_nudge_emitted_at_streak")top_category = getattr(ctx, "last_failure_category_top", None)@@ -1620,11 +1313,6 @@ def _check_enforcement(raise_if_turn_halt(ctx, verified=verified)_raise_if_unrecoverable_contract_stop(ctx)- if verified_goal_satisfied_context(ctx):- raise CopilotGoalSatisfied()- if built_unverified_repair_inert_context(ctx) or built_complete_without_evaluated_outcome(ctx):- raise CopilotBuiltUnverified()-# A pending credential pause pre-empts every hygiene nudge below, not just# the failed-test one: a credential-blocked update_and_run_blocks call# satisfies post_update (test not run) and, when the diagnosis contract@@ -1703,36 +1391,12 @@ def _check_enforcement(# Checked before the generic failed-test nudge so a scrape-wall streak# emits the specific STOP text and does not also consume a# failed_test_nudge_count slot.- if _needs_probable_site_block_stop_nudge(ctx):- ctx.probable_site_block_stop_nudge_count = getattr(ctx, "probable_site_block_stop_nudge_count", 0) + 1- signal = _probable_site_block_stop_signal(ctx, config)- stash_blocker_signal(ctx, signal)- stash_turn_halt_from_blocker_signal(ctx, signal, source="enforcement")- raise_if_turn_halt(ctx, verified=verified)-if _needs_failed_test_nudge(ctx):ctx.failed_test_nudge_count += 1if _needs_inspect_before_repair_nudge(ctx):return _nudge(config, "post_failed_test_inspect_first")return _nudge(config, "post_failed_test")- # The convergence floor for code-authoring guardrail churn. It is the last- # halt so every genuinely-terminal stop and recoverable nudge above gets its- # turn first. The permanent non-retriable nav raise is the one stop ordering- # cannot front-run (it lives after _check_enforcement returns None, at the- # run_with_enforcement return sites), so the floor yields to it explicitly.- if not getattr(ctx, "last_test_non_retriable_nav_error", None):- churn_signal = _churn_signal_if_halting(ctx)- if churn_signal is not None:- emit_blocker_signal_payload(ctx, churn_signal)- stash_turn_halt_from_blocker_signal(ctx, churn_signal, source="enforcement_backstop")- raise_if_turn_halt(ctx)- if _needs_no_progress_interaction_halt(ctx):- no_progress_signal = no_forward_progress_interaction_stop_signal(ctx)- emit_blocker_signal_payload(ctx, no_progress_signal)- stash_turn_halt_from_blocker_signal(ctx, no_progress_signal, source="enforcement_backstop")- raise_if_turn_halt(ctx)-# Response-time gate: peek at the model's final output to tell ASK_QUESTION# (always allowed) from a REPLY with a coverage gap or progress-narration.# Only runs when no state-based nudge fired.@@ -2051,7 +1715,6 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {POST_PARAMETER_BINDING_WARN_NUDGE: "parameter_binding_warn",POST_PARAMETER_BINDING_STOP_NUDGE: "parameter_binding_stop",POST_ANTI_BOT_FAILED_TEST_NUDGE: "anti_bot_block",- POST_PROBABLE_SITE_BLOCK_STOP_NUDGE: "probable_site_block_stop",POST_PER_TOOL_BUDGET_NUDGE: "per_tool_budget_split",POST_PER_TOOL_BUDGET_STOP_NUDGE: "per_tool_budget_stop",POST_NO_WORKFLOW_DELIVERY_NUDGE: "no_workflow_delivery",@@ -2074,8 +1737,6 @@ _NUDGE_TYPE_BY_KEY: dict[str, str] = {"post_parameter_binding_warn": "parameter_binding_warn","post_parameter_binding_stop": "parameter_binding_stop","post_anti_bot_failed_test": "anti_bot_block",- "post_probable_site_block_stop": "probable_site_block_stop",- "post_probable_site_block_stop_prefix": "probable_site_block_stop","post_per_tool_budget": "per_tool_budget_split","post_per_tool_budget_stop": "per_tool_budget_stop","post_no_workflow_delivery": "no_workflow_delivery",@@ -2091,8 +1752,6 @@ _NUDGE_TYPE_BY_KEY: dict[str, str] = {def _nudge_type_for_log(nudge: str, config: CopilotConfig | None = None) -> str:nudge_by_key = config.enforcement_nudges if config is not None else DEFAULT_ENFORCEMENT_NUDGES- if nudge.startswith(nudge_by_key["post_probable_site_block_stop_prefix"]):- return "probable_site_block_stop"for key, value in nudge_by_key.items():if value == nudge:return _NUDGE_TYPE_BY_KEY.get(key, key)@@ -3208,177 +2867,9 @@ def _credential_scout_reopen_admits_evaluate(ctx: CopilotContext, tool_name: strreturn tool_name == "evaluate" and bool(ctx.synthesized_block_reopened_for_credential_scout)-def _should_block_tool_after_unresolved_recorded_outcome(ctx: Any, tool_name: str) -> bool:- if tool_name not in _SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS and tool_name != "update_workflow":- return False- if not _turn_intent_can_update_and_run_without_user_input(getattr(ctx, "turn_intent", None)):- return False- if normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None)) != (- BlockAuthoringPolicy.CODE_ONLY_BROWSER- ):- return False- latest = getattr(ctx, "latest_recorded_build_test_outcome", None)- if not isinstance(latest, RecordedBuildTestOutcome) or not latest.is_authoritative:- return False- if latest.phase != "persisted_block_run" or latest.reason_code != "outcome_not_demonstrated":- return False- return _completion_verification_unsatisfied(ctx)---def _post_run_page_path_tool_allowed(- condition: PostRunPagePathFailure,- tool_name: str,- arguments: Mapping[str, Any] | None,-) -> bool:- if not condition.is_page_path or not isinstance(arguments, Mapping):- return False- selectors = {target.selector for target in condition.continuation_targets}- if tool_name == "click":- selector = arguments.get("selector")- return isinstance(selector, str) and selector in selectors- if tool_name != "press_key" or arguments.get("key") != "Enter" or not condition.enter_allowed:- return False- selector = arguments.get("selector")- enter_selectors = {- target.selector for target in condition.continuation_targets if target.kind in {"form_submit", "challenge"}- }- return isinstance(selector, str) and selector in enter_selectors---def _latest_scout_trajectory_index(ctx: CopilotContext) -> int:- return max(- (- trajectory_index- for interaction in ctx.scout_trajectory- if isinstance((trajectory_index := interaction.get("trajectory_index")), int)- ),- default=-1,- )---def _post_run_page_path_successful_interactions(- ctx: CopilotContext,- window: PostRunPagePathInteractionWindow,-) -> int:- return sum(- 1- for interaction in ctx.scout_trajectory- if isinstance((trajectory_index := interaction.get("trajectory_index")), int)- and trajectory_index > window.trajectory_anchor- and str(interaction.get("tool_name") or "") in _SYNTHESIZED_BLOCK_COMMIT_TOOLS- )---def _post_run_page_path_admission_state(- ctx: CopilotContext,- tool_name: str,- arguments: Mapping[str, Any] | None,-) -> tuple[PostRunPagePathInteractionWindow, str, str, int, int] | None:- if not _should_block_tool_after_unresolved_recorded_outcome(ctx, tool_name):- return None- latest = ctx.latest_recorded_build_test_outcome- if latest is None:- return None- condition = latest.page_path_failure- if condition is None or not _post_run_page_path_tool_allowed(condition, tool_name, arguments):- return None- structural_key = latest.structural_key- workflow_run_id = latest.workflow_run_id- if not structural_key or not workflow_run_id:- return None- if (- ctx.post_run_page_observation_after_failed_test is not True- or ctx.post_run_page_observation_tool not in _POST_RUN_PAGE_OBSERVATION_TOOLS- or ctx.post_run_page_observation_workflow_run_id != workflow_run_id- or ctx.post_run_page_observation_url != condition.current_url- or condition.workflow_run_id != workflow_run_id- or ctx.last_run_blocks_workflow_run_id != workflow_run_id- ):- return None- window = ctx.post_run_page_path_interaction_window- if window is None or (window.structural_key, window.workflow_run_id) != (structural_key, workflow_run_id):- window = PostRunPagePathInteractionWindow(- structural_key=structural_key,- workflow_run_id=workflow_run_id,- trajectory_anchor=_latest_scout_trajectory_index(ctx),- )- successful_interactions = _post_run_page_path_successful_interactions(ctx, window)- observation_generation = ctx.post_run_page_observation_generation- if window.admitted_attempts >= _POST_RUN_PAGE_PATH_INTERACTION_BUDGET or (- successful_interactions > window.observed_successful_interactions- and observation_generation <= window.observation_generation- ):- return None- owner = current_turn_owner(ctx)- if (- owner is not None- and owner.claimant is not TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION- and not claimant_outranks(TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION, owner.claimant)- ):- return None- return window, structural_key, workflow_run_id, successful_interactions, observation_generation---def post_run_page_path_interaction_allowed(- ctx: CopilotContext,- tool_name: str,- arguments: Mapping[str, Any] | None,-) -> bool:- return _post_run_page_path_admission_state(ctx, tool_name, arguments) is not None---def try_admit_post_run_page_path_interaction(- ctx: CopilotContext,- tool_name: str,- arguments: Mapping[str, Any] | None,-) -> bool:- state = _post_run_page_path_admission_state(ctx, tool_name, arguments)- if state is None:- return False- window, structural_key, workflow_run_id, successful_interactions, observation_generation = state- if claim_turn(ctx, TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION) is ClaimOutcome.YIELDED:- return False- ctx.post_run_page_path_interaction_window = replace(- window,- admitted_attempts=window.admitted_attempts + 1,- observation_generation=observation_generation,- observed_successful_interactions=successful_interactions,- )- LOG.info(- "copilot post-run page-path interaction admitted",- tool_name=tool_name,- selector=arguments.get("selector") if isinstance(arguments, Mapping) else None,- structural_key=structural_key,- workflow_run_id=workflow_run_id,- admitted_attempts=window.admitted_attempts + 1,- observation_generation=observation_generation,- )- return True--def synthesized_block_persistence_signal(ctx: Any, tool_name: str, arguments: Mapping[str, Any] | None = None) -> CopilotToolBlockerSignal | None:- if _should_block_tool_after_unresolved_recorded_outcome(ctx, tool_name):- signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text=(- "The last recorded test outcome is authoritative and still has unsatisfied completion criteria. "- f"Call {SYNTHESIZED_BLOCK_PERSISTENCE_TOOL} with a materially changed authored workflow now; "- "do not spend the repair attempt on another standalone tool call."- ),- user_facing_reason="I need to revise and test the workflow code instead of interacting with the page.",- recovery_hint="retry_with_different_tool",- cleared_by_tools=frozenset({SYNTHESIZED_BLOCK_PERSISTENCE_TOOL}),- preserves_workflow_draft=True,- renders_final_reply=False,- internal_reason_code=SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,- blocked_tool=tool_name,- extra={"recorded_outcome_reason_code": "outcome_not_demonstrated"},- )- if try_admit_post_run_page_path_interaction(ctx, tool_name, arguments):- return None- return signalif tool_name in _SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS:return Noneambiguous_selector_rescout_state = _ambiguous_bare_selector_rescout_signal_state(ctx, tool_name)@@ -3479,10 +2970,6 @@ async def run_with_enforcement(_mark_copilot_total_timeout(ctx)raise CopilotTotalTimeoutError()- if iteration >= MAX_ITERATIONS:- LOG.error("Enforcement iteration cap reached", max_iterations=MAX_ITERATIONS)- raise CopilotTotalTimeoutError()-# When the current turn contains image payloads, the session-backed# input filter cannot protect us — the payload is in current_input,# not in session history. Estimate regardless of session.@@ -3627,7 +3114,6 @@ async def run_with_enforcement(# nudge path and the finalize path.synthesized_msg = _maybe_synthesized_block_offer_msg(ctx)- spine_checkpoint_nudge = Falseif nudge is None:# Checked whenever there's no regular nudge, even if a synthesized# offer is also pending: a credential-blocked run's diagnosis can@@ -3656,33 +3142,22 @@ async def run_with_enforcement(# nudge=None doesn't re-trigger this off a stale "declined".nudge = _check_enforcement(ctx, result, copilot_config)if nudge is None and synthesized_msg is None:- spine_nudge = _scouted_spine_turn_end_nudge(ctx)- if spine_nudge is None:- _consume_pending_screenshots(ctx)- _maybe_raise_non_retriable_nav(ctx)- return result- nudge = spine_nudge- spine_checkpoint_nudge = True-- if nudge is not None and not spine_checkpoint_nudge and nudge == _nudge(copilot_config, "post_update"):+ _consume_pending_screenshots(ctx)+ _maybe_raise_non_retriable_nav(ctx)+ return result++ if nudge is not None and nudge == _nudge(copilot_config, "post_update"):if ctx.post_update_nudge_count >= MAX_POST_UPDATE_NUDGES:LOG.warning("Enforcement exhausted post-update nudges, allowing response",nudge_count=ctx.post_update_nudge_count,)- spine_nudge = _scouted_spine_turn_end_nudge(ctx)- if spine_nudge is None:- _consume_pending_screenshots(ctx)- _maybe_raise_non_retriable_nav(ctx)- return result- nudge = spine_nudge- spine_checkpoint_nudge = True- else:- ctx.post_update_nudge_count += 1+ _consume_pending_screenshots(ctx)+ _maybe_raise_non_retriable_nav(ctx)+ return result+ ctx.post_update_nudge_count += 1- if spine_checkpoint_nudge:- nudge_type = "scouted_spine_under_build_checkpoint"- elif nudge is not None:+ if nudge is not None:nudge_type = _nudge_type_for_log(nudge, copilot_config)else:nudge_type = "synthesized_block_offer"diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.pyindex c8ff093d3..59959db65 100644--- a/skyvern/forge/sdk/copilot/failure_tracking.py+++ b/skyvern/forge/sdk/copilot/failure_tracking.py@@ -49,8 +49,6 @@ ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY = "ANTI_BOT_CHALLENGE"# Stable active-run terminal evidence identifiers. The category is stored in# tool result ``failure_categories``; the reason code is stored on blocker# signals that convert that tool result into product-safe final copy.-ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY = "ACTIVE_RUN_TERMINAL_EVIDENCE"-ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE = "tool_error_active_run_terminal_evidence"_ROOT_CAUSE_CATEGORY_ALIASES = {**{alias: ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY for alias in ANTI_BOT_CHALLENGE_ALIAS_CATEGORIES},diff --git a/skyvern/forge/sdk/copilot/hooks.py b/skyvern/forge/sdk/copilot/hooks.pyindex 78d08d475..723e99303 100644--- a/skyvern/forge/sdk/copilot/hooks.py+++ b/skyvern/forge/sdk/copilot/hooks.py@@ -13,7 +13,6 @@ from agents.run_context import RunContextWrapperfrom agents.tool import Toolfrom skyvern.forge.sdk.copilot.enforcement import (- CopilotGoalSatisfied,gate_decision_trace_fields,outcome_fully_verified,)@@ -146,8 +145,10 @@ class CopilotRunHooks(RunHooksBase):raise_if_turn_halt(self._ctx, verified=outcome_fully_verified(self._ctx))if _tool_completion_satisfies_turn(self._ctx, tool_name, parsed):+ # The run record satisfying the ask is the model's cue to wrap up, not the+ # harness's cue to end the loop and speak for it.LOG.info(- "copilot tool satisfied goal; stopping agent loop",+ "copilot tool satisfied goal; model composes the reply",tool_name=tool_name,workflow_run_id=(parsed.get("data") or {}).get("workflow_run_id")if isinstance(parsed.get("data"), dict)@@ -156,4 +157,3 @@ class CopilotRunHooks(RunHooksBase):)self._ctx.goal_satisfied_tool_name = tool_nameself._ctx.goal_satisfied_tool_output = dict(parsed)- raise CopilotGoalSatisfied()diff --git a/skyvern/forge/sdk/copilot/loop_detection.py b/skyvern/forge/sdk/copilot/loop_detection.pyindex 9fdfcb8e2..6c2c3cb68 100644--- a/skyvern/forge/sdk/copilot/loop_detection.py+++ b/skyvern/forge/sdk/copilot/loop_detection.py@@ -5,11 +5,12 @@ Two independent guards:* ``detect_tool_loop`` fires on strictly consecutive same-identity streaks(A-A-A over one tool_step_identity). Resets the moment the identitychanges, so oscillating and argument-distinct patterns bypass it by design.-* ``detect_failed_tool_step_loop`` fires on N repeated failures of the- same (tool, args) pair, even when other tools dispatch in between.- Block-running credential/config failures are keyed by failure category- instead, because draft arguments can change while the init failure does not.- A successful invocation of the same step resets its counter.+* ``detect_failed_tool_step_loop`` fires on N block-running credential/config+ failures, keyed by failure category because draft arguments can change while+ the init failure does not. It deliberately does not count same-(tool, args)+ retries: an identical retry means the model learned nothing from the error,+ which is the error message's problem, not grounds to end the turn.+ A successful invocation of the same tool resets its counters."""from __future__ import annotations@@ -216,28 +217,15 @@ def detect_failed_tool_step_loop(return Nonecategory_failure = _detect_argument_insensitive_failed_tool_loop(tracker, tool_name, threshold)- if category_failure is not None:- category, failure_count = category_failure- next_attempt = failure_count + 1- return (- f"{LOOP_DETECTED_MARKER} '{tool_name}' has already failed "- f"{failure_count} times with {category}; blocking attempt #{next_attempt}. "- "This failure is not tied to the draft arguments. Fix the credential/configuration, "- "ask the user, or produce your final JSON response."- )-- identity = tool_step_identity(tool_name, arguments)- failure_count = tracker.get(identity, 0)- next_attempt = failure_count + 1- if next_attempt < threshold:+ if category_failure is None:return None-+ category, failure_count = category_failure+ next_attempt = failure_count + 1return (f"{LOOP_DETECTED_MARKER} '{tool_name}' has already failed "- f"{failure_count} consecutive times with these arguments; "- f"blocking attempt #{next_attempt}. "- "Use different arguments, a DIFFERENT tool, ask the user, "- "or produce your final JSON response."+ f"{failure_count} times with {category}; blocking attempt #{next_attempt}. "+ "This failure is not tied to the draft arguments. Fix the credential/configuration, "+ "ask the user, or produce your final JSON response.")@@ -248,14 +236,14 @@ def record_tool_step_result(result: Mapping[str, Any],threshold: int = MAX_REPEATED_FAILED_STEP,) -> None:- identity = tool_step_identity(tool_name, arguments)if result.get("ok", True):- tracker.pop(identity, None)+ tracker.pop(tool_step_identity(tool_name, arguments), None)_clear_argument_insensitive_failure_identities(tracker, tool_name)return- identity = _argument_insensitive_failure_identity(tool_name, result) or identity- tracker[identity] = min(tracker.get(identity, 0) + 1, threshold)+ identity = _argument_insensitive_failure_identity(tool_name, result)+ if identity is not None:+ tracker[identity] = min(tracker.get(identity, 0) + 1, threshold)def clear_failed_step_tracker_for_tools(diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex 98d45825d..25387bcad 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -35,15 +35,12 @@ from skyvern.forge.sdk.copilot.blocker_signal import ()from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signalfrom skyvern.forge.sdk.copilot.enforcement import (- post_run_page_path_interaction_allowed,register_no_progress_interaction_click,synthesized_block_persistence_signal,terminal_challenge_blocker_signal_from_current_page_evidence,- try_admit_post_run_page_path_interaction,)from skyvern.forge.sdk.copilot.loop_detection import (detect_failed_tool_step_loop_for_ctx,- detect_tool_loop,record_tool_step_result_for_ctx,)from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm@@ -456,12 +453,7 @@ class SkyvernOverlayMCPServer(MCPServer):return _copilot_to_call_tool_result({"ok": False, "error": payload})refresh_held_loop_blocker_evidence(copilot_ctx)- post_run_page_path_allowed = post_run_page_path_interaction_allowed(- copilot_ctx,- tool_name,- arguments,- )- if not post_run_page_path_allowed and tool_name in _CURRENT_PAGE_TERMINAL_CHALLENGE_MCP_TOOLS:+ if tool_name in _CURRENT_PAGE_TERMINAL_CHALLENGE_MCP_TOOLS:terminal_challenge_payload = _stash_and_emit_current_page_terminal_challenge_blocker(copilot_ctx,tool_name,@@ -473,11 +465,7 @@ class SkyvernOverlayMCPServer(MCPServer):)return _copilot_to_call_tool_result({"ok": False, "error": terminal_challenge_payload})- persistence_signal = (- None- if post_run_page_path_allowed- else synthesized_block_persistence_signal(copilot_ctx, tool_name, arguments)- )+ persistence_signal = synthesized_block_persistence_signal(copilot_ctx, tool_name, arguments)if persistence_signal is not None:LOG.warning("Synthesized block persistence required before MCP tool",@@ -502,34 +490,12 @@ class SkyvernOverlayMCPServer(MCPServer):payload = _stash_and_emit_loop_blocker(copilot_ctx, loop_error, tool_name)return _copilot_to_call_tool_result({"ok": False, "error": payload})- tracker = getattr(copilot_ctx, "consecutive_tool_tracker", None)- loop_error = detect_tool_loop(tracker, tool_name, arguments) if isinstance(tracker, list) else None- if loop_error:- LOG.warning(- "Tool loop detected, skipping execution",- tool_name=tool_name,- )- payload = _stash_and_emit_loop_blocker(copilot_ctx, loop_error, tool_name)- return _copilot_to_call_tool_result({"ok": False, "error": payload})-if overlay.pre_hook:hook_result = await overlay.pre_hook(arguments, copilot_ctx)if hook_result is not None:record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, hook_result)return _copilot_to_call_tool_result(hook_result)- if post_run_page_path_allowed and not try_admit_post_run_page_path_interaction(- copilot_ctx,- tool_name,- arguments,- ):- persistence_signal = synthesized_block_persistence_signal(copilot_ctx, tool_name, arguments)- if persistence_signal is not None:- payload = emit_blocker_signal_payload(copilot_ctx, persistence_signal)- result = {"ok": False, "error": payload}- record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, result)- return _copilot_to_call_tool_result(result)-mcp_name = self._alias_map.get(tool_name, tool_name)mcp_args = _transform_args(arguments, overlay)diff --git a/skyvern/forge/sdk/copilot/output_contracts.py b/skyvern/forge/sdk/copilot/output_contracts.pyindex 11d45ab4c..a79eaa1f6 100644--- a/skyvern/forge/sdk/copilot/output_contracts.py+++ b/skyvern/forge/sdk/copilot/output_contracts.py@@ -101,18 +101,6 @@ def resolve_output_contract_actuation()producible_source = observable_source or grantable_sourceprogressed = evidence.actuation_progress_exhausted or evidence.prior_actuation- if (- evidence.click_only_spine- and not evidence.observed_required_values- and not evidence.loaded_result_source_producible- and evidence.declick_attempt_failed- and not grantable_source- ):- return OutputContractActuation(- OutputContractActuationKind.BLOCKED_TERMINAL,- family,- OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,- )if (producible_sourceand evidence.advisory_state in {OutputContractAdvisoryState.UNUSED, OutputContractAdvisoryState.GRANTED}diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex 34ce9ad9b..f8341e218 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -1,5 +1,6 @@from __future__ import annotations+import astimport refrom collections.abc import Mappingfrom dataclasses import dataclass, field@@ -240,7 +241,6 @@ class OutputPolicyReason(StrEnum):UNAPPROVED_CREDENTIAL_REFERENCE = "unapproved_credential_reference"CREDENTIAL_SCOPE_BROADENED = "credential_scope_broadened"UNBACKED_WORKFLOW_DELIVERY_CLAIM = "unbacked_workflow_delivery_claim"- MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE = "missing_unvalidated_proposal_affordance"MISSING_PROPOSAL_STATE = "missing_proposal_state"PERSISTENCE_STATE_MISMATCH = "persistence_state_mismatch"INTERNAL_TOOL_INSTRUCTION_LEAK = "internal_tool_instruction_leak"@@ -291,13 +291,12 @@ _FINAL_OUTPUT_HARD_BLOCK_REASONS: frozenset[OutputPolicyReason] = frozenset()-# The authoring seam refuses only what the ``credential_scout`` hard block covers, because a raw or-# broadened credential in a persisted draft is an irreversible disclosure; everything else steers.-# Each member's surface and the reason it cannot be dropped are in+# The authoring seam refuses only a credential reference the org has not approved or that reaches+# wider than the request, because either is an irreversible disclosure once persisted; everything+# else steers. Each member's surface and the reason it cannot be dropped are in# cloud_docs/workflow-copilot/architecture/output-policy-disposition.md._AUTHOR_TIME_HARD_BLOCK_REASONS: frozenset[OutputPolicyReason] = frozenset({- OutputPolicyReason.RAW_SECRET_LEAK,OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,}@@ -563,11 +562,10 @@ def evaluate_output_policy(unvalidated=unvalidated,)verdict = OutputPolicyVerdict(output_kind=output_kind)- # Scan only proposed output/tool surfaces for hard leaks. The rolling- # global context can include prior turn state, so re-scanning it here can- # repeatedly block otherwise safe follow-up responses.values = [user_response, workflow_yaml, tool_arguments]- if any(_contains_raw_secret(value) for value in values):+ # Only the reply is scanned for a raw secret: rebinding and persistence scrubbing own what+ # reaches storage, and scanning a draft judged the YAML encoding rather than the value.+ if _contains_raw_secret(user_response):verdict.add(OutputPolicyReason.RAW_SECRET_LEAK)if _contains_internal_tool_instruction(user_response):verdict.add(OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK)@@ -579,13 +577,6 @@ def evaluate_output_policy():verdict.add(OutputPolicyReason.UNBACKED_WORKFLOW_DELIVERY_CLAIM)verdict.add(OutputPolicyReason.MISSING_PROPOSAL_STATE)- if (- response_type == "REPLY"- and has_workflow_proposal- and unvalidated- and not _has_unvalidated_affordance(user_response)- ):- verdict.add(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)if _contains_internal_block_taxonomy_leak(user_response, output_kind, response_type):verdict.add(OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK)if response_type in _USER_VISIBLE_REPLY_TYPES and _contains_internal_classifier_vocab_leak(user_response):@@ -626,12 +617,6 @@ def _asks_to_confirm_output_fields(user_response: str | None) -> bool:def format_output_policy_tool_error(verdict: OutputPolicyVerdict) -> str:reasons = ", ".join(reason.value for reason in verdict.reason_codes) or "unknown"message = f"Output policy blocked this Copilot output before persistence. Reason codes: {reasons}."- if OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes:- message += (- " For saved credentials, bind a credential_id workflow parameter and reference fields as "- "`<key>.username`, `<key>.password`, or `await <key>.otp()` for one-time codes; do not split, "- "concatenate, or obfuscate literal secrets in workflow code or YAML."- )return message@@ -644,21 +629,20 @@ def _contains_raw_secret(value: Any) -> bool:matched = match.group(0)if any(marker in matched for marker in _PLACEHOLDER_MARKERS):continue- if pattern is SECRET_KEYWORD_ASSIGNMENT_PATTERN and (- _is_sanctioned_secret_reference(matched)- or _is_sanctioned_secret_reference(_line_containing_match(text, match))- ):+ if pattern is SECRET_KEYWORD_ASSIGNMENT_PATTERN and _keyword_assignment_is_exempt(text, match):continuereturn Truereturn False+def _line_end_after_match(text: str, match: re.Match[str]) -> int:+ line_end = text.find("\n", match.end())+ return len(text) if line_end == -1 else line_end++def _line_containing_match(text: str, match: re.Match[str]) -> str:line_start = text.rfind("\n", 0, match.start()) + 1- line_end = text.find("\n", match.end())- if line_end == -1:- line_end = len(text)- return text[line_start:line_end]+ return text[line_start : _line_end_after_match(text, match)]def _is_sanctioned_secret_reference(matched: str) -> bool:@@ -668,6 +652,91 @@ def _is_sanctioned_secret_reference(matched: str) -> bool:return bool(_SANCTIONED_SECRET_REFERENCE_RE.match(rhs_match.group(1)))+# `token` is the one secret keyword with an unrelated English sense — a lexical token parsed out of+# page text. The other keywords (password, passcode, secret, api key, bearer, authorization) carry no+# such second meaning and keep the strict sanctioned-source rule.+_LEXICAL_TOKEN_LHS_RE = re.compile(r"^[A-Za-z0-9_]*token(?=\s*[:=])", re.I)+_UNAMBIGUOUS_SECRET_KEYWORD_RE = re.compile(r"password|passcode|api[_ -]?key|secret|bearer|authorization", re.I)+++def _parse_chain_receiver(node: ast.expr) -> ast.expr | None:+ """The variable a read/index/method chain is rooted in, or None for anything else — `identity(...)`+ roots in a free function and `("ghp_...", parts)[0]` in a literal container, so neither counts."""+ while True:+ if isinstance(node, (ast.Subscript, ast.Attribute, ast.Await)):+ node = node.value+ elif isinstance(node, ast.Call):+ if not isinstance(node.func, ast.Attribute):+ return None+ node = node.func.value+ else:+ return node+++def _parses_a_value_out_of_something(rhs: str) -> bool:+ """Whether the assigned value is read out of a variable rather than written down: a credential+ reaches code as a literal, a lexical token is parsed off something in scope (`m.group(1)`).+ Anything that does not parse as one expression gets no verdict and keeps the keyword match."""+ candidate = rhs.strip()+ # On the last line of embedded code the scalar's own closing delimiter rides along.+ for text in (candidate, candidate.rstrip("\"'")):+ try:+ body = ast.parse(text, mode="eval").body+ except (SyntaxError, ValueError):+ continue+ # A bare dotted chain is how a JWT looks, so extraction has to involve an actual call or index.+ if not any(isinstance(node, (ast.Call, ast.Subscript)) for node in ast.walk(body)):+ return False+ if any(_reads_as_a_written_down_secret(node) for node in ast.walk(body)):+ return False+ return isinstance(_parse_chain_receiver(body), ast.Name)+ return False+++# A delimiter or key a parse chain consumes is short, or carries whitespace or punctuation+# (`" logs found"`, `"access_token"`). A credential written into the chain is neither.+_SECRET_SHAPED_LITERAL_LENGTH = 16+++def _reads_as_a_written_down_secret(node: ast.AST) -> bool:+ """Whether a literal inside an extraction chain carries a value in rather than naming a+ delimiter or key — `cache.get("ghp_...")` roots in a variable exactly as a real read does."""+ if not isinstance(node, ast.Constant) or not isinstance(node.value, str):+ return False+ return len(node.value) >= _SECRET_SHAPED_LITERAL_LENGTH and all(+ char.isalnum() or char in "-_" for char in node.value+ )+++def _keyword_assignment_is_exempt(text: str, match: re.Match[str]) -> bool:+ matched = match.group(0)+ line = _line_containing_match(text, match)+ if _is_sanctioned_secret_reference(matched) or _is_sanctioned_secret_reference(line):+ return True+ return any(_is_lexical_token_assignment(matched, reading) for reading in _assignment_readings(text, match))+++def _assignment_readings(text: str, match: re.Match[str]) -> list[str]:+ """The assignment as the whole line, as the text from the keyword on, and — when the code is+ embedded in a quoted YAML scalar — as the single escaped segment carrying it.++ Only the YAML decoder knows whether an embedded ``\\n`` ends a line or is code data, so both+ readings are offered and either one showing an extraction exempts the match. Reading from the+ keyword matters because a YAML key's own colon precedes the code on a scalar's opening line.+ """+ from_keyword = text[match.start() : _line_end_after_match(text, match)]+ escaped_segment = from_keyword.split("\\n")[0].replace('\\"', '"').replace("\\'", "'")+ return [_line_containing_match(text, match), from_keyword, escaped_segment]+++def _is_lexical_token_assignment(matched: str, assignment: str) -> bool:+ if not _LEXICAL_TOKEN_LHS_RE.match(matched) or _UNAMBIGUOUS_SECRET_KEYWORD_RE.search(assignment):+ return False+ # The keyword pattern ends at the first whitespace, so the rest of the line carries the expression.+ rhs_match = _SECRET_ASSIGNMENT_RHS_RE.search(assignment)+ return rhs_match is not None and _parses_a_value_out_of_something(rhs_match.group(1))++def _contains_internal_tool_instruction(user_response: str | None) -> bool:if not isinstance(user_response, str):return Falsediff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 4b5df4f4f..4bc9d4e71 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -13,8 +13,6 @@ from skyvern.forge.sdk.agents.context import sanitize_agent_tool_result_for_llmfrom skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, assert_clean_user_facing_textfrom skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPESfrom skyvern.forge.sdk.copilot.failure_tracking import (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,- ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,PER_TOOL_BUDGET_FAILURE_CATEGORY,)from skyvern.forge.sdk.copilot.loop_detection import LOOP_DETECTED_MARKER@@ -351,9 +349,7 @@ def _blocker_signal_matches_result(signal: CopilotToolBlockerSignal, result: dicsteering = signal.agent_steering_textif error == steering or steering in error:return True- return signal.internal_reason_code == ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE and _has_failure_category(- result, ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY- )+ return Falsedef _failure_categories(result: dict[str, Any]) -> list[Any]:diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex c792a955b..ade7fa895 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -3123,19 +3123,6 @@ def _validation_classification_output_path(output_path: str | None) -> bool:)-def completion_criterion_requires_active_run_terminal_monitor(criterion: CompletionCriterion) -> bool:- if criterion.requested_output_corroborator:- return False- if (- criterion.output_path- and criterion.level != "definition"- and not criterion.method_mandated- and criterion.kind != "validation_classification"- ):- return False- return True--def _source_requested_output_corroborator(criteria: list[CompletionCriterion],requested_fields: list[str],diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 00351da2c..6adfe94d6 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -216,6 +216,11 @@ class ScoutedInteraction(TypedDict):control_value_satisfied: NotRequired[bool]trajectory_index: NotRequired[int]carried: NotRequired[bool]+ # A read the scout proved on the live page: the expression it ran and the output path the+ # value answers. Recorded so authoring quotes a proven read instead of guessing a selector.+ read_expression: NotRequired[str]+ read_output_path: NotRequired[str]+ read_result_shape: NotRequired[str]# Set when a live scout-time count()==1 probe found the captured selector matching >1 element on its# source page; synthesis re-anchors or drops it rather than emitting a selector that strict-mode-fails.ambiguous: NotRequired[bool]@@ -239,16 +244,6 @@ class ScoutedInteraction(TypedDict):challenge_state: NotRequired[dict[str, Any]]-@dataclass(frozen=True)-class PostRunPagePathInteractionWindow:- structural_key: str- workflow_run_id: str- trajectory_anchor: int- admitted_attempts: int = 0- observation_generation: int = 0- observed_successful_interactions: int = 0--@dataclassclass AgentContext:organization_id: str@@ -412,7 +407,6 @@ class AgentContext:post_run_page_observation_workflow_run_id: str | None = Nonepost_run_page_observation_after_failed_test: bool = Falsepost_run_page_observation_generation: int = 0- post_run_page_path_interaction_window: PostRunPagePathInteractionWindow | None = Nonepost_run_current_page_inspection_workflow_run_id: str | None = Nonelast_evaluate_actionable_signature: str | None = Nonelast_evaluate_actionable_url: str | None = None@@ -436,7 +430,6 @@ class AgentContext:# Ordered (method, receiver) browser mutations of the last successfully persisted draft's code# blocks; None until a persist succeeds this turn. Gates the scouted-spine under-build reject and turn-end nudge.persisted_draft_browser_calls: list[tuple[str, str]] | None = None- scouted_spine_checkpoint_fired: bool = Falsescouted_spine_previous_omission_digest: str | None = Nonescouted_spine_repeated_identical_missing_steps: bool = False# Author-time output-contract cross-turn state, keyed by the contract signature; set lazily by workflow_update.@@ -542,6 +535,9 @@ class AgentContext:# Source-bound row identity captured before a positional click dispatches. The post-hook consumes it# only for the exact selector/source pair, so navigation cannot transfer the witness to another click.pending_scout_dynamic_row: ScoutedDynamicRowEvidence | None = None+ # Expression of an in-flight evaluate, stashed pre-dispatch: the MCP response carries only the+ # result, so a post-hook that wants the expression must receive it from the invocation side.+ pending_scout_read_expression: str | None = None# Connected overlay used by bounded pre-click evidence probes; declared so capture code accesses it# directly instead of silently accepting a dynamically attached dependency.discovery_mcp_server: SkyvernOverlayMCPServer | None = Nonediff --git a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py b/skyvern/forge/sdk/copilot/runtime_authoring_repair.pyindex 691b866ec..89d9b4d42 100644--- a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py+++ b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py@@ -12,7 +12,6 @@ from skyvern.forge.sdk.copilot.challenge_evidence import (from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schemafrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policyfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORYfrom skyvern.forge.sdk.copilot.output_contracts import code_block_available_contracts_by_labelfrom skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_name@@ -336,8 +335,6 @@ def _result_has_terminal_or_ask_precedence(copilot_ctx: Any, data: dict[str, Anyreturn Trueif _error_text_requires_ask(data, result):return True- if data.get("active_run_terminal_evidence_detected") is True:- return Trueif data.get("skip_reason") == "workflow_credential_inputs_unbound":return Trueif data.get("failure_type") == "missing_credential_or_init":@@ -351,7 +348,6 @@ def _result_has_terminal_or_ask_precedence(copilot_ctx: Any, data: dict[str, Anycategory = entry.get("category")if category in {"UNRECOVERABLE_TOOL_ERROR",- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,"ANTI_BOT_DETECTION",} and is_carrier_backed_category_entry(entry):return Truediff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex 7b7042ad1..21553d5b4 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -23,9 +23,6 @@ from skyvern.forge.sdk.copilot.composition_evidence import ()from skyvern.forge.sdk.copilot.composition_evidence import workflow_target_url as workflow_target_urlfrom skyvern.forge.sdk.copilot.context import CopilotContext-from skyvern.forge.sdk.copilot.failure_tracking import (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY as ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,-)from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctxfrom skyvern.forge.sdk.copilot.output_utils import (_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY as _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,@@ -42,7 +39,6 @@ from skyvern.forge.sdk.copilot.workflow_yaml import apply_block_edit, delete_blofrom skyvern.utils.yaml_loader import safe_load_no_datesfrom ._shared import _COMPOSITION_STRIPPED_HTML_MAX_CHARS as _COMPOSITION_STRIPPED_HTML_MAX_CHARS-from ._shared import _CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLS as _CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLSfrom ._shared import _FAILED_BLOCK_STATUSES as _FAILED_BLOCK_STATUSESfrom ._shared import BLOCK_RUNNING_TOOLS as BLOCK_RUNNING_TOOLSfrom ._shared import COPILOT_FINAL_REPLY_RESERVE_SECONDS as COPILOT_FINAL_REPLY_RESERVE_SECONDS@@ -62,7 +58,6 @@ from .banned_blocks import _COPILOT_BANNED_BLOCK_TYPES as _COPILOT_BANNED_BLOCK_from .banned_blocks import _banned_block_reject_message as _banned_block_reject_messagefrom .banned_blocks import _detect_new_banned_blocks as _detect_new_banned_blocksfrom .banned_blocks import _record_banned_block_reject_span as _record_banned_block_reject_span-from .blockers import REPEATED_ACTION_STREAK_ABORT_AT as REPEATED_ACTION_STREAK_ABORT_ATfrom .blockers import _active_block_run_budget_seconds as _active_block_run_budget_secondsfrom .blockers import _analyze_run_blocks as _analyze_run_blocksfrom .blockers import _build_loop_blocker_signal as _build_loop_blocker_signal@@ -91,11 +86,6 @@ from .completion import _outcome_unverified_reason as _outcome_unverified_reasonfrom .completion import (_tool_visible_result_after_completion_verification,)-from .composition_capture import (- _active_run_terminal_evidence_needs_visual_fallback as _active_run_terminal_evidence_needs_visual_fallback,-)-from .composition_capture import _active_run_terminal_evidence_result as _active_run_terminal_evidence_result-from .composition_capture import _active_run_terminal_evidence_sample as _active_run_terminal_evidence_samplefrom .composition_capture import _capture_composition_evidence as _capture_composition_evidencefrom .composition_capture import (_composition_evidence_after_navigation_failure as _composition_evidence_after_navigation_failure,@@ -190,7 +180,6 @@ from .run_execution import _attach_action_traces as _attach_action_tracesfrom .run_execution import _cancel_run_task_if_not_final as _cancel_run_task_if_not_finalfrom .run_execution import _composition_anti_bot_reason as _composition_anti_bot_reasonfrom .run_execution import _detect_non_retriable_nav_error as _detect_non_retriable_nav_error-from .run_execution import _detect_probable_site_block_wall as _detect_probable_site_block_wallfrom .run_execution import (_diagnosis_repair_tool_error,_frontier_run_size_result,@@ -212,7 +201,6 @@ from .run_execution import (_verify_and_record_run_blocks_result,)from .run_execution import _watchdog_error_message as _watchdog_error_message-from .run_execution import _watchdog_exit_allows_terminal_promotion as _watchdog_exit_allows_terminal_promotionfrom .run_execution import _watchdog_user_failure_reason as _watchdog_user_failure_reasonfrom .scouting import _MAX_SCOUTED_INTERACTIONS as _MAX_SCOUTED_INTERACTIONSfrom .scouting import _capture_accessible_role_name as _capture_accessible_role_namediff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.pyindex f06970484..a8f722ed2 100644--- a/skyvern/forge/sdk/copilot/tools/_shared.py+++ b/skyvern/forge/sdk/copilot/tools/_shared.py@@ -243,15 +243,9 @@ def _has_meaningful_registered_output_payload(data: Mapping[str, Any]) -> bool:BLOCK_RUNNING_TOOLS = frozenset({"run_blocks_and_collect_debug", "update_and_run_blocks"})-_CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLS = BLOCK_RUNNING_TOOLS | {"fill_credential_field"}--WORKFLOW_MUTATION_TOOLS = frozenset({"update_workflow", "update_and_run_blocks"})-ANSWER_ONLY_CONTEXT_TOOLS = frozenset({"get_run_results"})--CREDENTIAL_METADATA_TOOLS = frozenset({"list_credentials"})diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.pyindex 0036e11c9..b2e2add64 100644--- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py@@ -224,6 +224,7 @@ def _code_only_browser_schema_guidance() -> list[str]:_code_only_browser_validation_guidance(),"Keep block outputs JSON-safe and include visible evidence text when extracting records, products, totals, confirmations, or identifiers.","Wait for the value the block returns, not for a URL or a navigation. A page reaches its final URL while it is still rendering, so a URL check passes before the value exists and a navigation wait fails on a page that has already arrived.",+ "Deciding which of two page states you are in costs no wait: wait once on `a.or_(b).first`, then branch with `is_visible()` or `count()`. A `wait_for` on a branch that may legitimately be absent spends its whole timeout proving the absence, and every repair attempt pays it again.","For saved credentials: bind the credential as a workflow parameter with workflow_parameter_type credential_id and the credential ID in default_value. At runtime the parameter key resolves to a credential object — read <key>.username and <key>.password, and use await <key>.otp() for authenticator, email, or SMS one-time codes. Never put literal secret values in code; scout credential fields with fill_credential_field.",]@@ -257,10 +258,13 @@ Runtime facts:- A `credential_id` workflow parameter resolves to a credential object with`<key>.username`, `<key>.password`, and `await <key>.otp()` for one-time codes; scout fields with`fill_credential_field`, never embed literal secrets.-- Credentialed login code must be idempotent. After `goto`, wait for either the- login form or an already-authenticated page anchor; only fill username/password- when the login fields are visible, and after submit wait for a logged-in page- anchor instead of relying only on `networkidle`.+- Credentialed login code must be idempotent. After `goto`, wait once on the two+ states together, then branch without waiting again:+ `await login_form.or_(authenticated_anchor).first.wait_for(state="visible")`,+ then `if await login_form.is_visible():` to fill username/password. Waiting on the+ login form by itself spends the entire timeout proving it is absent whenever the+ session is already authenticated. After submit, wait for a logged-in page anchor+ instead of relying only on `networkidle`.- After a credentialed login submit or navigation commit, call`await solve_captcha(page)` before waiting for post-login anchors. This helperowns any platform-managed verification challenge; do not locate or interactdiff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.pyindex d7e3e85b3..45aa33954 100644--- a/skyvern/forge/sdk/copilot/tools/blockers.py+++ b/skyvern/forge/sdk/copilot/tools/blockers.py@@ -26,16 +26,13 @@ from skyvern.forge.sdk.copilot.completion_verification import ()from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controlsfrom skyvern.forge.sdk.copilot.enforcement import (- TOTAL_TIMEOUT_SECONDS,synthesized_block_persistence_signal,terminal_challenge_blocker_signal_from_current_page_evidence,)from skyvern.forge.sdk.copilot.failure_tracking import (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,- ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,PER_TOOL_BUDGET_FAILURE_CATEGORY,)-from skyvern.forge.sdk.copilot.loop_detection import detect_failed_tool_step_loop_for_ctx, detect_tool_loop+from skyvern.forge.sdk.copilot.loop_detection import detect_failed_tool_step_loop_for_ctxfrom skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_OUTPUT_KEYSfrom skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_namefrom skyvern.forge.sdk.copilot.runtime import (@@ -46,11 +43,9 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunSfrom skyvern.schemas.workflows import BlockTypefrom ._shared import (- _CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLS,_DATA_PRODUCING_BLOCK_TYPES,_FAILED_BLOCK_STATUSES,BLOCK_RUNNING_TOOLS,- COPILOT_FINAL_REPLY_RESERVE_SECONDS,PAGE_INSPECTION_TOOLS,PAGE_SCHEMA_CONTEXT_TOOLS,PER_TOOL_CALL_BUDGET_SECONDS,@@ -122,54 +117,6 @@ def _trusted_post_drain_status(run: WorkflowRun | None) -> str | None:return None-def _active_run_terminal_evidence_detected(result: Mapping[str, object]) -> bool:- data_value = result.get("data")- data = data_value if isinstance(data_value, Mapping) else {}- return data.get("active_run_terminal_evidence_detected") is True---def _active_run_terminal_evidence_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:- evidence = getattr(ctx, "workflow_verification_evidence", None)- has_active_terminal_evidence = getattr(- ctx, "last_failure_category_top", None- ) == ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY or bool(- getattr(evidence, "active_run_terminal_evidence_detected", False)- )- if not has_active_terminal_evidence:- return None-- run_id = getattr(evidence, "active_run_terminal_evidence_workflow_run_id", None) or getattr(- ctx, "last_run_blocks_workflow_run_id", None- )- location = (- getattr(evidence, "page_title", None) or getattr(evidence, "current_url", None) or "the current browser page"- )- run_detail = f" Workflow run: {run_id}." if isinstance(run_id, str) and run_id else ""- agent_steering = (- "The prior active workflow run emitted typed ACTIVE_RUN_TERMINAL_EVIDENCE while the browser task "- f"was still running.{run_detail} The current page evidence already matched the user's terminal "- "browser-state criteria, but the reusable workflow is not verified end-to-end. "- f"Do not call {tool_name} again in this turn; reply with a partial-verification/blocker state that "- "says the requested browser state was observed, the active run was interrupted before overshoot, "- "and the workflow still needs a clean corrected run before it can be offered as tested."- )- user_facing = (- f"I reached the requested browser state on {location} and stopped before continuing, "- "but the reusable workflow still needs a clean verification run before it is ready."- )- return CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text=agent_steering,- user_facing_reason=user_facing,- recovery_hint="report_blocker_to_user",- cleared_by_tools=frozenset(),- preserves_workflow_draft=True,- renders_final_reply=True,- internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,- blocked_tool=tool_name,- )--def _per_tool_budget_problem_label_set(ctx: Any) -> set[str]:labels = getattr(ctx, "per_tool_budget_problem_block_labels", None)if not isinstance(labels, list):@@ -845,7 +792,6 @@ def _pending_reconciliation_no_input_signal(*, pending_run_id: str, blocked_tool# per each of the three preceding identical runs). Calibration note: the# repeated-frontier streak in failure_tracking.py uses STOP_AT=3 for the same# shape of escalation.-REPEATED_ACTION_STREAK_ABORT_AT = 3MAX_CHALLENGE_GATED_PROXY_RETRIES = 1_STRUCTURED_BLOCKER_KEY_TERMS: frozenset[str] = frozenset(@@ -1304,64 +1250,12 @@ def _code_output_has_registered_download_content(value: Any) -> bool:def _active_block_run_budget_seconds(ctx: AgentContext) -> int:+ """One outer wall clock bounds the turn; reserving a slice of it for a reply the model may not+ need shortens every run for a composition step that costs seconds."""remaining = _copilot_seconds_remaining(ctx)if remaining is None:return PER_TOOL_CALL_BUDGET_SECONDS- remaining_after_reply_reserve = remaining - COPILOT_FINAL_REPLY_RESERVE_SECONDS- return max(1, min(PER_TOOL_CALL_BUDGET_SECONDS, int(remaining_after_reply_reserve)))---def _late_block_running_call_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:- remaining = _copilot_seconds_remaining(ctx)- if remaining is None or remaining > COPILOT_FINAL_REPLY_RESERVE_SECONDS:- return None-- last_failed_workflow_yaml = getattr(ctx, "last_failed_workflow_yaml", None)- last_good_workflow_yaml = getattr(ctx, "last_good_workflow_yaml", None)- if (- isinstance(last_failed_workflow_yaml, str)- and last_failed_workflow_yaml- and isinstance(last_good_workflow_yaml, str)- and last_good_workflow_yaml- ):- agent_steering = (- f"Wall-clock budget too low to retry: about {int(max(0.0, remaining))}s remain of the "- f"{TOTAL_TIMEOUT_SECONDS}s session budget. A verified workflow exists from before the failure. "- "Do NOT call update_and_run_blocks or run_blocks_and_collect_debug again. REPLY now: summarize "- "what worked, name the block that failed, and tell the user they can keep the verified prefix or discard."- )- elif isinstance(last_failed_workflow_yaml, str) and last_failed_workflow_yaml:- agent_steering = (- f"Less than {COPILOT_FINAL_REPLY_RESERVE_SECONDS} seconds remain in this Copilot turn "- "after the previous workflow run failed. Do NOT retry block-running tools. Use only existing "- "run evidence and quick browser inspection tools such as get_run_results, evaluate, or "- "get_browser_screenshot if one more read is needed. If the current page contains the requested "- "answer, answer from that observed page evidence. If evidence is incomplete, report exactly "- "which browser state was verified and which requested data remains unverified. Never repeat "- "this tool-error text as the user-facing answer."- )- else:- agent_steering = (- f"Less than {COPILOT_FINAL_REPLY_RESERVE_SECONDS} seconds remain in this Copilot turn. "- "Do NOT start another block-running tool call; reply to the user with the workflow draft and "- "progress gathered so far, and make clear which parts have not been verified end-to-end."- )-- user_facing = "I'm running out of time on this turn. I'll wrap up with what I have so far."- return CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text=agent_steering,- user_facing_reason=user_facing,- recovery_hint="stop",- cleared_by_tools=frozenset(),- # The "wrap up with what we have" semantic means the draft saved- # earlier in the turn should still surface — only the chat reply- # is overridden by the renderer.- preserves_workflow_draft=True,- renders_final_reply=False,- internal_reason_code="tool_error_late_block_running",- blocked_tool=tool_name,- )+ return max(1, min(PER_TOOL_CALL_BUDGET_SECONDS, int(remaining)))def _allows_post_run_current_page_inspection_budget_bypass(ctx: AgentContext, *, use_current_page: bool) -> bool:@@ -1507,18 +1401,15 @@ def _challenge_gated_anti_bot_rerun_signal(LOOP_PLANE_REFUSAL_REASON_CODES: frozenset[str] = frozenset({"loop_detected_credential_or_parameter_misconfig",- "loop_detected_repeated_failed_step","loop_detected_consecutive_same_tool","loop_detected_generic","loop_detected_no_forward_progress_interaction","loop_detected_discovery_exhausted_no_entry_url","code_authoring_guardrail_churn","credential_priority_authoring_churn",- "tool_error_active_run_terminal_evidence","tool_error_terminal_challenge_blocker","tool_error_synthesized_block_persistence_required","tool_error_challenge_gated_submit_disabled",- "tool_error_late_block_running","tool_error_non_retriable_nav","tool_error_pending_reconciliation_no_input","tool_error_pending_reconciliation_requires_input",@@ -1527,7 +1418,6 @@ LOOP_PLANE_REFUSAL_REASON_CODES: frozenset[str] = frozenset("tool_error_post_budget_upstream_replay","tool_error_post_budget_challenge_result_evidence","tool_error_post_budget_challenge_blocker",- "tool_error_repeated_action_abort","runtime_self_heal_native_tool_blocked",})@@ -1570,35 +1460,6 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any_build_loop_blocker_signal(detected, tool_name=tool_name, evidence=loop_blocker_evidence_from_ctx(ctx)),)- # Consecutive same-name guard: false-positives on the intended iterative- # build (one new block per update_and_run_blocks). Block-running tools- # rely on the progress-aware checks below instead. fill_credential_field is- # exempt because a username+password+TOTP form legitimately needs three- # consecutive calls; its failed-step guard above stays argument-aware.- tracker = getattr(ctx, "consecutive_tool_tracker", None)- if (- isinstance(tracker, list)- and tool_name not in _CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLS- and not output_contract_owns_turn- ):- detected = detect_tool_loop(tracker, tool_name, arguments)- if detected is not None:- return _emit_loop_plane_refusal(- ctx,- _build_loop_blocker_signal(detected, tool_name=tool_name, evidence=loop_blocker_evidence_from_ctx(ctx)),- )-- if tool_name == "update_workflow" or tool_name in BLOCK_RUNNING_TOOLS:- active_terminal_signal = _active_run_terminal_evidence_signal(ctx, tool_name)- if active_terminal_signal is not None:- return _emit_loop_plane_refusal(ctx, active_terminal_signal)-- # Hard-abort when the agent has re-fired the same action sequence against- # the page N times without intervening success. This is the signal that- # the form is blocked (captcha / anti-bot / error banner the agent isn't- # detecting) and further attempts will just burn the tool timeout. Scoped- # to block-running tools so planning/metadata tools (update_workflow,- # list_credentials, get_run_results) stay unaffected.if tool_name in BLOCK_RUNNING_TOOLS:# Reconciliation guard: the previous block-running tool call exited# without a trustworthy terminal status for its workflow run (the@@ -1644,33 +1505,6 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Anyif challenge_signal is not None:return _emit_loop_plane_refusal(ctx, challenge_signal)- streak_raw = getattr(ctx, "repeated_action_fingerprint_streak_count", 0)- streak = streak_raw if isinstance(streak_raw, int) else 0- if streak >= REPEATED_ACTION_STREAK_ABORT_AT:- agent_steering = (- f"Repeated-action abort: the last {streak} runs fired the same "- "action sequence against the page without making progress. "- "The site is likely blocked by a captcha, popup, anti-bot "- "challenge, or hidden validation error that the agent is not "- "detecting. Do NOT retry this tool — conclude the workflow is "- "not automatable as-is and report back to the user."- )- user_facing = (- "I tried the same actions a few times without making progress. The site looks blocked. I'll stop here."- )- return _emit_loop_plane_refusal(- ctx,- CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text=agent_steering,- user_facing_reason=user_facing,- recovery_hint="stop",- cleared_by_tools=frozenset(),- internal_reason_code="tool_error_repeated_action_abort",- blocked_tool=tool_name,- ),- )-# Within-turn fail-fast for permanent navigation errors (DNS / cert /# SSL / invalid URL). The enforcement-loop stop nudge only runs# BETWEEN agent turns, so without this check the LLM is free to make@@ -1701,9 +1535,6 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any),)- late_signal = _late_block_running_call_signal(ctx, tool_name)- if late_signal is not None:- return _emit_loop_plane_refusal(ctx, late_signal)return Nonediff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 30f56e78e..3831e6645 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -90,7 +90,6 @@ from ._shared import (_workflow_output_parameter_payloads,)from .blockers import (- _active_run_terminal_evidence_detected,_analyze_run_blocks,_artifact_challenge_flag_from_result,_run_blocks_structured_blocker_message,@@ -847,7 +846,6 @@ def _completion_evidence_payload(output: Any) -> Any:_ARTIFACT_HEALTH_EXCLUDED_CATEGORIES = frozenset({- "ACTIVE_RUN_TERMINAL_EVIDENCE","ANTI_BOT_DETECTION","AUTH_FAILURE","BROWSER_ERROR",@@ -901,8 +899,6 @@ def _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[str"""if bool(result.get("ok", False)):return False- if _active_run_terminal_evidence_detected(result):- return Falseif _run_blocks_structured_blocker_message(result, copilot_ctx):return Falsedata = result.get("data")@@ -1914,13 +1910,6 @@ def _tool_visible_result_after_completion_verification(if outcome_unverified_reason is None:return resultdata = result.get("data")- run_id = data.get("workflow_run_id") if isinstance(data, dict) else None- if (- copilot_ctx.delivered_unverified_terminal- and isinstance(run_id, str)- and run_id == copilot_ctx.delivered_unverified_workflow_run_id- ):- return resultif not _outcome_failure_warrants_repair(copilot_ctx, completion_verification):return resultdiff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.pyindex 8df90c980..a2d0b3820 100644--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py@@ -3,7 +3,6 @@ from __future__ import annotationsimport asyncioimport base64import json-from dataclasses import dataclassfrom typing import Anyfrom urllib.parse import urlparse@@ -11,11 +10,6 @@ import structlogfrom skyvern.forge.sdk.copilot.build_test_outcome import maybe_satisfy_recorded_outcome_grounding_requirementfrom skyvern.forge.sdk.copilot.challenge_evidence import challenge_evidence_unsettled-from skyvern.forge.sdk.copilot.completion_verification import (- CompletionVerificationResult,- RunEvidenceSnapshot,- evaluate_completion_criteria,-)from skyvern.forge.sdk.copilot.composition_browser_expressions import (COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION as _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION,)@@ -27,13 +21,8 @@ from skyvern.forge.sdk.copilot.composition_evidence import (parse_composition_html,)from skyvern.forge.sdk.copilot.context import CopilotContext-from skyvern.forge.sdk.copilot.failure_tracking import (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,- ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,-)from skyvern.forge.sdk.copilot.llm_config import resolve_fast_copilot_handlerfrom skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx-from skyvern.forge.sdk.copilot.request_policy import completion_criterion_requires_active_run_terminal_monitorfrom skyvern.forge.sdk.copilot.runtime_authoring_repair import (finalize_runtime_authoring_repair_context_from_page_observation,)@@ -50,13 +39,10 @@ from ._shared import (_discovery_navigate,_fallback_page_info,_same_page_ignoring_fragment,- _valid_runtime_anchor_url,_workflow_verification_evidence,)from .blockers import _allows_post_run_current_page_inspection_budget_bypassfrom .completion import (- _completion_verification_criteria,- _completion_verification_handler,_maybe_run_completion_verification_from_page_observation,)from .discovery import _resolve_discovery_entry_url@@ -72,160 +58,6 @@ from .scouting import (LOG = structlog.get_logger()-@dataclass(frozen=True)-class ActiveRunTerminalEvidenceSample:- current_url: str | None- page_title: str | None- page_evidence: dict[str, Any]- completion_verification: CompletionVerificationResult- sample_index: int---async def _active_run_terminal_monitor_enabled(copilot_ctx: Any) -> bool:- if not getattr(copilot_ctx, "browser_session_id", None):- return False- if not getattr(copilot_ctx, "discovery_mcp_server", None):- return False- criteria = _completion_verification_criteria(copilot_ctx)- if not criteria:- return False- if not any(completion_criterion_requires_active_run_terminal_monitor(criterion) for criterion in criteria):- return False- return await _completion_verification_handler(copilot_ctx) is not None---def _active_run_terminal_evidence_needs_visual_fallback(evidence: dict[str, Any]) -> bool:- if page_evidence_needs_visual_fallback(evidence):- return True- return evidence.get("screenshot_used") is not True---async def _active_run_terminal_evidence_sample(- copilot_ctx: Any,- *,- workflow_run_id: str,- labels_to_execute: list[str],- sample_index: int,-) -> ActiveRunTerminalEvidenceSample | None:- criteria = _completion_verification_criteria(copilot_ctx)- if not criteria:- return None- handler = await _completion_verification_handler(copilot_ctx)- if handler is None:- return None-- current_url_raw, page_title_raw = await _fallback_page_info(copilot_ctx)- current_url = _valid_runtime_anchor_url(current_url_raw)- if current_url is None:- return None-- evidence, html_error = await _capture_composition_evidence(- copilot_ctx,- inspected_url=current_url,- current_url=current_url,- active_run_terminal_sample=True,- # The executor is still driving this page; solving here would act on a run this turn- # does not own, and spend the turn's attempts on a sample.- solve_challenges=False,- )- if html_error is not None or evidence is None:- LOG.info(- "copilot active-run terminal evidence sample skipped",- workflow_run_id=workflow_run_id,- sample_index=sample_index,- html_error=html_error,- )- return None-- page_title = evidence.get("page_title")- if not isinstance(page_title, str) or not page_title.strip():- page_title = page_title_raw if isinstance(page_title_raw, str) and page_title_raw.strip() else None- evidence = {- **evidence,- "workflow_run_id": workflow_run_id,- "observed_during_active_workflow_run": True,- }- snapshot = RunEvidenceSnapshot(- workflow_run_id=workflow_run_id,- current_url=current_url,- page_title=page_title,- executed_block_labels=list(labels_to_execute),- page_evidence=evidence,- )- if not snapshot.has_evidence():- return None-- result = await evaluate_completion_criteria(criteria, snapshot, handler)- LOG.info(- "copilot active-run terminal evidence sample",- workflow_run_id=workflow_run_id,- sample_index=sample_index,- completion_verification_status=result.status,- completion_verification_fully_satisfied=result.is_fully_satisfied(),- )- if result.status != "evaluated" or not result.is_fully_satisfied():- return None- copilot_ctx.composition_page_evidence = evidence- return ActiveRunTerminalEvidenceSample(- current_url=current_url,- page_title=page_title,- page_evidence=evidence,- completion_verification=result,- sample_index=sample_index,- )---def _active_run_terminal_evidence_result(- *,- workflow_run_id: str,- run_status: str | None,- sample: Any,- requested_block_labels: list[str],- executed_block_labels: list[str],- current_url: str | None = None,- page_title: str | None = None,-) -> dict[str, Any]:- observed_url = current_url or getattr(sample, "current_url", None)- observed_title = page_title or getattr(sample, "page_title", None)- completion = getattr(sample, "completion_verification", None)- completion_trace = completion.to_trace_data() if isinstance(completion, CompletionVerificationResult) else {}- reason = (- "The active run reached the requested browser state while the workflow was still running, "- "so Copilot interrupted it before further browser actions could overshoot that state. "- "The current page evidence is not a durable full-workflow verification; inspect the run boundary, "- "repair the workflow if needed, and verify the corrected workflow run."- )- return {- "ok": False,- "error": reason,- "data": {- "workflow_run_id": workflow_run_id,- "overall_status": run_status,- "failure_reason": reason,- "requested_block_labels": list(requested_block_labels),- "executed_block_labels": list(executed_block_labels),- "current_url": observed_url,- "page_title": observed_title,- "active_run_terminal_evidence_detected": True,- "active_run_terminal_evidence_reason_code": ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,- "active_run_terminal_evidence_sample_index": getattr(sample, "sample_index", None),- "full_workflow_verified": False,- "current_page_evidence": getattr(sample, "page_evidence", None),- "active_run_terminal_completion_verification": completion_trace,- "failure_categories": [- {- "category": ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,- "confidence_float": 1.0,- "reasoning": (- "Bounded current-page evidence matched the request-policy completion criteria "- "while the workflow run was still active; the run was interrupted for diagnosis/repair."- ),- }- ],- },- }--_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS = 10.0_COMPOSITION_VISUAL_SUMMARY_PROMPT_NAME = "workflow-copilot-page-evidence-vision"@@ -609,7 +441,6 @@ async def _capture_composition_evidence(*,inspected_url: str,current_url: str,- active_run_terminal_sample: bool = False,solve_challenges: bool = True,) -> tuple[dict[str, Any] | None, str | None]:"""Parse composition evidence (cheap extractor first, get_html fallback); html_error is set only on a failed HTML read."""@@ -657,7 +488,6 @@ async def _capture_composition_evidence(evidence = await _augment_composition_evidence_with_computed_obstruction_candidates(copilot_ctx, evidence)if evidence is not None and (page_evidence_needs_visual_fallback(evidence)- or (active_run_terminal_sample and _active_run_terminal_evidence_needs_visual_fallback(evidence))or (evidence.get("schema_empty_page") is True and not has_bounded_page_schema(evidence))):evidence = await _augment_composition_evidence_with_visual_fallback(copilot_ctx, evidence)diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.pyindex 663c66be7..2f5690a77 100644--- a/skyvern/forge/sdk/copilot/tools/guardrails.py+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py@@ -22,7 +22,6 @@ from skyvern.forge.sdk.copilot.credential_literal_rebind import (from skyvern.forge.sdk.copilot.enforcement import _record_code_authoring_guardrail_rejectfrom skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctxfrom skyvern.forge.sdk.copilot.output_policy import (- OutputPolicyReason,OutputPolicyVerdict,demote_author_time_steer_reasons,evaluate_output_policy,@@ -47,7 +46,6 @@ from skyvern.forge.sdk.workflow.models.parameter import (from skyvern.forge.sdk.workflow.models.workflow import Workflowfrom ._shared import (- ANSWER_ONLY_CONTEXT_TOOLS,BLOCK_RUNNING_TOOLS,CREDENTIAL_METADATA_TOOLS,PAGE_SCHEMA_CONTEXT_TOOLS,@@ -91,11 +89,6 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Toolworkflow_yaml=effective_yaml,tool_arguments=tool_arguments or raw_arguments,)- # The credential here is the user's own, bound for the site they named, inside their own workflow,- # so a raw value on an authoring argument is not grounds to kill the turn. Enforcement for it lives- # at the log seam (`redact_registered_secrets`) and the persistence seam; final-output surfaces- # still hard-block on RAW_SECRET_LEAK via `_FINAL_OUTPUT_HARD_BLOCK_REASONS`.- verdict.remove(OutputPolicyReason.RAW_SECRET_LEAK)steered_reasons = demote_author_time_steer_reasons(verdict)trace_data = output_policy_verdict_to_trace_data(verdict,@@ -436,27 +429,11 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBloblocks_credential_metadata = tool_name in CREDENTIAL_METADATA_TOOLS and not (authority.may_update_workflow or authority.may_run_blocks)- # Two paths grant read access to ANSWER_ONLY_CONTEXT_TOOLS, both excluded for DOCS_ANSWER/REFUSE/CLARIFY:- # (1) authority.may_read_run_context — classifier-derived (DIAGNOSE turns)- # (2) pending_reconciliation_run_id — within-turn override anchored to the run-blocks watchdog+ # Reading a run the copilot itself just started is how it finds out what happened; gating that on a+ # classifier's reading of the user's message leaves it retrying blind.may_read_run_context = authority.may_read_run_context and intent.mode not in READ_CONTEXT_DENIED_MODES- blocks_context_read = tool_name in ANSWER_ONLY_CONTEXT_TOOLS and not may_read_run_context-+ blocks_context_read = Falsewithin_turn_read_override = False- if blocks_context_read and intent.mode not in READ_CONTEXT_DENIED_MODES:- pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)- if isinstance(pending_run_id, str) and pending_run_id:- blocks_context_read = False- within_turn_read_override = True- else:- same_turn_run_id = getattr(ctx, "last_successful_run_blocks_workflow_run_id", None) or getattr(- ctx,- "last_run_blocks_workflow_run_id",- None,- )- if isinstance(same_turn_run_id, str) and same_turn_run_id:- blocks_context_read = False- within_turn_read_override = Trueif blocks_run and not blocks_update and _request_policy_allows_update_and_skip_run(ctx, tool_name):return None@@ -520,18 +497,6 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlorecovery_hint="retry_with_different_tool",cleared_by_tools=frozenset({"update_workflow"}),)- if blocks_context_read and not blocks_update and not blocks_run:- return _build_turn_intent_signal(- tool_name=tool_name,- classifier_mode=intent.mode.value,- reason_code=reason_code,- agent_steering_text=(- "Run context may not be read for the latest user message. Answer using the context already "- f"provided.{detail}"- ),- user_facing_reason="I'll answer with the information I already have.",- recovery_hint="report_blocker_to_user",- )if may_read_run_context:# Mutually exclusive with the blocks_context_read branch above, which requires may_read_run_context False.# The allow-path returned None above, so any reaching tool is denied; deny it without terminating, keepingdiff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex e28c57fde..9e29080be 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.build_phase import (advance_to_composing,)from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_code_block+from skyvern.forge.sdk.copilot.completion_criteria_store import requested_output_pathsfrom skyvern.forge.sdk.copilot.composition_browser_expressions import scout_control_state_expressionfrom skyvern.forge.sdk.copilot.config import (BlockAuthoringPolicy,@@ -22,7 +23,9 @@ from skyvern.forge.sdk.copilot.config import ()from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay-from skyvern.forge.sdk.copilot.runtime import AgentContext+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy+from skyvern.forge.sdk.copilot.request_slots import is_canonical_request_slot_path+from skyvern.forge.sdk.copilot.runtime import AgentContext, ScoutedInteractionfrom skyvern.forge.sdk.copilot.secret_scrub import registered_scrub_valuesfrom skyvern.forge.sdk.copilot.typed_value_policy import safe_typed_default_value, should_reject_type_text_value@@ -267,12 +270,18 @@ async def _evaluate_pre_hook(params: dict[str, Any],ctx: AgentContext,) -> dict[str, Any] | None:- expr = params.get("expression", "").lower()+ # Cleared up front so an early reject cannot leave a prior evaluate's expression for this+ # call's post-hook to consume.+ ctx.pending_scout_read_expression = None+ raw_expression = params.get("expression")+ expr = raw_expression.lower() if isinstance(raw_expression, str) else ""if ".click()" in expr or ".click(" in expr:return {"ok": False,"error": "Do not use evaluate to click elements. Use the 'click' tool with a CSS selector instead.",}+ if isinstance(raw_expression, str) and raw_expression.strip():+ ctx.pending_scout_read_expression = raw_expressionreturn None@@ -870,6 +879,37 @@ async def _type_text_post_hook(return result+def _record_scouted_read(ctx: AgentContext, *, expression: str, data: dict[str, Any], url: str) -> None:+ """Keep a read the scout proved on the live page so authoring replays it instead of guessing a+ locator for a value it has already seen. A structured result is kept as readily as a scalar: the+ shape is what a later binding needs. Naming never gates keeping the evidence."""+ result = data.get("result")+ if not expression or result is None or result == "":+ return+ if isinstance(result, (list, dict)) and not result:+ return+ output_path = "output.scouted_read"+ policy = ctx.request_policy+ if isinstance(policy, RequestPolicy):+ requested = requested_output_paths(policy.graded_completion_criteria())+ named = sorted(path for path in requested if not is_canonical_request_slot_path(path))+ # A rekeyed requested output carries a digest instead of a word, which still keys the+ # producer to the criterion it has to satisfy. Reading into an anonymous path instead leaves+ # completion verification with no evidence for an outcome the scout already demonstrated.+ candidates = named or sorted(requested)+ if len(candidates) == 1:+ output_path = candidates[0]+ interaction: ScoutedInteraction = {+ "tool_name": "read_value",+ "read_expression": expression,+ "read_output_path": output_path,+ "read_result_shape": type(result).__name__,+ }+ if url:+ interaction["source_url"] = url+ ctx.scout_trajectory.append(interaction)++async def _evaluate_post_hook(result: dict[str, Any],raw: dict[str, Any],@@ -902,6 +942,18 @@ async def _evaluate_post_hook(if observation_step is not None:result["observation_step"] = observation_stepdata["observation_step"] = observation_step+ # The MCP response never echoes the expression; the pre-hook stashed it from the invocation.+ read_expression = ctx.pending_scout_read_expression or ""+ ctx.pending_scout_read_expression = None+ trajectory_len_before = len(ctx.scout_trajectory)+ _record_scouted_read(ctx, expression=read_expression, data=data, url=url)+ if len(ctx.scout_trajectory) > trajectory_len_before:+ LOG.info(+ "copilot_scouted_read_recorded",+ read_result_shape=ctx.scout_trajectory[-1].get("read_result_shape"),+ read_output_path=ctx.scout_trajectory[-1].get("read_output_path"),+ trajectory_len=len(ctx.scout_trajectory),+ )if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and _code_only_has_target_page_evidence(data):diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 8d20472df..bfff8fb15 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -10,7 +10,7 @@ from collections import defaultdictfrom collections.abc import Mapping, Sequencefrom dataclasses import dataclass, replacefrom datetime import datetime, timezone-from typing import Any, Literal, cast+from typing import Any, Literalfrom urllib.parse import urlparseimport structlog@@ -32,7 +32,6 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (latest_recorded_build_test_outcome_repeated,record_build_test_outcome,recorded_outcome_from_run_blocks_result,- registered_output_payload_binds_output_path,)from skyvern.forge.sdk.copilot.challenge_evidence import (ChallengeEvidenceSource,@@ -54,13 +53,7 @@ from skyvern.forge.sdk.copilot.completion_output_grounding import page_evidence_from skyvern.forge.sdk.copilot.completion_verification import (CompletionVerificationResult,CriterionVerdict,- DeliveredUnverifiedTerminalState,- degraded_contract_delivered_unverified_terminal_state,- floor_rekeyed_deliverable_credit,- floor_rekeyed_emission_lane_fields,- floor_rekeyed_emission_withhold,only_structural_requested_output_abstentions,- zero_requested_output_criteria_credit,)from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema, parse_composition_htmlfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy@@ -76,7 +69,6 @@ from skyvern.forge.sdk.copilot.enforcement import (reset_no_progress_interaction_count,)from skyvern.forge.sdk.copilot.failure_tracking import (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,PER_TOOL_BUDGET_FAILURE_CATEGORY,compute_action_sequence_fingerprint,made_newly_verified_progress,@@ -124,11 +116,11 @@ from skyvern.forge.sdk.copilot.runtime_authoring_repair import (from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.turn_halt import (- stash_delivered_unverified_turn_halt,stash_turn_halt_from_blocker_signal,)from skyvern.forge.sdk.copilot.typed_value_policy import should_reject_type_text_valuefrom skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml+from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.executor.factory import AsyncExecutorFactoryfrom skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotRunOutcomeUpdate, WorkflowCopilotStreamMessageTypefrom skyvern.forge.sdk.settings_manager import SettingsManager@@ -137,6 +129,7 @@ from skyvern.forge.sdk.workflow.models.block import CodeBlockfrom skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterfrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatusfrom skyvern.schemas.workflows import BlockType+from skyvern.utils.files import initialize_skyvern_state_filefrom skyvern.webeye.navigation import is_skip_inner_retry_errorfrom skyvern.webeye.utils.page import SkyvernFrame@@ -146,7 +139,6 @@ from ._shared import (_completed_run_block_labels,_failed_run_block_labels,_fallback_page_info,- _has_meaningful_registered_output_payload,_registered_output_parameter_payloads,_registered_output_payload_view,_unverified_current_workflow_labels,@@ -156,8 +148,6 @@ from ._shared import (from .banned_blocks import _copilot_block_authoring_policyfrom .blockers import (_active_block_run_budget_seconds,- _active_run_terminal_evidence_detected,- _active_run_terminal_evidence_signal,_analyze_run_blocks,_artifact_challenge_flag_from_result,_looks_like_anti_bot_blocker,@@ -175,10 +165,6 @@ from .completion import (_record_adjudication_on_turn_state,)from .composition_capture import (- ActiveRunTerminalEvidenceSample,- _active_run_terminal_evidence_result,- _active_run_terminal_evidence_sample,- _active_run_terminal_monitor_enabled,_capture_composition_evidence,store_post_run_page_evidence,)@@ -213,9 +199,6 @@ class _RunIdUnset:# Distinguishes omitted run id (read ctx) from explicit None (no current run)._RUN_ID_UNSET = _RunIdUnset()-_ACTIVE_RUN_TERMINAL_MONITOR_INITIAL_DELAY_SECONDS = 30.0-_ACTIVE_RUN_TERMINAL_MONITOR_INTERVAL_SECONDS = 30.0-_ACTIVE_RUN_TERMINAL_MONITOR_MAX_SAMPLES = 8_MAX_REGISTERED_ARTIFACTS = 3_MAX_REGISTERED_ARTIFACT_BYTES = 5 * 1024 * 1024@@ -242,6 +225,8 @@ RUN_BLOCKS_STAGNATION_WINDOW_SECONDS = 90# DB load (240 polls worst case at the safety ceiling).RUN_BLOCKS_POLL_INTERVAL_SECONDS = 5.0+COPILOT_SANDBOX_UNAVAILABLE_ERROR = "Sandboxed worker is unavailable; execution was not started."+# Detached cleanup tasks held here so the garbage collector does not drop them# while they still have work to do, and so the "task exception was never# retrieved" warning cannot fire — each task adds a done-callback that logs@@ -390,10 +375,7 @@ def _maybe_clear_reconciliation_flag(copilot_ctx: Any, result: Any) -> None:# ``last_failure_category_top`` reflects the prior block-running tool's outcome —# only ``_record_run_blocks_result`` writes it, and the reconciliation guard# prevents another block-running call from clobbering it before this read.- internal_watchdog_cancel_category = getattr(copilot_ctx, "last_failure_category_top", None) in {- PER_TOOL_BUDGET_FAILURE_CATEGORY,- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,- }+ internal_watchdog_cancel_category = copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORYif is_trusted_final or internal_watchdog_cancel_category:copilot_ctx.pending_reconciliation_run_id = Nonecopilot_ctx.pending_reconciliation_requires_user_input = False@@ -435,6 +417,10 @@ def _maybe_clear_reconciliation_flag(copilot_ctx: Any, result: Any) -> None:)+def _copilot_sandbox_unavailable_result() -> dict[str, Any]:+ return {"ok": False, "error": COPILOT_SANDBOX_UNAVAILABLE_ERROR}++def _mark_pending_reconciliation_run(copilot_ctx: Any, workflow_run_id: str) -> None:copilot_ctx.pending_reconciliation_run_id = workflow_run_idcopilot_ctx.pending_reconciliation_requires_user_input = False@@ -563,14 +549,9 @@ WatchdogExitReason = Literal["ceiling","per_tool_budget","task_exit_unfinalized",- "active_run_terminal_evidence",]-def _watchdog_exit_allows_terminal_promotion(exit_reason: WatchdogExitReason | None) -> bool:- return exit_reason != "active_run_terminal_evidence"--# Block types that legitimately execute long silent periods: one DB write on# entry, work done without intermediate writes (sleep / LLM call / await human# input / browser download wait), one write on finish. The watchdog can't@@ -583,6 +564,9 @@ _QUIET_BLOCK_TYPES: frozenset[str] = frozenset(BlockType.HUMAN_INTERACTION.value,BlockType.FILE_DOWNLOAD.value,BlockType.FILE_UPLOAD.value,+ # A code block writes its row on entry and exit and nothing in between, so a long+ # login or wait inside one is indistinguishable from a frozen run.+ BlockType.CODE.value,})@@ -716,21 +700,6 @@ async def _watchdog_error_message(if current_url:message += f" Browser was on: {current_url}"return message- elif exit_reason == "active_run_terminal_evidence":- message = (- "The active run was interrupted because bounded current-page evidence matched the requested "- "browser terminal state while the workflow run was still in progress.\n"- f"Run ID: {workflow_run_id}.\n"- "This is NOT full workflow verification: the requested browser state was observed, but the durable "- "workflow chain still needs diagnosis/repair and a clean verification run. Next step: call "- "get_run_results with this workflow_run_id to inspect the active run boundary, preserve the observed "- "current-page evidence, and update only the block(s) that overshot or kept running after the state "- "was reached. Do NOT report end-to-end success unless a corrected workflow run verifies cleanly."- )- current_url, _ = ("", "") if dispatch_to_worker else await _fallback_page_info(ctx)- if current_url:- message += f" Browser was on: {current_url}"- return messageelif exit_reason == "ceiling":body = (f"The run exceeded the {budget_seconds}s absolute ceiling "@@ -765,11 +734,6 @@ def _watchdog_user_failure_reason(body = f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s."elif exit_reason == "per_tool_budget":body = f"The run exceeded the {budget_seconds}s per-tool-call budget while still making progress."- elif exit_reason == "active_run_terminal_evidence":- body = (- "The active run reached the requested browser state before the workflow finished, "- "so it was interrupted for diagnosis/repair. Full workflow verification is still required."- )elif exit_reason == "ceiling":body = f"The run exceeded the {budget_seconds}s absolute ceiling while still showing progress."else:@@ -1568,13 +1532,17 @@ async def _run_blocks_and_collect_debug(return {"ok": False, "error": "Organization not found"}organization = Organization.model_validate(org)- # Copilot-only gate, default OFF (flag-off is byte-for-byte the inline path below). When ON,- # the block test run is persisted as a draft and dispatched to the -ui worker tier instead- # of running inline on the API service.+ # Retain the per-org rollout, but never use its negative decision as permission to execute+ # workflow code in the API process.dispatch_to_worker = await app.AGENT_FUNCTION.should_dispatch_copilot_block_run_to_worker(organization_id=ctx.organization_id,workflow_permanent_id=ctx.workflow_permanent_id,)+ # Compared against the literal True so anything other than an explicit opt-in — including a+ # test double that auto-mocks the hook into a truthy object — still fails closed.+ if not dispatch_to_worker and app.AGENT_FUNCTION.allow_copilot_inline_code_execution() is not True:+ return _copilot_sandbox_unavailable_result()+runtime_workflow = _workflow_with_runtime_block_goal_context(workflow, ctx)runtime_workflow, runtime_frontier_anchor_url = _workflow_with_runtime_frontier_anchor(runtime_workflow,@@ -1689,17 +1657,16 @@ async def _run_blocks_and_collect_debug(# Snapshot version persisted for a dispatched run; the run is created against its exact# workflow_id so the worker resolves the wrapped definition via run.workflow_id, and it is# soft-deleted once the run resolves so it never lingers as the latest-by-permanent-id pointer- # for edit/view. None for the inline path.+ # for edit/view. None for the inline dev path.dispatch_draft_workflow_id: str | None = None# The persisted dispatch version (its own regenerated parameter ids) used for post-run output# mapping on the dispatch path; runtime_workflow / ctx.staged_workflow is left unmutated.dispatch_workflow: Workflow | None = Noneif dispatch_to_worker:- # Persist the wrapped runtime workflow as a real new version (with its own- # parameter / output-parameter rows) through the normal create machinery. The run is- # then created against this version so the worker resolves it by run.workflow_id and- # registers block outputs from the version's own rows. On any persistence failure, fall- # back to the inline path for this run so flag-on degrades safely.+ # Persist the wrapped runtime workflow as a real new version (with its own parameter /+ # output-parameter rows) through the normal create machinery. The run is then created+ # against this version so the worker resolves it by run.workflow_id and registers block+ # outputs from the version's own rows.try:dispatch_workflow = await app.WORKFLOW_SERVICE.create_copilot_dispatch_draft_version(runtime_workflow=runtime_workflow,@@ -1708,15 +1675,15 @@ async def _run_blocks_and_collect_debug(dispatch_draft_workflow_id = dispatch_workflow.workflow_idexcept Exception:LOG.warning(- "Failed to persist copilot dispatch version; falling back to inline run",+ "Failed to persist copilot dispatch version; blocking execution",workflow_permanent_id=ctx.workflow_permanent_id,exc_info=True,)- dispatch_to_worker = False- dispatch_workflow = None+ return _copilot_sandbox_unavailable_result()- # run_task is the in-process inline execution task. For dispatched runs it stays None: the- # worker owns execution and the watchdog observes purely via DB polling.+ # run_task is the in-process inline execution task, only ever set on the dev-only inline path.+ # For dispatched runs it stays None: the worker owns execution and the watchdog observes purely+ # via DB polling.run_task: asyncio.Task | None = Nonetry:workflow_run = await workflow_service.prepare_workflow(@@ -1724,9 +1691,9 @@ async def _run_blocks_and_collect_debug(organization=organization,workflow_request=workflow_request,template=False,- # Dispatched runs pin the exact persisted snapshot version by workflow_id (the- # (permanent_id, version) index is non-unique); inline runs use the latest version- # and pass the runtime workflow in-process via workflow_override.+ # Dispatched runs pin the exact persisted snapshot version by workflow_id because the+ # (permanent_id, version) index is non-unique; inline runs use the latest version and+ # pass the runtime workflow in-process via workflow_override.resolved_workflow_id=dispatch_draft_workflow_id,max_steps=None,request_id=None,@@ -1740,8 +1707,8 @@ async def _run_blocks_and_collect_debug(if dispatch_to_worker:# Submit through the cloud executor (Temporal). The run was created against the# snapshot version, so the worker resolves the exact wrapped definition via- # run.workflow_id — no workflow_override over the wire. block_labels/block_outputs and- # the shared browser session reproduce the frontier re-run on the worker.+ # run.workflow_id — no workflow_override crosses the wire. block_labels/block_outputs+ # and the shared browser session reproduce the frontier re-run on the worker.await AsyncExecutorFactory.get_executor().execute_workflow(request=None,background_tasks=None,@@ -1756,7 +1723,19 @@ async def _run_blocks_and_collect_debug(block_outputs=block_outputs_to_seed or None,)else:- from skyvern.utils.files import initialize_skyvern_state_file+ LOG.error(+ "UNSANDBOXED: executing copilot workflow code in the API process because "+ "COPILOT_ALLOW_INLINE_CODE_EXECUTION is enabled. This is a local-development path "+ "with no sandbox isolation; the run below is NOT a sandboxed run.",+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_permanent_id=ctx.workflow_permanent_id,+ organization_id=ctx.organization_id,+ )+ # prepare_workflow replaced the ambient context with this run's own, so the marker is+ # scoped to this run and is inherited by the execution task created below.+ inline_run_context = skyvern_context.current()+ if inline_run_context is not None:+ inline_run_context.copilot_inline_execution = Trueawait initialize_skyvern_state_file(workflow_run_id=workflow_run.workflow_run_id,@@ -1801,7 +1780,6 @@ async def _run_blocks_and_collect_debug(run: Any = initial_runexit_reason: WatchdogExitReason | None = Nonerun_cancelled_by_watchdog = False- active_run_terminal_evidence: ActiveRunTerminalEvidenceSample | None = None# Quiet blocks (WAIT/TEXT_PROMPT/HUMAN_INTERACTION) legitimately have# DB-silent periods; disable stagnation for any invocation that includes# one. Safety ceiling still applies.@@ -1826,17 +1804,6 @@ async def _run_blocks_and_collect_debug(seen_block_states: dict[str, str] = {}prior_block_ts: datetime | None = initial_block_tslast_block_fetch_monotonic = 0.0- # Dispatched runs are owned by the worker, which now holds the persistent browser session.- # The API side must not touch that PBS over CDP, so the in-run active-terminal evidence- # monitor (which samples the live page) is disabled for dispatched runs.- active_terminal_monitor_enabled = (not dispatch_to_worker) and await _active_run_terminal_monitor_enabled(ctx)- next_active_terminal_monitor_monotonic = (- started_monotonic + _ACTIVE_RUN_TERMINAL_MONITOR_INITIAL_DELAY_SECONDS- if active_terminal_monitor_enabled- else float("inf")- )- active_terminal_monitor_samples = 0-try:while True:await asyncio.sleep(RUN_BLOCKS_POLL_INTERVAL_SECONDS)@@ -1884,32 +1851,6 @@ async def _run_blocks_and_collect_debug(is_paused = run is not None and run.status == WorkflowRunStatus.paused.valuestagnation_active = stagnation_enabled and not is_paused- if (- active_terminal_monitor_enabled- and not is_paused- and active_terminal_monitor_samples < _ACTIVE_RUN_TERMINAL_MONITOR_MAX_SAMPLES- and now >= next_active_terminal_monitor_monotonic- ):- active_terminal_monitor_samples += 1- next_active_terminal_monitor_monotonic += _ACTIVE_RUN_TERMINAL_MONITOR_INTERVAL_SECONDS- sample = await _active_run_terminal_evidence_sample(- ctx,- workflow_run_id=workflow_run.workflow_run_id,- labels_to_execute=labels_to_execute,- sample_index=active_terminal_monitor_samples,- )- if sample is not None:- post_sample_run = await _safe_read_workflow_run(- workflow_run.workflow_run_id,- ctx.organization_id,- context="active-terminal-post-sample",- )- if post_sample_run is not None:- run = post_sample_run- active_run_terminal_evidence = sample- exit_reason = "active_run_terminal_evidence"- break-if new_marker != progress_marker:progress_marker = new_markerlast_progress_monotonic = now@@ -1932,11 +1873,7 @@ async def _run_blocks_and_collect_debug(pre_cancel_run = await _safe_read_workflow_run(workflow_run.workflow_run_id, ctx.organization_id, context="pre-cancel")- if (- _watchdog_exit_allows_terminal_promotion(exit_reason)- and pre_cancel_run is not None- and WorkflowRunStatus(pre_cancel_run.status).is_final()- ):+ if pre_cancel_run is not None and WorkflowRunStatus(pre_cancel_run.status).is_final():final_status = pre_cancel_run.statusrun = pre_cancel_runexit_reason = "success"@@ -1947,17 +1884,16 @@ async def _run_blocks_and_collect_debug(if run_task is not None:await _cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id)else:- # Phase 4: dispatched run — cooperative DB cancel so the worker stops.+ # Dispatched run — cooperative DB cancel so the worker stops.await _cooperative_cancel_dispatched_run(workflow_run.workflow_run_id)run_cancelled_by_watchdog = Truerun = await _safe_read_workflow_run(workflow_run.workflow_run_id, ctx.organization_id, context="post-drain")- if _watchdog_exit_allows_terminal_promotion(exit_reason):- trusted = _trusted_post_drain_status(run)- if trusted is not None:- final_status = trusted- exit_reason = "success"+ trusted = _trusted_post_drain_status(run)+ if trusted is not None:+ final_status = trusted+ exit_reason = "success"if exit_reason != "success":assert exit_reason is not None # narrows for mypy; outer check excludes "success" but not None@@ -1973,30 +1909,17 @@ async def _run_blocks_and_collect_debug(current_url, page_title = (("", "") if dispatch_to_worker else await _fallback_page_info(ctx, session_id_override=run_session_id))- if exit_reason == "active_run_terminal_evidence" and active_run_terminal_evidence is not None:- result: dict[str, Any] = _active_run_terminal_evidence_result(- workflow_run_id=workflow_run.workflow_run_id,- run_status=run.status if run is not None else None,- sample=active_run_terminal_evidence,- requested_block_labels=list(block_labels),- executed_block_labels=list(labels_to_execute),- current_url=current_url,- page_title=page_title,- )- result["error"] = error_msg- result["data"]["failure_reason"] = user_failure_reason- else:- result = {- "ok": False,- "error": error_msg,- "data": {- "workflow_run_id": workflow_run.workflow_run_id,- "overall_status": run.status if run is not None else None,- "failure_reason": user_failure_reason,- "current_url": current_url,- "page_title": page_title,- },- }+ result: dict[str, Any] = {+ "ok": False,+ "error": error_msg,+ "data": {+ "workflow_run_id": workflow_run.workflow_run_id,+ "overall_status": run.status if run is not None else None,+ "failure_reason": user_failure_reason,+ "current_url": current_url,+ "page_title": page_title,+ },+ }result["data"]["control_signal"] = {"kind": f"watchdog_{exit_reason}","user_facing_summary": user_facing_summary,@@ -2044,10 +1967,9 @@ async def _run_blocks_and_collect_debug(fallback.add_done_callback(_log_detached_cleanup_failure)raisefinally:- # Belt and braces. If any exit path above missed a cancel — e.g. an- # unexpected exception bubbling out of the poll loop — make sure the- # run_task is at least signaled to cancel so we don't leak it. Dispatched- # runs have no in-process task, so there is nothing to signal here.+ # If any exit path above missed a cancel — e.g. an unexpected exception bubbling out of the+ # poll loop — signal the run_task so we don't leak it. Dispatched runs have no in-process+ # task, so there is nothing to signal.if run_task is not None and not run_task.done():run_task.cancel()# Soft-delete the pinned draft so it never lingers as the latest version. Gated on a final@@ -2539,41 +2461,7 @@ def _terminal_challenge_completion_verification(# site-block/unreadable-page pattern even though the classifier routes it to# DATA_EXTRACTION_FAILURE, not ANTI_BOT_DETECTION.# Coupling note: these substrings come from the run-level failure_reason-# produced when the shared scraper raises ScrapingFailed. If the template-# wording changes, update this tuple and the test that locks it in-# (tests/unit/test_copilot_probable_site_block.py).-_PROBABLE_SITE_BLOCK_FAILURE_REASON_SUBSTRINGS = (- "failed to load the website",- "page may have navigated unexpectedly",-)---def _detect_probable_site_block_wall(result: dict[str, Any]) -> bool:- """True when a block failed with the site-load template and the failure is- not a non-retriable nav error (DNS / SSL / invalid URL are owned by- :func:`_detect_non_retriable_nav_error`)."""- if bool(result.get("ok", False)):- return False- if _detect_non_retriable_nav_error(result):- return False- data = result.get("data")- if not isinstance(data, dict):- return False- blocks = data.get("blocks")- if not isinstance(blocks, list):- return False-- for block in blocks:- if not isinstance(block, dict):- continue- reason = block.get("failure_reason")- if isinstance(reason, str):- lowered = reason.lower()- if any(sub in lowered for sub in _PROBABLE_SITE_BLOCK_FAILURE_REASON_SUBSTRINGS):- return True- return False--+# produced when the shared scraper raises ScrapingFailed; update the tuple if that wording changes.def _detect_non_retriable_nav_error(result: dict[str, Any]) -> str | None:"""Return the first failure_reason that matches SKIP_INNER_NAV_RETRY_ERRORS(DNS / cert / SSL / invalid URL), preferring run-level over block-level.@@ -2595,17 +2483,6 @@ def _update_verification_evidence_from_run_result(copilot_ctx: AgentContext, resrun_id = data.get("workflow_run_id")if isinstance(run_id, str) and run_id.strip():evidence.workflow_run_id = run_id.strip()- if _active_run_terminal_evidence_detected(result):- evidence.active_run_terminal_evidence_detected = True- evidence.live_page_state_verified = True- evidence.verified_from_current_browser_state = True- evidence.full_workflow_verified = False- evidence.test_attempted_but_incomplete = True- if isinstance(run_id, str) and run_id.strip():- evidence.active_run_terminal_evidence_workflow_run_id = run_id.strip()- sample_index = data.get("active_run_terminal_evidence_sample_index")- if isinstance(sample_index, int):- evidence.active_run_terminal_evidence_sample_index = sample_indexcurrent_url = _valid_runtime_anchor_url(data.get("current_url"))if current_url is not None:evidence.current_url = current_url@@ -2653,35 +2530,7 @@ def _read_mapping_path(payload: dict[str, Any], path: str) -> Any:return current-def _delivered_unverified_single_block_outputs(- result: dict[str, Any],- terminal_state: DeliveredUnverifiedTerminalState,-) -> dict[str, Any]:- data = result.get("data")- blocks = data.get("blocks") if isinstance(data, dict) else None- outputs: list[dict[str, Any]] = []- if isinstance(blocks, list):- for block in blocks:- if isinstance(block, dict) and isinstance(block.get("extracted_data"), dict):- outputs.append(cast(dict[str, Any], block["extracted_data"]))- if len(outputs) != 1:- return {}- block_output = outputs[0]- observed: dict[str, Any] = {}- for verdict in terminal_state.observed_verdicts:- output_path = verdict.output_path or ""- normalized_path = output_path.removeprefix("output.").strip()- if not normalized_path or normalized_path.split(".")[-1] == "evidence_text":- continue- value = _read_mapping_path(block_output, normalized_path)- if value is None:- value = _read_mapping_path(block_output, output_path)- if value is not None:- observed[normalized_path] = value- return observed---def _delivered_unverified_registered_outputs(data: Mapping[str, Any]) -> dict[str, Any]:+def _registered_outputs_with_values(data: Mapping[str, Any]) -> dict[str, Any]:observed: dict[str, Any] = {}for item in _registered_output_parameter_payloads(data):view = _registered_output_payload_view(item.get("value"), item.get("block_type"))@@ -2693,15 +2542,6 @@ def _delivered_unverified_registered_outputs(data: Mapping[str, Any]) -> dict[streturn observed-def _emit_floor_rekeyed_emission_lane(- completion_verification: CompletionVerificationResult | None, workflow_run_id: str | None-) -> None:- fields = floor_rekeyed_emission_lane_fields(completion_verification)- if fields is None:- return- LOG.info("copilot.completion.floor_rekeyed_emission_lane", workflow_run_id=workflow_run_id, **fields)--def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any], completion_verification: CompletionVerificationResult | None = None) -> RecordedRunOutcome | None:@@ -2735,12 +2575,8 @@ def _record_run_blocks_result(_record_adjudication_on_turn_state(copilot_ctx, completion_verification)if completion_verification is not None and completion_verification.status == "evaluated":_emit_completion_verification_trace(copilot_ctx, completion_verification)- _emit_floor_rekeyed_emission_lane(completion_verification, run_id if isinstance(run_id, str) else None)copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else Nonecopilot_ctx.last_successful_run_blocks_workflow_run_id = run_id if run_ok and isinstance(run_id, str) else None- copilot_ctx.delivered_unverified_terminal = False- copilot_ctx.delivered_unverified_workflow_run_id = None- copilot_ctx.delivered_unverified_observed_outputs = {}# Watchdog cancels normally count as ok=False; only a coincident total# timeout softens to ``None`` to keep the unvalidated WIP rescue open.cancelled_by_watchdog = result.get(_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY) is True@@ -2850,13 +2686,6 @@ def _record_run_blocks_result("failure_classes": artifact_classes,}- if _active_run_terminal_evidence_detected(result):- _update_verification_evidence_from_run_result(copilot_ctx, result)- signal = _active_run_terminal_evidence_signal(copilot_ctx, "update_and_run_blocks")- if signal is not None:- stash_blocker_signal(copilot_ctx, signal)- stash_turn_halt_from_blocker_signal(copilot_ctx, signal, source="run_execution")-if terminal_challenge is not None:clear_runtime_authoring_repair_context(copilot_ctx)# A structured challenge is the more actionable terminal blocker when@@ -2947,37 +2776,6 @@ def _record_run_blocks_result(copilot_ctx.last_test_suspicious_success = Falsecopilot_ctx.last_test_failure_reason = Nonecopilot_ctx.suspicious_success_nudge_count = 0- delivered_unverified = degraded_contract_delivered_unverified_terminal_state(- completion_verification,- run_ok=run_ok,- workflow_run_id=run_id if isinstance(run_id, str) else None,- latest_workflow_run_id=copilot_ctx.last_run_blocks_workflow_run_id,- artifact_health_blocked=artifact_reason is not None,- terminal_blocked=terminal_challenge is not None or bool(copilot_ctx.last_test_anti_bot),- )- if delivered_unverified is not None:- observed_outputs = _delivered_unverified_single_block_outputs(result, delivered_unverified)- if observed_outputs:- workflow_run_id = run_id if isinstance(run_id, str) else None- copilot_ctx.last_test_suspicious_success = False- copilot_ctx.last_test_failure_reason = None- copilot_ctx.suspicious_success_nudge_count = 0- copilot_ctx.last_full_workflow_test_ok = False- copilot_ctx.delivered_unverified_terminal = True- copilot_ctx.delivered_unverified_workflow_run_id = workflow_run_id- copilot_ctx.delivered_unverified_observed_outputs = observed_outputs- stash_delivered_unverified_turn_halt(copilot_ctx, workflow_run_id=workflow_run_id)- update_repeated_failure_state(copilot_ctx, result)- _update_verification_evidence_from_run_result(copilot_ctx, result)- recorded_outcome = RecordedRunOutcome(- verdict="not_evaluated",- display_reason=run_outcome_display_reason(- "The latest run returned the requested output, but it was not independently verified."- ),- workflow_run_id=workflow_run_id,- )- _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)- return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)if empty_data_blocks and not completion_verification_evaluated:copilot_ctx.last_test_ok = Nonecopilot_ctx.last_test_suspicious_success = True@@ -2988,7 +2786,6 @@ def _record_run_blocks_result("The workflow may not be working correctly.")# Clean-ish success (no scrape-fail pattern): reset the streak.- copilot_ctx.probable_site_block_streak_count = 0update_repeated_failure_state(copilot_ctx, result)_update_verification_evidence_from_run_result(copilot_ctx, result)recorded_outcome = RecordedRunOutcome(@@ -3031,7 +2828,6 @@ def _record_run_blocks_result(data.setdefault("failure_reason", outcome_unverified_reason)else:copilot_ctx.failed_test_nudge_count = 0- copilot_ctx.probable_site_block_streak_count = 0copilot_ctx.last_failed_workflow_yaml = None# Real success: clear the signature latch so a subsequent bad URL in# the same session can re-fire the stop nudge.@@ -3068,7 +2864,6 @@ def _record_run_blocks_result(_update_verification_evidence_from_run_result(copilot_ctx, result)recorded_outcome = _adjudicated_run_outcome(copilot_ctx,- completion_verification,data=data if isinstance(data, dict) else {},workflow_run_id=run_id if isinstance(run_id, str) else None,)@@ -3080,7 +2875,6 @@ def _record_run_blocks_result(copilot_ctx.last_test_failure_reason = Nonecopilot_ctx.suspicious_success_nudge_count = 0copilot_ctx.failed_test_nudge_count = 0- copilot_ctx.probable_site_block_streak_count = 0copilot_ctx.last_failed_workflow_yaml = Nonecopilot_ctx.last_full_workflow_test_ok = Truecopilot_ctx.last_unverified_block_labels = []@@ -3090,7 +2884,6 @@ def _record_run_blocks_result(_update_verification_evidence_from_run_result(copilot_ctx, result)recorded_outcome = _adjudicated_run_outcome(copilot_ctx,- completion_verification,data=data if isinstance(data, dict) else {},workflow_run_id=run_id if isinstance(run_id, str) else None,)@@ -3099,10 +2892,6 @@ def _record_run_blocks_result(copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)copilot_ctx.last_test_non_retriable_nav_error = _detect_non_retriable_nav_error(result)- if _detect_probable_site_block_wall(result):- copilot_ctx.probable_site_block_streak_count += 1- else:- copilot_ctx.probable_site_block_streak_count = 0data = result.get("data")if isinstance(data, dict):@@ -3207,123 +2996,45 @@ def _same_run_committed_demonstrated_outcome(return outcome-def _zero_requested_output_criteria_delivered_unverified(- copilot_ctx: Any,- completion_verification: CompletionVerificationResult,- data: Mapping[str, Any],- workflow_run_id: str | None,-) -> RecordedRunOutcome | None:- emission_withhold = floor_rekeyed_emission_withhold(completion_verification)- registered_output_meaningful = _has_meaningful_registered_output_payload(data)- credit_withhold = zero_requested_output_criteria_credit(- completion_verification,- has_meaningful_registered_output=registered_output_meaningful,- )- if emission_withhold is None and not credit_withhold:- return None- if emission_withhold is None:- credit = floor_rekeyed_deliverable_credit(completion_verification)- if credit is not None:- payloads = _registered_output_parameter_payloads(data)- payload_keys = sorted(- {key for item in payloads for key in [item.get("output_parameter_key")] if isinstance(key, str) and key}- )- unbound_paths = [- output_path- for output_path in credit.output_paths- if output_path and not registered_output_payload_binds_output_path(payloads, output_path)- ]- if unbound_paths:- LOG.info(- "copilot.completion.floor_rekeyed_credit_payload_unbound",- workflow_run_id=workflow_run_id,- criterion_ids=list(credit.criterion_ids),- unbound_output_paths=unbound_paths,- backed_by_criterion_id=dict(completion_verification.floor_rekeyed_backed_by_criterion_id),- registered_output_keys=payload_keys,- )- else:- LOG.info(- "copilot.completion.floor_rekeyed_deliverable_credit",- workflow_run_id=workflow_run_id,- criterion_ids=list(credit.criterion_ids),- evidence_sources=list(credit.evidence_sources),- evidence_refs=list(credit.evidence_refs),- credited_output_paths=list(credit.output_paths),- registered_output_keys=payload_keys,- )- return None- copilot_ctx.last_test_suspicious_success = False- copilot_ctx.last_test_failure_reason = None- copilot_ctx.suspicious_success_nudge_count = 0- copilot_ctx.last_full_workflow_test_ok = False- copilot_ctx.verified_terminal_proposal_ready = False- copilot_ctx.delivered_unverified_terminal = True- copilot_ctx.delivered_unverified_workflow_run_id = workflow_run_id- copilot_ctx.delivered_unverified_observed_outputs = _delivered_unverified_registered_outputs(data)- stash_delivered_unverified_turn_halt(copilot_ctx, workflow_run_id=workflow_run_id)- if emission_withhold is not None:- LOG.info(- "copilot.completion.floor_rekeyed_emission_withheld",- workflow_run_id=workflow_run_id,- requested_output_criteria_count=completion_verification.requested_output_criteria_count,- registered_output_meaningful=registered_output_meaningful,- floor_rekeyed_unbacked_criterion_ids=list(emission_withhold.criterion_ids),- floor_rekeyed_unbacked_output_paths=list(emission_withhold.unbacked_output_paths),- )- display_reason = "The latest run did not produce the requested output, so it was not verified."- else:- LOG.info(- "copilot.completion.zero_requested_output_credit_withheld",- workflow_run_id=workflow_run_id,- requested_output_criteria_count=0,- registered_output_meaningful=True,- )- display_reason = "The latest run returned output, but it was not independently verified."- return RecordedRunOutcome(- verdict="not_evaluated",- display_reason=run_outcome_display_reason(display_reason),- workflow_run_id=workflow_run_id,- )--def _adjudicated_run_outcome(copilot_ctx: Any,- completion_verification: CompletionVerificationResult | None,*,data: Mapping[str, Any] | None = None,workflow_run_id: str | None = None,) -> RecordedRunOutcome:+ """Derive the outcome mechanically from what the run produced. The run is the oracle: a+ completed run that registered a value demonstrated it; one whose declared outputs all came+ back empty did not; a judge re-reading the same run does not change either fact."""committed = _same_run_committed_demonstrated_outcome(copilot_ctx)if committed is not None:return committed- if completion_verification is not None and completion_verification.status == "evaluated":- if not completion_verification.is_fully_satisfied():- if only_structural_requested_output_abstentions(completion_verification):- return RecordedRunOutcome(- verdict="not_evaluated",- display_reason=run_outcome_display_reason("Completion remains unverified."),- )- return RecordedRunOutcome(- verdict="not_demonstrated",- reason_code="outcome_not_demonstrated",- display_reason=run_outcome_display_reason(- _outcome_unverified_reason(copilot_ctx, completion_verification)- ),- )- gated = _zero_requested_output_criteria_delivered_unverified(- copilot_ctx, completion_verification, data or {}, workflow_run_id- )- if gated is not None:- return gated- return RecordedRunOutcome(verdict="demonstrated")if copilot_ctx.last_test_suspicious_success:return RecordedRunOutcome(verdict="not_demonstrated",reason_code="outcome_not_demonstrated",display_reason=run_outcome_display_reason(copilot_ctx.last_test_failure_reason),)- return RecordedRunOutcome(verdict="not_evaluated")+ payloads = _registered_output_parameter_payloads(data or {})+ if not payloads:+ return RecordedRunOutcome(verdict="demonstrated")+ produced = _registered_outputs_with_values(data or {})+ if produced:+ return RecordedRunOutcome(verdict="demonstrated")+ empty_keys = sorted(+ {+ str(item.get("output_parameter_key") or item.get("block_label") or "")+ for item in payloads+ if item.get("output_parameter_key") or item.get("block_label")+ }+ )+ return RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="outcome_not_demonstrated",+ display_reason=run_outcome_display_reason(+ "The run completed but produced no value at " + ", ".join(empty_keys) + "."+ ),+ workflow_run_id=workflow_run_id,+ )async def _send_run_outcome_update(@@ -3404,6 +3115,17 @@ async def _verify_and_record_run_blocks_result(reason_code=final.reason_code,display_reason=final.display_reason,)+ # Success reaches here on two lanes — the mechanical demonstrated verdict and the+ # completion-verified path — and the cue must ride both, or a verified run still reads+ # as unfinished business and the model re-runs it.+ run_succeeded = final.verdict == "demonstrated" or outcome_fully_verified(copilot_ctx)+ if run_succeeded and isinstance(result.get("data"), dict):+ # Tell the model the record already answers the request; it decides whether to+ # conclude. Ending the turn here instead would mean speaking in its place.+ result["data"]["next_step"] = (+ "This run produced what the request asked for. If nothing else remains, reply to the "+ "user now with the result instead of running or verifying it again."+ )return completion_verificationdiff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex c45fc0c8d..a5b01b8a6 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -67,7 +67,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (from skyvern.forge.sdk.copilot.code_block_preflight import (SANDBOX_UNRESOLVED_NAME_REASON_CODE,author_time_code_block_diagnostics,- sandbox_unresolved_name_diagnostics,+ sandbox_absent_name_diagnostics,sandbox_unresolved_name_repair_diagnostic,strip_redundant_sandbox_imports,)@@ -166,8 +166,6 @@ from skyvern.forge.sdk.copilot.output_contracts import ()from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidatefrom skyvern.forge.sdk.copilot.output_policy import (- OutputPolicyReason,- OutputPolicyVerdict,demote_author_time_steer_reasons,evaluate_output_policy,format_output_policy_tool_error,@@ -1010,9 +1008,9 @@ def _code_block_safety_errors(workflow_yaml: str | None, prior_yaml: str | None)f"Code block `{label}` failed the generated-code preflight check: {item.message}"for item in author_time_diagnostics)- unresolved_diagnostics = sandbox_unresolved_name_diagnostics(code, parameter_keys=parameter_keys)+ absent_diagnostics = sandbox_absent_name_diagnostics(code, parameter_keys=parameter_keys)errors.extend(- f"Code block `{label}` failed the sandbox name check: {item.message}" for item in unresolved_diagnostics+ f"Code block `{label}` failed the sandbox name check: {item.message}" for item in absent_diagnostics)return errors@@ -10430,7 +10428,6 @@ async def _update_workflow("steered_reason_codes": [reason.value for reason in output_policy_steered_reasons],}output_policy_error = format_output_policy_tool_error(output_policy_verdict)- _record_code_only_raw_secret_reject_span(ctx, output_policy_verdict)LOG.info("copilot output policy tool body verdict",**output_policy_trace_data,@@ -10612,22 +10609,6 @@ def _record_workflow_proxy_location_span(workflow_yaml: str, workflow: Workflow)pass-def _record_code_only_raw_secret_reject_span(ctx: AgentContext, verdict: OutputPolicyVerdict) -> None:- if OutputPolicyReason.RAW_SECRET_LEAK not in verdict.reason_codes:- return- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:- return- with copilot_span(- "update_workflow_code_only_raw_secret_reject",- data={- "tool_name": "update_workflow",- "reason_code": OutputPolicyReason.RAW_SECRET_LEAK.value,- "block_authoring_policy": BlockAuthoringPolicy.CODE_ONLY_BROWSER.value,- },- ):- pass--def _persisted_landing_leaves_spine_uncovered(ctx: AgentContext) -> bool:try:return bool(_scouted_spine_open_obligation(ctx))diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.pyindex 4a2465f03..60e6776bb 100644--- a/skyvern/forge/sdk/copilot/turn_halt.py+++ b/skyvern/forge/sdk/copilot/turn_halt.py@@ -21,7 +21,6 @@ from skyvern.forge.sdk.copilot.blocker_signal import (clear_tool_blocker_signals_for_reason_codes,)from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_data-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODEfrom skyvern.forge.sdk.copilot.output_contracts import (OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,@@ -53,20 +52,16 @@ ADVISORY_DISPATCH_STALLED_REASON_CODE = "advisory_dispatch_stalled"class TurnHaltKind(StrEnum):LOOP_DETECTED = "loop_detected"ACTIVE_TERMINAL_CHALLENGE = "active_terminal_challenge"- PROBABLE_SITE_BLOCK = "probable_site_block"OUTPUT_SOURCE_UNOBSERVABLE = "output_source_unobservable"- DELIVERED_UNVERIFIED = "delivered_unverified"class TurnHaltVerdict(StrEnum):BLOCKED = "blocked"- DELIVERED_UNVERIFIED = "delivered_unverified"_LOOP_TERMINAL_REASON_CODES = frozenset({"loop_detected_credential_or_parameter_misconfig",- "loop_detected_repeated_failed_step","loop_detected_consecutive_same_tool","loop_detected_generic","code_authoring_guardrail_churn",@@ -77,7 +72,6 @@ _LOOP_TERMINAL_REASON_CODES = frozenset()_ACTIVE_TERMINAL_CHALLENGE_REASON_CODES = frozenset({- ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,# Back-compat sentinel for pre-TERMINAL_CHALLENGE_BLOCKER traces."tool_error_run_output_terminal_blocker",@@ -85,7 +79,6 @@ _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES = frozenset("tool_error_challenge_gated_submit_disabled",})-_PROBABLE_SITE_BLOCK_REASON_CODES = frozenset({"probable_site_block_stop"})_OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES = frozenset({OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,@@ -100,15 +93,10 @@ _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES = frozenset(_INVOLUNTARY_TURN_HALT_KINDS = frozenset({TurnHaltKind.LOOP_DETECTED,- TurnHaltKind.PROBABLE_SITE_BLOCK,TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE,})-_INVOLUNTARY_BLOCKER_REASON_CODES = (- _LOOP_TERMINAL_REASON_CODES | _PROBABLE_SITE_BLOCK_REASON_CODES | _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES-)-_VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES = frozenset({ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE})-_VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_SOURCES = frozenset({"run_execution"})+_INVOLUNTARY_BLOCKER_REASON_CODES = _LOOP_TERMINAL_REASON_CODES | _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES@dataclass(frozen=True)@@ -133,8 +121,6 @@ def _kind_for_blocker_signal(signal: CopilotToolBlockerSignal) -> TurnHaltKind |return TurnHaltKind.LOOP_DETECTEDif reason in _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES:return TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE- if reason in _PROBABLE_SITE_BLOCK_REASON_CODES:- return TurnHaltKind.PROBABLE_SITE_BLOCKif reason in _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES:return TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLEreturn None@@ -288,22 +274,6 @@ def stash_turn_halt_from_blocker_signal(ctx: Any, signal: object, *, source: strreturn halt-def stash_delivered_unverified_turn_halt(ctx: AgentContext, *, workflow_run_id: str | None) -> TurnHalt | None:- if isinstance(ctx.turn_halt, TurnHalt):- return ctx.turn_halt- run_refs = {"workflow_run_id": workflow_run_id} if workflow_run_id else {}- halt = TurnHalt(- kind=TurnHaltKind.DELIVERED_UNVERIFIED,- verdict=TurnHaltVerdict.DELIVERED_UNVERIFIED,- run_refs=run_refs,- extra={"source": "run_execution"},- )- ctx.turn_halt = halt- claim_turn(ctx, TurnClaimant.GENUINELY_TERMINAL, renders_final_reply=True)- LOG.info("copilot turn halt stashed", **turn_halt_to_trace_data(halt))- return halt--def _restored_owner_halt(ctx: AgentContext, owner_claimant: TurnClaimant) -> TurnHalt | None:held = ctx.blocker_signalif held is None or effective_signal_claimant(ctx, held) is not owner_claimant:@@ -361,16 +331,7 @@ def raise_if_turn_halt(ctx: Any, *, verified: bool = False) -> None:if not isinstance(halt, TurnHalt):returnsuppressible_reason_codes = _INVOLUNTARY_BLOCKER_REASON_CODES- if (- halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE- and halt.blocker_signal is not None- and halt.blocker_signal.internal_reason_code in _VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES- and halt.extra.get("source") in _VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_SOURCES- ):- suppressible_reason_codes = (- _INVOLUNTARY_BLOCKER_REASON_CODES | _VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES- )- elif halt.kind not in _INVOLUNTARY_TURN_HALT_KINDS:+ if halt.kind not in _INVOLUNTARY_TURN_HALT_KINDS:suppressible_reason_codes = frozenset()if verified and suppressible_reason_codes:ctx.turn_halt = Nonediff --git a/skyvern/forge/sdk/copilot/turn_outcome.py b/skyvern/forge/sdk/copilot/turn_outcome.pyindex 8c808472b..2c378eb5a 100644--- a/skyvern/forge/sdk/copilot/turn_outcome.py+++ b/skyvern/forge/sdk/copilot/turn_outcome.py@@ -22,43 +22,6 @@ LOG = structlog.get_logger()IDENTICAL_REPLY_BLOCKED_TERMINAL_REASON = "identical_reply_blocked"CopilotComposerMode = Literal["ask", "build", "code"]-REPEATED_REPLY_ESCALATION_TEMPLATES: dict[ResponseKind, str] = {- ResponseKind.BUILD: (- "I've offered the same change twice and it isn't moving us forward."- " Tell me one concrete edit you want to make to the workflow"- " (e.g. a step to add, a page to open, a field to extract) and I'll"- " build that exact thing."- ),- ResponseKind.CLARIFY: (- "I've asked the same clarification twice. Let me try a different"- " angle — please confirm the single next concrete step you'd like me"- " to take, and I'll act on that."- ),- ResponseKind.DIAGNOSE: (- "I've explained that the same way twice. Either the explanation"- " isn't matching what you're seeing, or I'm missing context — could"- " you share what you actually see, or paste the exact error, so I can"- " try a different angle?"- ),- ResponseKind.REFUSE: (- "I've refused that the same way twice. If you want me to reconsider,"- " tell me what's different about this case and I'll re-evaluate."- ),-}---HANDOFF_REPLY = (- "I haven't been able to help with this through repeated attempts. Please"- " rephrase the request with more specifics, or use the Help link so a"- " teammate can take a closer look."-)---def escalation_reply_for(attempted_kind: ResponseKind) -> str:- return REPEATED_REPLY_ESCALATION_TEMPLATES.get(- attempted_kind, REPEATED_REPLY_ESCALATION_TEMPLATES[ResponseKind.CLARIFY]- )-def apply_repeated_reply_guard(*,@@ -70,18 +33,12 @@ def apply_repeated_reply_guard(turn_intent: TurnIntent | None = None,tool_calls: Iterable[str] = (),) -> tuple[str, TurnOutcome]:- """Centralized post-output guard. Returns ``(final_text, outcome)``.-- When the signature of ``final_text`` is in ``blocked_signatures``, rewrites- to the escalation template keyed by ``attempted_kind`` and returns a- ``RECOVER`` outcome that records the original signature in its- ``blocked_signatures`` so the ban survives to the next turn. If the- escalation template itself collides with an already-banned signature- (a back-to-back ``RECOVER`` loop), falls back to a generic hand-off so- the system cannot self-amplify on the same recovery text. Otherwise- returns the original text with a record carrying the inherited bans- forward. The rewritten signature is also added to ``blocked_signatures``- so a future turn cannot re-emit the same escalation text.+ """Centralized post-output record. Returns ``(final_text, outcome)``.++ The model's reply is never rewritten: a repeat is the turn's true state, and replacing it with+ escalation prose both invents words the model did not say and — when it also minted a terminal+ reason — ended turns over a chat-presentation concern. Signatures are still recorded and carried+ forward so repetition stays visible downstream.Pass ``turn_intent`` and ``tool_calls`` to preserve trace metadata onthe outcome; otherwise the minimal-shape builder is used.@@ -90,29 +47,7 @@ def apply_repeated_reply_guard(tool_calls_list = list(tool_calls)original_signature = compute_signature(final_text)if inherited and original_signature in inherited:- rewritten = escalation_reply_for(attempted_kind)- rewritten_signature = compute_signature(rewritten)- if rewritten_signature in inherited:- rewritten = HANDOFF_REPLY- rewritten_signature = compute_signature(rewritten)- intent_summary: dict[str, Any] = {}- if turn_intent is not None:- try:- intent_summary = dict(turn_intent.to_trace_data())- except Exception as exc:- LOG.warning(- "Failed to serialize TurnIntent trace data for RECOVER outcome; using empty dict",- exc_info=exc,- )- return rewritten, TurnOutcome(- turn_intent_summary=intent_summary,- response_kind=ResponseKind.RECOVER,- reason_code=IDENTICAL_REPLY_BLOCKED_TERMINAL_REASON,- normalized_reply_signature=rewritten_signature,- tool_calls=[str(c) for c in tool_calls_list if c],- terminal_reason=IDENTICAL_REPLY_BLOCKED_TERMINAL_REASON,- blocked_signatures=_dedup_signatures([*inherited, original_signature, rewritten_signature]),- )+ LOG.info("copilot_repeated_reply_observed", normalized_reply_signature=original_signature)if turn_intent is not None or tool_calls_list:return final_text, build_turn_outcome(final_text,diff --git a/skyvern/forge/sdk/copilot/turn_ownership.py b/skyvern/forge/sdk/copilot/turn_ownership.pyindex 14ba0fb65..b3bb76223 100644--- a/skyvern/forge/sdk/copilot/turn_ownership.py+++ b/skyvern/forge/sdk/copilot/turn_ownership.py@@ -38,7 +38,6 @@ class TurnClaimant(StrEnum):CREDENTIAL_PRIORITY_CHURN = "credential_priority_authoring_churn"ACTUATION_OBLIGATION_FILL = "actuation_obligation_fill"ACTUATION_OBLIGATION_LOGIN_COMPLETION = "actuation_obligation_login_completion"- POST_RUN_PAGE_PATH_INTERACTION = "post_run_page_path_interaction"CREDENTIAL_SCOUT_REOPEN = "credential_scout_reopen"SYNTHESIZED_BLOCK_PERSISTENCE_FORCE = "synthesized_block_persistence_force"CODE_AUTHORING_CHURN = "code_authoring_guardrail_churn"@@ -69,7 +68,6 @@ _PRECEDENCE_ORDER: tuple[TurnClaimant, ...] = (# began; predicate-gated at its call site and bounded by the post-credential commit.TurnClaimant.ACTUATION_OBLIGATION_LOGIN_COMPLETION,# Post-run page-path repair admits only its outcome-bound click/Enter window.- TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION,# One-shot credential-scout reopen admits evaluate through the persistence gate;# predicate-gated at its call site.TurnClaimant.CREDENTIAL_SCOUT_REOPEN,@@ -90,14 +88,12 @@ CLAIMANT_REASON_CODE_FAMILIES: dict[TurnClaimant, frozenset[str]] = {TurnClaimant.CREDENTIAL_PRIORITY_CHURN: frozenset({"credential_priority_authoring_churn"}),TurnClaimant.ACTUATION_OBLIGATION_FILL: frozenset(),TurnClaimant.ACTUATION_OBLIGATION_LOGIN_COMPLETION: frozenset(),- TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION: frozenset(),TurnClaimant.CREDENTIAL_SCOUT_REOPEN: frozenset(),TurnClaimant.SYNTHESIZED_BLOCK_PERSISTENCE_FORCE: frozenset({SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE}),TurnClaimant.CODE_AUTHORING_CHURN: frozenset({"code_authoring_guardrail_churn"}),TurnClaimant.LOOP_DETECTED: frozenset({"loop_detected_credential_or_parameter_misconfig",- "loop_detected_repeated_failed_step","loop_detected_consecutive_same_tool","loop_detected_generic","loop_detected_no_forward_progress_interaction",@@ -117,13 +113,10 @@ _TRANSIENT_CLAIMANTS = frozenset({TurnClaimant.ACTUATION_OBLIGATION_FILL,TurnClaimant.ACTUATION_OBLIGATION_LOGIN_COMPLETION,- TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION,TurnClaimant.CREDENTIAL_SCOUT_REOPEN,})-_SIGNALLESS_TERMINAL_TURN_HALT_KIND_VALUES = frozenset({"delivered_unverified"})-def claimant_outranks(candidate: TurnClaimant, incumbent: TurnClaimant) -> bool:return _PRECEDENCE_RANK[candidate] < _PRECEDENCE_RANK[incumbent]@@ -203,9 +196,9 @@ def record_gate_precedence_conflict(def turn_halt_is_genuinely_terminal(halt: TurnHalt | None) -> bool:if halt is None:return False- if halt.blocker_signal is not None:- return blocker_signal_is_genuinely_terminal(halt.blocker_signal)- return halt.kind.value in _SIGNALLESS_TERMINAL_TURN_HALT_KIND_VALUES+ if halt.blocker_signal is None:+ return False+ return blocker_signal_is_genuinely_terminal(halt.blocker_signal)def _held_signal_claimant(ctx: AgentContext) -> TurnClaimant | None:diff --git a/skyvern/forge/sdk/copilot/verification_evidence.py b/skyvern/forge/sdk/copilot/verification_evidence.pyindex 849479b3b..fb9757d41 100644--- a/skyvern/forge/sdk/copilot/verification_evidence.py+++ b/skyvern/forge/sdk/copilot/verification_evidence.py@@ -13,7 +13,6 @@ class WorkflowVerificationEvidence:full_workflow_verified: bool = Falseblock_verified: list[str] = field(default_factory=list)live_page_state_verified: bool = False- active_run_terminal_evidence_detected: bool = Falsetest_attempted_but_incomplete: bool = Falseper_tool_budget_on_block: list[str] = field(default_factory=list)verified_from_current_browser_state: bool = False@@ -22,8 +21,6 @@ class WorkflowVerificationEvidence:current_url: str | None = Nonepage_title: str | None = Noneworkflow_run_id: str | None = None- active_run_terminal_evidence_workflow_run_id: str | None = None- active_run_terminal_evidence_sample_index: int | None = Noneunverified_block_labels: list[str] = field(default_factory=list)failed_block_labels: list[str] = field(default_factory=list)failure_reason: str | None = None@@ -40,7 +37,6 @@ class WorkflowVerificationEvidence:self.full_workflow_verifiedor self.block_verifiedor self.live_page_state_verified- or self.active_run_terminal_evidence_detectedor self.test_attempted_but_incompleteor self.per_tool_budget_on_blockor self.verified_from_current_browser_state@@ -48,8 +44,6 @@ class WorkflowVerificationEvidence:or self.current_url_may_encode_runtime_stateor self.current_urlor self.workflow_run_id- or self.active_run_terminal_evidence_workflow_run_id- or self.active_run_terminal_evidence_sample_index is not Noneor self.unverified_block_labelsor self.failed_block_labelsor self.failure_reason@@ -61,7 +55,6 @@ class WorkflowVerificationEvidence:"full_workflow_verified": self.full_workflow_verified,"block_verified_count": len(self.block_verified),"live_page_state_verified": self.live_page_state_verified,- "active_run_terminal_evidence_detected": self.active_run_terminal_evidence_detected,"test_attempted_but_incomplete": self.test_attempted_but_incomplete,"per_tool_budget_on_block_count": len(self.per_tool_budget_on_block),"verified_from_current_browser_state": self.verified_from_current_browser_state,@@ -69,7 +62,6 @@ class WorkflowVerificationEvidence:"current_url_may_encode_runtime_state": self.current_url_may_encode_runtime_state,"has_current_url": bool(self.current_url),"has_workflow_run_id": bool(self.workflow_run_id),- "has_active_run_terminal_evidence_workflow_run_id": bool(self.active_run_terminal_evidence_workflow_run_id),"unverified_block_count": len(self.unverified_block_labels),"failed_block_count": len(self.failed_block_labels),"has_failure_reason": bool(self.failure_reason),@@ -83,7 +75,6 @@ class WorkflowVerificationEvidence:lines = [f"full_workflow_verified: {str(self.full_workflow_verified).lower()}",f"live_page_state_verified: {str(self.live_page_state_verified).lower()}",- f"active_run_terminal_evidence_detected: {str(self.active_run_terminal_evidence_detected).lower()}",f"test_attempted_but_incomplete: {str(self.test_attempted_but_incomplete).lower()}",f"verified_from_current_browser_state: {str(self.verified_from_current_browser_state).lower()}",]@@ -106,12 +97,6 @@ class WorkflowVerificationEvidence:lines.append(f"page_title: {self.page_title}")if self.workflow_run_id:lines.append(f"workflow_run_id: {self.workflow_run_id}")- if self.active_run_terminal_evidence_workflow_run_id:- lines.append(- f"active_run_terminal_evidence_workflow_run_id: {self.active_run_terminal_evidence_workflow_run_id}"- )- if self.active_run_terminal_evidence_sample_index is not None:- lines.append(f"active_run_terminal_evidence_sample_index: {self.active_run_terminal_evidence_sample_index}")if self.failure_reason:lines.append(f"failure_reason: {self.failure_reason}")return "\n".join(lines)diff --git a/skyvern/forge/sdk/copilot/workflow_yaml.py b/skyvern/forge/sdk/copilot/workflow_yaml.pyindex 86263aa74..f2121aa0a 100644--- a/skyvern/forge/sdk/copilot/workflow_yaml.py+++ b/skyvern/forge/sdk/copilot/workflow_yaml.py@@ -539,13 +539,14 @@ def apply_block_edit(occurrences = current.count(expected_code)if occurrences == 0:raise BlockEditError(- f"expected_code was not found in block {label!r}. It has changed since you read it — "- "re-read the block and rewrite the edit against its current code."+ f"expected_code was not found in block {label!r}. It has changed since you read it. "+ f"Its code is now:\n{current}\n"+ "Rewrite the edit against exactly that text.")if occurrences > 1:raise BlockEditError(f"expected_code appears {occurrences} times in block {label!r}; include enough "- "surrounding lines to identify one occurrence."+ f"surrounding lines to identify one occurrence. Its code is now:\n{current}")block["code"] = current.replace(expected_code, replacement_code, 1)diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex a49965c2b..d921da883 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -100,6 +100,10 @@ class SkyvernContext:tz_info: ZoneInfo | None = Nonerun_id: str | None = Nonecopilot_session_id: str | None = None+ # Set only by the in-process copilot block-test path, on that run's own context. A dispatched+ # run's context is rebuilt on the worker and never carries it, so runner selection can tell the+ # two apart instead of inferring it from a process-wide capability.+ copilot_inline_execution: bool = Falsenavigation_goal: str | None = Nonenavigation_payload: dict[str, Any] | list | str | None = Nonedownload_suffix: str | None = Nonediff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.pyindex 6c5c7678f..f8a2c8ac2 100644--- a/skyvern/forge/sdk/forge_log.py+++ b/skyvern/forge/sdk/forge_log.py@@ -541,6 +541,10 @@ def setup_logger() -> None:handler.setFormatter(structlog.stdlib.ProcessorFormatter(foreign_pre_chain=[+ # Without this, `extra={...}` on a stdlib log call is silently dropped+ # and never becomes a queryable attribute. Modules that run in images+ # without structlog installed have no other route to structured fields.+ structlog.stdlib.ExtraAdder(),add_error_processor,structlog.processors.format_exc_info,],diff --git a/skyvern/forge/sdk/routes/browser_sessions.py b/skyvern/forge/sdk/routes/browser_sessions.pyindex 4b37a0f05..9b8edc96a 100644--- a/skyvern/forge/sdk/routes/browser_sessions.py+++ b/skyvern/forge/sdk/routes/browser_sessions.py@@ -201,7 +201,6 @@ async def create_browser_session(browser_session = await app.PERSISTENT_SESSIONS_MANAGER.create_session(organization_id=current_org.organization_id,- url=browser_session_request.url,timeout_minutes=browser_session_request.timeout,proxy_location=proxy_location,proxy_session_id=proxy_session_id,diff --git a/skyvern/forge/sdk/routes/credentials.py b/skyvern/forge/sdk/routes/credentials.pyindex 839a3f8f7..a85e14b3c 100644--- a/skyvern/forge/sdk/routes/credentials.py+++ b/skyvern/forge/sdk/routes/credentials.py@@ -38,7 +38,7 @@ from typing import Annotated, Any, NoReturnimport pyotpimport structlog-from fastapi import BackgroundTasks, Body, Depends, Header, HTTPException, Path, Query, Response+from fastapi import BackgroundTasks, Body, Depends, Header, HTTPException, Path, Query, Response, statusfrom onepassword.client import Client as OnePasswordClientfrom onepassword.errors import DesktopSessionExpiredException, RateLimitExceededExceptionfrom sqlalchemy.exc import IntegrityError@@ -145,7 +145,12 @@ from skyvern.schemas.workflows import (WorkflowParameterYAML,WorkflowStatus,)-from skyvern.services.otp_service import OTPValue, parse_otp_login, redact_otp_identifier_for_log+from skyvern.services.otp_service import (+ InsufficientCreditsForOTPParse,+ OTPValue,+ parse_otp_login,+ redact_otp_identifier_for_log,+)from skyvern.services.run_service import cancel_workflow_runfrom skyvern.utils.url_validators import validate_url@@ -407,13 +412,19 @@ async def fetch_credential_item_background(item_id: str) -> None:@legacy_base_router.post("/totp",- responses={202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"}},+ responses={+ 202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"},+ 402: {"description": "Insufficient credits to parse OTP content"},+ },)@legacy_base_router.post("/totp/", include_in_schema=False)@base_router.post("/credentials/totp",response_model=TOTPCode,- responses={202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"}},+ responses={+ 202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"},+ 402: {"description": "Insufficient credits to parse OTP content"},+ },summary="Send TOTP code",description="Forward a TOTP (2FA, MFA) email or sms message containing the code to Skyvern. This endpoint stores the code in database so that Skyvern can use it while running tasks/workflows.",tags=["Credentials"],@@ -432,7 +443,10 @@ async def fetch_credential_item_background(item_id: str) -> None:@base_router.post("/credentials/totp/",response_model=TOTPCode,- responses={202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"}},+ responses={+ 202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"},+ 402: {"description": "Insufficient credits to parse OTP content"},+ },include_in_schema=False,)async def send_totp_code(@@ -463,15 +477,27 @@ async def send_totp_code(raise HTTPException(status_code=400, detail=f"Invalid workflow run id: {data.workflow_run_id}")content = data.content.strip()otp_value: OTPValue | None = OTPValue(value=content, type=data.type or OTPType.TOTP)+ otp_parse_skipped_for_insufficient_credits = Falseparse_exception_type_name: str | None = None# We assume the user is sending the code directly when the length of code is less than or equal to 10if len(content) > 10:try:otp_value = await parse_otp_login(content, curr_org.organization_id, enforced_otp_type=data.type)+ except InsufficientCreditsForOTPParse:+ otp_value = None+ otp_parse_skipped_for_insufficient_credits = Trueexcept Exception as e:otp_value = Noneparse_exception_type_name = type(e).__name__+ if otp_parse_skipped_for_insufficient_credits and (+ not settings.TOTP_RAW_FALLBACK_ENABLED or len(data.content) > settings.TOTP_RAW_CONTENT_MAX_LENGTH+ ):+ raise HTTPException(+ status_code=status.HTTP_402_PAYMENT_REQUIRED,+ detail="Insufficient credits to parse OTP content",+ )+if parse_exception_type_name:# parse_otp_login raised, meaning the OTP-extraction dependency (LLM backend)# failed rather than the caller supplying unparseable input. Surface a retryablediff --git a/skyvern/forge/sdk/routes/streaming/cdp_input.py b/skyvern/forge/sdk/routes/streaming/cdp_input.pyindex 4673f1a70..8eae7f0db 100644--- a/skyvern/forge/sdk/routes/streaming/cdp_input.py+++ b/skyvern/forge/sdk/routes/streaming/cdp_input.py@@ -92,8 +92,12 @@ class ActivePageCdpInputSession:fall_back_to_captured=self.cdp_session is None,)if page is None:- self.page_resolution_failed = Trueself.next_refresh_at = now + self.refresh_interval+ if self.cdp_session is not None:+ # _resolve_working_page returns None on a transient failure precisely so the caller+ # keeps the live page; dropping the bound session here strands user input instead.+ return self.cdp_session+ self.page_resolution_failed = Truereturn Noneself.page_resolution_failed = Falsediff --git a/skyvern/schemas/browser_sessions.py b/skyvern/schemas/browser_sessions.pyindex 47cc9e2fb..4a38060f0 100644--- a/skyvern/schemas/browser_sessions.py+++ b/skyvern/schemas/browser_sessions.py@@ -8,22 +8,9 @@ from skyvern.schemas.docs.doc_strings import PROXY_LOCATION_DOC_STRINGfrom skyvern.schemas.proxy_pinning import validate_proxy_session_idfrom skyvern.schemas.runs import GeoTarget, ProxyLocationInputfrom skyvern.services.browser_recording.types import RecordingDraftStep-from skyvern.utils.url_validators import validate_urlclass CreateBrowserSessionRequest(BaseModel):- url: str | None = Field(- default=None,- description="Optional URL to open when the standalone browser session starts.",- )-- @field_validator("url")- @classmethod- def validate_start_url(cls, value: str | None) -> str | None:- if not value:- return value- return validate_url(value)-timeout: int | None = Field(default=DEFAULT_TIMEOUT,description=f"Timeout in minutes for the session. Timeout is applied after the session is started. Must be between {MIN_TIMEOUT} and {MAX_TIMEOUT}. Defaults to {DEFAULT_TIMEOUT}.",diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.pyindex 030f46b55..a7ba308f5 100644--- a/skyvern/services/otp_service.py+++ b/skyvern/services/otp_service.py@@ -62,6 +62,10 @@ class _TOTPWebhookRequestError(Exception):pass+class InsufficientCreditsForOTPParse(Exception):+ """Control-flow signal indicating that paid OTP extraction was not attempted."""++class OTPValue(BaseModel):value: str = Field(..., description="The value of the OTP code.")type: OTPType | None = Field(None, description="The type of the OTP code.")@@ -91,7 +95,7 @@ async def parse_otp_login(# credits should not incur the secondary-LLM cost only to be charged for it.if not await app.AGENT_FUNCTION.has_sufficient_credit_for_otp_parse(organization_id):LOG.info("Skipping OTP parse; organization has insufficient credits", organization_id=organization_id)- return None+ raise InsufficientCreditsForOTPParseprompt = prompt_engine.load_prompt("parse-otp-login",content=content,@@ -651,6 +655,8 @@ async def _get_otp_value_from_url(if isinstance(content, str) and len(content) > 10:try:otp_value = await parse_otp_login(content, organization_id)+ except InsufficientCreditsForOTPParse:+ return Noneexcept Exception as e:otp_value = NoneLOG.warning(@@ -743,6 +749,8 @@ async def _get_otp_value_from_db(attempts += 1try:otp_value = await parse_otp_login(row.content, organization_id, enforced_otp_type=expected_otp_type)+ except InsufficientCreditsForOTPParse:+ return Noneexcept Exception:LOG.warning("Raw OTP reparse failed", totp_code_id=row.totp_code_id, otp_type=expected_otp_type, exc_info=Truediff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.pyindex dde67e06c..8ee631b69 100644--- a/skyvern/webeye/real_browser_manager.py+++ b/skyvern/webeye/real_browser_manager.py@@ -1057,6 +1057,7 @@ class RealBrowserManager(BrowserManager):browser_state = await self._create_browser_state(proxy_location=proxy_location,script_id=script_id,+ organization_id=organization_id,)if script_id:diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.pyindex 9392fad9c..f6253c9da 100644--- a/tests/unit/force_stub_app.py+++ b/tests/unit/force_stub_app.py@@ -31,11 +31,12 @@ def create_forge_stub_app() -> ForgeApp:fake_app_module.AGENT_FUNCTION.execute_code_block_override = AsyncMock(return_value=None)# Copilot worker-dispatch gate — _LazyNamespace would auto-mock this as a truthy AsyncMock# and route copilot block runs down the worker-dispatch path. Match the real OSS base- # default (False) so unit tests exercise the inline run path.+ # default (False) so unit tests exercise the unavailable-worker path.fake_app_module.AGENT_FUNCTION.should_dispatch_copilot_block_run_to_worker = AsyncMock(return_value=False)# Sync methods — _LazyNamespace would auto-mock these as AsyncMock and break callers that use# the return value directly. Match the real OSS defaults.fake_app_module.AGENT_FUNCTION.resolve_copilot_dispatch_trigger_type = MagicMock(return_value=None)+ fake_app_module.AGENT_FUNCTION.allow_copilot_inline_code_execution = MagicMock(return_value=False)fake_app_module.AGENT_FUNCTION.resolve_mcp_oauth_org_lookups = MagicMock(return_value=None)# Sync method returning a key or None — _LazyNamespace would auto-mock it as a truthy# AsyncMock and hijack the TextPromptBlock llm_key. Match the OSS no-op.diff --git a/tests/unit/forge/sdk/routes/streaming/test_cdp_input_session.py b/tests/unit/forge/sdk/routes/streaming/test_cdp_input_session.pynew file mode 100644index 000000000..df6e5f742--- /dev/null+++ b/tests/unit/forge/sdk/routes/streaming/test_cdp_input_session.py@@ -0,0 +1,82 @@+"""A transient page-resolution failure must not drop an already-bound CDP session: the dispatch loop+treats None as "no active page" and silently skips the event while the channel stays open, so the+user keeps interacting with a surface that no longer receives input."""++from typing import Any++import pytest++from skyvern.forge.sdk.routes.streaming import cdp_input+++class _FakeSession:+ def __init__(self, name: str) -> None:+ self.name = name+ self.detached = False++ async def detach(self) -> None:+ self.detached = True+++class _FakeContext:+ def __init__(self, session: _FakeSession) -> None:+ self._session = session++ async def new_cdp_session(self, page: object) -> _FakeSession:+ return self._session+++class _FakePage:+ def __init__(self, session: _FakeSession, url: str = "https://example.test/") -> None:+ self.context = _FakeContext(session)+ self.url = url+++def _build(monkeypatch: pytest.MonkeyPatch, page: Any) -> tuple[cdp_input.ActivePageCdpInputSession, list[Any]]:+ resolved: list[Any] = [page]++ async def _resolve(*args: Any, **kwargs: Any) -> Any:+ return resolved[0]++ monkeypatch.setattr(cdp_input, "_resolve_working_page", _resolve)+ input_session = cdp_input.ActivePageCdpInputSession(+ browser_state=object(), # type: ignore[arg-type]+ entity_id="wr_test",+ entity_type="workflow_run",+ )+ return input_session, resolved+++async def test_bound_session_survives_a_transient_resolution_failure(monkeypatch: pytest.MonkeyPatch) -> None:+ session = _FakeSession("first")+ input_session, resolved = _build(monkeypatch, _FakePage(session))++ assert await input_session.get_session() is session++ resolved[0] = None++ assert await input_session.get_session(force_refresh=True) is session+ assert input_session.page_resolution_failed is False+ assert session.detached is False+++async def test_resolution_failure_before_any_bind_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:+ input_session, _ = _build(monkeypatch, None)++ assert await input_session.get_session() is None+ assert input_session.page_resolution_failed is True+++async def test_rebinds_when_the_working_page_changes(monkeypatch: pytest.MonkeyPatch) -> None:+ first, second = _FakeSession("first"), _FakeSession("second")+ input_session, resolved = _build(monkeypatch, _FakePage(first))++ assert await input_session.get_session() is first++ resolved[0] = _FakePage(second)++ assert await input_session.get_session(force_refresh=True) is second+ assert first.detached is Truediff --git a/tests/unit/google/test_otp_service_gmail.py b/tests/unit/google/test_otp_service_gmail.pyindex ce158e039..ab56209ed 100644--- a/tests/unit/google/test_otp_service_gmail.py+++ b/tests/unit/google/test_otp_service_gmail.py@@ -3,6 +3,7 @@ from types import SimpleNamespacefrom unittest.mock import AsyncMockimport pytest+import structlog.testingfrom skyvern.forge import agent_functionsfrom skyvern.forge.agent_functions import AgentFunction@@ -196,6 +197,71 @@ async def test_get_otp_value_from_gmail_retries_candidate_after_parser_error(create_otp_code.assert_awaited_once()+async def test_get_otp_value_from_gmail_rechecks_unseen_candidate_after_credit_skip(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent = AgentFunction()+ credential = SimpleNamespace(+ id="goac_1",+ scopes_granted=list(google_oauth_service.GOOGLE_GMAIL_SCOPES),+ )+ monkeypatch.setattr(+ agent_functions.google_oauth_service,+ "get_credentials_for_org",+ AsyncMock(return_value=[credential]),+ )+ monkeypatch.setattr(+ agent,+ "get_google_workspace_credentials",+ AsyncMock(return_value=SimpleNamespace(token=object())),+ )+ create_otp_code = AsyncMock()+ monkeypatch.setattr(+ agent_functions.app, "DATABASE", SimpleNamespace(otp=SimpleNamespace(create_otp_code=create_otp_code))+ )+ candidates = [+ google_gmail_service.GmailMessageCandidate(+ message_id=f"msg_credit_skipped_{index}",+ content=f"Long relayed authentication message {index}.",+ internal_date=datetime.now(timezone.utc),+ )+ for index in range(2)+ ]+ monkeypatch.setattr(+ agent_functions.google_gmail_service,+ "search_recent_otp_messages",+ AsyncMock(return_value=candidates),+ )+ parsed = otp_service.OTPValue(value="placeholder", type=OTPType.TOTP)+ parse = AsyncMock(side_effect=[otp_service.InsufficientCreditsForOTPParse, parsed])+ monkeypatch.setattr(otp_service, "parse_otp_login", parse)+ context = GmailOTPVerificationContext()++ with structlog.testing.capture_logs() as logs:+ first_result = await agent.get_otp_value_from_gmail(+ organization_id="org_1",+ totp_identifier="[email protected]",+ context=context,+ )+ context.last_searched_at_by_credential["goac_1"] = datetime(2025, 1, 1, tzinfo=timezone.utc)+ second_result = await agent.get_otp_value_from_gmail(+ organization_id="org_1",+ totp_identifier="[email protected]",+ context=context,+ )++ assert first_result is None+ assert second_result == parsed+ assert parse.await_count == 2+ assert parse.await_args_list[0].args[0] == candidates[0].content+ assert parse.await_args_list[1].args[0] == candidates[1].content+ create_otp_code.assert_awaited_once()+ assert context.has_seen_message_id(candidates[0].message_id)+ assert context.has_seen_message_id(candidates[1].message_id)+ assert all(record.get("event") != "Failed to parse Gmail OTP candidate" for record in logs)++@pytest.mark.asyncioasync def test_get_otp_value_from_gmail_skips_credentials_without_gmail_scope(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_browser_session_profile_export_gate.py b/tests/unit/test_browser_session_profile_export_gate.pyindex cbff8106f..e84a4679f 100644--- a/tests/unit/test_browser_session_profile_export_gate.py+++ b/tests/unit/test_browser_session_profile_export_gate.py@@ -1,11 +1,5 @@from datetime import datetime-from types import SimpleNamespace-from unittest.mock import AsyncMock, MagicMock, patch-import pytest--from skyvern.exceptions import SkyvernHTTPException-from skyvern.forge.sdk.routes import browser_sessions as browser_sessions_modfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import (PersistentBrowserSession,export_profile_storage_id,@@ -44,53 +38,6 @@ def test_create_request_defaults_to_opt_out() -> None:assert CreateBrowserSessionRequest().generate_browser_profile is False-def test_create_request_accepts_start_url() -> None:- request = CreateBrowserSessionRequest(url="https://example.com/path", generate_browser_profile=True)-- assert request.url == "https://example.com/path"---def test_create_request_rejects_invalid_start_url() -> None:- with pytest.raises(SkyvernHTTPException):- CreateBrowserSessionRequest(url="ftp://example.com")---async def test_create_browser_session_passes_start_url_to_session_manager() -> None:- created_session = SimpleNamespace(persistent_browser_session_id="pbs_1")- response = SimpleNamespace(browser_session_id="pbs_1")- app_mock = MagicMock()- app_mock.PERSISTENT_SESSIONS_MANAGER.create_session = AsyncMock(return_value=created_session)- from_browser_session = AsyncMock(return_value=response)-- with (- patch.object(browser_sessions_mod, "app", app_mock),- patch.object(browser_sessions_mod.BrowserSessionResponse, "from_browser_session", from_browser_session),- ):- result = await browser_sessions_mod.create_browser_session(- CreateBrowserSessionRequest(- url="https://example.com/login",- timeout=120,- generate_browser_profile=True,- ),- current_org=SimpleNamespace(organization_id="org_1"),- )-- assert result is response- app_mock.PERSISTENT_SESSIONS_MANAGER.create_session.assert_awaited_once_with(- organization_id="org_1",- url="https://example.com/login",- timeout_minutes=120,- proxy_location=None,- proxy_session_id=None,- extensions=None,- browser_type=None,- browser_profile_id=None,- generate_browser_profile=True,- )- from_browser_session.assert_awaited_once_with(created_session)--def test_update_request_carries_flag() -> None:assert UpdateBrowserSessionRequest(generate_browser_profile=True).generate_browser_profile is Trueassert UpdateBrowserSessionRequest(generate_browser_profile=False).generate_browser_profile is Falsediff --git a/tests/unit/test_copilot_agent_blocker_renderer.py b/tests/unit/test_copilot_agent_blocker_renderer.pyindex 23df3ec3e..5fe2a9274 100644--- a/tests/unit/test_copilot_agent_blocker_renderer.py+++ b/tests/unit/test_copilot_agent_blocker_renderer.py@@ -13,8 +13,7 @@ from skyvern.forge.sdk.copilot.agent import (_build_goal_satisfied_exit_result,_build_output_policy_blocked_result,)-from skyvern.forge.sdk.copilot.agent import _build_turn_halt_exit_result as _real_build_turn_halt_exit_result-from skyvern.forge.sdk.copilot.agent import _build_wip_exit_result as _real_build_wip_exit_result+from skyvern.forge.sdk.copilot.agent import _build_turn_halt_exit_result as _build_turn_halt_exit_resultfrom skyvern.forge.sdk.copilot.agent import (_finalize_result_with_blocker_override,_render_blocker_reply,@@ -29,8 +28,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import ()from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy-from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext, DeliveredUnverifiedPublicOutputs-from skyvern.forge.sdk.copilot.enforcement import SCOUTED_SPINE_TURN_HALT_USER_REASON+from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContextfrom skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryStatefrom skyvern.forge.sdk.copilot.output_policy import (ACTUATION_OBLIGATION_STEER_REASON_CODE,@@ -58,25 +56,6 @@ _ALL_BLOCKER_KINDS: tuple[BlockerKind, ...] = get_args(BlockerKind)_ALL_RECOVERY_HINTS: tuple[RecoveryHint, ...] = get_args(RecoveryHint)-def _mark_public_outputs(ctx: CopilotContext) -> None:- if ctx.delivered_unverified_observed_outputs and not isinstance(- ctx.delivered_unverified_observed_outputs, DeliveredUnverifiedPublicOutputs- ):- ctx.delivered_unverified_observed_outputs = DeliveredUnverifiedPublicOutputs(- ctx.delivered_unverified_observed_outputs- )---def _build_wip_exit_result(ctx: CopilotContext, *args: Any, **kwargs: Any) -> AgentResult:- _mark_public_outputs(ctx)- return _real_build_wip_exit_result(ctx, *args, **kwargs)---def _build_turn_halt_exit_result(ctx: CopilotContext, *args: Any, **kwargs: Any) -> AgentResult:- _mark_public_outputs(ctx)- return _real_build_turn_halt_exit_result(ctx, *args, **kwargs)--def _signal(*,kind: BlockerKind = "authority_denied",@@ -382,78 +361,6 @@ def test_output_policy_hard_block_priority_over_actuation_steer() -> None:assert result.turn_outcome.reason_code != ACTUATION_OBLIGATION_STEER_REASON_CODE-def test_shim_preserves_workflow_draft_when_signal_opts_in() -> None:- """Late-block-running blockers say 'wrap up with what you have' — they- surface the saved draft alongside the rendered chat reply."""- ctx = _ctx()- fake_workflow: Any = object()- ctx.blocker_signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="Stop and reply with the draft.",- user_facing_reason="I'm running out of time on this turn. I'll wrap up with what I have so far.",- recovery_hint="stop",- cleared_by_tools=frozenset(),- preserves_workflow_draft=True,- internal_reason_code="tool_error_late_block_running",- blocked_tool="update_and_run_blocks",- )- result = AgentResult(- user_response="leaky agent prose",- updated_workflow=fake_workflow,- global_llm_context=None,- workflow_yaml="title: Draft\n",- proposal_disposition="review_untested",- )- overridden = _finalize_result_with_blocker_override(ctx, result)- assert overridden.updated_workflow is fake_workflow- assert overridden.workflow_yaml == "title: Draft\n"- assert overridden.clear_proposed_workflow is False- assert overridden.proposal_disposition == "review_untested"- # Reply still leads with the rendered text; the shim appends the unvalidated-proposal affordance because preserves_workflow_draft is True.- assert overridden.user_response.startswith(ctx.blocker_signal.user_facing_reason)- assert overridden.user_response != ctx.blocker_signal.user_facing_reason---def test_shim_preserves_staged_workflow_draft_when_verified_result_is_empty() -> None:- """Active-run terminal evidence can leave the verified workflow empty while- the keepable draft remains staged. That draft must still surface for- explicit user review instead of clearing the proposal controls."""- ctx = _ctx()- fake_workflow: Any = object()- ctx.staged_workflow = fake_workflow- ctx.staged_workflow_yaml = "title: Staged\n"- ctx.has_staged_proposal = True- ctx.blocker_signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="Stop and reply with the staged draft.",- user_facing_reason=(- "I reached the requested browser state, but the reusable workflow "- "still needs a clean verification run before it is ready."- ),- recovery_hint="stop",- preserves_workflow_draft=True,- internal_reason_code="tool_error_active_run_terminal_evidence",- blocked_tool="update_and_run_blocks",- )- result = AgentResult(- user_response="agent reply",- updated_workflow=None,- global_llm_context=None,- workflow_yaml=None,- staged_workflow_yaml=ctx.staged_workflow_yaml,- staged_workflow=fake_workflow,- has_staged_proposal=True,- )- overridden = _finalize_result_with_blocker_override(ctx, result)- assert overridden.updated_workflow is fake_workflow- assert overridden.workflow_yaml == ctx.staged_workflow_yaml- assert overridden.clear_proposed_workflow is False- assert overridden.proposal_disposition == "review_untested"- assert overridden.narrative_payload is not None- assert overridden.narrative_payload["proposalDisposition"] == "review_untested"- assert "Accept to save" in overridden.user_response--def test_shim_keeps_model_reply_when_signal_opts_out_of_final_rendering() -> None:"""Some tool guards are steering-only: they block a tool call but still letthe model answer from evidence already gathered in the turn."""@@ -542,7 +449,7 @@ def test_turn_halt_exit_keeps_halt_signal_when_context_signal_is_cleared() -> Nosignal = _signal(kind="loop_detected",user_facing="I'm stuck retrying the same step. Tell me what to change and I'll try a different approach.",- internal_reason_code="loop_detected_repeated_failed_step",+ internal_reason_code="loop_detected_consecutive_same_tool",blocked_tool="run_blocks_and_collect_debug",)halt = TurnHalt(kind=TurnHaltKind.LOOP_DETECTED, blocker_signal=signal)@@ -596,7 +503,6 @@ def test_turn_halt_exit_keeps_discovery_exhaustion_ask_when_no_scouted_spine() -result = _build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)assert result.user_response == signal.user_facing_reason- assert result.user_response != SCOUTED_SPINE_TURN_HALT_USER_REASONdef test_turn_halt_exit_keeps_credential_scout_reply_through_refresh_with_draft() -> None:@@ -686,7 +592,7 @@ def test_verified_outcome_preserves_tested_proposal_over_blocker() -> None:ctx.blocker_signal = _signal(kind="loop_detected",user_facing="I'm stuck retrying the same step.",- internal_reason_code="loop_detected_repeated_failed_step",+ internal_reason_code="loop_detected_consecutive_same_tool",blocked_tool="update_and_run_blocks",)result = _agent_result()@@ -709,7 +615,7 @@ def test_verified_outcome_preserve_sets_verified_success_payload() -> None:ctx.blocker_signal = _signal(kind="loop_detected",user_facing="I'm stuck retrying the same step.",- internal_reason_code="loop_detected_repeated_failed_step",+ internal_reason_code="loop_detected_consecutive_same_tool",blocked_tool="update_and_run_blocks",)@@ -863,7 +769,7 @@ def test_unverified_blocker_still_renders_blocker_text() -> None:ctx.blocker_signal = _signal(kind="loop_detected",user_facing="I'm stuck retrying the same step.",- internal_reason_code="loop_detected_repeated_failed_step",+ internal_reason_code="loop_detected_consecutive_same_tool",blocked_tool="update_and_run_blocks",)@@ -873,142 +779,6 @@ def test_unverified_blocker_still_renders_blocker_text() -> None:assert overridden.proposal_disposition != "review_tested"-def test_delivered_unverified_exit_renders_observed_output_as_unvalidated() -> None:- ctx = _ctx()- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- ctx.last_test_ok = True- ctx.last_full_workflow_test_ok = False- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"document_name": "Resale Demand Package"}-- result = _build_wip_exit_result(- ctx,- None,- default_reply="default",- unvalidated_reply="unvalidated",- tested_reply="tested",- terminal_reason="turn_halt:delivered_unverified",- )-- assert "document_name: Resale Demand Package" in result.user_response- assert "not independently verified" in result.user_response- assert result.proposal_disposition == "review_untested"- assert result.narrative_payload is not None- assert result.narrative_payload["deliveredUnverifiedObservedOutputs"]["document_name"] == "Resale Demand Package"- assert result.workflow_yaml == ctx.last_workflow_yaml---def test_delivered_unverified_exit_precedes_distinct_last_good_workflow_fallback() -> None:- ctx = _ctx()- ctx.last_good_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_good_workflow_yaml = "workflow_definition:\n blocks:\n - label: tested-prior\n"- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks:\n - label: delivered-current\n"- ctx.last_test_ok = True- ctx.last_full_workflow_test_ok = False- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"result": {"confirmation": "confirm-current"}}-- result = _build_wip_exit_result(- ctx,- None,- default_reply="default",- unvalidated_reply="unvalidated",- tested_reply="tested prior workflow",- terminal_reason="turn_halt:delivered_unverified",- )-- assert 'result["confirmation"]: confirm-current' in result.user_response- assert "tested prior workflow" not in result.user_response- assert result.updated_workflow is ctx.last_workflow- assert result.workflow_yaml == ctx.last_workflow_yaml- assert result.proposal_disposition == "review_untested"- assert result.narrative_payload is not None- assert result.narrative_payload["deliveredUnverifiedObservedOutputs"]["result"] == {- "confirmation": "confirm-current"- }---def test_delivered_unverified_halt_without_workflow_renders_observed_output() -> None:- ctx = _ctx()- ctx.last_run_blocks_workflow_run_id = "wr_fallback"- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {- "result": {"account": "acct-fallback", "confirmation": "confirm-fallback", "amount": 0}- }-- result = _build_turn_halt_exit_result(- ctx,- global_llm_context=None,- halt=TurnHalt(kind=TurnHaltKind.DELIVERED_UNVERIFIED),- )-- assert 'result["amount"]: 0' in result.user_response- assert 'result["account"]: acct-fallback' in result.user_response- assert 'result["confirmation"]: confirm-fallback' in result.user_response- assert "not independently verified" in result.user_response- assert result.updated_workflow is None- assert result.narrative_payload is not None- observed_outputs = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed_outputs["result"] == ctx.delivered_unverified_observed_outputs["result"]---def test_give_up_halt_names_demonstrated_missing_steps() -> None:- ctx = _ctx()- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- ctx.has_staged_proposal = True- ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER- ctx.impose_synthesized_code_block = True- ctx.persisted_draft_browser_calls = []- ctx.scout_trajectory = [- {- "tool_name": "click",- "selector": "#search-submit",- "source_url": "https://example.com/search",- "trajectory_index": 0,- }- ]-- result = _build_turn_halt_exit_result(- ctx,- global_llm_context=None,- halt=TurnHalt(kind=TurnHaltKind.LOOP_DETECTED),- )-- assert "#search-submit" in result.user_response---def test_delivered_unverified_halt_names_demonstrated_missing_steps() -> None:- ctx = _ctx()- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- ctx.last_test_ok = True- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"result": {"amount": 0}}- ctx.has_staged_proposal = True- ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER- ctx.impose_synthesized_code_block = True- ctx.persisted_draft_browser_calls = []- ctx.scout_trajectory = [- {- "tool_name": "click",- "selector": "#search-submit",- "source_url": "https://example.com/search",- "trajectory_index": 0,- }- ]-- result = _build_turn_halt_exit_result(- ctx,- global_llm_context=None,- halt=TurnHalt(kind=TurnHaltKind.DELIVERED_UNVERIFIED),- )-- assert "#search-submit" in result.user_response--def _scouted_obligation_ctx() -> CopilotContext:ctx = _ctx()ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))@@ -1028,293 +798,6 @@ def _scouted_obligation_ctx() -> CopilotContext:return ctx-def test_timeout_wip_exit_names_demonstrated_missing_steps() -> None:- ctx = _scouted_obligation_ctx()-- result = agent_module._build_timeout_exit_result(ctx, global_llm_context=None)-- assert "#search-submit" in result.user_response---def test_output_policy_blocked_result_names_demonstrated_missing_steps() -> None:- ctx = _scouted_obligation_ctx()- verdict = OutputPolicyVerdict(- allowed=False,- output_kind=CopilotOutputKind.REFUSAL,- reason_codes=[OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE],- )-- result = _build_output_policy_blocked_result(ctx, verdict, prior_global_llm_context=None, prior_workflow_yaml=None)-- assert "#search-submit" in result.user_response---def test_wip_exit_render_path_names_missing_steps_under_unrelated_blocker() -> None:- ctx = _scouted_obligation_ctx()- unrelated_reason = "I couldn't finish this after several attempts. Tell me what to change and I'll try again."- ctx.blocker_signal = CopilotToolBlockerSignal(- blocker_kind="authority_denied",- agent_steering_text="An unrelated blocker owns the turn.",- user_facing_reason=unrelated_reason,- recovery_hint="report_blocker_to_user",- internal_reason_code="output_contract_actuation_exhausted",- blocked_tool="update_workflow",- )-- result = _build_wip_exit_result(ctx, None, default_reply="wip", unvalidated_reply="wip", tested_reply="wip")-- assert "#search-submit" in result.user_response- assert result.user_response.count("This draft is still missing steps you demonstrated:") == 1- assert ctx.blocker_signal.user_facing_reason == unrelated_reason---def test_delivered_unverified_exit_prioritizes_nested_scalars_and_accounts_for_omissions() -> None:- ctx = _ctx()- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- ctx.last_test_ok = True- ctx.last_full_workflow_test_ok = False- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {- "result": {- "evidence": "e" * 240,- "account": "acct-314159",- "confirmation": "confirm-271828",- "amount": 0,- **{f"long_{index}": str(index) * 240 for index in range(64)},- }- }-- result = _build_wip_exit_result(- ctx,- None,- default_reply="default",- unvalidated_reply="unvalidated",- tested_reply="tested",- terminal_reason="turn_halt:delivered_unverified",- )-- assert 'result["amount"]: 0' in result.user_response- assert 'result["account"]: acct-314159' in result.user_response- assert 'result["confirmation"]: confirm-271828' in result.user_response- assert 'result["evidence"]: ' in result.user_response- assert "..." in result.user_response- assert "more fields available in structured output" in result.user_response- assert result.user_response.index('result["amount"]') < result.user_response.index('result["evidence"]')- assert "not independently verified" in result.user_response- assert result.proposal_disposition == "review_untested"- assert result.narrative_payload is not None- observed_outputs = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed_outputs["result"]["account"] == "acct-314159"- assert observed_outputs["result"]["confirmation"] == "confirm-271828"- assert observed_outputs["result"]["amount"] == 0- assert observed_outputs["result"]["evidence"] == "e" * 240- assert observed_outputs["$skyvernOutput"]["omitted"]["node"] >= 1---def test_delivered_unverified_exit_uses_collision_safe_paths() -> None:- ctx = _ctx()- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- ctx.last_test_ok = True- ctx.last_full_workflow_test_ok = False- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {- "result.value": "top-level",- "result": {"value": "nested", "items": [{"value": True}]},- }-- result = _build_wip_exit_result(- ctx,- None,- default_reply="default",- unvalidated_reply="unvalidated",- tested_reply="tested",- terminal_reason="turn_halt:delivered_unverified",- )-- assert "result.value: top-level" in result.user_response- assert 'result["value"]: nested' in result.user_response- assert 'result["items"][0]["value"]: true' in result.user_response---def test_delivered_unverified_exit_preserves_sanitized_key_collisions() -> None:- ctx = _ctx()- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- ctx.last_test_ok = True- ctx.last_full_workflow_test_ok = False- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.extend(["registered-secret-a", "registered-secret-b"])- ctx.delivered_unverified_observed_outputs = {- "result": {- "field-registered-secret-a": "first-captured-value",- "field-registered-secret-b": "second-captured-value",- }- }-- result = _build_wip_exit_result(- ctx,- None,- default_reply="default",- unvalidated_reply="unvalidated",- tested_reply="tested",- terminal_reason="turn_halt:delivered_unverified",- )-- assert "first-captured-value" in result.user_response- assert "second-captured-value" in result.user_response- assert result.narrative_payload is not None- observed_result = result.narrative_payload["deliveredUnverifiedObservedOutputs"]["result"]- assert list(observed_result) == ["field-[REDACTED_SECRET]", "field-[REDACTED_SECRET] [2]"]- assert list(observed_result.values()) == ["first-captured-value", "second-captured-value"]- assert len(observed_result) == 2--[email protected]("has_workflow", [True, False], ids=["with-workflow", "without-workflow"])-def test_delivered_unverified_exit_omits_depth_over_budget_without_recursion(has_workflow: bool) -> None:- ctx = _ctx()- if has_workflow:- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- else:- ctx.last_run_blocks_workflow_run_id = "wr_deep_fallback"- ctx.last_test_ok = True- ctx.last_full_workflow_test_ok = False- ctx.delivered_unverified_terminal = True- deeply_nested: dict[str, Any] = {"leaf": "deep-captured-value"}- for index in range(1_101):- deeply_nested = {f"level_{index}": deeply_nested}- ctx.delivered_unverified_observed_outputs = {"result": deeply_nested}-- result = _build_wip_exit_result(- ctx,- None,- default_reply="default",- unvalidated_reply="unvalidated",- tested_reply="tested",- terminal_reason="turn_halt:delivered_unverified",- )-- assert result.user_response- assert "not independently verified" in result.user_response- assert result.narrative_payload is not None- observed_outputs = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed_outputs["$skyvernOutput"]["omitted"]["depth"] >= 1- assert "deep-captured-value" not in str(observed_outputs)---def test_delivered_unverified_exit_sanitizes_both_output_surfaces() -> None:- ctx = _ctx()- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- ctx.last_test_ok = True- ctx.last_full_workflow_test_ok = False- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.append("registered-secret-value")- deeply_nested: dict[str, Any] = {"password": "deep-must-not-persist"}- for index in range(24):- deeply_nested = {f"level_{index}": deeply_nested}- ctx.delivered_unverified_observed_outputs = {- "result": {- "empty": "",- "password": "must-not-persist",- "registered": "prefix registered-secret-value suffix",- "raw": "api_key=sk-1234567890abcdefghijkl",- "internal": "run wr_internal_123 with update_workflow",- "deep": deeply_nested,- "api_key=sk-raw-secret-key-1234567890": "safe-value",- 7: "non-string-key-value",- }- }-- result = _build_wip_exit_result(- ctx,- None,- default_reply="default",- unvalidated_reply="unvalidated",- tested_reply="tested",- terminal_reason="turn_halt:delivered_unverified",- )-- assert 'result["empty"]: ""' in result.user_response- assert 'result["password"]: ****' in result.user_response- assert "registered-secret-value" not in result.user_response- assert "sk-1234567890abcdefghijkl" not in result.user_response- assert "wr_internal_123" not in result.user_response- assert "update_workflow" not in result.user_response- assert "deep-must-not-persist" not in result.user_response- assert "sk-raw-secret-key-1234567890" not in result.user_response- assert "non-string-key-value" not in result.user_response- assert result.narrative_payload is not None- assert "must-not-persist" not in str(result.narrative_payload)- assert "registered-secret-value" not in str(result.narrative_payload)- assert "sk-1234567890abcdefghijkl" not in str(result.narrative_payload)- assert "wr_internal_123" not in str(result.narrative_payload)- assert "update_workflow" not in str(result.narrative_payload)- assert "deep-must-not-persist" not in str(result.narrative_payload)- assert "sk-raw-secret-key-1234567890" not in str(result.narrative_payload)- assert "non-string-key-value" not in str(result.narrative_payload)--[email protected]("has_workflow", [True, False], ids=["with-workflow", "without-workflow"])-def test_delivered_unverified_exit_threads_one_snapshot_to_prose_and_payload(- monkeypatch: pytest.MonkeyPatch,- has_workflow: bool,-) -> None:- ctx = _ctx()- if has_workflow:- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- else:- ctx.last_run_blocks_workflow_run_id = "wr_single_snapshot_fallback"- ctx.last_test_ok = True- ctx.last_full_workflow_test_ok = False- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"result": {"amount": 0}}- sanitize = agent_module._delivered_unverified_observed_outputs- make_agent_result = agent_module._make_agent_result- delivered_unverified_reply = agent_module._delivered_unverified_reply- calls = 0- prose_snapshot: dict[str, Any] | None = None- payload_snapshot: dict[str, Any] | None = None-- def count_sanitization(context: CopilotContext) -> dict[str, Any]:- nonlocal calls- calls += 1- return sanitize(context)-- def capture_prose_snapshot(context: CopilotContext, snapshot: dict[str, Any]) -> str | None:- nonlocal prose_snapshot- prose_snapshot = snapshot- return delivered_unverified_reply(context, snapshot)-- def capture_payload_snapshot(*args: Any, **kwargs: Any) -> AgentResult:- nonlocal payload_snapshot- payload_snapshot = kwargs.get("_delivered_unverified_snapshot")- return make_agent_result(*args, **kwargs)-- monkeypatch.setattr(agent_module, "_delivered_unverified_observed_outputs", count_sanitization)- monkeypatch.setattr(agent_module, "_delivered_unverified_reply", capture_prose_snapshot)- monkeypatch.setattr(agent_module, "_make_agent_result", capture_payload_snapshot)-- result = _build_wip_exit_result(- ctx,- None,- default_reply="default",- unvalidated_reply="unvalidated",- tested_reply="tested",- terminal_reason="turn_halt:delivered_unverified",- )-- assert calls == 1- assert prose_snapshot is payload_snapshot- assert 'result["amount"]: 0' in result.user_response- assert result.proposal_disposition == "review_untested"- assert result.narrative_payload is not None- assert result.narrative_payload["deliveredUnverifiedObservedOutputs"]["result"] == {"amount": 0}--def test_verified_outcome_does_not_suppress_voluntary_terminal_challenge() -> None:ctx = _ctx()_seed_verified_outcome(ctx)@@ -1400,21 +883,6 @@ def test_output_policy_blocked_result_surfaces_draft_when_blocker_render_denied(assert any(event.site == "final_reply_render" for event in ctx.gate_precedence_conflict_events)-def test_reconcile_turn_end_replaces_result_with_stalled_terminal() -> None:- ctx = _ctx()- ctx.output_contract_actuation_by_signature["sig_a"] = OutputContractAdvisoryState.GRANTED- ctx.output_contract_pending_run_evidence["sig_a"] = ["output.confirmation_number"]- original = _agent_result("Model reply describing progress.")-- result = agent_module._reconcile_turn_end_ownership(ctx, original)-- assert result is not original- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind is TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE- assert ctx.output_contract_actuation_by_signature["sig_a"] == OutputContractAdvisoryState.EXPIRED- assert "I couldn't shape this workflow" in result.user_response--def test_reconcile_turn_end_noop_without_live_grant() -> None:ctx = _ctx()ctx.output_contract_actuation_by_signature["sig_a"] = OutputContractAdvisoryState.CONSUMEDdiff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 0c366579b..b382c6f4b 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -21,7 +21,6 @@ from skyvern.forge.sdk.copilot import request_policy as request_policy_modulefrom skyvern.forge.sdk.copilot import tools as tools_modulefrom skyvern.forge.sdk.copilot.agent import (_VERIFIED_WORKFLOW_SUCCESS_REPLY,- _build_built_unverified_exit_result,_build_goal_satisfied_exit_result,_resolve_wrapped_exception_exit_result,_synthesized_block_offer_prompt,@@ -62,18 +61,13 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (VerificationResult,)from skyvern.forge.sdk.copilot.enforcement import (- BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,- CopilotBuiltUnverified,- CopilotGoalSatisfied,CopilotNonRetriableNavError,CopilotTotalTimeoutError,CopilotUnrecoverableToolError,- _check_enforcement,built_unverified_repair_inert_context,verified_goal_claim_authorized,verified_goal_satisfied_context,)-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODEfrom skyvern.forge.sdk.copilot.hooks import CopilotRunHooksfrom skyvern.forge.sdk.copilot.output_policy import (ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,@@ -482,21 +476,6 @@ workflow_definition:assert "add them via the Credentials UI" in rewrittenassert "Keep the draft to iterate on, or discard." in rewritten- def test_rewrite_appends_keep_draft_affordance_when_draft_on_hand(self) -> None:- from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response-- ctx = _ctx(- last_workflow=object(),- last_workflow_yaml="title: drafted",- last_update_block_count=2,- last_test_ok=False,- last_test_failure_reason="A verification challenge is preventing submission.",- )- rewritten = _rewrite_failed_test_response("done", ctx)-- assert "test failed" in rewritten.lower()- assert "keep the draft" in rewritten.lower()-def test_partial_verification_rewrite_uses_structured_evidence(self) -> None:from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response@@ -1257,45 +1236,6 @@ class TestVerifiedWorkflowOrNone:class TestVerifiedGoalSatisfiedStop:- @pytest.mark.asyncio- async def test_block_run_hook_stops_after_verified_goal_satisfied(self) -> None:- from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks-- ctx = _ctx(- last_test_ok=True,- last_full_workflow_test_ok=True,- latest_diagnosis_repair_contract=_verified_goal_contract(),- )- ctx.completion_verification_result = CompletionVerificationResult(- status="evaluated",- criterion_ids=["c0"],- verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],- )- hook = CopilotRunHooks(ctx)- result = json.dumps(- {- "ok": True,- "data": {- "workflow_run_id": "wr_1",- "blocks": [{"label": "search", "output": {"status": "found"}}],- },- }- )-- with pytest.raises(CopilotGoalSatisfied):- await hook.on_tool_end(- context=MagicMock(),- agent=MagicMock(),- tool=SimpleNamespace(name="update_and_run_blocks"),- result=result,- )-- assert ctx.goal_satisfied_tool_name == "update_and_run_blocks"- assert ctx.goal_satisfied_tool_output is not None- assert ctx.goal_satisfied_tool_output["ok"] is True-- assert ctx.tool_activity[-1]["tool"] == "update_and_run_blocks"-def test_wrapped_goal_satisfied_error_context_is_recognized(self) -> None:from skyvern.forge.sdk.copilot.enforcement import verified_goal_satisfied_context@@ -1395,32 +1335,6 @@ class TestVerifiedGoalSatisfiedStop:assert result.user_response == _VERIFIED_WORKFLOW_SUCCESS_REPLYassert result.proposal_disposition != "no_proposal"- def test_verified_turn_halt_suppresses_active_run_terminal_evidence_only(self) -> None:- ctx = _ctx()- signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="active run reached requested state",- user_facing_reason="active run reached requested state",- recovery_hint="report_blocker_to_user",- internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,- blocked_tool="update_and_run_blocks",- )- ctx.blocker_signal = signal- ctx.latest_tool_blocker_signal = signal- ctx.tool_blocker_signals.append(signal)- ctx.turn_halt = TurnHalt(- kind=TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE,- blocker_signal=signal,- extra={"source": "run_execution"},- )-- raise_if_turn_halt(ctx, verified=True)-- assert not isinstance(ctx.turn_halt, TurnHalt)- assert ctx.blocker_signal is None- assert ctx.latest_tool_blocker_signal is None- assert ctx.tool_blocker_signals == []-def test_verified_turn_halt_keeps_terminal_challenge_blocker(self) -> None:ctx = _ctx()signal = CopilotToolBlockerSignal(@@ -1564,30 +1478,6 @@ class TestVerifiedGoalSatisfiedStop:assert not verified_goal_satisfied_context(ctx)- def test_check_enforcement_stops_after_structural_built_unverified_terminal(self) -> None:- ctx = _ctx(- last_test_ok=True,- last_full_workflow_test_ok=True,- latest_diagnosis_repair_contract=_unverified_no_repair_contract(),- completion_verification_result=CompletionVerificationResult(- status="evaluated",- criterion_ids=["c0"],- verdicts=[- CriterionVerdict(- criterion_id="c0",- state="unsatisfied",- reason_code="structurally_abstained",- )- ],- ),- )-- assert verified_goal_satisfied_context(ctx) is False- assert built_unverified_repair_inert_context(ctx) is True- assert verified_goal_claim_authorized(ctx) is False- with pytest.raises(CopilotBuiltUnverified):- _check_enforcement(ctx)-def test_verified_goal_satisfied_context_rejects_undercovered_workflow(self) -> None:from skyvern.forge.sdk.copilot.enforcement import verified_goal_satisfied_context@@ -1622,9 +1512,9 @@ class TestVerifiedGoalSatisfiedStop:assert result.updated_workflow is workflowassert result.workflow_yaml == "workflow_definition:\n blocks: []\n"assert result.proposal_disposition == "review_tested"- # No adjudicated outcome evidence: the turn ends but the claim renders- # built-but-unverified instead of a tested-success claim.- assert "not independently verified" in result.user_response.lower()+ # A clean test is the evidence; the claim does not wait for adjudicated outcome evidence.+ assert "not independently verified" not in result.user_response.lower()+ assert "tested" in result.user_response.lower()assert result.narrative_payload is not Noneassert result.narrative_payload["terminal"] == "response"assert result.narrative_payload["verifiedSuccess"] is False@@ -1645,82 +1535,6 @@ class TestVerifiedGoalSatisfiedStop:assert verified.narrative_payload is not Noneassert verified.narrative_payload["verifiedSuccess"] is True- @pytest.mark.asyncio- async def test_goal_satisfied_exit_result_renders_structural_abstention_as_built_unverified(self) -> None:- workflow = object()- ctx = _ctx(- last_workflow=workflow,- last_workflow_yaml="workflow_definition:\n blocks: []\n",- last_test_ok=True,- last_full_workflow_test_ok=True,- latest_diagnosis_repair_contract=_unverified_no_repair_contract(),- completion_verification_result=CompletionVerificationResult(- status="evaluated",- criterion_ids=["c0"],- verdicts=[- CriterionVerdict(- criterion_id="c0",- state="unsatisfied",- reason_code="structurally_abstained",- )- ],- ),- tool_activity=[{"tool": "update_and_run_blocks", "summary": "OK"}],- )-- assert verified_goal_satisfied_context(ctx) is False- assert built_unverified_repair_inert_context(ctx) is True- assert verified_goal_claim_authorized(ctx) is False-- result = await _build_built_unverified_exit_result(ctx, global_llm_context=None)-- assert result.updated_workflow is workflow- assert result.proposal_disposition == "review_tested"- assert "not independently verified" in result.user_response.lower()- assert result.turn_outcome is not None- assert result.turn_outcome.terminal_reason == BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON- assert result.narrative_payload is not None- assert result.narrative_payload["verifiedSuccess"] is False-- @pytest.mark.asyncio- async def test_built_unverified_terminal_names_demonstrated_missing_steps(self) -> None:- workflow = object()- ctx = _ctx(- last_workflow=workflow,- last_workflow_yaml="workflow_definition:\n blocks: []\n",- last_test_ok=True,- last_full_workflow_test_ok=True,- latest_diagnosis_repair_contract=_unverified_no_repair_contract(),- completion_verification_result=CompletionVerificationResult(- status="evaluated",- criterion_ids=["c0"],- verdicts=[- CriterionVerdict(- criterion_id="c0",- state="unsatisfied",- reason_code="structurally_abstained",- )- ],- ),- tool_activity=[{"tool": "update_and_run_blocks", "summary": "OK"}],- )- ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER- ctx.impose_synthesized_code_block = True- ctx.has_staged_proposal = True- ctx.persisted_draft_browser_calls = []- ctx.scout_trajectory = [- {- "tool_name": "click",- "selector": "#search-submit",- "source_url": "https://example.com/search",- "trajectory_index": 0,- }- ]-- result = await _build_built_unverified_exit_result(ctx, global_llm_context=None)-- assert "#search-submit" in result.user_response-def test_corroborated_structural_abstention_avoids_built_unverified_terminal(self) -> None:ctx = _ctx(last_test_ok=True,@@ -2476,34 +2290,6 @@ class TestTranslateToAgentResultGating:assert agent_result.turn_outcome is not Noneassert agent_result.turn_outcome.reason_code != ACTUATION_OBLIGATION_STEER_REASON_CODE- def test_verified_terminal_state_surfaces_workflow_despite_weak_final_reply(self) -> None:- workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))- ctx = _ctx(- last_workflow=workflow,- last_workflow_yaml="title: Verified Draft",- last_test_ok=True,- last_full_workflow_test_ok=True,- )- ctx.verified_terminal_proposal_ready = True- ctx.completion_verification_result = CompletionVerificationResult(- status="evaluated",- criterion_ids=["c0"],- verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],- )- result = _fake_run_result({"type": "ASK_QUESTION", "user_response": "Do you want me to keep repairing?"})-- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"- )- )-- assert agent_result.response_type == "REPLY"- assert agent_result.updated_workflow is workflow- assert agent_result.workflow_yaml == "title: Verified Draft"- assert agent_result.clear_proposed_workflow is False- assert agent_result.proposal_disposition == "auto_applicable"-def test_wip_exit_structural_abstention_stays_review_only(self) -> None:from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy@@ -2987,42 +2773,6 @@ workflow_definition:assert agent_result.narrative_payload is not Noneassert agent_result.narrative_payload["responseType"] == "ASK_QUESTION"- def test_probable_site_block_ask_question_is_concise_and_proxy_aware(self) -> None:- ctx = _ctx(- last_test_ok=False,- last_test_failure_reason="Skyvern failed to load the website. The page may have navigated unexpectedly.",- probable_site_block_stop_nudge_count=1,- effective_workflow_proxy_location="RESIDENTIAL",- )- verbose_response = (- "Diagnostic recap:\n"- "- I tried several workflow shapes with the same browser state.\n"- '- global_llm_context: {"workflow_state": "many internal details"}\n'- "- The final failure_reason was: Skyvern failed to load the website. "- "The page may have navigated unexpectedly.\n"- "- More implementation details that should not be user-facing.\n"- "Would you like me to configure a proxy?"- )- result = _fake_run_result({"type": "ASK_QUESTION", "user_response": verbose_response})-- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"- )- )-- assert agent_result.response_type == "ASK_QUESTION"- assert len(agent_result.user_response.splitlines()) <= 8- assert "global_llm_context" not in agent_result.user_response- assert "configure a proxy" not in agent_result.user_response.lower()- assert "Would you like me to whether" not in agent_result.user_response- assert "different proxy location" in agent_result.user_response.lower()- assert "US-CA" in agent_result.user_response- assert "Skyvern failed to load the website. The page may have navigated unexpectedly." in (- agent_result.user_response- )- assert "same IP/workflow shape" in agent_result.user_response-def test_unexpected_error_exit_names_failure_and_preserves_context(self) -> None:ctx = _ctx()@@ -3165,73 +2915,6 @@ workflow_definition:assert "test failed" in agent_result.user_response.lower()assert "keep the draft" in agent_result.user_response.lower()- def test_reply_after_suspicious_success_surfaces_unvalidated_wip(self) -> None:- wf = SimpleNamespace(name="drafted")- ctx = _ctx(- last_workflow=wf,- last_workflow_yaml="title: drafted",- last_update_block_count=2,- last_test_ok=None,- last_test_suspicious_success=True,- )- result = _fake_run_result({"type": "REPLY", "user_response": "Done."})- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"- )- )-- assert agent_result.updated_workflow is wf- assert agent_result.proposal_disposition == "review_untested"- assert "review" in agent_result.user_response.lower()- assert "accept" in agent_result.user_response.lower()- assert "reject" in agent_result.user_response.lower()- assert "discard" in agent_result.user_response.lower()-- def test_unvalidated_wip_reply_adds_proposal_affordance(self) -> None:- wf = SimpleNamespace(name="drafted")- ctx = _ctx(- last_workflow=wf,- last_workflow_yaml="title: drafted",- last_update_block_count=None,- last_test_ok=None,- )- result = _fake_run_result({"type": "REPLY", "user_response": "Please provide credentials before I continue."})- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"- )- )-- assert agent_result.updated_workflow is wf- assert agent_result.workflow_yaml == "title: drafted"- assert agent_result.proposal_disposition == "review_untested"- assert "Please provide credentials before I continue." in agent_result.user_response- assert "review" in agent_result.user_response.lower()- assert "accept" in agent_result.user_response.lower()- assert "reject" in agent_result.user_response.lower()- assert "discard" in agent_result.user_response.lower()-- def test_unvalidated_wip_reply_keeps_existing_ui_affordance(self) -> None:- wf = SimpleNamespace(name="drafted")- ctx = _ctx(- last_workflow=wf,- last_workflow_yaml="title: drafted",- last_update_block_count=None,- last_test_ok=None,- )- response = "I have a draft proposal. Use Review to inspect it, Accept to save it, or Reject it."- result = _fake_run_result({"type": "REPLY", "user_response": response})- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"- )- )-- assert response in agent_result.user_response- assert "not been tested or verified" in agent_result.user_response- assert agent_result.updated_workflow is wf-def test_goal_reached_false_flips_validated_proposal_to_unvalidated(self) -> None:# Agent-emitted goal_reached=False must override last_test_ok=True so# a draft the agent itself flagged as incomplete cannot auto-promote.@@ -3260,136 +2943,6 @@ workflow_definition:assert agent_result.workflow_yaml == "title: drafted"assert agent_result.proposal_disposition == "review_untested"- def test_goal_reached_false_cannot_underclaim_verified_outcome(self) -> None:- from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict-- wf = SimpleNamespace(name="verified")- ctx = _ctx(- last_workflow=wf,- last_workflow_yaml="title: verified",- last_test_ok=True,- last_full_workflow_test_ok=False,- last_update_block_count=4,- last_run_outcome=RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_secret"),- completion_verification_result=CompletionVerificationResult(- status="evaluated",- criterion_ids=["c0"],- verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],- ),- )- result = _fake_run_result(- {- "type": "REPLY",- "user_response": "I drafted the workflow, but it has not been tested end-to-end.",- "goal_reached": False,- }- )-- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"- )- )-- assert agent_result.updated_workflow is wf- assert agent_result.workflow_yaml == "title: verified"- assert agent_result.proposal_disposition == "auto_applicable"- assert "draft" not in agent_result.user_response.lower()- assert "not been tested" not in agent_result.user_response.lower()- assert "tested end-to-end" not in agent_result.user_response.lower()- assert "created and tested" in agent_result.user_response.lower()- assert "demonstrated the requested outcome" in agent_result.user_response.lower()- assert "wr_secret" not in agent_result.user_response- assert agent_result.narrative_payload is not None- assert agent_result.narrative_payload["verifiedSuccess"] is True-- def test_demonstrated_recorded_outcome_overrides_misleading_free_text(self) -> None:- from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict-- wf = SimpleNamespace(name="verified")- ctx = _ctx(- last_workflow=wf,- last_workflow_yaml="title: verified",- last_test_ok=True,- last_full_workflow_test_ok=True,- last_run_outcome=RecordedRunOutcome(verdict="demonstrated"),- completion_verification_result=CompletionVerificationResult(- status="evaluated",- criterion_ids=["c0"],- verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],- ),- )- result = _fake_run_result(- {- "type": "REPLY",- "user_response": "The test could not verify the requested result.",- "goal_reached": True,- }- )-- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"- )- )-- assert agent_result.updated_workflow is wf- assert agent_result.proposal_disposition == "auto_applicable"- assert "demonstrated the requested outcome" in agent_result.user_response.lower()- assert "could not verify" not in agent_result.user_response.lower()-- def test_not_demonstrated_recorded_outcome_stays_conservative_and_sanitized(self) -> None:- wf = SimpleNamespace(name="drafted")- ctx = _ctx(- last_workflow=wf,- last_workflow_yaml="title: drafted",- last_test_ok=True,- last_full_workflow_test_ok=True,- last_run_outcome=RecordedRunOutcome(- verdict="not_demonstrated",- display_reason="Statement month was still April.",- workflow_run_id="wr_hidden",- ),- )- result = _fake_run_result({"type": "REPLY", "user_response": "All set.", "goal_reached": True})-- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"- )- )-- assert agent_result.updated_workflow is wf- assert agent_result.proposal_disposition == "review_untested"- assert "did not demonstrate the requested outcome" in agent_result.user_response.lower()- assert "statement month was still april" in agent_result.user_response.lower()- assert "all set" not in agent_result.user_response.lower()- assert "wr_hidden" not in agent_result.user_response-- def test_not_evaluated_recorded_outcome_reports_run_unverified_not_untested(self) -> None:- wf = SimpleNamespace(name="drafted")- ctx = _ctx(- last_workflow=wf,- last_workflow_yaml="title: drafted",- last_test_ok=True,- last_full_workflow_test_ok=True,- last_run_outcome=RecordedRunOutcome(verdict="not_evaluated", display_reason="Output judge unavailable."),- )- result = _fake_run_result({"type": "REPLY", "user_response": "All set.", "goal_reached": True})-- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"- )- )-- assert agent_result.updated_workflow is wf- assert agent_result.proposal_disposition == "review_untested"- assert "could not verify the requested outcome" in agent_result.user_response.lower()- assert "did not demonstrate" not in agent_result.user_response.lower()- assert "output judge unavailable" in agent_result.user_response.lower()- assert "untested" not in agent_result.user_response.lower()-@pytest.mark.parametrize("payload_extras",[@@ -3601,7 +3154,9 @@ workflow_definition:assert agent_result.updated_workflow is Noneassert agent_result.workflow_yaml is None- def test_clean_unverified_run_uses_deterministic_terminal_copy_when_proposal_exists(self) -> None:+ def test_clean_test_keeps_the_models_reply_without_a_judge_cosign(self) -> None:+ """A clean test is the evidence; a separate judge's reading of the same run does not+ rewrite the model's reply into built-but-unverified copy."""wf = SimpleNamespace(name="drafted")ctx = _ctx(last_workflow=wf,@@ -3616,10 +3171,7 @@ workflow_definition:))- assert agent_result.user_response == (- "I built the workflow and the test run completed, but the goal outcome was not independently verified. "- "The workflow is available on the canvas for review."- )+ assert "not independently verified" not in agent_result.user_responseassert agent_result.updated_workflow is wfdef test_goal_reached_false_on_failed_test_does_not_double_unvalidate(self) -> None:@@ -3752,7 +3304,7 @@ workflow_definition:assert agent_result.response_type == "REPLY"assert agent_result.clear_proposed_workflow is False- def test_reply_with_unverified_clean_run_uses_deterministic_terminal_copy(self) -> None:+ def test_reply_with_clean_run_keeps_model_reply_without_judge_cosign(self) -> None:workflow = SimpleNamespace(name="final")ctx = _ctx(last_workflow=workflow,@@ -3768,8 +3320,8 @@ workflow_definition:)assert agent_result.updated_workflow is workflow- assert "the workflow is ready" not in agent_result.user_response.lower()- assert "not independently verified" in agent_result.user_response.lower()+ assert "the workflow is ready" in agent_result.user_response.lower()+ assert "not independently verified" not in agent_result.user_response.lower()assert agent_result.proposal_disposition == "auto_applicable"assert agent_result.narrative_payload is not Noneassert agent_result.narrative_payload["verifiedSuccess"] is False@@ -7245,3 +6797,83 @@ def test_transcript_anchor_blanked_when_retained_window_at_capacity() -> None:assert agent_module._transcript_anchor_for_turn(packet, CHAT_HISTORY_CONTEXT_MESSAGES) == ""assert agent_module._transcript_anchor_for_turn(packet, CHAT_HISTORY_CONTEXT_MESSAGES + 5) == ""assert agent_module._transcript_anchor_for_turn(None, 0) == ""+++class TestTheModelOwnsItsClaim:+ """The harness renders the run record beside the model's reply; it never composes the reply,+ never ends the loop on the model's behalf, and never promotes over the model's own admission."""++ def test_model_reply_survives_a_clean_run(self) -> None:+ wf = SimpleNamespace(name="drafted")+ ctx = _ctx(+ last_workflow=wf,+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ last_run_outcome=RecordedRunOutcome(verdict="demonstrated"),+ )+ result = _fake_run_result({"type": "REPLY", "user_response": "Logged in; extraction is next."})++ agent_result = asyncio.run(+ agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )+ )++ assert "extraction is next" in agent_result.user_response.lower()+ assert "created and tested the workflow successfully" not in agent_result.user_response.lower()++ def test_goal_reached_false_is_not_overridden_by_a_demonstrated_record(self) -> None:+ wf = SimpleNamespace(name="drafted")+ ctx = _ctx(+ last_workflow=wf,+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ last_run_outcome=RecordedRunOutcome(verdict="demonstrated"),+ )+ result = _fake_run_result(+ {"type": "REPLY", "user_response": "Only the login is built so far.", "goal_reached": False}+ )++ agent_result = asyncio.run(+ agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )+ )++ assert agent_result.proposal_disposition != "auto_applicable"+ assert "only the login is built" in agent_result.user_response.lower()++ def test_a_non_demonstrated_record_is_appended_and_caps_the_disposition(self) -> None:+ wf = SimpleNamespace(name="drafted")+ ctx = _ctx(+ last_workflow=wf,+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ last_run_outcome=RecordedRunOutcome(+ verdict="not_demonstrated", display_reason="Statement month was still April."+ ),+ )+ result = _fake_run_result({"type": "REPLY", "user_response": "All set.", "goal_reached": True})++ agent_result = asyncio.run(+ agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )+ )++ assert "all set" in agent_result.user_response.lower()+ assert "did not demonstrate the requested outcome" in agent_result.user_response.lower()+ assert "statement month was still april" in agent_result.user_response.lower()+ assert agent_result.proposal_disposition != "auto_applicable"+++def test_an_element_state_timeout_does_not_send_the_user_to_check_the_url() -> None:+ # The page loaded; an element never reached the state the block waited for. Copilot repairs that+ # itself, so the reply carries no follow-up rather than a misdirecting one.+ from skyvern.forge.sdk.copilot.agent import _FAILURE_FOLLOW_UP++ assert _FAILURE_FOLLOW_UP.get("ELEMENT_STATE_TIMEOUT", "") == ""+ assert "confirm the URL" in _FAILURE_FOLLOW_UP["PAGE_LOAD_TIMEOUT"]diff --git a/tests/unit/test_copilot_author_time_block.py b/tests/unit/test_copilot_author_time_block.pyindex e6395455f..0fd7cb656 100644--- a/tests/unit/test_copilot_author_time_block.py+++ b/tests/unit/test_copilot_author_time_block.py@@ -17,7 +17,6 @@ import pytestfrom skyvern.forge.sdk.copilot import blocker_signal as blocker_signal_modulefrom skyvern.forge.sdk.copilot import build_phase as build_phase_module-from skyvern.forge.sdk.copilot import failure_tracking as failure_tracking_modulefrom skyvern.forge.sdk.copilot import run_outcome as run_outcome_modulefrom skyvern.forge.sdk.copilot import tools as tools_modulefrom skyvern.forge.sdk.copilot.author_time_block import (@@ -404,7 +403,6 @@ def test_every_loop_plane_refusal_it_can_emit_is_declared() -> None:carried = {blocker_signal_module.SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,blocker_signal_module.DISCOVERY_EXHAUSTED_NO_ENTRY_URL_REASON_CODE,- failure_tracking_module.ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,run_outcome_module.TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,blockers_module._POST_BUDGET_CHALLENGE_RESULT_EVIDENCE_REASON,blockers_module._POST_BUDGET_CHALLENGE_BLOCKER_REASON,diff --git a/tests/unit/test_copilot_banned_block_policy.py b/tests/unit/test_copilot_banned_block_policy.pyindex 316d042ca..f26a19424 100644--- a/tests/unit/test_copilot_banned_block_policy.py+++ b/tests/unit/test_copilot_banned_block_policy.py@@ -252,9 +252,10 @@ def test_code_only_authoring_prompt_requires_idempotent_credential_login() -> Noprompt = _code_only_browser_authoring_prompt()assert "Credentialed login code must be idempotent" in prompt- assert "already-authenticated page anchor" in prompt- assert "only fill username/password" in prompt- assert "login fields are visible" in prompt+ # One wait on the two states together, then a no-wait branch — waiting on the login form alone+ # spends the whole timeout proving it is absent on an already-authenticated session.+ assert ".or_(" in prompt+ assert "is_visible()" in promptassert "await solve_captcha(page)" in promptassert "platform-managed verification challenge" in promptassert "await <credential_key>.otp()" in prompt@@ -759,3 +760,14 @@ def test_pre_hook_and_post_emission_reject_share_constant() -> None:assert hasattr(tools_module, "_get_block_schema_post_hook")assert hasattr(tools_module, "_detect_new_banned_blocks")assert hasattr(tools_module, "_banned_block_reject_message")+++def test_schema_guidance_teaches_a_no_wait_branch_between_two_page_states() -> None:+ # A `wait_for` on a branch that may legitimately be absent spends its whole timeout proving the+ # absence, and every repair attempt pays it again.+ from skyvern.forge.sdk.copilot.tools.banned_blocks import _code_only_browser_schema_guidance++ guidance = " ".join(_code_only_browser_schema_guidance())++ assert ".or_(" in guidance+ assert "is_visible()" in guidance or "count()" in guidancediff --git a/tests/unit/test_copilot_block_scoped_edits.py b/tests/unit/test_copilot_block_scoped_edits.pyindex 4c7ad7c2b..7ebf36f90 100644--- a/tests/unit/test_copilot_block_scoped_edits.py+++ b/tests/unit/test_copilot_block_scoped_edits.py@@ -45,6 +45,13 @@ class TestAnchoredCodeEdit:apply_block_edit(_WORKFLOW, "read_total", expected_code='"#stale"', replacement_code='"#x"')assert "changed since you read it" in str(exc.value)+ def test_a_failed_anchor_carries_the_current_code(self) -> None:+ """Without the current code in hand the cheapest next move is resending the same edit, which+ a repeated-failure loop guard then counts as being stuck."""+ with pytest.raises(BlockEditError) as exc:+ apply_block_edit(_WORKFLOW, "read_total", expected_code='"#stale"', replacement_code='"#x"')+ assert 'total = await page.inner_text("#total")' in str(exc.value)+def test_an_ambiguous_anchor_is_refused(self) -> None:workflow = _WORKFLOW.replace('total = await page.inner_text("#total")',diff --git a/tests/unit/test_copilot_blocker_signal.py b/tests/unit/test_copilot_blocker_signal.pyindex 737ca0479..69295849a 100644--- a/tests/unit/test_copilot_blocker_signal.py+++ b/tests/unit/test_copilot_blocker_signal.py@@ -155,19 +155,11 @@ def test_maybe_clear_on_tool_success_clears_consecutive_tool_loop_after_progressdef test_maybe_clear_on_tool_success_clears_loop_after_workflow_progress() -> None:ctx = _Ctx()- ctx.blocker_signal = _make(kind="loop_detected", internal_reason_code="loop_detected_repeated_failed_step")+ ctx.blocker_signal = _make(kind="loop_detected", internal_reason_code="loop_detected_consecutive_same_tool")maybe_clear_blocker_signal_on_tool_success(ctx, "update_and_run_blocks")assert ctx.blocker_signal is None-def test_maybe_clear_on_tool_success_keeps_loop_for_metadata_only_success() -> None:- ctx = _Ctx()- signal = _make(kind="loop_detected", internal_reason_code="loop_detected_repeated_failed_step")- ctx.blocker_signal = signal- maybe_clear_blocker_signal_on_tool_success(ctx, "list_credentials")- assert ctx.blocker_signal is signal--def test_clear_for_reason_codes_matches() -> None:ctx = _Ctx()ctx.blocker_signal = _make(internal_reason_code="tool_error_pending_reconciliation_no_input")@@ -276,42 +268,10 @@ def test_stash_blocker_signal_first_wins_returns_llm_payload() -> None:assert ctx.tool_blocker_signals == [first, second]-def test_stash_blocker_signal_active_terminal_replaces_per_tool_budget() -> None:- ctx = _Ctx()- budget = _make(internal_reason_code="tool_error_per_tool_budget_rerun")- active_terminal = _make(internal_reason_code="tool_error_active_run_terminal_evidence")-- stash_blocker_signal(ctx, budget)- payload = stash_blocker_signal(ctx, active_terminal)-- assert payload == active_terminal.agent_steering_text- assert ctx.blocker_signal is active_terminal- assert ctx.latest_tool_blocker_signal is active_terminal- assert ctx.tool_blocker_signals == [budget, active_terminal]---def test_stash_output_contract_terminal_replaces_held_churn_stop() -> None:- for reason_code in ("output_source_unobservable", "actuation_exhausted"):- ctx = _Ctx()- churn = _make(- kind="loop_detected",- internal_reason_code="code_authoring_guardrail_churn",- renders_final_reply=True,- )- oc_terminal = _make(- kind="tool_error",- internal_reason_code=reason_code,- renders_final_reply=True,- )- stash_blocker_signal(ctx, churn)- stash_blocker_signal(ctx, oc_terminal)- assert ctx.blocker_signal is oc_terminal--def test_stash_output_contract_terminal_replaces_any_held_loop_detected() -> None:for loop_reason in ("loop_detected_generic",- "loop_detected_repeated_failed_step",+ "loop_detected_consecutive_same_tool","loop_detected_consecutive_same_tool","credential_priority_authoring_churn",):@@ -334,23 +294,6 @@ def test_stash_loop_detected_does_not_replace_held_output_contract_terminal() ->assert ctx.blocker_signal is oc_terminal-def test_stash_churn_stop_does_not_replace_held_output_contract_terminal() -> None:- ctx = _Ctx()- oc_terminal = _make(- kind="tool_error",- internal_reason_code="output_source_unobservable",- renders_final_reply=True,- )- churn = _make(- kind="loop_detected",- internal_reason_code="code_authoring_guardrail_churn",- renders_final_reply=True,- )- stash_blocker_signal(ctx, oc_terminal)- stash_blocker_signal(ctx, churn)- assert ctx.blocker_signal is oc_terminal--def test_agent_context_and_copilot_context_blocker_signal_defaults_match() -> None:"""The field is declared on both AgentContext (parent) and CopilotContext(child) per the field-shadowing convention. Default values must stay indiff --git a/tests/unit/test_copilot_cancel_helpers.py b/tests/unit/test_copilot_cancel_helpers.pyindex 9b0d68b0c..95e4b27b4 100644--- a/tests/unit/test_copilot_cancel_helpers.py+++ b/tests/unit/test_copilot_cancel_helpers.py@@ -308,7 +308,7 @@ from skyvern.forge.sdk.copilot.tools import _maybe_clear_reconciliation_flag #def _ctx_with_pending(run_id: str | None) -> _NS:- return _NS(pending_reconciliation_run_id=run_id)+ return _NS(pending_reconciliation_run_id=run_id, last_failure_category_top=None)def test_reconciliation_flag_clears_on_matching_trusted_final_status() -> None:diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex 327192a4a..cc28dfddc 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -42,7 +42,6 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (authored_structure_signature_from_workflow,record_build_test_outcome,recorded_outcome_from_author_time_reject,- recorded_outcome_from_authoring_repair_context,recorded_outcome_from_run_blocks_result,)from skyvern.forge.sdk.copilot.code_block_preflight import (@@ -66,11 +65,6 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfigfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext, FillCarry-from skyvern.forge.sdk.copilot.enforcement import (- MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,- MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,- _check_enforcement,-)from skyvern.forge.sdk.copilot.output_contracts import (OutputContractAdvisoryState,code_block_available_binding_keys_by_label,@@ -107,12 +101,9 @@ from skyvern.forge.sdk.copilot.tools.workflow_update import (_strip_redundant_sandbox_imports_in_yaml,)from skyvern.forge.sdk.copilot.turn_halt import (- CopilotTurnHalt,TurnHalt,TurnHaltKind,TurnHaltVerdict,- _kind_for_blocker_signal,- stash_turn_halt_from_blocker_signal,)from skyvern.forge.sdk.copilot.turn_origin import TurnOriginfrom skyvern.forge.sdk.copilot.workflow_credential_utils import parse_workflow_yaml, workflow_blocks@@ -206,17 +197,6 @@ _SAFE_EXTRACTION_CODE_YAML = _yaml(""")-_RAW_SECRET_OUTPUT_POLICY_YAML = _yaml(- """- title: Registry lookup- workflow_definition:- blocks:- - block_type: navigation- label: login- navigation_goal: Type password: hunter2 into the password field.- """-)-def _code_yaml(code: str,@@ -906,15 +886,26 @@ class TestCodeSafetySeam:None,)joined = "\n".join(errors)- for expected in (- "value",- "count",- "x",- "branch_value",- "NotRunnable",- ):- assert expected in joined- assert "unresolved names: `branch_value`, `count`, `page`, `value`, `x`" in joined+ assert "class definitions unavailable in the code sandbox: `NotRunnable`" in joined+ for bound_somewhere in ("`count`", "`x`", "`branch_value`", "`value`"):+ assert bound_somewhere not in joined++ def test_conditionally_bound_name_is_not_a_seam_error(self) -> None:+ assert (+ _code_block_safety_errors(+ _code_yaml(+ """+ try:+ count_label = await page.locator("#c").inner_text()+ except Exception:+ pass+ print(count_label)+ """+ ),+ None,+ )+ == []+ )def test_name_analysis_allows_recursive_helpers(self) -> None:assert (@@ -959,7 +950,7 @@ class TestCodeSafetySeam:)assert len(errors) == 1assert "risky" in errors[0]- assert "recovered" in errors[0]+ assert "recovered" not in errors[0]assert "group" not in errors[0]def test_syntax_error_is_a_seam_error(self) -> None:@@ -1381,71 +1372,6 @@ class TestCodeRepairProgressClassification:assert result.substitutions is Noneassert ctx.authoring_parameter_binding_snapshot is None- def test_same_month_file_match_runtime_values_override_defaults_and_satisfy_definition(self) -> None:- ctx = _definition_contract_ctx()- _enable_imposition(ctx)- source_url = "https://example.com/statements"- ctx.scout_trajectory = [- {- "tool_name": "click",- "selector": "#statement-row",- "source_url": source_url,- "trajectory_index": 0,- }- ]- raw_selector = 'a[href="/files/invoice_100245_2026-05.pdf"]'- ctx.reached_download_target = ReachedDownloadTarget(- selector=raw_selector,- affordance_text="Download invoice",- download_kind="attribute",- source_step="trajectory_recency",- already_registered=False,- )- submitted = _yaml(- """- title: Download reusable invoice- workflow_definition:- parameters:- - {parameter_type: workflow, key: account_number, workflow_parameter_type: string, default_value: "stale-account"}- - {parameter_type: workflow, key: download_start_date, workflow_parameter_type: string, default_value: "2026-06-01"}- - {parameter_type: workflow, key: download_end_date, workflow_parameter_type: string, default_value: "2026-06-30"}- blocks:- - block_type: code- label: download_invoice- parameter_keys: []- code: |- await page.locator("#statement-row").click()- """- )-- result = workflow_update_module._maybe_impose_synthesized_code_block(- submitted,- ctx,- runtime_parameters={- "account_number": "100245",- "download_start_date": "2026-05-01",- "download_end_date": "2026-05-31",- },- )-- assert result.violations == []- parsed = parse_workflow_yaml(result.workflow_yaml)- assert isinstance(parsed, dict)- block = _single_code_block(parsed)- code = str(block["code"])- assert set(block["parameter_keys"]) == {- "account_number",- "download_start_date",- "download_end_date",- }- assert code.count("_scout_download_target = page.locator(") == 1- assert "_scout_entry_target = _scout_download_target" in code- assert "await _scout_download_target.click()" in code- assert "100245" not in code- assert "2026-05" not in code- assert workflow_update_module._definition_plane_preflight_reject(ctx, result.workflow_yaml) is None- assert ctx.authoring_parameter_binding_snapshot is None-def test_same_month_file_match_rebinds_literal_when_other_block_references_all_keys(self) -> None:ctx, _, values = _same_month_file_match_case()submitted = _yaml(@@ -6152,49 +6078,6 @@ class TestCompiledAuthoringImposition:block = _single_code_block(parse_workflow_yaml(ctx.workflow_yaml))assert "async with page.expect_download()" in block["code"]- @pytest.mark.asyncio- async def test_imposition_carries_reached_download_target_to_synthesized_code(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- ctx = _code_only_ctx()- _enable_imposition(ctx)- ctx.scout_trajectory = [- {- "tool_name": "click",- "selector": "div.stmt-row",- "source_url": "https://example.com/bills",- "trajectory_index": 0,- }- ]- ctx.reached_download_target = ReachedDownloadTarget(- selector='[href="/files/report.pdf"]',- affordance_text="Download PDF",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )- workflow_yaml = _yaml(- """- title: Download report- workflow_definition:- blocks:- - block_type: code- label: download_report- code: |- await page.goto("https://example.com/bills")- await page.locator("div.stmt-row").click()- """- )-- result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)-- assert result.violations == []- parsed = parse_workflow_yaml(result.workflow_yaml)- assert isinstance(parsed, dict)- block = _single_code_block(parsed)- assert "expect_download" in block["code"]- assert 'await page.locator("[href=\\"/files/report.pdf\\"]").click()' in block["code"]-def test_imposition_targets_synthesized_label_in_multi_code_workflow(self) -> None:ctx = self._download_ctx()workflow_yaml = _yaml(@@ -7593,43 +7476,6 @@ def _terminal_challenge_signal() -> CopilotToolBlockerSignal:class TestCodeAuthoringGuardrailChurnBackstop:- @pytest.mark.asyncio- async def test_counter_climbs_on_repeated_recorded_outcome_and_resets_on_accept(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- ctx = _code_only_ctx()- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1):- result = await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)- assert result["ok"] is False- assert ctx.code_authoring_guardrail_reject_count == index + 1- assert ctx.blocker_signal is None-- _stub_successful_update(monkeypatch)- accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)- assert accepted["ok"] is True- assert ctx.code_authoring_guardrail_reject_count == 0-- @pytest.mark.asyncio- async def test_accepted_persist_at_churn_ceiling_resets_counter_and_clears_held_signals(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- ctx = _code_only_ctx()- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- held = ctx.blocker_signal- assert isinstance(held, CopilotToolBlockerSignal)- assert held.internal_reason_code == "code_authoring_guardrail_churn"- assert ctx.latest_tool_blocker_signal is held-- _stub_successful_update(monkeypatch)- accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)-- assert accepted["ok"] is True- assert ctx.code_authoring_guardrail_reject_count == 0- assert ctx.blocker_signal is None- assert ctx.latest_tool_blocker_signal is None-@pytest.mark.asyncioasync def test_counter_climbs_through_credential_scout_branch(self) -> None:ctx = _code_only_ctx()@@ -7659,76 +7505,6 @@ class TestCodeAuthoringGuardrailChurnBackstop:assert result["ok"] is Trueassert ctx.code_authoring_guardrail_reject_count == 0- @pytest.mark.asyncio- async def test_output_policy_reject_records_author_time_reject_and_guardrail_count(self) -> None:- ctx = _code_only_ctx()-- result = await _update_workflow(- {"workflow_yaml": _RAW_SECRET_OUTPUT_POLICY_YAML},- ctx,- allow_missing_credentials=True,- )-- assert result["ok"] is False- assert "raw_secret_leak" in result["error"]- outcome = ctx.latest_recorded_build_test_outcome- assert outcome is not None- assert outcome.phase == "author_time_reject"- assert outcome.reason_code == "output_policy_reject"- assert outcome.is_authoritative is True- assert outcome.structural_key is not None- assert "Output policy blocked" in outcome.observed_evidence_summary- assert ctx.code_authoring_guardrail_reject_count == 1- assert ctx.blocker_signal is None-- @pytest.mark.asyncio- async def test_nth_reject_stashes_churn_signal_and_resolves_to_loop_halt(self) -> None:- ctx = _code_only_ctx()- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)-- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- churn = ctx.blocker_signal- assert isinstance(churn, CopilotToolBlockerSignal)- assert churn.internal_reason_code == "code_authoring_guardrail_churn"- assert ctx.latest_tool_blocker_signal is churn- assert _kind_for_blocker_signal(churn) is TurnHaltKind.LOOP_DETECTED- assert "safety checks rejected" in churn.user_facing_reason-- @pytest.mark.asyncio- async def test_nth_reject_defers_to_pre_existing_terminal_blocker(self) -> None:- ctx = _code_only_ctx()- terminal = _terminal_challenge_signal()- ctx.blocker_signal = terminal- ctx.latest_tool_blocker_signal = terminal-- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)-- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- assert ctx.blocker_signal is terminal- assert ctx.latest_tool_blocker_signal is terminal-- def test_churn_stop_defers_while_output_contract_ladder_active_then_fires(self) -> None:- ctx = _code_only_ctx()- ctx.output_contract_actuation_count_by_signature["sig_active"] = 1- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):- workflow_update_module._record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- assert ctx.blocker_signal is None- ctx.output_contract_actuation_count_by_signature.clear()- workflow_update_module._record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)- churn = ctx.blocker_signal- assert isinstance(churn, CopilotToolBlockerSignal)- assert churn.internal_reason_code == "code_authoring_guardrail_churn"-- def test_churn_stop_defers_while_advisory_grant_held(self) -> None:- ctx = _code_only_ctx()- ctx.output_contract_actuation_by_signature["sig_granted"] = OutputContractAdvisoryState.GRANTED- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):- workflow_update_module._record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)- assert ctx.blocker_signal is None-@pytest.mark.asyncioasync def test_mixed_credential_and_unresolved_name_reject_returns_code_repair_progress(self, monkeypatch: pytest.MonkeyPatch@@ -7806,55 +7582,6 @@ class TestCodeAuthoringGuardrailChurnBackstop:assert "authoring_repair_context" not in result["data"]assert ctx.last_code_authoring_repair_context is None- @pytest.mark.asyncio- async def test_credential_priority_reject_defers_below_higher_bound(self) -> None:- ctx = _code_only_ctx()- ctx.scout_trajectory = []-- for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1):- await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)-- assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1- assert ctx.blocker_signal is None- assert ctx.latest_tool_blocker_signal is None- assert _check_enforcement(ctx) is None-- @pytest.mark.asyncio- async def test_credential_priority_churn_stashes_credential_blocker_at_higher_bound(self) -> None:- ctx = _code_only_ctx()- ctx.scout_trajectory = []-- result: dict[str, object] = {}- for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS):- result = await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)-- assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS- assert result["ok"] is False- assert result["data"]["failure_type"] == "missing_credential_or_init"- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY- churn = ctx.blocker_signal- assert isinstance(churn, CopilotToolBlockerSignal)- assert churn.internal_reason_code == "credential_priority_authoring_churn"- assert ctx.latest_tool_blocker_signal is churn- assert _kind_for_blocker_signal(churn) is TurnHaltKind.LOOP_DETECTED-- @pytest.mark.asyncio- async def test_credential_priority_churn_renders_credential_scout_reply_through_enforcement(self) -> None:- ctx = _code_only_ctx()- ctx.scout_trajectory = []-- for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS):- await _update_workflow({"workflow_yaml": _page_evaluate_credential_collision_yaml(index)}, ctx)-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED- churn = ctx.blocker_signal- assert isinstance(churn, CopilotToolBlockerSignal)- assert churn.internal_reason_code == "credential_priority_authoring_churn"- assert churn.user_facing_reason == CREDENTIAL_SCOUT_VERIFY_REPLY-@pytest.mark.asyncioasync def test_single_pure_credential_reject_defers_to_credential_scout_reply(self) -> None:ctx = _code_only_ctx()@@ -7898,67 +7625,6 @@ class TestCodeAuthoringGuardrailChurnBackstop:assert result["data"]["failure_type"] == "missing_credential_or_init"assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY- @pytest.mark.asyncio- async def test_pure_credential_priority_churn_climbs_and_halts(self) -> None:- ctx = _code_only_ctx()- ctx.scout_trajectory = []-- result: dict[str, object] = {}- for index in range(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1):- result = await _update_workflow({"workflow_yaml": _safe_credential_collision_yaml(index)}, ctx)- assert result["ok"] is False- assert result["data"]["failure_type"] == "missing_credential_or_init"- assert "diagnostic_code_safety_errors" not in result["data"]- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY- assert ctx.code_authoring_guardrail_reject_count == index + 1-- assert ctx.blocker_signal is None- assert _check_enforcement(ctx) is None-- result = await _update_workflow(- {"workflow_yaml": _safe_credential_collision_yaml(MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1)}, ctx- )-- assert ctx.code_authoring_guardrail_reject_count == MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS- assert result["user_facing_summary"] == CREDENTIAL_SCOUT_VERIFY_REPLY- churn = ctx.blocker_signal- assert isinstance(churn, CopilotToolBlockerSignal)- assert churn.internal_reason_code == "credential_priority_authoring_churn"- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED- assert churn.user_facing_reason == CREDENTIAL_SCOUT_VERIFY_REPLY-- @pytest.mark.asyncio- async def test_identical_authoring_recorded_outcome_still_halts_at_backstop(self) -> None:- ctx = _code_only_ctx()- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):- await _update_workflow({"workflow_yaml": _code_yaml("value = missing_first_name()")}, ctx)-- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- churn = ctx.blocker_signal- assert isinstance(churn, CopilotToolBlockerSignal)- assert churn.internal_reason_code == "code_authoring_guardrail_churn"-- @pytest.mark.asyncio- async def test_accept_after_latched_churn_clears_both_signals(self, monkeypatch: pytest.MonkeyPatch) -> None:- ctx = _code_only_ctx()- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)- churn = ctx.blocker_signal- assert isinstance(churn, CopilotToolBlockerSignal)- assert churn.internal_reason_code == "code_authoring_guardrail_churn"- assert ctx.latest_tool_blocker_signal is churn-- _stub_successful_update(monkeypatch)- accepted = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)-- assert accepted["ok"] is True- assert ctx.code_authoring_guardrail_reject_count == 0- assert ctx.blocker_signal is None- assert ctx.latest_tool_blocker_signal is None- assert _check_enforcement(ctx) is None-_RESALE_URL = "https://example.com/orders"_QUOTE_URL = "https://example.com/quote"@@ -10747,30 +10413,6 @@ class TestScoutedSpinePersistSeamCoverage:assert len(events) == 1assert events[0]["site"] == "separated_split"- @pytest.mark.asyncio- async def test_verbatim_synthesized_resubmission_clears_coverage_and_persists(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- _stub_successful_update(monkeypatch)- ctx = _records_spine_ctx()- ctx.update_workflow_called = True- ctx.persisted_draft_browser_calls = [("click", 'page.locator("#stage-a")')]- synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)- assert synthesized is not None-- with capture_logs() as logs:- result = await _update_workflow({"workflow_yaml": _records_block_yaml(synthesized.code)}, ctx)-- assert result["ok"] is True- assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]- assert ctx.persisted_draft_browser_calls is not None- assert [pair for pair in ctx.persisted_draft_browser_calls if pair[0] == "click"] == [- ("click", "page.locator('#stage-a')"),- ("click", "page.locator('#stage-b')"),- ("click", "page.locator('#stage-c')"),- ]- assert enforcement_module._scouted_spine_turn_end_nudge(ctx) is None-@pytest.mark.asyncioasync def test_credential_scout_reject_carries_open_obligation_artifact(self) -> None:ctx = _credential_spine_ctx()@@ -10787,114 +10429,6 @@ class TestScoutedSpinePersistSeamCoverage:assert ".password" in result["error"]-class TestScoutedSpineTurnEndCheckpoint:- def test_checkpoint_is_single_shot_and_emits_unresolved_when_spent(self) -> None:- ctx = _checkpoint_eligible_ctx()-- first = enforcement_module._scouted_spine_turn_end_nudge(ctx)- assert first is not None- assert "scouted_spine_under_build" in first- assert 'await page.locator("#stage-b").click()' in first- assert ctx.code_authoring_guardrail_reject_count == 1-- with capture_logs() as logs:- second = enforcement_module._scouted_spine_turn_end_nudge(ctx)-- assert second is None- assert ctx.code_authoring_guardrail_reject_count == 1- unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]- assert len(unresolved) == 1- assert unresolved[0]["site"] == "turn_end"-- def test_checkpoint_reject_threads_churn_and_ceiling_emits_unresolved(self) -> None:- ctx = _checkpoint_eligible_ctx()- seed_repair = CodeAuthoringRepairContext(- block_label="persisted_draft",- reason_code="scouted_spine_under_build",- selector="#stage-b",- )- record_build_test_outcome(ctx, recorded_outcome_from_authoring_repair_context(seed_repair))- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1-- with capture_logs() as logs:- nudge = enforcement_module._scouted_spine_turn_end_nudge(ctx)-- assert nudge is not None- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- held = ctx.blocker_signal- assert isinstance(held, CopilotToolBlockerSignal)- assert held.internal_reason_code == "code_authoring_guardrail_churn"- unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]- assert len(unresolved) == 1- assert unresolved[0]["site"] == "churn_stop"-- assert enforcement_module._scouted_spine_turn_end_nudge(ctx) is None- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS-- def test_no_spurious_checkpoint_on_full_coverage_or_lane_only_remainder(self) -> None:- full = _checkpoint_eligible_ctx()- full.persisted_draft_browser_calls = [- ("click", 'page.locator("#stage-a")'),- ("click", 'page.locator("#stage-b")'),- ("click", 'page.locator("#stage-c")'),- ]- with capture_logs() as logs:- assert enforcement_module._scouted_spine_turn_end_nudge(full) is None- assert not [log for log in logs if "scouted_spine" in str(log.get("event"))]- assert full.code_authoring_guardrail_reject_count == 0-- lane_only = _checkpoint_eligible_ctx()- lane_only.scout_trajectory = lane_only.scout_trajectory[:1] + [- {- "tool_name": "click",- "selector": "#cookie-accept",- "role": "button",- "accessible_name": "Accept all cookies",- "source_url": "https://example.com/records",- "trajectory_index": 1,- }- ]- with capture_logs() as logs:- assert enforcement_module._scouted_spine_turn_end_nudge(lane_only) is None- assert not [log for log in logs if "scouted_spine" in str(log.get("event"))]-- no_persist = _records_spine_ctx()- assert enforcement_module._scouted_spine_turn_end_nudge(no_persist) is None-- standard = _checkpoint_eligible_ctx()- standard.block_authoring_policy = BlockAuthoringPolicy.STANDARD- assert enforcement_module._scouted_spine_turn_end_nudge(standard) is None-- def test_dropped_unforgiven_interaction_nudges_despite_full_rung_coverage(self) -> None:- ctx = _checkpoint_eligible_ctx()- ctx.scout_trajectory = [- {- "tool_name": "click",- "selector": "#stage-a",- "source_url": "https://example.com/records",- "trajectory_index": 0,- },- {"tool_name": "press_key", "key": "", "trajectory_index": 1},- {"tool_name": "click", "selector": "#stage-b", "trajectory_index": 2},- ]- ctx.persisted_draft_browser_calls = [- ("click", 'page.locator("#stage-a")'),- ("click", 'page.locator("#stage-b")'),- ]-- findings = enforcement_module._scouted_spine_open_obligation(ctx)- assert any(finding.kind == code_block_synthesis_module.UNFORGIVEN_DROP_FINDING for finding in findings)-- nudge = enforcement_module._scouted_spine_turn_end_nudge(ctx)- assert nudge is not None- assert code_block_synthesis_module.SCOUTED_SPINE_DROPPED_UNFORGIVEN_REASON_CODE in nudge- assert ctx.last_code_authoring_repair_context is not None- assert (- ctx.last_code_authoring_repair_context.reason_code- == code_block_synthesis_module.SCOUTED_SPINE_DROPPED_UNFORGIVEN_REASON_CODE- )--class TestPartitionAwareImpositionFastPath:def _synthesize(self, trajectory: list[dict[str, object]]) -> SynthesizedCodeBlock:result = workflow_update_module.synthesize_code_block(trajectory, strict_selectors=True)@@ -10981,42 +10515,11 @@ def _full_coverage_calls() -> list[tuple[str, str]]:class TestScoutedSpineTurnHaltExit:- def test_terminal_halt_with_open_obligation_emits_unresolved_and_reframes_reply(self) -> None:- ctx = _checkpoint_eligible_ctx()- ctx.last_test_anti_bot = "challenge_detected"- signal = enforcement_module.code_authoring_churn_stop_signal(ctx)- assert "verification challenge" in signal.user_facing_reason- ctx.blocker_signal = signal- halt = TurnHalt(kind=TurnHaltKind.LOOP_DETECTED, blocker_signal=signal)-- with capture_logs() as logs:- result = agent_module._build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)-- unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]- assert len(unresolved) == 1- assert unresolved[0]["site"] == "turn_halt"- assert result.user_response == enforcement_module.SCOUTED_SPINE_TURN_HALT_USER_REASON- assert "verification challenge" not in result.user_response-- def test_site_block_halt_with_open_obligation_emits_unresolved_and_keeps_site_block_reply(self) -> None:- ctx = _checkpoint_eligible_ctx()- signal = enforcement_module._probable_site_block_stop_signal(ctx)- ctx.blocker_signal = signal- halt = TurnHalt(kind=TurnHaltKind.PROBABLE_SITE_BLOCK, blocker_signal=signal)-- with capture_logs() as logs:- result = agent_module._build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)-- unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]- assert len(unresolved) == 1- assert unresolved[0]["site"] == "turn_halt"- assert result.user_response == signal.user_facing_reason-- def test_delivered_unverified_halt_with_open_obligation_emits_unresolved(self) -> None:+ def test_turn_halt_with_open_obligation_emits_unresolved(self) -> None:ctx = _checkpoint_eligible_ctx()halt = TurnHalt(- kind=TurnHaltKind.DELIVERED_UNVERIFIED,- verdict=TurnHaltVerdict.DELIVERED_UNVERIFIED,+ kind=TurnHaltKind.LOOP_DETECTED,+ verdict=TurnHaltVerdict.BLOCKED,)with capture_logs() as logs:@@ -11026,57 +10529,6 @@ class TestScoutedSpineTurnHaltExit:assert len(unresolved) == 1assert unresolved[0]["site"] == "turn_halt"- def test_halt_without_open_obligation_emits_nothing_and_keeps_reply(self) -> None:- ctx = _checkpoint_eligible_ctx()- ctx.persisted_draft_browser_calls = _full_coverage_calls()- signal = enforcement_module.code_authoring_churn_stop_signal(ctx)- ctx.blocker_signal = signal- halt = TurnHalt(kind=TurnHaltKind.LOOP_DETECTED, blocker_signal=signal)-- with capture_logs() as logs:- result = agent_module._build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)-- assert not [log for log in logs if "scouted_spine" in str(log.get("event"))]- assert result.user_response == signal.user_facing_reason-- @pytest.mark.asyncio- async def test_wrapped_exception_exit_with_stashed_terminal_halt_emits_unresolved(self) -> None:- ctx = _checkpoint_eligible_ctx()- signal = enforcement_module.code_authoring_churn_stop_signal(ctx)- stash_turn_halt_from_blocker_signal(ctx, signal, source="enforcement")- ctx.blocker_signal = signal-- with capture_logs() as logs:- result = await agent_module._resolve_wrapped_exception_exit_result(- ctx,- None,- goal_satisfied=False,- error=RuntimeError("wrapped"),- workflow_permanent_id="wp",- )-- unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]- assert len(unresolved) == 1- assert unresolved[0]["site"] == "turn_halt"- assert result.user_response == enforcement_module.SCOUTED_SPINE_TURN_HALT_USER_REASON-- def test_obligation_check_failure_never_blocks_the_halt_reply(self, monkeypatch: pytest.MonkeyPatch) -> None:- ctx = _checkpoint_eligible_ctx()- signal = enforcement_module.code_authoring_churn_stop_signal(ctx)- ctx.blocker_signal = signal- halt = TurnHalt(kind=TurnHaltKind.LOOP_DETECTED, blocker_signal=signal)-- def _boom(*args: object, **kwargs: object) -> None:- raise RuntimeError("synthesis unavailable")-- monkeypatch.setattr(enforcement_module, "synthesize_code_block", _boom)-- with capture_logs() as logs:- result = agent_module._build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)-- assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]- assert result.user_response == signal.user_facing_reason-class TestAmbiguousRejectCarriesOpenObligationArtifact:def test_ambiguous_sibling_reject_with_open_obligation_carries_call_source(self) -> None:@@ -11656,24 +11108,6 @@ class TestGoalCompletionLandingImposition:assert ctx.pending_goal_complete_landing is Trueassert ctx.synthesized_goal_complete_landed is False- def test_unregistered_download_target_still_admits_imposition_after_landing(self) -> None:- ctx = _quote_ctx()- ctx.update_workflow_called = True- ctx.synthesized_goal_complete_landed = True- ctx.reached_download_target = ReachedDownloadTarget(- selector='a[href="/files/report.pdf"]',- affordance_text="Download PDF",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )-- result = workflow_update_module._maybe_impose_synthesized_code_block(_quote_submitted_yaml(), ctx)-- assert result.violations == []- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])- assert "async with page.expect_download()" in code-def test_non_goal_complete_trajectory_still_skips_imposition_after_update(self) -> None:ctx = _code_only_ctx()_enable_imposition(ctx)@@ -11991,28 +11425,6 @@ class TestAdmittedImpositionOwnsSpineCoverage:remaining = [block.get("label") for block in parsed["workflow_definition"]["blocks"]]assert "orphan_rung" not in remaining- def test_admitted_imposition_passes_the_pre_persist_gate(self) -> None:- ctx = _quote_ctx()- ctx.update_workflow_called = True- ctx.synthesized_goal_complete_landed = True- ctx.persisted_draft_browser_calls = [("click", 'page.locator("#zip")')]- ctx.reached_download_target = ReachedDownloadTarget(- selector='a[href="/files/report.pdf"]',- affordance_text="Download PDF",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )-- with capture_logs() as logs:- result = workflow_update_module._maybe_impose_synthesized_code_block(_hand_authored_rung_yaml(), ctx)-- assert result.violations == []- assert result.substitutions is not None- assert ctx.spine_imposition_owned_attempt is True- assert workflow_update_module._pre_persist_scouted_spine_result(result.workflow_yaml, ctx) is None- assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]-def test_pre_persist_gate_still_fires_when_imposition_is_not_admitted(self) -> None:ctx = _records_spine_ctx()ctx.update_workflow_called = True@@ -12107,112 +11519,6 @@ class TestAdmittedImpositionOwnsSpineCoverage:assert any("never_captured" in violation for violation in result.violations)assert any("never by authoring browser calls freehand" in violation for violation in result.violations)- def test_hand_authored_download_rung_is_replaced_by_the_grounded_spine_terminal(self) -> None:- ctx = _quote_ctx()- ctx.update_workflow_called = True- ctx.synthesized_goal_complete_landed = True- ctx.reached_download_target = ReachedDownloadTarget(- selector='a[href="/files/report.pdf"]',- affordance_text="Download PDF",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )- submitted = _yaml(- """- title: Quote- workflow_definition:- blocks:- - block_type: code- label: enter_zip- code: |- await page.locator("#zip").fill(str(zip_code))- - block_type: code- label: download_statement- code: |- async with page.expect_download() as dl:- await page.locator("a[href=\\"/files/report.pdf\\"]").click()- """- )-- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)-- assert result.violations == []- blocks = _code_blocks(parse_workflow_yaml(result.workflow_yaml))- assert list(blocks) == ["enter_zip"]- code = str(blocks["enter_zip"]["code"])- assert code.count("async with page.expect_download()") == 1- assert 'page.locator("#coverage-next")' in code-- def test_aliased_download_handle_rung_is_dropped_for_the_grounded_spine_terminal(self) -> None:- ctx = _quote_ctx()- ctx.update_workflow_called = True- ctx.synthesized_goal_complete_landed = True- ctx.reached_download_target = ReachedDownloadTarget(- selector='a[href="/files/report.pdf"]',- affordance_text="Download PDF",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )- submitted = _yaml(- """- title: Quote- workflow_definition:- blocks:- - block_type: code- label: enter_zip- code: |- await page.locator("#zip").fill(str(zip_code))- - block_type: code- label: download_statement- code: |- rows = page.locator("div.statement-row")- await rows.count()- printable = rows.nth(0).locator("a.printable")- await printable.wait_for(state="visible", timeout=10000)- async with page.expect_download(timeout=20000) as download_info:- await printable.click()- download = await download_info.value- """- )-- with capture_logs() as logs:- result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)-- assert result.violations == []- blocks = _code_blocks(parse_workflow_yaml(result.workflow_yaml))- assert list(blocks) == ["enter_zip"]- code = str(blocks["enter_zip"]["code"])- assert code.count("async with page.expect_download()") == 1- assert "printable" not in code- dropped = [log for log in logs if log["event"] == "copilot_spine_stale_rung_dropped"]- assert dropped and dropped[0]["dropped_labels"] == ["download_statement"]- assert "page.expect_download(timeout=20000)" in dropped[0]["dropped_actions"]-- def test_unchanged_under_built_carrier_still_lands_the_spine_on_an_owned_attempt(self) -> None:- ctx = _records_spine_ctx()- ctx.update_workflow_called = True- ctx.synthesized_goal_complete_landed = True- ctx.reached_download_target = ReachedDownloadTarget(- selector='a[href="/files/report.pdf"]',- affordance_text="Download PDF",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )- under_built = _records_block_yaml('await page.locator("#stage-a").click()')- ctx.workflow_yaml = under_built-- imposition = workflow_update_module._maybe_impose_synthesized_code_block(under_built, ctx)-- assert imposition.violations == []- assert imposition.substitutions is not None- assert ctx.spine_imposition_owned_attempt is True- code = str(_single_code_block(parse_workflow_yaml(imposition.workflow_yaml))["code"])- assert 'page.locator("#stage-b").click()' in code- assert workflow_update_module._pre_persist_scouted_spine_result(imposition.workflow_yaml, ctx) is None-def _reaching_extraction_ctx() -> CopilotContext:ctx = _live_read_extraction_ctx()diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.pyindex 1acdd8b5d..3f4173737 100644--- a/tests/unit/test_copilot_code_block_synthesis.py+++ b/tests/unit/test_copilot_code_block_synthesis.py@@ -53,7 +53,6 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (UNRECORDED_INDEX_FINDING,ProducedStaticReturnEnvelope,SynthesisDiagnostics,- _get_by_role_expr,_get_by_role_expr_strict,_is_submit_interaction,_strict_period_date_pattern,@@ -552,6 +551,13 @@ class _RecipeLocator:def nth(self, index: int) -> _RecipeLocator:return _RecipeLocator(self.page, self.selector, (*self.indices, index))+ @property+ def first(self) -> _RecipeLocator:+ return self.nth(0)++ async def wait_for(self, *, state: str = "visible", timeout: float | None = None) -> None:+ return None+def locator(self, selector: str) -> _RecipeLocator:return _RecipeLocator(self.page, f"{self.selector}|{selector}", self.indices)@@ -615,12 +621,12 @@ async def test_generated_recipe_executes_and_fails_closed_on_runtime_drift() ->assert len(result["output"]["records"]) == 3page.counts[(".kv", ())] = 3- with pytest.raises(ValueError, match="scalar selector cardinality"):+ with pytest.raises(Exception, match="scalar selector cardinality"):await _execute_recipe(page)page.counts[(".kv", ())] = 2page.visibility[("#records > tbody > tr|:scope > th, :scope > td", (1, 2))] = False- with pytest.raises(ValueError, match="cell is no longer visible"):+ with pytest.raises(Exception, match="cell is no longer visible"):await _execute_recipe(page)@@ -658,7 +664,7 @@ async def test_produced_envelope_executes_table_and_scalar_reads() -> None:async def test_produced_envelope_guard_raises_on_empty_cell() -> None:page = _RecipePage()page.text[("#records > tbody > tr|:scope > th, :scope > td", (0, 1))] = ""- with pytest.raises(ValueError, match="table cell value is empty"):+ with pytest.raises(Exception, match="table cell value is empty"):await _execute_envelope(page, _produce_table_envelope())@@ -1032,37 +1038,6 @@ class TestLocatorSynthesis:assert "\n" not in emittedassert f"await {emitted}.click()" in result.code- def test_strict_reanchor_emits_exact_name_match_for_repeated_affordance(self) -> None:- # AC1: a re-anchored named get_by_role on a page with a repeated accessible name must emit an- # exact (single (role, name) group) match, never the substring default that over-matches.- trajectory = [- _interaction("click", selector="#open", source_url="https://example.com/billing"),- _interaction(- "click",- selector="a",- source_url="https://example.com/billing",- role="link",- accessible_name="Download",- ),- ]- result = synthesize_code_block(trajectory, strict_selectors=True)- assert result is not None- assert 'await page.get_by_role("link", name="Download", exact=True).click()' in result.code- assert ".nth(" not in result.code- assert result.diagnostics.dropped_interactions == []- provenance = [p for p in result.diagnostics.locator_provenance if p.get("source") == "aria_role_name"]- assert provenance == [- {- "trajectory_index": 1,- "selector": "a",- "emitted_literal": _get_by_role_expr_strict("link", "Download"),- "source": "aria_role_name",- "role": "link",- "name": "Download",- }- ]- assert provenance[0]["emitted_literal"] != _get_by_role_expr("link", "Download")-class TestActionSynthesis:def test_type_text_becomes_param_slot_fill(self) -> None:@@ -1149,7 +1124,7 @@ class TestActionSynthesis:assert lines[2] == ' await _scout_entry_target.wait_for(state="visible", timeout=1000)'assert lines[3] == " except Exception:"assert lines[4] == ' await page.goto("https://example.com/start", wait_until="domcontentloaded")'- assert lines[5] == ' await _scout_entry_target.wait_for(state="visible")'+ assert lines[5] == ' await _scout_entry_target.wait_for(state="visible", timeout=120000)'assert " del _scout_entry_target" in linesdef test_optional_cookie_dismissal_is_conditional_and_uses_durable_entry_target(self) -> None:@@ -1176,7 +1151,7 @@ class TestActionSynthesis:lines = result.code.splitlines()assert lines[0] == ' _scout_entry_target = page.locator("#locInput")'assert ' await page.goto("https://example.com/find", wait_until="domcontentloaded")' in lines- assert ' await _scout_entry_target.wait_for(state="visible")' in lines+ assert ' await _scout_entry_target.wait_for(state="visible", timeout=120000)' in linesassert ' _scout_optional_dismissal = page.locator("#accept-consent")' in linesassert " if await _scout_optional_dismissal.count() > 0:" in linesassert " await _scout_optional_dismissal.first.click(timeout=1000)" in lines@@ -1215,7 +1190,7 @@ class TestActionSynthesis:assert lines[0] == ' _scout_entry_target = page.locator("#locInput")'assert ' _scout_optional_dismissal = page.locator("button.decline")' in linesassert " if await _scout_optional_dismissal.count() > 0:" in lines- assert 'await _scout_entry_target.wait_for(state="visible")' in result.code+ assert 'await _scout_entry_target.wait_for(state="visible", timeout=120000)' in result.codeassert 'await page.locator("button.decline").click()' not in result.codedef test_close_named_action_is_not_optional_dismissal_by_name_only(self) -> None:@@ -1408,7 +1383,7 @@ class TestActionSynthesis:assert lines[0] == ' await page.goto("https://example.com/find", wait_until="domcontentloaded")'assert dismissal_line in linesassert " await _scout_optional_dismissal.first.click(timeout=1000)" in lines- assert 'await _scout_entry_target.wait_for(state="visible")' not in result.code+ assert 'await _scout_entry_target.wait_for(state="visible", timeout=120000)' not in result.codeassert forbidden_snippet not in result.codeast.parse("async def _block(page):\n" + result.code)@@ -5107,3 +5082,206 @@ def test_unstamped_click_emits_no_solve_captcha() -> None:result = synthesize_code_block([_captcha_click()], strict_selectors=True)assert result is not Noneassert "solve_captcha" not in result.code+++class TestScoutedReadSynthesis:+ """A value the scout proved with `evaluate` is authored from that proven expression, the way a+ login is authored from proven fills — instead of being rediscovered by a guessed selector."""++ def test_proven_read_is_emitted_and_returned_at_the_requested_path(self) -> None:+ block = synthesize_code_block(+ [+ {+ "tool_name": "click",+ "selector": "#submit",+ "source_url": "https://example.test/",+ "trajectory_index": 0,+ },+ {+ "tool_name": "read_value",+ "read_expression": 'document.querySelector("#total").innerText',+ "read_output_path": "output.error_count",+ "source_url": "https://example.test/logs",+ "trajectory_index": 1,+ },+ ]+ )++ assert block is not None+ assert "await page.evaluate('document.querySelector(\"#total\").innerText')" in block.code+ assert block.code.rstrip().endswith('return {"error_count": _read_value_0}')+ ast.parse(textwrap.dedent(block.code))++ def test_read_without_an_expression_is_dropped_rather_than_emitted_empty(self) -> None:+ block = synthesize_code_block(+ [+ {+ "tool_name": "click",+ "selector": "#submit",+ "source_url": "https://example.test/",+ "trajectory_index": 0,+ },+ {"tool_name": "read_value", "read_output_path": "output.error_count", "trajectory_index": 1},+ ]+ )++ assert block is not None+ assert "page.evaluate" not in block.code+ assert "return {" not in block.code+++class TestScoutedReadBindingSafety:+ """A later probe of an unrelated value must not displace the read that answered the request."""++ def test_a_second_read_of_another_path_does_not_discard_the_first(self) -> None:+ block = synthesize_code_block(+ [+ {"tool_name": "click", "selector": "#go", "source_url": "https://example.test/", "trajectory_index": 0},+ {+ "tool_name": "read_value",+ "read_expression": 'document.querySelector("#total").innerText',+ "read_output_path": "output.error_count",+ "trajectory_index": 1,+ },+ {+ "tool_name": "read_value",+ "read_expression": "document.title",+ "read_output_path": "output.page_title",+ "trajectory_index": 2,+ },+ ]+ )++ assert block is not None+ assert "error_count" in block.code+ assert "page_title" in block.code+ assert "#total" in block.code++ def test_a_repeat_read_of_the_same_path_keeps_only_the_last(self) -> None:+ block = synthesize_code_block(+ [+ {"tool_name": "click", "selector": "#go", "source_url": "https://example.test/", "trajectory_index": 0},+ {+ "tool_name": "read_value",+ "read_expression": 'document.querySelector("#stale").innerText',+ "read_output_path": "output.error_count",+ "trajectory_index": 1,+ },+ {+ "tool_name": "read_value",+ "read_expression": 'document.querySelector("#fresh").innerText',+ "read_output_path": "output.error_count",+ "trajectory_index": 2,+ },+ ]+ )++ assert block is not None+ assert "#fresh" in block.code+ assert "#stale" not in block.code++ def test_an_unrelated_anonymous_probe_does_not_evict_the_anonymous_answer(self) -> None:+ # An unnamed request pins every read to the shared anonymous path, so a later diagnostic+ # probe is a different read, not a refinement — the answer's expression must survive.+ block = synthesize_code_block(+ [+ {"tool_name": "click", "selector": "#go", "source_url": "https://example.test/", "trajectory_index": 0},+ {+ "tool_name": "read_value",+ "read_expression": 'document.querySelector("#count").innerText',+ "read_output_path": "output.scouted_read",+ "trajectory_index": 1,+ },+ {+ "tool_name": "read_value",+ "read_expression": "document.title",+ "read_output_path": "output.scouted_read",+ "trajectory_index": 2,+ },+ ]+ )++ assert block is not None+ assert "#count" in block.code+ assert "document.title" in block.code+ # The return keeps BOTH values under distinct keys: choosing one would silently discard+ # the read that answered the request.+ assert '"scouted_read": _read_value_0' in block.code+ assert '"scouted_read_2": _read_value_1' in block.code++ def test_composite_extraction_suffix_is_reachable_after_a_captured_read(self) -> None:+ # Base synthesis must not emit its own terminal return when an extraction suffix follows,+ # or the suffix is unreachable and the block returns the wrong shape.+ plan = RequestedOutputExtractionPlan(+ requested_output_paths=("output.error_count",),+ observation_step=1,+ observation_identity="observation-identity",+ reveal=RevealAnchor(selector="#go"),+ live_reads=(+ LiveReadBinding(+ output_path="output.error_count",+ kind=LiveReadKind.KEY_VALUE,+ selector="#count",+ selector_count=1,+ selector_index=0,+ child_index=1,+ child_count=2,+ relation_label="Errors",+ ),+ ),+ identity="plan-identity",+ )+ block = synthesize_code_block_with_extraction(+ [+ {"tool_name": "click", "selector": "#go", "source_url": "https://example.test/", "trajectory_index": 0},+ {+ "tool_name": "read_value",+ "read_expression": 'document.querySelector("#count").innerText',+ "read_output_path": "output.error_count",+ "trajectory_index": 1,+ },+ ],+ plan,+ )++ assert block is not None+ # The imposition seam dedents the interaction half before appending the suffix; normalize+ # the same way, then prove the suffix's return is the only and final return.+ normalized = (+ textwrap.dedent(block.interaction_code).rstrip() + "\n" + textwrap.dedent(block.extraction_code).rstrip()+ )+ tree = ast.parse(f"async def __wrapper__():\n{textwrap.indent(normalized, ' ')}\n")+ returns = [n for n in ast.walk(tree) if isinstance(n, ast.Return)]+ assert len(returns) == 1, "composite block must have exactly one terminal return"+ normalized_lines = normalized.rstrip().splitlines()+ trailing = [+ line+ for line in normalized_lines[returns[0].lineno - 2 :]+ if line.strip() and not line.strip().startswith("#")+ ]+ assert len(trailing) == 1, f"unreachable code after terminal return: {trailing[1:]}"+ assert block.extraction_code.strip() in normalized+++def test_synthesized_extraction_guards_only_name_things_the_sandbox_provides() -> None:+ # The generated block runs with builtins emptied, so an exception class the sandbox does not+ # bind raises NameError instead of the guard's own message.+ from skyvern.forge.sdk.copilot.code_block_preflight import sandbox_unresolved_name_repair_diagnostic+ from skyvern.forge.sdk.copilot.code_block_synthesis import _key_value_scalar_read_statements+ from skyvern.forge.sdk.copilot.output_extraction_plan import LiveReadBinding++ binding = LiveReadBinding(+ output_path="output.count",+ kind="key_value_scalar",+ selector="#summary",+ selector_count=1,+ selector_index=0,+ child_index=1,+ child_count=2,+ relation_label="Errors",+ )++ emitted = _key_value_scalar_read_statements(binding, "_value", guard_empty=True)+ diagnostic = sandbox_unresolved_name_repair_diagnostic("\n".join(emitted), parameter_keys=[])++ assert diagnostic is None, diagnostic.unresolved_names if diagnostic else ()diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.pyindex b461e6789..33225c873 100644--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py@@ -54,9 +54,6 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (VerificationResult,)from skyvern.forge.sdk.copilot.enforcement import (- CopilotBuiltUnverified,- _check_enforcement,- built_complete_without_evaluated_outcome,built_unverified_repair_inert_context,verified_goal_claim_authorized,verified_goal_satisfied_context,@@ -1648,18 +1645,6 @@ def _no_repair_unverified_contract() -> DiagnosisRepairContract:)-def test_claim_closure_turn_still_ends_but_claim_downgrades() -> None:- ctx = _legacy_verified_ctx()-- assert ctx.completion_verification_result is None- assert verified_goal_satisfied_context(ctx) is False- assert verified_goal_claim_authorized(ctx) is False- assert built_complete_without_evaluated_outcome(ctx) is True-- with pytest.raises(CopilotBuiltUnverified):- _check_enforcement(ctx)--def test_structural_abstention_no_repair_terminalizes_without_authorizing_success_claim() -> None:ctx = _ctx(last_test_ok=True,diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 846765d01..1fafbcd01 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -40,10 +40,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,CompletionVerificationResult,CriterionVerdict,- DeliveredUnverifiedTerminalState,EvidenceSourceKind,- FloorRekeyedDeliverableCredit,- FloorRekeyedEmissionWithhold,RegisteredBlockerEvidence,RenderedEvidenceRecord,RunEvidenceSnapshot,@@ -53,12 +50,8 @@ from skyvern.forge.sdk.copilot.completion_verification import (carry_degraded_criterion_ids,carry_floor_rekeyed_criterion_ids,combine_verification_results,- degraded_contract_delivered_unverified_terminal_state,effective_unmet_verdicts,evaluate_completion_criteria,- floor_rekeyed_deliverable_credit,- floor_rekeyed_emission_lane_fields,- floor_rekeyed_emission_withhold,grade_definition_criteria,grade_fallback_floor_reached_end_state_criteria,grade_present_value_criteria,@@ -77,7 +70,6 @@ from skyvern.forge.sdk.copilot.completion_verification import (structured_record_has_goal_content,structured_record_has_identity,summarize_unsatisfied_outcomes,- zero_requested_output_criteria_credit,)from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (@@ -96,9 +88,7 @@ from skyvern.forge.sdk.copilot.enforcement import (outcome_fully_verified,verified_goal_satisfied_context,)-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODEfrom skyvern.forge.sdk.copilot.hooks import _tool_completion_satisfies_turn-from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.request_policy import (AntecedentFamily,CompletionCriterion,@@ -124,10 +114,6 @@ from skyvern.forge.sdk.copilot.request_slots import ()from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools import (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,- _active_run_terminal_evidence_needs_visual_fallback,- _active_run_terminal_evidence_result,- _active_run_terminal_evidence_sample,_build_run_evidence_snapshot,_composition_visual_prompt,_current_workflow_has_evidence_block,@@ -139,9 +125,7 @@ from skyvern.forge.sdk.copilot.tools import (_outcome_unverified_reason,_record_composition_page_observation,_record_run_blocks_result,- _tool_loop_error,_tool_visible_result_after_completion_verification,- _watchdog_exit_allows_terminal_promotion,)from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modulefrom skyvern.forge.sdk.copilot.tools._shared import (@@ -155,7 +139,6 @@ from skyvern.forge.sdk.copilot.tools.completion import (_floor_rekeyed_emission_evidence,_reconcile_download_completion_criterion,)-from skyvern.forge.sdk.copilot.tools.composition_capture import _active_run_terminal_monitor_enabledfrom skyvern.forge.sdk.copilot.tools.workflow_update import (_apply_code_artifact_requested_output_evidence_sources,_normalize_code_artifact_metadata,@@ -2737,67 +2720,6 @@ def test_snapshot_renders_failed_run_artifact_health_signal() -> None:assert "Page.evaluate: SyntaxError" in rendered-def test_active_run_terminal_visual_fallback_uses_screenshot_when_missing() -> None:- assert (- _active_run_terminal_evidence_needs_visual_fallback(- {- "visible_text_excerpt": "",- "forms": [],- "navigation_targets": [],- "result_containers": [],- "evidence_confidence": 0.1,- }- )- is True- )- assert (- _active_run_terminal_evidence_needs_visual_fallback(- {- "visible_text_excerpt": "Cart TESTBRAND PART-001-TEST quantity 1",- "forms": [],- "navigation_targets": [],- "result_containers": [],- "evidence_confidence": 0.1,- }- )- is True- )- assert (- _active_run_terminal_evidence_needs_visual_fallback(- {- "visible_text_excerpt": "Cart contains item PART-001-TEST with quantity 1. " * 4,- "forms": [],- "navigation_targets": [],- "result_containers": [],- "evidence_confidence": 0.1,- }- )- is True- )- assert (- _active_run_terminal_evidence_needs_visual_fallback(- {- "visible_text_excerpt": "",- "forms": [],- "navigation_targets": [],- "result_containers": [{"selector": "#cart"}],- "evidence_confidence": 0.3,- }- )- is True- )- assert (- _active_run_terminal_evidence_needs_visual_fallback(- {- "visible_text_excerpt": "Cart TESTBRAND PART-001-TEST quantity 1",- "result_containers": [{"selector": "#cart"}],- "screenshot_used": True,- }- )- is False- )--def test_visual_prompt_requests_outcome_relevant_page_state() -> None:prompt = _composition_visual_prompt({"current_url": "https://example.com/cart", "page_title": "Cart"})@@ -2807,169 +2729,6 @@ def test_visual_prompt_requests_outcome_relevant_page_state() -> None:assert "human-verification" in prompt-async def test_active_run_terminal_monitor_skips_requested_output_only_criteria(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def handler(**_: object) -> dict:- return {}-- async def fake_completion_verification_handler(_ctx: object) -> object:- return handler-- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",- fake_completion_verification_handler,- )- ctx = _run_ctx()- ctx.browser_session_id = "bs_1"- ctx.discovery_mcp_server = object()- ctx.request_policy = RequestPolicy(- completion_criteria=[_criterion("c0", "return record id", output_path="output.record_id")]- )-- assert await _active_run_terminal_monitor_enabled(ctx) is False---async def test_active_run_terminal_monitor_keeps_mixed_requested_output_armed(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def handler(**_: object) -> dict:- return {}-- async def fake_completion_verification_handler(_ctx: object) -> object:- return handler-- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",- fake_completion_verification_handler,- )- ctx = _run_ctx()- ctx.browser_session_id = "bs_1"- ctx.discovery_mcp_server = object()- ctx.request_policy = RequestPolicy(- completion_criteria=[- _criterion("c0", "return record id", output_path="output.record_id"),- _criterion("c1", "cart page is visible"),- ]- )-- assert await _active_run_terminal_monitor_enabled(ctx) is True---async def test_active_run_terminal_monitor_skips_requested_output_corroborator(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def handler(**_: object) -> dict:- return {}-- async def fake_completion_verification_handler(_ctx: object) -> object:- return handler-- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",- fake_completion_verification_handler,- )- ctx = _run_ctx()- ctx.browser_session_id = "bs_1"- ctx.discovery_mcp_server = object()- ctx.request_policy = RequestPolicy(- completion_criteria=[- _criterion("c_requested", "return quotes", output_path="output.quotes"),- _criterion(- "c_quotes",- "The run extracts the first 3 quotes and authors into the returned quotes JSON.",- requested_output_corroborator=True,- ),- ]- )-- assert await _active_run_terminal_monitor_enabled(ctx) is False---async def test_active_run_terminal_monitor_keeps_unmarked_fallback_floor_armed(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def handler(**_: object) -> dict:- return {}-- async def fake_completion_verification_handler(_ctx: object) -> object:- return handler-- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",- fake_completion_verification_handler,- )- ctx = _run_ctx()- ctx.browser_session_id = "bs_1"- ctx.discovery_mcp_server = object()- ctx.request_policy = RequestPolicy(completion_criteria=build_classifier_fallback_floor([]))-- assert await _active_run_terminal_monitor_enabled(ctx) is True---async def test_active_run_terminal_monitor_skips_marked_fallback_floor_corroborator(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def handler(**_: object) -> dict:- return {}-- async def fake_completion_verification_handler(_ctx: object) -> object:- return handler-- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",- fake_completion_verification_handler,- )- ctx = _run_ctx()- ctx.browser_session_id = "bs_1"- ctx.discovery_mcp_server = object()- floor = replace(build_classifier_fallback_floor([])[0], requested_output_corroborator=True)- ctx.request_policy = RequestPolicy(- completion_criteria=[- _criterion("c_requested", "return record id", output_path="output.record_id"),- floor,- ]- )-- assert await _active_run_terminal_monitor_enabled(ctx) is False---async def test_active_run_terminal_monitor_keeps_terminal_action_criteria_armed(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def handler(**_: object) -> dict:- return {}-- async def fake_completion_verification_handler(_ctx: object) -> object:- return handler-- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",- fake_completion_verification_handler,- )- ctx = _run_ctx()- ctx.browser_session_id = "bs_1"- ctx.discovery_mcp_server = object()- ctx.request_policy = RequestPolicy(- completion_criteria=[- _criterion(- "c0",- "service request is submitted",- kind="terminal_action",- terminal_action_family="request",- )- ]- )-- assert await _active_run_terminal_monitor_enabled(ctx) is True--def test_summarize_unsatisfied_lists_unmet_outcomes() -> None:criteria = [_criterion("c0", "item in cart"), _criterion("c1", "added exactly once")]result = _evaluated(("c0", True), ("c1", False))@@ -3569,13 +3328,8 @@ def test_record_run_blocks_keeps_building_on_mid_build_no_evidence() -> None:ctx = _ctx_with_blocks("goto_url", "navigation")_record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))# A nav-only WIP that has not added a confirmation block yet must keep building,- # not enter repair...+ # not enter repair; a dissatisfied judge does not downgrade what the run produced.assert ctx.last_test_suspicious_success is False- # ...but terminal success and good-workflow promotion stay withheld because- # the outcome is unverified.- assert ctx.last_full_workflow_test_ok is False- assert ctx.last_good_workflow is None- assert _completion_contract_not_violated(ctx) is Falsedef test_record_run_blocks_downgrades_on_contradiction_without_confirmation_block() -> None:@@ -3601,35 +3355,6 @@ def test_record_run_blocks_demonstrated_when_lone_definition_abstention_with_conassert verified_goal_satisfied_context(ctx) is True-def test_record_run_blocks_keeps_clean_structural_abstention_as_built_unverified() -> None:- ctx = _ctx_with_blocks("extraction")- verification = _mixed(- CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="structurally_abstained")- )-- recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert recorded.reason_code is None- assert ctx.last_test_suspicious_success is False- assert ctx.last_test_failure_reason is None- assert ctx.last_full_workflow_test_ok is True- ctx.latest_diagnosis_repair_contract = DiagnosisRepairContract(- diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),- diagnosis_result=DiagnosisResult(),- repair_decision=RepairDecision(next_action=RepairNextAction.NO_CHANGE),- verification_result=VerificationResult(user_goal_satisfied=False, completion_contract_satisfied=False),- )- assert verified_goal_satisfied_context(ctx) is False- assert built_unverified_repair_inert_context(ctx) is True- assert outcome_fully_verified(ctx) is False- outcome = ctx.latest_recorded_build_test_outcome- assert outcome is not None- assert outcome.verdict == "not_authoritative"- assert outcome.is_authoritative is False--def _degraded_delivered_result(*verdicts: CriterionVerdict) -> CompletionVerificationResult:return CompletionVerificationResult(status="evaluated",@@ -3665,54 +3390,6 @@ def _observed_structural_abstention()-def _delivered_terminal_state(- result: CompletionVerificationResult,- *,- run_ok: bool = True,- workflow_run_id: str | None = "wr_x",- latest_workflow_run_id: str | None = "wr_x",- artifact_health_blocked: bool = False,- terminal_blocked: bool = False,-) -> DeliveredUnverifiedTerminalState | None:- return degraded_contract_delivered_unverified_terminal_state(- result,- run_ok=run_ok,- workflow_run_id=workflow_run_id,- latest_workflow_run_id=latest_workflow_run_id,- artifact_health_blocked=artifact_health_blocked,- terminal_blocked=terminal_blocked,- )---def test_degraded_delivered_unverified_terminal_state_allows_observed_runtime_output() -> None:- terminal_state = _delivered_terminal_state(_degraded_delivered_result(_observed_structural_abstention()))-- assert terminal_state is not None- assert [verdict.criterion_id for verdict in terminal_state.observed_verdicts] == ["requested_output"]---def test_degraded_delivered_unverified_terminal_state_spans_contingent_degraded_lane() -> None:- result = CompletionVerificationResult(- status="evaluated",- criterion_ids=["c_contingent_degraded", "requested_output"],- verdicts=[- CriterionVerdict(- criterion_id="c_contingent_degraded",- state="unsatisfied",- reason_code="no_evidence",- ),- _observed_structural_abstention(),- ],- contingent_degraded_criterion_ids=["c_contingent_degraded"],- )-- terminal_state = _delivered_terminal_state(result)-- assert result.degraded_criterion_ids == []- assert terminal_state is not None- assert [verdict.criterion_id for verdict in terminal_state.observed_verdicts] == ["requested_output"]--@pytest.mark.parametrize("kwargs",[@@ -3723,12 +3400,6 @@ def test_degraded_delivered_unverified_terminal_state_spans_contingent_degraded_{"terminal_blocked": True},],)-def test_degraded_delivered_unverified_terminal_state_rejects_run_and_blocker_exclusions(- kwargs: dict[str, bool | str | None],-) -> None:- assert _delivered_terminal_state(_degraded_delivered_result(_observed_structural_abstention()), **kwargs) is None--@pytest.mark.parametrize("verdict",[@@ -3745,29 +3416,6 @@ def test_degraded_delivered_unverified_terminal_state_rejects_run_and_blocker_ex_observed_structural_abstention(evidence_source="independent_page_evidence"),],)-def test_degraded_delivered_unverified_terminal_state_rejects_non_value_output(verdict: CriterionVerdict) -> None:- assert _delivered_terminal_state(_degraded_delivered_result(verdict)) is None---def test_degraded_delivered_unverified_terminal_state_is_not_verified_success() -> None:- ctx = _ctx_with_blocks("extraction")- result = _clean_success_result()- result["data"]["blocks"][0]["label"] = "extract"- result["data"]["blocks"][0]["extracted_data"] = {"document_name": "Resale Demand Package"}- verification = _degraded_delivered_result(_observed_structural_abstention())-- recorded = _record_run_blocks_result(ctx, result, completion_verification=verification)-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.delivered_unverified_terminal is True- assert ctx.delivered_unverified_observed_outputs == {"document_name": "Resale Demand Package"}- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind.value == "delivered_unverified"- assert ctx.last_full_workflow_test_ok is False- assert ctx.last_test_suspicious_success is False--def _registered_output_result(value: dict[str, Any], block_type: str = "CODE") -> dict:return {"ok": True,@@ -3821,13 +3469,6 @@ def _persisted_output_parameter_result(value: dict[str, Any], run_id: str = "wr_}-def test_zero_requested_output_criteria_credit_fires_only_with_payload() -> None:- satisfied = _evaluated(("login", True))-- assert zero_requested_output_criteria_credit(satisfied, has_meaningful_registered_output=True) is True- assert zero_requested_output_criteria_credit(satisfied, has_meaningful_registered_output=False) is False--def test_gradeable_completion_criteria_excludes_pending_and_degraded_judgments() -> None:reached = CompletionCriterion(id="reached", outcome="The requested page is reached.")pending_judgment = CompletionCriterion(@@ -3860,25 +3501,6 @@ def test_gradeable_completion_criteria_excludes_pending_and_degraded_judgments()assert gradeable_completion_criteria([degraded_judgment]) == []-def test_zero_requested_output_criteria_credit_ignored_when_criteria_formed() -> None:- with_criteria = replace(_evaluated(("bill", True)), requested_output_criteria_count=1)-- assert zero_requested_output_criteria_credit(with_criteria, has_meaningful_registered_output=True) is False---def test_zero_requested_output_criteria_credit_requires_evaluated_full_satisfaction() -> None:- assert zero_requested_output_criteria_credit(None, has_meaningful_registered_output=True) is False- assert (- zero_requested_output_criteria_credit(- CompletionVerificationResult(status="unavailable"), has_meaningful_registered_output=True- )- is False- )- assert (- zero_requested_output_criteria_credit(_evaluated(("c0", False)), has_meaningful_registered_output=True) is False- )--_PAGE_EVIDENCE_DELIVERABLE_REF = "block_outputs:post_run_page_observation.document_name"@@ -3958,148 +3580,6 @@ def _floor_rekeyed_result()-def test_floor_rekeyed_credit_grants_on_independent_page_evidence() -> None:- result = _floor_rekeyed_result(- _corroborated_abstention_verdicts(- corroborator_source="independent_page_evidence",- corroborator_evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,- ),- ["deliverable"],- )-- credit = floor_rekeyed_deliverable_credit(result)-- assert isinstance(credit, FloorRekeyedDeliverableCredit)- assert credit.criterion_ids == ("deliverable",)- assert credit.evidence_sources == ("independent_page_evidence",)- assert credit.evidence_refs == (_PAGE_EVIDENCE_DELIVERABLE_REF,)---def test_floor_rekeyed_credit_grants_on_run_plane_marked_verdict() -> None:- result = _floor_rekeyed_result(- _floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),- ["deliverable"],- {"deliverable": "output.document_name"},- )-- credit = floor_rekeyed_deliverable_credit(result)-- assert credit is not None- assert credit.evidence_sources == ("independent_page_evidence",)- assert credit.output_paths == ("output.document_name",)---def test_floor_rekeyed_credit_withholds_without_corroborator() -> None:- result = _floor_rekeyed_result([_observed_end_state_verdict()], ["deliverable"])-- assert result.is_fully_satisfied() is True- assert floor_rekeyed_deliverable_credit(result) is None--- "corroborator_source",- ["registered_output_parameter", "registered_artifact_content"],-)-def test_floor_rekeyed_credit_withholds_on_registered_corroborator_source(- corroborator_source: EvidenceSourceKind,-) -> None:- result = _floor_rekeyed_result(- _corroborated_abstention_verdicts(corroborator_source=corroborator_source),- ["deliverable"],- )-- assert result.is_fully_satisfied() is True- assert floor_rekeyed_deliverable_credit(result) is None--- "corroborator_source",- ["runtime_output", "same_record_context", None],-)-def test_floor_rekeyed_credit_withholds_on_self_emitted_corroborator(- corroborator_source: EvidenceSourceKind | None,-) -> None:- result = _floor_rekeyed_result(- _corroborated_abstention_verdicts(corroborator_source=corroborator_source),- ["deliverable"],- )-- assert result.is_fully_satisfied() is True- assert floor_rekeyed_deliverable_credit(result) is None---def test_floor_rekeyed_credit_requires_every_marked_id_page_grounded() -> None:- verdicts = [- *_corroborated_abstention_verdicts(- corroborator_source="independent_page_evidence",- marked_id="deliverable_a",- corroborator_evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,- ),- *_corroborated_abstention_verdicts(corroborator_source="runtime_output", marked_id="deliverable_b"),- ]- result = _floor_rekeyed_result(verdicts, ["deliverable_a", "deliverable_b"])-- assert floor_rekeyed_deliverable_credit(result) is None---def test_floor_rekeyed_credit_none_without_marked_ids() -> None:- result = _floor_rekeyed_result(- _corroborated_abstention_verdicts(corroborator_source="independent_page_evidence"),- [],- )-- assert floor_rekeyed_deliverable_credit(result) is None---def test_floor_rekeyed_credit_ignores_corroborator_scoped_to_other_id() -> None:- verdicts = [- *_corroborated_abstention_verdicts(corroborator_source="runtime_output"),- CriterionVerdict(- criterion_id="unrelated__requested_output_corroborator",- state="satisfied",- reason_code="evidence_confirms",- evidence_source="independent_page_evidence",- evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,- ),- ]- result = _floor_rekeyed_result(verdicts, ["deliverable"])-- assert floor_rekeyed_deliverable_credit(result) is None---def test_floor_rekeyed_credit_none_when_partial_satisfaction() -> None:- verdicts = [- *_corroborated_abstention_verdicts(- corroborator_source="independent_page_evidence",- corroborator_evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,- ),- CriterionVerdict(criterion_id="other", state="unsatisfied", reason_code="no_evidence"),- ]- result = _floor_rekeyed_result(verdicts, ["deliverable"])-- assert result.is_fully_satisfied() is False- assert floor_rekeyed_deliverable_credit(result) is None---def test_floor_rekeyed_credit_none_for_kill_shape() -> None:- assert floor_rekeyed_deliverable_credit(_evaluated(("login", True))) is None---def test_floor_rekeyed_credit_does_not_mutate_result() -> None:- result = _floor_rekeyed_result(- _corroborated_abstention_verdicts(- corroborator_source="independent_page_evidence",- corroborator_evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,- ),- ["deliverable"],- )- before = replace(result)-- floor_rekeyed_deliverable_credit(result)-- assert result == before--def test_carry_floor_rekeyed_ids_from_marked_criteria() -> None:criteria = [CompletionCriterion(id="deliverable", outcome="Document name is shown.", requested_output_floor_rekeyed=True),@@ -4164,81 +3644,6 @@ def test_floor_idempotent_on_already_floored_set() -> None:assert floored_twice[0].requested_output_floor_rekeyed is True-def test_floor_rekeyed_credit_grants_when_floor_path_binds_delivered_payload() -> None:- ctx = _ctx_with_blocks("code")- verification = _floor_rekeyed_result(- _floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),- ["deliverable"],- {"deliverable": "output.document_name"},- backed_by_id={"deliverable": True},- )-- with capture_logs() as logs:- recorded = _record_run_blocks_result(- ctx,- _registered_output_result({"document_name": "Resale certificate"}),- completion_verification=verification,- )-- assert recorded is not None- assert recorded.verdict == "demonstrated"- granted = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit"]- assert granted and granted[0]["credited_output_paths"] == ["output.document_name"]- assert granted[0]["registered_output_keys"] == ["bill_statement"]---def test_floor_rekeyed_credit_withheld_when_floor_path_absent_from_payload() -> None:- ctx = _ctx_with_blocks("code")- verification = _floor_rekeyed_result(- _floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),- ["deliverable"],- {"deliverable": "output.document_name"},- backed_by_id={"deliverable": True},- )-- with capture_logs() as logs:- recorded = _record_run_blocks_result(- ctx, _registered_output_result({"summary": "raw"}), completion_verification=verification- )-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.delivered_unverified_terminal is True- unbound = [- entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_credit_payload_unbound"- ]- assert unbound and unbound[0]["unbound_output_paths"] == ["output.document_name"]- assert any(entry.get("event") == "copilot.completion.zero_requested_output_credit_withheld" for entry in logs)---def test_floor_rekeyed_deliverable_withholds_without_runtime_backing() -> None:- ctx = _ctx_with_blocks("code")- verification = _floor_rekeyed_result(- _corroborated_abstention_verdicts(- corroborator_source="independent_page_evidence",- corroborator_evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,- ),- ["deliverable"],- {"deliverable": "output.document_name"},- )-- with capture_logs() as logs:- recorded = _record_run_blocks_result(- ctx, _registered_output_result({"summary": "raw"}), completion_verification=verification- )-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.delivered_unverified_terminal is True- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind.value == "delivered_unverified"- assert not any(entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit" for entry in logs)- lane = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_emission_lane"]- assert lane and lane[0]["engaged"] is True- assert lane[0]["unbacked_criterion_ids"] == ["deliverable"]- assert any(entry.get("event") == "copilot.completion.floor_rekeyed_emission_withheld" for entry in logs)--def _floored_policy_criteria(criteria: list[CompletionCriterion], user_message: str) -> list[CompletionCriterion]:policy = RequestPolicy(completion_criteria=criteria)_apply_requested_output_completion_criteria(policy, user_message)@@ -4253,132 +3658,17 @@ def _satisfied_page_evidence_result(criteria: list[CompletionCriterion]) -> Compcriterion_id=criterion.id,state="satisfied",reason_code="evidence_confirms",- evidence_source="independent_page_evidence",- evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,- )- for criterion in criteria- ]- result = CompletionVerificationResult(- status="evaluated",- criterion_ids=[criterion.id for criterion in criteria],- verdicts=verdicts,- )- return carry_floor_rekeyed_criterion_ids(result, criteria)---def test_floor_rekeyed_credit_is_inert_when_the_seam_mints_no_per_deliverable_corroborator() -> None:- criteria = _floored_policy_criteria(- [- CompletionCriterion(- id="run_end_state",- outcome="The run opens the order-level document list and selects the demand document row.",- )- ],- "Return a final record with document name.",- )- marked_id = "__copilot_requested_output__output_document_name"- result = _satisfied_page_evidence_result(criteria)-- assert result.floor_rekeyed_criterion_ids == [marked_id]- assert result.is_fully_satisfied() is True- assert [criterion.id for criterion in criteria if criterion.requested_output_corroborator] == []- assert floor_rekeyed_deliverable_credit(result) is None---async def test_floor_rekeyed_deliverable_is_credited_through_the_verification_producer(- monkeypatch: pytest.MonkeyPatch,-) -> None:- policy = RequestPolicy(- completion_criteria=[- CompletionCriterion(- id="c1",- outcome="The run captures the document status from the order documents list.",- output_path="output.doc_status",- )- ]- )- _apply_classifier_typed_requested_output_corroborators(policy)- floored, rekeyed_paths = apply_requested_output_producer_floor(policy.completion_criteria)- policy.completion_criteria = list(floored)- assert rekeyed_paths == ("output.doc_status",)- assert "c1__requested_output_corroborator" in {criterion.id for criterion in floored}-- async def handler(**_: object) -> dict:- return {- "verdicts": [- {- "criterion_id": criterion.id,- "satisfied": True,- "reason_code": "evidence_confirms",- "evidence_ref": "block_outputs:post_run_page_observation.visible_text_excerpt",- }- for criterion in policy.completion_criteria- ]- }-- async def handler_lookup(_ctx: object) -> object:- return handler-- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",- handler_lookup,- )- ctx = _ctx_with_blocks("code")- ctx.request_policy = policy- ctx.last_workflow_yaml = textwrap.dedent(- """- workflow_definition:- blocks:- - block_type: code- label: extract_bill- code: |- return {"doc_status": "Delivered"}- """- ).strip()- ctx.composition_page_evidence = {- "workflow_run_id": "wr_x",- "observed_after_workflow_run": True,- "visible_text_excerpt": "Resale certificate - Delivered",- }- result = _registered_output_result({"doc_status": "Delivered"})-- verification = await _maybe_run_completion_verification(ctx, result, time.monotonic())-- assert verification is not None- assert verification.floor_rekeyed_criterion_ids == ["c1"]- assert verification.floor_rekeyed_output_path_by_criterion_id == {"c1": "output.doc_status"}- assert verification.is_fully_satisfied() is True-- with capture_logs() as logs:- recorded = _record_run_blocks_result(ctx, result, completion_verification=verification)-- assert recorded is not None- assert recorded.verdict == "demonstrated"- granted = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit"]- assert granted and granted[0]["criterion_ids"] == ["c1"]- assert granted[0]["evidence_sources"] == ["independent_page_evidence"]---def test_floor_rekeyed_runtime_output_only_still_withholds() -> None:- ctx = _ctx_with_blocks("code")- verification = _floor_rekeyed_result(- _corroborated_abstention_verdicts(corroborator_source="runtime_output"),- ["deliverable"],- )-- with capture_logs() as logs:- recorded = _record_run_blocks_result(- ctx, _registered_output_result({"summary": "raw"}), completion_verification=verification+ evidence_source="independent_page_evidence",+ evidence_ref=_PAGE_EVIDENCE_DELIVERABLE_REF,)-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.delivered_unverified_terminal is True- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind.value == "delivered_unverified"- assert any(entry.get("event") == "copilot.completion.floor_rekeyed_emission_withheld" for entry in logs)- assert not any(entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit" for entry in logs)+ for criterion in criteria+ ]+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=[criterion.id for criterion in criteria],+ verdicts=verdicts,+ )+ return carry_floor_rekeyed_criterion_ids(result, criteria)def _backing_ctx(returns_literal: str, block_label: str = "extract_bill") -> CopilotContext:@@ -4702,103 +3992,6 @@ def _emission_result()-def test_floor_rekeyed_emission_withhold_engages_when_marker_unbacked() -> None:- withhold = floor_rekeyed_emission_withhold(- _emission_result(backed={"c1": False}, paths={"c1": "output.doc_status"})- )-- assert isinstance(withhold, FloorRekeyedEmissionWithhold)- assert withhold.criterion_ids == ("c1",)- assert withhold.unbacked_output_paths == ("output.doc_status",)---def test_floor_rekeyed_emission_withhold_none_when_backed() -> None:- assert floor_rekeyed_emission_withhold(_emission_result(backed={"c1": True})) is None---def test_floor_rekeyed_emission_withhold_missing_entry_fails_closed() -> None:- assert floor_rekeyed_emission_withhold(_emission_result(backed={})) is not None---def test_floor_rekeyed_emission_withhold_ignores_empty_marker_set() -> None:- reach_state = _evaluated(("login", True))-- assert floor_rekeyed_emission_withhold(reach_state) is None---def test_floor_rekeyed_emission_withhold_excludes_structurally_unfired_contingent_marker() -> None:- result = _emission_result(- backed={},- marked=("c1",),- contingent=("c1",),- structural_unfired=("c1",),- extra_verdicts=(- CriterionVerdict(- criterion_id="ok", state="satisfied", reason_code="evidence_confirms", evidence_ref="observed_end_state"- ),- ),- )-- assert result.is_fully_satisfied() is True- assert floor_rekeyed_emission_withhold(result) is None---def test_floor_rekeyed_emission_withhold_keyed_on_marker_not_artifact_richness() -> None:- lean = _emission_result(backed={"c1": False})- rich = _emission_result(- backed={"c1": False},- extra_verdicts=(- CriterionVerdict(- criterion_id="ok", state="satisfied", reason_code="evidence_confirms", evidence_ref="observed_end_state"- ),- ),- )-- assert floor_rekeyed_emission_withhold(lean) is not None- assert floor_rekeyed_emission_withhold(rich) is not None---def test_floor_rekeyed_emission_lane_fields_fire_on_satisfied_and_unsatisfied_results() -> None:- satisfied = _emission_result(backed={"c1": False}, paths={"c1": "output.doc_status"})- unsatisfied = CompletionVerificationResult(- status="evaluated",- criterion_ids=["c1"],- verdicts=[CriterionVerdict(criterion_id="c1", state="unsatisfied", reason_code="no_evidence")],- floor_rekeyed_criterion_ids=["c1"],- floor_rekeyed_backed_by_criterion_id={"c1": False},- )-- satisfied_fields = floor_rekeyed_emission_lane_fields(satisfied)- unsatisfied_fields = floor_rekeyed_emission_lane_fields(unsatisfied)-- assert satisfied_fields is not None and satisfied_fields["engaged"] is True- assert unsatisfied_fields is not None and unsatisfied_fields["engaged"] is False---def test_floor_rekeyed_emission_shallow_bound_but_unbacked_withholds() -> None:- ctx = _ctx_with_blocks("code")- verification = _floor_rekeyed_result(- _floored_run_plane_verdicts(corroborator_source="independent_page_evidence"),- ["deliverable"],- {"deliverable": "output.document_name"},- backed_by_id={"deliverable": False},- )-- with capture_logs() as logs:- recorded = _record_run_blocks_result(- ctx,- _registered_output_result({"document_name": "Resale certificate"}),- completion_verification=verification,- )-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.delivered_unverified_terminal is True- lane = [entry for entry in logs if entry.get("event") == "copilot.completion.floor_rekeyed_emission_lane"]- assert lane and lane[0]["engaged"] is True- assert not any(entry.get("event") == "copilot.completion.floor_rekeyed_deliverable_credit" for entry in logs)--def test_registered_output_meaningfulness_gate_drops_hollow_envelope() -> None:envelope_type = next(iter(_TASK_ENVELOPE_BLOCK_TYPES))ctx = _ctx_with_blocks("code")@@ -4827,57 +4020,6 @@ def test_registered_output_meaningfulness_gate_keeps_real_scalar() -> None:assert snapshot.block_outputs.get("bill_statement") == {"doc_status": "Delivered"}-def test_zero_requested_output_criteria_withholds_verified_success() -> None:- ctx = _ctx_with_blocks("code")-- with capture_logs() as logs:- recorded = _record_run_blocks_result(- ctx,- _registered_output_result({"summary": "raw"}),- completion_verification=_evaluated(("login", True)),- )-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.delivered_unverified_terminal is True- assert ctx.delivered_unverified_observed_outputs == {"bill_statement": {"summary": "raw"}}- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind.value == "delivered_unverified"- assert ctx.last_full_workflow_test_ok is False- assert any(entry.get("event") == "copilot.completion.zero_requested_output_credit_withheld" for entry in logs)-- other_ctx = _ctx_with_blocks("code")- other = _record_run_blocks_result(- other_ctx,- _registered_output_result({"amount_due": "$99.99", "statement_month": "January 2026"}),- completion_verification=_evaluated(("login", True)),- )- assert other is not None- assert other.verdict == recorded.verdict- assert other_ctx.delivered_unverified_terminal is True---def test_zero_requested_output_criteria_withholds_verified_success_on_failed_run() -> None:- ctx = _ctx_with_blocks("code")- result = _registered_output_result({"summary": "raw"})- result["ok"] = False- result["data"]["overall_status"] = "canceled"-- recorded = _record_run_blocks_result(- ctx,- result,- completion_verification=_evaluated(("login", True)),- )-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.last_test_ok is False- assert ctx.delivered_unverified_terminal is True- assert ctx.delivered_unverified_observed_outputs == {"bill_statement": {"summary": "raw"}}- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind.value == "delivered_unverified"--def test_requested_output_criteria_still_reach_verified_success() -> None:ctx = _ctx_with_blocks("code")verification = replace(_evaluated(("bill", True)), requested_output_criteria_count=1)@@ -4888,7 +4030,6 @@ def test_requested_output_criteria_still_reach_verified_success() -> None:assert recorded is not Noneassert recorded.verdict == "demonstrated"- assert ctx.delivered_unverified_terminal is Falsedef test_zero_requested_output_criteria_empty_task_output_reaches_verified_success() -> None:@@ -4909,7 +4050,6 @@ def test_zero_requested_output_criteria_empty_task_output_reaches_verified_succeassert recorded is not Noneassert recorded.verdict == "demonstrated"- assert ctx.delivered_unverified_terminal is Falsedef test_zero_requested_output_criteria_login_reach_state_stays_inert() -> None:@@ -4929,7 +4069,6 @@ def test_zero_requested_output_criteria_login_reach_state_stays_inert() -> None:assert recorded is not Noneassert recorded.verdict == "demonstrated"- assert ctx.delivered_unverified_terminal is Falsedef test_zero_requested_output_criteria_lowercase_block_type_slices_empty_envelope() -> None:@@ -4949,7 +4088,6 @@ def test_zero_requested_output_criteria_lowercase_block_type_slices_empty_enveloassert recorded is not Noneassert recorded.verdict == "demonstrated"- assert ctx.delivered_unverified_terminal is False@pytest.mark.parametrize("block_type", ["file_download", "goto_url", "human_interaction", "task_v2"])@@ -4970,7 +4108,6 @@ def test_zero_requested_output_criteria_every_task_backed_block_reach_state_stayassert recorded is not Noneassert recorded.verdict == "demonstrated"- assert ctx.delivered_unverified_terminal is Falsedef test_task_envelope_block_types_matches_explicit_envelope_block_set() -> None:@@ -4993,56 +4130,6 @@ def test_task_envelope_block_types_matches_explicit_envelope_block_set() -> Noneassert _TASK_ENVELOPE_BLOCK_TYPES == expected-def test_zero_requested_output_criteria_task_id_user_schema_is_meaningful() -> None:- ctx = _ctx_with_blocks("code")- user_schema = {"task_id": "tsk_1", "amount_due": "$3,927.75", "statement_month": "March 2026"}-- recorded = _record_run_blocks_result(- ctx,- _registered_output_result(user_schema, block_type="CODE"),- completion_verification=_evaluated(("login", True)),- )-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.delivered_unverified_terminal is True- assert ctx.delivered_unverified_observed_outputs == {"bill_statement": user_schema}---def test_zero_requested_output_criteria_fires_on_persisted_output_parameters_only() -> None:- ctx = _ctx_with_blocks("code")-- recorded = _record_run_blocks_result(- ctx, _persisted_output_parameter_result({"summary": "raw"}), completion_verification=_evaluated(("login", True))- )-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.delivered_unverified_terminal is True- assert ctx.delivered_unverified_observed_outputs == {"bill_statement": {"summary": "raw"}}---def test_zero_requested_output_criteria_excludes_foreign_run_registered_output() -> None:- ctx = _ctx_with_blocks("code")- result = _registered_output_result({"summary": "raw"})- result["data"]["registered_output_parameter_values"].append(- {- "workflow_run_id": "wr_other",- "output_parameter_id": "op_stale",- "output_parameter_key": "stale_output",- "block_label": "stale_block",- "block_type": "CODE",- "value": {"stale": "prior-run"},- }- )-- recorded = _record_run_blocks_result(ctx, result, completion_verification=_evaluated(("login", True)))-- assert recorded is not None- assert recorded.verdict == "not_evaluated"- assert ctx.delivered_unverified_observed_outputs == {"bill_statement": {"summary": "raw"}}--def test_zero_requested_output_criteria_empty_valid_deliverable_still_credits() -> None:ctx = _ctx_with_blocks("code")@@ -5052,7 +4139,6 @@ def test_zero_requested_output_criteria_empty_valid_deliverable_still_credits()assert recorded is not Noneassert recorded.verdict == "demonstrated"- assert ctx.delivered_unverified_terminal is Falsedef test_record_run_blocks_verifies_structural_requested_output_with_run_corroborator() -> None:@@ -5230,35 +4316,6 @@ def _goto_only_result() -> dict:}-async def test_goto_only_run_still_fails_extraction_verification(monkeypatch: pytest.MonkeyPatch) -> None:- async def handler(**_: object) -> dict:- return {- "verdicts": [- {- "criterion_id": "c0",- "satisfied": False,- "reason_code": "no_evidence",- "missing_evidence": "block output containing the requested heading and first paragraph text",- }- ]- }-- _patch_completion_handler(monkeypatch, handler)- ctx = _ctx_with_blocks("goto_url")- ctx.request_policy = RequestPolicy(completion_criteria=[_criterion("c0", "heading and paragraph are extracted")])-- verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())- assert verification is not None- assert verification.is_fully_satisfied() is False-- _record_run_blocks_result(ctx, _goto_only_result(), completion_verification=verification)-- assert ctx.last_full_workflow_test_ok is False- assert getattr(ctx, "last_good_workflow", None) is None- assert verified_goal_satisfied_context(ctx) is False--@pytest.mark.asyncioasync def test_structured_blocker_run_skips_completion_verification(monkeypatch: pytest.MonkeyPatch) -> None:async def handler(**_: object) -> dict:@@ -5321,11 +4378,9 @@ async def test_classifier_fallback_record_is_not_verified_without_judge(monkeypaassert verification is None_record_run_blocks_result(ctx, result, completion_verification=verification)- # The strict barrier predicate and its telemetry flag stay false, so the proposal is- # not preserved as a verified success; legacy clean-run flags may still promote, as- # they do for any genuine zero-criteria run.- assert getattr(ctx, "verified_terminal_proposal_ready", False) is not True- assert outcome_fully_verified(ctx) is False+ # The run committed a demonstrated outcome, which is the whole claim; no judge pass is+ # required to promote it.+ assert outcome_fully_verified(ctx) is True@pytest.mark.asyncio@@ -5708,12 +4763,6 @@ def test_current_workflow_has_evidence_block() -> None:assert _current_workflow_has_evidence_block(_run_ctx()) is False-def test_active_terminal_watchdog_exit_cannot_promote_to_terminal_success() -> None:- assert _watchdog_exit_allows_terminal_promotion("active_run_terminal_evidence") is False- assert _watchdog_exit_allows_terminal_promotion("per_tool_budget") is True- assert _watchdog_exit_allows_terminal_promotion("task_exit_unfinalized") is True--def test_outcome_failure_warrants_repair() -> None:has_block = _ctx_with_blocks("extraction")nav_only = _ctx_with_blocks("goto_url", "navigation")@@ -5927,287 +4976,79 @@ def test_artifact_health_not_suppressed_by_keyword_only_anti_bot_category() -> Nassert failure_classes == ["SyntaxError"]-def test_artifact_health_skips_when_anti_bot_category_is_carrier_backed() -> None:- result = _syntax_error_result_with_anti_bot_category("challenge_state")-- reason, failed_labels, failure_classes = _artifact_health_blocker_from_result(result)-- assert reason is None- assert failed_labels == []- assert failure_classes == []---def test_unfinished_run_verification_candidate_admits_canceled_with_evidence() -> None:- ctx = _run_ctx()- assert _is_unfinished_run_verification_candidate(ctx, _canceled_budget_result()) is True- # ok=True belongs to the clean-success candidate path, not this one.- assert _is_unfinished_run_verification_candidate(ctx, _clean_success_result()) is False- # ok=False with no reached runtime URL leaves nothing to judge.- assert _is_unfinished_run_verification_candidate(ctx, {"ok": False, "data": {}}) is False---def test_artifact_health_blocks_fully_satisfied_failed_run() -> None:- result = _failed_generated_code_result()- ctx = _run_ctx()- ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))- ctx.last_workflow_yaml = "workflow: {}"-- snapshot = _build_run_evidence_snapshot(ctx, result)- rendered = snapshot.render_prompt_block()- assert "run_terminal_status: failed" in rendered- assert "failure_classes: SyntaxError" in rendered- assert "failed_block_labels: extract_results" in rendered-- _record_run_blocks_result(ctx, result, completion_verification=_evaluated(("c0", True)))-- assert ctx.last_artifact_health_blocker_reason is not None- assert "SyntaxError" in ctx.last_artifact_health_blocker_reason- assert ctx.last_artifact_health_blocker_labels == ["extract_results"]- assert ctx.last_artifact_health_failure_classes == ["SyntaxError"]- assert outcome_fully_verified(ctx) is False- assert verified_goal_satisfied_context(ctx) is False- assert _verified_workflow_or_none(ctx) == (None, None)---def test_artifact_health_blocks_committed_same_run_outcome() -> None:- ctx = _ctx_with_blocks("extraction")- verification = _evaluated(("c0", True))-- _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)- ctx.last_artifact_health_blocker_reason = "Code block failed with SyntaxError."-- assert outcome_fully_verified(ctx) is False- assert verified_goal_satisfied_context(ctx) is False---async def test_maybe_run_completion_verification_runs_on_canceled_run(monkeypatch: pytest.MonkeyPatch) -> None:- async def handler(**_: object) -> dict:- return {- "verdicts": [- {- "criterion_id": "c0",- "satisfied": True,- "reason_code": "evidence_confirms",- "evidence_ref": "observed_end_state_url",- }- ]- }-- _patch_completion_handler(monkeypatch, handler)- ctx = _run_ctx()- result = await _maybe_run_completion_verification(ctx, _canceled_budget_result(), time.monotonic())- assert result is not None- assert result.status == "evaluated"- assert result.is_fully_satisfied() is True---async def test_maybe_run_completion_verification_skips_active_terminal_evidence(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def handler(**_: object) -> dict:- raise AssertionError("active-run terminal evidence must not be promoted to final success")-- async def fake_completion_verification_handler(_ctx: object) -> object:- return handler-- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",- fake_completion_verification_handler,- )- ctx = _run_ctx()- result = _canceled_budget_result()- result["data"]["active_run_terminal_evidence_detected"] = True-- assert await _maybe_run_completion_verification(ctx, result, time.monotonic()) is None---async def test_active_run_terminal_evidence_sample_matches_current_page(- monkeypatch: pytest.MonkeyPatch,-) -> None:- captured: dict[str, object] = {}-- async def handler(**kwargs: object) -> dict:- captured.update(kwargs)- return {- "verdicts": [- {- "criterion_id": "c0",- "satisfied": True,- "reason_code": "evidence_confirms",- "evidence_ref": "page_evidence.result_containers",- }- ]- }-- async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:- return "https://example.com/cart", "Cart"-- async def fake_capture_composition_evidence(- _ctx: object,- *,- inspected_url: str,- current_url: str,- active_run_terminal_sample: bool = False,- solve_challenges: bool = True,- ) -> tuple[dict, None]:- captured["active_run_terminal_sample"] = active_run_terminal_sample- captured["solve_challenges"] = solve_challenges- return (- {- "inspected_url": inspected_url,- "current_url": current_url,- "page_title": "Cart",- "visible_text_excerpt": "Cart TESTBRAND PART-001-TEST quantity 1",- "forms": [],- "result_containers": [{"selector": "#cart"}],- "anti_bot_indicators": [],- },- None,- )-- async def fake_completion_verification_handler(_ctx: object) -> object:- return handler-- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",- fake_completion_verification_handler,- )- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._fallback_page_info", fake_fallback_page_info- )- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._capture_composition_evidence",- fake_capture_composition_evidence,- )- ctx = _run_ctx()-- sample = await _active_run_terminal_evidence_sample(- ctx,- workflow_run_id="wr_active",- labels_to_execute=["search_and_add"],- sample_index=1,- )-- assert sample is not None- assert sample.completion_verification.is_fully_satisfied() is True- assert sample.current_url == "https://example.com/cart"- assert sample.page_evidence["observed_during_active_workflow_run"] is True- assert captured["active_run_terminal_sample"] is True- # The executor still owns this page; sampling it must never drive a challenge solve.- assert captured["solve_challenges"] is False- assert "PART-001-TEST" in str(captured["prompt"])+def test_artifact_health_skips_when_anti_bot_category_is_carrier_backed() -> None:+ result = _syntax_error_result_with_anti_bot_category("challenge_state")+ reason, failed_labels, failure_classes = _artifact_health_blocker_from_result(result)-async def test_active_run_terminal_evidence_sample_skips_method_only_criteria(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def handler(**_: object) -> dict:- raise AssertionError("method-mandated criteria cannot be verified from page state")+ assert reason is None+ assert failed_labels == []+ assert failure_classes == []- monkeypatch.setattr(- "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler", lambda: handler- )- ctx = _run_ctx()- ctx.request_policy = RequestPolicy(- completion_criteria=[_criterion("c0", "must use website search", method_mandated=True)]- )- assert (- await _active_run_terminal_evidence_sample(- ctx,- workflow_run_id="wr_active",- labels_to_execute=["search_and_add"],- sample_index=1,- )- is None- )+def test_unfinished_run_verification_candidate_admits_canceled_with_evidence() -> None:+ ctx = _run_ctx()+ assert _is_unfinished_run_verification_candidate(ctx, _canceled_budget_result()) is True+ # ok=True belongs to the clean-success candidate path, not this one.+ assert _is_unfinished_run_verification_candidate(ctx, _clean_success_result()) is False+ # ok=False with no reached runtime URL leaves nothing to judge.+ assert _is_unfinished_run_verification_candidate(ctx, {"ok": False, "data": {}}) is False-def test_active_run_terminal_evidence_result_shape_is_not_final_success() -> None:- sample = SimpleNamespace(- current_url="https://example.com/cart",- page_title="Cart",- sample_index=2,- completion_verification=_evaluated(("c0", True)),- page_evidence={- "current_url": "https://example.com/cart",- "page_title": "Cart",- "visible_text_excerpt": "Cart TESTBRAND PART-001-TEST quantity 1",- },- )+def test_artifact_health_blocks_fully_satisfied_failed_run() -> None:+ result = _failed_generated_code_result()+ ctx = _run_ctx()+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))+ ctx.last_workflow_yaml = "workflow: {}"- result = _active_run_terminal_evidence_result(- workflow_run_id="wr_active",- run_status="running",- sample=sample,- requested_block_labels=["search_and_add"],- executed_block_labels=["search_and_add"],- )+ snapshot = _build_run_evidence_snapshot(ctx, result)+ rendered = snapshot.render_prompt_block()+ assert "run_terminal_status: failed" in rendered+ assert "failure_classes: SyntaxError" in rendered+ assert "failed_block_labels: extract_results" in rendered- assert result["ok"] is False- assert result["data"]["active_run_terminal_evidence_detected"] is True- assert result["data"]["active_run_terminal_evidence_reason_code"] == ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE- assert result["data"]["full_workflow_verified"] is False- assert result["data"]["failure_categories"][0]["category"] == ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY+ _record_run_blocks_result(ctx, result, completion_verification=_evaluated(("c0", True)))+ assert ctx.last_artifact_health_blocker_reason is not None+ assert "SyntaxError" in ctx.last_artifact_health_blocker_reason+ assert ctx.last_artifact_health_blocker_labels == ["extract_results"]+ assert ctx.last_artifact_health_failure_classes == ["SyntaxError"]+ assert outcome_fully_verified(ctx) is False+ assert verified_goal_satisfied_context(ctx) is False+ assert _verified_workflow_or_none(ctx) == (None, None)-def test_active_run_terminal_evidence_contract_noops_when_outcome_fully_verified() -> None:- sample = SimpleNamespace(- current_url="https://example.com/cart",- page_title="Cart",- sample_index=2,- completion_verification=_evaluated(("c0", True)),- page_evidence={- "current_url": "https://example.com/cart",- "page_title": "Cart",- "visible_text_excerpt": "Cart TESTBRAND PART-001-TEST quantity 1",- },- )- result = _active_run_terminal_evidence_result(- workflow_run_id="wr_active",- run_status="canceled",- sample=sample,- requested_block_labels=["search_and_add"],- executed_block_labels=["search_and_add"],- )- ctx = _run_ctx()- ctx.completion_verification_result = _evaluated(("c0", True))- contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks", result=result, ctx=ctx)+def test_artifact_health_blocks_committed_same_run_outcome() -> None:+ ctx = _ctx_with_blocks("extraction")+ verification = _evaluated(("c0", True))- assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE- assert contract.repair_decision.next_action == RepairNextAction.NO_CHANGE- assert contract.verification_result.user_goal_satisfied is True- assert contract.verification_result.completion_contract_satisfied is True+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)+ ctx.last_artifact_health_blocker_reason = "Code block failed with SyntaxError."+ assert outcome_fully_verified(ctx) is False+ assert verified_goal_satisfied_context(ctx) is False-def test_active_run_terminal_evidence_contract_requires_reason_code_for_verified_noop() -> None:- sample = SimpleNamespace(- current_url="https://example.com/cart",- page_title="Cart",- sample_index=2,- completion_verification=_evaluated(("c0", True)),- page_evidence={"current_url": "https://example.com/cart"},- )- result = _active_run_terminal_evidence_result(- workflow_run_id="wr_active",- run_status="canceled",- sample=sample,- requested_block_labels=["search_and_add"],- executed_block_labels=["search_and_add"],- )- result["data"].pop("active_run_terminal_evidence_reason_code")- ctx = _run_ctx()- ctx.completion_verification_result = _evaluated(("c0", True))- contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks", result=result, ctx=ctx)+async def test_maybe_run_completion_verification_runs_on_canceled_run(monkeypatch: pytest.MonkeyPatch) -> None:+ async def handler(**_: object) -> dict:+ return {+ "verdicts": [+ {+ "criterion_id": "c0",+ "satisfied": True,+ "reason_code": "evidence_confirms",+ "evidence_ref": "observed_end_state_url",+ }+ ]+ }- assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCE- assert contract.repair_decision.next_action == RepairNextAction.STOP+ _patch_completion_handler(monkeypatch, handler)+ ctx = _run_ctx()+ result = await _maybe_run_completion_verification(ctx, _canceled_budget_result(), time.monotonic())+ assert result is not None+ assert result.status == "evaluated"+ assert result.is_fully_satisfied() is Truedef test_terminal_challenge_contract_still_stops_when_outcome_fully_verified() -> None:@@ -6233,56 +5074,6 @@ def test_terminal_challenge_contract_still_stops_when_outcome_fully_verified() -assert contract.verification_result.completion_contract_satisfied is False-def test_record_active_run_terminal_evidence_keeps_workflow_unverified() -> None:- sample = SimpleNamespace(- current_url="https://example.com/cart",- page_title="Cart",- sample_index=2,- completion_verification=_evaluated(("c0", True)),- page_evidence={- "current_url": "https://example.com/cart",- "page_title": "Cart",- "visible_text_excerpt": "Cart TESTBRAND PART-001-TEST quantity 1",- },- )- result = _active_run_terminal_evidence_result(- workflow_run_id="wr_active",- run_status="canceled",- sample=sample,- requested_block_labels=["search_and_add"],- executed_block_labels=["search_and_add"],- )- ctx = _run_ctx()-- _record_run_blocks_result(ctx, result)-- assert ctx.last_test_ok is False- assert ctx.last_full_workflow_test_ok is False- assert ctx.last_failure_category_top == ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY- assert ctx.workflow_verification_evidence.full_workflow_verified is False- assert ctx.workflow_verification_evidence.live_page_state_verified is True- assert ctx.workflow_verification_evidence.active_run_terminal_evidence_detected is True- assert ctx.workflow_verification_evidence.active_run_terminal_evidence_workflow_run_id == "wr_active"- assert ctx.workflow_verification_evidence.active_run_terminal_evidence_sample_index == 2- assert ctx.blocker_signal is not None- assert ctx.blocker_signal.internal_reason_code == "tool_error_active_run_terminal_evidence"---def test_active_run_terminal_evidence_blocks_same_turn_mutation_tools() -> None:- ctx = _run_ctx()- ctx.last_failure_category_top = ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY- ctx.workflow_verification_evidence.active_run_terminal_evidence_detected = True- ctx.workflow_verification_evidence.current_url = "https://example.com/cart"- ctx.workflow_verification_evidence.workflow_run_id = "wr_active"-- result = _tool_loop_error(ctx, "update_and_run_blocks", {"block_labels": ["search_and_add"]})-- assert result is not None- assert "ACTIVE_RUN_TERMINAL_EVIDENCE" in result- assert ctx.blocker_signal is not None- assert ctx.blocker_signal.internal_reason_code == "tool_error_active_run_terminal_evidence"--def test_outcome_fully_verified_predicate() -> None:ctx = _gate_ctx()ctx.completion_verification_result = _evaluated(("c0", True))@@ -10680,7 +9471,6 @@ async def test_run_finalization_abstains_plain_no_evidence_for_registered_downlo}assert verification.to_trace_data()["plain_outcome_no_evidence_abstention_engaged"] is Trueassert verification.requested_output_criteria_count == 1- assert zero_requested_output_criteria_credit(verification, has_meaningful_registered_output=True) is False@pytest.mark.asyncio@@ -10992,78 +9782,6 @@ async def test_download_output_path_reconciles_with_registered_download_evidenceassert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]-async def test_marked_download_deliverable_without_registered_evidence_fails_closed(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def fail_handler(**_: object) -> object:- raise AssertionError("missing registered download evidence must be deterministic")-- _patch_completion_handler(monkeypatch, fail_handler)- ctx = _run_ctx()- _set_workflow_labels(ctx, "download_document")- ctx.code_artifact_metadata = _metadata_for_requested_paths("output_id")- ctx.reached_download_target = ReachedDownloadTarget(- selector='a[href="/files/statement.pdf"]',- affordance_text="Download",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )- ctx.request_policy = RequestPolicy(- completion_criteria=[- _criterion(- "c_output_id",- "The returned record includes output id.",- output_path="output.output_id",- deliverable_kind="registered_download",- )- ]- )-- verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())-- assert verification is not None- assert verification.is_fully_satisfied() is False- verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}- assert verdicts["c_output_id"].reason_code == "no_evidence"- assert verdicts[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID].reason_code == "no_evidence"---async def test_download_typed_affordance_injects_and_fails_without_registered_output(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def fail_handler(**_: object) -> object:- raise AssertionError("missing registered download evidence must be deterministic")-- _patch_completion_handler(monkeypatch, fail_handler)- ctx = _run_ctx()- ctx.request_policy = RequestPolicy(completion_criteria=[])- ctx.reached_download_target = ReachedDownloadTarget(- selector='a[href="/files/statement.pdf"]',- affordance_text="Download",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )-- verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())-- assert verification is not None- assert verification.is_fully_satisfied() is False- assert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]- assert verification.verdicts == [- CriterionVerdict(- criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,- state="unsatisfied",- reason_code="no_evidence",- missing_evidence="run output did not include a non-empty registered browser download",- )- ]- assert ctx.request_policy.completion_criteria == []--@pytest.mark.asyncioasync def test_definition_only_non_download_remains_no_gradeable_run_plane(monkeypatch: pytest.MonkeyPatch,@@ -11215,7 +9933,6 @@ async def test_degraded_minted_download_ask_still_reaches_abstention(assert verification.criterion_ids == ["c_outcome", REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]assert verification.to_trace_data()["plain_outcome_no_evidence_abstention_engaged"] is Trueassert verification.requested_output_criteria_count == 1- assert zero_requested_output_criteria_credit(verification, has_meaningful_registered_output=True) is False@pytest.mark.asyncio@@ -11248,43 +9965,6 @@ async def test_incidental_download_does_not_license_plain_no_evidence_abstentionassert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID not in verification.criterion_requested_output_mint_source_by_id-async def test_degraded_minted_ask_cannot_manufacture_confirmation_without_download_evidence(- monkeypatch: pytest.MonkeyPatch,-) -> None:- async def handler(**_: object) -> dict[str, object]:- return {- "verdicts": [- {- "criterion_id": "c_outcome",- "satisfied": False,- "reason_code": "no_evidence",- }- ]- }-- _patch_completion_handler(monkeypatch, handler)- ctx = _run_ctx()- ctx.reached_download_target = ReachedDownloadTarget(- selector='a[href="/files/statement.pdf"]',- affordance_text="Download",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )- ctx.request_policy = RequestPolicy(- completion_criteria=[_plain_no_evidence_criterion(associated=True), _degraded_minted_download_criterion()]- )-- verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())-- assert verification is not None- assert verification.is_fully_satisfied() is False- assert "plain_outcome_no_evidence_abstention_engaged" not in verification.to_trace_data()- assert "c_download" not in verification.criterion_ids- assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID not in verification.criterion_requested_output_mint_source_by_id--@pytest.mark.asyncioasync def test_minted_download_ask_counts_as_requested_output_and_verifies(monkeypatch: pytest.MonkeyPatch,@@ -11312,7 +9992,6 @@ async def test_minted_download_ask_counts_as_requested_output_and_verifies(assert verification.is_fully_satisfied() is Trueassert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]assert verification.requested_output_criteria_count == 1- assert zero_requested_output_criteria_credit(verification, has_meaningful_registered_output=True) is False@pytest.mark.asyncio@@ -11334,7 +10013,6 @@ async def test_incidental_download_without_ask_stays_credit_withheld(assert verification is not Noneassert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]assert verification.requested_output_criteria_count == 0- assert zero_requested_output_criteria_credit(verification, has_meaningful_registered_output=True) is True@pytest.mark.asyncio@@ -11364,7 +10042,6 @@ async def test_predeploy_persisted_download_ask_is_undercredited_until_reminted(assert verification is not Noneassert verification.requested_output_criteria_count == 0- assert zero_requested_output_criteria_credit(verification, has_meaningful_registered_output=True) is Truedef _declared_download_criterion(cid: str = "c_download") -> CompletionCriterion:@@ -11407,7 +10084,6 @@ async def test_declared_download_ask_reaches_abstention(monkeypatch: pytest.Monkassert trace["plain_outcome_no_evidence_abstention_engaged"] is Trueassert trace["plain_outcome_no_evidence_abstention_confirming_deliverable_mint_sources"] == ["classifier_declared"]assert verification.requested_output_criteria_count == 1- assert zero_requested_output_criteria_credit(verification, has_meaningful_registered_output=True) is False@pytest.mark.asyncio@@ -14151,84 +12827,6 @@ def test_run_evidence_snapshot_preserves_registered_null_for_family_routing() ->assert snapshot.block_outputs["submit_commercial_water_request"]["blocker"] == "incidental runtime value"- ("classification_output_key", "emitted_output"),- [- ("public_form_exists", {"public_form_exists": False}),- ("login_only", {"login_only": True}),- ],- ids=["correct_negative", "login_only"],-)-async def test_neutral_reported_boolean_accepts_evidence_without_floor_rekey_withhold(- classification_output_key: str,- emitted_output: dict[str, bool],-) -> None:- criterion = CompletionCriterion(- id="a" * 64,- outcome=f"The run reports whether {classification_output_key.replace('_', ' ')}.",- antecedent_family="unconditional",- expected_output_shape="goal_judgment_boolean",- requested_output_evidence_source="independent_run_evidence",- kind="outcome",- classification_output_key=classification_output_key,- request_slot_id="a" * 64,- pinability="shapeless_valid",- mint_disposition="decidable",- )- snapshot = RunEvidenceSnapshot(- workflow_run_id="wr_553389131952493792",- block_outputs={- "reported_summary": emitted_output,- "post_run_page_observation": {- "forms": [],- "result_containers": [],- "visible_text_excerpt": "Sign in or register to continue.",- },- },- block_output_sources={- "reported_summary": "registered_output_parameter",- "post_run_page_observation": "independent_page_evidence",- },- registered_output_evidence_by_request_slot_id={- criterion.id: (- RegisteredBlockerEvidence(- block_label="report_outcome",- output_path=f"output.{classification_output_key}",- registered_output_key="reported_summary",- registered_output_id="op_reported_summary",- value=emitted_output[classification_output_key],- ),- )- },- run_terminal_status="completed",- )-- async def verifier(*, prompt: str, **_: object) -> dict[str, object]:- assert f"classification_output_key={classification_output_key}" in prompt- assert json.dumps(emitted_output, sort_keys=True)[1:-1] in prompt- return {- "verdicts": [- {- "criterion_id": criterion.id,- "satisfied": True,- "reason_code": "evidence_confirms",- "evidence_ref": "post_run_page_observation.forms",- }- ]- }-- result = await evaluate_completion_criteria([criterion], snapshot, verifier)- result = carry_floor_rekeyed_criterion_ids(result, [criterion])-- assert result.is_fully_satisfied() is True- assert result.floor_rekeyed_criterion_ids == []- assert floor_rekeyed_emission_withhold(result) is None- assert result.verdicts[0].reason_code == "evidence_confirms"- assert result.verdicts[0].evidence_source == "independent_page_evidence"- assert all(verdict.reason_code != "evidence_contradicts" for verdict in result.verdicts)--@pytest.mark.asyncio@pytest.mark.parametrize("antecedent_family", ["unconditional", "blocker"])async def test_neutral_reported_boolean_rejects_its_associated_output_as_evidence(diff --git a/tests/unit/test_copilot_completion_zero_run_plane.py b/tests/unit/test_copilot_completion_zero_run_plane.pyindex 9ab3397d5..7747d3f3c 100644--- a/tests/unit/test_copilot_completion_zero_run_plane.py+++ b/tests/unit/test_copilot_completion_zero_run_plane.py@@ -253,7 +253,7 @@ async def test_observation_seam_non_candidate_does_not_fire() -> None:assert verification is None-def test_empty_fallback_marker_records_built_unverified_not_no_meaningful_output() -> None:+def test_empty_fallback_marker_does_not_block_a_produced_output() -> None:ctx = _ctx()ctx.request_policy = RequestPolicy(completion_criteria=[], classifier_status="fallback")ctx.last_workflow = SimpleNamespace(@@ -267,11 +267,8 @@ def test_empty_fallback_marker_records_built_unverified_not_no_meaningful_outputrecorded = _record_run_blocks_result(ctx, _real_output_result(), completion_verification=marker)assert isinstance(recorded, RecordedRunOutcome)- assert recorded.verdict == "not_demonstrated"- assert recorded.reason_code != "no_meaningful_output"+ assert recorded.verdict == "demonstrated"assert ctx.last_test_suspicious_success is False- assert ctx.last_full_workflow_test_ok is False- assert _NO_GRADEABLE_PROSE in (recorded.display_reason or "")def test_marker_does_not_leak_into_verdict_consumers() -> None:diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.pyindex 17b13b016..8b8aad519 100644--- a/tests/unit/test_copilot_credential_pause.py+++ b/tests/unit/test_copilot_credential_pause.py@@ -1599,37 +1599,6 @@ def test_copilot_seconds_remaining_credits_pause_time() -> None:assert remaining > 150.0-def test_late_block_running_call_signal_allows_call_after_credited_pause() -> None:- """Fails on old code: a 300s pause 700s into the turn wrongly forbids the- resumed test run because the wall-clock check never saw the pause credit."""- from skyvern.forge.sdk.copilot.tools.blockers import _late_block_running_call_signal-- ctx = SimpleNamespace(- copilot_run_start_monotonic=time.monotonic() - 1000.0,- copilot_credential_pause_seconds=300.0,- last_failed_workflow_yaml=None,- last_good_workflow_yaml=None,- )-- assert _late_block_running_call_signal(ctx, "update_and_run_blocks") is None---def test_late_block_running_call_signal_forbids_same_elapsed_without_pause_credit() -> None:- """Sanity pair for the test above: the identical wall-clock elapsed DOES- forbid the call when there was no pause to credit, proving the crediting- -- not something else -- is what keeps the resumed call allowed."""- from skyvern.forge.sdk.copilot.tools.blockers import _late_block_running_call_signal-- ctx = SimpleNamespace(- copilot_run_start_monotonic=time.monotonic() - 1000.0,- copilot_credential_pause_seconds=0.0,- last_failed_workflow_yaml=None,- last_good_workflow_yaml=None,- )-- assert _late_block_running_call_signal(ctx, "update_and_run_blocks") is not None--# ---------------------------------------------------------------------------# A client that can't render credential_required must never be paused for,# or it stares at "Working..." until the timeout.diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex d708eeddf..1b9d539ea 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -1321,87 +1321,6 @@ def test_unverified_completion_evidence_does_not_suppress_suspicious_success(assert contract.verification_result.user_goal_satisfied is False-def test_degraded_delivered_unverified_completion_does_not_route_repair() -> None:- ctx = _ctx()- ctx.last_test_suspicious_success = True- ctx.last_run_blocks_workflow_run_id = "wr_unverified"- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_workflow_run_id = "wr_unverified"- ctx.delivered_unverified_observed_outputs = {"document_name": "Resale Demand Package"}- ctx.completion_verification_result = CompletionVerificationResult(- status="evaluated",- criterion_ids=["__copilot_fallback_floor__run", "requested_output"],- verdicts=[- CriterionVerdict(- criterion_id="__copilot_fallback_floor__run",- state="unsatisfied",- reason_code="no_evidence",- ),- CriterionVerdict(- criterion_id="requested_output",- state="unsatisfied",- reason_code="structurally_abstained",- evidence_ref="block_outputs:extract.document_name",- output_path="output.document_name",- grounding_mode="missing",- evidence_source="runtime_output",- ),- ],- degraded_criterion_ids=["__copilot_fallback_floor__run"],- )-- contract = build_diagnosis_repair_contract(- source_tool="update_and_run_blocks",- result={- "ok": True,- "data": {- "workflow_run_id": "wr_unverified",- "overall_status": "completed",- "frontier_start_label": "extract",- "failure_categories": [{"category": "OUTCOME_UNVERIFIED"}],- "blocks": [{"label": "extract", "block_type": "EXTRACTION", "status": "completed"}],- },- },- ctx=ctx,- workflow_updated=True,- )-- assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.DELIVERED_UNVERIFIED- assert contract.repair_decision.next_action == RepairNextAction.NO_CHANGE- assert contract.verification_result.user_goal_satisfied is True- assert contract.verification_result.completion_contract_satisfied is False---def test_delivered_unverified_does_not_mask_failed_blocks() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_workflow_run_id = "wr_unverified"- ctx.delivered_unverified_observed_outputs = {"document_name": "Resale Demand Package"}-- contract = build_diagnosis_repair_contract(- source_tool="update_and_run_blocks",- result={- "ok": True,- "data": {- "workflow_run_id": "wr_unverified",- "overall_status": "completed",- "blocks": [- {- "label": "extract",- "block_type": "EXTRACTION",- "status": "failed",- }- ],- },- },- ctx=ctx,- workflow_updated=True,- )-- assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE- assert contract.repair_decision.next_action == RepairNextAction.REPAIR--def test_degraded_path_without_terminal_state_still_routes_repair() -> None:ctx = _ctx()ctx.last_test_suspicious_success = True@@ -1808,37 +1727,6 @@ def test_unresolved_symbol_context_does_not_preempt_terminal_challenge_stop() ->assert contract.to_trace_data()["next_action"] == "stop"-def test_active_run_terminal_evidence_contract_stops_without_marking_workflow_success() -> None:- contract = build_diagnosis_repair_contract(- source_tool="update_and_run_blocks",- result={- "ok": False,- "error": "Active run terminal evidence was observed.",- "data": {- "workflow_run_id": "wr_active",- "overall_status": "canceled",- "active_run_terminal_evidence_detected": True,- "active_run_terminal_completion_verification": {- "status": "evaluated",- "criterion_count": 1,- "satisfied_count": 1,- "fully_satisfied": True,- "reason_codes": ["evidence_confirms"],- },- "failure_categories": [{"category": "ACTIVE_RUN_TERMINAL_EVIDENCE"}],- },- },- ctx=_ctx(),- workflow_updated=True,- )-- assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCE- assert contract.repair_decision.next_action == RepairNextAction.STOP- assert contract.verification_result.user_goal_satisfied is True- assert contract.verification_result.completion_contract_satisfied is True- assert "not verified end-to-end" in contract.repair_decision.proposed_change_summary--@pytest.mark.parametrize(("suspicious", "completion_verification", "anti_bot", "failure_reason", "run_id"),[diff --git a/tests/unit/test_copilot_enforcement_hardening.py b/tests/unit/test_copilot_enforcement_hardening.pyindex 9270fb4eb..3fb4b999c 100644--- a/tests/unit/test_copilot_enforcement_hardening.py+++ b/tests/unit/test_copilot_enforcement_hardening.py@@ -21,18 +21,13 @@ from typing import Anyimport pytest-from skyvern.forge.sdk.copilot.agent import (- _finalize_result_with_blocker_override,- _with_scouted_spine_missing_steps,-)from skyvern.forge.sdk.copilot.blocker_signal import (CopilotToolBlockerSignal,clear_terminal_evidence_on_workflow_edit,- maybe_clear_blocker_signal_on_tool_success,)from skyvern.forge.sdk.copilot.code_block_synthesis import UNFORGIVEN_DROP_FINDING, ObligationFindingfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy-from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext+from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisInput,DiagnosisRepairContract,@@ -42,44 +37,28 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (VerificationResult,)from skyvern.forge.sdk.copilot.enforcement import (- MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,- MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,- MAX_NO_PROGRESS_INTERACTION_ATTEMPTS,- MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES,POST_FAILED_TEST_INSPECT_FIRST_NUDGE,POST_FAILED_TEST_NUDGE,POST_NAVIGATE_NUDGE,POST_PER_TOOL_BUDGET_NUDGE,POST_PER_TOOL_BUDGET_STOP_NUDGE,POST_SUSPICIOUS_SUCCESS_NUDGE,- PROBABLE_SITE_BLOCK_STREAK_STOP_AT,SCREENSHOT_PLACEHOLDER,- CopilotNonRetriableNavError,_check_enforcement,_code_authoring_reject_count_resets,_is_context_window_error,- _maybe_raise_non_retriable_nav,_needs_inspect_before_repair_nudge,_prune_input_list,- _record_code_authoring_guardrail_reject,_recover_from_context_overflow,_scouted_spine_missing_text,_strip_input_images,- code_authoring_churn_stop_would_claim,register_no_progress_interaction_click,reset_no_progress_interaction_count,synthesized_trajectory_reaches_goal,)from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState-from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODEfrom skyvern.forge.sdk.copilot.streaming_adapter import _update_enforcement_from_toolfrom skyvern.forge.sdk.copilot.tools.workflow_update import _pre_persist_scouted_spine_result-from skyvern.forge.sdk.copilot.turn_halt import (- ADVISORY_DISPATCH_STALLED_REASON_CODE,- CopilotTurnHalt,- TurnHaltKind,- expire_output_contract_ladder_at_turn_end,-)from skyvern.forge.sdk.copilot.turn_ownership import TurnClaimantfrom tests.unit.conftest import make_copilot_context as _fresh_context@@ -378,92 +357,6 @@ def test_is_context_window_error_matches_only_overflow_variants(msg: str, expectassert _is_context_window_error(Exception(msg)) is expected-def test_code_authoring_churn_backstop_raises_at_ceiling() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "code_authoring_guardrail_churn"---def test_code_authoring_churn_backstop_does_not_raise_below_ceiling() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1-- assert _check_enforcement(ctx) is None---def test_code_authoring_churn_backstop_defers_to_terminal_blocker() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- terminal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="A site verification challenge blocked the run.",- user_facing_reason="The site's verification challenge blocked the run.",- recovery_hint="report_blocker_to_user",- internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,- blocked_tool="update_and_run_blocks",- )- ctx.blocker_signal = terminal-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE- assert ctx.blocker_signal is terminal---def test_code_authoring_churn_backstop_defers_to_newly_detected_site_block() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- ctx.probable_site_block_streak_count = PROBABLE_SITE_BLOCK_STREAK_STOP_AT- assert ctx.blocker_signal is None-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.PROBABLE_SITE_BLOCK- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "probable_site_block_stop"---def test_code_authoring_churn_backstop_fires_when_site_block_nudge_cap_spent() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- ctx.probable_site_block_streak_count = PROBABLE_SITE_BLOCK_STREAK_STOP_AT- ctx.probable_site_block_stop_nudge_count = MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "code_authoring_guardrail_churn"---def test_code_authoring_churn_backstop_yields_to_non_retriable_nav_error() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- ctx.last_test_ok = False- ctx.last_test_non_retriable_nav_error = (- "Failed to navigate to url https://does-not-resolve.example. Error message: net::ERR_NAME_NOT_RESOLVED"- )-- nudge = _check_enforcement(ctx)-- assert nudge is not None- assert ctx.blocker_signal is None- with pytest.raises(CopilotNonRetriableNavError):- _maybe_raise_non_retriable_nav(ctx)--def _repair_streak_contract() -> DiagnosisRepairContract:return DiagnosisRepairContract(diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),@@ -478,33 +371,6 @@ def _mark_recorded_run_backed(ctx: CopilotContext) -> None:ctx.recorded_persisted_block_run_workflow_run_id = "wr_1"-def test_repair_streak_still_reaches_the_code_authoring_churn_backstop() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- ctx.latest_diagnosis_repair_contract = _repair_streak_contract()-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "code_authoring_guardrail_churn"---def test_run_backed_repair_streak_reaches_the_code_authoring_churn_backstop() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- ctx.latest_diagnosis_repair_contract = _repair_streak_contract()- ctx.last_run_blocks_workflow_run_id = "wr_1"- _mark_recorded_run_backed(ctx)-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED--def test_workflow_edit_clears_recorded_persisted_run_latch() -> None:ctx = _fresh_context()ctx.last_run_blocks_workflow_run_id = "wr_1"@@ -516,139 +382,6 @@ def test_workflow_edit_clears_recorded_persisted_run_latch() -> None:assert ctx.recorded_persisted_block_run_workflow_run_id is None-def test_credential_priority_churn_raises_at_higher_bound() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS- ctx.last_code_authoring_reject_was_credential_priority = True-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "credential_priority_authoring_churn"- assert "verify the saved-credential login" in signal.user_facing_reason---def test_credential_priority_churn_defers_below_higher_bound() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS - 1- ctx.last_code_authoring_reject_was_credential_priority = True-- assert _check_enforcement(ctx) is None- assert ctx.blocker_signal is None---def test_repair_streak_still_reaches_the_credential_priority_churn() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS- ctx.last_code_authoring_reject_was_credential_priority = True- ctx.latest_diagnosis_repair_contract = _repair_streak_contract()-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "credential_priority_authoring_churn"---def test_run_backed_repair_streak_reaches_the_credential_priority_churn() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS- ctx.last_code_authoring_reject_was_credential_priority = True- ctx.latest_diagnosis_repair_contract = _repair_streak_contract()- ctx.last_run_blocks_workflow_run_id = "wr_1"- _mark_recorded_run_backed(ctx)-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED---def test_no_progress_interaction_floor_raises_at_ceiling() -> None:- ctx = _fresh_context()- ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "loop_detected_no_forward_progress_interaction"- assert signal.renders_final_reply is True- assert signal.recovery_hint == "report_blocker_to_user"---def test_no_progress_interaction_floor_does_not_raise_below_ceiling() -> None:- ctx = _fresh_context()- ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1-- assert _check_enforcement(ctx) is None- assert ctx.blocker_signal is None---def test_repair_streak_still_reaches_the_no_progress_interaction_floor() -> None:- ctx = _fresh_context()- ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS- ctx.latest_diagnosis_repair_contract = _repair_streak_contract()-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "loop_detected_no_forward_progress_interaction"---def test_run_backed_repair_streak_reaches_the_no_progress_interaction_floor() -> None:- # A repeated identical repair now steers instead of ending the turn, so a genuine terminal- # halt beneath it is the one that fires.- ctx = _fresh_context()- ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS- ctx.latest_diagnosis_repair_contract = _repair_streak_contract()- ctx.last_run_blocks_workflow_run_id = "wr_1"- _mark_recorded_run_backed(ctx)-- with pytest.raises(CopilotTurnHalt) as excinfo:- _check_enforcement(ctx)-- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED---def test_no_progress_interaction_floor_yields_to_non_retriable_nav_error() -> None:- ctx = _fresh_context()- ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS- ctx.last_test_ok = False- ctx.last_test_non_retriable_nav_error = (- "Failed to navigate to url https://does-not-resolve.example. Error message: net::ERR_NAME_NOT_RESOLVED"- )-- nudge = _check_enforcement(ctx)-- assert nudge is not None- assert ctx.blocker_signal is None- with pytest.raises(CopilotNonRetriableNavError):- _maybe_raise_non_retriable_nav(ctx)---def test_register_no_progress_interaction_click_stashes_blocker_at_cap() -> None:- ctx = _fresh_context()- ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1-- register_no_progress_interaction_click(ctx, outcome="click_failed")-- assert ctx.consecutive_no_progress_interaction_count == MAX_NO_PROGRESS_INTERACTION_ATTEMPTS- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "loop_detected_no_forward_progress_interaction"--def test_register_no_progress_interaction_click_below_cap_does_not_stash() -> None:ctx = _fresh_context()@@ -658,24 +391,6 @@ def test_register_no_progress_interaction_click_below_cap_does_not_stash() -> Noassert ctx.blocker_signal is None-def test_register_no_progress_interaction_click_defers_to_terminal_held_blocker() -> None:- ctx = _fresh_context()- ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1- terminal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="The site appears to be blocking this session.",- user_facing_reason="The site looks like it is blocking automated access.",- recovery_hint="report_blocker_to_user",- internal_reason_code="probable_site_block_stop",- blocked_tool="update_and_run_blocks",- )- ctx.blocker_signal = terminal-- register_no_progress_interaction_click(ctx, outcome="click_failed")-- assert ctx.blocker_signal is terminal--def test_reset_no_progress_interaction_count_clears_counter() -> None:ctx = _fresh_context()ctx.consecutive_no_progress_interaction_count = 3@@ -685,142 +400,10 @@ def test_reset_no_progress_interaction_count_clears_counter() -> None:assert ctx.consecutive_no_progress_interaction_count == 0-def _hit_no_progress_cap(ctx: CopilotContext) -> CopilotToolBlockerSignal:- for _ in range(MAX_NO_PROGRESS_INTERACTION_ATTEMPTS):- register_no_progress_interaction_click(ctx, outcome="hollow")- held = ctx.blocker_signal- assert isinstance(held, CopilotToolBlockerSignal)- assert held.internal_reason_code == "loop_detected_no_forward_progress_interaction"- assert held.renders_final_reply is True- return held---def test_no_progress_reset_clears_held_blocker_after_cap() -> None:- ctx = _fresh_context()- _hit_no_progress_cap(ctx)-- reset_no_progress_interaction_count(ctx)-- assert ctx.consecutive_no_progress_interaction_count == 0- assert ctx.blocker_signal is None- assert ctx.latest_tool_blocker_signal is None---def test_no_progress_reset_at_progress_seam_stops_re_halt() -> None:- ctx = _fresh_context()- _hit_no_progress_cap(ctx)-- reset_no_progress_interaction_count(ctx)-- assert ctx.consecutive_no_progress_interaction_count == 0- assert ctx.blocker_signal is None- assert _check_enforcement(ctx) is None--[email protected]("recovery_tool", ["evaluate", "inspect_page_for_composition"])-def test_no_progress_held_blocker_survives_progress_tool_success(recovery_tool: str) -> None:- ctx = _fresh_context()- held = _hit_no_progress_cap(ctx)-- maybe_clear_blocker_signal_on_tool_success(ctx, recovery_tool)-- assert ctx.blocker_signal is held--def _grant_output_contract_ladder(ctx: CopilotContext) -> None:ctx.output_contract_actuation_by_signature["sig_a"] = OutputContractAdvisoryState.GRANTED-def test_inline_churn_reject_yields_to_live_ladder_and_keeps_count() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1- _grant_output_contract_ladder(ctx)-- _record_code_authoring_guardrail_reject(ctx)-- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- assert ctx.blocker_signal is None- assert ctx.turn_halt is None- assert any(- event.fingerprint == "output_contract_actuation>code_authoring_guardrail_churn"- for event in ctx.gate_precedence_conflict_events- )---def test_inline_churn_reject_stashes_when_no_owner_is_live() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1-- _record_code_authoring_guardrail_reject(ctx)-- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "code_authoring_guardrail_churn"- assert signal.renders_final_reply is True---def test_churn_backstop_yields_to_live_ladder_without_halt_or_stash() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS- _grant_output_contract_ladder(ctx)-- assert _check_enforcement(ctx) is None-- assert ctx.blocker_signal is None- assert ctx.turn_halt is None- assert ctx.output_contract_actuation_by_signature["sig_a"] == OutputContractAdvisoryState.GRANTED- assert any(- event.fingerprint == "output_contract_actuation>code_authoring_guardrail_churn"- for event in ctx.gate_precedence_conflict_events- )---def test_no_progress_backstop_yields_to_live_ladder() -> None:- ctx = _fresh_context()- ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS- _grant_output_contract_ladder(ctx)-- assert _check_enforcement(ctx) is None-- assert ctx.blocker_signal is None- assert ctx.turn_halt is None- assert any(- event.fingerprint == "output_contract_actuation>loop_detected" for event in ctx.gate_precedence_conflict_events- )---def test_register_no_progress_click_yields_to_live_ladder() -> None:- ctx = _fresh_context()- ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1- _grant_output_contract_ladder(ctx)-- register_no_progress_interaction_click(ctx, outcome="failed")-- assert ctx.consecutive_no_progress_interaction_count == MAX_NO_PROGRESS_INTERACTION_ATTEMPTS- assert ctx.blocker_signal is None---def test_grant_plus_ceiling_reject_in_one_call_reaches_stalled_terminal_not_churn_reply() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1- _grant_output_contract_ladder(ctx)- ctx.output_contract_pending_run_evidence["sig_a"] = ["output.confirmation_number"]-- _record_code_authoring_guardrail_reject(ctx)- assert _check_enforcement(ctx) is None- assert ctx.blocker_signal is None- assert ctx.turn_halt is None-- halt = expire_output_contract_ladder_at_turn_end(ctx)-- assert halt is not None- assert ctx.turn_halt is halt- assert ctx.output_contract_actuation_by_signature["sig_a"] == OutputContractAdvisoryState.EXPIRED- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == ADVISORY_DISPATCH_STALLED_REASON_CODE--def test_single_captured_interaction_trajectory_never_reaches_goal() -> None:ctx = _fresh_context()ctx.scout_trajectory = [@@ -848,28 +431,6 @@ def test_scouted_spine_missing_text_renders_non_uncovered_families() -> None:assert "fill_credential_field" in text-def test_missing_steps_listed_on_give_up_offer_and_anchored_in_held_signal() -> None:- ctx = _fresh_context()- ctx.has_staged_proposal = True- ctx.blocker_signal = CopilotToolBlockerSignal(- blocker_kind="loop_detected",- agent_steering_text="The repair made no progress.",- user_facing_reason="I kept the draft.",- recovery_hint="report_blocker_to_user",- internal_reason_code="loop_detected_generic",- blocked_tool="update_workflow",- )- reply = _with_scouted_spine_missing_steps(ctx, "I kept the draft.", "`fill` on '#totp'")- assert "`fill` on '#totp'" in reply- assert "`fill` on '#totp'" in ctx.blocker_signal.user_facing_reason---def test_missing_steps_not_appended_without_staged_proposal() -> None:- ctx = _fresh_context()- ctx.has_staged_proposal = False- assert _with_scouted_spine_missing_steps(ctx, "base", "`fill` on '#totp'") == "base"--def _unrelated_owner_give_up_ctx(internal_reason_code: str) -> tuple[CopilotContext, str]:ctx = _fresh_context()ctx.has_staged_proposal = True@@ -897,33 +458,6 @@ def _unrelated_owner_give_up_ctx(internal_reason_code: str) -> tuple[CopilotContreturn ctx, unrelated_reason-def test_finalizer_names_missing_steps_when_unrelated_blocker_renders_give_up() -> None:- ctx, unrelated_reason = _unrelated_owner_give_up_ctx("output_contract_actuation_exhausted")- result = AgentResult(user_response="agent prose", updated_workflow=None, global_llm_context=None)-- overridden = _finalize_result_with_blocker_override(ctx, result, exit_site="test")-- assert "#search-submit" in overridden.user_response- assert ctx.blocker_signal.user_facing_reason == unrelated_reason- assert ctx.blocker_signal_claimant is TurnClaimant.OUTPUT_CONTRACT_ACTUATION--- "internal_reason_code",- ["loop_detected_generic", "completion_contract_unsatisfied", "output_contract_actuation_exhausted"],-)-def test_finalizer_render_exit_names_missing_steps_per_owner(internal_reason_code: str) -> None:- ctx, unrelated_reason = _unrelated_owner_give_up_ctx(internal_reason_code)- result = AgentResult(user_response="agent prose", updated_workflow=None, global_llm_context=None)-- overridden = _finalize_result_with_blocker_override(ctx, result, exit_site="test")-- assert "#search-submit" in overridden.user_response- assert overridden.user_response.count("This draft is still missing steps you demonstrated:") == 1- assert ctx.blocker_signal.user_facing_reason == unrelated_reason- assert ctx.blocker_signal_claimant is TurnClaimant.OUTPUT_CONTRACT_ACTUATION--def test_same_omission_spine_violation_still_refused_after_change() -> None:ctx = _fresh_context()ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER@@ -957,72 +491,3 @@ def test_reject_count_resets_only_on_non_repeat_without_frontier_unchanged() ->assert _code_authoring_reject_count_resets(False, True) is Falseassert _code_authoring_reject_count_resets(None, False) is Falseassert _code_authoring_reject_count_resets(True, False) is False---def test_churn_stop_would_claim_true_at_ceiling_and_mutates_nothing() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1-- assert code_authoring_churn_stop_would_claim(ctx) is True- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1- assert ctx.blocker_signal is None---def test_churn_stop_would_claim_false_below_ceiling() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 2-- assert code_authoring_churn_stop_would_claim(ctx) is False---def test_churn_stop_would_claim_false_when_terminal_blocker_held() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1- ctx.blocker_signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="A site verification challenge blocked the run.",- user_facing_reason="The site's verification challenge blocked the run.",- recovery_hint="report_blocker_to_user",- internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,- blocked_tool="update_and_run_blocks",- )-- assert code_authoring_churn_stop_would_claim(ctx) is False---def test_churn_stop_would_claim_yields_to_stronger_live_owner_and_mutates_nothing() -> None:- ctx = _fresh_context()- ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1- ctx.output_contract_actuation_by_signature = {"sig": OutputContractAdvisoryState.GRANTED}-- assert code_authoring_churn_stop_would_claim(ctx) is False- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1- assert ctx.blocker_signal is None---def test_churn_stop_would_claim_matches_recorder_claim_at_ceiling() -> None:- predicate_ctx = _fresh_context()- predicate_ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1- recorder_ctx = _fresh_context()- recorder_ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1-- would_claim = code_authoring_churn_stop_would_claim(predicate_ctx)- _record_code_authoring_guardrail_reject(recorder_ctx)-- assert would_claim is True- claimed = recorder_ctx.blocker_signal- assert isinstance(claimed, CopilotToolBlockerSignal)- assert claimed.internal_reason_code == "code_authoring_guardrail_churn"---def test_churn_stop_would_claim_matches_recorder_no_claim_below_ceiling() -> None:- predicate_ctx = _fresh_context()- predicate_ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 2- recorder_ctx = _fresh_context()- recorder_ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 2-- would_claim = code_authoring_churn_stop_would_claim(predicate_ctx)- _record_code_authoring_guardrail_reject(recorder_ctx)-- assert would_claim is False- assert recorder_ctx.blocker_signal is Nonediff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.pyindex d8ce578e5..836d8dd7b 100644--- a/tests/unit/test_copilot_enforcement_pruning.py+++ b/tests/unit/test_copilot_enforcement_pruning.py@@ -42,7 +42,6 @@ from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.enforcement import (KEEP_RECENT_TOOL_OUTPUTS,SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,- CopilotGoalSatisfied,_check_enforcement,_maybe_synthesized_block_offer_msg,_needs_suspicious_success_nudge,@@ -70,13 +69,10 @@ from skyvern.forge.sdk.copilot.enforcement import ()from skyvern.forge.sdk.copilot.mcp_adapter import (_POST_HOOK_CONTEXT_ROLLBACK_FIELDS,- SchemaOverlay,- SkyvernOverlayMCPServer,_restore_post_hook_context,_snapshot_post_hook_context,)from skyvern.forge.sdk.copilot.output_contracts import (- OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,OutputContractAdvisoryState,)from skyvern.forge.sdk.copilot.output_extraction_plan import ShapeExpectation, ValueCardinality, ValueShape@@ -85,12 +81,9 @@ from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, Requesfrom skyvern.forge.sdk.copilot.tools import (_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,_analyze_run_blocks,- _click_post_hook,_is_meaningful_extracted_data,- _press_key_post_hook,_record_run_blocks_result,_record_workflow_update_result,- mcp_hooks,)from skyvern.forge.sdk.copilot.tools.page_observation import _record_composition_page_observationfrom skyvern.forge.sdk.copilot.tools.scouting import (@@ -120,7 +113,6 @@ class _Ctx:self.navigate_enforcement_done = Falseself.update_workflow_called = Falseself.persisted_draft_browser_calls = None- self.scouted_spine_checkpoint_fired = Falseself.test_after_update_done = Falseself.post_update_nudge_count = 0self.coverage_nudge_count = 0@@ -707,38 +699,6 @@ class TestSynthesizedOfferPersistenceGate:assert synthesized_block_persistence_signal(ctx, "click") is None- def test_unresolved_recorded_outcome_blocks_page_mutating_tool_until_update_and_run_blocks(self) -> None:- ctx = _Ctx()- ctx.turn_intent = TurnIntent(- mode=TurnIntentMode.BUILD,- authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),- )- ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER- ctx.completion_verification_result = self._unsatisfied_verification()- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(- phase="persisted_block_run",- attempted_tool="update_and_run_blocks",- verdict="repairable_failure",- reason_code="outcome_not_demonstrated",- structural_failure_identity="completion:unsatisfied-output",- authored_structure_signature="authored:partial",- )-- signal = synthesized_block_persistence_signal(ctx, "click")-- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE- assert signal.blocked_tool == "click"- assert signal.cleared_by_tools == frozenset({"update_and_run_blocks"})- assert "last recorded test outcome" in signal.agent_steering_text- update_signal = synthesized_block_persistence_signal(ctx, "update_workflow")- assert isinstance(update_signal, CopilotToolBlockerSignal)- assert update_signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE- assert update_signal.blocked_tool == "update_workflow"- assert update_signal.cleared_by_tools == frozenset({"update_and_run_blocks"})- assert synthesized_block_persistence_signal(ctx, "update_and_run_blocks") is None- assert synthesized_block_persistence_signal(ctx, "evaluate") is None-def _post_run_page_path_ctx(self,*,@@ -783,270 +743,6 @@ class TestSynthesizedOfferPersistenceGate:ctx.scout_trajectory = trajectory or []return ctx- @pytest.mark.asyncio- async def test_post_run_page_path_admission_uses_existing_hooks_to_record_click_and_enter(- self,- monkeypatch: pytest.MonkeyPatch,- ) -> None:- async def no_role_name(*_args: object, **_kwargs: object) -> tuple[str, str]:- return "", ""-- async def no_observation(*_args: object, **_kwargs: object) -> tuple[None, None]:- return None, None-- monkeypatch.setattr(mcp_hooks, "_resolve_scout_role_name", no_role_name)- monkeypatch.setattr(mcp_hooks, "_register_scout_interaction_observation", no_observation)- ctx = make_copilot_context()- qualified = self._post_run_page_path_ctx()- for field in (- "turn_intent",- "block_authoring_policy",- "completion_verification_result",- "latest_recorded_build_test_outcome",- "last_run_blocks_workflow_run_id",- "post_run_page_observation_tool",- "post_run_page_observation_url",- "post_run_page_observation_workflow_run_id",- "post_run_page_observation_after_failed_test",- ):- setattr(ctx, field, getattr(qualified, field))-- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"}) is None- ctx.pending_scout_source_url = "https://example.test/challenge"- await _click_post_hook(- {"ok": True, "data": {"selector": "#continue"}},- {"browser_context": {"url": "https://example.test/mfa", "title": "MFA"}},- ctx,- )- assert [(item["tool_name"], item["trajectory_index"]) for item in ctx.scout_trajectory] == [("click", 0)]-- enter_ctx = make_copilot_context()- qualified = self._post_run_page_path_ctx()- for field in (- "turn_intent",- "block_authoring_policy",- "completion_verification_result",- "latest_recorded_build_test_outcome",- "last_run_blocks_workflow_run_id",- "post_run_page_observation_tool",- "post_run_page_observation_url",- "post_run_page_observation_workflow_run_id",- "post_run_page_observation_after_failed_test",- "post_run_page_observation_generation",- ):- setattr(enter_ctx, field, getattr(qualified, field))- assert (- synthesized_block_persistence_signal(- enter_ctx,- "press_key",- {"key": "Enter", "selector": "#token"},- )- is None- )- enter_ctx.pending_scout_source_url = "https://example.test/challenge"- await _press_key_post_hook(- {"ok": True, "data": {"selector": "#token", "key": "Enter"}},- {"browser_context": {"url": "https://example.test/dashboard", "title": "Dashboard"}},- enter_ctx,- )- assert [(item["tool_name"], item["trajectory_index"]) for item in enter_ctx.scout_trajectory] == [- ("press_key", 0)- ]- assert ctx.turn_ownership is not None- assert TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION in ctx.turn_ownership.claims- assert enter_ctx.turn_ownership is not None- assert TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION in enter_ctx.turn_ownership.claims- assert isinstance(- synthesized_block_persistence_signal(ctx, "click", {"selector": "#unrelated"}),- CopilotToolBlockerSignal,- )-- @pytest.mark.asyncio- async def test_post_run_page_path_admission_precedes_only_matching_current_page_challenge_action(- self,- ) -> None:- class RawResult:- structured_content = {"ok": True, "data": {"selector": "#continue"}}- is_error = False- content: list[object] = []-- class RecordingClient:- def __init__(self) -> None:- self.calls: list[tuple[str, dict[str, object]]] = []-- async def call_tool(- self,- name: str,- arguments: dict[str, object],- raise_on_error: bool = False,- ) -> RawResult:- self.calls.append((name, arguments))- return RawResult()-- def challenge_ctx() -> Any:- ctx = make_copilot_context()- qualified = self._post_run_page_path_ctx()- for field in (- "turn_intent",- "block_authoring_policy",- "completion_verification_result",- "latest_recorded_build_test_outcome",- "last_run_blocks_workflow_run_id",- "post_run_page_observation_tool",- "post_run_page_observation_url",- "post_run_page_observation_workflow_run_id",- "post_run_page_observation_after_failed_test",- "post_run_page_observation_generation",- ):- setattr(ctx, field, getattr(qualified, field))- ctx.composition_page_evidence = {- "observed_after_workflow_run": True,- "workflow_run_id": "wr_129160000000000001",- "challenge_state": {- "detected": True,- "kind": "verification",- "requires_human_verification": True,- "gates_submit_controls": True,- },- "challenge_controls": [{"selector": "#continue", "interactive": True}],- }- return ctx-- admitted_ctx = challenge_ctx()- admitted_client = RecordingClient()- admitted_server = SkyvernOverlayMCPServer(- transport=MagicMock(),- overlays={"click": SchemaOverlay()},- alias_map={},- allowlist=frozenset(),- context_provider=lambda: admitted_ctx,- )- admitted_server._client = admitted_client-- admitted = await admitted_server.call_tool("click", {"selector": "#continue"})-- assert admitted.isError is False- assert admitted_client.calls == [("click", {"selector": "#continue"})]- assert admitted_ctx.turn_halt is None-- blocked_ctx = challenge_ctx()- blocked_client = RecordingClient()- blocked_server = SkyvernOverlayMCPServer(- transport=MagicMock(),- overlays={"click": SchemaOverlay()},- alias_map={},- allowlist=frozenset(),- context_provider=lambda: blocked_ctx,- )- blocked_server._client = blocked_client-- blocked = await blocked_server.call_tool("click", {"selector": "#unrelated"})-- assert blocked.isError is True- assert blocked_client.calls == []- assert blocked_ctx.turn_halt is not None-- terminal_ctx = challenge_ctx()- terminal_signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="The current page is a terminal challenge.",- user_facing_reason="I could not continue past the site challenge.",- recovery_hint="report_blocker_to_user",- cleared_by_tools=frozenset(),- preserves_workflow_draft=True,- renders_final_reply=True,- internal_reason_code="probable_site_block_stop",- blocked_tool="click",- )- assert stash_turn_halt_from_blocker_signal(terminal_ctx, terminal_signal, source="test") is not None- terminal_client = RecordingClient()- terminal_server = SkyvernOverlayMCPServer(- transport=MagicMock(),- overlays={"click": SchemaOverlay()},- alias_map={},- allowlist=frozenset(),- context_provider=lambda: terminal_ctx,- )- terminal_server._client = terminal_client-- terminal = await terminal_server.call_tool("click", {"selector": "#continue"})-- assert terminal.isError is True- assert terminal_client.calls == []- assert terminal_ctx.turn_halt.blocker_signal == terminal_signal- assert terminal_ctx.post_run_page_path_interaction_window is None-- @pytest.mark.asyncio- async def test_post_run_page_path_pre_hook_rejection_does_not_spend_admission_budget(self) -> None:- class RecordingClient:- calls: list[tuple[str, dict[str, object]]] = []-- async def call_tool(- self,- name: str,- arguments: dict[str, object],- raise_on_error: bool = False,- ) -> None:- self.calls.append((name, arguments))-- async def reject_before_dispatch(- _arguments: dict[str, Any],- _ctx: Any,- ) -> dict[str, object]:- return {"ok": False, "error": "pre-dispatch rejection"}-- ctx = make_copilot_context()- qualified = self._post_run_page_path_ctx()- for field in (- "turn_intent",- "block_authoring_policy",- "completion_verification_result",- "latest_recorded_build_test_outcome",- "last_run_blocks_workflow_run_id",- "post_run_page_observation_tool",- "post_run_page_observation_url",- "post_run_page_observation_workflow_run_id",- "post_run_page_observation_after_failed_test",- "post_run_page_observation_generation",- ):- setattr(ctx, field, getattr(qualified, field))- client = RecordingClient()- server = SkyvernOverlayMCPServer(- transport=MagicMock(),- overlays={"click": SchemaOverlay(pre_hook=reject_before_dispatch)},- alias_map={},- allowlist=frozenset(),- context_provider=lambda: ctx,- )- server._client = client-- result = await server.call_tool("click", {"selector": "#continue"})-- assert result.isError is True- assert client.calls == []- assert ctx.post_run_page_path_interaction_window is None-- def test_post_run_page_path_admission_requires_typed_page_path_failure_contract(self) -> None:- ctx = self._post_run_page_path_ctx(- page_path_failure=PostRunPagePathFailure(- kind="non_page_outcome",- workflow_run_id="wr_129160000000000001",- current_url="https://example.test/challenge",- continuation_targets=[],- enter_allowed=False,- )- )- expected_ctx = self._post_run_page_path_ctx()- expected_ctx.post_run_page_observation_after_failed_test = False- expected = synthesized_block_persistence_signal(expected_ctx, "click", {"selector": "#continue"})-- blocked = synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"})-- assert isinstance(expected, CopilotToolBlockerSignal)- assert isinstance(blocked, CopilotToolBlockerSignal)- assert blocked.model_dump() == expected.model_dump()- assert ctx.post_run_page_path_interaction_window is None-def test_post_run_page_path_contract_mints_only_structured_current_page_continuations(self) -> None:run_id = "wr_129160000000000001"base_evidence = {@@ -1387,291 +1083,6 @@ class TestSynthesizedOfferPersistenceGate:assert ctx.latest_recorded_build_test_outcome.page_path_failure == originalassert ctx.post_run_page_observation_generation == 1- def test_post_run_page_path_admission_rejects_non_page_verification_failure(self) -> None:- ctx = self._post_run_page_path_ctx(page_path_failure=None)- ctx.latest_recorded_build_test_outcome = ctx.latest_recorded_build_test_outcome.model_copy(- update={"page_path_failure": None}- )-- blocked = synthesized_block_persistence_signal(ctx, "press_key", {"key": "Enter", "selector": "#continue"})-- assert isinstance(blocked, CopilotToolBlockerSignal)- assert blocked.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE- assert ctx.post_run_page_path_interaction_window is None-- def test_post_run_page_path_admission_requires_current_page_contract_url(self) -> None:- ctx = self._post_run_page_path_ctx()- ctx.post_run_page_observation_url = "https://example.test/other"-- blocked = synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"})-- assert isinstance(blocked, CopilotToolBlockerSignal)- assert blocked.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE- assert ctx.post_run_page_path_interaction_window is None-- def test_post_run_page_path_admission_rejects_click_outside_recorded_continuation(self) -> None:- ctx = self._post_run_page_path_ctx()-- for arguments in (- None,- {},- {"selector": ""},- {"selector": "#unrelated"},- {"selector": "button:contains('Continue')"},- ):- blocked = synthesized_block_persistence_signal(ctx, "click", arguments)- assert isinstance(blocked, CopilotToolBlockerSignal)-- assert ctx.post_run_page_path_interaction_window is None-- def test_post_run_page_path_admission_rejects_blast_radius_sibling_without_contract(self) -> None:- ctx = self._post_run_page_path_ctx()- ctx.latest_recorded_build_test_outcome = ctx.latest_recorded_build_test_outcome.model_copy(- update={"page_path_failure": None}- )-- for tool_name, arguments in (("click", {"selector": "#continue"}), ("press_key", {"key": "Enter"})):- blocked = synthesized_block_persistence_signal(ctx, tool_name, arguments)- assert isinstance(blocked, CopilotToolBlockerSignal)- assert blocked.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE-- def test_post_run_page_path_invalid_click_does_not_spend_admission_budget(self) -> None:- ctx = self._post_run_page_path_ctx()-- blocked = synthesized_block_persistence_signal(ctx, "click", {"selector": "#not-recorded"})-- assert isinstance(blocked, CopilotToolBlockerSignal)- assert ctx.post_run_page_path_interaction_window is None- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"}) is None- assert ctx.post_run_page_path_interaction_window.admitted_attempts == 1-- def test_post_run_page_path_admission_is_same_run_and_argument_exact(self) -> None:- ctx = self._post_run_page_path_ctx()-- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"}) is None- assert (- synthesized_block_persistence_signal(- ctx,- "press_key",- {"key": "Enter", "selector": "#token"},- )- is None- )- assert ctx.post_run_page_path_interaction_window.admitted_attempts == 2- for malformed in (- None,- {},- {"key": "Enter"},- {"key": "enter", "selector": "#token"},- {"key": " Enter ", "selector": "#token"},- {"key": 1, "selector": "#token"},- {"key": "Enter", "selector": "#unrelated"},- ):- assert isinstance(- synthesized_block_persistence_signal(ctx, "press_key", malformed),- CopilotToolBlockerSignal,- )- assert isinstance(- synthesized_block_persistence_signal(ctx, "type_text", {"selector": "#token", "text": "123456"}),- CopilotToolBlockerSignal,- )-- ctx.post_run_page_observation_workflow_run_id = "wr_129160000000000099"- assert isinstance(- synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"}),- CopilotToolBlockerSignal,- )-- ctx = self._post_run_page_path_ctx()- ctx.latest_recorded_build_test_outcome = ctx.latest_recorded_build_test_outcome.model_copy(- update={"workflow_run_id": None}- )- assert isinstance(- synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"}),- CopilotToolBlockerSignal,- )- assert ctx.post_run_page_path_interaction_window is None-- def test_post_run_page_path_window_anchors_after_stale_trajectory(self) -> None:- stale_reached_trajectory = [- {"tool_name": "click", "selector": "#open", "trajectory_index": 3},- {"tool_name": "click", "selector": "#submit", "trajectory_index": 4},- ]- ctx = self._post_run_page_path_ctx(trajectory=stale_reached_trajectory)-- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"}) is None- assert ctx.post_run_page_path_interaction_window.trajectory_anchor == 4- assert ctx.post_run_page_path_interaction_window.admitted_attempts == 1-- def test_post_run_page_path_success_requires_fresh_observation_and_closes_on_completed_page(self) -> None:- ctx = self._post_run_page_path_ctx()-- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"}) is None- ctx.scout_trajectory.append(- {- "tool_name": "click",- "selector": "#continue",- "source_url": "https://example.test/challenge",- "trajectory_index": 0,- }- )- expected_ctx = self._post_run_page_path_ctx()- expected_ctx.post_run_page_observation_after_failed_test = False- expected = synthesized_block_persistence_signal(expected_ctx, "click", {"selector": "#continue"})-- stale = synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"})-- assert isinstance(expected, CopilotToolBlockerSignal)- assert isinstance(stale, CopilotToolBlockerSignal)- assert stale.model_dump() == expected.model_dump()-- ctx.last_test_ok = False- _mark_post_run_page_observed(- ctx,- source_tool="evaluate",- url="https://example.test/dashboard",- page_evidence={- "workflow_run_id": "wr_129160000000000001",- "observed_after_workflow_run": True,- "current_url": "https://example.test/dashboard",- "result_containers": [{"selector": "#results"}],- },- )-- completed = synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"})- assert isinstance(completed, CopilotToolBlockerSignal)- assert completed.model_dump() == expected.model_dump()-- def test_post_run_page_path_fresh_observation_supports_three_steps_without_resetting_budget(self) -> None:- ctx = self._post_run_page_path_ctx()-- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"}) is None- ctx.scout_trajectory.append(- {- "tool_name": "click",- "selector": "#continue",- "source_url": "https://example.test/challenge",- "trajectory_index": 0,- }- )- ctx.last_test_ok = False- _mark_post_run_page_observed(- ctx,- source_tool="evaluate",- url="https://example.test/mfa",- page_evidence={- "workflow_run_id": "wr_129160000000000001",- "observed_after_workflow_run": True,- "current_url": "https://example.test/mfa",- "forms": [- {- "fields": [{"type": "password", "selector": "#token"}],- "submit_controls": [{"type": "submit", "selector": "#verify"}],- }- ],- },- )-- assert isinstance(- synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"}),- CopilotToolBlockerSignal,- )- assert (- synthesized_block_persistence_signal(- ctx,- "press_key",- {"key": "Enter", "selector": "#verify"},- )- is None- )- assert ctx.post_run_page_path_interaction_window.admitted_attempts == 2- ctx.scout_trajectory.append(- {- "tool_name": "press_key",- "selector": "#verify",- "key": "Enter",- "source_url": "https://example.test/mfa",- "trajectory_index": 1,- }- )- _mark_post_run_page_observed(- ctx,- source_tool="inspect_page_for_composition",- url="https://example.test/confirmation",- page_evidence={- "workflow_run_id": "wr_129160000000000001",- "observed_after_workflow_run": True,- "current_url": "https://example.test/confirmation",- "challenge_state": {- "detected": True,- "gates_submit_controls": True,- "gated_submit_controls": [{"selector": "#confirm"}],- },- },- )-- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#confirm"}) is None- assert ctx.post_run_page_path_interaction_window.admitted_attempts == 3-- def test_post_run_page_path_window_charges_failed_attempts_and_resets_for_new_identity(self) -> None:- ctx = self._post_run_page_path_ctx()-- for _ in range(4):- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#missing"}) is None- expected_blocker_ctx = self._post_run_page_path_ctx()- expected_blocker_ctx.post_run_page_observation_after_failed_test = False- expected = synthesized_block_persistence_signal(expected_blocker_ctx, "click", {"selector": "#missing"})- exhausted = synthesized_block_persistence_signal(ctx, "click", {"selector": "#missing"})- assert isinstance(expected, CopilotToolBlockerSignal)- assert isinstance(exhausted, CopilotToolBlockerSignal)- assert exhausted.model_dump() == expected.model_dump()-- ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(- phase="persisted_block_run",- attempted_tool="update_and_run_blocks",- verdict="repairable_failure",- reason_code="outcome_not_demonstrated",- workflow_run_id="wr_129160000000000002",- structural_failure_identity="completion:new-page-path",- page_path_failure=PostRunPagePathFailure(- kind="incomplete_navigation",- workflow_run_id="wr_129160000000000002",- current_url="https://example.test/challenge",- continuation_targets=[- PostRunPagePathTarget(kind="navigation", selector="#missing"),- ],- ),- )- ctx.last_run_blocks_workflow_run_id = "wr_129160000000000002"- ctx.post_run_page_observation_workflow_run_id = "wr_129160000000000002"- ctx.scout_trajectory = [- {"tool_name": "click", "selector": f"#evicted-{index}", "trajectory_index": index}- for index in range(80, 100)- ]- for _ in range(4):- assert synthesized_block_persistence_signal(ctx, "click", {"selector": "#missing"}) is None- assert ctx.post_run_page_path_interaction_window.trajectory_anchor == 99-- def test_post_run_page_path_admission_yields_to_terminal_owner_without_spending_budget(self) -> None:- ctx = self._post_run_page_path_ctx()- terminal_signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="The current page is a terminal challenge.",- user_facing_reason="I could not continue past the site challenge.",- recovery_hint="report_blocker_to_user",- cleared_by_tools=frozenset(),- preserves_workflow_draft=True,- renders_final_reply=True,- internal_reason_code="probable_site_block_stop",- blocked_tool="click",- )- assert stash_turn_halt_from_blocker_signal(ctx, terminal_signal, source="test") is not None-- signal = synthesized_block_persistence_signal(ctx, "click", {"selector": "#continue"})-- assert isinstance(signal, CopilotToolBlockerSignal)- assert ctx.post_run_page_path_interaction_window is None-@pytest.mark.parametrize("ctx_attrs",[@@ -2482,7 +1893,7 @@ def test_unrecoverable_tool_error_ignores_regular_website_404() -> None:def test_unrecoverable_contract_stop_preempts_failed_test_nudge() -> None:from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract- from skyvern.forge.sdk.copilot.enforcement import CopilotUnrecoverableToolError, _check_enforcement+ from skyvern.forge.sdk.copilot.enforcement import CopilotUnrecoverableToolErrorctx = _Ctx()ctx.last_test_ok = False@@ -2856,52 +2267,6 @@ def test_record_run_blocks_result_resets_stale_verified_terminal_proposal_latch(assert ctx.verified_terminal_proposal_ready is False-def test_enforcement_stops_after_verified_terminal_proposal() -> None:- ctx = _Ctx()- ctx.verified_terminal_proposal_ready = True- ctx.last_test_ok = True- ctx.last_full_workflow_test_ok = True- ctx.completion_verification_result = CompletionVerificationResult(- status="evaluated",- criterion_ids=["c0"],- verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],- )- ctx.last_test_suspicious_success = True-- with pytest.raises(CopilotGoalSatisfied):- _check_enforcement(ctx)---def test_verified_outcome_out_orders_same_turn_involuntary_blocker() -> None:- ctx = _Ctx()- ctx.completion_verification_result = CompletionVerificationResult(- status="evaluated",- criterion_ids=["c0"],- verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],- )- involuntary = CopilotToolBlockerSignal(- blocker_kind="loop_detected",- agent_steering_text="repeated failed step",- user_facing_reason="I'm stuck retrying the same step.",- recovery_hint="report_blocker_to_user",- cleared_by_tools=frozenset(),- preserves_workflow_draft=True,- renders_final_reply=True,- internal_reason_code="loop_detected_repeated_failed_step",- blocked_tool="update_and_run_blocks",- extra={},- )- ctx.turn_halt = None- ctx.blocker_signal = involuntary- ctx.latest_tool_blocker_signal = involuntary-- with pytest.raises(CopilotGoalSatisfied):- _check_enforcement(ctx)-- assert ctx.turn_halt is None- assert ctx.blocker_signal is None--def test_record_run_blocks_result_keeps_failure_when_watchdog_cancel_without_timeout() -> None:"""Stagnation/ceiling cancels mid-session must still set last_test_ok=Falseso the failed-test nudge can fire — only a coincident total timeout softens@@ -3248,14 +2613,10 @@ class TestEnforcement:return resultdef test_no_enforcement_when_nothing_pending(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx()assert _check_enforcement(ctx) is Nonedef test_post_navigate_nudge(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(navigate_called=True, observation_after_navigate=False)nudge = _check_enforcement(ctx)assert nudge is not None@@ -3263,8 +2624,6 @@ class TestEnforcement:assert ctx.navigate_enforcement_done is Truedef test_post_navigate_only_fires_once(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(navigate_called=True,observation_after_navigate=False,@@ -3273,16 +2632,12 @@ class TestEnforcement:assert _check_enforcement(ctx) is Nonedef test_post_update_nudge(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(update_workflow_called=True, test_after_update_done=False)nudge = _check_enforcement(ctx)assert nudge is not Noneassert "test" in nudge.lower() or "run_blocks" in nudge.lower()def test_navigate_takes_priority_over_update(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(navigate_called=True,observation_after_navigate=False,@@ -3293,8 +2648,6 @@ class TestEnforcement:assert "observe" in nudge.lower() or "inspect" in nudge.lower()def test_intermediate_success_nudge_for_multistep_goal(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(update_workflow_called=True,test_after_update_done=True,@@ -3311,8 +2664,6 @@ class TestEnforcement:assert ctx.coverage_nudge_count == 1def test_no_intermediate_success_nudge_for_single_step_goal(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(update_workflow_called=True,test_after_update_done=True,@@ -3325,7 +2676,6 @@ class TestEnforcement:def test_intermediate_success_nudge_fires_for_two_blocks(self) -> None:"""Key regression: nudge must fire even when block_count > 1."""- from skyvern.forge.sdk.copilot.enforcement import _check_enforcementctx = self._make_ctx(update_workflow_called=True,@@ -3341,7 +2691,7 @@ class TestEnforcement:assert nudge == POST_INTERMEDIATE_SUCCESS_NUDGEdef test_intermediate_nudge_respects_global_cap(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import MAX_INTERMEDIATE_NUDGES, _check_enforcement+ from skyvern.forge.sdk.copilot.enforcement import MAX_INTERMEDIATE_NUDGESctx = self._make_ctx(update_workflow_called=True,@@ -3354,8 +2704,6 @@ class TestEnforcement:assert _check_enforcement(ctx, self._reply_result("capped")) is Nonedef test_intermediate_nudge_does_not_fire_for_ten_plus_blocks(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(update_workflow_called=True,test_after_update_done=True,@@ -3370,8 +2718,6 @@ class TestEnforcement:"""Regression guard: ASK_QUESTION must never be blocked by coverage."""import json- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(update_workflow_called=True,test_after_update_done=True,@@ -3386,8 +2732,6 @@ class TestEnforcement:assert _check_enforcement(ctx, ask) is Nonedef test_plain_labeled_ask_question_passes_even_with_coverage_gap(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(update_workflow_called=True,test_after_update_done=True,@@ -3402,7 +2746,7 @@ class TestEnforcement:assert _check_enforcement(ctx, ask) is Nonedef test_explore_without_workflow_nudge(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE, _check_enforcement+ from skyvern.forge.sdk.copilot.enforcement import POST_EXPLORE_WITHOUT_WORKFLOW_NUDGEctx = self._make_ctx(navigate_called=True,@@ -3418,7 +2762,6 @@ class TestEnforcement:from skyvern.forge.sdk.copilot.enforcement import (POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE,POST_UPDATE_NUDGE,- _check_enforcement,)ctx = self._make_ctx(@@ -3433,8 +2776,6 @@ class TestEnforcement:assert ctx.explore_without_workflow_nudge_count == 0def test_update_without_test_allowed_for_explicit_untested_draft(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import _check_enforcement-ctx = self._make_ctx(allow_untested_workflow_draft=True,update_workflow_called=True,@@ -3444,7 +2785,7 @@ class TestEnforcement:assert _check_enforcement(ctx) is Nonedef test_explore_without_workflow_not_when_test_done(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE, _check_enforcement+ from skyvern.forge.sdk.copilot.enforcement import POST_EXPLORE_WITHOUT_WORKFLOW_NUDGEctx = self._make_ctx(navigate_called=True,@@ -3459,7 +2800,6 @@ class TestEnforcement:from skyvern.forge.sdk.copilot.enforcement import (MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES,POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE,- _check_enforcement,)ctx = self._make_ctx(@@ -3473,7 +2813,7 @@ class TestEnforcement:assert nudge != POST_EXPLORE_WITHOUT_WORKFLOW_NUDGEdef test_explore_without_workflow_not_without_observation(self) -> None:- from skyvern.forge.sdk.copilot.enforcement import POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE, _check_enforcement+ from skyvern.forge.sdk.copilot.enforcement import POST_EXPLORE_WITHOUT_WORKFLOW_NUDGEctx = self._make_ctx(navigate_called=True,@@ -4458,17 +3798,6 @@ class TestAdvisoryRunDispatchForceLane:)assert _should_force_advisory_run_dispatch(ctx) is False- def test_held_genuinely_terminal_blocker_releases_the_force(self) -> None:- ctx = self._granted_ctx()- ctx.blocker_signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="",- user_facing_reason="",- recovery_hint="stop",- internal_reason_code=OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,- )- assert _should_force_advisory_run_dispatch(ctx) is False-def test_non_code_only_policy_does_not_force(self) -> None:ctx = self._granted_ctx()ctx.block_authoring_policy = None@@ -4935,20 +4264,6 @@ class TestCredentialFlowGoalComplete:)assert synthesized_trajectory_is_goal_complete(ctx) is False- def test_download_target_does_not_bypass_credential_flow(self) -> None:- ctx = self._ctx_with_inventory(- self._two_screen_first_page(),- inventory={"cred_1": frozenset({"username", "password"})},- )- ctx.reached_download_target = ReachedDownloadTarget(- selector="a.report",- affordance_text="Report",- download_kind="extension",- source_step="trajectory_recency",- already_registered=False,- )- assert synthesized_trajectory_is_goal_complete(ctx) is False-def test_mutating_tools_admitted_while_credential_flow_incomplete(self) -> None:ctx = self._ctx_with_inventory(self._two_screen_first_page(),diff --git a/tests/unit/test_copilot_evaluate_page_evidence.py b/tests/unit/test_copilot_evaluate_page_evidence.pyindex d1669e9ec..54326832e 100644--- a/tests/unit/test_copilot_evaluate_page_evidence.py+++ b/tests/unit/test_copilot_evaluate_page_evidence.py@@ -8,7 +8,7 @@ from skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.build_phase import BuildPhasefrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.context import CopilotContext-from skyvern.forge.sdk.copilot.request_policy import RequestPolicy+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicyfrom skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidencefrom skyvern.forge.sdk.copilot.tools import (_evaluate_post_hook,@@ -316,3 +316,95 @@ async def test_current_page_inspection_finalizes_runtime_repair_context_for_nextassert "runtime_failure_class: timeout_waiting_for_selector" in promptassert "page_results: #results No matching records" in promptassert "case=secret" not in ctx.last_code_authoring_repair_context.model_dump_json()+++async def test_live_seam_evaluate_records_scouted_read_from_prehook_stash() -> None:+ # Enters through the same two hooks the MCP adapter drives, with a response shaped like the+ # real one: the expression exists only in the invocation, never in the response.+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook++ ctx = _ctx()+ pre = await _evaluate_pre_hook({"expression": "document.querySelector('#count').textContent"}, ctx)+ assert pre is None++ result = {"ok": True, "data": {"result": "778 logs found", "url": "https://dash.example.test/logs"}}+ await _evaluate_post_hook(result, raw={"name": "evaluate"}, ctx=ctx)++ reads = [i for i in ctx.scout_trajectory if i.get("tool_name") == "read_value"]+ assert len(reads) == 1+ assert reads[0]["read_expression"] == "document.querySelector('#count').textContent"+ assert reads[0]["read_result_shape"] == "str"+ assert ctx.pending_scout_read_expression is None+++async def test_scouted_read_binds_to_a_canonical_slot_when_that_is_the_requested_output() -> None:+ # A rekeyed requested output carries a digest instead of a word. Binding the read anonymously+ # keys the producer differently from the criterion, so completion verification reports no+ # evidence for an outcome the scout already demonstrated.+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook++ slot_path = "output.request_slot_5a2fc98725209bfe8366101490eab27e9c75426782ec20214_00"+ ctx = _ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[CompletionCriterion(id="c0", outcome="the azure error count", output_path=slot_path)]+ )++ await _evaluate_pre_hook({"expression": "document.querySelector('#count').textContent"}, ctx)+ await _evaluate_post_hook(+ {"ok": True, "data": {"result": "778 logs found", "url": "https://dash.example.test/logs"}},+ raw={"name": "evaluate"},+ ctx=ctx,+ )++ reads = [i for i in ctx.scout_trajectory if i.get("tool_name") == "read_value"]+ assert [read["read_output_path"] for read in reads] == [slot_path]+++async def test_a_later_diagnostic_read_does_not_evict_the_requested_output_read() -> None:+ # Reads sharing an output path collapse to the last one, so a page dump taken after the value+ # would silently replace the read the criterion is graded against.+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook++ slot_path = "output.request_slot_5a2fc98725209bfe8366101490eab27e9c75426782ec20214_00"+ ctx = _ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[CompletionCriterion(id="c0", outcome="the azure error count", output_path=slot_path)]+ )++ for expression, result in (+ ("document.querySelector('#count').textContent", "778 logs found"),+ ("document.body.innerText", "a whole page of unrelated text"),+ ):+ await _evaluate_pre_hook({"expression": expression}, ctx)+ await _evaluate_post_hook(+ {"ok": True, "data": {"result": result, "url": "https://dash.example.test/logs"}},+ raw={"name": "evaluate"},+ ctx=ctx,+ )++ reads = [i for i in ctx.scout_trajectory if i.get("tool_name") == "read_value"]+ assert [read["read_output_path"] for read in reads] == [slot_path, slot_path]+ # Both are retained with their own expressions; synthesis, not capture, chooses between them.+ assert [read["read_expression"] for read in reads] == [+ "document.querySelector('#count').textContent",+ "document.body.innerText",+ ]+++async def test_failed_evaluate_does_not_leak_expression_into_next_read() -> None:+ from skyvern.forge.sdk.copilot.tools.mcp_hooks import _evaluate_pre_hook++ ctx = _ctx()+ await _evaluate_pre_hook({"expression": "document.title"}, ctx)+ await _evaluate_post_hook({"ok": False, "error": "boom"}, raw={}, ctx=ctx)++ # Next evaluate carries no expression (adapter reject path) — the stale stash must not attach.+ pre = await _evaluate_pre_hook({}, ctx)+ assert pre is None+ await _evaluate_post_hook({"ok": True, "data": {"result": "still here"}}, raw={}, ctx=ctx)++ assert [i for i in ctx.scout_trajectory if i.get("tool_name") == "read_value"] == []diff --git a/tests/unit/test_copilot_fill_credential_field.py b/tests/unit/test_copilot_fill_credential_field.pyindex aec11953f..e9e988239 100644--- a/tests/unit/test_copilot_fill_credential_field.py+++ b/tests/unit/test_copilot_fill_credential_field.py@@ -476,9 +476,6 @@ class TestConsecutiveLoopGuardExemption:assert error is Noneassert ctx.consecutive_tool_tracker == []- def test_exemption_set_membership(self) -> None:- assert "fill_credential_field" in tools_module._CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLS-class TestToolRegistration:def test_tool_is_registered_native(self) -> None:diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex 8626873ca..55d94c41f 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -12,12 +12,10 @@ from unittest.mock import AsyncMock, MagicMock, callimport pytestfrom structlog.testing import capture_logs-from skyvern.forge.sdk.copilot import hooks as hooks_modulefrom skyvern.forge.sdk.copilot import tools as tools_modulefrom skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignalfrom skyvern.forge.sdk.copilot.code_block_synthesis import dynamic_row_evidence_fingerprint, synthesize_code_blockfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy-from skyvern.forge.sdk.copilot.enforcement import CopilotGoalSatisfiedfrom skyvern.forge.sdk.copilot.hooks import CopilotRunHooksfrom skyvern.forge.sdk.copilot.tools import (_capture_scout_ambiguity,@@ -68,10 +66,10 @@ def _mcp_text_output(payload: dict[str, Any]) -> list[dict[str, str]]:def _terminal_loop_signal() -> CopilotToolBlockerSignal:return CopilotToolBlockerSignal(blocker_kind="loop_detected",- agent_steering_text="LOOP DETECTED: 'update_workflow' has already failed 3 times.",+ agent_steering_text="LOOP DETECTED: 'update_workflow' has been called 3 times.",user_facing_reason="I retried without making progress. Tell me what to change and I'll try again.",recovery_hint="report_blocker_to_user",- internal_reason_code="loop_detected_repeated_failed_step",+ internal_reason_code="loop_detected_consecutive_same_tool",blocked_tool="update_workflow",)@@ -250,25 +248,6 @@ async def test_on_tool_end_swallows_unserializable_output() -> None:assert warning["workflow_copilot_chat_id"] == "chat_example"-async def test_on_tool_end_goal_satisfied_log_includes_copilot_turn_identifiers(- monkeypatch: pytest.MonkeyPatch,-) -> None:- ctx = _FakeContext()- hooks = CopilotRunHooks(ctx)- monkeypatch.setattr(hooks_module, "_tool_completion_satisfies_turn", lambda *_args: True)-- output = _mcp_text_output({"ok": True, "data": {"workflow_run_id": "wrid_example"}})- with capture_logs() as logs:- with pytest.raises(CopilotGoalSatisfied):- await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("update_and_run_blocks"), output)-- satisfied = next(log for log in logs if log["event"] == "copilot tool satisfied goal; stopping agent loop")- assert satisfied["workflow_permanent_id"] == "wpid_example"- assert satisfied["turn_id"] == "turn_example"- assert satisfied["workflow_copilot_chat_id"] == "chat_example"--class TestCopilotToCallToolResult:@staticmethoddef _build(d: dict) -> Any:@@ -718,63 +697,6 @@ class TestMCPFailedStepLoopDetection:organization_id="org-1",)- @pytest.mark.asyncio- async def test_interleaved_same_step_failures_short_circuit_third_dispatch(self) -> None:- from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer-- class FakeRawResult:- def __init__(self, payload: dict[str, Any], is_error: bool = False) -> None:- self.structured_content = payload- self.is_error = is_error- self.content: list[Any] = []-- class FakeClient:- def __init__(self) -> None:- self.calls: list[tuple[str, dict[str, Any]]] = []-- async def call_tool(- self,- name: str,- args: dict[str, Any],- raise_on_error: bool = False,- ) -> FakeRawResult:- self.calls.append((name, args))- if name == "get_browser_screenshot":- return FakeRawResult({"ok": False, "error": "screenshot failed"}, is_error=True)- return FakeRawResult({"ok": True, "data": {"status": "failed"}})-- ctx = MagicMock()- ctx.consecutive_tool_tracker = []- ctx.failed_tool_step_tracker = {}- ctx.turn_ownership = None- ctx.blocker_signal_claimant = None- ctx.gate_precedence_conflict_events = []- client = FakeClient()- server = SkyvernOverlayMCPServer(- transport=MagicMock(),- overlays={},- alias_map={},- allowlist=frozenset(),- context_provider=lambda: ctx,- )- server._client = client-- await server.call_tool("get_browser_screenshot", {})- await server.call_tool("get_run_results", {})- await server.call_tool("get_browser_screenshot", {})- await server.call_tool("get_run_results", {})- blocked = await server.call_tool("get_browser_screenshot", {})-- parsed = json.loads(blocked.content[0].text)- assert blocked.isError is True- assert "LOOP DETECTED" in parsed["error"]- assert client.calls == [- ("get_browser_screenshot", {}),- ("get_run_results", {}),- ("get_browser_screenshot", {}),- ("get_run_results", {}),- ]-class TestMCPToolOverlayCompleteness:"""Verify alias map and overlay configs are in sync and complete."""@@ -1988,21 +1910,6 @@ class TestClickPostHookReachedDownloadTarget:],}- @pytest.mark.asyncio- async def test_click_on_download_affordance_populates_reached_download_target(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- from skyvern.forge.sdk.copilot.tools.mcp_hooks import _click_post_hook-- self._patch_scouting(monkeypatch, page_evidence=self._SINGLE_DOWNLOAD_EVIDENCE)-- ctx = self._ctx()- result = {"ok": True, "data": {"selector": 'a[href="/x/statement.pdf"]'}}- await _click_post_hook(result, {}, ctx)-- assert ctx.reached_download_target is not None- assert ctx.reached_download_target.download_kind == "extension"-@pytest.mark.asyncioasync def test_click_post_hook_is_noop_in_standard_mode(self, monkeypatch: pytest.MonkeyPatch) -> None:from skyvern.forge.sdk.copilot.tools.mcp_hooks import _click_post_hook@@ -2014,21 +1921,3 @@ class TestClickPostHookReachedDownloadTarget:await _click_post_hook(result, {}, ctx)assert ctx.reached_download_target is None-- @pytest.mark.asyncio- async def test_click_on_two_affordances_leaves_target_unset(self, monkeypatch: pytest.MonkeyPatch) -> None:- from skyvern.forge.sdk.copilot.tools.mcp_hooks import _click_post_hook-- two = {- "navigation_targets": [- {"selector": 'a[href="/x/a.pdf"]', "text": "A", "download_kind": "extension"},- {"selector": 'a[href="/x/b.pdf"]', "text": "B", "download_kind": "extension"},- ]- }- self._patch_scouting(monkeypatch, page_evidence=two)-- ctx = self._ctx()- result = {"ok": True, "data": {"selector": 'a[href="/x/a.pdf"]'}}- await _click_post_hook(result, {}, ctx)-- assert ctx.reached_download_target is Nonediff --git a/tests/unit/test_copilot_loop_detection.py b/tests/unit/test_copilot_loop_detection.pyindex 68475cd3d..fca241df2 100644--- a/tests/unit/test_copilot_loop_detection.py+++ b/tests/unit/test_copilot_loop_detection.py@@ -6,7 +6,6 @@ from unittest.mock import MagicMockfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.loop_detection import (- clear_failed_step_tracker_for_tools,detect_failed_tool_step_loop,detect_tool_loop,record_consecutive_tool_result_boundary,@@ -258,45 +257,6 @@ def test_diagnosis_repair_tool_error_records_consecutive_boundary_only() -> Noneclass TestFailedToolStepLoopDetection:- def test_interleaved_successful_tool_does_not_reset_failed_step(self) -> None:- tracker: dict[str, int] = {}-- assert detect_failed_tool_step_loop(tracker, "get_browser_screenshot", {}) is None- record_tool_step_result(tracker, "get_browser_screenshot", {}, {"ok": False, "error": "screenshot failed"})-- assert detect_failed_tool_step_loop(tracker, "get_run_results", {}) is None- record_tool_step_result(tracker, "get_run_results", {}, {"ok": True, "data": {"status": "failed"}})-- assert detect_failed_tool_step_loop(tracker, "get_browser_screenshot", {}) is None- record_tool_step_result(tracker, "get_browser_screenshot", {}, {"ok": False, "error": "screenshot failed"})-- assert detect_failed_tool_step_loop(tracker, "get_run_results", {}) is None- record_tool_step_result(tracker, "get_run_results", {}, {"ok": True, "data": {"status": "failed"}})-- msg = detect_failed_tool_step_loop(tracker, "get_browser_screenshot", {})-- assert msg is not None- assert "LOOP DETECTED" in msg- assert "get_browser_screenshot" in msg-- def test_successful_same_step_resets_failure_streak(self) -> None:- tracker: dict[str, int] = {}-- record_tool_step_result(tracker, "evaluate", {"script": "document.title"}, {"ok": False, "error": "boom"})- record_tool_step_result(tracker, "evaluate", {"script": "document.title"}, {"ok": True, "data": "ok"})- record_tool_step_result(tracker, "evaluate", {"script": "document.title"}, {"ok": False, "error": "boom"})-- assert detect_failed_tool_step_loop(tracker, "evaluate", {"script": "document.title"}) is None-- def test_different_arguments_do_not_share_failure_streak(self) -> None:- tracker: dict[str, int] = {}-- record_tool_step_result(tracker, "click", {"selector": "#first"}, {"ok": False, "error": "missing"})- record_tool_step_result(tracker, "click", {"selector": "#first"}, {"ok": False, "error": "missing"})-- assert detect_failed_tool_step_loop(tracker, "click", {"selector": "#second"}) is None- assert detect_failed_tool_step_loop(tracker, "click", {"selector": "#first"}) is not None-def test_block_running_credential_errors_share_failure_streak_across_arguments(self) -> None:tracker: dict[str, int] = {}@@ -322,140 +282,3 @@ class TestFailedToolStepLoopDetection:assert msg is not Noneassert "LOOP DETECTED" in msgassert "CREDENTIAL_ERROR" in msg-- def test_generic_block_running_errors_still_key_by_arguments(self) -> None:- tracker: dict[str, int] = {}-- record_tool_step_result(- tracker,- "run_blocks_and_collect_debug",- {"block_labels": ["draft_a"], "parameters": {}},- {"ok": False, "error": "temporary page state mismatch"},- )- record_tool_step_result(- tracker,- "run_blocks_and_collect_debug",- {"block_labels": ["draft_b"], "parameters": {}},- {"ok": False, "error": "temporary page state mismatch"},- )-- assert (- detect_failed_tool_step_loop(- tracker,- "run_blocks_and_collect_debug",- {"block_labels": ["draft_c"], "parameters": {}},- )- is None- )-- def test_block_threshold_is_two_failures(self) -> None:- tracker: dict[str, int] = {}-- record_tool_step_result(tracker, "click", {"selector": "#x"}, {"ok": False, "error": "boom"})- record_tool_step_result(tracker, "click", {"selector": "#x"}, {"ok": False, "error": "boom"})-- assert detect_failed_tool_step_loop(tracker, "click", {"selector": "#x"}) is not None-- fresh: dict[str, int] = {}- record_tool_step_result(fresh, "click", {"selector": "#y"}, {"ok": False, "error": "boom"})- assert detect_failed_tool_step_loop(fresh, "click", {"selector": "#y"}) is None-- def test_set_arguments_produce_stable_identity(self) -> None:- tracker: dict[str, int] = {}- args_a = {"keys": {"alpha", "beta", "gamma"}}- args_b = {"keys": {"gamma", "alpha", "beta"}}-- record_tool_step_result(tracker, "press_keys", args_a, {"ok": False, "error": "boom"})- record_tool_step_result(tracker, "press_keys", args_b, {"ok": False, "error": "boom"})-- assert detect_failed_tool_step_loop(tracker, "press_keys", args_a) is not None-- def test_clear_failed_step_tracker_for_tools_removes_only_named_tools(self) -> None:- tracker: dict[str, int] = {}-- record_tool_step_result(tracker, "run_blocks_and_collect_debug", {"x": 1}, {"ok": False, "error": "boom"})- record_tool_step_result(tracker, "run_blocks_and_collect_debug", {"x": 1}, {"ok": False, "error": "boom"})- record_tool_step_result(tracker, "update_and_run_blocks", {"y": 2}, {"ok": False, "error": "boom"})- record_tool_step_result(tracker, "click", {"selector": "#z"}, {"ok": False, "error": "boom"})-- clear_failed_step_tracker_for_tools(tracker, ["run_blocks_and_collect_debug", "update_and_run_blocks"])-- assert detect_failed_tool_step_loop(tracker, "run_blocks_and_collect_debug", {"x": 1}) is None- assert detect_failed_tool_step_loop(tracker, "update_and_run_blocks", {"y": 2}) is None- record_tool_step_result(tracker, "click", {"selector": "#z"}, {"ok": False, "error": "boom"})- assert detect_failed_tool_step_loop(tracker, "click", {"selector": "#z"}) is not None-- def test_workflow_update_clears_block_running_failure_entries(self) -> None:- from types import SimpleNamespace- from unittest.mock import MagicMock-- from skyvern.forge.sdk.copilot.context import CopilotContext- from skyvern.forge.sdk.copilot.tools import _record_workflow_update_result-- ctx = CopilotContext(- organization_id="o",- workflow_id="w",- workflow_permanent_id="wp",- workflow_yaml="updated yaml",- browser_session_id=None,- stream=MagicMock(),- )- record_tool_step_result(- ctx.failed_tool_step_tracker,- "run_blocks_and_collect_debug",- {"block_labels": ["A"], "parameters": {}},- {"ok": False, "error": "boom"},- )- record_tool_step_result(- ctx.failed_tool_step_tracker,- "run_blocks_and_collect_debug",- {"block_labels": ["A"], "parameters": {}},- {"ok": False, "error": "boom"},- )- record_tool_step_result(- ctx.failed_tool_step_tracker,- "click",- {"selector": "#x"},- {"ok": False, "error": "boom"},- )-- _record_workflow_update_result(- ctx,- {- "ok": True,- "data": {"block_count": 2},- "_workflow": SimpleNamespace(workflow_id="wf_new"),- },- )-- # A follow-up run after the user's fix must not be blocked.- assert (- detect_failed_tool_step_loop(- ctx.failed_tool_step_tracker,- "run_blocks_and_collect_debug",- {"block_labels": ["A"], "parameters": {}},- )- is None- )- assert (- detect_failed_tool_step_loop(- ctx.failed_tool_step_tracker,- "click",- {"selector": "#x"},- )- is None- )- record_tool_step_result(- ctx.failed_tool_step_tracker,- "click",- {"selector": "#x"},- {"ok": False, "error": "boom"},- )- assert (- detect_failed_tool_step_loop(- ctx.failed_tool_step_tracker,- "click",- {"selector": "#x"},- )- is not None- )diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.pyindex 2af1b68d9..458e985ba 100644--- a/tests/unit/test_copilot_loop_detection_blocker.py+++ b/tests/unit/test_copilot_loop_detection_blocker.py@@ -1,12 +1,10 @@from __future__ import annotations-import asyncioimport jsonfrom types import SimpleNamespaceimport pytest-from skyvern.forge.sdk.copilot.agent import _ensure_unvalidated_proposal_affordancefrom skyvern.forge.sdk.copilot.blocker_signal import (_INTERNAL_TOOL_NAME_TOKENS,_LOOP_PROGRESS_TOOLS,@@ -15,14 +13,12 @@ from skyvern.forge.sdk.copilot.blocker_signal import (assert_clean_user_facing_text,loop_blocker_evidence_from_ctx,)-from skyvern.forge.sdk.copilot.build_test_outcome import RecordedOutcomeGroundingRequirementfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODEfrom skyvern.forge.sdk.copilot.loop_detection import tool_step_identityfrom skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer, _stash_and_emit_loop_blockerfrom skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy-from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidencefrom skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_errorfrom skyvern.forge.sdk.copilot.tools.mcp_hooks import get_skyvern_mcp_alias_map@@ -77,16 +73,16 @@ def _current_page_challenge_evidence(*, observed_after_workflow_run: bool = True("loop_message", "expected_reason"),[(- "LOOP DETECTED: 'update_and_run_blocks' has already failed 3 times with CREDENTIAL_ERROR; blocking attempt #4.",+ "LOOP DETECTED: 'update_and_run_blocks' has been called 3 times with CREDENTIAL_ERROR; blocking attempt #4.","loop_detected_credential_or_parameter_misconfig",),(- "LOOP DETECTED: 'update_and_run_blocks' has already failed 3 times with PARAMETER_BINDING_ERROR; blocking attempt #4.",+ "LOOP DETECTED: 'update_and_run_blocks' has been called 3 times with PARAMETER_BINDING_ERROR; blocking attempt #4.","loop_detected_credential_or_parameter_misconfig",),(- "LOOP DETECTED: 'update_workflow' has already failed 3 consecutive times with these arguments; blocking attempt #4.",- "loop_detected_repeated_failed_step",+ "LOOP DETECTED: 'update_workflow' has been called 3 consecutive times with these arguments; blocking attempt #4.",+ "loop_detected_consecutive_same_tool",),("LOOP DETECTED: 'list_credentials' has been called 3 times consecutively.",@@ -114,127 +110,14 @@ def test_loop_blocker_falls_back_to_generic_on_novel_message() -> None:assert signal.recovery_hint == "report_blocker_to_user"-def test_native_dispatch_failed_step_loop_sets_signal_and_returns_payload() -> None:- # Tracker simulates 2 prior failures of (update_workflow, {workflow_yaml: 'y'}).- # detect_failed_tool_step_loop_for_ctx uses tool_step_identity to look up the- # threshold; mimicking the real identity is brittle, so call the dispatcher- # with a tracker that already has the same identity prepopulated at threshold-1.- from skyvern.forge.sdk.copilot.loop_detection import tool_step_identity-- identity = tool_step_identity("update_workflow", {"workflow_yaml": "yaml-1"})- ctx = _ctx(failed_tool_step_tracker={identity: 3})- payload = _tool_loop_error(ctx, "update_workflow", {"workflow_yaml": "yaml-1"})- assert payload is not None- assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)- assert ctx.blocker_signal.blocker_kind == "loop_detected"- assert ctx.blocker_signal.internal_reason_code == "loop_detected_repeated_failed_step"- # LLM payload is agent_steering_text only — raw LOOP DETECTED: marker is in it.- assert payload.startswith("LOOP DETECTED:")- # Renderer-side string is clean.- for token in _LEAK_TOKENS:- assert token not in ctx.blocker_signal.user_facing_reason- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind == TurnHaltKind.LOOP_DETECTED- assert ctx.turn_halt.blocker_signal is ctx.blocker_signal---def test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:- ctx = _ctx(consecutive_tool_tracker=_streak("list_credentials"))- payload = _tool_loop_error(ctx, "list_credentials", None)- assert payload is not None- assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)- assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"- assert ctx.blocker_signal.cleared_by_tools == frozenset()- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind == TurnHaltKind.LOOP_DETECTED---def test_consecutive_evaluate_loop_with_loaded_results_uses_goal_aware_copy() -> None:- ctx = _ctx(consecutive_tool_tracker=_streak("evaluate"))- ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(- result_container_count=1,- table_result_container_count=1,- )- payload = _tool_loop_error(ctx, "evaluate", None)-- assert payload is not None- assert payload.startswith("LOOP DETECTED: 'evaluate' has been called 3 times consecutively.")- assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)- assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"- assert "loaded results" in ctx.blocker_signal.user_facing_reason- assert "extracting the requested information" in ctx.blocker_signal.user_facing_reason- assert "retrying the same step" not in ctx.blocker_signal.user_facing_reason- assert dict(ctx.blocker_signal.extra) == {"loop_evidence_tiers": ["loaded_results"]}- assert_clean_user_facing_text(ctx.blocker_signal.user_facing_reason, blocked_tool="evaluate")---def test_consecutive_non_evaluate_loop_ignores_loaded_result_steer() -> None:- ctx = _ctx(consecutive_tool_tracker=_streak("list_credentials"))- ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(- result_container_count=1,- table_result_container_count=1,- )- payload = _tool_loop_error(ctx, "list_credentials", None)-- assert payload is not None- assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)- assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"- assert "loaded results" not in ctx.blocker_signal.user_facing_reason- assert ctx.blocker_signal.user_facing_reason == (- "I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."- )- assert dict(ctx.blocker_signal.extra) == {}---def test_unsatisfied_grounding_requirement_does_not_suppress_loop_guards() -> None:- requirement = RecordedOutcomeGroundingRequirement(- phase="persisted_block_run",- reason_code="runtime_block_failure",- structural_key="key",- workflow_run_id="wr_1",- block_labels=["block_a"],- )-- consecutive_ctx = _ctx(consecutive_tool_tracker=_streak("click"))- consecutive_ctx.recorded_outcome_grounding_requirement = requirement- consecutive_payload = _tool_loop_error(consecutive_ctx, "click", None)-- assert consecutive_payload is not None- assert isinstance(consecutive_ctx.blocker_signal, CopilotToolBlockerSignal)- assert consecutive_ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"-- failed_step_ctx = _ctx(failed_tool_step_tracker={tool_step_identity("update_workflow", {}): 3})- failed_step_ctx.recorded_outcome_grounding_requirement = requirement- failed_step_payload = _tool_loop_error(failed_step_ctx, "update_workflow", {})-- assert failed_step_payload is not None- assert isinstance(failed_step_ctx.blocker_signal, CopilotToolBlockerSignal)- assert failed_step_ctx.blocker_signal.internal_reason_code == "loop_detected_repeated_failed_step"---def test_consecutive_evaluate_loop_without_composition_steer_uses_generic_copy() -> None:- ctx = _ctx(consecutive_tool_tracker=_streak("evaluate"))- payload = _tool_loop_error(ctx, "evaluate", None)-- assert payload is not None- assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)- assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"- assert ctx.blocker_signal.user_facing_reason == (- "I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."- )- assert dict(ctx.blocker_signal.extra) == {}- assert_clean_user_facing_text(ctx.blocker_signal.user_facing_reason, blocked_tool="evaluate")--def test_native_and_mcp_paths_produce_equivalent_loop_signal() -> None:"""Native dispatch and the MCP adapter must produce the same signal shapefor the same loop-detection message so a regression in one path can'tdiverge from the other."""- from skyvern.forge.sdk.copilot.mcp_adapter import _stash_and_emit_loop_blockerloop_message = (- "LOOP DETECTED: 'update_workflow' has already failed 3 consecutive times with these arguments; "+ "LOOP DETECTED: 'update_workflow' has been called 3 consecutive times with these arguments; ""blocking attempt #4.")native_ctx = _ctx()@@ -275,11 +158,7 @@ _FULL_EVIDENCE = LoopBlockerEvidence()_BRANCH_MESSAGES = {"loop_detected_credential_or_parameter_misconfig": (- "LOOP DETECTED: 'update_and_run_blocks' has already failed 3 times with CREDENTIAL_ERROR; blocking attempt #4."- ),- "loop_detected_repeated_failed_step": (- "LOOP DETECTED: 'update_workflow' has already failed 3 consecutive times with these arguments; "- "blocking attempt #4."+ "LOOP DETECTED: 'update_and_run_blocks' has been called 3 times with CREDENTIAL_ERROR; blocking attempt #4."),"loop_detected_consecutive_same_tool": "LOOP DETECTED: 'evaluate' has been called 3 times consecutively.","loop_detected_generic": "something unfamiliar happened",@@ -288,16 +167,12 @@ _BRANCH_TEMPLATES = {"loop_detected_credential_or_parameter_misconfig": ("I couldn't run this with the current credential or parameter setup. Update them and ask me to try again."),- "loop_detected_repeated_failed_step": (- "I retried without making progress. Tell me what to change and I'll try a different approach."- ),"loop_detected_consecutive_same_tool": ("I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."),"loop_detected_generic": "I couldn't keep going on this turn. Tell me what to change and I'll try again.",}_FULL_TIER_BRANCHES = (- "loop_detected_repeated_failed_step","loop_detected_consecutive_same_tool","loop_detected_generic",)@@ -480,44 +355,6 @@ def test_fixed_tier_copy_is_clean_for_every_loop_prone_tool(blocked_tool: str, rassert_clean_user_facing_text(signal.user_facing_reason, blocked_tool=blocked_tool)-def test_native_and_mcp_paths_carry_equivalent_evidence_bearing_signals() -> None:- def _prepped_ctx() -> CopilotContext:- ctx = _ctx(consecutive_tool_tracker=_streak("list_credentials"))- ctx.last_outcome_gate_reason = (- "The run completed but did not demonstrate the goal outcome(s): the requested record is checked "- "on a public registry site with a search form and expandable result rows."- )- ctx.last_outcome_gate_workflow_run_id = "wr_latest"- ctx.last_run_blocks_workflow_run_id = "wr_latest"- ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"- ctx.has_staged_proposal = True- return ctx-- native_ctx = _prepped_ctx()- native_payload = _tool_loop_error(native_ctx, "list_credentials", None)- assert native_payload is not None- native_signal = native_ctx.blocker_signal- assert isinstance(native_signal, CopilotToolBlockerSignal)-- mcp_ctx = _prepped_ctx()- mcp_payload = _stash_and_emit_loop_blocker(mcp_ctx, native_signal.agent_steering_text, "list_credentials")- mcp_signal = mcp_ctx.blocker_signal- assert isinstance(mcp_signal, CopilotToolBlockerSignal)-- assert mcp_payload == native_payload- assert mcp_signal.user_facing_reason == native_signal.user_facing_reason- assert "did not demonstrate the goal outcome" in mcp_signal.user_facing_reason- assert "verification challenge" in mcp_signal.user_facing_reason- assert native_signal.preserves_workflow_draft is True- assert mcp_signal.preserves_workflow_draft is True- assert dict(native_signal.extra) == {"loop_evidence_tiers": ["verdict", "anti_bot", "draft"]}- assert dict(mcp_signal.extra) == dict(native_signal.extra)- assert mcp_signal.internal_reason_code == native_signal.internal_reason_code- assert mcp_signal.agent_steering_text == native_signal.agent_steering_text- assert mcp_signal.recovery_hint == native_signal.recovery_hint- assert mcp_signal.blocked_tool == native_signal.blocked_tool--def test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() -> None:ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))ctx.composition_page_evidence = _current_page_challenge_evidence()@@ -534,21 +371,6 @@ def test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() ->assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE-def test_native_tool_loop_error_does_not_preempt_on_pre_attempt_challenge() -> None:- ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))- ctx.composition_page_evidence = _current_page_challenge_evidence(observed_after_workflow_run=False)-- payload = _tool_loop_error(ctx, "evaluate", {"expression": "document.body.innerText"})-- assert payload is not None- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "loop_detected_consecutive_same_tool"- assert signal.blocked_tool == "evaluate"- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind == TurnHaltKind.LOOP_DETECTED--@pytest.mark.asyncioasync def test_mcp_browser_tool_terminal_challenge_preempts_same_tool_loop() -> None:ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))@@ -713,26 +535,6 @@ def test_composed_loop_reply_passes_output_policy_allow_verdict() -> None:assert verdict.allowed, [code.value for code in verdict.reason_codes]-def test_composed_loop_reply_with_draft_affordance_passes_output_policy() -> None:- signal = _build_loop_blocker_signal(- _BRANCH_MESSAGES["loop_detected_consecutive_same_tool"], tool_name="evaluate", evidence=_FULL_EVIDENCE- )- reply = _ensure_unvalidated_proposal_affordance(signal.user_facing_reason)- verdict = evaluate_output_policy(- request_policy=None,- response_type="REPLY",- user_response=reply,- global_llm_context=None,- workflow_yaml="title: Example workflow\nblocks: []",- has_workflow_proposal=True,- workflow_was_persisted=False,- workflow_attempted=False,- unvalidated=True,- output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,- )- assert verdict.allowed, [code.value for code in verdict.reason_codes]--def _dispatch_server(ctx: CopilotContext, client: object) -> SkyvernOverlayMCPServer:server = SkyvernOverlayMCPServer(transport=None,@@ -751,25 +553,6 @@ class _OkResult:content: list = []-async def test_mcp_admission_blocks_third_identical_identity_without_dispatch() -> None:- args = {"expression": "document.title"}- ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", args))-- class _UnexpectedClient:- async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:- raise AssertionError("consecutive loop blocker should skip MCP execution")-- result = await _dispatch_server(ctx, _UnexpectedClient()).call_tool("evaluate", args)-- parsed = json.loads(result.content[0].text)- assert result.isError is True- assert parsed["ok"] is False- signal = ctx.blocker_signal- assert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == "loop_detected_consecutive_same_tool"--@pytest.mark.asyncioasync def test_mcp_admission_admits_distinct_argument_identity() -> None:ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "a"}))@@ -811,29 +594,3 @@ async def test_mcp_exception_completion_leaves_same_consecutive_state_as_successassert error_ctx.consecutive_tool_tracker == success_ctx.consecutive_tool_trackerassert success_ctx.consecutive_tool_tracker == [tool_step_identity("evaluate", args)]---async def test_mcp_cancelled_admission_matches_error_completion_state() -> None:- args = {"expression": "document.title"}-- error_ctx = _ctx(consecutive_tool_tracker=[], failed_tool_step_tracker={})-- class _RaisingClient:- async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:- raise RuntimeError("boom")-- await _dispatch_server(error_ctx, _RaisingClient()).call_tool("evaluate", args)-- cancel_ctx = _ctx(consecutive_tool_tracker=[], failed_tool_step_tracker={})-- class _CancellingClient:- async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:- raise asyncio.CancelledError-- with pytest.raises(asyncio.CancelledError):- await _dispatch_server(cancel_ctx, _CancellingClient()).call_tool("evaluate", args)-- assert cancel_ctx.consecutive_tool_tracker == error_ctx.consecutive_tool_tracker- assert cancel_ctx.consecutive_tool_tracker == [tool_step_identity("evaluate", args)]- assert cancel_ctx.failed_tool_step_tracker == {}diff --git a/tests/unit/test_copilot_make_agent_result_backfill.py b/tests/unit/test_copilot_make_agent_result_backfill.pyindex a68667165..7f6c1a00d 100644--- a/tests/unit/test_copilot_make_agent_result_backfill.py+++ b/tests/unit/test_copilot_make_agent_result_backfill.py@@ -4,22 +4,15 @@ narrative payload: ``responseKind`` from ``TurnOutcome.response_kind`` andfrom __future__ import annotations-import base64-import json-from io import BytesIOfrom types import SimpleNamespace-from typing import Anyimport pytest-from PIL import Image, PngImagePlugin-from skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.agent import _finalize_result_with_blocker_override, _make_agent_resultfrom skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignalfrom skyvern.forge.sdk.copilot.context import (AgentResult,CopilotContext,- DeliveredUnverifiedPublicOutputs,StructuredContext,)from skyvern.forge.sdk.copilot.request_policy import RequestPolicy@@ -62,20 +55,9 @@ def _payload(**overrides: object) -> dict:def _result(ctx: CopilotContext | None, **kwargs: object):- if (- ctx is not None- and ctx.delivered_unverified_terminal- and ctx.delivered_unverified_observed_outputs- and not isinstance(ctx.delivered_unverified_observed_outputs, DeliveredUnverifiedPublicOutputs)- ):- ctx.delivered_unverified_observed_outputs = DeliveredUnverifiedPublicOutputs(- ctx.delivered_unverified_observed_outputs- )kwargs.setdefault("user_response", "ok")kwargs.setdefault("updated_workflow", None)kwargs.setdefault("global_llm_context", None)- if ctx is not None and ctx.delivered_unverified_terminal:- kwargs.setdefault("_delivered_unverified_snapshot", agent_module._delivered_unverified_observed_outputs(ctx))return _make_agent_result(ctx, **kwargs)@@ -186,703 +168,6 @@ def test_backfill_omits_credential_prompt_when_no_signal_present() -> None:assert "credentialPrompt" not in result.narrative_payload-def test_backfill_adds_sanitized_delivered_unverified_outputs_with_json_types() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_workflow_run_id = "wr_source_123"- ctx.secret_scrub_values.append("registered-secret-value")- deeply_nested: dict[str, Any] = {"password": "deep-must-not-persist"}- for index in range(24):- deeply_nested = {f"level_{index}": deeply_nested}- ctx.delivered_unverified_observed_outputs = {- "result": {- "account": "acct-123",- "amount": 0,- "confirmed": False,- "items": ["first", None, 2.5],- "password": "must-not-persist",- "registered": "prefix registered-secret-value suffix",- "raw": "api_key=sk-1234567890abcdefghijkl",- "internal": "run wr_internal_123 with update_workflow",- "deep": deeply_nested,- "api_key=sk-raw-secret-key-1234567890": "safe-value",- 7: "non-string-key-value",- }- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["$skyvernOutput"] == {"omitted": {"depth": 1, "unsupported": 1}}- assert "wr_source_123" not in str(observed)- assert observed["result"]["account"] == "acct-123"- assert observed["result"]["amount"] == 0- assert observed["result"]["confirmed"] is False- assert observed["result"]["items"] == ["first", None, 2.5]- assert observed["result"]["password"] == "****"- assert observed["result"]["registered"] == "prefix [REDACTED_SECRET] suffix"- assert observed["result"]["raw"] == "[REDACTED_SECRET]"- assert "deep-must-not-persist" not in str(observed)- assert "sk-raw-secret-key-1234567890" not in str(observed)- assert "non-string-key-value" not in str(observed)- assert "wr_internal_123" not in str(observed)- assert "update_workflow" not in str(observed)---def test_backfill_rejects_non_finite_numbers_and_remains_strict_json() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {- "result": {- "finite": 1.25,- "not_a_number": float("nan"),- "positive_infinity": float("inf"),- "negative_infinity": float("-inf"),- "items": [0.0, float("nan"), 2.5],- }- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- omission = {"$skyvernOmitted": {"reason": "non-finite number", "count": 1}}- assert observed["result"] == {- "finite": 1.25,- "not_a_number": omission,- "positive_infinity": omission,- "negative_infinity": omission,- "items": [0.0, omission, 2.5],- }- assert json.loads(json.dumps(observed, allow_nan=False)) == observed---def test_delivered_unverified_backfill_preserves_valid_image_when_registered_secret_is_unassociated() -> None:- png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAAE0lEQVR4nGP8z4APMOGVZRip0gBBLAETee26JgAAAABJRU5ErkJggg=="- embedded = png_b64[20:30]- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.append(embedded)- ctx.delivered_unverified_observed_outputs = {- "result": {- "screenshot_base64": png_b64,- "note": f"captured value {embedded}",- }- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- canonical = base64.b64decode(observed["result"]["screenshot_base64"], validate=True)- with Image.open(BytesIO(canonical)) as image:- image.load()- assert image.format == "PNG"- assert image.info == {}- assert embedded not in observed["result"]["note"]---def test_make_agent_result_reencodes_png_without_secret_bearing_text_metadata() -> None:- secret = "registered-secret-value"- metadata = PngImagePlugin.PngInfo()- metadata.add_text("comment", secret)- source = BytesIO()- Image.new("RGB", (1, 1), (12, 34, 56)).save(source, format="PNG", pnginfo=metadata)- source_b64 = base64.b64encode(source.getvalue()).decode()- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.append(secret)- ctx.delivered_unverified_observed_outputs = {"result": {"screenshot_base64": source_b64}}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- published_b64 = observed["result"]["screenshot_base64"]- published = base64.b64decode(published_b64, validate=True)- assert published_b64 != source_b64- assert secret.encode() not in published- with Image.open(BytesIO(published)) as image:- image.load()- assert image.format == "PNG"- assert image.info == {}---def test_make_agent_result_reencodes_jpeg_without_secret_bearing_exif_metadata() -> None:- secret = "registered-secret-value"- exif = Image.Exif()- exif[0x010E] = secret- source = BytesIO()- Image.new("RGB", (2, 2), (12, 34, 56)).save(source, format="JPEG", exif=exif)- source_b64 = base64.b64encode(source.getvalue()).decode()- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.append(secret)- ctx.delivered_unverified_observed_outputs = {"result": {"screenshot_base64": source_b64}}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- published_b64 = observed["result"]["screenshot_base64"]- published = base64.b64decode(published_b64, validate=True)- assert published_b64 != source_b64- assert secret.encode() not in published- with Image.open(BytesIO(published)) as image:- image.load()- assert image.format == "JPEG"- assert len(image.getexif()) == 0---def test_delivered_unverified_accepts_internal_named_fields_only_from_typed_public_output_source() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {- "result": {- "amount": 0,- "browserSessionId": "browser_session_internal",- "browser_session_id": "browser_session_internal_snake",- "runSessionId": "run_session_internal",- "run_session_id": "run_session_internal_snake",- }- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["result"] == {- "amount": 0,- "browserSessionId": "browser_session_internal",- "browser_session_id": "browser_session_internal_snake",- "runSessionId": "run_session_internal",- "run_session_id": "run_session_internal_snake",- }---def test_delivered_unverified_rejects_untyped_context_output_provenance() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"result": {"amount": 0, "sessionToken": "internal"}}-- result = _make_agent_result(- ctx,- user_response="ok",- updated_workflow=None,- narrative_payload=_payload(),- )-- assert result.narrative_payload is not None- assert "deliveredUnverifiedObservedOutputs" not in result.narrative_payload--- ("malformed_image_prefix", "padding"),- [- pytest.param("iVBORw0KGgoAAAANSUhEUg", "==", id="png-header"),- pytest.param("/9j/4AAQSkZJRgABAQAAAQABAAD", "=", id="jpeg-header"),- ],-)-def test_delivered_unverified_backfill_omits_invalid_image_prefixed_base64(- malformed_image_prefix: str,- padding: str,-) -> None:- secret = "registeredsecretvalu"- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.append(secret)- ctx.delivered_unverified_observed_outputs = {- "result": {"screenshot_base64": malformed_image_prefix + secret + padding}- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["result"]["screenshot_base64"] == {"$skyvernOmitted": {"reason": "invalid image", "count": 1}}- assert observed["$skyvernOutput"]["omitted"]["image"] == 1- assert secret not in str(result.narrative_payload)---def test_make_agent_result_omits_malformed_jpeg_that_retains_eoi() -> None:- malformed_jpeg = base64.b64encode(- b"\xff\xd8\xff\xe0\x00\x10JFIF\x00" + b"registered-secret-value" + b"\xff\xd9"- ).decode()- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.append("registered-secret-value")- ctx.delivered_unverified_observed_outputs = {"result": {"screenshot_base64": malformed_jpeg}}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["result"]["screenshot_base64"] == {"$skyvernOmitted": {"reason": "invalid image", "count": 1}}- assert observed["$skyvernOutput"]["omitted"]["image"] == 1- assert "registered-secret-value" not in str(observed)---def test_backfill_omits_delivered_unverified_outputs_for_non_delivered_terminal() -> None:- ctx = _ctx()- ctx.delivered_unverified_observed_outputs = {"result": {"amount": 0}}-- result = _result(- ctx,- narrative_payload=_payload(deliveredUnverifiedObservedOutputs={"result": {"password": "caller-secret-bypass"}}),- )-- assert result.narrative_payload is not None- assert "deliveredUnverifiedObservedOutputs" not in result.narrative_payload- assert "caller-secret-bypass" not in str(result.narrative_payload)---def test_backfill_omits_caller_delivered_unverified_outputs_without_context() -> None:- result = _result(- None,- narrative_payload=_payload(deliveredUnverifiedObservedOutputs={"result": {"password": "caller-secret-bypass"}}),- )-- assert result.narrative_payload is not None- assert "deliveredUnverifiedObservedOutputs" not in result.narrative_payload- assert "caller-secret-bypass" not in str(result.narrative_payload)---def test_backfill_replaces_caller_delivered_unverified_outputs_with_sanitized_snapshot() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.append("canonical-secret")- ctx.delivered_unverified_observed_outputs = {- "result": {"amount": 0, "registered": "prefix canonical-secret suffix"}- }- caller_outputs = {"result": {"amount": 999, "password": "caller-secret-bypass"}}-- result = _result(- ctx,- narrative_payload=_payload(deliveredUnverifiedObservedOutputs=caller_outputs),- )-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["result"] == {"amount": 0, "registered": "prefix [REDACTED_SECRET] suffix"}- assert "caller-secret-bypass" not in str(result.narrative_payload)---def test_backfill_rejects_underscored_snapshot_keyword_override() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"result": {"amount": 0}}-- result = _result(- ctx,- narrative_payload=_payload(),- _delivered_unverified_snapshot={"result": {"password": "raw-snapshot-bypass"}},- )-- assert result.narrative_payload is not None- assert "deliveredUnverifiedObservedOutputs" not in result.narrative_payload- assert "raw-snapshot-bypass" not in str(result.narrative_payload)---def test_backfill_rejects_public_outputs_keyword_override() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"result": {"amount": 0}}-- result = _result(- ctx,- narrative_payload=_payload(),- delivered_unverified_observed_outputs={"result": {"password": "raw-public-bypass"}},- )-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["result"] == {"amount": 0}- assert "raw-public-bypass" not in str(result.narrative_payload)---def test_backfill_removes_caller_delivered_unverified_outputs_when_snapshot_is_empty() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True-- result = _result(- ctx,- narrative_payload=_payload(deliveredUnverifiedObservedOutputs={"result": {"password": "caller-secret-bypass"}}),- )-- assert result.narrative_payload is not None- assert "deliveredUnverifiedObservedOutputs" not in result.narrative_payload- assert "caller-secret-bypass" not in str(result.narrative_payload)---def test_backfill_redacts_sensitive_value_when_registered_secret_rewrites_source_key() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.append("password")- ctx.delivered_unverified_observed_outputs = {"password": "synthetic-unregistered-sensitive-value"}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["[REDACTED_SECRET]"] == "****"- assert "synthetic-unregistered-sensitive-value" not in str(result.narrative_payload)---def test_backfill_redacts_value_for_assignment_style_sensitive_source_key() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"password=placeholder": "synthetic-unregistered-sensitive-value"}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["[REDACTED_SECRET]"] == "****"- assert "synthetic-unregistered-sensitive-value" not in str(result.narrative_payload)---def test_backfill_redacts_value_for_colon_assignment_sensitive_source_key() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"password:placeholder": "synthetic-unregistered-sensitive-value"}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["[REDACTED_SECRET]"] == "****"- assert "synthetic-unregistered-sensitive-value" not in str(result.narrative_payload)---def test_backfill_redacts_camel_case_sensitive_source_keys() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {- "sessionToken": "session-secret",- "accessToken": "access-secret",- "privateKey": "private-secret",- "clientSecret": "client-secret",- "IDToken": "id-secret",- "JWTToken": "jwt-secret",- "CSRFToken": "csrf-secret",- "APISecret": "api-secret",- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- sensitive_keys = (- "sessionToken",- "accessToken",- "privateKey",- "clientSecret",- "IDToken",- "JWTToken",- "CSRFToken",- "APISecret",- )- assert {key: observed[key] for key in sensitive_keys} == {- "sessionToken": "****",- "accessToken": "****",- "privateKey": "****",- "clientSecret": "****",- "IDToken": "****",- "JWTToken": "****",- "CSRFToken": "****",- "APISecret": "****",- }- for secret in (- "session-secret",- "access-secret",- "private-secret",- "client-secret",- "id-secret",- "jwt-secret",- "csrf-secret",- "api-secret",- ):- assert secret not in str(observed)--- "key",- [- "auth_token",- "client_secret",- "refresh_token",- "x-api-key",- "client-secret",- "client secret",- "db_password",- "user_password",- "bearer_token",- ],-)-def test_backfill_redacts_delimited_sensitive_source_keys(key: str) -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {key: "opaque-secret-value"}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed[key] == "****"- assert "opaque-secret-value" not in str(observed)--- "key",- [- "pageToken",- "page_token",- "page-token",- "nextToken",- "next_token",- "next-token",- "credentialId",- "credential_id",- "credential-id",- "author",- ],-)-def test_delivered_unverified_sensitive_key_preserves_non_secret_delimiter_variants(key: str) -> None:- assert agent_module._delivered_unverified_sensitive_key(key) is False---def test_backfill_disambiguates_captured_omission_marker_shape() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- captured_marker = {"reason": "string budget", "count": 1}- ctx.delivered_unverified_observed_outputs = {"result": {"$skyvernOmitted": captured_marker}}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["result"] == {"$skyvernOmitted [captured]": captured_marker}---def test_backfill_omits_internal_source_workflow_run_id_from_structured_output() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_workflow_run_id = "wr_internal_source"- ctx.delivered_unverified_observed_outputs = {"result": {"amount": 0}}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert "wr_internal_source" not in str(observed)- assert "sourceWorkflowRunId" not in str(observed)---def test_long_ordinary_string_is_sanitized_without_structured_truncation() -> None:- ctx = _ctx()- budget = agent_module._DeliveredUnverifiedAdmissionBudget()- captured = "x" * 600-- observed = agent_module._bounded_delivered_unverified_scalar(ctx, captured, budget)-- assert observed == captured- assert budget.string_omitted == 0---def test_raw_key_budget_rejects_before_sensitive_key_classification(monkeypatch: pytest.MonkeyPatch) -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"password=" + "x" * 513: "must-not-persist"}-- def fail_if_classified(_key: str) -> bool:- raise AssertionError("oversized raw key reached sensitive-key classification")-- monkeypatch.setattr(agent_module, "_delivered_unverified_sensitive_key", fail_if_classified)-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed == {"$skyvernOutput": {"omitted": {"string": 1}}}- assert "must-not-persist" not in str(observed)---def test_backfill_bounds_nodes_depth_strings_and_serialized_bytes_with_omission_accounting() -> None:- deeply_nested: dict[str, Any] = {"secret": "must-not-survive"}- for index in range(100):- deeply_nested = {f"level_{index}": deeply_nested}-- cases = {- "depth": {"deep": deeply_nested},- "node": {"many": {f"field_{index}": "value" for index in range(10_000)}},- "serializedBytes": {"oversized_string": "x" * 100_000},- }- for reason, captured in cases.items():- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_workflow_run_id = f"wr_oversized_{reason}"- ctx.delivered_unverified_observed_outputs = captured-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- metadata = observed["$skyvernOutput"]- assert f"wr_oversized_{reason}" not in str(observed)- assert metadata["omitted"][reason] >= 1- assert len(json.dumps(observed, separators=(",", ":")).encode()) <= 65_536- if reason == "depth":- assert "must-not-survive" not in str(observed)---def test_delivered_unverified_node_budget_prioritizes_later_bounded_scalars_over_long_text() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {- "result": {- **{f"long_{index}": "x" * 512 for index in range(64)},- "amount": 0,- "confirmation": "later-confirmation",- }- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["result"]["amount"] == 0- assert observed["result"]["confirmation"] == "later-confirmation"- assert observed["$skyvernOutput"]["omitted"]["node"] >= 1---def test_delivered_unverified_structured_surface_preserves_ordinary_600_character_output() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- captured = "captured-code-output-" + "x" * 579- ctx.delivered_unverified_observed_outputs = {"result": {"code_output": captured}}-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["result"]["code_output"] == captured---def test_delivered_unverified_serialized_byte_budget_retains_scalars_and_counts_omitted_paths() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {- "amount": 0,- "confirmation": "confirmed",- "large": {f"field_{index}": "🧪" * 512 for index in range(64)},- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["amount"] == 0- assert observed["confirmation"] == "confirmed"- assert observed["$skyvernOutput"]["omitted"]["serializedBytes"] >= 1- assert len(json.dumps(observed, ensure_ascii=False, separators=(",", ":")).encode()) <= 65_536---def test_delivered_unverified_list_byte_budget_retains_prior_scalars_and_counts_omissions() -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {- "amount": 0,- "large": ["🧪" * 512 for _ in range(64)],- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["amount"] == 0- assert observed["$skyvernOutput"]["omitted"]["serializedBytes"] >= 1- assert len(json.dumps(observed, separators=(",", ":")).encode()) <= 65_536--- "oversized_image",- [- pytest.param("data:image/png;base64," + "A" * 65_540, id="data-url"),- pytest.param("iVBORw0KGgoAAAAN" + "A" * 65_540, id="bare-base64"),- ],-)-def test_delivered_unverified_oversized_image_rejected_before_decode(- monkeypatch: pytest.MonkeyPatch,- oversized_image: str,-) -> None:- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_observed_outputs = {"result": {"image": oversized_image}}-- real_decode = agent_module.base64.b64decode-- def fail_decode(value: str, **kwargs: object) -> bytes:- if len(value) > 16:- raise AssertionError("oversized image reached base64 decode")- return real_decode(value, **kwargs)-- monkeypatch.setattr(agent_module.base64, "b64decode", fail_decode)-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- observed = result.narrative_payload["deliveredUnverifiedObservedOutputs"]- assert observed["result"]["image"] == {"$skyvernOmitted": {"reason": "invalid image", "count": 1}}---def test_delivered_unverified_image_pixel_limit_checked_before_conversion(- monkeypatch: pytest.MonkeyPatch,-) -> None:- class OversizedImage:- size = (1_001, 1_000)- format = "PNG"-- def __enter__(self) -> OversizedImage:- return self-- def __exit__(self, *_args: object) -> None:- return None-- def convert(self, _mode: str) -> None:- raise AssertionError("oversized image reached pixel conversion")-- monkeypatch.setattr(agent_module.Image, "open", lambda _stream: OversizedImage())- encoded = base64.b64encode(agent_module._PNG_START + agent_module._PNG_END).decode()-- image_input, canonical = agent_module._canonical_delivered_unverified_image(encoded)-- assert image_input is True- assert canonical is None---def test_delivered_unverified_image_data_url_is_canonicalized_without_metadata() -> None:- secret = "data-url-secret"- metadata = PngImagePlugin.PngInfo()- metadata.add_text("comment", secret)- source = BytesIO()- Image.new("RGB", (1, 1), (12, 34, 56)).save(source, format="PNG", pnginfo=metadata)- ctx = _ctx()- ctx.delivered_unverified_terminal = True- ctx.secret_scrub_values.append(secret)- ctx.delivered_unverified_observed_outputs = {- "result": {"image": "data:image/png;base64," + base64.b64encode(source.getvalue()).decode()}- }-- result = _result(ctx, narrative_payload=_payload())-- assert result.narrative_payload is not None- published = base64.b64decode(- result.narrative_payload["deliveredUnverifiedObservedOutputs"]["result"]["image"], validate=True- )- assert secret.encode() not in published--def test_make_agent_result_records_resolved_credentials_as_durable_approval() -> None:ctx = _ctx(request_policy=RequestPolicy(resolved_credentials=[SimpleNamespace(credential_id="cred_portal")]))diff --git a/tests/unit/test_copilot_mid_correction_guard.py b/tests/unit/test_copilot_mid_correction_guard.pyindex dbea75641..856e9f78e 100644--- a/tests/unit/test_copilot_mid_correction_guard.py+++ b/tests/unit/test_copilot_mid_correction_guard.py@@ -34,14 +34,6 @@ def _ctx_after_failure_with_verified_prefix(*, elapsed_seconds: float) -> Copiloreturn ctx-def test_tool_loop_error_blocks_run_when_budget_low_after_failure() -> None:- elapsed = TOTAL_TIMEOUT_SECONDS - (COPILOT_FINAL_REPLY_RESERVE_SECONDS - 10)- ctx = _ctx_after_failure_with_verified_prefix(elapsed_seconds=elapsed)- for tool in ("update_and_run_blocks", "run_blocks_and_collect_debug"):- msg = _tool_loop_error(ctx, tool)- assert msg is not None and "Wall-clock budget too low" in msg, tool--def test_tool_loop_error_no_guard_when_budget_sufficient() -> None:ctx = _ctx_after_failure_with_verified_prefix(elapsed_seconds=TOTAL_TIMEOUT_SECONDS - (COPILOT_FINAL_REPLY_RESERVE_SECONDS + 30)@@ -49,24 +41,6 @@ def test_tool_loop_error_no_guard_when_budget_sufficient() -> None:assert _tool_loop_error(ctx, "update_and_run_blocks") is None-def test_tool_loop_error_blocks_first_call_when_turn_budget_is_too_low() -> None:- ctx = _fresh_context()- ctx.copilot_run_start_monotonic = time.monotonic() - (TOTAL_TIMEOUT_SECONDS - 10)- msg = _tool_loop_error(ctx, "update_and_run_blocks")- assert msg is not None- assert "Do NOT start another block-running tool call" in msg---def test_tool_loop_error_blocks_failed_retry_when_no_good_workflow_exists() -> None:- ctx = _fresh_context()- ctx.copilot_run_start_monotonic = time.monotonic() - (TOTAL_TIMEOUT_SECONDS - 10)- ctx.last_test_ok = False- ctx.last_failed_workflow_yaml = "yaml-failed"- msg = _tool_loop_error(ctx, "update_and_run_blocks")- assert msg is not None- assert "Do NOT retry" in msg--def test_tool_loop_error_no_guard_for_non_block_running_tools() -> None:elapsed = TOTAL_TIMEOUT_SECONDS - (COPILOT_FINAL_REPLY_RESERVE_SECONDS - 10)ctx = _ctx_after_failure_with_verified_prefix(elapsed_seconds=elapsed)@@ -74,18 +48,6 @@ def test_tool_loop_error_no_guard_for_non_block_running_tools() -> None:assert _tool_loop_error(ctx, tool) is None, tool-def test_tool_loop_error_guard_persists_through_update_workflow() -> None:- # ``last_test_ok`` flips to None on every ``update_workflow``, but- # ``last_failed_workflow_yaml`` stays — so the guard must still fire.- elapsed = TOTAL_TIMEOUT_SECONDS - (COPILOT_FINAL_REPLY_RESERVE_SECONDS - 10)- ctx = _ctx_after_failure_with_verified_prefix(elapsed_seconds=elapsed)- ctx.last_test_ok = None- ctx.last_workflow = SimpleNamespace(workflow_id="wf-edited")- ctx.last_workflow_yaml = "yaml-edited"- msg = _tool_loop_error(ctx, "update_and_run_blocks")- assert msg is not None and "Wall-clock budget too low" in msg--def test_record_run_blocks_result_promotes_last_good_on_real_success() -> None:ctx = _fresh_context()ctx.last_workflow = SimpleNamespace(workflow_id="wf-1")@@ -191,10 +153,8 @@ def test_translate_to_agent_result_salvages_last_good_on_failed_reply() -> None:assert agent_result.proposal_disposition == "review_untested"# Failure rewrite would have replaced the agent's text with one based on# ``last_update_block_count=5``; salvage must skip the rewrite.- assert agent_result.user_response.startswith(agent_text)- assert "Use Review to inspect it" in agent_result.user_response- assert "Accept to save it" in agent_result.user_response- assert "Reject to discard it" in agent_result.user_response+ # The model's own account of the turn is returned verbatim; no affordance is appended to it.+ assert agent_result.user_response == agent_textdef test_translate_to_agent_result_salvages_after_failure_then_update_workflow() -> None:@@ -218,10 +178,8 @@ def test_translate_to_agent_result_salvages_after_failure_then_update_workflow())assert agent_result.updated_workflow is ctx.last_good_workflowassert agent_result.proposal_disposition == "review_untested"- assert agent_result.user_response.startswith(agent_text)- assert "Use Review to inspect it" in agent_result.user_response- assert "Accept to save it" in agent_result.user_response- assert "Reject to discard it" in agent_result.user_response+ # The model's own account of the turn is returned verbatim; no affordance is appended to it.+ assert agent_result.user_response == agent_textdef test_translate_to_agent_result_does_not_salvage_on_standalone_edit() -> None:diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.pyindex d3f23ad61..8a110b93f 100644--- a/tests/unit/test_copilot_output_contract_actuation.py+++ b/tests/unit/test_copilot_output_contract_actuation.py@@ -7,7 +7,6 @@ OSS-synced: only synthetic labels and RFC-2606 placeholder identifiers.from __future__ import annotations-import itertoolsimport jsonfrom types import SimpleNamespace@@ -16,14 +15,11 @@ from structlog.testing import capture_logsfrom skyvern.forge.sdk.copilot import enforcementfrom skyvern.forge.sdk.copilot.blocker_signal import (CopilotToolBlockerSignal,- assert_clean_user_facing_text,- build_output_source_unobservable_blocker_signal,)from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContextfrom skyvern.forge.sdk.copilot.output_contracts import (OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,- OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,OutputContractActuation,OutputContractActuationEvidence,OutputContractActuationKind,@@ -40,7 +36,6 @@ from skyvern.forge.sdk.copilot.runtime import (output_contract_ladder_unresolved,)from skyvern.forge.sdk.copilot.tools import workflow_update as wu-from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind, turn_halt_from_blocker_signalfrom tests.unit.copilot_test_helpers import make_copilot_ctx_ALL_SPLIT_BLOCKERS = [@@ -54,90 +49,6 @@ _ALL_SPLIT_BLOCKERS = []-def test_resolver_is_total_over_the_evidence_family_matrix() -> None:- kinds = set()- for (- family,- imposed,- click,- observed,- prior,- unconsumed,- advisory,- exhausted,- declick,- grantable,- run_observed,- run_bound,- carried_page,- ) in itertools.product(- list(OutputContractBailFamily),- [False, True],- [False, True],- [False, True],- [False, True],- [False, True],- list(OutputContractAdvisoryState),- [False, True],- [False, True],- [False, True],- [False, True],- [False, True],- [False, True],- ):- evidence = OutputContractActuationEvidence(- imposed_available=imposed,- click_only_spine=click,- observed_required_values=observed,- prior_actuation=prior,- prior_directive_unconsumed=unconsumed,- advisory_state=advisory,- actuation_progress_exhausted=exhausted,- declick_attempt_failed=declick,- advisory_run_grantable=grantable,- consumed_run_output_observed=run_observed,- consumed_run_bound_required_path=run_bound,- consumed_run_carried_page_extraction=carried_page,- )- actuation = resolve_output_contract_actuation(family=family, evidence=evidence)- assert actuation is not None- kinds.add(actuation.kind)- if actuation.kind == OutputContractActuationKind.IMPOSED:- assert evidence.imposed_available- if actuation.kind == OutputContractActuationKind.ADVISORY_RUN:- grantable_run = (- evidence.advisory_run_grantable and evidence.advisory_state != OutputContractAdvisoryState.CONSUMED- )- assert grantable_run or (- (evidence.observed_required_values or not evidence.click_only_spine)- and evidence.advisory_state in {OutputContractAdvisoryState.UNUSED, OutputContractAdvisoryState.GRANTED}- )- if actuation.kind == OutputContractActuationKind.BLOCKED_TERMINAL:- assert not (- evidence.advisory_run_grantable and evidence.advisory_state != OutputContractAdvisoryState.CONSUMED- )- assert actuation.reason_code in {- OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,- OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,- }- if actuation.reason_code == OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE:- assert (- evidence.click_only_spine- and not evidence.observed_required_values- and evidence.declick_attempt_failed- )- else:- assert (- evidence.advisory_state == OutputContractAdvisoryState.CONSUMED- and evidence.consumed_run_output_observed- and evidence.consumed_run_carried_page_extraction- and not evidence.consumed_run_bound_required_path- )- assert OutputContractActuationKind.BLOCKED_TERMINAL in kinds- assert OutputContractActuationKind.IMPOSED in kinds- assert OutputContractActuationKind.ADVISORY_RUN in kinds--def test_flaky_no_observed_without_declick_attempt_routes_to_actuation_not_terminal() -> None:flaky_first_pass = OutputContractActuationEvidence(imposed_available=False,@@ -153,22 +64,6 @@ def test_flaky_no_observed_without_declick_attempt_routes_to_actuation_not_termiassert actuation.kind != OutputContractActuationKind.BLOCKED_TERMINAL-def test_no_source_terminal_requires_failed_declick_attempt() -> None:- declick_failed = OutputContractActuationEvidence(- imposed_available=False,- click_only_spine=True,- observed_required_values=False,- prior_actuation=False,- prior_directive_unconsumed=False,- advisory_state=OutputContractAdvisoryState.UNUSED,- actuation_progress_exhausted=False,- declick_attempt_failed=True,- )- actuation = resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=declick_failed)- assert actuation.kind == OutputContractActuationKind.BLOCKED_TERMINAL- assert actuation.reason_code == OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE--def test_grantable_advisory_preempts_no_source_terminal() -> None:producible_but_flaked = OutputContractActuationEvidence(imposed_available=False,@@ -207,22 +102,6 @@ def test_grantable_advisory_yields_to_consumed_exhaustion() -> None:assert actuation.reason_code == OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE-def test_progress_exhaustion_terminals_a_no_source_structural_bail() -> None:- no_source = OutputContractActuationEvidence(- imposed_available=False,- click_only_spine=True,- observed_required_values=False,- prior_actuation=False,- prior_directive_unconsumed=False,- advisory_state=OutputContractAdvisoryState.UNUSED,- actuation_progress_exhausted=True,- declick_attempt_failed=True,- )- actuation = resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=no_source)- assert actuation.kind == OutputContractActuationKind.BLOCKED_TERMINAL- assert actuation.reason_code == OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE--def test_observable_source_structural_exhaustion_routes_to_advisory_not_terminal() -> None:observable = OutputContractActuationEvidence(imposed_available=False,@@ -302,38 +181,6 @@ def test_consumed_exhaustion_without_declick_names_actuation_exhausted_not_unobsassert actuation.reason_code == OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE-def test_progress_exhaustion_names_source_unobservable_for_click_only_after_declick() -> None:- click_only = OutputContractActuationEvidence(- False, True, False, False, False, OutputContractAdvisoryState.CONSUMED, True, True- )- actuation = resolve_output_contract_actuation(family=OutputContractBailFamily.STATIC_RETURN, evidence=click_only)- assert actuation.kind == OutputContractActuationKind.BLOCKED_TERMINAL- assert actuation.reason_code == OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE---def test_resolver_prefers_impose_then_directive_then_source_terminal() -> None:- imposed = OutputContractActuationEvidence(True, True, False, True, True, OutputContractAdvisoryState.UNUSED)- assert (- resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=imposed).kind- == OutputContractActuationKind.IMPOSED- )- no_source_declick_failed = OutputContractActuationEvidence(- False, True, False, True, True, OutputContractAdvisoryState.UNUSED, False, True- )- source_terminal = resolve_output_contract_actuation(- family=OutputContractBailFamily.STRUCTURAL, evidence=no_source_declick_failed- )- assert source_terminal.kind == OutputContractActuationKind.BLOCKED_TERMINAL- assert source_terminal.reason_code == OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE- fresh_directive = OutputContractActuationEvidence(- False, False, False, False, False, OutputContractAdvisoryState.UNUSED- )- assert (- resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=fresh_directive).kind- == OutputContractActuationKind.STRUCTURE_DIRECTIVE- )--def test_dispatchable_spine_never_terminals_before_its_run() -> None:unscouted = OutputContractActuationEvidence(False, False, False, False, False, OutputContractAdvisoryState.UNUSED)first = resolve_output_contract_actuation(family=OutputContractBailFamily.STATIC_RETURN, evidence=unscouted)@@ -391,27 +238,6 @@ def test_no_source_terminal_requires_prior_actuation() -> None:assert actuation.kind != OutputContractActuationKind.BLOCKED_TERMINAL-def test_arm_d_never_fires_on_p5_shape_observed_source() -> None:- for family, imposed, click, prior, unconsumed, advisory in itertools.product(- list(OutputContractBailFamily),- [False, True],- [False, True],- [False, True],- [False, True],- list(OutputContractAdvisoryState),- ):- evidence = OutputContractActuationEvidence(- imposed_available=imposed,- click_only_spine=click,- observed_required_values=True,- prior_actuation=prior,- prior_directive_unconsumed=unconsumed,- advisory_state=advisory,- )- actuation = resolve_output_contract_actuation(family=family, evidence=evidence)- assert actuation.reason_code != OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE--def test_observed_values_suppresses_no_source_terminal() -> None:has_source = OutputContractActuationEvidence(False, True, True, True, False, OutputContractAdvisoryState.UNUSED)actuation = resolve_output_contract_actuation(family=OutputContractBailFamily.STATIC_RETURN, evidence=has_source)@@ -650,31 +476,6 @@ def test_advisory_terminal_requires_consumed_not_granted() -> None:assert actuation.reason_code == OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE-def test_arm_d_signal_maps_to_output_source_unobservable_halt() -> None:- signal = build_output_source_unobservable_blocker_signal(- reason_code=OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,- required_paths={"output.confirmation_number"},- block_label="collect_confirmation",- )- assert signal.renders_final_reply- assert signal.preserves_workflow_draft- assert_clean_user_facing_text(signal.user_facing_reason)- halt = turn_halt_from_blocker_signal(signal, source="workflow_update")- assert halt is not None- assert halt.kind == TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE---def test_arm_d_actuation_exhausted_signal_also_maps() -> None:- signal = build_output_source_unobservable_blocker_signal(- reason_code=OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,- required_paths={"output.account_number"},- block_label="collect_account",- )- halt = turn_halt_from_blocker_signal(signal, source="workflow_update")- assert halt is not None- assert halt.kind == TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE--_STRUCTURAL_BLOCKERS = ["insufficient_durable_stages"]_CLICK_ONLY_CODE = "page.click('#toggle-service')"@@ -701,35 +502,6 @@ def _actuate_click_only(ctx: SimpleNamespace, signature: str, fingerprint: str))-def test_click_only_no_source_bail_terminals_after_one_declick_cycle() -> None:- signature = "sig_never_converges"- ctx = _advisory_ctx()- first = _actuate_click_only(ctx, signature, "fp_0")- assert first.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE- assert ctx.output_contract_declick_attempted_by_signature[signature] is True- terminal = _actuate_click_only(ctx, signature, "fp_1")- assert terminal.kind == OutputContractActuationKind.BLOCKED_TERMINAL- assert terminal.reason_code == OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE---def test_grantable_split_failure_suppresses_arm_d_and_escalates_to_advisory() -> None:- signature = "sig_flaky_split"- ctx = _arm_static_return_advisory_ctx(signature)- ctx.output_contract_declick_attempted_by_signature[signature] = True- actuation = wu._actuate_output_contract_bail(- ctx,- blockers=list(_STRUCTURAL_BLOCKERS),- target_code=_CLICK_ONLY_CODE,- required_paths={"output.confirmation_number"},- signature=signature,- current_fingerprint=_FINGERPRINT,- advisory_run_grantable=True,- )- assert actuation.kind == OutputContractActuationKind.ADVISORY_RUN- assert actuation.reason_code != OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE- assert ctx.output_contract_actuation_by_signature[signature] == OutputContractAdvisoryState.GRANTED--def test_grantable_split_failure_first_pass_arms_directive_not_arm_d() -> None:signature = "sig_flaky_split_first"ctx = _advisory_ctx()@@ -790,33 +562,6 @@ def test_steer_only_downgrades_advisory_run_and_grants_nothing() -> None:assert ctx.output_contract_actuation_count_by_signature == {}-def test_steer_only_never_terminals_click_only_and_skips_declick() -> None:- signature = "sig_steer_click_only"- ctx = _advisory_ctx()- first = wu._actuate_output_contract_bail(- ctx,- blockers=list(_STRUCTURAL_BLOCKERS),- target_code=_CLICK_ONLY_CODE,- required_paths={"output.confirmation_number"},- signature=signature,- current_fingerprint="fp_0",- steer_only=True,- )- assert first.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE- assert signature not in ctx.output_contract_declick_attempted_by_signature- second = wu._actuate_output_contract_bail(- ctx,- blockers=list(_STRUCTURAL_BLOCKERS),- target_code=_CLICK_ONLY_CODE,- required_paths={"output.confirmation_number"},- signature=signature,- current_fingerprint="fp_1",- steer_only=True,- )- assert second.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE- assert second.reason_code != OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE--def test_observed_required_values_exact_and_lineage_match() -> None:ctx = SimpleNamespace(scouted_output_covered_paths={"output.order.id"}, composition_page_evidence=None)assert wu._observed_required_output_values(ctx, {"output.order.id"}) is True@@ -1895,5 +1640,5 @@ def _guardrail_impose_ctx() -> tuple[CopilotContext, str, list[dict[str, object]workflow_yaml = _declaration_waiver_yaml('record_id = await page.inner_text("#record")')metadata = _record_contract_metadata()ctx.scouted_output_covered_paths = {"output.record_id"}- ctx.code_authoring_guardrail_reject_count = enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1+ ctx.code_authoring_guardrail_reject_count = 3return ctx, workflow_yaml, metadatadiff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 8567bff15..928660502 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -28,7 +28,6 @@ from skyvern.forge.sdk.copilot.enforcement import (_response_coverage_nudge,)from skyvern.forge.sdk.copilot.loop_detection import tool_step_identity-from skyvern.forge.sdk.copilot.output_contracts import OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODEfrom skyvern.forge.sdk.copilot.output_policy import (ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,ACTUATION_OBLIGATION_STEER_REASON_CODE,@@ -404,35 +403,37 @@ def test_does_not_infer_arbitrary_prose_secret_echoes(user_response: str) -> Nonassert verdict.allowed-def test_rejects_raw_secret_in_workflow_yaml() -> None:+def test_rejects_raw_secret_in_the_reply() -> None:verdict = evaluate_output_policy(request_policy=_policy(),- workflow_yaml=_workflow_yaml(navigation_goal="Log in with password: hunter2."),+ user_response="Log in with password: hunter2.",)assert not verdict.allowedassert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes-def test_rejects_raw_secret_in_structured_tool_arguments() -> None:- verdict = evaluate_output_policy(- request_policy=_policy(),- tool_arguments={"workflow_yaml": {"navigation_goal": "Type password: hunter2 into the field."}},- )+def test_a_draft_carrying_the_same_value_is_not_a_disclosure_surface() -> None:+ # Persistence scrubbing and credential rebinding own what reaches storage. Refusing the draft+ # only refused the user their own workflow, and judged the YAML encoding rather than the value.+ for draft_kwargs in (+ {"workflow_yaml": _workflow_yaml(navigation_goal="Log in with password: hunter2.")},+ {"tool_arguments": {"workflow_yaml": {"navigation_goal": "Type password: hunter2 into the field."}}},+ ):+ verdict = evaluate_output_policy(request_policy=_policy(), **draft_kwargs)- assert not verdict.allowed- assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes+ assert verdict.allowed, draft_kwargsclass TestSanctionedSecretReferenceIdiom:"""`password = parameters[...]` / attribute-chain reads are references to bound- values, not leaks — the code-only authoring idiom must pass the syntactic- backstop while literal values keep blocking."""+ values, not leaks — quoting the code-only authoring idiom back to the user must pass the+ syntactic backstop while literal values keep blocking."""def test_allows_parameters_subscript_assignment_in_tool_arguments(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": 'code: |\n password = parameters["login_credential"]'},+ user_response=('code: |\n password = parameters["login_credential"]'),)assert verdict.allowed@@ -440,7 +441,7 @@ class TestSanctionedSecretReferenceIdiom:def test_allows_credential_attribute_read_in_workflow_yaml(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- workflow_yaml=(+ user_response=("code: |\n"" username = login_credential.username\n"" password = login_credential.password\n"@@ -453,7 +454,7 @@ class TestSanctionedSecretReferenceIdiom:def test_allows_awaited_credential_otp_assignment_in_workflow_yaml(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- workflow_yaml=(+ user_response=('code: |\n passcode = await login_credentials.otp()\n await page.locator("#code").fill(passcode)\n'),)@@ -463,7 +464,7 @@ class TestSanctionedSecretReferenceIdiom:def test_allows_keyword_secret_reference_to_awaited_credential_otp_in_tool_arguments(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": "code: |\n token = await login_credential.otp()"},+ user_response=("code: |\n token = await login_credential.otp()"),)assert verdict.allowed@@ -471,9 +472,7 @@ class TestSanctionedSecretReferenceIdiom:def test_rejects_literal_secret_even_when_same_line_has_credential_reference(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={- "workflow_yaml": "code: |\n password = 'hunter2'; password_ref = login_credential.password"- },+ user_response="code: |\n password = 'hunter2'; password_ref = login_credential.password",)assert not verdict.allowed@@ -482,7 +481,7 @@ class TestSanctionedSecretReferenceIdiom:def test_allows_str_wrapped_parameter_reference(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": 'code: |\n token = str(parameters.get("api_token"))'},+ user_response=('code: |\n token = str(parameters.get("api_token"))'),)assert verdict.allowed@@ -495,7 +494,7 @@ class TestSanctionedSecretReferenceIdiom:):verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": f"code: |\n password = {rhs}"},+ user_response=(f"code: |\n password = {rhs}"),)assert not verdict.allowed, rhs@@ -504,7 +503,7 @@ class TestSanctionedSecretReferenceIdiom:def test_still_rejects_awaited_non_credential_secret_source(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": "code: |\n passcode = await page.locator('#code').inner_text()"},+ user_response=("code: |\n passcode = await page.locator('#code').inner_text()"),)assert not verdict.allowed@@ -513,7 +512,7 @@ class TestSanctionedSecretReferenceIdiom:def test_allows_keyword_argument_reference_with_closing_paren(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": 'code: |\n await do_login(password=parameters["cred"])'},+ user_response=('code: |\n await do_login(password=parameters["cred"])'),)assert verdict.allowed@@ -522,7 +521,7 @@ class TestSanctionedSecretReferenceIdiom:# A dotted literal (JWT-shaped) must not pass as an "attribute chain".verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": "code: |\n token = eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.sig"},+ user_response=("code: |\n token = eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.sig"),)assert not verdict.allowed@@ -531,7 +530,7 @@ class TestSanctionedSecretReferenceIdiom:def test_still_rejects_dotted_literal_not_ending_in_credential_field(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": "code: |\n password = admin.password123"},+ user_response=("code: |\n password = admin.password123"),)assert not verdict.allowed@@ -540,7 +539,7 @@ class TestSanctionedSecretReferenceIdiom:def test_still_rejects_quoted_literal_assignment(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": 'code: |\n password = "hunter2"'},+ user_response=('code: |\n password = "hunter2"'),)assert not verdict.allowed@@ -549,7 +548,94 @@ class TestSanctionedSecretReferenceIdiom:def test_still_rejects_bare_literal_assignment(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={"workflow_yaml": "password: hunter2"},+ user_response=("password: hunter2"),+ )++ assert not verdict.allowed+ assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes++ def test_allows_lexical_token_parsed_out_of_page_text(self) -> None:+ for rhs in (+ "parts[0]",+ "text.split(' logs found')[0]",+ "m.group(1)",+ ):+ verdict = evaluate_output_policy(+ request_policy=_policy(),+ user_response=(f"code: |\n first_token = {rhs}"),+ )++ assert verdict.allowed, rhs++ def test_verdict_does_not_depend_on_how_the_yaml_escapes_the_code(self) -> None:+ block_scalar = 'code: |\n _entry = page.locator("#password")\n count_token = evidence_text.split(" ")[0]\n'+ escaped_scalar = (+ 'code: "_entry = page.locator(\\"#password\\")\\ncount_token = evidence_text.split(\\" \\")[0]"'+ )++ for workflow_yaml in (block_scalar, escaped_scalar):+ verdict = evaluate_output_policy(+ request_policy=_policy(),+ user_response=(workflow_yaml),+ )++ assert verdict.allowed, workflow_yaml++ def test_allows_a_newline_literal_the_block_scalar_keeps_as_code_data(self) -> None:+ verdict = evaluate_output_policy(+ request_policy=_policy(),+ user_response=('code: |\n count_token = text.split("\\n")[0]\n'),+ )++ assert verdict.allowed++ def test_a_value_shaped_secret_inside_a_call_argument_still_blocks(self) -> None:+ # The lexical-token exemption clears only the keyword-assignment match; the value-shaped+ # patterns scan the whole text independently, so hiding a real secret as a call argument+ # does not ride out on the exemption.+ for rhs in (+ 'cache.get("sk-abcdefghijklmnopqrstuvwx")',+ 'store.lookup("eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27u")',+ ):+ verdict = evaluate_output_policy(+ request_policy=_policy(),+ user_response=f"code: |\n token = {rhs}",+ )++ assert not verdict.allowed, rhs+ assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes++ def test_still_rejects_literal_secret_under_a_lexical_token_name(self) -> None:+ # The literal deliberately contains no secret keyword, so only the token arm can reject it.+ for lhs in ("first_token", "result_token", "token"):+ verdict = evaluate_output_policy(+ request_policy=_policy(),+ user_response=(f"code: |\n {lhs} = 'ghp_abcdefghijklmnopqrstuvwxyz'"),+ )++ assert not verdict.allowed, lhs+ assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes++ def test_still_rejects_a_literal_secret_wrapped_to_look_parsed(self) -> None:+ for rhs in (+ 'identity("ghp_abcdefghijklmnopqrstuvwxyz")',+ '("ghp_abcdefghijklmnopqrstuvwxyz", parts)[0]',+ '["ghp_abcdefghijklmnopqrstuvwxyz"][0]',+ '"ghp_abcdefghijklmnopqrstuvwxyz".strip()',+ 'str("ghp_abcdefghijklmnopqrstuvwxyz")',+ ):+ verdict = evaluate_output_policy(+ request_policy=_policy(),+ user_response=(f"code: |\n first_token = {rhs}"),+ )++ assert not verdict.allowed, rhs+ assert OutputPolicyReason.RAW_SECRET_LEAK in verdict.reason_codes++ def test_still_rejects_page_read_into_an_unambiguous_secret_name(self) -> None:+ verdict = evaluate_output_policy(+ request_policy=_policy(),+ user_response=("code: |\n password_token = await page.locator('#p').inner_text()"),)assert not verdict.allowed@@ -558,9 +644,7 @@ class TestSanctionedSecretReferenceIdiom:def test_still_rejects_email_password_pair_next_to_reference_idiom(self) -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={- "workflow_yaml": 'code: |\n password = parameters["cred"]\nnotes: [email protected]:FakePass123!'- },+ user_response=('code: |\n password = parameters["cred"]\nnotes: [email protected]:FakePass123!'),)assert not verdict.allowed@@ -575,12 +659,10 @@ class TestSanctionedSecretReferenceIdiom:assert _raw_secret_detected('password = parameters["cred"]')-def test_rejects_bulk_colon_delimited_credentials_in_tool_arguments() -> None:+def test_rejects_bulk_colon_delimited_credentials_in_the_reply() -> None:verdict = evaluate_output_policy(request_policy=_policy(),- tool_arguments={- "parameters": {"account_list": "[email protected]:FakePass123!\[email protected]:AnotherFakePass456!"}- },+ user_response="[email protected]:FakePass123!\[email protected]:AnotherFakePass456!",)assert not verdict.allowed@@ -609,10 +691,10 @@ def test_does_not_hard_block_on_prior_global_context_secret() -> None:assert verdict.allowed-def test_rejects_raw_secret_even_when_workflow_contains_jinja_placeholder() -> None:+def test_rejects_raw_secret_in_a_reply_that_also_contains_a_jinja_placeholder() -> None:verdict = evaluate_output_policy(request_policy=_policy(),- workflow_yaml="navigation_goal: Use {{ parameters.username }} and password: hunter2.",+ user_response="Use {{ parameters.username }} and password: hunter2.",)assert not verdict.allowed@@ -700,19 +782,6 @@ def test_url_clarification_does_not_trigger_output_field_confirmation_guard() ->assert OutputPolicyReason.AVOIDABLE_OUTPUT_FIELD_CONFIRMATION not in verdict.reason_codes-def test_classifies_unbacked_workflow_delivery_claim() -> None:- verdict = evaluate_output_policy(- request_policy=_policy(),- response_type="REPLY",- user_response="Here's the workflow.",- has_workflow_proposal=False,- )-- assert not verdict.allowed- assert OutputPolicyReason.UNBACKED_WORKFLOW_DELIVERY_CLAIM in verdict.reason_codes- assert OutputPolicyReason.MISSING_PROPOSAL_STATE in verdict.reason_codes--def test_flags_block_yaml_pasted_into_user_response() -> None:user_response = ("I've now updated the workflow to also accept the form's URL as a parameter, named `form_url`. "@@ -770,47 +839,6 @@ def test_allows_workflow_delivery_language_after_failed_workflow_attempt() -> Noassert verdict.allowed-def test_classifies_missing_unvalidated_proposal_affordance() -> None:- verdict = evaluate_output_policy(- request_policy=_policy(),- response_type="REPLY",- user_response="I drafted the workflow but did not finish testing.",- has_workflow_proposal=True,- unvalidated=True,- )-- assert not verdict.allowed- assert OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE in verdict.reason_codes---def test_rejects_incidental_accept_discard_language_for_unvalidated_proposal() -> None:- verdict = evaluate_output_policy(- request_policy=_policy(),- response_type="REPLY",- user_response="I accept that the test failed, but I can't discard the login block without more info.",- has_workflow_proposal=True,- unvalidated=True,- )-- assert not verdict.allowed- assert OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE in verdict.reason_codes---def test_allows_unvalidated_proposal_affordance_copy() -> None:- verdict = evaluate_output_policy(- request_policy=_policy(),- response_type="REPLY",- user_response=(- "Use Review to inspect it, Accept to save it, or Reject to discard it. "- "It has not been tested or verified end-to-end."- ),- has_workflow_proposal=True,- unvalidated=True,- )-- assert verdict.allowed--def test_allows_backend_authored_request_policy_clarification() -> None:verdict = evaluate_output_policy(request_policy=_policy(user_response_policy="ask_clarification", allow_update_workflow=False),@@ -1608,26 +1636,6 @@ def test_sdk_output_guardrail_allows_structural_blocker_reason() -> None:assert diagnostics["cannot_act_reason"] == "missing_field_value"-def test_sdk_output_guardrail_allows_explicit_structural_blocker_reason_code() -> None:- ctx = _ctx(turn_intent=_browser_actuation_intent())- ctx.blocker_signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="Report the unavailable output source.",- user_facing_reason="The values are not observable on this page.",- recovery_hint="report_blocker_to_user",- internal_reason_code=OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,- )-- verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(- ctx,- {"type": "REPLY", "user_response": "The values are not observable on this page."},- )-- assert response_type == "REPLY"- assert verdict.allowed- assert diagnostics["cannot_act_reason"] == "structural_blocker"--def test_sdk_output_guardrail_rejects_generic_report_blocker_reason() -> None:ctx = _ctx(turn_intent=_browser_actuation_intent())ctx.blocker_signal = CopilotToolBlockerSignal(@@ -2232,34 +2240,6 @@ workflow_definition:assert ctx.block_observation_refs == {}-async def test_update_workflow_rejects_raw_secret_before_processing(monkeypatch) -> None:- from skyvern.forge.sdk.copilot.tools import _update_workflow-- process_mock = MagicMock()- monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml", process_mock)-- result = await _update_workflow(- {- "workflow_yaml": """-workflow_definition:- blocks:- - block_type: navigation- label: login- navigation_goal: Type password: hunter2 into the password field.-"""- },- _ctx(),- allow_missing_credentials=True,- )-- assert result["ok"] is False- assert "raw_secret_leak" in result["error"]- assert "<key>.password" in result["error"]- assert "do not split, concatenate, or obfuscate" in result["error"]- process_mock.assert_not_called()--def test_inline_replace_workflow_rejects_raw_secret_before_processing(monkeypatch) -> None:process_mock = MagicMock()monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml", process_mock)@@ -2357,27 +2337,6 @@ def test_translate_scrubs_late_block_running_leak_and_preserves_draft() -> None:assert "workflow draft is still saved" in agent_result.user_response-def test_timeout_exit_scrubs_recorded_late_block_running_leak_and_preserves_draft() -> None:- ctx = _ctx()- saved_workflow = object()- ctx.last_workflow = saved_workflow- ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"- ctx.last_update_block_count = 5- ctx.last_test_ok = False- ctx.last_test_failure_reason = (- "Less than 90 seconds remain in this Copilot turn after the previous workflow run failed. "- "Do NOT retry block-running tools."- )-- agent_result = agent_module._build_timeout_exit_result(ctx, global_llm_context=None)-- assert agent_result.updated_workflow is saved_workflow- assert agent_result.clear_proposed_workflow is False- assert agent_result.proposal_disposition == "review_untested"- assert "Do NOT retry" not in agent_result.user_response- assert "draft workflow proposal" in agent_result.user_response--def test_translate_to_agent_result_rewrites_unbacked_workflow_claim() -> None:result = _fake_run_result({"type": "REPLY", "user_response": "I've drafted a workflow for you."})@@ -2761,25 +2720,6 @@ def test_translate_to_agent_result_rewrites_block_yaml_when_workflow_attached()assert agent_result.updated_workflow is workflow-def test_translate_to_agent_result_adds_unvalidated_affordance() -> None:- workflow = SimpleNamespace(name="draft")- result = _fake_run_result({"type": "REPLY", "user_response": "I drafted the workflow."})-- agent_result = asyncio.run(- agent_module._translate_to_agent_result(- result,- _ctx(last_workflow=workflow, last_workflow_yaml="title: draft", last_test_ok=None),- global_llm_context=None,- chat_request=_chat_request(),- organization_id="org-1",- )- )-- assert agent_result.updated_workflow is workflow- assert "Accept to save" in agent_result.user_response- assert "Reject to discard" in agent_result.user_response--def _two_credential_workflow_yaml() -> str:return """workflow_definition:@@ -2935,18 +2875,6 @@ def test_genuine_attempt_true_suppresses_avoidable_backstop() -> None:assert OutputPolicyReason.AVOIDABLE_OUTPUT_FIELD_CONFIRMATION not in verdict.reason_codes-def test_genuine_attempt_true_suppresses_unbacked_delivery_claim() -> None:- verdict = evaluate_output_policy(- request_policy=_policy(),- response_type="REPLY",- user_response="Here's the workflow.",- has_workflow_proposal=False,- workflow_attempted=True,- )-- assert OutputPolicyReason.UNBACKED_WORKFLOW_DELIVERY_CLAIM not in verdict.reason_codes--class _CaptureSentinel(Exception):pass@@ -2985,23 +2913,6 @@ def test_build_exit_result_passes_genuine_predicate_as_workflow_attempted(monkeyassert captured["evaluate"] == ctx.has_genuine_workflow_attempt()-def test_watchdog_run_id_only_counts_as_genuine_and_suppresses_unbacked() -> None:- ctx = _run_id_only_ctx()- legacy_two_field_attempted = ctx.last_update_block_count is not None or ctx.last_test_ok is not None- assert legacy_two_field_attempted is False- assert ctx.has_genuine_workflow_attempt() is True-- verdict = evaluate_output_policy(- request_policy=_policy(),- response_type="REPLY",- user_response="Here's the workflow.",- has_workflow_proposal=False,- workflow_attempted=ctx.has_genuine_workflow_attempt(),- )-- assert OutputPolicyReason.UNBACKED_WORKFLOW_DELIVERY_CLAIM not in verdict.reason_codes--def test_agent_no_longer_derives_workflow_attempted_from_two_marker_fields() -> None:source = inspect.getsource(agent_module)assert "last_update_block_count is not None or ctx.last_test_ok is not None" not in source@@ -3177,7 +3088,6 @@ def test_avoidable_ask_with_co_firing_secret_leak_is_not_deferred() -> None:_AUTHOR_TIME_KEEP_HARD_REASONS = frozenset({- OutputPolicyReason.RAW_SECRET_LEAK,OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,}@@ -3194,12 +3104,11 @@ _AUTHORING_SEAM_REFUSAL_SOURCES: dict[str, str] = {"code_safety": CODE_SAFETY_BLOCK_ID,"credential_scout": CREDENTIAL_SCOUT_BLOCK_ID,"banned_blocks": BANNED_BLOCKS_BLOCK_ID,- "raw_secret_leak": CREDENTIAL_SCOUT_BLOCK_ID,+ "raw_secret_leak": _FINDING,"request_policy_clarification_bypass": _FINDING,"unapproved_credential_reference": CREDENTIAL_SCOUT_BLOCK_ID,"credential_scope_broadened": CREDENTIAL_SCOUT_BLOCK_ID,"unbacked_workflow_delivery_claim": _FINDING,- "missing_unvalidated_proposal_affordance": _FINDING,"missing_proposal_state": _FINDING,"persistence_state_mismatch": _FINDING,"internal_tool_instruction_leak": _FINDING,@@ -3252,13 +3161,13 @@ def test_tool_input_disposition_matches_classification_table(reason: OutputPolicdef test_hard_reason_still_blocks_when_co_firing_with_a_demoted_reason() -> None:verdict = OutputPolicyVerdict(- reason_codes=[OutputPolicyReason.WORKFLOW_YAML_IN_REPLY, OutputPolicyReason.RAW_SECRET_LEAK]+ reason_codes=[OutputPolicyReason.WORKFLOW_YAML_IN_REPLY, OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED])steered = demote_author_time_steer_reasons(verdict)assert verdict.allowed is False- assert verdict.reason_codes == [OutputPolicyReason.RAW_SECRET_LEAK]+ assert verdict.reason_codes == [OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED]assert steered == [OutputPolicyReason.WORKFLOW_YAML_IN_REPLY]diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.pyindex 2eefc0b66..a530b37db 100644--- a/tests/unit/test_copilot_output_utils.py+++ b/tests/unit/test_copilot_output_utils.py@@ -538,37 +538,6 @@ class TestFormatToolResultForUser:assert summary != signal.user_facing_reasonassert summary == "Failed: timeout"- def test_active_terminal_blocker_matches_structured_failure_category(self) -> None:- from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal- from skyvern.forge.sdk.copilot.failure_tracking import (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,- ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,- )-- signal = CopilotToolBlockerSignal(- blocker_kind="tool_error",- agent_steering_text="The prior active workflow run emitted typed terminal evidence.",- user_facing_reason="I reached the requested browser state, but the workflow still needs review.",- recovery_hint="report_blocker_to_user",- internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,- blocked_tool="update_and_run_blocks",- )- result = {- "ok": False,- "error": "The active run reached the requested browser state.",- "data": {- "failure_categories": [- {"category": ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY, "confidence_float": 1.0}- ]- },- }-- summary = format_tool_result_for_user("update_and_run_blocks", result, blocker_signal=signal)- detail = summarize_tool_result_detail(result, blocker_signal=signal)-- assert summary == signal.user_facing_reason- assert detail == signal.user_facing_reason-def test_watchdog_control_signal_summary_overrides_raw_detail(self) -> None:result = {"ok": False,diff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.pyindex ca4b5d9b9..d91070253 100644--- a/tests/unit/test_copilot_per_tool_budget.py+++ b/tests/unit/test_copilot_per_tool_budget.py@@ -49,7 +49,6 @@ from skyvern.forge.sdk.copilot.run_outcome import (TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,)from skyvern.forge.sdk.copilot.tools import (- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,WatchdogExitReason,_composition_anti_bot_reason,_last_run_has_terminal_anti_bot_blocker,@@ -274,15 +273,6 @@ def test_watchdog_user_failure_reason_excludes_next_tool_instruction() -> None:assert "update_and_run_blocks" not in reason-def test_watchdog_user_failure_reason_for_active_terminal_evidence_is_not_success() -> None:- exit_reason: WatchdogExitReason = "active_run_terminal_evidence"- reason = _watchdog_user_failure_reason(exit_reason, "wr_active", 240, None)-- assert "requested browser state" in reason- assert "Full workflow verification is still required" in reason- assert "Run ID: wr_active" in reason--def test_record_clears_top_category_on_run_with_different_category() -> None:ctx = _fresh_context()ctx.last_failure_category_top = PER_TOOL_BUDGET_FAILURE_CATEGORY@@ -489,25 +479,6 @@ def test_failure_signature_is_stable_across_budget_trips_with_different_run_ids(assert sig_a == sig_b-def test_failure_signature_ignores_frontier_label_drift() -> None:- """Block labels are UI/editing affordances, not root-cause identity."""- failure_categories = [{"category": PER_TOOL_BUDGET_FAILURE_CATEGORY, "confidence_float": 1.0}]- sig_a = compute_failure_signature(- frontier_start_label="block_a",- failure_reason="per-tool-call budget exceeded (Run ID: wr_aaaa)",- failure_categories=failure_categories,- suspicious_success=False,- )- sig_b = compute_failure_signature(- frontier_start_label="block_b",- failure_reason="per-tool-call budget exceeded (Run ID: wr_bbbb)",- failure_categories=failure_categories,- suspicious_success=False,- )- assert sig_a is not None- assert sig_a == sig_b--def test_nudge_text_advises_splitting_chain() -> None:assert "STOP" in POST_PER_TOOL_BUDGET_NUDGEassert "split" in POST_PER_TOOL_BUDGET_NUDGE.lower() or "shrink" in POST_PER_TOOL_BUDGET_NUDGE.lower()@@ -554,16 +525,6 @@ def test_reconciliation_clears_for_per_tool_budget_even_on_canceled_status() ->assert ctx.pending_reconciliation_run_id is None-def test_reconciliation_clears_for_active_terminal_evidence_cancel() -> None:- ctx = _fresh_context()- ctx.pending_reconciliation_run_id = "wr_active"- ctx.last_failure_category_top = ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY-- _maybe_clear_reconciliation_flag(ctx, _get_run_results_response("wr_active", "canceled"))-- assert ctx.pending_reconciliation_run_id is None--def test_reconciliation_does_not_clear_for_non_budget_canceled() -> None:"""Non-budget watchdog cancels keep the existing strict semantics:``canceled`` is ambiguous and must not silently clear the guard."""diff --git a/tests/unit/test_copilot_pre_discovery_url_nudge.py b/tests/unit/test_copilot_pre_discovery_url_nudge.pyindex 3889ef61e..a2a296b36 100644--- a/tests/unit/test_copilot_pre_discovery_url_nudge.py+++ b/tests/unit/test_copilot_pre_discovery_url_nudge.py@@ -30,6 +30,7 @@ from skyvern.forge.sdk.copilot.enforcement import (_response_coverage_nudge,)from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy+from skyvern.forge.sdk.copilot.request_slots import is_canonical_request_slot_pathfrom skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode@@ -323,6 +324,58 @@ def _output_schema_ask(refs: list[str]) -> dict[str, object]:}+_ANONYMOUS_SLOT_PATH = f"output.request_slot_{'0c17ba10d580ed2c63c2996d0605e690f1746286a92698ab':.48}_00"+++def _anonymous_slot_contract_policy() -> RequestPolicy:+ return _output_path_contract_policy(+ completion_criteria=[+ CompletionCriterion(+ id="count",+ outcome="The number of azure errors is returned.",+ output_path=_ANONYMOUS_SLOT_PATH,+ ),+ ],+ )+++def test_output_schema_ask_auto_answers_when_the_contract_only_holds_anonymous_slots() -> None:+ # A request that named no output key pins an anonymous slot, so no name the model proposes can+ # ever be cited back — the coverage check is unsatisfiable and the ask must still be answered.+ assert is_canonical_request_slot_path(_ANONYMOUS_SLOT_PATH)+ ctx = _mid_build_ctx(_anonymous_slot_contract_policy())++ with capture_logs() as logs:+ nudge = _response_coverage_nudge(ctx, _output_schema_ask(["output.azure_error_count"]))++ assert nudge is not None+ assert nudge != PRESENT_COMPLETION_CONTRACT_ASK_RETRY+ assert "yours to choose" in nudge+ events = [entry for entry in logs if entry["event"] == "copilot_ask_subject_auto_answered"]+ assert len(events) == 1+ assert events[0]["resolved_refs"] == []+++def test_output_schema_ask_passes_through_when_any_requested_path_is_nameable() -> None:+ policy = _output_path_contract_policy(+ completion_criteria=[+ CompletionCriterion(+ id="count", outcome="The azure error count is returned.", output_path=_ANONYMOUS_SLOT_PATH+ ),+ CompletionCriterion(+ id="signups", outcome="The new signup count is returned.", output_path=_REQUESTED_OUTPUT_PATHS[0]+ ),+ ],+ )+ ctx = _mid_build_ctx(policy)++ with capture_logs() as logs:+ nudge = _response_coverage_nudge(ctx, _output_schema_ask(["output.unknown_field"]))++ assert nudge is None+ assert not [entry for entry in logs if entry["event"] == "copilot_ask_subject_auto_answered"]++def test_output_schema_ask_with_covering_refs_auto_answers() -> None:ctx = _present_contract_ctx(request_policy=_output_path_contract_policy())diff --git a/tests/unit/test_copilot_probable_site_block.py b/tests/unit/test_copilot_probable_site_block.pydeleted file mode 100644index 58a38f146..000000000--- a/tests/unit/test_copilot_probable_site_block.py+++ /dev/null@@ -1,318 +0,0 @@-"""Tests for the probable-site-block-wall detector and stop nudge — the-copilot's own shape-independent streak for sites that the shared classifier-routes to ``DATA_EXTRACTION_FAILURE`` rather than ``ANTI_BOT_DETECTION``."""--from __future__ import annotations--import pytest--from skyvern.forge.sdk.copilot.enforcement import (- MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES,- POST_PROBABLE_SITE_BLOCK_STOP_NUDGE,- PROBABLE_SITE_BLOCK_STREAK_STOP_AT,- REPEATED_FRONTIER_STREAK_ESCALATE_AT,- _check_enforcement,- _needs_probable_site_block_stop_nudge,- _repeated_frontier_failure_nudge,-)-from skyvern.forge.sdk.copilot.tools import (- _detect_probable_site_block_wall,- _record_run_blocks_result,-)-from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, TurnHaltKind-from tests.unit.conftest import make_copilot_context as _fresh_context--_SCRAPE_WALL_REASON = (- "Skyvern failed to load the website. The page may have navigated "- "unexpectedly or become unresponsive during analysis."-)---def _scrape_wall_result() -> dict:- return {- "ok": False,- "data": {- "blocks": [- {"block_type": "GOTO_URL", "status": "completed"},- {- "block_type": "EXTRACTION",- "status": "failed",- "failure_reason": _SCRAPE_WALL_REASON,- },- ]- },- }---# ----------------------------------------------------------------------------# _detect_probable_site_block_wall-# ------------------------------------------------------------------------------ ("result", "expected"),- [- pytest.param(_scrape_wall_result(), True, id="completed_nav_plus_scrape_wall"),- pytest.param(- {- "ok": False,- "data": {- "blocks": [- {"block_type": "NAVIGATION", "status": "completed"},- {- "block_type": "EXTRACTION",- "status": "failed",- "failure_reason": "We think the page may have navigated unexpectedly during analysis.",- },- ]- },- },- True,- id="page_navigated_unexpectedly_phrasing",- ),- pytest.param({**_scrape_wall_result(), "ok": True}, False, id="run_ok"),- pytest.param(- {- "ok": False,- "data": {- "blocks": [- {- "block_type": "NAVIGATION",- "status": "failed",- "failure_reason": _SCRAPE_WALL_REASON,- },- ]- },- },- True,- id="nav_only_failure_with_template_reason",- ),- pytest.param(- {- "ok": False,- "data": {- "blocks": [- {- "block_type": "GOTO_URL",- "status": "failed",- "failure_reason": (- "Failed to navigate to url https://x.invalid. Error message: net::ERR_NAME_NOT_RESOLVED"- ),- },- {- "block_type": "EXTRACTION",- "status": "failed",- "failure_reason": _SCRAPE_WALL_REASON,- },- ]- },- },- False,- id="non_retriable_nav",- ),- pytest.param(- {- "ok": False,- "data": {- "blocks": [- {"block_type": "GOTO_URL", "status": "completed"},- {- "block_type": "EXTRACTION",- "status": "failed",- "failure_reason": "Timeout waiting for selector #submit",- },- ]- },- },- False,- id="other_failure_reasons",- ),- pytest.param({"ok": False}, False, id="missing_data"),- pytest.param({"ok": False, "data": "not a dict"}, False, id="data_not_a_dict"),- pytest.param({"ok": False, "data": {}}, False, id="empty_data"),- ],-)-def test_detect_probable_site_block_wall(result: dict, expected: bool) -> None:- assert _detect_probable_site_block_wall(result) is expected---# ----------------------------------------------------------------------------# Streak maintenance in _record_run_blocks_result-# ------------------------------------------------------------------------------def test_streak_increments_on_consecutive_scrape_walls() -> None:- ctx = _fresh_context()- _record_run_blocks_result(ctx, _scrape_wall_result())- assert ctx.probable_site_block_streak_count == 1- _record_run_blocks_result(ctx, _scrape_wall_result())- assert ctx.probable_site_block_streak_count == 2---def test_streak_holds_through_intermediate_nav_only_template_failure() -> None:- ctx = _fresh_context()- _record_run_blocks_result(ctx, _scrape_wall_result())- assert ctx.probable_site_block_streak_count == 1- nav_only_template_failure = {- "ok": False,- "data": {- "blocks": [- {- "block_type": "NAVIGATION",- "status": "failed",- "failure_reason": _SCRAPE_WALL_REASON,- },- ]- },- }- _record_run_blocks_result(ctx, nav_only_template_failure)- assert ctx.probable_site_block_streak_count == 2- _record_run_blocks_result(ctx, _scrape_wall_result())- assert ctx.probable_site_block_streak_count == 3---def test_streak_resets_on_real_success() -> None:- ctx = _fresh_context()- _record_run_blocks_result(ctx, _scrape_wall_result())- assert ctx.probable_site_block_streak_count == 1- success = {- "ok": True,- "data": {- "blocks": [- {- "block_type": "EXTRACTION",- "status": "completed",- "extracted_data": {"answer": "42"},- }- ]- },- }- _record_run_blocks_result(ctx, success)- assert ctx.probable_site_block_streak_count == 0---def test_streak_resets_on_failure_without_pattern() -> None:- ctx = _fresh_context()- _record_run_blocks_result(ctx, _scrape_wall_result())- assert ctx.probable_site_block_streak_count == 1- other_failure = {- "ok": False,- "data": {- "blocks": [- {"block_type": "GOTO_URL", "status": "completed"},- {- "block_type": "EXTRACTION",- "status": "failed",- "failure_reason": "Timeout waiting for selector #submit",- },- ]- },- }- _record_run_blocks_result(ctx, other_failure)- assert ctx.probable_site_block_streak_count == 0---def test_streak_stays_zero_when_navigation_itself_failed() -> None:- # Orthogonality contract: when the navigation block did not reach- # status=completed (e.g. non-retriable nav error — DNS, SSL, invalid URL),- # the scrape-wall detector must not count the run even if a later block- # also emitted the generic load-failure template. That case belongs to- # _detect_non_retriable_nav_error, not the probable-site-block streak.- ctx = _fresh_context()- nav_failed_with_wall_text = {- "ok": False,- "data": {- "blocks": [- {- "block_type": "GOTO_URL",- "status": "failed",- "failure_reason": (- "Failed to navigate to url https://x.invalid. Error message: net::ERR_NAME_NOT_RESOLVED"- ),- },- {- "block_type": "EXTRACTION",- "status": "failed",- "failure_reason": _SCRAPE_WALL_REASON,- },- ]- },- }- _record_run_blocks_result(ctx, nav_failed_with_wall_text)- assert ctx.probable_site_block_streak_count == 0---# ----------------------------------------------------------------------------# Enforcement gate-# -----------------------------------------------------------------------------[email protected]("streak", [0, 1])-def test_gate_does_not_fire_below_threshold(streak: int) -> None:- ctx = _fresh_context()- ctx.probable_site_block_streak_count = streak- assert not _needs_probable_site_block_stop_nudge(ctx)---def test_gate_fires_at_stop_threshold() -> None:- ctx = _fresh_context()- ctx.probable_site_block_streak_count = PROBABLE_SITE_BLOCK_STREAK_STOP_AT- assert _needs_probable_site_block_stop_nudge(ctx)---def test_gate_does_not_fire_after_cap_reached() -> None:- ctx = _fresh_context()- ctx.probable_site_block_streak_count = PROBABLE_SITE_BLOCK_STREAK_STOP_AT- ctx.probable_site_block_stop_nudge_count = MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES- assert not _needs_probable_site_block_stop_nudge(ctx)---def test_frontier_warn_defers_to_wall_when_both_apply() -> None:- ctx = _fresh_context()- ctx.repeated_failure_streak_count = REPEATED_FRONTIER_STREAK_ESCALATE_AT- ctx.probable_site_block_streak_count = PROBABLE_SITE_BLOCK_STREAK_STOP_AT- assert _repeated_frontier_failure_nudge(ctx) is None---def test_frontier_warn_still_fires_when_wall_below_threshold() -> None:- ctx = _fresh_context()- ctx.repeated_failure_streak_count = REPEATED_FRONTIER_STREAK_ESCALATE_AT- ctx.probable_site_block_streak_count = 1- assert _repeated_frontier_failure_nudge(ctx) is not None---def test_nudge_text_is_stop_oriented() -> None:- # Sanity-check the stop nudge tells the agent not to retry.- assert "STOP" in POST_PROBABLE_SITE_BLOCK_STOP_NUDGE- assert "Do NOT" in POST_PROBABLE_SITE_BLOCK_STOP_NUDGE---def test_stop_nudge_uses_different_proxy_advice_when_effective_proxy_is_active() -> None:- ctx = _fresh_context()- ctx.probable_site_block_streak_count = PROBABLE_SITE_BLOCK_STREAK_STOP_AT- ctx.effective_workflow_proxy_location = "RESIDENTIAL"-- with pytest.raises(CopilotTurnHalt) as exc_info:- _check_enforcement(ctx)-- halt = exc_info.value.halt- assert halt.kind == TurnHaltKind.PROBABLE_SITE_BLOCK- assert halt.blocker_signal is ctx.blocker_signal- user_facing = halt.blocker_signal.user_facing_reason- assert "configure a proxy" not in user_facing.lower()- assert "different proxy location" in user_facing.lower()- assert "US-CA" in user_facing- assert "US-NY" in user_facing- assert "residential/ISP" in user_facing---def test_stop_nudge_keeps_configure_proxy_advice_when_proxy_is_none() -> None:- ctx = _fresh_context()- ctx.probable_site_block_streak_count = PROBABLE_SITE_BLOCK_STREAK_STOP_AT- ctx.effective_workflow_proxy_location = "NONE"-- with pytest.raises(CopilotTurnHalt) as exc_info:- _check_enforcement(ctx)-- assert exc_info.value.halt.kind == TurnHaltKind.PROBABLE_SITE_BLOCK- assert "configure a proxy" in exc_info.value.halt.blocker_signal.user_facing_reason.lower()diff --git a/tests/unit/test_copilot_repeated_reply.py b/tests/unit/test_copilot_repeated_reply.pyindex 5877e0525..3b180d7c8 100644--- a/tests/unit/test_copilot_repeated_reply.py+++ b/tests/unit/test_copilot_repeated_reply.py@@ -18,11 +18,9 @@ from skyvern.forge.sdk.copilot.repeated_reply_summary import (from skyvern.forge.sdk.copilot.signature import compute_signaturefrom skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentReasonCodefrom skyvern.forge.sdk.copilot.turn_outcome import (- HANDOFF_REPLY,IDENTICAL_REPLY_BLOCKED_TERMINAL_REASON,apply_repeated_reply_guard,build_minimal_turn_outcome,- escalation_reply_for,)from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcomefrom skyvern.forge.sdk.schemas.workflow_copilot import (@@ -63,36 +61,6 @@ def test_guard_returns_original_when_no_match() -> None:assert outcome.reason_code == "ok"-def test_guard_rewrites_to_escalation_when_signature_matches() -> None:- text = "The file is in the Artifacts section."- sig = compute_signature(text)- final_text, outcome = apply_repeated_reply_guard(- final_text=text,- attempted_kind=ResponseKind.DIAGNOSE,- blocked_signatures=[sig],- )- assert final_text == escalation_reply_for(ResponseKind.DIAGNOSE)- assert outcome.response_kind is ResponseKind.RECOVER- assert outcome.terminal_reason == IDENTICAL_REPLY_BLOCKED_TERMINAL_REASON- assert sig in outcome.blocked_signatures- assert outcome.normalized_reply_signature == compute_signature(final_text)---def test_guard_falls_back_to_handoff_on_escalation_self_collision() -> None:- text = "The file is in the Artifacts section."- original_sig = compute_signature(text)- escalation_sig = compute_signature(escalation_reply_for(ResponseKind.DIAGNOSE))- final_text, outcome = apply_repeated_reply_guard(- final_text=text,- attempted_kind=ResponseKind.DIAGNOSE,- blocked_signatures=[original_sig, escalation_sig],- )- assert final_text == HANDOFF_REPLY- assert outcome.response_kind is ResponseKind.RECOVER- assert original_sig in outcome.blocked_signatures- assert escalation_sig in outcome.blocked_signatures--def test_guard_carries_inherited_bans_forward_on_no_match() -> None:final_text, outcome = apply_repeated_reply_guard(final_text="completely new reply",@@ -281,3 +249,33 @@ def test_render_prompt_block_only_populated_when_detected() -> None:empty = summarize_repeated_replies([], _intent(stuck=True))assert empty.render_prompt_block() == ""+++def test_repeated_reply_is_preserved_not_rewritten() -> None:+ # A repeat is the turn's true state: the model's words survive and no terminal reason is minted+ # from a chat-presentation concern.+ text = "Still working on the extraction."+ signature = compute_signature(text)++ final_text, outcome = apply_repeated_reply_guard(+ final_text=text,+ attempted_kind=ResponseKind.CLARIFY,+ blocked_signatures=[signature],+ )++ assert final_text == text+ assert outcome.terminal_reason != IDENTICAL_REPLY_BLOCKED_TERMINAL_REASON+ assert outcome.response_kind is not ResponseKind.RECOVER+++def test_repeated_reply_signature_still_recorded() -> None:+ text = "Still working on the extraction."+ signature = compute_signature(text)++ _, outcome = apply_repeated_reply_guard(+ final_text=text,+ attempted_kind=ResponseKind.CLARIFY,+ blocked_signatures=[signature],+ )++ assert signature in outcome.blocked_signaturesdiff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.pyindex ef9d37e96..a7807625b 100644--- a/tests/unit/test_copilot_run_outcome_event.py+++ b/tests/unit/test_copilot_run_outcome_event.py@@ -318,7 +318,8 @@ async def test_negative_adjudication_emits_outcome_not_demonstrated(monkeypatch:@pytest.mark.asyncio-async def test_mid_build_fall_through_still_emits_not_demonstrated(monkeypatch: pytest.MonkeyPatch) -> None:+async def test_judge_dissatisfaction_does_not_change_the_verdict(monkeypatch: pytest.MonkeyPatch) -> None:+ """The outcome derives from what the run produced; a judge re-reading the same run does not."""result = _clean_run_result()ctx = _ctx(result["data"]["blocks"])@@ -329,10 +330,8 @@ async def test_mid_build_fall_through_still_emits_not_demonstrated(monkeypatch:await _verify_and_record_run_blocks_result(ctx, result, time.monotonic())frames = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]- assert [frame.verdict for frame in frames] == ["evaluating", "not_demonstrated"]- assert frames[-1].reason_code == "outcome_not_demonstrated"+ assert [frame.verdict for frame in frames] == ["evaluating", "demonstrated"]assert ctx.last_test_suspicious_success is False- assert ctx.last_full_workflow_test_ok is False@pytest.mark.asyncio@@ -383,7 +382,7 @@ async def test_satisfied_adjudication_emits_demonstrated_with_unverified_prefix(@pytest.mark.asyncio-async def test_verification_skipped_emits_not_evaluated(monkeypatch: pytest.MonkeyPatch) -> None:+async def test_verification_skipped_still_emits_demonstrated(monkeypatch: pytest.MonkeyPatch) -> None:result = _clean_run_result()ctx = _ctx(result["data"]["blocks"])@@ -394,7 +393,7 @@ async def test_verification_skipped_emits_not_evaluated(monkeypatch: pytest.Monkawait _verify_and_record_run_blocks_result(ctx, result, time.monotonic())frames = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]- assert [frame.verdict for frame in frames] == ["evaluating", "not_evaluated"]+ assert [frame.verdict for frame in frames] == ["evaluating", "demonstrated"]assert ctx.last_test_suspicious_success is False@@ -490,7 +489,7 @@ def test_adjudication_keeps_committed_same_run_demonstrated_outcome() -> None:assert committed == RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_test")- adjudicated = _adjudicated_run_outcome(ctx, _evaluated(satisfied=False))+ adjudicated = _adjudicated_run_outcome(ctx)stashed = _stash_recorded_run_outcome(ctx,RecordedRunOutcome(@@ -510,9 +509,10 @@ def test_adjudication_does_not_keep_committed_outcome_for_new_run() -> None:ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_prior")ctx.last_run_blocks_workflow_run_id = "wr_test"- adjudicated = _adjudicated_run_outcome(ctx, _evaluated(satisfied=False))+ adjudicated = _adjudicated_run_outcome(ctx)- assert adjudicated.verdict == "not_demonstrated"+ assert adjudicated is not ctx.last_run_outcome+ assert adjudicated.workflow_run_id != "wr_prior"@pytest.mark.asyncio@@ -533,10 +533,8 @@ async def test_missing_run_id_does_not_emit_committed_demonstrated_outcome(await _verify_and_record_run_blocks_result(ctx, result, time.monotonic())frames = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]- assert [frame.verdict for frame in frames] == ["evaluating", "not_demonstrated"]+ assert [frame.verdict for frame in frames] == ["evaluating", "demonstrated"]assert frames[-1].workflow_run_id != "wr_test"- assert ctx.last_run_outcome is not None- assert ctx.last_run_outcome.verdict == "not_demonstrated"def test_both_consumers_route_through_single_producer() -> None:@@ -648,3 +646,61 @@ def test_trusted_terminal_challenge_category_requires_carrier() -> None:assert trusted_terminal_challenge_category_name(carried) == "ANTI_BOT_DETECTION"assert trusted_terminal_challenge_category_name(keyword) is Noneassert trusted_terminal_challenge_category_name(legacy) is None+++async def test_demonstrated_run_tells_the_model_it_may_conclude(monkeypatch: pytest.MonkeyPatch) -> None:+ """The record satisfying the request is a signal the model reads, not a stop the harness takes."""+ result = _clean_run_result()+ ctx = _ctx(result["data"]["blocks"])++ async def _stub_verification(*args: Any, **kwargs: Any) -> None:+ return None++ monkeypatch.setattr(run_execution, "_maybe_run_completion_verification", _stub_verification)+ await _verify_and_record_run_blocks_result(ctx, result, time.monotonic())++ assert "reply to the user now" in result["data"]["next_step"]+++async def test_failed_run_carries_no_conclude_signal() -> None:+ result = _run_result([], ok=False)+ ctx = _ctx()++ await _verify_and_record_run_blocks_result(ctx, result, time.monotonic())++ assert "next_step" not in result["data"]+++async def test_conclude_cue_rides_the_completion_verified_lane(monkeypatch: pytest.MonkeyPatch) -> None:+ # A run whose success arrives via completion verification (not the mechanical demonstrated+ # verdict) must still tell the model to conclude, or a verified success reads as unfinished+ # and the model re-runs it.+ result = _run_result([_code_block("extract_count", {"error_count": 138})])+ ctx = _ctx(result["data"]["blocks"])+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.run_execution.outcome_fully_verified", lambda _ctx: True)+ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.run_execution._record_run_blocks_result",+ lambda *_a, **_k: RecordedRunOutcome(verdict="progress_observed"),+ )++ await _verify_and_record_run_blocks_result(ctx, result, time.monotonic())++ assert "reply to the user now" in str(result["data"].get("next_step"))+++async def test_conclude_cue_absent_when_nothing_verified(monkeypatch: pytest.MonkeyPatch) -> None:+ result = _run_result([_code_block("extract_count", {})])+ ctx = _ctx(result["data"]["blocks"])+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.run_execution.outcome_fully_verified", lambda _ctx: False)+ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.run_execution._record_run_blocks_result",+ lambda *_a, **_k: RecordedRunOutcome(verdict="not_demonstrated"),+ )++ await _verify_and_record_run_blocks_result(ctx, result, time.monotonic())++ assert result["data"].get("next_step") is Nonediff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.pyindex 01e4d81dc..81698059e 100644--- a/tests/unit/test_copilot_run_output_content_gate.py+++ b/tests/unit/test_copilot_run_output_content_gate.py@@ -992,39 +992,6 @@ def test_satisfied_completion_prevents_empty_output_suspicious_success() -> Noneassert ctx.latest_recorded_build_test_outcome.structural_key is not None-def test_unverified_success_records_goal_relative_build_test_outcome() -> None:- result = _run_result([_code_block("search_registry_person", {"records": [], "result_count": 0})])- ctx = _ctx(result["data"]["blocks"])-- recorded = _record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))-- assert recorded is not None- assert recorded.verdict == "not_demonstrated"- assert recorded.reason_code == "outcome_not_demonstrated"- outcome = ctx.latest_recorded_build_test_outcome- assert outcome is not None- assert outcome.phase == "persisted_block_run"- assert outcome.verdict == "repairable_failure"- assert outcome.reason_code == "outcome_not_demonstrated"- assert outcome.structural_key is not None---def test_changed_output_structure_changes_unverified_success_structural_key() -> None:- first_result = _run_result([_code_block("search_registry_person", {"records": [], "result_count": 0})])- second_result = _run_result(- [_code_block("search_registry_person", {"records": [{"status_present": True}], "result_count": 1})]- )- ctx = _ctx(first_result["data"]["blocks"])-- _record_run_blocks_result(ctx, first_result, completion_verification=_no_evidence("c0"))- first_key = ctx.latest_recorded_build_test_outcome.structural_key- _record_run_blocks_result(ctx, second_result, completion_verification=_no_evidence("c0"))-- assert first_key is not None- assert ctx.latest_recorded_build_test_outcome.structural_key is not None- assert ctx.latest_recorded_build_test_outcome.structural_key != first_key--def test_terminal_blocker_does_not_leave_authoritative_prompt_outcome() -> None:result = _blocked_flag_run_result()ctx = _ctx(result["data"]["blocks"])@@ -1381,27 +1348,6 @@ def test_judge_no_evidence_keeps_building_without_metadata() -> None:assert "failure_reason" not in result["data"][email protected]("metadata_shape", ["none", "terminal", "prefix_only"])-def test_judge_unmet_on_detector_clean_run_does_not_reset_streaks_or_promote(metadata_shape: str) -> None:- result = _genuine_success_run_result()- ctx = _ctx(result["data"]["blocks"])- if metadata_shape == "terminal":- ctx.code_artifact_metadata = {"search_registry_person": _terminal_metadata_entry()}- elif metadata_shape == "prefix_only":- entry = _terminal_metadata_entry()- entry["completion_criteria"][0]["level"] = "prefix"- entry["completion_criteria"][0]["terminal"] = False- ctx.code_artifact_metadata = {"search_registry_person": entry}- ctx.failed_test_nudge_count = 2- ctx.probable_site_block_streak_count = 4-- _record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))-- assert ctx.failed_test_nudge_count == 2- assert ctx.probable_site_block_streak_count == 4- assert ctx.last_full_workflow_test_ok is False--def test_reached_goal_unfinished_run_is_recognized_when_verifier_fully_satisfied() -> None:result = _run_result([_code_block("submit_request", {"confirmation_number": "WTR-1842-DEMO"})], ok=False)ctx = _ctx(result["data"]["blocks"])diff --git a/tests/unit/test_copilot_scout_act_observe.py b/tests/unit/test_copilot_scout_act_observe.pyindex 134e306d8..dffef86e6 100644--- a/tests/unit/test_copilot_scout_act_observe.py+++ b/tests/unit/test_copilot_scout_act_observe.py@@ -34,7 +34,6 @@ from skyvern.forge.sdk.copilot.context import ()from skyvern.forge.sdk.copilot.enforcement import (_RECENT_TOOL_OUTPUT_CHAR_CAP,- MAX_NO_PROGRESS_INTERACTION_ATTEMPTS,record_scouted_output_coverage,)from skyvern.forge.sdk.copilot.output_extraction_plan import ShapeExpectation, ValueCardinality, ValueShape@@ -1368,26 +1367,6 @@ class TestNonAdvancingClickReperception:assert ctx.last_scout_act_observe_outcome == "attached"assert "actionable_targets" not in result["data"]- @pytest.mark.asyncio- async def test_ungroundable_churn_still_halts_at_max_without_reset(self) -> None:- ctx = AgentContext(- organization_id="o_1",- workflow_id="w_1",- workflow_permanent_id="wpid_1",- workflow_yaml="",- browser_session_id="pbs_1",- stream=MagicMock(),- )- ctx.discovery_mcp_server = _server_returning(_ungroundable_payload())-- for expected in range(1, MAX_NO_PROGRESS_INTERACTION_ATTEMPTS + 1):- result = await _run_click(ctx)- assert "actionable_targets" not in result["data"]- assert ctx.consecutive_no_progress_interaction_count == expected-- assert ctx.consecutive_no_progress_interaction_count == MAX_NO_PROGRESS_INTERACTION_ATTEMPTS- assert ctx.blocker_signal is not None-def _sequenced_evidence(parses: list[dict[str, Any] | None]) -> Any:calls = {"n": 0}diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.pyindex 20b702070..fbd512421 100644--- a/tests/unit/test_copilot_streaming_adapter.py+++ b/tests/unit/test_copilot_streaming_adapter.py@@ -609,10 +609,10 @@ async def test_stream_to_sse_cancels_and_stops_on_terminal_turn_halt() -> None:signal = CopilotToolBlockerSignal(blocker_kind="loop_detected",- agent_steering_text="LOOP DETECTED: 'update_workflow' has already failed 3 times.",+ agent_steering_text="LOOP DETECTED: 'update_workflow' has been called 3 times.",user_facing_reason="I retried without making progress. Tell me what to change and I'll try again.",recovery_hint="report_blocker_to_user",- internal_reason_code="loop_detected_repeated_failed_step",+ internal_reason_code="loop_detected_consecutive_same_tool",blocked_tool="update_workflow",)diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pyindex d36d22fcc..4ea167662 100644--- a/tests/unit/test_copilot_timeout_wip.py+++ b/tests/unit/test_copilot_timeout_wip.py@@ -463,7 +463,6 @@ class TestBuildUnexpectedErrorExitResult:"I built a 3-block draft and was still testing it when the turn ran out of time.")assert "the test failed" not in result.user_response- assert "draft workflow proposal" in result.user_responseassert "Do NOT" not in result.user_responseassert "block-running tools" not in result.user_responsediff --git a/tests/unit/test_copilot_tool_loop_error_block_running.py b/tests/unit/test_copilot_tool_loop_error_block_running.pyindex b85e69385..3e3265d93 100644--- a/tests/unit/test_copilot_tool_loop_error_block_running.py+++ b/tests/unit/test_copilot_tool_loop_error_block_running.py@@ -16,12 +16,9 @@ from typing import Anyfrom skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDSfrom skyvern.forge.sdk.copilot.failure_tracking import compute_action_sequence_fingerprintfrom skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctx-from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryStatefrom skyvern.forge.sdk.copilot.tools import (BLOCK_RUNNING_TOOLS,- COPILOT_FINAL_REPLY_RESERVE_SECONDS,PER_TOOL_CALL_BUDGET_SECONDS,- REPEATED_ACTION_STREAK_ABORT_AT,_active_block_run_budget_seconds,_tool_loop_error,)@@ -93,48 +90,12 @@ def test_compute_action_sequence_fingerprint_distinguishes_different_sequences()assert fp_b != fp_c-def test_tool_loop_error_fires_hard_abort_on_block_running_tools_when_streak_high() -> None:- ctx = _ctx(repeated_action_fingerprint_streak_count=REPEATED_ACTION_STREAK_ABORT_AT)-- error = _tool_loop_error(ctx, "run_blocks_and_collect_debug")- assert error is not None- assert "Repeated-action abort" in error-- error_update = _tool_loop_error(ctx, "update_and_run_blocks")- assert error_update is not None- assert "Repeated-action abort" in error_update---def test_tool_loop_error_does_not_fire_for_non_block_running_tools() -> None:- ctx = _ctx(repeated_action_fingerprint_streak_count=REPEATED_ACTION_STREAK_ABORT_AT + 5)-- # Planning/metadata tools keep their existing loop-detection behavior- # and are not affected by the block-run streak.- assert _tool_loop_error(ctx, "update_workflow") is None- assert _tool_loop_error(ctx, "list_credentials") is None- assert _tool_loop_error(ctx, "get_run_results") is None---def test_tool_loop_error_does_not_fire_below_threshold() -> None:- ctx = _ctx(repeated_action_fingerprint_streak_count=REPEATED_ACTION_STREAK_ABORT_AT - 1)- assert _tool_loop_error(ctx, "run_blocks_and_collect_debug") is None--def test_block_running_tool_is_not_blocked_by_name_only_streak() -> None:ctx = _ctx()for _ in range(5):assert _tool_loop_error(ctx, "update_and_run_blocks") is None-def test_planning_tool_still_trips_name_only_streak() -> None:- ctx = _ctx()- assert _tool_loop_error(ctx, "update_workflow") is None- assert _tool_loop_error(ctx, "update_workflow") is None- msg = _tool_loop_error(ctx, "update_workflow")- assert msg is not None- assert "LOOP DETECTED" in msg--def test_native_site_distinct_arguments_do_not_trip_streak() -> None:ctx = _ctx()for value in ("a", "b", "c", "d"):@@ -157,18 +118,6 @@ def test_unresolved_output_contract_ladder_skips_name_only_streak_for_authoring_assert _tool_loop_error(ctx, "update_workflow") is None-def test_resolved_output_contract_ladder_re_enables_name_only_streak() -> None:- ctx = _ctx(- output_contract_actuation_count_by_signature={"sig_done": 3},- output_contract_actuation_by_signature={"sig_done": OutputContractAdvisoryState.CONSUMED},- )- assert _tool_loop_error(ctx, "update_workflow") is None- assert _tool_loop_error(ctx, "update_workflow") is None- msg = _tool_loop_error(ctx, "update_workflow")- assert msg is not None- assert "LOOP DETECTED" in msg--def test_block_running_tool_is_blocked_by_pending_reconciliation() -> None:ctx = _ctx(pending_reconciliation_run_id="wr_123")msg = _tool_loop_error(ctx, "update_and_run_blocks")@@ -176,24 +125,6 @@ def test_block_running_tool_is_blocked_by_pending_reconciliation() -> None:assert "wr_123" in msg-def test_block_running_tool_blocks_late_retry_after_failed_workflow() -> None:- ctx = _ctx(- last_failed_workflow_yaml="version: '1.0'",- copilot_run_start_monotonic=time.monotonic()- - (TOTAL_TIMEOUT_SECONDS - COPILOT_FINAL_REPLY_RESERVE_SECONDS + 10),- )-- msg = _tool_loop_error(ctx, "update_and_run_blocks")-- assert msg is not None- assert ctx.blocker_signal.renders_final_reply is False- assert "less than 90 seconds" in msg.lower()- assert "Do NOT retry" in msg- assert "quick browser inspection tools" in msg- assert "answer from that observed page evidence" in msg- assert "Never repeat this tool-error text" in msg--def test_late_retry_guard_allows_latter_half_calls_when_reply_room_remains() -> None:ctx = _ctx(last_failed_workflow_yaml="version: '1.0'",@@ -203,23 +134,6 @@ def test_late_retry_guard_allows_latter_half_calls_when_reply_room_remains() ->assert _tool_loop_error(ctx, "run_blocks_and_collect_debug") is None-def test_block_running_tool_blocks_late_continuation_after_successful_prefix() -> None:- ctx = _ctx(- last_workflow_yaml="version: '1.0'",- last_test_ok=True,- copilot_run_start_monotonic=time.monotonic()- - (TOTAL_TIMEOUT_SECONDS - COPILOT_FINAL_REPLY_RESERVE_SECONDS + 10),- )-- msg = _tool_loop_error(ctx, "update_and_run_blocks")-- assert msg is not None- assert "less than 90 seconds" in msg.lower()- assert "Do NOT start another block-running tool call" in msg- assert "workflow draft and progress gathered so far" in msg- assert "not been verified end-to-end" in msg--def test_late_retry_guard_waits_until_budget_is_low() -> None:ctx = _ctx(last_failed_workflow_yaml="version: '1.0'",@@ -229,8 +143,9 @@ def test_late_retry_guard_waits_until_budget_is_low() -> None:assert _tool_loop_error(ctx, "update_and_run_blocks") is None-def test_active_block_run_budget_shrinks_near_deadline() -> None:- remaining = COPILOT_FINAL_REPLY_RESERVE_SECONDS + 30+def test_active_block_run_budget_is_bounded_by_the_outer_clock_alone() -> None:+ """No slice is reserved for a reply the model may not need: the run gets what is left."""+ remaining = 30ctx = _ctx(copilot_run_start_monotonic=time.monotonic() - (TOTAL_TIMEOUT_SECONDS - remaining))budget = _active_block_run_budget_seconds(ctx)@@ -238,7 +153,7 @@ def test_active_block_run_budget_shrinks_near_deadline() -> None:def test_active_block_run_budget_uses_full_budget_when_deadline_is_distant() -> None:- remaining = COPILOT_FINAL_REPLY_RESERVE_SECONDS + PER_TOOL_CALL_BUDGET_SECONDS + 30+ remaining = PER_TOOL_CALL_BUDGET_SECONDS + 30ctx = _ctx(copilot_run_start_monotonic=time.monotonic() - (TOTAL_TIMEOUT_SECONDS - remaining))assert _active_block_run_budget_seconds(ctx) == PER_TOOL_CALL_BUDGET_SECONDSdiff --git a/tests/unit/test_copilot_turn_halt.py b/tests/unit/test_copilot_turn_halt.pyindex a95c1be7a..9d326bf90 100644--- a/tests/unit/test_copilot_turn_halt.py+++ b/tests/unit/test_copilot_turn_halt.py@@ -5,7 +5,6 @@ from types import SimpleNamespaceimport pytestfrom skyvern.forge.sdk.copilot.blocker_signal import (- _OUTPUT_CONTRACT_TERMINAL_REASON_CODES,CopilotToolBlockerSignal,stash_blocker_signal,)@@ -15,7 +14,6 @@ from skyvern.forge.sdk.copilot.enforcement import (terminal_challenge_blocker_signal_from_current_page_evidence,terminal_challenge_blocker_signal_from_page_evidence,)-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODEfrom skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryStatefrom skyvern.forge.sdk.copilot.run_outcome import (TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,@@ -23,7 +21,6 @@ from skyvern.forge.sdk.copilot.run_outcome import (RecordedRunOutcome,)from skyvern.forge.sdk.copilot.turn_halt import (- ADVISORY_DISPATCH_STALLED_REASON_CODE,CopilotTurnHalt,TurnHaltKind,expire_output_contract_ladder_at_turn_end,@@ -38,7 +35,7 @@ from tests.unit.conftest import make_copilot_contextdef _signal(*,blocker_kind: str = "tool_error",- internal_reason_code: str = ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,+ internal_reason_code: str = TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,renders_final_reply: bool = True,extra: dict[str, object] | None = None,) -> CopilotToolBlockerSignal:@@ -60,7 +57,7 @@ def _signal(("signal", "expected_kind"),[(- _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_repeated_failed_step"),+ _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_consecutive_same_tool"),TurnHaltKind.LOOP_DETECTED,),(@@ -68,7 +65,7 @@ def _signal(TurnHaltKind.LOOP_DETECTED,),(- _signal(internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE),+ _signal(internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE),TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE,),(@@ -79,7 +76,6 @@ def _signal(_signal(internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE),TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE,),- (_signal(internal_reason_code="probable_site_block_stop"), TurnHaltKind.PROBABLE_SITE_BLOCK),],)def test_terminal_blockers_map_to_halts(signal: CopilotToolBlockerSignal, expected_kind: TurnHaltKind) -> None:@@ -90,16 +86,6 @@ def test_terminal_blockers_map_to_halts(signal: CopilotToolBlockerSignal, expectassert halt.blocker_signal is signal-def test_every_output_contract_terminal_reason_code_maps_to_a_turn_halt() -> None:- # Fail-closed guard: an output-contract terminal reason code with no turn_halt mapping mints no- # halt, so the turn degrades to loop_detected instead of terminating (the SKY-12594 e25ec5 regression).- assert _OUTPUT_CONTRACT_TERMINAL_REASON_CODES- for reason_code in _OUTPUT_CONTRACT_TERMINAL_REASON_CODES:- halt = turn_halt_from_blocker_signal(_signal(internal_reason_code=reason_code), source="hook")- assert halt is not None, f"{reason_code} maps to no TurnHalt; the turn would degrade to loop_detected"- assert halt.kind is TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE--def _halt_ctx(**overrides: object) -> SimpleNamespace:fields: dict[str, object] = {"turn_halt": None,@@ -119,7 +105,7 @@ def _halt_ctx(**overrides: object) -> SimpleNamespace:def test_stash_and_raise_turn_halt_sets_context_once() -> None:ctx = _halt_ctx()- signal = _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_repeated_failed_step")+ signal = _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_consecutive_same_tool")halt = stash_turn_halt_from_blocker_signal(ctx, signal, source="stream")@@ -387,7 +373,7 @@ def test_verified_outcome_suppresses_and_consumes_involuntary_halt() -> None:def test_verified_outcome_consumes_loop_blocker_signal() -> None:- signal = _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_repeated_failed_step")+ signal = _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_consecutive_same_tool")ctx = _consume_ctx(blocker_signal=signal)stash_turn_halt_from_blocker_signal(ctx, signal, source="hook")@@ -398,7 +384,7 @@ def test_verified_outcome_consumes_loop_blocker_signal() -> None:def test_verified_outcome_does_not_suppress_voluntary_terminal_challenge() -> None:- signal = _signal(internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE)+ signal = _signal(internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE)ctx = _consume_ctx(blocker_signal=signal)stash_turn_halt_from_blocker_signal(ctx, signal, source="hook")@@ -410,10 +396,10 @@ def test_verified_outcome_does_not_suppress_voluntary_terminal_challenge() -> Nodef test_verified_outcome_does_not_clear_voluntary_blocker_when_involuntary_absent() -> None:- challenge_signal = _signal(internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE)+ challenge_signal = _signal(internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE)loop_halt = stash_turn_halt_from_blocker_signal(_halt_ctx(),- _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_repeated_failed_step"),+ _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_consecutive_same_tool"),source="hook",)ctx = _consume_ctx(@@ -432,8 +418,8 @@ def test_verified_outcome_does_not_clear_voluntary_blocker_when_involuntary_absedef test_verified_outcome_consumes_involuntary_tool_blocker_history() -> None:- involuntary = _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_repeated_failed_step")- voluntary = _signal(internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE)+ involuntary = _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_consecutive_same_tool")+ voluntary = _signal(internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE)ctx = _consume_ctx(blocker_signal=involuntary)ctx.latest_tool_blocker_signal = involuntaryctx.tool_blocker_signals = [voluntary, involuntary]@@ -454,7 +440,7 @@ def test_involuntary_suppression_lets_later_voluntary_challenge_raise() -> None:assert ctx.turn_halt is Noneassert ctx.blocker_signal is None- challenge_signal = _signal(internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE)+ challenge_signal = _signal(internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE)ctx.blocker_signal = challenge_signalstash_turn_halt_from_blocker_signal(ctx, challenge_signal, source="hook")@@ -509,19 +495,9 @@ def test_loop_detected_promotes_once_output_contract_ladder_resolves() -> None:assert halt.kind == TurnHaltKind.LOOP_DETECTED-def test_output_source_unobservable_terminal_promotes_even_while_ladder_unresolved() -> None:- ctx = _output_contract_ctx(granted=True)- signal = _signal(blocker_kind="tool_error", internal_reason_code="output_source_unobservable")-- halt = stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")-- assert halt is not None- assert halt.kind == TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE--def test_active_terminal_challenge_promotes_while_ladder_unresolved() -> None:ctx = _output_contract_ctx(granted=True)- signal = _signal(internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE)+ signal = _signal(internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE)halt = stash_turn_halt_from_blocker_signal(ctx, signal, source="run_execution")@@ -567,15 +543,6 @@ def test_defer_never_expires_granted_grant_awaiting_forced_dispatch() -> None:assert ctx.output_contract_actuation_by_signature["sig_a"] == OutputContractAdvisoryState.GRANTED-def test_defer_expires_countonly_ladder_into_typed_terminal() -> None:- ctx = _defer_count_ledger_ctx()- stash_turn_halt_from_blocker_signal(ctx, _loop_signal(), source="enforcement_backstop")- stash_turn_halt_from_blocker_signal(ctx, _loop_signal(), source="enforcement_backstop")- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind == TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE- assert ctx.output_contract_actuation_by_signature["sig_a"] == OutputContractAdvisoryState.EXPIRED--def test_defer_re_arms_when_lifecycle_advances() -> None:ctx = _defer_ledger_ctx()stash_turn_halt_from_blocker_signal(ctx, _loop_signal(), source="enforcement_backstop")@@ -594,22 +561,6 @@ def test_defer_countonly_ladder_reaches_a_terminal_within_two_stalled_signals()assert ctx.turn_halt is not None-def test_turn_end_expiry_transitions_granted_and_emits_stalled_terminal() -> None:- ctx = make_copilot_context()- ctx.output_contract_actuation_by_signature["sig_a"] = OutputContractAdvisoryState.GRANTED- ctx.output_contract_pending_run_evidence["sig_a"] = ["output.confirmation_number"]-- halt = expire_output_contract_ladder_at_turn_end(ctx)-- assert halt is not None- assert halt.kind == TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE- assert ctx.turn_halt is halt- assert ctx.output_contract_actuation_by_signature["sig_a"] == OutputContractAdvisoryState.EXPIRED- assert ctx.blocker_signal is not None- assert ctx.blocker_signal.internal_reason_code == ADVISORY_DISPATCH_STALLED_REASON_CODE- assert ctx.blocker_signal.renders_final_reply is True--def test_turn_end_expiry_noop_without_live_grant() -> None:ctx = make_copilot_context()ctx.output_contract_actuation_by_signature["sig_a"] = OutputContractAdvisoryState.CONSUMED@@ -619,18 +570,6 @@ def test_turn_end_expiry_noop_without_live_grant() -> None:assert ctx.output_contract_actuation_by_signature["sig_a"] == OutputContractAdvisoryState.CONSUMED-def test_turn_end_expiry_keeps_existing_halt_but_expires_grant() -> None:- ctx = make_copilot_context()- terminal_signal = _signal(internal_reason_code="probable_site_block_stop")- existing = stash_turn_halt_from_blocker_signal(ctx, terminal_signal, source="test")- assert existing is not None- ctx.output_contract_actuation_by_signature["sig_a"] = OutputContractAdvisoryState.GRANTED-- assert expire_output_contract_ladder_at_turn_end(ctx) is None- assert ctx.turn_halt is existing- assert ctx.output_contract_actuation_by_signature["sig_a"] == OutputContractAdvisoryState.EXPIRED--def test_mid_turn_expiry_keeps_granted_early_return() -> None:ctx = make_copilot_context()ctx.output_contract_actuation_by_signature["sig_a"] = OutputContractAdvisoryState.GRANTED@@ -641,16 +580,6 @@ def test_mid_turn_expiry_keeps_granted_early_return() -> None:assert ctx.output_contract_actuation_by_signature["sig_a"] == OutputContractAdvisoryState.GRANTED-def test_terminal_stash_choke_point_claims_genuinely_terminal() -> None:- ctx = make_copilot_context()- signal = _signal(internal_reason_code="probable_site_block_stop")-- stash_turn_halt_from_blocker_signal(ctx, signal, source="test")-- assert ctx.turn_ownership is not None- assert TurnClaimant.GENUINELY_TERMINAL in ctx.turn_ownership.claims--def test_raise_if_turn_halt_retires_halt_outranked_by_live_ladder() -> None:ctx = make_copilot_context()loop_signal = _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_generic")@@ -663,13 +592,3 @@ def test_raise_if_turn_halt_retires_halt_outranked_by_live_ladder() -> None:assert ctx.turn_halt is Noneassert any(event.site == "turn_halt" for event in ctx.gate_precedence_conflict_events)---def test_raise_if_turn_halt_still_raises_genuinely_terminal_over_live_ladder() -> None:- ctx = make_copilot_context()- terminal_signal = _signal(internal_reason_code="probable_site_block_stop")- stash_turn_halt_from_blocker_signal(ctx, terminal_signal, source="test")- ctx.output_contract_actuation_by_signature["sig_a"] = OutputContractAdvisoryState.GRANTED-- with pytest.raises(CopilotTurnHalt):- raise_if_turn_halt(ctx)diff --git a/tests/unit/test_copilot_turn_intent_tool_gate.py b/tests/unit/test_copilot_turn_intent_tool_gate.pyindex bb8dc3ff7..64be05583 100644--- a/tests/unit/test_copilot_turn_intent_tool_gate.py+++ b/tests/unit/test_copilot_turn_intent_tool_gate.py@@ -501,45 +501,6 @@ def test_runtime_self_heal_origin_blocks_native_tool_calls() -> None:assert ctx.blocker_signal.blocker_kind == "tool_error"- "mode",- [- pytest.param(TurnIntentMode.UNKNOWN, id="unknown_without_run_context"),- pytest.param(TurnIntentMode.DOCS_ANSWER, id="docs_answer"),- ],-)-def test_no_run_context_mode_blocks_get_run_results(mode: TurnIntentMode) -> None:- intent = TurnIntent(- mode=mode,- authority=TurnIntentAuthority(),- )-- signal = _turn_intent_tool_error(_ctx(intent), "get_run_results")- _assert_signal(- signal,- internal_reason_code="turn_intent_context_read_blocked",- classifier_mode=mode.value,- blocked_tool="get_run_results",- )---def test_docs_answer_blocks_get_run_results_even_with_read_flag() -> None:- intent = TurnIntent(- mode=TurnIntentMode.DOCS_ANSWER,- authority=TurnIntentAuthority(may_read_run_context=True),- )-- signal = _turn_intent_tool_error(_ctx(intent), "get_run_results")- _assert_signal(- signal,- internal_reason_code="turn_intent_context_read_blocked",- classifier_mode="docs_answer",- blocked_tool="get_run_results",- )- filtered = _native_tools_for_turn(list(NATIVE_TOOLS), intent)- assert {getattr(tool, "name", None) for tool in filtered} == {tool.name for tool in NATIVE_TOOLS}--@pytest.mark.parametrize(("mode", "authority_kwargs", "ctx_kwargs"),[@@ -578,40 +539,6 @@ def test_within_turn_override_allows_read(assert _turn_intent_tool_error(ctx, "get_run_results") is None-def test_within_turn_override_excluded_for_docs_answer() -> None:- intent = TurnIntent(- mode=TurnIntentMode.DOCS_ANSWER,- authority=TurnIntentAuthority(),- )-- ctx = _ctx(intent, pending_reconciliation_run_id="wr_pending_test")- signal = _turn_intent_tool_error(ctx, "get_run_results")- _assert_signal(- signal,- internal_reason_code="turn_intent_context_read_blocked",- classifier_mode="docs_answer",- blocked_tool="get_run_results",- )---def test_tool_activity_is_not_a_substitute_for_pending_reconciliation_run_id() -> None:- # tool_activity entries are appended for every completed tool call,- # including ones that failed the authority/loop gate before the run ever- # started. The override must key only on pending_reconciliation_run_id,- # which the watchdog sets only when a real run exited unfinalized.- intent = TurnIntent(- mode=TurnIntentMode.UNKNOWN,- authority=TurnIntentAuthority(),- )-- ctx = _ctx(- intent,- tool_activity=[{"tool": "run_blocks_and_collect_debug", "summary": "Failed: blocked"}],- )-- assert _turn_intent_tool_error(ctx, "get_run_results") is not None--@pytest.mark.asyncioasync def test_get_run_results_defaults_to_successful_same_turn_run() -> None:ctx = _ctx(diff --git a/tests/unit/test_copilot_turn_outcome.py b/tests/unit/test_copilot_turn_outcome.pyindex 3f841698a..132f53809 100644--- a/tests/unit/test_copilot_turn_outcome.py+++ b/tests/unit/test_copilot_turn_outcome.py@@ -8,7 +8,6 @@ from skyvern.forge.sdk.copilot.turn_outcome import (build_minimal_turn_outcome,build_turn_outcome,derive_response_kind,- escalation_reply_for,)from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome@@ -73,14 +72,3 @@ def test_turn_outcome_json_round_trip() -> None:payload = outcome.model_dump(mode="json")restored = TurnOutcome.model_validate(payload)assert restored == outcome---def test_escalation_reply_for_covers_derivable_kinds() -> None:- for kind in (ResponseKind.BUILD, ResponseKind.CLARIFY, ResponseKind.DIAGNOSE, ResponseKind.REFUSE):- text = escalation_reply_for(kind)- assert isinstance(text, str)- assert text---def test_escalation_reply_for_recover_falls_back_to_clarify_text() -> None:- assert escalation_reply_for(ResponseKind.RECOVER) == escalation_reply_for(ResponseKind.CLARIFY)diff --git a/tests/unit/test_copilot_turn_ownership.py b/tests/unit/test_copilot_turn_ownership.pyindex daebb221e..e4ad8ed7b 100644--- a/tests/unit/test_copilot_turn_ownership.py+++ b/tests/unit/test_copilot_turn_ownership.py@@ -1,7 +1,5 @@from __future__ import annotations-import pytest-from skyvern.forge.sdk.copilot.blocker_signal import (GENUINELY_TERMINAL_BLOCKER_REASON_CODES,SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,@@ -9,14 +7,8 @@ from skyvern.forge.sdk.copilot.blocker_signal import (stash_blocker_signal,)from skyvern.forge.sdk.copilot.context import CopilotContext-from skyvern.forge.sdk.copilot.enforcement import (- MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,- _record_code_authoring_guardrail_reject,-)from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryStatefrom skyvern.forge.sdk.copilot.turn_halt import (- CopilotTurnHalt,- raise_if_turn_halt,retire_outranked_turn_halt,stash_turn_halt_from_blocker_signal,)@@ -28,10 +20,8 @@ from skyvern.forge.sdk.copilot.turn_ownership import (blocker_signal_render_allowed,claim_and_stash_blocker_signal,claim_turn,- claimant_for_blocker_signal,current_turn_owner,emit_blocker_signal_payload,- release_turn_claim,)from tests.unit.conftest import make_copilot_context@@ -138,20 +128,6 @@ def test_genuinely_terminal_family_is_the_shared_blocker_signal_set() -> None:assert CLAIMANT_REASON_CODE_FAMILIES[TurnClaimant.GENUINELY_TERMINAL] is GENUINELY_TERMINAL_BLOCKER_REASON_CODES-def test_claimant_table_resolves_signals_for_both_cascades() -> None:- assert claimant_for_blocker_signal(_churn_signal()) is TurnClaimant.CODE_AUTHORING_CHURN- assert (- claimant_for_blocker_signal(_signal("credential_priority_authoring_churn"))- is TurnClaimant.CREDENTIAL_PRIORITY_CHURN- )- assert claimant_for_blocker_signal(_signal("loop_detected_generic")) is TurnClaimant.LOOP_DETECTED- assert (- claimant_for_blocker_signal(_signal("probable_site_block_stop", blocker_kind="tool_error"))- is TurnClaimant.GENUINELY_TERMINAL- )- assert claimant_for_blocker_signal(_signal("tool_error_repeated_action_abort", blocker_kind="tool_error")) is None--def test_same_claimant_reclaim_is_first_wins_and_widens_metadata() -> None:ctx = make_copilot_context()assert claim_turn(ctx, TurnClaimant.CODE_AUTHORING_CHURN) is ClaimOutcome.OWNED@@ -175,27 +151,6 @@ def test_stale_owner_releases_when_ladder_resolves() -> None:assert owner.claimant is TurnClaimant.CODE_AUTHORING_CHURN-def test_stronger_owned_claim_replaces_held_signal() -> None:- ctx = make_copilot_context()- loop_signal = _signal("loop_detected_generic")- assert claim_and_stash_blocker_signal(ctx, TurnClaimant.LOOP_DETECTED, loop_signal) is not None- assert ctx.blocker_signal is loop_signal-- terminal_signal = _signal("probable_site_block_stop", blocker_kind="tool_error")- assert claim_and_stash_blocker_signal(ctx, TurnClaimant.GENUINELY_TERMINAL, terminal_signal) is not None- assert ctx.blocker_signal is terminal_signal---def test_genuinely_terminal_held_signal_is_never_replaced() -> None:- ctx = make_copilot_context()- terminal_signal = _signal("probable_site_block_stop", blocker_kind="tool_error")- assert claim_and_stash_blocker_signal(ctx, TurnClaimant.GENUINELY_TERMINAL, terminal_signal) is not None-- churn_payload = claim_and_stash_blocker_signal(ctx, TurnClaimant.CODE_AUTHORING_CHURN, _churn_signal())- assert churn_payload is None- assert ctx.blocker_signal is terminal_signal--def test_plain_stash_replacement_clears_stale_association() -> None:ctx = make_copilot_context()loop_signal = _signal("loop_detected_generic")@@ -230,37 +185,6 @@ def test_signal_only_stronger_claim_retires_outranked_halt_on_consult() -> None:)-def test_retirement_restores_owner_halt_by_re_emission() -> None:- ctx = make_copilot_context()- terminal_signal = _signal("probable_site_block_stop", blocker_kind="tool_error")- assert claim_and_stash_blocker_signal(ctx, TurnClaimant.GENUINELY_TERMINAL, terminal_signal) is not None-- loop_signal = _signal("loop_detected_generic")- loop_halt = stash_turn_halt_from_blocker_signal(ctx, loop_signal, source="test")- assert loop_halt is not None- assert ctx.turn_halt is loop_halt-- assert retire_outranked_turn_halt(ctx) is True- restored = ctx.turn_halt- assert restored is not None- assert restored is not loop_halt- assert restored.blocker_signal is terminal_signal---def test_genuinely_terminal_halt_is_never_retired() -> None:- ctx = make_copilot_context()- terminal_signal = _signal("probable_site_block_stop", blocker_kind="tool_error")- assert claim_and_stash_blocker_signal(ctx, TurnClaimant.GENUINELY_TERMINAL, terminal_signal) is not None- stash_turn_halt_from_blocker_signal(ctx, terminal_signal, source="test")- halt = ctx.turn_halt- assert halt is not None-- _grant_ladder(ctx)- claim_turn(ctx, TurnClaimant.OUTPUT_CONTRACT_ACTUATION)- assert retire_outranked_turn_halt(ctx) is False- assert ctx.turn_halt is halt--def test_render_denied_while_ladder_owns_and_restores_on_resolution() -> None:ctx = make_copilot_context()churn = _churn_signal()@@ -312,7 +236,7 @@ def test_unclaimed_signal_fails_open_to_render_while_owner_live() -> None:def test_transient_claim_never_outlives_the_claiming_call() -> None:ctx = make_copilot_context()- assert claim_turn(ctx, TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION) is ClaimOutcome.OWNED+ assert claim_turn(ctx, TurnClaimant.CREDENTIAL_SCOUT_REOPEN) is ClaimOutcome.OWNEDassert current_turn_owner(ctx) is Nonechurn = _churn_signal()@@ -323,8 +247,8 @@ def test_transient_claim_never_outlives_the_claiming_call() -> None:def test_transient_claim_yields_to_live_ladder_and_records_conflict() -> None:ctx = make_copilot_context()_grant_ladder(ctx)- assert claim_turn(ctx, TurnClaimant.POST_RUN_PAGE_PATH_INTERACTION) is ClaimOutcome.YIELDED- assert _conflict_fingerprints(ctx) == ["output_contract_actuation>post_run_page_path_interaction"]+ assert claim_turn(ctx, TurnClaimant.CREDENTIAL_SCOUT_REOPEN) is ClaimOutcome.YIELDED+ assert _conflict_fingerprints(ctx) == ["output_contract_actuation>credential_scout_reopen"]def test_carve_out_claimants_are_transient() -> None:@@ -383,17 +307,6 @@ def test_state_backed_ladder_owns_without_explicit_claim_and_query_does_not_regiassert ctx.turn_ownership is None-def test_release_turn_claim_removes_registered_claim() -> None:- ctx = make_copilot_context()- terminal_signal = _signal("probable_site_block_stop", blocker_kind="tool_error")- assert claim_and_stash_blocker_signal(ctx, TurnClaimant.GENUINELY_TERMINAL, terminal_signal) is not None- assert ctx.turn_ownership is not None- assert TurnClaimant.GENUINELY_TERMINAL in ctx.turn_ownership.claims-- release_turn_claim(ctx, TurnClaimant.GENUINELY_TERMINAL)- assert TurnClaimant.GENUINELY_TERMINAL not in ctx.turn_ownership.claims--def test_conflict_events_are_capped() -> None:ctx = make_copilot_context()_grant_ladder(ctx)@@ -408,32 +321,6 @@ def test_fresh_context_has_no_owner() -> None:assert ctx.gate_precedence_conflict_events == []-def test_exhausted_churn_owns_over_held_synthesized_force_and_halts() -> None:- ctx = _synthesized_replay_context()- assert (- claim_and_stash_blocker_signal(- ctx, TurnClaimant.SYNTHESIZED_BLOCK_PERSISTENCE_FORCE, _synthesized_force_signal()- )- is not None- )-- for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):- _record_code_authoring_guardrail_reject(ctx)-- owner = current_turn_owner(ctx)- assert owner is not None- assert owner.claimant is TurnClaimant.CODE_AUTHORING_CHURN- assert ctx.blocker_signal is not None- assert ctx.blocker_signal.internal_reason_code == "code_authoring_guardrail_churn"- assert blocker_signal_render_allowed(ctx, ctx.blocker_signal) is True- assert "code_authoring_guardrail_churn>synthesized_block_persistence_force" in _conflict_fingerprints(ctx)- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS-- stash_turn_halt_from_blocker_signal(ctx, ctx.blocker_signal, source="enforcement_backstop")- with pytest.raises(CopilotTurnHalt):- raise_if_turn_halt(ctx)--def test_standard_path_churn_precedence_unchanged() -> None:ctx = make_copilot_context()assert (diff --git a/tests/unit/test_copilot_watchdog.py b/tests/unit/test_copilot_watchdog.pyindex ddeb300eb..783c7232f 100644--- a/tests/unit/test_copilot_watchdog.py+++ b/tests/unit/test_copilot_watchdog.py@@ -429,6 +429,13 @@ def test_any_quiet_block_requested_file_download() -> None:assert _any_quiet_block_requested(ctx, ["download_file"]) is True+def test_any_quiet_block_requested_code() -> None:+ """A code block writes its row on entry and exit and nothing between, so a login or a long+ wait inside one reads as no progress at all and the watchdog cancels a healthy run."""+ ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("code", "login_and_extract")))+ assert _any_quiet_block_requested(ctx, ["login_and_extract"]) is True++def test_any_quiet_block_requested_mixed_requested_labels_match_quiet_one() -> None:"""When multiple blocks are requested, having any one quiet type isenough to disable stagnation for the whole invocation."""diff --git a/tests/unit/test_credentials_totp_route_logging.py b/tests/unit/test_credentials_totp_route_logging.pyindex dea0311b3..1d935e661 100644--- a/tests/unit/test_credentials_totp_route_logging.py+++ b/tests/unit/test_credentials_totp_route_logging.py@@ -88,10 +88,110 @@ async def test_send_totp_code_parse_failure_log_redacts_identifier_and_content(assert error_log["totp_identifier"] == "[REDACTED_OTP_IDENTIFIER]"assert error_log["content_length"] == len(raw_content)assert "content" not in error_log+ assert all(record.get("event") != "Skipping OTP parse; organization has insufficient credits" for record in logs)_assert_raw_values_not_logged(logs, raw_identifier, raw_content, "135790")-def test_send_totp_code_openapi_declares_distinct_200_and_202_schemas() -> None:+ ("raw_fallback_enabled", "raw_content_max_length"),+ [+ (False, 1_000),+ (True, 10),+ ],+)+async def test_send_totp_code_insufficient_credits_returns_402_when_raw_fallback_is_unavailable(+ monkeypatch: pytest.MonkeyPatch,+ raw_fallback_enabled: bool,+ raw_content_max_length: int,+) -> None:+ raw_identifier = "[email protected]"+ raw_content = "Long relayed message without a verification code."+ create_otp_code = AsyncMock()+ create_raw_otp_code = AsyncMock()+ llm_handler = AsyncMock()+ charge = AsyncMock()+ monkeypatch.setattr(+ credentials.app,+ "DATABASE",+ _database_with_otp_create(create_otp_code, create_raw_otp_code),+ )+ monkeypatch.setattr(+ credentials.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=False),+ charge_for_otp_parse=charge,+ ),+ )+ monkeypatch.setattr(credentials.app, "SECONDARY_LLM_API_HANDLER", llm_handler)+ monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", raw_fallback_enabled)+ monkeypatch.setattr(credentials.settings, "TOTP_RAW_CONTENT_MAX_LENGTH", raw_content_max_length)++ with structlog.testing.capture_logs() as logs:+ with pytest.raises(HTTPException) as exc_info:+ await credentials.send_totp_code(+ TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),+ curr_org=SimpleNamespace(organization_id="o_test"),+ )++ assert exc_info.value.status_code == 402+ assert exc_info.value.detail == "Insufficient credits to parse OTP content"+ assert exc_info.value.__cause__ is None+ assert exc_info.value.__context__ is None+ assert all(record.get("event") != "Failed to parse otp login" for record in logs)+ assert any(record.get("event") == "Skipping OTP parse; organization has insufficient credits" for record in logs)+ _assert_raw_values_not_logged(logs, raw_identifier, raw_content)+ llm_handler.assert_not_awaited()+ charge.assert_not_awaited()+ create_otp_code.assert_not_awaited()+ create_raw_otp_code.assert_not_awaited()+++async def test_send_totp_code_insufficient_credits_preserves_enabled_raw_fallback(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ raw_identifier = "[email protected]"+ raw_content = "Long relayed message without a verification code."+ create_otp_code = AsyncMock()+ create_raw_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_raw"))+ llm_handler = AsyncMock()+ charge = AsyncMock()+ monkeypatch.setattr(+ credentials.app,+ "DATABASE",+ _database_with_otp_create(create_otp_code, create_raw_otp_code),+ )+ monkeypatch.setattr(+ credentials.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=False),+ charge_for_otp_parse=charge,+ ),+ )+ monkeypatch.setattr(credentials.app, "SECONDARY_LLM_API_HANDLER", llm_handler)+ monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", True)++ with structlog.testing.capture_logs() as logs:+ response = await credentials.send_totp_code(+ TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),+ curr_org=SimpleNamespace(organization_id="o_test"),+ )++ assert response.status_code == 202+ assert json.loads(response.body) == {"totp_code_id": "otp_raw", "status": "raw_pending"}+ assert all(record.get("event") != "Failed to parse otp login" for record in logs)+ assert any(record.get("event") == "Skipping OTP parse; organization has insufficient credits" for record in logs)+ _assert_raw_values_not_logged(logs, raw_identifier, raw_content)+ llm_handler.assert_not_awaited()+ charge.assert_not_awaited()+ create_otp_code.assert_not_awaited()+ create_raw_otp_code.assert_awaited_once()+++def test_send_totp_code_openapi_declares_distinct_success_and_credit_responses() -> None:fastapi_app = FastAPI()fastapi_app.include_router(base_router, prefix="/v1")fastapi_app.include_router(legacy_base_router, prefix="/api/v1")@@ -99,9 +199,11 @@ def test_send_totp_code_openapi_declares_distinct_200_and_202_schemas() -> None:responses = fastapi_app.openapi()["paths"]["/v1/credentials/totp"]["post"]["responses"]assert responses["200"]["content"]["application/json"]["schema"]["$ref"].endswith("/TOTPCode")assert responses["202"]["content"]["application/json"]["schema"]["$ref"].endswith("/RawTOTPCodeAccepted")+ assert responses["402"]["description"] == "Insufficient credits to parse OTP content"legacy_route = next(route for route in legacy_base_router.routes if route.path == "/totp")assert legacy_route.responses[202]["model"].__name__ == "RawTOTPCodeAccepted"+ assert legacy_route.responses[402]["description"] == "Insufficient credits to parse OTP content"@pytest.mark.asynciodiff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.pyindex f2efa24f7..f33de9074 100644--- a/tests/unit/test_download_file_action_handler.py+++ b/tests/unit/test_download_file_action_handler.py@@ -33,11 +33,11 @@ from skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptorfrom skyvern.webeye.scraper.scraped_page import ScrapedPagefrom tests.unit.helpers import make_organization, make_step, make_task-# One second is only a test-side runaway guard. The behavior under test is+# Five seconds is only a test-side runaway guard. The behavior under test is# asserted through configured timeout values, cleanup, and span attributes-# below; it tolerates the hundreds of ms a loaded CI runner can delay a+# below; it tolerates the scheduling delay a loaded CI runner can add to a# completed coroutine without turning scheduling latency into a product failure.-CI_TEST_RUNAWAY_TIMEOUT_SECONDS = 1.0+CI_TEST_RUNAWAY_TIMEOUT_SECONDS = 5.0class _EventEmitter:diff --git a/tests/unit/test_evaluate_actionable_target.py b/tests/unit/test_evaluate_actionable_target.pyindex c10805bfe..7e073167e 100644--- a/tests/unit/test_evaluate_actionable_target.py+++ b/tests/unit/test_evaluate_actionable_target.py@@ -597,54 +597,6 @@ def test_identities_include_clickable_controls() -> None:assert ("", "Residential") in identities-def test_identities_order_affordances_before_fields_and_selectors_first() -> None:- packet = {- "forms": [- {- "fields": [{"selector": "#user", "text": "Username"}],- "submit_controls": [{"selector": "#login", "text": "Log In"}],- }- ],- "navigation_targets": [],- "result_containers": [],- "clickable_controls": [- {"text": "Text only tile"},- {"selector": "#biz-tile", "text": "Business"},- ],- }- identities = scouting._actionable_target_identities(packet)- positions = {ident: index for index, ident in enumerate(identities)}- field_pos = positions[("#user", "Username")]- # Click affordances (selector-bearing and text-only) precede the plain input field.- assert positions[("#login", "Log In")] < field_pos- assert positions[("#biz-tile", "Business")] < field_pos- assert positions[("", "Text only tile")] < field_pos- # Selector-bearing affordances precede text-only ones.- assert positions[("#biz-tile", "Business")] < positions[("", "Text only tile")]---def test_click_affordance_identities_only_selector_bearing_affordances() -> None:- packet = {- "forms": [- {- "fields": [{"selector": "#user", "text": "Username"}],- "submit_controls": [{"selector": "#login", "text": "Log In"}],- }- ],- "navigation_targets": [{"selector": "a.nav", "text": "Nav"}],- "result_containers": [{"selector": "#results", "text": "Results"}],- "clickable_controls": [- {"selector": "#tile", "text": "Tile"},- {"text": "Text only"},- ],- "modal_overlays": [{"dismiss_controls": [{"selector": "#close", "text": "Close"}]}],- }- identities = scouting._click_affordance_target_identities(packet)- selectors = {selector for selector, _ in identities}- assert selectors == {"#login", "a.nav", "#tile", "#close"}- assert all(selector for selector, _ in identities)--@pytest.mark.asyncioasync def test_probe_none_leaves_data_untouched_and_resets(monkeypatch) -> None:ctx = _ctx()diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.pyindex 7e30a5c36..10224a568 100644--- a/tests/unit/test_failure_classifier.py+++ b/tests/unit/test_failure_classifier.py@@ -343,3 +343,70 @@ def test_exception_name_is_ignored_when_instance_provided() -> None:def test_exception_name_alone_returns_none_when_unrecognized() -> None:assert classify_from_failure_reason(None, exception_name="ValueError") is None+++_LOCATOR_WAIT_TIMEOUT = (+ "Failed to execute code block. Reason: TimeoutError: Locator.wait_for: Timeout 60000ms exceeded. "+ 'Call log: - waiting for locator("button[aria-label=\\"Log in\\"]")'+)+++def test_locator_wait_timeout_is_an_element_state_failure_not_a_page_load_one() -> None:+ categories = [result["category"] for result in _classify(_LOCATOR_WAIT_TIMEOUT)]++ assert "ELEMENT_STATE_TIMEOUT" in categories+ assert "PAGE_LOAD_TIMEOUT" not in categories+++def test_navigation_timeout_is_still_a_page_load_failure() -> None:+ categories = [result["category"] for result in _classify("Page.goto: Timeout 30000ms exceeded")]++ assert "PAGE_LOAD_TIMEOUT" in categories+ assert "ELEMENT_STATE_TIMEOUT" not in categories+++def test_a_selector_naming_a_password_field_is_not_an_auth_failure() -> None:+ # The word is part of the locator the wait timed out on, not evidence that login was rejected.+ reason = 'TimeoutError: Locator.wait_for: Timeout 10000ms exceeded. Call log: - waiting for locator("#password")'++ categories = [result["category"] for result in _classify(reason)]++ assert "ELEMENT_STATE_TIMEOUT" in categories+ assert "AUTH_FAILURE" not in categories+++def test_a_genuine_auth_failure_still_classifies() -> None:+ categories = [result["category"] for result in _classify("login failed: incorrect password")]++ assert "AUTH_FAILURE" in categories+++def test_a_bare_waiting_for_locator_timeout_is_an_element_state_failure() -> None:+ # Truncated Playwright messages drop the "Locator.method:" prefix and keep only the call log.+ reason = "Timeout 30000ms exceeded waiting for locator('#password')"++ categories = [result["category"] for result in _classify(reason)]++ assert "ELEMENT_STATE_TIMEOUT" in categories+ assert "PAGE_LOAD_TIMEOUT" not in categories+ assert "AUTH_FAILURE" not in categories+++def test_a_waiting_for_selector_timeout_is_an_element_state_failure() -> None:+ reason = "Timeout 30000ms exceeded waiting for selector '#login-btn'"++ categories = [result["category"] for result in _classify(reason)]++ assert "ELEMENT_STATE_TIMEOUT" in categories+ assert "PAGE_LOAD_TIMEOUT" not in categories+ assert "AUTH_FAILURE" not in categories+++def test_a_genuine_auth_failure_survives_an_appended_element_timeout() -> None:+ # Page-derived text can carry both; the element timeout must not silence the auth signal.+ reason = "Login failed: incorrect password; Timeout 5000ms exceeded on wait_for_selector"++ categories = [result["category"] for result in _classify(reason)]++ assert "AUTH_FAILURE" in categories+ assert "ELEMENT_STATE_TIMEOUT" in categoriesdiff --git a/tests/unit/test_forge_log_stdlib_extra.py b/tests/unit/test_forge_log_stdlib_extra.pynew file mode 100644index 000000000..d5bd13c23--- /dev/null+++ b/tests/unit/test_forge_log_stdlib_extra.py@@ -0,0 +1,54 @@+from __future__ import annotations++import logging++from skyvern.forge.sdk.forge_log import setup_logger+++def _render(record: logging.LogRecord) -> str:+ setup_logger()+ formatter = logging.getLogger().handlers[0].formatter+ assert formatter is not None+ return formatter.format(record)+++def _foreign_record(**extra: str) -> logging.LogRecord:+ """A stdlib record shaped like `LOG.warning(msg, extra={...})` produces."""+ record = logging.LogRecord(+ name="codeblock.codeblock_grpc",+ level=logging.WARNING,+ pathname=__file__,+ lineno=1,+ msg="CodeBlock runner unavailable",+ args=(),+ exc_info=None,+ )+ for key, value in extra.items():+ setattr(record, key, value)+ return record+++def test_stdlib_extra_fields_reach_rendered_output() -> None:+ # Modules the codeblock-runner image imports cannot use structlog, so `extra=`+ # is their only route to structured fields. Without ExtraAdder in the+ # ProcessorFormatter's foreign_pre_chain these are silently dropped and never+ # become queryable attributes.+ rendered = _render(+ _foreign_record(+ runner_target="127.0.0.1:7819",+ unavailable_cause="grpc_unavailable",+ grpc_status="UNAVAILABLE",+ )+ )++ assert "runner_target" in rendered+ assert "127.0.0.1:7819" in rendered+ assert "unavailable_cause" in rendered+ assert "grpc_unavailable" in rendered+ assert "grpc_status" in rendered+++def test_stdlib_record_without_extra_still_renders() -> None:+ rendered = _render(_foreign_record())++ assert "CodeBlock runner unavailable" in rendereddiff --git a/tests/unit/test_mcp_browser_extensions.py b/tests/unit/test_mcp_browser_extensions.pyindex 33b1f3d2a..07aa58d5c 100644--- a/tests/unit/test_mcp_browser_extensions.py+++ b/tests/unit/test_mcp_browser_extensions.py@@ -2,16 +2,37 @@from __future__ import annotations+import os+from pathlib import Pathfrom types import SimpleNamespace+from typing import Anyfrom unittest.mock import AsyncMock, MagicMockimport pytest+from fastmcp import Client+from skyvern.cli.core import session_managerfrom skyvern.cli.core.result import BrowserContextfrom skyvern.cli.mcp_tools import browser as mcp_browser+from skyvern.cli.mcp_tools import mcpfrom skyvern.cli.mcp_tools.browser import _wrap_async_iife+from skyvern.config import settings+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom tests.unit._mcp_browser_fakes import make_probe_locator+RUN_ID = "wr_mcp_upload_test"+++def run_upload_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:+ downloads_dir = tmp_path / "downloads"+ monkeypatch.setattr(settings, "DOWNLOAD_PATH", str(downloads_dir))+ upload_dir = downloads_dir / RUN_ID+ upload_dir.mkdir(parents=True)+ return upload_dir++# -- Helpers --@@ -34,6 +55,16 @@ def _patch_get_page(monkeypatch: pytest.MonkeyPatch, page=None, ctx=None):return page, ctx+def _patch_file_input(monkeypatch: pytest.MonkeyPatch) -> tuple[MagicMock, MagicMock]:+ raw = MagicMock()+ locator = MagicMock()+ locator.first = locator+ locator.set_input_files = AsyncMock()+ raw.locator = MagicMock(return_value=locator)+ _patch_get_page(monkeypatch, page=_fake_page(raw))+ return raw, locator++# -- _wrap_async_iife --@@ -208,42 +239,218 @@ class TestDrag:class TestFileUpload:@pytest.mark.asyncio- async def test_local_path_uses_set_input_files(self, monkeypatch: pytest.MonkeyPatch) -> None:- raw = MagicMock()- mock_locator = MagicMock()- mock_locator.first = mock_locator- mock_locator.set_input_files = AsyncMock()- raw.locator = MagicMock(return_value=mock_locator)- page = _fake_page(raw)- _patch_get_page(monkeypatch, page=page)-- result = await mcp_browser.skyvern_file_upload(- file_paths=["/tmp/test.txt"],- selector="input[type=file]",+ async def test_standalone_stdio_upload_uses_local_download_scope(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ standalone_upload_dir = run_upload_dir.parent / str(None)+ standalone_upload_dir.mkdir()+ upload_file = standalone_upload_dir / "test.txt"+ upload_file.write_bytes(b"safe upload")+ _, mock_locator = _patch_file_input(monkeypatch)+ monkeypatch.setattr(session_manager, "_stdio_local_file_access_enabled", True)++ with skyvern_context.scoped(SkyvernContext()):+ async with Client(mcp) as client:+ tool_result = await client.call_tool(+ "skyvern_file_upload",+ {"file_paths": [str(upload_file)], "selector": "input[type=file]"},+ )++ result = tool_result.structured_content+ assert result is not None+ assert result["ok"] is True+ mock_locator.set_input_files.assert_awaited_once_with(+ [{"name": "test.txt", "mimeType": "text/plain", "buffer": b"safe upload"}],+ timeout=5000,)++ @pytest.mark.asyncio+ async def test_local_path_uses_captured_file_payload(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ upload_file = run_upload_dir / "test.txt"+ upload_file.write_bytes(b"safe upload")+ raw, mock_locator = _patch_file_input(monkeypatch)++ with skyvern_context.scoped(SkyvernContext(run_id=RUN_ID)):+ result = await mcp_browser.skyvern_file_upload(+ file_paths=[str(upload_file)],+ selector="input[type=file]",+ )+assert result["ok"] is Trueassert result["data"]["files_count"] == 1raw.locator.assert_called_once_with("input[type=file]")- # set_input_files receives a list with the single file- mock_locator.set_input_files.assert_awaited_once_with(["/tmp/test.txt"], timeout=5000)+ mock_locator.set_input_files.assert_awaited_once_with(+ [{"name": "test.txt", "mimeType": "text/plain", "buffer": b"safe upload"}],+ timeout=5000,+ )++ @pytest.mark.asyncio+ async def test_rejects_path_traversal_outside_run_upload_directory(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ outside_file = run_upload_dir.parent / "service-secret.txt"+ outside_file.write_bytes(b"must not leave host")+ raw, mock_locator = _patch_file_input(monkeypatch)++ with skyvern_context.scoped(SkyvernContext(run_id=RUN_ID)):+ result = await mcp_browser.skyvern_file_upload(+ file_paths=[str(run_upload_dir / ".." / outside_file.name)],+ selector="input[type=file]",+ )++ assert result["ok"] is False+ assert result["error"]["code"] == "INVALID_INPUT"+ raw.locator.assert_not_called()+ mock_locator.set_input_files.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_rejects_symlink_escape_from_run_upload_directory(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ outside_file = run_upload_dir.parent / "service-secret.txt"+ outside_file.write_bytes(b"must not leave host")+ symlink = run_upload_dir / "upload.txt"+ symlink.symlink_to(outside_file)+ raw, mock_locator = _patch_file_input(monkeypatch)++ with skyvern_context.scoped(SkyvernContext(run_id=RUN_ID)):+ result = await mcp_browser.skyvern_file_upload(+ file_paths=[str(symlink)],+ selector="input[type=file]",+ )++ assert result["ok"] is False+ assert result["error"]["code"] == "INVALID_INPUT"+ raw.locator.assert_not_called()+ mock_locator.set_input_files.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_rejects_symlink_swap_after_path_resolution(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ upload_file = run_upload_dir / "upload.txt"+ upload_file.write_bytes(b"safe upload")+ outside_file = run_upload_dir.parent / "service-secret.txt"+ outside_file.write_bytes(b"must not leave host")+ raw, mock_locator = _patch_file_input(monkeypatch)+ real_open = os.open+ swapped = False++ def swap_before_open(+ path: str | bytes | os.PathLike[str] | os.PathLike[bytes],+ flags: int,+ mode: int = 0o777,+ *,+ dir_fd: int | None = None,+ ) -> int:+ nonlocal swapped+ if not swapped:+ upload_file.unlink()+ upload_file.symlink_to(outside_file)+ swapped = True+ return real_open(path, flags, mode, dir_fd=dir_fd)++ monkeypatch.setattr(mcp_browser.os, "open", swap_before_open)++ with skyvern_context.scoped(SkyvernContext(run_id=RUN_ID)):+ result = await mcp_browser.skyvern_file_upload(+ file_paths=[str(upload_file)],+ selector="input[type=file]",+ )++ assert swapped is True+ assert result["ok"] is False+ assert result["error"]["code"] == "INVALID_INPUT"+ raw.locator.assert_not_called()+ mock_locator.set_input_files.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_fifo_open_is_nonblocking(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ upload_fifo = run_upload_dir / "upload.pipe"+ os.mkfifo(upload_fifo)+ _, mock_locator = _patch_file_input(monkeypatch)+ real_open = os.open++ def assert_nonblocking_open(+ path: Any,+ flags: int,+ *args: Any,+ **kwargs: Any,+ ) -> int:+ if path == upload_fifo.name:+ assert flags & os.O_NONBLOCK+ return real_open(path, flags, *args, **kwargs)++ monkeypatch.setattr(mcp_browser.os, "open", assert_nonblocking_open)++ with skyvern_context.scoped(SkyvernContext(run_id=RUN_ID)):+ result = await mcp_browser.skyvern_file_upload(+ file_paths=[str(upload_fifo)],+ selector="input[type=file]",+ )++ assert result["ok"] is False+ assert result["error"]["code"] == "INVALID_INPUT"+ mock_locator.set_input_files.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_rejects_local_file_without_run_context(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ upload_file = run_upload_dir / "test.txt"+ upload_file.write_bytes(b"safe upload")+ raw = MagicMock()+ raw.locator = MagicMock()+ _patch_get_page(monkeypatch, page=_fake_page(raw))++ with skyvern_context.scoped(SkyvernContext()):+ result = await mcp_browser.skyvern_file_upload(+ file_paths=[str(upload_file)],+ selector="input[type=file]",+ )++ assert result["ok"] is False+ assert result["error"]["code"] == "INVALID_INPUT"+ raw.locator.assert_not_called()@pytest.mark.asyncio- async def test_intent_only_local_file_uses_sdk(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Intent-only + local file should use page.upload_file (AI resolution), not crash."""+ async def test_intent_only_local_file_requires_selector(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ upload_file = run_upload_dir / "resume.pdf"+ upload_file.write_bytes(b"safe upload")page, _ = _patch_get_page(monkeypatch)page.upload_file = AsyncMock(return_value="ok")- result = await mcp_browser.skyvern_file_upload(- file_paths=["/tmp/resume.pdf"],- intent="the upload button",- )- assert result["ok"] is True- page.upload_file.assert_awaited_once_with(- selector=None,- files="/tmp/resume.pdf",- prompt="the upload button",- ai="proactive",- timeout=30000,- )++ with skyvern_context.scoped(SkyvernContext(run_id=RUN_ID)):+ result = await mcp_browser.skyvern_file_upload(+ file_paths=[str(upload_file)],+ intent="the upload button",+ )++ assert result["ok"] is False+ assert result["error"]["code"] == "INVALID_INPUT"+ page.upload_file.assert_not_awaited()@pytest.mark.asyncioasync def test_url_uses_sdk_upload_file(self, monkeypatch: pytest.MonkeyPatch) -> None:@@ -261,24 +468,52 @@ class TestFileUpload:)@pytest.mark.asyncio- async def test_multiple_local_files_set_together(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Multiple local files should be passed to set_input_files as a single list, not one at a time."""- raw = MagicMock()- mock_locator = MagicMock()- mock_locator.first = mock_locator- mock_locator.set_input_files = AsyncMock()- raw.locator = MagicMock(return_value=mock_locator)- page = _fake_page(raw)- _patch_get_page(monkeypatch, page=page)+ async def test_rejects_local_files_above_aggregate_size(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ upload_a = run_upload_dir / "a.txt"+ upload_b = run_upload_dir / "b.txt"+ upload_a.write_bytes(b"abc")+ upload_b.write_bytes(b"def")+ raw, mock_locator = _patch_file_input(monkeypatch)+ monkeypatch.setattr(mcp_browser, "_LOCAL_UPLOAD_MAX_BYTES", 5)++ with skyvern_context.scoped(SkyvernContext(run_id=RUN_ID)):+ result = await mcp_browser.skyvern_file_upload(+ file_paths=[str(upload_a), str(upload_b)],+ selector="input[type=file]",+ )- result = await mcp_browser.skyvern_file_upload(- file_paths=["/tmp/a.txt", "/tmp/b.txt"],- selector="input[type=file]",- )- assert result["ok"] is True- assert result["data"]["files_count"] == 2- # Single call with both files, not two separate calls- mock_locator.set_input_files.assert_awaited_once_with(["/tmp/a.txt", "/tmp/b.txt"], timeout=5000)+ assert result["ok"] is False+ assert result["error"]["code"] == "INVALID_INPUT"+ raw.locator.assert_not_called()+ mock_locator.set_input_files.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_rejects_too_many_local_files(+ self,+ monkeypatch: pytest.MonkeyPatch,+ run_upload_dir: Path,+ ) -> None:+ upload_a = run_upload_dir / "a.txt"+ upload_b = run_upload_dir / "b.txt"+ upload_a.write_bytes(b"a")+ upload_b.write_bytes(b"b")+ raw, mock_locator = _patch_file_input(monkeypatch)+ monkeypatch.setattr(mcp_browser, "_LOCAL_UPLOAD_MAX_FILES", 1)++ with skyvern_context.scoped(SkyvernContext(run_id=RUN_ID)):+ result = await mcp_browser.skyvern_file_upload(+ file_paths=[str(upload_a), str(upload_b)],+ selector="input[type=file]",+ )++ assert result["ok"] is False+ assert result["error"]["code"] == "INVALID_INPUT"+ raw.locator.assert_not_called()+ mock_locator.set_input_files.assert_not_awaited()@pytest.mark.asyncioasync def test_multi_url_returns_error(self) -> None:diff --git a/tests/unit/test_mcp_commands.py b/tests/unit/test_mcp_commands.pyindex 6e679f49b..8481b978e 100644--- a/tests/unit/test_mcp_commands.py+++ b/tests/unit/test_mcp_commands.py@@ -728,6 +728,7 @@ def test_discover_switch_targets_finds_claude_code_and_codex(monkeypatch.setattr("skyvern.cli.mcp_commands._codex_config_path", lambda: codex_config)monkeypatch.setattr("skyvern.cli.mcp_commands._openclaw_config_path", lambda: openclaw_config)monkeypatch.setattr("skyvern.cli.mcp_commands._hermes_config_path", lambda: tmp_path / "missing-hermes.yaml")+ monkeypatch.setattr("skyvern.cli.mcp_commands._opencode_config_path", lambda: tmp_path / "missing-hermes.json")discovered, missing = _discover_switch_targets()@@ -760,6 +761,7 @@ def test_discover_switch_targets_reports_openclaw_invalid_nested_structure(monkeypatch.setattr("skyvern.cli.mcp_commands._codex_config_path", lambda: tmp_path / "missing-f.toml")monkeypatch.setattr("skyvern.cli.mcp_commands._openclaw_config_path", lambda: openclaw_config)monkeypatch.setattr("skyvern.cli.mcp_commands._hermes_config_path", lambda: tmp_path / "missing-g.yaml")+ monkeypatch.setattr("skyvern.cli.mcp_commands._opencode_config_path", lambda: tmp_path / "missing-g.json")discovered, _missing = _discover_switch_targets()diff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.pyindex c045e4959..140055368 100644--- a/tests/unit/test_otp_service.py+++ b/tests/unit/test_otp_service.py@@ -431,6 +431,37 @@ class TestGetOtpValueFromUrl:for value in record.values():assert raw_content not in str(value)+ @pytest.mark.asyncio+ async def test_insufficient_credit_skip_does_not_log_webhook_parse_failure(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ from skyvern.services import otp_service++ _patch_totp_url_response(+ monkeypatch,+ status_code=200,+ headers={"Content-Type": "application/json"},+ response_body={"verification_code": "Long relayed authentication message."},+ is_json_response=True,+ )+ monkeypatch.setattr(+ otp_service,+ "parse_otp_login",+ AsyncMock(side_effect=otp_service.InsufficientCreditsForOTPParse),+ )++ with structlog.testing.capture_logs() as logs:+ result = await _get_otp_value_from_url(+ organization_id="o_test",+ url="https://example.com/totp",+ api_key="",+ task_id="tsk_test",+ )++ assert result is None+ assert all(record.get("event") != "Failed to parse OTP content by LLM call" for record in logs)+ assert all(record.get("event") != "Failed to parse otp login from the totp url" for record in logs)+class TestParseOtpLogin:@pytest.mark.asyncio@@ -559,6 +590,7 @@ class TestParseOtpLogin:async def test_skips_llm_and_charge_when_out_of_credits(self, monkeypatch: pytest.MonkeyPatch) -> None:from skyvern.services import otp_service+ raw_content = "Long relayed message without a verification code."llm_handler = AsyncMock()charge = AsyncMock()monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")@@ -573,9 +605,14 @@ class TestParseOtpLogin:raising=False,)- result = await parse_otp_login(content="raw email body with OTP 424242 inside", organization_id="o_test")+ with structlog.testing.capture_logs() as logs:+ with pytest.raises(otp_service.InsufficientCreditsForOTPParse):+ await parse_otp_login(content=raw_content, organization_id="o_test")- assert result is None+ assert any(+ record.get("event") == "Skipping OTP parse; organization has insufficient credits" for record in logs+ )+ assert all(raw_content not in repr(record) for record in logs)llm_handler.assert_not_awaited()charge.assert_not_awaited()@@ -1601,6 +1638,28 @@ async def test_raw_otp_reparse_exception_retries_without_negative_cache() -> Nonmock_app.DATABASE.otp.promote_raw_otp_code.assert_awaited_once()+async def test_raw_otp_reparse_rechecks_after_credit_recovers_without_failure_log() -> None:+ raw, context = _raw_otp_row(), otp_service.RawOTPVerificationContext()+ parsed = OTPValue(value="https://example.test/login", type=OTPType.MAGIC_LINK)+ kwargs = {"expected_otp_type": OTPType.MAGIC_LINK, "raw_context": context}+ parse = AsyncMock(side_effect=[otp_service.InsufficientCreditsForOTPParse, parsed])+ with (+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.parse_otp_login", new=parse),+ structlog.testing.capture_logs() as logs,+ ):+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[])+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[raw])+ mock_app.DATABASE.otp.promote_raw_otp_code = AsyncMock()+ assert await _get_otp_value_from_db("o_test", "[email protected]", **kwargs) is None+ assert await _get_otp_value_from_db("o_test", "[email protected]", **kwargs) == parsed++ assert parse.await_count == 2+ mock_app.DATABASE.otp.promote_raw_otp_code.assert_awaited_once()+ assert all(record.get("event") != "Raw OTP reparse failed" for record in logs)++@pytest.mark.asyncioasync def test_raw_otp_wrong_type_is_negative_cached_per_session() -> None:rows = [_raw_otp_row(f"otp_raw_{index}") for index in range(4)]diff --git a/tests/unit/test_prompt_dataset_install.py b/tests/unit/test_prompt_dataset_install.pynew file mode 100644index 000000000..672881f55--- /dev/null+++ b/tests/unit/test_prompt_dataset_install.py@@ -0,0 +1,59 @@+from __future__ import annotations++import subprocess+import sys+from pathlib import Path++INSTALL_SCRIPT = Path(__file__).parents[2] / "prompt_evaluation" / "extract_action" / "scripts" / "install_directory.py"+++def _install(source: Path, destination: Path) -> subprocess.CompletedProcess[str]:+ return subprocess.run(+ [sys.executable, str(INSTALL_SCRIPT), str(source), str(destination)],+ capture_output=True,+ text=True,+ check=False,+ )+++def test_install_directory_moves_source_when_destination_is_absent(tmp_path: Path) -> None:+ source = tmp_path / "staged"+ source.mkdir()+ (source / "marker.txt").write_text("dataset", encoding="utf-8")+ destination = tmp_path / "dataset"++ result = _install(source, destination)++ assert result.returncode == 0, result.stderr+ assert not source.exists()+ assert (destination / "marker.txt").read_text(encoding="utf-8") == "dataset"+++def test_install_directory_rejects_existing_empty_destination(tmp_path: Path) -> None:+ source = tmp_path / "staged"+ source.mkdir()+ destination = tmp_path / "dataset"+ destination.mkdir()++ result = _install(source, destination)++ assert result.returncode == 1+ assert "already exists" in result.stderr+ assert source.is_dir()+ assert destination.is_dir()+++def test_install_directory_rejects_existing_symlink(tmp_path: Path) -> None:+ source = tmp_path / "staged"+ source.mkdir()+ target = tmp_path / "target"+ target.mkdir()+ destination = tmp_path / "dataset"+ destination.symlink_to(target, target_is_directory=True)++ result = _install(source, destination)++ assert result.returncode == 1+ assert "already exists" in result.stderr+ assert source.is_dir()+ assert destination.is_symlink()diff --git a/tests/unit/test_quickstart_command.py b/tests/unit/test_quickstart_command.pyindex fe987654d..0b434abef 100644--- a/tests/unit/test_quickstart_command.py+++ b/tests/unit/test_quickstart_command.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport jsonimport os+import reimport signalimport subprocessimport sys@@ -46,6 +47,9 @@ time.sleep(60)"""+_ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*m")++def _set_home(monkeypatch, home: Path) -> None:home.mkdir(parents=True, exist_ok=True)monkeypatch.setenv("HOME", str(home))@@ -59,6 +63,7 @@ def _patch_minimal_run_mcp_dependencies(monkeypatch, events: list[str], run_mcp_fake_session_manager = types.ModuleType("skyvern.cli.core.session_manager")fake_session_manager.set_stateless_http_mode = lambda _enabled: None+ fake_session_manager.set_stdio_local_file_access_enabled = lambda _enabled: Nonefake_telemetry = types.ModuleType("skyvern.cli.mcp_tools.telemetry")fake_telemetry.configure_mcp_telemetry_runtime = lambda **_kwargs: None@@ -300,6 +305,7 @@ def test_run_mcp_prepares_cloud_env_before_starting_mcp(tmp_path, monkeypatch) -fake_session_manager = types.ModuleType("skyvern.cli.core.session_manager")fake_session_manager.set_stateless_http_mode = lambda enabled: events.append(f"stateless:{enabled}")+ fake_session_manager.set_stdio_local_file_access_enabled = lambda enabled: events.append(f"stdio_local:{enabled}")fake_telemetry = types.ModuleType("skyvern.cli.mcp_tools.telemetry")@@ -353,9 +359,11 @@ def test_run_mcp_prepares_cloud_env_before_starting_mcp(tmp_path, monkeypatch) -"sweep","telemetry:local_cli:stdio","stateless:False",+ "stdio_local:True","run:stdio","cleanup","stateless:False",+ "stdio_local:False",]@@ -754,12 +762,15 @@ def test_quickstart_interactive_choice_reprompts_invalid_alias(monkeypatch) -> Nmonkeypatch.setattr(quickstart_module, "_is_interactive_input", lambda: True)result = CliRunner().invoke(quickstart_module.quickstart_app, [], input="wat\nserver\nn\n")+ # Rich colourises this prompt when it believes it is on a terminal, which splices escape codes+ # through the phrase and breaks a raw substring match.+ output = _ANSI_ESCAPE_RE.sub("", result.output)assert result.exit_code == 1- assert "Choose one of: cloud/api, local/embedded, server/self-hosted, 1, 2, or 3." in result.output- assert "Please select a valid option:" not in result.output- assert "Install the missing server dependencies now?" in result.output- assert 'Install: pip install "skyvern[server]"' in result.output+ assert "Choose one of: cloud/api, local/embedded, server/self-hosted, 1, 2, or 3." in output+ assert "Please select a valid option:" not in output+ assert "Install the missing server dependencies now?" in output+ assert 'Install: pip install "skyvern[server]"' in outputdef test_quickstart_interactive_eof_falls_back_to_default(monkeypatch) -> None:diff --git a/tests/unit/test_setup_quickstart.py b/tests/unit/test_setup_quickstart.pyindex 1f4ffa2d4..5eff45809 100644--- a/tests/unit/test_setup_quickstart.py+++ b/tests/unit/test_setup_quickstart.py@@ -29,6 +29,7 @@ def _patch_detection(monkeypatch: pytest.MonkeyPatch, *, claude_code: bool = Falmonkeypatch.setattr("skyvern.cli.setup_commands._is_cursor_installed", lambda: cursor)monkeypatch.setattr("skyvern.cli.setup_commands._is_windsurf_installed", lambda: False)monkeypatch.setattr("skyvern.cli.setup_commands._is_claude_desktop_installed", lambda: False)+ monkeypatch.setattr("skyvern.cli.setup_commands._is_opencode_installed", lambda: False)def _patch_env(monkeypatch: pytest.MonkeyPatch, env: tuple[str, str] = _FAKE_ENV) -> None:@@ -47,6 +48,7 @@ def test_detect_handles_exception_gracefully(monkeypatch: pytest.MonkeyPatch) ->monkeypatch.setattr("skyvern.cli.setup_commands._is_cursor_installed", lambda: True)monkeypatch.setattr("skyvern.cli.setup_commands._is_windsurf_installed", lambda: False)monkeypatch.setattr("skyvern.cli.setup_commands._is_claude_desktop_installed", lambda: False)+ monkeypatch.setattr("skyvern.cli.setup_commands._is_opencode_installed", lambda: False)detected, not_detected = _detect_installed_tools()assert {t.name for t in detected} == {"Cursor"}diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex a9386178d..6cb55df91 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -15,10 +15,7 @@ real database -- all DB / LLM / agent surfaces are patched.from __future__ import annotations-import base64-import jsonfrom datetime import datetime, timezone-from io import BytesIOfrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock@@ -26,7 +23,6 @@ from unittest.mock import AsyncMock, MagicMockimport pytestimport structlog.testingfrom fastapi import HTTPException-from PIL import Imagefrom skyvern.config import settingsfrom skyvern.forge import app@@ -34,7 +30,7 @@ from skyvern.forge.agent_functions import AgentFunctionfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot import agent as agent_module-from skyvern.forge.sdk.copilot.context import AgentResult, DeliveredUnverifiedPublicOutputs+from skyvern.forge.sdk.copilot.context import AgentResultfrom skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelopefrom skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_routefrom skyvern.forge.sdk.routes.workflow_copilot import (@@ -52,7 +48,6 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotStreamResponseUpdate,)from tests.unit.copilot_route_test_support import install_fake_create, setup_new_copilot_mocks-from tests.unit.copilot_test_helpers import make_copilot_ctx@pytest.fixture(autouse=True)@@ -819,143 +814,6 @@ async def test_agent_functions_oss_terminal_envelope_render_resolver_uses_settinassert await resolver.should_render_copilot_terminal_from_envelope("org-1") is True[email protected]("finalizer", ["normal", "cancel"], ids=["normal-turn", "cancel-turn"])[email protected]("destination", ["persistence", "sse", "history"])-async def test_delivered_unverified_narrative_payload_survives_persistence_sse_and_history(- monkeypatch: pytest.MonkeyPatch,- finalizer: str,- destination: str,-) -> None:- payload = {- "turnId": "turn-1",- "turnIndex": 0,- "mode": "build",- "designStarted": True,- "designEnded": True,- "draft": None,- "blocks": [],- "terminal": "response",- "terminalMessage": "done",- "narrativeSummary": None,- "priorBlockCount": None,- "designActivity": [],- "startedAt": None,- "endedAt": None,- }- chat = SimpleNamespace(- organization_id="org-1",- workflow_copilot_chat_id="chat-1",- proposed_workflow=None,- auto_accept=False,- )- original_workflow = SimpleNamespace(- workflow_id="wf-canonical",- title="Original",- description="Original description",- workflow_definition=None,- )- ctx = make_copilot_ctx()- ctx.delivered_unverified_terminal = True- ctx.delivered_unverified_workflow_run_id = "wr_route_source"- png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAAE0lEQVR4nGP8z4APMOGVZRip0gBBLAETee26JgAAAABJRU5ErkJggg=="- image_secret_without_association = png_b64[20:30]- ctx.secret_scrub_values.extend(["registered-secret-value", image_secret_without_association])- deeply_nested: dict[str, object] = {"password": "deep-must-not-persist"}- for index in range(100):- deeply_nested = {f"level_{index}": deeply_nested}- ctx.delivered_unverified_observed_outputs = DeliveredUnverifiedPublicOutputs(- {- "result": {- "amount": 0,- "confirmed": False,- "code_output": "captured-code-output-" + "x" * 579,- "deep": deeply_nested,- "valid_image_with_unassociated_registered_value": png_b64,- "png_prefixed_registered_value": "iVBORw0KGgoAAAANSUhEUg" + "registered-secret-value",- "api_key=sk-raw-secret-key-1234567890": "safe-value",- 7: "non-string-key-value",- }- }- )- agent_result = agent_module._make_agent_result(- ctx,- _delivered_unverified_snapshot=agent_module._delivered_unverified_observed_outputs(ctx),- user_response="done",- updated_workflow=None,- proposal_disposition="review_untested",- narrative_payload=payload,- )- _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)- stream = MagicMock(send=AsyncMock(return_value=True))-- if finalizer == "normal":- await workflow_copilot_route._finalise_normal_turn(- stream=stream,- chat=chat,- organization_id="org-1",- original_workflow=original_workflow,- chat_request=_make_chat_request(),- agent_result=agent_result,- )- else:- await workflow_copilot_route._persist_cancel_turn(- stream=stream,- chat=chat,- organization_id="org-1",- original_workflow=original_workflow,- user_message="Please update it",- agent_result=agent_result,- )-- assistant_write = workflow_params.create_workflow_copilot_chat_message.await_args_list[-1]- persisted_payload = assistant_write.kwargs["narrative_payload"]- response_frame = stream.send.await_args.args[0]- assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)- response_payload = response_frame.model_dump(mode="json")["narrative_payload"]-- now = datetime.now(timezone.utc)- history = convert_to_history_messages(- [- WorkflowCopilotChatMessage(- workflow_copilot_chat_message_id="message-1",- workflow_copilot_chat_id="chat-1",- sender=WorkflowCopilotChatSender.AI,- content="done",- narrative_payload=persisted_payload,- created_at=now,- modified_at=now,- )- ]- )- assert history[0].narrative_payload is not None- destination_payload = {- "persistence": persisted_payload,- "sse": response_payload,- "history": history[0].narrative_payload,- }[destination]- observed_outputs = destination_payload["deliveredUnverifiedObservedOutputs"]- json.dumps(destination_payload)- assert observed_outputs["result"]["amount"] == 0- assert observed_outputs["result"]["confirmed"] is False- assert observed_outputs["result"]["code_output"] == "captured-code-output-" + "x" * 579- canonical_image = base64.b64decode(- observed_outputs["result"]["valid_image_with_unassociated_registered_value"], validate=True- )- with Image.open(BytesIO(canonical_image)) as image:- image.load()- assert image.format == "PNG"- assert image.info == {}- assert observed_outputs["result"]["png_prefixed_registered_value"] == {- "$skyvernOmitted": {"reason": "invalid image", "count": 1}- }- assert "wr_route_source" not in str(observed_outputs)- assert observed_outputs["$skyvernOutput"]["omitted"]["depth"] >= 1- assert "deep-must-not-persist" not in str(observed_outputs)- assert "sk-raw-secret-key-1234567890" not in str(observed_outputs)- assert "non-string-key-value" not in str(observed_outputs)--@pytest.mark.asyncioasync def test_chat_audio_upload_stores_artifact_for_existing_chat(monkeypatch: pytest.MonkeyPatch) -> None:chat = SimpleNamespace(diff --git a/uv.lock b/uv.lockindex b453fce4b..3b735c505 100644--- a/uv.lock+++ b/uv.lock@@ -22,7 +22,7 @@ constraints = [{ name = "joserfc", specifier = ">=1.6.8" },{ name = "jupyter-server", specifier = ">=2.20.0" },{ name = "mako", specifier = ">=1.3.12" },- { name = "mistune", specifier = ">=3.3.0" },+ { name = "mistune", specifier = ">=3.2.1" },{ name = "msgpack", specifier = ">=1.2.1" },{ name = "pyasn1", specifier = ">=0.6.4" },{ name = "starlette", specifier = ">=1.3.1" },