Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-8879) copilot-stack/06: MCP tools surface + orphan-task cancellation (#5517)
Andrew Neilsonfaa2b233cb09
diff --git a/skyvern-frontend/src/routes/workflows/editor/helpContent.ts b/skyvern-frontend/src/routes/workflows/editor/helpContent.tsindex 8a15103b6..1ac102e10 100644--- a/skyvern-frontend/src/routes/workflows/editor/helpContent.ts+++ b/skyvern-frontend/src/routes/workflows/editor/helpContent.ts@@ -111,6 +111,8 @@ export const helpTooltips = {...baseHelpTooltipContent,fileUrl:"Since we're in beta this section isn't fully customizable yet, contact us if you'd like to integrate it into your workflow.",+ fileType:+ "The format of the file to parse. Auto-detected from the URL extension when possible.",},wait: {...baseHelpTooltipContent,diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsxindex 16ec92f2a..b850d413a 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx@@ -1,8 +1,15 @@import { HelpTooltip } from "@/components/HelpTooltip";import { Label } from "@/components/ui/label";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";import { Handle, NodeProps, Position } from "@xyflow/react";import { helpTooltips } from "../../helpContent";-import { type FileParserNode } from "./types";+import { type FileParserNode, type FileParserFileType } from "./types";import { WorkflowBlockInput } from "@/components/WorkflowBlockInput";import { useIsFirstBlockInWorkflow } from "../../hooks/useIsFirstNodeInWorkflow";import { cn } from "@/util/utils";@@ -16,6 +23,45 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";import { ModelSelector } from "@/components/ModelSelector";import { useRecordingStore } from "@/store/useRecordingStore";+const FILE_TYPE_OPTIONS: Array<{ value: FileParserFileType; label: string }> = [+ { value: "csv", label: "CSV" },+ { value: "excel", label: "Excel" },+ { value: "pdf", label: "PDF" },+ { value: "image", label: "Image" },+ { value: "docx", label: "DOCX" },+];++const FILE_EXTENSION_TO_TYPE: Record<string, FileParserFileType> = {+ csv: "csv",+ xlsx: "excel",+ xls: "excel",+ pdf: "pdf",+ png: "image",+ jpg: "image",+ jpeg: "image",+ gif: "image",+ webp: "image",+ docx: "docx",+};++function detectFileTypeFromUrl(url: string): FileParserFileType | null {+ try {+ const urlObj = new URL(url);+ const pathname = urlObj.pathname;+ const ext = pathname.split(".").pop()?.toLowerCase();+ if (ext && ext in FILE_EXTENSION_TO_TYPE) {+ return FILE_EXTENSION_TO_TYPE[ext] ?? null;+ }+ } catch {+ // Not a valid URL; try plain extension match+ const ext = url.split(".").pop()?.toLowerCase().split("?")[0];+ if (ext && ext in FILE_EXTENSION_TO_TYPE) {+ return FILE_EXTENSION_TO_TYPE[ext] ?? null;+ }+ }+ return null;+}+function FileParserNode({ id, data }: NodeProps<FileParserNode>) {const { editable, label } = data;const { blockLabel: urlBlockLabel } = useParams();@@ -30,6 +76,15 @@ function FileParserNode({ id, data }: NodeProps<FileParserNode>) {const update = useUpdate<FileParserNode["data"]>({ id, editable });const recordingStore = useRecordingStore();+ function handleFileUrlChange(value: string) {+ const detected = detectFileTypeFromUrl(value);+ if (detected) {+ update({ fileUrl: value, fileType: detected });+ } else {+ update({ fileUrl: value });+ }+ }+return (<divclassName={cn({@@ -83,12 +138,38 @@ function FileParserNode({ id, data }: NodeProps<FileParserNode>) {<WorkflowBlockInputnodeId={id}value={data.fileUrl}- onChange={(value) => {- update({ fileUrl: value });- }}+ onChange={handleFileUrlChange}className="nopan text-xs"/></div>+ <div className="space-y-2">+ <div className="flex gap-2">+ <Label className="text-xs text-slate-300">File Type</Label>+ <HelpTooltip content={helpTooltips["fileParser"]["fileType"]} />+ </div>+ <Select+ value={data.fileType}+ onValueChange={(value) => {+ update({ fileType: value as FileParserFileType });+ }}+ disabled={!editable}+ >+ <SelectTrigger className="nopan w-36 text-xs">+ <SelectValue placeholder="Select type" />+ </SelectTrigger>+ <SelectContent>+ {FILE_TYPE_OPTIONS.map((option) => (+ <SelectItem+ key={option.value}+ value={option.value}+ className="text-xs"+ >+ {option.label}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </div><WorkflowDataSchemaInputGroupexampleValue={dataSchemaExampleForFileExtraction}value={data.jsonSchema}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/types.tsindex b514ff762..71d146259 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/types.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/types.ts@@ -6,8 +6,11 @@ import {WorkflowModel,} from "@/routes/workflows/types/workflowTypes";+export type FileParserFileType = "csv" | "excel" | "pdf" | "image" | "docx";+export type FileParserNodeData = NodeBaseData & {fileUrl: string;+ fileType: FileParserFileType;jsonSchema: string;model: WorkflowModel | null;};@@ -19,6 +22,7 @@ export const fileParserNodeDefaultData: FileParserNodeData = {editable: true,label: "",fileUrl: "",+ fileType: "csv",continueOnFailure: false,jsonSchema: "null",model: null,diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex 787211070..69b66d950 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -864,6 +864,7 @@ function convertToNode(data: {...commonData,fileUrl: block.file_url,+ fileType: block.file_type ?? "csv",jsonSchema: JSON.stringify(block.json_schema, null, 2),model: block.model,},@@ -2511,7 +2512,7 @@ function getWorkflowBlock(...base,block_type: "file_url_parser",file_url: node.data.fileUrl,- file_type: "csv", // Backend will auto-detect based on file extension+ file_type: node.data.fileType,json_schema: JSONParseSafe(node.data.jsonSchema),};}diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.tsindex dac6f8e51..db6e8ddbf 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts@@ -410,7 +410,7 @@ export type SendEmailBlock = WorkflowBlockBase & {export type FileURLParserBlock = WorkflowBlockBase & {block_type: "file_url_parser";file_url: string;- file_type: "csv" | "excel" | "pdf" | "image";+ file_type: "csv" | "excel" | "pdf" | "image" | "docx";json_schema: Record<string, unknown> | null;};diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.tsindex b4e816d9b..6ae6dd798 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts@@ -352,7 +352,7 @@ export type SendEmailBlockYAML = BlockYAMLBase & {export type FileUrlParserBlockYAML = BlockYAMLBase & {block_type: "file_url_parser";file_url: string;- file_type: "csv" | "excel" | "pdf" | "image";+ file_type: "csv" | "excel" | "pdf" | "image" | "docx";json_schema?: Record<string, unknown> | null;};diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.pyindex 215549a68..00f999002 100644--- a/skyvern/cli/mcp_tools/browser.py+++ b/skyvern/cli/mcp_tools/browser.py@@ -154,7 +154,14 @@ async def skyvern_click("Include visual cues, position, or surrounding text when the page has similar elements."),] = None,- selector: Annotated[str | None, Field(description="CSS selector or XPath for the element to click")] = None,+ selector: Annotated[+ str | None,+ Field(+ description="Standard CSS selector or XPath for the element to click. "+ "jQuery pseudo-selectors like :contains(), :eq(), :first are NOT valid. "+ "Use standard CSS: 'button.class', 'a[href*=\"pdf\"]', '#id', ':nth-of-type()'."+ ),+ ] = None,timeout: Annotated[int,Field(diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.pyindex 9d007810e..70f309586 100644--- a/skyvern/forge/sdk/copilot/failure_tracking.py+++ b/skyvern/forge/sdk/copilot/failure_tracking.py@@ -82,15 +82,45 @@ def compute_failure_signature(def _canonical_block_config(block: Any) -> dict[str, Any]:+ """Stable dict view of a block's material config, with fields that don't+ affect downstream behavior (``output_parameter``) dropped.+ """dump = getattr(block, "model_dump", None)if callable(dump):try:- return dump(mode="json", exclude_none=True)+ cfg = dump(mode="json", exclude_none=True)except TypeError:- return dump()- if isinstance(block, dict):- return block- return {"repr": repr(block)}+ cfg = dump()+ elif isinstance(block, dict):+ cfg = dict(block)+ else:+ return {"repr": repr(block)}+ cfg.pop("output_parameter", None)+ return cfg+++def compute_action_sequence_fingerprint(results: list[dict[str, Any]]) -> str | None:+ """Hash the ordered ``(action_type, element_id)`` pairs across every+ block's ``action_trace`` in ``results``. Returns ``None`` when the trace is+ empty (e.g. fully-successful run where ``_attach_action_traces`` did not+ attach anything). Stable across runs: a form-fill→click→re-fill loop that+ retargets the same elements will produce the same fingerprint.+ """+ pairs: list[str] = []+ for entry in results:+ trace = entry.get("action_trace")+ if not isinstance(trace, list):+ continue+ for action in trace:+ if not isinstance(action, dict):+ continue+ action_type = action.get("action") or ""+ element = action.get("element") or ""+ pairs.append(f"{action_type}\x1f{element}")+ if not pairs:+ return None+ payload = "\x1e".join(pairs).encode("utf-8")+ return hashlib.sha256(payload).hexdigest()def compute_frontier_fingerprint(@@ -118,9 +148,7 @@ def compute_frontier_fingerprint(if block is None:payload.append({"label": label, "missing": True})continue- config = _canonical_block_config(block)- config.pop("output_parameter", None)- payload.append({"label": label, "config": config})+ payload.append({"label": label, "config": _canonical_block_config(block)})try:serialized = json.dumps(payload, sort_keys=True, default=str, separators=(",", ":"))except (TypeError, ValueError):diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pynew file mode 100644index 000000000..ec37cf4fa--- /dev/null+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -0,0 +1,275 @@+"""SDK-native MCP server with schema overlays for the Skyvern copilot."""++from __future__ import annotations++import json+from contextlib import AsyncExitStack+from dataclasses import dataclass, field+from typing import Any, Awaitable, Callable++import structlog+from agents.agent import AgentBase+from agents.mcp.server import MCPServer+from agents.run_context import RunContextWrapper+from fastmcp import Client+from mcp import Tool as MCPTool+from mcp.types import (+ CallToolResult,+ GetPromptResult,+ ListPromptsResult,+ TextContent,+)++from skyvern.forge.sdk.copilot.loop_detection import detect_tool_loop+from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm+from skyvern.forge.sdk.copilot.runtime import (+ AgentContext,+ ensure_browser_session,+ mcp_browser_context,+ mcp_to_copilot,+)+from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result++PreHook = Callable[[dict[str, Any], AgentContext], Awaitable[dict[str, Any] | None]]+PostHook = Callable[[dict[str, Any], dict[str, Any], AgentContext], Awaitable[dict[str, Any]]]+++@dataclass+class SchemaOverlay:+ """Schema overlay for MCP tools — hides params, renames args, injects forced values."""++ description: str | None = None+ hide_params: frozenset[str] = frozenset()+ required_overrides: list[str] | None = None+ arg_transforms: dict[str, str] = field(default_factory=dict)+ forced_args: dict[str, Any] = field(default_factory=dict)+ requires_browser: bool = False+ timeout: int | None = None+ pre_hook: PreHook | None = None+ post_hook: PostHook | None = None+++LOG = structlog.get_logger()+++def _apply_schema_overlay(+ input_schema: dict[str, Any],+ overlay: SchemaOverlay,+) -> dict[str, Any]:+ props = dict(input_schema.get("properties", {}))+ required = list(input_schema.get("required", []))++ for p in overlay.hide_params | frozenset(overlay.forced_args):+ props.pop(p, None)+ if p in required:+ required.remove(p)++ for copilot_param, mcp_param in overlay.arg_transforms.items():+ if mcp_param in props:+ props[copilot_param] = props.pop(mcp_param)+ if mcp_param in required:+ required.remove(mcp_param)+ required.append(copilot_param)++ if overlay.required_overrides is not None:+ required = overlay.required_overrides++ return {+ "type": input_schema.get("type", "object"),+ "properties": props,+ "required": required,+ }+++def _transform_args(+ arguments: dict[str, Any],+ overlay: SchemaOverlay,+) -> dict[str, Any]:+ mcp_args = {k: v for k, v in arguments.items() if k not in overlay.hide_params}++ for copilot_param, mcp_param in overlay.arg_transforms.items():+ if copilot_param in mcp_args:+ mcp_args[mcp_param] = mcp_args.pop(copilot_param)++ mcp_args.update(overlay.forced_args)+ return mcp_args+++def _copilot_to_call_tool_result(+ copilot_result: dict[str, Any],+) -> CallToolResult:+ sanitized = sanitize_tool_result_for_llm("", copilot_result)+ content: list[TextContent] = [TextContent(type="text", text=json.dumps(sanitized))]+ is_error = not copilot_result.get("ok", True)+ return CallToolResult(content=content, isError=is_error)+++class SkyvernOverlayMCPServer(MCPServer):+ """MCP server that wraps a FastMCP transport with schema overlays and+ copilot-specific dispatch logic (loop detection, browser injection, hooks).+ """++ def __init__(+ self,+ transport: Any,+ overlays: dict[str, SchemaOverlay],+ alias_map: dict[str, str],+ allowlist: frozenset[str],+ context_provider: Callable[[], Any],+ ) -> None:+ super().__init__(use_structured_content=False)+ self._transport = transport+ self._overlays = overlays+ self._alias_map = alias_map # copilot_name -> mcp_name+ self._reverse_alias: dict[str, str] = {v: k for k, v in alias_map.items()}+ self._allowlist = allowlist+ self._context_provider = context_provider+ self._client: Client | None = None+ self._exit_stack: AsyncExitStack | None = None+ self._cached_tools: list[MCPTool] | None = None++ @property+ def name(self) -> str:+ return "skyvern"++ async def connect(self) -> None:+ stack = AsyncExitStack()+ await stack.__aenter__()+ client = Client(self._transport)+ await stack.enter_async_context(client)+ self._client = client+ self._exit_stack = stack++ async def cleanup(self) -> None:+ if self._exit_stack:+ await self._exit_stack.__aexit__(None, None, None)+ self._client = None+ self._exit_stack = None+ self._cached_tools = None++ async def list_tools(+ self,+ run_context: RunContextWrapper[Any] | None = None,+ agent: AgentBase | None = None,+ ) -> list[MCPTool]:+ if self._cached_tools is not None:+ return self._cached_tools++ if not self._client:+ raise RuntimeError("Not connected — call connect() first")++ raw_tools = await self._client.list_tools()+ result: list[MCPTool] = []++ for tool in raw_tools:+ if tool.name not in self._allowlist:+ continue++ copilot_name = self._reverse_alias.get(tool.name, tool.name)+ overlay = self._overlays.get(copilot_name, SchemaOverlay())++ schema = _apply_schema_overlay(tool.inputSchema, overlay)+ description = overlay.description or tool.description or ""++ result.append(+ MCPTool(+ name=copilot_name,+ description=description,+ inputSchema=schema,+ )+ )+ self._cached_tools = result+ return result++ async def call_tool(+ self,+ tool_name: str,+ arguments: dict[str, Any] | None,+ meta: dict[str, Any] | None = None,+ ) -> CallToolResult:+ if not self._client:+ raise RuntimeError("Not connected — call connect() first")++ arguments = arguments or {}+ copilot_ctx = self._context_provider()+ overlay = self._overlays.get(tool_name, SchemaOverlay())++ tracker = getattr(copilot_ctx, "consecutive_tool_tracker", None)+ loop_error = detect_tool_loop(tracker, tool_name) if isinstance(tracker, list) else None+ if loop_error:+ LOG.warning(+ "Tool loop detected, skipping execution",+ tool_name=tool_name,+ )+ return CallToolResult(+ content=[+ TextContent(+ type="text",+ text=json.dumps(+ {+ "ok": False,+ "error": loop_error,+ }+ ),+ )+ ],+ isError=True,+ )++ if overlay.pre_hook:+ hook_result = await overlay.pre_hook(arguments, copilot_ctx)+ if hook_result is not None:+ return _copilot_to_call_tool_result(hook_result)++ mcp_name = self._alias_map.get(tool_name, tool_name)+ mcp_args = _transform_args(arguments, overlay)++ if overlay.requires_browser:+ err = await ensure_browser_session(copilot_ctx)+ if err:+ return _copilot_to_call_tool_result(err)+ mcp_args["session_id"] = copilot_ctx.browser_session_id++ try:+ call = self._client.call_tool(mcp_name, mcp_args, raise_on_error=False)+ if overlay.requires_browser:+ async with mcp_browser_context(copilot_ctx):+ raw_result = await call+ else:+ raw_result = await call+ except Exception as e:+ LOG.warning(+ "MCP tool call failed",+ tool=tool_name,+ error=str(e),+ exc_info=True,+ )+ return _copilot_to_call_tool_result({"ok": False, "error": f"{tool_name} failed: {e}"})++ # Copy fastmcp's structured_content so mutations below stay local to+ # this call — the client may reuse or cache the response object.+ raw_mcp = dict(raw_result.structured_content or {})+ if raw_result.is_error:+ raw_mcp["ok"] = False+ if not raw_result.structured_content and raw_result.content:+ text_parts = [c.text for c in raw_result.content if hasattr(c, "text")]+ raw_mcp["error"] = " ".join(text_parts) if text_parts else "Unknown MCP error"+ else:+ raw_mcp["error"] = raw_mcp.get("error") or "Unknown MCP error"+ copilot_result = mcp_to_copilot(raw_mcp)++ if overlay.post_hook:+ copilot_result = await overlay.post_hook(copilot_result, raw_mcp, copilot_ctx)++ enqueue_screenshot_from_result(copilot_ctx, copilot_result)+ return _copilot_to_call_tool_result(copilot_result)++ async def list_prompts(self) -> ListPromptsResult:+ return ListPromptsResult(prompts=[])++ async def get_prompt(+ self,+ name: str,+ arguments: dict[str, Any] | None = None,+ ) -> GetPromptResult:+ raise ValueError(f"Prompts not supported: {name}")diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 74437e85d..143b1ddac 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -50,6 +50,18 @@ class AgentContext:pending_screenshots: list[ScreenshotEntry] = field(default_factory=list)tool_activity: list[dict[str, Any]] = field(default_factory=list)+ # Cross-turn agent state accumulated by tools.py as the agent runs.+ # Read back by failure_tracking / loop_detection to detect stuck loops,+ # preserve verified prefixes across partial runs, etc. All optional —+ # downstream accessors use ``getattr(ctx, name, default)`` where+ # tolerant-to-unset is the right default.+ last_requested_block_labels: list[str] = field(default_factory=list)+ last_executed_block_labels: list[str] = field(default_factory=list)+ last_frontier_start_label: str | None = None+ pending_action_sequence_fingerprint: str | None = None+ verified_block_outputs: dict[str, Any] = field(default_factory=dict)+ verified_prefix_labels: list[str] = field(default_factory=list)+def mcp_to_copilot(mcp_result: dict[str, Any]) -> dict[str, Any]:"""Convert an MCP result dict to the copilot {ok, data, error} format."""diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pynew file mode 100644index 000000000..bf5cca9c6--- /dev/null+++ b/skyvern/forge/sdk/copilot/tools.py@@ -0,0 +1,1753 @@+"""Copilot agent tools — native handlers, hooks, and registration."""++from __future__ import annotations++import asyncio+import base64+import json+import re+from collections import defaultdict+from typing import Any++import structlog+import yaml+from agents import function_tool+from agents.run_context import RunContextWrapper+from pydantic import ValidationError++from skyvern.forge import app+from skyvern.forge.failure_classifier import classify_from_failure_reason+from skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.copilot.failure_tracking import (+ _canonical_block_config,+ compute_action_sequence_fingerprint,+ update_repeated_failure_state,+)+from skyvern.forge.sdk.copilot.loop_detection import detect_tool_loop+from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay+from skyvern.forge.sdk.copilot.output_utils import (+ sanitize_tool_result_for_llm,+ truncate_output,+)+from skyvern.forge.sdk.copilot.runtime import AgentContext+from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span+from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml+from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException+from skyvern.forge.sdk.workflow.models.parameter import (+ OutputParameter,+ WorkflowParameter,+ WorkflowParameterType,+)+from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRunStatus++LOG = structlog.get_logger()++_FAILED_BLOCK_STATUSES: frozenset[str] = frozenset(+ {+ WorkflowRunStatus.failed.value,+ WorkflowRunStatus.terminated.value,+ WorkflowRunStatus.canceled.value,+ WorkflowRunStatus.timed_out.value,+ }+)+_DATA_PRODUCING_BLOCK_TYPES = frozenset({"EXTRACTION", "TEXT_PROMPT"})+RUN_BLOCKS_DEBUG_TIMEOUT_SECONDS = 180++# Detached cleanup tasks held here so the garbage collector does not drop them+# while they still have work to do, and so the "task exception was never+# retrieved" warning cannot fire — each task adds a done-callback that logs+# exceptions and removes itself from this set.+_DETACHED_CLEANUP_TASKS: set[asyncio.Task] = set()+++async def _cancel_run_task_if_not_final(+ run_task: asyncio.Task,+ workflow_run_id: str,+) -> None:+ """Cancel ``run_task`` and reconcile the workflow run row to a terminal+ state.++ ``run_task.cancel()`` is synchronous — it just flips the cancel flag. We+ then wait briefly for ``execute_workflow``'s outer ``finally`` to drain+ its shielded ``_finalize_workflow_run_status`` call, which restores the+ real terminal status (``failed``/``terminated``/``timed_out``) even when+ we cancel mid-flight. After that we issue a conditional DB cancel that+ is a no-op when the row is already terminal — so a run whose finally+ block produced a proper terminal status keeps it, and a run that truly+ never finalized (e.g. cancel landed before block execution captured a+ ``pre_finally_status``) lands as ``canceled``. All awaits are+ exception-contained so teardown of the enclosing tool task doesn't+ surface a secondary error over the original cancellation.+ """+ run_task.cancel()+ try:+ # Shield run_task so OUR wait timeout does not send another cancel+ # through to it — the cancel we want is already pending.+ await asyncio.wait_for(asyncio.shield(run_task), timeout=5.0)+ except (asyncio.CancelledError, asyncio.TimeoutError):+ pass+ except Exception:+ LOG.warning(+ "Run task raised during cancellation grace window",+ workflow_run_id=workflow_run_id,+ exc_info=True,+ )+ try:+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final(+ workflow_run_id=workflow_run_id,+ )+ except Exception:+ LOG.warning(+ "Conditional cancel write failed",+ workflow_run_id=workflow_run_id,+ exc_info=True,+ )+++def _log_detached_cleanup_failure(task: asyncio.Task) -> None:+ exc = task.exception() if task.done() and not task.cancelled() else None+ if exc is not None:+ LOG.warning("Detached cancel fallback failed", exc_info=exc)+++# Streak threshold at which the copilot hard-aborts a tool call because the+# same action sequence has repeated run-over-run with no intervening success.+# The streak counter is incremented in ``update_repeated_failure_state`` AFTER+# each run, so the abort fires when the 4th consecutive run against the same+# action fingerprint enters ``_tool_loop_error`` (streak == 3 at entry, one+# per each of the three preceding identical runs). Calibration note: the+# repeated-frontier streak in failure_tracking.py uses STOP_AT=3 for the same+# shape of escalation.+REPEATED_ACTION_STREAK_ABORT_AT = 3+++def _is_meaningful_extracted_data(extracted: Any) -> bool:+ """Return True when extracted data contains at least one non-null, non-empty value.++ A dict like ``{"price": None}`` is technically present but carries no signal —+ treat it the same as no output at all so enforcement can nudge the agent to+ investigate instead of declaring success.+ """+ if extracted is None:+ return False+ if isinstance(extracted, (str, bytes)):+ return bool(extracted)+ if isinstance(extracted, dict):+ return any(_is_meaningful_extracted_data(v) for v in extracted.values())+ if isinstance(extracted, (list, tuple, set)):+ return any(_is_meaningful_extracted_data(v) for v in extracted)+ # Numbers, booleans, and other scalars count as meaningful output.+ return True+++async def _attach_action_traces(+ blocks: list,+ results: list[dict[str, Any]],+ organization_id: str,+) -> None:+ """For non-success blocks with a task_id, fetch and attach a compact action trace."""+ failed_task_ids = [+ b.task_id for b, r in zip(blocks, results) if b.task_id and r.get("status") in _FAILED_BLOCK_STATUSES+ ]+ if not failed_task_ids:+ return++ rows = await app.DATABASE.tasks.get_recent_actions_for_tasks(+ task_ids=failed_task_ids,+ organization_id=organization_id,+ )++ actions_by_task: dict[str, list] = defaultdict(list)+ for row in rows:+ if row.task_id is not None:+ actions_by_task[row.task_id].append(row)++ for block, block_result in zip(blocks, results):+ if block_result.get("status") not in _FAILED_BLOCK_STATUSES or not block.task_id:+ continue+ task_actions = actions_by_task.get(block.task_id, [])+ block_result["action_trace"] = [+ {+ "action": a.action_type,+ "status": a.status,+ "reasoning": a.reasoning[:150] if a.reasoning else None,+ "element": a.element_id,+ }+ for a in task_actions+ ]+++async def _fetch_last_screenshot_b64(task_id: str, organization_id: str) -> str | None:+ try:+ artifacts = await app.DATABASE.artifacts.get_artifacts_for_task_v2(+ task_v2_id=task_id,+ organization_id=organization_id,+ artifact_types=[ArtifactType.SCREENSHOT_LLM],+ )+ if not artifacts:+ return None+ # The last artifact is the one captured closest to the failure.+ artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifacts[-1])+ if not artifact_bytes:+ return None+ return base64.b64encode(artifact_bytes).decode("utf-8")+ except Exception:+ LOG.debug("Failed to fetch screenshot for failed block", task_id=task_id, exc_info=True)+ return None+++async def _attach_failed_block_screenshots(+ blocks: list,+ results: list[dict[str, Any]],+ organization_id: str,+) -> None:+ """For failed blocks with a task_id, fetch the last SCREENSHOT_LLM artifact."""+ task_id_to_block: dict[str, dict] = {+ block.task_id: block_result+ for block, block_result in zip(blocks, results)+ if block.task_id and block_result.get("status") in _FAILED_BLOCK_STATUSES+ }+ if not task_id_to_block:+ return++ task_ids = list(task_id_to_block.keys())+ screenshots = await asyncio.gather(+ *(_fetch_last_screenshot_b64(task_id, organization_id) for task_id in task_ids),+ )+ for task_id, b64 in zip(task_ids, screenshots):+ if b64 is not None:+ task_id_to_block[task_id]["screenshot_b64"] = b64+++_BLOCK_RUNNING_TOOLS = frozenset({"run_blocks_and_collect_debug", "update_and_run_blocks"})+++def _tool_loop_error(ctx: AgentContext, tool_name: str) -> str | None:+ tracker = getattr(ctx, "consecutive_tool_tracker", None)+ if isinstance(tracker, list):+ detected = detect_tool_loop(tracker, tool_name)+ if detected is not None:+ return detected++ # Hard-abort when the agent has re-fired the same action sequence against+ # the page N times without intervening success. This is the signal that+ # the form is blocked (captcha / anti-bot / error banner the agent isn't+ # detecting) and further attempts will just burn the tool timeout. Scoped+ # to block-running tools so planning/metadata tools (update_workflow,+ # list_credentials, get_run_results) stay unaffected.+ if tool_name in _BLOCK_RUNNING_TOOLS:+ streak_raw = getattr(ctx, "repeated_action_fingerprint_streak_count", 0)+ streak = streak_raw if isinstance(streak_raw, int) else 0+ if streak >= REPEATED_ACTION_STREAK_ABORT_AT:+ return (+ f"Repeated-action abort: the last {streak} runs fired the same "+ "action sequence against the page without making progress. "+ "The site is likely blocked by a captcha, popup, anti-bot "+ "challenge, or hidden validation error that the agent is not "+ "detecting. Do NOT retry this tool — conclude the workflow is "+ "not automatable as-is and report back to the user."+ )+ return None+++_PARAMETER_TYPE_PLACEHOLDERS: dict[WorkflowParameterType, Any] = {+ WorkflowParameterType.STRING: "",+ WorkflowParameterType.INTEGER: 0,+ WorkflowParameterType.FLOAT: 0.0,+ WorkflowParameterType.BOOLEAN: False,+ WorkflowParameterType.JSON: {},+ WorkflowParameterType.FILE_URL: "",+}+++def _placeholder_for_parameter_type(param_type: WorkflowParameterType) -> Any:+ return _PARAMETER_TYPE_PLACEHOLDERS.get(param_type)+++def _parameter_binding_invariant_error(+ workflow: Workflow,+ persisted_workflow_params: list[WorkflowParameter],+ persisted_output_params: list[OutputParameter],+) -> tuple[str, dict[str, list[str]], dict[str, list[str]]] | None:+ """Return a ``(summary, missing_persisted, missing_from_definition)`` tuple+ when ``workflow.workflow_definition`` disagrees with persisted+ definition-parameter rows for runtime-relevant classes. Returns ``None``+ when aligned.++ Compares ``WorkflowParameter`` rows by ``(key, workflow_parameter_type)``+ and ``OutputParameter`` rows by ``key``. Secret/credential and context+ parameters are intentionally out of scope — runtime reads those from the+ definition JSON.+ """+ definition = getattr(workflow, "workflow_definition", None)+ parameters = getattr(definition, "parameters", None) if definition else None+ parameters = list(parameters) if parameters else []++ def_workflow_ids: set[tuple[str, str]] = set()+ def_output_keys: set[str] = set()+ for parameter in parameters:+ if isinstance(parameter, WorkflowParameter):+ def_workflow_ids.add((parameter.key, parameter.workflow_parameter_type.value))+ elif isinstance(parameter, OutputParameter):+ def_output_keys.add(parameter.key)++ persisted_workflow_ids: set[tuple[str, str]] = {+ (wp.key, wp.workflow_parameter_type.value) for wp in persisted_workflow_params+ }+ persisted_output_keys: set[str] = {op.key for op in persisted_output_params}++ missing_persisted_workflow = sorted(+ f"{key} ({ptype})" for (key, ptype) in def_workflow_ids - persisted_workflow_ids+ )+ extra_persisted_workflow = sorted(f"{key} ({ptype})" for (key, ptype) in persisted_workflow_ids - def_workflow_ids)+ missing_persisted_output = sorted(def_output_keys - persisted_output_keys)+ extra_persisted_output = sorted(persisted_output_keys - def_output_keys)++ if (+ not missing_persisted_workflow+ and not extra_persisted_workflow+ and not missing_persisted_output+ and not extra_persisted_output+ ):+ return None++ summary = (+ "Pre-run invariant: workflow_definition and persisted parameter rows disagree. "+ f"workflow missing persisted: {missing_persisted_workflow or '[]'}; "+ f"workflow missing from definition: {extra_persisted_workflow or '[]'}; "+ f"output missing persisted: {missing_persisted_output or '[]'}; "+ f"output missing from definition: {extra_persisted_output or '[]'}"+ )+ return (+ summary,+ {"workflow": missing_persisted_workflow, "output": missing_persisted_output},+ {"workflow": extra_persisted_workflow, "output": extra_persisted_output},+ )+++async def _update_workflow(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:+ workflow_yaml = params["workflow_yaml"]+ try:+ workflow = _process_workflow_yaml(+ workflow_id=ctx.workflow_id,+ workflow_permanent_id=ctx.workflow_permanent_id,+ organization_id=ctx.organization_id,+ workflow_yaml=workflow_yaml,+ )+ await app.WORKFLOW_SERVICE.update_workflow_definition(+ workflow_id=ctx.workflow_id,+ organization_id=ctx.organization_id,+ title=workflow.title,+ description=workflow.description,+ workflow_definition=workflow.workflow_definition,+ proxy_location=workflow.proxy_location,+ webhook_callback_url=workflow.webhook_callback_url,+ persist_browser_session=workflow.persist_browser_session,+ model=workflow.model,+ max_screenshot_scrolling_times=workflow.max_screenshot_scrolls,+ extra_http_headers=workflow.extra_http_headers,+ run_with=workflow.run_with,+ ai_fallback=workflow.ai_fallback,+ cache_key=workflow.cache_key,+ run_sequentially=workflow.run_sequentially,+ sequential_key=workflow.sequential_key,+ )+ ctx.workflow_yaml = workflow_yaml+ return {+ "ok": True,+ "data": {+ "message": "Workflow updated successfully.",+ "block_count": len(workflow.workflow_definition.blocks) if workflow.workflow_definition else 0,+ },+ "_workflow": workflow,+ }+ except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException) as e:+ return {+ "ok": False,+ "error": f"Workflow validation failed: {e}",+ }+++async def _list_credentials(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:+ page = params.get("page", 1)+ page_size = min(params.get("page_size", 10), 50)+ credentials = await app.DATABASE.credentials.get_credentials(+ organization_id=ctx.organization_id,+ page=page,+ page_size=page_size,+ )+ serialized = []+ for cred in credentials:+ entry: dict[str, Any] = {+ "credential_id": cred.credential_id,+ "name": cred.name,+ "credential_type": str(cred.credential_type),+ }+ if cred.username:+ entry["username"] = cred.username+ entry["totp_type"] = str(cred.totp_type) if cred.totp_type else None+ elif cred.card_last4:+ entry["card_last_four"] = cred.card_last4+ entry["card_brand"] = cred.card_brand+ elif cred.secret_label:+ entry["secret_label"] = cred.secret_label+ serialized.append(entry)+ return {+ "ok": True,+ "data": {+ "credentials": serialized,+ "page": page,+ "page_size": page_size,+ "count": len(serialized),+ "has_more": len(serialized) == page_size,+ },+ }+++# Block types that establish browser state (loaded page / authenticated+# session / navigation target). These are valid upstream anchors to walk back+# to when a downstream edit invalidates part of the chain.+#+# We intentionally do NOT maintain a companion "rerunnable from current+# browser state" set. We have no signal that the persistent browser session+# is actually anchored at the frontier boundary — after a successful+# [A, B, C] the browser is at post-C state, not pre-C — so rerunning only+# an edited block is unsafe even for read-only types. Every edit walks back+# to an upstream state-establisher, or falls back to the full requested list.+_BLOCK_TYPES_STATE_ESTABLISHER = frozenset({"navigation", "login", "goto_url"})++_OUTPUT_REF_RE = re.compile(r"\{\{\s*([A-Za-z0-9_]+)_output\s*[\.}|]")+++def _block_type_name(block: Any) -> str:+ """Lowercase string name of a block's type, for both YAML and runtime blocks."""+ bt = getattr(block, "block_type", None)+ if bt is None:+ return ""+ name = getattr(bt, "value", None) or getattr(bt, "name", None) or str(bt)+ return str(name).lower()+++def _blocks_by_label(workflow_definition: Any) -> dict[str, Any]:+ blocks = getattr(workflow_definition, "blocks", None) if workflow_definition else None+ by_label: dict[str, Any] = {}+ if not blocks:+ return by_label+ for block in blocks:+ label = getattr(block, "label", None)+ if isinstance(label, str):+ by_label[label] = block+ return by_label+++def _find_invalidated_labels(+ old_definition: Any,+ new_definition: Any,+ requested_labels: list[str],+) -> set[str]:+ """Return the set of requested labels whose behavior is invalidated.++ A label is invalidated when its own config changed or when any upstream+ label in the requested chain was invalidated (downstream trust propagates+ forward).+ """+ old_by_label = _blocks_by_label(old_definition)+ new_by_label = _blocks_by_label(new_definition)+ invalidated: set[str] = set()+ upstream_invalidated = False+ for label in requested_labels:+ if upstream_invalidated:+ invalidated.add(label)+ continue+ old_block = old_by_label.get(label)+ new_block = new_by_label.get(label)+ if old_block is None or new_block is None:+ invalidated.add(label)+ upstream_invalidated = True+ continue+ if _canonical_block_config(old_block) != _canonical_block_config(new_block):+ invalidated.add(label)+ upstream_invalidated = True+ return invalidated+++def _earliest_invalidated(requested_labels: list[str], invalidated: set[str]) -> str | None:+ for label in requested_labels:+ if label in invalidated:+ return label+ return None+++def _nearest_upstream_state_establisher(+ requested_labels: list[str], target_label: str, new_definition: Any+) -> str | None:+ by_label = _blocks_by_label(new_definition)+ try:+ idx = requested_labels.index(target_label)+ except ValueError:+ return None+ for candidate in reversed(requested_labels[:idx]):+ block = by_label.get(candidate)+ if block is None:+ continue+ if _block_type_name(block) in _BLOCK_TYPES_STATE_ESTABLISHER:+ return candidate+ return None+++def _referenced_output_labels(frontier_labels: list[str], new_definition: Any) -> set[str]:+ by_label = _blocks_by_label(new_definition)+ needed: set[str] = set()+ for label in frontier_labels:+ block = by_label.get(label)+ if block is None:+ continue+ try:+ serialized = json.dumps(_canonical_block_config(block), default=str, separators=(",", ":"))+ except (TypeError, ValueError):+ serialized = repr(block)+ for match in _OUTPUT_REF_RE.findall(serialized):+ needed.add(match)+ return needed+++def _summarize_action_trace(action_trace: list[dict[str, Any]] | None) -> list[str]:+ """Compact, stringified summary of action entries for the compact packet."""+ if not action_trace:+ return []+ summary: list[str] = []+ for entry in action_trace[-6:]:+ if not isinstance(entry, dict):+ continue+ action = entry.get("action") or "?"+ status = entry.get("status") or ""+ element = entry.get("element")+ bits = [str(action)]+ if element:+ bits.append(str(element))+ if status:+ bits.append(str(status))+ summary.append(" ".join(bits).strip())+ return summary+++async def _get_prior_workflow_definition(ctx: AgentContext) -> Any:+ """Hybrid: prefer ctx.last_workflow, fall back to DB fetch on cold start."""+ last_workflow = getattr(ctx, "last_workflow", None)+ if last_workflow is not None:+ definition = getattr(last_workflow, "workflow_definition", None)+ if definition is not None:+ return definition+ last_yaml = getattr(ctx, "last_workflow_yaml", None)+ if last_yaml:+ try:+ workflow = _process_workflow_yaml(+ workflow_id=ctx.workflow_id,+ workflow_permanent_id=ctx.workflow_permanent_id,+ organization_id=ctx.organization_id,+ workflow_yaml=last_yaml,+ )+ return workflow.workflow_definition+ except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException):+ pass+ try:+ fetched = await app.DATABASE.workflows.get_workflow_by_permanent_id(+ workflow_permanent_id=ctx.workflow_permanent_id,+ organization_id=ctx.organization_id,+ )+ if fetched is not None:+ return fetched.workflow_definition+ except Exception:+ LOG.debug("Failed to fetch prior workflow definition for frontier diff", exc_info=True)+ return None+++def _plan_frontier(+ ctx: Any,+ requested_labels: list[str],+ old_definition: Any,+ new_definition: Any,+) -> tuple[list[str], dict[str, Any], str | None]:+ """Plan the frontier execution.++ Returns ``(labels_to_execute, block_outputs_to_seed, frontier_start_label)``.++ Falls back to the full requested list on any ambiguity. When there is no+ workflow change (plain run path) the frontier is the first requested label+ and we seed any verified outputs referenced by the suffix.+ """+ if not requested_labels:+ return requested_labels, {}, None+ if new_definition is None:+ return requested_labels, {}, requested_labels[0]++ verified_outputs: dict[str, Any] = dict(getattr(ctx, "verified_block_outputs", {}) or {})+ verified_prefix: list[str] = list(getattr(ctx, "verified_prefix_labels", []) or [])+ verified_prefix_set = set(verified_prefix)++ # No old definition (cold start or parse failure) OR no diff signal → plain path.+ if old_definition is None:+ frontier = requested_labels[0]+ return _seed_for_frontier(requested_labels, frontier, verified_outputs, new_definition)++ try:+ invalidated = _find_invalidated_labels(old_definition, new_definition, requested_labels)+ except Exception:+ LOG.debug("Frontier diff failed, falling back to full run", exc_info=True)+ return requested_labels, {}, requested_labels[0]++ earliest = _earliest_invalidated(requested_labels, invalidated)+ if earliest is None:+ # No invalidation at all — unchanged request. Use first requested as frontier.+ frontier = requested_labels[0]+ return _seed_for_frontier(requested_labels, frontier, verified_outputs, new_definition)++ # Ensure the prefix before the earliest invalidated label is all in the+ # verified prefix from a successful prior run. Otherwise we have no+ # trusted anchor — fall back to the full requested list.+ prefix_in_requested = [label for label in requested_labels if label != earliest]+ prefix_in_requested = prefix_in_requested[: requested_labels.index(earliest)]+ if not all(label in verified_prefix_set for label in prefix_in_requested):+ return requested_labels, {}, requested_labels[0]++ old_by_label = _blocks_by_label(old_definition)+ is_append_only = earliest not in old_by_label+ if is_append_only:+ # Case A — append-after-success. The earliest invalidated label is a+ # new block that didn't exist in the prior definition, so the verified+ # prefix represents the browser state just before it. Start there.+ return _seed_for_frontier(requested_labels, earliest, verified_outputs, new_definition)++ # Edit-in-place. We lack a browser-anchor signal, so we cannot safely+ # rerun just the edited block (the browser is at post-prefix state, not+ # pre-edit state). Walk back to the nearest upstream state-establishing+ # block within the requested chain. Falls back to the full requested list+ # if no safe upstream anchor can be identified.+ anchor = _nearest_upstream_state_establisher(requested_labels, earliest, new_definition)+ if anchor is None:+ return requested_labels, {}, requested_labels[0]+ return _seed_for_frontier(requested_labels, anchor, verified_outputs, new_definition)+++def _seed_for_frontier(+ requested_labels: list[str],+ frontier: str,+ verified_outputs: dict[str, Any],+ new_definition: Any,+) -> tuple[list[str], dict[str, Any], str]:+ try:+ idx = requested_labels.index(frontier)+ except ValueError:+ return requested_labels, {}, requested_labels[0]+ labels_to_execute = requested_labels[idx:]+ prefix_labels = requested_labels[:idx]+ if not prefix_labels:+ return labels_to_execute, {}, frontier+ # Only seed outputs that are actually referenced by the frontier suffix.+ # Over-seeding would weaken the "seed what downstream needs" discipline+ # and risks masking bugs where a block references an output that doesn't+ # flow through the normal {{label_output}} path.+ needed = _referenced_output_labels(labels_to_execute, new_definition)+ prefix_needed = [label for label in prefix_labels if label in needed]+ seed: dict[str, Any] = {}+ for label in prefix_needed:+ if label in verified_outputs:+ seed[label] = verified_outputs[label]+ else:+ # A referenced output is missing from the verified cache — we+ # can't safely seed just the suffix. Fall back to the full list.+ return requested_labels, {}, requested_labels[0]+ return labels_to_execute, seed, frontier+++async def _run_blocks_and_collect_debug(+ params: dict[str, Any],+ ctx: AgentContext,+ *,+ labels_to_execute: list[str] | None = None,+ block_outputs_to_seed: dict[str, Any] | None = None,+ frontier_start_label: str | None = None,+) -> dict[str, Any]:+ block_labels = params["block_labels"]+ if not block_labels:+ return {"ok": False, "error": "block_labels must not be empty"}++ labels_to_execute = list(labels_to_execute) if labels_to_execute else list(block_labels)+ block_outputs_to_seed = block_outputs_to_seed or {}+ if frontier_start_label is None:+ frontier_start_label = labels_to_execute[0] if labels_to_execute else None++ ctx.last_requested_block_labels = list(block_labels)+ ctx.last_executed_block_labels = list(labels_to_execute)+ ctx.last_frontier_start_label = frontier_start_label++ # Verified state is NOT invalidated pre-run. On a failed / partial run we+ # want the prior verified prefix preserved so the next edit can still use+ # the optimization. YAML-diff-based invalidation for edited/downstream+ # labels happens in update_and_run_blocks_tool at edit time, which is the+ # right moment to drop stale outputs. Full success at the end of this+ # function updates verified state in place (overwriting re-run labels).++ workflow = await app.DATABASE.workflows.get_workflow_by_permanent_id(+ workflow_permanent_id=ctx.workflow_permanent_id,+ organization_id=ctx.organization_id,+ )+ if not workflow:+ return {"ok": False, "error": f"Workflow not found: {ctx.workflow_permanent_id}"}++ for label in block_labels:+ if not workflow.get_output_parameter(label):+ return {"ok": False, "error": f"Block label not found in saved workflow: {label!r}"}++ from skyvern.forge.sdk.schemas.organizations import Organization+ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRequestBody+ from skyvern.services import workflow_service++ org = await app.DATABASE.organizations.get_organization(organization_id=ctx.organization_id)+ if not org:+ return {"ok": False, "error": "Organization not found"}++ organization = Organization.model_validate(org)++ user_params: dict[str, Any] = params.get("parameters") or {}+ all_workflow_params, all_output_params = await asyncio.gather(+ app.WORKFLOW_SERVICE.get_workflow_parameters(workflow_id=workflow.workflow_id),+ app.DATABASE.workflow_params.get_workflow_output_parameters(workflow_id=workflow.workflow_id),+ )++ # Short-circuit before a wasted workflow execution when the definition+ # JSON has drifted from the persisted parameter rows that runtime reads.+ invariant_error = _parameter_binding_invariant_error(workflow, all_workflow_params, all_output_params)+ if invariant_error is not None:+ summary, missing_persisted, missing_from_definition = invariant_error+ return {+ "ok": False,+ "error": summary,+ "data": {+ "workflow_run_id": None,+ "overall_status": "failed",+ "failure_reason": summary,+ "requested_block_labels": list(block_labels),+ "executed_block_labels": [],+ "frontier_start_label": None,+ "blocks": [],+ "failure_categories": [+ {+ "category": "PARAMETER_BINDING_ERROR",+ "confidence_float": 0.99,+ "reasoning": "Pre-run invariant: workflow_definition and persisted parameter rows disagree",+ }+ ],+ "missing_persisted": missing_persisted,+ "missing_from_definition": missing_from_definition,+ },+ }++ data: dict[str, Any] = {}+ for wp in all_workflow_params:+ if wp.key in user_params:+ data[wp.key] = user_params[wp.key]+ elif wp.default_value is not None:+ data[wp.key] = wp.default_value+ else:+ placeholder = _placeholder_for_parameter_type(wp.workflow_parameter_type)+ if placeholder is not None:+ data[wp.key] = placeholder+ LOG.info(+ "Auto-filled missing workflow parameter for copilot test run",+ parameter_key=wp.key,+ parameter_type=str(wp.workflow_parameter_type),+ )++ workflow_request = WorkflowRequestBody(+ data=data if data else None,+ browser_session_id=ctx.browser_session_id,+ # Copilot test runs don't need scrolling post-action screenshots;+ # the ForgeAgent's split screenshots (used for LLM context) are unaffected.+ max_screenshot_scrolls=0,+ )++ workflow_run = await workflow_service.prepare_workflow(+ workflow_id=ctx.workflow_permanent_id,+ organization=organization,+ workflow_request=workflow_request,+ template=False,+ version=None,+ max_steps=None,+ request_id=None,+ )++ from skyvern.utils.files import initialize_skyvern_state_file++ await initialize_skyvern_state_file(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=ctx.organization_id,+ )++ run_task = asyncio.create_task(+ app.WORKFLOW_SERVICE.execute_workflow(+ workflow_run_id=workflow_run.workflow_run_id,+ api_key="copilot-agent",+ organization=organization,+ browser_session_id=ctx.browser_session_id,+ block_labels=labels_to_execute,+ block_outputs=block_outputs_to_seed or None,+ )+ )++ # Internal poll budget strictly below the SDK timeout. The OpenAI Agents+ # SDK wraps this tool in ``asyncio.wait_for(..., timeout=RUN_BLOCKS_DEBUG_TIMEOUT_SECONDS)``+ # (see openai-agents-python tool.py:invoke_function_tool). If our poll and+ # the SDK timeout share the same budget, the SDK wins the race, cancels+ # the tool coroutine, and our orderly cleanup branch below never runs —+ # leaving ``run_task`` as an orphan that runs to natural completion.+ # Leaving 10 s headroom ensures the orderly path fires first in the+ # normal-slow case; the outer ``except asyncio.CancelledError`` covers+ # the abnormal case where we get cancelled anyway.+ max_poll = max(1, RUN_BLOCKS_DEBUG_TIMEOUT_SECONDS - 10)+ poll_interval = 2.0+ elapsed = 0.0+ final_status = None+ run: Any = None++ try:+ while elapsed < max_poll:+ await asyncio.sleep(poll_interval)+ elapsed += poll_interval++ if run_task.done():+ run = await app.DATABASE.workflow_runs.get_workflow_run(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=ctx.organization_id,+ )+ if run and WorkflowRunStatus(run.status).is_final():+ final_status = run.status+ break++ if await ctx.stream.is_disconnected():+ await _cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id)+ return {"ok": False, "error": "Client disconnected during block execution."}++ run = await app.DATABASE.workflow_runs.get_workflow_run(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=ctx.organization_id,+ )+ if run and WorkflowRunStatus(run.status).is_final():+ final_status = run.status+ break++ if final_status is None:+ await _cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id)+ timeout_msg = (+ f"Block execution timed out after {max_poll}s. "+ f"Run ID: {workflow_run.workflow_run_id}. "+ f"The task was likely stuck repeating failing actions. "+ f"Consider: simplifying the navigation_goal, using a more specific URL, "+ f"adding a dismiss-popup step, or concluding the site is not automatable."+ )+ if ctx.browser_session_id:+ try:+ browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(+ session_id=ctx.browser_session_id,+ organization_id=ctx.organization_id,+ )+ if browser_state:+ page = await browser_state.get_or_create_page()+ timeout_msg += f" Browser was on: {page.url}"+ except Exception:+ pass+ return {"ok": False, "error": timeout_msg}+ except asyncio.CancelledError:+ # The SDK's @function_tool(timeout=...) cancelled us mid-poll. Shield+ # the cleanup so the parent cancellation can't interrupt it mid-await.+ # If the shield itself is cancelled, fall back to a detached task+ # that outlives tool teardown and still reconciles workflow state.+ try:+ await asyncio.shield(_cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id))+ except asyncio.CancelledError:+ fallback = asyncio.ensure_future(_cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id))+ _DETACHED_CLEANUP_TASKS.add(fallback)+ fallback.add_done_callback(_DETACHED_CLEANUP_TASKS.discard)+ fallback.add_done_callback(_log_detached_cleanup_failure)+ raise+ finally:+ # Belt and braces. If any exit path above missed a cancel — e.g. an+ # unexpected exception bubbling out of the poll loop — make sure the+ # run_task is at least signaled to cancel so we don't leak it.+ if not run_task.done():+ run_task.cancel()++ if run and run.browser_session_id:+ ctx.browser_session_id = run.browser_session_id++ blocks = await app.DATABASE.observer.get_workflow_run_blocks(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=ctx.organization_id,+ )++ results = []+ block_outputs_by_label: dict[str, Any] = {}+ for block in blocks:+ block_result: dict[str, Any] = {+ "label": block.label,+ "block_type": block.block_type.name if hasattr(block.block_type, "name") else str(block.block_type),+ "status": block.status,+ }+ if block.failure_reason:+ block_result["failure_reason"] = block.failure_reason+ if hasattr(block, "output") and block.output:+ block_result["extracted_data"] = block.output+ if block.label is not None:+ block_outputs_by_label[block.label] = block.output+ results.append(block_result)++ await _attach_action_traces(blocks, results, ctx.organization_id)++ run_ok = WorkflowRunStatus(final_status) == WorkflowRunStatus.completed++ action_trace_summary: list[str] = []+ first_failed = next(+ (r for r in results if r.get("status") in _FAILED_BLOCK_STATUSES and r.get("action_trace")),+ None,+ )+ if first_failed is not None:+ action_trace_summary = _summarize_action_trace(first_failed.get("action_trace"))++ # Compute the action-sequence fingerprint BEFORE we strip action_trace.+ # Stash it on a pending ctx field so update_repeated_failure_state can+ # compare the NEW fingerprint against ctx.last_action_sequence_fingerprint+ # (the PRIOR value) and increment the streak. Never enters the LLM-visible+ # packet. Drives the repeated-action streak that hard-aborts a stuck+ # fill→click→re-fill loop in _tool_loop_error.+ ctx.pending_action_sequence_fingerprint = compute_action_sequence_fingerprint(results)++ # Per-block action_trace is for derivation only — keep it out of the+ # compact packet. get_run_results remains the heavier inspection path.+ for entry in results:+ entry.pop("action_trace", None)++ current_url, page_title = await _fallback_page_info(ctx)++ screenshot_b64: str | None = None+ if not run_ok and ctx.browser_session_id:+ try:+ browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(+ session_id=ctx.browser_session_id,+ organization_id=ctx.organization_id,+ )+ if browser_state:+ page = await browser_state.get_or_create_page()+ screenshot_bytes = await page.screenshot(type="png")+ screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")+ except Exception:+ LOG.debug("Failed to capture post-run screenshot", exc_info=True)++ result_data: dict[str, Any] = {+ "workflow_run_id": workflow_run.workflow_run_id,+ "browser_session_id": ctx.browser_session_id,+ "overall_status": final_status,+ "requested_block_labels": list(block_labels),+ "executed_block_labels": list(labels_to_execute),+ "frontier_start_label": frontier_start_label,+ "blocks": results,+ "current_url": current_url,+ "page_title": page_title,+ "action_trace_summary": action_trace_summary,+ }+ if screenshot_b64 is not None:+ result_data["screenshot_base64"] = screenshot_b64+ if not run_ok and run and getattr(run, "failure_reason", None):+ result_data["failure_reason"] = run.failure_reason++ # Update verified prefix state ONLY on a fully-successful run. A failed+ # suffix run leaves the browser in post-failure state, so we must not+ # trust blocks that individually succeeded inside it.+ if run_ok and all(r.get("status") == "completed" for r in results):+ for label, output in block_outputs_by_label.items():+ ctx.verified_block_outputs[label] = output+ existing_prefix = list(getattr(ctx, "verified_prefix_labels", []) or [])+ existing_set = set(existing_prefix)+ for label in labels_to_execute:+ if label not in existing_set:+ existing_prefix.append(label)+ existing_set.add(label)+ ctx.verified_prefix_labels = existing_prefix++ return {+ "ok": run_ok,+ "data": result_data,+ }+++async def _get_run_results(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:+ workflow_run_id = params.get("workflow_run_id")++ if not workflow_run_id:+ # Include every final state so the agent can inspect failures via the+ # fallback. Non-final states (created/queued/running/paused) remain+ # excluded — reading block records from an in-flight run is unsafe.+ runs = await app.WORKFLOW_SERVICE.get_workflow_runs_for_workflow_permanent_id(+ workflow_permanent_id=ctx.workflow_permanent_id,+ organization_id=ctx.organization_id,+ page=1,+ page_size=1,+ status=[+ WorkflowRunStatus.completed,+ WorkflowRunStatus.failed,+ WorkflowRunStatus.terminated,+ WorkflowRunStatus.canceled,+ WorkflowRunStatus.timed_out,+ ],+ )+ if not runs:+ return {"ok": False, "error": "No runs found for this workflow."}+ workflow_run_id = runs[0].workflow_run_id++ run = await app.DATABASE.workflow_runs.get_workflow_run(+ workflow_run_id=workflow_run_id,+ organization_id=ctx.organization_id,+ )+ if not run:+ return {"ok": False, "error": f"Workflow run not found: {workflow_run_id}"}++ blocks = await app.DATABASE.observer.get_workflow_run_blocks(+ workflow_run_id=workflow_run_id,+ organization_id=ctx.organization_id,+ )++ results = []+ for block in blocks:+ block_result: dict[str, Any] = {+ "label": block.label,+ "block_type": block.block_type.name if hasattr(block.block_type, "name") else str(block.block_type),+ "status": block.status,+ }+ if block.failure_reason:+ block_result["failure_reason"] = block.failure_reason+ output = truncate_output(getattr(block, "output", None))+ if output:+ block_result["output"] = output+ results.append(block_result)++ await _attach_action_traces(blocks, results, ctx.organization_id)+ await _attach_failed_block_screenshots(blocks, results, ctx.organization_id)++ result_data: dict[str, Any] = {+ "workflow_run_id": workflow_run_id,+ "overall_status": run.status,+ "blocks": results,+ }+ if getattr(run, "failure_reason", None):+ result_data["failure_reason"] = run.failure_reason++ return {+ "ok": True,+ "data": result_data,+ }+++async def _fallback_page_info(ctx: AgentContext) -> tuple[str, str]:+ if not ctx.browser_session_id:+ return "", ""+ try:+ from skyvern.cli.core.session_manager import get_page++ page, _ = await get_page(session_id=ctx.browser_session_id)+ if page:+ return page.url, await page.title()+ except Exception:+ pass+ return "", ""+++async def _resolve_url_title(raw: dict[str, Any], ctx: AgentContext) -> tuple[str, str]:+ """Extract URL and title from raw MCP result, falling back to live page info."""+ browser_ctx = raw.get("browser_context", {})+ url = browser_ctx.get("url", "")+ title = browser_ctx.get("title", "")+ if not url:+ url, fallback_title = await _fallback_page_info(ctx)+ if fallback_title:+ title = fallback_title+ return url, title+++async def _evaluate_pre_hook(+ params: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any] | None:+ expr = params.get("expression", "").lower()+ if ".click()" in expr or ".click(" in expr:+ return {+ "ok": False,+ "error": "Do not use evaluate to click elements. Use the 'click' tool with a CSS selector instead.",+ }+ return None+++_JQUERY_SELECTOR_RE = re.compile(+ r":(?:contains|eq|first|last|gt|lt|nth|has|visible|hidden|checked)\s*\(", re.IGNORECASE+)+++async def _click_pre_hook(+ params: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any] | None:+ selector = params.get("selector", "")+ if not selector:+ return None+ if _JQUERY_SELECTOR_RE.search(selector):+ return {+ "ok": False,+ "error": (+ f"Invalid selector: {selector!r}. "+ "jQuery pseudo-selectors like :contains(), :eq(), :first, :visible are NOT valid CSS. "+ "Use standard CSS selectors instead. Examples: "+ "nth-of-type() instead of :eq(), "+ "[data-attr] or tag.class for filtering, "+ "or use the 'evaluate' tool with JS: "+ "document.querySelectorAll('button').forEach("+ "b => {{ if (b.textContent.includes('Download')) b.click() }})"+ ),+ }+ return None+++async def _navigate_post_hook(+ result: dict[str, Any],+ raw: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any]:+ if result.get("ok"):+ data = result.pop("data", {})+ result["url"] = data.get("url", "")+ result["next_step"] = (+ "Page loaded. You MUST now use evaluate, "+ "get_browser_screenshot, or click to inspect page content "+ "before responding."+ )+ return result+++async def _screenshot_post_hook(+ result: dict[str, Any],+ raw: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any]:+ if result.get("ok") and result.get("data"):+ data = result["data"]+ url, title = await _resolve_url_title(raw, ctx)+ result["data"] = {+ "screenshot_base64": data.get("data", ""),+ "url": url,+ "title": title,+ }+ return result+++async def _click_post_hook(+ result: dict[str, Any],+ raw: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any]:+ if result.get("ok") and result.get("data"):+ data = result["data"]+ url, title = await _resolve_url_title(raw, ctx)+ result["data"] = {+ "selector": data.get("selector", ""),+ "url": url,+ "title": title,+ }+ return result+++async def _type_text_post_hook(+ result: dict[str, Any],+ raw: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any]:+ if result.get("ok") and result.get("data"):+ data = result["data"]+ url, _ = await _resolve_url_title(raw, ctx)+ result["data"] = {+ "selector": data.get("selector", ""),+ "typed_length": data.get("text_length", 0),+ "url": url,+ }+ return result+++async def _evaluate_post_hook(+ result: dict[str, Any],+ raw: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any]:+ if result.get("ok") and result.get("data"):+ result["data"].pop("sdk_equivalent", None)+ if "url" not in result["data"]:+ url, _ = await _resolve_url_title(raw, ctx)+ if url:+ result["data"]["url"] = url+ return result+++async def _scroll_post_hook(+ result: dict[str, Any],+ raw: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any]:+ if result.get("ok") and result.get("data"):+ data = result["data"]+ url, _ = await _resolve_url_title(raw, ctx)+ result["data"] = {+ "direction": data.get("direction", ""),+ "amount": data.get("pixels") or data.get("amount"),+ "url": url,+ }+ return result+++async def _select_option_post_hook(+ result: dict[str, Any],+ raw: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any]:+ if result.get("ok") and result.get("data"):+ data = result["data"]+ url, _ = await _resolve_url_title(raw, ctx)+ result["data"] = {+ "selector": data.get("selector", ""),+ "value": data.get("value", ""),+ "url": url,+ }+ return result+++async def _press_key_post_hook(+ result: dict[str, Any],+ raw: dict[str, Any],+ ctx: AgentContext,+) -> dict[str, Any]:+ if result.get("ok") and result.get("data"):+ data = result["data"]+ url, _ = await _resolve_url_title(raw, ctx)+ result["data"] = {+ "key": data.get("key", ""),+ "selector": data.get("selector", ""),+ "url": url,+ }+ return result+++def get_skyvern_mcp_alias_map() -> dict[str, str]:+ return {+ "get_block_schema": "skyvern_block_schema",+ "validate_block": "skyvern_block_validate",+ "navigate_browser": "skyvern_navigate",+ "get_browser_screenshot": "skyvern_screenshot",+ "evaluate": "skyvern_evaluate",+ "click": "skyvern_click",+ "type_text": "skyvern_type",+ "scroll": "skyvern_scroll",+ "console_messages": "skyvern_console_messages",+ "select_option": "skyvern_select_option",+ "press_key": "skyvern_press_key",+ }+++def _build_skyvern_mcp_overlays() -> dict[str, SchemaOverlay]:+ return {+ "get_block_schema": SchemaOverlay(),+ "validate_block": SchemaOverlay(),+ "navigate_browser": SchemaOverlay(+ description=(+ "Navigate the debug browser to a URL. "+ "Use this to reset browser state or navigate to a starting page before running blocks."+ ),+ hide_params=frozenset({"session_id", "cdp_url"}),+ requires_browser=True,+ post_hook=_navigate_post_hook,+ ),+ "get_browser_screenshot": SchemaOverlay(+ description=(+ "Take a screenshot of the current debug browser session. "+ "Returns a base64-encoded PNG image. "+ "Use this to see what the browser looks like after running blocks."+ ),+ hide_params=frozenset({"session_id", "cdp_url", "selector"}),+ forced_args={"inline": True},+ requires_browser=True,+ post_hook=_screenshot_post_hook,+ ),+ "evaluate": SchemaOverlay(+ description=(+ "Execute JavaScript in the browser and return the result. "+ "Use this to inspect DOM state, read values, or run arbitrary JS."+ ),+ hide_params=frozenset({"session_id", "cdp_url"}),+ requires_browser=True,+ timeout=30,+ pre_hook=_evaluate_pre_hook,+ post_hook=_evaluate_post_hook,+ ),+ "click": SchemaOverlay(+ description=(+ "Click an element in the browser. Use a CSS selector, an AI intent "+ "description, or both for resilient targeting. "+ "IMPORTANT: jQuery pseudo-selectors like :contains(), :eq(), :first, "+ ":visible are NOT valid CSS. Use standard selectors: "+ "'button.download', 'a[href*=\"pdf\"]', '#submit-btn', "+ "'table tr:nth-of-type(2) td a'."+ ),+ hide_params=frozenset({"session_id", "cdp_url", "button", "click_count"}),+ requires_browser=True,+ timeout=15,+ pre_hook=_click_pre_hook,+ post_hook=_click_post_hook,+ ),+ "type_text": SchemaOverlay(+ description=(+ "Type text into an input element. Use a CSS selector, an AI intent "+ "description, or both to target the field. "+ "Optionally clear the field first. Use this for form filling."+ ),+ hide_params=frozenset({"session_id", "cdp_url", "delay"}),+ required_overrides=["text"],+ arg_transforms={"clear_first": "clear"},+ requires_browser=True,+ timeout=15,+ post_hook=_type_text_post_hook,+ ),+ "scroll": SchemaOverlay(+ description=(+ "Scroll the page in a direction (up/down/left/right) by pixel amount, "+ "or scroll a specific element into view using intent or selector. "+ "Use this to reveal content below the fold."+ ),+ hide_params=frozenset({"session_id", "cdp_url"}),+ requires_browser=True,+ post_hook=_scroll_post_hook,+ ),+ "console_messages": SchemaOverlay(+ description=(+ "Read console log messages from the browser. "+ "Use level='error' to find JavaScript errors. "+ "This is a read-only diagnostic tool."+ ),+ hide_params=frozenset({"session_id", "cdp_url"}),+ requires_browser=True,+ ),+ "select_option": SchemaOverlay(+ description=(+ "Select an option from a <select> dropdown. Provide the value to select "+ "and use selector or intent to target the element. "+ "For free-text inputs, use type_text instead."+ ),+ hide_params=frozenset({"session_id", "cdp_url", "timeout"}),+ required_overrides=["value"],+ requires_browser=True,+ timeout=15,+ post_hook=_select_option_post_hook,+ ),+ "press_key": SchemaOverlay(+ description=(+ "Press a keyboard key (Enter, Tab, Escape, ArrowDown, etc.). "+ "Optionally focus an element first via selector or intent. "+ "Use for form submission, tab navigation, or closing dialogs."+ ),+ hide_params=frozenset({"session_id", "cdp_url"}),+ required_overrides=["key"],+ requires_browser=True,+ post_hook=_press_key_post_hook,+ ),+ }+++def _record_workflow_update_result(copilot_ctx: Any, result: dict[str, Any]) -> None:+ if not (result.get("ok") and "_workflow" in result):+ return++ wf = result["_workflow"]+ copilot_ctx.last_workflow = wf+ copilot_ctx.last_workflow_yaml = copilot_ctx.workflow_yaml or None+ data = result.get("data")+ if isinstance(data, dict):+ block_count = data.get("block_count")+ if isinstance(block_count, int):+ copilot_ctx.last_update_block_count = block_count+ copilot_ctx.last_test_ok = None+ copilot_ctx.last_test_failure_reason = None+ copilot_ctx.workflow_persisted = True+++def _analyze_run_blocks(result: dict[str, Any]) -> tuple[str | None, bool, list[dict] | None]:+ """Single-pass analysis of run result blocks.++ Returns ``(anti_bot_match, has_empty_data_blocks, failure_categories)``+ by iterating the block list once. Classification delegates to+ :func:`~skyvern.forge.failure_classifier.classify_from_failure_reason`.+ When ``data["failure_categories"]`` is already populated (pre-run+ short-circuit with no blocks), honor it instead of re-classifying.+ """+ data = result.get("data")+ if not isinstance(data, dict):+ return None, False, None++ anti_bot_match: str | None = None++ precomputed_categories = data.get("failure_categories")+ if isinstance(precomputed_categories, list) and precomputed_categories:+ for cat in precomputed_categories:+ if isinstance(cat, dict) and cat.get("category") == "ANTI_BOT_DETECTION":+ anti_bot_match = cat.get("reasoning", "anti-bot pattern detected")+ break+ return anti_bot_match, False, precomputed_categories++ # Collect texts for scanning and data-block stats in one pass+ texts_to_scan: list[str] = []+ html = data.get("visible_elements_html")+ if isinstance(html, str):+ texts_to_scan.append(html)++ has_data_blocks = False+ any_data_output = False++ blocks = data.get("blocks")+ if isinstance(blocks, list):+ for block in blocks:+ if not isinstance(block, dict):+ continue+ reason = block.get("failure_reason")+ if isinstance(reason, str):+ texts_to_scan.append(reason)+ if block.get("block_type") in _DATA_PRODUCING_BLOCK_TYPES and block.get("status") == "completed":+ has_data_blocks = True+ if _is_meaningful_extracted_data(block.get("extracted_data")):+ any_data_output = True++ combined = "\n".join(texts_to_scan)+ categories = classify_from_failure_reason(combined)+ if categories:+ for cat in categories:+ if cat.get("category") == "ANTI_BOT_DETECTION":+ anti_bot_match = cat.get("reasoning", "anti-bot pattern detected")+ break++ empty_data_blocks = has_data_blocks and not any_data_output+ return anti_bot_match, empty_data_blocks, categories+++def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:+ run_ok = bool(result.get("ok", False))+ copilot_ctx.last_test_ok = run_ok+ copilot_ctx.last_test_failure_reason = None+ copilot_ctx.last_test_suspicious_success = False+ copilot_ctx.last_test_anti_bot = None+ copilot_ctx.last_failure_category_top = None++ anti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result)+ if anti_bot_match:+ copilot_ctx.last_test_anti_bot = anti_bot_match+ if failure_categories:+ top = failure_categories[0]+ if isinstance(top, dict):+ top_category = top.get("category")+ if isinstance(top_category, str):+ copilot_ctx.last_failure_category_top = top_category++ # Expose full failure classification in tool output for agent reasoning+ if failure_categories:+ data = result.get("data")+ if isinstance(data, dict):+ data["failure_categories"] = failure_categories++ if run_ok:+ if empty_data_blocks:+ copilot_ctx.last_test_ok = None+ copilot_ctx.last_test_suspicious_success = True+ copilot_ctx.null_data_streak_count = getattr(copilot_ctx, "null_data_streak_count", 0) + 1+ copilot_ctx.last_test_failure_reason = (+ "All blocks completed but data-producing blocks "+ "(extraction/text_prompt) produced no meaningful output "+ "(missing, empty, or all-null fields). "+ "The workflow may not be working correctly."+ )+ update_repeated_failure_state(copilot_ctx, result)+ return+ copilot_ctx.failed_test_nudge_count = 0+ copilot_ctx.null_data_streak_count = 0+ copilot_ctx.last_failed_workflow_yaml = None+ update_repeated_failure_state(copilot_ctx, result)+ return++ copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)++ data = result.get("data")+ if isinstance(data, dict):+ blocks = data.get("blocks")+ if isinstance(blocks, list):+ for block in blocks:+ if isinstance(block, dict) and block.get("failure_reason"):+ copilot_ctx.last_test_failure_reason = str(block["failure_reason"])+ break+ if result.get("error") and copilot_ctx.last_test_failure_reason is None:+ copilot_ctx.last_test_failure_reason = str(result["error"])+ update_repeated_failure_state(copilot_ctx, result)+++@function_tool(name_override="update_workflow")+async def update_workflow_tool(+ ctx: RunContextWrapper,+ workflow_yaml: str,+) -> str:+ """Validate and update the workflow YAML definition.+ Provide the complete workflow YAML as a string.+ Returns the validated workflow or validation errors.+ """+ copilot_ctx = ctx.context+ loop_error = _tool_loop_error(copilot_ctx, "update_workflow")+ if loop_error:+ return json.dumps({"ok": False, "error": loop_error})++ with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, copilot_ctx)+ _record_workflow_update_result(copilot_ctx, result)+ sanitized = sanitize_tool_result_for_llm("update_workflow", result)+ return json.dumps(sanitized)+++@function_tool(name_override="list_credentials")+async def list_credentials_tool(+ ctx: RunContextWrapper,+ page: int = 1,+ page_size: int = 10,+) -> str:+ """List stored credentials (metadata only — never passwords or secrets).+ Use this to find credential IDs for login blocks.+ """+ copilot_ctx = ctx.context+ loop_error = _tool_loop_error(copilot_ctx, "list_credentials")+ if loop_error:+ return json.dumps({"ok": False, "error": loop_error})++ result = await _list_credentials({"page": page, "page_size": page_size}, copilot_ctx)+ sanitized = sanitize_tool_result_for_llm("list_credentials", result)+ return json.dumps(sanitized)+++async def _frontier_plan_for_current_workflow(+ ctx: AgentContext, block_labels: list[str]+) -> tuple[list[str], dict[str, Any], str | None]:+ """Plan execution for the plain (no YAML update) path."""+ prior_definition = await _get_prior_workflow_definition(ctx)+ return _plan_frontier(ctx, block_labels, prior_definition, prior_definition)+++def _run_blocks_span_data(+ block_labels: list[str],+ labels_to_execute: list[str],+ frontier_start_label: str | None,+ seeded_outputs: dict[str, Any],+ ctx: Any,+) -> dict[str, Any]:+ return {+ "requested_block_labels": block_labels,+ "executed_block_labels": labels_to_execute,+ "frontier_start_label": frontier_start_label,+ "seeded_output_count": len(seeded_outputs or {}),+ "repeated_failure_streak_count": int(getattr(ctx, "repeated_failure_streak_count", 0) or 0),+ "block_count": len(block_labels),+ }+++@function_tool(+ name_override="run_blocks_and_collect_debug",+ timeout=RUN_BLOCKS_DEBUG_TIMEOUT_SECONDS,+ strict_mode=False,+)+async def run_blocks_tool(+ ctx: RunContextWrapper,+ block_labels: list[str],+ parameters: dict[str, Any] | None = None,+) -> Any:+ """Run one or more blocks of the current workflow, wait for completion,+ and return compact debug output (status, failure reason, visible elements).+ The workflow must be saved before running blocks.+ Block labels must match labels in the saved workflow.++ Pass runtime values for workflow parameters via the `parameters` dict —+ keys must match the workflow parameter `key` field. When the user has+ supplied concrete values in their message (names, emails, IDs), pass them+ on the first call rather than letting the workflow fall back to+ placeholders. For sensitive values (password, secret, token, api_key,+ credential, totp, otp, one_time_code, private_key, auth), prefer calling+ list_credentials first; if a stored credential matches, reference its+ credential_id via a credential parameter. If no matching credential+ exists and the user supplied an inline value, you may pass it via+ `parameters` — values under those sensitive key names are redacted from+ the outbound client stream and the tool-execution trace.+ """+ copilot_ctx = ctx.context+ loop_error = _tool_loop_error(copilot_ctx, "run_blocks_and_collect_debug")+ if loop_error:+ return json.dumps({"ok": False, "error": loop_error})++ labels_to_execute, block_outputs_to_seed, frontier_start_label = await _frontier_plan_for_current_workflow(+ copilot_ctx, block_labels+ )++ with copilot_span(+ "run_blocks",+ data=_run_blocks_span_data(+ block_labels,+ labels_to_execute,+ frontier_start_label,+ block_outputs_to_seed,+ copilot_ctx,+ ),+ ):+ result = await _run_blocks_and_collect_debug(+ {"block_labels": block_labels, "parameters": parameters or {}},+ copilot_ctx,+ labels_to_execute=labels_to_execute,+ block_outputs_to_seed=block_outputs_to_seed,+ frontier_start_label=frontier_start_label,+ )+ _record_run_blocks_result(copilot_ctx, result)+ enqueue_screenshot_from_result(copilot_ctx, result)++ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)+ return json.dumps(sanitized)+++@function_tool(name_override="get_run_results")+async def get_run_results_tool(+ ctx: RunContextWrapper,+ workflow_run_id: str | None = None,+) -> str:+ """Fetch results from a previous workflow run.+ Returns block statuses, failure reasons, and output data.+ If workflow_run_id is omitted, fetches the most recently created finished+ run (completed, failed, canceled, terminated, or timed_out — excludes+ in-flight runs). For unambiguous results in concurrent-run scenarios,+ pass an explicit workflow_run_id from a prior tool response.+ """+ copilot_ctx = ctx.context+ loop_error = _tool_loop_error(copilot_ctx, "get_run_results")+ if loop_error:+ return json.dumps({"ok": False, "error": loop_error})++ params: dict[str, Any] = {}+ if workflow_run_id:+ params["workflow_run_id"] = workflow_run_id+ result = await _get_run_results(params, copilot_ctx)+ sanitized = sanitize_tool_result_for_llm("get_run_results", result)+ return json.dumps(sanitized)+++@function_tool(+ name_override="update_and_run_blocks",+ timeout=RUN_BLOCKS_DEBUG_TIMEOUT_SECONDS,+ strict_mode=False,+)+async def update_and_run_blocks_tool(+ ctx: RunContextWrapper,+ workflow_yaml: str,+ block_labels: list[str],+ parameters: dict[str, Any] | None = None,+) -> Any:+ """Update the workflow YAML and immediately run the specified blocks in one step.+ Use this instead of calling update_workflow and run_blocks_and_collect_debug separately.+ The workflow must validate successfully before blocks are run.++ Pass runtime values for workflow parameters via the `parameters` dict —+ keys must match the workflow parameter `key` field. When the user has+ supplied concrete values in their message (names, emails, IDs), pass them+ on the first call rather than letting the workflow fall back to+ placeholders. For sensitive values (password, secret, token, api_key,+ credential, totp, otp, one_time_code, private_key, auth), prefer calling+ list_credentials first; if a stored credential matches, reference its+ credential_id via a credential parameter. If no matching credential+ exists and the user supplied an inline value, you may pass it via+ `parameters` — values under those sensitive key names are redacted from+ the outbound client stream and the tool-execution trace.+ """+ copilot_ctx = ctx.context+ loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks")+ if loop_error:+ return json.dumps({"ok": False, "error": loop_error})++ # Snapshot the prior workflow definition BEFORE _update_workflow saves+ # the new one — we need the pre-update state to diff against.+ prior_definition = await _get_prior_workflow_definition(copilot_ctx)++ # Step 1: Update the workflow+ with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):+ update_result = await _update_workflow({"workflow_yaml": workflow_yaml}, copilot_ctx)+ _record_workflow_update_result(copilot_ctx, update_result)++ if not update_result.get("ok"):+ sanitized = sanitize_tool_result_for_llm("update_workflow", update_result)+ return json.dumps(sanitized)++ # Step 2: Compute frontier and run the blocks+ new_definition = None+ if copilot_ctx.last_workflow is not None:+ new_definition = getattr(copilot_ctx.last_workflow, "workflow_definition", None)++ # YAML-diff-based invalidation: drop any verified-prefix entries whose+ # block config changed (or are downstream of a change) BEFORE the next+ # run plans the frontier. This is the only point where we shrink verified+ # state — failed runs leave it intact so subsequent edits can still use+ # the optimization. On append-only diffs this is a no-op.+ if prior_definition is not None and new_definition is not None and copilot_ctx.verified_prefix_labels:+ try:+ edit_invalidated = _find_invalidated_labels(+ prior_definition, new_definition, list(copilot_ctx.verified_prefix_labels)+ )+ except Exception:+ edit_invalidated = set()+ if edit_invalidated:+ for label in edit_invalidated:+ copilot_ctx.verified_block_outputs.pop(label, None)+ copilot_ctx.verified_prefix_labels = [+ label for label in copilot_ctx.verified_prefix_labels if label not in edit_invalidated+ ]++ labels_to_execute, block_outputs_to_seed, frontier_start_label = _plan_frontier(+ copilot_ctx, block_labels, prior_definition, new_definition+ )++ with copilot_span(+ "run_blocks",+ data=_run_blocks_span_data(+ block_labels,+ labels_to_execute,+ frontier_start_label,+ block_outputs_to_seed,+ copilot_ctx,+ ),+ ):+ run_result = await _run_blocks_and_collect_debug(+ {"block_labels": block_labels, "parameters": parameters or {}},+ copilot_ctx,+ labels_to_execute=labels_to_execute,+ block_outputs_to_seed=block_outputs_to_seed,+ frontier_start_label=frontier_start_label,+ )+ _record_run_blocks_result(copilot_ctx, run_result)+ enqueue_screenshot_from_result(copilot_ctx, run_result)++ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", run_result)+ return json.dumps(sanitized)+++NATIVE_TOOLS = [+ update_workflow_tool,+ list_credentials_tool,+ run_blocks_tool,+ get_run_results_tool,+ update_and_run_blocks_tool,+]diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 0451e8115..cae134615 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -1077,6 +1077,13 @@ class WorkflowService:name=f"browser_session_renewal_{workflow_run_id}",)+ # Captured inside the try and consumed in the outer finally so status+ # finalization runs even when the body is cancelled or raises. Stays+ # None if we were cancelled before the block-execution step completed;+ # in that case there's no terminal-state intent to restore.+ pre_finally_status: WorkflowRunStatus | None = None+ pre_finally_failure_reason: str | None = None+try:# Check if there's a related workflow script that should be used insteadworkflow_script, _, script_is_pinned = await workflow_script_service.get_workflow_script(@@ -1205,14 +1212,37 @@ class WorkflowService:organization=organization,browser_session_id=browser_session_id,)-- workflow_run = await self._finalize_workflow_run_status(- workflow_run_id=workflow_run_id,- workflow_run=workflow_run,- pre_finally_status=pre_finally_status,- pre_finally_failure_reason=pre_finally_failure_reason,- )finally:+ # Shielded finalize runs even when the try body was cancelled+ # mid-flight (e.g. the copilot tool's orphan-task cancel path, or+ # any outer caller that cancels execute_workflow). Without this,+ # cancellation between the temporary ``running`` write above and+ # the original finalize call leaked ``running``/``canceled`` rows+ # in place of the real terminal reason. When pre_finally_status is+ # still ``None`` (cancellation landed before block execution+ # completed), there's no captured intent to restore and we skip.+ if pre_finally_status is not None:+ try:+ workflow_run = await asyncio.shield(+ self._finalize_workflow_run_status(+ workflow_run_id=workflow_run_id,+ workflow_run=workflow_run,+ pre_finally_status=pre_finally_status,+ pre_finally_failure_reason=pre_finally_failure_reason,+ )+ )+ except BaseException:+ # Catch BaseException (not Exception) so a second+ # ``CancelledError`` arriving during the shielded await —+ # plausible when the copilot's detached cancellation+ # fallback re-cancels ``run_task`` — does not escape this+ # block and skip ``clean_up_workflow`` below.+ LOG.warning(+ "Finalize failed during execute_workflow cleanup",+ workflow_run_id=workflow_run_id,+ exc_info=True,+ )+if renewal_task is not None and not renewal_task.done():renewal_task.cancel()try:diff --git a/tests/unit/test_copilot_action_fingerprint.py b/tests/unit/test_copilot_action_fingerprint.pynew file mode 100644index 000000000..dfd677641--- /dev/null+++ b/tests/unit/test_copilot_action_fingerprint.py@@ -0,0 +1,96 @@+"""Tests for the action-sequence fingerprint compute and the hard-abort+short-circuit in ``_tool_loop_error``.++The streak counter that drives the abort is owned by+``failure_tracking.update_repeated_failure_state`` (stack 03). These tests+cover only the tools.py-side behavior:++- ``compute_action_sequence_fingerprint`` is stable across runs that fire+ the same action shape (independent of reasoning text / status).+- ``compute_action_sequence_fingerprint`` distinguishes different sequences.+- ``_tool_loop_error`` returns a hard-abort message when the streak crosses+ ``REPEATED_ACTION_STREAK_ABORT_AT`` for a block-running tool.+- The hard abort does NOT fire for non-block-running tools, regardless of+ streak height.+"""++from __future__ import annotations++from types import SimpleNamespace++from skyvern.forge.sdk.copilot.failure_tracking import compute_action_sequence_fingerprint+from skyvern.forge.sdk.copilot.tools import (+ REPEATED_ACTION_STREAK_ABORT_AT,+ _tool_loop_error,+)+++def _ctx_with_streak(streak: int) -> SimpleNamespace:+ return SimpleNamespace(+ consecutive_tool_tracker=[],+ repeated_action_fingerprint_streak_count=streak,+ )+++def test_compute_action_sequence_fingerprint_stable_for_same_sequence() -> None:+ trace_a = [+ {"action": "input_text", "element": "elem-name", "reasoning": "r1", "status": "failed"},+ {"action": "input_text", "element": "elem-email", "reasoning": "r2", "status": "failed"},+ {"action": "click", "element": "elem-submit", "reasoning": "r3", "status": "failed"},+ ]+ trace_b = [+ {"action": "input_text", "element": "elem-name", "reasoning": "different_text", "status": "failed"},+ {"action": "input_text", "element": "elem-email", "reasoning": "other", "status": "failed"},+ {"action": "click", "element": "elem-submit", "reasoning": "third", "status": "failed"},+ ]+ fp_a = compute_action_sequence_fingerprint([{"action_trace": trace_a}])+ fp_b = compute_action_sequence_fingerprint([{"action_trace": trace_b}])+ assert fp_a is not None+ # Reasoning / status are excluded from the fingerprint on purpose — only+ # the (action, element) shape matters for detecting a retry loop.+ assert fp_a == fp_b+++def test_compute_action_sequence_fingerprint_none_when_trace_missing() -> None:+ assert compute_action_sequence_fingerprint([]) is None+ assert compute_action_sequence_fingerprint([{"status": "completed"}]) is None+ assert compute_action_sequence_fingerprint([{"action_trace": []}]) is None+++def test_compute_action_sequence_fingerprint_distinguishes_different_sequences() -> None:+ trace_a = [{"action": "click", "element": "btn-a"}]+ trace_b = [{"action": "click", "element": "btn-b"}]+ trace_c = [{"action": "input_text", "element": "btn-a"}]+ fp_a = compute_action_sequence_fingerprint([{"action_trace": trace_a}])+ fp_b = compute_action_sequence_fingerprint([{"action_trace": trace_b}])+ fp_c = compute_action_sequence_fingerprint([{"action_trace": trace_c}])+ assert fp_a != fp_b+ assert fp_a != fp_c+ assert fp_b != fp_c+++def test_tool_loop_error_fires_hard_abort_on_block_running_tools_when_streak_high() -> None:+ ctx = _ctx_with_streak(REPEATED_ACTION_STREAK_ABORT_AT)++ error = _tool_loop_error(ctx, "run_blocks_and_collect_debug")+ assert error is not None+ assert "Repeated-action abort" in error++ error_update = _tool_loop_error(ctx, "update_and_run_blocks")+ assert error_update is not None+ assert "Repeated-action abort" in error_update+++def test_tool_loop_error_does_not_fire_for_non_block_running_tools() -> None:+ ctx = _ctx_with_streak(REPEATED_ACTION_STREAK_ABORT_AT + 5)++ # Planning/metadata tools keep their existing loop-detection behavior+ # and are not affected by the block-run streak.+ assert _tool_loop_error(ctx, "update_workflow") is None+ assert _tool_loop_error(ctx, "list_credentials") is None+ assert _tool_loop_error(ctx, "get_run_results") is None+++def test_tool_loop_error_does_not_fire_below_threshold() -> None:+ ctx = _ctx_with_streak(REPEATED_ACTION_STREAK_ABORT_AT - 1)+ assert _tool_loop_error(ctx, "run_blocks_and_collect_debug") is Nonediff --git a/tests/unit/test_copilot_cancel_helpers.py b/tests/unit/test_copilot_cancel_helpers.pynew file mode 100644index 000000000..ac12898d3--- /dev/null+++ b/tests/unit/test_copilot_cancel_helpers.py@@ -0,0 +1,148 @@+"""Tests for the copilot orphan-workflow cancellation helpers.++Covers:+- ``_cancel_run_task_if_not_final`` cancels ``run_task`` and writes the+ conditional cancel exactly once.+- A SUCCESS path (run_task completes on its own) still calls the conditional+ cancel, but because the row is terminal the real helper would be a no-op.+- An SDK-realistic ``asyncio.wait_for`` timeout around the tool coroutine does+ not leave ``run_task`` running in the background.+"""++from __future__ import annotations++import asyncio+from typing import Any++import pytest++from skyvern.forge.sdk.copilot.tools import _cancel_run_task_if_not_final+++class _FakeService:+ def __init__(self) -> None:+ self.mark_calls: list[str] = []+ self.raise_on_mark: Exception | None = None++ async def mark_workflow_run_as_canceled_if_not_final(+ self,+ workflow_run_id: str,+ ) -> Any:+ self.mark_calls.append(workflow_run_id)+ if self.raise_on_mark is not None:+ raise self.raise_on_mark+ return None+++async def test_cancel_helper_cancels_task_and_writes_conditional_cancel(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from skyvern.forge import app as forge_app++ service = _FakeService()+ monkeypatch.setattr(forge_app, "WORKFLOW_SERVICE", service)++ async def long_running() -> None:+ await asyncio.sleep(60)++ run_task = asyncio.create_task(long_running())++ await _cancel_run_task_if_not_final(run_task, workflow_run_id="wr_1")++ assert run_task.cancelled() or run_task.done()+ assert service.mark_calls == ["wr_1"]+++async def test_cancel_helper_does_not_raise_on_mark_failure(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Secondary errors during cleanup are logged, not propagated — otherwise+ they would replace the original timeout/cancellation surface."""+ from skyvern.forge import app as forge_app++ service = _FakeService()+ service.raise_on_mark = RuntimeError("DB is down")+ monkeypatch.setattr(forge_app, "WORKFLOW_SERVICE", service)++ async def long_running() -> None:+ await asyncio.sleep(60)++ run_task = asyncio.create_task(long_running())++ # Must not raise despite the mark raising.+ await _cancel_run_task_if_not_final(run_task, workflow_run_id="wr_2")+++async def test_cancel_helper_handles_already_completed_run_task(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """When run_task has already finished (natural completion), the helper+ should still issue the conditional cancel — it is a no-op at the DB layer+ if the row is already terminal, so the result is harmless."""+ from skyvern.forge import app as forge_app++ service = _FakeService()+ monkeypatch.setattr(forge_app, "WORKFLOW_SERVICE", service)++ async def quick() -> None:+ return++ run_task = asyncio.create_task(quick())+ await run_task++ await _cancel_run_task_if_not_final(run_task, workflow_run_id="wr_3")+ assert service.mark_calls == ["wr_3"]+++async def test_sdk_style_wait_for_timeout_does_not_leak_background_work(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Exercises the production failure mode: the OpenAI Agents SDK wraps the+ tool coroutine in ``asyncio.wait_for(..., timeout=N)`` and cancels it on+ timeout. Our CancelledError branch must cancel ``run_task`` through the+ helper so no orphan work is left behind."""+ from skyvern.forge import app as forge_app++ service = _FakeService()+ monkeypatch.setattr(forge_app, "WORKFLOW_SERVICE", service)++ run_task_ref: dict[str, asyncio.Task] = {}+ workflow_work_completed = asyncio.Event()++ async def tool_body() -> None:+ async def workflow_body() -> None:+ try:+ await asyncio.sleep(60)+ except asyncio.CancelledError:+ workflow_work_completed.set()+ raise++ run_task = asyncio.create_task(workflow_body())+ run_task_ref["run_task"] = run_task+ try:+ # Simulate the inner poll loop.+ while True:+ await asyncio.sleep(0.05)+ except asyncio.CancelledError:+ try:+ await asyncio.shield(_cancel_run_task_if_not_final(run_task, workflow_run_id="wr_sdk"))+ except asyncio.CancelledError:+ # Detached fallback mirror of the production path.+ fallback = asyncio.ensure_future(_cancel_run_task_if_not_final(run_task, workflow_run_id="wr_sdk"))+ await asyncio.wait_for(asyncio.shield(fallback), timeout=5.0)+ raise++ tool_task = asyncio.ensure_future(tool_body())+ with pytest.raises(asyncio.TimeoutError):+ await asyncio.wait_for(tool_task, timeout=0.2)++ # Workflow's CancelledError handler should have fired via our helper.+ await asyncio.wait_for(workflow_work_completed.wait(), timeout=1.0)++ assert "run_task" in run_task_ref+ assert run_task_ref["run_task"].cancelled() or run_task_ref["run_task"].done()+ assert service.mark_calls == ["wr_sdk"]diff --git a/tests/unit/test_workflow_service_cancel.py b/tests/unit/test_workflow_service_cancel.pyindex 694482f9b..edec58410 100644--- a/tests/unit/test_workflow_service_cancel.py+++ b/tests/unit/test_workflow_service_cancel.py@@ -6,10 +6,14 @@- ``execute_workflow_webhook`` returns cleanly when the workflow row has beensoft-deleted mid-run — it must not raise ``WorkflowNotFound`` from thecleanup path.+- The cancellation-safe finalize pattern used in ``execute_workflow``'s outer+ ``finally`` runs ``_finalize_workflow_run_status`` via ``asyncio.shield``+ so an outer cancel mid-body still restores the real terminal status."""from __future__ import annotations+import asynciofrom unittest.mock import AsyncMock, MagicMockimport pytest@@ -113,3 +117,89 @@ async def test_build_status_response_uses_filter_deleted_false_when_allowed(by_wpid.assert_awaited_once()assert by_wpid.call_args.kwargs["filter_deleted"] is False+++async def test_shielded_finalize_runs_when_outer_cancelled_mid_body() -> None:+ """Contract test for the ``execute_workflow`` cancellation-safe pattern:+ when the try body is cancelled after ``pre_finally_status`` is captured,+ the outer ``finally`` must still run ``_finalize_workflow_run_status``+ via ``asyncio.shield`` so the row ends up terminal rather than stuck as+ transient ``running``. Mirrors the structure of+ ``WorkflowService.execute_workflow``; if anyone removes the ``shield`` or+ moves finalize back into the try, this test breaks.+ """++ finalize_calls: list[WorkflowRunStatus] = []+ clean_up_called = False+ body_entered = asyncio.Event()++ async def finalize(status: WorkflowRunStatus) -> None:+ # Simulate a non-trivial DB write so shield cancellation-protection+ # matters rather than being invisible.+ await asyncio.sleep(0.05)+ finalize_calls.append(status)++ async def clean_up() -> None:+ nonlocal clean_up_called+ clean_up_called = True++ async def simulated_execute_workflow() -> None:+ pre_finally_status: WorkflowRunStatus | None = None+ try:+ pre_finally_status = WorkflowRunStatus.failed+ body_entered.set()+ # Simulate the finally-block execution phase that our copilot+ # cancel lands inside of.+ await asyncio.sleep(10)+ finally:+ if pre_finally_status is not None:+ try:+ await asyncio.shield(finalize(pre_finally_status))+ except Exception:+ pass+ await clean_up()++ task = asyncio.create_task(simulated_execute_workflow())+ await body_entered.wait()+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await task++ assert finalize_calls == [WorkflowRunStatus.failed], (+ "shielded finalize must run with the captured pre_finally_status"+ )+ assert clean_up_called, "clean_up_workflow must still run in the outer finally"+++async def test_shielded_finalize_skipped_when_pre_finally_status_unset() -> None:+ """If cancellation lands before block execution captures+ ``pre_finally_status``, there's no intended terminal state to restore —+ the outer ``finally`` must skip finalize, not call it with ``None``.+ """++ finalize_called = False++ async def finalize(status: WorkflowRunStatus) -> None:+ nonlocal finalize_called+ finalize_called = True++ async def simulated_execute_workflow() -> None:+ pre_finally_status: WorkflowRunStatus | None = None+ try:+ await asyncio.sleep(10)+ pre_finally_status = WorkflowRunStatus.failed # pragma: no cover+ finally:+ if pre_finally_status is not None:+ await asyncio.shield(finalize(pre_finally_status))++ task = asyncio.create_task(simulated_execute_workflow())+ await asyncio.sleep(0) # let the task enter its body+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await task++ assert not finalize_called, "finalize must not run when pre_finally_status is unset"diff --git a/tests/unit/test_yaml_loader_no_dates.py b/tests/unit/test_yaml_loader_no_dates.pyindex 00636f7b2..c654b31c3 100644--- a/tests/unit/test_yaml_loader_no_dates.py+++ b/tests/unit/test_yaml_loader_no_dates.py@@ -16,6 +16,7 @@ import jsonimport yaml+from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yamlfrom skyvern.utils.yaml_loader import safe_load_no_datesISO_BLOB = """@@ -75,3 +76,40 @@ def test_safe_load_no_dates_preserves_other_implicit_types() -> None:assert parsed["a_bool"] is Trueassert parsed["a_null"] is Noneassert parsed["a_list"] == [1, 2, 3]+++def test_process_workflow_yaml_keeps_json_parameter_iso_strings() -> None:+ workflow = _process_workflow_yaml(+ workflow_id="wf-123",+ workflow_permanent_id="wfp-123",+ organization_id="org-123",+ workflow_yaml="""+title: Test+workflow_definition:+ parameters:+ - parameter_type: workflow+ key: payload+ workflow_parameter_type: json+ default_value:+ id: "12345"+ metadata:+ created_at: 2023-10-27T10:00:00Z+ updated_at: 2023-10-28T14:30:00Z+ blocks:+ - block_type: navigation+ label: step1+ url: https://example.com+ title: Step 1+ navigation_goal: Open the page+""",+ )++ parameter = workflow.get_parameter("payload")+ assert parameter is not None+ assert parameter.default_value is not None++ metadata = parameter.default_value["metadata"]+ assert metadata["created_at"] == "2023-10-27T10:00:00Z"+ assert metadata["updated_at"] == "2023-10-28T14:30:00Z"+ assert isinstance(metadata["created_at"], str)+ assert isinstance(metadata["updated_at"], str)