Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Copilot writes code block Goals as outcome then route, and the AI fallback no longer crashes [SKY-16386] (#8607)
Andrew Neilson1f622501e503
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 1d28bd3ad..a6dfab432 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -4241,7 +4241,9 @@ async def _run_agent_loop_with_surface(allowlist=frozenset(alias_map.values()),context_provider=lambda: ctx,ordered_allowlist=(tuple(alias_map.values()) if ctx.eval_mode == CopilotEvalMode.BROWSER_ABLATION else None),- enforce_dispatch_allowlist=(ctx.eval_mode == CopilotEvalMode.BROWSER_ABLATION),+ enforce_dispatch_allowlist=(+ ctx.eval_mode == CopilotEvalMode.BROWSER_ABLATION or ctx.turn_origin == TurnOrigin.runtime_self_heal+ ),)ctx.discovery_mcp_server = mcp_serveragent = Agent(diff --git a/skyvern/forge/sdk/copilot/code_block_steps.py b/skyvern/forge/sdk/copilot/code_block_steps.pyindex 7f68b250a..bfb604c16 100644--- a/skyvern/forge/sdk/copilot/code_block_steps.py+++ b/skyvern/forge/sdk/copilot/code_block_steps.py@@ -545,57 +545,6 @@ def derive_code_block_steps_in_yaml(workflow_yaml: str) -> str:return yaml.safe_dump(data, sort_keys=False)-def fill_code_block_prompts_in_yaml(- workflow_yaml: str,- *,- prior_yaml: str | None = None,- fallback_goals: dict[str, str] | None = None,-) -> str:- """Return workflow_yaml with each code block's `prompt` (goal) filled when absent.-- The editor treats a code block as code-first (plain view + steps) only when it- carries a `prompt`; the model authors the goal as artifact `declared_goal`, not on- the block, and code regeneration replaces the whole block YAML and drops it. Prefer- the prior block's prompt by label (exact user text, preserved across regen), then a- fallback goal by label (e.g. the model's `declared_goal`)."""- try:- data = yaml.safe_load(workflow_yaml)- except yaml.YAMLError:- return workflow_yaml- if not isinstance(data, (dict, list)):- return workflow_yaml-- prior_prompts: dict[str, str] = {}- if prior_yaml:- try:- prior_data = yaml.safe_load(prior_yaml)- except yaml.YAMLError:- prior_data = None- if isinstance(prior_data, (dict, list)):- for block in _iter_code_block_dicts(prior_data):- label = block.get("label")- prompt = block.get("prompt")- if isinstance(label, str) and isinstance(prompt, str) and prompt:- prior_prompts[label] = prompt-- fallback_goals = fallback_goals or {}- changed = False- for block in _iter_code_block_dicts(data):- if block.get("prompt"):- continue- label = block.get("label")- if not isinstance(label, str):- continue- goal = prior_prompts.get(label) or fallback_goals.get(label)- if goal:- block["prompt"] = goal- changed = True-- if not changed:- return workflow_yaml- return yaml.safe_dump(data, sort_keys=False)--def fill_code_block_error_code_mappings_in_yaml(workflow_yaml: str, *, prior_yaml: str | None = None) -> str:"""Preserve omitted code-block manifests by label while honoring explicit removal.diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex dd18ad65b..91ce5bea9 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -17,7 +17,7 @@ from typing_extensions import NotRequired, TypedDictfrom skyvern.forge.sdk.browser_action_policy import canonicalize_originfrom skyvern.forge.sdk.copilot.authoring_parameter_binding import AuthoringParameterBindingDirective-from skyvern.forge.sdk.copilot.browser_ablation import BrowserAblationMetadata, CopilotEvalMode+from skyvern.forge.sdk.copilot.browser_ablation import BrowserAblationMetadatafrom skyvern.forge.sdk.copilot.budget_expiry import BudgetExpiryStatefrom skyvern.forge.sdk.copilot.code_block_synthesis import CREDENTIAL_FILL_TOOL_NAMEfrom skyvern.forge.sdk.copilot.code_write_diff import TURN_PATCH_CHAR_BUDGET, CodeWriteDiff@@ -1118,7 +1118,6 @@ class CopilotContext(AgentContext):copilot_question_pause_seconds: float = 0.0human_input_wait: HumanInputWait = field(default_factory=HumanInputWait)eval_capture_case_id: str | None = None- eval_mode: CopilotEvalMode | None = Noneeval_prompt_sha256: str | None = Noneeval_tool_surface_sha256: str | None = Noneeval_native_tool_names: tuple[str, ...] = ()@@ -1166,9 +1165,6 @@ class CopilotContext(AgentContext):credential_recovery_token_digest: str | None = field(default=None, repr=False)credential_recovery_armed: bool = Falsecredential_pause_used: bool = False- # True only while a card is on screen. credential_pause_used stays true for the rest of the- # turn once one has been raised, which cannot tell a concurrent sibling ask from a later one.- credential_ask_in_flight: bool = False# A tool ask the user did not answer with a credential spends the one-card budget on a guess.# A run that then hits a real login wall has evidence the guess did not, so it gets the budget# back once.diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex d536be12b..3049c37a0 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -31,6 +31,7 @@ from skyvern.cli.core.session_manager import ()from skyvern.config import settingsfrom skyvern.forge import app+from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalModefrom skyvern.forge.sdk.copilot.budget_expiry import BudgetExpiryStatefrom skyvern.forge.sdk.copilot.build_test_connect_failure import (SUPERSEDED_BY_NEWER_TEST_REASON,@@ -410,6 +411,10 @@ class AgentContext:# finalizer before those tasks finish unwinding, and the manager refuses to retire a busy generation.admitted_browser_operations: dict[str, set[asyncio.Task[object]]] = field(default_factory=dict)heal_workflow_run_id: str | None = None+ eval_mode: CopilotEvalMode | None = None+ # True only while a card is on screen. credential_pause_used stays true for the rest of the+ # turn once one has been raised, which cannot tell a concurrent sibling ask from a later one.+ credential_ask_in_flight: bool = False# The deadline the current model stream runs under, published by the enforcement loop so a tool# that parks on a user decision can suspend it instead of being cancelled mid-question.model_stream_deadline: asyncio.Timeout | None = Nonediff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex 649c6729d..6c4dadf06 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -491,7 +491,16 @@ async def _get_block_schema_post_hook("type": "string","description": ("Every new or wholly rewritten code block must include this non-null string: "- "the model-authored plain-language Goal shown in the editor."+ "the model-authored plain-language Goal shown in the editor, written for this code. "+ "Open with one sentence naming only what a person sees on the page once the block has "+ "succeeded, showing every value the block returns as it appears there, with no actions, "+ "conditions, 'after' or 'returns' clauses. Then give the route that reaches "+ "it: the site or page the block works on, the controls it uses by their visible label "+ "or role, the order it acts in, the inputs it reads by workflow parameter name (never "+ "a value), and what the block returns, naming only page content that first sentence "+ "already shows, never a data structure. A few sentences, enough for an agent to redo "+ "this block on a live browser without reading the code. Rewrite the Goal whenever you "+ "rewrite the code."),}required = schema.get("required")diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 66730a30e..f72fd822e 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -316,7 +316,16 @@ class CodeArtifactMetadata(BaseModel):default=None, description="Label of the authored `code` block this artifact describes.")block_id: str | None = None- declared_goal: str = Field(default="", description="The durable goal this block accomplishes; model-owned.")+ declared_goal: str = Field(+ default="",+ description=(+ "The durable goal this block accomplishes; model-owned. First name only the visible page state a person "+ "sees once the block has succeeded, showing every value it returns, with no actions, 'after' or 'returns' "+ "clauses, then the route that reaches it: page, controls by visible label or role, action order, inputs "+ "by workflow parameter name (never a value), and what the block returns, naming only page content the "+ "first sentence already shows, never a data structure."+ ),+ )claimed_outcomes: list[CodeArtifactClaimedOutcome] = Field(default_factory=list,description=(diff --git a/tests/unit/copilot/test_self_heal_recovery.py b/tests/unit/copilot/test_self_heal_recovery.pyindex ed8de1032..b8f1c6bf2 100644--- a/tests/unit/copilot/test_self_heal_recovery.py+++ b/tests/unit/copilot/test_self_heal_recovery.py@@ -2,15 +2,19 @@ from __future__ import annotationsimport asyncioimport json+from collections.abc import AsyncIteratorfrom types import SimpleNamespacefrom typing import castfrom unittest.mock import AsyncMock, MagicMockimport pytest+from agents import Agent+from agents.stream_events import StreamEventfrom skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.runtime import AgentContextfrom skyvern.forge.sdk.copilot.self_heal_recovery import run_self_heal_recoveryfrom skyvern.forge.sdk.copilot.tools import _authority_tool_errorfrom skyvern.forge.sdk.copilot.turn_origin import TurnOrigin@@ -635,6 +639,65 @@ async def test_recovery_fails_closed_on_wall_clock_budget(monkeypatch: pytest.Moassert result.failure_note == "wall_clock_budget_exhausted"+async def test_recovery_runs_real_agent_loop_on_its_own_context(monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_recovery_common(monkeypatch, verified=True)+ agents_run: list[Agent[AgentContext]] = []+ rejected_dispatches: list[str] = []++ async def _stream_events_probing_dispatch() -> AsyncIterator[StreamEvent]:+ for server in agents_run[0].mcp_servers:+ if (await server.call_tool("skyvern_block_schema", {})).isError:+ rejected_dispatches.append(server.name)+ for event in ():+ yield event++ def _run_streamed(agent: Agent[AgentContext], *_a: object, **_kw: object) -> MagicMock:+ agents_run.append(agent)+ return model_result++ model_result = MagicMock()+ model_result.final_output = '{"type":"REPLY","user_response":"The recovery result is visible."}'+ model_result.new_items = []+ model_result.raw_responses = []+ model_result.to_input_list.return_value = []+ model_result.stream_events = _stream_events_probing_dispatch+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", _run_streamed)+ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.self_heal_recovery._run_post_loop_verification_from_browser_state",+ AsyncMock(return_value=None),+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.self_heal_recovery.llm_config.resolve_main_copilot_handler",+ AsyncMock(return_value=None),+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.model_resolver.resolve_model_config",+ lambda *_args, **_kwargs: ("gpt-test", _fake_run_config(), "llm_key", True),+ )+ monkeypatch.setattr("skyvern.forge.sdk.copilot.self_heal_recovery.app", SimpleNamespace(AGENT_FUNCTION=MagicMock()))+ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.self_heal_recovery.app.AGENT_FUNCTION.get_copilot_config",+ lambda: CopilotConfig(),+ )++ result = await run_self_heal_recovery(+ block=_FakeCodeBlock(),+ workflow_run_context=_fake_context(),+ workflow_run_id="wr_1",+ workflow_run_block_id="wrb_1",+ organization_id="org_1",+ browser_state=object(),+ failing_line=8,+ api_key="sk-test",+ max_actions=15,+ wall_clock_budget_seconds=10,+ )++ assert result.failure_note == "no_action_progress"+ assert rejected_dispatches and rejected_dispatches == [server.name for server in agents_run[0].mcp_servers]++def test_runtime_self_heal_guardrail_rejects_native_tool_call() -> None:ctx = CopilotContext(organization_id="org_1",diff --git a/tests/unit/test_code_block_steps.py b/tests/unit/test_code_block_steps.pyindex 0bbca6b4b..d5887f5bb 100644--- a/tests/unit/test_code_block_steps.py+++ b/tests/unit/test_code_block_steps.py@@ -9,7 +9,6 @@ from skyvern.forge.sdk.copilot.code_block_steps import (bind_referenced_parameters_in_yaml,derive_code_block_steps,derive_code_block_steps_in_yaml,- fill_code_block_prompts_in_yaml,)from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_code_blockfrom skyvern.webeye.actions.action_types import ActionType@@ -312,65 +311,6 @@ def test_bind_noop_on_unparseable():assert bind_referenced_parameters_in_yaml("::not yaml::") == "::not yaml::"-def test_fill_prompts_preserves_prior_block_prompt_across_regen():- # Regenerating a code block replaces the whole block YAML, dropping the goal.- # Without the prompt the editor renders the legacy code-only layout, so the- # block's prior prompt must be carried forward (exact user text).- prior = (- "workflow_definition:\n"- " blocks:\n"- " - block_type: code\n"- " label: block_1\n"- " prompt: Build an agent to find the top post on the site\n"- " code: 'x = 1'\n"- )- regenerated = (- "workflow_definition:\n"- " blocks:\n"- " - block_type: code\n"- " label: block_1\n"- " code: |\n"- " await page.goto('https://example.com/')\n"- )- out = yaml.safe_load(fill_code_block_prompts_in_yaml(regenerated, prior_yaml=prior))- assert out["workflow_definition"]["blocks"][0]["prompt"] == "Build an agent to find the top post on the site"---def test_fill_prompts_falls_back_to_declared_goal_for_new_block():- new = "workflow_definition:\n blocks:\n - block_type: code\n label: block_2\n code: 'x = 1'\n"- out = yaml.safe_load(- fill_code_block_prompts_in_yaml(new, prior_yaml=None, fallback_goals={"block_2": "Search the catalog"})- )- assert out["workflow_definition"]["blocks"][0]["prompt"] == "Search the catalog"---def test_fill_prompts_prefers_prior_over_fallback_and_preserves_existing():- prior = "workflow_definition:\n blocks:\n - block_type: code\n label: b\n prompt: Exact user text\n code: 'x=1'\n"- new = (- "workflow_definition:\n"- " blocks:\n"- " - block_type: code\n"- " label: b\n"- " code: 'y=2'\n"- " - block_type: code\n"- " label: c\n"- " prompt: Already set\n"- " code: 'z=3'\n"- )- out = yaml.safe_load(- fill_code_block_prompts_in_yaml(new, prior_yaml=prior, fallback_goals={"b": "model goal", "c": "ignored"})- )- blocks = out["workflow_definition"]["blocks"]- assert blocks[0]["prompt"] == "Exact user text" # prior beats fallback- assert blocks[1]["prompt"] == "Already set" # existing prompt untouched---def test_fill_prompts_noop_without_sources():- new = "workflow_definition:\n blocks:\n - block_type: code\n label: b\n code: 'x=1'\n"- out = yaml.safe_load(fill_code_block_prompts_in_yaml(new))- assert "prompt" not in out["workflow_definition"]["blocks"][0]--@pytest.mark.asyncioasync def test_process_workflow_yaml_derives_code_block_steps_for_replace_path():# Regression: the inline REPLACE_WORKFLOW path (v1 and v2) builds the