Skip to content

Automate browser based workflows with AI

Copilot writes code block Goals as outcome then route, and the AI fallback no longer crashes [SKY-16386] (#8607)

Andrew Neilson1f622501e503
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 1d28bd3ad..a6dfab432 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -4241,7 +4241,9 @@ async def _run_agent_loop_with_surface(
allowlist=frozenset(alias_map.values()),
context_provider=lambda: ctx,
ordered_allowlist=(tuple(alias_map.values()) if ctx.eval_mode == CopilotEvalMode.BROWSER_ABLATION else None),
- enforce_dispatch_allowlist=(ctx.eval_mode == CopilotEvalMode.BROWSER_ABLATION),
+ enforce_dispatch_allowlist=(
+ ctx.eval_mode == CopilotEvalMode.BROWSER_ABLATION or ctx.turn_origin == TurnOrigin.runtime_self_heal
+ ),
)
ctx.discovery_mcp_server = mcp_server
agent = Agent(
diff --git a/skyvern/forge/sdk/copilot/code_block_steps.py b/skyvern/forge/sdk/copilot/code_block_steps.py
index 7f68b250a..bfb604c16 100644
--- a/skyvern/forge/sdk/copilot/code_block_steps.py
+++ b/skyvern/forge/sdk/copilot/code_block_steps.py
@@ -545,57 +545,6 @@ def derive_code_block_steps_in_yaml(workflow_yaml: str) -> str:
return yaml.safe_dump(data, sort_keys=False)
-def fill_code_block_prompts_in_yaml(
- workflow_yaml: str,
- *,
- prior_yaml: str | None = None,
- fallback_goals: dict[str, str] | None = None,
-) -> str:
- """Return workflow_yaml with each code block's `prompt` (goal) filled when absent.
-
- The editor treats a code block as code-first (plain view + steps) only when it
- carries a `prompt`; the model authors the goal as artifact `declared_goal`, not on
- the block, and code regeneration replaces the whole block YAML and drops it. Prefer
- the prior block's prompt by label (exact user text, preserved across regen), then a
- fallback goal by label (e.g. the model's `declared_goal`)."""
- try:
- data = yaml.safe_load(workflow_yaml)
- except yaml.YAMLError:
- return workflow_yaml
- if not isinstance(data, (dict, list)):
- return workflow_yaml
-
- prior_prompts: dict[str, str] = {}
- if prior_yaml:
- try:
- prior_data = yaml.safe_load(prior_yaml)
- except yaml.YAMLError:
- prior_data = None
- if isinstance(prior_data, (dict, list)):
- for block in _iter_code_block_dicts(prior_data):
- label = block.get("label")
- prompt = block.get("prompt")
- if isinstance(label, str) and isinstance(prompt, str) and prompt:
- prior_prompts[label] = prompt
-
- fallback_goals = fallback_goals or {}
- changed = False
- for block in _iter_code_block_dicts(data):
- if block.get("prompt"):
- continue
- label = block.get("label")
- if not isinstance(label, str):
- continue
- goal = prior_prompts.get(label) or fallback_goals.get(label)
- if goal:
- block["prompt"] = goal
- changed = True
-
- if not changed:
- return workflow_yaml
- return yaml.safe_dump(data, sort_keys=False)
-
-
def fill_code_block_error_code_mappings_in_yaml(workflow_yaml: str, *, prior_yaml: str | None = None) -> str:
"""Preserve omitted code-block manifests by label while honoring explicit removal.
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index dd18ad65b..91ce5bea9 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -17,7 +17,7 @@ from typing_extensions import NotRequired, TypedDict
from skyvern.forge.sdk.browser_action_policy import canonicalize_origin
from skyvern.forge.sdk.copilot.authoring_parameter_binding import AuthoringParameterBindingDirective
-from skyvern.forge.sdk.copilot.browser_ablation import BrowserAblationMetadata, CopilotEvalMode
+from skyvern.forge.sdk.copilot.browser_ablation import BrowserAblationMetadata
from skyvern.forge.sdk.copilot.budget_expiry import BudgetExpiryState
from skyvern.forge.sdk.copilot.code_block_synthesis import CREDENTIAL_FILL_TOOL_NAME
from skyvern.forge.sdk.copilot.code_write_diff import TURN_PATCH_CHAR_BUDGET, CodeWriteDiff
@@ -1118,7 +1118,6 @@ class CopilotContext(AgentContext):
copilot_question_pause_seconds: float = 0.0
human_input_wait: HumanInputWait = field(default_factory=HumanInputWait)
eval_capture_case_id: str | None = None
- eval_mode: CopilotEvalMode | None = None
eval_prompt_sha256: str | None = None
eval_tool_surface_sha256: str | None = None
eval_native_tool_names: tuple[str, ...] = ()
@@ -1166,9 +1165,6 @@ class CopilotContext(AgentContext):
credential_recovery_token_digest: str | None = field(default=None, repr=False)
credential_recovery_armed: bool = False
credential_pause_used: bool = False
- # True only while a card is on screen. credential_pause_used stays true for the rest of the
- # turn once one has been raised, which cannot tell a concurrent sibling ask from a later one.
- credential_ask_in_flight: bool = False
# A tool ask the user did not answer with a credential spends the one-card budget on a guess.
# A run that then hits a real login wall has evidence the guess did not, so it gets the budget
# back once.
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index d536be12b..3049c37a0 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -31,6 +31,7 @@ from skyvern.cli.core.session_manager import (
)
from skyvern.config import settings
from skyvern.forge import app
+from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode
from skyvern.forge.sdk.copilot.budget_expiry import BudgetExpiryState
from skyvern.forge.sdk.copilot.build_test_connect_failure import (
SUPERSEDED_BY_NEWER_TEST_REASON,
@@ -410,6 +411,10 @@ class AgentContext:
# finalizer before those tasks finish unwinding, and the manager refuses to retire a busy generation.
admitted_browser_operations: dict[str, set[asyncio.Task[object]]] = field(default_factory=dict)
heal_workflow_run_id: str | None = None
+ eval_mode: CopilotEvalMode | None = None
+ # True only while a card is on screen. credential_pause_used stays true for the rest of the
+ # turn once one has been raised, which cannot tell a concurrent sibling ask from a later one.
+ credential_ask_in_flight: bool = False
# The deadline the current model stream runs under, published by the enforcement loop so a tool
# that parks on a user decision can suspend it instead of being cancelled mid-question.
model_stream_deadline: asyncio.Timeout | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 649c6729d..6c4dadf06 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -491,7 +491,16 @@ async def _get_block_schema_post_hook(
"type": "string",
"description": (
"Every new or wholly rewritten code block must include this non-null string: "
- "the model-authored plain-language Goal shown in the editor."
+ "the model-authored plain-language Goal shown in the editor, written for this code. "
+ "Open with one sentence naming only what a person sees on the page once the block has "
+ "succeeded, showing every value the block returns as it appears there, with no actions, "
+ "conditions, 'after' or 'returns' clauses. Then give the route that reaches "
+ "it: the site or page the block works on, the controls it uses by their visible label "
+ "or role, the order it acts in, the inputs it reads by workflow parameter name (never "
+ "a value), and what the block returns, naming only page content that first sentence "
+ "already shows, never a data structure. A few sentences, enough for an agent to redo "
+ "this block on a live browser without reading the code. Rewrite the Goal whenever you "
+ "rewrite the code."
),
}
required = schema.get("required")
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 66730a30e..f72fd822e 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -316,7 +316,16 @@ class CodeArtifactMetadata(BaseModel):
default=None, description="Label of the authored `code` block this artifact describes."
)
block_id: str | None = None
- declared_goal: str = Field(default="", description="The durable goal this block accomplishes; model-owned.")
+ declared_goal: str = Field(
+ default="",
+ description=(
+ "The durable goal this block accomplishes; model-owned. First name only the visible page state a person "
+ "sees once the block has succeeded, showing every value it returns, with no actions, 'after' or 'returns' "
+ "clauses, then the route that reaches it: page, controls by visible label or role, action order, inputs "
+ "by workflow parameter name (never a value), and what the block returns, naming only page content the "
+ "first sentence already shows, never a data structure."
+ ),
+ )
claimed_outcomes: list[CodeArtifactClaimedOutcome] = Field(
default_factory=list,
description=(
diff --git a/tests/unit/copilot/test_self_heal_recovery.py b/tests/unit/copilot/test_self_heal_recovery.py
index ed8de1032..b8f1c6bf2 100644
--- a/tests/unit/copilot/test_self_heal_recovery.py
+++ b/tests/unit/copilot/test_self_heal_recovery.py
@@ -2,15 +2,19 @@ from __future__ import annotations
import asyncio
import json
+from collections.abc import AsyncIterator
from types import SimpleNamespace
from typing import cast
from unittest.mock import AsyncMock, MagicMock
import pytest
+from agents import Agent
+from agents.stream_events import StreamEvent
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.self_heal_recovery import run_self_heal_recovery
from skyvern.forge.sdk.copilot.tools import _authority_tool_error
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
@@ -635,6 +639,65 @@ async def test_recovery_fails_closed_on_wall_clock_budget(monkeypatch: pytest.Mo
assert result.failure_note == "wall_clock_budget_exhausted"
+async def test_recovery_runs_real_agent_loop_on_its_own_context(monkeypatch: pytest.MonkeyPatch) -> None:
+ _patch_recovery_common(monkeypatch, verified=True)
+ agents_run: list[Agent[AgentContext]] = []
+ rejected_dispatches: list[str] = []
+
+ async def _stream_events_probing_dispatch() -> AsyncIterator[StreamEvent]:
+ for server in agents_run[0].mcp_servers:
+ if (await server.call_tool("skyvern_block_schema", {})).isError:
+ rejected_dispatches.append(server.name)
+ for event in ():
+ yield event
+
+ def _run_streamed(agent: Agent[AgentContext], *_a: object, **_kw: object) -> MagicMock:
+ agents_run.append(agent)
+ return model_result
+
+ model_result = MagicMock()
+ model_result.final_output = '{"type":"REPLY","user_response":"The recovery result is visible."}'
+ model_result.new_items = []
+ model_result.raw_responses = []
+ model_result.to_input_list.return_value = []
+ model_result.stream_events = _stream_events_probing_dispatch
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", _run_streamed)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.self_heal_recovery._run_post_loop_verification_from_browser_state",
+ AsyncMock(return_value=None),
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.self_heal_recovery.llm_config.resolve_main_copilot_handler",
+ AsyncMock(return_value=None),
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.model_resolver.resolve_model_config",
+ lambda *_args, **_kwargs: ("gpt-test", _fake_run_config(), "llm_key", True),
+ )
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.self_heal_recovery.app", SimpleNamespace(AGENT_FUNCTION=MagicMock()))
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.self_heal_recovery.app.AGENT_FUNCTION.get_copilot_config",
+ lambda: CopilotConfig(),
+ )
+
+ result = await run_self_heal_recovery(
+ block=_FakeCodeBlock(),
+ workflow_run_context=_fake_context(),
+ workflow_run_id="wr_1",
+ workflow_run_block_id="wrb_1",
+ organization_id="org_1",
+ browser_state=object(),
+ failing_line=8,
+ api_key="sk-test",
+ max_actions=15,
+ wall_clock_budget_seconds=10,
+ )
+
+ assert result.failure_note == "no_action_progress"
+ assert rejected_dispatches and rejected_dispatches == [server.name for server in agents_run[0].mcp_servers]
+
+
def test_runtime_self_heal_guardrail_rejects_native_tool_call() -> None:
ctx = CopilotContext(
organization_id="org_1",
diff --git a/tests/unit/test_code_block_steps.py b/tests/unit/test_code_block_steps.py
index 0bbca6b4b..d5887f5bb 100644
--- a/tests/unit/test_code_block_steps.py
+++ b/tests/unit/test_code_block_steps.py
@@ -9,7 +9,6 @@ from skyvern.forge.sdk.copilot.code_block_steps import (
bind_referenced_parameters_in_yaml,
derive_code_block_steps,
derive_code_block_steps_in_yaml,
- fill_code_block_prompts_in_yaml,
)
from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_code_block
from skyvern.webeye.actions.action_types import ActionType
@@ -312,65 +311,6 @@ def test_bind_noop_on_unparseable():
assert bind_referenced_parameters_in_yaml("::not yaml::") == "::not yaml::"
-def test_fill_prompts_preserves_prior_block_prompt_across_regen():
- # Regenerating a code block replaces the whole block YAML, dropping the goal.
- # Without the prompt the editor renders the legacy code-only layout, so the
- # block's prior prompt must be carried forward (exact user text).
- prior = (
- "workflow_definition:\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: block_1\n"
- " prompt: Build an agent to find the top post on the site\n"
- " code: 'x = 1'\n"
- )
- regenerated = (
- "workflow_definition:\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: block_1\n"
- " code: |\n"
- " await page.goto('https://example.com/')\n"
- )
- out = yaml.safe_load(fill_code_block_prompts_in_yaml(regenerated, prior_yaml=prior))
- assert out["workflow_definition"]["blocks"][0]["prompt"] == "Build an agent to find the top post on the site"
-
-
-def test_fill_prompts_falls_back_to_declared_goal_for_new_block():
- new = "workflow_definition:\n blocks:\n - block_type: code\n label: block_2\n code: 'x = 1'\n"
- out = yaml.safe_load(
- fill_code_block_prompts_in_yaml(new, prior_yaml=None, fallback_goals={"block_2": "Search the catalog"})
- )
- assert out["workflow_definition"]["blocks"][0]["prompt"] == "Search the catalog"
-
-
-def test_fill_prompts_prefers_prior_over_fallback_and_preserves_existing():
- prior = "workflow_definition:\n blocks:\n - block_type: code\n label: b\n prompt: Exact user text\n code: 'x=1'\n"
- new = (
- "workflow_definition:\n"
- " blocks:\n"
- " - block_type: code\n"
- " label: b\n"
- " code: 'y=2'\n"
- " - block_type: code\n"
- " label: c\n"
- " prompt: Already set\n"
- " code: 'z=3'\n"
- )
- out = yaml.safe_load(
- fill_code_block_prompts_in_yaml(new, prior_yaml=prior, fallback_goals={"b": "model goal", "c": "ignored"})
- )
- blocks = out["workflow_definition"]["blocks"]
- assert blocks[0]["prompt"] == "Exact user text" # prior beats fallback
- assert blocks[1]["prompt"] == "Already set" # existing prompt untouched
-
-
-def test_fill_prompts_noop_without_sources():
- new = "workflow_definition:\n blocks:\n - block_type: code\n label: b\n code: 'x=1'\n"
- out = yaml.safe_load(fill_code_block_prompts_in_yaml(new))
- assert "prompt" not in out["workflow_definition"]["blocks"][0]
-
-
@pytest.mark.asyncio
async def test_process_workflow_yaml_derives_code_block_steps_for_replace_path():
# Regression: the inline REPLACE_WORKFLOW path (v1 and v2) builds the