Skip to content

Automate browser based workflows with AI

SKY-11081: sunset copilot scout flags (#6635)

Andrew Neilsoncdcd669919a3
diff --git a/skyvern/config.py b/skyvern/config.py
index 7852198ec..a8c49d3d9 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -158,9 +158,6 @@ class Settings(BaseSettings):
COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS: float = 6.0
COPILOT_TURN_INTENT_CLASSIFIER_TIMEOUT_SECONDS: float = 12.0
COPILOT_COMPLETION_JUDGE_TIMEOUT_SECONDS: float = 12.0
- # Capture bounded page evidence synchronously in scout interaction post-hooks.
- # Off restores schema-less interaction packets plus standalone page inspects.
- COPILOT_SCOUT_ACT_OBSERVE_ENABLED: bool = True
# Consecutive repair runs that make no newly-verified forward progress before the
# copilot stops re-running and escalates honestly. Set very high to disable the ceiling.
COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL: int = 3
@@ -175,11 +172,6 @@ class Settings(BaseSettings):
# On, a reached-download affordance compiles into the expect_download idiom in the code-block
# synthesizer; off (default) restores today's bare-click synthesis. Per-rung gate.
COPILOT_DOWNLOAD_RUNG_SYNTHESIS_ENABLED: bool = False
- # A download fires only when its affordance is clicked, so a download-intent code block must
- # be scout-acted (skyvern_evaluate) before it can be authored: the guardrail rejects authoring
- # one with no scout_interaction this turn, and the evaluate tool is framed as the scout-act
- # path for interaction-gated affordances. Off (default) restores today's inspect-then-author.
- COPILOT_DOWNLOAD_SCOUT_ACT_REQUIRED_ENABLED: bool = False
# Staged rollout for treating omitted runtime workflow proxy values as direct/no-proxy.
# Off preserves the historical implicit residential default for anti-bot-sensitive traffic.
RUNTIME_PROXY_DEFAULT_NONE_ENABLED: bool = False
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 333d33a2d..72bafabb2 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -3186,7 +3186,7 @@ async def _run_copilot_turn_impl(
output_guardrails = _build_copilot_output_guardrails(OutputGuardrail, GuardrailFunctionOutput)
alias_map = get_skyvern_mcp_alias_map()
- overlays = _build_skyvern_mcp_overlays()
+ overlays = _build_skyvern_mcp_overlays(copilot_config.block_authoring_policy)
if _turn_intent_disables_tools(ctx.turn_intent):
alias_map = {}
overlays = {}
diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.py
index f7e416d5f..300ac60bd 100644
--- a/skyvern/forge/sdk/copilot/config.py
+++ b/skyvern/forge/sdk/copilot/config.py
@@ -28,6 +28,10 @@ def block_authoring_policy_from_code_only_mode(enabled: bool) -> BlockAuthoringP
return BlockAuthoringPolicy.CODE_ONLY_BROWSER if enabled else BlockAuthoringPolicy.STANDARD
+def download_scout_act_required_for_policy(block_authoring_policy: BlockAuthoringPolicy | str | None) -> bool:
+ return normalize_block_authoring_policy(block_authoring_policy) == BlockAuthoringPolicy.CODE_ONLY_BROWSER
+
+
DEFAULT_PROMPT_TEMPLATE = "workflow-copilot-agent.j2"
DEFAULT_MAX_TURNS = 35
DEFAULT_TOKEN_BUDGET = 90_000
diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.py
index 946889295..db178782f 100644
--- a/skyvern/forge/sdk/copilot/tools/guardrails.py
+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py
@@ -14,6 +14,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
turn_has_scout_interaction,
workflow_target_url,
)
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctx
from skyvern.forge.sdk.copilot.output_policy import (
evaluate_output_policy,
@@ -46,6 +47,7 @@ from ._shared import (
_emit_tool_blocker_signal,
_workflow_yaml_blocks_by_label,
)
+from .banned_blocks import _copilot_block_authoring_policy
LOG = structlog.get_logger()
@@ -172,8 +174,8 @@ def _download_intent_block_labels(workflow_yaml: str | None) -> list[str]:
def _download_scout_required_error(copilot_ctx: Any, workflow_yaml: str | None) -> str | None:
"""Reject authoring a download-intent code block until the affordance has been scout-acted
this turn, so the skyvern_evaluate post-hook can populate the reached-download target and the
- synthesizer can compile the terminal download step. Off unless the scout-act flag is set."""
- if not settings.COPILOT_DOWNLOAD_SCOUT_ACT_REQUIRED_ENABLED:
+ synthesizer can compile the terminal download step. Active in code-only browser mode."""
+ if _copilot_block_authoring_policy(copilot_ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
return None
if not settings.COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED:
# The gate is cleared by the reached-download scout interaction, which is only registered
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 077ccf840..be89e9349 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -7,13 +7,15 @@ from typing import Any
import structlog
-from skyvern.config import settings
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
from skyvern.forge.sdk.copilot.build_phase import (
BuildPhase,
advance_to_composing,
)
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.config import (
+ BlockAuthoringPolicy,
+ download_scout_act_required_for_policy,
+)
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay
from skyvern.forge.sdk.copilot.runtime import AgentContext
@@ -398,7 +400,7 @@ async def _click_post_hook(
result["data"]["observation_step"] = observation_step
if page_evidence is not None:
_attach_scout_page_summary(result, page_evidence)
- if settings.COPILOT_DOWNLOAD_SCOUT_ACT_REQUIRED_ENABLED:
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
await _maybe_attach_reached_download_target(ctx, result, url=url, page_evidence=page_evidence)
return result
@@ -677,13 +679,17 @@ _EVALUATE_SCOUT_ACT_DESCRIPTION = (
)
-def _evaluate_overlay_description() -> str:
- if settings.COPILOT_DOWNLOAD_SCOUT_ACT_REQUIRED_ENABLED:
+def _evaluate_overlay_description(
+ block_authoring_policy: BlockAuthoringPolicy | str | None = BlockAuthoringPolicy.STANDARD,
+) -> str:
+ if download_scout_act_required_for_policy(block_authoring_policy):
return _EVALUATE_SCOUT_ACT_DESCRIPTION
return _EVALUATE_BASE_DESCRIPTION
-def _build_skyvern_mcp_overlays() -> dict[str, SchemaOverlay]:
+def _build_skyvern_mcp_overlays(
+ block_authoring_policy: BlockAuthoringPolicy | str | None = BlockAuthoringPolicy.STANDARD,
+) -> dict[str, SchemaOverlay]:
return {
"get_block_schema": SchemaOverlay(
pre_hook=_get_block_schema_pre_hook,
@@ -711,7 +717,7 @@ def _build_skyvern_mcp_overlays() -> dict[str, SchemaOverlay]:
post_hook=_screenshot_post_hook,
),
"evaluate": SchemaOverlay(
- description=_evaluate_overlay_description(),
+ description=_evaluate_overlay_description(block_authoring_policy),
hide_params=frozenset({"session_id", "cdp_url"}),
requires_browser=True,
timeout=30,
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index c58366374..433e2425b 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
SCOUT_INTERACTION_EVIDENCE_TOOL,
has_bounded_page_schema,
)
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP
from skyvern.forge.sdk.copilot.reached_download_target import (
ReachedDownloadTarget,
@@ -40,6 +41,7 @@ from ._shared import (
_same_page_ignoring_fragment,
_workflow_verification_evidence,
)
+from .banned_blocks import _copilot_block_authoring_policy
LOG = structlog.get_logger()
@@ -866,7 +868,10 @@ async def _maybe_attach_reached_download_target(
# non-download idiom. Reopen the latch once so the post-turn fallback re-fires carrying it.
ctx.synthesized_block_offered = False
LOG.info("copilot_synthesized_block_offer_latch_reset_for_download", url=url)
- if settings.COPILOT_DOWNLOAD_SCOUT_ACT_REQUIRED_ENABLED and not target.already_registered:
+ if (
+ _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ and not target.already_registered
+ ):
_register_reached_download_scout_interaction(ctx, target, url=url)
LOG.info(
"copilot_reached_download_target_steer",
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index dada4fdaa..26e1f5228 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -14,6 +14,7 @@ from structlog.testing import capture_logs
from skyvern.forge.sdk.copilot import hooks as hooks_module
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.enforcement import CopilotGoalSatisfied
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, turn_halt_from_blocker_signal
@@ -1522,7 +1523,7 @@ class TestClickPostHookReachedDownloadTarget:
monkeypatch.setattr(mh, "_register_scout_interaction_observation", fake_register)
@staticmethod
- def _ctx() -> Any:
+ def _ctx(policy: BlockAuthoringPolicy = BlockAuthoringPolicy.CODE_ONLY_BROWSER) -> Any:
from skyvern.forge.sdk.copilot.runtime import AgentContext
return AgentContext(
@@ -1532,6 +1533,7 @@ class TestClickPostHookReachedDownloadTarget:
workflow_yaml="",
browser_session_id="pbs_copilot",
stream=MagicMock(is_disconnected=AsyncMock(return_value=False)),
+ block_authoring_policy=policy,
)
_SINGLE_DOWNLOAD_EVIDENCE = {
@@ -1552,7 +1554,6 @@ class TestClickPostHookReachedDownloadTarget:
from skyvern.forge.sdk.copilot.tools.mcp_hooks import _click_post_hook
self._patch_scouting(monkeypatch, page_evidence=self._SINGLE_DOWNLOAD_EVIDENCE)
- monkeypatch.setattr(settings, "COPILOT_DOWNLOAD_SCOUT_ACT_REQUIRED_ENABLED", True)
monkeypatch.setattr(settings, "COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED", True)
monkeypatch.setattr(settings, "COPILOT_DOWNLOAD_RUNG_SYNTHESIS_ENABLED", True)
@@ -1564,16 +1565,15 @@ class TestClickPostHookReachedDownloadTarget:
assert ctx.reached_download_target.download_kind == "extension"
@pytest.mark.asyncio
- async def test_click_post_hook_is_noop_when_flag_off(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_click_post_hook_is_noop_in_standard_mode(self, monkeypatch: pytest.MonkeyPatch) -> None:
from skyvern.config import settings
from skyvern.forge.sdk.copilot.tools.mcp_hooks import _click_post_hook
self._patch_scouting(monkeypatch, page_evidence=self._SINGLE_DOWNLOAD_EVIDENCE)
- monkeypatch.setattr(settings, "COPILOT_DOWNLOAD_SCOUT_ACT_REQUIRED_ENABLED", False)
monkeypatch.setattr(settings, "COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED", True)
monkeypatch.setattr(settings, "COPILOT_DOWNLOAD_RUNG_SYNTHESIS_ENABLED", True)
- ctx = self._ctx()
+ ctx = self._ctx(BlockAuthoringPolicy.STANDARD)
result = {"ok": True, "data": {"selector": 'a[href="/x/statement.pdf"]'}}
await _click_post_hook(result, {}, ctx)
@@ -1591,7 +1591,6 @@ class TestClickPostHookReachedDownloadTarget:
]
}
self._patch_scouting(monkeypatch, page_evidence=two)
- monkeypatch.setattr(settings, "COPILOT_DOWNLOAD_SCOUT_ACT_REQUIRED_ENABLED", True)
monkeypatch.setattr(settings, "COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED", True)
monkeypatch.setattr(settings, "COPILOT_DOWNLOAD_RUNG_SYNTHESIS_ENABLED", True)
diff --git a/tests/unit/test_workflow_copilot_code_block_config.py b/tests/unit/test_workflow_copilot_code_block_config.py
index ef7f750cd..a5ce982ff 100644
--- a/tests/unit/test_workflow_copilot_code_block_config.py
+++ b/tests/unit/test_workflow_copilot_code_block_config.py
@@ -10,6 +10,7 @@ from skyvern.forge.sdk.copilot.config import (
BlockAuthoringPolicy,
CopilotConfig,
block_authoring_policy_from_code_only_mode,
+ download_scout_act_required_for_policy,
)
@@ -22,6 +23,13 @@ def test_code_block_settings_helper_selects_policy() -> None:
assert block_authoring_policy_from_code_only_mode(False) == BlockAuthoringPolicy.STANDARD
+def test_download_scout_act_requirement_follows_code_only_policy() -> None:
+ assert download_scout_act_required_for_policy(BlockAuthoringPolicy.CODE_ONLY_BROWSER) is True
+ assert download_scout_act_required_for_policy("code_only_browser") is True
+ assert download_scout_act_required_for_policy(None) is False
+ assert download_scout_act_required_for_policy(BlockAuthoringPolicy.STANDARD) is False
+
+
def test_base_agent_function_honors_code_block_mode_setting(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "WORKFLOW_COPILOT_CODE_BLOCK_MODE", True)