Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11030: promote scout-act-observe + evaluate-target-steer to unconditional defaults (#6588)
Andrew Neilson340f4857ed87
diff --git a/skyvern/config.py b/skyvern/config.pyindex 598025101..7c560a99f 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -154,15 +154,9 @@ class Settings(BaseSettings):# Gate copilot verified-success on per-criterion outcome evidence.# Off restores the prior run-status/suspicious-success gate and classifier prompt.COPILOT_OUTCOME_VERIFICATION_ENABLED: bool = True- # Capture bounded page evidence synchronously in scout interaction post-hooks.- # Off restores schema-less interaction packets plus standalone page inspects.- COPILOT_SCOUT_ACT_OBSERVE_ENABLED: bool = TrueCOPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS: float = 4.0- # Steer a repeat evaluate on an unchanged page toward acting on an already-identified target.- # Off restores the raw evaluate result with no actionable-target naming or act instruction.- COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED: bool = True# On a repeat evaluate of an unchanged page with exactly one safe target, click it in-process- # instead of only advising. Off (default) keeps the advisory-only steer above.+ # instead of only advising. Off (default) leaves the advisory steer as the only response.COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED: bool = False# Persist copilot completion criteria per goal epoch with tri-state verdicts# and definition-plane grading. Off restores per-turn re-derivation, the legacydiff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.pyindex 294d53270..574b78261 100644--- a/skyvern/forge/sdk/copilot/tools/scouting.py+++ b/skyvern/forge/sdk/copilot/tools/scouting.py@@ -317,7 +317,7 @@ async def _register_scout_interaction_observation(if source_url and source_url.strip():evidence["interaction_source_url"] = source_url.strip()page_evidence: dict[str, Any] | None = None- if settings.COPILOT_SCOUT_ACT_OBSERVE_ENABLED and tool_name in _ACT_OBSERVE_TOOLS:+ if tool_name in _ACT_OBSERVE_TOOLS:parsed = await _scout_act_observe_page_evidence(ctx, url=url)if parsed is not None:# Identity keys overwrite the parsed packet so the entry stays a@@ -724,8 +724,6 @@ async def _auto_act_on_repeat(ctx: AgentContext, result: dict[str, Any], *, url:async def _maybe_steer_evaluate_to_action(ctx: AgentContext, result: dict[str, Any], *, url: str) -> None:- if not settings.COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED:- returndata = result.get("data")if not isinstance(data, dict):returndiff --git a/tests/unit/test_copilot_scout_act_observe.py b/tests/unit/test_copilot_scout_act_observe.pyindex 32b36be8d..1a8b589e1 100644--- a/tests/unit/test_copilot_scout_act_observe.py+++ b/tests/unit/test_copilot_scout_act_observe.py@@ -96,8 +96,7 @@ def _flow_by_step(ctx: SimpleNamespace) -> dict[int, tuple[dict[str, Any], str]]class TestActObserveSuccess:@pytest.mark.asyncio- async def test_schema_merged_into_interaction_packet_before_append(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)+ async def test_schema_merged_into_interaction_packet_before_append(self) -> None:ctx = _ctx(server=_server_returning(_bounded_extractor_payload()))result = await _run_click(ctx)@@ -118,8 +117,7 @@ class TestActObserveSuccess:assert evidence["forms"][0]["fields"][0]["label"] == "NPI number"@pytest.mark.asyncio- async def test_pending_marker_cleared_and_result_carries_summary(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)+ async def test_pending_marker_cleared_and_result_carries_summary(self) -> None:ctx = _ctx(server=_server_returning(_bounded_extractor_payload()))result = await _run_click(ctx)@@ -149,7 +147,6 @@ class TestActObserveDegrade:async def test_timeout_degrades_to_schema_less_packet_and_keeps_marker(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS", 0.05)async def slow_extract(*_args: object, **_kwargs: object) -> dict[str, Any]:@@ -178,8 +175,7 @@ class TestActObserveDegrade:assert ctx.pending_browser_interaction_observation.url == _LANDING_URL@pytest.mark.asyncio- async def test_hollow_parse_degrades_to_schema_less_packet(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)+ async def test_hollow_parse_degrades_to_schema_less_packet(self) -> None:ctx = _ctx(server=_server_returning({"page_title": "Loading", "forms": []}))result = await _run_click(ctx)@@ -191,8 +187,7 @@ class TestActObserveDegrade:assert ctx.pending_browser_interaction_observation is not None@pytest.mark.asyncio- async def test_extractor_error_never_fails_the_click(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)+ async def test_extractor_error_never_fails_the_click(self) -> None:server = SimpleNamespace()server.call_internal_tool = AsyncMock(side_effect=RuntimeError("browser gone"))ctx = _ctx(server=server)@@ -207,7 +202,6 @@ class TestActObserveDegrade:class TestActObserveNoRace:@pytest.mark.asyncioasync def test_appended_entry_never_mutates_after_hook_returns(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS", 0.05)async def slow_extract(*_args: object, **_kwargs: object) -> dict[str, Any]:@@ -229,8 +223,7 @@ class TestActObserveNoRace:assert not has_bounded_page_schema(ctx.flow_evidence[0]["evidence"])@pytest.mark.asyncio- async def test_successful_attach_credits_exactly_once(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)+ async def test_successful_attach_credits_exactly_once(self) -> None:ctx = _ctx(server=_server_returning(_bounded_extractor_payload()))await _run_click(ctx)@@ -256,8 +249,7 @@ class TestActObserveNoRace:assert _auto_credit_interaction_observation(by_step, consumed) is False@pytest.mark.asyncio- async def test_degraded_path_preserves_pending_upgrade(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)+ async def test_degraded_path_preserves_pending_upgrade(self) -> None:ctx = _ctx(server=_server_returning({"page_title": "Loading", "forms": []}))await _run_click(ctx)@@ -277,8 +269,7 @@ class TestActObserveNoRace:class TestActObserveSummaryBound:@pytest.mark.asyncio- async def test_adversarial_page_never_clips_serialized_result(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)+ async def test_adversarial_page_never_clips_serialized_result(self) -> None:payload = {"page_title": "R" * 300,"forms": [@@ -333,34 +324,11 @@ class TestActObserveSummaryBound:assert len(ctx.flow_evidence[0]["evidence"]["forms"]) == 5-class TestActObserveKillSwitch:- @pytest.mark.asyncio- async def test_disabled_restores_two_call_behavior(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", False)- server = _server_returning(_bounded_extractor_payload())- ctx = _ctx(server=server)-- result = await _run_click(ctx)-- server.call_internal_tool.assert_not_awaited()- assert result["ok"] is True- assert result["data"] == {- "selector": "#open-details",- "url": _LANDING_URL,- "title": "Results",- "observation_step": ctx.flow_evidence[0]["step"],- }- entry = ctx.flow_evidence[0]- assert entry["had_bounded_schema"] is False- assert set(entry["evidence"].keys()) == _SCHEMA_LESS_PACKET_KEYS- assert ctx.pending_browser_interaction_observation is not None-+class TestActObserveToolGate:@pytest.mark.asyncioasync def test_non_click_tools_do_not_capture(self, monkeypatch: pytest.MonkeyPatch) -> None:from skyvern.forge.sdk.copilot import tools as tools_module- monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)-async def passes(*_args: object, **_kwargs: object) -> None:return Nonediff --git a/tests/unit/test_evaluate_actionable_target.py b/tests/unit/test_evaluate_actionable_target.pyindex 256c78c3a..51439c82f 100644--- a/tests/unit/test_evaluate_actionable_target.py+++ b/tests/unit/test_evaluate_actionable_target.py@@ -64,8 +64,6 @@ def test_signature_changes_when_controls_change() -> None:@pytest.mark.asyncioasync def test_first_evaluate_names_targets_without_next_action(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)-async def fake_evidence(ctx, *, url):return _packet()@@ -81,8 +79,6 @@ async def test_first_evaluate_names_targets_without_next_action(monkeypatch) ->@pytest.mark.asyncioasync def test_repeat_evaluate_same_page_steers_to_click(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)-async def fake_evidence(ctx, *, url):return _packet()@@ -101,7 +97,6 @@ async def test_repeat_evaluate_same_page_steers_to_click(monkeypatch) -> None:@pytest.mark.asyncioasync def test_changed_signature_not_steered(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)packets = [_packet(),{"navigation_targets": [{"selector": "#other", "text": "Next"}], "forms": [], "result_containers": []},@@ -125,21 +120,8 @@ async def test_changed_signature_not_steered(monkeypatch) -> None:assert "next_action" not in second["data"]-async def test_flag_off_restores_raw_result(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", False)- ctx = _ctx()- ctx.last_evaluate_actionable_signature = None- ctx.last_evaluate_actionable_url = None- result = {"ok": True, "data": {"url": "https://example.com/bill"}}- await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://example.com/bill")- assert "actionable_targets" not in result["data"]- assert "next_action" not in result["data"]--@pytest.mark.asyncioasync def test_same_signature_different_url_not_steered(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = Nonectx.last_evaluate_actionable_url = None@@ -150,7 +132,6 @@ async def test_same_signature_different_url_not_steered(monkeypatch) -> None:@pytest.mark.asyncioasync def test_hash_route_change_not_steered(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = Nonectx.last_evaluate_actionable_url = None@@ -161,7 +142,6 @@ async def test_hash_route_change_not_steered(monkeypatch) -> None:@pytest.mark.asyncioasync def test_repeat_scalar_result_sheds_under_cap(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = Nonectx.last_evaluate_actionable_url = None@@ -185,7 +165,6 @@ async def test_repeat_scalar_result_sheds_under_cap(monkeypatch) -> None:@pytest.mark.asyncioasync def test_repeat_nested_result_sheds_under_cap(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = Nonectx.last_evaluate_actionable_url = None@@ -220,7 +199,6 @@ async def test_repeat_nested_result_sheds_under_cap(monkeypatch) -> None:@pytest.mark.asyncioasync def test_first_evaluate_large_result_not_shed(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = Nonectx.last_evaluate_actionable_url = None@@ -239,7 +217,6 @@ async def test_first_evaluate_large_result_not_shed(monkeypatch) -> None:@pytest.mark.asyncioasync def test_non_serializable_data_does_not_raise(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = "seed"ctx.last_evaluate_actionable_url = "https://app.example.com/bill"@@ -263,7 +240,6 @@ def test_identities_cover_all_collections() -> None:@pytest.mark.asyncioasync def test_probe_none_leaves_data_untouched_and_resets(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = "seed"ctx.last_evaluate_actionable_url = "https://app.example.com/x"@@ -280,7 +256,6 @@ async def test_probe_none_leaves_data_untouched_and_resets(monkeypatch) -> None:@pytest.mark.asyncioasync def test_probe_empty_collections_no_steer(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = Nonectx.last_evaluate_actionable_url = None@@ -296,7 +271,6 @@ async def test_probe_empty_collections_no_steer(monkeypatch) -> None:@pytest.mark.asyncioasync def test_interaction_between_evaluates_suppresses_steer(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = Nonectx.last_evaluate_actionable_url = None@@ -313,7 +287,6 @@ async def test_interaction_between_evaluates_suppresses_steer(monkeypatch) -> Noasync def test_steer_front_runs_loop_guard(monkeypatch) -> None:from skyvern.forge.sdk.copilot.loop_detection import MAX_CONSECUTIVE_SAME_TOOL, detect_tool_loop- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)ctx = _ctx()ctx.last_evaluate_actionable_signature = Nonectx.last_evaluate_actionable_url = None@@ -468,7 +441,6 @@ def test_auto_act_candidate_rejects_javascript_href() -> None:@pytest.mark.asyncioasync def test_multiple_candidates_no_auto_act(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {}})ctx = _auto_act_ctx(server)@@ -489,7 +461,6 @@ async def test_multiple_candidates_no_auto_act(monkeypatch) -> None:@pytest.mark.asyncioasync def test_high_tier_no_auto_act(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {}})ctx = _auto_act_ctx(server)@@ -510,7 +481,6 @@ async def test_high_tier_no_auto_act(monkeypatch) -> None:@pytest.mark.asyncioasync def test_bare_button_submit_no_auto_act(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {}})ctx = _auto_act_ctx(server)@@ -531,7 +501,6 @@ async def test_bare_button_submit_no_auto_act(monkeypatch) -> None:@pytest.mark.asyncioasync def test_empty_text_link_no_auto_act(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {}})ctx = _auto_act_ctx(server)@@ -552,7 +521,6 @@ async def test_empty_text_link_no_auto_act(monkeypatch) -> None:@pytest.mark.asyncioasync def test_single_low_tier_link_auto_acts(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})ctx = _auto_act_ctx(server)@@ -581,7 +549,6 @@ async def test_single_low_tier_link_auto_acts(monkeypatch) -> None:@pytest.mark.asyncioasync def test_auto_act_post_evidence_none_sheds_bulky_result_under_cap(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})ctx = _auto_act_ctx(server)@@ -611,7 +578,6 @@ async def test_auto_act_post_evidence_none_sheds_bulky_result_under_cap(monkeypa@pytest.mark.asyncioasync def test_auto_act_success_branch_sheds_oversized_page_under_cap(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})ctx = _auto_act_ctx(server)@@ -659,7 +625,6 @@ async def test_auto_act_success_branch_sheds_oversized_page_under_cap(monkeypatc@pytest.mark.asyncioasync def test_auto_act_re_arms_on_changed_signature(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})ctx = _auto_act_ctx(server)@@ -701,7 +666,6 @@ async def test_auto_act_re_arms_on_changed_signature(monkeypatch) -> None:@pytest.mark.asyncioasync def test_auto_act_idempotent_on_unchanged_signature(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})ctx = _auto_act_ctx(server)@@ -719,7 +683,6 @@ async def test_auto_act_idempotent_on_unchanged_signature(monkeypatch) -> None:@pytest.mark.asyncioasync def test_auto_act_flag_off_pure_advisory(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", False)server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})ctx = _auto_act_ctx(server)@@ -740,7 +703,6 @@ async def test_auto_act_flag_off_pure_advisory(monkeypatch) -> None:@pytest.mark.asyncioasync def test_auto_act_click_failure_degrades_to_advisory(monkeypatch) -> None:- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": False, "error": "element not found"})ctx = _auto_act_ctx(server)@@ -764,7 +726,6 @@ async def test_auto_act_click_failure_degrades_to_advisory(monkeypatch) -> None:async def test_auto_act_front_runs_the_third_evaluate(monkeypatch) -> None:from skyvern.forge.sdk.copilot.loop_detection import detect_tool_loop- monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})ctx = _auto_act_ctx(server)