Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10823: nudge entry-point discovery before a pre-discovery 'provide the URL' ask (#6473)
Andrew Neilson62b0a0c22528
diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.pyindex d7390ab8f..6e74fe3d4 100644--- a/skyvern/forge/sdk/copilot/config.py+++ b/skyvern/forge/sdk/copilot/config.py@@ -271,6 +271,14 @@ POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE = ("and only when a separate required non-URL input is still missing.")+PRE_DISCOVERY_URL_QUESTION_NUDGE = (+ "STOP — you are asking the user for an entry-point URL before resolving it yourself. "+ "discover_workflow_entrypoint has not run this turn. Call "+ "discover_workflow_entrypoint(site_or_url, intent_hint) to resolve the entrypoint from the "+ "site the user named, then compose from the resolved page. Only ask for a URL if discovery "+ "runs and cannot resolve a site."+)+PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX = ("STOP — the target site has failed to scrape on every attempt across ""multiple workflow shapes. Every run navigated successfully but the "@@ -346,6 +354,7 @@ DEFAULT_ENFORCEMENT_NUDGES: dict[str, str] = {"post_per_tool_budget_stop": POST_PER_TOOL_BUDGET_STOP_NUDGE,"post_no_workflow_delivery": POST_NO_WORKFLOW_DELIVERY_NUDGE,"post_discovery_entrypoint_url_question": POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE,+ "pre_discovery_url_question": PRE_DISCOVERY_URL_QUESTION_NUDGE,"post_probable_site_block_stop_prefix": PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX,"post_probable_site_block_stop": POST_PROBABLE_SITE_BLOCK_STOP_NUDGE,"post_anti_bot_failed_test": POST_ANTI_BOT_FAILED_TEST_NUDGE,diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex fd2557a6f..6a4b4ddce 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -486,6 +486,7 @@ class CopilotContext(AgentContext):resolved_discovery_failure_reason: str | None = Noneresolved_discovery_entrypoint_inspection_baseline: int = 0discovery_entrypoint_url_question_nudge_count: int = 0+ pre_discovery_url_question_nudge_count: int = 0# Set in `_run_attempt` after SkyvernOverlayMCPServer is constructed.# The discovery tool reaches the connected FastMCP client through this.discovery_mcp_server: Any | None = Nonediff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex d5741b44a..02693fad4 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -15,6 +15,7 @@ from agents.run import Runnerfrom skyvern.config import settingsfrom skyvern.forge.sdk.copilot import config as copilot_config_defaults+from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASESfrom skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_blockfrom skyvern.forge.sdk.copilot.config import (DEFAULT_ENFORCEMENT_NUDGES,@@ -37,6 +38,7 @@ from skyvern.forge.sdk.copilot.config import (POST_REPEATED_NULL_DATA_NUDGE,POST_SUSPICIOUS_SUCCESS_NUDGE,POST_UPDATE_NUDGE,+ PRE_DISCOVERY_URL_QUESTION_NUDGE,SCREENSHOT_DROPPED_NUDGE,BlockAuthoringPolicy,CopilotConfig,@@ -74,6 +76,7 @@ MAX_FAILED_TEST_NUDGES = 2MAX_FORMAT_NUDGES = 2MAX_NO_WORKFLOW_NUDGES = 2MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES = 2+MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES = 2MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES = 2# Stops the suspicious-success nudge from re-firing forever when the agent has# correctly diagnosed an unrecoverable block (anti-bot, paywall) and is no@@ -572,6 +575,46 @@ def _has_candidate_bound_page_evidence(ctx: Any, candidate_url: str) -> bool:return False+def _pre_discovery_url_question_nudge(+ ctx: Any,+ parsed: dict[str, Any],+ config: CopilotConfig | None = None,+) -> str | None:+ """Steer the model to discovery when it asks before discovery has run.++ INITIAL/DISCOVERING phase with zero discovery calls means the model went+ straight to asking instead of resolving the entrypoint itself. Credential,+ loop, and conditional clarifications carry a non-default+ request_policy.clarification_reason and are let through; the structural+ triple (phase + zero discovery calls + default clarification_reason) already+ excludes them. The post-discovery could-not-resolve ask happens after+ discovery ran (discovery_calls_this_turn > 0) and so never reaches this gate.+ Steering any remaining pre-discovery ASK to discovery is correct: discovery+ is cheap, and if the site cannot resolve the model re-asks afterward.+ """+ if parsed.get("type") != "ASK_QUESTION":+ return None+ if getattr(ctx, "build_phase", None) not in DISCOVERY_PERMITTED_PHASES:+ return None+ if _get_int(ctx, "discovery_calls_this_turn") != 0:+ return None+ request_policy = getattr(ctx, "request_policy", None)+ clarification_reason = getattr(request_policy, "clarification_reason", "none")+ if clarification_reason not in (None, "none"):+ return None+ nudge_count = _get_int(ctx, "pre_discovery_url_question_nudge_count")+ if nudge_count >= MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES:+ return None+ ctx.pre_discovery_url_question_nudge_count = nudge_count + 1+ LOG.info(+ "copilot.pre_discovery_url_question_nudge",+ reason_code="pre_discovery_url_question_steer_to_discovery",+ build_phase=getattr(getattr(ctx, "build_phase", None), "value", None),+ nudge_count=ctx.pre_discovery_url_question_nudge_count,+ )+ return _nudge(config, "pre_discovery_url_question")++def _post_discovery_entrypoint_url_question_nudge(ctx: Any,parsed: dict[str, Any],@@ -604,6 +647,10 @@ def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any], config: CopilotCoReturns the nudge string to inject, or None to let the response through."""response_type = parsed.get("type")+ pre_discovery_nudge = _pre_discovery_url_question_nudge(ctx, parsed, config)+ if pre_discovery_nudge is not None:+ return pre_discovery_nudge+discovery_entrypoint_nudge = _post_discovery_entrypoint_url_question_nudge(ctx, parsed, config)if discovery_entrypoint_nudge is not None:return discovery_entrypoint_nudge@@ -1249,6 +1296,7 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {POST_PER_TOOL_BUDGET_STOP_NUDGE: "per_tool_budget_stop",POST_NO_WORKFLOW_DELIVERY_NUDGE: "no_workflow_delivery",POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE: "discovery_entrypoint_url_question",+ PRE_DISCOVERY_URL_QUESTION_NUDGE: "pre_discovery_url_question",POST_FAILED_TEST_NUDGE: "post_failed_test",POST_FAILED_TEST_INSPECT_FIRST_NUDGE: "post_failed_test_inspect_first",SCREENSHOT_DROPPED_NUDGE: "screenshot_dropped_on_recovery",@@ -1273,6 +1321,7 @@ _NUDGE_TYPE_BY_KEY: dict[str, str] = {"post_per_tool_budget_stop": "per_tool_budget_stop","post_no_workflow_delivery": "no_workflow_delivery","post_discovery_entrypoint_url_question": "discovery_entrypoint_url_question",+ "pre_discovery_url_question": "pre_discovery_url_question","post_failed_test": "post_failed_test","post_failed_test_inspect_first": "post_failed_test_inspect_first","screenshot_dropped": "screenshot_dropped_on_recovery",diff --git a/tests/unit/test_copilot_pre_discovery_url_nudge.py b/tests/unit/test_copilot_pre_discovery_url_nudge.pynew file mode 100644index 000000000..5c7d297fb--- /dev/null+++ b/tests/unit/test_copilot_pre_discovery_url_nudge.py@@ -0,0 +1,144 @@+"""Tests for the pre-discovery entry-point-URL ASK gate in _check_enforcement.++A fresh BUILD turn that names a site but no exact URL enters BuildPhase.INITIAL.+The model is supposed to call discover_workflow_entrypoint, but at temperature=1+it occasionally samples straight to an ASK_QUESTION demanding the exact URL. The+gate fires on the structural triple (phase in DISCOVERY_PERMITTED_PHASES,+discovery_calls_this_turn == 0, default clarification_reason) with no text+matching, steering the model to discovery while leaving credential / loop /+conditional clarifications and the post-discovery could-not-resolve ask+untouched.+"""++from __future__ import annotations++from types import SimpleNamespace++from skyvern.forge.sdk.copilot.build_phase import BuildPhase+from skyvern.forge.sdk.copilot.enforcement import (+ MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES,+ PRE_DISCOVERY_URL_QUESTION_NUDGE,+ _pre_discovery_url_question_nudge,+ _response_coverage_nudge,+)+++class _Ctx:+ """Minimal stand-in for CopilotContext used in the pre-discovery gate."""++ def __init__(self) -> None:+ self.build_phase = BuildPhase.INITIAL+ self.discovery_calls_this_turn = 0+ self.pre_discovery_url_question_nudge_count = 0+ self.request_policy = None+ # Fields the post-discovery sibling reads when _response_coverage_nudge runs.+ self.resolved_discovery_entrypoint_url = None+ self.resolved_discovery_failure_reason = None+ self.update_workflow_called = False+ self.no_workflow_nudge_count = 0+ self.coverage_nudge_count = 0+ self.format_nudge_count = 0+++_URL_ASK = {+ "type": "ASK_QUESTION",+ "user_response": "I can build that, but I need the exact URL to use as the entry point. Please provide the URL.",+}+++def test_pre_discovery_url_ask_in_initial_phase_steers_to_discovery() -> None:+ ctx = _Ctx()+ nudge = _pre_discovery_url_question_nudge(ctx, _URL_ASK)+ assert nudge == PRE_DISCOVERY_URL_QUESTION_NUDGE+ assert ctx.pre_discovery_url_question_nudge_count == 1+++def test_pre_discovery_url_ask_fires_through_response_coverage_gate() -> None:+ ctx = _Ctx()+ assert _response_coverage_nudge(ctx, _URL_ASK) == PRE_DISCOVERY_URL_QUESTION_NUDGE+++def test_pre_discovery_url_ask_in_discovering_phase_steers_to_discovery() -> None:+ ctx = _Ctx()+ ctx.build_phase = BuildPhase.DISCOVERING+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) == PRE_DISCOVERY_URL_QUESTION_NUDGE+++def test_no_nudge_after_discovery_ran_this_turn() -> None:+ ctx = _Ctx()+ ctx.discovery_calls_this_turn = 1+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None+++def test_no_nudge_outside_initial_or_discovering_phase() -> None:+ ctx = _Ctx()+ ctx.build_phase = BuildPhase.COMPOSING+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None+++def test_no_nudge_for_non_ask_question() -> None:+ ctx = _Ctx()+ reply = {"type": "REPLY", "user_response": "Please provide the URL to start from."}+ assert _pre_discovery_url_question_nudge(ctx, reply) is None+++# ---------------------------------------------------------------------------+# No over-suppression — legitimate INITIAL asks are let through.+# ---------------------------------------------------------------------------+++def test_credential_clarification_passes_through() -> None:+ ctx = _Ctx()+ ctx.request_policy = SimpleNamespace(clarification_reason="credential_name_unresolved")+ ask = {+ "type": "ASK_QUESTION",+ "user_response": "Which saved credential should I use for the login page?",+ }+ assert _pre_discovery_url_question_nudge(ctx, ask) is None+++def test_loop_clarification_passes_through() -> None:+ ctx = _Ctx()+ ctx.request_policy = SimpleNamespace(clarification_reason="ambiguous_loop_edit")+ ask = {"type": "ASK_QUESTION", "user_response": "Which loop block on the page should I edit?"}+ assert _pre_discovery_url_question_nudge(ctx, ask) is None+++def test_conditional_clarification_passes_through() -> None:+ ctx = _Ctx()+ ctx.request_policy = SimpleNamespace(clarification_reason="missing_conditional_condition")+ ask = {"type": "ASK_QUESTION", "user_response": "What condition should gate this site visit?"}+ assert _pre_discovery_url_question_nudge(ctx, ask) is None+++def test_post_discovery_could_not_resolve_site_name_ask_passes_through() -> None:+ # Discovery ran and failed to resolve a site → discovery_calls_this_turn > 0,+ # so the genuine "which URL?" ask is let through to the user.+ ctx = _Ctx()+ ctx.discovery_calls_this_turn = 1+ ctx.resolved_discovery_failure_reason = "could_not_resolve_site_name"+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None+++def test_any_pre_discovery_ask_on_structural_triple_steers_to_discovery() -> None:+ # No text matching: any ASK_QUESTION on the structural triple (INITIAL/0+ # discovery / default clarification_reason) is steered to discovery, even+ # when the phrasing names neither a URL nor a site.+ ctx = _Ctx()+ ask = {"type": "ASK_QUESTION", "user_response": "Where should the agent begin?"}+ assert _pre_discovery_url_question_nudge(ctx, ask) == PRE_DISCOVERY_URL_QUESTION_NUDGE+ assert ctx.pre_discovery_url_question_nudge_count == 1+++# ---------------------------------------------------------------------------+# Bounded — the (N+1)th ask is let through.+# ---------------------------------------------------------------------------+++def test_pre_discovery_nudge_is_bounded() -> None:+ ctx = _Ctx()+ for expected_count in range(1, MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES + 1):+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) == PRE_DISCOVERY_URL_QUESTION_NUDGE+ assert ctx.pre_discovery_url_question_nudge_count == expected_count+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None+ assert ctx.pre_discovery_url_question_nudge_count == MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES