Skip to content

Automate browser based workflows with AI

SKY-10823: nudge entry-point discovery before a pre-discovery 'provide the URL' ask (#6473)

Andrew Neilson62b0a0c22528
diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.py
index d7390ab8f..6e74fe3d4 100644
--- a/skyvern/forge/sdk/copilot/config.py
+++ b/skyvern/forge/sdk/copilot/config.py
@@ -271,6 +271,14 @@ POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE = (
"and only when a separate required non-URL input is still missing."
)
+PRE_DISCOVERY_URL_QUESTION_NUDGE = (
+ "STOP — you are asking the user for an entry-point URL before resolving it yourself. "
+ "discover_workflow_entrypoint has not run this turn. Call "
+ "discover_workflow_entrypoint(site_or_url, intent_hint) to resolve the entrypoint from the "
+ "site the user named, then compose from the resolved page. Only ask for a URL if discovery "
+ "runs and cannot resolve a site."
+)
+
PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX = (
"STOP — the target site has failed to scrape on every attempt across "
"multiple workflow shapes. Every run navigated successfully but the "
@@ -346,6 +354,7 @@ DEFAULT_ENFORCEMENT_NUDGES: dict[str, str] = {
"post_per_tool_budget_stop": POST_PER_TOOL_BUDGET_STOP_NUDGE,
"post_no_workflow_delivery": POST_NO_WORKFLOW_DELIVERY_NUDGE,
"post_discovery_entrypoint_url_question": POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE,
+ "pre_discovery_url_question": PRE_DISCOVERY_URL_QUESTION_NUDGE,
"post_probable_site_block_stop_prefix": PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX,
"post_probable_site_block_stop": POST_PROBABLE_SITE_BLOCK_STOP_NUDGE,
"post_anti_bot_failed_test": POST_ANTI_BOT_FAILED_TEST_NUDGE,
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index fd2557a6f..6a4b4ddce 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -486,6 +486,7 @@ class CopilotContext(AgentContext):
resolved_discovery_failure_reason: str | None = None
resolved_discovery_entrypoint_inspection_baseline: int = 0
discovery_entrypoint_url_question_nudge_count: int = 0
+ pre_discovery_url_question_nudge_count: int = 0
# Set in `_run_attempt` after SkyvernOverlayMCPServer is constructed.
# The discovery tool reaches the connected FastMCP client through this.
discovery_mcp_server: Any | None = None
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index d5741b44a..02693fad4 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -15,6 +15,7 @@ from agents.run import Runner
from skyvern.config import settings
from skyvern.forge.sdk.copilot import config as copilot_config_defaults
+from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASES
from skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_block
from skyvern.forge.sdk.copilot.config import (
DEFAULT_ENFORCEMENT_NUDGES,
@@ -37,6 +38,7 @@ from skyvern.forge.sdk.copilot.config import (
POST_REPEATED_NULL_DATA_NUDGE,
POST_SUSPICIOUS_SUCCESS_NUDGE,
POST_UPDATE_NUDGE,
+ PRE_DISCOVERY_URL_QUESTION_NUDGE,
SCREENSHOT_DROPPED_NUDGE,
BlockAuthoringPolicy,
CopilotConfig,
@@ -74,6 +76,7 @@ MAX_FAILED_TEST_NUDGES = 2
MAX_FORMAT_NUDGES = 2
MAX_NO_WORKFLOW_NUDGES = 2
MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES = 2
+MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES = 2
MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES = 2
# Stops the suspicious-success nudge from re-firing forever when the agent has
# correctly diagnosed an unrecoverable block (anti-bot, paywall) and is no
@@ -572,6 +575,46 @@ def _has_candidate_bound_page_evidence(ctx: Any, candidate_url: str) -> bool:
return False
+def _pre_discovery_url_question_nudge(
+ ctx: Any,
+ parsed: dict[str, Any],
+ config: CopilotConfig | None = None,
+) -> str | None:
+ """Steer the model to discovery when it asks before discovery has run.
+
+ INITIAL/DISCOVERING phase with zero discovery calls means the model went
+ straight to asking instead of resolving the entrypoint itself. Credential,
+ loop, and conditional clarifications carry a non-default
+ request_policy.clarification_reason and are let through; the structural
+ triple (phase + zero discovery calls + default clarification_reason) already
+ excludes them. The post-discovery could-not-resolve ask happens after
+ discovery ran (discovery_calls_this_turn > 0) and so never reaches this gate.
+ Steering any remaining pre-discovery ASK to discovery is correct: discovery
+ is cheap, and if the site cannot resolve the model re-asks afterward.
+ """
+ if parsed.get("type") != "ASK_QUESTION":
+ return None
+ if getattr(ctx, "build_phase", None) not in DISCOVERY_PERMITTED_PHASES:
+ return None
+ if _get_int(ctx, "discovery_calls_this_turn") != 0:
+ return None
+ request_policy = getattr(ctx, "request_policy", None)
+ clarification_reason = getattr(request_policy, "clarification_reason", "none")
+ if clarification_reason not in (None, "none"):
+ return None
+ nudge_count = _get_int(ctx, "pre_discovery_url_question_nudge_count")
+ if nudge_count >= MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES:
+ return None
+ ctx.pre_discovery_url_question_nudge_count = nudge_count + 1
+ LOG.info(
+ "copilot.pre_discovery_url_question_nudge",
+ reason_code="pre_discovery_url_question_steer_to_discovery",
+ build_phase=getattr(getattr(ctx, "build_phase", None), "value", None),
+ nudge_count=ctx.pre_discovery_url_question_nudge_count,
+ )
+ return _nudge(config, "pre_discovery_url_question")
+
+
def _post_discovery_entrypoint_url_question_nudge(
ctx: Any,
parsed: dict[str, Any],
@@ -604,6 +647,10 @@ def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any], config: CopilotCo
Returns the nudge string to inject, or None to let the response through.
"""
response_type = parsed.get("type")
+ pre_discovery_nudge = _pre_discovery_url_question_nudge(ctx, parsed, config)
+ if pre_discovery_nudge is not None:
+ return pre_discovery_nudge
+
discovery_entrypoint_nudge = _post_discovery_entrypoint_url_question_nudge(ctx, parsed, config)
if discovery_entrypoint_nudge is not None:
return discovery_entrypoint_nudge
@@ -1249,6 +1296,7 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {
POST_PER_TOOL_BUDGET_STOP_NUDGE: "per_tool_budget_stop",
POST_NO_WORKFLOW_DELIVERY_NUDGE: "no_workflow_delivery",
POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE: "discovery_entrypoint_url_question",
+ PRE_DISCOVERY_URL_QUESTION_NUDGE: "pre_discovery_url_question",
POST_FAILED_TEST_NUDGE: "post_failed_test",
POST_FAILED_TEST_INSPECT_FIRST_NUDGE: "post_failed_test_inspect_first",
SCREENSHOT_DROPPED_NUDGE: "screenshot_dropped_on_recovery",
@@ -1273,6 +1321,7 @@ _NUDGE_TYPE_BY_KEY: dict[str, str] = {
"post_per_tool_budget_stop": "per_tool_budget_stop",
"post_no_workflow_delivery": "no_workflow_delivery",
"post_discovery_entrypoint_url_question": "discovery_entrypoint_url_question",
+ "pre_discovery_url_question": "pre_discovery_url_question",
"post_failed_test": "post_failed_test",
"post_failed_test_inspect_first": "post_failed_test_inspect_first",
"screenshot_dropped": "screenshot_dropped_on_recovery",
diff --git a/tests/unit/test_copilot_pre_discovery_url_nudge.py b/tests/unit/test_copilot_pre_discovery_url_nudge.py
new file mode 100644
index 000000000..5c7d297fb
--- /dev/null
+++ b/tests/unit/test_copilot_pre_discovery_url_nudge.py
@@ -0,0 +1,144 @@
+"""Tests for the pre-discovery entry-point-URL ASK gate in _check_enforcement.
+
+A fresh BUILD turn that names a site but no exact URL enters BuildPhase.INITIAL.
+The model is supposed to call discover_workflow_entrypoint, but at temperature=1
+it occasionally samples straight to an ASK_QUESTION demanding the exact URL. The
+gate fires on the structural triple (phase in DISCOVERY_PERMITTED_PHASES,
+discovery_calls_this_turn == 0, default clarification_reason) with no text
+matching, steering the model to discovery while leaving credential / loop /
+conditional clarifications and the post-discovery could-not-resolve ask
+untouched.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from skyvern.forge.sdk.copilot.build_phase import BuildPhase
+from skyvern.forge.sdk.copilot.enforcement import (
+ MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES,
+ PRE_DISCOVERY_URL_QUESTION_NUDGE,
+ _pre_discovery_url_question_nudge,
+ _response_coverage_nudge,
+)
+
+
+class _Ctx:
+ """Minimal stand-in for CopilotContext used in the pre-discovery gate."""
+
+ def __init__(self) -> None:
+ self.build_phase = BuildPhase.INITIAL
+ self.discovery_calls_this_turn = 0
+ self.pre_discovery_url_question_nudge_count = 0
+ self.request_policy = None
+ # Fields the post-discovery sibling reads when _response_coverage_nudge runs.
+ self.resolved_discovery_entrypoint_url = None
+ self.resolved_discovery_failure_reason = None
+ self.update_workflow_called = False
+ self.no_workflow_nudge_count = 0
+ self.coverage_nudge_count = 0
+ self.format_nudge_count = 0
+
+
+_URL_ASK = {
+ "type": "ASK_QUESTION",
+ "user_response": "I can build that, but I need the exact URL to use as the entry point. Please provide the URL.",
+}
+
+
+def test_pre_discovery_url_ask_in_initial_phase_steers_to_discovery() -> None:
+ ctx = _Ctx()
+ nudge = _pre_discovery_url_question_nudge(ctx, _URL_ASK)
+ assert nudge == PRE_DISCOVERY_URL_QUESTION_NUDGE
+ assert ctx.pre_discovery_url_question_nudge_count == 1
+
+
+def test_pre_discovery_url_ask_fires_through_response_coverage_gate() -> None:
+ ctx = _Ctx()
+ assert _response_coverage_nudge(ctx, _URL_ASK) == PRE_DISCOVERY_URL_QUESTION_NUDGE
+
+
+def test_pre_discovery_url_ask_in_discovering_phase_steers_to_discovery() -> None:
+ ctx = _Ctx()
+ ctx.build_phase = BuildPhase.DISCOVERING
+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) == PRE_DISCOVERY_URL_QUESTION_NUDGE
+
+
+def test_no_nudge_after_discovery_ran_this_turn() -> None:
+ ctx = _Ctx()
+ ctx.discovery_calls_this_turn = 1
+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None
+
+
+def test_no_nudge_outside_initial_or_discovering_phase() -> None:
+ ctx = _Ctx()
+ ctx.build_phase = BuildPhase.COMPOSING
+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None
+
+
+def test_no_nudge_for_non_ask_question() -> None:
+ ctx = _Ctx()
+ reply = {"type": "REPLY", "user_response": "Please provide the URL to start from."}
+ assert _pre_discovery_url_question_nudge(ctx, reply) is None
+
+
+# ---------------------------------------------------------------------------
+# No over-suppression — legitimate INITIAL asks are let through.
+# ---------------------------------------------------------------------------
+
+
+def test_credential_clarification_passes_through() -> None:
+ ctx = _Ctx()
+ ctx.request_policy = SimpleNamespace(clarification_reason="credential_name_unresolved")
+ ask = {
+ "type": "ASK_QUESTION",
+ "user_response": "Which saved credential should I use for the login page?",
+ }
+ assert _pre_discovery_url_question_nudge(ctx, ask) is None
+
+
+def test_loop_clarification_passes_through() -> None:
+ ctx = _Ctx()
+ ctx.request_policy = SimpleNamespace(clarification_reason="ambiguous_loop_edit")
+ ask = {"type": "ASK_QUESTION", "user_response": "Which loop block on the page should I edit?"}
+ assert _pre_discovery_url_question_nudge(ctx, ask) is None
+
+
+def test_conditional_clarification_passes_through() -> None:
+ ctx = _Ctx()
+ ctx.request_policy = SimpleNamespace(clarification_reason="missing_conditional_condition")
+ ask = {"type": "ASK_QUESTION", "user_response": "What condition should gate this site visit?"}
+ assert _pre_discovery_url_question_nudge(ctx, ask) is None
+
+
+def test_post_discovery_could_not_resolve_site_name_ask_passes_through() -> None:
+ # Discovery ran and failed to resolve a site → discovery_calls_this_turn > 0,
+ # so the genuine "which URL?" ask is let through to the user.
+ ctx = _Ctx()
+ ctx.discovery_calls_this_turn = 1
+ ctx.resolved_discovery_failure_reason = "could_not_resolve_site_name"
+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None
+
+
+def test_any_pre_discovery_ask_on_structural_triple_steers_to_discovery() -> None:
+ # No text matching: any ASK_QUESTION on the structural triple (INITIAL/0
+ # discovery / default clarification_reason) is steered to discovery, even
+ # when the phrasing names neither a URL nor a site.
+ ctx = _Ctx()
+ ask = {"type": "ASK_QUESTION", "user_response": "Where should the agent begin?"}
+ assert _pre_discovery_url_question_nudge(ctx, ask) == PRE_DISCOVERY_URL_QUESTION_NUDGE
+ assert ctx.pre_discovery_url_question_nudge_count == 1
+
+
+# ---------------------------------------------------------------------------
+# Bounded — the (N+1)th ask is let through.
+# ---------------------------------------------------------------------------
+
+
+def test_pre_discovery_nudge_is_bounded() -> None:
+ ctx = _Ctx()
+ for expected_count in range(1, MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES + 1):
+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) == PRE_DISCOVERY_URL_QUESTION_NUDGE
+ assert ctx.pre_discovery_url_question_nudge_count == expected_count
+ assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None
+ assert ctx.pre_discovery_url_question_nudge_count == MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES