Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10328): compose after discovered entrypoint (#6336)
Andrew Neilsonf9fc0421c590
diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.pyindex add2ca340..eaeb73029 100644--- a/skyvern/forge/sdk/copilot/config.py+++ b/skyvern/forge/sdk/copilot/config.py@@ -241,6 +241,14 @@ POST_NO_WORKFLOW_DELIVERY_NUDGE = ("workflow proposal behind the response.")+POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE = (+ "STOP — discover_workflow_entrypoint already resolved a candidate_url for this build turn, "+ "but you have not inspected the resolved page or composed from it yet. Use the resolved "+ "candidate_url as the goto_url entrypoint, inspect the page if needed, then call "+ "update_and_run_blocks. Only ask a clarifying question after using the resolved page evidence "+ "and only when a separate required non-URL input is still missing."+)+PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX = ("STOP — the target site has failed to scrape on every attempt across ""multiple workflow shapes. Every run navigated successfully but the "@@ -315,6 +323,7 @@ DEFAULT_ENFORCEMENT_NUDGES: dict[str, str] = {"post_per_tool_budget": POST_PER_TOOL_BUDGET_NUDGE,"post_per_tool_budget_stop": POST_PER_TOOL_BUDGET_STOP_NUDGE,"post_no_workflow_delivery": POST_NO_WORKFLOW_DELIVERY_NUDGE,+ "post_discovery_entrypoint_url_question": POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE,"post_probable_site_block_stop_prefix": PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX,"post_probable_site_block_stop": POST_PROBABLE_SITE_BLOCK_STOP_NUDGE,"post_anti_bot_failed_test": POST_ANTI_BOT_FAILED_TEST_NUDGE,diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex f8dffe2ee..ff7845168 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -452,6 +452,10 @@ class CopilotContext(AgentContext):discovery_step_count: int = 0discovery_started_monotonic: float | None = Nonediscovery_evidence_trail: list[dict[str, Any]] = field(default_factory=list)+ resolved_discovery_entrypoint_url: str | None = None+ resolved_discovery_failure_reason: str | None = None+ resolved_discovery_entrypoint_inspection_baseline: int = 0+ discovery_entrypoint_url_question_nudge_count: int = 0# Set in `_run_attempt` after SkyvernOverlayMCPServer is constructed.# The discovery tool reaches the connected FastMCP client through this.discovery_mcp_server: Any | None = Nonediff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 70db226d1..17ca00a84 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -8,6 +8,7 @@ import jsonimport reimport timefrom typing import TYPE_CHECKING, Any+from urllib.parse import urlparseimport structlogfrom agents.run import Runner@@ -18,6 +19,7 @@ from skyvern.forge.sdk.copilot.config import (DEFAULT_ENFORCEMENT_NUDGES,DEFAULT_TOKEN_BUDGET,POST_ANTI_BOT_FAILED_TEST_NUDGE,+ POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE,POST_EXPLORE_WITHOUT_WORKFLOW_NUDGE,POST_FAILED_TEST_INSPECT_FIRST_NUDGE,POST_FAILED_TEST_NUDGE,@@ -68,6 +70,7 @@ MAX_INTERMEDIATE_NUDGES = 8MAX_FAILED_TEST_NUDGES = 2MAX_FORMAT_NUDGES = 2MAX_NO_WORKFLOW_NUDGES = 2+MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES = 2MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES = 2# Stops the suspicious-success nudge from re-firing forever when the agent has# correctly diagnosed an unrecoverable block (anti-bot, paywall) and is no@@ -532,14 +535,76 @@ def _goal_likely_needs_more_blocks(user_message: Any, block_count: int, completireturn block_count < estimated_min_blocks+def _same_page(left: str | None, right: str | None) -> bool:+ if not left or not right:+ return False+ try:+ left_parsed = urlparse(left)+ right_parsed = urlparse(right)+ except Exception:+ return False+ if not left_parsed.netloc or not right_parsed.netloc:+ return False+ if left_parsed.netloc.lower() != right_parsed.netloc.lower():+ return False+ left_path = (left_parsed.path or "/").rstrip("/") or "/"+ right_path = (right_parsed.path or "/").rstrip("/") or "/"+ return left_path == right_path+++def _has_candidate_bound_page_evidence(ctx: Any, candidate_url: str) -> bool:+ inspection_count = int(getattr(ctx, "page_inspection_calls_this_turn", 0) or 0)+ inspection_baseline = int(getattr(ctx, "resolved_discovery_entrypoint_inspection_baseline", 0) or 0)+ if inspection_count <= inspection_baseline:+ return False+ evidence = getattr(ctx, "composition_page_evidence", None)+ if not isinstance(evidence, dict):+ return False+ if evidence.get("source_tool") != "inspect_page_for_composition":+ return False+ for key in ("inspected_url", "current_url"):+ value = evidence.get(key)+ if isinstance(value, str) and _same_page(candidate_url, value):+ return True+ return False+++def _post_discovery_entrypoint_url_question_nudge(+ ctx: Any,+ parsed: dict[str, Any],+ config: CopilotConfig | None = None,+) -> str | None:+ if parsed.get("type") != "ASK_QUESTION":+ return None+ candidate_url = getattr(ctx, "resolved_discovery_entrypoint_url", None)+ failure_reason = getattr(ctx, "resolved_discovery_failure_reason", None)+ if not isinstance(candidate_url, str) or not candidate_url or failure_reason:+ return None+ inspected_after_discovery = _has_candidate_bound_page_evidence(ctx, candidate_url)+ mutated_after_discovery = bool(getattr(ctx, "update_workflow_called", False))+ if inspected_after_discovery or mutated_after_discovery:+ return None+ nudge_count = getattr(ctx, "discovery_entrypoint_url_question_nudge_count", 0)+ if nudge_count >= MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES:+ return None+ ctx.discovery_entrypoint_url_question_nudge_count = nudge_count + 1+ return f"{_nudge(config, 'post_discovery_entrypoint_url_question')} Resolved candidate_url: {candidate_url}"++def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any], config: CopilotConfig | None = None) -> str | None:"""Peek at the model's final output and return a nudge for coverage gaps- or progress-narration format. ASK_QUESTION is always let through so the- agent can request missing credentials or disambiguation.+ or progress-narration format. ASK_QUESTION is let through so the agent+ can request missing credentials or disambiguation, except when discovery+ resolved a candidate and the agent has not yet inspected or composed from+ that candidate.Returns the nudge string to inject, or None to let the response through."""response_type = parsed.get("type")+ discovery_entrypoint_nudge = _post_discovery_entrypoint_url_question_nudge(ctx, parsed, config)+ if discovery_entrypoint_nudge is not None:+ return discovery_entrypoint_nudge+if response_type not in ("REPLY", "REPLACE_WORKFLOW"):return None@@ -1180,6 +1245,7 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {POST_PER_TOOL_BUDGET_NUDGE: "per_tool_budget_split",POST_PER_TOOL_BUDGET_STOP_NUDGE: "per_tool_budget_stop",POST_NO_WORKFLOW_DELIVERY_NUDGE: "no_workflow_delivery",+ POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE: "discovery_entrypoint_url_question",POST_FAILED_TEST_NUDGE: "post_failed_test",POST_FAILED_TEST_INSPECT_FIRST_NUDGE: "post_failed_test_inspect_first",SCREENSHOT_DROPPED_NUDGE: "screenshot_dropped_on_recovery",@@ -1203,6 +1269,7 @@ _NUDGE_TYPE_BY_KEY: dict[str, str] = {"post_per_tool_budget": "per_tool_budget_split","post_per_tool_budget_stop": "per_tool_budget_stop","post_no_workflow_delivery": "no_workflow_delivery",+ "post_discovery_entrypoint_url_question": "discovery_entrypoint_url_question","post_failed_test": "post_failed_test","post_failed_test_inspect_first": "post_failed_test_inspect_first","screenshot_dropped": "screenshot_dropped_on_recovery",diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 633e7065d..ec25d8910 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -18,6 +18,7 @@ from urllib.parse import urljoin, urlparseimport structlogimport yamlfrom agents import ToolGuardrailFunctionOutput, ToolInputGuardrail, ToolInputGuardrailData, function_tool+from opentelemetry import trace as otel_tracetry:from bs4 import BeautifulSoup # type: ignore[import-not-found]@@ -6120,6 +6121,7 @@ _DISCOVERY_STEP_CAP = 8_DISCOVERY_EVIDENCE_TRAIL_MAX = 8_DISCOVERY_CANDIDATE_FORM_FIELDS_MAX = 10_DISCOVERY_HTML_BYTES_MAX = 200_000+_DISCOVERY_CONCRETE_HOMEPAGE_CONFIDENCE = 0.6_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS = 10.0_COMPOSITION_VISUAL_SUMMARY_PROMPT_NAME = "workflow-copilot-page-evidence-vision"@@ -6168,6 +6170,20 @@ def _resolve_discovery_entry_url(site_or_url: str) -> tuple[str | None, str]:return None, "unresolved"+def _concrete_homepage_entrypoint(entry_url: str | None, kind: str) -> str | None:+ if kind not in {"domain", "url"} or not entry_url:+ return None+ try:+ parsed = urlparse(entry_url)+ except Exception:+ return None+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:+ return None+ if parsed.path not in {"", "/"} or parsed.params or parsed.query or parsed.fragment:+ return None+ return f"{parsed.scheme}://{parsed.netloc}/"++def _discovery_anchor_score(anchor_text: str,anchor_title: str,@@ -6244,6 +6260,40 @@ def _discovery_build_result(}+def _record_discovery_resolution_on_ctx(ctx: Any, result: Mapping[str, Any]) -> None:+ data_payload = result.get("data")+ data: Mapping[str, Any] = data_payload if isinstance(data_payload, Mapping) else {}+ candidate_url = data.get("candidate_url")+ failure_reason = data.get("failure_reason")+ if isinstance(candidate_url, str) and candidate_url:+ prior_candidate_url = getattr(ctx, "resolved_discovery_entrypoint_url", None)+ ctx.resolved_discovery_entrypoint_url = candidate_url+ ctx.resolved_discovery_failure_reason = (+ failure_reason if isinstance(failure_reason, str) and failure_reason else None+ )+ if prior_candidate_url != candidate_url:+ ctx.resolved_discovery_entrypoint_inspection_baseline = int(+ getattr(ctx, "page_inspection_calls_this_turn", 0) or 0+ )+ ctx.discovery_entrypoint_url_question_nudge_count = 0+ # Prior successful candidates remain authoritative over later no-candidate failures.+ elif not getattr(ctx, "resolved_discovery_entrypoint_url", None):+ # No prior candidate and no new one: clear URL state while recording the failure.+ ctx.resolved_discovery_entrypoint_url = None+ ctx.resolved_discovery_failure_reason = (+ failure_reason if isinstance(failure_reason, str) and failure_reason else None+ )+ ctx.resolved_discovery_entrypoint_inspection_baseline = 0+ try:+ current_span = otel_trace.get_current_span()+ if ctx.resolved_discovery_entrypoint_url is not None:+ current_span.set_attribute("copilot.discovery_candidate_url", ctx.resolved_discovery_entrypoint_url)+ if ctx.resolved_discovery_failure_reason is not None:+ current_span.set_attribute("copilot.discovery_failure_reason", ctx.resolved_discovery_failure_reason)+ except Exception:+ LOG.debug("Unable to set discovery resolution span attributes", exc_info=True)++def _discovery_parse_html(html: str) -> tuple[str, list[dict[str, str]], list[dict[str, str]]]:"""Parse HTML for page title, link anchors, and form-field metadata.@@ -7150,11 +7200,29 @@ async def _discover_workflow_entrypoint_impl("""arguments = {"site_or_url": site_or_url, "intent_hint": intent_hint}+ def finish(result: dict[str, Any], *, site_or_url_kind: str | None = None) -> dict[str, Any]:+ _record_discovery_resolution_on_ctx(copilot_ctx, result)+ record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)+ data_payload = result.get("data")+ data = data_payload if isinstance(data_payload, Mapping) else {}+ candidate_url = data.get("candidate_url")+ failure_reason = data.get("failure_reason")+ if not isinstance(failure_reason, str) or not failure_reason:+ error = result.get("error")+ failure_reason = error if isinstance(error, str) and error else None+ LOG.info(+ "discover_workflow_entrypoint completed",+ ok=result.get("ok"),+ candidate_url=candidate_url if isinstance(candidate_url, str) and candidate_url else None,+ failure_reason=failure_reason,+ site_or_url_kind=site_or_url_kind,+ )+ return result+authority_error = _authority_tool_error(copilot_ctx, "discover_workflow_entrypoint")if authority_error:result = {"ok": False, "error": authority_error}- record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)- return result+ return finish(result)if copilot_ctx.discovery_calls_this_turn >= _DISCOVERY_PER_TURN_BUDGET:result = _discovery_build_result(@@ -7164,8 +7232,7 @@ async def _discover_workflow_entrypoint_impl(confidence=0.0,failure_reason="discovery_already_completed_this_turn",)- record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)- return result+ return finish(result)cumulative = copilot_ctx.prior_discovery_calls_made + copilot_ctx.discovery_calls_this_turnif cumulative >= _DISCOVERY_PER_CHAT_BUDGET:@@ -7176,8 +7243,7 @@ async def _discover_workflow_entrypoint_impl(confidence=0.0,failure_reason="discovery_budget_exhausted_for_chat",)- record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)- return result+ return finish(result)entry_url, kind = _resolve_discovery_entry_url(site_or_url)if entry_url is None:@@ -7188,8 +7254,7 @@ async def _discover_workflow_entrypoint_impl(confidence=0.0,failure_reason="could_not_resolve_site_name",)- record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)- return result+ return finish(result, site_or_url_kind=kind)if copilot_ctx.build_phase == BuildPhase.INITIAL:try:@@ -7204,6 +7269,34 @@ async def _discover_workflow_entrypoint_impl()copilot_ctx.discovery_calls_this_turn += 1+ concrete_homepage_url = _concrete_homepage_entrypoint(entry_url, kind)+ if concrete_homepage_url is not None:+ evidence_trail = [+ {+ "url": concrete_homepage_url,+ "page_title": "",+ "transition_reason": "concrete_domain_homepage",+ }+ ]+ result = _discovery_build_result(+ candidate_url=concrete_homepage_url,+ candidate_form_fields=[],+ evidence_trail=evidence_trail,+ # Lower than a scraped-page match because the fast path skips page inspection.+ confidence=_DISCOVERY_CONCRETE_HOMEPAGE_CONFIDENCE,+ failure_reason=None,+ )+ copilot_ctx.discovery_evidence_trail = list(evidence_trail)+ try:+ advance_to_composing(copilot_ctx, reason="discovery_concrete_domain_homepage")+ except ValueError as exc:+ LOG.warning(+ "discover_workflow_entrypoint phase transition to composing rejected",+ error=str(exc),+ build_phase=copilot_ctx.build_phase.value,+ )+ return finish(result, site_or_url_kind=kind)+with copilot_span("discover_workflow_entrypoint",data={@@ -7245,8 +7338,7 @@ async def _discover_workflow_entrypoint_impl(build_phase=copilot_ctx.build_phase.value,)- record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)- return result+ return finish(result, site_or_url_kind=kind)@function_tool(name_override="discover_workflow_entrypoint", strict_mode=False)diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.pyindex bed985a31..fb0cceefe 100644--- a/tests/unit/test_copilot_response_coverage_gate.py+++ b/tests/unit/test_copilot_response_coverage_gate.py@@ -16,9 +16,11 @@ from types import SimpleNamespacefrom typing import Anyfrom skyvern.forge.sdk.copilot.enforcement import (+ MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES,MAX_FORMAT_NUDGES,MAX_INTERMEDIATE_NUDGES,MAX_NO_WORKFLOW_NUDGES,+ POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE,POST_FORMAT_NUDGE,POST_INTERMEDIATE_SUCCESS_NUDGE,POST_NO_WORKFLOW_DELIVERY_NUDGE,@@ -41,8 +43,14 @@ class _Ctx:self.coverage_nudge_count = 0self.format_nudge_count = 0self.no_workflow_nudge_count = 0+ self.discovery_entrypoint_url_question_nudge_count = 0self.user_message = ""self.request_policy = None+ self.resolved_discovery_entrypoint_url = None+ self.resolved_discovery_failure_reason = None+ self.resolved_discovery_entrypoint_inspection_baseline = 0+ self.page_inspection_calls_this_turn = 0+ self.composition_page_evidence = Noneself.last_update_block_count = Noneself.last_test_ok = Noneself.last_test_failure_reason = None@@ -118,13 +126,102 @@ def test_coverage_nudge_respects_counter_cap() -> None:assert _response_coverage_nudge(ctx, parsed) is None-def test_ask_question_always_passes_through_even_with_coverage_gap() -> None:+def test_ask_question_passes_through_even_with_coverage_gap() -> None:ctx = _post_success_ctx("go to site and download file")parsed = {"type": "ASK_QUESTION", "user_response": "Which file do you mean?"}assert _response_coverage_nudge(ctx, parsed) is Noneassert ctx.coverage_nudge_count == 0+def test_ask_question_before_acting_on_discovery_candidate_fires_nudge() -> None:+ ctx = _Ctx()+ ctx.resolved_discovery_entrypoint_url = "https://example.com/"+ parsed = {+ "type": "ASK_QUESTION",+ "user_response": "Which file should I download?",+ }++ nudge = _response_coverage_nudge(ctx, parsed)++ assert nudge is not None+ assert nudge.startswith(POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE)+ assert "https://example.com/" in nudge+ assert ctx.discovery_entrypoint_url_question_nudge_count == 1+++def test_ask_question_after_discovery_failure_still_passes_through() -> None:+ ctx = _Ctx()+ ctx.resolved_discovery_entrypoint_url = "https://example.com/"+ ctx.resolved_discovery_failure_reason = "could_not_resolve_site_name"+ parsed = {"type": "ASK_QUESTION", "user_response": "Which URL should I use?"}++ assert _response_coverage_nudge(ctx, parsed) is None+++def test_ask_question_before_acting_on_discovery_candidate_caps_nudges() -> None:+ ctx = _Ctx()+ ctx.resolved_discovery_entrypoint_url = "https://example.com/"+ parsed = {"type": "ASK_QUESTION", "user_response": "Which file should I download?"}++ for expected_count in range(1, MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES + 1):+ assert _response_coverage_nudge(ctx, parsed) is not None+ assert ctx.discovery_entrypoint_url_question_nudge_count == expected_count+ assert _response_coverage_nudge(ctx, parsed) is None+ assert ctx.discovery_entrypoint_url_question_nudge_count == MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES+++def test_ask_question_after_page_inspection_of_discovery_candidate_passes_through() -> None:+ ctx = _Ctx()+ ctx.resolved_discovery_entrypoint_url = "https://example.com/"+ ctx.page_inspection_calls_this_turn = 1+ ctx.composition_page_evidence = {+ "source_tool": "inspect_page_for_composition",+ "inspected_url": "https://example.com/",+ "current_url": "https://example.com/",+ }+ parsed = {"type": "ASK_QUESTION", "user_response": "Which account should I use?"}++ assert _response_coverage_nudge(ctx, parsed) is None+++def test_ask_question_after_unrelated_page_inspection_of_discovery_candidate_still_fires_nudge() -> None:+ ctx = _Ctx()+ ctx.resolved_discovery_entrypoint_url = "https://example.com/"+ ctx.page_inspection_calls_this_turn = 1+ ctx.composition_page_evidence = {+ "source_tool": "inspect_page_for_composition",+ "inspected_url": "https://other.example/",+ "current_url": "https://other.example/",+ }+ parsed = {"type": "ASK_QUESTION", "user_response": "Which account should I use?"}++ assert _response_coverage_nudge(ctx, parsed) is not None+++def test_ask_question_after_stale_candidate_page_inspection_still_fires_nudge() -> None:+ ctx = _Ctx()+ ctx.resolved_discovery_entrypoint_url = "https://example.com/"+ ctx.resolved_discovery_entrypoint_inspection_baseline = 1+ ctx.page_inspection_calls_this_turn = 1+ ctx.composition_page_evidence = {+ "source_tool": "inspect_page_for_composition",+ "inspected_url": "https://example.com/",+ "current_url": "https://example.com/",+ }+ parsed = {"type": "ASK_QUESTION", "user_response": "Which account should I use?"}++ assert _response_coverage_nudge(ctx, parsed) is not None+++def test_ask_question_after_mutating_from_discovery_candidate_passes_through() -> None:+ ctx = _Ctx()+ ctx.resolved_discovery_entrypoint_url = "https://example.com/"+ ctx.update_workflow_called = True+ parsed = {"type": "ASK_QUESTION", "user_response": "Which account should I use?"}++ assert _response_coverage_nudge(ctx, parsed) is None++def test_reply_without_coverage_gap_passes_through() -> None:# 2-action goal and 2 blocks — no gap.ctx = _post_success_ctx("go to X and download Y", block_count=2)