Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11024: steer copilot to author a terminal download block on a reached affordance (#6592)
Andrew Neilsonb98eded6f324
diff --git a/skyvern/config.py b/skyvern/config.pyindex daa107a1b..72e442e64 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -158,6 +158,10 @@ class Settings(BaseSettings):# On a repeat evaluate of an unchanged page with exactly one safe target, click it in-process# instead of only advising. Off (default) leaves the advisory steer as the only response.COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED: bool = False+ # After a correct click reaches a same-host download affordance (or a browser download has+ # already registered into a block output), attach a typed reached-download target and steer+ # the agent to author one terminal download code block. Off (default) restores today's behavior.+ COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED: bool = False# Dispatch flag for the workflow copilot v2 (openai-agents-SDK rewrite).# Off = existing direct-LLM copilot at workflow_copilot_chat_post.# On = new agent-SDK path under skyvern.forge.sdk.copilot.diff --git a/skyvern/forge/sdk/copilot/composition_browser_expressions.py b/skyvern/forge/sdk/copilot/composition_browser_expressions.pyindex e0a0c963f..e2c5e80b8 100644--- a/skyvern/forge/sdk/copilot/composition_browser_expressions.py+++ b/skyvern/forge/sdk/copilot/composition_browser_expressions.py@@ -325,7 +325,9 @@ for (const link of document.querySelectorAll('a[href]')) {let resolved; try { resolved = new URL(rawHref, location.href).href; } catch (e) { continue; }let host; try { host = new URL(resolved).host.toLowerCase(); } catch (e) { continue; }if (!host || host !== baseHost) continue;- navTargets.push({ text: nodeText(link), href: resolved, selector: selectorFor(link) });+ const entry = { text: nodeText(link), href: resolved, selector: selectorFor(link) };+ if (link.hasAttribute('download')) entry.has_download_attr = true;+ navTargets.push(entry);}const resultContainers = [];diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.pyindex 0baf3119f..c51d0db55 100644--- a/skyvern/forge/sdk/copilot/composition_evidence.py+++ b/skyvern/forge/sdk/copilot/composition_evidence.py@@ -15,6 +15,10 @@ except ImportError: # pragma: no cover - bs4 is a transitive dep but inspectionBeautifulSoup = None # type: ignore[assignment, misc]from skyvern.forge.sdk.copilot.build_phase import BuildPhase+from skyvern.forge.sdk.copilot.reached_download_target import (+ NAV_TARGET_DOWNLOAD_KIND_KEY,+ classify_download_affordance,+)from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidencefrom skyvern.utils.yaml_loader import safe_load_no_dates@@ -1680,13 +1684,18 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -if not _same_origin(resolved_href, current_url or inspected_url):continuetext = _node_text(link)- navigation_targets.append(- {- "text": _schema_text(text, 160),- "href": resolved_href[:300],- "selector": _selector_for(link)[:160],- }+ nav_entry: dict[str, Any] = {+ "text": _schema_text(text, 160),+ "href": resolved_href[:300],+ "selector": _selector_for(link)[:160],+ }+ download_kind = classify_download_affordance(+ href=resolved_href,+ has_download_attr=link.has_attr("download"),)+ if download_kind is not None:+ nav_entry[NAV_TARGET_DOWNLOAD_KIND_KEY] = download_kind+ navigation_targets.append(nav_entry)result_containers: list[dict[str, Any]] = []for node in all_nodes:@@ -1828,13 +1837,18 @@ def _structured_navigation_targets(value: Any, *, base_url: str) -> list[dict[stcontinueif not _same_origin(href, base_url):continue- targets.append(- {- "text": _schema_text(_structured_str(link.get("text")), 160),- "href": href[:300],- "selector": _structured_str(link.get("selector"))[:160],- }+ entry: dict[str, Any] = {+ "text": _schema_text(_structured_str(link.get("text")), 160),+ "href": href[:300],+ "selector": _structured_str(link.get("selector"))[:160],+ }+ download_kind = classify_download_affordance(+ href=href,+ has_download_attr=link.get("has_download_attr") is True,)+ if download_kind is not None:+ entry[NAV_TARGET_DOWNLOAD_KIND_KEY] = download_kind+ targets.append(entry)return targetsdiff --git a/skyvern/forge/sdk/copilot/reached_download_target.py b/skyvern/forge/sdk/copilot/reached_download_target.pynew file mode 100644index 000000000..1b87097fb--- /dev/null+++ b/skyvern/forge/sdk/copilot/reached_download_target.py@@ -0,0 +1,181 @@+"""Typed reached-download-target signal for the copilot compose surface, matched on the captured+selector and trajectory recency (never URL identity — a browser download does not change the SPA URL)."""++from __future__ import annotations++from dataclasses import dataclass+from enum import Enum+from typing import Any, Literal, cast++DownloadKind = Literal["registered", "attribute", "extension"]++# ``registered`` is the only kind backed by an actual browser download having fired (S1);+# ``attribute``/``extension`` are S2 predictions from a scouted link.+DOWNLOAD_KIND_REGISTERED: DownloadKind = "registered"+DOWNLOAD_KIND_ATTRIBUTE: DownloadKind = "attribute"+DOWNLOAD_KIND_EXTENSION: DownloadKind = "extension"++# S2 may only mint a prediction; ``registered`` is S1-only and must never come from a nav target.+_PREDICTED_DOWNLOAD_KINDS: frozenset[str] = frozenset({DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})++# File extensions that, when ending an href path, mark the link as a direct file download.+_DOWNLOADABLE_EXTENSIONS: frozenset[str] = frozenset(+ {+ "pdf",+ "csv",+ "tsv",+ "xls",+ "xlsx",+ "xlsm",+ "doc",+ "docx",+ "ppt",+ "pptx",+ "txt",+ "rtf",+ "json",+ "xml",+ "zip",+ "gz",+ "tar",+ "rar",+ "7z",+ "ofx",+ "qfx",+ "qbo",+ "ics",+ "eml",+ }+)++# Block output keys written by the execution-layer download registration when a browser+# download fired inside a code block. Presence of any of these is hard proof of a download.+REGISTERED_DOWNLOAD_OUTPUT_KEYS: tuple[str, ...] = (+ "downloaded_files",+ "downloaded_file_urls",+ "downloaded_file_artifact_ids",+)++# The keys whose typed-affordance hints flow through the scout navTargets capture.+NAV_TARGET_DOWNLOAD_KIND_KEY = "download_kind"+++class _SourceStepKind(str, Enum):+ trajectory_recency = "trajectory_recency"+ registered_output = "registered_output"+++@dataclass(frozen=True)+class ReachedDownloadTarget:+ selector: str+ affordance_text: str+ download_kind: DownloadKind+ source_step: str+ already_registered: bool++ def to_dict(self) -> dict[str, Any]:+ return {+ "selector": self.selector,+ "affordance_text": self.affordance_text,+ "download_kind": self.download_kind,+ "source_step": self.source_step,+ "already_registered": self.already_registered,+ }+++def _href_path_extension(href: str) -> str:+ candidate = href.split("?", 1)[0].split("#", 1)[0]+ last_segment = candidate.rsplit("/", 1)[-1]+ if "." not in last_segment:+ return ""+ return last_segment.rsplit(".", 1)[-1].strip().lower()+++def classify_download_affordance(*, href: str | None, has_download_attr: bool = False) -> DownloadKind | None:+ """Type a same-host ``<a href>`` as a download affordance, or None if it is plain navigation.++ A ``download`` attribute wins over a downloadable file extension on the href path."""+ if has_download_attr:+ return DOWNLOAD_KIND_ATTRIBUTE+ if href and _href_path_extension(href) in _DOWNLOADABLE_EXTENSIONS:+ return DOWNLOAD_KIND_EXTENSION+ return None+++def _summary_str(value: Any) -> str:+ return value.strip() if isinstance(value, str) else ""+++def derive_from_navigation_targets(navigation_targets: Any) -> ReachedDownloadTarget | None:+ """S2: derive a single unambiguous typed download target from scouted navigation targets.++ Returns None when zero or more than one same-host download affordance is present, mirroring+ the auto-act single-survivor rule so an ambiguous page never over-fires the steer."""+ if not isinstance(navigation_targets, list):+ return None+ candidates: list[ReachedDownloadTarget] = []+ for target in navigation_targets:+ if not isinstance(target, dict):+ continue+ download_kind = _summary_str(target.get(NAV_TARGET_DOWNLOAD_KIND_KEY))+ selector = _summary_str(target.get("selector"))+ if download_kind not in _PREDICTED_DOWNLOAD_KINDS or not selector:+ continue+ candidates.append(+ ReachedDownloadTarget(+ selector=selector,+ affordance_text=_summary_str(target.get("text")),+ download_kind=cast(DownloadKind, download_kind),+ source_step=_SourceStepKind.trajectory_recency.value,+ already_registered=False,+ )+ )+ if len(candidates) != 1:+ return None+ return candidates[0]+++def block_output_has_registered_download(block_output: Any) -> bool:+ if not isinstance(block_output, dict):+ return False+ return any(bool(block_output.get(key)) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)+++def derive_from_block_outputs(block_outputs_by_label: Any) -> ReachedDownloadTarget | None:+ """S1: confirm a reached download from a browser download already registered into a block output.++ This is hard proof a download fired; the typed field carries no selector because the affordance+ has already been exercised — the agent only needs to know the run converged on a real download."""+ if not isinstance(block_outputs_by_label, dict):+ return None+ for label, output in block_outputs_by_label.items():+ if block_output_has_registered_download(output):+ return ReachedDownloadTarget(+ selector="",+ affordance_text="",+ download_kind=DOWNLOAD_KIND_REGISTERED,+ source_step=_summary_str(label) or _SourceStepKind.registered_output.value,+ already_registered=True,+ )+ return None+++_AUTHOR_DOWNLOAD_GUIDANCE = (+ "A correct click reached a download affordance on the current page. Author ONE terminal "+ "download code block that fires the browser download for the captured target using the "+ "expect_download idiom (async with page.expect_download() as dl_info: await "+ "page.click(<selector>)), not a static-fetch request and not another page re-evaluation. "+ "The downloaded file is registered to the workflow output surface (downloaded_files); never "+ "place file bytes or URLs in the chat reply."+)++_CONFIRMED_DOWNLOAD_GUIDANCE = (+ "A browser download already registered into the workflow output surface (downloaded_files). "+ "The download flow is reached — finalize one terminal download code block rather than "+ "re-evaluating the page or re-authoring static-fetch scout blocks. Never place file bytes or "+ "URLs in the chat reply."+)+++def guidance_for(target: ReachedDownloadTarget) -> str:+ return _CONFIRMED_DOWNLOAD_GUIDANCE if target.already_registered else _AUTHOR_DOWNLOAD_GUIDANCEdiff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex b8eb3e1bd..76c9ff13c 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -42,10 +42,10 @@ from .scouting import (_consume_scout_source_url,_mark_page_inspected,_mark_pending_browser_interaction_observation,- _maybe_steer_evaluate_to_action,_record_scouted_interaction,_register_scout_interaction_observation,_resolve_scout_role_name,+ _steer_evaluate_result,)LOG = structlog.get_logger()@@ -537,7 +537,7 @@ async def _evaluate_post_hook(title=title,observed_data=result["data"],)- await _maybe_steer_evaluate_to_action(ctx, result, url=url)+ await _steer_evaluate_result(ctx, result, url=url)return resultdiff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 1897a6d0d..36957cff5 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -47,6 +47,10 @@ from skyvern.forge.sdk.copilot.output_utils import (iter_failure_reasons,truncate_output,)+from skyvern.forge.sdk.copilot.reached_download_target import (+ derive_from_block_outputs as _derive_reached_download_from_block_outputs,+)+from skyvern.forge.sdk.copilot.reached_download_target import guidance_for as _reached_download_guidance_forfrom skyvern.forge.sdk.copilot.run_outcome import (RecordedRunOutcome,RunOutcomeReasonCode,@@ -1187,6 +1191,11 @@ async def _run_blocks_and_collect_debug("page_title": page_title,"action_trace_summary": action_trace_summary,}+ if SettingsManager.get_settings().COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED:+ reached_download = _derive_reached_download_from_block_outputs(block_outputs_by_label)+ if reached_download is not None:+ result_data["reached_download_target"] = reached_download.to_dict()+ result_data["reached_download_guidance"] = _reached_download_guidance_for(reached_download)if runtime_frontier_anchor_url is not None:result_data["runtime_frontier_anchor_url"] = runtime_frontier_anchor_urlif runtime_frontier_starter_url_seeded:diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.pyindex 574b78261..b678ee981 100644--- a/skyvern/forge/sdk/copilot/tools/scouting.py+++ b/skyvern/forge/sdk/copilot/tools/scouting.py@@ -20,6 +20,10 @@ from skyvern.forge.sdk.copilot.composition_evidence import (has_bounded_page_schema,)from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP+from skyvern.forge.sdk.copilot.reached_download_target import (+ derive_from_navigation_targets as _derive_reached_download_from_nav_targets,+)+from skyvern.forge.sdk.copilot.reached_download_target import guidance_for as _reached_download_guidance_forfrom skyvern.forge.sdk.copilot.runtime import (AgentContext,PendingBrowserInteractionObservation,@@ -723,19 +727,36 @@ async def _auto_act_on_repeat(ctx: AgentContext, result: dict[str, Any], *, url:return True-async def _maybe_steer_evaluate_to_action(ctx: AgentContext, result: dict[str, Any], *, url: str) -> None:+class _UnsetEvidence:+ pass+++_EVIDENCE_UNSET = _UnsetEvidence()+++async def _maybe_steer_evaluate_to_action(+ ctx: AgentContext,+ result: dict[str, Any],+ *,+ url: str,+ page_evidence: dict[str, Any] | None | _UnsetEvidence = _EVIDENCE_UNSET,+) -> bool:data = result.get("data")if not isinstance(data, dict):- return+ return Falsetry:- parsed = await _scout_act_observe_page_evidence(ctx, url=url)+ parsed = (+ await _scout_act_observe_page_evidence(ctx, url=url)+ if isinstance(page_evidence, _UnsetEvidence)+ else page_evidence+ )if parsed is None or not has_bounded_page_schema(parsed):_reset_evaluate_actionable_tracker(ctx)- return+ return Falseidentities = _actionable_target_identities(parsed)if not identities:_reset_evaluate_actionable_tracker(ctx)- return+ return Falsesignature = _actionable_target_signature(identities)# Strict full-URL match (fragment included): on an SPA a hash-route change# is a navigation, so a differing fragment must read as a different page.@@ -755,7 +776,7 @@ async def _maybe_steer_evaluate_to_action(ctx: AgentContext, result: dict[str, Actx.last_auto_acted_signature = signatureif await _auto_act_on_repeat(ctx, result, url=url, target=candidate):LOG.info("copilot_evaluate_actionable_target_steer", url=url, is_repeat=True, steered=True)- return+ return Trueif targets:data["actionable_targets"] = targetsif is_repeat:@@ -775,6 +796,58 @@ async def _maybe_steer_evaluate_to_action(ctx: AgentContext, result: dict[str, Adata.pop("next_action_reason", None)_reset_evaluate_actionable_tracker(ctx)LOG.warning("copilot_evaluate_actionable_target_steer_failed", exc_info=True)+ return False+++async def _maybe_attach_reached_download_target(+ ctx: AgentContext,+ result: dict[str, Any],+ *,+ url: str,+ page_evidence: dict[str, Any] | None | _UnsetEvidence = _EVIDENCE_UNSET,+) -> None:+ """Attach a typed reached-download target + guidance when the page exposes exactly one same-host+ download affordance, matched on the captured selector (never URL — a download does not change the SPA URL)."""+ if not settings.COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED:+ return+ data = result.get("data")+ if not isinstance(data, dict):+ return+ try:+ parsed = (+ await _scout_act_observe_page_evidence(ctx, url=url)+ if isinstance(page_evidence, _UnsetEvidence)+ else page_evidence+ )+ if parsed is None:+ return+ target = _derive_reached_download_from_nav_targets(parsed.get("navigation_targets"))+ if target is None:+ return+ data["reached_download_target"] = target.to_dict()+ data["reached_download_guidance"] = _reached_download_guidance_for(target)+ LOG.info(+ "copilot_reached_download_target_steer",+ url=url,+ download_kind=target.download_kind,+ already_registered=target.already_registered,+ )+ except Exception:+ data.pop("reached_download_target", None)+ data.pop("reached_download_guidance", None)+ LOG.warning("copilot_reached_download_target_steer_failed", exc_info=True)+++async def _steer_evaluate_result(ctx: AgentContext, result: dict[str, Any], *, url: str) -> None:+ # Observe the bounded page evidence once and feed both evaluate steers; re-observe for the+ # download steer only when the actionable steer auto-acted and may have changed the page.+ if not isinstance(result.get("data"), dict):+ return+ page_evidence = await _scout_act_observe_page_evidence(ctx, url=url)+ acted = await _maybe_steer_evaluate_to_action(ctx, result, url=url, page_evidence=page_evidence)+ await _maybe_attach_reached_download_target(+ ctx, result, url=url, page_evidence=_EVIDENCE_UNSET if acted else page_evidence+ )def _mark_post_run_page_observed(ctx: AgentContext, *, source_tool: str, url: str) -> None: