Skip to content

Automate browser based workflows with AI

SKY-11024: steer copilot to author a terminal download block on a reached affordance (#6592)

Andrew Neilsonb98eded6f324
diff --git a/skyvern/config.py b/skyvern/config.py
index daa107a1b..72e442e64 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -158,6 +158,10 @@ class Settings(BaseSettings):
# On a repeat evaluate of an unchanged page with exactly one safe target, click it in-process
# instead of only advising. Off (default) leaves the advisory steer as the only response.
COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED: bool = False
+ # After a correct click reaches a same-host download affordance (or a browser download has
+ # already registered into a block output), attach a typed reached-download target and steer
+ # the agent to author one terminal download code block. Off (default) restores today's behavior.
+ COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED: bool = False
# Dispatch flag for the workflow copilot v2 (openai-agents-SDK rewrite).
# Off = existing direct-LLM copilot at workflow_copilot_chat_post.
# On = new agent-SDK path under skyvern.forge.sdk.copilot.
diff --git a/skyvern/forge/sdk/copilot/composition_browser_expressions.py b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
index e0a0c963f..e2c5e80b8 100644
--- a/skyvern/forge/sdk/copilot/composition_browser_expressions.py
+++ b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
@@ -325,7 +325,9 @@ for (const link of document.querySelectorAll('a[href]')) {
let resolved; try { resolved = new URL(rawHref, location.href).href; } catch (e) { continue; }
let host; try { host = new URL(resolved).host.toLowerCase(); } catch (e) { continue; }
if (!host || host !== baseHost) continue;
- navTargets.push({ text: nodeText(link), href: resolved, selector: selectorFor(link) });
+ const entry = { text: nodeText(link), href: resolved, selector: selectorFor(link) };
+ if (link.hasAttribute('download')) entry.has_download_attr = true;
+ navTargets.push(entry);
}
const resultContainers = [];
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.py
index 0baf3119f..c51d0db55 100644
--- a/skyvern/forge/sdk/copilot/composition_evidence.py
+++ b/skyvern/forge/sdk/copilot/composition_evidence.py
@@ -15,6 +15,10 @@ except ImportError: # pragma: no cover - bs4 is a transitive dep but inspection
BeautifulSoup = None # type: ignore[assignment, misc]
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
+from skyvern.forge.sdk.copilot.reached_download_target import (
+ NAV_TARGET_DOWNLOAD_KIND_KEY,
+ classify_download_affordance,
+)
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from skyvern.utils.yaml_loader import safe_load_no_dates
@@ -1680,13 +1684,18 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
if not _same_origin(resolved_href, current_url or inspected_url):
continue
text = _node_text(link)
- navigation_targets.append(
- {
- "text": _schema_text(text, 160),
- "href": resolved_href[:300],
- "selector": _selector_for(link)[:160],
- }
+ nav_entry: dict[str, Any] = {
+ "text": _schema_text(text, 160),
+ "href": resolved_href[:300],
+ "selector": _selector_for(link)[:160],
+ }
+ download_kind = classify_download_affordance(
+ href=resolved_href,
+ has_download_attr=link.has_attr("download"),
)
+ if download_kind is not None:
+ nav_entry[NAV_TARGET_DOWNLOAD_KIND_KEY] = download_kind
+ navigation_targets.append(nav_entry)
result_containers: list[dict[str, Any]] = []
for node in all_nodes:
@@ -1828,13 +1837,18 @@ def _structured_navigation_targets(value: Any, *, base_url: str) -> list[dict[st
continue
if not _same_origin(href, base_url):
continue
- targets.append(
- {
- "text": _schema_text(_structured_str(link.get("text")), 160),
- "href": href[:300],
- "selector": _structured_str(link.get("selector"))[:160],
- }
+ entry: dict[str, Any] = {
+ "text": _schema_text(_structured_str(link.get("text")), 160),
+ "href": href[:300],
+ "selector": _structured_str(link.get("selector"))[:160],
+ }
+ download_kind = classify_download_affordance(
+ href=href,
+ has_download_attr=link.get("has_download_attr") is True,
)
+ if download_kind is not None:
+ entry[NAV_TARGET_DOWNLOAD_KIND_KEY] = download_kind
+ targets.append(entry)
return targets
diff --git a/skyvern/forge/sdk/copilot/reached_download_target.py b/skyvern/forge/sdk/copilot/reached_download_target.py
new file mode 100644
index 000000000..1b87097fb
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/reached_download_target.py
@@ -0,0 +1,181 @@
+"""Typed reached-download-target signal for the copilot compose surface, matched on the captured
+selector and trajectory recency (never URL identity — a browser download does not change the SPA URL)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Literal, cast
+
+DownloadKind = Literal["registered", "attribute", "extension"]
+
+# ``registered`` is the only kind backed by an actual browser download having fired (S1);
+# ``attribute``/``extension`` are S2 predictions from a scouted link.
+DOWNLOAD_KIND_REGISTERED: DownloadKind = "registered"
+DOWNLOAD_KIND_ATTRIBUTE: DownloadKind = "attribute"
+DOWNLOAD_KIND_EXTENSION: DownloadKind = "extension"
+
+# S2 may only mint a prediction; ``registered`` is S1-only and must never come from a nav target.
+_PREDICTED_DOWNLOAD_KINDS: frozenset[str] = frozenset({DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})
+
+# File extensions that, when ending an href path, mark the link as a direct file download.
+_DOWNLOADABLE_EXTENSIONS: frozenset[str] = frozenset(
+ {
+ "pdf",
+ "csv",
+ "tsv",
+ "xls",
+ "xlsx",
+ "xlsm",
+ "doc",
+ "docx",
+ "ppt",
+ "pptx",
+ "txt",
+ "rtf",
+ "json",
+ "xml",
+ "zip",
+ "gz",
+ "tar",
+ "rar",
+ "7z",
+ "ofx",
+ "qfx",
+ "qbo",
+ "ics",
+ "eml",
+ }
+)
+
+# Block output keys written by the execution-layer download registration when a browser
+# download fired inside a code block. Presence of any of these is hard proof of a download.
+REGISTERED_DOWNLOAD_OUTPUT_KEYS: tuple[str, ...] = (
+ "downloaded_files",
+ "downloaded_file_urls",
+ "downloaded_file_artifact_ids",
+)
+
+# The keys whose typed-affordance hints flow through the scout navTargets capture.
+NAV_TARGET_DOWNLOAD_KIND_KEY = "download_kind"
+
+
+class _SourceStepKind(str, Enum):
+ trajectory_recency = "trajectory_recency"
+ registered_output = "registered_output"
+
+
+@dataclass(frozen=True)
+class ReachedDownloadTarget:
+ selector: str
+ affordance_text: str
+ download_kind: DownloadKind
+ source_step: str
+ already_registered: bool
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "selector": self.selector,
+ "affordance_text": self.affordance_text,
+ "download_kind": self.download_kind,
+ "source_step": self.source_step,
+ "already_registered": self.already_registered,
+ }
+
+
+def _href_path_extension(href: str) -> str:
+ candidate = href.split("?", 1)[0].split("#", 1)[0]
+ last_segment = candidate.rsplit("/", 1)[-1]
+ if "." not in last_segment:
+ return ""
+ return last_segment.rsplit(".", 1)[-1].strip().lower()
+
+
+def classify_download_affordance(*, href: str | None, has_download_attr: bool = False) -> DownloadKind | None:
+ """Type a same-host ``<a href>`` as a download affordance, or None if it is plain navigation.
+
+ A ``download`` attribute wins over a downloadable file extension on the href path."""
+ if has_download_attr:
+ return DOWNLOAD_KIND_ATTRIBUTE
+ if href and _href_path_extension(href) in _DOWNLOADABLE_EXTENSIONS:
+ return DOWNLOAD_KIND_EXTENSION
+ return None
+
+
+def _summary_str(value: Any) -> str:
+ return value.strip() if isinstance(value, str) else ""
+
+
+def derive_from_navigation_targets(navigation_targets: Any) -> ReachedDownloadTarget | None:
+ """S2: derive a single unambiguous typed download target from scouted navigation targets.
+
+ Returns None when zero or more than one same-host download affordance is present, mirroring
+ the auto-act single-survivor rule so an ambiguous page never over-fires the steer."""
+ if not isinstance(navigation_targets, list):
+ return None
+ candidates: list[ReachedDownloadTarget] = []
+ for target in navigation_targets:
+ if not isinstance(target, dict):
+ continue
+ download_kind = _summary_str(target.get(NAV_TARGET_DOWNLOAD_KIND_KEY))
+ selector = _summary_str(target.get("selector"))
+ if download_kind not in _PREDICTED_DOWNLOAD_KINDS or not selector:
+ continue
+ candidates.append(
+ ReachedDownloadTarget(
+ selector=selector,
+ affordance_text=_summary_str(target.get("text")),
+ download_kind=cast(DownloadKind, download_kind),
+ source_step=_SourceStepKind.trajectory_recency.value,
+ already_registered=False,
+ )
+ )
+ if len(candidates) != 1:
+ return None
+ return candidates[0]
+
+
+def block_output_has_registered_download(block_output: Any) -> bool:
+ if not isinstance(block_output, dict):
+ return False
+ return any(bool(block_output.get(key)) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)
+
+
+def derive_from_block_outputs(block_outputs_by_label: Any) -> ReachedDownloadTarget | None:
+ """S1: confirm a reached download from a browser download already registered into a block output.
+
+ This is hard proof a download fired; the typed field carries no selector because the affordance
+ has already been exercised — the agent only needs to know the run converged on a real download."""
+ if not isinstance(block_outputs_by_label, dict):
+ return None
+ for label, output in block_outputs_by_label.items():
+ if block_output_has_registered_download(output):
+ return ReachedDownloadTarget(
+ selector="",
+ affordance_text="",
+ download_kind=DOWNLOAD_KIND_REGISTERED,
+ source_step=_summary_str(label) or _SourceStepKind.registered_output.value,
+ already_registered=True,
+ )
+ return None
+
+
+_AUTHOR_DOWNLOAD_GUIDANCE = (
+ "A correct click reached a download affordance on the current page. Author ONE terminal "
+ "download code block that fires the browser download for the captured target using the "
+ "expect_download idiom (async with page.expect_download() as dl_info: await "
+ "page.click(<selector>)), not a static-fetch request and not another page re-evaluation. "
+ "The downloaded file is registered to the workflow output surface (downloaded_files); never "
+ "place file bytes or URLs in the chat reply."
+)
+
+_CONFIRMED_DOWNLOAD_GUIDANCE = (
+ "A browser download already registered into the workflow output surface (downloaded_files). "
+ "The download flow is reached — finalize one terminal download code block rather than "
+ "re-evaluating the page or re-authoring static-fetch scout blocks. Never place file bytes or "
+ "URLs in the chat reply."
+)
+
+
+def guidance_for(target: ReachedDownloadTarget) -> str:
+ return _CONFIRMED_DOWNLOAD_GUIDANCE if target.already_registered else _AUTHOR_DOWNLOAD_GUIDANCE
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index b8eb3e1bd..76c9ff13c 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -42,10 +42,10 @@ from .scouting import (
_consume_scout_source_url,
_mark_page_inspected,
_mark_pending_browser_interaction_observation,
- _maybe_steer_evaluate_to_action,
_record_scouted_interaction,
_register_scout_interaction_observation,
_resolve_scout_role_name,
+ _steer_evaluate_result,
)
LOG = structlog.get_logger()
@@ -537,7 +537,7 @@ async def _evaluate_post_hook(
title=title,
observed_data=result["data"],
)
- await _maybe_steer_evaluate_to_action(ctx, result, url=url)
+ await _steer_evaluate_result(ctx, result, url=url)
return result
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 1897a6d0d..36957cff5 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -47,6 +47,10 @@ from skyvern.forge.sdk.copilot.output_utils import (
iter_failure_reasons,
truncate_output,
)
+from skyvern.forge.sdk.copilot.reached_download_target import (
+ derive_from_block_outputs as _derive_reached_download_from_block_outputs,
+)
+from skyvern.forge.sdk.copilot.reached_download_target import guidance_for as _reached_download_guidance_for
from skyvern.forge.sdk.copilot.run_outcome import (
RecordedRunOutcome,
RunOutcomeReasonCode,
@@ -1187,6 +1191,11 @@ async def _run_blocks_and_collect_debug(
"page_title": page_title,
"action_trace_summary": action_trace_summary,
}
+ if SettingsManager.get_settings().COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED:
+ reached_download = _derive_reached_download_from_block_outputs(block_outputs_by_label)
+ if reached_download is not None:
+ result_data["reached_download_target"] = reached_download.to_dict()
+ result_data["reached_download_guidance"] = _reached_download_guidance_for(reached_download)
if runtime_frontier_anchor_url is not None:
result_data["runtime_frontier_anchor_url"] = runtime_frontier_anchor_url
if runtime_frontier_starter_url_seeded:
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 574b78261..b678ee981 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -20,6 +20,10 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
has_bounded_page_schema,
)
from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP
+from skyvern.forge.sdk.copilot.reached_download_target import (
+ derive_from_navigation_targets as _derive_reached_download_from_nav_targets,
+)
+from skyvern.forge.sdk.copilot.reached_download_target import guidance_for as _reached_download_guidance_for
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
PendingBrowserInteractionObservation,
@@ -723,19 +727,36 @@ async def _auto_act_on_repeat(ctx: AgentContext, result: dict[str, Any], *, url:
return True
-async def _maybe_steer_evaluate_to_action(ctx: AgentContext, result: dict[str, Any], *, url: str) -> None:
+class _UnsetEvidence:
+ pass
+
+
+_EVIDENCE_UNSET = _UnsetEvidence()
+
+
+async def _maybe_steer_evaluate_to_action(
+ ctx: AgentContext,
+ result: dict[str, Any],
+ *,
+ url: str,
+ page_evidence: dict[str, Any] | None | _UnsetEvidence = _EVIDENCE_UNSET,
+) -> bool:
data = result.get("data")
if not isinstance(data, dict):
- return
+ return False
try:
- parsed = await _scout_act_observe_page_evidence(ctx, url=url)
+ parsed = (
+ await _scout_act_observe_page_evidence(ctx, url=url)
+ if isinstance(page_evidence, _UnsetEvidence)
+ else page_evidence
+ )
if parsed is None or not has_bounded_page_schema(parsed):
_reset_evaluate_actionable_tracker(ctx)
- return
+ return False
identities = _actionable_target_identities(parsed)
if not identities:
_reset_evaluate_actionable_tracker(ctx)
- return
+ return False
signature = _actionable_target_signature(identities)
# Strict full-URL match (fragment included): on an SPA a hash-route change
# is a navigation, so a differing fragment must read as a different page.
@@ -755,7 +776,7 @@ async def _maybe_steer_evaluate_to_action(ctx: AgentContext, result: dict[str, A
ctx.last_auto_acted_signature = signature
if await _auto_act_on_repeat(ctx, result, url=url, target=candidate):
LOG.info("copilot_evaluate_actionable_target_steer", url=url, is_repeat=True, steered=True)
- return
+ return True
if targets:
data["actionable_targets"] = targets
if is_repeat:
@@ -775,6 +796,58 @@ async def _maybe_steer_evaluate_to_action(ctx: AgentContext, result: dict[str, A
data.pop("next_action_reason", None)
_reset_evaluate_actionable_tracker(ctx)
LOG.warning("copilot_evaluate_actionable_target_steer_failed", exc_info=True)
+ return False
+
+
+async def _maybe_attach_reached_download_target(
+ ctx: AgentContext,
+ result: dict[str, Any],
+ *,
+ url: str,
+ page_evidence: dict[str, Any] | None | _UnsetEvidence = _EVIDENCE_UNSET,
+) -> None:
+ """Attach a typed reached-download target + guidance when the page exposes exactly one same-host
+ download affordance, matched on the captured selector (never URL — a download does not change the SPA URL)."""
+ if not settings.COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED:
+ return
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return
+ try:
+ parsed = (
+ await _scout_act_observe_page_evidence(ctx, url=url)
+ if isinstance(page_evidence, _UnsetEvidence)
+ else page_evidence
+ )
+ if parsed is None:
+ return
+ target = _derive_reached_download_from_nav_targets(parsed.get("navigation_targets"))
+ if target is None:
+ return
+ data["reached_download_target"] = target.to_dict()
+ data["reached_download_guidance"] = _reached_download_guidance_for(target)
+ LOG.info(
+ "copilot_reached_download_target_steer",
+ url=url,
+ download_kind=target.download_kind,
+ already_registered=target.already_registered,
+ )
+ except Exception:
+ data.pop("reached_download_target", None)
+ data.pop("reached_download_guidance", None)
+ LOG.warning("copilot_reached_download_target_steer_failed", exc_info=True)
+
+
+async def _steer_evaluate_result(ctx: AgentContext, result: dict[str, Any], *, url: str) -> None:
+ # Observe the bounded page evidence once and feed both evaluate steers; re-observe for the
+ # download steer only when the actionable steer auto-acted and may have changed the page.
+ if not isinstance(result.get("data"), dict):
+ return
+ page_evidence = await _scout_act_observe_page_evidence(ctx, url=url)
+ acted = await _maybe_steer_evaluate_to_action(ctx, result, url=url, page_evidence=page_evidence)
+ await _maybe_attach_reached_download_target(
+ ctx, result, url=url, page_evidence=_EVIDENCE_UNSET if acted else page_evidence
+ )
def _mark_post_run_page_observed(ctx: AgentContext, *, source_tool: str, url: str) -> None: