Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-14836 Pair fresh page screenshots with their evidence (#8279)
Andrew Neilson58e02e0e2d19
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex bba92cba6..dc54b11ef 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -82,7 +82,7 @@ from skyvern.forge.sdk.copilot.result_evidence import (from skyvern.forge.sdk.copilot.runtime import (AgentContext,)-from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry+from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotActionRelation, ScreenshotEntryfrom skyvern.forge.sdk.copilot.terminal_predicates import (artifact_health_blocked,outcome_criteria_evaluated,@@ -117,6 +117,7 @@ TOTAL_TIMEOUT_SECONDS = settings.WORKFLOW_COPILOT_TOTAL_TIMEOUT_SECONDS or 900# constant so tests can shrink it instead of paying a full second per deadline.MIN_DEADLINE_REMAINING_SECONDS = 1.0SCREENSHOT_SENTINEL = "[copilot:screenshot] "+PAIRED_OBSERVATION_MARKER = "[copilot:paired-observation] "NUDGE_SENTINEL = "[copilot:nudge] "SCREENSHOT_PLACEHOLDER = SCREENSHOT_SENTINEL + "[prior screenshot removed to save context]"TOKEN_BUDGET = DEFAULT_TOKEN_BUDGET@@ -450,13 +451,42 @@ def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:if not isinstance(pending, list) or not pending:return Nonescreenshots: list[ScreenshotEntry] = list(pending)+ provenance_lines: list[str] = []+ for entry in screenshots:+ provenance = entry.provenance+ fields = {+ "capture_id": entry.capture_id,+ "source_tool": provenance.source_tool,+ "captured_url": provenance.captured_url or "unavailable",+ "dispatch_url": provenance.dispatch_url or "unavailable",+ "observation_step": provenance.observation_step+ if provenance.observation_step is not None+ else "unavailable",+ "browser_session_id": provenance.browser_session_id or "unavailable",+ "dispatch_browser_session_id": provenance.dispatch_browser_session_id or "unavailable",+ "producer_browser_session_id": provenance.producer_browser_session_id or "unavailable",+ "session_binding": provenance.session_binding.value,+ "workflow_run_id": provenance.workflow_run_id or "unavailable",+ "action_relation": provenance.action_relation.value,+ }+ rendered = "; ".join(f"{key}={value}" for key, value in fields.items())+ relation = (+ "This frame was captured during the named page observation."+ if provenance.action_relation is ScreenshotActionRelation.SAME_PAGE_OBSERVATION+ else "This frame records the named source at its stated action relation."+ )+ provenance_lines.append(+ f"Frame provenance: {rendered}. {relation} Its provenance does not claim freshness after later actions."+ )+ paired_marker = (+ PAIRED_OBSERVATION_MARKER+ if screenshots[0].provenance.action_relation is ScreenshotActionRelation.SAME_PAGE_OBSERVATION+ else ""+ )content: list[dict[str, Any]] = [{"type": "input_text",- "text": (- SCREENSHOT_SENTINEL + "Here is the most recent screenshot captured this turn. "- "It shows the page as of that capture and may predate later actions."- ),+ "text": SCREENSHOT_SENTINEL + paired_marker + "\n".join(provenance_lines),},]for entry in screenshots:@@ -476,6 +506,8 @@ def _consume_pending_screenshots(ctx: Any) -> dict[str, Any] | None:pending = getattr(ctx, "pending_screenshots", None)if isinstance(pending, list):pending.clear()+ if hasattr(ctx, "pending_frame_lease"):+ ctx.pending_frame_lease = Nonereturn message@@ -536,6 +568,22 @@ def is_screenshot_message(item: Any) -> bool:return False+def is_paired_observation_message(item: Any) -> bool:+ """Return True only for explicitly marked observation-bound frames."""+ if _item_field(item, "role") != "user":+ return False+ prefix = SCREENSHOT_SENTINEL + PAIRED_OBSERVATION_MARKER+ content = _item_field(item, "content")+ if isinstance(content, str):+ return content.startswith(prefix)+ if not isinstance(content, list):+ return False+ return any(+ isinstance(_item_field(block, "text"), str) and _item_field(block, "text").startswith(prefix)+ for block in content+ )++def _is_nudge_message(item: Any) -> bool:"""Return True if the item is a synthetic enforcement nudge."""if _item_field(item, "role") != "user":diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex 05f92a0b4..0e78da02c 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -52,7 +52,11 @@ from skyvern.forge.sdk.copilot.runtime import (resolve_browser_state_for_context,retire_browser_session_id,)-from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result+from skyvern.forge.sdk.copilot.screenshot_utils import (+ ScreenshotActionRelation,+ ScreenshotProvenance,+ enqueue_screenshot_from_result,+)from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structurefrom skyvern.forge.sdk.copilot.turn_origin import TurnOriginfrom skyvern.utils.contained_effects import contained_effect@@ -1177,7 +1181,27 @@ class SkyvernOverlayMCPServer(MCPServer):copilot_result = _scrub_tool_result(copilot_ctx, copilot_result)record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, copilot_result)- enqueue_screenshot_from_result(copilot_ctx, copilot_result)+ screenshot_data = copilot_result.get("data")+ screenshot_data = screenshot_data if isinstance(screenshot_data, dict) else {}+ captured_url = screenshot_data.get("url") or screenshot_data.get("current_url")+ observation_step = copilot_result.get("observation_step")+ if observation_step is None:+ observation_step = screenshot_data.get("observation_step")+ workflow_run_id = copilot_result.get("workflow_run_id") or screenshot_data.get("workflow_run_id")+ enqueue_screenshot_from_result(+ copilot_ctx,+ copilot_result,+ provenance=ScreenshotProvenance(+ source_tool=tool_name,+ captured_url=captured_url if isinstance(captured_url, str) and captured_url else None,+ observation_step=observation_step+ if isinstance(observation_step, int) and not isinstance(observation_step, bool)+ else None,+ browser_session_id=call_browser_session_id,+ workflow_run_id=workflow_run_id if isinstance(workflow_run_id, str) and workflow_run_id else None,+ action_relation=ScreenshotActionRelation.TOOL_RESULT,+ ),+ )return _copilot_to_call_tool_result(copilot_result)async def call_internal_tool(@@ -1214,6 +1238,9 @@ class SkyvernOverlayMCPServer(MCPServer):phases = _PhaseClock()copilot_name = self._reverse_alias.get(mcp_tool_name, mcp_tool_name)ctx = self._context_provider()+ requested_session_id = mcp_args.get("session_id")+ if not isinstance(requested_session_id, str) or not requested_session_id:+ requested_session_id = Noneobserved_continuity_generation = getattr(ctx, "browser_session_continuity_generation", 0)if not self._client:_log_mcp_timing(ctx, copilot_name, mcp_tool_name, phases, {}, "internal", "not_connected")@@ -1238,8 +1265,10 @@ class SkyvernOverlayMCPServer(MCPServer):if continuity_result is not None:_log_mcp_timing(ctx, copilot_name, mcp_tool_name, phases, {}, "internal", "session_error")return _scrub_tool_result(ctx, continuity_result)- merged_args = {**mcp_args, "session_id": ctx.browser_session_id}- call_browser_session_id = ctx.browser_session_id+ # An explicit session is a dispatch lease chosen before session preparation awaits. Do not+ # relabel that call with mutable ambient context if another concurrent tool replaces it.+ call_browser_session_id = requested_session_id or ctx.browser_session_id+ merged_args = {**mcp_args, "session_id": call_browser_session_id}try:phases.enter("context_enter")async with mcp_browser_context(ctx):diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex ce0ac333f..4b66a0eaf 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -29,7 +29,7 @@ from skyvern.cli.core.session_manager import (from skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy-from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry+from skyvern.forge.sdk.copilot.screenshot_utils import PendingFrameLease, ScreenshotEntryfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.turn_origin import (HealAdoptionFailed,@@ -325,6 +325,7 @@ class AgentContext:browser_session_continuity_disposition: str | None = Nonesupports_vision: bool = Truepending_screenshots: list[ScreenshotEntry] = field(default_factory=list)+ pending_frame_lease: PendingFrameLease | None = Nonetool_activity: list[dict[str, Any]] = field(default_factory=list)unrecoverable_tool_error_streak_count: int = 0unrecoverable_tool_error_signature: str | None = Nonediff --git a/skyvern/forge/sdk/copilot/screenshot_utils.py b/skyvern/forge/sdk/copilot/screenshot_utils.pyindex 0e4a9b324..6af89e6ef 100644--- a/skyvern/forge/sdk/copilot/screenshot_utils.py+++ b/skyvern/forge/sdk/copilot/screenshot_utils.py@@ -3,8 +3,13 @@from __future__ import annotationsimport base64+import hashlibimport io-from dataclasses import dataclass+import json+import time+import uuid+from dataclasses import dataclass, field+from enum import Enumfrom pathlib import Pathfrom typing import Any@@ -18,19 +23,146 @@ COPILOT_JPEG_QUALITY = 60LOG = structlog.get_logger()-@dataclass+class ScreenshotActionRelation(str, Enum):+ SAME_PAGE_OBSERVATION = "same_page_observation"+ AFTER_SOURCE_ACTION = "after_source_action"+ WORKFLOW_RUN_RESULT = "workflow_run_result"+ TOOL_RESULT = "tool_result"+++class ProvenanceBinding(str, Enum):+ AGREE = "agree"+ DISAGREE = "disagree"+ UNAVAILABLE = "unavailable"+++@dataclass(frozen=True)+class ScreenshotProvenance:+ source_tool: str+ captured_url: str | None+ observation_step: int | None+ browser_session_id: str | None+ workflow_run_id: str | None+ action_relation: ScreenshotActionRelation+ dispatch_url: str | None = None+ dispatch_browser_session_id: str | None = None+ producer_browser_session_id: str | None = None+ session_binding: ProvenanceBinding = ProvenanceBinding.UNAVAILABLE++ @classmethod+ def unknown(cls, *, source_tool: str) -> ScreenshotProvenance:+ return cls(+ source_tool=source_tool,+ captured_url=None,+ observation_step=None,+ browser_session_id=None,+ workflow_run_id=None,+ action_relation=ScreenshotActionRelation.TOOL_RESULT,+ )+++@dataclass(frozen=True)class ScreenshotEntry:b64: strmime: str # "image/jpeg" or "image/png"+ capture_id: str+ provenance: ScreenshotProvenance+ captured_at: float = field(default_factory=time.monotonic)+ capture_event_id: str = field(default_factory=lambda: uuid.uuid4().hex)-def resize_screenshot_b64(b64_png: str) -> ScreenshotEntry:+@dataclass(frozen=True)+class CapturedFrame:+ b64: str+ captured_at: float+ captured_url: str | None = None+ browser_session_id: str | None = None+ dispatch_url: str | None = None+ dispatch_browser_session_id: str | None = None+ producer_browser_session_id: str | None = None+ session_binding: ProvenanceBinding = ProvenanceBinding.UNAVAILABLE+++@dataclass(frozen=True)+class PendingFrameLease:+ capture_event_id: str+ capture_id: str+ input_fingerprint: str+++def screenshot_result_facts(+ result: dict[str, Any],+ *,+ dispatch_url: str | None,+ dispatch_browser_session_id: str | None,+) -> tuple[str | None, str | None, ProvenanceBinding]:+ """Return only facts reported by the screenshot producer, plus their dispatch binding.++ Internal MCP responses may run in concise mode, where ``browser_context`` is deliberately+ omitted. In that case the binding is unavailable; the caller's dispatch target remains+ separately labelled and is never promoted to a producer fact.+ """+ data = result.get("data")+ producer_url: str | None = None+ if isinstance(data, dict):+ value = data.get("current_url") or data.get("url")+ producer_url = value if isinstance(value, str) and value else None+ browser_context = result.get("browser_context")+ producer_session_id: str | None = None+ if isinstance(browser_context, dict):+ value = browser_context.get("session_id")+ producer_session_id = value if isinstance(value, str) and value else None+ if dispatch_browser_session_id is None or producer_session_id is None:+ binding = ProvenanceBinding.UNAVAILABLE+ elif dispatch_browser_session_id == producer_session_id:+ binding = ProvenanceBinding.AGREE+ else:+ binding = ProvenanceBinding.DISAGREE+ return producer_url, producer_session_id, binding+++def _fingerprint_projection(value: Any) -> Any:+ if hasattr(value, "model_dump"):+ value = value.model_dump(mode="json")+ if isinstance(value, dict):+ if value.get("type") in {"input_image", "image_url"}:+ return {"type": value.get("type")}+ return {+ str(key): _fingerprint_projection(item)+ for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))+ }+ if isinstance(value, (list, tuple)):+ return [_fingerprint_projection(item) for item in value]+ if isinstance(value, (str, int, float, bool)) or value is None:+ return value+ return repr(value)+++def model_input_fingerprint(items: list[Any]) -> str:+ """Stable, order-sensitive identity for one logical model request.++ Image payloads project to their block type so pixels never enter the serialized+ fingerprint material. The digest distinguishes a provider retry from a later+ request whose model-visible conversation has advanced.+ """+ projected = _fingerprint_projection(items)+ encoded = json.dumps(projected, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()+ return hashlib.sha256(encoded).hexdigest()+++def resize_screenshot_b64(+ b64_png: str,+ *,+ provenance: ScreenshotProvenance,+ captured_at: float | None = None,+) -> ScreenshotEntry:"""Resize a base64 PNG to 1024x768 max and compress to JPEG.Uses thumbnail() to preserve aspect ratio — only shrinks, never enlarges.Converts RGBA/P modes to RGB for JPEG compatibility."""raw = base64.b64decode(b64_png)+ capture_id = f"sha256:{hashlib.sha256(raw).hexdigest()}"img = Image.open(io.BytesIO(raw))img.thumbnail((COPILOT_SCREENSHOT_MAX_WIDTH, COPILOT_SCREENSHOT_MAX_HEIGHT),@@ -43,29 +175,51 @@ def resize_screenshot_b64(b64_png: str) -> ScreenshotEntry:return ScreenshotEntry(b64=base64.b64encode(buf.getvalue()).decode("ascii"),mime="image/jpeg",+ capture_id=capture_id,+ provenance=provenance,+ captured_at=captured_at if captured_at is not None else time.monotonic(),)-def enqueue_screenshot(ctx: Any, b64_png: str) -> bool:+def enqueue_screenshot(+ ctx: Any,+ b64_png: str,+ *,+ provenance: ScreenshotProvenance,+ captured_at: float | None = None,+) -> bool:"""Resize, compress, and store a screenshot for later LLM injection, reporting whether this callleft an entry on the queue. Replaces any previously pending screenshot (cap to 1)."""if not getattr(ctx, "supports_vision", False):return Falsetry:- entry = resize_screenshot_b64(b64_png)+ entry = resize_screenshot_b64(b64_png, provenance=provenance, captured_at=captured_at)except Exception:LOG.warning("Failed to resize copilot screenshot", exc_info=True)return Falsepending = getattr(ctx, "pending_screenshots", None)if not isinstance(pending, list):return False+ if pending and isinstance(pending[0], ScreenshotEntry) and pending[0].captured_at > entry.captured_at:+ LOG.info(+ "copilot_older_screenshot_rejected",+ source_tool=entry.provenance.source_tool,+ pending_source_tool=pending[0].provenance.source_tool,+ )+ return Falsepending.clear()pending.append(entry)return True-def stage_screenshot_from_artifact(ctx: Any, result: dict[str, Any]) -> bool:+def stage_screenshot_from_artifact(+ ctx: Any,+ result: dict[str, Any],+ *,+ provenance: ScreenshotProvenance,+ captured_at: float | None = None,+) -> bool:"""Stage the frame a non-inline screenshot tool call wrote to disk, reporting whether this callleft an entry on the queue rather than whether the queue is merely non-empty."""@@ -78,13 +232,24 @@ def stage_screenshot_from_artifact(ctx: Any, result: dict[str, Any]) -> bool:except (OSError, ValueError):LOG.info("Copilot screenshot artifact could not be read", path=path)return False- return enqueue_screenshot(ctx, base64.b64encode(raw).decode("ascii"))+ return enqueue_screenshot(+ ctx,+ base64.b64encode(raw).decode("ascii"),+ provenance=provenance,+ captured_at=captured_at if captured_at is not None else time.monotonic(),+ )-def enqueue_screenshot_from_result(ctx: Any, result: dict[str, Any]) -> None:+def enqueue_screenshot_from_result(+ ctx: Any,+ result: dict[str, Any],+ *,+ provenance: ScreenshotProvenance,+) -> None:"""Extract, validate, and enqueue a screenshot from a copilot result dict."""from skyvern.forge.sdk.copilot.output_utils import extract_screenshot_b64, is_valid_image_base64+ captured_at = time.monotonic()screenshot_b64 = extract_screenshot_b64(result)if screenshot_b64 and is_valid_image_base64(screenshot_b64):- enqueue_screenshot(ctx, screenshot_b64)+ enqueue_screenshot(ctx, screenshot_b64, provenance=provenance, captured_at=captured_at)diff --git a/skyvern/forge/sdk/copilot/session_factory.py b/skyvern/forge/sdk/copilot/session_factory.pyindex 8d3e09b38..076069239 100644--- a/skyvern/forge/sdk/copilot/session_factory.py+++ b/skyvern/forge/sdk/copilot/session_factory.py@@ -33,11 +33,17 @@ from skyvern.forge.sdk.copilot.enforcement import (_summarize_tool_output,aggressive_prune,estimate_tokens,+ is_paired_observation_message,is_screenshot_message,is_synthetic_user_message,log_recent_tool_output_truncation,pending_screenshot_message,)+from skyvern.forge.sdk.copilot.screenshot_utils import (+ PendingFrameLease,+ ScreenshotActionRelation,+ model_input_fingerprint,+)LOG = structlog.get_logger()@@ -123,12 +129,21 @@ def copilot_session_input_callback(# The newest frame must survive the merge: with every injected message classified# synthetic, the whole post-goal history is middle, and blanket-replacing would# blind the agent to the frame it just captured once it rotates into history.- middle_screenshots = [i for i, item in enumerate(pruned_middle) if _screenshot_with_image(item)]+ trailing_items = list(recent) + list(new_items)+ stale_paired_screenshots = {+ i+ for i, item in enumerate(pruned_middle)+ if is_paired_observation_message(item)+ and any(not is_synthetic_user_message(following) for following in list(pruned_middle[i + 1 :]) + trailing_items)+ }+ middle_screenshots = [+ i for i, item in enumerate(pruned_middle) if _screenshot_with_image(item) and i not in stale_paired_screenshots+ ]has_newer_screenshot = any(_screenshot_with_image(item) for item in list(recent) + list(new_items))keep_screenshot = middle_screenshots[-1] if middle_screenshots and not has_newer_screenshot else Nonepruned_middle = [{"role": "user", "content": SCREENSHOT_PLACEHOLDER}- if is_screenshot_message(item) and i != keep_screenshot+ if is_screenshot_message(item) and (i != keep_screenshot or i in stale_paired_screenshots)else itemfor i, item in enumerate(pruned_middle)]@@ -158,9 +173,36 @@ def _run_context(data: CallModelData[Any]) -> Any:def _copilot_call_model_input_filter(data: CallModelData[Any], *, token_budget: int) -> ModelInputData:items = list(data.model_data.input)+ ctx = _run_context(data)# Read-only peek: the end-of-turn drain in enforcement stays the only clear, so a provider# retry or model fallback re-running this filter still carries the frame.- screenshot_msg = pending_screenshot_message(_run_context(data))+ screenshot_msg = pending_screenshot_message(ctx)+ pending = getattr(ctx, "pending_screenshots", None)+ entry = pending[0] if isinstance(pending, list) and pending else None+ if (+ screenshot_msg is not None+ and entry is not None+ and entry.provenance.action_relation is ScreenshotActionRelation.SAME_PAGE_OBSERVATION+ ):+ fingerprint = model_input_fingerprint(items)+ lease = getattr(ctx, "pending_frame_lease", None)+ if isinstance(lease, PendingFrameLease) and lease.capture_event_id == entry.capture_event_id:+ if lease.input_fingerprint != fingerprint:+ if isinstance(pending, list):+ pending.clear()+ ctx.pending_frame_lease = None+ screenshot_msg = None+ LOG.info(+ "copilot_frame_lease_invalidated",+ capture_id=entry.capture_id,+ input_fingerprint=fingerprint[:12],+ )+ else:+ ctx.pending_frame_lease = PendingFrameLease(+ capture_event_id=entry.capture_event_id,+ capture_id=entry.capture_id,+ input_fingerprint=fingerprint,+ )if screenshot_msg is not None:LOG.info("Injecting screenshot user message",diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex a69fa4c61..3a69761cf 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -31,7 +31,11 @@ from skyvern.forge.sdk.copilot.output_utils import (sanitize_tool_result_for_llm,)from skyvern.forge.sdk.copilot.pending_operation import pending_operation-from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result+from skyvern.forge.sdk.copilot.screenshot_utils import (+ ScreenshotActionRelation,+ ScreenshotProvenance,+ enqueue_screenshot_from_result,+)from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structurefrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.workflow_yaml import (@@ -807,7 +811,11 @@ async def run_blocks_tool(source_tool="run_blocks_and_collect_debug",result=result,)- enqueue_screenshot_from_result(copilot_ctx, result)+ enqueue_screenshot_from_result(+ copilot_ctx,+ result,+ provenance=_run_result_screenshot_provenance(result, source_tool="run_blocks_and_collect_debug"),+ )sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)return json.dumps(sanitized)@@ -1067,11 +1075,41 @@ async def _run_updated_workflow_blocks(result=run_result,workflow_updated=True,)- enqueue_screenshot_from_result(copilot_ctx, run_result)+ enqueue_screenshot_from_result(+ copilot_ctx,+ run_result,+ provenance=_run_result_screenshot_provenance(run_result, source_tool=tool_name),+ )sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", run_result)return json.dumps(sanitized)+def _run_result_screenshot_provenance(result: dict[str, Any], *, source_tool: str) -> ScreenshotProvenance:+ """Bind a run-result frame to the facts carried by the run payload.++ Run execution returns its identity fields under ``data``. Some older call+ sites and error paths still expose them at the top level, so retain that as+ a factual fallback rather than stamping the frame as unidentified.+ """+ data = result.get("data")+ facts = data if isinstance(data, dict) else {}++ def _string_fact(key: str) -> str | None:+ value = facts.get(key)+ if not isinstance(value, str) or not value:+ value = result.get(key)+ return value if isinstance(value, str) and value else None++ return ScreenshotProvenance(+ source_tool=source_tool,+ captured_url=_string_fact("current_url"),+ observation_step=None,+ browser_session_id=_string_fact("browser_session_id"),+ workflow_run_id=_string_fact("workflow_run_id"),+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,+ )++@function_tool(name_override="discover_workflow_entrypoint", strict_mode=False)async def discover_workflow_entrypoint_tool(ctx: RunContextWrapper,diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.pyindex 2d75d5082..bb0c893b3 100644--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py@@ -4,6 +4,9 @@ import asyncioimport base64import copyimport json+import time+from collections.abc import Iterator+from dataclasses import dataclassfrom typing import Anyfrom urllib.parse import urlparse@@ -31,6 +34,13 @@ from skyvern.forge.sdk.copilot.runtime_authoring_repair import (post_run_inspection_cleanly_matches,repair_page_evidence_is_admissible,)+from skyvern.forge.sdk.copilot.screenshot_utils import (+ CapturedFrame,+ ScreenshotActionRelation,+ ScreenshotProvenance,+ enqueue_screenshot,+ screenshot_result_facts,+)from skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom ._shared import (@@ -60,6 +70,27 @@ from .scouting import (LOG = structlog.get_logger()+@dataclass(frozen=True)+class CompositionEvidenceCapture:+ evidence: dict[str, Any] | None+ error: str | None+ frame: CapturedFrame | None = None++ def __iter__(self) -> Iterator[dict[str, Any] | str | None]:+ """Keep legacy two-value test/caller unpacking while frame-aware funnels migrate explicitly."""+ yield self.evidence+ yield self.error+++def _capture_result_parts(+ capture: CompositionEvidenceCapture | tuple[dict[str, Any] | None, str | None],+) -> tuple[dict[str, Any] | None, str | None, CapturedFrame | None]:+ if isinstance(capture, CompositionEvidenceCapture):+ return capture.evidence, capture.error, capture.frame+ evidence, error = capture+ return evidence, error, None++_POST_RUN_REPAIR_CAPTURE_TIMEOUT_SECONDS = 30.0_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS = 10.0_COMPOSITION_VISUAL_SUMMARY_PROMPT_NAME = "workflow-copilot-page-evidence-vision"@@ -96,13 +127,16 @@ def _model_facing_inspect_result(result: dict[str, Any]) -> dict[str, Any]:return shaped-async def _composition_get_screenshot(ctx: CopilotContext) -> dict[str, Any]:+async def _composition_get_screenshot(ctx: CopilotContext, *, dispatch_session_id: str | None) -> dict[str, Any]:server = getattr(ctx, "discovery_mcp_server", None)if server is None:return {"ok": False, "error": "discovery MCP server not attached to context"}try:return await asyncio.wait_for(- server.call_internal_tool("skyvern_screenshot", {"inline": True}),+ server.call_internal_tool(+ "skyvern_screenshot",+ {"inline": True, **({"session_id": dispatch_session_id} if dispatch_session_id else {})},+ ),timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,)except asyncio.TimeoutError:@@ -253,20 +287,46 @@ async def _composition_summarize_screenshot(async def _augment_composition_evidence_with_visual_fallback(ctx: CopilotContext,evidence: dict[str, Any],-) -> dict[str, Any]:- screenshot_result = await _composition_get_screenshot(ctx)+) -> tuple[dict[str, Any], CapturedFrame | None]:+ capture_started_at = time.monotonic()+ capture_url = evidence.get("current_url") or evidence.get("inspected_url")+ capture_session_id = getattr(ctx, "browser_session_id", None)+ screenshot_result = await _composition_get_screenshot(ctx, dispatch_session_id=capture_session_id)if not screenshot_result.get("ok"):- return _composition_add_evidence_omission(- evidence,- f"screenshot_capture_failed: {screenshot_result.get('error', 'unknown')}",+ return (+ _composition_add_evidence_omission(+ evidence,+ f"screenshot_capture_failed: {screenshot_result.get('error', 'unknown')}",+ ),+ None,)screenshot_b64 = _composition_extract_screenshot_b64(screenshot_result)+ producer_url, producer_session_id, session_binding = screenshot_result_facts(+ screenshot_result,+ dispatch_url=str(capture_url) if capture_url else None,+ dispatch_browser_session_id=capture_session_id,+ )+ captured_frame = (+ CapturedFrame(+ b64=screenshot_b64,+ captured_at=capture_started_at,+ captured_url=producer_url,+ browser_session_id=producer_session_id,+ dispatch_url=str(capture_url) if capture_url else None,+ dispatch_browser_session_id=capture_session_id,+ producer_browser_session_id=producer_session_id,+ session_binding=session_binding,+ )+ if screenshot_b64+ else None+ )visual_summary, visual_error = await _composition_summarize_screenshot(ctx,evidence=evidence,screenshot_b64=screenshot_b64,)- return merge_visual_composition_evidence(evidence, visual_summary=visual_summary, visual_error=visual_error)+ merged = merge_visual_composition_evidence(evidence, visual_summary=visual_summary, visual_error=visual_error)+ return merged, captured_framedef _composition_add_evidence_omission(evidence: dict[str, Any], message: str) -> dict[str, Any]:@@ -292,7 +352,7 @@ async def _composition_evidence_after_navigation_failure(*,inspected_url: str,navigation_error: str,-) -> dict[str, Any] | None:+) -> tuple[dict[str, Any], CapturedFrame | None] | None:current_url, _ = await _fallback_page_info(ctx)current_url = current_url or inspected_urlstructured, structured_error = await _composition_get_structured_evidence_result(@@ -303,9 +363,10 @@ async def _composition_evidence_after_navigation_failure(structured,"navigation_error_before_html_capture",)+ frame = Noneif page_evidence_needs_visual_fallback(evidence):- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)- return evidence+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)+ return evidence, frameif structured_error is not None:# A navigation failure is the exceptional path where a screenshot can still# observe the page even though the bounded DOM execution context was lost.@@ -319,8 +380,8 @@ async def _composition_evidence_after_navigation_failure(evidence,"structured_evidence_unavailable_after_navigation_error",)- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)- return evidence if evidence.get("screenshot_used") else None+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)+ return (evidence, frame) if evidence.get("screenshot_used") else None# Same size-cap survival as the success path: a heavy page that rendered before the nav# error still parses via the stripped-body evaluate instead of yielding hollow evidence.html, html_error, html_truncated, _ = await _composition_get_html(ctx)@@ -338,9 +399,10 @@ async def _composition_evidence_after_navigation_failure(if html_truncated:evidence = _composition_add_inspection_warning(evidence, "html_sliced_at_cap")evidence = await _augment_composition_evidence_with_computed_obstruction_candidates(ctx, evidence)+ frame = Noneif page_evidence_needs_visual_fallback(evidence):- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)- return evidence+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)+ return evidence, frameevidence = parse_composition_html("", inspected_url=inspected_url, current_url=current_url)evidence = _composition_add_inspection_warning(@@ -351,8 +413,8 @@ async def _composition_evidence_after_navigation_failure(evidence,"html_capture_failed_after_navigation_error",)- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)- return evidence if evidence.get("screenshot_used") else None+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)+ return (evidence, frame) if evidence.get("screenshot_used") else Nonedef _inspection_reached_via(*, use_current_page: bool, post_run: bool, earned_interaction: bool) -> str:@@ -504,7 +566,7 @@ async def _capture_composition_evidence(*,inspected_url: str,current_url: str,-) -> tuple[dict[str, Any] | None, str | None]:+) -> CompositionEvidenceCapture:"""Capture page evidence, using HTML only to enrich a valid but unsettled structured packet."""evidence: dict[str, Any] | None = Nonehtml_truncated = False@@ -517,7 +579,7 @@ async def _capture_composition_evidence(if structured_error is not None:if evidence is not None:break- return None, structured_error+ return CompositionEvidenceCapture(None, structured_error)if structured is not None:evidence = structuredused_structured = True@@ -535,7 +597,7 @@ async def _capture_composition_evidence(if html_error is not None:if evidence is not None:break- return None, html_error+ return CompositionEvidenceCapture(None, html_error)# On a heavy page the raw get_html serialization is dropped over the MCP size cap and# falls back to the stripped read; once that happens, settle-and-recapture via the# stripped path only so a slow page is still retried without re-serializing the full DOM.@@ -561,8 +623,9 @@ async def _capture_composition_evidence(page_evidence_needs_visual_fallback(evidence)or (evidence.get("schema_empty_page") is True and not has_bounded_page_schema(evidence))):- evidence = await _augment_composition_evidence_with_visual_fallback(copilot_ctx, evidence)- return evidence, None+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(copilot_ctx, evidence)+ return CompositionEvidenceCapture(evidence, None, frame)+ return CompositionEvidenceCapture(evidence, None)async def _read_run_session_page_evidence(@@ -570,7 +633,7 @@ async def _read_run_session_page_evidence(*,run_session_id: str,current_url: str,-) -> tuple[dict[str, Any] | None, str | None, str | None]:+) -> tuple[dict[str, Any] | None, str | None, str | None, CapturedFrame | None]:"""The discovery extractor reads ctx.browser_session_id per call, so the rebind targets the runsession and is restored in a finally. The browser layer can substitute a replacement session for aclosed one, so the id the capture actually observed is read back before the restore."""@@ -579,11 +642,13 @@ async def _read_run_session_page_evidence(observed_session_id: str | None = run_session_idobservation_error: str | None = Noneevidence: dict[str, Any] | None = None+ frame: CapturedFrame | None = Nonetry:- evidence, observation_error = await asyncio.wait_for(+ capture = await asyncio.wait_for(_capture_composition_evidence(ctx, inspected_url=current_url, current_url=current_url),timeout=_POST_RUN_REPAIR_CAPTURE_TIMEOUT_SECONDS,)+ evidence, observation_error, frame = _capture_result_parts(capture)except Exception:LOG.debug("Post-run run-session page capture failed", exc_info=True)observation_error = observation_error or "Post-run page capture against the run session failed."@@ -596,8 +661,8 @@ async def _read_run_session_page_evidence(if not observed_session_id:# A mid-capture session create that fails clears the id, and an unknown source id grants# post-run identity, so an unprovable source has to drop the packet rather than launder it.- return None, None, observation_error or "Post-run page capture lost its browser session."- return (evidence if isinstance(evidence, dict) else None), observed_session_id, observation_error+ return None, None, observation_error or "Post-run page capture lost its browser session.", None+ return (evidence if isinstance(evidence, dict) else None), observed_session_id, observation_error, framedef _post_run_page_source_session_id(ctx: CopilotContext, run_id: str | None) -> str | None:@@ -758,6 +823,7 @@ async def _inspect_page_for_composition_impl(# handful of structured looks, and the turn's wall clock spent without understanding the page.# The calls stay counted for telemetry.evidence = None+ visual_fallback_frame: CapturedFrame | None = Noneobservation_error: str | None = Nonewith copilot_span("inspect_page_for_composition",@@ -767,13 +833,19 @@ async def _inspect_page_for_composition_impl(# current_page, or a URL target the agent is already on — capture without navigating.current_url = inspect_target_url or entry_urlif run_page_source_session_id:- evidence, observed_run_session_id, observation_error = await _read_run_session_page_evidence(+ (+ evidence,+ observed_run_session_id,+ observation_error,+ visual_fallback_frame,+ ) = await _read_run_session_page_evidence(copilot_ctx, run_session_id=run_page_source_session_id, current_url=current_url)else:- evidence, observation_error = await _capture_composition_evidence(+ capture = await _capture_composition_evidence(copilot_ctx, inspected_url=entry_url, current_url=current_url)+ evidence, observation_error, visual_fallback_frame = _capture_result_parts(capture)else:nav_result = await _discovery_navigate(copilot_ctx,@@ -783,12 +855,12 @@ async def _inspect_page_for_composition_impl()if not nav_result.get("ok"):nav_error = str(nav_result.get("error") or "unknown")- failure_evidence = await _composition_evidence_after_navigation_failure(+ failure_capture = await _composition_evidence_after_navigation_failure(copilot_ctx,inspected_url=entry_url,navigation_error=nav_error,)- if failure_evidence is None:+ if failure_capture is None:result = {"ok": False,"data": None,@@ -796,13 +868,14 @@ async def _inspect_page_for_composition_impl(}record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)return result- evidence = failure_evidence+ evidence, visual_fallback_frame = failure_capturecurrent_url = str(evidence.get("current_url") or entry_url)else:current_url = _discovery_extract_current_url(nav_result, entry_url)- evidence, observation_error = await _capture_composition_evidence(+ capture = await _capture_composition_evidence(copilot_ctx, inspected_url=entry_url, current_url=current_url)+ evidence, observation_error, visual_fallback_frame = _capture_result_parts(capture)if observation_error is not None:result = {@@ -834,6 +907,8 @@ async def _inspect_page_for_composition_impl(source_browser_session_id=source_browser_session_id,run_browser_session_id=copilot_ctx.last_run_blocks_browser_session_id,)+ if preserved_stored_evidence:+ visual_fallback_frame = None# A capture too hollow or too foreign to store is also too weak to move the post-run# observation marker; leaving the marker put keeps it describing the packet that is stored.if not preserved_stored_evidence:@@ -881,4 +956,23 @@ async def _inspect_page_for_composition_impl(result["observation_step"] = observation_stepresult = _model_facing_inspect_result(result)record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)+ if visual_fallback_frame is not None:+ workflow_run_id = evidence.get("workflow_run_id")+ enqueue_screenshot(+ copilot_ctx,+ visual_fallback_frame.b64,+ provenance=ScreenshotProvenance(+ source_tool="inspect_page_for_composition",+ captured_url=visual_fallback_frame.captured_url,+ observation_step=observation_step,+ browser_session_id=visual_fallback_frame.browser_session_id,+ workflow_run_id=workflow_run_id if isinstance(workflow_run_id, str) else None,+ action_relation=ScreenshotActionRelation.SAME_PAGE_OBSERVATION,+ dispatch_url=visual_fallback_frame.dispatch_url,+ dispatch_browser_session_id=visual_fallback_frame.dispatch_browser_session_id,+ producer_browser_session_id=visual_fallback_frame.producer_browser_session_id,+ session_binding=visual_fallback_frame.session_binding,+ ),+ captured_at=visual_fallback_frame.captured_at,+ )return resultdiff --git a/skyvern/forge/sdk/copilot/tools/credential_fill.py b/skyvern/forge/sdk/copilot/tools/credential_fill.pyindex ccc56cbbf..a8d620ac1 100644--- a/skyvern/forge/sdk/copilot/tools/credential_fill.py+++ b/skyvern/forge/sdk/copilot/tools/credential_fill.py@@ -976,7 +976,12 @@ async def _fill_credential_field_impl_serial(form_submits = await _capture_enclosing_form_submits(copilot_ctx, selector)if form_submits:data["form_submit_controls"] = form_submits- await _capture_post_interaction_screenshot(copilot_ctx)+ await _capture_post_interaction_screenshot(+ copilot_ctx,+ source_tool=observed_tool,+ captured_url=observed_url,+ observation_step=observation_step,+ )LOG.info("copilot fill_credential_field filled a saved credential field",selector=selector,diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex 67d96477b..3a11c6c46 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -562,14 +562,22 @@ async def _navigate_post_hook(result_url=result["url"],)await _bind_login_credential_for_observed_url(ctx, result["url"], result)- staged = await _capture_post_interaction_screenshot(ctx)+ staged = await _capture_post_interaction_screenshot(+ ctx,+ source_tool="navigate_browser",+ captured_url=result["url"],+ )attached = " A screenshot is attached." if staged else ""result["next_step"] = (f"Page loaded.{attached} Use evaluate or inspect_page_for_composition when you need the ""page's structure or selectors before responding.")else:- await _capture_post_interaction_screenshot(ctx)+ await _capture_post_interaction_screenshot(+ ctx,+ source_tool="navigate_browser",+ captured_url=source_url,+ )return result@@ -633,6 +641,8 @@ async def _click_post_hook(ctx.last_scout_act_observe_outcome = Nonectx.last_scout_act_observe_packet = Nonepage_evidence: dict[str, Any] | None = None+ captured_url: str | None = None+ observation_step: int | None = None_clear_pending_browser_interaction_observation(ctx)source_url = _consume_scout_source_url(ctx)pending_role_name = ctx.pending_scout_role_name@@ -706,6 +716,7 @@ async def _click_post_hook(if observation_step is not None:result["observation_step"] = observation_stepresult["data"]["observation_step"] = observation_step+ captured_url = url or Noneif _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:# A download this click produced is proof the affordance works, so it outranks the# href-shape prediction — and is the only source that sees a command-URL download.@@ -723,7 +734,12 @@ async def _click_post_hook(# The round-trip is skipped only when the evidence positively names the obstruction a frame# would have shown; evidence that merely parsed is not a substitute for looking at the page.if ctx.last_scout_act_observe_outcome != "attached" or not _page_evidence_names_obstruction(page_evidence):- await _capture_post_interaction_screenshot(ctx)+ await _capture_post_interaction_screenshot(+ ctx,+ source_tool="click",+ captured_url=captured_url or source_url,+ observation_step=observation_step,+ )return resultdiff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 4e94d8a06..4a877ff35 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -102,6 +102,11 @@ from skyvern.forge.sdk.copilot.runtime_authoring_repair import (repair_page_evidence_is_admissible,same_run_typed_challenge_kind,)+from skyvern.forge.sdk.copilot.screenshot_utils import (+ ScreenshotActionRelation,+ ScreenshotProvenance,+ enqueue_screenshot,+)from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal@@ -1500,11 +1505,11 @@ async def _capture_and_store_post_run_page() -> None:"""A failed or hollow capture neutralizes stale evidence to None only when it would not cleanly matchthis run_id, so the matcher's destructive clear cannot fire on the pending failure-string context."""- evidence, observed_session_id, _ = await _read_run_session_page_evidence(+ evidence, observed_session_id, _, captured_frame = await _read_run_session_page_evidence(ctx, run_session_id=run_session_id, current_url=current_url)if evidence is not None and repair_page_evidence_is_admissible(evidence):- store_post_run_page_evidence(+ _, preserved_stored_evidence = store_post_run_page_evidence(ctx,evidence,run_id=run_id,@@ -1512,6 +1517,24 @@ async def _capture_and_store_post_run_page(source_browser_session_id=observed_session_id,run_browser_session_id=run_session_id,)+ if captured_frame is not None and not preserved_stored_evidence:+ enqueue_screenshot(+ ctx,+ captured_frame.b64,+ provenance=ScreenshotProvenance(+ source_tool="post_run_page_capture",+ captured_url=captured_frame.captured_url,+ observation_step=None,+ browser_session_id=captured_frame.browser_session_id,+ workflow_run_id=run_id,+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,+ dispatch_url=captured_frame.dispatch_url,+ dispatch_browser_session_id=captured_frame.dispatch_browser_session_id,+ producer_browser_session_id=captured_frame.producer_browser_session_id,+ session_binding=captured_frame.session_binding,+ ),+ captured_at=captured_frame.captured_at,+ )returnif not post_run_inspection_cleanly_matches(ctx.composition_page_evidence, run_id):ctx.composition_page_evidence = None@@ -1590,7 +1613,7 @@ async def _capture_dispatched_terminal_page_evidence(if _pre_run_baseline_is_provenance_valid(ctx.composition_page_evidence):_pin_pre_run_page_reference(ctx, run_id)source = "cdp_run_session"- evidence, source_session_id, _ = await _read_run_session_page_evidence(+ evidence, source_session_id, _, captured_frame = await _read_run_session_page_evidence(ctx, run_session_id=run_session_id, current_url=current_url)# A capture that landed on a substituted session can still look usable (a blank replacement page@@ -1602,6 +1625,7 @@ async def _capture_dispatched_terminal_page_evidence(or not _dispatched_terminal_page_evidence_is_usable(evidence)):source = "worker_artifact"+ captured_frame = Nonesource_session_id = run_session_idevidence = await _fetch_dispatched_terminal_page_evidence(run_id=run_id, organization_id=organization_id, current_url=current_url@@ -1616,6 +1640,24 @@ async def _capture_dispatched_terminal_page_evidence(source_browser_session_id=source_session_id,run_browser_session_id=run_session_id,)+ if captured_frame is not None and not preserved_stored_evidence:+ enqueue_screenshot(+ ctx,+ captured_frame.b64,+ provenance=ScreenshotProvenance(+ source_tool="dispatched_terminal_page_capture",+ captured_url=captured_frame.captured_url,+ observation_step=None,+ browser_session_id=captured_frame.browser_session_id,+ workflow_run_id=run_id,+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,+ dispatch_url=captured_frame.dispatch_url,+ dispatch_browser_session_id=captured_frame.dispatch_browser_session_id,+ producer_browser_session_id=captured_frame.producer_browser_session_id,+ session_binding=captured_frame.session_binding,+ ),+ captured_at=captured_frame.captured_at,+ )LOG.info("copilot_dispatched_terminal_page_evidence_captured",workflow_run_id=run_id,diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.pyindex 34cc4a314..2ef754fb0 100644--- a/skyvern/forge/sdk/copilot/tools/scouting.py+++ b/skyvern/forge/sdk/copilot/tools/scouting.py@@ -67,7 +67,12 @@ from skyvern.forge.sdk.copilot.runtime import (ScoutedSelectorCandidate,resolve_browser_state_for_context,)-from skyvern.forge.sdk.copilot.screenshot_utils import stage_screenshot_from_artifact+from skyvern.forge.sdk.copilot.screenshot_utils import (+ ScreenshotActionRelation,+ ScreenshotProvenance,+ screenshot_result_facts,+ stage_screenshot_from_artifact,+)from ._shared import (_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,@@ -519,7 +524,12 @@ async def _capture_element_fingerprint(async def _capture_post_interaction_screenshot(- ctx: AgentContext, *, timeout_seconds: float = _DISCOVERY_PER_CALL_TIMEOUT_SECONDS+ ctx: AgentContext,+ *,+ source_tool: str,+ captured_url: str | None,+ observation_step: int | None = None,+ timeout_seconds: float = _DISCOVERY_PER_CALL_TIMEOUT_SECONDS,) -> bool:"""Attach a look at the page after a state-changing action, reporting whether a frame staged.Reading the DOM answers "what is on the page" but not "did that work" -- a filled password reads@@ -533,16 +543,40 @@ async def _capture_post_interaction_screenshot(server = getattr(ctx, "discovery_mcp_server", None)if server is None:return False+ capture_started_at = time.monotonic()+ capture_session_id = ctx.browser_session_id+ screenshot_arguments = {"session_id": capture_session_id} if capture_session_id else {}try:result = await asyncio.wait_for(- server.call_internal_tool("skyvern_screenshot", {}),+ server.call_internal_tool("skyvern_screenshot", screenshot_arguments),timeout=timeout_seconds,)except Exception:return Falseif not isinstance(result, dict) or not result.get("ok"):return False- return stage_screenshot_from_artifact(ctx, result)+ producer_url, producer_session_id, session_binding = screenshot_result_facts(+ result,+ dispatch_url=captured_url,+ dispatch_browser_session_id=capture_session_id,+ )+ return stage_screenshot_from_artifact(+ ctx,+ result,+ provenance=ScreenshotProvenance(+ source_tool=source_tool,+ captured_url=producer_url,+ observation_step=observation_step,+ browser_session_id=producer_session_id,+ workflow_run_id=None,+ action_relation=ScreenshotActionRelation.AFTER_SOURCE_ACTION,+ dispatch_url=captured_url,+ dispatch_browser_session_id=capture_session_id,+ producer_browser_session_id=producer_session_id,+ session_binding=session_binding,+ ),+ captured_at=capture_started_at,+ )async def _capture_enclosing_form_submits(diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.pyindex 7d0747af6..05b7e43c6 100644--- a/tests/unit/test_copilot_composition_evidence.py+++ b/tests/unit/test_copilot_composition_evidence.py@@ -3955,11 +3955,13 @@ async def test_navigation_failure_uses_structured_evidence_when_bounded() -> Nonserver = _RecordingCompositionServer(structured_json=json.dumps(_structured_form_payload()))ctx = SimpleNamespace(discovery_mcp_server=server, browser_session_id=None, organization_id="o_test")- evidence = await tools_module._composition_evidence_after_navigation_failure(+ capture = await tools_module._composition_evidence_after_navigation_failure(ctx, inspected_url="https://example.com/lookup", navigation_error="boom")- assert evidence is not None+ assert capture is not None+ evidence, frame = capture+ assert frame is Noneassert evidence["forms"]assert server.calls.count("skyvern_get_html") == 0assert any("navigation_error_before_html_capture" in warning for warning in evidence["inspection_warnings"])@@ -3972,8 +3974,8 @@ async def test_navigation_failure_uses_visual_fallback_after_structured_failure_server = _RecordingCompositionServer(structured_json=None, html=_HTML_FORM_PAGE)ctx = SimpleNamespace(discovery_mcp_server=server, browser_session_id=None, organization_id="o_test")- async def attach_visual(_ctx: Any, evidence: dict[str, Any]) -> dict[str, Any]:- return {**evidence, "screenshot_used": True, "visual_evidence_summary": "A login form is visible."}+ async def attach_visual(_ctx: Any, evidence: dict[str, Any]) -> tuple[dict[str, Any], None]:+ return {**evidence, "screenshot_used": True, "visual_evidence_summary": "A login form is visible."}, Nonemonkeypatch.setattr(tools_module.composition_capture,@@ -3981,13 +3983,15 @@ async def test_navigation_failure_uses_visual_fallback_after_structured_failure_attach_visual,)- evidence = await tools_module._composition_evidence_after_navigation_failure(+ capture = await tools_module._composition_evidence_after_navigation_failure(ctx,inspected_url="https://example.com/login",navigation_error="https://example.com/callback?token=raw-secret",)- assert evidence is not None+ assert capture is not None+ evidence, frame = capture+ assert frame is Noneassert evidence["screenshot_used"] is Trueassert evidence["visual_evidence_summary"] == "A login form is visible."assert "raw-secret" not in json.dumps(evidence)diff --git a/tests/unit/test_copilot_composition_scrape.py b/tests/unit/test_copilot_composition_scrape.pyindex 7a6b1f3e9..33652383d 100644--- a/tests/unit/test_copilot_composition_scrape.py+++ b/tests/unit/test_copilot_composition_scrape.py@@ -183,6 +183,9 @@ async def test_unrendered_challenge_keeps_structured_packet_when_relooks_run_outasync def identity(ctx: object, evidence: dict) -> dict:return evidence+ async def visual_identity(ctx: object, evidence: dict) -> tuple[dict, None]:+ return evidence, None+# Body-only read: the anti-bot token lives in <title>, so it is absent here by construction.get_html = AsyncMock(return_value=(_BOUNDED_HTML, None, False, False))monkeypatch.setattr(tools.composition_capture, "_composition_get_structured_evidence_result", fake_structured)@@ -190,7 +193,9 @@ async def test_unrendered_challenge_keeps_structured_packet_when_relooks_run_outmonkeypatch.setattr(tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity)- monkeypatch.setattr(tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", identity)+ monkeypatch.setattr(+ tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", visual_identity+ )settle_sleep = AsyncMock()monkeypatch.setattr(tools.composition_capture, "asyncio", _AsyncioSleepProxy(settle_sleep))@@ -267,13 +272,18 @@ async def test_signalled_packet_survives_extractor_blinking_mid_loop(monkeypatchasync def identity(ctx: object, evidence: dict) -> dict:return evidence+ async def visual_identity(ctx: object, evidence: dict) -> tuple[dict, None]:+ return evidence, None+get_html = AsyncMock(return_value=(_BOUNDED_HTML, None, False, False))monkeypatch.setattr(tools.composition_capture, "_composition_get_structured_evidence_result", fake_structured)monkeypatch.setattr(tools.composition_capture, "_composition_get_html", get_html)monkeypatch.setattr(tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity)- monkeypatch.setattr(tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", identity)+ monkeypatch.setattr(+ tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", visual_identity+ )monkeypatch.setattr(tools.composition_capture, "asyncio", _AsyncioSleepProxy(AsyncMock()))evidence, html_error = await tools._capture_composition_evidence(diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex f30a22063..6bb67ba5a 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -3453,8 +3453,8 @@ async def test_obstruction_only_packet_survives_the_automatic_post_run_capture(async def _read(_ctx_arg: CopilotContext, *, run_session_id: str, current_url: str- ) -> tuple[dict[str, Any], str, None]:- return packet, run_session_id, None+ ) -> tuple[dict[str, Any], str, None, None]:+ return packet, run_session_id, None, Nonemonkeypatch.setattr(run_execution_module, "_read_run_session_page_evidence", _read)diff --git a/tests/unit/test_copilot_discovery.py b/tests/unit/test_copilot_discovery.pyindex edc271a34..b902081ec 100644--- a/tests/unit/test_copilot_discovery.py+++ b/tests/unit/test_copilot_discovery.py@@ -2,6 +2,8 @@from __future__ import annotations+import base64+import hashlibimport jsonfrom types import SimpleNamespacefrom typing import Any@@ -34,6 +36,10 @@ from skyvern.forge.sdk.copilot.tools.discovery import (from skyvern.forge.sdk.copilot.turn_origin import TurnOriginfrom skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence+_VALID_PNG_B64 = (+ "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAAE0lEQVR4nGP8z4APMOGVZRip0gBBLAETee26JgAAAABJRU5ErkJggg=="+)+class _Ctx:def __init__(self, server: object) -> None:@@ -51,6 +57,8 @@ class _Ctx:self.last_run_blocks_browser_session_id = Noneself.request_policy = Noneself.org_credentials_for_turn = None+ self.supports_vision = True+ self.pending_screenshots: list[Any] = []def _dense_structured_page() -> dict[str, Any]:@@ -356,8 +364,13 @@ class _GenericBarrierServer:assert arguments["expression"] == COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSIONreturn {"ok": True, "data": {"result": _structured_search_page(with_obstruction=True)}}if tool_name == "skyvern_screenshot":- assert arguments == {"inline": True}- return {"ok": True, "data": {"screenshot_base64": "aGVsbG8="}}+ expected_session = arguments.get("session_id")+ assert arguments == {"inline": True, **({"session_id": expected_session} if expected_session else {})}+ return {+ "ok": True,+ "browser_context": {"session_id": expected_session},+ "data": {"screenshot_base64": _VALID_PNG_B64},+ }raise AssertionError(f"unexpected tool: {tool_name}")@@ -795,7 +808,7 @@ async def test_target_url_inspection_uses_visual_summary_for_generic_obstructionevidence: dict[str, Any],screenshot_b64: str,) -> tuple[dict[str, Any], None]:- assert screenshot_b64 == "aGVsbG8="+ assert screenshot_b64 == _VALID_PNG_B64assert evidence["visual_obstruction_candidates"][0]["coverage"] == "viewport"return {"summary": "A checkpoint panel blocks the search form.",@@ -822,6 +835,8 @@ async def test_target_url_inspection_uses_visual_summary_for_generic_obstructionassert "skyvern_evaluate" in server.callsassert "skyvern_screenshot" in server.callsassert result["data"]["screenshot_used"] is True+ json.dumps(result["data"])+ assert all("CapturedFrame" not in repr(value) for value in result["data"].values())assert result["data"]["page_obstructions"] == [{"kind": "checkpoint_panel",@@ -831,6 +846,74 @@ async def test_target_url_inspection_uses_visual_summary_for_generic_obstruction"underlying_page_blocked": True,}]+ assert len(ctx.pending_screenshots) == 1+ frame = ctx.pending_screenshots[0]+ assert frame.provenance.source_tool == "inspect_page_for_composition"+ assert frame.provenance.captured_url is None+ assert frame.provenance.dispatch_url == "https://www.example.com/search"+ assert frame.provenance.observation_step == result["observation_step"]+ assert frame.provenance.browser_session_id is None+ assert frame.provenance.session_binding.value == "unavailable"+ assert frame.provenance.workflow_run_id is None+ assert frame.provenance.action_relation.value == "same_page_observation"+ assert frame.capture_id == f"sha256:{hashlib.sha256(base64.b64decode(_VALID_PNG_B64)).hexdigest()}"+++async def test_post_run_visual_fallback_binds_the_observed_run_session(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ server = _GenericBarrierServer()+ ctx = _Ctx(server)+ ctx.browser_session_id = "pbs_scout"+ ctx.last_run_blocks_browser_session_id = "pbs_run"+ ctx.last_run_blocks_workflow_run_id = "wr_123" # type: ignore[attr-defined]++ async def fake_page_info(_ctx: object, session_id: str | None = None) -> tuple[str, str]:+ assert session_id == "pbs_run"+ return "https://www.example.com/search", "Search"++ async def fake_visual_summary(+ _ctx: object,+ *,+ evidence: dict[str, Any],+ screenshot_b64: str,+ ) -> tuple[dict[str, Any], None]:+ assert screenshot_b64 == _VALID_PNG_B64+ return {+ "summary": "A checkpoint panel blocks the search form.",+ "challenge_detected": False,+ "challenge_kind": "",+ "challenge_location": "",+ "submit_blocked": False,+ "blocked_submit_controls": [],+ "empty_page_visible": False,+ "loading_state_visible": False,+ "page_obstruction_detected": True,+ "obstruction_kind": "checkpoint_panel",+ "obstruction_location": "Centered over the form.",+ "underlying_page_blocked": True,+ "visible_dismiss_controls": ["Continue"],+ "omissions": [],+ }, None++ monkeypatch.setattr(tools_module.composition_capture, "_fallback_page_info", fake_page_info)+ monkeypatch.setattr(tools_module.composition_capture, "_composition_summarize_screenshot", fake_visual_summary)++ result = await _inspect_page_for_composition_impl(ctx, "current_page")++ assert result["data"]["source_browser_session_id"] == "pbs_run"+ assert result["data"]["workflow_run_id"] == "wr_123"+ frame = ctx.pending_screenshots[0]+ assert frame.provenance.captured_url is None+ assert frame.provenance.dispatch_url == "https://www.example.com/search"+ assert frame.provenance.observation_step == result["observation_step"]+ assert frame.provenance.browser_session_id == "pbs_run"+ assert frame.provenance.dispatch_browser_session_id == "pbs_run"+ assert frame.provenance.producer_browser_session_id == "pbs_run"+ assert frame.provenance.session_binding.value == "agree"+ assert frame.provenance.workflow_run_id == "wr_123"+ assert ctx.browser_session_id == "pbs_scout"@pytest.mark.asynciodiff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex a5d475fce..286275201 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -405,7 +405,7 @@ class TestMCPFailedStepLoopDetection:monkeypatch.setattr(mcp_adapter,"enqueue_screenshot_from_result",- lambda _ctx, result: screenshots.append(dict(result)),+ lambda _ctx, result, **_kwargs: screenshots.append(dict(result)),)initial_scouted_interactions = [{"tool_name": "click", "selector": "#existing"}]diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.pyindex 70ec4b06b..35eca04b6 100644--- a/tests/unit/test_copilot_mcp_adapter.py+++ b/tests/unit/test_copilot_mcp_adapter.py@@ -197,6 +197,50 @@ def _server_whose_call_takes_time()+async def test_internal_call_preserves_explicit_session_across_session_prepare(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = make_copilot_ctx(browser_session_id="pbs_snapshot")+ server = _make_server(+ ctx,+ {"ok": True},+ SchemaOverlay(requires_browser=True),+ alias_map=get_skyvern_mcp_alias_map(),+ )+ dispatched: list[dict[str, Any]] = []++ class _CapturingClient:+ async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> Any:+ dispatched.append(dict(args))+ return SimpleNamespace(+ structured_content={"ok": True},+ is_error=False,+ content=[],+ )++ async def _replace_ambient_session(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None]:+ _ctx.browser_session_id = "pbs_replacement"+ return None, None++ @asynccontextmanager+ async def _scope(_ctx: AgentContext) -> AsyncIterator[None]:+ yield++ server._client = _CapturingClient()+ monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _replace_ambient_session)+ monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)++ result = await server.call_internal_tool(+ "skyvern_evaluate",+ {"expression": "scan()", "session_id": "pbs_snapshot"},+ )++ assert result["ok"] is True+ assert dispatched == [{"expression": "scan()", "session_id": "pbs_snapshot"}]+ assert ctx.browser_session_id == "pbs_replacement"++@pytest.mark.usefixtures("_stub_browser_session")class TestMCPToolTiming:@pytest.mark.asynciodiff --git a/tests/unit/test_copilot_positive_evidence_carriers.py b/tests/unit/test_copilot_positive_evidence_carriers.pyindex 4214d7297..235c3c01f 100644--- a/tests/unit/test_copilot_positive_evidence_carriers.py+++ b/tests/unit/test_copilot_positive_evidence_carriers.py@@ -632,8 +632,8 @@ async def test_dispatched_producer_prefers_worker_artifact_over_a_substituted_seasync def fake_read(inner_ctx: object, *, run_session_id: str, current_url: str- ) -> tuple[dict[str, object], str, None]:- return {"observed_empty_page": True, "current_url": current_url}, "pbs_replacement", None+ ) -> tuple[dict[str, object], str, None, None]:+ return {"observed_empty_page": True, "current_url": current_url}, "pbs_replacement", None, Nonemonkeypatch.setattr(run_execution_module, "_read_run_session_page_evidence", fake_read)diff --git a/tests/unit/test_copilot_screenshot_handling.py b/tests/unit/test_copilot_screenshot_handling.pyindex 2b003e4d5..2cdb6495d 100644--- a/tests/unit/test_copilot_screenshot_handling.py+++ b/tests/unit/test_copilot_screenshot_handling.py@@ -16,7 +16,10 @@ from skyvern.forge.sdk.copilot.enforcement import _consume_pending_screenshotsfrom skyvern.forge.sdk.copilot.screenshot_utils import (COPILOT_SCREENSHOT_MAX_HEIGHT,COPILOT_SCREENSHOT_MAX_WIDTH,+ ScreenshotActionRelation,ScreenshotEntry,+ ScreenshotProvenance,+ enqueue_screenshot,stage_screenshot_from_artifact,)from skyvern.forge.sdk.copilot.session_factory import copilot_call_model_input_filter@@ -50,6 +53,15 @@ def _install_mock_database(monkeypatch: pytest.MonkeyPatch, mock_db: Any) -> Nonmonkeypatch.setattr(run_execution_module, "app", _AppStub())+def _screenshot_entry(b64: str) -> ScreenshotEntry:+ return ScreenshotEntry(+ b64=b64,+ mime="image/jpeg",+ capture_id="sha256:test-frame",+ provenance=ScreenshotProvenance.unknown(source_tool="test_capture"),+ )++class TestIsValidPngBase64:VALID_PNG_B64 = ("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAAElFTkSuQmCC"@@ -93,11 +105,35 @@ class TestEnqueueScreenshot:ctx = MagicMock()ctx.supports_vision = Truectx.pending_screenshots = []- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}})+ provenance = ScreenshotProvenance(+ source_tool="get_browser_screenshot",+ captured_url="https://example.com/current",+ observation_step=3,+ browser_session_id="pbs_123",+ workflow_run_id=None,+ action_relation=ScreenshotActionRelation.TOOL_RESULT,+ )+ enqueue_screenshot_from_result(+ ctx,+ {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}},+ provenance=provenance,+ )assert len(ctx.pending_screenshots) == 1entry = ctx.pending_screenshots[0]assert isinstance(entry, ScreenshotEntry)assert entry.mime == "image/jpeg"+ assert entry.capture_id.startswith("sha256:")+ assert entry.provenance == provenance++ def test_older_capture_cannot_replace_newer_pending_frame(self) -> None:+ ctx = SimpleNamespace(supports_vision=True, pending_screenshots=[])+ provenance = ScreenshotProvenance.unknown(source_tool="inspect_page_for_composition")++ assert enqueue_screenshot(ctx, self.VALID_PNG_B64, provenance=provenance, captured_at=20.0) is True+ newest = ctx.pending_screenshots[0]+ assert enqueue_screenshot(ctx, self.VALID_PNG_B64, provenance=provenance, captured_at=10.0) is False++ assert ctx.pending_screenshots == [newest]def test_skips_when_no_vision(self) -> None:from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result@@ -105,7 +141,11 @@ class TestEnqueueScreenshot:ctx = MagicMock()ctx.supports_vision = Falsectx.pending_screenshots = []- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}})+ enqueue_screenshot_from_result(+ ctx,+ {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}},+ provenance=ScreenshotProvenance.unknown(source_tool="get_browser_screenshot"),+ )assert len(ctx.pending_screenshots) == 0def test_skips_invalid_image(self) -> None:@@ -114,7 +154,11 @@ class TestEnqueueScreenshot:ctx = MagicMock()ctx.supports_vision = Truectx.pending_screenshots = []- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": "not-valid"}})+ enqueue_screenshot_from_result(+ ctx,+ {"ok": True, "data": {"screenshot_base64": "not-valid"}},+ provenance=ScreenshotProvenance.unknown(source_tool="get_browser_screenshot"),+ )assert len(ctx.pending_screenshots) == 0def test_skips_corrupt_header_valid_image(self) -> None:@@ -126,7 +170,11 @@ class TestEnqueueScreenshot:ctx.supports_vision = Truectx.pending_screenshots = []truncated_png = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"broken-image-data").decode()- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": truncated_png + "A" * 100}})+ enqueue_screenshot_from_result(+ ctx,+ {"ok": True, "data": {"screenshot_base64": truncated_png + "A" * 100}},+ provenance=ScreenshotProvenance.unknown(source_tool="get_browser_screenshot"),+ )assert len(ctx.pending_screenshots) == 0def test_second_enqueue_replaces_first_pending_entry(self) -> None:@@ -137,15 +185,49 @@ class TestEnqueueScreenshot:ctx.supports_vision = Truectx.pending_screenshots = []- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}})+ provenance = ScreenshotProvenance.unknown(source_tool="get_browser_screenshot")+ enqueue_screenshot_from_result(+ ctx,+ {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}},+ provenance=provenance,+ )first_entry = ctx.pending_screenshots[0]- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}})+ enqueue_screenshot_from_result(+ ctx,+ {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}},+ provenance=provenance,+ )assert len(ctx.pending_screenshots) == 1assert ctx.pending_screenshots[0] is not first_entry+def test_run_result_screenshot_provenance_reads_nested_run_facts() -> None:+ from skyvern.forge.sdk.copilot.tools import _run_result_screenshot_provenance++ provenance = _run_result_screenshot_provenance(+ {+ "ok": False,+ "data": {+ "current_url": "https://example.com/after-run",+ "browser_session_id": "pbs_nested",+ "workflow_run_id": "wr_123",+ },+ },+ source_tool="update_and_run_blocks",+ )++ assert provenance == ScreenshotProvenance(+ source_tool="update_and_run_blocks",+ captured_url="https://example.com/after-run",+ observation_step=None,+ browser_session_id="pbs_nested",+ workflow_run_id="wr_123",+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,+ )++class TestStageScreenshotFromArtifact:"""The non-inline screenshot tool returns a local artifact path, not image bytes."""@@ -158,11 +240,15 @@ class TestStageScreenshotFromArtifact:def _ctx() -> SimpleNamespace:return SimpleNamespace(supports_vision=True, pending_screenshots=[])+ @staticmethod+ def _provenance() -> ScreenshotProvenance:+ return ScreenshotProvenance.unknown(source_tool="click")+def test_stages_the_artifact_the_path_names(self, tmp_path: Path) -> None:ctx = self._ctx()result = {"ok": True, "data": {"path": self._png(tmp_path / "frame.png", (400, 300))}}- assert stage_screenshot_from_artifact(ctx, result) is True+ assert stage_screenshot_from_artifact(ctx, result, provenance=self._provenance()) is Trueassert len(ctx.pending_screenshots) == 1assert ctx.pending_screenshots[0].mime == "image/jpeg"@@ -170,7 +256,7 @@ class TestStageScreenshotFromArtifact:ctx = self._ctx()result = {"ok": True, "data": {"path": self._png(tmp_path / "big.png", (2400, 1800))}}- assert stage_screenshot_from_artifact(ctx, result) is True+ assert stage_screenshot_from_artifact(ctx, result, provenance=self._provenance()) is Truewidth, height = Image.open(io.BytesIO(base64.b64decode(ctx.pending_screenshots[0].b64))).sizeassert width <= COPILOT_SCREENSHOT_MAX_WIDTHassert height <= COPILOT_SCREENSHOT_MAX_HEIGHT@@ -186,17 +272,31 @@ class TestStageScreenshotFromArtifact:def test_unresolvable_artifact_is_no_frame_not_an_error(self, data: dict[str, Any]) -> None:ctx = self._ctx()- assert stage_screenshot_from_artifact(ctx, {"ok": True, "data": data}) is False+ assert (+ stage_screenshot_from_artifact(+ ctx,+ {"ok": True, "data": data},+ provenance=self._provenance(),+ )+ is False+ )assert ctx.pending_screenshots == []def test_failed_capture_over_a_stale_entry_does_not_report_staged(self, tmp_path: Path) -> None:"""A queue-length check would call this staged: the earlier frame is still pending."""- stale = ScreenshotEntry(b64="stale", mime="image/jpeg")+ stale = _screenshot_entry("stale")ctx = SimpleNamespace(supports_vision=True, pending_screenshots=[stale])corrupt = tmp_path / "corrupt.png"corrupt.write_bytes(b"\x89PNG\r\n\x1a\n" + b"not-an-image" * 40)- assert stage_screenshot_from_artifact(ctx, {"ok": True, "data": {"path": str(corrupt)}}) is False+ assert (+ stage_screenshot_from_artifact(+ ctx,+ {"ok": True, "data": {"path": str(corrupt)}},+ provenance=self._provenance(),+ )+ is False+ )assert ctx.pending_screenshots == [stale]@@ -204,8 +304,8 @@ class TestCapturePostInteractionScreenshot:@staticmethoddef _server(result: dict[str, Any]) -> SimpleNamespace:async def call_internal_tool(name: str, arguments: dict[str, Any]) -> dict[str, Any]:- assert arguments == {}, "the copilot capture path must not request an inline frame"- return result+ assert arguments == {"session_id": "pbs_123"}, "capture must dispatch to the snapshotted session"+ return {**result, "browser_context": {"session_id": "pbs_123"}}return SimpleNamespace(call_internal_tool=call_internal_tool)@@ -215,6 +315,7 @@ class TestCapturePostInteractionScreenshot:"supports_vision": True,"pending_screenshots": [],"discovery_mcp_server": self._server(result),+ "browser_session_id": "pbs_123",}base.update(overrides)return SimpleNamespace(**base)@@ -225,8 +326,50 @@ class TestCapturePostInteractionScreenshot:Image.new("RGB", (300, 200), (0, 0, 0)).save(frame, format="PNG")ctx = self._ctx({"ok": True, "data": {"path": str(frame)}})- assert await _capture_post_interaction_screenshot(ctx) is True+ assert (+ await _capture_post_interaction_screenshot(+ ctx,+ source_tool="click",+ captured_url="https://example.com/results",+ observation_step=2,+ )+ is True+ )assert len(ctx.pending_screenshots) == 1+ provenance = ctx.pending_screenshots[0].provenance+ assert provenance.observation_step == 2+ assert provenance.captured_url is None+ assert provenance.dispatch_url == "https://example.com/results"+ assert provenance.dispatch_browser_session_id == "pbs_123"+ assert provenance.producer_browser_session_id == "pbs_123"+ assert provenance.session_binding.value == "agree"++ @pytest.mark.asyncio+ async def test_context_mutation_during_capture_does_not_relabel_dispatch(self, tmp_path: Path) -> None:+ frame = tmp_path / "frame.png"+ Image.new("RGB", (300, 200), (0, 0, 0)).save(frame, format="PNG")+ ctx = self._ctx({"ok": True, "data": {"path": str(frame)}})++ async def call_internal_tool(name: str, arguments: dict[str, Any]) -> dict[str, Any]:+ assert arguments == {"session_id": "pbs_123"}+ ctx.browser_session_id = "pbs_replacement"+ return {+ "ok": True,+ "browser_context": {"session_id": "pbs_producer"},+ "data": {"path": str(frame)},+ }++ ctx.discovery_mcp_server = SimpleNamespace(call_internal_tool=call_internal_tool)+ assert await _capture_post_interaction_screenshot(+ ctx,+ source_tool="click",+ captured_url="https://example.com/results",+ )+ provenance = ctx.pending_screenshots[0].provenance+ assert provenance.dispatch_browser_session_id == "pbs_123"+ assert provenance.producer_browser_session_id == "pbs_producer"+ assert provenance.browser_session_id == "pbs_producer"+ assert provenance.session_binding.value == "disagree"@pytest.mark.parametrize("overrides",@@ -242,14 +385,28 @@ class TestCapturePostInteractionScreenshot:Image.new("RGB", (300, 200), (0, 0, 0)).save(frame, format="PNG")ctx = self._ctx({"ok": True, "data": {"path": str(frame)}}, **overrides)- assert await _capture_post_interaction_screenshot(ctx) is False+ assert (+ await _capture_post_interaction_screenshot(+ ctx,+ source_tool="click",+ captured_url="https://example.com/results",+ )+ is False+ )assert ctx.pending_screenshots == []@pytest.mark.asyncioasync def test_failed_tool_call_stages_nothing(self) -> None:ctx = self._ctx({"ok": False, "error": "no page"})- assert await _capture_post_interaction_screenshot(ctx) is False+ assert (+ await _capture_post_interaction_screenshot(+ ctx,+ source_tool="click",+ captured_url=None,+ )+ is False+ )assert ctx.pending_screenshots == []@@ -265,7 +422,19 @@ class TestConsumePendingScreenshots:from skyvern.forge.sdk.copilot.enforcement import SCREENSHOT_SENTINEL, _consume_pending_screenshotsfrom skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry- entry = ScreenshotEntry(b64="dGVzdA==", mime="image/jpeg")+ entry = ScreenshotEntry(+ b64="dGVzdA==",+ mime="image/jpeg",+ capture_id="sha256:frame-123",+ provenance=ScreenshotProvenance(+ source_tool="inspect_page_for_composition",+ captured_url="https://example.com/results",+ observation_step=4,+ browser_session_id="pbs_123",+ workflow_run_id="wr_123",+ action_relation=ScreenshotActionRelation.SAME_PAGE_OBSERVATION,+ ),+ )ctx = MagicMock()ctx.pending_screenshots = [entry]msg = _consume_pending_screenshots(ctx)@@ -275,6 +444,13 @@ class TestConsumePendingScreenshots:assert len(content) == 2assert content[0]["type"] == "input_text"assert content[0]["text"].startswith(SCREENSHOT_SENTINEL)+ assert "capture_id=sha256:frame-123" in content[0]["text"]+ assert "captured_url=https://example.com/results" in content[0]["text"]+ assert "observation_step=4" in content[0]["text"]+ assert "browser_session_id=pbs_123" in content[0]["text"]+ assert "workflow_run_id=wr_123" in content[0]["text"]+ assert "action_relation=same_page_observation" in content[0]["text"]+ assert "may predate later actions" not in content[0]["text"]assert content[1]["type"] == "input_image"assert "image/jpeg" in content[1]["image_url"]assert content[1]["detail"] == "high"@@ -283,10 +459,9 @@ class TestConsumePendingScreenshots:def test_handles_multiple_screenshots(self) -> None:from skyvern.forge.sdk.copilot.enforcement import _consume_pending_screenshots- from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry- entry1 = ScreenshotEntry(b64="abc=", mime="image/jpeg")- entry2 = ScreenshotEntry(b64="def=", mime="image/jpeg")+ entry1 = _screenshot_entry("abc=")+ entry2 = _screenshot_entry("def=")ctx = MagicMock()ctx.pending_screenshots = [entry1, entry2]msg = _consume_pending_screenshots(ctx)@@ -305,7 +480,7 @@ class TestConsumePendingScreenshots:class TestNudgeDrainAndFilterExclusivity:def test_nudge_delivery_binds_the_frame_and_the_next_filter_pass_adds_nothing(self) -> None:ctx = SimpleNamespace(- pending_screenshots=[ScreenshotEntry(b64="dGVzdA==", mime="image/jpeg")],+ pending_screenshots=[_screenshot_entry("dGVzdA==")],supports_vision=True,)@@ -324,7 +499,7 @@ class TestNudgeDrainAndFilterExclusivity:# The vision check lives in the shared builder so every delivery path inherits it;# the drain still empties the queue.ctx = SimpleNamespace(- pending_screenshots=[ScreenshotEntry(b64="dGVzdA==", mime="image/jpeg")],+ pending_screenshots=[_screenshot_entry("dGVzdA==")],supports_vision=False,)@@ -1080,7 +1255,18 @@ class TestRunScreenshotResolution:ctx = MagicMock()ctx.supports_vision = Truectx.pending_screenshots = []- enqueue_screenshot_from_result(ctx, result)+ enqueue_screenshot_from_result(+ ctx,+ result,+ provenance=ScreenshotProvenance(+ source_tool="run_blocks_and_collect_debug",+ captured_url="https://example.com/failure",+ observation_step=None,+ browser_session_id="pbs_123",+ workflow_run_id="wr_123",+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,+ ),+ )assert len(ctx.pending_screenshots) == 1diff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.pyindex 59641fa43..cf87c30b8 100644--- a/tests/unit/test_copilot_session_factory.py+++ b/tests/unit/test_copilot_session_factory.py@@ -10,7 +10,11 @@ from typing import Anyimport pytestfrom structlog.testing import capture_logs-from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry+from skyvern.forge.sdk.copilot.screenshot_utils import (+ ScreenshotActionRelation,+ ScreenshotEntry,+ ScreenshotProvenance,+)from skyvern.forge.sdk.copilot.session_factory import (copilot_call_model_input_filter,make_copilot_call_model_input_filter,@@ -27,8 +31,23 @@ def _image_parts(item: Any) -> list[Any]:def _ctx_with_staged_frame(*, supports_vision: bool = True) -> SimpleNamespace:return SimpleNamespace(- pending_screenshots=[ScreenshotEntry(b64="dGVzdA==", mime="image/jpeg")],+ pending_screenshots=[+ ScreenshotEntry(+ b64="dGVzdA==",+ mime="image/jpeg",+ capture_id="sha256:frame-123",+ provenance=ScreenshotProvenance(+ source_tool="inspect_page_for_composition",+ captured_url="https://example.com/results",+ observation_step=2,+ browser_session_id="pbs_123",+ workflow_run_id="wr_123",+ action_relation=ScreenshotActionRelation.SAME_PAGE_OBSERVATION,+ ),+ )+ ],supports_vision=supports_vision,+ pending_frame_lease=None,)@@ -171,6 +190,44 @@ class TestSessionInputCallback:new_items = [{"role": "user", "content": "hello"}]assert copilot_session_input_callback([], new_items) == new_items+ def test_materialized_paired_frame_is_placeholdered_after_tool_output(self) -> None:+ from skyvern.forge.sdk.copilot.enforcement import SCREENSHOT_PLACEHOLDER+ from skyvern.forge.sdk.copilot.session_factory import copilot_session_input_callback++ goal = {"role": "user", "content": "build a workflow"}+ paired = {+ "role": "user",+ "content": [+ {+ "type": "input_text",+ "text": "[copilot:screenshot] [copilot:paired-observation] Frame provenance",+ },+ {"type": "input_image", "image_url": "data:image/jpeg;base64,dGVzdA=="},+ ],+ }+ output = {"type": "function_call_output", "call_id": "click-1", "output": '{"ok":true}'}++ combined = copilot_session_input_callback([goal, paired, output], [])++ assert combined[1] == {"role": "user", "content": SCREENSHOT_PLACEHOLDER}++ def test_unmarked_generic_frame_keeps_existing_retention_behavior(self) -> None:+ from skyvern.forge.sdk.copilot.session_factory import copilot_session_input_callback++ goal = {"role": "user", "content": "build a workflow"}+ generic = {+ "role": "user",+ "content": [+ {"type": "input_text", "text": "[copilot:screenshot] generic tool frame"},+ {"type": "input_image", "image_url": "data:image/jpeg;base64,dGVzdA=="},+ ],+ }+ output = {"type": "function_call_output", "call_id": "read-1", "output": '{"ok":true}'}++ combined = copilot_session_input_callback([goal, generic, output], [])++ assert combined[1] == generic+def test_preserves_original_goal_and_applies_compaction_to_middle(self) -> None:"""First-turn shape (one real user, several tool iterations): thegoal at index 0 is preserved; older function_call_output items get@@ -303,6 +360,32 @@ class TestStagedScreenshotBinding:assert [len(_image_parts(item)) for item in result.input] == [0, 1]assert any(log["event"] == "Injecting screenshot user message" for log in logs)+ def test_linked_structural_result_precedes_the_frame_and_its_provenance(self) -> None:+ ctx = _ctx_with_staged_frame()+ items = [+ {"role": "user", "content": "inspect this page"},+ {+ "type": "function_call_output",+ "call_id": "inspect-1",+ "output": json.dumps(+ {+ "ok": True,+ "current_url": "https://example.com/results",+ "observation_step": 2,+ "data": {"source_tool": "inspect_page_for_composition"},+ }+ ),+ },+ ]++ result = copilot_call_model_input_filter(_mk_input_data(items, context=ctx))++ assert result.input[-2] == items[-1]+ assert len(_image_parts(result.input[-1])) == 1+ provenance_text = result.input[-1]["content"][0]["text"]+ assert "source_tool=inspect_page_for_composition" in provenance_text+ assert "observation_step=2" in provenance_text+def test_a_second_pass_over_the_same_context_still_carries_the_frame(self) -> None:ctx = _ctx_with_staged_frame()items = [{"role": "user", "content": "clear the modal on this page"}]@@ -314,6 +397,63 @@ class TestStagedScreenshotBinding:assert len(_image_parts(second.input[-1])) == 1assert len(ctx.pending_screenshots) == 1+ def test_paired_frame_is_not_redelivered_after_model_input_advances(self) -> None:+ ctx = _ctx_with_staged_frame()+ initial = [{"role": "user", "content": "clear the modal on this page"}]++ first = copilot_call_model_input_filter(_mk_input_data(initial, context=ctx))+ advanced = [+ *initial,+ {"type": "function_call", "call_id": "click-1", "name": "click", "arguments": "{}"},+ {"type": "function_call_output", "call_id": "click-1", "output": '{"ok":true}'},+ ]+ second = copilot_call_model_input_filter(_mk_input_data(advanced, context=ctx))++ assert len(_image_parts(first.input[-1])) == 1+ assert not any(_image_parts(item) for item in second.input)+ assert ctx.pending_screenshots == []+ assert ctx.pending_frame_lease is None++ def test_new_paired_capture_gets_a_fresh_lease_after_invalidation(self) -> None:+ ctx = _ctx_with_staged_frame()+ initial = [{"role": "user", "content": "clear the modal on this page"}]+ copilot_call_model_input_filter(_mk_input_data(initial, context=ctx))+ copilot_call_model_input_filter(+ _mk_input_data([*initial, {"role": "assistant", "content": "acted"}], context=ctx)+ )+ ctx.pending_screenshots = _ctx_with_staged_frame().pending_screenshots++ delivered = copilot_call_model_input_filter(+ _mk_input_data([*initial, {"role": "assistant", "content": "acted"}], context=ctx)+ )++ assert len(_image_parts(delivered.input[-1])) == 1+ assert ctx.pending_frame_lease.capture_id == "sha256:frame-123"++ def test_same_pixels_from_a_new_capture_event_get_a_fresh_lease(self) -> None:+ ctx = _ctx_with_staged_frame()+ initial = [{"role": "user", "content": "inspect this page"}]+ copilot_call_model_input_filter(_mk_input_data(initial, context=ctx))+ first_event_id = ctx.pending_frame_lease.capture_event_id++ recaptured = _ctx_with_staged_frame().pending_screenshots[0]+ assert recaptured.capture_id == ctx.pending_screenshots[0].capture_id+ assert recaptured.capture_event_id != first_event_id+ ctx.pending_screenshots = [recaptured]+ advanced = [*initial, {"role": "assistant", "content": "the page was inspected again"}]++ delivered = copilot_call_model_input_filter(_mk_input_data(advanced, context=ctx))++ assert len(_image_parts(delivered.input[-1])) == 1+ assert ctx.pending_frame_lease.capture_event_id == recaptured.capture_event_id++ def test_paired_message_has_machine_readable_marker(self) -> None:+ ctx = _ctx_with_staged_frame()++ result = copilot_call_model_input_filter(_mk_input_data([{"role": "user", "content": "inspect"}], context=ctx))++ assert result.input[-1]["content"][0]["text"].startswith("[copilot:screenshot] [copilot:paired-observation] ")+def test_a_non_vision_fallback_model_gets_no_image(self) -> None:# The frame was staged while the primary was still vision-capable; a retriable failure# can swap in a fallback that cannot accept images.