Skip to content

Automate browser based workflows with AI

SKY-14836 Pair fresh page screenshots with their evidence (#8279)

Andrew Neilson58e02e0e2d19
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index bba92cba6..dc54b11ef 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -82,7 +82,7 @@ from skyvern.forge.sdk.copilot.result_evidence import (
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
)
-from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
+from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotActionRelation, ScreenshotEntry
from skyvern.forge.sdk.copilot.terminal_predicates import (
artifact_health_blocked,
outcome_criteria_evaluated,
@@ -117,6 +117,7 @@ TOTAL_TIMEOUT_SECONDS = settings.WORKFLOW_COPILOT_TOTAL_TIMEOUT_SECONDS or 900
# constant so tests can shrink it instead of paying a full second per deadline.
MIN_DEADLINE_REMAINING_SECONDS = 1.0
SCREENSHOT_SENTINEL = "[copilot:screenshot] "
+PAIRED_OBSERVATION_MARKER = "[copilot:paired-observation] "
NUDGE_SENTINEL = "[copilot:nudge] "
SCREENSHOT_PLACEHOLDER = SCREENSHOT_SENTINEL + "[prior screenshot removed to save context]"
TOKEN_BUDGET = DEFAULT_TOKEN_BUDGET
@@ -450,13 +451,42 @@ def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:
if not isinstance(pending, list) or not pending:
return None
screenshots: list[ScreenshotEntry] = list(pending)
+ provenance_lines: list[str] = []
+ for entry in screenshots:
+ provenance = entry.provenance
+ fields = {
+ "capture_id": entry.capture_id,
+ "source_tool": provenance.source_tool,
+ "captured_url": provenance.captured_url or "unavailable",
+ "dispatch_url": provenance.dispatch_url or "unavailable",
+ "observation_step": provenance.observation_step
+ if provenance.observation_step is not None
+ else "unavailable",
+ "browser_session_id": provenance.browser_session_id or "unavailable",
+ "dispatch_browser_session_id": provenance.dispatch_browser_session_id or "unavailable",
+ "producer_browser_session_id": provenance.producer_browser_session_id or "unavailable",
+ "session_binding": provenance.session_binding.value,
+ "workflow_run_id": provenance.workflow_run_id or "unavailable",
+ "action_relation": provenance.action_relation.value,
+ }
+ rendered = "; ".join(f"{key}={value}" for key, value in fields.items())
+ relation = (
+ "This frame was captured during the named page observation."
+ if provenance.action_relation is ScreenshotActionRelation.SAME_PAGE_OBSERVATION
+ else "This frame records the named source at its stated action relation."
+ )
+ provenance_lines.append(
+ f"Frame provenance: {rendered}. {relation} Its provenance does not claim freshness after later actions."
+ )
+ paired_marker = (
+ PAIRED_OBSERVATION_MARKER
+ if screenshots[0].provenance.action_relation is ScreenshotActionRelation.SAME_PAGE_OBSERVATION
+ else ""
+ )
content: list[dict[str, Any]] = [
{
"type": "input_text",
- "text": (
- SCREENSHOT_SENTINEL + "Here is the most recent screenshot captured this turn. "
- "It shows the page as of that capture and may predate later actions."
- ),
+ "text": SCREENSHOT_SENTINEL + paired_marker + "\n".join(provenance_lines),
},
]
for entry in screenshots:
@@ -476,6 +506,8 @@ def _consume_pending_screenshots(ctx: Any) -> dict[str, Any] | None:
pending = getattr(ctx, "pending_screenshots", None)
if isinstance(pending, list):
pending.clear()
+ if hasattr(ctx, "pending_frame_lease"):
+ ctx.pending_frame_lease = None
return message
@@ -536,6 +568,22 @@ def is_screenshot_message(item: Any) -> bool:
return False
+def is_paired_observation_message(item: Any) -> bool:
+ """Return True only for explicitly marked observation-bound frames."""
+ if _item_field(item, "role") != "user":
+ return False
+ prefix = SCREENSHOT_SENTINEL + PAIRED_OBSERVATION_MARKER
+ content = _item_field(item, "content")
+ if isinstance(content, str):
+ return content.startswith(prefix)
+ if not isinstance(content, list):
+ return False
+ return any(
+ isinstance(_item_field(block, "text"), str) and _item_field(block, "text").startswith(prefix)
+ for block in content
+ )
+
+
def _is_nudge_message(item: Any) -> bool:
"""Return True if the item is a synthetic enforcement nudge."""
if _item_field(item, "role") != "user":
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 05f92a0b4..0e78da02c 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -52,7 +52,11 @@ from skyvern.forge.sdk.copilot.runtime import (
resolve_browser_state_for_context,
retire_browser_session_id,
)
-from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
+from skyvern.forge.sdk.copilot.screenshot_utils import (
+ ScreenshotActionRelation,
+ ScreenshotProvenance,
+ enqueue_screenshot_from_result,
+)
from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structure
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from skyvern.utils.contained_effects import contained_effect
@@ -1177,7 +1181,27 @@ class SkyvernOverlayMCPServer(MCPServer):
copilot_result = _scrub_tool_result(copilot_ctx, copilot_result)
record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, copilot_result)
- enqueue_screenshot_from_result(copilot_ctx, copilot_result)
+ screenshot_data = copilot_result.get("data")
+ screenshot_data = screenshot_data if isinstance(screenshot_data, dict) else {}
+ captured_url = screenshot_data.get("url") or screenshot_data.get("current_url")
+ observation_step = copilot_result.get("observation_step")
+ if observation_step is None:
+ observation_step = screenshot_data.get("observation_step")
+ workflow_run_id = copilot_result.get("workflow_run_id") or screenshot_data.get("workflow_run_id")
+ enqueue_screenshot_from_result(
+ copilot_ctx,
+ copilot_result,
+ provenance=ScreenshotProvenance(
+ source_tool=tool_name,
+ captured_url=captured_url if isinstance(captured_url, str) and captured_url else None,
+ observation_step=observation_step
+ if isinstance(observation_step, int) and not isinstance(observation_step, bool)
+ else None,
+ browser_session_id=call_browser_session_id,
+ workflow_run_id=workflow_run_id if isinstance(workflow_run_id, str) and workflow_run_id else None,
+ action_relation=ScreenshotActionRelation.TOOL_RESULT,
+ ),
+ )
return _copilot_to_call_tool_result(copilot_result)
async def call_internal_tool(
@@ -1214,6 +1238,9 @@ class SkyvernOverlayMCPServer(MCPServer):
phases = _PhaseClock()
copilot_name = self._reverse_alias.get(mcp_tool_name, mcp_tool_name)
ctx = self._context_provider()
+ requested_session_id = mcp_args.get("session_id")
+ if not isinstance(requested_session_id, str) or not requested_session_id:
+ requested_session_id = None
observed_continuity_generation = getattr(ctx, "browser_session_continuity_generation", 0)
if not self._client:
_log_mcp_timing(ctx, copilot_name, mcp_tool_name, phases, {}, "internal", "not_connected")
@@ -1238,8 +1265,10 @@ class SkyvernOverlayMCPServer(MCPServer):
if continuity_result is not None:
_log_mcp_timing(ctx, copilot_name, mcp_tool_name, phases, {}, "internal", "session_error")
return _scrub_tool_result(ctx, continuity_result)
- merged_args = {**mcp_args, "session_id": ctx.browser_session_id}
- call_browser_session_id = ctx.browser_session_id
+ # An explicit session is a dispatch lease chosen before session preparation awaits. Do not
+ # relabel that call with mutable ambient context if another concurrent tool replaces it.
+ call_browser_session_id = requested_session_id or ctx.browser_session_id
+ merged_args = {**mcp_args, "session_id": call_browser_session_id}
try:
phases.enter("context_enter")
async with mcp_browser_context(ctx):
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index ce0ac333f..4b66a0eaf 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -29,7 +29,7 @@ from skyvern.cli.core.session_manager import (
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
-from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
+from skyvern.forge.sdk.copilot.screenshot_utils import PendingFrameLease, ScreenshotEntry
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.turn_origin import (
HealAdoptionFailed,
@@ -325,6 +325,7 @@ class AgentContext:
browser_session_continuity_disposition: str | None = None
supports_vision: bool = True
pending_screenshots: list[ScreenshotEntry] = field(default_factory=list)
+ pending_frame_lease: PendingFrameLease | None = None
tool_activity: list[dict[str, Any]] = field(default_factory=list)
unrecoverable_tool_error_streak_count: int = 0
unrecoverable_tool_error_signature: str | None = None
diff --git a/skyvern/forge/sdk/copilot/screenshot_utils.py b/skyvern/forge/sdk/copilot/screenshot_utils.py
index 0e4a9b324..6af89e6ef 100644
--- a/skyvern/forge/sdk/copilot/screenshot_utils.py
+++ b/skyvern/forge/sdk/copilot/screenshot_utils.py
@@ -3,8 +3,13 @@
from __future__ import annotations
import base64
+import hashlib
import io
-from dataclasses import dataclass
+import json
+import time
+import uuid
+from dataclasses import dataclass, field
+from enum import Enum
from pathlib import Path
from typing import Any
@@ -18,19 +23,146 @@ COPILOT_JPEG_QUALITY = 60
LOG = structlog.get_logger()
-@dataclass
+class ScreenshotActionRelation(str, Enum):
+ SAME_PAGE_OBSERVATION = "same_page_observation"
+ AFTER_SOURCE_ACTION = "after_source_action"
+ WORKFLOW_RUN_RESULT = "workflow_run_result"
+ TOOL_RESULT = "tool_result"
+
+
+class ProvenanceBinding(str, Enum):
+ AGREE = "agree"
+ DISAGREE = "disagree"
+ UNAVAILABLE = "unavailable"
+
+
+@dataclass(frozen=True)
+class ScreenshotProvenance:
+ source_tool: str
+ captured_url: str | None
+ observation_step: int | None
+ browser_session_id: str | None
+ workflow_run_id: str | None
+ action_relation: ScreenshotActionRelation
+ dispatch_url: str | None = None
+ dispatch_browser_session_id: str | None = None
+ producer_browser_session_id: str | None = None
+ session_binding: ProvenanceBinding = ProvenanceBinding.UNAVAILABLE
+
+ @classmethod
+ def unknown(cls, *, source_tool: str) -> ScreenshotProvenance:
+ return cls(
+ source_tool=source_tool,
+ captured_url=None,
+ observation_step=None,
+ browser_session_id=None,
+ workflow_run_id=None,
+ action_relation=ScreenshotActionRelation.TOOL_RESULT,
+ )
+
+
+@dataclass(frozen=True)
class ScreenshotEntry:
b64: str
mime: str # "image/jpeg" or "image/png"
+ capture_id: str
+ provenance: ScreenshotProvenance
+ captured_at: float = field(default_factory=time.monotonic)
+ capture_event_id: str = field(default_factory=lambda: uuid.uuid4().hex)
-def resize_screenshot_b64(b64_png: str) -> ScreenshotEntry:
+@dataclass(frozen=True)
+class CapturedFrame:
+ b64: str
+ captured_at: float
+ captured_url: str | None = None
+ browser_session_id: str | None = None
+ dispatch_url: str | None = None
+ dispatch_browser_session_id: str | None = None
+ producer_browser_session_id: str | None = None
+ session_binding: ProvenanceBinding = ProvenanceBinding.UNAVAILABLE
+
+
+@dataclass(frozen=True)
+class PendingFrameLease:
+ capture_event_id: str
+ capture_id: str
+ input_fingerprint: str
+
+
+def screenshot_result_facts(
+ result: dict[str, Any],
+ *,
+ dispatch_url: str | None,
+ dispatch_browser_session_id: str | None,
+) -> tuple[str | None, str | None, ProvenanceBinding]:
+ """Return only facts reported by the screenshot producer, plus their dispatch binding.
+
+ Internal MCP responses may run in concise mode, where ``browser_context`` is deliberately
+ omitted. In that case the binding is unavailable; the caller's dispatch target remains
+ separately labelled and is never promoted to a producer fact.
+ """
+ data = result.get("data")
+ producer_url: str | None = None
+ if isinstance(data, dict):
+ value = data.get("current_url") or data.get("url")
+ producer_url = value if isinstance(value, str) and value else None
+ browser_context = result.get("browser_context")
+ producer_session_id: str | None = None
+ if isinstance(browser_context, dict):
+ value = browser_context.get("session_id")
+ producer_session_id = value if isinstance(value, str) and value else None
+ if dispatch_browser_session_id is None or producer_session_id is None:
+ binding = ProvenanceBinding.UNAVAILABLE
+ elif dispatch_browser_session_id == producer_session_id:
+ binding = ProvenanceBinding.AGREE
+ else:
+ binding = ProvenanceBinding.DISAGREE
+ return producer_url, producer_session_id, binding
+
+
+def _fingerprint_projection(value: Any) -> Any:
+ if hasattr(value, "model_dump"):
+ value = value.model_dump(mode="json")
+ if isinstance(value, dict):
+ if value.get("type") in {"input_image", "image_url"}:
+ return {"type": value.get("type")}
+ return {
+ str(key): _fingerprint_projection(item)
+ for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))
+ }
+ if isinstance(value, (list, tuple)):
+ return [_fingerprint_projection(item) for item in value]
+ if isinstance(value, (str, int, float, bool)) or value is None:
+ return value
+ return repr(value)
+
+
+def model_input_fingerprint(items: list[Any]) -> str:
+ """Stable, order-sensitive identity for one logical model request.
+
+ Image payloads project to their block type so pixels never enter the serialized
+ fingerprint material. The digest distinguishes a provider retry from a later
+ request whose model-visible conversation has advanced.
+ """
+ projected = _fingerprint_projection(items)
+ encoded = json.dumps(projected, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()
+ return hashlib.sha256(encoded).hexdigest()
+
+
+def resize_screenshot_b64(
+ b64_png: str,
+ *,
+ provenance: ScreenshotProvenance,
+ captured_at: float | None = None,
+) -> ScreenshotEntry:
"""Resize a base64 PNG to 1024x768 max and compress to JPEG.
Uses thumbnail() to preserve aspect ratio — only shrinks, never enlarges.
Converts RGBA/P modes to RGB for JPEG compatibility.
"""
raw = base64.b64decode(b64_png)
+ capture_id = f"sha256:{hashlib.sha256(raw).hexdigest()}"
img = Image.open(io.BytesIO(raw))
img.thumbnail(
(COPILOT_SCREENSHOT_MAX_WIDTH, COPILOT_SCREENSHOT_MAX_HEIGHT),
@@ -43,29 +175,51 @@ def resize_screenshot_b64(b64_png: str) -> ScreenshotEntry:
return ScreenshotEntry(
b64=base64.b64encode(buf.getvalue()).decode("ascii"),
mime="image/jpeg",
+ capture_id=capture_id,
+ provenance=provenance,
+ captured_at=captured_at if captured_at is not None else time.monotonic(),
)
-def enqueue_screenshot(ctx: Any, b64_png: str) -> bool:
+def enqueue_screenshot(
+ ctx: Any,
+ b64_png: str,
+ *,
+ provenance: ScreenshotProvenance,
+ captured_at: float | None = None,
+) -> bool:
"""Resize, compress, and store a screenshot for later LLM injection, reporting whether this call
left an entry on the queue. Replaces any previously pending screenshot (cap to 1).
"""
if not getattr(ctx, "supports_vision", False):
return False
try:
- entry = resize_screenshot_b64(b64_png)
+ entry = resize_screenshot_b64(b64_png, provenance=provenance, captured_at=captured_at)
except Exception:
LOG.warning("Failed to resize copilot screenshot", exc_info=True)
return False
pending = getattr(ctx, "pending_screenshots", None)
if not isinstance(pending, list):
return False
+ if pending and isinstance(pending[0], ScreenshotEntry) and pending[0].captured_at > entry.captured_at:
+ LOG.info(
+ "copilot_older_screenshot_rejected",
+ source_tool=entry.provenance.source_tool,
+ pending_source_tool=pending[0].provenance.source_tool,
+ )
+ return False
pending.clear()
pending.append(entry)
return True
-def stage_screenshot_from_artifact(ctx: Any, result: dict[str, Any]) -> bool:
+def stage_screenshot_from_artifact(
+ ctx: Any,
+ result: dict[str, Any],
+ *,
+ provenance: ScreenshotProvenance,
+ captured_at: float | None = None,
+) -> bool:
"""Stage the frame a non-inline screenshot tool call wrote to disk, reporting whether this call
left an entry on the queue rather than whether the queue is merely non-empty.
"""
@@ -78,13 +232,24 @@ def stage_screenshot_from_artifact(ctx: Any, result: dict[str, Any]) -> bool:
except (OSError, ValueError):
LOG.info("Copilot screenshot artifact could not be read", path=path)
return False
- return enqueue_screenshot(ctx, base64.b64encode(raw).decode("ascii"))
+ return enqueue_screenshot(
+ ctx,
+ base64.b64encode(raw).decode("ascii"),
+ provenance=provenance,
+ captured_at=captured_at if captured_at is not None else time.monotonic(),
+ )
-def enqueue_screenshot_from_result(ctx: Any, result: dict[str, Any]) -> None:
+def enqueue_screenshot_from_result(
+ ctx: Any,
+ result: dict[str, Any],
+ *,
+ provenance: ScreenshotProvenance,
+) -> None:
"""Extract, validate, and enqueue a screenshot from a copilot result dict."""
from skyvern.forge.sdk.copilot.output_utils import extract_screenshot_b64, is_valid_image_base64
+ captured_at = time.monotonic()
screenshot_b64 = extract_screenshot_b64(result)
if screenshot_b64 and is_valid_image_base64(screenshot_b64):
- enqueue_screenshot(ctx, screenshot_b64)
+ enqueue_screenshot(ctx, screenshot_b64, provenance=provenance, captured_at=captured_at)
diff --git a/skyvern/forge/sdk/copilot/session_factory.py b/skyvern/forge/sdk/copilot/session_factory.py
index 8d3e09b38..076069239 100644
--- a/skyvern/forge/sdk/copilot/session_factory.py
+++ b/skyvern/forge/sdk/copilot/session_factory.py
@@ -33,11 +33,17 @@ from skyvern.forge.sdk.copilot.enforcement import (
_summarize_tool_output,
aggressive_prune,
estimate_tokens,
+ is_paired_observation_message,
is_screenshot_message,
is_synthetic_user_message,
log_recent_tool_output_truncation,
pending_screenshot_message,
)
+from skyvern.forge.sdk.copilot.screenshot_utils import (
+ PendingFrameLease,
+ ScreenshotActionRelation,
+ model_input_fingerprint,
+)
LOG = structlog.get_logger()
@@ -123,12 +129,21 @@ def copilot_session_input_callback(
# The newest frame must survive the merge: with every injected message classified
# synthetic, the whole post-goal history is middle, and blanket-replacing would
# blind the agent to the frame it just captured once it rotates into history.
- middle_screenshots = [i for i, item in enumerate(pruned_middle) if _screenshot_with_image(item)]
+ trailing_items = list(recent) + list(new_items)
+ stale_paired_screenshots = {
+ i
+ for i, item in enumerate(pruned_middle)
+ if is_paired_observation_message(item)
+ and any(not is_synthetic_user_message(following) for following in list(pruned_middle[i + 1 :]) + trailing_items)
+ }
+ middle_screenshots = [
+ i for i, item in enumerate(pruned_middle) if _screenshot_with_image(item) and i not in stale_paired_screenshots
+ ]
has_newer_screenshot = any(_screenshot_with_image(item) for item in list(recent) + list(new_items))
keep_screenshot = middle_screenshots[-1] if middle_screenshots and not has_newer_screenshot else None
pruned_middle = [
{"role": "user", "content": SCREENSHOT_PLACEHOLDER}
- if is_screenshot_message(item) and i != keep_screenshot
+ if is_screenshot_message(item) and (i != keep_screenshot or i in stale_paired_screenshots)
else item
for i, item in enumerate(pruned_middle)
]
@@ -158,9 +173,36 @@ def _run_context(data: CallModelData[Any]) -> Any:
def _copilot_call_model_input_filter(data: CallModelData[Any], *, token_budget: int) -> ModelInputData:
items = list(data.model_data.input)
+ ctx = _run_context(data)
# Read-only peek: the end-of-turn drain in enforcement stays the only clear, so a provider
# retry or model fallback re-running this filter still carries the frame.
- screenshot_msg = pending_screenshot_message(_run_context(data))
+ screenshot_msg = pending_screenshot_message(ctx)
+ pending = getattr(ctx, "pending_screenshots", None)
+ entry = pending[0] if isinstance(pending, list) and pending else None
+ if (
+ screenshot_msg is not None
+ and entry is not None
+ and entry.provenance.action_relation is ScreenshotActionRelation.SAME_PAGE_OBSERVATION
+ ):
+ fingerprint = model_input_fingerprint(items)
+ lease = getattr(ctx, "pending_frame_lease", None)
+ if isinstance(lease, PendingFrameLease) and lease.capture_event_id == entry.capture_event_id:
+ if lease.input_fingerprint != fingerprint:
+ if isinstance(pending, list):
+ pending.clear()
+ ctx.pending_frame_lease = None
+ screenshot_msg = None
+ LOG.info(
+ "copilot_frame_lease_invalidated",
+ capture_id=entry.capture_id,
+ input_fingerprint=fingerprint[:12],
+ )
+ else:
+ ctx.pending_frame_lease = PendingFrameLease(
+ capture_event_id=entry.capture_event_id,
+ capture_id=entry.capture_id,
+ input_fingerprint=fingerprint,
+ )
if screenshot_msg is not None:
LOG.info(
"Injecting screenshot user message",
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index a69fa4c61..3a69761cf 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -31,7 +31,11 @@ from skyvern.forge.sdk.copilot.output_utils import (
sanitize_tool_result_for_llm,
)
from skyvern.forge.sdk.copilot.pending_operation import pending_operation
-from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
+from skyvern.forge.sdk.copilot.screenshot_utils import (
+ ScreenshotActionRelation,
+ ScreenshotProvenance,
+ enqueue_screenshot_from_result,
+)
from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structure
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.workflow_yaml import (
@@ -807,7 +811,11 @@ async def run_blocks_tool(
source_tool="run_blocks_and_collect_debug",
result=result,
)
- enqueue_screenshot_from_result(copilot_ctx, result)
+ enqueue_screenshot_from_result(
+ copilot_ctx,
+ result,
+ provenance=_run_result_screenshot_provenance(result, source_tool="run_blocks_and_collect_debug"),
+ )
sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)
return json.dumps(sanitized)
@@ -1067,11 +1075,41 @@ async def _run_updated_workflow_blocks(
result=run_result,
workflow_updated=True,
)
- enqueue_screenshot_from_result(copilot_ctx, run_result)
+ enqueue_screenshot_from_result(
+ copilot_ctx,
+ run_result,
+ provenance=_run_result_screenshot_provenance(run_result, source_tool=tool_name),
+ )
sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", run_result)
return json.dumps(sanitized)
+def _run_result_screenshot_provenance(result: dict[str, Any], *, source_tool: str) -> ScreenshotProvenance:
+ """Bind a run-result frame to the facts carried by the run payload.
+
+ Run execution returns its identity fields under ``data``. Some older call
+ sites and error paths still expose them at the top level, so retain that as
+ a factual fallback rather than stamping the frame as unidentified.
+ """
+ data = result.get("data")
+ facts = data if isinstance(data, dict) else {}
+
+ def _string_fact(key: str) -> str | None:
+ value = facts.get(key)
+ if not isinstance(value, str) or not value:
+ value = result.get(key)
+ return value if isinstance(value, str) and value else None
+
+ return ScreenshotProvenance(
+ source_tool=source_tool,
+ captured_url=_string_fact("current_url"),
+ observation_step=None,
+ browser_session_id=_string_fact("browser_session_id"),
+ workflow_run_id=_string_fact("workflow_run_id"),
+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,
+ )
+
+
@function_tool(name_override="discover_workflow_entrypoint", strict_mode=False)
async def discover_workflow_entrypoint_tool(
ctx: RunContextWrapper,
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index 2d75d5082..bb0c893b3 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -4,6 +4,9 @@ import asyncio
import base64
import copy
import json
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass
from typing import Any
from urllib.parse import urlparse
@@ -31,6 +34,13 @@ from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
post_run_inspection_cleanly_matches,
repair_page_evidence_is_admissible,
)
+from skyvern.forge.sdk.copilot.screenshot_utils import (
+ CapturedFrame,
+ ScreenshotActionRelation,
+ ScreenshotProvenance,
+ enqueue_screenshot,
+ screenshot_result_facts,
+)
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from ._shared import (
@@ -60,6 +70,27 @@ from .scouting import (
LOG = structlog.get_logger()
+@dataclass(frozen=True)
+class CompositionEvidenceCapture:
+ evidence: dict[str, Any] | None
+ error: str | None
+ frame: CapturedFrame | None = None
+
+ def __iter__(self) -> Iterator[dict[str, Any] | str | None]:
+ """Keep legacy two-value test/caller unpacking while frame-aware funnels migrate explicitly."""
+ yield self.evidence
+ yield self.error
+
+
+def _capture_result_parts(
+ capture: CompositionEvidenceCapture | tuple[dict[str, Any] | None, str | None],
+) -> tuple[dict[str, Any] | None, str | None, CapturedFrame | None]:
+ if isinstance(capture, CompositionEvidenceCapture):
+ return capture.evidence, capture.error, capture.frame
+ evidence, error = capture
+ return evidence, error, None
+
+
_POST_RUN_REPAIR_CAPTURE_TIMEOUT_SECONDS = 30.0
_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS = 10.0
_COMPOSITION_VISUAL_SUMMARY_PROMPT_NAME = "workflow-copilot-page-evidence-vision"
@@ -96,13 +127,16 @@ def _model_facing_inspect_result(result: dict[str, Any]) -> dict[str, Any]:
return shaped
-async def _composition_get_screenshot(ctx: CopilotContext) -> dict[str, Any]:
+async def _composition_get_screenshot(ctx: CopilotContext, *, dispatch_session_id: str | None) -> dict[str, Any]:
server = getattr(ctx, "discovery_mcp_server", None)
if server is None:
return {"ok": False, "error": "discovery MCP server not attached to context"}
try:
return await asyncio.wait_for(
- server.call_internal_tool("skyvern_screenshot", {"inline": True}),
+ server.call_internal_tool(
+ "skyvern_screenshot",
+ {"inline": True, **({"session_id": dispatch_session_id} if dispatch_session_id else {})},
+ ),
timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
)
except asyncio.TimeoutError:
@@ -253,20 +287,46 @@ async def _composition_summarize_screenshot(
async def _augment_composition_evidence_with_visual_fallback(
ctx: CopilotContext,
evidence: dict[str, Any],
-) -> dict[str, Any]:
- screenshot_result = await _composition_get_screenshot(ctx)
+) -> tuple[dict[str, Any], CapturedFrame | None]:
+ capture_started_at = time.monotonic()
+ capture_url = evidence.get("current_url") or evidence.get("inspected_url")
+ capture_session_id = getattr(ctx, "browser_session_id", None)
+ screenshot_result = await _composition_get_screenshot(ctx, dispatch_session_id=capture_session_id)
if not screenshot_result.get("ok"):
- return _composition_add_evidence_omission(
- evidence,
- f"screenshot_capture_failed: {screenshot_result.get('error', 'unknown')}",
+ return (
+ _composition_add_evidence_omission(
+ evidence,
+ f"screenshot_capture_failed: {screenshot_result.get('error', 'unknown')}",
+ ),
+ None,
)
screenshot_b64 = _composition_extract_screenshot_b64(screenshot_result)
+ producer_url, producer_session_id, session_binding = screenshot_result_facts(
+ screenshot_result,
+ dispatch_url=str(capture_url) if capture_url else None,
+ dispatch_browser_session_id=capture_session_id,
+ )
+ captured_frame = (
+ CapturedFrame(
+ b64=screenshot_b64,
+ captured_at=capture_started_at,
+ captured_url=producer_url,
+ browser_session_id=producer_session_id,
+ dispatch_url=str(capture_url) if capture_url else None,
+ dispatch_browser_session_id=capture_session_id,
+ producer_browser_session_id=producer_session_id,
+ session_binding=session_binding,
+ )
+ if screenshot_b64
+ else None
+ )
visual_summary, visual_error = await _composition_summarize_screenshot(
ctx,
evidence=evidence,
screenshot_b64=screenshot_b64,
)
- return merge_visual_composition_evidence(evidence, visual_summary=visual_summary, visual_error=visual_error)
+ merged = merge_visual_composition_evidence(evidence, visual_summary=visual_summary, visual_error=visual_error)
+ return merged, captured_frame
def _composition_add_evidence_omission(evidence: dict[str, Any], message: str) -> dict[str, Any]:
@@ -292,7 +352,7 @@ async def _composition_evidence_after_navigation_failure(
*,
inspected_url: str,
navigation_error: str,
-) -> dict[str, Any] | None:
+) -> tuple[dict[str, Any], CapturedFrame | None] | None:
current_url, _ = await _fallback_page_info(ctx)
current_url = current_url or inspected_url
structured, structured_error = await _composition_get_structured_evidence_result(
@@ -303,9 +363,10 @@ async def _composition_evidence_after_navigation_failure(
structured,
"navigation_error_before_html_capture",
)
+ frame = None
if page_evidence_needs_visual_fallback(evidence):
- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
- return evidence
+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
+ return evidence, frame
if structured_error is not None:
# A navigation failure is the exceptional path where a screenshot can still
# observe the page even though the bounded DOM execution context was lost.
@@ -319,8 +380,8 @@ async def _composition_evidence_after_navigation_failure(
evidence,
"structured_evidence_unavailable_after_navigation_error",
)
- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
- return evidence if evidence.get("screenshot_used") else None
+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
+ return (evidence, frame) if evidence.get("screenshot_used") else None
# Same size-cap survival as the success path: a heavy page that rendered before the nav
# error still parses via the stripped-body evaluate instead of yielding hollow evidence.
html, html_error, html_truncated, _ = await _composition_get_html(ctx)
@@ -338,9 +399,10 @@ async def _composition_evidence_after_navigation_failure(
if html_truncated:
evidence = _composition_add_inspection_warning(evidence, "html_sliced_at_cap")
evidence = await _augment_composition_evidence_with_computed_obstruction_candidates(ctx, evidence)
+ frame = None
if page_evidence_needs_visual_fallback(evidence):
- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
- return evidence
+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
+ return evidence, frame
evidence = parse_composition_html("", inspected_url=inspected_url, current_url=current_url)
evidence = _composition_add_inspection_warning(
@@ -351,8 +413,8 @@ async def _composition_evidence_after_navigation_failure(
evidence,
"html_capture_failed_after_navigation_error",
)
- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
- return evidence if evidence.get("screenshot_used") else None
+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
+ return (evidence, frame) if evidence.get("screenshot_used") else None
def _inspection_reached_via(*, use_current_page: bool, post_run: bool, earned_interaction: bool) -> str:
@@ -504,7 +566,7 @@ async def _capture_composition_evidence(
*,
inspected_url: str,
current_url: str,
-) -> tuple[dict[str, Any] | None, str | None]:
+) -> CompositionEvidenceCapture:
"""Capture page evidence, using HTML only to enrich a valid but unsettled structured packet."""
evidence: dict[str, Any] | None = None
html_truncated = False
@@ -517,7 +579,7 @@ async def _capture_composition_evidence(
if structured_error is not None:
if evidence is not None:
break
- return None, structured_error
+ return CompositionEvidenceCapture(None, structured_error)
if structured is not None:
evidence = structured
used_structured = True
@@ -535,7 +597,7 @@ async def _capture_composition_evidence(
if html_error is not None:
if evidence is not None:
break
- return None, html_error
+ return CompositionEvidenceCapture(None, html_error)
# On a heavy page the raw get_html serialization is dropped over the MCP size cap and
# falls back to the stripped read; once that happens, settle-and-recapture via the
# stripped path only so a slow page is still retried without re-serializing the full DOM.
@@ -561,8 +623,9 @@ async def _capture_composition_evidence(
page_evidence_needs_visual_fallback(evidence)
or (evidence.get("schema_empty_page") is True and not has_bounded_page_schema(evidence))
):
- evidence = await _augment_composition_evidence_with_visual_fallback(copilot_ctx, evidence)
- return evidence, None
+ evidence, frame = await _augment_composition_evidence_with_visual_fallback(copilot_ctx, evidence)
+ return CompositionEvidenceCapture(evidence, None, frame)
+ return CompositionEvidenceCapture(evidence, None)
async def _read_run_session_page_evidence(
@@ -570,7 +633,7 @@ async def _read_run_session_page_evidence(
*,
run_session_id: str,
current_url: str,
-) -> tuple[dict[str, Any] | None, str | None, str | None]:
+) -> tuple[dict[str, Any] | None, str | None, str | None, CapturedFrame | None]:
"""The discovery extractor reads ctx.browser_session_id per call, so the rebind targets the run
session and is restored in a finally. The browser layer can substitute a replacement session for a
closed one, so the id the capture actually observed is read back before the restore."""
@@ -579,11 +642,13 @@ async def _read_run_session_page_evidence(
observed_session_id: str | None = run_session_id
observation_error: str | None = None
evidence: dict[str, Any] | None = None
+ frame: CapturedFrame | None = None
try:
- evidence, observation_error = await asyncio.wait_for(
+ capture = await asyncio.wait_for(
_capture_composition_evidence(ctx, inspected_url=current_url, current_url=current_url),
timeout=_POST_RUN_REPAIR_CAPTURE_TIMEOUT_SECONDS,
)
+ evidence, observation_error, frame = _capture_result_parts(capture)
except Exception:
LOG.debug("Post-run run-session page capture failed", exc_info=True)
observation_error = observation_error or "Post-run page capture against the run session failed."
@@ -596,8 +661,8 @@ async def _read_run_session_page_evidence(
if not observed_session_id:
# A mid-capture session create that fails clears the id, and an unknown source id grants
# post-run identity, so an unprovable source has to drop the packet rather than launder it.
- return None, None, observation_error or "Post-run page capture lost its browser session."
- return (evidence if isinstance(evidence, dict) else None), observed_session_id, observation_error
+ return None, None, observation_error or "Post-run page capture lost its browser session.", None
+ return (evidence if isinstance(evidence, dict) else None), observed_session_id, observation_error, frame
def _post_run_page_source_session_id(ctx: CopilotContext, run_id: str | None) -> str | None:
@@ -758,6 +823,7 @@ async def _inspect_page_for_composition_impl(
# handful of structured looks, and the turn's wall clock spent without understanding the page.
# The calls stay counted for telemetry.
evidence = None
+ visual_fallback_frame: CapturedFrame | None = None
observation_error: str | None = None
with copilot_span(
"inspect_page_for_composition",
@@ -767,13 +833,19 @@ async def _inspect_page_for_composition_impl(
# current_page, or a URL target the agent is already on — capture without navigating.
current_url = inspect_target_url or entry_url
if run_page_source_session_id:
- evidence, observed_run_session_id, observation_error = await _read_run_session_page_evidence(
+ (
+ evidence,
+ observed_run_session_id,
+ observation_error,
+ visual_fallback_frame,
+ ) = await _read_run_session_page_evidence(
copilot_ctx, run_session_id=run_page_source_session_id, current_url=current_url
)
else:
- evidence, observation_error = await _capture_composition_evidence(
+ capture = await _capture_composition_evidence(
copilot_ctx, inspected_url=entry_url, current_url=current_url
)
+ evidence, observation_error, visual_fallback_frame = _capture_result_parts(capture)
else:
nav_result = await _discovery_navigate(
copilot_ctx,
@@ -783,12 +855,12 @@ async def _inspect_page_for_composition_impl(
)
if not nav_result.get("ok"):
nav_error = str(nav_result.get("error") or "unknown")
- failure_evidence = await _composition_evidence_after_navigation_failure(
+ failure_capture = await _composition_evidence_after_navigation_failure(
copilot_ctx,
inspected_url=entry_url,
navigation_error=nav_error,
)
- if failure_evidence is None:
+ if failure_capture is None:
result = {
"ok": False,
"data": None,
@@ -796,13 +868,14 @@ async def _inspect_page_for_composition_impl(
}
record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
return result
- evidence = failure_evidence
+ evidence, visual_fallback_frame = failure_capture
current_url = str(evidence.get("current_url") or entry_url)
else:
current_url = _discovery_extract_current_url(nav_result, entry_url)
- evidence, observation_error = await _capture_composition_evidence(
+ capture = await _capture_composition_evidence(
copilot_ctx, inspected_url=entry_url, current_url=current_url
)
+ evidence, observation_error, visual_fallback_frame = _capture_result_parts(capture)
if observation_error is not None:
result = {
@@ -834,6 +907,8 @@ async def _inspect_page_for_composition_impl(
source_browser_session_id=source_browser_session_id,
run_browser_session_id=copilot_ctx.last_run_blocks_browser_session_id,
)
+ if preserved_stored_evidence:
+ visual_fallback_frame = None
# A capture too hollow or too foreign to store is also too weak to move the post-run
# observation marker; leaving the marker put keeps it describing the packet that is stored.
if not preserved_stored_evidence:
@@ -881,4 +956,23 @@ async def _inspect_page_for_composition_impl(
result["observation_step"] = observation_step
result = _model_facing_inspect_result(result)
record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
+ if visual_fallback_frame is not None:
+ workflow_run_id = evidence.get("workflow_run_id")
+ enqueue_screenshot(
+ copilot_ctx,
+ visual_fallback_frame.b64,
+ provenance=ScreenshotProvenance(
+ source_tool="inspect_page_for_composition",
+ captured_url=visual_fallback_frame.captured_url,
+ observation_step=observation_step,
+ browser_session_id=visual_fallback_frame.browser_session_id,
+ workflow_run_id=workflow_run_id if isinstance(workflow_run_id, str) else None,
+ action_relation=ScreenshotActionRelation.SAME_PAGE_OBSERVATION,
+ dispatch_url=visual_fallback_frame.dispatch_url,
+ dispatch_browser_session_id=visual_fallback_frame.dispatch_browser_session_id,
+ producer_browser_session_id=visual_fallback_frame.producer_browser_session_id,
+ session_binding=visual_fallback_frame.session_binding,
+ ),
+ captured_at=visual_fallback_frame.captured_at,
+ )
return result
diff --git a/skyvern/forge/sdk/copilot/tools/credential_fill.py b/skyvern/forge/sdk/copilot/tools/credential_fill.py
index ccc56cbbf..a8d620ac1 100644
--- a/skyvern/forge/sdk/copilot/tools/credential_fill.py
+++ b/skyvern/forge/sdk/copilot/tools/credential_fill.py
@@ -976,7 +976,12 @@ async def _fill_credential_field_impl_serial(
form_submits = await _capture_enclosing_form_submits(copilot_ctx, selector)
if form_submits:
data["form_submit_controls"] = form_submits
- await _capture_post_interaction_screenshot(copilot_ctx)
+ await _capture_post_interaction_screenshot(
+ copilot_ctx,
+ source_tool=observed_tool,
+ captured_url=observed_url,
+ observation_step=observation_step,
+ )
LOG.info(
"copilot fill_credential_field filled a saved credential field",
selector=selector,
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 67d96477b..3a11c6c46 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -562,14 +562,22 @@ async def _navigate_post_hook(
result_url=result["url"],
)
await _bind_login_credential_for_observed_url(ctx, result["url"], result)
- staged = await _capture_post_interaction_screenshot(ctx)
+ staged = await _capture_post_interaction_screenshot(
+ ctx,
+ source_tool="navigate_browser",
+ captured_url=result["url"],
+ )
attached = " A screenshot is attached." if staged else ""
result["next_step"] = (
f"Page loaded.{attached} Use evaluate or inspect_page_for_composition when you need the "
"page's structure or selectors before responding."
)
else:
- await _capture_post_interaction_screenshot(ctx)
+ await _capture_post_interaction_screenshot(
+ ctx,
+ source_tool="navigate_browser",
+ captured_url=source_url,
+ )
return result
@@ -633,6 +641,8 @@ async def _click_post_hook(
ctx.last_scout_act_observe_outcome = None
ctx.last_scout_act_observe_packet = None
page_evidence: dict[str, Any] | None = None
+ captured_url: str | None = None
+ observation_step: int | None = None
_clear_pending_browser_interaction_observation(ctx)
source_url = _consume_scout_source_url(ctx)
pending_role_name = ctx.pending_scout_role_name
@@ -706,6 +716,7 @@ async def _click_post_hook(
if observation_step is not None:
result["observation_step"] = observation_step
result["data"]["observation_step"] = observation_step
+ captured_url = url or None
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
# A download this click produced is proof the affordance works, so it outranks the
# href-shape prediction — and is the only source that sees a command-URL download.
@@ -723,7 +734,12 @@ async def _click_post_hook(
# The round-trip is skipped only when the evidence positively names the obstruction a frame
# would have shown; evidence that merely parsed is not a substitute for looking at the page.
if ctx.last_scout_act_observe_outcome != "attached" or not _page_evidence_names_obstruction(page_evidence):
- await _capture_post_interaction_screenshot(ctx)
+ await _capture_post_interaction_screenshot(
+ ctx,
+ source_tool="click",
+ captured_url=captured_url or source_url,
+ observation_step=observation_step,
+ )
return result
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 4e94d8a06..4a877ff35 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -102,6 +102,11 @@ from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
repair_page_evidence_is_admissible,
same_run_typed_challenge_kind,
)
+from skyvern.forge.sdk.copilot.screenshot_utils import (
+ ScreenshotActionRelation,
+ ScreenshotProvenance,
+ enqueue_screenshot,
+)
from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal
@@ -1500,11 +1505,11 @@ async def _capture_and_store_post_run_page(
) -> None:
"""A failed or hollow capture neutralizes stale evidence to None only when it would not cleanly match
this run_id, so the matcher's destructive clear cannot fire on the pending failure-string context."""
- evidence, observed_session_id, _ = await _read_run_session_page_evidence(
+ evidence, observed_session_id, _, captured_frame = await _read_run_session_page_evidence(
ctx, run_session_id=run_session_id, current_url=current_url
)
if evidence is not None and repair_page_evidence_is_admissible(evidence):
- store_post_run_page_evidence(
+ _, preserved_stored_evidence = store_post_run_page_evidence(
ctx,
evidence,
run_id=run_id,
@@ -1512,6 +1517,24 @@ async def _capture_and_store_post_run_page(
source_browser_session_id=observed_session_id,
run_browser_session_id=run_session_id,
)
+ if captured_frame is not None and not preserved_stored_evidence:
+ enqueue_screenshot(
+ ctx,
+ captured_frame.b64,
+ provenance=ScreenshotProvenance(
+ source_tool="post_run_page_capture",
+ captured_url=captured_frame.captured_url,
+ observation_step=None,
+ browser_session_id=captured_frame.browser_session_id,
+ workflow_run_id=run_id,
+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,
+ dispatch_url=captured_frame.dispatch_url,
+ dispatch_browser_session_id=captured_frame.dispatch_browser_session_id,
+ producer_browser_session_id=captured_frame.producer_browser_session_id,
+ session_binding=captured_frame.session_binding,
+ ),
+ captured_at=captured_frame.captured_at,
+ )
return
if not post_run_inspection_cleanly_matches(ctx.composition_page_evidence, run_id):
ctx.composition_page_evidence = None
@@ -1590,7 +1613,7 @@ async def _capture_dispatched_terminal_page_evidence(
if _pre_run_baseline_is_provenance_valid(ctx.composition_page_evidence):
_pin_pre_run_page_reference(ctx, run_id)
source = "cdp_run_session"
- evidence, source_session_id, _ = await _read_run_session_page_evidence(
+ evidence, source_session_id, _, captured_frame = await _read_run_session_page_evidence(
ctx, run_session_id=run_session_id, current_url=current_url
)
# A capture that landed on a substituted session can still look usable (a blank replacement page
@@ -1602,6 +1625,7 @@ async def _capture_dispatched_terminal_page_evidence(
or not _dispatched_terminal_page_evidence_is_usable(evidence)
):
source = "worker_artifact"
+ captured_frame = None
source_session_id = run_session_id
evidence = await _fetch_dispatched_terminal_page_evidence(
run_id=run_id, organization_id=organization_id, current_url=current_url
@@ -1616,6 +1640,24 @@ async def _capture_dispatched_terminal_page_evidence(
source_browser_session_id=source_session_id,
run_browser_session_id=run_session_id,
)
+ if captured_frame is not None and not preserved_stored_evidence:
+ enqueue_screenshot(
+ ctx,
+ captured_frame.b64,
+ provenance=ScreenshotProvenance(
+ source_tool="dispatched_terminal_page_capture",
+ captured_url=captured_frame.captured_url,
+ observation_step=None,
+ browser_session_id=captured_frame.browser_session_id,
+ workflow_run_id=run_id,
+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,
+ dispatch_url=captured_frame.dispatch_url,
+ dispatch_browser_session_id=captured_frame.dispatch_browser_session_id,
+ producer_browser_session_id=captured_frame.producer_browser_session_id,
+ session_binding=captured_frame.session_binding,
+ ),
+ captured_at=captured_frame.captured_at,
+ )
LOG.info(
"copilot_dispatched_terminal_page_evidence_captured",
workflow_run_id=run_id,
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 34cc4a314..2ef754fb0 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -67,7 +67,12 @@ from skyvern.forge.sdk.copilot.runtime import (
ScoutedSelectorCandidate,
resolve_browser_state_for_context,
)
-from skyvern.forge.sdk.copilot.screenshot_utils import stage_screenshot_from_artifact
+from skyvern.forge.sdk.copilot.screenshot_utils import (
+ ScreenshotActionRelation,
+ ScreenshotProvenance,
+ screenshot_result_facts,
+ stage_screenshot_from_artifact,
+)
from ._shared import (
_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
@@ -519,7 +524,12 @@ async def _capture_element_fingerprint(
async def _capture_post_interaction_screenshot(
- ctx: AgentContext, *, timeout_seconds: float = _DISCOVERY_PER_CALL_TIMEOUT_SECONDS
+ ctx: AgentContext,
+ *,
+ source_tool: str,
+ captured_url: str | None,
+ observation_step: int | None = None,
+ timeout_seconds: float = _DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
) -> bool:
"""Attach a look at the page after a state-changing action, reporting whether a frame staged.
Reading the DOM answers "what is on the page" but not "did that work" -- a filled password reads
@@ -533,16 +543,40 @@ async def _capture_post_interaction_screenshot(
server = getattr(ctx, "discovery_mcp_server", None)
if server is None:
return False
+ capture_started_at = time.monotonic()
+ capture_session_id = ctx.browser_session_id
+ screenshot_arguments = {"session_id": capture_session_id} if capture_session_id else {}
try:
result = await asyncio.wait_for(
- server.call_internal_tool("skyvern_screenshot", {}),
+ server.call_internal_tool("skyvern_screenshot", screenshot_arguments),
timeout=timeout_seconds,
)
except Exception:
return False
if not isinstance(result, dict) or not result.get("ok"):
return False
- return stage_screenshot_from_artifact(ctx, result)
+ producer_url, producer_session_id, session_binding = screenshot_result_facts(
+ result,
+ dispatch_url=captured_url,
+ dispatch_browser_session_id=capture_session_id,
+ )
+ return stage_screenshot_from_artifact(
+ ctx,
+ result,
+ provenance=ScreenshotProvenance(
+ source_tool=source_tool,
+ captured_url=producer_url,
+ observation_step=observation_step,
+ browser_session_id=producer_session_id,
+ workflow_run_id=None,
+ action_relation=ScreenshotActionRelation.AFTER_SOURCE_ACTION,
+ dispatch_url=captured_url,
+ dispatch_browser_session_id=capture_session_id,
+ producer_browser_session_id=producer_session_id,
+ session_binding=session_binding,
+ ),
+ captured_at=capture_started_at,
+ )
async def _capture_enclosing_form_submits(
diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.py
index 7d0747af6..05b7e43c6 100644
--- a/tests/unit/test_copilot_composition_evidence.py
+++ b/tests/unit/test_copilot_composition_evidence.py
@@ -3955,11 +3955,13 @@ async def test_navigation_failure_uses_structured_evidence_when_bounded() -> Non
server = _RecordingCompositionServer(structured_json=json.dumps(_structured_form_payload()))
ctx = SimpleNamespace(discovery_mcp_server=server, browser_session_id=None, organization_id="o_test")
- evidence = await tools_module._composition_evidence_after_navigation_failure(
+ capture = await tools_module._composition_evidence_after_navigation_failure(
ctx, inspected_url="https://example.com/lookup", navigation_error="boom"
)
- assert evidence is not None
+ assert capture is not None
+ evidence, frame = capture
+ assert frame is None
assert evidence["forms"]
assert server.calls.count("skyvern_get_html") == 0
assert any("navigation_error_before_html_capture" in warning for warning in evidence["inspection_warnings"])
@@ -3972,8 +3974,8 @@ async def test_navigation_failure_uses_visual_fallback_after_structured_failure_
server = _RecordingCompositionServer(structured_json=None, html=_HTML_FORM_PAGE)
ctx = SimpleNamespace(discovery_mcp_server=server, browser_session_id=None, organization_id="o_test")
- async def attach_visual(_ctx: Any, evidence: dict[str, Any]) -> dict[str, Any]:
- return {**evidence, "screenshot_used": True, "visual_evidence_summary": "A login form is visible."}
+ async def attach_visual(_ctx: Any, evidence: dict[str, Any]) -> tuple[dict[str, Any], None]:
+ return {**evidence, "screenshot_used": True, "visual_evidence_summary": "A login form is visible."}, None
monkeypatch.setattr(
tools_module.composition_capture,
@@ -3981,13 +3983,15 @@ async def test_navigation_failure_uses_visual_fallback_after_structured_failure_
attach_visual,
)
- evidence = await tools_module._composition_evidence_after_navigation_failure(
+ capture = await tools_module._composition_evidence_after_navigation_failure(
ctx,
inspected_url="https://example.com/login",
navigation_error="https://example.com/callback?token=raw-secret",
)
- assert evidence is not None
+ assert capture is not None
+ evidence, frame = capture
+ assert frame is None
assert evidence["screenshot_used"] is True
assert evidence["visual_evidence_summary"] == "A login form is visible."
assert "raw-secret" not in json.dumps(evidence)
diff --git a/tests/unit/test_copilot_composition_scrape.py b/tests/unit/test_copilot_composition_scrape.py
index 7a6b1f3e9..33652383d 100644
--- a/tests/unit/test_copilot_composition_scrape.py
+++ b/tests/unit/test_copilot_composition_scrape.py
@@ -183,6 +183,9 @@ async def test_unrendered_challenge_keeps_structured_packet_when_relooks_run_out
async def identity(ctx: object, evidence: dict) -> dict:
return evidence
+ async def visual_identity(ctx: object, evidence: dict) -> tuple[dict, None]:
+ return evidence, None
+
# Body-only read: the anti-bot token lives in <title>, so it is absent here by construction.
get_html = AsyncMock(return_value=(_BOUNDED_HTML, None, False, False))
monkeypatch.setattr(tools.composition_capture, "_composition_get_structured_evidence_result", fake_structured)
@@ -190,7 +193,9 @@ async def test_unrendered_challenge_keeps_structured_packet_when_relooks_run_out
monkeypatch.setattr(
tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity
)
- monkeypatch.setattr(tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", identity)
+ monkeypatch.setattr(
+ tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", visual_identity
+ )
settle_sleep = AsyncMock()
monkeypatch.setattr(tools.composition_capture, "asyncio", _AsyncioSleepProxy(settle_sleep))
@@ -267,13 +272,18 @@ async def test_signalled_packet_survives_extractor_blinking_mid_loop(monkeypatch
async def identity(ctx: object, evidence: dict) -> dict:
return evidence
+ async def visual_identity(ctx: object, evidence: dict) -> tuple[dict, None]:
+ return evidence, None
+
get_html = AsyncMock(return_value=(_BOUNDED_HTML, None, False, False))
monkeypatch.setattr(tools.composition_capture, "_composition_get_structured_evidence_result", fake_structured)
monkeypatch.setattr(tools.composition_capture, "_composition_get_html", get_html)
monkeypatch.setattr(
tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity
)
- monkeypatch.setattr(tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", identity)
+ monkeypatch.setattr(
+ tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", visual_identity
+ )
monkeypatch.setattr(tools.composition_capture, "asyncio", _AsyncioSleepProxy(AsyncMock()))
evidence, html_error = await tools._capture_composition_evidence(
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index f30a22063..6bb67ba5a 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -3453,8 +3453,8 @@ async def test_obstruction_only_packet_survives_the_automatic_post_run_capture(
async def _read(
_ctx_arg: CopilotContext, *, run_session_id: str, current_url: str
- ) -> tuple[dict[str, Any], str, None]:
- return packet, run_session_id, None
+ ) -> tuple[dict[str, Any], str, None, None]:
+ return packet, run_session_id, None, None
monkeypatch.setattr(run_execution_module, "_read_run_session_page_evidence", _read)
diff --git a/tests/unit/test_copilot_discovery.py b/tests/unit/test_copilot_discovery.py
index edc271a34..b902081ec 100644
--- a/tests/unit/test_copilot_discovery.py
+++ b/tests/unit/test_copilot_discovery.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import base64
+import hashlib
import json
from types import SimpleNamespace
from typing import Any
@@ -34,6 +36,10 @@ from skyvern.forge.sdk.copilot.tools.discovery import (
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
+_VALID_PNG_B64 = (
+ "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAAE0lEQVR4nGP8z4APMOGVZRip0gBBLAETee26JgAAAABJRU5ErkJggg=="
+)
+
class _Ctx:
def __init__(self, server: object) -> None:
@@ -51,6 +57,8 @@ class _Ctx:
self.last_run_blocks_browser_session_id = None
self.request_policy = None
self.org_credentials_for_turn = None
+ self.supports_vision = True
+ self.pending_screenshots: list[Any] = []
def _dense_structured_page() -> dict[str, Any]:
@@ -356,8 +364,13 @@ class _GenericBarrierServer:
assert arguments["expression"] == COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION
return {"ok": True, "data": {"result": _structured_search_page(with_obstruction=True)}}
if tool_name == "skyvern_screenshot":
- assert arguments == {"inline": True}
- return {"ok": True, "data": {"screenshot_base64": "aGVsbG8="}}
+ expected_session = arguments.get("session_id")
+ assert arguments == {"inline": True, **({"session_id": expected_session} if expected_session else {})}
+ return {
+ "ok": True,
+ "browser_context": {"session_id": expected_session},
+ "data": {"screenshot_base64": _VALID_PNG_B64},
+ }
raise AssertionError(f"unexpected tool: {tool_name}")
@@ -795,7 +808,7 @@ async def test_target_url_inspection_uses_visual_summary_for_generic_obstruction
evidence: dict[str, Any],
screenshot_b64: str,
) -> tuple[dict[str, Any], None]:
- assert screenshot_b64 == "aGVsbG8="
+ assert screenshot_b64 == _VALID_PNG_B64
assert evidence["visual_obstruction_candidates"][0]["coverage"] == "viewport"
return {
"summary": "A checkpoint panel blocks the search form.",
@@ -822,6 +835,8 @@ async def test_target_url_inspection_uses_visual_summary_for_generic_obstruction
assert "skyvern_evaluate" in server.calls
assert "skyvern_screenshot" in server.calls
assert result["data"]["screenshot_used"] is True
+ json.dumps(result["data"])
+ assert all("CapturedFrame" not in repr(value) for value in result["data"].values())
assert result["data"]["page_obstructions"] == [
{
"kind": "checkpoint_panel",
@@ -831,6 +846,74 @@ async def test_target_url_inspection_uses_visual_summary_for_generic_obstruction
"underlying_page_blocked": True,
}
]
+ assert len(ctx.pending_screenshots) == 1
+ frame = ctx.pending_screenshots[0]
+ assert frame.provenance.source_tool == "inspect_page_for_composition"
+ assert frame.provenance.captured_url is None
+ assert frame.provenance.dispatch_url == "https://www.example.com/search"
+ assert frame.provenance.observation_step == result["observation_step"]
+ assert frame.provenance.browser_session_id is None
+ assert frame.provenance.session_binding.value == "unavailable"
+ assert frame.provenance.workflow_run_id is None
+ assert frame.provenance.action_relation.value == "same_page_observation"
+ assert frame.capture_id == f"sha256:{hashlib.sha256(base64.b64decode(_VALID_PNG_B64)).hexdigest()}"
+
+
+async def test_post_run_visual_fallback_binds_the_observed_run_session(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ server = _GenericBarrierServer()
+ ctx = _Ctx(server)
+ ctx.browser_session_id = "pbs_scout"
+ ctx.last_run_blocks_browser_session_id = "pbs_run"
+ ctx.last_run_blocks_workflow_run_id = "wr_123" # type: ignore[attr-defined]
+
+ async def fake_page_info(_ctx: object, session_id: str | None = None) -> tuple[str, str]:
+ assert session_id == "pbs_run"
+ return "https://www.example.com/search", "Search"
+
+ async def fake_visual_summary(
+ _ctx: object,
+ *,
+ evidence: dict[str, Any],
+ screenshot_b64: str,
+ ) -> tuple[dict[str, Any], None]:
+ assert screenshot_b64 == _VALID_PNG_B64
+ return {
+ "summary": "A checkpoint panel blocks the search form.",
+ "challenge_detected": False,
+ "challenge_kind": "",
+ "challenge_location": "",
+ "submit_blocked": False,
+ "blocked_submit_controls": [],
+ "empty_page_visible": False,
+ "loading_state_visible": False,
+ "page_obstruction_detected": True,
+ "obstruction_kind": "checkpoint_panel",
+ "obstruction_location": "Centered over the form.",
+ "underlying_page_blocked": True,
+ "visible_dismiss_controls": ["Continue"],
+ "omissions": [],
+ }, None
+
+ monkeypatch.setattr(tools_module.composition_capture, "_fallback_page_info", fake_page_info)
+ monkeypatch.setattr(tools_module.composition_capture, "_composition_summarize_screenshot", fake_visual_summary)
+
+ result = await _inspect_page_for_composition_impl(ctx, "current_page")
+
+ assert result["data"]["source_browser_session_id"] == "pbs_run"
+ assert result["data"]["workflow_run_id"] == "wr_123"
+ frame = ctx.pending_screenshots[0]
+ assert frame.provenance.captured_url is None
+ assert frame.provenance.dispatch_url == "https://www.example.com/search"
+ assert frame.provenance.observation_step == result["observation_step"]
+ assert frame.provenance.browser_session_id == "pbs_run"
+ assert frame.provenance.dispatch_browser_session_id == "pbs_run"
+ assert frame.provenance.producer_browser_session_id == "pbs_run"
+ assert frame.provenance.session_binding.value == "agree"
+ assert frame.provenance.workflow_run_id == "wr_123"
+ assert ctx.browser_session_id == "pbs_scout"
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index a5d475fce..286275201 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -405,7 +405,7 @@ class TestMCPFailedStepLoopDetection:
monkeypatch.setattr(
mcp_adapter,
"enqueue_screenshot_from_result",
- lambda _ctx, result: screenshots.append(dict(result)),
+ lambda _ctx, result, **_kwargs: screenshots.append(dict(result)),
)
initial_scouted_interactions = [{"tool_name": "click", "selector": "#existing"}]
diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.py
index 70ec4b06b..35eca04b6 100644
--- a/tests/unit/test_copilot_mcp_adapter.py
+++ b/tests/unit/test_copilot_mcp_adapter.py
@@ -197,6 +197,50 @@ def _server_whose_call_takes_time(
)
+async def test_internal_call_preserves_explicit_session_across_session_prepare(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = make_copilot_ctx(browser_session_id="pbs_snapshot")
+ server = _make_server(
+ ctx,
+ {"ok": True},
+ SchemaOverlay(requires_browser=True),
+ alias_map=get_skyvern_mcp_alias_map(),
+ )
+ dispatched: list[dict[str, Any]] = []
+
+ class _CapturingClient:
+ async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> Any:
+ dispatched.append(dict(args))
+ return SimpleNamespace(
+ structured_content={"ok": True},
+ is_error=False,
+ content=[],
+ )
+
+ async def _replace_ambient_session(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None]:
+ _ctx.browser_session_id = "pbs_replacement"
+ return None, None
+
+ @asynccontextmanager
+ async def _scope(_ctx: AgentContext) -> AsyncIterator[None]:
+ yield
+
+ server._client = _CapturingClient()
+ monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _replace_ambient_session)
+ monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
+
+ result = await server.call_internal_tool(
+ "skyvern_evaluate",
+ {"expression": "scan()", "session_id": "pbs_snapshot"},
+ )
+
+ assert result["ok"] is True
+ assert dispatched == [{"expression": "scan()", "session_id": "pbs_snapshot"}]
+ assert ctx.browser_session_id == "pbs_replacement"
+
+
@pytest.mark.usefixtures("_stub_browser_session")
class TestMCPToolTiming:
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_positive_evidence_carriers.py b/tests/unit/test_copilot_positive_evidence_carriers.py
index 4214d7297..235c3c01f 100644
--- a/tests/unit/test_copilot_positive_evidence_carriers.py
+++ b/tests/unit/test_copilot_positive_evidence_carriers.py
@@ -632,8 +632,8 @@ async def test_dispatched_producer_prefers_worker_artifact_over_a_substituted_se
async def fake_read(
inner_ctx: object, *, run_session_id: str, current_url: str
- ) -> tuple[dict[str, object], str, None]:
- return {"observed_empty_page": True, "current_url": current_url}, "pbs_replacement", None
+ ) -> tuple[dict[str, object], str, None, None]:
+ return {"observed_empty_page": True, "current_url": current_url}, "pbs_replacement", None, None
monkeypatch.setattr(run_execution_module, "_read_run_session_page_evidence", fake_read)
diff --git a/tests/unit/test_copilot_screenshot_handling.py b/tests/unit/test_copilot_screenshot_handling.py
index 2b003e4d5..2cdb6495d 100644
--- a/tests/unit/test_copilot_screenshot_handling.py
+++ b/tests/unit/test_copilot_screenshot_handling.py
@@ -16,7 +16,10 @@ from skyvern.forge.sdk.copilot.enforcement import _consume_pending_screenshots
from skyvern.forge.sdk.copilot.screenshot_utils import (
COPILOT_SCREENSHOT_MAX_HEIGHT,
COPILOT_SCREENSHOT_MAX_WIDTH,
+ ScreenshotActionRelation,
ScreenshotEntry,
+ ScreenshotProvenance,
+ enqueue_screenshot,
stage_screenshot_from_artifact,
)
from skyvern.forge.sdk.copilot.session_factory import copilot_call_model_input_filter
@@ -50,6 +53,15 @@ def _install_mock_database(monkeypatch: pytest.MonkeyPatch, mock_db: Any) -> Non
monkeypatch.setattr(run_execution_module, "app", _AppStub())
+def _screenshot_entry(b64: str) -> ScreenshotEntry:
+ return ScreenshotEntry(
+ b64=b64,
+ mime="image/jpeg",
+ capture_id="sha256:test-frame",
+ provenance=ScreenshotProvenance.unknown(source_tool="test_capture"),
+ )
+
+
class TestIsValidPngBase64:
VALID_PNG_B64 = (
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAAElFTkSuQmCC"
@@ -93,11 +105,35 @@ class TestEnqueueScreenshot:
ctx = MagicMock()
ctx.supports_vision = True
ctx.pending_screenshots = []
- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}})
+ provenance = ScreenshotProvenance(
+ source_tool="get_browser_screenshot",
+ captured_url="https://example.com/current",
+ observation_step=3,
+ browser_session_id="pbs_123",
+ workflow_run_id=None,
+ action_relation=ScreenshotActionRelation.TOOL_RESULT,
+ )
+ enqueue_screenshot_from_result(
+ ctx,
+ {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}},
+ provenance=provenance,
+ )
assert len(ctx.pending_screenshots) == 1
entry = ctx.pending_screenshots[0]
assert isinstance(entry, ScreenshotEntry)
assert entry.mime == "image/jpeg"
+ assert entry.capture_id.startswith("sha256:")
+ assert entry.provenance == provenance
+
+ def test_older_capture_cannot_replace_newer_pending_frame(self) -> None:
+ ctx = SimpleNamespace(supports_vision=True, pending_screenshots=[])
+ provenance = ScreenshotProvenance.unknown(source_tool="inspect_page_for_composition")
+
+ assert enqueue_screenshot(ctx, self.VALID_PNG_B64, provenance=provenance, captured_at=20.0) is True
+ newest = ctx.pending_screenshots[0]
+ assert enqueue_screenshot(ctx, self.VALID_PNG_B64, provenance=provenance, captured_at=10.0) is False
+
+ assert ctx.pending_screenshots == [newest]
def test_skips_when_no_vision(self) -> None:
from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
@@ -105,7 +141,11 @@ class TestEnqueueScreenshot:
ctx = MagicMock()
ctx.supports_vision = False
ctx.pending_screenshots = []
- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}})
+ enqueue_screenshot_from_result(
+ ctx,
+ {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}},
+ provenance=ScreenshotProvenance.unknown(source_tool="get_browser_screenshot"),
+ )
assert len(ctx.pending_screenshots) == 0
def test_skips_invalid_image(self) -> None:
@@ -114,7 +154,11 @@ class TestEnqueueScreenshot:
ctx = MagicMock()
ctx.supports_vision = True
ctx.pending_screenshots = []
- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": "not-valid"}})
+ enqueue_screenshot_from_result(
+ ctx,
+ {"ok": True, "data": {"screenshot_base64": "not-valid"}},
+ provenance=ScreenshotProvenance.unknown(source_tool="get_browser_screenshot"),
+ )
assert len(ctx.pending_screenshots) == 0
def test_skips_corrupt_header_valid_image(self) -> None:
@@ -126,7 +170,11 @@ class TestEnqueueScreenshot:
ctx.supports_vision = True
ctx.pending_screenshots = []
truncated_png = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"broken-image-data").decode()
- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": truncated_png + "A" * 100}})
+ enqueue_screenshot_from_result(
+ ctx,
+ {"ok": True, "data": {"screenshot_base64": truncated_png + "A" * 100}},
+ provenance=ScreenshotProvenance.unknown(source_tool="get_browser_screenshot"),
+ )
assert len(ctx.pending_screenshots) == 0
def test_second_enqueue_replaces_first_pending_entry(self) -> None:
@@ -137,15 +185,49 @@ class TestEnqueueScreenshot:
ctx.supports_vision = True
ctx.pending_screenshots = []
- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}})
+ provenance = ScreenshotProvenance.unknown(source_tool="get_browser_screenshot")
+ enqueue_screenshot_from_result(
+ ctx,
+ {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}},
+ provenance=provenance,
+ )
first_entry = ctx.pending_screenshots[0]
- enqueue_screenshot_from_result(ctx, {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}})
+ enqueue_screenshot_from_result(
+ ctx,
+ {"ok": True, "data": {"screenshot_base64": self.VALID_PNG_B64}},
+ provenance=provenance,
+ )
assert len(ctx.pending_screenshots) == 1
assert ctx.pending_screenshots[0] is not first_entry
+def test_run_result_screenshot_provenance_reads_nested_run_facts() -> None:
+ from skyvern.forge.sdk.copilot.tools import _run_result_screenshot_provenance
+
+ provenance = _run_result_screenshot_provenance(
+ {
+ "ok": False,
+ "data": {
+ "current_url": "https://example.com/after-run",
+ "browser_session_id": "pbs_nested",
+ "workflow_run_id": "wr_123",
+ },
+ },
+ source_tool="update_and_run_blocks",
+ )
+
+ assert provenance == ScreenshotProvenance(
+ source_tool="update_and_run_blocks",
+ captured_url="https://example.com/after-run",
+ observation_step=None,
+ browser_session_id="pbs_nested",
+ workflow_run_id="wr_123",
+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,
+ )
+
+
class TestStageScreenshotFromArtifact:
"""The non-inline screenshot tool returns a local artifact path, not image bytes."""
@@ -158,11 +240,15 @@ class TestStageScreenshotFromArtifact:
def _ctx() -> SimpleNamespace:
return SimpleNamespace(supports_vision=True, pending_screenshots=[])
+ @staticmethod
+ def _provenance() -> ScreenshotProvenance:
+ return ScreenshotProvenance.unknown(source_tool="click")
+
def test_stages_the_artifact_the_path_names(self, tmp_path: Path) -> None:
ctx = self._ctx()
result = {"ok": True, "data": {"path": self._png(tmp_path / "frame.png", (400, 300))}}
- assert stage_screenshot_from_artifact(ctx, result) is True
+ assert stage_screenshot_from_artifact(ctx, result, provenance=self._provenance()) is True
assert len(ctx.pending_screenshots) == 1
assert ctx.pending_screenshots[0].mime == "image/jpeg"
@@ -170,7 +256,7 @@ class TestStageScreenshotFromArtifact:
ctx = self._ctx()
result = {"ok": True, "data": {"path": self._png(tmp_path / "big.png", (2400, 1800))}}
- assert stage_screenshot_from_artifact(ctx, result) is True
+ assert stage_screenshot_from_artifact(ctx, result, provenance=self._provenance()) is True
width, height = Image.open(io.BytesIO(base64.b64decode(ctx.pending_screenshots[0].b64))).size
assert width <= COPILOT_SCREENSHOT_MAX_WIDTH
assert height <= COPILOT_SCREENSHOT_MAX_HEIGHT
@@ -186,17 +272,31 @@ class TestStageScreenshotFromArtifact:
def test_unresolvable_artifact_is_no_frame_not_an_error(self, data: dict[str, Any]) -> None:
ctx = self._ctx()
- assert stage_screenshot_from_artifact(ctx, {"ok": True, "data": data}) is False
+ assert (
+ stage_screenshot_from_artifact(
+ ctx,
+ {"ok": True, "data": data},
+ provenance=self._provenance(),
+ )
+ is False
+ )
assert ctx.pending_screenshots == []
def test_failed_capture_over_a_stale_entry_does_not_report_staged(self, tmp_path: Path) -> None:
"""A queue-length check would call this staged: the earlier frame is still pending."""
- stale = ScreenshotEntry(b64="stale", mime="image/jpeg")
+ stale = _screenshot_entry("stale")
ctx = SimpleNamespace(supports_vision=True, pending_screenshots=[stale])
corrupt = tmp_path / "corrupt.png"
corrupt.write_bytes(b"\x89PNG\r\n\x1a\n" + b"not-an-image" * 40)
- assert stage_screenshot_from_artifact(ctx, {"ok": True, "data": {"path": str(corrupt)}}) is False
+ assert (
+ stage_screenshot_from_artifact(
+ ctx,
+ {"ok": True, "data": {"path": str(corrupt)}},
+ provenance=self._provenance(),
+ )
+ is False
+ )
assert ctx.pending_screenshots == [stale]
@@ -204,8 +304,8 @@ class TestCapturePostInteractionScreenshot:
@staticmethod
def _server(result: dict[str, Any]) -> SimpleNamespace:
async def call_internal_tool(name: str, arguments: dict[str, Any]) -> dict[str, Any]:
- assert arguments == {}, "the copilot capture path must not request an inline frame"
- return result
+ assert arguments == {"session_id": "pbs_123"}, "capture must dispatch to the snapshotted session"
+ return {**result, "browser_context": {"session_id": "pbs_123"}}
return SimpleNamespace(call_internal_tool=call_internal_tool)
@@ -215,6 +315,7 @@ class TestCapturePostInteractionScreenshot:
"supports_vision": True,
"pending_screenshots": [],
"discovery_mcp_server": self._server(result),
+ "browser_session_id": "pbs_123",
}
base.update(overrides)
return SimpleNamespace(**base)
@@ -225,8 +326,50 @@ class TestCapturePostInteractionScreenshot:
Image.new("RGB", (300, 200), (0, 0, 0)).save(frame, format="PNG")
ctx = self._ctx({"ok": True, "data": {"path": str(frame)}})
- assert await _capture_post_interaction_screenshot(ctx) is True
+ assert (
+ await _capture_post_interaction_screenshot(
+ ctx,
+ source_tool="click",
+ captured_url="https://example.com/results",
+ observation_step=2,
+ )
+ is True
+ )
assert len(ctx.pending_screenshots) == 1
+ provenance = ctx.pending_screenshots[0].provenance
+ assert provenance.observation_step == 2
+ assert provenance.captured_url is None
+ assert provenance.dispatch_url == "https://example.com/results"
+ assert provenance.dispatch_browser_session_id == "pbs_123"
+ assert provenance.producer_browser_session_id == "pbs_123"
+ assert provenance.session_binding.value == "agree"
+
+ @pytest.mark.asyncio
+ async def test_context_mutation_during_capture_does_not_relabel_dispatch(self, tmp_path: Path) -> None:
+ frame = tmp_path / "frame.png"
+ Image.new("RGB", (300, 200), (0, 0, 0)).save(frame, format="PNG")
+ ctx = self._ctx({"ok": True, "data": {"path": str(frame)}})
+
+ async def call_internal_tool(name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+ assert arguments == {"session_id": "pbs_123"}
+ ctx.browser_session_id = "pbs_replacement"
+ return {
+ "ok": True,
+ "browser_context": {"session_id": "pbs_producer"},
+ "data": {"path": str(frame)},
+ }
+
+ ctx.discovery_mcp_server = SimpleNamespace(call_internal_tool=call_internal_tool)
+ assert await _capture_post_interaction_screenshot(
+ ctx,
+ source_tool="click",
+ captured_url="https://example.com/results",
+ )
+ provenance = ctx.pending_screenshots[0].provenance
+ assert provenance.dispatch_browser_session_id == "pbs_123"
+ assert provenance.producer_browser_session_id == "pbs_producer"
+ assert provenance.browser_session_id == "pbs_producer"
+ assert provenance.session_binding.value == "disagree"
@pytest.mark.parametrize(
"overrides",
@@ -242,14 +385,28 @@ class TestCapturePostInteractionScreenshot:
Image.new("RGB", (300, 200), (0, 0, 0)).save(frame, format="PNG")
ctx = self._ctx({"ok": True, "data": {"path": str(frame)}}, **overrides)
- assert await _capture_post_interaction_screenshot(ctx) is False
+ assert (
+ await _capture_post_interaction_screenshot(
+ ctx,
+ source_tool="click",
+ captured_url="https://example.com/results",
+ )
+ is False
+ )
assert ctx.pending_screenshots == []
@pytest.mark.asyncio
async def test_failed_tool_call_stages_nothing(self) -> None:
ctx = self._ctx({"ok": False, "error": "no page"})
- assert await _capture_post_interaction_screenshot(ctx) is False
+ assert (
+ await _capture_post_interaction_screenshot(
+ ctx,
+ source_tool="click",
+ captured_url=None,
+ )
+ is False
+ )
assert ctx.pending_screenshots == []
@@ -265,7 +422,19 @@ class TestConsumePendingScreenshots:
from skyvern.forge.sdk.copilot.enforcement import SCREENSHOT_SENTINEL, _consume_pending_screenshots
from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
- entry = ScreenshotEntry(b64="dGVzdA==", mime="image/jpeg")
+ entry = ScreenshotEntry(
+ b64="dGVzdA==",
+ mime="image/jpeg",
+ capture_id="sha256:frame-123",
+ provenance=ScreenshotProvenance(
+ source_tool="inspect_page_for_composition",
+ captured_url="https://example.com/results",
+ observation_step=4,
+ browser_session_id="pbs_123",
+ workflow_run_id="wr_123",
+ action_relation=ScreenshotActionRelation.SAME_PAGE_OBSERVATION,
+ ),
+ )
ctx = MagicMock()
ctx.pending_screenshots = [entry]
msg = _consume_pending_screenshots(ctx)
@@ -275,6 +444,13 @@ class TestConsumePendingScreenshots:
assert len(content) == 2
assert content[0]["type"] == "input_text"
assert content[0]["text"].startswith(SCREENSHOT_SENTINEL)
+ assert "capture_id=sha256:frame-123" in content[0]["text"]
+ assert "captured_url=https://example.com/results" in content[0]["text"]
+ assert "observation_step=4" in content[0]["text"]
+ assert "browser_session_id=pbs_123" in content[0]["text"]
+ assert "workflow_run_id=wr_123" in content[0]["text"]
+ assert "action_relation=same_page_observation" in content[0]["text"]
+ assert "may predate later actions" not in content[0]["text"]
assert content[1]["type"] == "input_image"
assert "image/jpeg" in content[1]["image_url"]
assert content[1]["detail"] == "high"
@@ -283,10 +459,9 @@ class TestConsumePendingScreenshots:
def test_handles_multiple_screenshots(self) -> None:
from skyvern.forge.sdk.copilot.enforcement import _consume_pending_screenshots
- from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
- entry1 = ScreenshotEntry(b64="abc=", mime="image/jpeg")
- entry2 = ScreenshotEntry(b64="def=", mime="image/jpeg")
+ entry1 = _screenshot_entry("abc=")
+ entry2 = _screenshot_entry("def=")
ctx = MagicMock()
ctx.pending_screenshots = [entry1, entry2]
msg = _consume_pending_screenshots(ctx)
@@ -305,7 +480,7 @@ class TestConsumePendingScreenshots:
class TestNudgeDrainAndFilterExclusivity:
def test_nudge_delivery_binds_the_frame_and_the_next_filter_pass_adds_nothing(self) -> None:
ctx = SimpleNamespace(
- pending_screenshots=[ScreenshotEntry(b64="dGVzdA==", mime="image/jpeg")],
+ pending_screenshots=[_screenshot_entry("dGVzdA==")],
supports_vision=True,
)
@@ -324,7 +499,7 @@ class TestNudgeDrainAndFilterExclusivity:
# The vision check lives in the shared builder so every delivery path inherits it;
# the drain still empties the queue.
ctx = SimpleNamespace(
- pending_screenshots=[ScreenshotEntry(b64="dGVzdA==", mime="image/jpeg")],
+ pending_screenshots=[_screenshot_entry("dGVzdA==")],
supports_vision=False,
)
@@ -1080,7 +1255,18 @@ class TestRunScreenshotResolution:
ctx = MagicMock()
ctx.supports_vision = True
ctx.pending_screenshots = []
- enqueue_screenshot_from_result(ctx, result)
+ enqueue_screenshot_from_result(
+ ctx,
+ result,
+ provenance=ScreenshotProvenance(
+ source_tool="run_blocks_and_collect_debug",
+ captured_url="https://example.com/failure",
+ observation_step=None,
+ browser_session_id="pbs_123",
+ workflow_run_id="wr_123",
+ action_relation=ScreenshotActionRelation.WORKFLOW_RUN_RESULT,
+ ),
+ )
assert len(ctx.pending_screenshots) == 1
diff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.py
index 59641fa43..cf87c30b8 100644
--- a/tests/unit/test_copilot_session_factory.py
+++ b/tests/unit/test_copilot_session_factory.py
@@ -10,7 +10,11 @@ from typing import Any
import pytest
from structlog.testing import capture_logs
-from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
+from skyvern.forge.sdk.copilot.screenshot_utils import (
+ ScreenshotActionRelation,
+ ScreenshotEntry,
+ ScreenshotProvenance,
+)
from skyvern.forge.sdk.copilot.session_factory import (
copilot_call_model_input_filter,
make_copilot_call_model_input_filter,
@@ -27,8 +31,23 @@ def _image_parts(item: Any) -> list[Any]:
def _ctx_with_staged_frame(*, supports_vision: bool = True) -> SimpleNamespace:
return SimpleNamespace(
- pending_screenshots=[ScreenshotEntry(b64="dGVzdA==", mime="image/jpeg")],
+ pending_screenshots=[
+ ScreenshotEntry(
+ b64="dGVzdA==",
+ mime="image/jpeg",
+ capture_id="sha256:frame-123",
+ provenance=ScreenshotProvenance(
+ source_tool="inspect_page_for_composition",
+ captured_url="https://example.com/results",
+ observation_step=2,
+ browser_session_id="pbs_123",
+ workflow_run_id="wr_123",
+ action_relation=ScreenshotActionRelation.SAME_PAGE_OBSERVATION,
+ ),
+ )
+ ],
supports_vision=supports_vision,
+ pending_frame_lease=None,
)
@@ -171,6 +190,44 @@ class TestSessionInputCallback:
new_items = [{"role": "user", "content": "hello"}]
assert copilot_session_input_callback([], new_items) == new_items
+ def test_materialized_paired_frame_is_placeholdered_after_tool_output(self) -> None:
+ from skyvern.forge.sdk.copilot.enforcement import SCREENSHOT_PLACEHOLDER
+ from skyvern.forge.sdk.copilot.session_factory import copilot_session_input_callback
+
+ goal = {"role": "user", "content": "build a workflow"}
+ paired = {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_text",
+ "text": "[copilot:screenshot] [copilot:paired-observation] Frame provenance",
+ },
+ {"type": "input_image", "image_url": "data:image/jpeg;base64,dGVzdA=="},
+ ],
+ }
+ output = {"type": "function_call_output", "call_id": "click-1", "output": '{"ok":true}'}
+
+ combined = copilot_session_input_callback([goal, paired, output], [])
+
+ assert combined[1] == {"role": "user", "content": SCREENSHOT_PLACEHOLDER}
+
+ def test_unmarked_generic_frame_keeps_existing_retention_behavior(self) -> None:
+ from skyvern.forge.sdk.copilot.session_factory import copilot_session_input_callback
+
+ goal = {"role": "user", "content": "build a workflow"}
+ generic = {
+ "role": "user",
+ "content": [
+ {"type": "input_text", "text": "[copilot:screenshot] generic tool frame"},
+ {"type": "input_image", "image_url": "data:image/jpeg;base64,dGVzdA=="},
+ ],
+ }
+ output = {"type": "function_call_output", "call_id": "read-1", "output": '{"ok":true}'}
+
+ combined = copilot_session_input_callback([goal, generic, output], [])
+
+ assert combined[1] == generic
+
def test_preserves_original_goal_and_applies_compaction_to_middle(self) -> None:
"""First-turn shape (one real user, several tool iterations): the
goal at index 0 is preserved; older function_call_output items get
@@ -303,6 +360,32 @@ class TestStagedScreenshotBinding:
assert [len(_image_parts(item)) for item in result.input] == [0, 1]
assert any(log["event"] == "Injecting screenshot user message" for log in logs)
+ def test_linked_structural_result_precedes_the_frame_and_its_provenance(self) -> None:
+ ctx = _ctx_with_staged_frame()
+ items = [
+ {"role": "user", "content": "inspect this page"},
+ {
+ "type": "function_call_output",
+ "call_id": "inspect-1",
+ "output": json.dumps(
+ {
+ "ok": True,
+ "current_url": "https://example.com/results",
+ "observation_step": 2,
+ "data": {"source_tool": "inspect_page_for_composition"},
+ }
+ ),
+ },
+ ]
+
+ result = copilot_call_model_input_filter(_mk_input_data(items, context=ctx))
+
+ assert result.input[-2] == items[-1]
+ assert len(_image_parts(result.input[-1])) == 1
+ provenance_text = result.input[-1]["content"][0]["text"]
+ assert "source_tool=inspect_page_for_composition" in provenance_text
+ assert "observation_step=2" in provenance_text
+
def test_a_second_pass_over_the_same_context_still_carries_the_frame(self) -> None:
ctx = _ctx_with_staged_frame()
items = [{"role": "user", "content": "clear the modal on this page"}]
@@ -314,6 +397,63 @@ class TestStagedScreenshotBinding:
assert len(_image_parts(second.input[-1])) == 1
assert len(ctx.pending_screenshots) == 1
+ def test_paired_frame_is_not_redelivered_after_model_input_advances(self) -> None:
+ ctx = _ctx_with_staged_frame()
+ initial = [{"role": "user", "content": "clear the modal on this page"}]
+
+ first = copilot_call_model_input_filter(_mk_input_data(initial, context=ctx))
+ advanced = [
+ *initial,
+ {"type": "function_call", "call_id": "click-1", "name": "click", "arguments": "{}"},
+ {"type": "function_call_output", "call_id": "click-1", "output": '{"ok":true}'},
+ ]
+ second = copilot_call_model_input_filter(_mk_input_data(advanced, context=ctx))
+
+ assert len(_image_parts(first.input[-1])) == 1
+ assert not any(_image_parts(item) for item in second.input)
+ assert ctx.pending_screenshots == []
+ assert ctx.pending_frame_lease is None
+
+ def test_new_paired_capture_gets_a_fresh_lease_after_invalidation(self) -> None:
+ ctx = _ctx_with_staged_frame()
+ initial = [{"role": "user", "content": "clear the modal on this page"}]
+ copilot_call_model_input_filter(_mk_input_data(initial, context=ctx))
+ copilot_call_model_input_filter(
+ _mk_input_data([*initial, {"role": "assistant", "content": "acted"}], context=ctx)
+ )
+ ctx.pending_screenshots = _ctx_with_staged_frame().pending_screenshots
+
+ delivered = copilot_call_model_input_filter(
+ _mk_input_data([*initial, {"role": "assistant", "content": "acted"}], context=ctx)
+ )
+
+ assert len(_image_parts(delivered.input[-1])) == 1
+ assert ctx.pending_frame_lease.capture_id == "sha256:frame-123"
+
+ def test_same_pixels_from_a_new_capture_event_get_a_fresh_lease(self) -> None:
+ ctx = _ctx_with_staged_frame()
+ initial = [{"role": "user", "content": "inspect this page"}]
+ copilot_call_model_input_filter(_mk_input_data(initial, context=ctx))
+ first_event_id = ctx.pending_frame_lease.capture_event_id
+
+ recaptured = _ctx_with_staged_frame().pending_screenshots[0]
+ assert recaptured.capture_id == ctx.pending_screenshots[0].capture_id
+ assert recaptured.capture_event_id != first_event_id
+ ctx.pending_screenshots = [recaptured]
+ advanced = [*initial, {"role": "assistant", "content": "the page was inspected again"}]
+
+ delivered = copilot_call_model_input_filter(_mk_input_data(advanced, context=ctx))
+
+ assert len(_image_parts(delivered.input[-1])) == 1
+ assert ctx.pending_frame_lease.capture_event_id == recaptured.capture_event_id
+
+ def test_paired_message_has_machine_readable_marker(self) -> None:
+ ctx = _ctx_with_staged_frame()
+
+ result = copilot_call_model_input_filter(_mk_input_data([{"role": "user", "content": "inspect"}], context=ctx))
+
+ assert result.input[-1]["content"][0]["text"].startswith("[copilot:screenshot] [copilot:paired-observation] ")
+
def test_a_non_vision_fallback_model_gets_no_image(self) -> None:
# The frame was staged while the primary was still vision-capable; a retriable failure
# can swap in a fallback that cannot accept images.