Skip to content

Automate browser based workflows with AI

Attach at-failure screenshot and final URL to failed code blocks (SKY-13250) (#7943)

Andrew Neilsonccc2ee5d02ba
diff --git a/alembic/versions/2026_08_01_0519-04647fa4df41_add_workflow_run_blocks_final_url.py b/alembic/versions/2026_08_01_0519-04647fa4df41_add_workflow_run_blocks_final_url.py
new file mode 100644
index 000000000..b40ee571a
--- /dev/null
+++ b/alembic/versions/2026_08_01_0519-04647fa4df41_add_workflow_run_blocks_final_url.py
@@ -0,0 +1,29 @@
+"""add workflow_run_blocks.final_url
+
+Revision ID: 04647fa4df41
+Revises: f3a7c1d9e482
+Create Date: 2026-08-01T05:19:15.224002+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "04647fa4df41"
+down_revision: Union[str, None] = "f3a7c1d9e482"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.execute("SET lock_timeout = '5s'")
+ op.add_column("workflow_run_blocks", sa.Column("final_url", sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+ op.execute("SET lock_timeout = '5s'")
+ op.drop_column("workflow_run_blocks", "final_url")
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 294a98f45..992f17785 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -28608,6 +28608,17 @@
],
"title": "Url"
},
+ "final_url": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Final Url"
+ },
"navigation_goal": {
"anyOf": [
{
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index f2881df7e..a39ba4268 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -274,10 +274,12 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict
else block
for block in blocks
]
- if tool_name == "get_run_results":
- # _attach_failed_block_screenshots puts base64 bytes on each failed
- # block. They would otherwise flow straight into the LLM context as
- # raw image data — strip them while preserving the existence signal.
+ if tool_name in {"get_run_results", "run_blocks_and_collect_debug"}:
+ # _attach_failed_block_screenshots puts base64 bytes on each failed block. They would
+ # otherwise flow straight into the LLM context as raw image data — strip them while
+ # preserving the existence signal. The image itself reaches the model through
+ # data.screenshot_base64; leaving the bytes here also crowds out later fields such as
+ # final_url under downstream truncation.
blocks = data.get("blocks")
if isinstance(blocks, list):
data["blocks"] = [
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index affe23e98..c4dca3f06 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -483,27 +483,94 @@ async def _fetch_last_screenshot_b64(task_id: str, organization_id: str) -> str
return None
+async def _fetch_run_block_screenshot_b64(workflow_run_block_id: str, organization_id: str) -> str | None:
+ try:
+ artifact = await app.DATABASE.artifacts.get_artifact_by_entity_id(
+ artifact_type=ArtifactType.SCREENSHOT_LLM,
+ organization_id=organization_id,
+ workflow_run_block_id=workflow_run_block_id,
+ )
+ if not artifact:
+ return None
+ artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)
+ if not artifact_bytes:
+ return None
+ return base64.b64encode(artifact_bytes).decode("utf-8")
+ except Exception:
+ LOG.debug(
+ "Failed to fetch run-block screenshot for failed block",
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+ return None
+
+
+async def _fetch_failed_block_screenshot_b64(block: Any, organization_id: str) -> str | None:
+ """Resolve a failed block's at-failure screenshot.
+
+ Code blocks persist theirs on the workflow_run_block, so read that first. The task_v2
+ lookup filters on observer_cruise_id — it can never match a code block — and is kept only
+ so runs whose screenshots resolved through it before keep resolving.
+ """
+ if block.workflow_run_block_id:
+ b64 = await _fetch_run_block_screenshot_b64(block.workflow_run_block_id, organization_id)
+ if b64 is not None:
+ return b64
+ if block.task_id:
+ return await _fetch_last_screenshot_b64(block.task_id, organization_id)
+ return None
+
+
async def _attach_failed_block_screenshots(
blocks: list,
results: list[dict[str, Any]],
organization_id: str,
) -> None:
- """For failed blocks with a task_id, fetch the last SCREENSHOT_LLM artifact."""
- task_id_to_block: dict[str, dict] = {
- block.task_id: block_result
+ """Attach the at-failure screenshot and final URL to every failed block that has them."""
+ failed = [
+ (block, block_result)
for block, block_result in zip(blocks, results)
- if block.task_id and block_result.get("status") in _FAILED_BLOCK_STATUSES
- }
- if not task_id_to_block:
+ if block_result.get("status") in _FAILED_BLOCK_STATUSES
+ ]
+ if not failed:
return
- task_ids = list(task_id_to_block.keys())
screenshots = await asyncio.gather(
- *(_fetch_last_screenshot_b64(task_id, organization_id) for task_id in task_ids),
+ *(_fetch_failed_block_screenshot_b64(block, organization_id) for block, _ in failed),
)
- for task_id, b64 in zip(task_ids, screenshots):
+ for (block, block_result), b64 in zip(failed, screenshots):
if b64 is not None:
- task_id_to_block[task_id]["screenshot_b64"] = b64
+ block_result["screenshot_b64"] = b64
+ if block.final_url:
+ block_result["final_url"] = block.final_url
+ if b64 is not None or block.final_url:
+ LOG.info(
+ "Attached at-failure evidence to failed block",
+ workflow_run_block_id=block.workflow_run_block_id,
+ block_type=block_result.get("block_type"),
+ has_screenshot=b64 is not None,
+ final_url=block.final_url,
+ )
+
+
+def _resolve_run_screenshot_b64(
+ *,
+ live_capture: str | None,
+ results: list[dict[str, Any]],
+ run_ok: bool,
+) -> str | None:
+ """Pick the screenshot the model sees for this run.
+
+ Only data.screenshot_base64 becomes a model-visible image, and the live capture that fills
+ it is skipped on dispatched runs — so a failed run falls back to its first failed block's
+ at-failure screenshot. A successful run never promotes one: a healed or continue_on_failure
+ block must not put a failure image in front of the model.
+ """
+ if live_capture is not None:
+ return live_capture
+ if run_ok:
+ return None
+ return next((r["screenshot_b64"] for r in results if r.get("screenshot_b64")), None)
# Block types that establish browser state (loaded page / authenticated
@@ -2066,6 +2133,7 @@ async def _run_blocks_and_collect_debug(
ctx.last_run_blocks_block_labels = list(dict.fromkeys(block.label for block in run_block_rows if block.label))
await _attach_action_traces(blocks, results, ctx.organization_id)
+ await _attach_failed_block_screenshots(blocks, results, ctx.organization_id)
# final_status is guaranteed set here: every non-success exit returns
# above, and the success path always populates final_status.
@@ -2187,6 +2255,7 @@ async def _run_blocks_and_collect_debug(
result_data["runtime_frontier_anchor_url"] = runtime_frontier_anchor_url
if runtime_frontier_starter_url_seeded:
result_data["runtime_frontier_starter_url_seeded"] = True
+ screenshot_b64 = _resolve_run_screenshot_b64(live_capture=screenshot_b64, results=results, run_ok=run_ok)
if screenshot_b64 is not None:
result_data["screenshot_base64"] = screenshot_b64
if not run_ok and run and getattr(run, "failure_reason", None):
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 27c928b99..1b19fb81a 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -1164,6 +1164,9 @@ class WorkflowRunBlockModel(Base):
output = Column(JSON, nullable=True)
continue_on_failure = Column(Boolean, nullable=False, default=False)
failure_reason = Column(String, nullable=True)
+ # Page URL at the moment the block failed. Distinct from the task's target url, which is
+ # where the block was aimed rather than where it ended up.
+ final_url = Column(String, nullable=True)
error_codes = Column(JSON, nullable=True)
engine = Column(String, nullable=True)
diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.py
index 354581213..7843a97d1 100644
--- a/skyvern/forge/sdk/db/repositories/observer.py
+++ b/skyvern/forge/sdk/db/repositories/observer.py
@@ -469,6 +469,7 @@ class ObserverRepository(BaseRepository):
status: BlockStatus | None = None,
output: dict | list | str | None = None,
failure_reason: str | None = None,
+ final_url: str | None = None,
task_id: str | None = None,
loop_values: list | None = None,
current_value: str | None = None,
@@ -528,6 +529,8 @@ class ObserverRepository(BaseRepository):
workflow_run_block.task_id = task_id
if failure_reason:
workflow_run_block.failure_reason = failure_reason
+ if final_url:
+ workflow_run_block.final_url = final_url
# Use `is not None` instead of truthiness checks so that falsy
# values like current_index=0, empty loop_values=[], or
# current_value="" are correctly persisted. Without this,
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index e512e7479..357a2d9f2 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -842,6 +842,7 @@ def convert_to_workflow_run_block(
output=workflow_run_block_model.output,
continue_on_failure=workflow_run_block_model.continue_on_failure,
failure_reason=workflow_run_block_model.failure_reason,
+ final_url=workflow_run_block_model.final_url,
error_codes=workflow_run_block_model.error_codes or [],
engine=workflow_run_block_model.engine,
task_id=workflow_run_block_model.task_id,
diff --git a/skyvern/forge/sdk/schemas/workflow_runs.py b/skyvern/forge/sdk/schemas/workflow_runs.py
index d89ad2b00..e240f3c52 100644
--- a/skyvern/forge/sdk/schemas/workflow_runs.py
+++ b/skyvern/forge/sdk/schemas/workflow_runs.py
@@ -29,6 +29,7 @@ class WorkflowRunBlock(BaseModel):
engine: RunEngine | None = None
task_id: str | None = None
url: str | None = None
+ final_url: str | None = None
navigation_goal: str | None = None
navigation_payload: dict[str, Any] | list | str | None = None
data_extraction_goal: str | None = None
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 01212c239..e97092d45 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -5037,6 +5037,104 @@ async def wrapper({default_args}):
exc_info=True,
)
+ async def _capture_failure_evidence(
+ self,
+ *,
+ workflow_run_context: WorkflowRunContext,
+ workflow_run_id: str,
+ workflow_run_block_id: str,
+ organization_id: str | None,
+ browser_state: BrowserState | None,
+ page: Page | None,
+ ) -> None:
+ """Persist the page the block died on, so the copilot repair loop can see it.
+
+ Entirely best-effort: this runs inside the failure path, so it must never raise and
+ never change the block outcome.
+ """
+ if not organization_id:
+ return
+ live_browser_state = browser_state or app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id=workflow_run_id)
+ if not live_browser_state:
+ return
+ try:
+ screenshot = await live_browser_state.take_fullpage_screenshot()
+ except Exception:
+ LOG.warning(
+ "Failed to capture the at-failure screenshot for a code block",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+ screenshot = None
+ final_url: str | None = None
+ try:
+ failure_page = page or await live_browser_state.get_or_create_page()
+ # A failing login/MFA step can leave a credential or a generated OTP in the query
+ # string, and this URL is persisted and logged; mask before it leaves the page.
+ final_url = workflow_run_context.mask_secrets_in_data(failure_page.url) if failure_page else None
+ except Exception:
+ LOG.warning(
+ "Failed to read the at-failure URL for a code block",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+
+ try:
+ workflow_run_block = await app.DATABASE.observer.update_workflow_run_block(
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ final_url=final_url,
+ )
+ if screenshot:
+ await app.ARTIFACT_MANAGER.create_workflow_run_block_artifact(
+ workflow_run_block=workflow_run_block,
+ artifact_type=ArtifactType.SCREENSHOT_LLM,
+ data=screenshot,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to persist the at-failure evidence for a code block",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+
+ async def _failed_result_with_evidence(
+ self,
+ *,
+ failure_reason: str,
+ status: BlockStatus,
+ workflow_run_context: WorkflowRunContext,
+ workflow_run_id: str,
+ workflow_run_block_id: str,
+ organization_id: str | None,
+ browser_state: BrowserState | None,
+ page: Page | None,
+ ) -> BlockResult:
+ """Fail a code block after recording the page it died on.
+
+ Every non-healable exit routes through here so a new one inherits the capture instead of
+ silently reopening the blindness this closes.
+ """
+ await self._capture_failure_evidence(
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ browser_state=browser_state,
+ page=page,
+ )
+ return await self.build_block_result(
+ success=False,
+ failure_reason=failure_reason,
+ output_parameter_value=None,
+ status=status,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+
async def _resolve_failure_with_heal(
self,
*,
@@ -5053,6 +5151,17 @@ async def wrapper({default_args}):
browser_state: BrowserState | None = None,
page: Page | None = None,
) -> BlockResult:
+ # Capture before the healable branch: a non-healable failure is exactly the case the
+ # repair loop has the least to go on.
+ await self._capture_failure_evidence(
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ browser_state=browser_state,
+ page=page,
+ )
+
async def _finalize_heal_result(result: BlockResult | None) -> BlockResult:
if result is not None:
output_obligation = self._output_obligation_for_heal_result(result)
@@ -5596,17 +5705,29 @@ async def wrapper({default_args}):
block_label=self.label,
)
await recorder.finalize(success=False)
- return await self.build_block_result(
- success=False,
+ return await self._failed_result_with_evidence(
failure_reason="Secure code block runner returned no result",
- output_parameter_value=None,
status=BlockStatus.failed,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
+ browser_state=browser_state,
+ page=page,
)
# failure=None does not imply success: infra arms (no browser/page, runner raise,
# invalid output) return a failed block_result with no healable failure metadata.
if not secure_code_block_result.block_result.success:
+ # No healable metadata means the repair loop has even less to go on here
+ # than on the healable path, so the page evidence matters more, not less.
+ await self._capture_failure_evidence(
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ browser_state=browser_state,
+ page=page,
+ )
await recorder.finalize(success=False)
return secure_code_block_result.block_result
await recorder.finalize(success=True)
@@ -5651,27 +5772,31 @@ async def wrapper({default_args}):
except InsecureCodeDetected as e:
await recorder.persist(recorder.recorded_actions())
await recorder.finalize(success=False)
- return await self.build_block_result(
- success=False,
+ return await self._failed_result_with_evidence(
failure_reason=str(e),
- output_parameter_value=None,
status=BlockStatus.failed,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
+ browser_state=browser_state,
+ page=page,
)
except asyncio.TimeoutError:
await recorder.persist(recorder.recorded_actions())
await recorder.finalize(success=False)
- return await self.build_block_result(
- success=False,
+ return await self._failed_result_with_evidence(
failure_reason=(
"Failed to execute code block. Reason: TimeoutError: code block exceeded "
f"{settings.CODE_BLOCK_EXECUTION_TIMEOUT_SECONDS} seconds"
),
- output_parameter_value=None,
status=BlockStatus.failed,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
+ browser_state=browser_state,
+ page=page,
)
except Exception as e:
# Exact type, not isinstance: the IllegitCompleteScriptTermination subclass means the
@@ -5679,13 +5804,15 @@ async def wrapper({default_args}):
if type(e) is ScriptTerminationException:
await recorder.persist(recorder.recorded_actions())
await recorder.finalize(success=False)
- return await self.build_block_result(
- success=False,
+ return await self._failed_result_with_evidence(
failure_reason=workflow_run_context.mask_secrets_in_data(str(e)),
- output_parameter_value=None,
status=BlockStatus.terminated,
+ workflow_run_context=workflow_run_context,
+ workflow_run_id=workflow_run_id,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
+ browser_state=browser_state,
+ page=page,
)
exc = CustomizedCodeException(e)
failing_line = user_code_line_from_exception(e)
diff --git a/tests/unit/test_code_block_recording_artifact.py b/tests/unit/test_code_block_recording_artifact.py
index c977ebf50..186fa1d00 100644
--- a/tests/unit/test_code_block_recording_artifact.py
+++ b/tests/unit/test_code_block_recording_artifact.py
@@ -14,6 +14,7 @@ import asyncio
from datetime import datetime, timezone
from pathlib import Path
from types import SimpleNamespace
+from typing import Any
from unittest.mock import AsyncMock, patch
import pytest
@@ -24,6 +25,7 @@ from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
from skyvern.forge.sdk.workflow.service import WorkflowService
from skyvern.webeye.browser_artifacts import BrowserArtifacts, VideoArtifact
+from tests.unit.fake_workflow_run_context import FakeWorkflowRunContext
_BLOCK_PATH = "skyvern.forge.sdk.workflow.models.block.app"
_MANAGER_PATH = "skyvern.forge.sdk.artifact.manager.app"
@@ -557,3 +559,39 @@ class TestPersistVideoDataPathFallback:
update_data.assert_awaited_once()
wait_for_uploads.assert_awaited_once_with(["wrb_1"])
assert video_artifacts[0].video_artifact_id == "a_existing"
+
+
+async def test_at_failure_url_masks_registered_secrets() -> None:
+ """A failing MFA step can leave the generated OTP in the query string.
+
+ The at-failure URL is persisted and logged, so registered secrets must be masked before
+ it leaves the page (SKY-13250).
+ """
+
+ class _MaskingContext(FakeWorkflowRunContext):
+ def mask_secrets_in_data(self, data: Any, mask: str = "*****") -> Any:
+ return data.replace("123456", mask) if isinstance(data, str) else data
+
+ page = SimpleNamespace(url="https://portal.example.com/mfa?token=123456&step=verify")
+ browser_state = SimpleNamespace(take_fullpage_screenshot=AsyncMock(return_value=b"png-bytes"))
+ fake_block = SimpleNamespace(workflow_run_block_id="wrb_1", workflow_run_id="wr_1", organization_id="o_1")
+ update_block = AsyncMock(return_value=fake_block)
+
+ with (
+ patch(f"{_BLOCK_PATH}.DATABASE.observer.update_workflow_run_block", update_block),
+ patch(f"{_BLOCK_PATH}.ARTIFACT_MANAGER.create_workflow_run_block_artifact", AsyncMock()),
+ ):
+ await _code_block()._capture_failure_evidence(
+ workflow_run_context=_MaskingContext(values={}, secrets={"otp": "123456"}),
+ workflow_run_id="wr_1",
+ workflow_run_block_id="wrb_1",
+ organization_id="o_1",
+ browser_state=browser_state,
+ page=page,
+ )
+
+ persisted_url = update_block.call_args.kwargs["final_url"]
+ assert "123456" not in persisted_url
+ # The rest of the URL is the diagnostic payload and must survive masking.
+ assert persisted_url == "https://portal.example.com/mfa?token=*****&step=verify"
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index f32390771..7e1e6f97b 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -115,9 +115,32 @@ def test_sanitize_run_blocks_debug_does_not_mutate_extracted_data() -> None:
assert original_block["extracted_data"] is original_extracted
-def test_sanitize_other_tools_do_not_touch_block_screenshot_b64() -> None:
- # `run_blocks_and_collect_debug` does not attach nested `screenshot_b64`;
- # if one somehow shows up there, leave it alone so behavior is scoped.
+def test_sanitize_run_blocks_debug_strips_block_screenshot_b64() -> None:
+ # `run_blocks_and_collect_debug` now attaches at-failure `screenshot_b64` to failed blocks
+ # (SKY-13250). The image reaches the model through `data.screenshot_base64`, so the raw bytes
+ # are stripped here as they are for `get_run_results` — leaving them crowds out the sibling
+ # fields, `final_url` among them.
+ result = {
+ "ok": False,
+ "data": {
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "a",
+ "status": "failed",
+ "screenshot_b64": "raw_base64_bytes",
+ "final_url": "https://portal.example.com/mfa",
+ }
+ ],
+ },
+ }
+ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)
+ assert sanitized["data"]["blocks"][0]["screenshot_b64"].startswith("[base64 image omitted")
+ assert sanitized["data"]["blocks"][0]["final_url"] == "https://portal.example.com/mfa"
+
+
+def test_sanitize_unrelated_tools_do_not_touch_block_screenshot_b64() -> None:
+ # The strip is scoped to the two tools that carry failed-block payloads.
result = {
"ok": True,
"data": {
@@ -131,7 +154,7 @@ def test_sanitize_other_tools_do_not_touch_block_screenshot_b64() -> None:
],
},
}
- sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)
+ sanitized = sanitize_tool_result_for_llm("update_workflow", result)
assert sanitized["data"]["blocks"][0]["screenshot_b64"] == "stays_here"
diff --git a/tests/unit/test_copilot_screenshot_handling.py b/tests/unit/test_copilot_screenshot_handling.py
index 8952adb79..92e736312 100644
--- a/tests/unit/test_copilot_screenshot_handling.py
+++ b/tests/unit/test_copilot_screenshot_handling.py
@@ -336,3 +336,196 @@ class TestSyntheticScreenshotPlaceholders:
]
assert _find_real_user_boundary(items, recent_turns=2) == 2
+
+
+class TestAttachFailedBlockScreenshots:
+ """A failed CODE block must surface its at-failure page, not just exception text (SKY-13250)."""
+
+ PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"\x00" * 32
+
+ @staticmethod
+ def _make_block(
+ *,
+ workflow_run_block_id: str | None,
+ task_id: str | None,
+ final_url: str | None = None,
+ ) -> MagicMock:
+ block = MagicMock()
+ block.workflow_run_block_id = workflow_run_block_id
+ block.task_id = task_id
+ block.final_url = final_url
+ return block
+
+ def _install_app(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ run_block_artifact: object | None,
+ task_v2_artifacts: list[object],
+ ) -> None:
+ import skyvern.forge.sdk.copilot.tools.run_execution as run_execution_module
+
+ artifacts = MagicMock()
+ artifacts.get_artifact_by_entity_id = AsyncMock(return_value=run_block_artifact)
+ artifacts.get_artifacts_for_task_v2 = AsyncMock(return_value=task_v2_artifacts)
+
+ class _AppStub:
+ DATABASE = MagicMock(artifacts=artifacts)
+ ARTIFACT_MANAGER = MagicMock(retrieve_artifact=AsyncMock(return_value=self.PNG_BYTES))
+
+ monkeypatch.setattr(run_execution_module, "app", _AppStub())
+
+ @pytest.mark.asyncio
+ async def test_failed_code_block_carries_screenshot_and_final_url(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """The regression: a code block's screenshot lives on the workflow_run_block.
+
+ Its container task is a task v1, so the task_v2 lookup returns nothing — before this
+ fix the failed block's result carried neither the screenshot nor the URL.
+ """
+ from skyvern.forge.sdk.copilot.tools.run_execution import _attach_failed_block_screenshots
+
+ block = self._make_block(
+ workflow_run_block_id="wrb-1",
+ task_id="tsk-v1-container",
+ final_url="https://auth.example.com/otp",
+ )
+ result: dict[str, Any] = {"label": "login", "status": "failed", "failure_reason": "Timeout 30000ms"}
+
+ self._install_app(monkeypatch, run_block_artifact=MagicMock(), task_v2_artifacts=[])
+
+ await _attach_failed_block_screenshots([block], [result], "org-1")
+
+ assert result["screenshot_b64"] == base64.b64encode(self.PNG_BYTES).decode("utf-8")
+ assert result["final_url"] == "https://auth.example.com/otp"
+
+ @pytest.mark.asyncio
+ async def test_legacy_task_v2_lookup_still_resolves_when_run_block_has_no_artifact(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from skyvern.forge.sdk.copilot.tools.run_execution import _attach_failed_block_screenshots
+
+ block = self._make_block(workflow_run_block_id="wrb-2", task_id="tsk-v2")
+ result: dict[str, Any] = {"label": "extract", "status": "failed"}
+
+ self._install_app(monkeypatch, run_block_artifact=None, task_v2_artifacts=[MagicMock()])
+
+ await _attach_failed_block_screenshots([block], [result], "org-1")
+
+ assert result["screenshot_b64"] == base64.b64encode(self.PNG_BYTES).decode("utf-8")
+ assert "final_url" not in result
+
+ @pytest.mark.asyncio
+ async def test_successful_block_gets_no_evidence(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.copilot.tools.run_execution import _attach_failed_block_screenshots
+
+ block = self._make_block(
+ workflow_run_block_id="wrb-3",
+ task_id="tsk-3",
+ final_url="https://example.com/done",
+ )
+ result: dict[str, Any] = {"label": "login", "status": "completed"}
+
+ self._install_app(monkeypatch, run_block_artifact=MagicMock(), task_v2_artifacts=[MagicMock()])
+
+ await _attach_failed_block_screenshots([block], [result], "org-1")
+
+ assert "screenshot_b64" not in result
+ assert "final_url" not in result
+
+
+class TestRunScreenshotResolution:
+ """Only data.screenshot_base64 becomes a model-visible image, so choosing it correctly
+ decides what the repair loop actually sees (SKY-13250)."""
+
+ VALID_PNG_B64 = (
+ "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAAE0lEQVR4nGP8z4APMOGVZRip0gBBLAETee26JgAAAABJRU5ErkJggg=="
+ )
+
+ @staticmethod
+ def _resolve(live_capture: str | None, results: list[dict[str, Any]], run_ok: bool) -> str | None:
+ from skyvern.forge.sdk.copilot.tools.run_execution import _resolve_run_screenshot_b64
+
+ return _resolve_run_screenshot_b64(live_capture=live_capture, results=results, run_ok=run_ok)
+
+ def test_live_capture_wins_when_present(self) -> None:
+ results = [{"status": "failed", "screenshot_b64": "block-b64"}]
+ assert self._resolve("live-b64", results, run_ok=False) == "live-b64"
+
+ def test_failed_run_promotes_first_failed_block(self) -> None:
+ """The dispatched path: no live capture, so the block's at-failure shot is all there is."""
+ results = [
+ {"label": "start", "status": "completed"},
+ {"label": "login", "status": "failed", "screenshot_b64": "first-b64"},
+ {"label": "extract", "status": "failed", "screenshot_b64": "second-b64"},
+ ]
+ assert self._resolve(None, results, run_ok=False) == "first-b64"
+
+ def test_successful_run_never_promotes_a_failure_screenshot(self) -> None:
+ """A healed or continue_on_failure block leaves a failure screenshot behind on a run
+ that succeeded; showing it would put a stale, contradicted page in front of the model."""
+ results = [
+ {"label": "login", "status": "failed", "screenshot_b64": "healed-failure-b64"},
+ {"label": "extract", "status": "completed"},
+ ]
+ assert self._resolve(None, results, run_ok=True) is None
+
+ def test_failed_run_with_no_block_screenshot_resolves_none(self) -> None:
+ assert self._resolve(None, [{"label": "login", "status": "failed"}], run_ok=False) is None
+
+ def test_promoted_screenshot_reaches_the_image_enqueue_path(self) -> None:
+ """End of the chain: the promoted bytes must arrive as a pending image, not as text."""
+ from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
+
+ results = [{"label": "login", "status": "failed", "screenshot_b64": self.VALID_PNG_B64}]
+ promoted = self._resolve(None, results, run_ok=False)
+ result = {"ok": False, "data": {"screenshot_base64": promoted}}
+
+ ctx = MagicMock()
+ ctx.supports_vision = True
+ ctx.pending_screenshots = []
+ enqueue_screenshot_from_result(ctx, result)
+
+ assert len(ctx.pending_screenshots) == 1
+
+
+class TestSanitizerStripsPerBlockScreenshots:
+ """Raw base64 in the text channel crowds out the fields beside it. The image reaches the
+ model through data.screenshot_base64, so the per-block copy is stripped on every tool that
+ carries failed blocks — including the primary repair tool (SKY-13250)."""
+
+ @staticmethod
+ def _sanitize(tool_name: str, result: dict[str, Any]) -> dict[str, Any]:
+ from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm
+
+ return sanitize_tool_result_for_llm(tool_name, result)
+
+ @pytest.mark.parametrize("tool_name", ["run_blocks_and_collect_debug", "get_run_results"])
+ def test_strips_base64_but_keeps_final_url(self, tool_name: str) -> None:
+ from skyvern.forge.sdk.copilot.output_utils import _BASE64_IMAGE_OMITTED_MESSAGE
+
+ result = {
+ "ok": False,
+ "data": {
+ "blocks": [
+ {
+ "label": "login",
+ "status": "failed",
+ "screenshot_b64": "A" * 5000,
+ "final_url": "https://portal.example.com/mfa",
+ }
+ ]
+ },
+ }
+
+ block = self._sanitize(tool_name, result)["data"]["blocks"][0]
+
+ assert block["screenshot_b64"] == _BASE64_IMAGE_OMITTED_MESSAGE
+ # final_url is the only URL evidence on the dispatched path; it must survive.
+ assert block["final_url"] == "https://portal.example.com/mfa"
+
+ def test_leaves_blocks_without_screenshots_untouched(self) -> None:
+ result = {"ok": True, "data": {"blocks": [{"label": "login", "status": "completed"}]}}
+ assert self._sanitize("run_blocks_and_collect_debug", result)["data"]["blocks"][0] == {
+ "label": "login",
+ "status": "completed",
+ }
diff --git a/tests/unit/workflow/test_workflow_run_block_script_run_exposure.py b/tests/unit/workflow/test_workflow_run_block_script_run_exposure.py
index db52fe288..3e32e5613 100644
--- a/tests/unit/workflow/test_workflow_run_block_script_run_exposure.py
+++ b/tests/unit/workflow/test_workflow_run_block_script_run_exposure.py
@@ -32,6 +32,7 @@ def _fake_block_model(script_run_value: dict | None, prompt_value: str | None =
model.output = None
model.continue_on_failure = False
model.failure_reason = None
+ model.final_url = None
model.error_codes = None
model.engine = None
model.task_id = None