Skip to content

Automate browser based workflows with AI

Show Copilot the recorded error of a page call a completed block caught (SKY-18206) (#8970)

Andrew Neilson0fc4361986b1
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 60bd74ce6..019c5c5bc 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -41,6 +41,7 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (
)
from skyvern.schemas.workflows import BlockStatus, BlockType
from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import ActionStatus
from skyvern.webeye.utils.captcha_solver import ChallengeArm, ChallengePageState, ChallengeStatus
LOG = structlog.get_logger()
@@ -606,6 +607,17 @@ _ValueT = TypeVar("_ValueT")
BLOCK_FACT_MAX_LABELS = 12
BLOCK_FACT_MAX_OBSERVATIONS = 6
BLOCK_FACT_MAX_CHARS = 300
+
+
+_RECORDED_FAILURE_OBSERVATION = re.compile(rf"\S+ {ActionStatus.failed.value} code_line=\d+[^:]*: ")
+
+
+def recorded_failure_observation(observation: str) -> bool:
+ """Whether a rendered action observation carries a code-block operation's recorded error, which native task
+ rows never do."""
+ return _RECORDED_FAILURE_OBSERVATION.match(observation) is not None
+
+
# An over-long URL is refused rather than cut everywhere it is bounded: a cut URL still parses, so
# it would be reported as a page the run reached while being unresumable.
BLOCK_FACT_URL_MAX_CHARS = 2000
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 1423017f4..1115a507a 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -40,6 +40,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
append_omission_notice,
coerce_block_action_observations,
coerce_block_end_urls,
+ recorded_failure_observation,
)
from skyvern.forge.sdk.copilot.composition_evidence import (
INTERNAL_VALIDATION_FAILURE_PREFIX,
@@ -355,19 +356,54 @@ def _compacted(
return values[-limit:] if keep == "last" else values[:limit]
+def _newest_indexes_preferring(preferred: Sequence[bool], limit: int) -> list[int]:
+ """The ``limit`` newest positions, preferred ones first, back in chronological order."""
+ newest_first = range(len(preferred) - 1, -1, -1)
+ ranked = [index for index in newest_first if preferred[index]] + [
+ index for index in newest_first if not preferred[index]
+ ]
+ return sorted(ranked[:limit])
+
+
+def _compacted_failures_first(values: list[str], limit: int, *, field_name: str, notices: list[str]) -> list[str]:
+ """Keep the newest action observations, a recorded code-block failure ahead of any other."""
+ if len(values) <= limit:
+ return values
+ kept = _newest_indexes_preferring([recorded_failure_observation(value) for value in values], limit)
+ which = "oldest " if kept == list(range(len(values) - limit, len(values))) else ""
+ append_omission_notice(
+ notices,
+ f"{field_name} shortened at the aggregate packet limit: {len(values) - limit} {which}item(s) omitted, "
+ "recorded code-block failures kept first.",
+ )
+ return [values[index] for index in kept]
+
+
def _compacted_newest_labelled(
- entries: list[tuple[str, _ItemT]], limit: int, *, field_name: str, notices: list[str]
+ entries: list[tuple[str, _ItemT]],
+ limit: int,
+ *,
+ field_name: str,
+ notices: list[str],
+ prefer: Callable[[_ItemT], bool] | None = None,
) -> list[tuple[str, _ItemT]]:
- """Keeping the newest entries relies on producers inserting labels in execution order. Name the
- blocks withheld, so a block this bound dropped is not read as one that recorded nothing."""
+ """Keeping the newest entries relies on producers inserting labels in execution order; ``prefer`` keeps its
+ entries ahead of newer ones. Name the blocks withheld, so a dropped block is not read as one that recorded
+ nothing."""
if len(entries) <= limit:
return entries
- named = _bounded_labelled_names([(label, None) for label, _ in entries[:-limit]])
+ kept = _newest_indexes_preferring([prefer is not None and prefer(value) for _, value in entries], limit)
+ named = _bounded_labelled_names([(label, None) for index, (label, _) in enumerate(entries) if index not in kept])
+ which = "the oldest " if kept == list(range(len(entries) - limit, len(entries))) else ""
append_omission_notice(
notices,
- f"{field_name} shortened at the aggregate packet limit: dropped the oldest block(s) {named}.",
+ f"{field_name} shortened at the aggregate packet limit: dropped {which}block(s) {named}.",
)
- return entries[-limit:]
+ return [entries[index] for index in kept]
+
+
+def _reports_a_recorded_failure(observations: list[str]) -> bool:
+ return any(recorded_failure_observation(observation) for observation in observations)
def screened_recorded_url(url: str) -> tuple[str | None, str | None]:
@@ -854,6 +890,7 @@ def _compact_block_fact_maps(packet: BuildTestEvidencePacket, notices: list[str]
2,
field_name="per_block_action_observations",
notices=notices,
+ prefer=_reports_a_recorded_failure,
)
),
"omission_notices": notices,
@@ -971,8 +1008,8 @@ def _compact_packet_for_aggregate_limit(
),
"action_observations": [
value[:117] + "..." if len(value) > 120 else value
- for value in _compacted(
- packet.action_observations, 2, field_name="action_observations", notices=notices, keep="last"
+ for value in _compacted_failures_first(
+ packet.action_observations, 2, field_name="action_observations", notices=notices
)
],
"observed_block_end_urls": dict(
@@ -984,14 +1021,15 @@ def _compact_packet_for_aggregate_limit(
)
),
"per_block_action_observations": {
- label: _compacted(
- values, 1, field_name=f"per_block_action_observations[{label}]", notices=notices, keep="last"
+ label: _compacted_failures_first(
+ values, 1, field_name=f"per_block_action_observations[{label}]", notices=notices
)
for label, values in _compacted_newest_labelled(
list(packet.per_block_action_observations.items()),
2,
field_name="per_block_action_observations",
notices=notices,
+ prefer=_reports_a_recorded_failure,
)
},
"failure": failure,
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index a53ccdd52..89aff80a3 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -7,6 +7,7 @@ import copy
import hashlib
import json
import os
+import re
import tempfile
import time
import uuid
@@ -57,6 +58,7 @@ from skyvern.forge.sdk.copilot.build_test_connect_failure import (
from skyvern.forge.sdk.copilot.build_test_outcome import (
ACTION_OBSERVATIONS_EMPTY,
ACTION_TRACE_PER_TASK_LIMIT,
+ BLOCK_FACT_MAX_CHARS,
INFRASTRUCTURE_RUNNER_ERROR_CODES,
OBSERVED_BLOCK_END_URLS_UNREPORTABLE,
OBSERVED_BLOCK_END_URLS_WITHHELD,
@@ -645,6 +647,75 @@ def _reconcile_narrative_block_attempts(ctx: CopilotContext, blocks: list[Workfl
attempt["startedAt"], attempt["endedAt"] = started_at, ended_at
+# Playwright 1.46 writes ` - fill("…")`; 1.58 indents it and may prefix a repeat count, ` 2 × fill("…")`.
+# Either echoes the typed text raw, newlines included, so no line can be shown to end it. After an echo only whole
+# lines matching a message Playwright's dom.js logs (same in 1.46 and 1.58) are kept, with their numbers masked.
+_TYPED_INPUT_ECHO = re.compile(r'^\s*(?:-\s*)?(?:\d+ × )?(?:elementHandle\.)?(?:fill|type)\("')
+_PLAYWRIGHT_ACTION = r"(?:click|dblclick|hover|tap|fill|select option|scroll into view)"
+_CALL_LOG_STEP = re.compile(
+ r"^\s*(?:-\s*)?(?:\d+ × )?(?:"
+ rf"(?:attempting|retrying|performing) {_PLAYWRIGHT_ACTION} action(?: \(trial run\))?(?:, attempt #\d+)?"
+ rf"|(?:trial )?{_PLAYWRIGHT_ACTION} action done|forcing action|waiting \d+ms|done scrolling"
+ r"|scrolling into view if needed"
+ r"|waiting for element to be (?:visible, enabled and (?:editable|stable)|visible and (?:stable|enabled)"
+ r"|stable|visible)"
+ r"|element is (?:not (?:visible|enabled|editable|stable|attached)|visible, enabled and stable"
+ r"|visible and stable|outside of the viewport)"
+ r"|waiting for scheduled navigations to finish|navigations have finished"
+ r")\s*$"
+)
+# An element preview cuts a long attribute with "…" and no closing quote.
+_VALUE_ATTRIBUTE = re.compile(r'\svalue="[^"\n]*"?')
+
+
+def _without_typed_input(text: str) -> str:
+ """Playwright's call log echoes the text a fill or type call entered, and its element previews carry an input's
+ value. Persistence masks only registered secrets and parameters, not a value the block obtained at runtime."""
+ kept: list[str] = []
+ after_echo = False
+ for line in text.split("\n"):
+ if _TYPED_INPUT_ECHO.match(line):
+ after_echo = True
+ elif not after_echo:
+ kept.append(line)
+ elif _CALL_LOG_STEP.match(line):
+ kept.append(re.sub(r"\d+", "N", line))
+ return _VALUE_ATTRIBUTE.sub(' value="[omitted]"', "\n".join(kept))
+
+
+def _screened_failure_output(output: Any) -> Any:
+ """A failed block's output repeats its failure text as ``failure_reason`` and as each error's ``reasoning``."""
+ if not isinstance(output, dict) or output.get("status") != BlockStatus.failed.value:
+ return output
+ screened = dict(output)
+ if isinstance(screened.get("failure_reason"), str):
+ screened["failure_reason"] = _without_typed_input(screened["failure_reason"])
+ if isinstance(screened.get("errors"), list):
+ screened["errors"] = [
+ {**error, "reasoning": _without_typed_input(error["reasoning"])}
+ if isinstance(error, dict) and isinstance(error.get("reasoning"), str)
+ else error
+ for error in screened["errors"]
+ ]
+ return screened
+
+
+# An apostrophe is legal in a URL, credentials included; Playwright quotes the URLs it logs with double quotes.
+_RECORDED_URL = re.compile(r"https?://[^\s\"<>]+")
+
+
+def _screened_recorded_url_text(match: re.Match[str]) -> str:
+ url, _ = screened_recorded_url(match.group())
+ return url or "[URL withheld]"
+
+
+def _screened_recorded_failure(response: str) -> str:
+ """Playwright repeats a navigation's whole target URL, credentials and query included, so each URL gets the
+ recorded-URL screen."""
+ text = _RECORDED_URL.sub(_screened_recorded_url_text, _without_typed_input(response))
+ return redact_raw_secrets_for_prompt(text)[:RECORDED_FAILURE_RESPONSE_MAX_CHARS]
+
+
async def _attach_action_traces(
blocks: list[WorkflowRunBlock],
results: list[dict[str, Any]],
@@ -726,11 +797,7 @@ async def _attach_action_traces(
# response, which is user data and must not reach the packet.
entry["code_line"] = code_line
if action.response:
- # Persistence masks registered secrets and parameters; a token the block
- # obtained at runtime is in neither registry, so screen it here too.
- entry["response"] = redact_raw_secrets_for_prompt(action.response)[
- :RECORDED_FAILURE_RESPONSE_MAX_CHARS
- ]
+ entry["response"] = _screened_recorded_failure(action.response)
action_trace.append(entry)
block_result["action_trace"] = action_trace
return actions_by_task
@@ -766,7 +833,7 @@ def _code_failure_text(task_actions: Sequence[Action]) -> str | None:
if not failure_rows:
return None
response = max(failure_rows, key=lambda action: action.action_order or 0).response
- return redact_raw_secrets_for_prompt(response)[:RECORDED_FAILURE_RESPONSE_MAX_CHARS] if response else None
+ return _screened_recorded_failure(response) if response else None
async def _attach_ai_fallback_facts(
@@ -781,6 +848,7 @@ async def _attach_ai_fallback_facts(
code_block_labels = {
block.workflow_run_block_id: block.label for block in blocks if block.block_type == BlockType.CODE
}
+ results_by_block_id = {block.workflow_run_block_id: result for block, result in zip(blocks, results)}
for block, result in zip(blocks, results):
if parent_label := code_block_labels.get(block.parent_workflow_run_block_id or ""):
result["parent_block_label"] = parent_label
@@ -792,6 +860,8 @@ async def _attach_ai_fallback_facts(
LOG.warning(
"Failed to read AI fallback episodes for run results", workflow_run_id=workflow_run_id, exc_info=True
)
+ for block_id in code_block_labels:
+ results_by_block_id[block_id]["ai_fallback_unread"] = True
return
fired: dict[str, HealEpisodeDetail] = {}
for episode in episodes:
@@ -806,6 +876,8 @@ async def _attach_ai_fallback_facts(
workflow_run_block_id=episode.workflow_run_block_id,
exc_info=True,
)
+ if (unread := results_by_block_id.get(episode.workflow_run_block_id)) is not None:
+ unread["ai_fallback_unread"] = True
for block, result in zip(blocks, results):
detail = fired.get(block.workflow_run_block_id)
if detail is None:
@@ -834,12 +906,12 @@ def _recorded_run_block_result(block: WorkflowRunBlock) -> dict[str, Any]:
if block.task_id:
result["task_id"] = block.task_id
if block.failure_reason:
- result["failure_reason"] = redact_totp_runtime_values(block.failure_reason)
+ result["failure_reason"] = redact_totp_runtime_values(_without_typed_input(block.failure_reason))
if block.error_codes:
result["error_codes"] = list(block.error_codes)
# The persisted row owns null too. Keeping the key distinguishes an explicit null from an
# older result that carried no block-output fact at all.
- result["output"] = redact_totp_runtime_values(block.output)
+ result["output"] = redact_totp_runtime_values(_screened_failure_output(block.output))
return result
@@ -991,7 +1063,11 @@ async def recorded_open_step_runs(ctx: CopilotContext) -> list[RecordedRunState]
)
rows = []
blocks = [
- RecordedBlockState(label=row.label, status=row.status, failure_reason=row.failure_reason)
+ RecordedBlockState(
+ label=row.label,
+ status=row.status,
+ failure_reason=_without_typed_input(row.failure_reason) if row.failure_reason else None,
+ )
for row in rows
if row.label and row.status
]
@@ -999,7 +1075,7 @@ async def recorded_open_step_runs(ctx: CopilotContext) -> list[RecordedRunState]
state = RecordedRunState(
workflow_run_id=workflow_run_id,
status=str(run.status) if run is not None else None,
- failure_reason=run.failure_reason if run is not None else None,
+ failure_reason=_without_typed_input(run.failure_reason) if run is not None and run.failure_reason else None,
blocks=shown,
earlier_blocks_not_shown=len(blocks) - len(shown),
left_running=ctx.budget_expiry_state.hard_backstop_reached
@@ -1213,7 +1289,23 @@ def _result_current_url(result: Mapping[str, Any] | None) -> str | None:
return str(url) if isinstance(url, str) and url.strip() else None
-def _action_observation(entry: Mapping[str, Any]) -> str | None:
+def _recorded_error_digest(response: str) -> str:
+ """Playwright's call log repeats every retry's steps after a one-line header. Drop the repeats and lead with the
+ steps stating why the action could not proceed, so the cause survives the per-observation cut."""
+ header, _, call_log = response.partition("Call log:")
+ steps = list(dict.fromkeys(step for line in call_log.splitlines() if (step := line.strip().lstrip("- ").strip())))
+ causes = [
+ step
+ for step in steps
+ if "intercepts pointer events" in step
+ or "intercepted by another element" in step
+ or step.startswith(("element is not ", "element is outside"))
+ ]
+ parts = [*causes, " ".join(header.split()), *(step for step in steps if step not in causes)]
+ return "; ".join(part for part in parts if part)
+
+
+def _action_observation(entry: Mapping[str, Any], *, block_continued: bool) -> str | None:
raw_action = entry.get("action")
raw_status = entry.get("status")
if not isinstance(raw_action, str) or not isinstance(raw_status, str):
@@ -1225,9 +1317,36 @@ def _action_observation(entry: Mapping[str, Any]) -> str | None:
return None
observation = f"{action.value} {status.value}"
code_line = entry.get("code_line")
- if type(code_line) is int:
- observation += f" code_line={code_line}"
- return observation
+ if type(code_line) is not int:
+ return observation
+ observation += f" code_line={code_line}"
+ if status != ActionStatus.failed:
+ return observation
+ if block_continued:
+ observation += " (block continued)"
+ # Only the code-block recorder's own failed rows carry code_line and a response, already screened on attach.
+ response = entry.get("response")
+ if isinstance(response, str) and response:
+ observation = f"{observation}: {_recorded_error_digest(response)}"
+ return observation if len(observation) <= BLOCK_FACT_MAX_CHARS else observation[: BLOCK_FACT_MAX_CHARS - 3] + "..."
+
+
+def _block_continued(block_result: Mapping[str, Any]) -> bool:
+ """The block finished on its own code: completed, and not by an AI fallback taking over after the code raised."""
+ return (
+ block_result.get("status") == BlockStatus.completed.value
+ and "ai_fallback" not in block_result
+ and not block_result.get("ai_fallback_unread")
+ )
+
+
+def _recorded_code_failure(entry: Mapping[str, Any]) -> bool:
+ return (
+ entry.get("status") == ActionStatus.failed.value
+ and type(entry.get("code_line")) is int
+ and isinstance(entry.get("response"), str)
+ and bool(entry["response"])
+ )
class _ActionObservationSelection(NamedTuple):
@@ -1238,9 +1357,8 @@ class _ActionObservationSelection(NamedTuple):
def _selected_action_observations(results: Sequence[Mapping[str, Any]]) -> _ActionObservationSelection:
"""Pick the retained action facts chronologically as (block index, observation) pairs, plus the
predecessor blocks the budget left with no entry at all.
- ``results`` is chronological and each ``action_trace`` is newest-first, so a block's terminal
- action is entry 0; the budget reserves one such entry for each of the newest predecessor blocks
- and the newest block fills the rest."""
+ Recorded code-block failures go first, then each newest unreported predecessor's terminal entry (entry 0 of its
+ newest-first ``action_trace``), then the newest block fills the rest."""
traces: list[list[Mapping[str, Any]]] = []
for block_result in results:
action_trace = block_result.get("action_trace")
@@ -1249,12 +1367,26 @@ def _selected_action_observations(results: Sequence[Mapping[str, Any]]) -> _Acti
)
newest_block_index = len(traces) - 1
+ newest_trace = traces[newest_block_index] if traces else []
+ # An older block's failures must not take every slot from a newest block that recorded none of its own.
+ failure_budget = _ACTION_OBSERVATION_BUDGET - (
+ 1 if newest_trace and not any(_recorded_code_failure(entry) for entry in newest_trace) else 0
+ )
selected: set[tuple[int, int]] = set()
+ # A program retrying one call in a loop records the same failure each time; only the newest copy takes a slot.
+ seen_failures: set[tuple[int, Any, Any]] = set()
+ for block_index in range(newest_block_index, -1, -1):
+ for entry_index, entry in enumerate(traces[block_index]):
+ failure_key = (block_index, entry.get("code_line"), entry.get("response"))
+ if len(selected) < failure_budget and _recorded_code_failure(entry) and failure_key not in seen_failures:
+ seen_failures.add(failure_key)
+ selected.add((block_index, entry_index))
+ newest_block_slot = 0 if any(block_index == newest_block_index for block_index, _ in selected) else 1
unreserved: list[int] = []
for block_index in range(newest_block_index - 1, -1, -1):
- if not traces[block_index]:
+ if not traces[block_index] or any(selected_block == block_index for selected_block, _ in selected):
continue
- if len(selected) >= _ACTION_OBSERVATION_BUDGET - 1:
+ if len(selected) >= _ACTION_OBSERVATION_BUDGET - newest_block_slot:
unreserved.append(block_index)
continue
selected.add((block_index, 0))
@@ -1268,10 +1400,11 @@ def _selected_action_observations(results: Sequence[Mapping[str, Any]]) -> _Acti
retained: list[tuple[int, str]] = []
for block_index in range(len(traces)):
+ block_continued = _block_continued(results[block_index])
for entry_index in range(len(traces[block_index]) - 1, -1, -1):
if (block_index, entry_index) not in selected:
continue
- observation = _action_observation(traces[block_index][entry_index])
+ observation = _action_observation(traces[block_index][entry_index], block_continued=block_continued)
if observation is not None:
retained.append((block_index, observation))
reported = {block_index for block_index, _ in retained}
@@ -2312,7 +2445,7 @@ async def _attach_registered_output_parameter_values(
block_info.update(index_by_key.get(output_parameter_key, {}))
if block_info.get("block_label") in excluded_block_labels:
continue
- value = without_output_only_labels(getattr(row, "value", None))
+ value = _screened_failure_output(without_output_only_labels(getattr(row, "value", None)))
item = {
"workflow_run_id": workflow_run_id,
"output_parameter_id": output_parameter_id,
@@ -5110,7 +5243,7 @@ async def _run_blocks_and_collect_debug(
for block in blocks:
block_result = _recorded_run_block_result(block)
if hasattr(block, "output") and block.output:
- block_result["extracted_data"] = block.output
+ block_result["extracted_data"] = redact_totp_runtime_values(_screened_failure_output(block.output))
if block.label is not None:
block_outputs_by_label[block.label] = block.output
results.append(block_result)
@@ -5121,7 +5254,14 @@ async def _run_blocks_and_collect_debug(
)
ctx.last_run_blocks_block_labels = list(dict.fromkeys(block.label for block in run_block_rows if block.label))
- await _attach_action_traces(blocks, results, ctx.organization_id, include_completed=True)
+ actions_by_task = await _attach_action_traces(blocks, results, ctx.organization_id, include_completed=True)
+ await _attach_ai_fallback_facts(
+ blocks,
+ results,
+ actions_by_task,
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=ctx.organization_id,
+ )
origin_registry = (
await _complete_origin_run_redaction_registry_from_runtime(
ctx, workflow_run.workflow_run_id, origin_registry
@@ -5176,7 +5316,7 @@ async def _run_blocks_and_collect_debug(
result_data["reused_origin_input_keys"] = list(execution.reused_origin_input_keys)
_attach_reused_origin_outputs(result_data, execution)
if not run_ok and run and run.failure_reason:
- result_data["failure_reason"] = redact_totp_runtime_values(run.failure_reason)
+ result_data["failure_reason"] = redact_totp_runtime_values(_without_typed_input(run.failure_reason))
if not run_ok and run and run.failure_category:
failure_category = without_output_only_categories(run.failure_category)
if failure_category:
@@ -5597,7 +5737,7 @@ async def _get_run_results(
if dispatch_to_worker and dispatched_end_url is None and not secrets_unscrubbable:
result_data["current_url_evidence"] = NO_PERSISTED_END_URL
if getattr(run, "failure_reason", None):
- result_data["failure_reason"] = redact_totp_runtime_values(run.failure_reason)
+ result_data["failure_reason"] = redact_totp_runtime_values(_without_typed_input(run.failure_reason))
await _halt_turn_if_superseded(
ctx,
run,
diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.py
index eadc0691e..76f65fbcd 100644
--- a/tests/unit/copilot_test_helpers.py
+++ b/tests/unit/copilot_test_helpers.py
@@ -564,6 +564,7 @@ async def install_run_blocks_harness(
database.workflow_params.get_workflow_output_parameters = AsyncMock(return_value=persisted_output_params)
database.observer.get_workflow_run_blocks = AsyncMock(return_value=terminal_blocks or [])
database.tasks.get_recent_actions_for_tasks = AsyncMock(return_value=list(recent_actions or []))
+ database.self_heal.get_heal_episodes_for_run = AsyncMock(return_value=[])
database.workflow_runs.get_workflow_run = AsyncMock(return_value=_fake_workflow_run(status=polled_status))
database.workflow_runs.get_workflow_run_output_parameters = AsyncMock(return_value=[])
database.workflow_runs.get_workflow_runs_for_workflow_permanent_id = AsyncMock(return_value=[])
diff --git a/tests/unit/fixtures/copilot/caught_failure_repair/consent_toggle_caught_click.json b/tests/unit/fixtures/copilot/caught_failure_repair/consent_toggle_caught_click.json
new file mode 100644
index 000000000..e515bcc15
--- /dev/null
+++ b/tests/unit/fixtures/copilot/caught_failure_repair/consent_toggle_caught_click.json
@@ -0,0 +1,536 @@
+{
+ "schema": "generated-code-exception-packet-v1",
+ "provenance": "a fixed caught-click draft run by tests/browser_e2e/test_workflow_copilot_code_first_failure_attribution.py against a live Chromium page, with its recorded rows projected by the run tool's own attachment; no model was called",
+ "label": "read_order_documents",
+ "context_slug": "caught-intercepted-click",
+ "system": "You are authoring the browser automation code block for the user's request. If a test run of your block is reported, submit the block that should be saved: the same block if that run satisfied the request, otherwise a repaired one. Return only the Python statements of the block body, with no markdown fence and no commentary; `page` is already in scope and the body must return a dict with a 'documents' key.",
+ "ask": "Open https://documents.fixture.test/orders?intercepted_toggle=1 and return every row of the order documents table under the documents key.",
+ "captured_turn": 0,
+ "routes": {
+ "GET /orders": "<!doctype html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n <title>Order Documents Portal</title>\n <style>\n :root {\n color-scheme: light;\n --ink: #18212f;\n --muted: #657187;\n --line: #d7dce5;\n --panel: #f5f7fb;\n --accent: #0f766e;\n --paper: #ffffff;\n }\n\n * {\n box-sizing: border-box;\n }\n\n body {\n margin: 0;\n min-height: 100vh;\n font-family: Inter, ui-sans-serif, system-ui, -apple-system, \"Segoe UI\", sans-serif;\n color: var(--ink);\n background: var(--panel);\n }\n\n header {\n display: flex;\n align-items: center;\n min-height: 64px;\n padding: 0 32px;\n border-bottom: 1px solid var(--line);\n background: var(--paper);\n font-weight: 700;\n }\n\n main {\n max-width: 860px;\n margin: 32px auto;\n padding: 0 16px;\n }\n\n .card {\n background: var(--paper);\n border: 1px solid var(--line);\n border-radius: 10px;\n padding: 24px;\n margin-bottom: 20px;\n }\n\n .check-status {\n display: flex;\n align-items: center;\n gap: 10px;\n font-weight: 600;\n }\n\n .check-status .ok {\n color: var(--accent);\n }\n\n table {\n width: 100%;\n border-collapse: collapse;\n }\n\n th,\n td {\n text-align: left;\n padding: 10px 12px;\n border-bottom: 1px solid var(--line);\n }\n\n .overlay {\n position: fixed;\n inset: 0;\n background: rgba(24, 33, 47, 0.55);\n display: flex;\n align-items: center;\n justify-content: center;\n z-index: 60;\n }\n\n .modal {\n background: var(--paper);\n border-radius: 12px;\n max-width: 460px;\n width: calc(100% - 32px);\n padding: 28px;\n box-shadow: 0 18px 44px rgba(24, 33, 47, 0.28);\n }\n\n .modal h2 {\n margin-top: 0;\n font-size: 19px;\n }\n\n .modal p {\n color: var(--muted);\n font-size: 14px;\n line-height: 1.5;\n }\n\n .consent-row {\n display: flex;\n gap: 10px;\n align-items: flex-start;\n position: relative;\n margin: 18px 0;\n font-size: 14px;\n }\n\n .partial-coverer {\n position: absolute;\n top: -5px;\n left: -5px;\n z-index: 2;\n width: 30px;\n height: 30px;\n border: 1px solid #b45309;\n border-radius: 6px;\n background: #fbbf24;\n cursor: pointer;\n }\n\n .pref-toggle {\n position: relative;\n display: block;\n min-width: 260px;\n margin-top: 16px;\n padding: 10px 14px;\n border: 1px solid var(--line);\n border-radius: 18px;\n cursor: pointer;\n font-size: 14px;\n }\n\n .pref-toggle-heading {\n position: absolute;\n inset: 0;\n display: flex;\n align-items: center;\n justify-content: flex-end;\n padding-right: 14px;\n color: var(--muted);\n }\n\n button[type=\"submit\"] {\n width: 100%;\n padding: 11px 16px;\n font-size: 15px;\n font-weight: 600;\n color: white;\n background: var(--accent);\n border: 0;\n border-radius: 8px;\n cursor: pointer;\n }\n\n button[type=\"submit\"]:disabled {\n background: #9db4b1;\n cursor: not-allowed;\n }\n\n [hidden] {\n display: none !important;\n }\n </style>\n </head>\n <body>\n <header>Order Documents Portal</header>\n <main>\n <section class=\"card\" id=\"bot-check-card\">\n <div class=\"check-status\" id=\"bot-check-status\">\n <span aria-hidden=\"true\">&#9203;</span>\n <span>Checking your browser before granting access&hellip;</span>\n </div>\n <div class=\"check-status\" id=\"bot-check-passed\" hidden>\n <span class=\"ok\" aria-hidden=\"true\">&#10003;</span>\n <span class=\"ok\">Success! Browser check passed.</span>\n </div>\n <div id=\"verification-widget-wrapper\" hidden>\n <div id=\"verification-widget\" class=\"challenge-widget human-verification\" data-state=\"solved\"></div>\n </div>\n </section>\n\n <section class=\"card\" id=\"documents-card\" hidden>\n <h1>Your order documents</h1>\n <table id=\"order-documents\">\n <thead>\n <tr>\n <th>Document ID</th>\n <th>Title</th>\n <th>Order date</th>\n <th>Status</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <td>DOC-58231</td>\n <td>Purchase confirmation</td>\n <td>2026-06-02</td>\n <td>Available</td>\n </tr>\n <tr>\n <td>DOC-58232</td>\n <td>Shipping manifest</td>\n <td>2026-06-05</td>\n <td>Available</td>\n </tr>\n <tr>\n <td>DOC-58233</td>\n <td>Warranty certificate</td>\n <td>2026-06-11</td>\n <td>Processing</td>\n </tr>\n </tbody>\n </table>\n </section>\n </main>\n\n <div class=\"overlay\" id=\"terms-overlay\" hidden>\n <form class=\"modal\" id=\"terms-modal\" role=\"dialog\" aria-modal=\"true\" aria-labelledby=\"terms-title\">\n <h2 id=\"terms-title\">Terms of Service &amp; Privacy</h2>\n <p>\n Before viewing your order documents, please review and accept our Terms of Service and Privacy Policy.\n We process personal data in accordance with GDPR; see the privacy policy for details on cookies and\n data retention.\n </p>\n <label class=\"consent-row\">\n <input type=\"checkbox\" id=\"accept-terms\" name=\"accept_terms\" />\n <span>I have read and agree to the Terms of Service and the Privacy Policy.</span>\n </label>\n <button type=\"submit\" id=\"btn-continue\" disabled>Continue</button>\n <div class=\"pref-toggle\" id=\"reject-toggle\" role=\"switch\" aria-checked=\"false\" hidden>\n <span class=\"pref-toggle-label\">Reject All</span>\n <span class=\"pref-toggle-heading\">Optional cookies</span>\n </div>\n </form>\n </div>\n\n <script>\n (function () {\n var acceptTerms = document.getElementById(\"accept-terms\");\n var btnContinue = document.getElementById(\"btn-continue\");\n var query = new URLSearchParams(window.location.search);\n var partialCover = query.get(\"partial_cover\") === \"1\";\n\n // The heading span lies over its sibling label, so a click aimed at the label lands on the heading.\n // A script-dispatched click is not trusted, so the page records whether a pointer made the choice.\n if (query.get(\"intercepted_toggle\") === \"1\") {\n var toggle = document.getElementById(\"reject-toggle\");\n document.getElementById(\"terms-title\").textContent = \"Cookie preferences\";\n document.querySelector(\"#terms-modal p\").textContent =\n \"We use optional cookies for analytics. Make your choice to continue to your order documents.\";\n acceptTerms.parentElement.remove();\n btnContinue.remove();\n toggle.hidden = false;\n toggle.addEventListener(\"click\", function (event) {\n sessionStorage.setItem(\n \"cookie_choice_click\",\n JSON.stringify({ trusted: event.isTrusted, receiver: event.currentTarget.id }),\n );\n toggle.setAttribute(\"aria-checked\", \"true\");\n document.getElementById(\"terms-overlay\").hidden = true;\n document.getElementById(\"documents-card\").hidden = false;\n });\n }\n\n setTimeout(function () {\n document.getElementById(\"bot-check-status\").hidden = true;\n document.getElementById(\"bot-check-passed\").hidden = false;\n document.getElementById(\"terms-overlay\").hidden = false;\n if (partialCover) {\n var coverer = document.createElement(\"div\");\n coverer.id = \"target-coverer\";\n coverer.className = \"partial-coverer\";\n coverer.tabIndex = 0;\n coverer.setAttribute(\"role\", \"button\");\n coverer.setAttribute(\"aria-label\", \"Dismiss target cover\");\n var dismissCoverer = function (event) {\n event.preventDefault();\n event.stopPropagation();\n coverer.remove();\n };\n coverer.addEventListener(\"click\", dismissCoverer);\n coverer.addEventListener(\"keydown\", function (event) {\n if (event.key === \"Enter\" || event.key === \" \") {\n dismissCoverer(event);\n }\n });\n acceptTerms.parentElement.appendChild(coverer);\n }\n }, 1200);\n\n acceptTerms.addEventListener(\"change\", function () {\n btnContinue.disabled = !acceptTerms.checked;\n });\n\n document.getElementById(\"terms-modal\").addEventListener(\"submit\", function (event) {\n event.preventDefault();\n if (!acceptTerms.checked) {\n return;\n }\n document.getElementById(\"terms-overlay\").hidden = true;\n document.getElementById(\"documents-card\").hidden = false;\n });\n })();\n </script>\n </body>\n</html>\n"
+ },
+ "start_state": {
+ "cookies": [],
+ "origins": []
+ },
+ "state_reader": "() => {\n let choice = null;\n try {\n choice = JSON.parse(sessionStorage.getItem('cookie_choice_click') || 'null');\n } catch (error) {}\n return {\n cover_present: !document.getElementById('terms-overlay').hidden,\n documents_visible: !document.getElementById('documents-card').hidden,\n dismissed_by_trusted_click: !!choice && choice.trusted === true,\n };\n}",
+ "expected_state": {
+ "cover_present": false,
+ "documents_visible": true,
+ "dismissed_by_trusted_click": true
+ },
+ "expected_repair_facts": [
+ {
+ "name": "caught_click.intercepting_element",
+ "text": "<span class=\"pref-toggle-heading\">Optional cookies</span> intercepts pointer events",
+ "baseline": "absent"
+ }
+ ],
+ "turns": [
+ {
+ "turn": 0,
+ "kind": "repair",
+ "model": null,
+ "user": "CURRENT WORKFLOW YAML:\n\n\nThe user's current workflow definition is:\n\n```\nworkflow_definition:\n parameters: []\n blocks:\n - block_type: code\n label: read_order_documents\n code: |\n await page.goto(\"https://documents.fixture.test/orders?intercepted_toggle=1\")\n reject = page.get_by_text(\"Reject All\")\n try:\n await reject.wait_for(state=\"visible\", timeout=5000)\n await reject.click(timeout=3000)\n except Exception:\n pass\n rows = page.locator(\"#order-documents tbody tr\")\n return {\"documents\": await rows.all_text_contents()}\n\n```\n\n\n\nWorkflow block summary:\n\n```\n- read_order_documents (code)\n```\n\nUse this summary as a block-label index, then inspect the full workflow definition above for exact block details.\n\n\n\n\n\n\n\n\n\n\n\n\nPREVIOUS CONTEXT:\n\n\n\n\n\n\n\n\nNo previous context available.\n\n\nDEBUGGER RUN INFORMATION:\n\n\nThe user has run the workflow in the debugger. Here's the most recent block execution information:\n\n```\n{\n \"data\": {\n \"build_test_packet\": {\n \"contract_version\": \"build_test_evidence_packet_v1\",\n \"workflow_permanent_id\": \"wpid-caught-intercepted-click\",\n \"canonical_workflow_yaml\": \"workflow_definition:\\n parameters: []\\n blocks:\\n - block_type: code\\n label: read_order_documents\\n code: |\\n await page.goto(\\\"https://documents.fixture.test/orders?intercepted_toggle=1\\\")\\n reject = page.get_by_text(\\\"Reject All\\\")\\n try:\\n await reject.wait_for(state=\\\"visible\\\", timeout=5000)\\n await reject.click(timeout=3000)\\n except Exception:\\n pass\\n rows = page.locator(\\\"#order-documents tbody tr\\\")\\n return {\\\"documents\\\": await rows.all_text_contents()}\\n\",\n \"canonical_workflow_source\": \"turn_start_persisted_readback\",\n \"canonical_workflow_yaml_complete\": true,\n \"attempted_block_labels\": [\n \"read_order_documents\"\n ],\n \"executed_block_labels\": [\n \"read_order_documents\"\n ],\n \"run\": {\n \"workflow_run_id\": \"wr_caught_intercepted_click\",\n \"browser_session_id\": \"pbs_caught_intercepted_click\",\n \"status\": \"completed\"\n },\n \"action_observations\": [\n \"goto_url completed\",\n \"click failed code_line=5\"\n ],\n \"observed_block_end_urls\": {},\n \"per_block_action_observations\": {\n \"read_order_documents\": [\n \"goto_url completed\",\n \"click failed code_line=5\"\n ]\n },\n \"page_state\": {\n \"current_origin\": \"https://documents.fixture.test\",\n \"current_url\": \"https://documents.fixture.test/orders\",\n \"title\": \"Order Documents Portal\",\n \"evidence_source\": \"inspect_page_for_composition\",\n \"observed_after_workflow_run\": true,\n \"rendered_value_excerpt\": \"Order Documents Portal \\u2713 Success! Browser check passed. Cookie preferences We use optional cookies for analytics. Make your choice to continue to your order documents. Reject All Optional cookies\",\n \"form_summaries\": [],\n \"value_bindings\": [\n \"Reject All=Optional cookies\"\n ],\n \"result_summaries\": [\n \"DOC-58231 Purchase confirmation 2026-06-02 Available\",\n \"DOC-58232 Shipping manifest 2026-06-05 Available\",\n \"DOC-58233 Warranty certificate 2026-06-11 Processing\"\n ],\n \"action_summaries\": [],\n \"challenge_summaries\": [],\n \"obstruction_summaries\": [\n \"modal_overlay Cookie preferences We use optional cookies for analytics. Make your choice to continue to your order documents. Reject All Optional cookies obstruction present, no dismiss control found in page evidence\"\n ],\n \"obstructions\": [\n {\n \"kind\": \"modal_overlay\",\n \"source\": \"dom_html\",\n \"selector_candidates\": [\n {\n \"selector\": \"#terms-modal\",\n \"source\": \"id\"\n },\n {\n \"selector\": \"form.modal\",\n \"source\": \"class\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Cookie preferences\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Reject All\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Optional cookies\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Cookie preferences\\\") form.modal\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Reject All\\\") form.modal\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Optional cookies\\\") form.modal\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"#terms-overlay > form:nth-of-type(1)\",\n \"source\": \"structural\"\n }\n ],\n \"identity\": {\n \"tag\": \"form\",\n \"role\": \"dialog\",\n \"label_context\": \"Cookie preferences\"\n },\n \"text\": \"Cookie preferences We use optional cookies for analytics. Make your choice to continue to your order documents. Reject All Optional cookies\",\n \"visible_controls\": []\n }\n ]\n },\n \"page_capture\": {\n \"status\": \"captured\"\n },\n \"requested_outputs\": [],\n \"registered_outputs\": [\n {\n \"label\": \"read_order_documents\",\n \"status\": \"completed\",\n \"output\": {\n \"documents\": [\n \"\\n DOC-58231\\n Purchase confirmation\\n 2026-06-02\\n Available\\n \",\n \"\\n DOC-58232\\n Shipping manifest\\n 2026-06-05\\n Available\\n \",\n \"\\n DOC-58233\\n Warranty certificate\\n 2026-06-11\\n Processing\\n \"\n ]\n },\n \"value_complete\": true\n }\n ],\n \"downloads\": [],\n \"screenshot\": {\n \"present\": false\n },\n \"unfinished_items\": [],\n \"omission_notices\": [\n \"failure omitted: no failed run or failed block was recorded.\",\n \"downloads empty: no registered download artifact was recorded.\",\n \"screenshot omitted: no final or failed-block screenshot was recorded.\",\n \"unfinished_items empty: recorded outcome and workflow evidence identify none.\",\n \"observed_block_end_urls empty: no per-block end URL was recorded for this run.\"\n ],\n \"levers\": [],\n \"challenge_notices\": []\n },\n \"workflow_run_id\": \"wr_caught_intercepted_click\",\n \"browser_session_id\": \"pbs_caught_intercepted_click\",\n \"overall_status\": \"completed\",\n \"requested_block_labels\": [\n \"read_order_documents\"\n ],\n \"executed_block_labels\": [\n \"read_order_documents\"\n ],\n \"post_run_page_evidence\": {\n \"inspected_url\": \"https://documents.fixture.test/orders?intercepted_toggle=1\",\n \"current_url\": \"https://documents.fixture.test/orders?intercepted_toggle=1\",\n \"page_title\": \"Order Documents Portal\",\n \"forms\": [\n {\n \"id\": \"terms-modal\",\n \"name\": \"\",\n \"action\": \"\",\n \"method\": \"\",\n \"fields\": [],\n \"submit_controls\": []\n }\n ],\n \"navigation_targets_truncated\": false,\n \"navigation_targets\": [],\n \"result_containers\": [\n {\n \"tag\": \"table\",\n \"id\": \"order-documents\",\n \"visible\": false,\n \"headers\": [\n {\n \"text\": \"Document ID\",\n \"column_index\": 0\n },\n {\n \"text\": \"Title\",\n \"column_index\": 1\n },\n {\n \"text\": \"Order date\",\n \"column_index\": 2\n },\n {\n \"text\": \"Status\",\n \"column_index\": 3\n }\n ],\n \"row_count\": 3,\n \"rows_truncated\": false,\n \"span_free\": true,\n \"nested_table_free\": true,\n \"rows\": [\n {\n \"row_index\": 0,\n \"visible\": false,\n \"has_row_header\": false,\n \"cells\": [\n {\n \"column_index\": 0,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"DOC-58231\"\n },\n {\n \"column_index\": 1,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"Purchase confirmation\"\n },\n {\n \"column_index\": 2,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"2026-06-02\"\n },\n {\n \"column_index\": 3,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"Available\"\n }\n ]\n },\n {\n \"row_index\": 1,\n \"visible\": false,\n \"has_row_header\": false,\n \"cells\": [\n {\n \"column_index\": 0,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"DOC-58232\"\n },\n {\n \"column_index\": 1,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"Shipping manifest\"\n },\n {\n \"column_index\": 2,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"2026-06-05\"\n },\n {\n \"column_index\": 3,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"Available\"\n }\n ]\n },\n {\n \"row_index\": 2,\n \"visible\": false,\n \"has_row_header\": false,\n \"cells\": [\n {\n \"column_index\": 0,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"DOC-58233\"\n },\n {\n \"column_index\": 1,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"Warranty certificate\"\n },\n {\n \"column_index\": 2,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"2026-06-11\"\n },\n {\n \"column_index\": 3,\n \"visible\": false,\n \"has_text\": true,\n \"text\": \"Processing\"\n }\n ]\n }\n ],\n \"sample_rows\": [\n \"DOC-58231 Purchase confirmation 2026-06-02 Available\",\n \"DOC-58232 Shipping manifest 2026-06-05 Available\",\n \"DOC-58233 Warranty certificate 2026-06-11 Processing\"\n ],\n \"selector_candidates\": [\n {\n \"selector\": \"#order-documents\",\n \"source\": \"id\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"table:has-text(\\\"Document ID\\\")\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"table:has-text(\\\"Title\\\")\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"table:has-text(\\\"Order date\\\")\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"table:has-text(\\\"Status\\\")\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"section.card:has-text(\\\"Your order documents\\\") table\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"section.card:has-text(\\\"Document ID\\\") table\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"section.card:has-text(\\\"Title\\\") table\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"section.card:has-text(\\\"Order date\\\") table\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"main:has-text(\\\"Checking your browser before granting access\\u2026\\\") table\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"main:has-text(\\\"Success! Browser check passed.\\\") table\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"main:has-text(\\\"Your order documents\\\") table\",\n \"source\": \"text_anchor\",\n \"visible_match_count\": 0\n },\n {\n \"selector\": \"#documents-card > table:nth-of-type(1)\",\n \"source\": \"structural\",\n \"visible_match_count\": 0\n }\n ],\n \"identity\": {\n \"tag\": \"table\",\n \"role\": \"\",\n \"label_context\": \"Your order documents\"\n }\n }\n ],\n \"result_containers_truncated\": false,\n \"key_value_relations\": [\n {\n \"key_text\": \"Reject All\",\n \"value_text\": \"Optional cookies\",\n \"container_position\": 0,\n \"value_child_index\": 1,\n \"direct_child_count\": 2,\n \"visible\": true,\n \"value_visible\": true,\n \"key_text_walked_count\": 1,\n \"value_text_walked_count\": 1,\n \"selector_candidates\": [\n {\n \"selector\": \"#reject-toggle\",\n \"source\": \"id\"\n },\n {\n \"selector\": \"div.pref-toggle\",\n \"source\": \"class\"\n },\n {\n \"selector\": \"div.pref-toggle:has-text(\\\"Reject All\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.pref-toggle:has-text(\\\"Optional cookies\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Cookie preferences\\\") div.pref-toggle\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Reject All\\\") div.pref-toggle\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Optional cookies\\\") div.pref-toggle\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Cookie preferences\\\") div.pref-toggle\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Reject All\\\") div.pref-toggle\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Optional cookies\\\") div.pref-toggle\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"#terms-modal > div:nth-of-type(1)\",\n \"source\": \"structural\"\n }\n ],\n \"identity\": {\n \"tag\": \"div\",\n \"role\": \"switch\",\n \"label_context\": \"Reject All\"\n }\n }\n ],\n \"key_value_relations_truncated\": false,\n \"clickable_controls\": [],\n \"visible_text_excerpt\": \"Order Documents Portal \\u2713 Success! Browser check passed. Cookie preferences We use optional cookies for analytics. Make your choice to continue to your order documents. Reject All Optional cookies\",\n \"anti_bot_indicators\": [\n \"challenge\",\n \"human-verification\"\n ],\n \"challenge_controls\": [],\n \"modal_overlays\": [\n {\n \"role\": \"dialog\",\n \"aria_modal\": true,\n \"id\": \"terms-modal\",\n \"class\": \"modal\",\n \"text\": \"Cookie preferences We use optional cookies for analytics. Make your choice to continue to your order documents. Reject All Optional cookies\",\n \"dismiss_controls\": [],\n \"selector_candidates\": [\n {\n \"selector\": \"#terms-modal\",\n \"source\": \"id\"\n },\n {\n \"selector\": \"form.modal\",\n \"source\": \"class\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Cookie preferences\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Reject All\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Optional cookies\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Cookie preferences\\\") form.modal\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Reject All\\\") form.modal\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Optional cookies\\\") form.modal\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"#terms-overlay > form:nth-of-type(1)\",\n \"source\": \"structural\"\n }\n ],\n \"identity\": {\n \"tag\": \"form\",\n \"role\": \"dialog\",\n \"label_context\": \"Cookie preferences\"\n }\n }\n ],\n \"page_obstructions\": [\n {\n \"kind\": \"modal_overlay\",\n \"source\": \"dom_html\",\n \"selector_candidates\": [\n {\n \"selector\": \"#terms-modal\",\n \"source\": \"id\"\n },\n {\n \"selector\": \"form.modal\",\n \"source\": \"class\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Cookie preferences\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Reject All\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"form.modal:has-text(\\\"Optional cookies\\\")\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Cookie preferences\\\") form.modal\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Reject All\\\") form.modal\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"div.overlay:has-text(\\\"Optional cookies\\\") form.modal\",\n \"source\": \"text_anchor\"\n },\n {\n \"selector\": \"#terms-overlay > form:nth-of-type(1)\",\n \"source\": \"structural\"\n }\n ],\n \"identity\": {\n \"tag\": \"form\",\n \"role\": \"dialog\",\n \"label_context\": \"Cookie preferences\"\n },\n \"text\": \"Cookie preferences We use optional cookies for analytics. Make your choice to continue to your order documents. Reject All Optional cookies\",\n \"visible_controls\": []\n }\n ],\n \"visual_obstruction_candidates\": [\n {\n \"source\": \"computed_style\",\n \"position\": \"fixed\",\n \"coverage\": \"viewport\",\n \"has_visible_controls\": false\n }\n ],\n \"schema_empty_page\": false,\n \"observed_empty_page\": false,\n \"empty_page_visual_state\": null,\n \"evidence_confidence\": 0.3,\n \"source_tool\": \"inspect_page_for_composition\",\n \"evidence_sources\": [\n \"dom_html\"\n ],\n \"screenshot_used\": false,\n \"visual_evidence_summary\": \"\",\n \"visual_evidence_omissions\": [],\n \"inspection_warnings\": [],\n \"challenge_state\": {\n \"detected\": true,\n \"kind\": \"captcha\",\n \"source\": \"dom_html\",\n \"indicators\": [\n \"challenge\",\n \"human-verification\"\n ],\n \"requires_human_verification\": false,\n \"visual_location\": \"\",\n \"gates_submit_controls\": false,\n \"gated_submit_controls\": []\n },\n \"workflow_run_id\": \"wr_caught_intercepted_click\",\n \"source_browser_session_id\": \"pbs_caught_intercepted_click\",\n \"observed_after_workflow_run\": true\n },\n \"blocks\": [\n {\n \"workflow_run_block_id\": \"wrb_wr_caught_intercepted_click\",\n \"label\": \"read_order_documents\",\n \"block_type\": \"CODE\",\n \"status\": \"completed\",\n \"task_id\": \"tsk_caught_intercepted_click\"\n }\n ]\n },\n \"ok\": true\n}\n```\n\nUse this information to help diagnose issues, suggest fixes, or explain what might be going wrong.\nIf there's a failure, analyze the failure reason, the error code and the failing line to provide specific guidance. Look at the page with a tool if you need to see it.\n\n\n\nUSER MESSAGE:\n\nThe user says:\n\n```\nOpen https://documents.fixture.test/orders?intercepted_toggle=1 and return every row of the order documents table under the documents key.\n```\n",
+ "inputs": {
+ "kind": "repair",
+ "page_raw": {
+ "page_title": "Order Documents Portal",
+ "forms": [
+ {
+ "id": "terms-modal",
+ "name": "",
+ "action": "",
+ "method": "",
+ "fields": [],
+ "submit_controls": []
+ }
+ ],
+ "navigation_targets_truncated": false,
+ "navigation_targets": [],
+ "result_containers": [
+ {
+ "tag": "table",
+ "id": "order-documents",
+ "selector": "#order-documents",
+ "selector_candidates": [
+ {
+ "selector": "#order-documents",
+ "source": "id",
+ "match_count": 1,
+ "visible_match_count": 0,
+ "first_match_visible": false
+ },
+ {
+ "selector": "table:has-text(\"Document ID\")",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "table:has-text(\"Title\")",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "table:has-text(\"Order date\")",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "table:has-text(\"Status\")",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "section.card:has-text(\"Your order documents\") table",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "section.card:has-text(\"Document ID\") table",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "section.card:has-text(\"Title\") table",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "section.card:has-text(\"Order date\") table",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "main:has-text(\"Checking your browser before granting access\u2026\") table",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "main:has-text(\"Success! Browser check passed.\") table",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "main:has-text(\"Your order documents\") table",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 0
+ },
+ {
+ "selector": "#documents-card > table:nth-of-type(1)",
+ "source": "structural",
+ "match_count": 1,
+ "visible_match_count": 0,
+ "first_match_visible": false
+ }
+ ],
+ "identity": {
+ "tag": "table",
+ "role": "",
+ "label_context": "Your order documents"
+ },
+ "selector_match_count": 1,
+ "visible": false,
+ "is_table": true,
+ "row_count": 3,
+ "rows_truncated": false,
+ "headers": [
+ {
+ "text": "Document ID",
+ "column_index": 0
+ },
+ {
+ "text": "Title",
+ "column_index": 1
+ },
+ {
+ "text": "Order date",
+ "column_index": 2
+ },
+ {
+ "text": "Status",
+ "column_index": 3
+ }
+ ],
+ "span_free": true,
+ "nested_table_free": true,
+ "row_selector": "#order-documents > tbody > tr",
+ "rows": [
+ {
+ "row_index": 0,
+ "visible": false,
+ "has_row_header": false,
+ "cells": [
+ {
+ "column_index": 0,
+ "visible": false,
+ "has_text": true,
+ "text": "DOC-58231"
+ },
+ {
+ "column_index": 1,
+ "visible": false,
+ "has_text": true,
+ "text": "Purchase confirmation"
+ },
+ {
+ "column_index": 2,
+ "visible": false,
+ "has_text": true,
+ "text": "2026-06-02"
+ },
+ {
+ "column_index": 3,
+ "visible": false,
+ "has_text": true,
+ "text": "Available"
+ }
+ ]
+ },
+ {
+ "row_index": 1,
+ "visible": false,
+ "has_row_header": false,
+ "cells": [
+ {
+ "column_index": 0,
+ "visible": false,
+ "has_text": true,
+ "text": "DOC-58232"
+ },
+ {
+ "column_index": 1,
+ "visible": false,
+ "has_text": true,
+ "text": "Shipping manifest"
+ },
+ {
+ "column_index": 2,
+ "visible": false,
+ "has_text": true,
+ "text": "2026-06-05"
+ },
+ {
+ "column_index": 3,
+ "visible": false,
+ "has_text": true,
+ "text": "Available"
+ }
+ ]
+ },
+ {
+ "row_index": 2,
+ "visible": false,
+ "has_row_header": false,
+ "cells": [
+ {
+ "column_index": 0,
+ "visible": false,
+ "has_text": true,
+ "text": "DOC-58233"
+ },
+ {
+ "column_index": 1,
+ "visible": false,
+ "has_text": true,
+ "text": "Warranty certificate"
+ },
+ {
+ "column_index": 2,
+ "visible": false,
+ "has_text": true,
+ "text": "2026-06-11"
+ },
+ {
+ "column_index": 3,
+ "visible": false,
+ "has_text": true,
+ "text": "Processing"
+ }
+ ]
+ }
+ ],
+ "sample_rows": [
+ "DOC-58231 Purchase confirmation 2026-06-02 Available",
+ "DOC-58232 Shipping manifest 2026-06-05 Available",
+ "DOC-58233 Warranty certificate 2026-06-11 Processing"
+ ]
+ }
+ ],
+ "result_containers_truncated": false,
+ "key_value_relations": [
+ {
+ "key_text": "Reject All",
+ "selector_candidates": [
+ {
+ "selector": "#reject-toggle",
+ "source": "id",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "div.pref-toggle",
+ "source": "class",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "div.pref-toggle:has-text(\"Reject All\")",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "div.pref-toggle:has-text(\"Optional cookies\")",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "form.modal:has-text(\"Cookie preferences\") div.pref-toggle",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "form.modal:has-text(\"Reject All\") div.pref-toggle",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "form.modal:has-text(\"Optional cookies\") div.pref-toggle",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "div.overlay:has-text(\"Cookie preferences\") div.pref-toggle",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "div.overlay:has-text(\"Reject All\") div.pref-toggle",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "div.overlay:has-text(\"Optional cookies\") div.pref-toggle",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "#terms-modal > div:nth-of-type(1)",
+ "source": "structural",
+ "match_count": 1,
+ "visible_match_count": 1
+ }
+ ],
+ "identity": {
+ "tag": "div",
+ "role": "switch",
+ "label_context": "Reject All"
+ },
+ "value_text": "Optional cookies",
+ "container_selector": "#reject-toggle",
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 1,
+ "direct_child_count": 2,
+ "visible": true,
+ "value_visible": true,
+ "key_text_walked_count": 1,
+ "value_text_walked_count": 1
+ }
+ ],
+ "key_value_relations_truncated": false,
+ "reveal_relations_truncated": false,
+ "clickable_controls": [],
+ "challenge_controls": [],
+ "modal_overlays": [
+ {
+ "role": "dialog",
+ "aria_modal": true,
+ "id": "terms-modal",
+ "class": [
+ "modal"
+ ],
+ "selector": "#terms-modal",
+ "selector_candidates": [
+ {
+ "selector": "#terms-modal",
+ "source": "id",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "form.modal",
+ "source": "class",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "form.modal:has-text(\"Cookie preferences\")",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "form.modal:has-text(\"Reject All\")",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "form.modal:has-text(\"Optional cookies\")",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "div.overlay:has-text(\"Cookie preferences\") form.modal",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "div.overlay:has-text(\"Reject All\") form.modal",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "div.overlay:has-text(\"Optional cookies\") form.modal",
+ "source": "text_anchor",
+ "match_count": 1,
+ "visible_match_count": 1
+ },
+ {
+ "selector": "#terms-overlay > form:nth-of-type(1)",
+ "source": "structural",
+ "match_count": 1,
+ "visible_match_count": 1
+ }
+ ],
+ "identity": {
+ "tag": "form",
+ "role": "dialog",
+ "label_context": "Cookie preferences"
+ },
+ "text": "Cookie preferences We use optional cookies for analytics. Make your choice to continue to your order documents. Reject All Optional cookies",
+ "dismiss_controls": []
+ }
+ ],
+ "page_obstructions": [],
+ "visual_obstruction_candidates": [
+ {
+ "source": "computed_style",
+ "position": "fixed",
+ "coverage": "viewport",
+ "has_visible_controls": false
+ }
+ ],
+ "visible_text_excerpt": "Order Documents Portal\n\u2713\nSuccess! Browser check passed.\nCookie preferences\n\nWe use optional cookies for analytics. Make your choice to continue to your order documents.\n\nReject All\nOptional cookies",
+ "body_has_markup": true,
+ "anti_bot_indicators": [
+ "challenge",
+ "human-verification"
+ ],
+ "workflow_run_id": "wr_caught_intercepted_click",
+ "source_browser_session_id": "pbs_caught_intercepted_click",
+ "source_tool": "inspect_page_for_composition",
+ "observed_after_workflow_run": true,
+ "current_url": "https://documents.fixture.test/orders?intercepted_toggle=1"
+ },
+ "page_state": {
+ "url": "https://documents.fixture.test/orders?intercepted_toggle=1",
+ "storage_state": {
+ "cookies": [],
+ "origins": []
+ },
+ "content": "<!DOCTYPE html><html lang=\"en\"><head>\n <meta charset=\"utf-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n <title>Order Documents Portal</title>\n <style>\n :root {\n color-scheme: light;\n --ink: #18212f;\n --muted: #657187;\n --line: #d7dce5;\n --panel: #f5f7fb;\n --accent: #0f766e;\n --paper: #ffffff;\n }\n\n * {\n box-sizing: border-box;\n }\n\n body {\n margin: 0;\n min-height: 100vh;\n font-family: Inter, ui-sans-serif, system-ui, -apple-system, \"Segoe UI\", sans-serif;\n color: var(--ink);\n background: var(--panel);\n }\n\n header {\n display: flex;\n align-items: center;\n min-height: 64px;\n padding: 0 32px;\n border-bottom: 1px solid var(--line);\n background: var(--paper);\n font-weight: 700;\n }\n\n main {\n max-width: 860px;\n margin: 32px auto;\n padding: 0 16px;\n }\n\n .card {\n background: var(--paper);\n border: 1px solid var(--line);\n border-radius: 10px;\n padding: 24px;\n margin-bottom: 20px;\n }\n\n .check-status {\n display: flex;\n align-items: center;\n gap: 10px;\n font-weight: 600;\n }\n\n .check-status .ok {\n color: var(--accent);\n }\n\n table {\n width: 100%;\n border-collapse: collapse;\n }\n\n th,\n td {\n text-align: left;\n padding: 10px 12px;\n border-bottom: 1px solid var(--line);\n }\n\n .overlay {\n position: fixed;\n inset: 0;\n background: rgba(24, 33, 47, 0.55);\n display: flex;\n align-items: center;\n justify-content: center;\n z-index: 60;\n }\n\n .modal {\n background: var(--paper);\n border-radius: 12px;\n max-width: 460px;\n width: calc(100% - 32px);\n padding: 28px;\n box-shadow: 0 18px 44px rgba(24, 33, 47, 0.28);\n }\n\n .modal h2 {\n margin-top: 0;\n font-size: 19px;\n }\n\n .modal p {\n color: var(--muted);\n font-size: 14px;\n line-height: 1.5;\n }\n\n .consent-row {\n display: flex;\n gap: 10px;\n align-items: flex-start;\n position: relative;\n margin: 18px 0;\n font-size: 14px;\n }\n\n .partial-coverer {\n position: absolute;\n top: -5px;\n left: -5px;\n z-index: 2;\n width: 30px;\n height: 30px;\n border: 1px solid #b45309;\n border-radius: 6px;\n background: #fbbf24;\n cursor: pointer;\n }\n\n .pref-toggle {\n position: relative;\n display: block;\n min-width: 260px;\n margin-top: 16px;\n padding: 10px 14px;\n border: 1px solid var(--line);\n border-radius: 18px;\n cursor: pointer;\n font-size: 14px;\n }\n\n .pref-toggle-heading {\n position: absolute;\n inset: 0;\n display: flex;\n align-items: center;\n justify-content: flex-end;\n padding-right: 14px;\n color: var(--muted);\n }\n\n button[type=\"submit\"] {\n width: 100%;\n padding: 11px 16px;\n font-size: 15px;\n font-weight: 600;\n color: white;\n background: var(--accent);\n border: 0;\n border-radius: 8px;\n cursor: pointer;\n }\n\n button[type=\"submit\"]:disabled {\n background: #9db4b1;\n cursor: not-allowed;\n }\n\n [hidden] {\n display: none !important;\n }\n </style>\n </head>\n <body>\n <header>Order Documents Portal</header>\n <main>\n <section class=\"card\" id=\"bot-check-card\">\n <div class=\"check-status\" id=\"bot-check-status\" hidden=\"\">\n <span aria-hidden=\"true\">\u23f3</span>\n <span>Checking your browser before granting access\u2026</span>\n </div>\n <div class=\"check-status\" id=\"bot-check-passed\">\n <span class=\"ok\" aria-hidden=\"true\">\u2713</span>\n <span class=\"ok\">Success! Browser check passed.</span>\n </div>\n <div id=\"verification-widget-wrapper\" hidden=\"\">\n <div id=\"verification-widget\" class=\"challenge-widget human-verification\" data-state=\"solved\"></div>\n </div>\n </section>\n\n <section class=\"card\" id=\"documents-card\" hidden=\"\">\n <h1>Your order documents</h1>\n <table id=\"order-documents\">\n <thead>\n <tr>\n <th>Document ID</th>\n <th>Title</th>\n <th>Order date</th>\n <th>Status</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <td>DOC-58231</td>\n <td>Purchase confirmation</td>\n <td>2026-06-02</td>\n <td>Available</td>\n </tr>\n <tr>\n <td>DOC-58232</td>\n <td>Shipping manifest</td>\n <td>2026-06-05</td>\n <td>Available</td>\n </tr>\n <tr>\n <td>DOC-58233</td>\n <td>Warranty certificate</td>\n <td>2026-06-11</td>\n <td>Processing</td>\n </tr>\n </tbody>\n </table>\n </section>\n </main>\n\n <div class=\"overlay\" id=\"terms-overlay\">\n <form class=\"modal\" id=\"terms-modal\" role=\"dialog\" aria-modal=\"true\" aria-labelledby=\"terms-title\">\n <h2 id=\"terms-title\">Cookie preferences</h2>\n <p>We use optional cookies for analytics. Make your choice to continue to your order documents.</p>\n \n \n <div class=\"pref-toggle\" id=\"reject-toggle\" role=\"switch\" aria-checked=\"false\">\n <span class=\"pref-toggle-label\">Reject All</span>\n <span class=\"pref-toggle-heading\">Optional cookies</span>\n </div>\n </form>\n </div>\n\n <script>\n (function () {\n var acceptTerms = document.getElementById(\"accept-terms\");\n var btnContinue = document.getElementById(\"btn-continue\");\n var query = new URLSearchParams(window.location.search);\n var partialCover = query.get(\"partial_cover\") === \"1\";\n\n // The heading span lies over its sibling label, so a click aimed at the label lands on the heading.\n // A script-dispatched click is not trusted, so the page records whether a pointer made the choice.\n if (query.get(\"intercepted_toggle\") === \"1\") {\n var toggle = document.getElementById(\"reject-toggle\");\n document.getElementById(\"terms-title\").textContent = \"Cookie preferences\";\n document.querySelector(\"#terms-modal p\").textContent =\n \"We use optional cookies for analytics. Make your choice to continue to your order documents.\";\n acceptTerms.parentElement.remove();\n btnContinue.remove();\n toggle.hidden = false;\n toggle.addEventListener(\"click\", function (event) {\n sessionStorage.setItem(\n \"cookie_choice_click\",\n JSON.stringify({ trusted: event.isTrusted, receiver: event.currentTarget.id }),\n );\n toggle.setAttribute(\"aria-checked\", \"true\");\n document.getElementById(\"terms-overlay\").hidden = true;\n document.getElementById(\"documents-card\").hidden = false;\n });\n }\n\n setTimeout(function () {\n document.getElementById(\"bot-check-status\").hidden = true;\n document.getElementById(\"bot-check-passed\").hidden = false;\n document.getElementById(\"terms-overlay\").hidden = false;\n if (partialCover) {\n var coverer = document.createElement(\"div\");\n coverer.id = \"target-coverer\";\n coverer.className = \"partial-coverer\";\n coverer.tabIndex = 0;\n coverer.setAttribute(\"role\", \"button\");\n coverer.setAttribute(\"aria-label\", \"Dismiss target cover\");\n var dismissCoverer = function (event) {\n event.preventDefault();\n event.stopPropagation();\n coverer.remove();\n };\n coverer.addEventListener(\"click\", dismissCoverer);\n coverer.addEventListener(\"keydown\", function (event) {\n if (event.key === \"Enter\" || event.key === \" \") {\n dismissCoverer(event);\n }\n });\n acceptTerms.parentElement.appendChild(coverer);\n }\n }, 1200);\n\n acceptTerms.addEventListener(\"change\", function () {\n btnContinue.disabled = !acceptTerms.checked;\n });\n\n document.getElementById(\"terms-modal\").addEventListener(\"submit\", function (event) {\n event.preventDefault();\n if (!acceptTerms.checked) {\n return;\n }\n document.getElementById(\"terms-overlay\").hidden = true;\n document.getElementById(\"documents-card\").hidden = false;\n });\n })();\n </script>\n \n\n</body></html>"
+ },
+ "code": "await page.goto(\"https://documents.fixture.test/orders?intercepted_toggle=1\")\nreject = page.get_by_text(\"Reject All\")\ntry:\n await reject.wait_for(state=\"visible\", timeout=5000)\n await reject.click(timeout=3000)\nexcept Exception:\n pass\nrows = page.locator(\"#order-documents tbody tr\")\nreturn {\"documents\": await rows.all_text_contents()}\n",
+ "result": {
+ "ok": true,
+ "data": {
+ "workflow_run_id": "wr_caught_intercepted_click",
+ "browser_session_id": "pbs_caught_intercepted_click",
+ "overall_status": "completed",
+ "requested_block_labels": [
+ "read_order_documents"
+ ],
+ "executed_block_labels": [
+ "read_order_documents"
+ ],
+ "post_run_page_evidence": {},
+ "blocks": [
+ {
+ "workflow_run_block_id": "wrb_wr_caught_intercepted_click",
+ "label": "read_order_documents",
+ "block_type": "CODE",
+ "status": "completed",
+ "extracted_data": {
+ "documents": [
+ "\n DOC-58231\n Purchase confirmation\n 2026-06-02\n Available\n ",
+ "\n DOC-58232\n Shipping manifest\n 2026-06-05\n Available\n ",
+ "\n DOC-58233\n Warranty certificate\n 2026-06-11\n Processing\n "
+ ]
+ },
+ "output": {
+ "documents": [
+ "\n DOC-58231\n Purchase confirmation\n 2026-06-02\n Available\n ",
+ "\n DOC-58232\n Shipping manifest\n 2026-06-05\n Available\n ",
+ "\n DOC-58233\n Warranty certificate\n 2026-06-11\n Processing\n "
+ ]
+ },
+ "task_id": "tsk_caught_intercepted_click",
+ "action_trace": [
+ {
+ "action": "click",
+ "status": "failed",
+ "reasoning": null,
+ "element": "get_by_text(Reject All)",
+ "code_line": 5,
+ "response": "Locator.click: Timeout 3000ms exceeded.\nCall log:\nwaiting for get_by_text(\"Reject All\")\n - locator resolved to <span class=\"pref-toggle-label\">Reject All</span>\n - attempting click action\n - waiting for element to be visible, enabled and stable\n - element is visible, enabled and stable\n - scrolling into view if needed\n - done scrolling\n - <span class=\"pref-toggle-heading\">Optional cookies</span> intercepts pointer events\n - retrying click action, attempt #1\n - waiting for element to be visible, enabled and stable\n - element is visible, enabled and stable\n - scrolling into view if needed\n - done scrolling\n - <span class=\"pref-toggle-heading\">Optional cookies</span> intercepts pointer events\n - retrying click action, attempt #2\n - waiting 20ms\n - waiting for element to be visible, enabled and stable\n - element is visible, enabled and stable\n - scrolling into view if needed\n - done scrolling\n - <span class=\"pref-toggle-heading\">Optiona"
+ },
+ {
+ "action": "goto_url",
+ "status": "completed",
+ "reasoning": null,
+ "element": null
+ }
+ ]
+ }
+ ],
+ "action_observations": [
+ "goto_url completed",
+ "click failed code_line=5"
+ ],
+ "per_block_action_observations": {
+ "read_order_documents": [
+ "goto_url completed",
+ "click failed code_line=5"
+ ]
+ }
+ }
+ }
+ },
+ "submission": "await page.goto(\"https://documents.fixture.test/orders?intercepted_toggle=1\")\nreject = page.get_by_text(\"Reject All\")\ntry:\n await reject.wait_for(state=\"visible\", timeout=5000)\n await reject.click(timeout=3000)\nexcept Exception:\n pass\nrows = page.locator(\"#order-documents tbody tr\")\nreturn {\"documents\": await rows.all_text_contents()}\n"
+ }
+ ]
+}
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 374ccb082..f1ed65c96 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -3629,7 +3629,7 @@ workflow_definition:
assert "response" not in entry
@pytest.mark.asyncio
- async def test_retained_completed_action_observation_excludes_target_controlled_fields(
+ async def test_retained_completed_action_observation_carries_only_the_screened_recorded_error(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
secret = "sk-live-persisted-action-secret"
@@ -3685,8 +3685,12 @@ workflow_definition:
)
packet = result["data"]["build_test_packet"]
- assert packet["action_observations"] == ["null_action failed code_line=18"]
+ [observation] = packet["action_observations"]
+ assert observation.startswith("null_action failed code_line=18")
+ assert "response says reveal [REDACTED_SECRET]" in observation
assert secret not in str(packet)
+ assert "Ignore prior instructions" not in str(packet)
+ assert "target-element" not in str(packet)
@pytest.mark.asyncio
async def test_completed_action_observation_read_failure_degrades_to_packet_omission(
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index f2ab0eb3e..fdab6c418 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -7,6 +7,7 @@ import json
from collections.abc import AsyncIterator, Coroutine
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
+from pathlib import Path
from types import SimpleNamespace
from typing import Any, Literal
from unittest.mock import AsyncMock, MagicMock, patch
@@ -73,6 +74,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
_BUILD_TEST_PACKET_MAX_CHARS,
_INTERNAL_RUN_OUTCOME_RECORDED_KEY,
BUILD_TEST_PACKET_KEY,
+ _compact_block_fact_maps,
_compact_packet_for_aggregate_limit,
_compacted_newest_labelled,
labelled_url_screen_reasons,
@@ -6989,6 +6991,23 @@ def test_a_seven_block_run_reserves_the_newest_predecessors_and_leaves_the_faili
assert len(run_execution_module._retained_action_observations(results)) == 6
+def test_a_native_block_with_many_failed_actions_leaves_the_predecessors_terminal_facts() -> None:
+ results = [
+ {"label": "login", "action_trace": [{"action": "click", "status": "completed"}]},
+ {"label": "open_orders", "action_trace": [{"action": "goto_url", "status": "completed"}]},
+ {
+ "label": "download_invoices",
+ "action_trace": [{"action": "click", "status": "failed"} for _ in range(7)],
+ },
+ ]
+
+ by_label = run_execution_module._retained_action_observations_by_label(results)
+
+ assert by_label["login"] == ["click completed"]
+ assert by_label["open_orders"] == ["goto_url completed"]
+ assert by_label["download_invoices"] == ["click failed"] * 4
+
+
def test_predecessor_blocks_the_action_budget_drops_are_named_in_an_omission_notice() -> None:
results = [
{
@@ -7635,6 +7654,473 @@ async def test_a_secret_shaped_token_in_the_code_error_never_reaches_the_run_res
assert "[REDACTED_SECRET]" in ai_fallback["failure_text"]
+_CAUGHT_CLICK_INTERCEPT = '<span class="cc-toggle-heading">…</span> intercepts pointer events'
+_CAPTURED_CAUGHT_CLICK_ERROR = json.loads(
+ (Path(__file__).parent / "fixtures/copilot/caught_failure_repair/consent_toggle_caught_click.json").read_text()
+)["turns"][0]["inputs"]["result"]["data"]["blocks"][0]["action_trace"][0]["response"]
+_CAUGHT_CLICK_ERROR = "\n".join(
+ [
+ "Locator.click: Timeout 4000ms exceeded.",
+ "Call log:",
+ ' - waiting for get_by_text("Reject All").first',
+ ' - locator resolved to <span class="cc-toggle-label">Reject All</span>',
+ " - attempting click action",
+ " 2 × waiting for element to be visible, enabled and stable",
+ " - element is visible, enabled and stable",
+ " - scrolling into view if needed",
+ " - done scrolling",
+ f" - {_CAUGHT_CLICK_INTERCEPT}",
+ " - retrying click action",
+ " - waiting 20ms",
+ " 2 × waiting for element to be visible, enabled and stable",
+ " - element is visible, enabled and stable",
+ " - scrolling into view if needed",
+ " - done scrolling",
+ f" - {_CAUGHT_CLICK_INTERCEPT}",
+ " - retrying click action",
+ " - waiting 100ms",
+ ]
+)
+
+
+_DISABLED_FIELD_VALUE = "harbor lights 2291"
+_DISABLED_FIELD_FILL_ERROR = "\n - ".join(
+ [
+ 'Locator.fill: Timeout 3000ms exceeded.\nCall log:\nwaiting for locator("#account-note")',
+ ' locator resolved to <input disabled id="account-note"/>',
+ f' fill("{_DISABLED_FIELD_VALUE}")',
+ "attempting fill action",
+ " waiting for element to be visible, enabled and editable",
+ " element is not enabled",
+ "retrying fill action, attempt #1",
+ ]
+)
+# Playwright 1.58 indents call-log lines and prefixes repeats with a count. Its element preview carries the input's
+# value and cuts a long attribute with "…" and no closing quote.
+_DISABLED_FIELD_FILL_ERROR_1_58 = "\n".join(
+ [
+ "Locator.fill: Timeout 3000ms exceeded.",
+ "Call log:",
+ ' - waiting for locator("#account-note")',
+ f' - locator resolved to <input disabled class="{" ".join(["form-field"] * 30)}" '
+ f'value="{"x" * 470}{_DISABLED_FIELD_VALUE}…/>',
+ f' - fill("{_DISABLED_FIELD_VALUE}")',
+ " - attempting fill action",
+ " 2 × waiting for element to be visible, enabled and editable",
+ " - element is not enabled",
+ " - retrying fill action",
+ ]
+)
+# A multi-line value is echoed raw, so its later lines carry no call-log prefix.
+_DISABLED_FIELD_FILL_ERROR_MULTILINE = "\n".join(
+ [
+ "Locator.fill: Timeout 3000ms exceeded.",
+ "Call log:",
+ 'waiting for locator("#account-note")',
+ ' - locator resolved to <textarea disabled id="account-note"></textarea>',
+ f' - fill("Account note\n{_DISABLED_FIELD_VALUE}")',
+ " - attempting fill action",
+ " - waiting for element to be visible, enabled and editable",
+ " - element is not enabled",
+ ]
+)
+
+
+async def _caught_click_run(
+ monkeypatch: pytest.MonkeyPatch,
+ response: str,
+ action_type: ActionType = ActionType.CLICK,
+ heal_episodes: list[HealEpisode] | None = None,
+) -> tuple[dict[str, Any], CopilotContext]:
+ """A completed block whose program caught one failed page call and then made seven more calls, newest first as
+ the repository returns them."""
+ caught = run_result_action_row("tsk_code", action_type, ActionStatus.failed, code_line=7)
+ caught.response = response
+ later = [
+ run_result_action_row("tsk_code", ActionType.CLICK, ActionStatus.completed, code_line=line)
+ for line in range(14, 7, -1)
+ ]
+ await install_run_blocks_harness(
+ monkeypatch,
+ workflow_yaml=HANDBACK_WORKFLOW_YAML,
+ polled_status="completed",
+ terminal_blocks=[terminal_extraction_block("completed", task_id="tsk_code")],
+ recent_actions=[*later, caught],
+ )
+ if heal_episodes:
+ monkeypatch.setattr(
+ run_execution_module.app.DATABASE.self_heal,
+ "get_heal_episodes_for_run",
+ AsyncMock(return_value=heal_episodes),
+ )
+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")
+ ctx.frontier_resume_session_id = "pbs_run"
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+ return result, ctx
+
+
+async def test_a_value_a_failed_fill_typed_stays_out_of_a_test_runs_extracted_data(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ block = terminal_extraction_block("failed", failure_reason=_DISABLED_FIELD_FILL_ERROR_1_58, task_id="tsk_code")
+ block.output = {
+ "status": "failed",
+ "failure_reason": _DISABLED_FIELD_FILL_ERROR_1_58,
+ "errors": [{"error_code": "page_operation_failed", "reasoning": _DISABLED_FIELD_FILL_ERROR_1_58}],
+ }
+ await install_run_blocks_harness(
+ monkeypatch, workflow_yaml=HANDBACK_WORKFLOW_YAML, polled_status="failed", terminal_blocks=[block]
+ )
+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")
+ ctx.frontier_resume_session_id = "pbs_run"
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+ assert _DISABLED_FIELD_VALUE not in json.dumps(result, default=str)
+ assert "element is not enabled" in json.dumps(result["data"]["blocks"][0]["extracted_data"])
+
+
+async def test_a_caught_page_failure_in_a_block_the_ai_fallback_rescued_during_a_test_run_is_not_reported_as_continued(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ rescued = _ai_fallback_episode(HealStatus.fired_completed, failing_line=7).model_copy(
+ update={"workflow_run_block_id": "wrb_extract_heading"}
+ )
+ result, ctx = await _caught_click_run(monkeypatch, _CAUGHT_CLICK_ERROR, heal_episodes=[rescued])
+
+ projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
+
+ failed = [observation for observation in projected.action_observations if observation.startswith("click failed")]
+ assert len(failed) == 1 and _CAUGHT_CLICK_INTERCEPT in failed[0], projected.action_observations
+ assert "continued" not in failed[0]
+
+
+async def test_a_page_failure_the_program_caught_reaches_the_packet_with_its_error_and_that_the_block_went_on(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ result, ctx = await _caught_click_run(monkeypatch, _CAUGHT_CLICK_ERROR)
+
+ projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
+
+ failed = [observation for observation in projected.action_observations if observation.startswith("click failed")]
+ assert len(failed) == 1, projected.action_observations
+ for fact in ("code_line=7", _CAUGHT_CLICK_INTERCEPT, "Timeout 4000ms exceeded", "continued"):
+ assert fact in failed[0]
+ assert len(failed[0]) <= 300
+ assert failed[0] in projected.per_block_action_observations["extract_heading"]
+ assert projected.action_observations.count("click completed") == 5
+
+
+async def test_a_secret_shaped_token_in_a_caught_page_failure_stays_out_of_the_packet(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Joined at runtime so no source line holds a whole JWT for secret scanners to flag.
+ token = ".".join(("eyJhbGciOiJIUzI1NiJ9", "eyJzdWIiOiJ0ZXN0LXVzZXIifQ", "c2lnbmF0dXJlLXZhbHVlLXg"))
+ result, ctx = await _caught_click_run(
+ monkeypatch,
+ f"Locator.click: Timeout 4000ms exceeded.\nCall log:\n - waiting for locator(\"#s[value='{token}']\")",
+ )
+
+ projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
+ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)
+
+ assert token not in projected.model_dump_json()
+ assert token not in json.dumps(sanitized)
+ assert any("[REDACTED_SECRET]" in observation for observation in projected.action_observations)
+
+
+ "fill_error",
+ [_DISABLED_FIELD_FILL_ERROR, _DISABLED_FIELD_FILL_ERROR_1_58, _DISABLED_FIELD_FILL_ERROR_MULTILINE],
+ ids=["playwright-1.46", "playwright-1.58", "multi-line-value"],
+)
+async def test_a_value_a_caught_fill_typed_stays_out_of_the_packet_while_its_cause_reaches_it(
+ monkeypatch: pytest.MonkeyPatch, fill_error: str
+) -> None:
+ result, ctx = await _caught_click_run(monkeypatch, fill_error, ActionType.INPUT_TEXT)
+
+ projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
+
+ assert _DISABLED_FIELD_VALUE not in projected.model_dump_json()
+ assert _DISABLED_FIELD_VALUE not in json.dumps(sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result))
+ failed = [observation for observation in projected.action_observations if observation.startswith("input_text")]
+ assert len(failed) == 1 and "element is not enabled" in failed[0], projected.action_observations
+
+
+ "password", ["hunter2pass", "hun'ter2pass"], ids=["plain-password", "password-with-apostrophe"]
+)
+async def test_credentials_and_query_in_a_caught_navigation_failure_stay_out_of_the_packet(
+ monkeypatch: pytest.MonkeyPatch, password: str
+) -> None:
+ url = f"https://alice:{password}@nohost.invalid/path?token=abc123#frag"
+ navigation_error = (
+ f'Page.goto: net::ERR_NAME_NOT_RESOLVED at {url}\nCall log:\nnavigating to "{url}", waiting until "load"\n'
+ )
+ result, ctx = await _caught_click_run(monkeypatch, navigation_error, ActionType.GOTO_URL)
+
+ projected = project_build_test_packet_for_llm(build_test_evidence_packet(ctx, result))
+
+ for secret in ("ter2pass", "token=abc123"):
+ assert secret not in projected.model_dump_json()
+ failed = [observation for observation in projected.action_observations if observation.startswith("goto_url")]
+ assert len(failed) == 1 and "ERR_NAME_NOT_RESOLVED" in failed[0], projected.action_observations
+
+
+async def test_a_value_a_failed_fill_typed_stays_out_of_the_ai_fallback_code_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _install_rescued_run(
+ monkeypatch,
+ [_ai_fallback_episode(HealStatus.fired_completed, failing_line=5)],
+ code_error=_DISABLED_FIELD_FILL_ERROR,
+ )
+
+ page = await _run_results_page(ctx)
+
+ assert _DISABLED_FIELD_VALUE not in json.dumps(page)
+ ai_fallback = next(row for row in page["data"]["blocks"] if row["label"] == "collect_metrics")["ai_fallback"]
+ assert "element is not enabled" in ai_fallback["failure_text"]
+
+
+# Typed lines that look like call-log lines: a bulleted note (1.46 format) and a count-prefixed one (1.58 format).
+_BULLETED_NOTE_FILL_ERROR = "\n".join(
+ [
+ "Locator.fill: Timeout 3000ms exceeded.",
+ "Call log:",
+ 'waiting for locator("#account-note")',
+ ' - locator resolved to <textarea disabled id="account-note"></textarea>',
+ ' - fill("Call back Monday\n- SYNTHETIC two\n- SYNTHETIC three")',
+ " - attempting fill action",
+ " - waiting for element to be visible, enabled and editable",
+ " - element is not enabled",
+ ]
+)
+_COUNTED_NOTE_FILL_ERROR = "\n".join(
+ [
+ "Locator.fill: Timeout 3000ms exceeded.",
+ "Call log:",
+ ' - waiting for locator("#account-note")',
+ ' - locator resolved to <textarea disabled id="account-note"></textarea>',
+ ' - fill("first\n3 × SYNTHETIC\nlast")',
+ " - attempting fill action",
+ " 2 × waiting for element to be visible, enabled and editable",
+ " - element is not enabled",
+ ]
+)
+_STEP_PHRASE_NOTE_FILL_ERROR = "\n".join(
+ [
+ "Locator.fill: Timeout 3000ms exceeded.",
+ "Call log:",
+ 'waiting for locator("#account-note")',
+ ' - locator resolved to <textarea disabled id="account-note"></textarea>',
+ ' - fill("first\nattempting click action\nprivate customer text")',
+ " - attempting fill action",
+ " - waiting for element to be visible, enabled and editable",
+ " - element is not enabled",
+ ]
+)
+_EMBEDDED_CLOSER_NOTE_FILL_ERROR = _STEP_PHRASE_NOTE_FILL_ERROR.replace('fill("first\n', 'fill("first")\n')
+_PARAMETER_NOTE_FILL_ERROR = _STEP_PHRASE_NOTE_FILL_ERROR.replace(
+ "attempting click action\nprivate customer text",
+ "retrying secretword action, attempt #987654321\n999 × waiting 123456ms\nlast",
+)
+
+
+ ("fill_error", "typed_text"),
+ [
+ (_DISABLED_FIELD_FILL_ERROR_1_58, [_DISABLED_FIELD_VALUE]),
+ (_BULLETED_NOTE_FILL_ERROR, ["Call back Monday", "SYNTHETIC"]),
+ (_COUNTED_NOTE_FILL_ERROR, ["SYNTHETIC", 'last")']),
+ (_STEP_PHRASE_NOTE_FILL_ERROR, ["private customer text"]),
+ (_EMBEDDED_CLOSER_NOTE_FILL_ERROR, ["private customer text"]),
+ (_PARAMETER_NOTE_FILL_ERROR, ["secretword", "987654321", "123456"]),
+ ],
+ ids=[
+ "playwright-1.58",
+ "bulleted-note",
+ "count-prefixed-note",
+ "step-phrase-note",
+ "embedded-closer-note",
+ "step-parameters-note",
+ ],
+)
+def test_a_value_a_failed_fill_typed_stays_out_of_the_block_failure_reason_and_output(
+ fill_error: str, typed_text: list[str]
+) -> None:
+ block = terminal_extraction_block("failed", failure_reason=fill_error)
+ block.output = {
+ "status": "failed",
+ "failure_reason": fill_error,
+ "errors": [{"error_code": "page_operation_failed", "reasoning": fill_error}],
+ }
+
+ result = _recorded_run_block_result(block)
+
+ for text in typed_text:
+ assert text not in json.dumps(result)
+ assert "element is not enabled" in result["failure_reason"]
+ assert "element is not enabled" in result["output"]["errors"][0]["reasoning"]
+
+
+async def test_a_caught_page_failure_is_not_reported_as_continued_when_the_ai_fallback_record_is_unreadable(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def caught_click_observations() -> list[str]:
+ return result["data"]["per_block_action_observations"]["collect_metrics"]
+
+ ctx = _install_rescued_run(monkeypatch, [])
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+ assert any("(block continued)" in observation for observation in caught_click_observations())
+
+ ctx = _install_rescued_run(monkeypatch, [])
+ monkeypatch.setattr(
+ run_execution_module.app.DATABASE.self_heal,
+ "get_heal_episodes_for_run",
+ AsyncMock(side_effect=RuntimeError("self-heal store unavailable")),
+ )
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+ observations = caught_click_observations()
+ assert any("Timeout 3000ms exceeded." in observation for observation in observations), observations
+ assert not any("continued" in observation for observation in observations)
+
+ ctx = _install_rescued_run(monkeypatch, [_ai_fallback_episode(HealStatus.fired_completed, failing_line=5)])
+ monkeypatch.setattr(
+ run_execution_module, "build_heal_episode_detail", MagicMock(side_effect=ValueError("unreadable episode"))
+ )
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+ assert not any("continued" in observation for observation in caught_click_observations())
+
+
+def test_an_older_blocks_caught_failures_leave_the_newest_block_a_slot() -> None:
+ caught = {"action": "click", "status": "failed", "code_line": 4, "response": "Timeout 3000ms exceeded."}
+ results = [
+ {
+ "label": "dismiss_cover",
+ "status": "completed",
+ "action_trace": [{**caught, "code_line": line} for line in range(4, 11)],
+ },
+ {
+ "label": "read_orders",
+ "status": "completed",
+ "action_trace": [{"action": "click", "status": "completed"}, {"action": "goto_url", "status": "completed"}],
+ },
+ ]
+
+ by_label = run_execution_module._retained_action_observations_by_label(results)
+
+ assert by_label["read_orders"], by_label
+ assert len(by_label["dismiss_cover"]) == 5
+
+
+def test_a_failure_a_program_retried_takes_one_slot_and_leaves_the_predecessors_terminal_facts() -> None:
+ caught = {"action": "click", "status": "failed", "code_line": 4, "response": "Timeout 3000ms exceeded."}
+ predecessors = [
+ {"label": f"step_{index}", "status": "completed", "action_trace": [{"action": "click", "status": "completed"}]}
+ for index in range(5)
+ ]
+ results = [*predecessors, {"label": "dismiss_cover", "status": "completed", "action_trace": [caught] * 6}]
+
+ selection = run_execution_module._selected_action_observations(results)
+
+ by_label = run_execution_module._retained_action_observations_by_label(results)
+ assert all(by_label[f"step_{index}"] == ["click completed"] for index in range(5)), by_label
+ assert len(by_label["dismiss_cover"]) == 1
+ assert selection.unreported_predecessor_indexes == []
+
+
+async def test_a_page_failure_the_ai_fallback_recovered_is_reported_without_the_block_going_on(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _install_rescued_run(monkeypatch, [_ai_fallback_episode(HealStatus.fired_completed, failing_line=5)])
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+
+ observations = result["data"]["per_block_action_observations"]["collect_metrics"]
+ assert any("Timeout 3000ms exceeded." in observation for observation in observations), observations
+ assert not any("continued" in observation for observation in observations)
+
+
+def test_a_caught_page_failure_in_the_oldest_block_survives_the_aggregate_limit_with_its_intercepting_element() -> None:
+ results = [
+ {
+ "label": "dismiss_cover",
+ "status": "completed",
+ "action_trace": [
+ {"action": "click", "status": "completed", "code_line": 9},
+ {"action": "click", "status": "failed", "code_line": 3, "response": _CAPTURED_CAUGHT_CLICK_ERROR},
+ ],
+ },
+ {"label": "run_search", "status": "completed", "action_trace": [{"action": "click", "status": "completed"}]},
+ {
+ "label": "read_results",
+ "status": "completed",
+ "action_trace": [{"action": "scroll", "status": "completed"}, {"action": "click", "status": "completed"}],
+ },
+ ]
+ packet = build_test_evidence_packet(
+ make_copilot_ctx(),
+ {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_1",
+ "overall_status": "completed",
+ "blocks": [{"label": result["label"], "status": "completed"} for result in results],
+ "action_observations": run_execution_module._retained_action_observations(results),
+ "per_block_action_observations": run_execution_module._retained_action_observations_by_label(results),
+ },
+ },
+ )
+
+ compacted = _compact_packet_for_aggregate_limit(packet, [])
+ by_label = _compact_block_fact_maps(packet, []).per_block_action_observations
+
+ failed = [observation for observation in compacted.action_observations if observation.startswith("click failed")]
+ assert len(failed) == 1, compacted.action_observations
+ assert '<span class="pref-toggle-heading">Optional cookies</span>' in failed[0]
+ assert failed[0].endswith("...") and len(failed[0]) == 120
+ assert next(iter(compacted.per_block_action_observations)) == "dismiss_cover"
+ assert compacted.per_block_action_observations["dismiss_cover"][0].startswith("click failed")
+ assert "dismiss_cover" in by_label
+
+
+def test_a_native_failed_action_takes_no_priority_at_the_aggregate_limit() -> None:
+ packet = build_test_evidence_packet(
+ make_copilot_ctx(),
+ {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_1",
+ "overall_status": "completed",
+ "blocks": [{"label": label, "status": "completed"} for label in ("login", "open_orders", "search")],
+ "action_observations": ["click failed", "goto_url completed", "scroll completed"],
+ "per_block_action_observations": {
+ "login": ["click failed"],
+ "open_orders": ["goto_url completed"],
+ "search": ["scroll completed"],
+ },
+ },
+ },
+ )
+
+ compacted = _compact_packet_for_aggregate_limit(packet, [])
+ by_label = _compact_block_fact_maps(packet, []).per_block_action_observations
+
+ assert compacted.action_observations == ["goto_url completed", "scroll completed"]
+ assert list(compacted.per_block_action_observations) == ["open_orders", "search"]
+ assert list(by_label) == ["open_orders", "search"]
+
+
def test_the_packet_keeps_the_newest_ai_fallback_records_and_says_how_many_it_left_out() -> None:
ctx = make_copilot_ctx()
ctx.registered_artifact_evidence = None