Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Compile the attach worker's two wheel-less C extensions in a builder stage (#8436)
Shuchang Zheng8eacacc60d9c
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex 5ec2a1fa1..5afc75791 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -102,6 +102,7 @@ _UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"_BROWSER_OPERATION_FAILED: BuildTestFailedOperationKind = "browser_operation_failed"_EXECUTED_BLOCK_STATUSES = frozenset(status.value for status in BlockStatus if status != BlockStatus.skipped)_FAILED_BLOCK_STATUSES = frozenset({"failed", "terminated", "canceled", "timed_out"})+_COMPLETED_BLOCK_STATUSES = frozenset({BlockStatus.completed.value})# Sandbox-process faults, not authored-code faults. ``timeout`` and ``user_code_error`` stay# out: both are repairable despite also carrying ``runner_internal_error``. ``busy`` is in —# a saturated runner gate says nothing about the code, so rewriting it cannot help.@@ -409,6 +410,7 @@ class RecordedBuildTestOutcome(BaseModel):failed_operation_call_signature: str | None = Field(default=None, exclude=True, repr=False)failed_operation_code_signature: str | None = Field(default=None, exclude=True, repr=False)executed_block_associations: tuple[str, ...] = Field(default=(), exclude=True, repr=False)+ completed_block_associations: tuple[str, ...] = Field(default=(), exclude=True, repr=False)authored_structure_signature: str | None = Nonedisplay_text: str = ""observed_page_value_excerpt: str = ""@@ -501,6 +503,24 @@ def _history_entry(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuild}+def _releasing_associations(+ ctx: _RecordedBuildTestOutcomeContext, failed_operation: BuildTestFailedOperation+) -> set[str]:+ """Identities whose completion resolves this failure.++ A full replacement mints a fresh identity for the same label, so the recorded association can+ never run again; without the label's current identity that repair shape has no way to clear.+ """+ releasing: set[str] = set()+ if failed_operation.block_association:+ releasing.add(failed_operation.block_association)+ if failed_operation.block_label:+ current = ctx.runner_code_block_associations_by_label.get(failed_operation.block_label)+ if current:+ releasing.add(current)+ return releasing++def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome | None) -> None:if outcome is None:latest = getattr(ctx, "latest_recorded_build_test_outcome", None)@@ -523,22 +543,13 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re})elif prior is not None and prior.failed_operation is not None:- source_yaml = _executed_workflow_yaml(ctx)- association = prior.failed_operation.block_association- tested_code = _code_for_runner_association(ctx, source_yaml, association)- changed_attempt_was_tested = (+ releasing = _releasing_associations(ctx, prior.failed_operation)+ failed_block_completed_later = (outcome.phase == "persisted_block_run"- and outcome.verdict == "progress_observed"and outcome.workflow_run_id not in (None, prior.failed_operation.workflow_run_id)- and association is not None- and association in outcome.executed_block_associations- and _failed_operation_changed(- tested_code,- prior.failed_operation_call_signature,- prior.failed_operation_code_signature,- )+ and bool(releasing & set(outcome.completed_block_associations)))- if not changed_attempt_was_tested:+ if not failed_block_completed_later:outcome = outcome.model_copy(update={"failed_operation": prior.failed_operation,@@ -729,27 +740,6 @@ def _failed_operation_call_signature(code: str | None, failing_line: int | None)return _stable_hash(ast.dump(outermost, include_attributes=False))-def _failed_operation_changed(- tested_code: str | None,- prior_call_signature: str | None,- prior_code_signature: str | None,-) -> bool:- """Prove the recorded operation changed, or use whole-block change for a typed line omission."""- if not tested_code:- return False- wrapper = _parse_block_code(tested_code)- if wrapper is None:- return False- current_call_signatures = {- _stable_hash(ast.dump(node, include_attributes=False))- for node in ast.walk(wrapper)- if isinstance(node, ast.Call)- }- if prior_call_signature is not None:- return prior_call_signature not in current_call_signatures- return prior_code_signature is not None and _stable_hash(tested_code) != prior_code_signature--# Every construct that can reach the end of a block without running something inside it, including# expression-level ones: a zero-length comprehension skips its body exactly as an unentered loop does.# Over-detection only costs a redundant note; a miss silently drops a real failure.@@ -1405,14 +1395,8 @@ def recorded_outcome_from_run_blocks_result(if (redacted := _redacted_terminal_text(_safe_str(block.get("label"))))])- executed_block_associations = tuple(- dict.fromkeys(- association- for block in blocks- if _safe_str(block.get("status")) in _EXECUTED_BLOCK_STATUSES- if (association := (block_associations_by_label or {}).get(_safe_str(block.get("label"))))- )- )+ executed_block_associations = _block_associations(blocks, block_associations_by_label, _EXECUTED_BLOCK_STATUSES)+ completed_block_associations = _block_associations(blocks, block_associations_by_label, _COMPLETED_BLOCK_STATUSES)block_shape_hashes = dict(block_shape_hashes or {})referenced_unbound_keys = _referenced_unbound_input_keys(result,@@ -1493,6 +1477,7 @@ def recorded_outcome_from_run_blocks_result(executed_block_labels=executed_block_labels,block_shape_hashes=block_shape_hashes,executed_block_associations=executed_block_associations,+ completed_block_associations=completed_block_associations,page_capture=page_capture,authored_structure_signature=authored_structure_signature,failed_operation=failed_operation,@@ -1511,6 +1496,7 @@ def recorded_outcome_from_run_blocks_result(executed_block_labels=executed_block_labels,block_shape_hashes=block_shape_hashes,executed_block_associations=executed_block_associations,+ completed_block_associations=completed_block_associations,verified_progress_marker=verification_identity or "run_completed_verified",page_capture=page_capture,evidence_refs=output_refs,@@ -1535,6 +1521,7 @@ def recorded_outcome_from_run_blocks_result(failed_block_labels=_failed_block_labels(blocks),block_shape_hashes=block_shape_hashes,executed_block_associations=executed_block_associations,+ completed_block_associations=completed_block_associations,page_capture=page_capture,authored_structure_signature=authored_structure_signature,failed_operation=failed_operation,@@ -1568,6 +1555,7 @@ def recorded_outcome_from_run_blocks_result(executed_block_labels=executed_block_labels,block_shape_hashes=block_shape_hashes,executed_block_associations=executed_block_associations,+ completed_block_associations=completed_block_associations,page_capture=page_capture,authored_structure_signature=authored_structure_signature,failed_operation=failed_operation,@@ -1590,6 +1578,7 @@ def recorded_outcome_from_run_blocks_result(executed_block_labels=executed_block_labels,block_shape_hashes=block_shape_hashes,executed_block_associations=executed_block_associations,+ completed_block_associations=completed_block_associations,page_capture=page_capture,authored_structure_signature=authored_structure_signature,failed_operation=failed_operation,@@ -1616,6 +1605,7 @@ def recorded_outcome_from_run_blocks_result(authored_structure_signature=authored_structure_signature,failed_operation=failed_operation,executed_block_associations=executed_block_associations,+ completed_block_associations=completed_block_associations,observed_evidence_summary=recorded_run_outcome.display_reason or "",key_provenance={"structural_failure_identity": (@@ -1740,6 +1730,7 @@ def recorded_outcome_from_run_blocks_result(authored_structure_signature=authored_structure_signature,failed_operation=failed_operation,executed_block_associations=executed_block_associations,+ completed_block_associations=completed_block_associations,observed_page_value_excerpt=_observed_page_value_excerpt(graded_page_evidence),# The failed block's recorded reason carries the exception and its line; the run status is# the word "failed" and says nothing a repair can act on.@@ -1989,6 +1980,21 @@ def _block_dicts(value: object) -> list[Mapping[str, object]]:return [item for item in value if isinstance(item, Mapping)] if isinstance(value, list) else []+def _block_associations(+ blocks: Sequence[Mapping[str, object]],+ block_associations_by_label: Mapping[str, str] | None,+ statuses: frozenset[str],+) -> tuple[str, ...]:+ return tuple(+ dict.fromkeys(+ association+ for block in blocks+ if _safe_str(block.get("status")) in statuses+ if (association := (block_associations_by_label or {}).get(_safe_str(block.get("label"))))+ )+ )++def _first_failed_block(blocks: Sequence[Mapping[str, object]]) -> Mapping[str, object] | None:for block in blocks:if _safe_str(block.get("status")).lower() in _FAILED_BLOCK_STATUSES:diff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.pyindex 96453b435..0eea8d211 100644--- a/skyvern/forge/sdk/copilot/terminal_envelope.py+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py@@ -294,7 +294,12 @@ def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: stif envelope.interruption is not None and not cancelled:return render_interrupted_message(envelope.interruption, proposal_present=envelope.proposal_present), Trueif envelope.failed_operation is not None and not cancelled:- message = "I stopped after a browser operation failed while testing the workflow."+ block_label = envelope.failed_operation.block_label+ message = (+ f"I stopped after a browser operation failed in `{block_label}` while testing the workflow."+ if block_label+ else "I stopped after a browser operation failed while testing the workflow."+ )if envelope.proposal_present:message = _append_sentence(message,diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex c4f7f518c..1cd360e02 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -444,6 +444,14 @@ CANONICAL_LOOP_TOUCHES = 4PROGRESS_LEDGER_SHADOW_EVENT = "taskv3 loop progress ledger would fail-fast"PROGRESS_LEDGER_FINAL_EVENT = "taskv3 loop progress ledger final"+# The ledger's survival record is gated on ever_armed, so it describes ONLY runs that saw a+# validation error. Search / filter / navigate / extract runs emitted nothing at all, which makes any+# fire count taken off that population a floor and never a prevalence. This is the complement record,+# keyed on the canonical tracker because its counters are the only progress signal defined without a+# form. The two partition the population: exactly one is emitted per run, so their union is the+# denominator. Additive — the ledger record's own population and fields are unchanged, so survival+# data collected before this change stays comparable with data collected after it.+CANONICAL_SURVIVAL_EVENT = "taskv3 canonical progress final"@dataclass@@ -942,6 +950,11 @@ class _CanonicalProgressTracker:# on a fire rung while churn evicts other keys — a rung is one threshold crossing, not one# event per touch spent at it._fired: set[tuple[str, int]] = field(default_factory=set)+ # Run-level high-water marks, deliberately NOT cleared by progress(): the survival record needs+ # the worst churn the run ever reached, the same way the ledger keeps peak_actions_since_progress+ # across its own clears. Clearing these with the ring would report the last streak, not the peak.+ peak_same_touches: int = 0+ peak_same_errors: int = 0def record_touch(self, target_key: str, is_error: bool) -> tuple[int, int]:"""Record one billable touch; returns (same-target touches, same-target errors) in-window."""@@ -952,7 +965,10 @@ class _CanonicalProgressTracker:# A rung re-arms once the in-window count dips below it: a fresh streak after full eviction# is a genuine re-crossing, distinct from a saturated window PARKED on a fire count.self._fired = {entry for entry in self._fired if entry[0] != target_key or entry[1] <= len(same)}- return len(same), sum(1 for err in same if err)+ same_errors = sum(1 for err in same if err)+ self.peak_same_touches = max(self.peak_same_touches, len(same))+ self.peak_same_errors = max(self.peak_same_errors, same_errors)+ return len(same), same_errorsdef progress(self, evidence: _ProgressEvidence) -> None:"""The one clear choke point: a caller must name its evidence class, so no site can wipe@@ -978,6 +994,15 @@ class _CanonicalProgressTracker:self._touches = [touch for touch in self._touches if not touch[0].startswith("mark=")]self._fired = {entry for entry in self._fired if not entry[0].startswith("mark=")}+ def survival_fields(self) -> dict[str, int]:+ """The per-run survival numbers, read through the class for the same reason progress() is+ the one write choke point: the ring's state has no references outside this body."""+ return {+ "peak_same_touches": self.peak_same_touches,+ "peak_same_errors": self.peak_same_errors,+ "looping_targets": self.looping_targets(),+ }+def looping_targets(self) -> int:counts: dict[str, list[bool]] = {}for key, err in self._touches:@@ -2604,6 +2629,20 @@ async def run_agent_tool_loop(outcome_status=outcome.status,turns=turns,)+ else:+ # The complement: a run that never armed a form. Same terminal join key and the same+ # outcome/turns tagging as above, so the two read as one population split in two, not as two+ # unrelated streams. `looping_targets` is the end-state count; the peaks are the run's worst.+ # Deliberately NOT gated on `progress`: the canonical tracker is built and updated+ # unconditionally, so this record does not need the ledger to exist. Gating it would drop+ # BOTH records whenever progress_window is None and silently restore the very hole this+ # record closes — the partition has to be total for the union to be a denominator.+ LOG.info(+ CANONICAL_SURVIVAL_EVENT,+ **canonical.survival_fields(),+ outcome_status=outcome.status,+ turns=turns,+ )outcome.turns = turnsoutcome.no_tool_call_turns = no_tool_call_turnsdiff --git a/skyvern/webeye/cdp_download_interceptor.py b/skyvern/webeye/cdp_download_interceptor.pyindex ddc27fc81..54dfafe94 100644--- a/skyvern/webeye/cdp_download_interceptor.py+++ b/skyvern/webeye/cdp_download_interceptor.py@@ -38,6 +38,7 @@ from pathlib import Pathfrom typing import IO, TYPE_CHECKING, Any, Literalfrom urllib.parse import unquote, urlparse+import aiohttpimport structlogfrom playwright.async_api import Browser, BrowserContext, CDPSession, Page@@ -46,6 +47,7 @@ from skyvern.constants import (BROWSER_INTERCEPTOR_DISABLE_TIMEOUT,BROWSER_PAGE_CLOSE_TIMEOUT,)+from skyvern.exceptions import DownloadFileMaxSizeExceededfrom skyvern.forge.sdk.api import files as file_apifrom skyvern.forge.sdk.core.hashing import diagnostic_fingerprintfrom skyvern.forge.sdk.core.http_request_authorization import (@@ -1701,7 +1703,15 @@ class CDPDownloadInterceptor:except Exception as exc:# The download URL is credential-bearing and can reappear inside an exception message# (aiohttp embeds it), so the raise site stands in for the message and traceback.- LOG.error(+ #+ # This try also covers _validated_download_basename and _cookie_header_for_url, so a+ # first-party defect lands here too. Only the transport failure set is site-caused and+ # safe to lower; anything else keeps error, because nothing watches warn on this service+ # and a defect downgraded here would be invisible. Unknown types stay at error by+ # construction -- the test is a whitelist, so it fails toward visibility.+ site_caused = isinstance(exc, (aiohttp.ClientError, asyncio.TimeoutError, DownloadFileMaxSizeExceeded))+ emit = LOG.warning if site_caused else LOG.error+ emit("Guarded direct download failed",error_type=type(exc).__name__,error_origin=redacted_exception_origin(exc),diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex ea3d3a1ad..a64ee3a63 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -34,6 +34,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (RecordedBuildTestOutcome,authored_block_signatures_from_workflow,authored_structure_signature_from_workflow,+ bind_post_run_page_evidence,observed_value_extraction_scaffold_lines,record_build_test_outcome,recorded_outcome_from_author_time_reject,@@ -56,6 +57,7 @@ from skyvern.forge.sdk.copilot.output_utils import (from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.runtime_authoring_repair import inject_runtime_authoring_repair_contextfrom skyvern.forge.sdk.copilot.secret_scrub import clear_session_scrub_values, register_secret_scrub_value+from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope, render_terminal_messagefrom skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modulefrom skyvern.forge.sdk.copilot.tools.composition_capture import store_post_run_page_evidencefrom skyvern.forge.sdk.copilot.tools.run_execution import (@@ -3004,7 +3006,7 @@ def test_browser_operation_structural_key_ignores_transient_run_and_block_ids()assert first_outcome.structural_key == second_outcome.structural_key-def test_browser_operation_failure_survives_non_clearing_outcomes_until_changed_attempt_is_tested() -> None:+def test_browser_operation_failure_survives_non_clearing_outcomes_until_the_block_completes() -> None:failed_workflow = """title: retain browser failureworkflow_definition:parameters: []@@ -3014,7 +3016,6 @@ workflow_definition:code: |return {"failure_rate": await page.locator("canvas.failure-rate").inner_text()}"""- repaired_workflow = failed_workflow.replace("canvas.failure-rate", "[data-testid='failure-rate']")ctx = _locator_packet_ctx()ctx.workflow_yaml = failed_workflowctx.persisted_workflow_yaml = failed_workflow@@ -3070,20 +3071,18 @@ workflow_definition:assert ctx.latest_recorded_build_test_outcome is not Noneassert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation- ctx.workflow_yaml = repaired_workflow- ctx.persisted_workflow_yaml = repaired_workflowrecord_build_test_outcome(ctx,RecordedBuildTestOutcome(phase="persisted_block_run",attempted_tool="update_and_run_blocks",- verdict="progress_observed",- reason_code="verified_success",- workflow_run_id="wr_changed_but_not_executed",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_block_failed_again",block_labels=["collect_failure_rate"],requested_block_labels=["collect_failure_rate"],- executed_block_labels=[],- verified_progress_marker="run-completed",+ executed_block_labels=["collect_failure_rate"],+ executed_block_associations=("cba_collect_failure_rate",),),)assert ctx.latest_recorded_build_test_outcome is not None@@ -3094,75 +3093,21 @@ workflow_definition:RecordedBuildTestOutcome(phase="persisted_block_run",attempted_tool="update_and_run_blocks",- verdict="progress_observed",- reason_code="verified_success",- workflow_run_id="wr_changed_and_tested",+ verdict="not_authoritative",+ reason_code="run_completed_unevaluated",+ workflow_run_id="wr_block_completed",block_labels=["collect_failure_rate"],requested_block_labels=["collect_failure_rate"],executed_block_labels=["collect_failure_rate"],executed_block_associations=("cba_collect_failure_rate",),- verified_progress_marker="run-completed",- ),- )- assert ctx.latest_recorded_build_test_outcome is not None- assert ctx.latest_recorded_build_test_outcome.failed_operation is None---def test_existing_workflow_failure_and_retest_use_the_exact_unmasked_staged_snapshots() -> None:- turn_start_workflow = """workflow_definition:- parameters: []- blocks:- - block_type: code- label: collect_failure_rate- code: |- return await page.locator("[data-testid='turn-start']").inner_text()-"""- failed_snapshot = turn_start_workflow.replace("[data-testid='turn-start']", "canvas.failure-rate")- repaired_snapshot = turn_start_workflow.replace("[data-testid='turn-start']", "[data-testid='failure-rate']")- ctx = _locator_packet_ctx()- ctx.persisted_workflow_yaml = turn_start_workflow- ctx.workflow_yaml = failed_snapshot- ctx.staged_workflow_yaml = failed_snapshot- ctx.runner_code_block_associations_by_label = {"collect_failure_rate": "cba_collect_failure_rate"}- failed_operation = BuildTestFailedOperation(- kind="browser_operation_failed",- workflow_run_id="wr_failed",- block_label="collect_failure_rate",- failing_line=1,- block_association="cba_collect_failure_rate",- )-- record_build_test_outcome(- ctx,- RecordedBuildTestOutcome(- phase="persisted_block_run",- verdict="repairable_failure",- reason_code="runtime_block_failure",- workflow_run_id="wr_failed",- structural_failure_identity="browser-operation",- failed_operation=failed_operation,- ),- )- ctx.workflow_yaml = repaired_snapshot- ctx.staged_workflow_yaml = repaired_snapshot- record_build_test_outcome(- ctx,- RecordedBuildTestOutcome(- phase="persisted_block_run",- verdict="progress_observed",- reason_code="verified_success",- workflow_run_id="wr_retested",- executed_block_labels=["collect_failure_rate"],- executed_block_associations=("cba_collect_failure_rate",),- verified_progress_marker="run-completed",+ completed_block_associations=("cba_collect_failure_rate",),),)-assert ctx.latest_recorded_build_test_outcome is not Noneassert ctx.latest_recorded_build_test_outcome.failed_operation is None-def test_verified_retest_clears_a_changed_outer_browser_operation_not_its_nested_locator() -> None:+def test_unevaluated_later_run_that_completed_the_failed_block_clears_the_stop() -> None:failed_workflow = """workflow_definition:parameters: []blocks:@@ -3192,17 +3137,15 @@ def test_verified_retest_clears_a_changed_outer_browser_operation_not_its_nested),),)- ctx.workflow_yaml = failed_workflow.replace("timeout=1000", "timeout=5000")- ctx.persisted_workflow_yaml = ctx.workflow_yamlrecord_build_test_outcome(ctx,RecordedBuildTestOutcome(phase="persisted_block_run",- verdict="progress_observed",- reason_code="verified_success",+ verdict="not_authoritative",+ reason_code="run_completed_unevaluated",workflow_run_id="wr_retested",executed_block_associations=("cba_submit",),- verified_progress_marker="run-completed",+ completed_block_associations=("cba_submit",),),)@@ -3210,7 +3153,7 @@ def test_verified_retest_clears_a_changed_outer_browser_operation_not_its_nestedassert ctx.latest_recorded_build_test_outcome.failed_operation is None-def test_browser_operation_failure_is_not_cleared_when_full_replacement_copies_its_old_association() -> None:+def test_browser_operation_failure_clears_when_a_full_replacement_completes_the_block() -> None:failed_workflow = """workflow_definition:parameters: []blocks:@@ -3269,6 +3212,22 @@ def test_browser_operation_failure_is_not_cleared_when_full_replacement_copies_i)replacement_association = ctx.runner_code_block_associations_by_label["collect_failure_rate"]assert replacement_association != "cba_original"++ # Running the replacement without completing it resolves nothing.+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ verdict="not_authoritative",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_replacement_failed",+ executed_block_labels=["collect_failure_rate"],+ executed_block_associations=(replacement_association,),+ ),+ )+ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation+record_build_test_outcome(ctx,RecordedBuildTestOutcome(@@ -3278,12 +3237,15 @@ def test_browser_operation_failure_is_not_cleared_when_full_replacement_copies_iworkflow_run_id="wr_replacement",executed_block_labels=["collect_failure_rate"],executed_block_associations=(replacement_association,),+ completed_block_associations=(replacement_association,),verified_progress_marker="run-completed",),)+ # The replacement carries a fresh identity, so the recorded association can never run again;+ # the label's current identity completing is the only route by which this repair can clear.assert ctx.latest_recorded_build_test_outcome is not None- assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation+ assert ctx.latest_recorded_build_test_outcome.failed_operation is Nonedef test_code_block_association_covers_conditional_blocks_without_entering_packet_projection() -> None:@@ -3356,7 +3318,7 @@ def test_recorded_outcome_execution_receipts_come_from_block_status_not_requesteassert outcome.executed_block_labels == ["executed"]-def test_browser_operation_failure_is_not_cleared_by_unrelated_edit_in_executed_block() -> None:+def test_browser_operation_failure_without_a_block_association_is_never_cleared() -> None:failed_workflow = """title: retain browser failureworkflow_definition:parameters: []@@ -3389,22 +3351,17 @@ workflow_definition:),)- unrelated_edit = failed_workflow.replace(- 'return {"failure_rate": value}',- 'return {"failure_rate": value, "note": "unchanged locator"}',- )- ctx.workflow_yaml = unrelated_edit- ctx.persisted_workflow_yaml = unrelated_editrecord_build_test_outcome(ctx,RecordedBuildTestOutcome(phase="persisted_block_run",- verdict="progress_observed",- reason_code="verified_success",- workflow_run_id="wr_unrelated_edit",+ verdict="not_authoritative",+ reason_code="run_completed_unevaluated",+ workflow_run_id="wr_later_completed_run",requested_block_labels=["collect_failure_rate"],executed_block_labels=["collect_failure_rate"],- verified_progress_marker="run-completed",+ executed_block_associations=("cba_collect_failure_rate",),+ completed_block_associations=("cba_collect_failure_rate",),),)@@ -3453,6 +3410,203 @@ workflow_definition:assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation+def test_failure_in_the_latest_run_keeps_the_stop_and_names_its_block() -> None:+ ctx = _locator_packet_ctx()+ ctx.runner_code_block_associations_by_label = {"continue_to_payment": "cba_continue_to_payment"}+ failed_operation = BuildTestFailedOperation(+ kind="browser_operation_failed",+ workflow_run_id="wr_latest",+ workflow_run_block_id="wrb_latest",+ block_label="continue_to_payment",+ failing_line=3,+ block_association="cba_continue_to_payment",+ )+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_latest",+ attempted_block_label="continue_to_payment",+ block_labels=["search_tomorrow_trip", "continue_to_payment"],+ structural_failure_identity="browser-operation",+ failed_operation=failed_operation,+ ),+ )+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="not_authoritative",+ reason_code="blocker_reported",+ workflow_run_id="wr_latest",+ block_labels=["search_tomorrow_trip", "continue_to_payment"],+ ),+ )++ latest = ctx.latest_recorded_build_test_outcome+ assert latest is not None+ assert latest.failed_operation == failed_operation+ assert latest.completed_block_associations == ()++ envelope = assemble_terminal_envelope(+ response_type="REPLY",+ verified=False,+ workflow_applied=False,+ proposal_disposition="review_untested",+ run_outcomes=[],+ blocker_reason=None,+ halt_kind=None,+ attempted=None,+ workflow_mutated=True,+ workflow_attempted=True,+ failed_operation=latest.failed_operation,+ proposal_present=True,+ interruption=None,+ )+ assert envelope is not None+ assert envelope.terminal_cause == "browser_operation_failed"+ message, replaced = render_terminal_message(envelope, "End-to-end test did not complete.", cancelled=False)++ assert replaced is True+ assert message.startswith(+ "I stopped after a browser operation failed in `continue_to_payment` while testing the workflow."+ )+++def test_stop_clears_when_the_failed_block_completed_even_though_a_later_block_failed() -> None:+ ctx = _locator_packet_ctx()+ ctx.runner_code_block_associations_by_label = {"log_in": "cba_log_in", "collect_titles": "cba_collect_titles"}+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_login_failed",+ block_labels=["log_in", "collect_titles"],+ structural_failure_identity="browser-operation",+ failed_operation=BuildTestFailedOperation(+ kind="browser_operation_failed",+ workflow_run_id="wr_login_failed",+ block_label="log_in",+ failing_line=2,+ block_association="cba_log_in",+ ),+ ),+ )+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_collect_failed",+ block_labels=["log_in", "collect_titles"],+ executed_block_labels=["log_in", "collect_titles"],+ executed_block_associations=("cba_log_in", "cba_collect_titles"),+ completed_block_associations=("cba_log_in",),+ ),+ )++ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.failed_operation is None+++def test_post_run_enrichment_does_not_resurrect_a_released_stop() -> None:+ ctx = _locator_packet_ctx()+ ctx.runner_code_block_associations_by_label = {"log_in": "cba_log_in"}+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_login_failed",+ block_labels=["log_in"],+ structural_failure_identity="browser-operation",+ failed_operation=BuildTestFailedOperation(+ kind="browser_operation_failed",+ workflow_run_id="wr_login_failed",+ block_label="log_in",+ failing_line=2,+ block_association="cba_log_in",+ ),+ ),+ )+ released = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="not_authoritative",+ reason_code="run_completed_unevaluated",+ workflow_run_id="wr_login_completed",+ block_labels=["log_in"],+ executed_block_labels=["log_in"],+ executed_block_associations=("cba_log_in",),+ completed_block_associations=("cba_log_in",),+ )+ record_build_test_outcome(ctx, released)+ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.failed_operation is None++ assert bind_post_run_page_evidence(+ ctx,+ {"workflow_run_id": "wr_login_completed"},+ None,+ regraded=released,+ )++ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.failed_operation is None+++def test_stop_holds_when_the_failed_block_only_terminated_in_the_later_run() -> None:+ ctx = _locator_packet_ctx()+ ctx.runner_code_block_associations_by_label = {"log_in": "cba_log_in"}+ failed_operation = BuildTestFailedOperation(+ kind="browser_operation_failed",+ workflow_run_id="wr_login_failed",+ block_label="log_in",+ failing_line=2,+ block_association="cba_log_in",+ )+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_login_failed",+ block_labels=["log_in"],+ structural_failure_identity="browser-operation",+ failed_operation=failed_operation,+ ),+ )+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="not_authoritative",+ reason_code="terminal_challenge_blocker",+ workflow_run_id="wr_login_terminated",+ block_labels=["log_in"],+ executed_block_labels=["log_in"],+ executed_block_associations=("cba_log_in",),+ ),+ )++ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation++@pytest.mark.asyncioasync def test_browser_operation_outcome_clears_after_server_composed_verified_retest(monkeypatch: pytest.MonkeyPatch,@@ -3562,6 +3716,7 @@ workflow_definition:reason_code="verified_success",workflow_run_id="wr_retested_repair",executed_block_associations=("cba_collect_failure_rate",),+ completed_block_associations=("cba_collect_failure_rate",),verified_progress_marker="run-completed",),)diff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.pyindex 48d46101c..c664c7fa7 100644--- a/tests/unit/test_copilot_terminal_envelope.py+++ b/tests/unit/test_copilot_terminal_envelope.py@@ -304,10 +304,28 @@ def test_failed_operation_without_a_crash_keeps_its_own_sentence() -> None:message, replaced = render_terminal_message(envelope, "The code run failed.", cancelled=False)assert replaced is True- assert "browser operation failed" in message.lower()+ assert message.startswith("I stopped after a browser operation failed while testing the workflow.")+ assert "`" not in messageassert INTERRUPTED_TERMINAL_HEADLINE not in message+def test_failed_operation_sentence_names_the_block_that_failed() -> None:+ envelope = _assemble(+ proposal_disposition="review_untested",+ failed_operation=BuildTestFailedOperation(+ kind="browser_operation_failed",+ block_label="continue_to_payment",+ ),+ proposal_present=True,+ )+ message, replaced = render_terminal_message(envelope, "The code run failed.", cancelled=False)++ assert replaced is True+ assert message.startswith(+ "I stopped after a browser operation failed in `continue_to_payment` while testing the workflow."+ )++def test_unexpected_error_exit_renders_interrupted_and_keeps_its_terminal_reason() -> None:ctx = make_copilot_ctx(latest_recorded_build_test_outcome=RecordedBuildTestOutcome(diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex 749a1ddd8..7a76ba94c 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -29,6 +29,7 @@ from skyvern.forge.taskv3.loop import (ACTION_BUDGET_EXTENDED_EVENT,ACTION_BUDGET_EXTENSION_REFUSED_EVENT,ACTION_LOOP_REASON_PREFIX,+ CANONICAL_SURVIVAL_EVENT,FAILURE_EVIDENCE_MIN_TOOL_CALLS,FAILURE_EVIDENCE_MIN_TURNS,NAV_DEAD_END_REASON_PREFIX,@@ -6447,6 +6448,73 @@ async def test_progress_ledger_emits_terminal_survival_record() -> None:assert final[0]["would_fire"] is True+async def test_non_form_run_emits_its_own_survival_record_keyed_on_canonical_touches() -> None:+ # The ledger's survival record is gated on ever_armed, so it covers only runs that saw a+ # validation error. Every search / filter / navigate / extract run — the whole non-form half of+ # the product — emitted NOTHING, which is why a fire count off that population is a floor and+ # never a prevalence. This is the complement record, keyed on the canonical tracker's counters+ # because they are the only progress signal defined without a form.+ rounds = 6+ clicks: list[tuple[str, dict[str, Any]]] = []+ tools = [_form_observe("observe", [0] * rounds), _billable_tool("click", clicks), make_finish_tool()]+ script: list[list[tuple[str, dict[str, Any]]]] = [[("observe", {})]]+ # One batch against ONE target: touches accumulate inside the turn, so the peak does not depend+ # on whether a later observe clears the ring.+ script.append([("click", {"selector": "#stuck"}) for _ in range(4)])+ script.append([("finish", {"status": "completed", "reason": "done"})])+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=200, max_tool_calls=500)++ assert outcome.status == "completed"+ survival = [e for e in logs if e.get("event") == CANONICAL_SURVIVAL_EVENT]+ assert len(survival) == 1, survival+ assert survival[0]["outcome_status"] == "completed"+ # Load-bearing: the record has to carry the same-target churn, or it is an empty denominator.+ assert survival[0]["peak_same_touches"] >= 4, survival[0]+ # The two records partition the population — a non-form run must not also emit the form one, or+ # the union double-counts and the denominator is wrong in the other direction.+ assert not [e for e in logs if e.get("event") == PROGRESS_LEDGER_FINAL_EVENT]+++async def test_a_run_still_gets_exactly_one_survival_record_with_the_ledger_disabled() -> None:+ # The partition must be TOTAL, not conditional on an unrelated flag. The canonical tracker is+ # built and updated unconditionally, so its record does not depend on the ledger existing —+ # gating it on `progress` would drop BOTH records whenever progress_window is None and silently+ # restore the "floor, not prevalence" hole this record exists to close.+ rounds = 4+ clicks: list[tuple[str, dict[str, Any]]] = []+ tools = [_form_observe("observe", [3] * rounds), _billable_tool("click", clicks), make_finish_tool()]+ script = [[("observe", {}), ("click", {"selector": f"#f{i}"})] for i in range(rounds)]+ script.append([("finish", {"status": "completed", "reason": "done"})])+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=200, max_tool_calls=500, progress_window=None)++ assert outcome.status == "completed"+ records = [e for e in logs if e.get("event") in (PROGRESS_LEDGER_FINAL_EVENT, CANONICAL_SURVIVAL_EVENT)]+ assert len(records) == 1, records+ assert records[0]["event"] == CANONICAL_SURVIVAL_EVENT+++async def test_form_run_still_emits_only_the_ledger_record_and_not_the_non_form_one() -> None:+ # The other half of the partition, and the compatibility guarantee: the existing record's+ # population and shape are untouched, so the survival data already collected stays comparable+ # with everything collected after this change.+ rounds = 10+ clicks: list[tuple[str, dict[str, Any]]] = []+ tools = [_form_observe("observe", [3] * rounds), _billable_tool("click", clicks), make_finish_tool()]+ script = [[("observe", {}), ("click", {"selector": f"#f{i}"})] for i in range(rounds)]+ script.append([("finish", {"status": "completed", "reason": "done"})])+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=200, max_tool_calls=500)++ assert outcome.status == "completed"+ assert len([e for e in logs if e.get("event") == PROGRESS_LEDGER_FINAL_EVENT]) == 1+ assert not [e for e in logs if e.get("event") == CANONICAL_SURVIVAL_EVENT]++@pytest.mark.asyncioasync def test_progress_ledger_silent_when_actions_batch_before_a_confirming_observe() -> None:# Healthy batch: the model fixes several fields in one turn before re-observing (markers stay