Skip to content

Automate browser based workflows with AI

Compile the attach worker's two wheel-less C extensions in a builder stage (#8436)

Shuchang Zheng8eacacc60d9c
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 5ec2a1fa1..5afc75791 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -102,6 +102,7 @@ _UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"
_BROWSER_OPERATION_FAILED: BuildTestFailedOperationKind = "browser_operation_failed"
_EXECUTED_BLOCK_STATUSES = frozenset(status.value for status in BlockStatus if status != BlockStatus.skipped)
_FAILED_BLOCK_STATUSES = frozenset({"failed", "terminated", "canceled", "timed_out"})
+_COMPLETED_BLOCK_STATUSES = frozenset({BlockStatus.completed.value})
# Sandbox-process faults, not authored-code faults. ``timeout`` and ``user_code_error`` stay
# out: both are repairable despite also carrying ``runner_internal_error``. ``busy`` is in —
# a saturated runner gate says nothing about the code, so rewriting it cannot help.
@@ -409,6 +410,7 @@ class RecordedBuildTestOutcome(BaseModel):
failed_operation_call_signature: str | None = Field(default=None, exclude=True, repr=False)
failed_operation_code_signature: str | None = Field(default=None, exclude=True, repr=False)
executed_block_associations: tuple[str, ...] = Field(default=(), exclude=True, repr=False)
+ completed_block_associations: tuple[str, ...] = Field(default=(), exclude=True, repr=False)
authored_structure_signature: str | None = None
display_text: str = ""
observed_page_value_excerpt: str = ""
@@ -501,6 +503,24 @@ def _history_entry(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuild
}
+def _releasing_associations(
+ ctx: _RecordedBuildTestOutcomeContext, failed_operation: BuildTestFailedOperation
+) -> set[str]:
+ """Identities whose completion resolves this failure.
+
+ A full replacement mints a fresh identity for the same label, so the recorded association can
+ never run again; without the label's current identity that repair shape has no way to clear.
+ """
+ releasing: set[str] = set()
+ if failed_operation.block_association:
+ releasing.add(failed_operation.block_association)
+ if failed_operation.block_label:
+ current = ctx.runner_code_block_associations_by_label.get(failed_operation.block_label)
+ if current:
+ releasing.add(current)
+ return releasing
+
+
def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome | None) -> None:
if outcome is None:
latest = getattr(ctx, "latest_recorded_build_test_outcome", None)
@@ -523,22 +543,13 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re
}
)
elif prior is not None and prior.failed_operation is not None:
- source_yaml = _executed_workflow_yaml(ctx)
- association = prior.failed_operation.block_association
- tested_code = _code_for_runner_association(ctx, source_yaml, association)
- changed_attempt_was_tested = (
+ releasing = _releasing_associations(ctx, prior.failed_operation)
+ failed_block_completed_later = (
outcome.phase == "persisted_block_run"
- and outcome.verdict == "progress_observed"
and outcome.workflow_run_id not in (None, prior.failed_operation.workflow_run_id)
- and association is not None
- and association in outcome.executed_block_associations
- and _failed_operation_changed(
- tested_code,
- prior.failed_operation_call_signature,
- prior.failed_operation_code_signature,
- )
+ and bool(releasing & set(outcome.completed_block_associations))
)
- if not changed_attempt_was_tested:
+ if not failed_block_completed_later:
outcome = outcome.model_copy(
update={
"failed_operation": prior.failed_operation,
@@ -729,27 +740,6 @@ def _failed_operation_call_signature(code: str | None, failing_line: int | None)
return _stable_hash(ast.dump(outermost, include_attributes=False))
-def _failed_operation_changed(
- tested_code: str | None,
- prior_call_signature: str | None,
- prior_code_signature: str | None,
-) -> bool:
- """Prove the recorded operation changed, or use whole-block change for a typed line omission."""
- if not tested_code:
- return False
- wrapper = _parse_block_code(tested_code)
- if wrapper is None:
- return False
- current_call_signatures = {
- _stable_hash(ast.dump(node, include_attributes=False))
- for node in ast.walk(wrapper)
- if isinstance(node, ast.Call)
- }
- if prior_call_signature is not None:
- return prior_call_signature not in current_call_signatures
- return prior_code_signature is not None and _stable_hash(tested_code) != prior_code_signature
-
-
# Every construct that can reach the end of a block without running something inside it, including
# expression-level ones: a zero-length comprehension skips its body exactly as an unentered loop does.
# Over-detection only costs a redundant note; a miss silently drops a real failure.
@@ -1405,14 +1395,8 @@ def recorded_outcome_from_run_blocks_result(
if (redacted := _redacted_terminal_text(_safe_str(block.get("label"))))
]
)
- executed_block_associations = tuple(
- dict.fromkeys(
- association
- for block in blocks
- if _safe_str(block.get("status")) in _EXECUTED_BLOCK_STATUSES
- if (association := (block_associations_by_label or {}).get(_safe_str(block.get("label"))))
- )
- )
+ executed_block_associations = _block_associations(blocks, block_associations_by_label, _EXECUTED_BLOCK_STATUSES)
+ completed_block_associations = _block_associations(blocks, block_associations_by_label, _COMPLETED_BLOCK_STATUSES)
block_shape_hashes = dict(block_shape_hashes or {})
referenced_unbound_keys = _referenced_unbound_input_keys(
result,
@@ -1493,6 +1477,7 @@ def recorded_outcome_from_run_blocks_result(
executed_block_labels=executed_block_labels,
block_shape_hashes=block_shape_hashes,
executed_block_associations=executed_block_associations,
+ completed_block_associations=completed_block_associations,
page_capture=page_capture,
authored_structure_signature=authored_structure_signature,
failed_operation=failed_operation,
@@ -1511,6 +1496,7 @@ def recorded_outcome_from_run_blocks_result(
executed_block_labels=executed_block_labels,
block_shape_hashes=block_shape_hashes,
executed_block_associations=executed_block_associations,
+ completed_block_associations=completed_block_associations,
verified_progress_marker=verification_identity or "run_completed_verified",
page_capture=page_capture,
evidence_refs=output_refs,
@@ -1535,6 +1521,7 @@ def recorded_outcome_from_run_blocks_result(
failed_block_labels=_failed_block_labels(blocks),
block_shape_hashes=block_shape_hashes,
executed_block_associations=executed_block_associations,
+ completed_block_associations=completed_block_associations,
page_capture=page_capture,
authored_structure_signature=authored_structure_signature,
failed_operation=failed_operation,
@@ -1568,6 +1555,7 @@ def recorded_outcome_from_run_blocks_result(
executed_block_labels=executed_block_labels,
block_shape_hashes=block_shape_hashes,
executed_block_associations=executed_block_associations,
+ completed_block_associations=completed_block_associations,
page_capture=page_capture,
authored_structure_signature=authored_structure_signature,
failed_operation=failed_operation,
@@ -1590,6 +1578,7 @@ def recorded_outcome_from_run_blocks_result(
executed_block_labels=executed_block_labels,
block_shape_hashes=block_shape_hashes,
executed_block_associations=executed_block_associations,
+ completed_block_associations=completed_block_associations,
page_capture=page_capture,
authored_structure_signature=authored_structure_signature,
failed_operation=failed_operation,
@@ -1616,6 +1605,7 @@ def recorded_outcome_from_run_blocks_result(
authored_structure_signature=authored_structure_signature,
failed_operation=failed_operation,
executed_block_associations=executed_block_associations,
+ completed_block_associations=completed_block_associations,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={
"structural_failure_identity": (
@@ -1740,6 +1730,7 @@ def recorded_outcome_from_run_blocks_result(
authored_structure_signature=authored_structure_signature,
failed_operation=failed_operation,
executed_block_associations=executed_block_associations,
+ completed_block_associations=completed_block_associations,
observed_page_value_excerpt=_observed_page_value_excerpt(graded_page_evidence),
# The failed block's recorded reason carries the exception and its line; the run status is
# the word "failed" and says nothing a repair can act on.
@@ -1989,6 +1980,21 @@ def _block_dicts(value: object) -> list[Mapping[str, object]]:
return [item for item in value if isinstance(item, Mapping)] if isinstance(value, list) else []
+def _block_associations(
+ blocks: Sequence[Mapping[str, object]],
+ block_associations_by_label: Mapping[str, str] | None,
+ statuses: frozenset[str],
+) -> tuple[str, ...]:
+ return tuple(
+ dict.fromkeys(
+ association
+ for block in blocks
+ if _safe_str(block.get("status")) in statuses
+ if (association := (block_associations_by_label or {}).get(_safe_str(block.get("label"))))
+ )
+ )
+
+
def _first_failed_block(blocks: Sequence[Mapping[str, object]]) -> Mapping[str, object] | None:
for block in blocks:
if _safe_str(block.get("status")).lower() in _FAILED_BLOCK_STATUSES:
diff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.py
index 96453b435..0eea8d211 100644
--- a/skyvern/forge/sdk/copilot/terminal_envelope.py
+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py
@@ -294,7 +294,12 @@ def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: st
if envelope.interruption is not None and not cancelled:
return render_interrupted_message(envelope.interruption, proposal_present=envelope.proposal_present), True
if envelope.failed_operation is not None and not cancelled:
- message = "I stopped after a browser operation failed while testing the workflow."
+ block_label = envelope.failed_operation.block_label
+ message = (
+ f"I stopped after a browser operation failed in `{block_label}` while testing the workflow."
+ if block_label
+ else "I stopped after a browser operation failed while testing the workflow."
+ )
if envelope.proposal_present:
message = _append_sentence(
message,
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index c4f7f518c..1cd360e02 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -444,6 +444,14 @@ CANONICAL_LOOP_TOUCHES = 4
PROGRESS_LEDGER_SHADOW_EVENT = "taskv3 loop progress ledger would fail-fast"
PROGRESS_LEDGER_FINAL_EVENT = "taskv3 loop progress ledger final"
+# The ledger's survival record is gated on ever_armed, so it describes ONLY runs that saw a
+# validation error. Search / filter / navigate / extract runs emitted nothing at all, which makes any
+# fire count taken off that population a floor and never a prevalence. This is the complement record,
+# keyed on the canonical tracker because its counters are the only progress signal defined without a
+# form. The two partition the population: exactly one is emitted per run, so their union is the
+# denominator. Additive — the ledger record's own population and fields are unchanged, so survival
+# data collected before this change stays comparable with data collected after it.
+CANONICAL_SURVIVAL_EVENT = "taskv3 canonical progress final"
@dataclass
@@ -942,6 +950,11 @@ class _CanonicalProgressTracker:
# on a fire rung while churn evicts other keys — a rung is one threshold crossing, not one
# event per touch spent at it.
_fired: set[tuple[str, int]] = field(default_factory=set)
+ # Run-level high-water marks, deliberately NOT cleared by progress(): the survival record needs
+ # the worst churn the run ever reached, the same way the ledger keeps peak_actions_since_progress
+ # across its own clears. Clearing these with the ring would report the last streak, not the peak.
+ peak_same_touches: int = 0
+ peak_same_errors: int = 0
def record_touch(self, target_key: str, is_error: bool) -> tuple[int, int]:
"""Record one billable touch; returns (same-target touches, same-target errors) in-window."""
@@ -952,7 +965,10 @@ class _CanonicalProgressTracker:
# A rung re-arms once the in-window count dips below it: a fresh streak after full eviction
# is a genuine re-crossing, distinct from a saturated window PARKED on a fire count.
self._fired = {entry for entry in self._fired if entry[0] != target_key or entry[1] <= len(same)}
- return len(same), sum(1 for err in same if err)
+ same_errors = sum(1 for err in same if err)
+ self.peak_same_touches = max(self.peak_same_touches, len(same))
+ self.peak_same_errors = max(self.peak_same_errors, same_errors)
+ return len(same), same_errors
def progress(self, evidence: _ProgressEvidence) -> None:
"""The one clear choke point: a caller must name its evidence class, so no site can wipe
@@ -978,6 +994,15 @@ class _CanonicalProgressTracker:
self._touches = [touch for touch in self._touches if not touch[0].startswith("mark=")]
self._fired = {entry for entry in self._fired if not entry[0].startswith("mark=")}
+ def survival_fields(self) -> dict[str, int]:
+ """The per-run survival numbers, read through the class for the same reason progress() is
+ the one write choke point: the ring's state has no references outside this body."""
+ return {
+ "peak_same_touches": self.peak_same_touches,
+ "peak_same_errors": self.peak_same_errors,
+ "looping_targets": self.looping_targets(),
+ }
+
def looping_targets(self) -> int:
counts: dict[str, list[bool]] = {}
for key, err in self._touches:
@@ -2604,6 +2629,20 @@ async def run_agent_tool_loop(
outcome_status=outcome.status,
turns=turns,
)
+ else:
+ # The complement: a run that never armed a form. Same terminal join key and the same
+ # outcome/turns tagging as above, so the two read as one population split in two, not as two
+ # unrelated streams. `looping_targets` is the end-state count; the peaks are the run's worst.
+ # Deliberately NOT gated on `progress`: the canonical tracker is built and updated
+ # unconditionally, so this record does not need the ledger to exist. Gating it would drop
+ # BOTH records whenever progress_window is None and silently restore the very hole this
+ # record closes — the partition has to be total for the union to be a denominator.
+ LOG.info(
+ CANONICAL_SURVIVAL_EVENT,
+ **canonical.survival_fields(),
+ outcome_status=outcome.status,
+ turns=turns,
+ )
outcome.turns = turns
outcome.no_tool_call_turns = no_tool_call_turns
diff --git a/skyvern/webeye/cdp_download_interceptor.py b/skyvern/webeye/cdp_download_interceptor.py
index ddc27fc81..54dfafe94 100644
--- a/skyvern/webeye/cdp_download_interceptor.py
+++ b/skyvern/webeye/cdp_download_interceptor.py
@@ -38,6 +38,7 @@ from pathlib import Path
from typing import IO, TYPE_CHECKING, Any, Literal
from urllib.parse import unquote, urlparse
+import aiohttp
import structlog
from playwright.async_api import Browser, BrowserContext, CDPSession, Page
@@ -46,6 +47,7 @@ from skyvern.constants import (
BROWSER_INTERCEPTOR_DISABLE_TIMEOUT,
BROWSER_PAGE_CLOSE_TIMEOUT,
)
+from skyvern.exceptions import DownloadFileMaxSizeExceeded
from skyvern.forge.sdk.api import files as file_api
from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint
from skyvern.forge.sdk.core.http_request_authorization import (
@@ -1701,7 +1703,15 @@ class CDPDownloadInterceptor:
except Exception as exc:
# The download URL is credential-bearing and can reappear inside an exception message
# (aiohttp embeds it), so the raise site stands in for the message and traceback.
- LOG.error(
+ #
+ # This try also covers _validated_download_basename and _cookie_header_for_url, so a
+ # first-party defect lands here too. Only the transport failure set is site-caused and
+ # safe to lower; anything else keeps error, because nothing watches warn on this service
+ # and a defect downgraded here would be invisible. Unknown types stay at error by
+ # construction -- the test is a whitelist, so it fails toward visibility.
+ site_caused = isinstance(exc, (aiohttp.ClientError, asyncio.TimeoutError, DownloadFileMaxSizeExceeded))
+ emit = LOG.warning if site_caused else LOG.error
+ emit(
"Guarded direct download failed",
error_type=type(exc).__name__,
error_origin=redacted_exception_origin(exc),
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index ea3d3a1ad..a64ee3a63 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -34,6 +34,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
+ bind_post_run_page_evidence,
observed_value_extraction_scaffold_lines,
record_build_test_outcome,
recorded_outcome_from_author_time_reject,
@@ -56,6 +57,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.runtime_authoring_repair import inject_runtime_authoring_repair_context
from skyvern.forge.sdk.copilot.secret_scrub import clear_session_scrub_values, register_secret_scrub_value
+from skyvern.forge.sdk.copilot.terminal_envelope import assemble_terminal_envelope, render_terminal_message
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.tools.composition_capture import store_post_run_page_evidence
from skyvern.forge.sdk.copilot.tools.run_execution import (
@@ -3004,7 +3006,7 @@ def test_browser_operation_structural_key_ignores_transient_run_and_block_ids()
assert first_outcome.structural_key == second_outcome.structural_key
-def test_browser_operation_failure_survives_non_clearing_outcomes_until_changed_attempt_is_tested() -> None:
+def test_browser_operation_failure_survives_non_clearing_outcomes_until_the_block_completes() -> None:
failed_workflow = """title: retain browser failure
workflow_definition:
parameters: []
@@ -3014,7 +3016,6 @@ workflow_definition:
code: |
return {"failure_rate": await page.locator("canvas.failure-rate").inner_text()}
"""
- repaired_workflow = failed_workflow.replace("canvas.failure-rate", "[data-testid='failure-rate']")
ctx = _locator_packet_ctx()
ctx.workflow_yaml = failed_workflow
ctx.persisted_workflow_yaml = failed_workflow
@@ -3070,20 +3071,18 @@ workflow_definition:
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
- ctx.workflow_yaml = repaired_workflow
- ctx.persisted_workflow_yaml = repaired_workflow
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
- verdict="progress_observed",
- reason_code="verified_success",
- workflow_run_id="wr_changed_but_not_executed",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_block_failed_again",
block_labels=["collect_failure_rate"],
requested_block_labels=["collect_failure_rate"],
- executed_block_labels=[],
- verified_progress_marker="run-completed",
+ executed_block_labels=["collect_failure_rate"],
+ executed_block_associations=("cba_collect_failure_rate",),
),
)
assert ctx.latest_recorded_build_test_outcome is not None
@@ -3094,75 +3093,21 @@ workflow_definition:
RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
- verdict="progress_observed",
- reason_code="verified_success",
- workflow_run_id="wr_changed_and_tested",
+ verdict="not_authoritative",
+ reason_code="run_completed_unevaluated",
+ workflow_run_id="wr_block_completed",
block_labels=["collect_failure_rate"],
requested_block_labels=["collect_failure_rate"],
executed_block_labels=["collect_failure_rate"],
executed_block_associations=("cba_collect_failure_rate",),
- verified_progress_marker="run-completed",
- ),
- )
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.failed_operation is None
-
-
-def test_existing_workflow_failure_and_retest_use_the_exact_unmasked_staged_snapshots() -> None:
- turn_start_workflow = """workflow_definition:
- parameters: []
- blocks:
- - block_type: code
- label: collect_failure_rate
- code: |
- return await page.locator("[data-testid='turn-start']").inner_text()
-"""
- failed_snapshot = turn_start_workflow.replace("[data-testid='turn-start']", "canvas.failure-rate")
- repaired_snapshot = turn_start_workflow.replace("[data-testid='turn-start']", "[data-testid='failure-rate']")
- ctx = _locator_packet_ctx()
- ctx.persisted_workflow_yaml = turn_start_workflow
- ctx.workflow_yaml = failed_snapshot
- ctx.staged_workflow_yaml = failed_snapshot
- ctx.runner_code_block_associations_by_label = {"collect_failure_rate": "cba_collect_failure_rate"}
- failed_operation = BuildTestFailedOperation(
- kind="browser_operation_failed",
- workflow_run_id="wr_failed",
- block_label="collect_failure_rate",
- failing_line=1,
- block_association="cba_collect_failure_rate",
- )
-
- record_build_test_outcome(
- ctx,
- RecordedBuildTestOutcome(
- phase="persisted_block_run",
- verdict="repairable_failure",
- reason_code="runtime_block_failure",
- workflow_run_id="wr_failed",
- structural_failure_identity="browser-operation",
- failed_operation=failed_operation,
- ),
- )
- ctx.workflow_yaml = repaired_snapshot
- ctx.staged_workflow_yaml = repaired_snapshot
- record_build_test_outcome(
- ctx,
- RecordedBuildTestOutcome(
- phase="persisted_block_run",
- verdict="progress_observed",
- reason_code="verified_success",
- workflow_run_id="wr_retested",
- executed_block_labels=["collect_failure_rate"],
- executed_block_associations=("cba_collect_failure_rate",),
- verified_progress_marker="run-completed",
+ completed_block_associations=("cba_collect_failure_rate",),
),
)
-
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.failed_operation is None
-def test_verified_retest_clears_a_changed_outer_browser_operation_not_its_nested_locator() -> None:
+def test_unevaluated_later_run_that_completed_the_failed_block_clears_the_stop() -> None:
failed_workflow = """workflow_definition:
parameters: []
blocks:
@@ -3192,17 +3137,15 @@ def test_verified_retest_clears_a_changed_outer_browser_operation_not_its_nested
),
),
)
- ctx.workflow_yaml = failed_workflow.replace("timeout=1000", "timeout=5000")
- ctx.persisted_workflow_yaml = ctx.workflow_yaml
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
- verdict="progress_observed",
- reason_code="verified_success",
+ verdict="not_authoritative",
+ reason_code="run_completed_unevaluated",
workflow_run_id="wr_retested",
executed_block_associations=("cba_submit",),
- verified_progress_marker="run-completed",
+ completed_block_associations=("cba_submit",),
),
)
@@ -3210,7 +3153,7 @@ def test_verified_retest_clears_a_changed_outer_browser_operation_not_its_nested
assert ctx.latest_recorded_build_test_outcome.failed_operation is None
-def test_browser_operation_failure_is_not_cleared_when_full_replacement_copies_its_old_association() -> None:
+def test_browser_operation_failure_clears_when_a_full_replacement_completes_the_block() -> None:
failed_workflow = """workflow_definition:
parameters: []
blocks:
@@ -3269,6 +3212,22 @@ def test_browser_operation_failure_is_not_cleared_when_full_replacement_copies_i
)
replacement_association = ctx.runner_code_block_associations_by_label["collect_failure_rate"]
assert replacement_association != "cba_original"
+
+ # Running the replacement without completing it resolves nothing.
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_replacement_failed",
+ executed_block_labels=["collect_failure_rate"],
+ executed_block_associations=(replacement_association,),
+ ),
+ )
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
+
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
@@ -3278,12 +3237,15 @@ def test_browser_operation_failure_is_not_cleared_when_full_replacement_copies_i
workflow_run_id="wr_replacement",
executed_block_labels=["collect_failure_rate"],
executed_block_associations=(replacement_association,),
+ completed_block_associations=(replacement_association,),
verified_progress_marker="run-completed",
),
)
+ # The replacement carries a fresh identity, so the recorded association can never run again;
+ # the label's current identity completing is the only route by which this repair can clear.
assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
+ assert ctx.latest_recorded_build_test_outcome.failed_operation is None
def test_code_block_association_covers_conditional_blocks_without_entering_packet_projection() -> None:
@@ -3356,7 +3318,7 @@ def test_recorded_outcome_execution_receipts_come_from_block_status_not_requeste
assert outcome.executed_block_labels == ["executed"]
-def test_browser_operation_failure_is_not_cleared_by_unrelated_edit_in_executed_block() -> None:
+def test_browser_operation_failure_without_a_block_association_is_never_cleared() -> None:
failed_workflow = """title: retain browser failure
workflow_definition:
parameters: []
@@ -3389,22 +3351,17 @@ workflow_definition:
),
)
- unrelated_edit = failed_workflow.replace(
- 'return {"failure_rate": value}',
- 'return {"failure_rate": value, "note": "unchanged locator"}',
- )
- ctx.workflow_yaml = unrelated_edit
- ctx.persisted_workflow_yaml = unrelated_edit
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
phase="persisted_block_run",
- verdict="progress_observed",
- reason_code="verified_success",
- workflow_run_id="wr_unrelated_edit",
+ verdict="not_authoritative",
+ reason_code="run_completed_unevaluated",
+ workflow_run_id="wr_later_completed_run",
requested_block_labels=["collect_failure_rate"],
executed_block_labels=["collect_failure_rate"],
- verified_progress_marker="run-completed",
+ executed_block_associations=("cba_collect_failure_rate",),
+ completed_block_associations=("cba_collect_failure_rate",),
),
)
@@ -3453,6 +3410,203 @@ workflow_definition:
assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
+def test_failure_in_the_latest_run_keeps_the_stop_and_names_its_block() -> None:
+ ctx = _locator_packet_ctx()
+ ctx.runner_code_block_associations_by_label = {"continue_to_payment": "cba_continue_to_payment"}
+ failed_operation = BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_latest",
+ workflow_run_block_id="wrb_latest",
+ block_label="continue_to_payment",
+ failing_line=3,
+ block_association="cba_continue_to_payment",
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_latest",
+ attempted_block_label="continue_to_payment",
+ block_labels=["search_tomorrow_trip", "continue_to_payment"],
+ structural_failure_identity="browser-operation",
+ failed_operation=failed_operation,
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="not_authoritative",
+ reason_code="blocker_reported",
+ workflow_run_id="wr_latest",
+ block_labels=["search_tomorrow_trip", "continue_to_payment"],
+ ),
+ )
+
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ assert latest.failed_operation == failed_operation
+ assert latest.completed_block_associations == ()
+
+ envelope = assemble_terminal_envelope(
+ response_type="REPLY",
+ verified=False,
+ workflow_applied=False,
+ proposal_disposition="review_untested",
+ run_outcomes=[],
+ blocker_reason=None,
+ halt_kind=None,
+ attempted=None,
+ workflow_mutated=True,
+ workflow_attempted=True,
+ failed_operation=latest.failed_operation,
+ proposal_present=True,
+ interruption=None,
+ )
+ assert envelope is not None
+ assert envelope.terminal_cause == "browser_operation_failed"
+ message, replaced = render_terminal_message(envelope, "End-to-end test did not complete.", cancelled=False)
+
+ assert replaced is True
+ assert message.startswith(
+ "I stopped after a browser operation failed in `continue_to_payment` while testing the workflow."
+ )
+
+
+def test_stop_clears_when_the_failed_block_completed_even_though_a_later_block_failed() -> None:
+ ctx = _locator_packet_ctx()
+ ctx.runner_code_block_associations_by_label = {"log_in": "cba_log_in", "collect_titles": "cba_collect_titles"}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_login_failed",
+ block_labels=["log_in", "collect_titles"],
+ structural_failure_identity="browser-operation",
+ failed_operation=BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_login_failed",
+ block_label="log_in",
+ failing_line=2,
+ block_association="cba_log_in",
+ ),
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_collect_failed",
+ block_labels=["log_in", "collect_titles"],
+ executed_block_labels=["log_in", "collect_titles"],
+ executed_block_associations=("cba_log_in", "cba_collect_titles"),
+ completed_block_associations=("cba_log_in",),
+ ),
+ )
+
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.failed_operation is None
+
+
+def test_post_run_enrichment_does_not_resurrect_a_released_stop() -> None:
+ ctx = _locator_packet_ctx()
+ ctx.runner_code_block_associations_by_label = {"log_in": "cba_log_in"}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_login_failed",
+ block_labels=["log_in"],
+ structural_failure_identity="browser-operation",
+ failed_operation=BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_login_failed",
+ block_label="log_in",
+ failing_line=2,
+ block_association="cba_log_in",
+ ),
+ ),
+ )
+ released = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="not_authoritative",
+ reason_code="run_completed_unevaluated",
+ workflow_run_id="wr_login_completed",
+ block_labels=["log_in"],
+ executed_block_labels=["log_in"],
+ executed_block_associations=("cba_log_in",),
+ completed_block_associations=("cba_log_in",),
+ )
+ record_build_test_outcome(ctx, released)
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.failed_operation is None
+
+ assert bind_post_run_page_evidence(
+ ctx,
+ {"workflow_run_id": "wr_login_completed"},
+ None,
+ regraded=released,
+ )
+
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.failed_operation is None
+
+
+def test_stop_holds_when_the_failed_block_only_terminated_in_the_later_run() -> None:
+ ctx = _locator_packet_ctx()
+ ctx.runner_code_block_associations_by_label = {"log_in": "cba_log_in"}
+ failed_operation = BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_login_failed",
+ block_label="log_in",
+ failing_line=2,
+ block_association="cba_log_in",
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_login_failed",
+ block_labels=["log_in"],
+ structural_failure_identity="browser-operation",
+ failed_operation=failed_operation,
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="not_authoritative",
+ reason_code="terminal_challenge_blocker",
+ workflow_run_id="wr_login_terminated",
+ block_labels=["log_in"],
+ executed_block_labels=["log_in"],
+ executed_block_associations=("cba_log_in",),
+ ),
+ )
+
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.failed_operation == failed_operation
+
+
@pytest.mark.asyncio
async def test_browser_operation_outcome_clears_after_server_composed_verified_retest(
monkeypatch: pytest.MonkeyPatch,
@@ -3562,6 +3716,7 @@ workflow_definition:
reason_code="verified_success",
workflow_run_id="wr_retested_repair",
executed_block_associations=("cba_collect_failure_rate",),
+ completed_block_associations=("cba_collect_failure_rate",),
verified_progress_marker="run-completed",
),
)
diff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.py
index 48d46101c..c664c7fa7 100644
--- a/tests/unit/test_copilot_terminal_envelope.py
+++ b/tests/unit/test_copilot_terminal_envelope.py
@@ -304,10 +304,28 @@ def test_failed_operation_without_a_crash_keeps_its_own_sentence() -> None:
message, replaced = render_terminal_message(envelope, "The code run failed.", cancelled=False)
assert replaced is True
- assert "browser operation failed" in message.lower()
+ assert message.startswith("I stopped after a browser operation failed while testing the workflow.")
+ assert "`" not in message
assert INTERRUPTED_TERMINAL_HEADLINE not in message
+def test_failed_operation_sentence_names_the_block_that_failed() -> None:
+ envelope = _assemble(
+ proposal_disposition="review_untested",
+ failed_operation=BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ block_label="continue_to_payment",
+ ),
+ proposal_present=True,
+ )
+ message, replaced = render_terminal_message(envelope, "The code run failed.", cancelled=False)
+
+ assert replaced is True
+ assert message.startswith(
+ "I stopped after a browser operation failed in `continue_to_payment` while testing the workflow."
+ )
+
+
def test_unexpected_error_exit_renders_interrupted_and_keeps_its_terminal_reason() -> None:
ctx = make_copilot_ctx(
latest_recorded_build_test_outcome=RecordedBuildTestOutcome(
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 749a1ddd8..7a76ba94c 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -29,6 +29,7 @@ from skyvern.forge.taskv3.loop import (
ACTION_BUDGET_EXTENDED_EVENT,
ACTION_BUDGET_EXTENSION_REFUSED_EVENT,
ACTION_LOOP_REASON_PREFIX,
+ CANONICAL_SURVIVAL_EVENT,
FAILURE_EVIDENCE_MIN_TOOL_CALLS,
FAILURE_EVIDENCE_MIN_TURNS,
NAV_DEAD_END_REASON_PREFIX,
@@ -6447,6 +6448,73 @@ async def test_progress_ledger_emits_terminal_survival_record() -> None:
assert final[0]["would_fire"] is True
+async def test_non_form_run_emits_its_own_survival_record_keyed_on_canonical_touches() -> None:
+ # The ledger's survival record is gated on ever_armed, so it covers only runs that saw a
+ # validation error. Every search / filter / navigate / extract run — the whole non-form half of
+ # the product — emitted NOTHING, which is why a fire count off that population is a floor and
+ # never a prevalence. This is the complement record, keyed on the canonical tracker's counters
+ # because they are the only progress signal defined without a form.
+ rounds = 6
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ tools = [_form_observe("observe", [0] * rounds), _billable_tool("click", clicks), make_finish_tool()]
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("observe", {})]]
+ # One batch against ONE target: touches accumulate inside the turn, so the peak does not depend
+ # on whether a later observe clears the ring.
+ script.append([("click", {"selector": "#stuck"}) for _ in range(4)])
+ script.append([("finish", {"status": "completed", "reason": "done"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=200, max_tool_calls=500)
+
+ assert outcome.status == "completed"
+ survival = [e for e in logs if e.get("event") == CANONICAL_SURVIVAL_EVENT]
+ assert len(survival) == 1, survival
+ assert survival[0]["outcome_status"] == "completed"
+ # Load-bearing: the record has to carry the same-target churn, or it is an empty denominator.
+ assert survival[0]["peak_same_touches"] >= 4, survival[0]
+ # The two records partition the population — a non-form run must not also emit the form one, or
+ # the union double-counts and the denominator is wrong in the other direction.
+ assert not [e for e in logs if e.get("event") == PROGRESS_LEDGER_FINAL_EVENT]
+
+
+async def test_a_run_still_gets_exactly_one_survival_record_with_the_ledger_disabled() -> None:
+ # The partition must be TOTAL, not conditional on an unrelated flag. The canonical tracker is
+ # built and updated unconditionally, so its record does not depend on the ledger existing —
+ # gating it on `progress` would drop BOTH records whenever progress_window is None and silently
+ # restore the "floor, not prevalence" hole this record exists to close.
+ rounds = 4
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ tools = [_form_observe("observe", [3] * rounds), _billable_tool("click", clicks), make_finish_tool()]
+ script = [[("observe", {}), ("click", {"selector": f"#f{i}"})] for i in range(rounds)]
+ script.append([("finish", {"status": "completed", "reason": "done"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=200, max_tool_calls=500, progress_window=None)
+
+ assert outcome.status == "completed"
+ records = [e for e in logs if e.get("event") in (PROGRESS_LEDGER_FINAL_EVENT, CANONICAL_SURVIVAL_EVENT)]
+ assert len(records) == 1, records
+ assert records[0]["event"] == CANONICAL_SURVIVAL_EVENT
+
+
+async def test_form_run_still_emits_only_the_ledger_record_and_not_the_non_form_one() -> None:
+ # The other half of the partition, and the compatibility guarantee: the existing record's
+ # population and shape are untouched, so the survival data already collected stays comparable
+ # with everything collected after this change.
+ rounds = 10
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ tools = [_form_observe("observe", [3] * rounds), _billable_tool("click", clicks), make_finish_tool()]
+ script = [[("observe", {}), ("click", {"selector": f"#f{i}"})] for i in range(rounds)]
+ script.append([("finish", {"status": "completed", "reason": "done"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=200, max_tool_calls=500)
+
+ assert outcome.status == "completed"
+ assert len([e for e in logs if e.get("event") == PROGRESS_LEDGER_FINAL_EVENT]) == 1
+ assert not [e for e in logs if e.get("event") == CANONICAL_SURVIVAL_EVENT]
+
+
@pytest.mark.asyncio
async def test_progress_ledger_silent_when_actions_batch_before_a_confirming_observe() -> None:
# Healthy batch: the model fixes several fields in one turn before re-observing (markers stay