Skip to content

Automate browser based workflows with AI

Keep a run's repairable block failure when a later record concludes nothing (SKY-15902) (#8495)

Andrew Neilson491da161949c
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 62c2f5d02..cb9a6de6c 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -55,7 +55,10 @@ from skyvern.forge.sdk.copilot.browser_ablation import (
resolve_copilot_tool_surface,
)
from skyvern.forge.sdk.copilot.budget_expiry import BudgetExpiryState, serialize_prior_budget_expiry
-from skyvern.forge.sdk.copilot.build_test_connect_failure import BuildTestConnectFailure
+from skyvern.forge.sdk.copilot.build_test_connect_failure import (
+ BuildTestConnectFailure,
+ build_test_connect_failure_sentence,
+)
from skyvern.forge.sdk.copilot.build_test_outcome import (
_TEXT_MAX,
_VALUE_EXCERPT_MAX,
@@ -1165,6 +1168,15 @@ def _recorded_build_test_outcome_prompt(ctx: CopilotContext | None) -> str:
"Repair the persisted code at this evidenced block/line, then test the changed attempt with "
"edit_block_and_run before reporting it."
)
+ # Rendered from the typed fact rather than folded into observed_evidence, which clips at 160
+ # characters and would drop whichever of the two failures came second.
+ connect_failure = outcome.connect_failure
+ if connect_failure is not None:
+ # The typed sentence, not the bare state: it says whether to wait or take a fresh session.
+ # The occupying run id is cleared first -- prompt atoms carry no raw run ids, and leaving it
+ # in makes the leak guard drop the whole line rather than redact the id.
+ sentence = build_test_connect_failure_sentence(connect_failure.model_copy(update={"occupier_run_id": None}))
+ lines.append(f"browser_acquisition_failed: {_clean_authoring_repair_prompt_atom(sentence)}")
# Facts render for every outcome; the two post-run page-path directives bind the model's next
# action, so they keep the authority check that gated this whole section before.
page_path_failure = outcome.page_path_failure
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 46c4bcb45..aa7ba17d4 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -759,7 +759,31 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re
prior = getattr(ctx, "latest_recorded_build_test_outcome", None)
if not isinstance(prior, RecordedBuildTestOutcome):
prior = None
- if outcome.failed_operation is not None:
+ # Keyed on the failure itself, not on ``failed_operation``: that projection exists only for
+ # browser-operation failures, so a block that died on an ordinary Python exception would
+ # otherwise keep none of its diagnosis.
+ # An acquisition failure that never reached a run names none, so it cannot contradict the prior
+ # failure and merges into it; the merged record keeps that run's identity, since the rebase below
+ # starts from it. Only a record naming a *different* run is a separate outcome.
+ prior_same_run_block_failure = (
+ prior is not None
+ and prior.reason_code == "runtime_block_failure"
+ and bool(prior.workflow_run_id)
+ and outcome.workflow_run_id in (None, "", prior.workflow_run_id)
+ )
+ merged_into_prior_failure = False
+ if outcome.connect_failure is not None and prior is not None and prior_same_run_block_failure:
+ # An acquisition failure concluded nothing about the blocks, so the run's own failure record
+ # stands and the acquisition fact rides along with it. Rebased on that record rather than
+ # copied field by field: the repair prompt reads a dozen diagnostic fields -- evidence refs,
+ # page capture, observed summary -- and a copy list would silently drop each one it grows.
+ # The acquisition fact stays typed in ``connect_failure``; the repair prompt renders it from
+ # there. Folding it into the summary would put two failures through one 160-character clip.
+ # The page-path continuation is dropped: it binds the model to act on the page the prior run
+ # left behind, and the browser holding that page is the one this record says was not acquired.
+ outcome = prior.model_copy(update={"connect_failure": outcome.connect_failure, "page_path_failure": None})
+ merged_into_prior_failure = True
+ elif outcome.failed_operation is not None:
source_yaml = _executed_workflow_yaml(ctx)
association = outcome.failed_operation.block_association
failed_code = _code_for_runner_association(ctx, source_yaml, association)
@@ -789,7 +813,11 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re
ctx.latest_recorded_build_test_outcome = outcome
raw_history = getattr(ctx, "recorded_build_test_outcome_history", None)
history: list[dict[str, object]] = raw_history if isinstance(raw_history, list) else []
- history.append(_history_entry(ctx, outcome))
+ # A merged record adds no block outcome -- the prior entry already holds this failure, snapshotted
+ # against the code the run executed. Appending would re-snapshot it against the current draft,
+ # so a later clearance check would compare the failure to code it never ran.
+ if not merged_into_prior_failure:
+ history.append(_history_entry(ctx, outcome))
del history[:-_HISTORY_LIMIT]
ctx.recorded_build_test_outcome_history = history
if outcome.phase == "persisted_block_run" and outcome.is_authoritative and outcome.workflow_run_id:
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 23525bf79..f4fe27036 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -42,6 +42,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
)
from skyvern.forge.sdk.copilot.build_test_outcome import (
+ BuildTestConnectFailure,
ChallengeEffects,
Lever,
PostRunPagePathFailure,
@@ -752,6 +753,59 @@ workflow_definition:
assert "Declare code_artifact_metadata goal_value_paths" in prompt
assert "Coastal" not in prompt
+ def test_recorded_build_test_outcome_prompt_keeps_both_failures_past_the_summary_clip(self) -> None:
+ """A long block diagnosis must not push the browser loss out of the prompt: the summary
+ clips at 160 characters, so the acquisition fact renders from its own typed field."""
+ long_diagnosis = "select_option timed out waiting for the state dropdown; " + "detail " * 30
+ assert len(long_diagnosis) > 160
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ latest_recorded_build_test_outcome=RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_failed",
+ attempted_block_label="collect_credentials",
+ observed_evidence_summary=long_diagnosis,
+ connect_failure=BuildTestConnectFailure(
+ state="cdp_connect_failed",
+ browser_session_id="pbs_stale",
+ ),
+ ),
+ )
+
+ prompt = agent_module._recorded_build_test_outcome_prompt(ctx)
+
+ assert "select_option timed out" in prompt
+ assert "browser_acquisition_failed:" in prompt
+ assert "cdp_connect_failed" in prompt
+
+ def test_recorded_build_test_outcome_prompt_says_what_to_do_about_an_occupied_browser(self) -> None:
+ """The bare state does not say what to do. The occupying run id stays out of prompt text, and
+ must not take the guidance with it: the leak guard blanks any atom that carries one."""
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ latest_recorded_build_test_outcome=RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_failed",
+ attempted_block_label="collect_credentials",
+ connect_failure=BuildTestConnectFailure(
+ state="occupied",
+ occupier_run_id="wr_holder",
+ ),
+ ),
+ )
+
+ prompt = agent_module._recorded_build_test_outcome_prompt(ctx)
+
+ assert "fresh session" in prompt
+ assert "already running another test" in prompt
+ assert "wr_holder" not in prompt
+
def test_recorded_build_test_outcome_prompt_does_not_offer_page_actions_for_non_page_outcome(self) -> None:
ctx = _ctx(
block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 89cee6474..2bdfaf3ad 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -42,6 +42,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestPacketRegisteredOutput,
BuildTestPacketRequestedOutput,
BuildTestPacketUnfinishedItem,
+ PostRunPagePathFailure,
RecordedBuildTestOutcome,
authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
@@ -328,6 +329,405 @@ def test_connect_failure_clears_when_a_later_real_run_records_recovery() -> None
assert ctx.recorded_build_test_outcome_history[-1]["connect_failure"] is None
+def test_post_run_connect_failure_keeps_the_runs_repairable_block_failure() -> None:
+ ctx = _run_history_ctx(two_page_login_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ failed_operation = BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_1",
+ workflow_run_block_id="wrb_1",
+ block_label="collect_credentials",
+ failing_line=12,
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_1",
+ attempted_block_label="collect_credentials",
+ failed_block_labels=["collect_credentials"],
+ failed_operation=failed_operation,
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="unrecoverable_tool_error",
+ workflow_run_id="wr_1",
+ connect_failure=BuildTestConnectFailure(
+ state="cdp_connect_failed",
+ browser_session_id="pbs_stale",
+ ),
+ ),
+ )
+
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ assert latest.failed_operation == failed_operation
+ assert latest.verdict == "repairable_failure"
+ assert latest.reason_code == "runtime_block_failure"
+ assert latest.attempted_block_label == "collect_credentials"
+ assert latest.failed_block_labels == ["collect_credentials"]
+ assert latest.connect_failure is not None
+ assert latest.connect_failure.state == "cdp_connect_failed"
+
+
+def test_acquisition_failure_without_a_run_id_still_leaves_the_earlier_failure_findable() -> None:
+ ctx = _run_history_ctx(two_page_login_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_1",
+ attempted_block_label="collect_credentials",
+ failed_block_labels=["collect_credentials"],
+ failed_operation=BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_1",
+ block_label="collect_credentials",
+ failing_line=12,
+ ),
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="unrecoverable_tool_error",
+ connect_failure=BuildTestConnectFailure(
+ state="cdp_connect_failed",
+ browser_session_id="pbs_stale",
+ ),
+ ),
+ )
+
+ # An acquisition failure that never reached a run names none, so it merges into the failure it
+ # follows and keeps that run's identity. The record must never carry runtime_block_failure under
+ # an empty run id: the scan reads that as an incomplete record and stops, reporting nothing.
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ assert latest.workflow_run_id == "wr_1"
+ assert latest.attempted_block_label == "collect_credentials"
+ assert latest.connect_failure is not None
+ assert ctx.recorded_build_test_outcome_history[-1]["workflow_run_id"] == "wr_1"
+ assert unresolved_runtime_block_failure_with_disposition(ctx, reported_workflow_is_persisted=True)[1] != (
+ "incomplete_failure_record"
+ )
+
+
+def test_a_post_run_connect_failure_keeps_the_block_failures_diagnostic_evidence() -> None:
+ ctx = _run_history_ctx(two_page_login_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_1",
+ attempted_block_label="collect_credentials",
+ failed_block_labels=["collect_credentials"],
+ observed_evidence_summary="select_option timed out waiting for the state dropdown",
+ structural_failure_identity="locator-timeout-identity",
+ page_evidence_refs=["pe_1"],
+ evidence_refs=["ev_1"],
+ failed_operation=BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_1",
+ block_label="collect_credentials",
+ failing_line=12,
+ ),
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="unrecoverable_tool_error",
+ workflow_run_id="wr_1",
+ structural_failure_identity="build_test_connect:cdp_connect_failed",
+ observed_evidence_summary="Build-test browser acquisition stopped: cdp_connect_failed.",
+ connect_failure=BuildTestConnectFailure(
+ state="cdp_connect_failed",
+ browser_session_id="pbs_stale",
+ ),
+ ),
+ )
+
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ # The diagnosis the model repairs from, not just the label naming what failed.
+ assert "select_option timed out" in latest.observed_evidence_summary
+ assert latest.page_evidence_refs == ["pe_1"]
+ assert latest.evidence_refs == ["ev_1"]
+ assert latest.structural_failure_identity == "locator-timeout-identity"
+ # The acquisition fact is additional, not a replacement. It stays typed rather than folded into
+ # the summary, which the repair prompt clips to 160 characters.
+ assert latest.connect_failure is not None
+ assert latest.connect_failure.state == "cdp_connect_failed"
+ assert "cdp_connect_failed" not in latest.observed_evidence_summary
+
+
+def test_a_block_that_died_on_a_plain_exception_is_preserved_like_a_browser_failure() -> None:
+ """``failed_operation`` is populated only for browser-operation failures, so keying the merge on
+ it would leave an ordinary Python exception with none of its diagnosis."""
+ ctx = _run_history_ctx(two_page_login_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_1",
+ attempted_block_label="collect_credentials",
+ failed_block_labels=["collect_credentials"],
+ observed_evidence_summary="KeyError: 'state' while building the payload",
+ page_evidence_refs=["pe_1"],
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="unrecoverable_tool_error",
+ workflow_run_id="wr_1",
+ observed_evidence_summary="Build-test browser acquisition stopped: cdp_connect_failed.",
+ connect_failure=BuildTestConnectFailure(
+ state="cdp_connect_failed",
+ browser_session_id="pbs_stale",
+ ),
+ ),
+ )
+
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ assert latest.failed_operation is None
+ assert latest.verdict == "repairable_failure"
+ assert latest.reason_code == "runtime_block_failure"
+ assert latest.attempted_block_label == "collect_credentials"
+ assert "KeyError" in latest.observed_evidence_summary
+ assert latest.page_evidence_refs == ["pe_1"]
+ assert latest.connect_failure is not None
+
+
+def test_merging_an_acquisition_failure_drops_the_stale_page_continuation() -> None:
+ """The continuation binds the model to act on the prior run's page, in the browser this same
+ record says could not be acquired."""
+ ctx = _run_history_ctx(two_page_login_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_1",
+ attempted_block_label="collect_credentials",
+ page_path_failure=PostRunPagePathFailure(
+ kind="login",
+ workflow_run_id="wr_1",
+ current_url="https://example.test/login",
+ continuation_targets=(),
+ ),
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="unrecoverable_tool_error",
+ workflow_run_id="wr_1",
+ connect_failure=BuildTestConnectFailure(
+ state="cdp_connect_failed",
+ browser_session_id="pbs_stale",
+ ),
+ ),
+ )
+
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ assert latest.page_path_failure is None
+ assert latest.reason_code == "runtime_block_failure"
+ assert latest.connect_failure is not None
+
+
+def test_a_terminal_challenge_keeps_its_own_reason_over_an_earlier_block_failure() -> None:
+ """A terminal challenge outranks a failed block to suppress code repair. Restating it as that
+ failure would send the model to edit code that was never the obstacle."""
+ ctx = _run_history_ctx(two_page_login_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_1",
+ attempted_block_label="collect_credentials",
+ failed_block_labels=["collect_credentials"],
+ failed_operation=BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_1",
+ block_label="collect_credentials",
+ failing_line=12,
+ ),
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="terminal_challenge_blocker",
+ workflow_run_id="wr_1",
+ ),
+ )
+
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ assert latest.reason_code == "terminal_challenge_blocker"
+ assert latest.verdict == "not_authoritative"
+ assert latest.attempted_block_label == ""
+ assert ctx.recorded_build_test_outcome_history[-1]["reason_code"] == "terminal_challenge_blocker"
+
+
+def test_a_block_failure_with_no_run_id_is_never_merged_into_an_acquisition_failure() -> None:
+ """The merged record must carry a real run id. Stamped runtime_block_failure under an empty one,
+ the unresolved-failure scan reads it as an incomplete record and stops, reporting nothing."""
+ ctx = _run_history_ctx(two_page_login_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ attempted_block_label="collect_credentials",
+ failed_block_labels=["collect_credentials"],
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="unrecoverable_tool_error",
+ connect_failure=BuildTestConnectFailure(
+ state="cdp_connect_failed",
+ browser_session_id="pbs_stale",
+ ),
+ ),
+ )
+
+ newest = ctx.recorded_build_test_outcome_history[-1]
+ assert not (newest["reason_code"] == "runtime_block_failure" and not newest["workflow_run_id"])
+
+
+def test_a_different_runs_blocker_is_not_restated_as_the_earlier_runs_block_failure() -> None:
+ ctx = _run_history_ctx(two_page_login_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_1",
+ attempted_block_label="collect_credentials",
+ failed_block_labels=["collect_credentials"],
+ failed_operation=BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_1",
+ block_label="collect_credentials",
+ failing_line=12,
+ ),
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="terminal_challenge_blocker",
+ workflow_run_id="wr_2",
+ ),
+ )
+
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ assert latest.workflow_run_id == "wr_2"
+ assert latest.verdict == "not_authoritative"
+ assert latest.reason_code == "terminal_challenge_blocker"
+ assert latest.attempted_block_label == ""
+ assert latest.failed_block_labels == []
+ # The earlier run's failure is still carried, and is still the one the unresolved-failure scan
+ # finds: wr_2's own entry must not be stamped as a runtime block failure under wr_2's id.
+ assert latest.failed_operation is not None
+ assert latest.failed_operation.workflow_run_id == "wr_1"
+ assert ctx.recorded_build_test_outcome_history[-1]["reason_code"] == "terminal_challenge_blocker"
+ unresolved = unresolved_runtime_block_failure_with_disposition(ctx, reported_workflow_is_persisted=True)[0]
+ assert unresolved is not None
+ assert unresolved.workflow_run_id == "wr_1"
+
+
+def test_later_run_with_its_own_failed_block_replaces_the_earlier_attribution() -> None:
+ ctx = _run_history_ctx(two_page_login_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_1",
+ attempted_block_label="collect_credentials",
+ failed_block_labels=["collect_credentials"],
+ failed_operation=BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_1",
+ block_label="collect_credentials",
+ failing_line=12,
+ ),
+ ),
+ )
+ second_failure = BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_2",
+ block_label="sign_in_and_read",
+ failing_line=3,
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_2",
+ attempted_block_label="sign_in_and_read",
+ failed_block_labels=["sign_in_and_read"],
+ failed_operation=second_failure,
+ ),
+ )
+
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ assert latest.failed_operation == second_failure
+ assert latest.attempted_block_label == "sign_in_and_read"
+ assert latest.failed_block_labels == ["sign_in_and_read"]
+
+
def _failed_run_result_with_categories(categories: list[dict]) -> dict:
return {
"ok": False,
@@ -2183,6 +2583,86 @@ def test_a_templated_selector_does_not_read_as_the_call_being_removed() -> None:
assert open_failure.block_label == "sign_in_and_read"
+def test_merging_an_acquisition_failure_does_not_resnapshot_the_failure_against_edited_code() -> None:
+ """History snapshots are taken from the current draft, so re-appending the carried failure after
+ an edit would record it against code the run never executed."""
+ ctx = _run_history_ctx(_templated_selector_yaml())
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ failed_second_factor_run("wr_1").model_copy(update={"attempted_call_ref": ""}),
+ )
+ entries_before = len(ctx.recorded_build_test_outcome_history)
+ signature_at_failure = ctx.recorded_build_test_outcome_history[-1]["attempted_block_signature"]
+ assert signature_at_failure
+
+ # The model repairs the block, then the next dispatch dies before a run exists.
+ edited = _templated_selector_yaml(templated=False)
+ ctx.workflow_yaml = edited
+ ctx.persisted_workflow_yaml = edited
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="unrecoverable_tool_error",
+ connect_failure=BuildTestConnectFailure(
+ state="cdp_connect_failed",
+ browser_session_id="pbs_stale",
+ ),
+ ),
+ )
+
+ assert len(ctx.recorded_build_test_outcome_history) == entries_before
+ assert ctx.recorded_build_test_outcome_history[-1]["attempted_block_signature"] == signature_at_failure
+ # The diagnosis still reaches the repair prompt, which reads only the newest outcome.
+ latest = ctx.latest_recorded_build_test_outcome
+ assert latest is not None
+ assert latest.reason_code == "runtime_block_failure"
+ assert latest.connect_failure is not None
+
+
+def test_a_carried_failure_keeps_the_call_reference_so_an_edit_alone_cannot_retire_it() -> None:
+ """Carrying the block label without its call reference would downgrade clearance to a
+ whole-block signature comparison, which any unrelated edit trips."""
+ delivered = _templated_selector_yaml()
+ ctx = _run_history_ctx(delivered)
+ ctx.runner_code_block_associations_by_label = {}
+ record_build_test_outcome(
+ ctx,
+ failed_second_factor_run("wr_1").model_copy(
+ update={
+ "attempted_call_ref": "locator:#submit-btn",
+ "failed_operation": BuildTestFailedOperation(
+ kind="browser_operation_failed",
+ workflow_run_id="wr_1",
+ block_label="sign_in_and_read",
+ failing_line=2,
+ ),
+ }
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ verdict="not_authoritative",
+ reason_code="unrecoverable_tool_error",
+ workflow_run_id="wr_1",
+ connect_failure=BuildTestConnectFailure(
+ state="cdp_connect_failed",
+ browser_session_id="pbs_stale",
+ ),
+ ),
+ )
+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
+
+ assert ctx.recorded_build_test_outcome_history[-2]["attempted_call_ref"] == "locator:#submit-btn"
+ open_failure = unresolved_runtime_block_failure(ctx, reported_workflow_yaml=delivered)
+ assert open_failure is not None
+ assert open_failure.block_label == "sign_in_and_read"
+
+
def test_an_all_literal_block_still_retires_when_the_call_is_gone() -> None:
"""The pass path: with every selector a literal, absence really is proof of removal."""
delivered = _templated_selector_yaml(templated=False)