Skip to content

Automate browser based workflows with AI

fix(SKY-12073): make the copilot repair ceiling reachable after a failed run (#7244)

Andrew Neilsonf2acc9293914
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index d43bb8094..f82e620b7 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -247,6 +247,7 @@ class _BlockerSignalCtx(_LoopEvidenceCtx, Protocol):
class _TerminalEvidenceResetCtx(Protocol):
last_run_blocks_workflow_run_id: str | None
last_successful_run_blocks_workflow_run_id: str | None
+ recorded_persisted_block_run_workflow_run_id: str | None
last_run_blocks_block_ids: list[str]
last_run_blocks_block_labels: list[str]
last_run_outcome: RecordedRunOutcome | None
@@ -282,6 +283,7 @@ def terminal_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> TerminalEvidence:
def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) -> None:
ctx.last_run_blocks_workflow_run_id = None
ctx.last_successful_run_blocks_workflow_run_id = None
+ ctx.recorded_persisted_block_run_workflow_run_id = None
ctx.last_run_blocks_block_ids = []
ctx.last_run_blocks_block_labels = []
ctx.last_run_outcome = None
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index b0e17d970..9c9737c61 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -228,6 +228,7 @@ class RecordedOutcomeBindingConstraint(BaseModel):
class _RecordedBuildTestOutcomeContext(Protocol):
latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None
recorded_build_test_outcome_history: list[dict[str, object]]
+ recorded_persisted_block_run_workflow_run_id: str | None
recorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None
recorded_outcome_binding_constraint: RecordedOutcomeBindingConstraint | None
@@ -253,6 +254,8 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re
)
del history[:-_HISTORY_LIMIT]
ctx.recorded_build_test_outcome_history = history
+ if outcome.phase == "persisted_block_run" and outcome.is_authoritative and outcome.workflow_run_id:
+ ctx.recorded_persisted_block_run_workflow_run_id = outcome.workflow_run_id
LOG.info(
"copilot recorded build-test outcome stored",
phase=outcome.phase,
@@ -398,14 +401,10 @@ def latest_recorded_build_test_outcome_repeated(ctx: object) -> bool | None:
def run_backed_repair_evidence_exists(ctx: object) -> bool:
- fallback_run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
- latest = getattr(ctx, "latest_recorded_build_test_outcome", None)
- if isinstance(latest, RecordedBuildTestOutcome):
- # An author-time reject is never run-backed even if a stale run id lingers on ctx; only a persisted run counts.
- if latest.phase != "persisted_block_run":
- return False
- return bool(latest.workflow_run_id or fallback_run_id)
- return bool(fallback_run_id)
+ # Reached from the enforcement belt with an untyped ctx; a ctx without the latch must read as
+ # "no run-backed evidence" so the guardrail fails safe instead of raising.
+ run_id = getattr(ctx, "recorded_persisted_block_run_workflow_run_id", None)
+ return isinstance(run_id, str) and bool(run_id)
def arm_recorded_outcome_grounding_requirement(ctx: object) -> RecordedOutcomeGroundingRequirement | None:
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index b518762c3..91c8f1e27 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -758,6 +758,7 @@ class CopilotContext(AgentContext):
last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None
recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)
+ recorded_persisted_block_run_workflow_run_id: str | None = None
recorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None = None
recorded_outcome_binding_constraint: RecordedOutcomeBindingConstraint | None = None
# Turn-scoped monotonic marks of verified forward progress: the union of
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index e9be08224..ca39fbca5 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -297,6 +297,7 @@ class AgentContext:
last_run_outcome_block_labels: list[str] = field(default_factory=list)
latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None
recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)
+ recorded_persisted_block_run_workflow_run_id: str | None = None
recorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None = None
recorded_outcome_binding_constraint: RecordedOutcomeBindingConstraint | None = None
consecutive_non_converging_repair_count: int = 0
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index fb852942e..60d240e08 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -3295,6 +3295,9 @@ def _update_repair_loop_state(copilot_ctx: CopilotContext, contract: DiagnosisRe
arm_recorded_outcome_grounding_requirement(copilot_ctx)
if contract.repair_loop_state.ceiling_reached and run_backed_repair_evidence_exists(copilot_ctx):
signal = repair_ceiling_stop_signal(copilot_ctx, contract)
+ contract.repair_decision = contract.repair_decision.model_copy(
+ update={"next_action": RepairNextAction.STOP, "target_blocks": []}
+ )
stash_blocker_signal(copilot_ctx, signal)
stash_repair_ceiling_turn_halt(copilot_ctx, signal, consecutive_identical_repair_count=count)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index f18d63f2b..a6f42c8c6 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -8359,6 +8359,16 @@ async def _update_workflow(
block_labels=missing_labels,
missing_requested_output_facts=missing_metadata_output_facts,
)
+ credential_scout_errors = (
+ []
+ if _request_policy_allows_untested_code_block_draft(ctx)
+ else _credentialed_code_block_scout_gate_errors(
+ workflow_yaml,
+ ctx,
+ block_labels=params.get("block_labels"),
+ )
+ )
+ _record_code_authoring_guardrail_reject(ctx, defer_churn_stop=bool(credential_scout_errors))
return reject(
error=missing_metadata_error,
user_facing_summary=_compiled_authoring_user_summary(),
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 18aa86d7f..cb8760f4b 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
recorded_outcome_from_loaded_result_evidence,
recorded_outcome_from_run_blocks_result,
recorded_outcome_from_scout_act_observe_hollow,
+ run_backed_repair_evidence_exists,
)
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
@@ -302,7 +303,11 @@ def test_prose_or_label_only_typed_outcome_is_not_authoritative() -> None:
def test_record_none_clears_stale_latest_outcome() -> None:
- ctx = SimpleNamespace(latest_recorded_build_test_outcome=None, recorded_build_test_outcome_history=[])
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=None,
+ recorded_build_test_outcome_history=[],
+ recorded_persisted_block_run_workflow_run_id=None,
+ )
record_build_test_outcome(
ctx,
RecordedBuildTestOutcome(
@@ -320,8 +325,76 @@ def test_record_none_clears_stale_latest_outcome() -> None:
assert ctx.latest_recorded_build_test_outcome is None
+def test_record_authoritative_persisted_run_latches_run_backed_evidence() -> None:
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=None,
+ recorded_build_test_outcome_history=[],
+ recorded_persisted_block_run_workflow_run_id=None,
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_recorded",
+ structural_failure_identity="runtime:failed",
+ ),
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="author_time_reject",
+ attempted_tool="update_workflow",
+ verdict="authoring_rejected",
+ reason_code="metadata_reject",
+ structural_failure_identity="metadata:missing",
+ ),
+ )
+
+ assert ctx.recorded_persisted_block_run_workflow_run_id == "wr_recorded"
+ assert run_backed_repair_evidence_exists(ctx) is True
+
+
+def test_fallback_run_id_without_recorded_persisted_outcome_is_not_run_backed() -> None:
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=None,
+ recorded_build_test_outcome_history=[],
+ recorded_persisted_block_run_workflow_run_id=None,
+ last_run_blocks_workflow_run_id="wr_stale",
+ )
+
+ assert run_backed_repair_evidence_exists(ctx) is False
+
+
+def test_non_authoritative_persisted_run_does_not_latch_run_backed_evidence() -> None:
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=None,
+ recorded_build_test_outcome_history=[],
+ recorded_persisted_block_run_workflow_run_id=None,
+ )
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_hollow",
+ ),
+ )
+
+ assert ctx.recorded_persisted_block_run_workflow_run_id is None
+ assert run_backed_repair_evidence_exists(ctx) is False
+
+
def test_repeated_outcome_ignores_intervening_scout_evaluate_history() -> None:
- ctx = SimpleNamespace(latest_recorded_build_test_outcome=None, recorded_build_test_outcome_history=[])
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=None,
+ recorded_build_test_outcome_history=[],
+ recorded_persisted_block_run_workflow_run_id=None,
+ )
author_reject = RecordedBuildTestOutcome(
phase="author_time_reject",
verdict="authoring_rejected",
@@ -343,7 +416,11 @@ def test_repeated_outcome_ignores_intervening_scout_evaluate_history() -> None:
def test_repeated_outcome_still_detects_different_author_reject_after_scout_evaluate() -> None:
- ctx = SimpleNamespace(latest_recorded_build_test_outcome=None, recorded_build_test_outcome_history=[])
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=None,
+ recorded_build_test_outcome_history=[],
+ recorded_persisted_block_run_workflow_run_id=None,
+ )
first_reject = RecordedBuildTestOutcome(
phase="author_time_reject",
verdict="authoring_rejected",
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index d1f4d2e5b..803c04e62 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -3910,6 +3910,7 @@ class TestCodeRepairProgressClassification:
structural_failure_identity="completion:typed-output",
authored_structure_signature="previous-authoring-signature",
)
+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_recorded"
recorded_sig = authored_block_signatures_from_workflow(workflow_yaml, metadata)["submit_service_request"]
ctx.recorded_outcome_binding_constraint = RecordedOutcomeBindingConstraint(
repeated_structural_key=ctx.latest_recorded_build_test_outcome.structural_key or "",
@@ -7309,7 +7310,9 @@ class TestOutputContractBudgetRunGuard:
verdict="repairable_failure",
reason_code="failed_run",
workflow_run_id="wr_1",
+ structural_failure_identity="runtime:failed",
)
+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_1"
payload = workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="x")
@@ -7834,6 +7837,42 @@ class TestCompiledAuthoringImposition:
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ assert ctx.last_code_authoring_reject_was_credential_priority is False
+
+ @pytest.mark.asyncio
+ async def test_missing_metadata_with_credential_gap_uses_credential_priority_counter(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+ submitted = _yaml(
+ """
+ title: Credentialed lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: credential_id, key: login_credential, default_value: cred_missing}
+ - {parameter_type: output, key: lookup_result}
+ blocks:
+ - block_type: code
+ label: lookup_with_saved_credential
+ parameter_keys: [login_credential]
+ prompt: Sign in and return the lookup result.
+ code: |
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ return {"lookup_result": "sample"}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "must pass `code_artifact_metadata`" in result["error"]
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ assert ctx.last_code_authoring_reject_was_credential_priority is True
+ assert ctx.blocker_signal is None
@pytest.mark.asyncio
async def test_output_intent_rejects_partial_metadata_without_output_label(
diff --git a/tests/unit/test_copilot_enforcement_hardening.py b/tests/unit/test_copilot_enforcement_hardening.py
index 6ad4f062e..697ca7839 100644
--- a/tests/unit/test_copilot_enforcement_hardening.py
+++ b/tests/unit/test_copilot_enforcement_hardening.py
@@ -23,6 +23,7 @@ import pytest
from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
+ clear_terminal_evidence_on_workflow_edit,
maybe_clear_blocker_signal_on_tool_success,
)
from skyvern.forge.sdk.copilot.context import CopilotContext
@@ -454,6 +455,10 @@ def _ceiling_reached_contract() -> DiagnosisRepairContract:
)
+def _mark_recorded_run_backed(ctx: CopilotContext) -> None:
+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_1"
+
+
def test_zero_run_ceiling_yields_to_code_authoring_churn_backstop() -> None:
ctx = _fresh_context()
ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
@@ -473,6 +478,7 @@ def test_run_backed_ceiling_precedes_code_authoring_churn_backstop() -> None:
ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
ctx.latest_diagnosis_repair_contract = _ceiling_reached_contract()
ctx.last_run_blocks_workflow_run_id = "wr_1"
+ _mark_recorded_run_backed(ctx)
with pytest.raises(CopilotTurnHalt) as excinfo:
_check_enforcement(ctx)
@@ -483,6 +489,32 @@ def test_run_backed_ceiling_precedes_code_authoring_churn_backstop() -> None:
assert signal.internal_reason_code == "repair_ceiling_reached"
+def test_stale_fallback_run_id_does_not_make_ceiling_run_backed() -> None:
+ ctx = _fresh_context()
+ ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ ctx.latest_diagnosis_repair_contract = _ceiling_reached_contract()
+ ctx.last_run_blocks_workflow_run_id = "wr_stale"
+
+ with pytest.raises(CopilotTurnHalt) as excinfo:
+ _check_enforcement(ctx)
+
+ assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED
+ signal = ctx.blocker_signal
+ assert isinstance(signal, CopilotToolBlockerSignal)
+ assert signal.internal_reason_code == "code_authoring_guardrail_churn"
+
+
+def test_workflow_edit_clears_recorded_persisted_run_latch() -> None:
+ ctx = _fresh_context()
+ ctx.last_run_blocks_workflow_run_id = "wr_1"
+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_1"
+
+ clear_terminal_evidence_on_workflow_edit(ctx)
+
+ assert ctx.last_run_blocks_workflow_run_id is None
+ assert ctx.recorded_persisted_block_run_workflow_run_id is None
+
+
def test_credential_priority_churn_raises_at_higher_bound() -> None:
ctx = _fresh_context()
ctx.code_authoring_guardrail_reject_count = MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS
@@ -528,6 +560,7 @@ def test_run_backed_ceiling_precedes_credential_priority_churn() -> None:
ctx.last_code_authoring_reject_was_credential_priority = True
ctx.latest_diagnosis_repair_contract = _ceiling_reached_contract()
ctx.last_run_blocks_workflow_run_id = "wr_1"
+ _mark_recorded_run_backed(ctx)
with pytest.raises(CopilotTurnHalt) as excinfo:
_check_enforcement(ctx)
@@ -580,6 +613,7 @@ def test_run_backed_ceiling_precedes_no_progress_interaction_floor() -> None:
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
ctx.latest_diagnosis_repair_contract = _ceiling_reached_contract()
ctx.last_run_blocks_workflow_run_id = "wr_1"
+ _mark_recorded_run_backed(ctx)
with pytest.raises(CopilotTurnHalt) as excinfo:
_check_enforcement(ctx)
diff --git a/tests/unit/test_copilot_recorded_outcome_grounding.py b/tests/unit/test_copilot_recorded_outcome_grounding.py
index 583d9581d..13f7b81f0 100644
--- a/tests/unit/test_copilot_recorded_outcome_grounding.py
+++ b/tests/unit/test_copilot_recorded_outcome_grounding.py
@@ -240,6 +240,56 @@ def test_authoritative_persisted_outcome_arms_without_recorded_signature_prefix(
assert requirement.satisfied is False
+def test_run_backed_author_time_reject_stashes_repair_ceiling_on_first_crossing() -> None:
+ outcome = RecordedBuildTestOutcome(
+ phase="author_time_reject",
+ attempted_tool="update_workflow",
+ verdict="authoring_rejected",
+ reason_code="metadata_reject",
+ block_labels=["search_records"],
+ structural_failure_identity="metadata:missing-output",
+ )
+ ctx = _ctx(outcome)
+ key = outcome.structural_key
+ assert key is not None
+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_prior"
+ ctx.last_repair_non_convergence_signature = f"recorded_build_test_outcome:{key}"
+ ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1
+ contract = _contract()
+
+ _update_repair_loop_state(ctx, contract)
+
+ assert contract.repair_decision.next_action is RepairNextAction.STOP
+ assert contract.repair_loop_state.ceiling_reached is True
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind is TurnHaltKind.REPAIR_CEILING_REACHED
+
+
+def test_zero_run_author_time_reject_does_not_stash_repair_ceiling_on_first_crossing() -> None:
+ outcome = RecordedBuildTestOutcome(
+ phase="author_time_reject",
+ attempted_tool="update_workflow",
+ verdict="authoring_rejected",
+ reason_code="metadata_reject",
+ block_labels=["search_records"],
+ structural_failure_identity="metadata:missing-output",
+ )
+ ctx = _ctx(outcome)
+ key = outcome.structural_key
+ assert key is not None
+ ctx.last_run_blocks_workflow_run_id = "wr_stale"
+ ctx.last_repair_non_convergence_signature = f"recorded_build_test_outcome:{key}"
+ ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1
+ contract = _contract()
+
+ _update_repair_loop_state(ctx, contract)
+
+ assert contract.repair_loop_state.ceiling_reached is True
+ assert contract.repair_decision.next_action is RepairNextAction.REPAIR
+ assert ctx.turn_halt is None
+ assert ctx.blocker_signal is None
+
+
def test_progress_observed_outcome_does_not_arm_from_executed_run(
monkeypatch: pytest.MonkeyPatch,
) -> None: