Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12073): make the copilot repair ceiling reachable after a failed run (#7244)
Andrew Neilsonf2acc9293914
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex d43bb8094..f82e620b7 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -247,6 +247,7 @@ class _BlockerSignalCtx(_LoopEvidenceCtx, Protocol):class _TerminalEvidenceResetCtx(Protocol):last_run_blocks_workflow_run_id: str | Nonelast_successful_run_blocks_workflow_run_id: str | None+ recorded_persisted_block_run_workflow_run_id: str | Nonelast_run_blocks_block_ids: list[str]last_run_blocks_block_labels: list[str]last_run_outcome: RecordedRunOutcome | None@@ -282,6 +283,7 @@ def terminal_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> TerminalEvidence:def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) -> None:ctx.last_run_blocks_workflow_run_id = Nonectx.last_successful_run_blocks_workflow_run_id = None+ ctx.recorded_persisted_block_run_workflow_run_id = Nonectx.last_run_blocks_block_ids = []ctx.last_run_blocks_block_labels = []ctx.last_run_outcome = Nonediff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex b0e17d970..9c9737c61 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -228,6 +228,7 @@ class RecordedOutcomeBindingConstraint(BaseModel):class _RecordedBuildTestOutcomeContext(Protocol):latest_recorded_build_test_outcome: RecordedBuildTestOutcome | Nonerecorded_build_test_outcome_history: list[dict[str, object]]+ recorded_persisted_block_run_workflow_run_id: str | Nonerecorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | Nonerecorded_outcome_binding_constraint: RecordedOutcomeBindingConstraint | None@@ -253,6 +254,8 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re)del history[:-_HISTORY_LIMIT]ctx.recorded_build_test_outcome_history = history+ if outcome.phase == "persisted_block_run" and outcome.is_authoritative and outcome.workflow_run_id:+ ctx.recorded_persisted_block_run_workflow_run_id = outcome.workflow_run_idLOG.info("copilot recorded build-test outcome stored",phase=outcome.phase,@@ -398,14 +401,10 @@ def latest_recorded_build_test_outcome_repeated(ctx: object) -> bool | None:def run_backed_repair_evidence_exists(ctx: object) -> bool:- fallback_run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)- latest = getattr(ctx, "latest_recorded_build_test_outcome", None)- if isinstance(latest, RecordedBuildTestOutcome):- # An author-time reject is never run-backed even if a stale run id lingers on ctx; only a persisted run counts.- if latest.phase != "persisted_block_run":- return False- return bool(latest.workflow_run_id or fallback_run_id)- return bool(fallback_run_id)+ # Reached from the enforcement belt with an untyped ctx; a ctx without the latch must read as+ # "no run-backed evidence" so the guardrail fails safe instead of raising.+ run_id = getattr(ctx, "recorded_persisted_block_run_workflow_run_id", None)+ return isinstance(run_id, str) and bool(run_id)def arm_recorded_outcome_grounding_requirement(ctx: object) -> RecordedOutcomeGroundingRequirement | None:diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex b518762c3..91c8f1e27 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -758,6 +758,7 @@ class CopilotContext(AgentContext):last_code_authoring_repair_context: CodeAuthoringRepairContext | None = Nonelatest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = Nonerecorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)+ recorded_persisted_block_run_workflow_run_id: str | None = Nonerecorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None = Nonerecorded_outcome_binding_constraint: RecordedOutcomeBindingConstraint | None = None# Turn-scoped monotonic marks of verified forward progress: the union ofdiff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex e9be08224..ca39fbca5 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -297,6 +297,7 @@ class AgentContext:last_run_outcome_block_labels: list[str] = field(default_factory=list)latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = Nonerecorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)+ recorded_persisted_block_run_workflow_run_id: str | None = Nonerecorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None = Nonerecorded_outcome_binding_constraint: RecordedOutcomeBindingConstraint | None = Noneconsecutive_non_converging_repair_count: int = 0diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex fb852942e..60d240e08 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -3295,6 +3295,9 @@ def _update_repair_loop_state(copilot_ctx: CopilotContext, contract: DiagnosisRearm_recorded_outcome_grounding_requirement(copilot_ctx)if contract.repair_loop_state.ceiling_reached and run_backed_repair_evidence_exists(copilot_ctx):signal = repair_ceiling_stop_signal(copilot_ctx, contract)+ contract.repair_decision = contract.repair_decision.model_copy(+ update={"next_action": RepairNextAction.STOP, "target_blocks": []}+ )stash_blocker_signal(copilot_ctx, signal)stash_repair_ceiling_turn_halt(copilot_ctx, signal, consecutive_identical_repair_count=count)diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex f18d63f2b..a6f42c8c6 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -8359,6 +8359,16 @@ async def _update_workflow(block_labels=missing_labels,missing_requested_output_facts=missing_metadata_output_facts,)+ credential_scout_errors = (+ []+ if _request_policy_allows_untested_code_block_draft(ctx)+ else _credentialed_code_block_scout_gate_errors(+ workflow_yaml,+ ctx,+ block_labels=params.get("block_labels"),+ )+ )+ _record_code_authoring_guardrail_reject(ctx, defer_churn_stop=bool(credential_scout_errors))return reject(error=missing_metadata_error,user_facing_summary=_compiled_authoring_user_summary(),diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex 18aa86d7f..cb8760f4b 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (recorded_outcome_from_loaded_result_evidence,recorded_outcome_from_run_blocks_result,recorded_outcome_from_scout_act_observe_hollow,+ run_backed_repair_evidence_exists,)from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODEfrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict@@ -302,7 +303,11 @@ def test_prose_or_label_only_typed_outcome_is_not_authoritative() -> None:def test_record_none_clears_stale_latest_outcome() -> None:- ctx = SimpleNamespace(latest_recorded_build_test_outcome=None, recorded_build_test_outcome_history=[])+ ctx = SimpleNamespace(+ latest_recorded_build_test_outcome=None,+ recorded_build_test_outcome_history=[],+ recorded_persisted_block_run_workflow_run_id=None,+ )record_build_test_outcome(ctx,RecordedBuildTestOutcome(@@ -320,8 +325,76 @@ def test_record_none_clears_stale_latest_outcome() -> None:assert ctx.latest_recorded_build_test_outcome is None+def test_record_authoritative_persisted_run_latches_run_backed_evidence() -> None:+ ctx = SimpleNamespace(+ latest_recorded_build_test_outcome=None,+ recorded_build_test_outcome_history=[],+ recorded_persisted_block_run_workflow_run_id=None,+ )+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_recorded",+ structural_failure_identity="runtime:failed",+ ),+ )+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="author_time_reject",+ attempted_tool="update_workflow",+ verdict="authoring_rejected",+ reason_code="metadata_reject",+ structural_failure_identity="metadata:missing",+ ),+ )++ assert ctx.recorded_persisted_block_run_workflow_run_id == "wr_recorded"+ assert run_backed_repair_evidence_exists(ctx) is True+++def test_fallback_run_id_without_recorded_persisted_outcome_is_not_run_backed() -> None:+ ctx = SimpleNamespace(+ latest_recorded_build_test_outcome=None,+ recorded_build_test_outcome_history=[],+ recorded_persisted_block_run_workflow_run_id=None,+ last_run_blocks_workflow_run_id="wr_stale",+ )++ assert run_backed_repair_evidence_exists(ctx) is False+++def test_non_authoritative_persisted_run_does_not_latch_run_backed_evidence() -> None:+ ctx = SimpleNamespace(+ latest_recorded_build_test_outcome=None,+ recorded_build_test_outcome_history=[],+ recorded_persisted_block_run_workflow_run_id=None,+ )+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_hollow",+ ),+ )++ assert ctx.recorded_persisted_block_run_workflow_run_id is None+ assert run_backed_repair_evidence_exists(ctx) is False++def test_repeated_outcome_ignores_intervening_scout_evaluate_history() -> None:- ctx = SimpleNamespace(latest_recorded_build_test_outcome=None, recorded_build_test_outcome_history=[])+ ctx = SimpleNamespace(+ latest_recorded_build_test_outcome=None,+ recorded_build_test_outcome_history=[],+ recorded_persisted_block_run_workflow_run_id=None,+ )author_reject = RecordedBuildTestOutcome(phase="author_time_reject",verdict="authoring_rejected",@@ -343,7 +416,11 @@ def test_repeated_outcome_ignores_intervening_scout_evaluate_history() -> None:def test_repeated_outcome_still_detects_different_author_reject_after_scout_evaluate() -> None:- ctx = SimpleNamespace(latest_recorded_build_test_outcome=None, recorded_build_test_outcome_history=[])+ ctx = SimpleNamespace(+ latest_recorded_build_test_outcome=None,+ recorded_build_test_outcome_history=[],+ recorded_persisted_block_run_workflow_run_id=None,+ )first_reject = RecordedBuildTestOutcome(phase="author_time_reject",verdict="authoring_rejected",diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex d1f4d2e5b..803c04e62 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -3910,6 +3910,7 @@ class TestCodeRepairProgressClassification:structural_failure_identity="completion:typed-output",authored_structure_signature="previous-authoring-signature",)+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_recorded"recorded_sig = authored_block_signatures_from_workflow(workflow_yaml, metadata)["submit_service_request"]ctx.recorded_outcome_binding_constraint = RecordedOutcomeBindingConstraint(repeated_structural_key=ctx.latest_recorded_build_test_outcome.structural_key or "",@@ -7309,7 +7310,9 @@ class TestOutputContractBudgetRunGuard:verdict="repairable_failure",reason_code="failed_run",workflow_run_id="wr_1",+ structural_failure_identity="runtime:failed",)+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_1"payload = workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="x")@@ -7834,6 +7837,42 @@ class TestCompiledAuthoringImposition:assert ctx.latest_recorded_build_test_outcome is not Noneassert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"assert ctx.latest_recorded_build_test_outcome.is_authoritative is True+ assert ctx.code_authoring_guardrail_reject_count == 1+ assert ctx.last_code_authoring_reject_was_credential_priority is False++ @pytest.mark.asyncio+ async def test_missing_metadata_with_credential_gap_uses_credential_priority_counter(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ ctx.scout_trajectory = []+ submitted = _yaml(+ """+ title: Credentialed lookup+ workflow_definition:+ parameters:+ - {parameter_type: workflow, workflow_parameter_type: credential_id, key: login_credential, default_value: cred_missing}+ - {parameter_type: output, key: lookup_result}+ blocks:+ - block_type: code+ label: lookup_with_saved_credential+ parameter_keys: [login_credential]+ prompt: Sign in and return the lookup result.+ code: |+ await page.locator("#email").fill(login_credential.username)+ await page.locator("input[type='password']").fill(login_credential.password)+ return {"lookup_result": "sample"}+ """+ )++ result = await _update_workflow({"workflow_yaml": submitted}, ctx)++ assert result["ok"] is False+ assert "must pass `code_artifact_metadata`" in result["error"]+ assert ctx.code_authoring_guardrail_reject_count == 1+ assert ctx.last_code_authoring_reject_was_credential_priority is True+ assert ctx.blocker_signal is None@pytest.mark.asyncioasync def test_output_intent_rejects_partial_metadata_without_output_label(diff --git a/tests/unit/test_copilot_enforcement_hardening.py b/tests/unit/test_copilot_enforcement_hardening.pyindex 6ad4f062e..697ca7839 100644--- a/tests/unit/test_copilot_enforcement_hardening.py+++ b/tests/unit/test_copilot_enforcement_hardening.py@@ -23,6 +23,7 @@ import pytestfrom skyvern.forge.sdk.copilot.blocker_signal import (CopilotToolBlockerSignal,+ clear_terminal_evidence_on_workflow_edit,maybe_clear_blocker_signal_on_tool_success,)from skyvern.forge.sdk.copilot.context import CopilotContext@@ -454,6 +455,10 @@ def _ceiling_reached_contract() -> DiagnosisRepairContract:)+def _mark_recorded_run_backed(ctx: CopilotContext) -> None:+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_1"++def test_zero_run_ceiling_yields_to_code_authoring_churn_backstop() -> None:ctx = _fresh_context()ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS@@ -473,6 +478,7 @@ def test_run_backed_ceiling_precedes_code_authoring_churn_backstop() -> None:ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTSctx.latest_diagnosis_repair_contract = _ceiling_reached_contract()ctx.last_run_blocks_workflow_run_id = "wr_1"+ _mark_recorded_run_backed(ctx)with pytest.raises(CopilotTurnHalt) as excinfo:_check_enforcement(ctx)@@ -483,6 +489,32 @@ def test_run_backed_ceiling_precedes_code_authoring_churn_backstop() -> None:assert signal.internal_reason_code == "repair_ceiling_reached"+def test_stale_fallback_run_id_does_not_make_ceiling_run_backed() -> None:+ ctx = _fresh_context()+ ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS+ ctx.latest_diagnosis_repair_contract = _ceiling_reached_contract()+ ctx.last_run_blocks_workflow_run_id = "wr_stale"++ with pytest.raises(CopilotTurnHalt) as excinfo:+ _check_enforcement(ctx)++ assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED+ signal = ctx.blocker_signal+ assert isinstance(signal, CopilotToolBlockerSignal)+ assert signal.internal_reason_code == "code_authoring_guardrail_churn"+++def test_workflow_edit_clears_recorded_persisted_run_latch() -> None:+ ctx = _fresh_context()+ ctx.last_run_blocks_workflow_run_id = "wr_1"+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_1"++ clear_terminal_evidence_on_workflow_edit(ctx)++ assert ctx.last_run_blocks_workflow_run_id is None+ assert ctx.recorded_persisted_block_run_workflow_run_id is None++def test_credential_priority_churn_raises_at_higher_bound() -> None:ctx = _fresh_context()ctx.code_authoring_guardrail_reject_count = MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS@@ -528,6 +560,7 @@ def test_run_backed_ceiling_precedes_credential_priority_churn() -> None:ctx.last_code_authoring_reject_was_credential_priority = Truectx.latest_diagnosis_repair_contract = _ceiling_reached_contract()ctx.last_run_blocks_workflow_run_id = "wr_1"+ _mark_recorded_run_backed(ctx)with pytest.raises(CopilotTurnHalt) as excinfo:_check_enforcement(ctx)@@ -580,6 +613,7 @@ def test_run_backed_ceiling_precedes_no_progress_interaction_floor() -> None:ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTSctx.latest_diagnosis_repair_contract = _ceiling_reached_contract()ctx.last_run_blocks_workflow_run_id = "wr_1"+ _mark_recorded_run_backed(ctx)with pytest.raises(CopilotTurnHalt) as excinfo:_check_enforcement(ctx)diff --git a/tests/unit/test_copilot_recorded_outcome_grounding.py b/tests/unit/test_copilot_recorded_outcome_grounding.pyindex 583d9581d..13f7b81f0 100644--- a/tests/unit/test_copilot_recorded_outcome_grounding.py+++ b/tests/unit/test_copilot_recorded_outcome_grounding.py@@ -240,6 +240,56 @@ def test_authoritative_persisted_outcome_arms_without_recorded_signature_prefix(assert requirement.satisfied is False+def test_run_backed_author_time_reject_stashes_repair_ceiling_on_first_crossing() -> None:+ outcome = RecordedBuildTestOutcome(+ phase="author_time_reject",+ attempted_tool="update_workflow",+ verdict="authoring_rejected",+ reason_code="metadata_reject",+ block_labels=["search_records"],+ structural_failure_identity="metadata:missing-output",+ )+ ctx = _ctx(outcome)+ key = outcome.structural_key+ assert key is not None+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_prior"+ ctx.last_repair_non_convergence_signature = f"recorded_build_test_outcome:{key}"+ ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1+ contract = _contract()++ _update_repair_loop_state(ctx, contract)++ assert contract.repair_decision.next_action is RepairNextAction.STOP+ assert contract.repair_loop_state.ceiling_reached is True+ assert ctx.turn_halt is not None+ assert ctx.turn_halt.kind is TurnHaltKind.REPAIR_CEILING_REACHED+++def test_zero_run_author_time_reject_does_not_stash_repair_ceiling_on_first_crossing() -> None:+ outcome = RecordedBuildTestOutcome(+ phase="author_time_reject",+ attempted_tool="update_workflow",+ verdict="authoring_rejected",+ reason_code="metadata_reject",+ block_labels=["search_records"],+ structural_failure_identity="metadata:missing-output",+ )+ ctx = _ctx(outcome)+ key = outcome.structural_key+ assert key is not None+ ctx.last_run_blocks_workflow_run_id = "wr_stale"+ ctx.last_repair_non_convergence_signature = f"recorded_build_test_outcome:{key}"+ ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1+ contract = _contract()++ _update_repair_loop_state(ctx, contract)++ assert contract.repair_loop_state.ceiling_reached is True+ assert contract.repair_decision.next_action is RepairNextAction.REPAIR+ assert ctx.turn_halt is None+ assert ctx.blocker_signal is None++def test_progress_observed_outcome_does_not_arm_from_executed_run(monkeypatch: pytest.MonkeyPatch,) -> None: