Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12307): make the copilot repair-ceiling streak progress-gated (#7314)
Andrew Neilson9d8d3a0906d2
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex facaf9f45..4239dca21 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -761,7 +761,7 @@ class CopilotContext(AgentContext):repeated_failure_streak_count: int = 0last_repair_non_convergence_signature: str | None = Noneconsecutive_non_converging_repair_count: int = 0- # Unlike the identity-keyed repair ceiling, this climbs even when every+ # Unlike the progress-gated repair ceiling, this climbs even when every# rejection is different; it resets only on an accepted persist.code_authoring_guardrail_reject_count: int = 0# True when the most-recent such rejection deferred to the credential-scoutdiff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex b1c322bed..3a05a913b 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -3425,9 +3425,8 @@ def _update_repair_loop_state(copilot_ctx: CopilotContext, contract: DiagnosisReceiling_reached=False,)return- prior_signature = copilot_ctx.last_repair_non_convergence_signatureprior_count = copilot_ctx.consecutive_non_converging_repair_count- count = prior_count + 1 if signature == prior_signature else 1+ count = prior_count + 1requirement = copilot_ctx.recorded_outcome_grounding_requirementif isinstance(requirement, RecordedOutcomeGroundingRequirement):latest = copilot_ctx.latest_recorded_build_test_outcomediff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 9d912e568..74369e281 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -399,7 +399,9 @@ def test_runtime_authoring_repair_context_identity_includes_bounded_page_state())-def test_repair_loop_state_resets_when_authoring_repair_context_identity_changes() -> None:+def test_repair_loop_state_counts_through_authoring_repair_context_identity_change() -> None:+ # An identity change rotates the streak token but is not progress; the zero-run+ # ceiling still does not terminalize.ctx = _ctx()ambiguous = CodeAuthoringRepairContext(block_label="retrieve_document_link",@@ -431,9 +433,11 @@ def test_repair_loop_state_resets_when_authoring_repair_context_identity_changesrun_execution_module._update_repair_loop_state(ctx, sandbox_contract)assert sandbox_contract.repair_decision.next_action == RepairNextAction.REPAIR- assert sandbox_contract.repair_loop_state.consecutive_identical_repair_count == 1- assert sandbox_contract.repair_loop_state.ceiling_reached is False+ assert sandbox_contract.repair_loop_state.streak_token != contract.repair_loop_state.streak_token+ assert sandbox_contract.repair_loop_state.consecutive_identical_repair_count == 3+ assert sandbox_contract.repair_loop_state.ceiling_reached is Trueassert getattr(ctx, "blocker_signal", None) is None+ assert ctx.turn_halt is Nonedef _uncovered_output_turn_state(output_path: str) -> SimpleNamespace:@@ -502,6 +506,25 @@ def test_persisted_run_outcome_is_not_excluded_from_repair_streak() -> None:assert state.consecutive_identical_repair_count == 1assert ctx.synthesized_block_reopened_for_output_coverage is False+ repeat = _run_repair_loop_state(ctx)+ assert repeat.consecutive_identical_repair_count == 2+ assert ctx.synthesized_block_reopened_for_output_coverage is False+++def test_uncovered_output_reopen_preserves_prior_streak_count() -> None:+ ctx = _ctx()+ assert _run_repair_loop_state(ctx).consecutive_identical_repair_count == 1+ assert _run_repair_loop_state(ctx).consecutive_identical_repair_count == 2++ ctx.completion_criteria_turn_state = _uncovered_output_turn_state("output.document_name")+ ctx.latest_recorded_build_test_outcome = _uncovered_output_author_reject("output.document_name")+ reopened = _run_repair_loop_state(ctx)++ assert ctx.synthesized_block_reopened_for_output_coverage is True+ assert reopened.consecutive_identical_repair_count == 2+ assert reopened.ceiling_reached is False+ assert ctx.consecutive_non_converging_repair_count == 2+def test_failed_run_finalizes_runtime_authoring_repair_context_after_matching_page_observation() -> None:ctx = _ctx()