Skip to content

Automate browser based workflows with AI

fix(SKY-12307): make the copilot repair-ceiling streak progress-gated (#7314)

Andrew Neilson9d8d3a0906d2
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index facaf9f45..4239dca21 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -761,7 +761,7 @@ class CopilotContext(AgentContext):
repeated_failure_streak_count: int = 0
last_repair_non_convergence_signature: str | None = None
consecutive_non_converging_repair_count: int = 0
- # Unlike the identity-keyed repair ceiling, this climbs even when every
+ # Unlike the progress-gated repair ceiling, this climbs even when every
# rejection is different; it resets only on an accepted persist.
code_authoring_guardrail_reject_count: int = 0
# True when the most-recent such rejection deferred to the credential-scout
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index b1c322bed..3a05a913b 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -3425,9 +3425,8 @@ def _update_repair_loop_state(copilot_ctx: CopilotContext, contract: DiagnosisRe
ceiling_reached=False,
)
return
- prior_signature = copilot_ctx.last_repair_non_convergence_signature
prior_count = copilot_ctx.consecutive_non_converging_repair_count
- count = prior_count + 1 if signature == prior_signature else 1
+ count = prior_count + 1
requirement = copilot_ctx.recorded_outcome_grounding_requirement
if isinstance(requirement, RecordedOutcomeGroundingRequirement):
latest = copilot_ctx.latest_recorded_build_test_outcome
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 9d912e568..74369e281 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -399,7 +399,9 @@ def test_runtime_authoring_repair_context_identity_includes_bounded_page_state()
)
-def test_repair_loop_state_resets_when_authoring_repair_context_identity_changes() -> None:
+def test_repair_loop_state_counts_through_authoring_repair_context_identity_change() -> None:
+ # An identity change rotates the streak token but is not progress; the zero-run
+ # ceiling still does not terminalize.
ctx = _ctx()
ambiguous = CodeAuthoringRepairContext(
block_label="retrieve_document_link",
@@ -431,9 +433,11 @@ def test_repair_loop_state_resets_when_authoring_repair_context_identity_changes
run_execution_module._update_repair_loop_state(ctx, sandbox_contract)
assert sandbox_contract.repair_decision.next_action == RepairNextAction.REPAIR
- assert sandbox_contract.repair_loop_state.consecutive_identical_repair_count == 1
- assert sandbox_contract.repair_loop_state.ceiling_reached is False
+ assert sandbox_contract.repair_loop_state.streak_token != contract.repair_loop_state.streak_token
+ assert sandbox_contract.repair_loop_state.consecutive_identical_repair_count == 3
+ assert sandbox_contract.repair_loop_state.ceiling_reached is True
assert getattr(ctx, "blocker_signal", None) is None
+ assert ctx.turn_halt is None
def _uncovered_output_turn_state(output_path: str) -> SimpleNamespace:
@@ -502,6 +506,25 @@ def test_persisted_run_outcome_is_not_excluded_from_repair_streak() -> None:
assert state.consecutive_identical_repair_count == 1
assert ctx.synthesized_block_reopened_for_output_coverage is False
+ repeat = _run_repair_loop_state(ctx)
+ assert repeat.consecutive_identical_repair_count == 2
+ assert ctx.synthesized_block_reopened_for_output_coverage is False
+
+
+def test_uncovered_output_reopen_preserves_prior_streak_count() -> None:
+ ctx = _ctx()
+ assert _run_repair_loop_state(ctx).consecutive_identical_repair_count == 1
+ assert _run_repair_loop_state(ctx).consecutive_identical_repair_count == 2
+
+ ctx.completion_criteria_turn_state = _uncovered_output_turn_state("output.document_name")
+ ctx.latest_recorded_build_test_outcome = _uncovered_output_author_reject("output.document_name")
+ reopened = _run_repair_loop_state(ctx)
+
+ assert ctx.synthesized_block_reopened_for_output_coverage is True
+ assert reopened.consecutive_identical_repair_count == 2
+ assert reopened.ceiling_reached is False
+ assert ctx.consecutive_non_converging_repair_count == 2
+
def test_failed_run_finalizes_runtime_authoring_repair_context_after_matching_page_observation() -> None:
ctx = _ctx()