Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11028: halt the non-converging repair loop and escalate honestly (#6608)
Andrew Neilson34145d36a701
diff --git a/skyvern/config.py b/skyvern/config.pyindex df91a14a4..44054c6b4 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -154,6 +154,9 @@ class Settings(BaseSettings):# Capture bounded page evidence synchronously in scout interaction post-hooks.# Off restores schema-less interaction packets plus standalone page inspects.COPILOT_SCOUT_ACT_OBSERVE_ENABLED: bool = True+ # Consecutive repair runs that make no newly-verified forward progress before the+ # copilot stops re-running and escalates honestly. Set very high to disable the ceiling.+ COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL: int = 3COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS: float = 4.0# On a repeat evaluate of an unchanged page with exactly one safe target, click it in-process# instead of only advising. Off (default) leaves the advisory steer as the only response.diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex 13b4d6dd7..1f9978c3a 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -359,6 +359,10 @@ _LOOP_BRANCH_COPY: dict[str, tuple[str, str]] = {"I couldn't keep going on this turn.","Tell me what to change and I'll try again.",),+ "repair_ceiling_reached": (+ "I couldn't get past the same problem after several attempts.",+ "Tell me what to change and I'll try a different approach.",+ ),}_LOOP_ANTI_BOT_BLOCKER_COPY = "The site's verification challenge was still keeping the submit/search control disabled."_LOOP_VERDICT_MAX_CHARS = 240diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 086adc7a8..04c8dff28 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -471,6 +471,18 @@ class CopilotContext(AgentContext):last_frontier_fingerprint: str | None = Nonelast_failure_signature: str | None = Nonerepeated_failure_streak_count: int = 0+ last_repair_non_convergence_signature: str | None = None+ consecutive_non_converging_repair_count: int = 0+ # Turn-scoped monotonic marks of verified forward progress: the union of+ # completion criteria the judge confirmed satisfied so far this turn, and the+ # high-water length of the verified block prefix. A repair that grows either+ # made progress and resets the non-convergence streak.+ verified_criteria_high_water: frozenset[str] = frozenset()+ verified_prefix_high_water_len: int = 0+ # A fresh clean full-workflow pass counts as progress exactly once: latched+ # here so a stale carry-over ``last_full_workflow_test_ok`` on a later non-run+ # REPAIR verdict cannot keep resetting the non-convergence streak.+ verified_full_pass_consumed: bool = False# Highest streak level at which we've already emitted a repeated-failure# nudge. Prevents the warn nudge from re-firing every turn while the# streak is still at 2, and guarantees the stop nudge fires exactly oncediff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex 8b2e040b6..6ccbc7f9a 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -85,11 +85,18 @@ class VerificationResult(StrictModel):remaining_blocker: str | None = None+class RepairLoopState(StrictModel):+ streak_token: str | None = None+ consecutive_identical_repair_count: int = 0+ ceiling_reached: bool = False++class DiagnosisRepairContract(StrictModel):diagnosis_input: DiagnosisInputdiagnosis_result: DiagnosisResultrepair_decision: RepairDecisionverification_result: VerificationResult+ repair_loop_state: RepairLoopState = Field(default_factory=RepairLoopState)def to_trace_data(self) -> dict[str, Any]:return {@@ -105,6 +112,9 @@ class DiagnosisRepairContract(StrictModel):"missing_context": list(self.diagnosis_result.missing_context),"user_goal_satisfied": self.verification_result.user_goal_satisfied,"completion_contract_satisfied": self.verification_result.completion_contract_satisfied,+ "consecutive_identical_repair_count": self.repair_loop_state.consecutive_identical_repair_count,+ "ceiling_reached": self.repair_loop_state.ceiling_reached,+ "streak_token": self.repair_loop_state.streak_token,}diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex ca57552ae..4a3bbcd9c 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -14,7 +14,12 @@ import structlogfrom agents.run import Runnerfrom skyvern.forge.sdk.copilot import config as copilot_config_defaults-from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal+from skyvern.forge.sdk.copilot.blocker_signal import (+ CopilotToolBlockerSignal,+ compose_loop_blocker_user_facing_reason,+ loop_blocker_evidence_from_ctx,+ stash_blocker_signal,+)from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASESfrom skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_blockfrom skyvern.forge.sdk.copilot.config import (@@ -44,7 +49,11 @@ from skyvern.forge.sdk.copilot.config import (CopilotConfig,normalize_block_authoring_policy,)-from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (+ DiagnosisRepairContract,+ RepairLoopState,+ RepairNextAction,+)from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reasonfrom skyvern.forge.sdk.copilot.narration import TransitionKindfrom skyvern.forge.sdk.copilot.output_policy import normalize_response_scaffolding@@ -55,7 +64,11 @@ from skyvern.forge.sdk.copilot.output_utils import ()from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntryfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span-from skyvern.forge.sdk.copilot.turn_halt import raise_if_turn_halt, stash_turn_halt_from_blocker_signal+from skyvern.forge.sdk.copilot.turn_halt import (+ raise_if_turn_halt,+ stash_repair_ceiling_turn_halt,+ stash_turn_halt_from_blocker_signal,+)from skyvern.utils.token_counter import count_tokensif TYPE_CHECKING:@@ -245,6 +258,45 @@ def _probable_site_block_stop_signal(ctx: Any, config: CopilotConfig | None = No)+def _repair_loop_state(ctx: Any) -> RepairLoopState | None:+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)+ state = getattr(contract, "repair_loop_state", None)+ return state if isinstance(state, RepairLoopState) else None+++def _needs_repair_ceiling_halt(ctx: Any) -> bool:+ state = _repair_loop_state(ctx)+ return state is not None and state.ceiling_reached is True+++def repair_ceiling_stop_signal(+ ctx: Any,+ contract: DiagnosisRepairContract | None,+ config: CopilotConfig | None = None,+) -> CopilotToolBlockerSignal:+ state = contract.repair_loop_state if isinstance(contract, DiagnosisRepairContract) else None+ count = state.consecutive_identical_repair_count if state is not None else 0+ evidence = loop_blocker_evidence_from_ctx(ctx)+ user_facing, _tiers = compose_loop_blocker_user_facing_reason(+ "repair_ceiling_reached", evidence, blocked_tool="update_and_run_blocks"+ )+ agent_steering = (+ f"This repair has made no verified progress across {count} attempts; "+ "stop retrying and report the recorded blocker from the preserved draft."+ )+ return CopilotToolBlockerSignal(+ blocker_kind="loop_detected",+ agent_steering_text=agent_steering,+ user_facing_reason=user_facing,+ recovery_hint="report_blocker_to_user",+ cleared_by_tools=frozenset(),+ preserves_workflow_draft=evidence.has_draft,+ renders_final_reply=True,+ internal_reason_code="repair_ceiling_reached",+ blocked_tool="update_and_run_blocks",+ )++class CopilotTotalTimeoutError(Exception):"""Raised when the copilot agent exceeds the total allowed runtime."""@@ -969,6 +1021,19 @@ def _check_enforcement(raise_if_turn_halt(ctx)_raise_if_unrecoverable_contract_stop(ctx)+ if _needs_repair_ceiling_halt(ctx):+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)+ state = _repair_loop_state(ctx)+ # Backstop: the detection-time latch normally raises this above; this catches any increment path that bypassed the latch.+ signal = repair_ceiling_stop_signal(ctx, contract, config)+ stash_blocker_signal(ctx, signal)+ stash_repair_ceiling_turn_halt(+ ctx,+ signal,+ consecutive_identical_repair_count=(state.consecutive_identical_repair_count if state is not None else 0),+ )+ raise_if_turn_halt(ctx)+# A permanent navigation error (DNS / cert / SSL / invalid URL) cannot be# resolved by observing a prior navigate or by testing an updated# workflow against the same bad URL, so let it speak first.diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.pyindex d136ba165..e44dcb364 100644--- a/skyvern/forge/sdk/copilot/failure_tracking.py+++ b/skyvern/forge/sdk/copilot/failure_tracking.py@@ -25,6 +25,7 @@ import jsonfrom typing import TYPE_CHECKING, Anyif TYPE_CHECKING:+ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResultfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition@@ -60,6 +61,26 @@ def _top_failure_category(failure_categories: list[dict] | None) -> str:return ""+def _failure_identity_terms(+ failure_reason: str | None,+ failure_categories: list[dict] | None,+ suspicious_success: bool,+) -> list[str] | None:+ """Category-stable failure half shared by both signatures, or ``None`` on a+ real success. Per-call-data categories collapse to a constant so consecutive+ trips hash identically."""+ normalized = normalize_failure_reason(failure_reason)+ has_signal = bool(normalized) or suspicious_success+ if not has_signal:+ return None+ top_category = _top_failure_category(failure_categories)+ if top_category == "PARAMETER_BINDING_ERROR":+ normalized = "parameter_binding_error"+ elif top_category == PER_TOOL_BUDGET_FAILURE_CATEGORY:+ normalized = "per_tool_budget"+ return [normalized, top_category, "suspicious" if suspicious_success else "failed"]++def compute_failure_signature(frontier_start_label: str | None,failure_reason: str | None,@@ -72,26 +93,30 @@ def compute_failure_signature(run (status=completed but data-producing blocks produced no output) stillgenerates a signature so repeated no-data runs can be counted as repeats."""- normalized = normalize_failure_reason(failure_reason)- has_signal = bool(normalized) or suspicious_success- if not has_signal:+ terms = _failure_identity_terms(failure_reason, failure_categories, suspicious_success)+ if terms is None:return Nonesafe_label = frontier_start_label if isinstance(frontier_start_label, str) else ""- top_category = _top_failure_category(failure_categories)- # Categories whose failure_reason embeds per-call data (key name, run id):- # collapse to a stable constant so consecutive trips hash to the same- # signature instead of each one looking unique.- if top_category == "PARAMETER_BINDING_ERROR":- normalized = "parameter_binding_error"- elif top_category == PER_TOOL_BUDGET_FAILURE_CATEGORY:- normalized = "per_tool_budget"- parts = [- safe_label,- normalized,- top_category,- "suspicious" if suspicious_success else "failed",- ]- return "|".join(parts)+ return "|".join([safe_label, *terms])+++def satisfied_criterion_ids(result: CompletionVerificationResult | None) -> frozenset[str]:+ """Criterion ids the outcome-verification judge confirmed satisfied (evidence_confirms),+ or empty when no judge result is available for this run."""+ if result is None or result.status != "evaluated":+ return frozenset()+ return frozenset(verdict.criterion_id for verdict in result.verdicts if verdict.reason_code == "evidence_confirms")+++def made_newly_verified_progress(+ current_satisfied: frozenset[str],+ high_water: frozenset[str],+ full_workflow_verified_this_run: bool,+ verified_prefix_grew: bool,+) -> bool:+ """Whether this run advanced verified progress past the turn high-water: a newly+ confirmed criterion, a clean end-to-end run, or a grown verified block prefix."""+ return bool(current_satisfied - high_water) or full_workflow_verified_this_run or verified_prefix_grewdef _canonical_block_config(block: Any) -> dict[str, Any]:diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 36957cff5..783c1944b 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -15,6 +15,7 @@ import structlogimport yamlfrom pydantic import ValidationError+from skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot.blocker_signal import (@@ -28,12 +29,17 @@ from skyvern.forge.sdk.copilot.completion_verification import (from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisRepairContract,+ RepairLoopState,+ RepairNextAction,build_diagnosis_repair_contract,)+from skyvern.forge.sdk.copilot.enforcement import repair_ceiling_stop_signalfrom skyvern.forge.sdk.copilot.failure_tracking import (ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,PER_TOOL_BUDGET_FAILURE_CATEGORY,compute_action_sequence_fingerprint,+ made_newly_verified_progress,+ satisfied_criterion_ids,update_repeated_failure_state,)from skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctx@@ -62,7 +68,10 @@ from skyvern.forge.sdk.copilot.runtime import (ensure_browser_session,)from skyvern.forge.sdk.copilot.tracing_setup import copilot_span-from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal+from skyvern.forge.sdk.copilot.turn_halt import (+ stash_repair_ceiling_turn_halt,+ stash_turn_halt_from_blocker_signal,+)from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yamlfrom skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotRunOutcomeUpdate, WorkflowCopilotStreamMessageTypefrom skyvern.forge.sdk.settings_manager import SettingsManager@@ -1772,6 +1781,73 @@ async def _verify_and_record_run_blocks_result(return completion_verification+def _repair_non_convergence_signature(contract: DiagnosisRepairContract) -> str | None:+ """A constant streak token for any REPAIR verdict, ``None`` otherwise. The trigger+ is non-convergence (REPAIR with no verified progress), so the only thing the signature+ must do is tell a contiguous repair streak from a reset on a non-REPAIR verdict."""+ if contract.repair_decision.next_action is not RepairNextAction.REPAIR:+ return None+ # Constant by design: the streak only distinguishes a REPAIR run from a reset, so the failure specifics are ignored.+ return "repair_no_verified_progress"+++def _update_repair_loop_state(copilot_ctx: Any, contract: DiagnosisRepairContract) -> None:+ """Count consecutive REPAIR verdicts that made no newly-verified forward progress.++ Progress is growth in the turn-scoped set of judge-confirmed completion criteria, or a+ clean end-to-end run, or a grown verified block prefix — never the failure prose, the+ failure_type, or which block label failed. The high-water marks must be read BEFORE the+ current run's confirmations are folded in, else this run's own wins would already be+ banked and never read as new.+ """+ current = satisfied_criterion_ids(getattr(copilot_ctx, "completion_verification_result", None))+ high_water_raw: Any = getattr(copilot_ctx, "verified_criteria_high_water", frozenset())+ high_water = high_water_raw if isinstance(high_water_raw, frozenset) else frozenset(high_water_raw or [])+ prefix_len = len(getattr(copilot_ctx, "verified_prefix_labels", []) or [])+ prefix_high_raw = getattr(copilot_ctx, "verified_prefix_high_water_len", 0)+ prefix_high = prefix_high_raw if isinstance(prefix_high_raw, int) else 0+ # A run-tied REPAIR verdict always sees this False (the failing run cleared it in+ # _record_run_blocks_result); a True here is a stale carry-over from a prior clean+ # pass on a non-run path, so latch it consumed and count it as progress only once.+ full_pass = getattr(copilot_ctx, "last_full_workflow_test_ok", False) is True+ consumed = getattr(copilot_ctx, "verified_full_pass_consumed", False) is True+ progressed = made_newly_verified_progress(+ current_satisfied=current,+ high_water=high_water,+ full_workflow_verified_this_run=full_pass and not consumed,+ verified_prefix_grew=prefix_len > prefix_high,+ )+ copilot_ctx.verified_criteria_high_water = high_water | current+ copilot_ctx.verified_prefix_high_water_len = max(prefix_high, prefix_len)+ copilot_ctx.verified_full_pass_consumed = full_pass++ signature = _repair_non_convergence_signature(contract)+ if signature is None or progressed:+ copilot_ctx.consecutive_non_converging_repair_count = 0+ copilot_ctx.last_repair_non_convergence_signature = None+ contract.repair_loop_state = RepairLoopState(+ streak_token=None,+ consecutive_identical_repair_count=0,+ ceiling_reached=False,+ )+ return+ prior_signature = getattr(copilot_ctx, "last_repair_non_convergence_signature", None)+ prior_count = getattr(copilot_ctx, "consecutive_non_converging_repair_count", 0)+ prior_count = prior_count if isinstance(prior_count, int) else 0+ count = prior_count + 1 if signature == prior_signature else 1+ copilot_ctx.consecutive_non_converging_repair_count = count+ copilot_ctx.last_repair_non_convergence_signature = signature+ contract.repair_loop_state = RepairLoopState(+ streak_token=signature,+ consecutive_identical_repair_count=count,+ ceiling_reached=count >= settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL,+ )+ if contract.repair_loop_state.ceiling_reached:+ signal = repair_ceiling_stop_signal(copilot_ctx, contract)+ stash_blocker_signal(copilot_ctx, signal)+ stash_repair_ceiling_turn_halt(copilot_ctx, signal, consecutive_identical_repair_count=count)++def _record_diagnosis_repair_contract(copilot_ctx: Any,*,@@ -1785,6 +1861,7 @@ def _record_diagnosis_repair_contract(ctx=copilot_ctx,workflow_updated=workflow_updated,)+ _update_repair_loop_state(copilot_ctx, contract)copilot_ctx.latest_diagnosis_repair_contract = contracttrace_data = contract.to_trace_data()LOG.info(diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.pyindex c62f22900..7e6b4b492 100644--- a/skyvern/forge/sdk/copilot/turn_halt.py+++ b/skyvern/forge/sdk/copilot/turn_halt.py@@ -19,6 +19,7 @@ class TurnHaltKind(StrEnum):LOOP_DETECTED = "loop_detected"ACTIVE_TERMINAL_CHALLENGE = "active_terminal_challenge"PROBABLE_SITE_BLOCK = "probable_site_block"+ REPAIR_CEILING_REACHED = "repair_ceiling_reached"class TurnHaltVerdict(StrEnum):@@ -99,6 +100,29 @@ def stash_turn_halt_from_blocker_signal(ctx: Any, signal: object, *, source: strreturn halt+def stash_repair_ceiling_turn_halt(+ ctx: Any,+ signal: CopilotToolBlockerSignal,+ *,+ consecutive_identical_repair_count: int,+) -> TurnHalt | None:+ existing = getattr(ctx, "turn_halt", None)+ if isinstance(existing, TurnHalt):+ return existing+ halt = TurnHalt(+ kind=TurnHaltKind.REPAIR_CEILING_REACHED,+ blocker_signal=signal,+ draft_state={"preserves_workflow_draft": signal.preserves_workflow_draft},+ extra={+ "source": "enforcement",+ "consecutive_identical_repair_count": consecutive_identical_repair_count,+ },+ )+ ctx.turn_halt = halt+ LOG.info("copilot turn halt stashed", **turn_halt_to_trace_data(halt))+ return halt++def raise_if_turn_halt(ctx: Any) -> None:halt = getattr(ctx, "turn_halt", None)if isinstance(halt, TurnHalt):