Skip to content

Automate browser based workflows with AI

SKY-11028: halt the non-converging repair loop and escalate honestly (#6608)

Andrew Neilson34145d36a701
diff --git a/skyvern/config.py b/skyvern/config.py
index df91a14a4..44054c6b4 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -154,6 +154,9 @@ class Settings(BaseSettings):
# Capture bounded page evidence synchronously in scout interaction post-hooks.
# Off restores schema-less interaction packets plus standalone page inspects.
COPILOT_SCOUT_ACT_OBSERVE_ENABLED: bool = True
+ # Consecutive repair runs that make no newly-verified forward progress before the
+ # copilot stops re-running and escalates honestly. Set very high to disable the ceiling.
+ COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL: int = 3
COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS: float = 4.0
# On a repeat evaluate of an unchanged page with exactly one safe target, click it in-process
# instead of only advising. Off (default) leaves the advisory steer as the only response.
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index 13b4d6dd7..1f9978c3a 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -359,6 +359,10 @@ _LOOP_BRANCH_COPY: dict[str, tuple[str, str]] = {
"I couldn't keep going on this turn.",
"Tell me what to change and I'll try again.",
),
+ "repair_ceiling_reached": (
+ "I couldn't get past the same problem after several attempts.",
+ "Tell me what to change and I'll try a different approach.",
+ ),
}
_LOOP_ANTI_BOT_BLOCKER_COPY = "The site's verification challenge was still keeping the submit/search control disabled."
_LOOP_VERDICT_MAX_CHARS = 240
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 086adc7a8..04c8dff28 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -471,6 +471,18 @@ class CopilotContext(AgentContext):
last_frontier_fingerprint: str | None = None
last_failure_signature: str | None = None
repeated_failure_streak_count: int = 0
+ last_repair_non_convergence_signature: str | None = None
+ consecutive_non_converging_repair_count: int = 0
+ # Turn-scoped monotonic marks of verified forward progress: the union of
+ # completion criteria the judge confirmed satisfied so far this turn, and the
+ # high-water length of the verified block prefix. A repair that grows either
+ # made progress and resets the non-convergence streak.
+ verified_criteria_high_water: frozenset[str] = frozenset()
+ verified_prefix_high_water_len: int = 0
+ # A fresh clean full-workflow pass counts as progress exactly once: latched
+ # here so a stale carry-over ``last_full_workflow_test_ok`` on a later non-run
+ # REPAIR verdict cannot keep resetting the non-convergence streak.
+ verified_full_pass_consumed: bool = False
# Highest streak level at which we've already emitted a repeated-failure
# nudge. Prevents the warn nudge from re-firing every turn while the
# streak is still at 2, and guarantees the stop nudge fires exactly once
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index 8b2e040b6..6ccbc7f9a 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -85,11 +85,18 @@ class VerificationResult(StrictModel):
remaining_blocker: str | None = None
+class RepairLoopState(StrictModel):
+ streak_token: str | None = None
+ consecutive_identical_repair_count: int = 0
+ ceiling_reached: bool = False
+
+
class DiagnosisRepairContract(StrictModel):
diagnosis_input: DiagnosisInput
diagnosis_result: DiagnosisResult
repair_decision: RepairDecision
verification_result: VerificationResult
+ repair_loop_state: RepairLoopState = Field(default_factory=RepairLoopState)
def to_trace_data(self) -> dict[str, Any]:
return {
@@ -105,6 +112,9 @@ class DiagnosisRepairContract(StrictModel):
"missing_context": list(self.diagnosis_result.missing_context),
"user_goal_satisfied": self.verification_result.user_goal_satisfied,
"completion_contract_satisfied": self.verification_result.completion_contract_satisfied,
+ "consecutive_identical_repair_count": self.repair_loop_state.consecutive_identical_repair_count,
+ "ceiling_reached": self.repair_loop_state.ceiling_reached,
+ "streak_token": self.repair_loop_state.streak_token,
}
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index ca57552ae..4a3bbcd9c 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -14,7 +14,12 @@ import structlog
from agents.run import Runner
from skyvern.forge.sdk.copilot import config as copilot_config_defaults
-from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ CopilotToolBlockerSignal,
+ compose_loop_blocker_user_facing_reason,
+ loop_blocker_evidence_from_ctx,
+ stash_blocker_signal,
+)
from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASES
from skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_block
from skyvern.forge.sdk.copilot.config import (
@@ -44,7 +49,11 @@ from skyvern.forge.sdk.copilot.config import (
CopilotConfig,
normalize_block_authoring_policy,
)
-from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ DiagnosisRepairContract,
+ RepairLoopState,
+ RepairNextAction,
+)
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason
from skyvern.forge.sdk.copilot.narration import TransitionKind
from skyvern.forge.sdk.copilot.output_policy import normalize_response_scaffolding
@@ -55,7 +64,11 @@ from skyvern.forge.sdk.copilot.output_utils import (
)
from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
-from skyvern.forge.sdk.copilot.turn_halt import raise_if_turn_halt, stash_turn_halt_from_blocker_signal
+from skyvern.forge.sdk.copilot.turn_halt import (
+ raise_if_turn_halt,
+ stash_repair_ceiling_turn_halt,
+ stash_turn_halt_from_blocker_signal,
+)
from skyvern.utils.token_counter import count_tokens
if TYPE_CHECKING:
@@ -245,6 +258,45 @@ def _probable_site_block_stop_signal(ctx: Any, config: CopilotConfig | None = No
)
+def _repair_loop_state(ctx: Any) -> RepairLoopState | None:
+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)
+ state = getattr(contract, "repair_loop_state", None)
+ return state if isinstance(state, RepairLoopState) else None
+
+
+def _needs_repair_ceiling_halt(ctx: Any) -> bool:
+ state = _repair_loop_state(ctx)
+ return state is not None and state.ceiling_reached is True
+
+
+def repair_ceiling_stop_signal(
+ ctx: Any,
+ contract: DiagnosisRepairContract | None,
+ config: CopilotConfig | None = None,
+) -> CopilotToolBlockerSignal:
+ state = contract.repair_loop_state if isinstance(contract, DiagnosisRepairContract) else None
+ count = state.consecutive_identical_repair_count if state is not None else 0
+ evidence = loop_blocker_evidence_from_ctx(ctx)
+ user_facing, _tiers = compose_loop_blocker_user_facing_reason(
+ "repair_ceiling_reached", evidence, blocked_tool="update_and_run_blocks"
+ )
+ agent_steering = (
+ f"This repair has made no verified progress across {count} attempts; "
+ "stop retrying and report the recorded blocker from the preserved draft."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="loop_detected",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=evidence.has_draft,
+ renders_final_reply=True,
+ internal_reason_code="repair_ceiling_reached",
+ blocked_tool="update_and_run_blocks",
+ )
+
+
class CopilotTotalTimeoutError(Exception):
"""Raised when the copilot agent exceeds the total allowed runtime."""
@@ -969,6 +1021,19 @@ def _check_enforcement(
raise_if_turn_halt(ctx)
_raise_if_unrecoverable_contract_stop(ctx)
+ if _needs_repair_ceiling_halt(ctx):
+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)
+ state = _repair_loop_state(ctx)
+ # Backstop: the detection-time latch normally raises this above; this catches any increment path that bypassed the latch.
+ signal = repair_ceiling_stop_signal(ctx, contract, config)
+ stash_blocker_signal(ctx, signal)
+ stash_repair_ceiling_turn_halt(
+ ctx,
+ signal,
+ consecutive_identical_repair_count=(state.consecutive_identical_repair_count if state is not None else 0),
+ )
+ raise_if_turn_halt(ctx)
+
# A permanent navigation error (DNS / cert / SSL / invalid URL) cannot be
# resolved by observing a prior navigate or by testing an updated
# workflow against the same bad URL, so let it speak first.
diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.py
index d136ba165..e44dcb364 100644
--- a/skyvern/forge/sdk/copilot/failure_tracking.py
+++ b/skyvern/forge/sdk/copilot/failure_tracking.py
@@ -25,6 +25,7 @@ import json
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
+ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition
@@ -60,6 +61,26 @@ def _top_failure_category(failure_categories: list[dict] | None) -> str:
return ""
+def _failure_identity_terms(
+ failure_reason: str | None,
+ failure_categories: list[dict] | None,
+ suspicious_success: bool,
+) -> list[str] | None:
+ """Category-stable failure half shared by both signatures, or ``None`` on a
+ real success. Per-call-data categories collapse to a constant so consecutive
+ trips hash identically."""
+ normalized = normalize_failure_reason(failure_reason)
+ has_signal = bool(normalized) or suspicious_success
+ if not has_signal:
+ return None
+ top_category = _top_failure_category(failure_categories)
+ if top_category == "PARAMETER_BINDING_ERROR":
+ normalized = "parameter_binding_error"
+ elif top_category == PER_TOOL_BUDGET_FAILURE_CATEGORY:
+ normalized = "per_tool_budget"
+ return [normalized, top_category, "suspicious" if suspicious_success else "failed"]
+
+
def compute_failure_signature(
frontier_start_label: str | None,
failure_reason: str | None,
@@ -72,26 +93,30 @@ def compute_failure_signature(
run (status=completed but data-producing blocks produced no output) still
generates a signature so repeated no-data runs can be counted as repeats.
"""
- normalized = normalize_failure_reason(failure_reason)
- has_signal = bool(normalized) or suspicious_success
- if not has_signal:
+ terms = _failure_identity_terms(failure_reason, failure_categories, suspicious_success)
+ if terms is None:
return None
safe_label = frontier_start_label if isinstance(frontier_start_label, str) else ""
- top_category = _top_failure_category(failure_categories)
- # Categories whose failure_reason embeds per-call data (key name, run id):
- # collapse to a stable constant so consecutive trips hash to the same
- # signature instead of each one looking unique.
- if top_category == "PARAMETER_BINDING_ERROR":
- normalized = "parameter_binding_error"
- elif top_category == PER_TOOL_BUDGET_FAILURE_CATEGORY:
- normalized = "per_tool_budget"
- parts = [
- safe_label,
- normalized,
- top_category,
- "suspicious" if suspicious_success else "failed",
- ]
- return "|".join(parts)
+ return "|".join([safe_label, *terms])
+
+
+def satisfied_criterion_ids(result: CompletionVerificationResult | None) -> frozenset[str]:
+ """Criterion ids the outcome-verification judge confirmed satisfied (evidence_confirms),
+ or empty when no judge result is available for this run."""
+ if result is None or result.status != "evaluated":
+ return frozenset()
+ return frozenset(verdict.criterion_id for verdict in result.verdicts if verdict.reason_code == "evidence_confirms")
+
+
+def made_newly_verified_progress(
+ current_satisfied: frozenset[str],
+ high_water: frozenset[str],
+ full_workflow_verified_this_run: bool,
+ verified_prefix_grew: bool,
+) -> bool:
+ """Whether this run advanced verified progress past the turn high-water: a newly
+ confirmed criterion, a clean end-to-end run, or a grown verified block prefix."""
+ return bool(current_satisfied - high_water) or full_workflow_verified_this_run or verified_prefix_grew
def _canonical_block_config(block: Any) -> dict[str, Any]:
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 36957cff5..783c1944b 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -15,6 +15,7 @@ import structlog
import yaml
from pydantic import ValidationError
+from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot.blocker_signal import (
@@ -28,12 +29,17 @@ from skyvern.forge.sdk.copilot.completion_verification import (
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisRepairContract,
+ RepairLoopState,
+ RepairNextAction,
build_diagnosis_repair_contract,
)
+from skyvern.forge.sdk.copilot.enforcement import repair_ceiling_stop_signal
from skyvern.forge.sdk.copilot.failure_tracking import (
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
PER_TOOL_BUDGET_FAILURE_CATEGORY,
compute_action_sequence_fingerprint,
+ made_newly_verified_progress,
+ satisfied_criterion_ids,
update_repeated_failure_state,
)
from skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctx
@@ -62,7 +68,10 @@ from skyvern.forge.sdk.copilot.runtime import (
ensure_browser_session,
)
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
-from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal
+from skyvern.forge.sdk.copilot.turn_halt import (
+ stash_repair_ceiling_turn_halt,
+ stash_turn_halt_from_blocker_signal,
+)
from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotRunOutcomeUpdate, WorkflowCopilotStreamMessageType
from skyvern.forge.sdk.settings_manager import SettingsManager
@@ -1772,6 +1781,73 @@ async def _verify_and_record_run_blocks_result(
return completion_verification
+def _repair_non_convergence_signature(contract: DiagnosisRepairContract) -> str | None:
+ """A constant streak token for any REPAIR verdict, ``None`` otherwise. The trigger
+ is non-convergence (REPAIR with no verified progress), so the only thing the signature
+ must do is tell a contiguous repair streak from a reset on a non-REPAIR verdict."""
+ if contract.repair_decision.next_action is not RepairNextAction.REPAIR:
+ return None
+ # Constant by design: the streak only distinguishes a REPAIR run from a reset, so the failure specifics are ignored.
+ return "repair_no_verified_progress"
+
+
+def _update_repair_loop_state(copilot_ctx: Any, contract: DiagnosisRepairContract) -> None:
+ """Count consecutive REPAIR verdicts that made no newly-verified forward progress.
+
+ Progress is growth in the turn-scoped set of judge-confirmed completion criteria, or a
+ clean end-to-end run, or a grown verified block prefix — never the failure prose, the
+ failure_type, or which block label failed. The high-water marks must be read BEFORE the
+ current run's confirmations are folded in, else this run's own wins would already be
+ banked and never read as new.
+ """
+ current = satisfied_criterion_ids(getattr(copilot_ctx, "completion_verification_result", None))
+ high_water_raw: Any = getattr(copilot_ctx, "verified_criteria_high_water", frozenset())
+ high_water = high_water_raw if isinstance(high_water_raw, frozenset) else frozenset(high_water_raw or [])
+ prefix_len = len(getattr(copilot_ctx, "verified_prefix_labels", []) or [])
+ prefix_high_raw = getattr(copilot_ctx, "verified_prefix_high_water_len", 0)
+ prefix_high = prefix_high_raw if isinstance(prefix_high_raw, int) else 0
+ # A run-tied REPAIR verdict always sees this False (the failing run cleared it in
+ # _record_run_blocks_result); a True here is a stale carry-over from a prior clean
+ # pass on a non-run path, so latch it consumed and count it as progress only once.
+ full_pass = getattr(copilot_ctx, "last_full_workflow_test_ok", False) is True
+ consumed = getattr(copilot_ctx, "verified_full_pass_consumed", False) is True
+ progressed = made_newly_verified_progress(
+ current_satisfied=current,
+ high_water=high_water,
+ full_workflow_verified_this_run=full_pass and not consumed,
+ verified_prefix_grew=prefix_len > prefix_high,
+ )
+ copilot_ctx.verified_criteria_high_water = high_water | current
+ copilot_ctx.verified_prefix_high_water_len = max(prefix_high, prefix_len)
+ copilot_ctx.verified_full_pass_consumed = full_pass
+
+ signature = _repair_non_convergence_signature(contract)
+ if signature is None or progressed:
+ copilot_ctx.consecutive_non_converging_repair_count = 0
+ copilot_ctx.last_repair_non_convergence_signature = None
+ contract.repair_loop_state = RepairLoopState(
+ streak_token=None,
+ consecutive_identical_repair_count=0,
+ ceiling_reached=False,
+ )
+ return
+ prior_signature = getattr(copilot_ctx, "last_repair_non_convergence_signature", None)
+ prior_count = getattr(copilot_ctx, "consecutive_non_converging_repair_count", 0)
+ prior_count = prior_count if isinstance(prior_count, int) else 0
+ count = prior_count + 1 if signature == prior_signature else 1
+ copilot_ctx.consecutive_non_converging_repair_count = count
+ copilot_ctx.last_repair_non_convergence_signature = signature
+ contract.repair_loop_state = RepairLoopState(
+ streak_token=signature,
+ consecutive_identical_repair_count=count,
+ ceiling_reached=count >= settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL,
+ )
+ if contract.repair_loop_state.ceiling_reached:
+ signal = repair_ceiling_stop_signal(copilot_ctx, contract)
+ stash_blocker_signal(copilot_ctx, signal)
+ stash_repair_ceiling_turn_halt(copilot_ctx, signal, consecutive_identical_repair_count=count)
+
+
def _record_diagnosis_repair_contract(
copilot_ctx: Any,
*,
@@ -1785,6 +1861,7 @@ def _record_diagnosis_repair_contract(
ctx=copilot_ctx,
workflow_updated=workflow_updated,
)
+ _update_repair_loop_state(copilot_ctx, contract)
copilot_ctx.latest_diagnosis_repair_contract = contract
trace_data = contract.to_trace_data()
LOG.info(
diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.py
index c62f22900..7e6b4b492 100644
--- a/skyvern/forge/sdk/copilot/turn_halt.py
+++ b/skyvern/forge/sdk/copilot/turn_halt.py
@@ -19,6 +19,7 @@ class TurnHaltKind(StrEnum):
LOOP_DETECTED = "loop_detected"
ACTIVE_TERMINAL_CHALLENGE = "active_terminal_challenge"
PROBABLE_SITE_BLOCK = "probable_site_block"
+ REPAIR_CEILING_REACHED = "repair_ceiling_reached"
class TurnHaltVerdict(StrEnum):
@@ -99,6 +100,29 @@ def stash_turn_halt_from_blocker_signal(ctx: Any, signal: object, *, source: str
return halt
+def stash_repair_ceiling_turn_halt(
+ ctx: Any,
+ signal: CopilotToolBlockerSignal,
+ *,
+ consecutive_identical_repair_count: int,
+) -> TurnHalt | None:
+ existing = getattr(ctx, "turn_halt", None)
+ if isinstance(existing, TurnHalt):
+ return existing
+ halt = TurnHalt(
+ kind=TurnHaltKind.REPAIR_CEILING_REACHED,
+ blocker_signal=signal,
+ draft_state={"preserves_workflow_draft": signal.preserves_workflow_draft},
+ extra={
+ "source": "enforcement",
+ "consecutive_identical_repair_count": consecutive_identical_repair_count,
+ },
+ )
+ ctx.turn_halt = halt
+ LOG.info("copilot turn halt stashed", **turn_halt_to_trace_data(halt))
+ return halt
+
+
def raise_if_turn_halt(ctx: Any) -> None:
halt = getattr(ctx, "turn_halt", None)
if isinstance(halt, TurnHalt):