Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11067 stabilize Copilot repair root-cause signatures (#6678)
Andrew Neilsonb844eb4e9632
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex 6ccbc7f9a..e829c2542 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -5,7 +5,11 @@ from typing import TYPE_CHECKING, Anyfrom pydantic import BaseModel, ConfigDict, Field-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY+from skyvern.forge.sdk.copilot.failure_tracking import (+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,+ RepairRootCauseIdentity,+ compute_repair_root_cause_signature,+)from skyvern.forge.sdk.copilot.output_policy import url_originfrom skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_RE@@ -65,6 +69,7 @@ class DiagnosisInput(StrictModel):class DiagnosisResult(StrictModel):suspected_failure_type: DiagnosisFailureType = DiagnosisFailureType.UNKNOWNroot_cause_summary: str = ""+ root_cause_identity: RepairRootCauseIdentity = Field(default_factory=RepairRootCauseIdentity)confidence: float = Field(default=0.0, ge=0.0, le=1.0)evidence_references: list[str] = Field(default_factory=list)missing_context: list[str] = Field(default_factory=list)@@ -99,8 +104,15 @@ class DiagnosisRepairContract(StrictModel):repair_loop_state: RepairLoopState = Field(default_factory=RepairLoopState)def to_trace_data(self) -> dict[str, Any]:+ identity = self.diagnosis_result.root_cause_identityreturn {"failure_type": self.diagnosis_result.suspected_failure_type.value,+ "root_cause_signature": identity.root_cause_signature,+ "root_cause_primary_category": identity.primary_category,+ "root_cause_categories": list(identity.failure_categories),+ "root_cause_error_class": identity.error_class,+ "root_cause_selector_kind": identity.selector_kind,+ "root_cause_selector": identity.selector,"next_action": self.repair_decision.next_action.value,"confidence": self.diagnosis_result.confidence,"source_tool": self.diagnosis_input.source_tool,@@ -136,6 +148,13 @@ def build_diagnosis_repair_contract(run_status = _safe_str(data.get("overall_status"))workflow_run_id = _safe_str(data.get("workflow_run_id"))summary = _failure_summary(result, data, blocks)+ root_cause_identity = compute_repair_root_cause_signature(+ failure_categories=categories,+ failure_reason=_safe_str(data.get("failure_reason")),+ error_texts=[_safe_str(result.get("error"))],+ blocks=[block for block in blocks if isinstance(block, dict)],+ detected_challenge=bool(getattr(ctx, "last_test_anti_bot", None)),+ )failure_type = _failure_type(run_ok, suspicious, failed_blocks, categories, result, data)next_action = _next_action(failure_type, ctx, data)frontier = _safe_str(data.get("frontier_start_label"))@@ -207,6 +226,7 @@ def build_diagnosis_repair_contract(diagnosis_result=DiagnosisResult(suspected_failure_type=failure_type,root_cause_summary=summary,+ root_cause_identity=root_cause_identity,confidence=confidence,evidence_references=(([f"workflow_run:{workflow_run_id}"] if workflow_run_id else [])diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.pyindex e44dcb364..dda2e5053 100644--- a/skyvern/forge/sdk/copilot/failure_tracking.py+++ b/skyvern/forge/sdk/copilot/failure_tracking.py@@ -3,8 +3,8 @@Computes two normalized keys per run:- **failure signature**: "is this the same failure as last time?" —- frontier_start_label + normalized failure reason + top failure category +- suspicious-success flag.+ structural root-cause identity when available, otherwise normalized failure+ reason + top failure category + suspicious-success flag.- **frontier fingerprint**: SHA256 of the executed blocks' canonical config.Changes whenever the agent edits any block in the executed suffix.@@ -22,6 +22,9 @@ from __future__ import annotationsimport hashlibimport json+import re+from collections.abc import Mapping, Sequence+from dataclasses import dataclassfrom typing import TYPE_CHECKING, Anyif TYPE_CHECKING:@@ -30,11 +33,13 @@ if TYPE_CHECKING:from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition_FAILURE_REASON_MAX_CHARS = 200+_ROOT_CAUSE_SIGNATURE_VERSION = "repair_root_cause:v1"# Stable identifier for the per-tool budget failure category written into# ``failure_categories`` by the watchdog. Used by enforcement, reconciliation,# and signature normalization as a single source of truth.PER_TOOL_BUDGET_FAILURE_CATEGORY = "PER_TOOL_BUDGET"+ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY = "ANTI_BOT_CHALLENGE"# Stable active-run terminal evidence identifiers. The category is stored in# tool result ``failure_categories``; the reason code is stored on blocker@@ -42,6 +47,64 @@ PER_TOOL_BUDGET_FAILURE_CATEGORY = "PER_TOOL_BUDGET"ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY = "ACTIVE_RUN_TERMINAL_EVIDENCE"ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE = "tool_error_active_run_terminal_evidence"+_ROOT_CAUSE_CATEGORY_ALIASES = {+ "ANTI_BOT_DETECTION": ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY,+ "CHALLENGE_DETECTION": ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY,+ "HUMAN_VERIFICATION_CHALLENGE": ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY,+ # Self-mappings document categories whose noisy failure prose must not+ # affect repeat identity.+ "PARAMETER_BINDING_ERROR": "PARAMETER_BINDING_ERROR",+ PER_TOOL_BUDGET_FAILURE_CATEGORY: PER_TOOL_BUDGET_FAILURE_CATEGORY,+}+_CATEGORY_ONLY_ROOT_CAUSES = frozenset(+ {+ ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY,+ "PARAMETER_BINDING_ERROR",+ PER_TOOL_BUDGET_FAILURE_CATEGORY,+ }+)+_LOCATOR_RE = re.compile(r"""(?:page\.)?locator\(\s*(["'])(?P<selector>.+?)\1""")+_SELECTOR_QUOTED_RE = re.compile(+ r"""(?:selector|css selector|target selector|resolved selector)\s*[:=]\s*(["'])(?P<selector>.+?)\1""",+ re.IGNORECASE,+)+_SELECTOR_UNQUOTED_RE = re.compile(+ r"""(?:selector|css selector|target selector|resolved selector)\s*[:=]\s*(?P<selector>[#.\[\]:=\w\-/]+)""",+ re.IGNORECASE,+)+_ROLE_RE = re.compile(+ r"""get_by_role\(\s*(["'])(?P<role>.+?)\1(?:\s*,\s*name\s*=\s*(["'])(?P<name>.*?)\3)?""",+ re.IGNORECASE,+)+_EXCEPTION_CLASS_RE = re.compile(r"""\b(?P<class>[A-Za-z_][\w.]*?(?:Error|Exception))\s*:""")+# ``browser_context.mode=none`` is the stable enum-backed value surfaced by+# local browser-state failures, unlike surrounding natural-language prose.+_BROWSER_SESSION_ERROR_RE = re.compile(+ r"\b(browser session not found|no browser context|no active browser|browser_context\.mode=none|"+ r"no_active_browser|browser_not_found)\b",+ re.IGNORECASE,+)+_TIMEOUT_ERROR_RE = re.compile(r"\b(timeout(?:error)?|timed out)\b", re.IGNORECASE)+_CODE_BLOCK_IMPOSITION_DROPPED_RE = re.compile(+ r"unable to impose synthesized code block:\s*dropped scout interaction\s+\d+\s+"+ r"from\s+`?(?P<tool>[A-Za-z0-9_]+)`?\s+\((?P<reason>[A-Za-z0-9_]+)\)",+ re.IGNORECASE,+)+_CODE_BLOCK_IMPOSITION_RE = re.compile(+ r"unable to impose synthesized code block:\s*(?P<reason>[^\n.]+)",+ re.IGNORECASE,+)+++@dataclass(frozen=True)+class RepairRootCauseIdentity:+ root_cause_signature: str | None = None+ primary_category: str = ""+ failure_categories: tuple[str, ...] = ()+ error_class: str = ""+ selector_kind: str = ""+ selector: str = ""+def normalize_failure_reason(raw: str | None) -> str:if not raw:@@ -52,6 +115,138 @@ def normalize_failure_reason(raw: str | None) -> str:return collapsed.lower()+def _category_name(value: Mapping[str, Any] | str) -> str:+ raw = value.get("category") if isinstance(value, Mapping) else value+ return str(raw or "").strip().upper()+++def _normalized_root_cause_categories(+ failure_categories: Sequence[Mapping[str, Any] | str] | None,+ detected_challenge: bool,+) -> tuple[str, ...]:+ categories: set[str] = set()+ for entry in failure_categories or ():+ category = _category_name(entry)+ if not category:+ continue+ categories.add(_ROOT_CAUSE_CATEGORY_ALIASES.get(category, category))+ if detected_challenge:+ categories.add(ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY)+ return tuple(sorted(categories))+++def _normalize_signature_text(value: str) -> str:+ return " ".join(value.strip().split()).rstrip(".,;")+++def _selector_from_text(text: str) -> tuple[str, str]:+ for pattern, kind in (+ (_LOCATOR_RE, "locator"),+ (_SELECTOR_QUOTED_RE, "selector"),+ (_SELECTOR_UNQUOTED_RE, "selector"),+ ):+ match = pattern.search(text)+ if match:+ return kind, _normalize_signature_text(match.group("selector"))+ role_match = _ROLE_RE.search(text)+ if role_match:+ role = _normalize_signature_text(role_match.group("role"))+ name = _normalize_signature_text(role_match.group("name") or "")+ return "role", f"{role}:{name}" if name else role+ return "", ""+++def _error_class_from_text(text: str) -> str:+ code_block_imposition = _code_block_imposition_error_class(text)+ if code_block_imposition:+ return code_block_imposition+ if _BROWSER_SESSION_ERROR_RE.search(text):+ return "browser_session_not_found"+ match = _EXCEPTION_CLASS_RE.search(text)+ if match:+ return re.sub(r"(?<!^)(?=[A-Z])", "_", match.group("class").split(".")[-1]).lower()+ if _TIMEOUT_ERROR_RE.search(text):+ return "timeout_error"+ return ""+++def _code_block_imposition_error_class(text: str) -> str:+ dropped_match = _CODE_BLOCK_IMPOSITION_DROPPED_RE.search(text)+ if dropped_match:+ return f"code_block_synthesis_{_snake_token(dropped_match.group('reason'))}"+ match = _CODE_BLOCK_IMPOSITION_RE.search(text)+ if match:+ return f"code_block_synthesis_{_snake_token(match.group('reason'))}"+ return ""+++def _snake_token(value: str) -> str:+ normalized = re.sub(r"[^a-zA-Z0-9]+", "_", value.strip().lower()).strip("_")+ return normalized or "unknown"+++def _root_cause_texts(+ failure_reason: str | None,+ error_texts: Sequence[str | None] | None,+ blocks: Sequence[Mapping[str, Any]] | None,+) -> list[str]:+ texts = [text for text in ([failure_reason] + list(error_texts or ())) if isinstance(text, str) and text.strip()]+ for block in blocks or ():+ for key in ("failure_reason", "error", "message"):+ value = block.get(key)+ if isinstance(value, str) and value.strip():+ texts.append(value)+ return texts+++def compute_repair_root_cause_signature(+ *,+ failure_categories: Sequence[Mapping[str, Any] | str] | None = None,+ failure_reason: str | None = None,+ error_texts: Sequence[str | None] | None = None,+ blocks: Sequence[Mapping[str, Any]] | None = None,+ detected_challenge: bool = False,+) -> RepairRootCauseIdentity:+ categories = _normalized_root_cause_categories(failure_categories, detected_challenge)+ primary_category = categories[0] if categories else ""+ texts = _root_cause_texts(failure_reason, error_texts, blocks)+ error_class = ""+ selector_kind = ""+ selector = ""+ for text in texts:+ if not error_class:+ error_class = _error_class_from_text(text)+ if not selector:+ selector_kind, selector = _selector_from_text(text)+ if error_class and selector:+ break++ if set(categories) & _CATEGORY_ONLY_ROOT_CAUSES:+ error_class = ""+ selector_kind = ""+ selector = ""+ if not categories and not error_class and not selector:+ return RepairRootCauseIdentity()++ payload = {+ "version": _ROOT_CAUSE_SIGNATURE_VERSION,+ "failure_categories": categories,+ "error_class": error_class,+ "selector_kind": selector_kind,+ "selector": selector,+ }+ serialized = json.dumps(payload, sort_keys=True, separators=(",", ":"))+ signature = hashlib.sha256(serialized.encode("utf-8")).hexdigest()+ return RepairRootCauseIdentity(+ root_cause_signature=signature,+ primary_category=primary_category,+ failure_categories=categories,+ error_class=error_class,+ selector_kind=selector_kind,+ selector=selector,+ )++def _top_failure_category(failure_categories: list[dict] | None) -> str:if not failure_categories:return ""@@ -65,20 +260,29 @@ def _failure_identity_terms(failure_reason: str | None,failure_categories: list[dict] | None,suspicious_success: bool,+ detected_challenge: bool = False,) -> list[str] | None:"""Category-stable failure half shared by both signatures, or ``None`` on areal success. Per-call-data categories collapse to a constant so consecutivetrips hash identically."""normalized = normalize_failure_reason(failure_reason)- has_signal = bool(normalized) or suspicious_success+ has_signal = bool(normalized) or bool(failure_categories) or suspicious_success or detected_challengeif not has_signal:return None+ structural_identity = compute_repair_root_cause_signature(+ failure_categories=failure_categories,+ failure_reason=failure_reason,+ detected_challenge=detected_challenge,+ )+ terminal_state = "suspicious" if suspicious_success else "failed"+ if structural_identity.root_cause_signature:+ return [structural_identity.root_cause_signature]top_category = _top_failure_category(failure_categories)if top_category == "PARAMETER_BINDING_ERROR":normalized = "parameter_binding_error"elif top_category == PER_TOOL_BUDGET_FAILURE_CATEGORY:normalized = "per_tool_budget"- return [normalized, top_category, "suspicious" if suspicious_success else "failed"]+ return [normalized, top_category, terminal_state]def compute_failure_signature(@@ -86,18 +290,22 @@ def compute_failure_signature(failure_reason: str | None,failure_categories: list[dict] | None,suspicious_success: bool,+ detected_challenge: bool = False,) -> str | None:"""Return a normalized signature for the current failure, or ``None`` on success.``None`` means "no signature — this was a real success". A suspicious-successrun (status=completed but data-producing blocks produced no output) stillgenerates a signature so repeated no-data runs can be counted as repeats.++ ``frontier_start_label`` is kept for call-site compatibility. Labels are+ intentionally excluded from the signature because block renames are not a+ new root cause."""- terms = _failure_identity_terms(failure_reason, failure_categories, suspicious_success)+ terms = _failure_identity_terms(failure_reason, failure_categories, suspicious_success, detected_challenge)if terms is None:return None- safe_label = frontier_start_label if isinstance(frontier_start_label, str) else ""- return "|".join([safe_label, *terms])+ return "|".join(terms)def satisfied_criterion_ids(result: CompletionVerificationResult | None) -> frozenset[str]:@@ -219,6 +427,7 @@ def update_repeated_failure_state(suspicious_success_raw = getattr(ctx, "last_test_suspicious_success", False)suspicious_success = bool(suspicious_success_raw) if isinstance(suspicious_success_raw, (bool, int)) else False+ detected_challenge = bool(getattr(ctx, "last_test_anti_bot", None))failure_reason_raw = getattr(ctx, "last_test_failure_reason", None)failure_reason = failure_reason_raw if isinstance(failure_reason_raw, str) else Nonefrontier_start_raw = getattr(ctx, "last_frontier_start_label", None)@@ -258,6 +467,7 @@ def update_repeated_failure_state(failure_reason=failure_reason,failure_categories=failure_categories,suspicious_success=suspicious_success,+ detected_challenge=detected_challenge,)fingerprint = compute_frontier_fingerprint(executed_labels, workflow_definition)diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex 5ea21b90c..d66d409f9 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -352,6 +352,12 @@ async def update_workflow_tool()_record_workflow_update_result(copilot_ctx, result, prior_definition)record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)+ if result.get("ok") is False:+ _record_diagnosis_repair_contract(+ copilot_ctx,+ source_tool="update_workflow",+ result=result,+ )sanitized = sanitize_tool_result_for_llm("update_workflow", result)return json.dumps(sanitized)diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 4ca20edb2..1f3e24e85 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -20,6 +20,7 @@ from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocksfrom skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handlerfrom skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verificationfrom skyvern.forge.sdk.copilot.output_utils import iter_failure_reasons+from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_OUTPUT_KEYSfrom skyvern.forge.sdk.copilot.request_policy import CompletionCriterionfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span@@ -185,6 +186,46 @@ async def _maybe_run_completion_verification_from_page_observation(_COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS = 5.0+_DOWNLOAD_EVIDENCE_FILE_NAME_KEYS = ("filename", "file_name", "name", "path")+_MAX_EVIDENCE_FILE_NAMES = 5+++def _download_file_name(value: Any) -> str | None:+ if isinstance(value, dict):+ for key in _DOWNLOAD_EVIDENCE_FILE_NAME_KEYS:+ if name := _download_file_name(value.get(key)):+ return name+ return None+ if not isinstance(value, str) or not value.strip():+ return None+ name = value.split("?", 1)[0].rstrip("/").rsplit("/", 1)[-1].strip()+ return name or None+++def _completion_evidence_payload(output: Any) -> Any:+ if not isinstance(output, dict) or not any(output.get(key) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS):+ return output+ names: list[str] = []+ if name := _download_file_name(output.get("downloaded_file_name")):+ names.append(name)+ files = output.get("downloaded_files")+ if isinstance(files, list):+ names.extend(name for item in files[:_MAX_EVIDENCE_FILE_NAMES] if (name := _download_file_name(item)))+ urls = output.get("downloaded_file_urls")+ if isinstance(urls, list):+ names.extend(name for item in urls[:_MAX_EVIDENCE_FILE_NAMES] if (name := _download_file_name(item)))+ payload: dict[str, Any] = {"download_registered": True}+ if isinstance(files, list):+ payload["downloaded_file_count"] = len(files)+ if isinstance(urls, list):+ payload["downloaded_file_url_count"] = len(urls)+ artifacts = output.get("downloaded_file_artifact_ids")+ if isinstance(artifacts, list):+ payload["downloaded_file_artifact_count"] = len(artifacts)+ if names:+ payload["downloaded_file_names"] = list(dict.fromkeys(names))[:_MAX_EVIDENCE_FILE_NAMES]+ return payload+_ARTIFACT_HEALTH_EXCLUDED_CATEGORIES = frozenset({@@ -329,8 +370,9 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Rucontinuelabel = block.get("label")output = block.get("extracted_data")- if isinstance(label, str) and label in current_labels and _is_meaningful_extracted_data(output):- block_outputs[label] = output+ evidence_output = _completion_evidence_payload(output)+ if isinstance(label, str) and label in current_labels and _is_meaningful_extracted_data(evidence_output):+ block_outputs[label] = evidence_outputexecuted = data.get("executed_block_labels")executed_block_labels = [str(label) for label in executed] if isinstance(executed, list) else []page_title = data.get("page_title")diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex ce37bea81..1b3938479 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -1562,6 +1562,47 @@ def test_snapshot_uses_current_run_blocks_not_stale_outputs() -> None:assert snap2.block_outputs.get("b3") == {"extracted_information": {"price": "9.99"}}+def test_snapshot_summarizes_registered_download_outputs() -> None:+ ctx = _run_ctx()+ ctx.last_workflow = SimpleNamespace(+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(label="download_statement")])+ )+ run = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_download",+ "blocks": [+ {+ "label": "download_statement",+ "extracted_data": {+ "page": "<RecordingLocator>",+ "download": "<Download>",+ "downloaded_file_name": "apexbiz_100245_2026-05.pdf",+ "downloaded_files": [{"filename": "apexbiz_100245_2026-05.pdf"}],+ "downloaded_file_urls": [+ "https://local.test/downloads/apexbiz_100245_2026-05.pdf?token=secret"+ ],+ "downloaded_file_artifact_ids": ["artifact_1"],+ },+ }+ ],+ },+ }++ snapshot = _build_run_evidence_snapshot(ctx, run)+ rendered = snapshot.render_prompt_block()++ assert snapshot.block_outputs["download_statement"] == {+ "download_registered": True,+ "downloaded_file_count": 1,+ "downloaded_file_url_count": 1,+ "downloaded_file_artifact_count": 1,+ "downloaded_file_names": ["apexbiz_100245_2026-05.pdf"],+ }+ assert "apexbiz_100245_2026-05.pdf" in rendered+ assert "RecordingLocator" not in rendered and "Download" not in rendered and "secret" not in rendered++def test_snapshot_includes_verified_context_labels_without_prior_outputs() -> None:ctx = _run_ctx()labels = [diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 6c2d41dc8..773be572c 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -1,6 +1,10 @@from __future__ import annotations+import jsonfrom types import SimpleNamespace+from unittest.mock import AsyncMock++import pytestfrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.context import CopilotContext@@ -390,3 +394,78 @@ def test_unrecoverable_browser_session_contract_stops_with_blocker() -> None:assert contract.repair_decision.next_action == RepairNextAction.STOPassert contract.verification_result.user_goal_satisfied is Falseassert contract.verification_result.remaining_blocker == reason+++def test_contract_trace_exposes_stable_root_cause_identity() -> None:+ def result(reason: str, status: str, label: str) -> dict[str, object]:+ return {+ "ok": False,+ "error": reason,+ "data": {+ "workflow_run_id": f"wr_{label}",+ "overall_status": status,+ "failure_reason": reason,+ "frontier_start_label": label,+ "failure_categories": [{"category": "UNRECOVERABLE_TOOL_ERROR"}],+ "blocks": [{"label": label, "status": status, "failure_reason": reason}],+ },+ }++ base = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result=result('Browser session not found while waiting for locator("#submit")', "failed", "login_v1"),+ ctx=_ctx(),+ )+ renamed = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result=result('No browser context while waiting for locator("#submit")', "terminated", "login_v2"),+ ctx=_ctx(),+ )++ base_trace = base.to_trace_data()+ renamed_trace = renamed.to_trace_data()+ assert base_trace["root_cause_signature"] == renamed_trace["root_cause_signature"]+ assert base_trace["root_cause_error_class"] == "browser_session_not_found"+ assert base_trace["root_cause_selector_kind"] == "locator"+ assert base_trace["root_cause_selector"] == "#submit"+ assert (+ base.model_dump()["diagnosis_result"]["root_cause_identity"]["root_cause_signature"]+ == base_trace["root_cause_signature"]+ )+ assert base_trace["run_status"] != renamed_trace["run_status"]+ assert {base.diagnosis_result.suspected_failure_type, renamed.diagnosis_result.suspected_failure_type} == {+ DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR+ }+++async def test_update_workflow_failure_records_diagnosis_contract(monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.copilot import tools as tools_module++ error = (+ "Unable to impose synthesized code block: dropped scout interaction 0 from `click` (ambiguous_bare_selector)."+ )+ recorded: list[tuple[str, dict[str, object], str]] = []++ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "_request_policy_allows_credential_deferred_draft", lambda *args: False)+ monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *args: None)+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))+ monkeypatch.setattr(tools_module, "_record_workflow_update_result", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "_update_workflow", AsyncMock(return_value={"ok": False, "error": error}))++ def fake_record_contract(ctx: CopilotContext, *, source_tool: str, result: dict[str, object]) -> None:+ contract = build_diagnosis_repair_contract(source_tool=source_tool, result=result, ctx=ctx)+ recorded.append((source_tool, result, contract.to_trace_data()["root_cause_error_class"]))++ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", fake_record_contract)++ result = await tools_module.update_workflow_tool.on_invoke_tool(+ SimpleNamespace(context=_ctx(), tool_name="update_workflow"),+ json.dumps({"workflow_yaml": "title: Test\nworkflow_definition:\n parameters: []\n blocks: []\n"}),+ )++ assert json.loads(result)["ok"] is False+ assert recorded == [+ ("update_workflow", {"ok": False, "error": error}, "code_block_synthesis_ambiguous_bare_selector")+ ]diff --git a/tests/unit/test_copilot_failure_tracking.py b/tests/unit/test_copilot_failure_tracking.pyindex 691395506..3fa33391a 100644--- a/tests/unit/test_copilot_failure_tracking.py+++ b/tests/unit/test_copilot_failure_tracking.py@@ -8,6 +8,7 @@ import pytestfrom skyvern.forge.sdk.copilot.failure_tracking import (compute_failure_signature,compute_frontier_fingerprint,+ compute_repair_root_cause_signature,update_repeated_failure_state,)@@ -30,6 +31,7 @@ def _make_ctx(*,suspicious: bool = False,failure_reason: str | None = "Timeout on element",+ anti_bot: str | None = None,frontier_label: str = "extract",executed_labels: list[str] | None = None,workflow: Any = None,@@ -40,6 +42,7 @@ def _make_ctx() -> SimpleNamespace:return SimpleNamespace(last_test_suspicious_success=suspicious,+ last_test_anti_bot=anti_bot,last_test_failure_reason=failure_reason,last_frontier_start_label=frontier_label,last_executed_block_labels=executed_labels or ["open", "extract"],@@ -63,6 +66,32 @@ def test_signature_differs_on_reason_change() -> None:assert a != b+def test_signature_uses_root_cause_identity_for_repeat_detection() -> None:+ categories = [{"category": "UNRECOVERABLE_TOOL_ERROR"}]+ signatures = {+ compute_failure_signature(+ "login_v1", 'Browser session not found while waiting for locator("#submit")', categories, False+ ),+ compute_failure_signature(+ "login_v2", 'No browser context while waiting for locator("#submit")', categories, False+ ),+ compute_failure_signature(+ "login_v3", 'No browser context while waiting for locator("#submit")', categories, True+ ),+ compute_failure_signature(+ "login_v4", 'browser_context.mode=none while waiting for locator("#submit")', categories, True+ ),+ }+ parameter_signature = compute_failure_signature("block_a", None, [{"category": "PARAMETER_BINDING_ERROR"}], False)++ assert len(signatures) == 1+ assert next(iter(signatures)) is not None+ assert parameter_signature is not None+ assert parameter_signature == compute_failure_signature(+ "renamed_block", None, [{"category": "PARAMETER_BINDING_ERROR"}], False+ )++def test_signature_collapses_parameter_binding_error() -> None:# PARAMETER_BINDING_ERROR embeds offending key names in failure_reason;# different keys must still hash to the same signature so repeats count.@@ -75,6 +104,33 @@ def test_signature_collapses_parameter_binding_error() -> None:assert a == b+def test_category_only_and_code_imposition_root_causes_collapse_noise() -> None:+ category_detected = compute_repair_root_cause_signature(+ failure_categories=[{"category": "ANTI_BOT_DETECTION"}],+ failure_reason="Verify you are human before continuing.",+ )+ context_detected = compute_repair_root_cause_signature(+ failure_categories=[],+ failure_reason="Access page showed different challenge copy this time.",+ detected_challenge=True,+ )+ first = compute_repair_root_cause_signature(+ error_texts=[+ "Unable to impose synthesized code block: dropped scout interaction 0 from `click` (ambiguous_bare_selector)."+ ],+ )+ retried = compute_repair_root_cause_signature(+ error_texts=[+ "Unable to impose synthesized code block: dropped scout interaction 4 from `click` (ambiguous_bare_selector)."+ ],+ )++ assert category_detected.root_cause_signature == context_detected.root_cause_signature+ assert category_detected.primary_category == "ANTI_BOT_CHALLENGE"+ assert first.root_cause_signature == retried.root_cause_signature+ assert first.error_class == "code_block_synthesis_ambiguous_bare_selector"++def test_fingerprint_empty_without_workflow_definition() -> None:assert compute_frontier_fingerprint(["open"], None) == ""@@ -106,6 +162,33 @@ def test_streak_increments_when_signature_and_fingerprint_repeat() -> None:assert ctx.last_frontier_fingerprint == fingerprint+def test_streak_increments_across_challenge_prose_and_frontier_drift() -> None:+ wf = _make_workflow([_Block("open", url="x"), _Block("extract", schema="s")])+ fingerprint = compute_frontier_fingerprint(["open", "extract"], wf.workflow_definition)+ signature = compute_failure_signature(+ "extract_v1",+ "Run completed, but extracted data reported a blocker: Verify you are human",+ None,+ True,+ detected_challenge=True,+ )++ ctx = _make_ctx(+ suspicious=True,+ anti_bot="Extracted data reported anti-bot blocker: Verify you are human",+ failure_reason="Run completed, but extracted data reported a blocker: Cloudflare challenge still present",+ frontier_label="extract_v2",+ workflow=wf,+ last_signature=signature,+ last_fingerprint=fingerprint,+ streak=5,+ )+ update_repeated_failure_state(ctx, {"ok": True, "data": {}})++ assert ctx.repeated_failure_streak_count == 6+ assert ctx.last_failure_signature == signature++def test_streak_resets_on_meaningful_success() -> None:wf = _make_workflow([_Block("open", url="x")])ctx = _make_ctx(@@ -259,6 +342,7 @@ def test_action_fingerprint_streak_independent_of_frontier_streak() -> None:ctx = SimpleNamespace(last_test_suspicious_success=False,+ last_test_anti_bot=None,last_test_failure_reason="Validation failed: name is required",last_frontier_start_label="open",last_executed_block_labels=["open"],diff --git a/tests/unit/test_copilot_parameter_binding.py b/tests/unit/test_copilot_parameter_binding.pyindex dc1cf6942..7ce4b0cb1 100644--- a/tests/unit/test_copilot_parameter_binding.py+++ b/tests/unit/test_copilot_parameter_binding.py@@ -259,16 +259,16 @@ def test_signature_parameter_binding_ignores_key_name() -> None:assert sig_a == sig_b-def test_signature_non_parameter_binding_preserves_text() -> None:+def test_signature_non_category_only_failure_preserves_selector_shape() -> None:sig_a = compute_failure_signature(frontier_start_label="extract",- failure_reason="Element not found for selector #foo",+ failure_reason="Element not found for selector: #foo",failure_categories=[{"category": "ELEMENT_NOT_FOUND", "confidence_float": 0.8}],suspicious_success=False,)sig_b = compute_failure_signature(frontier_start_label="extract",- failure_reason="Element not found for selector #bar",+ failure_reason="Element not found for selector: #bar",failure_categories=[{"category": "ELEMENT_NOT_FOUND", "confidence_float": 0.8}],suspicious_success=False,)diff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.pyindex 1221cdd3f..c8837fb3e 100644--- a/tests/unit/test_copilot_per_tool_budget.py+++ b/tests/unit/test_copilot_per_tool_budget.py@@ -394,9 +394,8 @@ def test_failure_signature_is_stable_across_budget_trips_with_different_run_ids(assert sig_a == sig_b-def test_failure_signature_changes_when_frontier_label_differs() -> None:- """A budget trip on a different frontier should still produce a different- signature so the streak resets when the agent meaningfully changes shape."""+def test_failure_signature_ignores_frontier_label_drift() -> None:+ """Block labels are UI/editing affordances, not root-cause identity."""failure_categories = [{"category": PER_TOOL_BUDGET_FAILURE_CATEGORY, "confidence_float": 1.0}]sig_a = compute_failure_signature(frontier_start_label="block_a",@@ -410,7 +409,8 @@ def test_failure_signature_changes_when_frontier_label_differs() -> None:failure_categories=failure_categories,suspicious_success=False,)- assert sig_a != sig_b+ assert sig_a is not None+ assert sig_a == sig_bdef test_nudge_text_advises_splitting_chain() -> None: