Skip to content

Automate browser based workflows with AI

SKY-11067 stabilize Copilot repair root-cause signatures (#6678)

Andrew Neilsonb844eb4e9632
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index 6ccbc7f9a..e829c2542 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -5,7 +5,11 @@ from typing import TYPE_CHECKING, Any
from pydantic import BaseModel, ConfigDict, Field
-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY
+from skyvern.forge.sdk.copilot.failure_tracking import (
+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
+ RepairRootCauseIdentity,
+ compute_repair_root_cause_signature,
+)
from skyvern.forge.sdk.copilot.output_policy import url_origin
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_RE
@@ -65,6 +69,7 @@ class DiagnosisInput(StrictModel):
class DiagnosisResult(StrictModel):
suspected_failure_type: DiagnosisFailureType = DiagnosisFailureType.UNKNOWN
root_cause_summary: str = ""
+ root_cause_identity: RepairRootCauseIdentity = Field(default_factory=RepairRootCauseIdentity)
confidence: float = Field(default=0.0, ge=0.0, le=1.0)
evidence_references: list[str] = Field(default_factory=list)
missing_context: list[str] = Field(default_factory=list)
@@ -99,8 +104,15 @@ class DiagnosisRepairContract(StrictModel):
repair_loop_state: RepairLoopState = Field(default_factory=RepairLoopState)
def to_trace_data(self) -> dict[str, Any]:
+ identity = self.diagnosis_result.root_cause_identity
return {
"failure_type": self.diagnosis_result.suspected_failure_type.value,
+ "root_cause_signature": identity.root_cause_signature,
+ "root_cause_primary_category": identity.primary_category,
+ "root_cause_categories": list(identity.failure_categories),
+ "root_cause_error_class": identity.error_class,
+ "root_cause_selector_kind": identity.selector_kind,
+ "root_cause_selector": identity.selector,
"next_action": self.repair_decision.next_action.value,
"confidence": self.diagnosis_result.confidence,
"source_tool": self.diagnosis_input.source_tool,
@@ -136,6 +148,13 @@ def build_diagnosis_repair_contract(
run_status = _safe_str(data.get("overall_status"))
workflow_run_id = _safe_str(data.get("workflow_run_id"))
summary = _failure_summary(result, data, blocks)
+ root_cause_identity = compute_repair_root_cause_signature(
+ failure_categories=categories,
+ failure_reason=_safe_str(data.get("failure_reason")),
+ error_texts=[_safe_str(result.get("error"))],
+ blocks=[block for block in blocks if isinstance(block, dict)],
+ detected_challenge=bool(getattr(ctx, "last_test_anti_bot", None)),
+ )
failure_type = _failure_type(run_ok, suspicious, failed_blocks, categories, result, data)
next_action = _next_action(failure_type, ctx, data)
frontier = _safe_str(data.get("frontier_start_label"))
@@ -207,6 +226,7 @@ def build_diagnosis_repair_contract(
diagnosis_result=DiagnosisResult(
suspected_failure_type=failure_type,
root_cause_summary=summary,
+ root_cause_identity=root_cause_identity,
confidence=confidence,
evidence_references=(
([f"workflow_run:{workflow_run_id}"] if workflow_run_id else [])
diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.py
index e44dcb364..dda2e5053 100644
--- a/skyvern/forge/sdk/copilot/failure_tracking.py
+++ b/skyvern/forge/sdk/copilot/failure_tracking.py
@@ -3,8 +3,8 @@
Computes two normalized keys per run:
- **failure signature**: "is this the same failure as last time?" —
- frontier_start_label + normalized failure reason + top failure category +
- suspicious-success flag.
+ structural root-cause identity when available, otherwise normalized failure
+ reason + top failure category + suspicious-success flag.
- **frontier fingerprint**: SHA256 of the executed blocks' canonical config.
Changes whenever the agent edits any block in the executed suffix.
@@ -22,6 +22,9 @@ from __future__ import annotations
import hashlib
import json
+import re
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
@@ -30,11 +33,13 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition
_FAILURE_REASON_MAX_CHARS = 200
+_ROOT_CAUSE_SIGNATURE_VERSION = "repair_root_cause:v1"
# Stable identifier for the per-tool budget failure category written into
# ``failure_categories`` by the watchdog. Used by enforcement, reconciliation,
# and signature normalization as a single source of truth.
PER_TOOL_BUDGET_FAILURE_CATEGORY = "PER_TOOL_BUDGET"
+ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY = "ANTI_BOT_CHALLENGE"
# Stable active-run terminal evidence identifiers. The category is stored in
# tool result ``failure_categories``; the reason code is stored on blocker
@@ -42,6 +47,64 @@ PER_TOOL_BUDGET_FAILURE_CATEGORY = "PER_TOOL_BUDGET"
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY = "ACTIVE_RUN_TERMINAL_EVIDENCE"
ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE = "tool_error_active_run_terminal_evidence"
+_ROOT_CAUSE_CATEGORY_ALIASES = {
+ "ANTI_BOT_DETECTION": ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY,
+ "CHALLENGE_DETECTION": ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY,
+ "HUMAN_VERIFICATION_CHALLENGE": ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY,
+ # Self-mappings document categories whose noisy failure prose must not
+ # affect repeat identity.
+ "PARAMETER_BINDING_ERROR": "PARAMETER_BINDING_ERROR",
+ PER_TOOL_BUDGET_FAILURE_CATEGORY: PER_TOOL_BUDGET_FAILURE_CATEGORY,
+}
+_CATEGORY_ONLY_ROOT_CAUSES = frozenset(
+ {
+ ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY,
+ "PARAMETER_BINDING_ERROR",
+ PER_TOOL_BUDGET_FAILURE_CATEGORY,
+ }
+)
+_LOCATOR_RE = re.compile(r"""(?:page\.)?locator\(\s*(["'])(?P<selector>.+?)\1""")
+_SELECTOR_QUOTED_RE = re.compile(
+ r"""(?:selector|css selector|target selector|resolved selector)\s*[:=]\s*(["'])(?P<selector>.+?)\1""",
+ re.IGNORECASE,
+)
+_SELECTOR_UNQUOTED_RE = re.compile(
+ r"""(?:selector|css selector|target selector|resolved selector)\s*[:=]\s*(?P<selector>[#.\[\]:=\w\-/]+)""",
+ re.IGNORECASE,
+)
+_ROLE_RE = re.compile(
+ r"""get_by_role\(\s*(["'])(?P<role>.+?)\1(?:\s*,\s*name\s*=\s*(["'])(?P<name>.*?)\3)?""",
+ re.IGNORECASE,
+)
+_EXCEPTION_CLASS_RE = re.compile(r"""\b(?P<class>[A-Za-z_][\w.]*?(?:Error|Exception))\s*:""")
+# ``browser_context.mode=none`` is the stable enum-backed value surfaced by
+# local browser-state failures, unlike surrounding natural-language prose.
+_BROWSER_SESSION_ERROR_RE = re.compile(
+ r"\b(browser session not found|no browser context|no active browser|browser_context\.mode=none|"
+ r"no_active_browser|browser_not_found)\b",
+ re.IGNORECASE,
+)
+_TIMEOUT_ERROR_RE = re.compile(r"\b(timeout(?:error)?|timed out)\b", re.IGNORECASE)
+_CODE_BLOCK_IMPOSITION_DROPPED_RE = re.compile(
+ r"unable to impose synthesized code block:\s*dropped scout interaction\s+\d+\s+"
+ r"from\s+`?(?P<tool>[A-Za-z0-9_]+)`?\s+\((?P<reason>[A-Za-z0-9_]+)\)",
+ re.IGNORECASE,
+)
+_CODE_BLOCK_IMPOSITION_RE = re.compile(
+ r"unable to impose synthesized code block:\s*(?P<reason>[^\n.]+)",
+ re.IGNORECASE,
+)
+
+
+@dataclass(frozen=True)
+class RepairRootCauseIdentity:
+ root_cause_signature: str | None = None
+ primary_category: str = ""
+ failure_categories: tuple[str, ...] = ()
+ error_class: str = ""
+ selector_kind: str = ""
+ selector: str = ""
+
def normalize_failure_reason(raw: str | None) -> str:
if not raw:
@@ -52,6 +115,138 @@ def normalize_failure_reason(raw: str | None) -> str:
return collapsed.lower()
+def _category_name(value: Mapping[str, Any] | str) -> str:
+ raw = value.get("category") if isinstance(value, Mapping) else value
+ return str(raw or "").strip().upper()
+
+
+def _normalized_root_cause_categories(
+ failure_categories: Sequence[Mapping[str, Any] | str] | None,
+ detected_challenge: bool,
+) -> tuple[str, ...]:
+ categories: set[str] = set()
+ for entry in failure_categories or ():
+ category = _category_name(entry)
+ if not category:
+ continue
+ categories.add(_ROOT_CAUSE_CATEGORY_ALIASES.get(category, category))
+ if detected_challenge:
+ categories.add(ANTI_BOT_CHALLENGE_ROOT_CAUSE_CATEGORY)
+ return tuple(sorted(categories))
+
+
+def _normalize_signature_text(value: str) -> str:
+ return " ".join(value.strip().split()).rstrip(".,;")
+
+
+def _selector_from_text(text: str) -> tuple[str, str]:
+ for pattern, kind in (
+ (_LOCATOR_RE, "locator"),
+ (_SELECTOR_QUOTED_RE, "selector"),
+ (_SELECTOR_UNQUOTED_RE, "selector"),
+ ):
+ match = pattern.search(text)
+ if match:
+ return kind, _normalize_signature_text(match.group("selector"))
+ role_match = _ROLE_RE.search(text)
+ if role_match:
+ role = _normalize_signature_text(role_match.group("role"))
+ name = _normalize_signature_text(role_match.group("name") or "")
+ return "role", f"{role}:{name}" if name else role
+ return "", ""
+
+
+def _error_class_from_text(text: str) -> str:
+ code_block_imposition = _code_block_imposition_error_class(text)
+ if code_block_imposition:
+ return code_block_imposition
+ if _BROWSER_SESSION_ERROR_RE.search(text):
+ return "browser_session_not_found"
+ match = _EXCEPTION_CLASS_RE.search(text)
+ if match:
+ return re.sub(r"(?<!^)(?=[A-Z])", "_", match.group("class").split(".")[-1]).lower()
+ if _TIMEOUT_ERROR_RE.search(text):
+ return "timeout_error"
+ return ""
+
+
+def _code_block_imposition_error_class(text: str) -> str:
+ dropped_match = _CODE_BLOCK_IMPOSITION_DROPPED_RE.search(text)
+ if dropped_match:
+ return f"code_block_synthesis_{_snake_token(dropped_match.group('reason'))}"
+ match = _CODE_BLOCK_IMPOSITION_RE.search(text)
+ if match:
+ return f"code_block_synthesis_{_snake_token(match.group('reason'))}"
+ return ""
+
+
+def _snake_token(value: str) -> str:
+ normalized = re.sub(r"[^a-zA-Z0-9]+", "_", value.strip().lower()).strip("_")
+ return normalized or "unknown"
+
+
+def _root_cause_texts(
+ failure_reason: str | None,
+ error_texts: Sequence[str | None] | None,
+ blocks: Sequence[Mapping[str, Any]] | None,
+) -> list[str]:
+ texts = [text for text in ([failure_reason] + list(error_texts or ())) if isinstance(text, str) and text.strip()]
+ for block in blocks or ():
+ for key in ("failure_reason", "error", "message"):
+ value = block.get(key)
+ if isinstance(value, str) and value.strip():
+ texts.append(value)
+ return texts
+
+
+def compute_repair_root_cause_signature(
+ *,
+ failure_categories: Sequence[Mapping[str, Any] | str] | None = None,
+ failure_reason: str | None = None,
+ error_texts: Sequence[str | None] | None = None,
+ blocks: Sequence[Mapping[str, Any]] | None = None,
+ detected_challenge: bool = False,
+) -> RepairRootCauseIdentity:
+ categories = _normalized_root_cause_categories(failure_categories, detected_challenge)
+ primary_category = categories[0] if categories else ""
+ texts = _root_cause_texts(failure_reason, error_texts, blocks)
+ error_class = ""
+ selector_kind = ""
+ selector = ""
+ for text in texts:
+ if not error_class:
+ error_class = _error_class_from_text(text)
+ if not selector:
+ selector_kind, selector = _selector_from_text(text)
+ if error_class and selector:
+ break
+
+ if set(categories) & _CATEGORY_ONLY_ROOT_CAUSES:
+ error_class = ""
+ selector_kind = ""
+ selector = ""
+ if not categories and not error_class and not selector:
+ return RepairRootCauseIdentity()
+
+ payload = {
+ "version": _ROOT_CAUSE_SIGNATURE_VERSION,
+ "failure_categories": categories,
+ "error_class": error_class,
+ "selector_kind": selector_kind,
+ "selector": selector,
+ }
+ serialized = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+ signature = hashlib.sha256(serialized.encode("utf-8")).hexdigest()
+ return RepairRootCauseIdentity(
+ root_cause_signature=signature,
+ primary_category=primary_category,
+ failure_categories=categories,
+ error_class=error_class,
+ selector_kind=selector_kind,
+ selector=selector,
+ )
+
+
def _top_failure_category(failure_categories: list[dict] | None) -> str:
if not failure_categories:
return ""
@@ -65,20 +260,29 @@ def _failure_identity_terms(
failure_reason: str | None,
failure_categories: list[dict] | None,
suspicious_success: bool,
+ detected_challenge: bool = False,
) -> list[str] | None:
"""Category-stable failure half shared by both signatures, or ``None`` on a
real success. Per-call-data categories collapse to a constant so consecutive
trips hash identically."""
normalized = normalize_failure_reason(failure_reason)
- has_signal = bool(normalized) or suspicious_success
+ has_signal = bool(normalized) or bool(failure_categories) or suspicious_success or detected_challenge
if not has_signal:
return None
+ structural_identity = compute_repair_root_cause_signature(
+ failure_categories=failure_categories,
+ failure_reason=failure_reason,
+ detected_challenge=detected_challenge,
+ )
+ terminal_state = "suspicious" if suspicious_success else "failed"
+ if structural_identity.root_cause_signature:
+ return [structural_identity.root_cause_signature]
top_category = _top_failure_category(failure_categories)
if top_category == "PARAMETER_BINDING_ERROR":
normalized = "parameter_binding_error"
elif top_category == PER_TOOL_BUDGET_FAILURE_CATEGORY:
normalized = "per_tool_budget"
- return [normalized, top_category, "suspicious" if suspicious_success else "failed"]
+ return [normalized, top_category, terminal_state]
def compute_failure_signature(
@@ -86,18 +290,22 @@ def compute_failure_signature(
failure_reason: str | None,
failure_categories: list[dict] | None,
suspicious_success: bool,
+ detected_challenge: bool = False,
) -> str | None:
"""Return a normalized signature for the current failure, or ``None`` on success.
``None`` means "no signature — this was a real success". A suspicious-success
run (status=completed but data-producing blocks produced no output) still
generates a signature so repeated no-data runs can be counted as repeats.
+
+ ``frontier_start_label`` is kept for call-site compatibility. Labels are
+ intentionally excluded from the signature because block renames are not a
+ new root cause.
"""
- terms = _failure_identity_terms(failure_reason, failure_categories, suspicious_success)
+ terms = _failure_identity_terms(failure_reason, failure_categories, suspicious_success, detected_challenge)
if terms is None:
return None
- safe_label = frontier_start_label if isinstance(frontier_start_label, str) else ""
- return "|".join([safe_label, *terms])
+ return "|".join(terms)
def satisfied_criterion_ids(result: CompletionVerificationResult | None) -> frozenset[str]:
@@ -219,6 +427,7 @@ def update_repeated_failure_state(
suspicious_success_raw = getattr(ctx, "last_test_suspicious_success", False)
suspicious_success = bool(suspicious_success_raw) if isinstance(suspicious_success_raw, (bool, int)) else False
+ detected_challenge = bool(getattr(ctx, "last_test_anti_bot", None))
failure_reason_raw = getattr(ctx, "last_test_failure_reason", None)
failure_reason = failure_reason_raw if isinstance(failure_reason_raw, str) else None
frontier_start_raw = getattr(ctx, "last_frontier_start_label", None)
@@ -258,6 +467,7 @@ def update_repeated_failure_state(
failure_reason=failure_reason,
failure_categories=failure_categories,
suspicious_success=suspicious_success,
+ detected_challenge=detected_challenge,
)
fingerprint = compute_frontier_fingerprint(executed_labels, workflow_definition)
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 5ea21b90c..d66d409f9 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -352,6 +352,12 @@ async def update_workflow_tool(
)
_record_workflow_update_result(copilot_ctx, result, prior_definition)
record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
+ if result.get("ok") is False:
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_workflow",
+ result=result,
+ )
sanitized = sanitize_tool_result_for_llm("update_workflow", result)
return json.dumps(sanitized)
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 4ca20edb2..1f3e24e85 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -20,6 +20,7 @@ from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocks
from skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handler
from skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification
from skyvern.forge.sdk.copilot.output_utils import iter_failure_reasons
+from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_OUTPUT_KEYS
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -185,6 +186,46 @@ async def _maybe_run_completion_verification_from_page_observation(
_COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS = 5.0
+_DOWNLOAD_EVIDENCE_FILE_NAME_KEYS = ("filename", "file_name", "name", "path")
+_MAX_EVIDENCE_FILE_NAMES = 5
+
+
+def _download_file_name(value: Any) -> str | None:
+ if isinstance(value, dict):
+ for key in _DOWNLOAD_EVIDENCE_FILE_NAME_KEYS:
+ if name := _download_file_name(value.get(key)):
+ return name
+ return None
+ if not isinstance(value, str) or not value.strip():
+ return None
+ name = value.split("?", 1)[0].rstrip("/").rsplit("/", 1)[-1].strip()
+ return name or None
+
+
+def _completion_evidence_payload(output: Any) -> Any:
+ if not isinstance(output, dict) or not any(output.get(key) for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS):
+ return output
+ names: list[str] = []
+ if name := _download_file_name(output.get("downloaded_file_name")):
+ names.append(name)
+ files = output.get("downloaded_files")
+ if isinstance(files, list):
+ names.extend(name for item in files[:_MAX_EVIDENCE_FILE_NAMES] if (name := _download_file_name(item)))
+ urls = output.get("downloaded_file_urls")
+ if isinstance(urls, list):
+ names.extend(name for item in urls[:_MAX_EVIDENCE_FILE_NAMES] if (name := _download_file_name(item)))
+ payload: dict[str, Any] = {"download_registered": True}
+ if isinstance(files, list):
+ payload["downloaded_file_count"] = len(files)
+ if isinstance(urls, list):
+ payload["downloaded_file_url_count"] = len(urls)
+ artifacts = output.get("downloaded_file_artifact_ids")
+ if isinstance(artifacts, list):
+ payload["downloaded_file_artifact_count"] = len(artifacts)
+ if names:
+ payload["downloaded_file_names"] = list(dict.fromkeys(names))[:_MAX_EVIDENCE_FILE_NAMES]
+ return payload
+
_ARTIFACT_HEALTH_EXCLUDED_CATEGORIES = frozenset(
{
@@ -329,8 +370,9 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
continue
label = block.get("label")
output = block.get("extracted_data")
- if isinstance(label, str) and label in current_labels and _is_meaningful_extracted_data(output):
- block_outputs[label] = output
+ evidence_output = _completion_evidence_payload(output)
+ if isinstance(label, str) and label in current_labels and _is_meaningful_extracted_data(evidence_output):
+ block_outputs[label] = evidence_output
executed = data.get("executed_block_labels")
executed_block_labels = [str(label) for label in executed] if isinstance(executed, list) else []
page_title = data.get("page_title")
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index ce37bea81..1b3938479 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -1562,6 +1562,47 @@ def test_snapshot_uses_current_run_blocks_not_stale_outputs() -> None:
assert snap2.block_outputs.get("b3") == {"extracted_information": {"price": "9.99"}}
+def test_snapshot_summarizes_registered_download_outputs() -> None:
+ ctx = _run_ctx()
+ ctx.last_workflow = SimpleNamespace(
+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(label="download_statement")])
+ )
+ run = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_download",
+ "blocks": [
+ {
+ "label": "download_statement",
+ "extracted_data": {
+ "page": "<RecordingLocator>",
+ "download": "<Download>",
+ "downloaded_file_name": "apexbiz_100245_2026-05.pdf",
+ "downloaded_files": [{"filename": "apexbiz_100245_2026-05.pdf"}],
+ "downloaded_file_urls": [
+ "https://local.test/downloads/apexbiz_100245_2026-05.pdf?token=secret"
+ ],
+ "downloaded_file_artifact_ids": ["artifact_1"],
+ },
+ }
+ ],
+ },
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, run)
+ rendered = snapshot.render_prompt_block()
+
+ assert snapshot.block_outputs["download_statement"] == {
+ "download_registered": True,
+ "downloaded_file_count": 1,
+ "downloaded_file_url_count": 1,
+ "downloaded_file_artifact_count": 1,
+ "downloaded_file_names": ["apexbiz_100245_2026-05.pdf"],
+ }
+ assert "apexbiz_100245_2026-05.pdf" in rendered
+ assert "RecordingLocator" not in rendered and "Download" not in rendered and "secret" not in rendered
+
+
def test_snapshot_includes_verified_context_labels_without_prior_outputs() -> None:
ctx = _run_ctx()
labels = [
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 6c2d41dc8..773be572c 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -1,6 +1,10 @@
from __future__ import annotations
+import json
from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.context import CopilotContext
@@ -390,3 +394,78 @@ def test_unrecoverable_browser_session_contract_stops_with_blocker() -> None:
assert contract.repair_decision.next_action == RepairNextAction.STOP
assert contract.verification_result.user_goal_satisfied is False
assert contract.verification_result.remaining_blocker == reason
+
+
+def test_contract_trace_exposes_stable_root_cause_identity() -> None:
+ def result(reason: str, status: str, label: str) -> dict[str, object]:
+ return {
+ "ok": False,
+ "error": reason,
+ "data": {
+ "workflow_run_id": f"wr_{label}",
+ "overall_status": status,
+ "failure_reason": reason,
+ "frontier_start_label": label,
+ "failure_categories": [{"category": "UNRECOVERABLE_TOOL_ERROR"}],
+ "blocks": [{"label": label, "status": status, "failure_reason": reason}],
+ },
+ }
+
+ base = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result=result('Browser session not found while waiting for locator("#submit")', "failed", "login_v1"),
+ ctx=_ctx(),
+ )
+ renamed = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result=result('No browser context while waiting for locator("#submit")', "terminated", "login_v2"),
+ ctx=_ctx(),
+ )
+
+ base_trace = base.to_trace_data()
+ renamed_trace = renamed.to_trace_data()
+ assert base_trace["root_cause_signature"] == renamed_trace["root_cause_signature"]
+ assert base_trace["root_cause_error_class"] == "browser_session_not_found"
+ assert base_trace["root_cause_selector_kind"] == "locator"
+ assert base_trace["root_cause_selector"] == "#submit"
+ assert (
+ base.model_dump()["diagnosis_result"]["root_cause_identity"]["root_cause_signature"]
+ == base_trace["root_cause_signature"]
+ )
+ assert base_trace["run_status"] != renamed_trace["run_status"]
+ assert {base.diagnosis_result.suspected_failure_type, renamed.diagnosis_result.suspected_failure_type} == {
+ DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR
+ }
+
+
+async def test_update_workflow_failure_records_diagnosis_contract(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.copilot import tools as tools_module
+
+ error = (
+ "Unable to impose synthesized code block: dropped scout interaction 0 from `click` (ambiguous_bare_selector)."
+ )
+ recorded: list[tuple[str, dict[str, object], str]] = []
+
+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_request_policy_allows_credential_deferred_draft", lambda *args: False)
+ monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *args: None)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
+ monkeypatch.setattr(tools_module, "_record_workflow_update_result", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_update_workflow", AsyncMock(return_value={"ok": False, "error": error}))
+
+ def fake_record_contract(ctx: CopilotContext, *, source_tool: str, result: dict[str, object]) -> None:
+ contract = build_diagnosis_repair_contract(source_tool=source_tool, result=result, ctx=ctx)
+ recorded.append((source_tool, result, contract.to_trace_data()["root_cause_error_class"]))
+
+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", fake_record_contract)
+
+ result = await tools_module.update_workflow_tool.on_invoke_tool(
+ SimpleNamespace(context=_ctx(), tool_name="update_workflow"),
+ json.dumps({"workflow_yaml": "title: Test\nworkflow_definition:\n parameters: []\n blocks: []\n"}),
+ )
+
+ assert json.loads(result)["ok"] is False
+ assert recorded == [
+ ("update_workflow", {"ok": False, "error": error}, "code_block_synthesis_ambiguous_bare_selector")
+ ]
diff --git a/tests/unit/test_copilot_failure_tracking.py b/tests/unit/test_copilot_failure_tracking.py
index 691395506..3fa33391a 100644
--- a/tests/unit/test_copilot_failure_tracking.py
+++ b/tests/unit/test_copilot_failure_tracking.py
@@ -8,6 +8,7 @@ import pytest
from skyvern.forge.sdk.copilot.failure_tracking import (
compute_failure_signature,
compute_frontier_fingerprint,
+ compute_repair_root_cause_signature,
update_repeated_failure_state,
)
@@ -30,6 +31,7 @@ def _make_ctx(
*,
suspicious: bool = False,
failure_reason: str | None = "Timeout on element",
+ anti_bot: str | None = None,
frontier_label: str = "extract",
executed_labels: list[str] | None = None,
workflow: Any = None,
@@ -40,6 +42,7 @@ def _make_ctx(
) -> SimpleNamespace:
return SimpleNamespace(
last_test_suspicious_success=suspicious,
+ last_test_anti_bot=anti_bot,
last_test_failure_reason=failure_reason,
last_frontier_start_label=frontier_label,
last_executed_block_labels=executed_labels or ["open", "extract"],
@@ -63,6 +66,32 @@ def test_signature_differs_on_reason_change() -> None:
assert a != b
+def test_signature_uses_root_cause_identity_for_repeat_detection() -> None:
+ categories = [{"category": "UNRECOVERABLE_TOOL_ERROR"}]
+ signatures = {
+ compute_failure_signature(
+ "login_v1", 'Browser session not found while waiting for locator("#submit")', categories, False
+ ),
+ compute_failure_signature(
+ "login_v2", 'No browser context while waiting for locator("#submit")', categories, False
+ ),
+ compute_failure_signature(
+ "login_v3", 'No browser context while waiting for locator("#submit")', categories, True
+ ),
+ compute_failure_signature(
+ "login_v4", 'browser_context.mode=none while waiting for locator("#submit")', categories, True
+ ),
+ }
+ parameter_signature = compute_failure_signature("block_a", None, [{"category": "PARAMETER_BINDING_ERROR"}], False)
+
+ assert len(signatures) == 1
+ assert next(iter(signatures)) is not None
+ assert parameter_signature is not None
+ assert parameter_signature == compute_failure_signature(
+ "renamed_block", None, [{"category": "PARAMETER_BINDING_ERROR"}], False
+ )
+
+
def test_signature_collapses_parameter_binding_error() -> None:
# PARAMETER_BINDING_ERROR embeds offending key names in failure_reason;
# different keys must still hash to the same signature so repeats count.
@@ -75,6 +104,33 @@ def test_signature_collapses_parameter_binding_error() -> None:
assert a == b
+def test_category_only_and_code_imposition_root_causes_collapse_noise() -> None:
+ category_detected = compute_repair_root_cause_signature(
+ failure_categories=[{"category": "ANTI_BOT_DETECTION"}],
+ failure_reason="Verify you are human before continuing.",
+ )
+ context_detected = compute_repair_root_cause_signature(
+ failure_categories=[],
+ failure_reason="Access page showed different challenge copy this time.",
+ detected_challenge=True,
+ )
+ first = compute_repair_root_cause_signature(
+ error_texts=[
+ "Unable to impose synthesized code block: dropped scout interaction 0 from `click` (ambiguous_bare_selector)."
+ ],
+ )
+ retried = compute_repair_root_cause_signature(
+ error_texts=[
+ "Unable to impose synthesized code block: dropped scout interaction 4 from `click` (ambiguous_bare_selector)."
+ ],
+ )
+
+ assert category_detected.root_cause_signature == context_detected.root_cause_signature
+ assert category_detected.primary_category == "ANTI_BOT_CHALLENGE"
+ assert first.root_cause_signature == retried.root_cause_signature
+ assert first.error_class == "code_block_synthesis_ambiguous_bare_selector"
+
+
def test_fingerprint_empty_without_workflow_definition() -> None:
assert compute_frontier_fingerprint(["open"], None) == ""
@@ -106,6 +162,33 @@ def test_streak_increments_when_signature_and_fingerprint_repeat() -> None:
assert ctx.last_frontier_fingerprint == fingerprint
+def test_streak_increments_across_challenge_prose_and_frontier_drift() -> None:
+ wf = _make_workflow([_Block("open", url="x"), _Block("extract", schema="s")])
+ fingerprint = compute_frontier_fingerprint(["open", "extract"], wf.workflow_definition)
+ signature = compute_failure_signature(
+ "extract_v1",
+ "Run completed, but extracted data reported a blocker: Verify you are human",
+ None,
+ True,
+ detected_challenge=True,
+ )
+
+ ctx = _make_ctx(
+ suspicious=True,
+ anti_bot="Extracted data reported anti-bot blocker: Verify you are human",
+ failure_reason="Run completed, but extracted data reported a blocker: Cloudflare challenge still present",
+ frontier_label="extract_v2",
+ workflow=wf,
+ last_signature=signature,
+ last_fingerprint=fingerprint,
+ streak=5,
+ )
+ update_repeated_failure_state(ctx, {"ok": True, "data": {}})
+
+ assert ctx.repeated_failure_streak_count == 6
+ assert ctx.last_failure_signature == signature
+
+
def test_streak_resets_on_meaningful_success() -> None:
wf = _make_workflow([_Block("open", url="x")])
ctx = _make_ctx(
@@ -259,6 +342,7 @@ def test_action_fingerprint_streak_independent_of_frontier_streak() -> None:
ctx = SimpleNamespace(
last_test_suspicious_success=False,
+ last_test_anti_bot=None,
last_test_failure_reason="Validation failed: name is required",
last_frontier_start_label="open",
last_executed_block_labels=["open"],
diff --git a/tests/unit/test_copilot_parameter_binding.py b/tests/unit/test_copilot_parameter_binding.py
index dc1cf6942..7ce4b0cb1 100644
--- a/tests/unit/test_copilot_parameter_binding.py
+++ b/tests/unit/test_copilot_parameter_binding.py
@@ -259,16 +259,16 @@ def test_signature_parameter_binding_ignores_key_name() -> None:
assert sig_a == sig_b
-def test_signature_non_parameter_binding_preserves_text() -> None:
+def test_signature_non_category_only_failure_preserves_selector_shape() -> None:
sig_a = compute_failure_signature(
frontier_start_label="extract",
- failure_reason="Element not found for selector #foo",
+ failure_reason="Element not found for selector: #foo",
failure_categories=[{"category": "ELEMENT_NOT_FOUND", "confidence_float": 0.8}],
suspicious_success=False,
)
sig_b = compute_failure_signature(
frontier_start_label="extract",
- failure_reason="Element not found for selector #bar",
+ failure_reason="Element not found for selector: #bar",
failure_categories=[{"category": "ELEMENT_NOT_FOUND", "confidence_float": 0.8}],
suspicious_success=False,
)
diff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.py
index 1221cdd3f..c8837fb3e 100644
--- a/tests/unit/test_copilot_per_tool_budget.py
+++ b/tests/unit/test_copilot_per_tool_budget.py
@@ -394,9 +394,8 @@ def test_failure_signature_is_stable_across_budget_trips_with_different_run_ids(
assert sig_a == sig_b
-def test_failure_signature_changes_when_frontier_label_differs() -> None:
- """A budget trip on a different frontier should still produce a different
- signature so the streak resets when the agent meaningfully changes shape."""
+def test_failure_signature_ignores_frontier_label_drift() -> None:
+ """Block labels are UI/editing affordances, not root-cause identity."""
failure_categories = [{"category": PER_TOOL_BUDGET_FAILURE_CATEGORY, "confidence_float": 1.0}]
sig_a = compute_failure_signature(
frontier_start_label="block_a",
@@ -410,7 +409,8 @@ def test_failure_signature_changes_when_frontier_label_differs() -> None:
failure_categories=failure_categories,
suspicious_success=False,
)
- assert sig_a != sig_b
+ assert sig_a is not None
+ assert sig_a == sig_b
def test_nudge_text_advises_splitting_chain() -> None: