Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-9980 Add diagnosis repair contract shadow logging (#6070)
Andrew Neilson42ccf45e87fe
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 7919d1b49..1f6bdb626 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -15,6 +15,7 @@ ResponseType = Literal["REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"]COPILOT_RESPONSE_TYPES: tuple[ResponseType, ...] = get_args(ResponseType)if TYPE_CHECKING:+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContractfrom skyvern.forge.sdk.copilot.narration import NarratorStatefrom skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.copilot.turn_context import TurnContextPacket@@ -171,6 +172,7 @@ class CopilotContext(AgentContext):request_policy: RequestPolicy | None = Noneturn_intent: TurnIntent | None = Noneturn_context_packet: TurnContextPacket | None = None+ latest_diagnosis_repair_contract: DiagnosisRepairContract | None = None# Tool trackingconsecutive_tool_tracker: list[str] = field(default_factory=list)diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pynew file mode 100644index 000000000..636b62d9b--- /dev/null+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -0,0 +1,382 @@+from __future__ import annotations++import re+from enum import StrEnum+from typing import Any++from pydantic import BaseModel, ConfigDict, Field++from skyvern.forge.sdk.copilot.output_policy import url_origin+from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt++_TEXT_MAX = 240+_SUMMARY_MAX = 180+_MAX_ITEMS = 20+_FAILED_STATUSES = {"failed", "terminated", "canceled", "timed_out"}+_URL_CANDIDATE_RE = re.compile(r"https?://[^\s)>,]+")+++class StrictModel(BaseModel):+ model_config = ConfigDict(extra="forbid")+++class DiagnosisFailureType(StrEnum):+ NO_FAILURE = "no_failure"+ FAILED_RUN = "failed_run"+ SUSPICIOUS_SUCCESS = "suspicious_success"+ MISSING_CREDENTIAL_OR_INIT = "missing_credential_or_init"+ REPAIRABLE_BLOCK_FAILURE = "repairable_block_failure"+ UNKNOWN = "unknown"+++class RepairNextAction(StrEnum):+ REPAIR = "repair"+ ASK = "ask"+ STOP = "stop"+ ESCALATE = "escalate"+ NO_CHANGE = "no_change"+++class DiagnosisInput(StrictModel):+ user_goal: str = ""+ turn_intent: dict[str, Any] = Field(default_factory=dict)+ source_tool: str+ workflow_updated: bool = False+ workflow_run_id: str | None = None+ run_status: str | None = None+ requested_block_labels: list[str] = Field(default_factory=list)+ executed_block_labels: list[str] = Field(default_factory=list)+ frontier_start_label: str | None = None+ failed_block_labels: list[str] = Field(default_factory=list)+ failure_categories: list[str] = Field(default_factory=list)+ browser_page_state: dict[str, Any] = Field(default_factory=dict)+ prior_repair_attempts: dict[str, int] = Field(default_factory=dict)+++class DiagnosisResult(StrictModel):+ suspected_failure_type: DiagnosisFailureType = DiagnosisFailureType.UNKNOWN+ root_cause_summary: str = ""+ confidence: float = Field(default=0.0, ge=0.0, le=1.0)+ evidence_references: list[str] = Field(default_factory=list)+ missing_context: list[str] = Field(default_factory=list)+++class RepairDecision(StrictModel):+ next_action: RepairNextAction = RepairNextAction.NO_CHANGE+ target_blocks: list[str] = Field(default_factory=list)+ proposed_change_summary: str = ""+ required_authority: list[str] = Field(default_factory=list)+ completion_check: str = ""+++class VerificationResult(StrictModel):+ run_status: str | None = None+ user_goal_satisfied: bool | None = None+ completion_contract_satisfied: bool | None = None+ remaining_blocker: str | None = None+++class DiagnosisRepairContract(StrictModel):+ diagnosis_input: DiagnosisInput+ diagnosis_result: DiagnosisResult+ repair_decision: RepairDecision+ verification_result: VerificationResult++ def to_trace_data(self) -> dict[str, Any]:+ return {+ "failure_type": self.diagnosis_result.suspected_failure_type.value,+ "next_action": self.repair_decision.next_action.value,+ "confidence": self.diagnosis_result.confidence,+ "source_tool": self.diagnosis_input.source_tool,+ "workflow_updated": self.diagnosis_input.workflow_updated,+ "run_status": self.verification_result.run_status,+ "failed_block_count": len(self.diagnosis_input.failed_block_labels),+ "failure_categories": list(self.diagnosis_input.failure_categories),+ "target_block_count": len(self.repair_decision.target_blocks),+ "missing_context": list(self.diagnosis_result.missing_context),+ "user_goal_satisfied": self.verification_result.user_goal_satisfied,+ "completion_contract_satisfied": self.verification_result.completion_contract_satisfied,+ }+++def build_diagnosis_repair_contract(+ *,+ source_tool: str,+ result: dict[str, Any],+ ctx: Any,+ workflow_updated: bool = False,+) -> DiagnosisRepairContract:+ data = _dict(result.get("data")) if isinstance(result, dict) else {}+ raw_blocks = data.get("blocks")+ blocks: list[Any] = raw_blocks[:_MAX_ITEMS] if isinstance(raw_blocks, list) else []++ run_ok = bool(result.get("ok", False))+ suspicious = run_ok and bool(getattr(ctx, "last_test_suspicious_success", False))+ failed_blocks = _failed_block_labels(blocks)+ categories = _failure_categories(data)+ run_status = _safe_str(data.get("overall_status"))+ workflow_run_id = _safe_str(data.get("workflow_run_id"))+ summary = _failure_summary(result, data, blocks)+ failure_type = _failure_type(run_ok, suspicious, failed_blocks, categories, result, data)+ next_action = _next_action(failure_type, ctx, data)+ frontier = _safe_str(data.get("frontier_start_label"))+ target_blocks = failed_blocks or ([frontier] if frontier else []) if next_action == RepairNextAction.REPAIR else []+ user_goal_satisfied, completion_contract_satisfied = _verification_satisfaction(run_ok, suspicious, run_status)+ confidence = (+ 0.9+ if failure_type == DiagnosisFailureType.NO_FAILURE+ else 0.85+ if categories+ else 0.75+ if failed_blocks+ else 0.65+ if run_status+ else 0.2+ if failure_type == DiagnosisFailureType.UNKNOWN+ else 0.55+ )+ decision_summary = {+ RepairNextAction.NO_CHANGE: "No repair needed.",+ RepairNextAction.ASK: "Ask the user for the missing authority or context before changing the workflow.",+ RepairNextAction.STOP: "Stop retrying the current failure and report the blocker.",+ RepairNextAction.ESCALATE: "Escalate because the current evidence is insufficient for an autonomous repair.",+ }.get(next_action, _safe_text(f"Repair the workflow based on: {summary}", _SUMMARY_MAX))+ if next_action == RepairNextAction.REPAIR and failure_type == DiagnosisFailureType.SUSPICIOUS_SUCCESS:+ decision_summary = "Repair the data-producing block so completion is proven by meaningful output."+ completion_check = {+ RepairNextAction.NO_CHANGE: "Current run already satisfies the goal.",+ RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",+ RepairNextAction.STOP: "Do not rerun unchanged; user-visible blocker must be resolved first.",+ }.get(+ next_action,+ f"Run repaired block labels and confirm success: {', '.join(target_blocks)}"+ if target_blocks+ else "Run the repaired workflow path and confirm the requested goal is satisfied.",+ )+ required_authority: list[str] = []+ if next_action == RepairNextAction.ASK:+ required_authority = ["may_answer_without_mutation"]+ elif next_action == RepairNextAction.REPAIR:+ authority = getattr(getattr(ctx, "turn_intent", None), "authority", None)+ required_authority = ["may_update_workflow"] + (+ ["may_run_blocks"] if getattr(authority, "may_run_blocks", True) else []+ )++ return DiagnosisRepairContract(+ diagnosis_input=DiagnosisInput(+ user_goal=_safe_text(_ctx_user_goal(ctx)),+ turn_intent=_turn_intent_summary(ctx),+ source_tool=source_tool,+ workflow_updated=workflow_updated,+ workflow_run_id=workflow_run_id,+ run_status=run_status,+ requested_block_labels=_str_list(data.get("requested_block_labels")),+ executed_block_labels=_str_list(data.get("executed_block_labels")),+ frontier_start_label=_safe_str(data.get("frontier_start_label")),+ failed_block_labels=failed_blocks,+ failure_categories=categories,+ browser_page_state=_browser_page_state(data),+ prior_repair_attempts=_prior_repair_attempts(ctx),+ ),+ diagnosis_result=DiagnosisResult(+ suspected_failure_type=failure_type,+ root_cause_summary=summary,+ confidence=confidence,+ evidence_references=(+ ([f"workflow_run:{workflow_run_id}"] if workflow_run_id else [])+ + [f"failed_block:{label}" for label in failed_blocks[:_MAX_ITEMS]]+ + [f"failure_category:{category}" for category in categories[:_MAX_ITEMS]]+ ),+ missing_context=_missing_context(result, data, failure_type),+ ),+ repair_decision=RepairDecision(+ next_action=next_action,+ target_blocks=target_blocks,+ proposed_change_summary=decision_summary,+ required_authority=required_authority,+ completion_check=completion_check,+ ),+ verification_result=VerificationResult(+ run_status=run_status,+ user_goal_satisfied=user_goal_satisfied,+ completion_contract_satisfied=completion_contract_satisfied,+ remaining_blocker=None if run_ok and not suspicious else summary or "Run did not pass.",+ ),+ )+++def _dict(value: Any) -> dict[str, Any]:+ return value if isinstance(value, dict) else {}+++def _safe_str(value: Any) -> str | None:+ text = "" if value is None else str(value).strip()+ return text or None+++def _safe_text(value: str | None, max_chars: int = _TEXT_MAX) -> str:+ text = redact_raw_secrets_for_prompt((value or "").strip())+ text = _URL_CANDIDATE_RE.sub(lambda m: url_origin(m.group(0)) or "[URL]", text)+ return text if len(text) <= max_chars else text[: max_chars - 3].rstrip() + "..."+++def _str_list(value: Any) -> list[str]:+ return (+ [_safe_text(str(item), 80) for item in value[:_MAX_ITEMS] if str(item).strip()]+ if isinstance(value, list)+ else []+ )+++def _ctx_user_goal(ctx: Any) -> str:+ intent = getattr(ctx, "turn_intent", None)+ goal = getattr(intent, "user_goal", None)+ if isinstance(goal, str) and goal.strip():+ return goal+ user_message = getattr(ctx, "user_message", None)+ return user_message if isinstance(user_message, str) else ""+++def _turn_intent_summary(ctx: Any) -> dict[str, Any]:+ to_trace_data = getattr(getattr(ctx, "turn_intent", None), "to_trace_data", None)+ if not callable(to_trace_data):+ return {}+ try:+ return dict(to_trace_data())+ except Exception:+ return {}+++def _failure_categories(data: dict[str, Any]) -> list[str]:+ raw = data.get("failure_categories")+ if not isinstance(raw, list):+ return []+ return list(+ dict.fromkeys(+ category+ for entry in raw[:_MAX_ITEMS]+ if isinstance(entry, dict)+ for category in [_safe_str(entry.get("category"))]+ if category+ )+ )+++def _failed_block_labels(blocks: list[Any]) -> list[str]:+ return list(+ dict.fromkeys(+ label+ for block in blocks+ if isinstance(block, dict) and str(block.get("status") or "").lower() in _FAILED_STATUSES+ for label in [_safe_str(block.get("label"))]+ if label+ )+ )+++def _failure_summary(result: dict[str, Any], data: dict[str, Any], blocks: list[Any]) -> str:+ candidates = [result.get("error"), data.get("failure_reason")]+ candidates += [block.get("failure_reason") for block in blocks if isinstance(block, dict)]+ for candidate in candidates:+ if isinstance(candidate, str) and candidate.strip():+ return _safe_text(candidate, _SUMMARY_MAX)+ return "Run completed." if bool(result.get("ok", False)) else "No failure reason was provided."+++def _failure_type(+ run_ok: bool,+ suspicious: bool,+ failed_blocks: list[str],+ categories: list[str],+ result: dict[str, Any],+ data: dict[str, Any],+) -> DiagnosisFailureType:+ if run_ok:+ return DiagnosisFailureType.SUSPICIOUS_SUCCESS if suspicious else DiagnosisFailureType.NO_FAILURE+ error_text = " ".join(+ str(value).lower()+ for value in (result.get("error"), data.get("failure_reason"), data.get("skip_reason"))+ if value+ )+ if (+ data.get("skip_reason") == "workflow_credential_inputs_unbound"+ or "credential" in error_text+ or "organization not found" in error_text+ or "workflow not found" in error_text+ or "browser session" in error_text+ or "PARAMETER_BINDING_ERROR" in categories+ ):+ return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ if failed_blocks:+ return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE+ return DiagnosisFailureType.FAILED_RUN if result.get("ok") is False else DiagnosisFailureType.UNKNOWN+++def _next_action(failure_type: DiagnosisFailureType, ctx: Any, data: dict[str, Any]) -> RepairNextAction:+ if failure_type == DiagnosisFailureType.NO_FAILURE:+ return RepairNextAction.NO_CHANGE+ if (+ data.get("skip_reason") == "workflow_credential_inputs_unbound"+ or failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ ):+ return RepairNextAction.ASK+ if getattr(ctx, "last_test_non_retriable_nav_error", None):+ return RepairNextAction.STOP+ authority = getattr(getattr(ctx, "turn_intent", None), "authority", None)+ if getattr(authority, "requires_user_input", False) or getattr(authority, "may_update_workflow", True) is False:+ return RepairNextAction.ASK+ if failure_type in {+ DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE,+ DiagnosisFailureType.SUSPICIOUS_SUCCESS,+ DiagnosisFailureType.FAILED_RUN,+ }:+ return RepairNextAction.REPAIR+ return RepairNextAction.ESCALATE+++def _verification_satisfaction(+ run_ok: bool,+ suspicious: bool,+ run_status: str | None,+) -> tuple[bool | None, bool | None]:+ user_goal_satisfied = (not suspicious) if run_ok else None if run_status is None else False+ # Current tool evidence has one completion signal: whether the tested run+ # can be trusted. Keep these fields separate for future richer contracts,+ # but mirror them until an explicit completion-contract signal exists.+ completion_contract_satisfied = user_goal_satisfied+ return user_goal_satisfied, completion_contract_satisfied+++def _missing_context(result: dict[str, Any], data: dict[str, Any], failure_type: DiagnosisFailureType) -> list[str]:+ missing: list[str] = []+ if data.get("workflow_run_id") is None and failure_type not in {+ DiagnosisFailureType.NO_FAILURE,+ DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT,+ }:+ missing.append("workflow_run_id")+ if not data.get("blocks") and failure_type == DiagnosisFailureType.FAILED_RUN:+ missing.append("block_results")+ if not result.get("error") and not data.get("failure_reason") and failure_type != DiagnosisFailureType.NO_FAILURE:+ missing.append("failure_reason")+ return missing+++def _browser_page_state(data: dict[str, Any]) -> dict[str, Any]:+ raw_url = _safe_str(data.get("current_url"))+ return {+ "current_origin": url_origin(raw_url) if raw_url else None,+ "has_current_url": bool(raw_url),+ "has_page_title": bool(_safe_str(data.get("page_title"))),+ }+++def _prior_repair_attempts(ctx: Any) -> dict[str, int]:+ keys = (+ "repeated_failure_streak_count",+ "failed_test_nudge_count",+ "null_data_streak_count",+ "probable_site_block_streak_count",+ "per_tool_budget_nudge_count",+ "repeated_action_fingerprint_streak_count",+ )+ return {key: int(getattr(ctx, key, 0) or 0) for key in keys}diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex e9807e659..4ec22aabb 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -516,13 +516,13 @@ def _approved_origins_by_id(request_policy: RequestPolicy) -> dict[str, set[str]credential_id = getattr(credential, "credential_id", None)tested_url = getattr(credential, "tested_url", None)if isinstance(credential_id, str) and isinstance(tested_url, str):- origin = _origin(tested_url)+ origin = url_origin(tested_url)if origin:origins.setdefault(credential_id, set()).add(origin)return origins-def _origin(url: str) -> str | None:+def url_origin(url: str) -> str | None:parsed = urlparse(url if "://" in url else f"https://{url}")if not parsed.netloc:return None@@ -586,7 +586,7 @@ def _block_broadens_credential_scope(block_url = block.get("url")if not isinstance(block_url, str) or not block_url.strip():return False- origin = _origin(block_url)+ origin = url_origin(block_url)if not origin:return Truediff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 28dc8e0f2..ee788f8b4 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -26,6 +26,10 @@ from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stafrom skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goalsfrom skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_aliasfrom skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (+ DiagnosisRepairContract,+ build_diagnosis_repair_contract,+)from skyvern.forge.sdk.copilot.enforcement import (POST_INTERMEDIATE_SUCCESS_NUDGE,PROBABLE_SITE_BLOCK_STREAK_STOP_AT,@@ -3716,6 +3720,36 @@ def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:update_repeated_failure_state(copilot_ctx, result)+def _record_diagnosis_repair_contract(+ copilot_ctx: Any,+ *,+ source_tool: str,+ result: dict[str, Any],+ workflow_updated: bool = False,+) -> DiagnosisRepairContract:+ contract = build_diagnosis_repair_contract(+ source_tool=source_tool,+ result=result,+ ctx=copilot_ctx,+ workflow_updated=workflow_updated,+ )+ copilot_ctx.latest_diagnosis_repair_contract = contract+ trace_data = contract.to_trace_data()+ LOG.info(+ "copilot diagnosis repair contract shadow",+ **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},+ )+ with copilot_span("diagnosis_repair_contract", data=trace_data):+ pass+ return contract+++def _diagnosis_repair_tool_error(copilot_ctx: Any, source_tool: str, error: str) -> str:+ result = {"ok": False, "error": error}+ _record_diagnosis_repair_contract(copilot_ctx, source_tool=source_tool, result=result)+ return json.dumps(result)++@function_tool(name_override="update_workflow",tool_input_guardrails=[_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL],@@ -3852,11 +3886,11 @@ async def run_blocks_tool(arguments = {"block_labels": block_labels, "parameters": parameters or {}}policy_error = _request_policy_tool_error(copilot_ctx, "run_blocks_and_collect_debug")if policy_error:- return json.dumps({"ok": False, "error": policy_error})+ return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", policy_error)loop_error = _tool_loop_error(copilot_ctx, "run_blocks_and_collect_debug", arguments)if loop_error:- return json.dumps({"ok": False, "error": loop_error})+ return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", loop_error)labels_to_execute, block_outputs_to_seed, frontier_start_label = await _frontier_plan_for_current_workflow(copilot_ctx, block_labels@@ -3881,6 +3915,11 @@ async def run_blocks_tool()_record_run_blocks_result(copilot_ctx, result)record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, result)+ _record_diagnosis_repair_contract(+ copilot_ctx,+ source_tool="run_blocks_and_collect_debug",+ result=result,+ )enqueue_screenshot_from_result(copilot_ctx, result)sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)@@ -3974,11 +4013,11 @@ async def update_and_run_blocks_tool()policy_error = _request_policy_tool_error(copilot_ctx, "update_and_run_blocks")if not skip_run_after_update and policy_error:- return json.dumps({"ok": False, "error": policy_error})+ return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", policy_error)loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks", arguments)if loop_error:- return json.dumps({"ok": False, "error": loop_error})+ return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", loop_error)# Snapshot the prior workflow definition BEFORE _update_workflow saves# the new one — we need the pre-update state to diff against.@@ -4004,6 +4043,11 @@ async def update_and_run_blocks_tool(if not update_result.get("ok"):record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, update_result)+ _record_diagnosis_repair_contract(+ copilot_ctx,+ source_tool="update_and_run_blocks",+ result=update_result,+ )sanitized = sanitize_tool_result_for_llm("update_workflow", update_result)return json.dumps(sanitized)@@ -4019,6 +4063,12 @@ async def update_and_run_blocks_tool(},}record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)+ _record_diagnosis_repair_contract(+ copilot_ctx,+ source_tool="update_and_run_blocks",+ result=result,+ workflow_updated=True,+ )return json.dumps(result)if skip_run_after_update:@@ -4034,6 +4084,12 @@ async def update_and_run_blocks_tool(},}record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, skip_result)+ _record_diagnosis_repair_contract(+ copilot_ctx,+ source_tool="update_and_run_blocks",+ result=skip_result,+ workflow_updated=True,+ )LOG.info("update_and_run_blocks skipped run on unbound credential workflow inputs",workflow_permanent_id=copilot_ctx.workflow_permanent_id,@@ -4087,6 +4143,12 @@ async def update_and_run_blocks_tool()_record_run_blocks_result(copilot_ctx, run_result)record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, run_result)+ _record_diagnosis_repair_contract(+ copilot_ctx,+ source_tool="update_and_run_blocks",+ result=run_result,+ workflow_updated=True,+ )enqueue_screenshot_from_result(copilot_ctx, run_result)sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", run_result)diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pynew file mode 100644index 000000000..2367e3b09--- /dev/null+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -0,0 +1,176 @@+from __future__ import annotations++from types import SimpleNamespace++from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (+ DiagnosisFailureType,+ RepairNextAction,+ build_diagnosis_repair_contract,+)+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode+++def _ctx() -> CopilotContext:+ return CopilotContext(+ organization_id="o",+ workflow_id="w",+ workflow_permanent_id="wp",+ workflow_yaml="",+ browser_session_id=None,+ stream=SimpleNamespace(), # type: ignore[arg-type]+ user_message="Fix the workflow with password=hunter2",+ turn_intent=TurnIntent(+ mode=TurnIntentMode.EDIT,+ user_goal="Fix the workflow with password=hunter2",+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),+ ),+ )+++def test_contract_shapes_for_failed_suspicious_and_missing_credential_cases() -> None:+ failed = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result={+ "ok": False,+ "error": "The run ended before recording a trustworthy terminal status.",+ "data": {"workflow_run_id": "wr_1", "overall_status": "running", "failure_reason": "uncertain"},+ },+ ctx=_ctx(),+ )+ suspicious_ctx = _ctx()+ suspicious_ctx.last_test_suspicious_success = True+ suspicious = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": True,+ "data": {+ "workflow_run_id": "wr_2",+ "overall_status": "completed",+ "frontier_start_label": "extract",+ "blocks": [{"label": "extract", "block_type": "EXTRACTION", "status": "completed"}],+ },+ },+ ctx=suspicious_ctx,+ workflow_updated=True,+ )+ missing = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": False,+ "error": "Skipped test run: required credentials are not configured.",+ "data": {"workflow_updated": True, "skip_reason": "workflow_credential_inputs_unbound"},+ },+ ctx=_ctx(),+ workflow_updated=True,+ )++ assert (+ failed.diagnosis_result.suspected_failure_type,+ failed.repair_decision.next_action,+ failed.diagnosis_result.missing_context,+ ) == (DiagnosisFailureType.FAILED_RUN, RepairNextAction.REPAIR, ["block_results"])+ assert (+ suspicious.diagnosis_result.suspected_failure_type,+ suspicious.repair_decision.next_action,+ suspicious.repair_decision.target_blocks,+ suspicious.verification_result.user_goal_satisfied,+ ) == (DiagnosisFailureType.SUSPICIOUS_SUCCESS, RepairNextAction.REPAIR, ["extract"], False)+ assert (+ missing.diagnosis_result.suspected_failure_type,+ missing.repair_decision.next_action,+ missing.repair_decision.required_authority,+ ) == (DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT, RepairNextAction.ASK, ["may_answer_without_mutation"])+++def test_repairable_block_failure_contract_is_queryable_and_safe() -> None:+ contract = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result={+ "ok": False,+ "data": {+ "workflow_run_id": "wr_3",+ "overall_status": "failed",+ "requested_block_labels": ["login", "extract"],+ "executed_block_labels": ["extract"],+ "frontier_start_label": "extract",+ "current_url": "https://example.test/account?id=secret",+ "page_title": "Account page",+ "failure_categories": [{"category": "DATA_EXTRACTION_FAILURE", "reasoning": "missing fields"}],+ "blocks": [+ {"label": "extract", "block_type": "EXTRACTION", "status": "failed", "failure_reason": "No rows"}+ ],+ },+ },+ ctx=_ctx(),+ )++ trace = contract.to_trace_data()+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE+ assert contract.repair_decision.target_blocks == ["extract"]+ assert trace["failure_type"] == "repairable_block_failure"+ assert trace["next_action"] == "repair"+ assert trace["failure_categories"] == ["DATA_EXTRACTION_FAILURE"]+ assert contract.diagnosis_input.browser_page_state["current_origin"] == "https://example.test"+ assert "secret" not in contract.model_dump_json()+ assert "hunter2" not in contract.model_dump_json()+++def test_user_goal_urls_are_reduced_to_origins() -> None:+ ctx = _ctx()+ ctx.turn_intent.user_goal = "Fix https://example.test/account?id=secret now"++ contract = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result={"ok": False, "error": "failed", "data": {"overall_status": "failed"}},+ ctx=ctx,+ )++ assert contract.diagnosis_input.user_goal == "Fix https://example.test now"+ assert "id=secret" not in contract.model_dump_json()+++def test_suspicious_success_flag_does_not_override_failed_run() -> None:+ ctx = _ctx()+ ctx.last_test_suspicious_success = True++ contract = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result={+ "ok": False,+ "error": "The run failed before output validation.",+ "data": {"workflow_run_id": "wr_failed", "overall_status": "failed"},+ },+ ctx=ctx,+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.FAILED_RUN+ assert contract.repair_decision.proposed_change_summary == (+ "Repair the workflow based on: The run failed before output validation."+ )+++def test_stop_and_no_change_decisions_preserve_current_behavior_shadow_only() -> None:+ stop_ctx = _ctx()+ stop_ctx.last_test_non_retriable_nav_error = "net::ERR_NAME_NOT_RESOLVED"+ stop_contract = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result={+ "ok": False,+ "data": {+ "overall_status": "failed",+ "blocks": [{"label": "open", "status": "failed", "failure_reason": "net::ERR_NAME_NOT_RESOLVED"}],+ },+ },+ ctx=stop_ctx,+ )+ no_change = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result={"ok": True, "data": {"workflow_run_id": "wr_4", "overall_status": "completed", "blocks": []}},+ ctx=_ctx(),+ )++ assert stop_contract.repair_decision.next_action == RepairNextAction.STOP+ assert no_change.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE+ assert no_change.repair_decision.next_action == RepairNextAction.NO_CHANGE+ assert no_change.verification_result.completion_contract_satisfied is True