Skip to content

Automate browser based workflows with AI

SKY-9980 Add diagnosis repair contract shadow logging (#6070)

Andrew Neilson42ccf45e87fe
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 7919d1b49..1f6bdb626 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -15,6 +15,7 @@ ResponseType = Literal["REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"]
COPILOT_RESPONSE_TYPES: tuple[ResponseType, ...] = get_args(ResponseType)
if TYPE_CHECKING:
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContract
from skyvern.forge.sdk.copilot.narration import NarratorState
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.turn_context import TurnContextPacket
@@ -171,6 +172,7 @@ class CopilotContext(AgentContext):
request_policy: RequestPolicy | None = None
turn_intent: TurnIntent | None = None
turn_context_packet: TurnContextPacket | None = None
+ latest_diagnosis_repair_contract: DiagnosisRepairContract | None = None
# Tool tracking
consecutive_tool_tracker: list[str] = field(default_factory=list)
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
new file mode 100644
index 000000000..636b62d9b
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -0,0 +1,382 @@
+from __future__ import annotations
+
+import re
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from skyvern.forge.sdk.copilot.output_policy import url_origin
+from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
+
+_TEXT_MAX = 240
+_SUMMARY_MAX = 180
+_MAX_ITEMS = 20
+_FAILED_STATUSES = {"failed", "terminated", "canceled", "timed_out"}
+_URL_CANDIDATE_RE = re.compile(r"https?://[^\s)>,]+")
+
+
+class StrictModel(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+
+class DiagnosisFailureType(StrEnum):
+ NO_FAILURE = "no_failure"
+ FAILED_RUN = "failed_run"
+ SUSPICIOUS_SUCCESS = "suspicious_success"
+ MISSING_CREDENTIAL_OR_INIT = "missing_credential_or_init"
+ REPAIRABLE_BLOCK_FAILURE = "repairable_block_failure"
+ UNKNOWN = "unknown"
+
+
+class RepairNextAction(StrEnum):
+ REPAIR = "repair"
+ ASK = "ask"
+ STOP = "stop"
+ ESCALATE = "escalate"
+ NO_CHANGE = "no_change"
+
+
+class DiagnosisInput(StrictModel):
+ user_goal: str = ""
+ turn_intent: dict[str, Any] = Field(default_factory=dict)
+ source_tool: str
+ workflow_updated: bool = False
+ workflow_run_id: str | None = None
+ run_status: str | None = None
+ requested_block_labels: list[str] = Field(default_factory=list)
+ executed_block_labels: list[str] = Field(default_factory=list)
+ frontier_start_label: str | None = None
+ failed_block_labels: list[str] = Field(default_factory=list)
+ failure_categories: list[str] = Field(default_factory=list)
+ browser_page_state: dict[str, Any] = Field(default_factory=dict)
+ prior_repair_attempts: dict[str, int] = Field(default_factory=dict)
+
+
+class DiagnosisResult(StrictModel):
+ suspected_failure_type: DiagnosisFailureType = DiagnosisFailureType.UNKNOWN
+ root_cause_summary: str = ""
+ confidence: float = Field(default=0.0, ge=0.0, le=1.0)
+ evidence_references: list[str] = Field(default_factory=list)
+ missing_context: list[str] = Field(default_factory=list)
+
+
+class RepairDecision(StrictModel):
+ next_action: RepairNextAction = RepairNextAction.NO_CHANGE
+ target_blocks: list[str] = Field(default_factory=list)
+ proposed_change_summary: str = ""
+ required_authority: list[str] = Field(default_factory=list)
+ completion_check: str = ""
+
+
+class VerificationResult(StrictModel):
+ run_status: str | None = None
+ user_goal_satisfied: bool | None = None
+ completion_contract_satisfied: bool | None = None
+ remaining_blocker: str | None = None
+
+
+class DiagnosisRepairContract(StrictModel):
+ diagnosis_input: DiagnosisInput
+ diagnosis_result: DiagnosisResult
+ repair_decision: RepairDecision
+ verification_result: VerificationResult
+
+ def to_trace_data(self) -> dict[str, Any]:
+ return {
+ "failure_type": self.diagnosis_result.suspected_failure_type.value,
+ "next_action": self.repair_decision.next_action.value,
+ "confidence": self.diagnosis_result.confidence,
+ "source_tool": self.diagnosis_input.source_tool,
+ "workflow_updated": self.diagnosis_input.workflow_updated,
+ "run_status": self.verification_result.run_status,
+ "failed_block_count": len(self.diagnosis_input.failed_block_labels),
+ "failure_categories": list(self.diagnosis_input.failure_categories),
+ "target_block_count": len(self.repair_decision.target_blocks),
+ "missing_context": list(self.diagnosis_result.missing_context),
+ "user_goal_satisfied": self.verification_result.user_goal_satisfied,
+ "completion_contract_satisfied": self.verification_result.completion_contract_satisfied,
+ }
+
+
+def build_diagnosis_repair_contract(
+ *,
+ source_tool: str,
+ result: dict[str, Any],
+ ctx: Any,
+ workflow_updated: bool = False,
+) -> DiagnosisRepairContract:
+ data = _dict(result.get("data")) if isinstance(result, dict) else {}
+ raw_blocks = data.get("blocks")
+ blocks: list[Any] = raw_blocks[:_MAX_ITEMS] if isinstance(raw_blocks, list) else []
+
+ run_ok = bool(result.get("ok", False))
+ suspicious = run_ok and bool(getattr(ctx, "last_test_suspicious_success", False))
+ failed_blocks = _failed_block_labels(blocks)
+ categories = _failure_categories(data)
+ run_status = _safe_str(data.get("overall_status"))
+ workflow_run_id = _safe_str(data.get("workflow_run_id"))
+ summary = _failure_summary(result, data, blocks)
+ failure_type = _failure_type(run_ok, suspicious, failed_blocks, categories, result, data)
+ next_action = _next_action(failure_type, ctx, data)
+ frontier = _safe_str(data.get("frontier_start_label"))
+ target_blocks = failed_blocks or ([frontier] if frontier else []) if next_action == RepairNextAction.REPAIR else []
+ user_goal_satisfied, completion_contract_satisfied = _verification_satisfaction(run_ok, suspicious, run_status)
+ confidence = (
+ 0.9
+ if failure_type == DiagnosisFailureType.NO_FAILURE
+ else 0.85
+ if categories
+ else 0.75
+ if failed_blocks
+ else 0.65
+ if run_status
+ else 0.2
+ if failure_type == DiagnosisFailureType.UNKNOWN
+ else 0.55
+ )
+ decision_summary = {
+ RepairNextAction.NO_CHANGE: "No repair needed.",
+ RepairNextAction.ASK: "Ask the user for the missing authority or context before changing the workflow.",
+ RepairNextAction.STOP: "Stop retrying the current failure and report the blocker.",
+ RepairNextAction.ESCALATE: "Escalate because the current evidence is insufficient for an autonomous repair.",
+ }.get(next_action, _safe_text(f"Repair the workflow based on: {summary}", _SUMMARY_MAX))
+ if next_action == RepairNextAction.REPAIR and failure_type == DiagnosisFailureType.SUSPICIOUS_SUCCESS:
+ decision_summary = "Repair the data-producing block so completion is proven by meaningful output."
+ completion_check = {
+ RepairNextAction.NO_CHANGE: "Current run already satisfies the goal.",
+ RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",
+ RepairNextAction.STOP: "Do not rerun unchanged; user-visible blocker must be resolved first.",
+ }.get(
+ next_action,
+ f"Run repaired block labels and confirm success: {', '.join(target_blocks)}"
+ if target_blocks
+ else "Run the repaired workflow path and confirm the requested goal is satisfied.",
+ )
+ required_authority: list[str] = []
+ if next_action == RepairNextAction.ASK:
+ required_authority = ["may_answer_without_mutation"]
+ elif next_action == RepairNextAction.REPAIR:
+ authority = getattr(getattr(ctx, "turn_intent", None), "authority", None)
+ required_authority = ["may_update_workflow"] + (
+ ["may_run_blocks"] if getattr(authority, "may_run_blocks", True) else []
+ )
+
+ return DiagnosisRepairContract(
+ diagnosis_input=DiagnosisInput(
+ user_goal=_safe_text(_ctx_user_goal(ctx)),
+ turn_intent=_turn_intent_summary(ctx),
+ source_tool=source_tool,
+ workflow_updated=workflow_updated,
+ workflow_run_id=workflow_run_id,
+ run_status=run_status,
+ requested_block_labels=_str_list(data.get("requested_block_labels")),
+ executed_block_labels=_str_list(data.get("executed_block_labels")),
+ frontier_start_label=_safe_str(data.get("frontier_start_label")),
+ failed_block_labels=failed_blocks,
+ failure_categories=categories,
+ browser_page_state=_browser_page_state(data),
+ prior_repair_attempts=_prior_repair_attempts(ctx),
+ ),
+ diagnosis_result=DiagnosisResult(
+ suspected_failure_type=failure_type,
+ root_cause_summary=summary,
+ confidence=confidence,
+ evidence_references=(
+ ([f"workflow_run:{workflow_run_id}"] if workflow_run_id else [])
+ + [f"failed_block:{label}" for label in failed_blocks[:_MAX_ITEMS]]
+ + [f"failure_category:{category}" for category in categories[:_MAX_ITEMS]]
+ ),
+ missing_context=_missing_context(result, data, failure_type),
+ ),
+ repair_decision=RepairDecision(
+ next_action=next_action,
+ target_blocks=target_blocks,
+ proposed_change_summary=decision_summary,
+ required_authority=required_authority,
+ completion_check=completion_check,
+ ),
+ verification_result=VerificationResult(
+ run_status=run_status,
+ user_goal_satisfied=user_goal_satisfied,
+ completion_contract_satisfied=completion_contract_satisfied,
+ remaining_blocker=None if run_ok and not suspicious else summary or "Run did not pass.",
+ ),
+ )
+
+
+def _dict(value: Any) -> dict[str, Any]:
+ return value if isinstance(value, dict) else {}
+
+
+def _safe_str(value: Any) -> str | None:
+ text = "" if value is None else str(value).strip()
+ return text or None
+
+
+def _safe_text(value: str | None, max_chars: int = _TEXT_MAX) -> str:
+ text = redact_raw_secrets_for_prompt((value or "").strip())
+ text = _URL_CANDIDATE_RE.sub(lambda m: url_origin(m.group(0)) or "[URL]", text)
+ return text if len(text) <= max_chars else text[: max_chars - 3].rstrip() + "..."
+
+
+def _str_list(value: Any) -> list[str]:
+ return (
+ [_safe_text(str(item), 80) for item in value[:_MAX_ITEMS] if str(item).strip()]
+ if isinstance(value, list)
+ else []
+ )
+
+
+def _ctx_user_goal(ctx: Any) -> str:
+ intent = getattr(ctx, "turn_intent", None)
+ goal = getattr(intent, "user_goal", None)
+ if isinstance(goal, str) and goal.strip():
+ return goal
+ user_message = getattr(ctx, "user_message", None)
+ return user_message if isinstance(user_message, str) else ""
+
+
+def _turn_intent_summary(ctx: Any) -> dict[str, Any]:
+ to_trace_data = getattr(getattr(ctx, "turn_intent", None), "to_trace_data", None)
+ if not callable(to_trace_data):
+ return {}
+ try:
+ return dict(to_trace_data())
+ except Exception:
+ return {}
+
+
+def _failure_categories(data: dict[str, Any]) -> list[str]:
+ raw = data.get("failure_categories")
+ if not isinstance(raw, list):
+ return []
+ return list(
+ dict.fromkeys(
+ category
+ for entry in raw[:_MAX_ITEMS]
+ if isinstance(entry, dict)
+ for category in [_safe_str(entry.get("category"))]
+ if category
+ )
+ )
+
+
+def _failed_block_labels(blocks: list[Any]) -> list[str]:
+ return list(
+ dict.fromkeys(
+ label
+ for block in blocks
+ if isinstance(block, dict) and str(block.get("status") or "").lower() in _FAILED_STATUSES
+ for label in [_safe_str(block.get("label"))]
+ if label
+ )
+ )
+
+
+def _failure_summary(result: dict[str, Any], data: dict[str, Any], blocks: list[Any]) -> str:
+ candidates = [result.get("error"), data.get("failure_reason")]
+ candidates += [block.get("failure_reason") for block in blocks if isinstance(block, dict)]
+ for candidate in candidates:
+ if isinstance(candidate, str) and candidate.strip():
+ return _safe_text(candidate, _SUMMARY_MAX)
+ return "Run completed." if bool(result.get("ok", False)) else "No failure reason was provided."
+
+
+def _failure_type(
+ run_ok: bool,
+ suspicious: bool,
+ failed_blocks: list[str],
+ categories: list[str],
+ result: dict[str, Any],
+ data: dict[str, Any],
+) -> DiagnosisFailureType:
+ if run_ok:
+ return DiagnosisFailureType.SUSPICIOUS_SUCCESS if suspicious else DiagnosisFailureType.NO_FAILURE
+ error_text = " ".join(
+ str(value).lower()
+ for value in (result.get("error"), data.get("failure_reason"), data.get("skip_reason"))
+ if value
+ )
+ if (
+ data.get("skip_reason") == "workflow_credential_inputs_unbound"
+ or "credential" in error_text
+ or "organization not found" in error_text
+ or "workflow not found" in error_text
+ or "browser session" in error_text
+ or "PARAMETER_BINDING_ERROR" in categories
+ ):
+ return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ if failed_blocks:
+ return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
+ return DiagnosisFailureType.FAILED_RUN if result.get("ok") is False else DiagnosisFailureType.UNKNOWN
+
+
+def _next_action(failure_type: DiagnosisFailureType, ctx: Any, data: dict[str, Any]) -> RepairNextAction:
+ if failure_type == DiagnosisFailureType.NO_FAILURE:
+ return RepairNextAction.NO_CHANGE
+ if (
+ data.get("skip_reason") == "workflow_credential_inputs_unbound"
+ or failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ ):
+ return RepairNextAction.ASK
+ if getattr(ctx, "last_test_non_retriable_nav_error", None):
+ return RepairNextAction.STOP
+ authority = getattr(getattr(ctx, "turn_intent", None), "authority", None)
+ if getattr(authority, "requires_user_input", False) or getattr(authority, "may_update_workflow", True) is False:
+ return RepairNextAction.ASK
+ if failure_type in {
+ DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE,
+ DiagnosisFailureType.SUSPICIOUS_SUCCESS,
+ DiagnosisFailureType.FAILED_RUN,
+ }:
+ return RepairNextAction.REPAIR
+ return RepairNextAction.ESCALATE
+
+
+def _verification_satisfaction(
+ run_ok: bool,
+ suspicious: bool,
+ run_status: str | None,
+) -> tuple[bool | None, bool | None]:
+ user_goal_satisfied = (not suspicious) if run_ok else None if run_status is None else False
+ # Current tool evidence has one completion signal: whether the tested run
+ # can be trusted. Keep these fields separate for future richer contracts,
+ # but mirror them until an explicit completion-contract signal exists.
+ completion_contract_satisfied = user_goal_satisfied
+ return user_goal_satisfied, completion_contract_satisfied
+
+
+def _missing_context(result: dict[str, Any], data: dict[str, Any], failure_type: DiagnosisFailureType) -> list[str]:
+ missing: list[str] = []
+ if data.get("workflow_run_id") is None and failure_type not in {
+ DiagnosisFailureType.NO_FAILURE,
+ DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT,
+ }:
+ missing.append("workflow_run_id")
+ if not data.get("blocks") and failure_type == DiagnosisFailureType.FAILED_RUN:
+ missing.append("block_results")
+ if not result.get("error") and not data.get("failure_reason") and failure_type != DiagnosisFailureType.NO_FAILURE:
+ missing.append("failure_reason")
+ return missing
+
+
+def _browser_page_state(data: dict[str, Any]) -> dict[str, Any]:
+ raw_url = _safe_str(data.get("current_url"))
+ return {
+ "current_origin": url_origin(raw_url) if raw_url else None,
+ "has_current_url": bool(raw_url),
+ "has_page_title": bool(_safe_str(data.get("page_title"))),
+ }
+
+
+def _prior_repair_attempts(ctx: Any) -> dict[str, int]:
+ keys = (
+ "repeated_failure_streak_count",
+ "failed_test_nudge_count",
+ "null_data_streak_count",
+ "probable_site_block_streak_count",
+ "per_tool_budget_nudge_count",
+ "repeated_action_fingerprint_streak_count",
+ )
+ return {key: int(getattr(ctx, key, 0) or 0) for key in keys}
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index e9807e659..4ec22aabb 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -516,13 +516,13 @@ def _approved_origins_by_id(request_policy: RequestPolicy) -> dict[str, set[str]
credential_id = getattr(credential, "credential_id", None)
tested_url = getattr(credential, "tested_url", None)
if isinstance(credential_id, str) and isinstance(tested_url, str):
- origin = _origin(tested_url)
+ origin = url_origin(tested_url)
if origin:
origins.setdefault(credential_id, set()).add(origin)
return origins
-def _origin(url: str) -> str | None:
+def url_origin(url: str) -> str | None:
parsed = urlparse(url if "://" in url else f"https://{url}")
if not parsed.netloc:
return None
@@ -586,7 +586,7 @@ def _block_broadens_credential_scope(
block_url = block.get("url")
if not isinstance(block_url, str) or not block_url.strip():
return False
- origin = _origin(block_url)
+ origin = url_origin(block_url)
if not origin:
return True
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 28dc8e0f2..ee788f8b4 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -26,6 +26,10 @@ from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_sta
from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ DiagnosisRepairContract,
+ build_diagnosis_repair_contract,
+)
from skyvern.forge.sdk.copilot.enforcement import (
POST_INTERMEDIATE_SUCCESS_NUDGE,
PROBABLE_SITE_BLOCK_STREAK_STOP_AT,
@@ -3716,6 +3720,36 @@ def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:
update_repeated_failure_state(copilot_ctx, result)
+def _record_diagnosis_repair_contract(
+ copilot_ctx: Any,
+ *,
+ source_tool: str,
+ result: dict[str, Any],
+ workflow_updated: bool = False,
+) -> DiagnosisRepairContract:
+ contract = build_diagnosis_repair_contract(
+ source_tool=source_tool,
+ result=result,
+ ctx=copilot_ctx,
+ workflow_updated=workflow_updated,
+ )
+ copilot_ctx.latest_diagnosis_repair_contract = contract
+ trace_data = contract.to_trace_data()
+ LOG.info(
+ "copilot diagnosis repair contract shadow",
+ **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},
+ )
+ with copilot_span("diagnosis_repair_contract", data=trace_data):
+ pass
+ return contract
+
+
+def _diagnosis_repair_tool_error(copilot_ctx: Any, source_tool: str, error: str) -> str:
+ result = {"ok": False, "error": error}
+ _record_diagnosis_repair_contract(copilot_ctx, source_tool=source_tool, result=result)
+ return json.dumps(result)
+
+
@function_tool(
name_override="update_workflow",
tool_input_guardrails=[_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL],
@@ -3852,11 +3886,11 @@ async def run_blocks_tool(
arguments = {"block_labels": block_labels, "parameters": parameters or {}}
policy_error = _request_policy_tool_error(copilot_ctx, "run_blocks_and_collect_debug")
if policy_error:
- return json.dumps({"ok": False, "error": policy_error})
+ return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", policy_error)
loop_error = _tool_loop_error(copilot_ctx, "run_blocks_and_collect_debug", arguments)
if loop_error:
- return json.dumps({"ok": False, "error": loop_error})
+ return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", loop_error)
labels_to_execute, block_outputs_to_seed, frontier_start_label = await _frontier_plan_for_current_workflow(
copilot_ctx, block_labels
@@ -3881,6 +3915,11 @@ async def run_blocks_tool(
)
_record_run_blocks_result(copilot_ctx, result)
record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="run_blocks_and_collect_debug",
+ result=result,
+ )
enqueue_screenshot_from_result(copilot_ctx, result)
sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)
@@ -3974,11 +4013,11 @@ async def update_and_run_blocks_tool(
)
policy_error = _request_policy_tool_error(copilot_ctx, "update_and_run_blocks")
if not skip_run_after_update and policy_error:
- return json.dumps({"ok": False, "error": policy_error})
+ return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", policy_error)
loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks", arguments)
if loop_error:
- return json.dumps({"ok": False, "error": loop_error})
+ return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", loop_error)
# Snapshot the prior workflow definition BEFORE _update_workflow saves
# the new one — we need the pre-update state to diff against.
@@ -4004,6 +4043,11 @@ async def update_and_run_blocks_tool(
if not update_result.get("ok"):
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, update_result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=update_result,
+ )
sanitized = sanitize_tool_result_for_llm("update_workflow", update_result)
return json.dumps(sanitized)
@@ -4019,6 +4063,12 @@ async def update_and_run_blocks_tool(
},
}
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=result,
+ workflow_updated=True,
+ )
return json.dumps(result)
if skip_run_after_update:
@@ -4034,6 +4084,12 @@ async def update_and_run_blocks_tool(
},
}
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, skip_result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=skip_result,
+ workflow_updated=True,
+ )
LOG.info(
"update_and_run_blocks skipped run on unbound credential workflow inputs",
workflow_permanent_id=copilot_ctx.workflow_permanent_id,
@@ -4087,6 +4143,12 @@ async def update_and_run_blocks_tool(
)
_record_run_blocks_result(copilot_ctx, run_result)
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, run_result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=run_result,
+ workflow_updated=True,
+ )
enqueue_screenshot_from_result(copilot_ctx, run_result)
sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", run_result)
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
new file mode 100644
index 000000000..2367e3b09
--- /dev/null
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ DiagnosisFailureType,
+ RepairNextAction,
+ build_diagnosis_repair_contract,
+)
+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
+
+
+def _ctx() -> CopilotContext:
+ return CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ user_message="Fix the workflow with password=hunter2",
+ turn_intent=TurnIntent(
+ mode=TurnIntentMode.EDIT,
+ user_goal="Fix the workflow with password=hunter2",
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ ),
+ )
+
+
+def test_contract_shapes_for_failed_suspicious_and_missing_credential_cases() -> None:
+ failed = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result={
+ "ok": False,
+ "error": "The run ended before recording a trustworthy terminal status.",
+ "data": {"workflow_run_id": "wr_1", "overall_status": "running", "failure_reason": "uncertain"},
+ },
+ ctx=_ctx(),
+ )
+ suspicious_ctx = _ctx()
+ suspicious_ctx.last_test_suspicious_success = True
+ suspicious = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_2",
+ "overall_status": "completed",
+ "frontier_start_label": "extract",
+ "blocks": [{"label": "extract", "block_type": "EXTRACTION", "status": "completed"}],
+ },
+ },
+ ctx=suspicious_ctx,
+ workflow_updated=True,
+ )
+ missing = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": False,
+ "error": "Skipped test run: required credentials are not configured.",
+ "data": {"workflow_updated": True, "skip_reason": "workflow_credential_inputs_unbound"},
+ },
+ ctx=_ctx(),
+ workflow_updated=True,
+ )
+
+ assert (
+ failed.diagnosis_result.suspected_failure_type,
+ failed.repair_decision.next_action,
+ failed.diagnosis_result.missing_context,
+ ) == (DiagnosisFailureType.FAILED_RUN, RepairNextAction.REPAIR, ["block_results"])
+ assert (
+ suspicious.diagnosis_result.suspected_failure_type,
+ suspicious.repair_decision.next_action,
+ suspicious.repair_decision.target_blocks,
+ suspicious.verification_result.user_goal_satisfied,
+ ) == (DiagnosisFailureType.SUSPICIOUS_SUCCESS, RepairNextAction.REPAIR, ["extract"], False)
+ assert (
+ missing.diagnosis_result.suspected_failure_type,
+ missing.repair_decision.next_action,
+ missing.repair_decision.required_authority,
+ ) == (DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT, RepairNextAction.ASK, ["may_answer_without_mutation"])
+
+
+def test_repairable_block_failure_contract_is_queryable_and_safe() -> None:
+ contract = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result={
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_3",
+ "overall_status": "failed",
+ "requested_block_labels": ["login", "extract"],
+ "executed_block_labels": ["extract"],
+ "frontier_start_label": "extract",
+ "current_url": "https://example.test/account?id=secret",
+ "page_title": "Account page",
+ "failure_categories": [{"category": "DATA_EXTRACTION_FAILURE", "reasoning": "missing fields"}],
+ "blocks": [
+ {"label": "extract", "block_type": "EXTRACTION", "status": "failed", "failure_reason": "No rows"}
+ ],
+ },
+ },
+ ctx=_ctx(),
+ )
+
+ trace = contract.to_trace_data()
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
+ assert contract.repair_decision.target_blocks == ["extract"]
+ assert trace["failure_type"] == "repairable_block_failure"
+ assert trace["next_action"] == "repair"
+ assert trace["failure_categories"] == ["DATA_EXTRACTION_FAILURE"]
+ assert contract.diagnosis_input.browser_page_state["current_origin"] == "https://example.test"
+ assert "secret" not in contract.model_dump_json()
+ assert "hunter2" not in contract.model_dump_json()
+
+
+def test_user_goal_urls_are_reduced_to_origins() -> None:
+ ctx = _ctx()
+ ctx.turn_intent.user_goal = "Fix https://example.test/account?id=secret now"
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result={"ok": False, "error": "failed", "data": {"overall_status": "failed"}},
+ ctx=ctx,
+ )
+
+ assert contract.diagnosis_input.user_goal == "Fix https://example.test now"
+ assert "id=secret" not in contract.model_dump_json()
+
+
+def test_suspicious_success_flag_does_not_override_failed_run() -> None:
+ ctx = _ctx()
+ ctx.last_test_suspicious_success = True
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result={
+ "ok": False,
+ "error": "The run failed before output validation.",
+ "data": {"workflow_run_id": "wr_failed", "overall_status": "failed"},
+ },
+ ctx=ctx,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.FAILED_RUN
+ assert contract.repair_decision.proposed_change_summary == (
+ "Repair the workflow based on: The run failed before output validation."
+ )
+
+
+def test_stop_and_no_change_decisions_preserve_current_behavior_shadow_only() -> None:
+ stop_ctx = _ctx()
+ stop_ctx.last_test_non_retriable_nav_error = "net::ERR_NAME_NOT_RESOLVED"
+ stop_contract = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result={
+ "ok": False,
+ "data": {
+ "overall_status": "failed",
+ "blocks": [{"label": "open", "status": "failed", "failure_reason": "net::ERR_NAME_NOT_RESOLVED"}],
+ },
+ },
+ ctx=stop_ctx,
+ )
+ no_change = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result={"ok": True, "data": {"workflow_run_id": "wr_4", "overall_status": "completed", "blocks": []}},
+ ctx=_ctx(),
+ )
+
+ assert stop_contract.repair_decision.next_action == RepairNextAction.STOP
+ assert no_change.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE
+ assert no_change.repair_decision.next_action == RepairNextAction.NO_CHANGE
+ assert no_change.verification_result.completion_contract_satisfied is True