Skip to content

Automate browser based workflows with AI

SKY-10888: Repair credentialed runtime diagnosis (#6526)

Andrew Neilson46b4f2e25b5c
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index 2235e7836..8b2e040b6 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -18,6 +18,9 @@ _TEXT_MAX = 240
_SUMMARY_MAX = 180
_MAX_ITEMS = 20
_FAILED_STATUSES = {"failed", "terminated", "canceled", "timed_out"}
+_CREDENTIAL_INPUT_MISSING_SKIP_REASONS = {"workflow_credential_inputs_unbound", "credential_name_unresolved"}
+_PRE_RUN_CREDENTIAL_FAILURE_CATEGORIES = {"CREDENTIAL_ERROR", "PARAMETER_BINDING_ERROR"}
+_REPAIRABLE_RUNTIME_CATEGORIES = {"AUTH_FAILURE", "OUTCOME_UNVERIFIED"}
class StrictModel(BaseModel):
@@ -304,6 +307,9 @@ def _failure_type(
result: dict[str, Any],
data: dict[str, Any],
) -> DiagnosisFailureType:
+ skip_reason = _safe_str(data.get("skip_reason"))
+ if skip_reason in _CREDENTIAL_INPUT_MISSING_SKIP_REASONS:
+ return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
if run_ok:
return DiagnosisFailureType.SUSPICIOUS_SUCCESS if suspicious else DiagnosisFailureType.NO_FAILURE
error_text = " ".join(
@@ -311,6 +317,7 @@ def _failure_type(
for value in (result.get("error"), data.get("failure_reason"), data.get("skip_reason"))
if value
)
+ category_set = set(categories)
if (
"UNRECOVERABLE_TOOL_ERROR" in categories
or "browser session not found" in error_text
@@ -325,16 +332,16 @@ def _failure_type(
):
return DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCE
if (
- data.get("skip_reason") == "workflow_credential_inputs_unbound"
- or "credential" in error_text
+ category_set & _PRE_RUN_CREDENTIAL_FAILURE_CATEGORIES
or "organization not found" in error_text
or "workflow not found" in error_text
or "browser session" in error_text
- or "PARAMETER_BINDING_ERROR" in categories
):
return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
- if failed_blocks:
+ if failed_blocks or category_set & _REPAIRABLE_RUNTIME_CATEGORIES:
return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
+ if "credential" in error_text:
+ return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
return DiagnosisFailureType.FAILED_RUN if result.get("ok") is False else DiagnosisFailureType.UNKNOWN
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index d9e5c63b1..e9bcc61ff 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -2,6 +2,7 @@ from __future__ import annotations
from types import SimpleNamespace
+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisFailureType,
@@ -116,6 +117,135 @@ def test_repairable_block_failure_contract_is_queryable_and_safe() -> None:
assert "hunter2" not in contract.model_dump_json()
+def test_credentialed_runtime_auth_failure_repairs_failed_code_block() -> None:
+ contract = build_diagnosis_repair_contract(
+ source_tool="run_blocks_and_collect_debug",
+ result={
+ "ok": False,
+ "error": "The code block used saved credentials but the browser ended on Login Failure.",
+ "data": {
+ "workflow_run_id": "wr_auth",
+ "overall_status": "failed",
+ "requested_block_labels": ["login"],
+ "executed_block_labels": ["login"],
+ "frontier_start_label": "login",
+ "current_url": "https://example.test/loginFail/",
+ "page_title": "Login Failure",
+ "failure_categories": [{"category": "AUTH_FAILURE", "reasoning": "login rejected"}],
+ "blocks": [
+ {
+ "label": "login",
+ "block_type": "CODE",
+ "status": "failed",
+ "failure_reason": "Saved credentials were submitted, but the page showed Login Failure.",
+ }
+ ],
+ },
+ },
+ ctx=_ctx(),
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert contract.repair_decision.target_blocks == ["login"]
+ assert contract.diagnosis_result.suspected_failure_type != DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ assert contract.repair_decision.next_action != RepairNextAction.ASK
+
+
+def test_contradictory_completion_auth_evidence_repairs_frontier_block() -> None:
+ ctx = _ctx()
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ satisfied=False,
+ reason_code="evidence_contradicts",
+ evidence_ref="current_url,page_title",
+ )
+ ],
+ )
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": False,
+ "error": (
+ "Completion verification contradicted code output: login_succeeded=True, "
+ "but saved credentials landed on /loginFail/ with Login Failure page evidence."
+ ),
+ "data": {
+ "workflow_run_id": "wr_outcome",
+ "overall_status": "completed",
+ "frontier_start_label": "login",
+ "current_url": "https://example.test/loginFail/",
+ "page_title": "Login Failure",
+ "failure_categories": [
+ {
+ "category": "OUTCOME_UNVERIFIED",
+ "reasoning": "success flag contradicted by current page evidence",
+ }
+ ],
+ "completion_verification": ctx.completion_verification_result.to_trace_data(),
+ "blocks": [{"label": "login", "block_type": "CODE", "status": "completed"}],
+ },
+ },
+ ctx=ctx,
+ workflow_updated=True,
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE
+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert contract.repair_decision.target_blocks == ["login"]
+ assert contract.verification_result.user_goal_satisfied is False
+ assert contract.verification_result.completion_contract_satisfied is False
+ assert contract.diagnosis_result.suspected_failure_type != DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ assert contract.repair_decision.next_action != RepairNextAction.ASK
+
+
+def test_unbound_credential_skip_and_parameter_binding_errors_still_ask() -> None:
+ unbound = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": True,
+ "message": "Skipped test run: required credentials are not configured.",
+ "data": {
+ "workflow_updated": True,
+ "skipped_run": True,
+ "skip_reason": "workflow_credential_inputs_unbound",
+ },
+ },
+ ctx=_ctx(),
+ workflow_updated=True,
+ )
+ binding_error = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": False,
+ "error": "Missing required workflow parameter for credential binding.",
+ "data": {
+ "overall_status": "failed",
+ "failure_categories": [{"category": "PARAMETER_BINDING_ERROR"}],
+ },
+ },
+ ctx=_ctx(),
+ workflow_updated=True,
+ )
+
+ assert (
+ unbound.diagnosis_result.suspected_failure_type,
+ unbound.repair_decision.next_action,
+ binding_error.diagnosis_result.suspected_failure_type,
+ binding_error.repair_decision.next_action,
+ ) == (
+ DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT,
+ RepairNextAction.ASK,
+ DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT,
+ RepairNextAction.ASK,
+ )
+
+
def test_active_run_terminal_evidence_contract_stops_without_marking_workflow_success() -> None:
contract = build_diagnosis_repair_contract(
source_tool="update_and_run_blocks",