Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10888: Repair credentialed runtime diagnosis (#6526)
Andrew Neilson46b4f2e25b5c
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex 2235e7836..8b2e040b6 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -18,6 +18,9 @@ _TEXT_MAX = 240_SUMMARY_MAX = 180_MAX_ITEMS = 20_FAILED_STATUSES = {"failed", "terminated", "canceled", "timed_out"}+_CREDENTIAL_INPUT_MISSING_SKIP_REASONS = {"workflow_credential_inputs_unbound", "credential_name_unresolved"}+_PRE_RUN_CREDENTIAL_FAILURE_CATEGORIES = {"CREDENTIAL_ERROR", "PARAMETER_BINDING_ERROR"}+_REPAIRABLE_RUNTIME_CATEGORIES = {"AUTH_FAILURE", "OUTCOME_UNVERIFIED"}class StrictModel(BaseModel):@@ -304,6 +307,9 @@ def _failure_type(result: dict[str, Any],data: dict[str, Any],) -> DiagnosisFailureType:+ skip_reason = _safe_str(data.get("skip_reason"))+ if skip_reason in _CREDENTIAL_INPUT_MISSING_SKIP_REASONS:+ return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INITif run_ok:return DiagnosisFailureType.SUSPICIOUS_SUCCESS if suspicious else DiagnosisFailureType.NO_FAILUREerror_text = " ".join(@@ -311,6 +317,7 @@ def _failure_type(for value in (result.get("error"), data.get("failure_reason"), data.get("skip_reason"))if value)+ category_set = set(categories)if ("UNRECOVERABLE_TOOL_ERROR" in categoriesor "browser session not found" in error_text@@ -325,16 +332,16 @@ def _failure_type():return DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCEif (- data.get("skip_reason") == "workflow_credential_inputs_unbound"- or "credential" in error_text+ category_set & _PRE_RUN_CREDENTIAL_FAILURE_CATEGORIESor "organization not found" in error_textor "workflow not found" in error_textor "browser session" in error_text- or "PARAMETER_BINDING_ERROR" in categories):return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT- if failed_blocks:+ if failed_blocks or category_set & _REPAIRABLE_RUNTIME_CATEGORIES:return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE+ if "credential" in error_text:+ return DiagnosisFailureType.MISSING_CREDENTIAL_OR_INITreturn DiagnosisFailureType.FAILED_RUN if result.get("ok") is False else DiagnosisFailureType.UNKNOWNdiff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex d9e5c63b1..e9bcc61ff 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -2,6 +2,7 @@ from __future__ import annotationsfrom types import SimpleNamespace+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisFailureType,@@ -116,6 +117,135 @@ def test_repairable_block_failure_contract_is_queryable_and_safe() -> None:assert "hunter2" not in contract.model_dump_json()+def test_credentialed_runtime_auth_failure_repairs_failed_code_block() -> None:+ contract = build_diagnosis_repair_contract(+ source_tool="run_blocks_and_collect_debug",+ result={+ "ok": False,+ "error": "The code block used saved credentials but the browser ended on Login Failure.",+ "data": {+ "workflow_run_id": "wr_auth",+ "overall_status": "failed",+ "requested_block_labels": ["login"],+ "executed_block_labels": ["login"],+ "frontier_start_label": "login",+ "current_url": "https://example.test/loginFail/",+ "page_title": "Login Failure",+ "failure_categories": [{"category": "AUTH_FAILURE", "reasoning": "login rejected"}],+ "blocks": [+ {+ "label": "login",+ "block_type": "CODE",+ "status": "failed",+ "failure_reason": "Saved credentials were submitted, but the page showed Login Failure.",+ }+ ],+ },+ },+ ctx=_ctx(),+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR+ assert contract.repair_decision.target_blocks == ["login"]+ assert contract.diagnosis_result.suspected_failure_type != DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ assert contract.repair_decision.next_action != RepairNextAction.ASK+++def test_contradictory_completion_auth_evidence_repairs_frontier_block() -> None:+ ctx = _ctx()+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c0",+ satisfied=False,+ reason_code="evidence_contradicts",+ evidence_ref="current_url,page_title",+ )+ ],+ )++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": False,+ "error": (+ "Completion verification contradicted code output: login_succeeded=True, "+ "but saved credentials landed on /loginFail/ with Login Failure page evidence."+ ),+ "data": {+ "workflow_run_id": "wr_outcome",+ "overall_status": "completed",+ "frontier_start_label": "login",+ "current_url": "https://example.test/loginFail/",+ "page_title": "Login Failure",+ "failure_categories": [+ {+ "category": "OUTCOME_UNVERIFIED",+ "reasoning": "success flag contradicted by current page evidence",+ }+ ],+ "completion_verification": ctx.completion_verification_result.to_trace_data(),+ "blocks": [{"label": "login", "block_type": "CODE", "status": "completed"}],+ },+ },+ ctx=ctx,+ workflow_updated=True,+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR+ assert contract.repair_decision.target_blocks == ["login"]+ assert contract.verification_result.user_goal_satisfied is False+ assert contract.verification_result.completion_contract_satisfied is False+ assert contract.diagnosis_result.suspected_failure_type != DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ assert contract.repair_decision.next_action != RepairNextAction.ASK+++def test_unbound_credential_skip_and_parameter_binding_errors_still_ask() -> None:+ unbound = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": True,+ "message": "Skipped test run: required credentials are not configured.",+ "data": {+ "workflow_updated": True,+ "skipped_run": True,+ "skip_reason": "workflow_credential_inputs_unbound",+ },+ },+ ctx=_ctx(),+ workflow_updated=True,+ )+ binding_error = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": False,+ "error": "Missing required workflow parameter for credential binding.",+ "data": {+ "overall_status": "failed",+ "failure_categories": [{"category": "PARAMETER_BINDING_ERROR"}],+ },+ },+ ctx=_ctx(),+ workflow_updated=True,+ )++ assert (+ unbound.diagnosis_result.suspected_failure_type,+ unbound.repair_decision.next_action,+ binding_error.diagnosis_result.suspected_failure_type,+ binding_error.repair_decision.next_action,+ ) == (+ DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT,+ RepairNextAction.ASK,+ DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT,+ RepairNextAction.ASK,+ )++def test_active_run_terminal_evidence_contract_stops_without_marking_workflow_success() -> None:contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks",