Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fail a workflow run within seconds when its worker container restarts (SKY-17683) (#8828)
Aaron Perez44de8cd1af21
diff --git a/skyvern/cli/core/mcp_http_auth.py b/skyvern/cli/core/mcp_http_auth.pyindex 85905d12c..6a46863b1 100644--- a/skyvern/cli/core/mcp_http_auth.py+++ b/skyvern/cli/core/mcp_http_auth.py@@ -25,10 +25,12 @@ from starlette.types import ASGIApp, Receive, Scope, Sendfrom skyvern.config import settingsfrom skyvern.forge import app+from skyvern.forge.agent_functions import AuditEventfrom skyvern.forge.sdk.core.aiohttp_helper import aiohttp_requestfrom skyvern.forge.sdk.db.agent_db import AgentDBfrom skyvern.forge.sdk.db.enums import OrganizationAuthTokenTypefrom skyvern.forge.sdk.services.org_auth_service import resolve_org_from_api_key+from skyvern.forge.sdk.services.request_principal import AuthKind, RequestPrincipalfrom .api_key_hash import hash_api_key_for_cachefrom .client import reset_api_key_override, set_api_key_override@@ -338,6 +340,12 @@ async def _resolve_oauth_subject_to_org(api_token = await org_auth_token_service.create_org_api_token(org_id)LOG.info("MCP OAuth: auto-created API key for org", organization_id=org_id)+ await app.AGENT_FUNCTION.record_audit_event(+ RequestPrincipal(organization_id=org_id, auth_kind=AuthKind.bearer, user_id=subject),+ AuditEvent(+ organization_id=org_id, action="api_key.create", resource_type="api_key", resource_id=api_token.id+ ),+ )validation = MCPAPIKeyValidation(organization_id=org_id, token_type=OrganizationAuthTokenType.api)return _OAuthResolution(api_key=api_token.token, validation=validation)diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.pyindex da494170d..f5c94a27b 100644--- a/skyvern/forge/failure_classifier.py+++ b/skyvern/forge/failure_classifier.py@@ -383,6 +383,19 @@ def classify_from_failure_reason(})+ # The worker process running the run died mid-flight (container restart; in practice an OOM+ # kill) and its replacement failed the orphaned activity with this typed cause. Keyed on the cause,+ # never on reason text, so a reason that merely quotes the wording is not pinned on the worker.+ if exc_name == WORKER_CONTAINER_RESTARTED_REASON_CODE:+ categories.append(+ {+ "category": FailureCategory.INFRASTRUCTURE_ERROR.value,+ "confidence_float": 0.95,+ "reason_code": WORKER_CONTAINER_RESTARTED_REASON_CODE,+ "reasoning": "Worker container restarted mid-run",+ }+ )+# Runner-slot contention, not a fault; the distinct reason_code keeps it separable from# real runner failures in analytics.if "codeblock runner is already executing another codeblock" in reason:@@ -577,7 +590,7 @@ def classify_from_failure_reason(# Bump when the taxonomy or the category->component mapping below changes, so a frozen# coverage baseline stays reproducible per classifier_version.-CLASSIFIER_VERSION = 7+CLASSIFIER_VERSION = 8FAILURE_ATTRIBUTION_SCHEMA_VERSION = 1# Bounded sentinels — neither is an infra component id.@@ -597,6 +610,7 @@ BROWSER_SESSION_CLOSED_REASON_CODE = "browser_session_closed"BROWSER_SESSION_STARTUP_TIMEOUT_REASON_CODE = "browser_session_startup_timeout"BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE = "browser_session_expired_before_run"PROXY_TRANSPORT_FAILED_REASON_CODE = "proxy_transport_failed"+WORKER_CONTAINER_RESTARTED_REASON_CODE = "worker_container_restarted"UNSOLVED_CAPTCHA_BEFORE_BROWSER_LOSS_REASON_CODE = "unsolved_captcha_before_browser_loss"SITE_THROTTLE_REASON_CODE = "site_throttle"EXTERNAL_SERVICE_THROTTLE_REASON_CODE = "external_service_throttle"@@ -673,6 +687,7 @@ _REASON_CODE_LITERALS = frozenset(SITE_THROTTLE_REASON_CODE,EXTERNAL_SERVICE_THROTTLE_REASON_CODE,UNSOLVED_CAPTCHA_BEFORE_BROWSER_LOSS_REASON_CODE,+ WORKER_CONTAINER_RESTARTED_REASON_CODE,})diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.pyindex e60368859..0bbabebb5 100644--- a/tests/unit/test_failure_classifier.py+++ b/tests/unit/test_failure_classifier.py@@ -18,6 +18,7 @@ from skyvern.forge.failure_classifier import (CLASSIFIER_VERSION,FAILURE_ATTRIBUTION_SCHEMA_VERSION,PROXY_TRANSPORT_FAILED_REASON_CODE,+ WORKER_CONTAINER_RESTARTED_REASON_CODE,FailureCategory,classify_from_failure_reason,derive_failure_attribution,@@ -552,6 +553,24 @@ def test_secure_codeblock_runner_unavailable_carries_a_reason_code() -> None:assert infra[0]["reason_code"] == "secure_codeblock_runner_unavailable"+def test_worker_restart_is_attributed_from_the_typed_cause_not_from_reason_text() -> None:+ """Only the typed cause the restart recovery raises names a worker loss; a reason that merely+ quotes worker-restart wording (page text, user code) must not be pinned on the worker."""+ reason = "Workflow run failed: the worker running this run was restarted before the run finished."+ result = classify_from_failure_reason(+ reason, exception_name=WORKER_CONTAINER_RESTARTED_REASON_CODE, fallback_to_unknown=True+ )++ assert result is not None+ assert [entry["category"] for entry in result] == ["INFRASTRUCTURE_ERROR"]+ attribution = derive_failure_attribution(result)+ assert attribution["primary_infra_component"] == "worker"+ assert attribution["reason_code"] == WORKER_CONTAINER_RESTARTED_REASON_CODE++ quoted = _classify("Element text: 'the worker running it was restarted'", fallback_to_unknown=True)+ assert WORKER_CONTAINER_RESTARTED_REASON_CODE not in [entry.get("reason_code") for entry in quoted]++def test_ordinary_user_code_failure_is_not_infrastructure() -> None:# A block that reached the sandbox and raised is a user-code fault, not a deploy fault.reason = "code block failed. failure reason: CodeBlock failed while running user code."