Skip to content

Automate browser based workflows with AI

Fail a workflow run within seconds when its worker container restarts (SKY-17683) (#8828)

Aaron Perez44de8cd1af21
diff --git a/skyvern/cli/core/mcp_http_auth.py b/skyvern/cli/core/mcp_http_auth.py
index 85905d12c..6a46863b1 100644
--- a/skyvern/cli/core/mcp_http_auth.py
+++ b/skyvern/cli/core/mcp_http_auth.py
@@ -25,10 +25,12 @@ from starlette.types import ASGIApp, Receive, Scope, Send
from skyvern.config import settings
from skyvern.forge import app
+from skyvern.forge.agent_functions import AuditEvent
from skyvern.forge.sdk.core.aiohttp_helper import aiohttp_request
from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
from skyvern.forge.sdk.services.org_auth_service import resolve_org_from_api_key
+from skyvern.forge.sdk.services.request_principal import AuthKind, RequestPrincipal
from .api_key_hash import hash_api_key_for_cache
from .client import reset_api_key_override, set_api_key_override
@@ -338,6 +340,12 @@ async def _resolve_oauth_subject_to_org(
api_token = await org_auth_token_service.create_org_api_token(org_id)
LOG.info("MCP OAuth: auto-created API key for org", organization_id=org_id)
+ await app.AGENT_FUNCTION.record_audit_event(
+ RequestPrincipal(organization_id=org_id, auth_kind=AuthKind.bearer, user_id=subject),
+ AuditEvent(
+ organization_id=org_id, action="api_key.create", resource_type="api_key", resource_id=api_token.id
+ ),
+ )
validation = MCPAPIKeyValidation(organization_id=org_id, token_type=OrganizationAuthTokenType.api)
return _OAuthResolution(api_key=api_token.token, validation=validation)
diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.py
index da494170d..f5c94a27b 100644
--- a/skyvern/forge/failure_classifier.py
+++ b/skyvern/forge/failure_classifier.py
@@ -383,6 +383,19 @@ def classify_from_failure_reason(
}
)
+ # The worker process running the run died mid-flight (container restart; in practice an OOM
+ # kill) and its replacement failed the orphaned activity with this typed cause. Keyed on the cause,
+ # never on reason text, so a reason that merely quotes the wording is not pinned on the worker.
+ if exc_name == WORKER_CONTAINER_RESTARTED_REASON_CODE:
+ categories.append(
+ {
+ "category": FailureCategory.INFRASTRUCTURE_ERROR.value,
+ "confidence_float": 0.95,
+ "reason_code": WORKER_CONTAINER_RESTARTED_REASON_CODE,
+ "reasoning": "Worker container restarted mid-run",
+ }
+ )
+
# Runner-slot contention, not a fault; the distinct reason_code keeps it separable from
# real runner failures in analytics.
if "codeblock runner is already executing another codeblock" in reason:
@@ -577,7 +590,7 @@ def classify_from_failure_reason(
# Bump when the taxonomy or the category->component mapping below changes, so a frozen
# coverage baseline stays reproducible per classifier_version.
-CLASSIFIER_VERSION = 7
+CLASSIFIER_VERSION = 8
FAILURE_ATTRIBUTION_SCHEMA_VERSION = 1
# Bounded sentinels — neither is an infra component id.
@@ -597,6 +610,7 @@ BROWSER_SESSION_CLOSED_REASON_CODE = "browser_session_closed"
BROWSER_SESSION_STARTUP_TIMEOUT_REASON_CODE = "browser_session_startup_timeout"
BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE = "browser_session_expired_before_run"
PROXY_TRANSPORT_FAILED_REASON_CODE = "proxy_transport_failed"
+WORKER_CONTAINER_RESTARTED_REASON_CODE = "worker_container_restarted"
UNSOLVED_CAPTCHA_BEFORE_BROWSER_LOSS_REASON_CODE = "unsolved_captcha_before_browser_loss"
SITE_THROTTLE_REASON_CODE = "site_throttle"
EXTERNAL_SERVICE_THROTTLE_REASON_CODE = "external_service_throttle"
@@ -673,6 +687,7 @@ _REASON_CODE_LITERALS = frozenset(
SITE_THROTTLE_REASON_CODE,
EXTERNAL_SERVICE_THROTTLE_REASON_CODE,
UNSOLVED_CAPTCHA_BEFORE_BROWSER_LOSS_REASON_CODE,
+ WORKER_CONTAINER_RESTARTED_REASON_CODE,
}
)
diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.py
index e60368859..0bbabebb5 100644
--- a/tests/unit/test_failure_classifier.py
+++ b/tests/unit/test_failure_classifier.py
@@ -18,6 +18,7 @@ from skyvern.forge.failure_classifier import (
CLASSIFIER_VERSION,
FAILURE_ATTRIBUTION_SCHEMA_VERSION,
PROXY_TRANSPORT_FAILED_REASON_CODE,
+ WORKER_CONTAINER_RESTARTED_REASON_CODE,
FailureCategory,
classify_from_failure_reason,
derive_failure_attribution,
@@ -552,6 +553,24 @@ def test_secure_codeblock_runner_unavailable_carries_a_reason_code() -> None:
assert infra[0]["reason_code"] == "secure_codeblock_runner_unavailable"
+def test_worker_restart_is_attributed_from_the_typed_cause_not_from_reason_text() -> None:
+ """Only the typed cause the restart recovery raises names a worker loss; a reason that merely
+ quotes worker-restart wording (page text, user code) must not be pinned on the worker."""
+ reason = "Workflow run failed: the worker running this run was restarted before the run finished."
+ result = classify_from_failure_reason(
+ reason, exception_name=WORKER_CONTAINER_RESTARTED_REASON_CODE, fallback_to_unknown=True
+ )
+
+ assert result is not None
+ assert [entry["category"] for entry in result] == ["INFRASTRUCTURE_ERROR"]
+ attribution = derive_failure_attribution(result)
+ assert attribution["primary_infra_component"] == "worker"
+ assert attribution["reason_code"] == WORKER_CONTAINER_RESTARTED_REASON_CODE
+
+ quoted = _classify("Element text: 'the worker running it was restarted'", fallback_to_unknown=True)
+ assert WORKER_CONTAINER_RESTARTED_REASON_CODE not in [entry.get("reason_code") for entry in quoted]
+
+
def test_ordinary_user_code_failure_is_not_infrastructure() -> None:
# A block that reached the sandbox and raised is a user-code fault, not a deploy fault.
reason = "code block failed. failure reason: CodeBlock failed while running user code."