Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Medwig/sky 16553 agent waits for runs (#8806)
Shuchang Zhengfacec046ea69
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex 5002d2707..b9112d7d5 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -193,6 +193,17 @@"400": {"description": "Invalid agent run request"},+ "503": {+ "description": "The server is busy dispatching other runs. No run was created; retry after Retry-After seconds.",+ "headers": {+ "Retry-After": {+ "description": "Seconds to wait before retrying",+ "schema": {+ "type": "integer"+ }+ }+ }+ },"422": {"description": "Validation Error","content": {@@ -6628,6 +6639,17 @@"404": {"description": "Workflow run not found"},+ "503": {+ "description": "The server is busy dispatching other runs. No run was created; retry after Retry-After seconds.",+ "headers": {+ "Retry-After": {+ "description": "Seconds to wait before retrying",+ "schema": {+ "type": "integer"+ }+ }+ }+ },"422": {"description": "Validation Error","content": {@@ -19339,6 +19361,22 @@"title": "JinjaBranchCriteria","description": "Jinja2-templated branch criteria (only supported criteria type for now)."},+ "JobRecipeOutcome": {+ "type": "string",+ "enum": [+ "completed",+ "canceled",+ "job_unavailable",+ "login_required",+ "application_refused",+ "blocked_by_site",+ "invalid_input",+ "insufficient_credits",+ "incomplete",+ "failed"+ ],+ "title": "JobRecipeOutcome"+ },"JobRecipeRunListResponse": {"properties": {"runs": {@@ -19435,6 +19473,29 @@}],"title": "Failure Reason"+ },+ "outcome": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/JobRecipeOutcome"+ },+ {+ "type": "null"+ }+ ],+ "description": "Typed result of a finished run; null while it is still running. Apply failures name their cause (job_unavailable, login_required, application_refused, blocked_by_site, invalid_input, insufficient_credits, incomplete); a cause Skyvern cannot classify is failed, and so is every Extract failure except one Skyvern itself reports, such as running out of credits."+ },+ "next_step": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Next Step",+ "description": "What to do about an unsuccessful outcome; null for completed, canceled, and running runs."}},"additionalProperties": false,@@ -19446,7 +19507,9 @@"status","created_at","job_url",- "failure_reason"+ "failure_reason",+ "outcome",+ "next_step"],"title": "JobRecipeRunSummary"},diff --git a/docs/cookbooks/job-application-filler.mdx b/docs/cookbooks/job-application-filler.mdxindex ee2dc2f81..819a28274 100644--- a/docs/cookbooks/job-application-filler.mdx+++ b/docs/cookbooks/job-application-filler.mdx@@ -206,6 +206,21 @@ curl 'https://api.skyvern.com/v1/recipes/jobs/runs?page=1&page_size=25' \Each entry includes the run ID, recipe (`apply` or `extract`), status, creation time, job URL, and failure reason when available. The response also includes `page`, `page_size`, and `has_next`. Increment `page` while `has_next` is `true`. Page size must be between 1 and 100.+Each entry also has an `outcome` and a `next_step`. `outcome` is `null` until the run finishes. `next_step` says what to do about an unsuccessful outcome and is `null` otherwise.++| `outcome` | Meaning |+| ---------------------- | ------------------------------------------------------------------------------- |+| `completed` | The run finished successfully. |+| `canceled` | The run was canceled. |+| `job_unavailable` | Skyvern found the posting closed, or could not find it at this URL. |+| `login_required` | The site needed an account, a sign-in, or a verification code, or rejected the sign-in. |+| `application_refused` | The site refused the application, for example as a duplicate. |+| `blocked_by_site` | Bot protection, a CAPTCHA, a rate limit, or a locked account blocked the run. |+| `invalid_input` | A request value could not be used, so the run stopped before opening the site. |+| `insufficient_credits` | The organization ran out of credits. |+| `incomplete` | The run ran out of steps before it could submit the application. |+| `failed` | Any other failure. Read `failure_reason`. An unsuccessful Extract run is `failed` unless it ran out of credits. |+<Note>Each Apply API request creates a new run. Repeating a request can submit another application, so check the original run before retrying.</Note>diff --git a/skyvern/config.py b/skyvern/config.pyindex ed8f9a24d..0728eb2ee 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -301,6 +301,13 @@ class Settings(BaseSettings):# /stream sockets all count -- so size it against connections per task, not request concurrency.# Set it empty or 0 to disable shedding.API_LIMIT_CONCURRENCY: int | None = Field(default=512, gt=0)+ # Run submissions one API process dispatches at once; later ones wait without holding a pooled connection, and+ # get a retryable 503 before anything is written if no slot frees. 0 (the default) leaves them unbounded and 32+ # is the suggested first value, checked against the skyvern.run_submission.in_flight gauge, which records either+ # way; the DISABLE_RUN_SUBMISSION_GATE feature flag switches an enabled gate off without a restart.+ RUN_SUBMISSION_MAX_CONCURRENCY: int = Field(default=0, ge=0)+ # Below the SDK's 60 s client timeout, so a waiting caller is answered before it gives up.+ RUN_SUBMISSION_SLOT_WAIT_SECONDS: float = Field(default=20.0, gt=0)# Must exceed the load balancer's idle timeout (infra/terraform/production/alb.tf); otherwise# the ALB reuses a connection the server already closed and answers the client with a 502.UVICORN_TIMEOUT_KEEP_ALIVE: int = 125diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex 3fc696e34..5eba4c340 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -7,6 +7,8 @@ from http import HTTPStatusfrom importlib.util import find_specfrom typing import TYPE_CHECKING, Literal, NoReturn+from skyvern.schemas.browser_session_timeouts import MAX_EXTENDED_TIMEOUT+if TYPE_CHECKING:from skyvern.errors.errors import UserDefinedError@@ -1783,6 +1785,53 @@ class BrowserSessionClosed(SkyvernHTTPException):)+def _utc_minute(moment: datetime) -> str:+ aware = moment.replace(tzinfo=UTC) if moment.tzinfo is None else moment.astimezone(UTC)+ return aware.strftime("%Y-%m-%d %H:%M UTC")+++class BrowserSessionExpired(BrowserSessionClosed):+ """The session ran out its configured lifetime. Carries the session facts that decide whether a run sent to it+ is the caller's to fix (see the browser-lease seam in the workflow service)."""++ def __init__(+ self,+ browser_session_id: str,+ *,+ started_at: datetime | None,+ ended_at: datetime | None,+ timeout_minutes: int | None,+ created_by: str | None,+ bound: bool = False,+ refused_at_submission: bool = False,+ ) -> None:+ self.browser_session_id = browser_session_id+ self.started_at = started_at+ self.ended_at = ended_at+ self.timeout_minutes = timeout_minutes+ self.created_by = created_by+ self.bound = bound+ if refused_at_submission:+ SkyvernHTTPException.__init__(+ self, self.expired_before_run_message(run_created=False), status_code=HTTPStatus.GONE+ )+ else:+ super().__init__(browser_session_id, reason="expired after reaching its configured lifetime")++ def expired_before_run_message(self, *, run_created: bool) -> str:+ """States that the session ended before the run was submitted, so use it only once that is established."""+ ended = f" at {_utc_minute(self.ended_at)}" if self.ended_at is not None else ""+ limit = f"its {self.timeout_minutes}-minute limit" if self.timeout_minutes else "its configured lifetime"+ outcome = "so the run did not start" if run_created else "so no run was created"+ return (+ f"Browser session {self.browser_session_id} expired{ended} after {limit}, before this run was submitted, "+ f"{outcome}. To fix it: create a new browser session and pass its id, or leave browser_session_id empty "+ "to start a fresh browser. To reuse one session for many runs, extend it before it ends "+ "(POST /v1/browser_sessions/{browser_session_id}/extend, "+ f"up to {MAX_EXTENDED_TIMEOUT // 60} hours in total)."+ )++class BrowserSessionNotFound(SkyvernHTTPException):def __init__(self, browser_session_id: str) -> None:super().__init__(diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 7df96e4c5..c8a6cf367 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -168,6 +168,7 @@ from skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logsfrom skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatusfrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.schemas.persistent_browser_sessions import unusable_browser_session_errorfrom skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskResponse, TaskStatusfrom skyvern.forge.sdk.schemas.totp_codes import OTPTypefrom skyvern.forge.sdk.services.credentials import parse_totp_config@@ -2179,6 +2180,9 @@ class ForgeAgent:)if not browser_session:raise BrowserSessionNotFound(browser_session_id=task_request.browser_session_id)+ unusable = unusable_browser_session_error(browser_session, refused_at_submission=True)+ if unusable is not None:+ raise unusabletask = await app.DATABASE.tasks.create_task(url=str(task_request.url),diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.pyindex 742877bde..9a038b9e7 100644--- a/skyvern/forge/failure_classifier.py+++ b/skyvern/forge/failure_classifier.py@@ -17,6 +17,7 @@ class FailureCategory(StrEnum):ANTI_BOT_DETECTION = "ANTI_BOT_DETECTION"PROXY_ERROR = "PROXY_ERROR"BROWSER_ERROR = "BROWSER_ERROR"+ BROWSER_SESSION_EXPIRED = "BROWSER_SESSION_EXPIRED"NAVIGATION_FAILURE = "NAVIGATION_FAILURE"PAGE_LOAD_TIMEOUT = "PAGE_LOAD_TIMEOUT"ELEMENT_STATE_TIMEOUT = "ELEMENT_STATE_TIMEOUT"@@ -472,7 +473,7 @@ def classify_from_failure_reason(# Bump when the taxonomy or the category->component mapping below changes, so a frozen# coverage baseline stays reproducible per classifier_version.-CLASSIFIER_VERSION = 4+CLASSIFIER_VERSION = 5FAILURE_ATTRIBUTION_SCHEMA_VERSION = 1# Bounded sentinels — neither is an infra component id.@@ -490,6 +491,7 @@ _FAILURE_CATEGORY_LITERALS = frozenset(category.value for category in FailureCat# (skyvern/forge/sdk/workflow/service.py::_browser_lease_failure_category) imports these.BROWSER_SESSION_CLOSED_REASON_CODE = "browser_session_closed"BROWSER_SESSION_STARTUP_TIMEOUT_REASON_CODE = "browser_session_startup_timeout"+BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE = "browser_session_expired_before_run"PROXY_TRANSPORT_FAILED_REASON_CODE = "proxy_transport_failed"UNSOLVED_CAPTCHA_BEFORE_BROWSER_LOSS_REASON_CODE = "unsolved_captcha_before_browser_loss"@@ -558,6 +560,7 @@ _REASON_CODE_LITERALS = frozenset("locator_wait_for_timeout",BROWSER_SESSION_CLOSED_REASON_CODE,BROWSER_SESSION_STARTUP_TIMEOUT_REASON_CODE,+ BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE,PROXY_TRANSPORT_FAILED_REASON_CODE,UNSOLVED_CAPTCHA_BEFORE_BROWSER_LOSS_REASON_CODE,}@@ -582,6 +585,7 @@ _NON_INFRA_CATEGORIES = {"DATA_EXTRACTION_FAILURE","LLM_REASONING_ERROR","WRONG_PAGE_STATE",+ "BROWSER_SESSION_EXPIRED",}diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.pyindex 630874e31..5dfe32e20 100644--- a/skyvern/forge/request_logging.py+++ b/skyvern/forge/request_logging.py@@ -99,6 +99,7 @@ class _RequestIdentity:principal_resolution_conflict: bool = Falsebearer_identity_status: str | None = Noneprincipal_resolution_lock: asyncio.Lock = field(default_factory=asyncio.Lock)+ load_shed: bool = False_request_identity: ContextVar[_RequestIdentity | None] = ContextVar("raw_request_identity", default=None)@@ -133,6 +134,13 @@ def set_request_organization(identity.org_age = org_age+def mark_request_load_shed() -> None:+ """Log this request's 503 at warning level: the server shed it on purpose under load, before doing any work."""+ identity = _request_identity.get()+ if identity is not None:+ identity.load_shed = True++def set_request_principal(principal: RequestPrincipal,resolution_key: tuple[str, bytes | None],@@ -309,7 +317,10 @@ def _log_request(headers: dict[str, str],start_time: float,) -> None:- if status_code >= 500:+ identity = _request_identity.get()+ if status_code == 503 and identity is not None and identity.load_shed:+ log_method = LOG.warning+ elif status_code >= 500:log_method = LOG.errorelif status_code >= 400 and status_code not in _ROUTINE_CLIENT_ERROR_STATUSES:log_method = LOG.warningdiff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 7c434c2db..2b4833590 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -825,6 +825,7 @@ _ARTIFACT_HEALTH_EXCLUDED_CATEGORIES = frozenset("ANTI_BOT_DETECTION","AUTH_FAILURE","BROWSER_ERROR",+ "BROWSER_SESSION_EXPIRED","CREDENTIAL_ERROR","INFRASTRUCTURE_ERROR","NAVIGATION_FAILURE",diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 509aad3c9..d5fadb9c5 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -23,7 +23,11 @@ import structlogimport yamlfrom skyvern.constants import SCRUBBED_VALUE-from skyvern.exceptions import CopilotInlineSequentialCredentialUnsupported+from skyvern.exceptions import (+ BrowserSessionClosed,+ BrowserSessionStartupTimeout,+ CopilotInlineSequentialCredentialUnsupported,+)from skyvern.forge import appfrom skyvern.forge.failure_classifier import without_output_only_anti_bot, without_output_only_anti_bot_categoriesfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactType@@ -4397,23 +4401,41 @@ async def _run_blocks_and_collect_debug()current_context = skyvern_context.current()- workflow_run = await workflow_service.prepare_workflow(- workflow_id=ctx.workflow_permanent_id,- organization=organization,- workflow_request=workflow_request,- template=False,- # Both execution paths pin the exact version. A permanent-id lookup could- # select a newer proposal while this run still carries the previous definition.- resolved_workflow_id=snapshot.workflow.workflow_id,- max_steps=None,- request_id=current_context.request_id if current_context is not None else None,- # The trigger type (and the -ui queue routing it implies) is a cloud contract; ask the- # AgentFunction for it rather than hardcoding "manual == -ui pool" in OSS. OSS base- # returns None (no routing hint); cloud returns the value its executor routes to -ui.- trigger_type=(app.AGENT_FUNCTION.resolve_copilot_dispatch_trigger_type() if dispatch_to_worker else None),- copilot_session_id=ctx.workflow_copilot_chat_id,- created_by="copilot",- )+ try:+ workflow_run = await workflow_service.prepare_workflow(+ workflow_id=ctx.workflow_permanent_id,+ organization=organization,+ workflow_request=workflow_request,+ template=False,+ # Both execution paths pin the exact version. A permanent-id lookup could+ # select a newer proposal while this run still carries the previous definition.+ resolved_workflow_id=snapshot.workflow.workflow_id,+ max_steps=None,+ request_id=current_context.request_id if current_context is not None else None,+ # The trigger type (and the -ui queue routing it implies) is a cloud contract; ask the+ # AgentFunction for it rather than hardcoding "manual == -ui pool" in OSS. OSS base+ # returns None (no routing hint); cloud returns the value its executor routes to -ui.+ trigger_type=(+ app.AGENT_FUNCTION.resolve_copilot_dispatch_trigger_type() if dispatch_to_worker else None+ ),+ copilot_session_id=ctx.workflow_copilot_chat_id,+ created_by="copilot",+ )+ except (BrowserSessionClosed, BrowserSessionStartupTimeout) as refused:+ # Submission refuses a session that has already ended, so no run exists to carry the+ # lease-seam reason code; report the same typed browser loss that run would have.+ if dispatch_draft_workflow_id is not None:+ await _delete_dispatch_draft(dispatch_draft_workflow_id, ctx.organization_id)+ dispatch_draft_workflow_id = None+ state: BuildTestConnectFailureState = (+ "provisioning_unavailable" if isinstance(refused, BrowserSessionStartupTimeout) else "already_closed"+ )+ return _with_build_test_acquisition_context(+ _build_test_connect_failure_result(+ BuildTestConnectFailure(state=state, browser_session_id=run_session_id)+ ),+ requested_block_labels=block_labels,+ )if explicit_blank and workflow_run.browser_session_id != run_session_id:await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(workflow_run_id=workflow_run.workflow_run_id,@@ -6832,14 +6854,16 @@ def _safe_reason_code(value: object) -> str:_RUN_SIDE_CONNECT_STATES: dict[str, BuildTestConnectFailureState] = {"browser_session_closed": "already_closed","browser_session_startup_timeout": "provisioning_unavailable",+ "browser_session_expired_before_run": "already_closed",}+_RUN_SIDE_CONNECT_CATEGORIES = frozenset({"BROWSER_ERROR", "BROWSER_SESSION_EXPIRED"})def _run_side_connect_state(failure_category: object) -> BuildTestConnectFailureState | None:if not isinstance(failure_category, list):return Nonefor entry in failure_category:- if isinstance(entry, dict) and entry.get("category") == "BROWSER_ERROR":+ if isinstance(entry, dict) and entry.get("category") in _RUN_SIDE_CONNECT_CATEGORIES:state = _RUN_SIDE_CONNECT_STATES.get(str(entry.get("reason_code") or ""))if state is not None:return statediff --git a/skyvern/forge/sdk/core/run_submission_gate.py b/skyvern/forge/sdk/core/run_submission_gate.pynew file mode 100644index 000000000..9245f1fd7--- /dev/null+++ b/skyvern/forge/sdk/core/run_submission_gate.py@@ -0,0 +1,189 @@+import asyncio+import functools+import time+from collections.abc import AsyncIterator, Iterable+from contextlib import asynccontextmanager+from typing import Any++import structlog+from fastapi import HTTPException, status+from opentelemetry import metrics, trace++from skyvern.config import settings+from skyvern.forge import app+from skyvern.forge.request_logging import mark_request_load_shed+from skyvern.utils.contained_effects import contained_effect+from skyvern.utils.metric_attributes import task_metric_attributes++LOG = structlog.get_logger()++RETRY_AFTER_SECONDS = 5+SLOT_WAIT_SPAN_ATTRIBUTE = "run_submission.slot_wait_ms"+DISABLE_RUN_SUBMISSION_GATE_FLAG = "DISABLE_RUN_SUBMISSION_GATE"+RUN_SUBMISSION_METRIC_PREFIX = "skyvern.run_submission."+RUN_SUBMISSION_REJECTED_COUNTER = f"{RUN_SUBMISSION_METRIC_PREFIX}rejected"+RUN_SUBMISSION_IN_FLIGHT_GAUGE = f"{RUN_SUBMISSION_METRIC_PREFIX}in_flight"+RUN_SUBMISSION_WAITING_GAUGE = f"{RUN_SUBMISSION_METRIC_PREFIX}waiting"+_KILL_SWITCH_LOOKUP_TIMEOUT_SECONDS = 1.0+OPENAPI_SHED_RESPONSE: dict[str, Any] = {+ "description": "The server is busy dispatching other runs. No run was created; retry after Retry-After seconds.",+ "headers": {"Retry-After": {"description": "Seconds to wait before retrying", "schema": {"type": "integer"}}},+}+++def _rejected_counter() -> Any | None:+ # Cached on first use rather than at import: the meter must be created after otel_setup has+ # installed the real MeterProvider or every add() is a no-op.+ if not settings.OTEL_METRICS_ENABLED:+ return None+ try:+ meter = metrics.get_meter("skyvern.run_submission_gate")+ return meter.create_counter(+ RUN_SUBMISSION_REJECTED_COUNTER,+ unit="{request}",+ description="Run submissions answered 503 because no dispatch slot freed in time",+ )+ except Exception as e:+ LOG.warning("Failed to initialize run submission rejection counter", error=str(e))+ return None+++class RunSubmissionGate:+ """A dispatching submission holds a pooled connection for most of its life, so an unbounded burst+ on one process drains the pool for every route on it; a caller waiting here holds no connection."""++ def __init__(self, limit: int, wait_seconds: float) -> None:+ self._limit = limit+ self._wait_seconds = wait_seconds+ self._semaphore: asyncio.Semaphore | None = None+ self._loop: asyncio.AbstractEventLoop | None = None+ self._in_flight = 0+ self._waiting = 0++ @property+ def enabled(self) -> bool:+ return self._limit > 0++ @property+ def in_flight(self) -> int:+ return self._in_flight++ @property+ def waiting(self) -> int:+ return self._waiting++ def _semaphore_for_running_loop(self) -> asyncio.Semaphore:+ # A semaphore binds to the loop that first waits on it, and tests run each case on a new loop.+ loop = asyncio.get_running_loop()+ if self._semaphore is None or self._loop is not loop:+ self._semaphore = asyncio.Semaphore(self._limit)+ self._loop = loop+ return self._semaphore++ @asynccontextmanager+ async def slot(self, enforce: bool = True) -> AsyncIterator[None]:+ # Counted even when not enforced, so the in-flight gauge can size a limit before one is set.+ semaphore = await self._acquire() if enforce and self.enabled else None+ self._in_flight += 1+ try:+ yield+ finally:+ self._in_flight -= 1+ if semaphore is not None:+ semaphore.release()++ async def _acquire(self) -> asyncio.Semaphore:+ semaphore = self._semaphore_for_running_loop()+ wait_started = time.monotonic()+ self._waiting += 1+ try:+ async with asyncio.timeout(self._wait_seconds):+ await semaphore.acquire()+ except TimeoutError:+ self._record_rejection()+ mark_request_load_shed()+ raise HTTPException(+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,+ detail="The server is busy dispatching other runs. No run was created; retry this request.",+ headers={"Retry-After": str(RETRY_AFTER_SECONDS)},+ ) from None+ finally:+ self._waiting -= 1+ trace.get_current_span().set_attribute(+ SLOT_WAIT_SPAN_ATTRIBUTE, round((time.monotonic() - wait_started) * 1000, 1)+ )+ return semaphore++ def _record_rejection(self) -> None:+ with contained_effect("record run submission rejection"):+ counter = _rejected_counter()+ if counter is not None:+ counter.add(1, task_metric_attributes())+++_GATE = RunSubmissionGate(settings.RUN_SUBMISSION_MAX_CONCURRENCY, settings.RUN_SUBMISSION_SLOT_WAIT_SECONDS)+++def _observe_in_flight(_options: Any) -> Iterable[metrics.Observation]:+ return [metrics.Observation(_GATE.in_flight, task_metric_attributes())]+++def _observe_waiting(_options: Any) -> Iterable[metrics.Observation]:+ return [metrics.Observation(_GATE.waiting, task_metric_attributes())]+++def _register_gauges() -> None:+ # Registered on first use for the same reason as the counter, and once per process: the meter+ # reports a second instrument under the same name as a conflict.+ if not settings.OTEL_METRICS_ENABLED:+ return+ try:+ meter = metrics.get_meter("skyvern.run_submission_gate")+ meter.create_observable_gauge(+ RUN_SUBMISSION_IN_FLIGHT_GAUGE,+ callbacks=[_observe_in_flight],+ unit="{request}",+ description="Run submissions dispatching on this process, whether or not the gate is on",+ )+ meter.create_observable_gauge(+ RUN_SUBMISSION_WAITING_GAUGE,+ callbacks=[_observe_waiting],+ unit="{request}",+ description="Run submissions waiting for a dispatch slot on this process",+ )+ except Exception as e:+ LOG.warning("Failed to initialize run submission gauges", error=str(e))+++async def _disabled_by_kill_switch(organization_id: str) -> bool:+ """Runtime off switch: if a fleet-wide dispatch slowdown (Temporal, Redis) holds every slot, the gate+ would shed every submission above the limit, and changing the setting needs a restart.+ Anything other than an explicit True leaves the setting in charge."""+ try:+ # A cache miss can read the flag snapshot from the database, which is the resource a burst exhausts.+ disabled = await asyncio.wait_for(+ app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(+ DISABLE_RUN_SUBMISSION_GATE_FLAG,+ organization_id,+ properties={"organization_id": organization_id},+ ),+ _KILL_SWITCH_LOOKUP_TIMEOUT_SECONDS,+ )+ except Exception:+ LOG.warning(+ "Failed to check the run submission gate kill switch; the gate stays on",+ organization_id=organization_id,+ exc_info=True,+ )+ return False+ return disabled is True+++@asynccontextmanager+async def run_submission_slot(organization_id: str) -> AsyncIterator[None]:+ _register_gauges()+ enforce = _GATE.enabled and not await _disabled_by_kill_switch(organization_id)+ async with _GATE.slot(enforce=enforce):+ yielddiff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex f8c60c1bf..04c25da46 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -65,6 +65,7 @@ from skyvern.forge.sdk.artifact.storage.base import artifact_filename_from_urifrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.curl_converter import curl_to_http_request_block_paramsfrom skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory+from skyvern.forge.sdk.core.run_submission_gate import OPENAPI_SHED_RESPONSE, run_submission_slotfrom skyvern.forge.sdk.core.security import generate_skyvern_signaturefrom skyvern.forge.sdk.db.enums import (OrganizationAuthTokenType,@@ -676,6 +677,7 @@ def _hydrate_run_request_for_response(responses={200: {"description": "Successfully ran agent"},400: {"description": "Invalid agent run request"},+ 503: OPENAPI_SHED_RESPONSE,},)@base_router.post("/run/agents/", include_in_schema=False)@@ -709,53 +711,56 @@ async def run_workflow(legacy_workflow_request = _workflow_run_request_to_legacy_request(workflow_run_request)trigger_type = workflow_run_trigger_type_from_user_agent(x_user_agent)- try:- workflow_run = await workflow_service.run_workflow(- workflow_id=workflow_id,- organization=current_org,- workflow_request=legacy_workflow_request,- template=template,- version=None,- max_steps=x_max_steps_override,- api_key=x_api_key,- request_id=request_id,- request=request,- background_tasks=background_tasks,- trigger_type=trigger_type,- tag_write_context=_tag_write_context_from_caller(caller),- created_by=user_id,- refuse_unusable_parameters_before_create=True,+ # The slot covers every database read after dispatch too: a checkout timeout there would answer 500 for+ # a run that is already queued, and the client's retry would start it twice.+ async with run_submission_slot(current_org.organization_id):+ try:+ workflow_run = await workflow_service.run_workflow(+ workflow_id=workflow_id,+ organization=current_org,+ workflow_request=legacy_workflow_request,+ template=template,+ version=None,+ max_steps=x_max_steps_override,+ api_key=x_api_key,+ request_id=request_id,+ request=request,+ background_tasks=background_tasks,+ trigger_type=trigger_type,+ tag_write_context=_tag_write_context_from_caller(caller),+ created_by=user_id,+ refuse_unusable_parameters_before_create=True,+ )+ except MissingBrowserAddressError as e:+ raise HTTPException(status_code=400, detail=str(e)) from e+ await uploaded_file_service.attach_files_to_run(+ file_ids=workflow_run_request.file_ids or [],+ organization_id=current_org.organization_id,+ run_id=workflow_run.workflow_run_id,+ )+ background_tasks.add_task(+ app.AGENT_FUNCTION.on_run_created,+ organization_id=current_org.organization_id,+ run_id=workflow_run.workflow_run_id,+ run_type=RunType.workflow_run,+ caller_type=caller.caller_type,)- except MissingBrowserAddressError as e:- raise HTTPException(status_code=400, detail=str(e)) from e- await uploaded_file_service.attach_files_to_run(- file_ids=workflow_run_request.file_ids or [],- organization_id=current_org.organization_id,- run_id=workflow_run.workflow_run_id,- )- background_tasks.add_task(- app.AGENT_FUNCTION.on_run_created,- organization_id=current_org.organization_id,- run_id=workflow_run.workflow_run_id,- run_type=RunType.workflow_run,- caller_type=caller.caller_type,- )-- if settings.OTEL_ENABLED:- span = trace.get_current_span()- if span:- if workflow_run.workflow_run_id:- span.set_attribute("workflow_run_id", workflow_run.workflow_run_id)- if workflow_run.workflow_id:- span.set_attribute("workflow_id", workflow_run.workflow_id)- # Hydrate the returned request from the persisted run: workflow title (when the workflow exists) and- # the effective browser_type, so a run that omitted browser_type reports the inherited engine- # instead of the request's null.- workflow = await app.WORKFLOW_SERVICE.get_workflow(- workflow_id=workflow_run.workflow_id,- organization_id=current_org.organization_id,- )+ if settings.OTEL_ENABLED:+ span = trace.get_current_span()+ if span:+ if workflow_run.workflow_run_id:+ span.set_attribute("workflow_run_id", workflow_run.workflow_run_id)+ if workflow_run.workflow_id:+ span.set_attribute("workflow_id", workflow_run.workflow_id)++ # Hydrate the returned request from the persisted run: workflow title (when the workflow exists) and+ # the effective browser_type, so a run that omitted browser_type reports the inherited engine+ # instead of the request's null.+ workflow = await app.WORKFLOW_SERVICE.get_workflow(+ workflow_id=workflow_run.workflow_id,+ organization_id=current_org.organization_id,+ )workflow_run_request_hydrated = _hydrate_run_request_for_response(workflow_run_request, workflow_run, workflow)return WorkflowRunResponse(@@ -4102,6 +4107,7 @@ def _user_writable_run_metadata(run_metadata: dict[str, str] | None) -> dict[str200: {"description": "Successfully retried workflow run"},400: {"description": "Workflow run is not retryable"},404: {"description": "Workflow run not found"},+ 503: OPENAPI_SHED_RESPONSE,},)@base_router.post("/workflows/runs/{workflow_run_id}/retry/", include_in_schema=False)@@ -4198,25 +4204,26 @@ async def retry_workflow_run(if is_job_recipe_workflow_run_trigger_type(original_trigger_type)else workflow_run_trigger_type_from_user_agent(x_user_agent))- try:- workflow_run = await workflow_service.run_workflow(- workflow_id=original_workflow_run.workflow_permanent_id,- organization=current_org,- workflow_request=legacy_workflow_request,- template=template,- version=original_workflow.version,- max_steps=x_max_steps_override,- api_key=x_api_key,- request_id=context.request_id,- request=request,- background_tasks=background_tasks,- trigger_type=trigger_type,- ignore_inherited_workflow_system_prompt=original_workflow_run.ignore_inherited_workflow_system_prompt,- tag_write_context=_tag_write_context_from_caller(caller),- created_by=user_id,- )- except MissingBrowserAddressError as e:- raise HTTPException(status_code=400, detail=str(e)) from e+ async with run_submission_slot(current_org.organization_id):+ try:+ workflow_run = await workflow_service.run_workflow(+ workflow_id=original_workflow_run.workflow_permanent_id,+ organization=current_org,+ workflow_request=legacy_workflow_request,+ template=template,+ version=original_workflow.version,+ max_steps=x_max_steps_override,+ api_key=x_api_key,+ request_id=context.request_id,+ request=request,+ background_tasks=background_tasks,+ trigger_type=trigger_type,+ ignore_inherited_workflow_system_prompt=original_workflow_run.ignore_inherited_workflow_system_prompt,+ tag_write_context=_tag_write_context_from_caller(caller),+ created_by=user_id,+ )+ except MissingBrowserAddressError as e:+ raise HTTPException(status_code=400, detail=str(e)) from ebackground_tasks.add_task(app.AGENT_FUNCTION.on_run_created,organization_id=current_org.organization_id,@@ -4802,6 +4809,7 @@ async def get_actions(openapi_extra={"x-fern-sdk-method-name": "run_workflow_legacy",},+ responses={503: OPENAPI_SHED_RESPONSE},)@legacy_base_router.post("/workflows/{workflow_id}/run/",@@ -4836,25 +4844,26 @@ async def run_workflow_legacy(await app.RATE_LIMITER.rate_limit_submit_run(current_org.organization_id)legacy_trigger_type = workflow_run_trigger_type_from_user_agent(x_user_agent)- try:- workflow_run = await workflow_service.run_workflow(- workflow_id=workflow_id,- organization=current_org,- workflow_request=workflow_request,- template=template,- version=version,- max_steps=x_max_steps_override,- api_key=x_api_key,- request_id=request_id,- request=request,- background_tasks=background_tasks,- trigger_type=legacy_trigger_type,- tag_write_context=_tag_write_context_from_caller(caller),- created_by=user_id,- refuse_unusable_parameters_before_create=True,- )- except MissingBrowserAddressError as e:- raise HTTPException(status_code=400, detail=str(e)) from e+ async with run_submission_slot(current_org.organization_id):+ try:+ workflow_run = await workflow_service.run_workflow(+ workflow_id=workflow_id,+ organization=current_org,+ workflow_request=workflow_request,+ template=template,+ version=version,+ max_steps=x_max_steps_override,+ api_key=x_api_key,+ request_id=request_id,+ request=request,+ background_tasks=background_tasks,+ trigger_type=legacy_trigger_type,+ tag_write_context=_tag_write_context_from_caller(caller),+ created_by=user_id,+ refuse_unusable_parameters_before_create=True,+ )+ except MissingBrowserAddressError as e:+ raise HTTPException(status_code=400, detail=str(e)) from ebackground_tasks.add_task(app.AGENT_FUNCTION.on_run_created,organization_id=current_org.organization_id,diff --git a/skyvern/forge/sdk/schemas/persistent_browser_sessions.py b/skyvern/forge/sdk/schemas/persistent_browser_sessions.pyindex 11d40447c..6f7d808b7 100644--- a/skyvern/forge/sdk/schemas/persistent_browser_sessions.py+++ b/skyvern/forge/sdk/schemas/persistent_browser_sessions.py@@ -4,8 +4,15 @@ from enum import StrEnumfrom pydantic import BaseModel, ConfigDict, field_validator+from skyvern.exceptions import (+ BrowserSessionClosed,+ BrowserSessionExpired,+ BrowserSessionStartupTimeout,+ SkyvernHTTPException,+)from skyvern.forge.sdk.db.utils import deserialize_proxy_locationfrom skyvern.schemas.browser_session_close import BrowserSessionCloseReason+from skyvern.schemas.browser_session_timeouts import lived_full_lifetimefrom skyvern.schemas.proxy_pinning import validate_proxy_session_idfrom skyvern.schemas.runs import GeoTarget, ProxyLocation, ProxyLocationInput@@ -160,6 +167,40 @@ class PersistentBrowserSession(BaseModel):return not self.profile_read_only and bool(self.generate_browser_profile or self.browser_profile_id)+def unusable_browser_session_error(+ session: PersistentBrowserSession, *, refused_at_submission: bool = False+) -> SkyvernHTTPException | None:+ """The error a run sent to this session fails (or, at submission, is refused) with, or None while it is usable.++ The single source for both run submission and the session managers' lease, so the two cannot disagree."""+ if not (+ is_final_status(session.status) or session.completed_at is not None or session.close_requested_at is not None+ ):+ return None+ session_id = session.persistent_browser_session_id+ timed_out = session.status == PersistentBrowserSessionStatus.timeout+ if timed_out and session.started_at is None:+ return BrowserSessionStartupTimeout(session_id)+ # A session that reaches its lifetime ends with status timeout, or is closed as completed with close reason+ # expired, a reason a retirement can also stamp before the lifetime is up.+ if timed_out or (+ session.close_reason == BrowserSessionCloseReason.expired+ and lived_full_lifetime(+ started_at=session.started_at, ended_at=session.completed_at, timeout_minutes=session.timeout_minutes+ )+ ):+ return BrowserSessionExpired(+ session_id,+ started_at=session.started_at,+ ended_at=session.completed_at,+ timeout_minutes=session.timeout_minutes,+ created_by=session.created_by,+ bound=session.bound_key is not None or session.bound_workflow_permanent_id is not None,+ refused_at_submission=refused_at_submission,+ )+ return BrowserSessionClosed(session_id, reason=f"is no longer usable (status: {session.status})")++class FreshExitOutcome(StrEnum):distinct_verified = "distinct_verified"no_alternate = "no_alternate"diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 353a67514..956615468 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -57,6 +57,7 @@ from skyvern.exceptions import (BrowserProfileNotFound,BrowserSessionAlreadyOccupiedError,BrowserSessionClosed,+ BrowserSessionExpired,BrowserSessionNotFound,BrowserSessionNotRenewable,BrowserSessionStartupTimeout,@@ -83,6 +84,7 @@ from skyvern.exceptions import (from skyvern.forge import appfrom skyvern.forge.failure_classifier import (BROWSER_SESSION_CLOSED_REASON_CODE,+ BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE,BROWSER_SESSION_STARTUP_TIMEOUT_REASON_CODE,FailureCategory,classify_from_failure_reason,@@ -133,10 +135,12 @@ from skyvern.forge.sdk.schemas.credentials import Credential, credential_auto_prfrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import (+ API_BROWSER_SESSION_CREATED_BY,FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE,SESSION_RETIREMENT_RUNNABLE_TYPE,PersistentBrowserSession,is_final_status,+ unusable_browser_session_error,)from skyvern.forge.sdk.schemas.tasks import Task, TaskStatusfrom skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock, WorkflowRunTimeline, WorkflowRunTimelineType@@ -286,7 +290,7 @@ from skyvern.forge.sdk.workflow.status_mapping import (from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definitionfrom skyvern.schemas.browser_session_close import BrowserSessionCloseReasonfrom skyvern.schemas.browser_session_kind import BrowserSessionKind-from skyvern.schemas.browser_session_timeouts import REUSE_MIN_REMAINING_LIFETIME_SECONDS+from skyvern.schemas.browser_session_timeouts import REUSE_MIN_REMAINING_LIFETIME_SECONDS, lived_full_lifetimefrom skyvern.schemas.proxy_pinning import (derive_proxy_session_id,redact_proxy_session_id,@@ -1704,10 +1708,45 @@ def _build_managed_browser_profile_name(workflow_title: str | None, rendered_keyreturn f"{title}{suffix}"-def _browser_lease_failure_category(exc: Exception) -> list[dict] | None:+BROWSER_SESSION_EXPIRED_EXPLANATION = (+ "The browser session this run was sent to had already reached its time limit, so the run never started. "+ "Running it again with the same browser session will fail the same way. Create a new browser session and run "+ "again, or leave the browser session empty to start a fresh browser."+)+++def _session_expired_before_run(exc: Exception, workflow_run: WorkflowRun) -> BrowserSessionExpired | None:+ """The session, when the caller sent this run to a session of their own that had already run out its lifetime.++ Sessions Skyvern created (reuse and editor prewarm sessions carry a binding and no creator), sessions closed+ early, and runs that queued while the session was still alive stay platform-attributed. So do sessions a+ signed-in user made in the UI, whose creator is their user id: only creator null or api counts as the caller's."""+ if not isinstance(exc, BrowserSessionExpired):+ return None+ if exc.bound or exc.created_by not in (None, API_BROWSER_SESSION_CREATED_BY):+ return None+ if exc.ended_at is None or not lived_full_lifetime(+ started_at=exc.started_at, ended_at=exc.ended_at, timeout_minutes=exc.timeout_minutes+ ):+ return None+ if _as_utc(workflow_run.created_at) <= _as_utc(exc.ended_at):+ return None+ return exc+++def _browser_lease_failure_category(exc: Exception, workflow_run: WorkflowRun) -> list[dict] | None:"""The lease seam still holds the typed exception; persist its identity so a reader does nothave to rediscover it from prose or from the session row, which closes the same way afterevery run."""+ if _session_expired_before_run(exc, workflow_run) is not None:+ return [+ {+ "category": FailureCategory.BROWSER_SESSION_EXPIRED.value,+ "confidence_float": 1.0,+ "reason_code": BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE,+ "reasoning": BROWSER_SESSION_EXPIRED_EXPLANATION,+ }+ ]if isinstance(exc, BrowserSessionClosed):reason_code = BROWSER_SESSION_CLOSED_REASON_CODEreasoning = "The browser session had already closed before the run could lease it"@@ -7342,9 +7381,13 @@ class WorkflowService:browser_session_id=browser_session_id,workflow_run_id=workflow_run_id,)+ expired_before_run = _session_expired_before_run(e, workflow_run)failure_reason = (- f"Failed to begin browser session for workflow run: {get_user_facing_exception_message(e)}"+ expired_before_run.expired_before_run_message(run_created=True)+ if expired_before_run is not None+ else f"Failed to begin browser session for workflow run: {get_user_facing_exception_message(e)}")+ lease_failure_category = _browser_lease_failure_category(e, workflow_run)if browser_dispatch:failed = await self._mark_workflow_run_as_failed_for_dispatch(workflow_run_id=workflow_run_id,@@ -7352,7 +7395,7 @@ class WorkflowService:attempt_number=attempt_number,dispatch_claim_started_at=dispatch_claim_started_at,failure_reason=failure_reason,- failure_category=_browser_lease_failure_category(e),+ failure_category=lease_failure_category,)if failed is None:return await self.get_workflow_run(workflow_run_id, organization_id)@@ -7361,7 +7404,7 @@ class WorkflowService:workflow_run = await self.mark_workflow_run_as_failed(workflow_run_id=workflow_run_id,failure_reason=failure_reason,- failure_category=_browser_lease_failure_category(e),+ failure_category=lease_failure_category,)await self.clean_up_workflow(workflow=workflow,@@ -11656,6 +11699,11 @@ class WorkflowService:)if not browser_session:raise BrowserSessionNotFound(browser_session_id=workflow_request.browser_session_id)+ # Nothing between here and the lease swaps or revives a caller's session, so a run sent to+ # one that has already ended can only fail; refuse it before a run row exists.+ unusable = unusable_browser_session_error(browser_session, refused_at_submission=True)+ if unusable is not None:+ raise unusable# Auto-propagate profile from session when not explicitly providedif not browser_profile_id and browser_session.browser_profile_id:browser_profile_id = browser_session.browser_profile_iddiff --git a/skyvern/schemas/browser_session_timeouts.py b/skyvern/schemas/browser_session_timeouts.pyindex fea7412b9..5be19760c 100644--- a/skyvern/schemas/browser_session_timeouts.py+++ b/skyvern/schemas/browser_session_timeouts.py@@ -7,6 +7,7 @@ share these rules without pulling the heavier ``browser_sessions`` schema modulefrom __future__ import annotations+from datetime import UTC, datetimefrom typing import overloadMIN_TIMEOUT = 5@@ -24,6 +25,9 @@ RENEWAL_MIN_REMAINING_SECONDS = 60EXTENSION_MIN_REMAINING_SECONDS = 120# A found reusable session with less lifetime left than this is retired and replaced rather than handed to a run.REUSE_MIN_REMAINING_LIFETIME_SECONDS = 30 * 60+# How far short of its budget a session may end and still count as having run it out: a renewal closes a+# session with under RENEWAL_MIN_REMAINING_SECONDS left, and a deadline is noticed on a periodic heartbeat.+LIFETIME_END_TOLERANCE_SECONDS = 120DEFAULT_TIMEOUT = 60MAX_TIMEOUT_EXCEEDED_MESSAGE = (@@ -72,6 +76,18 @@ def creation_timeout_minutes(requested_minutes: int | None) -> int | None:return min(requested_minutes, MAX_TIMEOUT)+def _as_utc(moment: datetime) -> datetime:+ return moment.replace(tzinfo=UTC) if moment.tzinfo is None else moment.astimezone(UTC)+++def lived_full_lifetime(*, started_at: datetime | None, ended_at: datetime | None, timeout_minutes: int | None) -> bool:+ """Whether a session ended only after running out its whole budget, rather than being closed early."""+ if started_at is None or ended_at is None or not timeout_minutes:+ return False+ lived_seconds = (_as_utc(ended_at) - _as_utc(started_at)).total_seconds()+ return lived_seconds >= timeout_minutes * 60 - LIFETIME_END_TOLERANCE_SECONDS++def lifetime_cap_seconds(base_timeout_seconds: float, *, floor_seconds: float = MAX_LIFETIME_SECONDS) -> float:"""The hard cap on an extendable session's life.diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex 8fd3d7270..312b8ae66 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -14,6 +14,7 @@ from sqlalchemy.exc import OperationalErrorfrom skyvern.config import settingsfrom skyvern.constants import MINI_GOAL_TEMPLATEfrom skyvern.exceptions import (+ BrowserSessionNotFound,FailedToSendWebhook,ScreenshotTargetClosed,TaskTerminationError,@@ -39,6 +40,7 @@ from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signaturefrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, WorkflowRunTriggerTypefrom skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.schemas.persistent_browser_sessions import unusable_browser_session_errorfrom skyvern.forge.sdk.schemas.task_v2 import (TASK_V2_TIMEOUT_WEBHOOK_DELIVERED_SENTINEL,TaskV2,@@ -349,6 +351,17 @@ async def initialize_task_v2(await _validate_task_v2_model_for_org(organization, model)if user_url:user_url = await asyncio.to_thread(validate_fetch_url, user_url)+ if browser_session_id:+ # The workflow run created below refuses this too, but only after the task and its workflow exist.+ browser_session = await app.DATABASE.browser_sessions.get_persistent_browser_session(+ session_id=browser_session_id,+ organization_id=organization.organization_id,+ )+ if not browser_session:+ raise BrowserSessionNotFound(browser_session_id=browser_session_id)+ unusable = unusable_browser_session_error(browser_session, refused_at_submission=True)+ if unusable is not None:+ raise unusabletask_v2 = await app.DATABASE.observer.create_task_v2(prompt=user_prompt,diff --git a/skyvern/utils/metric_attributes.py b/skyvern/utils/metric_attributes.pynew file mode 100644index 000000000..eae7c706e--- /dev/null+++ b/skyvern/utils/metric_attributes.py@@ -0,0 +1,18 @@+"""Attributes that identify the process emitting a per-process OTEL metric."""++import os+++def task_metric_attributes() -> dict[str, str]:+ """Identify the emitting process so a per-process gauge does not collapse into one+ fleet-wide series. ``HOSTNAME`` is the fallback: it is unique per ECS ``awsvpc`` task+ and per Kubernetes pod."""+ container = os.getenv("ECS_CONTAINER_NAME")+ if not container:+ task_arn = os.getenv("ECS_TASK_ARN")+ if task_arn:+ container = task_arn.split("/")[-1]+ if not container:+ container = os.getenv("HOSTNAME")++ return {"container": container} if container else {}diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.pyindex 7a3b912e8..5721134e8 100644--- a/skyvern/webeye/default_persistent_sessions_manager.py+++ b/skyvern/webeye/default_persistent_sessions_manager.py@@ -39,6 +39,7 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import (PersistentBrowserSessionStatus,PersistentBrowserType,is_final_status,+ unusable_browser_session_error,)from skyvern.forge.sdk.streaming.registries import stream_tombstone_holds_session_leasefrom skyvern.schemas.browser_session_close import BrowserSessionCloseReason@@ -471,8 +472,9 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):if persistent_browser_session is None:raise Exception(f"Persistent browser session not found for {browser_session_id}")- if is_final_status(persistent_browser_session.status):- raise BrowserSessionClosed(browser_session_id)+ unusable = unusable_browser_session_error(persistent_browser_session)+ if unusable is not None:+ raise unusablerunnable_generation_id = expected_runnable_generation_id or uuid.uuid4().hexawait self.occupy_browser_session(@@ -904,12 +906,12 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):current = await self.database.browser_sessions.get_persistent_browser_session(session_id, organization_id)- if current is not None and (- current.completed_at is not None- or current.close_requested_at is not None- or is_final_status(current.status)- ):- raise BrowserSessionClosed(session_id) from None+ unusable = unusable_browser_session_error(current) if current is not None else None+ if unusable is not None:+ # A startup timeout stays a BrowserSessionClosed here: the reuse path catches only that type.+ raise (+ unusable if isinstance(unusable, BrowserSessionClosed) else BrowserSessionClosed(session_id)+ ) from Noneexcept BaseException as classification_error:if BROWSER_RETIREMENT_DENIED_NOTE in getattr(error, "__notes__", ()):classification_error.add_note(BROWSER_RETIREMENT_DENIED_NOTE)diff --git a/tests/unit/test_browser_session_timeouts.py b/tests/unit/test_browser_session_timeouts.pyindex cc6374c18..b6f9f1d3a 100644--- a/tests/unit/test_browser_session_timeouts.py+++ b/tests/unit/test_browser_session_timeouts.py@@ -1,13 +1,20 @@-from datetime import datetime+from datetime import UTC, datetime, timedeltafrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom pydantic import ValidationError+from skyvern.exceptions import BrowserSessionClosed, BrowserSessionExpired, BrowserSessionStartupTimeoutfrom skyvern.forge.sdk.routes import browser_sessions as browser_sessions_routes+from skyvern.forge.sdk.schemas.persistent_browser_sessions import (+ PersistentBrowserSession,+ unusable_browser_session_error,+)+from skyvern.schemas.browser_session_close import BrowserSessionCloseReasonfrom skyvern.schemas.browser_session_timeouts import (DEFAULT_TIMEOUT,+ LIFETIME_END_TOLERANCE_SECONDS,MAX_EXTENDED_LIFETIME_SECONDS,MAX_EXTENDED_TIMEOUT,MAX_LIFETIME_SECONDS,@@ -16,6 +23,7 @@ from skyvern.schemas.browser_session_timeouts import (MIN_TIMEOUT,creation_timeout_minutes,lifetime_cap_seconds,+ lived_full_lifetime,max_lifetime_exceeded_warning,max_timeout_exceeded_warning,seconds_until_expiry,@@ -290,3 +298,73 @@ def test_requested_timeout_below_the_minimum_is_still_rejected() -> None:def test_timeout_defaults_and_explicit_none_are_untouched() -> None:assert CreateBrowserSessionRequest().timeout == DEFAULT_TIMEOUTassert CreateBrowserSessionRequest(timeout=None).timeout is None+++ ("lived_seconds", "expected"),+ [+ (60 * 60, True),+ (60 * 60 - LIFETIME_END_TOLERANCE_SECONDS, True),+ (60 * 60 - LIFETIME_END_TOLERANCE_SECONDS - 1, False),+ (60, False),+ ],+)+def test_a_session_lived_its_full_lifetime_only_within_the_end_tolerance(lived_seconds: int, expected: bool) -> None:+ started_at = datetime(2026, 9, 28, 13, 0, tzinfo=UTC)+ ended_at = started_at + timedelta(seconds=lived_seconds)++ assert lived_full_lifetime(started_at=started_at, ended_at=ended_at, timeout_minutes=60) is expected+ assert not lived_full_lifetime(started_at=None, ended_at=ended_at, timeout_minutes=60)+ assert not lived_full_lifetime(started_at=started_at, ended_at=ended_at, timeout_minutes=None)+++def _ended_session(**overrides: object) -> PersistentBrowserSession:+ started_at = datetime(2026, 9, 28, 13, 0, tzinfo=UTC)+ fields: dict[str, object] = {+ "persistent_browser_session_id": "pbs_x",+ "organization_id": "o_test",+ "status": "timeout",+ "timeout_minutes": 60,+ "started_at": started_at,+ "completed_at": started_at + timedelta(minutes=60),+ "created_at": started_at,+ "modified_at": started_at,+ }+ return PersistentBrowserSession(**{**fields, **overrides})+++ ("overrides", "expected"),+ [+ pytest.param({}, BrowserSessionExpired, id="timeout"),+ pytest.param({"started_at": None}, BrowserSessionStartupTimeout, id="timeout-before-start"),+ pytest.param(+ {"status": "completed", "close_reason": BrowserSessionCloseReason.expired},+ BrowserSessionExpired,+ id="oss-reaper-full-lifetime",+ ),+ pytest.param(+ {+ "status": "completed",+ "close_reason": BrowserSessionCloseReason.expired,+ "completed_at": datetime(2026, 9, 28, 13, 10, tzinfo=UTC),+ },+ BrowserSessionClosed,+ id="expired-reason-stamped-early",+ ),+ pytest.param({"status": "completed"}, BrowserSessionClosed, id="completed-by-the-caller"),+ pytest.param({"status": "failed"}, BrowserSessionClosed, id="failed"),+ pytest.param(+ {"status": "running", "completed_at": None, "close_requested_at": datetime(2026, 9, 28, 13, 5, tzinfo=UTC)},+ BrowserSessionClosed,+ id="close-requested",+ ),+ pytest.param({"status": "running", "completed_at": None}, type(None), id="live"),+ ],+)+def test_unusable_browser_session_error_tells_expiry_from_an_early_close(+ overrides: dict[str, object], expected: type+) -> None:+ error = unusable_browser_session_error(_ended_session(**overrides))++ assert type(error) is expecteddiff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 8e31f721e..77215e8cc 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -7,7 +7,7 @@ import jsonimport timefrom collections.abc import AsyncIterator, Callable, Sequencefrom dataclasses import dataclass-from datetime import datetime, timezone+from datetime import datetime, timedelta, timezonefrom pathlib import Pathfrom types import SimpleNamespacefrom typing import Any, ClassVar, Self@@ -48,6 +48,7 @@ from openai.types.responses.response_prompt_param import ResponsePromptParamfrom pydantic import BaseModelfrom structlog.testing import capture_logs+from skyvern.exceptions import BrowserSessionExpiredfrom skyvern.forge import app as forge_appfrom skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigError, LLMProviderErrorfrom skyvern.forge.sdk.copilot import agent as agent_module@@ -6064,6 +6065,53 @@ class TestRunBlocksCredentialApproval:assert dispatched["browser_session_id"] == "pbs_carried"+ @pytest.mark.asyncio+ async def test_run_blocks_reports_a_resumed_browser_refused_at_submission_as_already_closed(+ self, monkeypatch+ ) -> None:+ # Submission now refuses a session that has ended, so no run exists to carry the lease-seam+ # reason code; the turn still needs the typed browser loss that offers the fresh-browser retry.+ from skyvern.forge.sdk.copilot.tools import _run_blocks_and_collect_debug+ from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module+ from skyvern.services import workflow_service as workflow_service_module++ workflow = self._typed_resume_workflow()+ organization = Organization(+ organization_id="org-1",+ organization_name="org",+ created_at=datetime.now(timezone.utc),+ modified_at=datetime.now(timezone.utc),+ )+ database = self._db(workflow=workflow, organization_lookup=organization)+ database.workflow_params = SimpleNamespace(+ get_workflow_output_parameters=AsyncMock(return_value=[workflow.get_output_parameter("login")])+ )+ ended_at = datetime.now(timezone.utc) - timedelta(minutes=5)+ refused = BrowserSessionExpired(+ "pbs_carried",+ started_at=ended_at - timedelta(minutes=60),+ ended_at=ended_at,+ timeout_minutes=60,+ created_by="copilot",+ refused_at_submission=True,+ )+ monkeypatch.setattr(run_execution_module.app, "DATABASE", database)+ monkeypatch.setattr(+ run_execution_module.app,+ "WORKFLOW_SERVICE",+ SimpleNamespace(get_workflow_parameters=AsyncMock(return_value=[])),+ )+ monkeypatch.setattr(workflow_service_module, "prepare_workflow", AsyncMock(side_effect=refused))++ ctx = _ctx(browser_session_id=None)+ ctx.frontier_resume_session_id = "pbs_carried"++ result = await _run_blocks_and_collect_debug({"block_labels": ["login"], "parameters": {}}, ctx)++ assert result["ok"] is False+ assert result["data"]["build_test_connect_failure"]["state"] == "already_closed"+ assert result["data"]["browser_session_id"] == "pbs_carried"+@pytest.mark.asyncioasync def test_run_blocks_rejects_unapproved_workflow_credential_before_dispatch(self, monkeypatch) -> None:from skyvern.forge.sdk.copilot.tools import _run_blocks_and_collect_debugdiff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.pyindex 86bb5fc6d..961706ccd 100644--- a/tests/unit/test_failure_classifier.py+++ b/tests/unit/test_failure_classifier.py@@ -2,6 +2,8 @@ from __future__ import annotationsimport jsonimport math+from datetime import UTC, datetime, timedelta+from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMockimport pytest@@ -1025,16 +1027,26 @@ def test_derive_is_never_raises_by_construction_for_pathological_primary() -> Nodef test_browser_lease_producer_reason_codes_round_trip_through_attribution() -> None:# Drift guard: the real producer's emitted reason codes must survive derivation. If a new# browser-lease code is added without updating the shared allowlist, this fails.- from skyvern.exceptions import BrowserSessionClosed, BrowserSessionStartupTimeout+ from skyvern.exceptions import BrowserSessionClosed, BrowserSessionExpired, BrowserSessionStartupTimeoutfrom skyvern.forge.sdk.workflow.service import _browser_lease_failure_category- for exc in (- BrowserSessionClosed(browser_session_id="pbs_x"),- BrowserSessionStartupTimeout(browser_session_id="pbs_x"),+ started_at = datetime(2026, 9, 28, 13, 1, tzinfo=UTC)+ ended_at = started_at + timedelta(minutes=240)+ workflow_run = SimpleNamespace(created_at=ended_at + timedelta(hours=6))+ expired = BrowserSessionExpired(+ "pbs_x", started_at=started_at, ended_at=ended_at, timeout_minutes=240, created_by="api"+ )+ for exc, component in (+ (BrowserSessionClosed(browser_session_id="pbs_x"), "browser"),+ (BrowserSessionStartupTimeout(browser_session_id="pbs_x"), "browser"),+ (expired, "non_infra"),):- fc = _browser_lease_failure_category(exc)+ fc = _browser_lease_failure_category(exc, workflow_run)assert fc is not Nonedoc = derive_failure_attribution(fc)assert doc["reason_code"] == fc[0]["reason_code"]assert doc["evidence_source"] == "reason_code"- assert doc["primary_infra_component"] == "browser"+ assert doc["primary_infra_component"] == component+ assert derive_failure_attribution(_browser_lease_failure_category(expired, workflow_run))["failure_category"] == (+ FailureCategory.BROWSER_SESSION_EXPIRED+ )diff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.pyindex 15d671aa0..c409af5b4 100644--- a/tests/unit/test_request_logging.py+++ b/tests/unit/test_request_logging.py@@ -37,6 +37,7 @@ from skyvern.forge.request_logging import (log_raw_request_middleware,set_request_organization,)+from skyvern.forge.sdk.core.run_submission_gate import RunSubmissionGatefrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.services.org_auth_service import apply_request_org_context@@ -721,6 +722,16 @@ def _make_app(unhandled_exception_status: int = 500) -> FastAPI:async def payment_required() -> dict:raise HTTPException(status_code=402, detail="Payment Required")+ @app.post("/runs/shed")+ async def shed_run_submission() -> dict:+ gate = RunSubmissionGate(limit=1, wait_seconds=0.01)+ async with gate.slot(), gate.slot():+ return {"dispatched": True}++ @app.post("/runs/unavailable")+ async def unavailable() -> dict:+ raise HTTPException(status_code=503, detail="Service unavailable")+@app.post("/post-only")async def post_only() -> dict:return {"ok": True}@@ -1063,6 +1074,15 @@ class TestMiddlewareLogVolume:assert log_mock.warning.call_args.args[0] == "api.raw_request"assert log_mock.warning.call_args.kwargs["status_code"] == 403+ @pytest.mark.parametrize(("path", "level"), [("/runs/shed", "warning"), ("/runs/unavailable", "error")])+ def test_shed_run_submission_logs_at_warning_while_other_503s_stay_errors(+ self, log_mock: MagicMock, path: str, level: str+ ) -> None:+ response = TestClient(_make_app()).post(path)++ assert response.status_code == 503+ assert [(call[0], call.kwargs["status_code"]) for call in log_mock.mock_calls] == [(level, 503)]+def test_artifact_url_queries_are_redacted_from_logged_bodies(self, log_mock: MagicMock) -> None:client = TestClient(_make_app())response = client.post(diff --git a/tests/unit/test_run_submission_gate.py b/tests/unit/test_run_submission_gate.pynew file mode 100644index 000000000..f2720f7b8--- /dev/null+++ b/tests/unit/test_run_submission_gate.py@@ -0,0 +1,160 @@+import asyncio++import pytest+from fastapi import HTTPException++from skyvern.config import Settings+from skyvern.forge import app+from skyvern.forge.sdk.core import run_submission_gate+from skyvern.forge.sdk.core.run_submission_gate import (+ DISABLE_RUN_SUBMISSION_GATE_FLAG,+ RETRY_AFTER_SECONDS,+ RunSubmissionGate,+ run_submission_slot,+)+from tests.unit._workflow_block_engine_fakes import FakeExperimentationProvider++# A leaked slot must fail the test, not hang it: no pytest timeout is configured.+_BOUNDED_WAIT_SECONDS = 1+++async def _hold_slot(gate: RunSubmissionGate, entered: asyncio.Event, release: asyncio.Event) -> None:+ async with gate.slot():+ entered.set()+ await release.wait()+++async def test_submission_over_the_cap_waits_for_a_slot_instead_of_failing() -> None:+ gate = RunSubmissionGate(limit=1, wait_seconds=5)+ entered, release = asyncio.Event(), asyncio.Event()+ holder = asyncio.create_task(_hold_slot(gate, entered, release))+ await asyncio.wait_for(entered.wait(), _BOUNDED_WAIT_SECONDS)++ admitted = asyncio.Event()++ async def queued_submission() -> None:+ async with gate.slot():+ admitted.set()++ waiter = asyncio.create_task(queued_submission())+ with pytest.raises(TimeoutError):+ await asyncio.wait_for(asyncio.shield(admitted.wait()), timeout=0.05)+ assert (gate.in_flight, gate.waiting) == (1, 1)++ release.set()+ await asyncio.wait_for(asyncio.gather(holder, waiter), _BOUNDED_WAIT_SECONDS)+ assert admitted.is_set()+ assert (gate.in_flight, gate.waiting) == (0, 0)+++async def test_submission_that_gets_no_slot_in_time_is_rejected_before_it_dispatches() -> None:+ gate = RunSubmissionGate(limit=1, wait_seconds=0.05)+ dispatched = False++ async with gate.slot():+ with pytest.raises(HTTPException) as rejected:+ async with gate.slot():+ dispatched = True+ assert gate.waiting == 0++ assert not dispatched+ assert rejected.value.status_code == 503+ assert rejected.value.headers == {"Retry-After": str(RETRY_AFTER_SECONDS)}+ # The timed-out waiter must not have consumed the slot the holder then released.+ async with gate.slot():+ pass+++async def test_slot_is_released_when_dispatch_fails_or_the_request_is_cancelled() -> None:+ gate = RunSubmissionGate(limit=1, wait_seconds=0.05)++ with pytest.raises(RuntimeError):+ async with gate.slot():+ raise RuntimeError("dispatch failed")++ entered = asyncio.Event()+ cancelled = asyncio.create_task(_hold_slot(gate, entered, asyncio.Event()))+ await asyncio.wait_for(entered.wait(), _BOUNDED_WAIT_SECONDS)+ cancelled.cancel()+ with pytest.raises(asyncio.CancelledError):+ await cancelled+ assert gate.in_flight == 0++ # A leaked slot would make this wait out its 50 ms and raise the 503.+ async with gate.slot():+ pass+++ ("provider", "admitted"),+ [+ (FakeExperimentationProvider({DISABLE_RUN_SUBMISSION_GATE_FLAG: True}), True),+ (FakeExperimentationProvider(), False),+ (FakeExperimentationProvider(raise_error=True), False),+ ],+ ids=["kill_switch_on", "flag_absent", "flag_lookup_fails"],+)+async def test_kill_switch_lets_submissions_past_a_full_gate_and_anything_else_keeps_the_setting(+ monkeypatch: pytest.MonkeyPatch, provider: FakeExperimentationProvider, admitted: bool+) -> None:+ full_gate = RunSubmissionGate(limit=1, wait_seconds=0.05)+ monkeypatch.setattr(run_submission_gate, "_GATE", full_gate)+ monkeypatch.setattr(app, "EXPERIMENTATION_PROVIDER", provider)+ dispatched = False++ async with full_gate.slot():+ try:+ async with run_submission_slot("o_123"):+ dispatched = True+ except HTTPException as rejected:+ assert rejected.status_code == 503++ assert dispatched is admitted+++ ("limit", "provider"),+ [+ (Settings.model_fields["RUN_SUBMISSION_MAX_CONCURRENCY"].default, FakeExperimentationProvider()),+ (1, FakeExperimentationProvider({DISABLE_RUN_SUBMISSION_GATE_FLAG: True})),+ ],+ ids=["default_limit", "kill_switch_on"],+)+async def test_gate_that_is_off_never_waits_or_rejects_but_still_counts_submissions_in_flight(+ monkeypatch: pytest.MonkeyPatch, limit: int, provider: FakeExperimentationProvider+) -> None:+ gate = RunSubmissionGate(limit=limit, wait_seconds=0.05)+ monkeypatch.setattr(run_submission_gate, "_GATE", gate)+ monkeypatch.setattr(app, "EXPERIMENTATION_PROVIDER", provider)+ # More submissions than the limit, so a gate that enforced it would leave some waiting.+ entered = [asyncio.Event() for _ in range(limit + 5)]+ release = asyncio.Event()++ async def submission(index: int) -> None:+ async with run_submission_slot("o_123"):+ entered[index].set()+ await release.wait()+ if index == 0:+ raise RuntimeError("dispatch failed")++ submissions = [asyncio.create_task(submission(index)) for index in range(len(entered))]+ await asyncio.wait_for(asyncio.gather(*(event.wait() for event in entered)), _BOUNDED_WAIT_SECONDS)+ assert (gate.in_flight, gate.waiting) == (len(entered), 0)++ submissions[1].cancel()+ with pytest.raises(asyncio.CancelledError):+ await submissions[1]+ assert gate.in_flight == len(entered) - 1++ release.set()+ outcomes = await asyncio.wait_for(+ asyncio.gather(submissions[0], *submissions[2:], return_exceptions=True), _BOUNDED_WAIT_SECONDS+ )+ assert isinstance(outcomes[0], RuntimeError)+ assert outcomes[1:] == [None] * (len(entered) - 2)+ assert (gate.in_flight, gate.waiting) == (0, 0)diff --git a/tests/unit/test_saved_session_proxy_pin.py b/tests/unit/test_saved_session_proxy_pin.pyindex 370dd5e16..bf75ffa58 100644--- a/tests/unit/test_saved_session_proxy_pin.py+++ b/tests/unit/test_saved_session_proxy_pin.py@@ -793,7 +793,11 @@ async def test_create_workflow_run_non_force_path_single_create_no_update(monkeymonkeypatch.setattr(app.DATABASE.browser_sessions,"get_persistent_browser_session",- AsyncMock(return_value=SimpleNamespace(browser_profile_id=None)),+ AsyncMock(+ return_value=SimpleNamespace(+ browser_profile_id=None, status="running", completed_at=None, close_requested_at=None+ )+ ),)monkeypatch.setattr(app.DATABASE.workflow_runs, "create_workflow_run", create_workflow_run)monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run", update_workflow_run)diff --git a/tests/unit/test_workflow_service_cancel.py b/tests/unit/test_workflow_service_cancel.pyindex ac20e09d7..256f7d273 100644--- a/tests/unit/test_workflow_service_cancel.py+++ b/tests/unit/test_workflow_service_cancel.py@@ -728,10 +728,14 @@ async def test_shielded_finalize_skipped_when_pre_finally_status_unset() -> Nonedef test_lease_failure_category_persists_only_typed_browser_loss() -> None:- closed = _browser_lease_failure_category(BrowserSessionClosed("pbs_x"))- timeout = _browser_lease_failure_category(BrowserSessionStartupTimeout("pbs_x"))+ workflow_run = SimpleNamespace(created_at=datetime.now(UTC))+ closed = _browser_lease_failure_category(BrowserSessionClosed("pbs_x"), workflow_run)+ timeout = _browser_lease_failure_category(BrowserSessionStartupTimeout("pbs_x"), workflow_run)assert closed is not None and closed[0]["category"] == "BROWSER_ERROR"assert closed[0]["reason_code"] == "browser_session_closed"assert timeout is not None and timeout[0]["reason_code"] == "browser_session_startup_timeout"- assert _browser_lease_failure_category(RuntimeError("a required workflow parameter was not provided")) is None+ assert (+ _browser_lease_failure_category(RuntimeError("a required workflow parameter was not provided"), workflow_run)+ is None+ )diff --git a/tests/unit/test_workflow_service_managed_browser_profile.py b/tests/unit/test_workflow_service_managed_browser_profile.pyindex 82bf7e99a..9d691228e 100644--- a/tests/unit/test_workflow_service_managed_browser_profile.py+++ b/tests/unit/test_workflow_service_managed_browser_profile.py@@ -12,7 +12,9 @@ import pytest_asynciofrom sqlalchemy import select, updatefrom sqlalchemy.ext.asyncio import AsyncEngine+from skyvern.exceptions import BrowserSessionExpiredfrom skyvern.forge import app+from skyvern.forge.agent import ForgeAgentfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.agent_db import AgentDB@@ -21,6 +23,7 @@ from skyvern.forge.sdk.db.models import (BrowserProfileModel,CredentialModel,PersistentBrowserSessionModel,+ TaskModel,TaskRunModel,WorkflowModel,WorkflowRunAttemptModel,@@ -31,6 +34,7 @@ from skyvern.forge.sdk.db.repositories import workflow_runs as workflow_runs_repfrom skyvern.forge.sdk.db.repositories.workflow_runs import PrepareNextAttemptResult, WorkflowRunDispatchFinalizationfrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.schemas.persistent_browser_sessions import BrowserSessionCloseReason+from skyvern.forge.sdk.schemas.tasks import TaskRequestfrom skyvern.forge.sdk.workflow import service as service_modulefrom skyvern.forge.sdk.workflow.browser_profile_key import (build_browser_profile_key_digest,@@ -1833,6 +1837,60 @@ async def test_execute_workflow_closes_owned_session_when_begin_session_fails(close_session.assert_awaited_once_with("o_test", "pbs_human", reason=BrowserSessionCloseReason.run_ended)+ ("created_by", "bound", "ended_after_run_created", "expected_category", "expected_reason_code"),+ [+ pytest.param(+ "api", False, False, "BROWSER_SESSION_EXPIRED", "browser_session_expired_before_run", id="customer"+ ),+ pytest.param(+ None, False, False, "BROWSER_SESSION_EXPIRED", "browser_session_expired_before_run", id="legacy-null"+ ),+ pytest.param(None, True, False, "BROWSER_ERROR", "browser_session_closed", id="reuse-bound-session"),+ pytest.param("copilot", False, False, "BROWSER_ERROR", "browser_session_closed", id="copilot-session"),+ pytest.param("api", False, True, "BROWSER_ERROR", "browser_session_closed", id="expired-while-queued"),+ ],+)+async def test_lease_of_an_expired_session_is_the_callers_only_when_sent_after_it_ended(+ monkeypatch: pytest.MonkeyPatch,+ created_by: str | None,+ bound: bool,+ ended_after_run_created: bool,+ expected_category: str,+ expected_reason_code: str,+) -> None:+ workflow = _execute_workflow()+ failed_run = _execute_workflow_run(WorkflowRunStatus.failed)+ svc = WorkflowService()+ _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))+ _patch_session_backed_run(monkeypatch, svc)+ run_created_at = _execute_workflow_run(WorkflowRunStatus.running).created_at+ ended_at = run_created_at + timedelta(minutes=1) if ended_after_run_created else run_created_at - timedelta(hours=6)+ expired = BrowserSessionExpired(+ "pbs_caller",+ started_at=ended_at - timedelta(minutes=240),+ ended_at=ended_at,+ timeout_minutes=240,+ created_by=created_by,+ bound=bound,+ )+ monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "begin_session", AsyncMock(side_effect=expired))+ mark_failed = AsyncMock(return_value=failed_run)+ monkeypatch.setattr(svc, "mark_workflow_run_as_failed", mark_failed)+ _patch_browser_cleanup(monkeypatch, svc, [])++ assert await _run_execute_workflow(svc, browser_session_id="pbs_caller") is failed_run++ failure_category = mark_failed.await_args.kwargs["failure_category"]+ failure_reason = mark_failed.await_args.kwargs["failure_reason"]+ assert (failure_category[0]["category"], failure_category[0]["reason_code"]) == (+ expected_category,+ expected_reason_code,+ )+ assert "pbs_caller" in failure_reason++@pytest.mark.asyncioasync def test_owned_session_close_deferred_while_child_runs_active(monkeypatch: pytest.MonkeyPatch,@@ -2364,7 +2422,15 @@ async def test_retry_preparation_preserves_session_after_transient_forced_pin_famonkeypatch.setattr(database.browser_sessions,"get_persistent_browser_session",- AsyncMock(return_value=SimpleNamespace(browser_profile_id=None, runnable_id="wr_other")),+ AsyncMock(+ return_value=SimpleNamespace(+ browser_profile_id=None,+ runnable_id="wr_other",+ status="running",+ completed_at=None,+ close_requested_at=None,+ )+ ),)run = await service.create_workflow_run(workflow_request=WorkflowRequestBody(@@ -2400,6 +2466,66 @@ async def test_retry_preparation_preserves_session_after_transient_forced_pin_faassert pin_calls == (2 if session_source == "forced" else 0)[email protected]("entry_point", ["workflow_run", "task"])+async def test_submission_refuses_a_session_past_its_lifetime_and_accepts_a_live_one(+ monkeypatch: pytest.MonkeyPatch,+ forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],+ entry_point: str,+) -> None:+ database, service, workflow = forced_session_setup+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False))+ ended_at = datetime(2026, 9, 28, 17, 1)+ async with database.Session() as session:+ session.add_all(+ [+ PersistentBrowserSessionModel(+ persistent_browser_session_id="pbs_expired",+ organization_id="o_test",+ status="timeout",+ timeout_minutes=240,+ started_at=ended_at - timedelta(minutes=240),+ completed_at=ended_at,+ created_by="api",+ ),+ PersistentBrowserSessionModel(+ persistent_browser_session_id="pbs_live",+ organization_id="o_test",+ status="running",+ timeout_minutes=240,+ started_at=datetime.now(UTC).replace(tzinfo=None),+ created_by="api",+ ),+ ]+ )+ await session.commit()++ async def submit(browser_session_id: str) -> str | None:+ if entry_point == "task":+ task = await ForgeAgent().create_task(+ TaskRequest(url="https://example.com", browser_session_id=browser_session_id), "o_test"+ )+ return task.browser_session_id+ run = await service.create_workflow_run(+ workflow_request=WorkflowRequestBody(browser_session_id=browser_session_id),+ workflow_permanent_id="wpid_test",+ workflow_id="wf_1",+ organization_id="o_test",+ workflow=workflow,+ )+ return run.browser_session_id++ with pytest.raises(BrowserSessionExpired) as refused:+ await submit("pbs_expired")++ assert refused.value.status_code == 410+ assert "pbs_expired" in refused.value.message+ async with database.Session() as session:+ assert (await session.scalars(select(WorkflowRunModel))).all() == []+ assert (await session.scalars(select(TaskModel))).all() == []+ assert await submit("pbs_live") == "pbs_live"++@pytest.mark.asyncio@pytest.mark.parametrize("failure_point", ["assignment", "pin"])async def test_forced_session_assignment_precedes_required_pin(