Skip to content

Automate browser based workflows with AI

Medwig/sky 16553 agent waits for runs (#8806)

Shuchang Zhengfacec046ea69
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 5002d2707..b9112d7d5 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -193,6 +193,17 @@
"400": {
"description": "Invalid agent run request"
},
+ "503": {
+ "description": "The server is busy dispatching other runs. No run was created; retry after Retry-After seconds.",
+ "headers": {
+ "Retry-After": {
+ "description": "Seconds to wait before retrying",
+ "schema": {
+ "type": "integer"
+ }
+ }
+ }
+ },
"422": {
"description": "Validation Error",
"content": {
@@ -6628,6 +6639,17 @@
"404": {
"description": "Workflow run not found"
},
+ "503": {
+ "description": "The server is busy dispatching other runs. No run was created; retry after Retry-After seconds.",
+ "headers": {
+ "Retry-After": {
+ "description": "Seconds to wait before retrying",
+ "schema": {
+ "type": "integer"
+ }
+ }
+ }
+ },
"422": {
"description": "Validation Error",
"content": {
@@ -19339,6 +19361,22 @@
"title": "JinjaBranchCriteria",
"description": "Jinja2-templated branch criteria (only supported criteria type for now)."
},
+ "JobRecipeOutcome": {
+ "type": "string",
+ "enum": [
+ "completed",
+ "canceled",
+ "job_unavailable",
+ "login_required",
+ "application_refused",
+ "blocked_by_site",
+ "invalid_input",
+ "insufficient_credits",
+ "incomplete",
+ "failed"
+ ],
+ "title": "JobRecipeOutcome"
+ },
"JobRecipeRunListResponse": {
"properties": {
"runs": {
@@ -19435,6 +19473,29 @@
}
],
"title": "Failure Reason"
+ },
+ "outcome": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/JobRecipeOutcome"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Typed result of a finished run; null while it is still running. Apply failures name their cause (job_unavailable, login_required, application_refused, blocked_by_site, invalid_input, insufficient_credits, incomplete); a cause Skyvern cannot classify is failed, and so is every Extract failure except one Skyvern itself reports, such as running out of credits."
+ },
+ "next_step": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Next Step",
+ "description": "What to do about an unsuccessful outcome; null for completed, canceled, and running runs."
}
},
"additionalProperties": false,
@@ -19446,7 +19507,9 @@
"status",
"created_at",
"job_url",
- "failure_reason"
+ "failure_reason",
+ "outcome",
+ "next_step"
],
"title": "JobRecipeRunSummary"
},
diff --git a/docs/cookbooks/job-application-filler.mdx b/docs/cookbooks/job-application-filler.mdx
index ee2dc2f81..819a28274 100644
--- a/docs/cookbooks/job-application-filler.mdx
+++ b/docs/cookbooks/job-application-filler.mdx
@@ -206,6 +206,21 @@ curl 'https://api.skyvern.com/v1/recipes/jobs/runs?page=1&page_size=25' \
Each entry includes the run ID, recipe (`apply` or `extract`), status, creation time, job URL, and failure reason when available. The response also includes `page`, `page_size`, and `has_next`. Increment `page` while `has_next` is `true`. Page size must be between 1 and 100.
+Each entry also has an `outcome` and a `next_step`. `outcome` is `null` until the run finishes. `next_step` says what to do about an unsuccessful outcome and is `null` otherwise.
+
+| `outcome` | Meaning |
+| ---------------------- | ------------------------------------------------------------------------------- |
+| `completed` | The run finished successfully. |
+| `canceled` | The run was canceled. |
+| `job_unavailable` | Skyvern found the posting closed, or could not find it at this URL. |
+| `login_required` | The site needed an account, a sign-in, or a verification code, or rejected the sign-in. |
+| `application_refused` | The site refused the application, for example as a duplicate. |
+| `blocked_by_site` | Bot protection, a CAPTCHA, a rate limit, or a locked account blocked the run. |
+| `invalid_input` | A request value could not be used, so the run stopped before opening the site. |
+| `insufficient_credits` | The organization ran out of credits. |
+| `incomplete` | The run ran out of steps before it could submit the application. |
+| `failed` | Any other failure. Read `failure_reason`. An unsuccessful Extract run is `failed` unless it ran out of credits. |
+
<Note>
Each Apply API request creates a new run. Repeating a request can submit another application, so check the original run before retrying.
</Note>
diff --git a/skyvern/config.py b/skyvern/config.py
index ed8f9a24d..0728eb2ee 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -301,6 +301,13 @@ class Settings(BaseSettings):
# /stream sockets all count -- so size it against connections per task, not request concurrency.
# Set it empty or 0 to disable shedding.
API_LIMIT_CONCURRENCY: int | None = Field(default=512, gt=0)
+ # Run submissions one API process dispatches at once; later ones wait without holding a pooled connection, and
+ # get a retryable 503 before anything is written if no slot frees. 0 (the default) leaves them unbounded and 32
+ # is the suggested first value, checked against the skyvern.run_submission.in_flight gauge, which records either
+ # way; the DISABLE_RUN_SUBMISSION_GATE feature flag switches an enabled gate off without a restart.
+ RUN_SUBMISSION_MAX_CONCURRENCY: int = Field(default=0, ge=0)
+ # Below the SDK's 60 s client timeout, so a waiting caller is answered before it gives up.
+ RUN_SUBMISSION_SLOT_WAIT_SECONDS: float = Field(default=20.0, gt=0)
# Must exceed the load balancer's idle timeout (infra/terraform/production/alb.tf); otherwise
# the ALB reuses a connection the server already closed and answers the client with a 502.
UVICORN_TIMEOUT_KEEP_ALIVE: int = 125
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 3fc696e34..5eba4c340 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -7,6 +7,8 @@ from http import HTTPStatus
from importlib.util import find_spec
from typing import TYPE_CHECKING, Literal, NoReturn
+from skyvern.schemas.browser_session_timeouts import MAX_EXTENDED_TIMEOUT
+
if TYPE_CHECKING:
from skyvern.errors.errors import UserDefinedError
@@ -1783,6 +1785,53 @@ class BrowserSessionClosed(SkyvernHTTPException):
)
+def _utc_minute(moment: datetime) -> str:
+ aware = moment.replace(tzinfo=UTC) if moment.tzinfo is None else moment.astimezone(UTC)
+ return aware.strftime("%Y-%m-%d %H:%M UTC")
+
+
+class BrowserSessionExpired(BrowserSessionClosed):
+ """The session ran out its configured lifetime. Carries the session facts that decide whether a run sent to it
+ is the caller's to fix (see the browser-lease seam in the workflow service)."""
+
+ def __init__(
+ self,
+ browser_session_id: str,
+ *,
+ started_at: datetime | None,
+ ended_at: datetime | None,
+ timeout_minutes: int | None,
+ created_by: str | None,
+ bound: bool = False,
+ refused_at_submission: bool = False,
+ ) -> None:
+ self.browser_session_id = browser_session_id
+ self.started_at = started_at
+ self.ended_at = ended_at
+ self.timeout_minutes = timeout_minutes
+ self.created_by = created_by
+ self.bound = bound
+ if refused_at_submission:
+ SkyvernHTTPException.__init__(
+ self, self.expired_before_run_message(run_created=False), status_code=HTTPStatus.GONE
+ )
+ else:
+ super().__init__(browser_session_id, reason="expired after reaching its configured lifetime")
+
+ def expired_before_run_message(self, *, run_created: bool) -> str:
+ """States that the session ended before the run was submitted, so use it only once that is established."""
+ ended = f" at {_utc_minute(self.ended_at)}" if self.ended_at is not None else ""
+ limit = f"its {self.timeout_minutes}-minute limit" if self.timeout_minutes else "its configured lifetime"
+ outcome = "so the run did not start" if run_created else "so no run was created"
+ return (
+ f"Browser session {self.browser_session_id} expired{ended} after {limit}, before this run was submitted, "
+ f"{outcome}. To fix it: create a new browser session and pass its id, or leave browser_session_id empty "
+ "to start a fresh browser. To reuse one session for many runs, extend it before it ends "
+ "(POST /v1/browser_sessions/{browser_session_id}/extend, "
+ f"up to {MAX_EXTENDED_TIMEOUT // 60} hours in total)."
+ )
+
+
class BrowserSessionNotFound(SkyvernHTTPException):
def __init__(self, browser_session_id: str) -> None:
super().__init__(
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 7df96e4c5..c8a6cf367 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -168,6 +168,7 @@ from skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logs
from skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatus
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import unusable_browser_session_error
from skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskResponse, TaskStatus
from skyvern.forge.sdk.schemas.totp_codes import OTPType
from skyvern.forge.sdk.services.credentials import parse_totp_config
@@ -2179,6 +2180,9 @@ class ForgeAgent:
)
if not browser_session:
raise BrowserSessionNotFound(browser_session_id=task_request.browser_session_id)
+ unusable = unusable_browser_session_error(browser_session, refused_at_submission=True)
+ if unusable is not None:
+ raise unusable
task = await app.DATABASE.tasks.create_task(
url=str(task_request.url),
diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.py
index 742877bde..9a038b9e7 100644
--- a/skyvern/forge/failure_classifier.py
+++ b/skyvern/forge/failure_classifier.py
@@ -17,6 +17,7 @@ class FailureCategory(StrEnum):
ANTI_BOT_DETECTION = "ANTI_BOT_DETECTION"
PROXY_ERROR = "PROXY_ERROR"
BROWSER_ERROR = "BROWSER_ERROR"
+ BROWSER_SESSION_EXPIRED = "BROWSER_SESSION_EXPIRED"
NAVIGATION_FAILURE = "NAVIGATION_FAILURE"
PAGE_LOAD_TIMEOUT = "PAGE_LOAD_TIMEOUT"
ELEMENT_STATE_TIMEOUT = "ELEMENT_STATE_TIMEOUT"
@@ -472,7 +473,7 @@ def classify_from_failure_reason(
# Bump when the taxonomy or the category->component mapping below changes, so a frozen
# coverage baseline stays reproducible per classifier_version.
-CLASSIFIER_VERSION = 4
+CLASSIFIER_VERSION = 5
FAILURE_ATTRIBUTION_SCHEMA_VERSION = 1
# Bounded sentinels — neither is an infra component id.
@@ -490,6 +491,7 @@ _FAILURE_CATEGORY_LITERALS = frozenset(category.value for category in FailureCat
# (skyvern/forge/sdk/workflow/service.py::_browser_lease_failure_category) imports these.
BROWSER_SESSION_CLOSED_REASON_CODE = "browser_session_closed"
BROWSER_SESSION_STARTUP_TIMEOUT_REASON_CODE = "browser_session_startup_timeout"
+BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE = "browser_session_expired_before_run"
PROXY_TRANSPORT_FAILED_REASON_CODE = "proxy_transport_failed"
UNSOLVED_CAPTCHA_BEFORE_BROWSER_LOSS_REASON_CODE = "unsolved_captcha_before_browser_loss"
@@ -558,6 +560,7 @@ _REASON_CODE_LITERALS = frozenset(
"locator_wait_for_timeout",
BROWSER_SESSION_CLOSED_REASON_CODE,
BROWSER_SESSION_STARTUP_TIMEOUT_REASON_CODE,
+ BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE,
PROXY_TRANSPORT_FAILED_REASON_CODE,
UNSOLVED_CAPTCHA_BEFORE_BROWSER_LOSS_REASON_CODE,
}
@@ -582,6 +585,7 @@ _NON_INFRA_CATEGORIES = {
"DATA_EXTRACTION_FAILURE",
"LLM_REASONING_ERROR",
"WRONG_PAGE_STATE",
+ "BROWSER_SESSION_EXPIRED",
}
diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.py
index 630874e31..5dfe32e20 100644
--- a/skyvern/forge/request_logging.py
+++ b/skyvern/forge/request_logging.py
@@ -99,6 +99,7 @@ class _RequestIdentity:
principal_resolution_conflict: bool = False
bearer_identity_status: str | None = None
principal_resolution_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+ load_shed: bool = False
_request_identity: ContextVar[_RequestIdentity | None] = ContextVar("raw_request_identity", default=None)
@@ -133,6 +134,13 @@ def set_request_organization(
identity.org_age = org_age
+def mark_request_load_shed() -> None:
+ """Log this request's 503 at warning level: the server shed it on purpose under load, before doing any work."""
+ identity = _request_identity.get()
+ if identity is not None:
+ identity.load_shed = True
+
+
def set_request_principal(
principal: RequestPrincipal,
resolution_key: tuple[str, bytes | None],
@@ -309,7 +317,10 @@ def _log_request(
headers: dict[str, str],
start_time: float,
) -> None:
- if status_code >= 500:
+ identity = _request_identity.get()
+ if status_code == 503 and identity is not None and identity.load_shed:
+ log_method = LOG.warning
+ elif status_code >= 500:
log_method = LOG.error
elif status_code >= 400 and status_code not in _ROUTINE_CLIENT_ERROR_STATUSES:
log_method = LOG.warning
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 7c434c2db..2b4833590 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -825,6 +825,7 @@ _ARTIFACT_HEALTH_EXCLUDED_CATEGORIES = frozenset(
"ANTI_BOT_DETECTION",
"AUTH_FAILURE",
"BROWSER_ERROR",
+ "BROWSER_SESSION_EXPIRED",
"CREDENTIAL_ERROR",
"INFRASTRUCTURE_ERROR",
"NAVIGATION_FAILURE",
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 509aad3c9..d5fadb9c5 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -23,7 +23,11 @@ import structlog
import yaml
from skyvern.constants import SCRUBBED_VALUE
-from skyvern.exceptions import CopilotInlineSequentialCredentialUnsupported
+from skyvern.exceptions import (
+ BrowserSessionClosed,
+ BrowserSessionStartupTimeout,
+ CopilotInlineSequentialCredentialUnsupported,
+)
from skyvern.forge import app
from skyvern.forge.failure_classifier import without_output_only_anti_bot, without_output_only_anti_bot_categories
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
@@ -4397,23 +4401,41 @@ async def _run_blocks_and_collect_debug(
)
current_context = skyvern_context.current()
- workflow_run = await workflow_service.prepare_workflow(
- workflow_id=ctx.workflow_permanent_id,
- organization=organization,
- workflow_request=workflow_request,
- template=False,
- # Both execution paths pin the exact version. A permanent-id lookup could
- # select a newer proposal while this run still carries the previous definition.
- resolved_workflow_id=snapshot.workflow.workflow_id,
- max_steps=None,
- request_id=current_context.request_id if current_context is not None else None,
- # The trigger type (and the -ui queue routing it implies) is a cloud contract; ask the
- # AgentFunction for it rather than hardcoding "manual == -ui pool" in OSS. OSS base
- # returns None (no routing hint); cloud returns the value its executor routes to -ui.
- trigger_type=(app.AGENT_FUNCTION.resolve_copilot_dispatch_trigger_type() if dispatch_to_worker else None),
- copilot_session_id=ctx.workflow_copilot_chat_id,
- created_by="copilot",
- )
+ try:
+ workflow_run = await workflow_service.prepare_workflow(
+ workflow_id=ctx.workflow_permanent_id,
+ organization=organization,
+ workflow_request=workflow_request,
+ template=False,
+ # Both execution paths pin the exact version. A permanent-id lookup could
+ # select a newer proposal while this run still carries the previous definition.
+ resolved_workflow_id=snapshot.workflow.workflow_id,
+ max_steps=None,
+ request_id=current_context.request_id if current_context is not None else None,
+ # The trigger type (and the -ui queue routing it implies) is a cloud contract; ask the
+ # AgentFunction for it rather than hardcoding "manual == -ui pool" in OSS. OSS base
+ # returns None (no routing hint); cloud returns the value its executor routes to -ui.
+ trigger_type=(
+ app.AGENT_FUNCTION.resolve_copilot_dispatch_trigger_type() if dispatch_to_worker else None
+ ),
+ copilot_session_id=ctx.workflow_copilot_chat_id,
+ created_by="copilot",
+ )
+ except (BrowserSessionClosed, BrowserSessionStartupTimeout) as refused:
+ # Submission refuses a session that has already ended, so no run exists to carry the
+ # lease-seam reason code; report the same typed browser loss that run would have.
+ if dispatch_draft_workflow_id is not None:
+ await _delete_dispatch_draft(dispatch_draft_workflow_id, ctx.organization_id)
+ dispatch_draft_workflow_id = None
+ state: BuildTestConnectFailureState = (
+ "provisioning_unavailable" if isinstance(refused, BrowserSessionStartupTimeout) else "already_closed"
+ )
+ return _with_build_test_acquisition_context(
+ _build_test_connect_failure_result(
+ BuildTestConnectFailure(state=state, browser_session_id=run_session_id)
+ ),
+ requested_block_labels=block_labels,
+ )
if explicit_blank and workflow_run.browser_session_id != run_session_id:
await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
workflow_run_id=workflow_run.workflow_run_id,
@@ -6832,14 +6854,16 @@ def _safe_reason_code(value: object) -> str:
_RUN_SIDE_CONNECT_STATES: dict[str, BuildTestConnectFailureState] = {
"browser_session_closed": "already_closed",
"browser_session_startup_timeout": "provisioning_unavailable",
+ "browser_session_expired_before_run": "already_closed",
}
+_RUN_SIDE_CONNECT_CATEGORIES = frozenset({"BROWSER_ERROR", "BROWSER_SESSION_EXPIRED"})
def _run_side_connect_state(failure_category: object) -> BuildTestConnectFailureState | None:
if not isinstance(failure_category, list):
return None
for entry in failure_category:
- if isinstance(entry, dict) and entry.get("category") == "BROWSER_ERROR":
+ if isinstance(entry, dict) and entry.get("category") in _RUN_SIDE_CONNECT_CATEGORIES:
state = _RUN_SIDE_CONNECT_STATES.get(str(entry.get("reason_code") or ""))
if state is not None:
return state
diff --git a/skyvern/forge/sdk/core/run_submission_gate.py b/skyvern/forge/sdk/core/run_submission_gate.py
new file mode 100644
index 000000000..9245f1fd7
--- /dev/null
+++ b/skyvern/forge/sdk/core/run_submission_gate.py
@@ -0,0 +1,189 @@
+import asyncio
+import functools
+import time
+from collections.abc import AsyncIterator, Iterable
+from contextlib import asynccontextmanager
+from typing import Any
+
+import structlog
+from fastapi import HTTPException, status
+from opentelemetry import metrics, trace
+
+from skyvern.config import settings
+from skyvern.forge import app
+from skyvern.forge.request_logging import mark_request_load_shed
+from skyvern.utils.contained_effects import contained_effect
+from skyvern.utils.metric_attributes import task_metric_attributes
+
+LOG = structlog.get_logger()
+
+RETRY_AFTER_SECONDS = 5
+SLOT_WAIT_SPAN_ATTRIBUTE = "run_submission.slot_wait_ms"
+DISABLE_RUN_SUBMISSION_GATE_FLAG = "DISABLE_RUN_SUBMISSION_GATE"
+RUN_SUBMISSION_METRIC_PREFIX = "skyvern.run_submission."
+RUN_SUBMISSION_REJECTED_COUNTER = f"{RUN_SUBMISSION_METRIC_PREFIX}rejected"
+RUN_SUBMISSION_IN_FLIGHT_GAUGE = f"{RUN_SUBMISSION_METRIC_PREFIX}in_flight"
+RUN_SUBMISSION_WAITING_GAUGE = f"{RUN_SUBMISSION_METRIC_PREFIX}waiting"
+_KILL_SWITCH_LOOKUP_TIMEOUT_SECONDS = 1.0
+OPENAPI_SHED_RESPONSE: dict[str, Any] = {
+ "description": "The server is busy dispatching other runs. No run was created; retry after Retry-After seconds.",
+ "headers": {"Retry-After": {"description": "Seconds to wait before retrying", "schema": {"type": "integer"}}},
+}
+
+
+def _rejected_counter() -> Any | None:
+ # Cached on first use rather than at import: the meter must be created after otel_setup has
+ # installed the real MeterProvider or every add() is a no-op.
+ if not settings.OTEL_METRICS_ENABLED:
+ return None
+ try:
+ meter = metrics.get_meter("skyvern.run_submission_gate")
+ return meter.create_counter(
+ RUN_SUBMISSION_REJECTED_COUNTER,
+ unit="{request}",
+ description="Run submissions answered 503 because no dispatch slot freed in time",
+ )
+ except Exception as e:
+ LOG.warning("Failed to initialize run submission rejection counter", error=str(e))
+ return None
+
+
+class RunSubmissionGate:
+ """A dispatching submission holds a pooled connection for most of its life, so an unbounded burst
+ on one process drains the pool for every route on it; a caller waiting here holds no connection."""
+
+ def __init__(self, limit: int, wait_seconds: float) -> None:
+ self._limit = limit
+ self._wait_seconds = wait_seconds
+ self._semaphore: asyncio.Semaphore | None = None
+ self._loop: asyncio.AbstractEventLoop | None = None
+ self._in_flight = 0
+ self._waiting = 0
+
+ @property
+ def enabled(self) -> bool:
+ return self._limit > 0
+
+ @property
+ def in_flight(self) -> int:
+ return self._in_flight
+
+ @property
+ def waiting(self) -> int:
+ return self._waiting
+
+ def _semaphore_for_running_loop(self) -> asyncio.Semaphore:
+ # A semaphore binds to the loop that first waits on it, and tests run each case on a new loop.
+ loop = asyncio.get_running_loop()
+ if self._semaphore is None or self._loop is not loop:
+ self._semaphore = asyncio.Semaphore(self._limit)
+ self._loop = loop
+ return self._semaphore
+
+ @asynccontextmanager
+ async def slot(self, enforce: bool = True) -> AsyncIterator[None]:
+ # Counted even when not enforced, so the in-flight gauge can size a limit before one is set.
+ semaphore = await self._acquire() if enforce and self.enabled else None
+ self._in_flight += 1
+ try:
+ yield
+ finally:
+ self._in_flight -= 1
+ if semaphore is not None:
+ semaphore.release()
+
+ async def _acquire(self) -> asyncio.Semaphore:
+ semaphore = self._semaphore_for_running_loop()
+ wait_started = time.monotonic()
+ self._waiting += 1
+ try:
+ async with asyncio.timeout(self._wait_seconds):
+ await semaphore.acquire()
+ except TimeoutError:
+ self._record_rejection()
+ mark_request_load_shed()
+ raise HTTPException(
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+ detail="The server is busy dispatching other runs. No run was created; retry this request.",
+ headers={"Retry-After": str(RETRY_AFTER_SECONDS)},
+ ) from None
+ finally:
+ self._waiting -= 1
+ trace.get_current_span().set_attribute(
+ SLOT_WAIT_SPAN_ATTRIBUTE, round((time.monotonic() - wait_started) * 1000, 1)
+ )
+ return semaphore
+
+ def _record_rejection(self) -> None:
+ with contained_effect("record run submission rejection"):
+ counter = _rejected_counter()
+ if counter is not None:
+ counter.add(1, task_metric_attributes())
+
+
+_GATE = RunSubmissionGate(settings.RUN_SUBMISSION_MAX_CONCURRENCY, settings.RUN_SUBMISSION_SLOT_WAIT_SECONDS)
+
+
+def _observe_in_flight(_options: Any) -> Iterable[metrics.Observation]:
+ return [metrics.Observation(_GATE.in_flight, task_metric_attributes())]
+
+
+def _observe_waiting(_options: Any) -> Iterable[metrics.Observation]:
+ return [metrics.Observation(_GATE.waiting, task_metric_attributes())]
+
+
+def _register_gauges() -> None:
+ # Registered on first use for the same reason as the counter, and once per process: the meter
+ # reports a second instrument under the same name as a conflict.
+ if not settings.OTEL_METRICS_ENABLED:
+ return
+ try:
+ meter = metrics.get_meter("skyvern.run_submission_gate")
+ meter.create_observable_gauge(
+ RUN_SUBMISSION_IN_FLIGHT_GAUGE,
+ callbacks=[_observe_in_flight],
+ unit="{request}",
+ description="Run submissions dispatching on this process, whether or not the gate is on",
+ )
+ meter.create_observable_gauge(
+ RUN_SUBMISSION_WAITING_GAUGE,
+ callbacks=[_observe_waiting],
+ unit="{request}",
+ description="Run submissions waiting for a dispatch slot on this process",
+ )
+ except Exception as e:
+ LOG.warning("Failed to initialize run submission gauges", error=str(e))
+
+
+async def _disabled_by_kill_switch(organization_id: str) -> bool:
+ """Runtime off switch: if a fleet-wide dispatch slowdown (Temporal, Redis) holds every slot, the gate
+ would shed every submission above the limit, and changing the setting needs a restart.
+ Anything other than an explicit True leaves the setting in charge."""
+ try:
+ # A cache miss can read the flag snapshot from the database, which is the resource a burst exhausts.
+ disabled = await asyncio.wait_for(
+ app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ DISABLE_RUN_SUBMISSION_GATE_FLAG,
+ organization_id,
+ properties={"organization_id": organization_id},
+ ),
+ _KILL_SWITCH_LOOKUP_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to check the run submission gate kill switch; the gate stays on",
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ return False
+ return disabled is True
+
+
+@asynccontextmanager
+async def run_submission_slot(organization_id: str) -> AsyncIterator[None]:
+ _register_gauges()
+ enforce = _GATE.enabled and not await _disabled_by_kill_switch(organization_id)
+ async with _GATE.slot(enforce=enforce):
+ yield
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index f8c60c1bf..04c25da46 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -65,6 +65,7 @@ from skyvern.forge.sdk.artifact.storage.base import artifact_filename_from_uri
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.curl_converter import curl_to_http_request_block_params
from skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory
+from skyvern.forge.sdk.core.run_submission_gate import OPENAPI_SHED_RESPONSE, run_submission_slot
from skyvern.forge.sdk.core.security import generate_skyvern_signature
from skyvern.forge.sdk.db.enums import (
OrganizationAuthTokenType,
@@ -676,6 +677,7 @@ def _hydrate_run_request_for_response(
responses={
200: {"description": "Successfully ran agent"},
400: {"description": "Invalid agent run request"},
+ 503: OPENAPI_SHED_RESPONSE,
},
)
@base_router.post("/run/agents/", include_in_schema=False)
@@ -709,53 +711,56 @@ async def run_workflow(
legacy_workflow_request = _workflow_run_request_to_legacy_request(workflow_run_request)
trigger_type = workflow_run_trigger_type_from_user_agent(x_user_agent)
- try:
- workflow_run = await workflow_service.run_workflow(
- workflow_id=workflow_id,
- organization=current_org,
- workflow_request=legacy_workflow_request,
- template=template,
- version=None,
- max_steps=x_max_steps_override,
- api_key=x_api_key,
- request_id=request_id,
- request=request,
- background_tasks=background_tasks,
- trigger_type=trigger_type,
- tag_write_context=_tag_write_context_from_caller(caller),
- created_by=user_id,
- refuse_unusable_parameters_before_create=True,
+ # The slot covers every database read after dispatch too: a checkout timeout there would answer 500 for
+ # a run that is already queued, and the client's retry would start it twice.
+ async with run_submission_slot(current_org.organization_id):
+ try:
+ workflow_run = await workflow_service.run_workflow(
+ workflow_id=workflow_id,
+ organization=current_org,
+ workflow_request=legacy_workflow_request,
+ template=template,
+ version=None,
+ max_steps=x_max_steps_override,
+ api_key=x_api_key,
+ request_id=request_id,
+ request=request,
+ background_tasks=background_tasks,
+ trigger_type=trigger_type,
+ tag_write_context=_tag_write_context_from_caller(caller),
+ created_by=user_id,
+ refuse_unusable_parameters_before_create=True,
+ )
+ except MissingBrowserAddressError as e:
+ raise HTTPException(status_code=400, detail=str(e)) from e
+ await uploaded_file_service.attach_files_to_run(
+ file_ids=workflow_run_request.file_ids or [],
+ organization_id=current_org.organization_id,
+ run_id=workflow_run.workflow_run_id,
+ )
+ background_tasks.add_task(
+ app.AGENT_FUNCTION.on_run_created,
+ organization_id=current_org.organization_id,
+ run_id=workflow_run.workflow_run_id,
+ run_type=RunType.workflow_run,
+ caller_type=caller.caller_type,
)
- except MissingBrowserAddressError as e:
- raise HTTPException(status_code=400, detail=str(e)) from e
- await uploaded_file_service.attach_files_to_run(
- file_ids=workflow_run_request.file_ids or [],
- organization_id=current_org.organization_id,
- run_id=workflow_run.workflow_run_id,
- )
- background_tasks.add_task(
- app.AGENT_FUNCTION.on_run_created,
- organization_id=current_org.organization_id,
- run_id=workflow_run.workflow_run_id,
- run_type=RunType.workflow_run,
- caller_type=caller.caller_type,
- )
-
- if settings.OTEL_ENABLED:
- span = trace.get_current_span()
- if span:
- if workflow_run.workflow_run_id:
- span.set_attribute("workflow_run_id", workflow_run.workflow_run_id)
- if workflow_run.workflow_id:
- span.set_attribute("workflow_id", workflow_run.workflow_id)
- # Hydrate the returned request from the persisted run: workflow title (when the workflow exists) and
- # the effective browser_type, so a run that omitted browser_type reports the inherited engine
- # instead of the request's null.
- workflow = await app.WORKFLOW_SERVICE.get_workflow(
- workflow_id=workflow_run.workflow_id,
- organization_id=current_org.organization_id,
- )
+ if settings.OTEL_ENABLED:
+ span = trace.get_current_span()
+ if span:
+ if workflow_run.workflow_run_id:
+ span.set_attribute("workflow_run_id", workflow_run.workflow_run_id)
+ if workflow_run.workflow_id:
+ span.set_attribute("workflow_id", workflow_run.workflow_id)
+
+ # Hydrate the returned request from the persisted run: workflow title (when the workflow exists) and
+ # the effective browser_type, so a run that omitted browser_type reports the inherited engine
+ # instead of the request's null.
+ workflow = await app.WORKFLOW_SERVICE.get_workflow(
+ workflow_id=workflow_run.workflow_id,
+ organization_id=current_org.organization_id,
+ )
workflow_run_request_hydrated = _hydrate_run_request_for_response(workflow_run_request, workflow_run, workflow)
return WorkflowRunResponse(
@@ -4102,6 +4107,7 @@ def _user_writable_run_metadata(run_metadata: dict[str, str] | None) -> dict[str
200: {"description": "Successfully retried workflow run"},
400: {"description": "Workflow run is not retryable"},
404: {"description": "Workflow run not found"},
+ 503: OPENAPI_SHED_RESPONSE,
},
)
@base_router.post("/workflows/runs/{workflow_run_id}/retry/", include_in_schema=False)
@@ -4198,25 +4204,26 @@ async def retry_workflow_run(
if is_job_recipe_workflow_run_trigger_type(original_trigger_type)
else workflow_run_trigger_type_from_user_agent(x_user_agent)
)
- try:
- workflow_run = await workflow_service.run_workflow(
- workflow_id=original_workflow_run.workflow_permanent_id,
- organization=current_org,
- workflow_request=legacy_workflow_request,
- template=template,
- version=original_workflow.version,
- max_steps=x_max_steps_override,
- api_key=x_api_key,
- request_id=context.request_id,
- request=request,
- background_tasks=background_tasks,
- trigger_type=trigger_type,
- ignore_inherited_workflow_system_prompt=original_workflow_run.ignore_inherited_workflow_system_prompt,
- tag_write_context=_tag_write_context_from_caller(caller),
- created_by=user_id,
- )
- except MissingBrowserAddressError as e:
- raise HTTPException(status_code=400, detail=str(e)) from e
+ async with run_submission_slot(current_org.organization_id):
+ try:
+ workflow_run = await workflow_service.run_workflow(
+ workflow_id=original_workflow_run.workflow_permanent_id,
+ organization=current_org,
+ workflow_request=legacy_workflow_request,
+ template=template,
+ version=original_workflow.version,
+ max_steps=x_max_steps_override,
+ api_key=x_api_key,
+ request_id=context.request_id,
+ request=request,
+ background_tasks=background_tasks,
+ trigger_type=trigger_type,
+ ignore_inherited_workflow_system_prompt=original_workflow_run.ignore_inherited_workflow_system_prompt,
+ tag_write_context=_tag_write_context_from_caller(caller),
+ created_by=user_id,
+ )
+ except MissingBrowserAddressError as e:
+ raise HTTPException(status_code=400, detail=str(e)) from e
background_tasks.add_task(
app.AGENT_FUNCTION.on_run_created,
organization_id=current_org.organization_id,
@@ -4802,6 +4809,7 @@ async def get_actions(
openapi_extra={
"x-fern-sdk-method-name": "run_workflow_legacy",
},
+ responses={503: OPENAPI_SHED_RESPONSE},
)
@legacy_base_router.post(
"/workflows/{workflow_id}/run/",
@@ -4836,25 +4844,26 @@ async def run_workflow_legacy(
await app.RATE_LIMITER.rate_limit_submit_run(current_org.organization_id)
legacy_trigger_type = workflow_run_trigger_type_from_user_agent(x_user_agent)
- try:
- workflow_run = await workflow_service.run_workflow(
- workflow_id=workflow_id,
- organization=current_org,
- workflow_request=workflow_request,
- template=template,
- version=version,
- max_steps=x_max_steps_override,
- api_key=x_api_key,
- request_id=request_id,
- request=request,
- background_tasks=background_tasks,
- trigger_type=legacy_trigger_type,
- tag_write_context=_tag_write_context_from_caller(caller),
- created_by=user_id,
- refuse_unusable_parameters_before_create=True,
- )
- except MissingBrowserAddressError as e:
- raise HTTPException(status_code=400, detail=str(e)) from e
+ async with run_submission_slot(current_org.organization_id):
+ try:
+ workflow_run = await workflow_service.run_workflow(
+ workflow_id=workflow_id,
+ organization=current_org,
+ workflow_request=workflow_request,
+ template=template,
+ version=version,
+ max_steps=x_max_steps_override,
+ api_key=x_api_key,
+ request_id=request_id,
+ request=request,
+ background_tasks=background_tasks,
+ trigger_type=legacy_trigger_type,
+ tag_write_context=_tag_write_context_from_caller(caller),
+ created_by=user_id,
+ refuse_unusable_parameters_before_create=True,
+ )
+ except MissingBrowserAddressError as e:
+ raise HTTPException(status_code=400, detail=str(e)) from e
background_tasks.add_task(
app.AGENT_FUNCTION.on_run_created,
organization_id=current_org.organization_id,
diff --git a/skyvern/forge/sdk/schemas/persistent_browser_sessions.py b/skyvern/forge/sdk/schemas/persistent_browser_sessions.py
index 11d40447c..6f7d808b7 100644
--- a/skyvern/forge/sdk/schemas/persistent_browser_sessions.py
+++ b/skyvern/forge/sdk/schemas/persistent_browser_sessions.py
@@ -4,8 +4,15 @@ from enum import StrEnum
from pydantic import BaseModel, ConfigDict, field_validator
+from skyvern.exceptions import (
+ BrowserSessionClosed,
+ BrowserSessionExpired,
+ BrowserSessionStartupTimeout,
+ SkyvernHTTPException,
+)
from skyvern.forge.sdk.db.utils import deserialize_proxy_location
from skyvern.schemas.browser_session_close import BrowserSessionCloseReason
+from skyvern.schemas.browser_session_timeouts import lived_full_lifetime
from skyvern.schemas.proxy_pinning import validate_proxy_session_id
from skyvern.schemas.runs import GeoTarget, ProxyLocation, ProxyLocationInput
@@ -160,6 +167,40 @@ class PersistentBrowserSession(BaseModel):
return not self.profile_read_only and bool(self.generate_browser_profile or self.browser_profile_id)
+def unusable_browser_session_error(
+ session: PersistentBrowserSession, *, refused_at_submission: bool = False
+) -> SkyvernHTTPException | None:
+ """The error a run sent to this session fails (or, at submission, is refused) with, or None while it is usable.
+
+ The single source for both run submission and the session managers' lease, so the two cannot disagree."""
+ if not (
+ is_final_status(session.status) or session.completed_at is not None or session.close_requested_at is not None
+ ):
+ return None
+ session_id = session.persistent_browser_session_id
+ timed_out = session.status == PersistentBrowserSessionStatus.timeout
+ if timed_out and session.started_at is None:
+ return BrowserSessionStartupTimeout(session_id)
+ # A session that reaches its lifetime ends with status timeout, or is closed as completed with close reason
+ # expired, a reason a retirement can also stamp before the lifetime is up.
+ if timed_out or (
+ session.close_reason == BrowserSessionCloseReason.expired
+ and lived_full_lifetime(
+ started_at=session.started_at, ended_at=session.completed_at, timeout_minutes=session.timeout_minutes
+ )
+ ):
+ return BrowserSessionExpired(
+ session_id,
+ started_at=session.started_at,
+ ended_at=session.completed_at,
+ timeout_minutes=session.timeout_minutes,
+ created_by=session.created_by,
+ bound=session.bound_key is not None or session.bound_workflow_permanent_id is not None,
+ refused_at_submission=refused_at_submission,
+ )
+ return BrowserSessionClosed(session_id, reason=f"is no longer usable (status: {session.status})")
+
+
class FreshExitOutcome(StrEnum):
distinct_verified = "distinct_verified"
no_alternate = "no_alternate"
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 353a67514..956615468 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -57,6 +57,7 @@ from skyvern.exceptions import (
BrowserProfileNotFound,
BrowserSessionAlreadyOccupiedError,
BrowserSessionClosed,
+ BrowserSessionExpired,
BrowserSessionNotFound,
BrowserSessionNotRenewable,
BrowserSessionStartupTimeout,
@@ -83,6 +84,7 @@ from skyvern.exceptions import (
from skyvern.forge import app
from skyvern.forge.failure_classifier import (
BROWSER_SESSION_CLOSED_REASON_CODE,
+ BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE,
BROWSER_SESSION_STARTUP_TIMEOUT_REASON_CODE,
FailureCategory,
classify_from_failure_reason,
@@ -133,10 +135,12 @@ from skyvern.forge.sdk.schemas.credentials import Credential, credential_auto_pr
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
+ API_BROWSER_SESSION_CREATED_BY,
FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE,
SESSION_RETIREMENT_RUNNABLE_TYPE,
PersistentBrowserSession,
is_final_status,
+ unusable_browser_session_error,
)
from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock, WorkflowRunTimeline, WorkflowRunTimelineType
@@ -286,7 +290,7 @@ from skyvern.forge.sdk.workflow.status_mapping import (
from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
from skyvern.schemas.browser_session_close import BrowserSessionCloseReason
from skyvern.schemas.browser_session_kind import BrowserSessionKind
-from skyvern.schemas.browser_session_timeouts import REUSE_MIN_REMAINING_LIFETIME_SECONDS
+from skyvern.schemas.browser_session_timeouts import REUSE_MIN_REMAINING_LIFETIME_SECONDS, lived_full_lifetime
from skyvern.schemas.proxy_pinning import (
derive_proxy_session_id,
redact_proxy_session_id,
@@ -1704,10 +1708,45 @@ def _build_managed_browser_profile_name(workflow_title: str | None, rendered_key
return f"{title}{suffix}"
-def _browser_lease_failure_category(exc: Exception) -> list[dict] | None:
+BROWSER_SESSION_EXPIRED_EXPLANATION = (
+ "The browser session this run was sent to had already reached its time limit, so the run never started. "
+ "Running it again with the same browser session will fail the same way. Create a new browser session and run "
+ "again, or leave the browser session empty to start a fresh browser."
+)
+
+
+def _session_expired_before_run(exc: Exception, workflow_run: WorkflowRun) -> BrowserSessionExpired | None:
+ """The session, when the caller sent this run to a session of their own that had already run out its lifetime.
+
+ Sessions Skyvern created (reuse and editor prewarm sessions carry a binding and no creator), sessions closed
+ early, and runs that queued while the session was still alive stay platform-attributed. So do sessions a
+ signed-in user made in the UI, whose creator is their user id: only creator null or api counts as the caller's."""
+ if not isinstance(exc, BrowserSessionExpired):
+ return None
+ if exc.bound or exc.created_by not in (None, API_BROWSER_SESSION_CREATED_BY):
+ return None
+ if exc.ended_at is None or not lived_full_lifetime(
+ started_at=exc.started_at, ended_at=exc.ended_at, timeout_minutes=exc.timeout_minutes
+ ):
+ return None
+ if _as_utc(workflow_run.created_at) <= _as_utc(exc.ended_at):
+ return None
+ return exc
+
+
+def _browser_lease_failure_category(exc: Exception, workflow_run: WorkflowRun) -> list[dict] | None:
"""The lease seam still holds the typed exception; persist its identity so a reader does not
have to rediscover it from prose or from the session row, which closes the same way after
every run."""
+ if _session_expired_before_run(exc, workflow_run) is not None:
+ return [
+ {
+ "category": FailureCategory.BROWSER_SESSION_EXPIRED.value,
+ "confidence_float": 1.0,
+ "reason_code": BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE,
+ "reasoning": BROWSER_SESSION_EXPIRED_EXPLANATION,
+ }
+ ]
if isinstance(exc, BrowserSessionClosed):
reason_code = BROWSER_SESSION_CLOSED_REASON_CODE
reasoning = "The browser session had already closed before the run could lease it"
@@ -7342,9 +7381,13 @@ class WorkflowService:
browser_session_id=browser_session_id,
workflow_run_id=workflow_run_id,
)
+ expired_before_run = _session_expired_before_run(e, workflow_run)
failure_reason = (
- f"Failed to begin browser session for workflow run: {get_user_facing_exception_message(e)}"
+ expired_before_run.expired_before_run_message(run_created=True)
+ if expired_before_run is not None
+ else f"Failed to begin browser session for workflow run: {get_user_facing_exception_message(e)}"
)
+ lease_failure_category = _browser_lease_failure_category(e, workflow_run)
if browser_dispatch:
failed = await self._mark_workflow_run_as_failed_for_dispatch(
workflow_run_id=workflow_run_id,
@@ -7352,7 +7395,7 @@ class WorkflowService:
attempt_number=attempt_number,
dispatch_claim_started_at=dispatch_claim_started_at,
failure_reason=failure_reason,
- failure_category=_browser_lease_failure_category(e),
+ failure_category=lease_failure_category,
)
if failed is None:
return await self.get_workflow_run(workflow_run_id, organization_id)
@@ -7361,7 +7404,7 @@ class WorkflowService:
workflow_run = await self.mark_workflow_run_as_failed(
workflow_run_id=workflow_run_id,
failure_reason=failure_reason,
- failure_category=_browser_lease_failure_category(e),
+ failure_category=lease_failure_category,
)
await self.clean_up_workflow(
workflow=workflow,
@@ -11656,6 +11699,11 @@ class WorkflowService:
)
if not browser_session:
raise BrowserSessionNotFound(browser_session_id=workflow_request.browser_session_id)
+ # Nothing between here and the lease swaps or revives a caller's session, so a run sent to
+ # one that has already ended can only fail; refuse it before a run row exists.
+ unusable = unusable_browser_session_error(browser_session, refused_at_submission=True)
+ if unusable is not None:
+ raise unusable
# Auto-propagate profile from session when not explicitly provided
if not browser_profile_id and browser_session.browser_profile_id:
browser_profile_id = browser_session.browser_profile_id
diff --git a/skyvern/schemas/browser_session_timeouts.py b/skyvern/schemas/browser_session_timeouts.py
index fea7412b9..5be19760c 100644
--- a/skyvern/schemas/browser_session_timeouts.py
+++ b/skyvern/schemas/browser_session_timeouts.py
@@ -7,6 +7,7 @@ share these rules without pulling the heavier ``browser_sessions`` schema module
from __future__ import annotations
+from datetime import UTC, datetime
from typing import overload
MIN_TIMEOUT = 5
@@ -24,6 +25,9 @@ RENEWAL_MIN_REMAINING_SECONDS = 60
EXTENSION_MIN_REMAINING_SECONDS = 120
# A found reusable session with less lifetime left than this is retired and replaced rather than handed to a run.
REUSE_MIN_REMAINING_LIFETIME_SECONDS = 30 * 60
+# How far short of its budget a session may end and still count as having run it out: a renewal closes a
+# session with under RENEWAL_MIN_REMAINING_SECONDS left, and a deadline is noticed on a periodic heartbeat.
+LIFETIME_END_TOLERANCE_SECONDS = 120
DEFAULT_TIMEOUT = 60
MAX_TIMEOUT_EXCEEDED_MESSAGE = (
@@ -72,6 +76,18 @@ def creation_timeout_minutes(requested_minutes: int | None) -> int | None:
return min(requested_minutes, MAX_TIMEOUT)
+def _as_utc(moment: datetime) -> datetime:
+ return moment.replace(tzinfo=UTC) if moment.tzinfo is None else moment.astimezone(UTC)
+
+
+def lived_full_lifetime(*, started_at: datetime | None, ended_at: datetime | None, timeout_minutes: int | None) -> bool:
+ """Whether a session ended only after running out its whole budget, rather than being closed early."""
+ if started_at is None or ended_at is None or not timeout_minutes:
+ return False
+ lived_seconds = (_as_utc(ended_at) - _as_utc(started_at)).total_seconds()
+ return lived_seconds >= timeout_minutes * 60 - LIFETIME_END_TOLERANCE_SECONDS
+
+
def lifetime_cap_seconds(base_timeout_seconds: float, *, floor_seconds: float = MAX_LIFETIME_SECONDS) -> float:
"""The hard cap on an extendable session's life.
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index 8fd3d7270..312b8ae66 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -14,6 +14,7 @@ from sqlalchemy.exc import OperationalError
from skyvern.config import settings
from skyvern.constants import MINI_GOAL_TEMPLATE
from skyvern.exceptions import (
+ BrowserSessionNotFound,
FailedToSendWebhook,
ScreenshotTargetClosed,
TaskTerminationError,
@@ -39,6 +40,7 @@ from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, WorkflowRunTriggerType
from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import unusable_browser_session_error
from skyvern.forge.sdk.schemas.task_v2 import (
TASK_V2_TIMEOUT_WEBHOOK_DELIVERED_SENTINEL,
TaskV2,
@@ -349,6 +351,17 @@ async def initialize_task_v2(
await _validate_task_v2_model_for_org(organization, model)
if user_url:
user_url = await asyncio.to_thread(validate_fetch_url, user_url)
+ if browser_session_id:
+ # The workflow run created below refuses this too, but only after the task and its workflow exist.
+ browser_session = await app.DATABASE.browser_sessions.get_persistent_browser_session(
+ session_id=browser_session_id,
+ organization_id=organization.organization_id,
+ )
+ if not browser_session:
+ raise BrowserSessionNotFound(browser_session_id=browser_session_id)
+ unusable = unusable_browser_session_error(browser_session, refused_at_submission=True)
+ if unusable is not None:
+ raise unusable
task_v2 = await app.DATABASE.observer.create_task_v2(
prompt=user_prompt,
diff --git a/skyvern/utils/metric_attributes.py b/skyvern/utils/metric_attributes.py
new file mode 100644
index 000000000..eae7c706e
--- /dev/null
+++ b/skyvern/utils/metric_attributes.py
@@ -0,0 +1,18 @@
+"""Attributes that identify the process emitting a per-process OTEL metric."""
+
+import os
+
+
+def task_metric_attributes() -> dict[str, str]:
+ """Identify the emitting process so a per-process gauge does not collapse into one
+ fleet-wide series. ``HOSTNAME`` is the fallback: it is unique per ECS ``awsvpc`` task
+ and per Kubernetes pod."""
+ container = os.getenv("ECS_CONTAINER_NAME")
+ if not container:
+ task_arn = os.getenv("ECS_TASK_ARN")
+ if task_arn:
+ container = task_arn.split("/")[-1]
+ if not container:
+ container = os.getenv("HOSTNAME")
+
+ return {"container": container} if container else {}
diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.py
index 7a3b912e8..5721134e8 100644
--- a/skyvern/webeye/default_persistent_sessions_manager.py
+++ b/skyvern/webeye/default_persistent_sessions_manager.py
@@ -39,6 +39,7 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
PersistentBrowserSessionStatus,
PersistentBrowserType,
is_final_status,
+ unusable_browser_session_error,
)
from skyvern.forge.sdk.streaming.registries import stream_tombstone_holds_session_lease
from skyvern.schemas.browser_session_close import BrowserSessionCloseReason
@@ -471,8 +472,9 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
if persistent_browser_session is None:
raise Exception(f"Persistent browser session not found for {browser_session_id}")
- if is_final_status(persistent_browser_session.status):
- raise BrowserSessionClosed(browser_session_id)
+ unusable = unusable_browser_session_error(persistent_browser_session)
+ if unusable is not None:
+ raise unusable
runnable_generation_id = expected_runnable_generation_id or uuid.uuid4().hex
await self.occupy_browser_session(
@@ -904,12 +906,12 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
current = await self.database.browser_sessions.get_persistent_browser_session(
session_id, organization_id
)
- if current is not None and (
- current.completed_at is not None
- or current.close_requested_at is not None
- or is_final_status(current.status)
- ):
- raise BrowserSessionClosed(session_id) from None
+ unusable = unusable_browser_session_error(current) if current is not None else None
+ if unusable is not None:
+ # A startup timeout stays a BrowserSessionClosed here: the reuse path catches only that type.
+ raise (
+ unusable if isinstance(unusable, BrowserSessionClosed) else BrowserSessionClosed(session_id)
+ ) from None
except BaseException as classification_error:
if BROWSER_RETIREMENT_DENIED_NOTE in getattr(error, "__notes__", ()):
classification_error.add_note(BROWSER_RETIREMENT_DENIED_NOTE)
diff --git a/tests/unit/test_browser_session_timeouts.py b/tests/unit/test_browser_session_timeouts.py
index cc6374c18..b6f9f1d3a 100644
--- a/tests/unit/test_browser_session_timeouts.py
+++ b/tests/unit/test_browser_session_timeouts.py
@@ -1,13 +1,20 @@
-from datetime import datetime
+from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import ValidationError
+from skyvern.exceptions import BrowserSessionClosed, BrowserSessionExpired, BrowserSessionStartupTimeout
from skyvern.forge.sdk.routes import browser_sessions as browser_sessions_routes
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
+ PersistentBrowserSession,
+ unusable_browser_session_error,
+)
+from skyvern.schemas.browser_session_close import BrowserSessionCloseReason
from skyvern.schemas.browser_session_timeouts import (
DEFAULT_TIMEOUT,
+ LIFETIME_END_TOLERANCE_SECONDS,
MAX_EXTENDED_LIFETIME_SECONDS,
MAX_EXTENDED_TIMEOUT,
MAX_LIFETIME_SECONDS,
@@ -16,6 +23,7 @@ from skyvern.schemas.browser_session_timeouts import (
MIN_TIMEOUT,
creation_timeout_minutes,
lifetime_cap_seconds,
+ lived_full_lifetime,
max_lifetime_exceeded_warning,
max_timeout_exceeded_warning,
seconds_until_expiry,
@@ -290,3 +298,73 @@ def test_requested_timeout_below_the_minimum_is_still_rejected() -> None:
def test_timeout_defaults_and_explicit_none_are_untouched() -> None:
assert CreateBrowserSessionRequest().timeout == DEFAULT_TIMEOUT
assert CreateBrowserSessionRequest(timeout=None).timeout is None
+
+
+ ("lived_seconds", "expected"),
+ [
+ (60 * 60, True),
+ (60 * 60 - LIFETIME_END_TOLERANCE_SECONDS, True),
+ (60 * 60 - LIFETIME_END_TOLERANCE_SECONDS - 1, False),
+ (60, False),
+ ],
+)
+def test_a_session_lived_its_full_lifetime_only_within_the_end_tolerance(lived_seconds: int, expected: bool) -> None:
+ started_at = datetime(2026, 9, 28, 13, 0, tzinfo=UTC)
+ ended_at = started_at + timedelta(seconds=lived_seconds)
+
+ assert lived_full_lifetime(started_at=started_at, ended_at=ended_at, timeout_minutes=60) is expected
+ assert not lived_full_lifetime(started_at=None, ended_at=ended_at, timeout_minutes=60)
+ assert not lived_full_lifetime(started_at=started_at, ended_at=ended_at, timeout_minutes=None)
+
+
+def _ended_session(**overrides: object) -> PersistentBrowserSession:
+ started_at = datetime(2026, 9, 28, 13, 0, tzinfo=UTC)
+ fields: dict[str, object] = {
+ "persistent_browser_session_id": "pbs_x",
+ "organization_id": "o_test",
+ "status": "timeout",
+ "timeout_minutes": 60,
+ "started_at": started_at,
+ "completed_at": started_at + timedelta(minutes=60),
+ "created_at": started_at,
+ "modified_at": started_at,
+ }
+ return PersistentBrowserSession(**{**fields, **overrides})
+
+
+ ("overrides", "expected"),
+ [
+ pytest.param({}, BrowserSessionExpired, id="timeout"),
+ pytest.param({"started_at": None}, BrowserSessionStartupTimeout, id="timeout-before-start"),
+ pytest.param(
+ {"status": "completed", "close_reason": BrowserSessionCloseReason.expired},
+ BrowserSessionExpired,
+ id="oss-reaper-full-lifetime",
+ ),
+ pytest.param(
+ {
+ "status": "completed",
+ "close_reason": BrowserSessionCloseReason.expired,
+ "completed_at": datetime(2026, 9, 28, 13, 10, tzinfo=UTC),
+ },
+ BrowserSessionClosed,
+ id="expired-reason-stamped-early",
+ ),
+ pytest.param({"status": "completed"}, BrowserSessionClosed, id="completed-by-the-caller"),
+ pytest.param({"status": "failed"}, BrowserSessionClosed, id="failed"),
+ pytest.param(
+ {"status": "running", "completed_at": None, "close_requested_at": datetime(2026, 9, 28, 13, 5, tzinfo=UTC)},
+ BrowserSessionClosed,
+ id="close-requested",
+ ),
+ pytest.param({"status": "running", "completed_at": None}, type(None), id="live"),
+ ],
+)
+def test_unusable_browser_session_error_tells_expiry_from_an_early_close(
+ overrides: dict[str, object], expected: type
+) -> None:
+ error = unusable_browser_session_error(_ended_session(**overrides))
+
+ assert type(error) is expected
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 8e31f721e..77215e8cc 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -7,7 +7,7 @@ import json
import time
from collections.abc import AsyncIterator, Callable, Sequence
from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from typing import Any, ClassVar, Self
@@ -48,6 +48,7 @@ from openai.types.responses.response_prompt_param import ResponsePromptParam
from pydantic import BaseModel
from structlog.testing import capture_logs
+from skyvern.exceptions import BrowserSessionExpired
from skyvern.forge import app as forge_app
from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigError, LLMProviderError
from skyvern.forge.sdk.copilot import agent as agent_module
@@ -6064,6 +6065,53 @@ class TestRunBlocksCredentialApproval:
assert dispatched["browser_session_id"] == "pbs_carried"
+ @pytest.mark.asyncio
+ async def test_run_blocks_reports_a_resumed_browser_refused_at_submission_as_already_closed(
+ self, monkeypatch
+ ) -> None:
+ # Submission now refuses a session that has ended, so no run exists to carry the lease-seam
+ # reason code; the turn still needs the typed browser loss that offers the fresh-browser retry.
+ from skyvern.forge.sdk.copilot.tools import _run_blocks_and_collect_debug
+ from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
+ from skyvern.services import workflow_service as workflow_service_module
+
+ workflow = self._typed_resume_workflow()
+ organization = Organization(
+ organization_id="org-1",
+ organization_name="org",
+ created_at=datetime.now(timezone.utc),
+ modified_at=datetime.now(timezone.utc),
+ )
+ database = self._db(workflow=workflow, organization_lookup=organization)
+ database.workflow_params = SimpleNamespace(
+ get_workflow_output_parameters=AsyncMock(return_value=[workflow.get_output_parameter("login")])
+ )
+ ended_at = datetime.now(timezone.utc) - timedelta(minutes=5)
+ refused = BrowserSessionExpired(
+ "pbs_carried",
+ started_at=ended_at - timedelta(minutes=60),
+ ended_at=ended_at,
+ timeout_minutes=60,
+ created_by="copilot",
+ refused_at_submission=True,
+ )
+ monkeypatch.setattr(run_execution_module.app, "DATABASE", database)
+ monkeypatch.setattr(
+ run_execution_module.app,
+ "WORKFLOW_SERVICE",
+ SimpleNamespace(get_workflow_parameters=AsyncMock(return_value=[])),
+ )
+ monkeypatch.setattr(workflow_service_module, "prepare_workflow", AsyncMock(side_effect=refused))
+
+ ctx = _ctx(browser_session_id=None)
+ ctx.frontier_resume_session_id = "pbs_carried"
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["login"], "parameters": {}}, ctx)
+
+ assert result["ok"] is False
+ assert result["data"]["build_test_connect_failure"]["state"] == "already_closed"
+ assert result["data"]["browser_session_id"] == "pbs_carried"
+
@pytest.mark.asyncio
async def test_run_blocks_rejects_unapproved_workflow_credential_before_dispatch(self, monkeypatch) -> None:
from skyvern.forge.sdk.copilot.tools import _run_blocks_and_collect_debug
diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.py
index 86bb5fc6d..961706ccd 100644
--- a/tests/unit/test_failure_classifier.py
+++ b/tests/unit/test_failure_classifier.py
@@ -2,6 +2,8 @@ from __future__ import annotations
import json
import math
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -1025,16 +1027,26 @@ def test_derive_is_never_raises_by_construction_for_pathological_primary() -> No
def test_browser_lease_producer_reason_codes_round_trip_through_attribution() -> None:
# Drift guard: the real producer's emitted reason codes must survive derivation. If a new
# browser-lease code is added without updating the shared allowlist, this fails.
- from skyvern.exceptions import BrowserSessionClosed, BrowserSessionStartupTimeout
+ from skyvern.exceptions import BrowserSessionClosed, BrowserSessionExpired, BrowserSessionStartupTimeout
from skyvern.forge.sdk.workflow.service import _browser_lease_failure_category
- for exc in (
- BrowserSessionClosed(browser_session_id="pbs_x"),
- BrowserSessionStartupTimeout(browser_session_id="pbs_x"),
+ started_at = datetime(2026, 9, 28, 13, 1, tzinfo=UTC)
+ ended_at = started_at + timedelta(minutes=240)
+ workflow_run = SimpleNamespace(created_at=ended_at + timedelta(hours=6))
+ expired = BrowserSessionExpired(
+ "pbs_x", started_at=started_at, ended_at=ended_at, timeout_minutes=240, created_by="api"
+ )
+ for exc, component in (
+ (BrowserSessionClosed(browser_session_id="pbs_x"), "browser"),
+ (BrowserSessionStartupTimeout(browser_session_id="pbs_x"), "browser"),
+ (expired, "non_infra"),
):
- fc = _browser_lease_failure_category(exc)
+ fc = _browser_lease_failure_category(exc, workflow_run)
assert fc is not None
doc = derive_failure_attribution(fc)
assert doc["reason_code"] == fc[0]["reason_code"]
assert doc["evidence_source"] == "reason_code"
- assert doc["primary_infra_component"] == "browser"
+ assert doc["primary_infra_component"] == component
+ assert derive_failure_attribution(_browser_lease_failure_category(expired, workflow_run))["failure_category"] == (
+ FailureCategory.BROWSER_SESSION_EXPIRED
+ )
diff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.py
index 15d671aa0..c409af5b4 100644
--- a/tests/unit/test_request_logging.py
+++ b/tests/unit/test_request_logging.py
@@ -37,6 +37,7 @@ from skyvern.forge.request_logging import (
log_raw_request_middleware,
set_request_organization,
)
+from skyvern.forge.sdk.core.run_submission_gate import RunSubmissionGate
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.services.org_auth_service import apply_request_org_context
@@ -721,6 +722,16 @@ def _make_app(unhandled_exception_status: int = 500) -> FastAPI:
async def payment_required() -> dict:
raise HTTPException(status_code=402, detail="Payment Required")
+ @app.post("/runs/shed")
+ async def shed_run_submission() -> dict:
+ gate = RunSubmissionGate(limit=1, wait_seconds=0.01)
+ async with gate.slot(), gate.slot():
+ return {"dispatched": True}
+
+ @app.post("/runs/unavailable")
+ async def unavailable() -> dict:
+ raise HTTPException(status_code=503, detail="Service unavailable")
+
@app.post("/post-only")
async def post_only() -> dict:
return {"ok": True}
@@ -1063,6 +1074,15 @@ class TestMiddlewareLogVolume:
assert log_mock.warning.call_args.args[0] == "api.raw_request"
assert log_mock.warning.call_args.kwargs["status_code"] == 403
+ @pytest.mark.parametrize(("path", "level"), [("/runs/shed", "warning"), ("/runs/unavailable", "error")])
+ def test_shed_run_submission_logs_at_warning_while_other_503s_stay_errors(
+ self, log_mock: MagicMock, path: str, level: str
+ ) -> None:
+ response = TestClient(_make_app()).post(path)
+
+ assert response.status_code == 503
+ assert [(call[0], call.kwargs["status_code"]) for call in log_mock.mock_calls] == [(level, 503)]
+
def test_artifact_url_queries_are_redacted_from_logged_bodies(self, log_mock: MagicMock) -> None:
client = TestClient(_make_app())
response = client.post(
diff --git a/tests/unit/test_run_submission_gate.py b/tests/unit/test_run_submission_gate.py
new file mode 100644
index 000000000..f2720f7b8
--- /dev/null
+++ b/tests/unit/test_run_submission_gate.py
@@ -0,0 +1,160 @@
+import asyncio
+
+import pytest
+from fastapi import HTTPException
+
+from skyvern.config import Settings
+from skyvern.forge import app
+from skyvern.forge.sdk.core import run_submission_gate
+from skyvern.forge.sdk.core.run_submission_gate import (
+ DISABLE_RUN_SUBMISSION_GATE_FLAG,
+ RETRY_AFTER_SECONDS,
+ RunSubmissionGate,
+ run_submission_slot,
+)
+from tests.unit._workflow_block_engine_fakes import FakeExperimentationProvider
+
+# A leaked slot must fail the test, not hang it: no pytest timeout is configured.
+_BOUNDED_WAIT_SECONDS = 1
+
+
+async def _hold_slot(gate: RunSubmissionGate, entered: asyncio.Event, release: asyncio.Event) -> None:
+ async with gate.slot():
+ entered.set()
+ await release.wait()
+
+
+async def test_submission_over_the_cap_waits_for_a_slot_instead_of_failing() -> None:
+ gate = RunSubmissionGate(limit=1, wait_seconds=5)
+ entered, release = asyncio.Event(), asyncio.Event()
+ holder = asyncio.create_task(_hold_slot(gate, entered, release))
+ await asyncio.wait_for(entered.wait(), _BOUNDED_WAIT_SECONDS)
+
+ admitted = asyncio.Event()
+
+ async def queued_submission() -> None:
+ async with gate.slot():
+ admitted.set()
+
+ waiter = asyncio.create_task(queued_submission())
+ with pytest.raises(TimeoutError):
+ await asyncio.wait_for(asyncio.shield(admitted.wait()), timeout=0.05)
+ assert (gate.in_flight, gate.waiting) == (1, 1)
+
+ release.set()
+ await asyncio.wait_for(asyncio.gather(holder, waiter), _BOUNDED_WAIT_SECONDS)
+ assert admitted.is_set()
+ assert (gate.in_flight, gate.waiting) == (0, 0)
+
+
+async def test_submission_that_gets_no_slot_in_time_is_rejected_before_it_dispatches() -> None:
+ gate = RunSubmissionGate(limit=1, wait_seconds=0.05)
+ dispatched = False
+
+ async with gate.slot():
+ with pytest.raises(HTTPException) as rejected:
+ async with gate.slot():
+ dispatched = True
+ assert gate.waiting == 0
+
+ assert not dispatched
+ assert rejected.value.status_code == 503
+ assert rejected.value.headers == {"Retry-After": str(RETRY_AFTER_SECONDS)}
+ # The timed-out waiter must not have consumed the slot the holder then released.
+ async with gate.slot():
+ pass
+
+
+async def test_slot_is_released_when_dispatch_fails_or_the_request_is_cancelled() -> None:
+ gate = RunSubmissionGate(limit=1, wait_seconds=0.05)
+
+ with pytest.raises(RuntimeError):
+ async with gate.slot():
+ raise RuntimeError("dispatch failed")
+
+ entered = asyncio.Event()
+ cancelled = asyncio.create_task(_hold_slot(gate, entered, asyncio.Event()))
+ await asyncio.wait_for(entered.wait(), _BOUNDED_WAIT_SECONDS)
+ cancelled.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await cancelled
+ assert gate.in_flight == 0
+
+ # A leaked slot would make this wait out its 50 ms and raise the 503.
+ async with gate.slot():
+ pass
+
+
+ ("provider", "admitted"),
+ [
+ (FakeExperimentationProvider({DISABLE_RUN_SUBMISSION_GATE_FLAG: True}), True),
+ (FakeExperimentationProvider(), False),
+ (FakeExperimentationProvider(raise_error=True), False),
+ ],
+ ids=["kill_switch_on", "flag_absent", "flag_lookup_fails"],
+)
+async def test_kill_switch_lets_submissions_past_a_full_gate_and_anything_else_keeps_the_setting(
+ monkeypatch: pytest.MonkeyPatch, provider: FakeExperimentationProvider, admitted: bool
+) -> None:
+ full_gate = RunSubmissionGate(limit=1, wait_seconds=0.05)
+ monkeypatch.setattr(run_submission_gate, "_GATE", full_gate)
+ monkeypatch.setattr(app, "EXPERIMENTATION_PROVIDER", provider)
+ dispatched = False
+
+ async with full_gate.slot():
+ try:
+ async with run_submission_slot("o_123"):
+ dispatched = True
+ except HTTPException as rejected:
+ assert rejected.status_code == 503
+
+ assert dispatched is admitted
+
+
+ ("limit", "provider"),
+ [
+ (Settings.model_fields["RUN_SUBMISSION_MAX_CONCURRENCY"].default, FakeExperimentationProvider()),
+ (1, FakeExperimentationProvider({DISABLE_RUN_SUBMISSION_GATE_FLAG: True})),
+ ],
+ ids=["default_limit", "kill_switch_on"],
+)
+async def test_gate_that_is_off_never_waits_or_rejects_but_still_counts_submissions_in_flight(
+ monkeypatch: pytest.MonkeyPatch, limit: int, provider: FakeExperimentationProvider
+) -> None:
+ gate = RunSubmissionGate(limit=limit, wait_seconds=0.05)
+ monkeypatch.setattr(run_submission_gate, "_GATE", gate)
+ monkeypatch.setattr(app, "EXPERIMENTATION_PROVIDER", provider)
+ # More submissions than the limit, so a gate that enforced it would leave some waiting.
+ entered = [asyncio.Event() for _ in range(limit + 5)]
+ release = asyncio.Event()
+
+ async def submission(index: int) -> None:
+ async with run_submission_slot("o_123"):
+ entered[index].set()
+ await release.wait()
+ if index == 0:
+ raise RuntimeError("dispatch failed")
+
+ submissions = [asyncio.create_task(submission(index)) for index in range(len(entered))]
+ await asyncio.wait_for(asyncio.gather(*(event.wait() for event in entered)), _BOUNDED_WAIT_SECONDS)
+ assert (gate.in_flight, gate.waiting) == (len(entered), 0)
+
+ submissions[1].cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await submissions[1]
+ assert gate.in_flight == len(entered) - 1
+
+ release.set()
+ outcomes = await asyncio.wait_for(
+ asyncio.gather(submissions[0], *submissions[2:], return_exceptions=True), _BOUNDED_WAIT_SECONDS
+ )
+ assert isinstance(outcomes[0], RuntimeError)
+ assert outcomes[1:] == [None] * (len(entered) - 2)
+ assert (gate.in_flight, gate.waiting) == (0, 0)
diff --git a/tests/unit/test_saved_session_proxy_pin.py b/tests/unit/test_saved_session_proxy_pin.py
index 370dd5e16..bf75ffa58 100644
--- a/tests/unit/test_saved_session_proxy_pin.py
+++ b/tests/unit/test_saved_session_proxy_pin.py
@@ -793,7 +793,11 @@ async def test_create_workflow_run_non_force_path_single_create_no_update(monkey
monkeypatch.setattr(
app.DATABASE.browser_sessions,
"get_persistent_browser_session",
- AsyncMock(return_value=SimpleNamespace(browser_profile_id=None)),
+ AsyncMock(
+ return_value=SimpleNamespace(
+ browser_profile_id=None, status="running", completed_at=None, close_requested_at=None
+ )
+ ),
)
monkeypatch.setattr(app.DATABASE.workflow_runs, "create_workflow_run", create_workflow_run)
monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run", update_workflow_run)
diff --git a/tests/unit/test_workflow_service_cancel.py b/tests/unit/test_workflow_service_cancel.py
index ac20e09d7..256f7d273 100644
--- a/tests/unit/test_workflow_service_cancel.py
+++ b/tests/unit/test_workflow_service_cancel.py
@@ -728,10 +728,14 @@ async def test_shielded_finalize_skipped_when_pre_finally_status_unset() -> None
def test_lease_failure_category_persists_only_typed_browser_loss() -> None:
- closed = _browser_lease_failure_category(BrowserSessionClosed("pbs_x"))
- timeout = _browser_lease_failure_category(BrowserSessionStartupTimeout("pbs_x"))
+ workflow_run = SimpleNamespace(created_at=datetime.now(UTC))
+ closed = _browser_lease_failure_category(BrowserSessionClosed("pbs_x"), workflow_run)
+ timeout = _browser_lease_failure_category(BrowserSessionStartupTimeout("pbs_x"), workflow_run)
assert closed is not None and closed[0]["category"] == "BROWSER_ERROR"
assert closed[0]["reason_code"] == "browser_session_closed"
assert timeout is not None and timeout[0]["reason_code"] == "browser_session_startup_timeout"
- assert _browser_lease_failure_category(RuntimeError("a required workflow parameter was not provided")) is None
+ assert (
+ _browser_lease_failure_category(RuntimeError("a required workflow parameter was not provided"), workflow_run)
+ is None
+ )
diff --git a/tests/unit/test_workflow_service_managed_browser_profile.py b/tests/unit/test_workflow_service_managed_browser_profile.py
index 82bf7e99a..9d691228e 100644
--- a/tests/unit/test_workflow_service_managed_browser_profile.py
+++ b/tests/unit/test_workflow_service_managed_browser_profile.py
@@ -12,7 +12,9 @@ import pytest_asyncio
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncEngine
+from skyvern.exceptions import BrowserSessionExpired
from skyvern.forge import app
+from skyvern.forge.agent import ForgeAgent
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.agent_db import AgentDB
@@ -21,6 +23,7 @@ from skyvern.forge.sdk.db.models import (
BrowserProfileModel,
CredentialModel,
PersistentBrowserSessionModel,
+ TaskModel,
TaskRunModel,
WorkflowModel,
WorkflowRunAttemptModel,
@@ -31,6 +34,7 @@ from skyvern.forge.sdk.db.repositories import workflow_runs as workflow_runs_rep
from skyvern.forge.sdk.db.repositories.workflow_runs import PrepareNextAttemptResult, WorkflowRunDispatchFinalization
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.persistent_browser_sessions import BrowserSessionCloseReason
+from skyvern.forge.sdk.schemas.tasks import TaskRequest
from skyvern.forge.sdk.workflow import service as service_module
from skyvern.forge.sdk.workflow.browser_profile_key import (
build_browser_profile_key_digest,
@@ -1833,6 +1837,60 @@ async def test_execute_workflow_closes_owned_session_when_begin_session_fails(
close_session.assert_awaited_once_with("o_test", "pbs_human", reason=BrowserSessionCloseReason.run_ended)
+ ("created_by", "bound", "ended_after_run_created", "expected_category", "expected_reason_code"),
+ [
+ pytest.param(
+ "api", False, False, "BROWSER_SESSION_EXPIRED", "browser_session_expired_before_run", id="customer"
+ ),
+ pytest.param(
+ None, False, False, "BROWSER_SESSION_EXPIRED", "browser_session_expired_before_run", id="legacy-null"
+ ),
+ pytest.param(None, True, False, "BROWSER_ERROR", "browser_session_closed", id="reuse-bound-session"),
+ pytest.param("copilot", False, False, "BROWSER_ERROR", "browser_session_closed", id="copilot-session"),
+ pytest.param("api", False, True, "BROWSER_ERROR", "browser_session_closed", id="expired-while-queued"),
+ ],
+)
+async def test_lease_of_an_expired_session_is_the_callers_only_when_sent_after_it_ended(
+ monkeypatch: pytest.MonkeyPatch,
+ created_by: str | None,
+ bound: bool,
+ ended_after_run_created: bool,
+ expected_category: str,
+ expected_reason_code: str,
+) -> None:
+ workflow = _execute_workflow()
+ failed_run = _execute_workflow_run(WorkflowRunStatus.failed)
+ svc = WorkflowService()
+ _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
+ _patch_session_backed_run(monkeypatch, svc)
+ run_created_at = _execute_workflow_run(WorkflowRunStatus.running).created_at
+ ended_at = run_created_at + timedelta(minutes=1) if ended_after_run_created else run_created_at - timedelta(hours=6)
+ expired = BrowserSessionExpired(
+ "pbs_caller",
+ started_at=ended_at - timedelta(minutes=240),
+ ended_at=ended_at,
+ timeout_minutes=240,
+ created_by=created_by,
+ bound=bound,
+ )
+ monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "begin_session", AsyncMock(side_effect=expired))
+ mark_failed = AsyncMock(return_value=failed_run)
+ monkeypatch.setattr(svc, "mark_workflow_run_as_failed", mark_failed)
+ _patch_browser_cleanup(monkeypatch, svc, [])
+
+ assert await _run_execute_workflow(svc, browser_session_id="pbs_caller") is failed_run
+
+ failure_category = mark_failed.await_args.kwargs["failure_category"]
+ failure_reason = mark_failed.await_args.kwargs["failure_reason"]
+ assert (failure_category[0]["category"], failure_category[0]["reason_code"]) == (
+ expected_category,
+ expected_reason_code,
+ )
+ assert "pbs_caller" in failure_reason
+
+
@pytest.mark.asyncio
async def test_owned_session_close_deferred_while_child_runs_active(
monkeypatch: pytest.MonkeyPatch,
@@ -2364,7 +2422,15 @@ async def test_retry_preparation_preserves_session_after_transient_forced_pin_fa
monkeypatch.setattr(
database.browser_sessions,
"get_persistent_browser_session",
- AsyncMock(return_value=SimpleNamespace(browser_profile_id=None, runnable_id="wr_other")),
+ AsyncMock(
+ return_value=SimpleNamespace(
+ browser_profile_id=None,
+ runnable_id="wr_other",
+ status="running",
+ completed_at=None,
+ close_requested_at=None,
+ )
+ ),
)
run = await service.create_workflow_run(
workflow_request=WorkflowRequestBody(
@@ -2400,6 +2466,66 @@ async def test_retry_preparation_preserves_session_after_transient_forced_pin_fa
assert pin_calls == (2 if session_source == "forced" else 0)
[email protected]("entry_point", ["workflow_run", "task"])
+async def test_submission_refuses_a_session_past_its_lifetime_and_accepts_a_live_one(
+ monkeypatch: pytest.MonkeyPatch,
+ forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
+ entry_point: str,
+) -> None:
+ database, service, workflow = forced_session_setup
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False))
+ ended_at = datetime(2026, 9, 28, 17, 1)
+ async with database.Session() as session:
+ session.add_all(
+ [
+ PersistentBrowserSessionModel(
+ persistent_browser_session_id="pbs_expired",
+ organization_id="o_test",
+ status="timeout",
+ timeout_minutes=240,
+ started_at=ended_at - timedelta(minutes=240),
+ completed_at=ended_at,
+ created_by="api",
+ ),
+ PersistentBrowserSessionModel(
+ persistent_browser_session_id="pbs_live",
+ organization_id="o_test",
+ status="running",
+ timeout_minutes=240,
+ started_at=datetime.now(UTC).replace(tzinfo=None),
+ created_by="api",
+ ),
+ ]
+ )
+ await session.commit()
+
+ async def submit(browser_session_id: str) -> str | None:
+ if entry_point == "task":
+ task = await ForgeAgent().create_task(
+ TaskRequest(url="https://example.com", browser_session_id=browser_session_id), "o_test"
+ )
+ return task.browser_session_id
+ run = await service.create_workflow_run(
+ workflow_request=WorkflowRequestBody(browser_session_id=browser_session_id),
+ workflow_permanent_id="wpid_test",
+ workflow_id="wf_1",
+ organization_id="o_test",
+ workflow=workflow,
+ )
+ return run.browser_session_id
+
+ with pytest.raises(BrowserSessionExpired) as refused:
+ await submit("pbs_expired")
+
+ assert refused.value.status_code == 410
+ assert "pbs_expired" in refused.value.message
+ async with database.Session() as session:
+ assert (await session.scalars(select(WorkflowRunModel))).all() == []
+ assert (await session.scalars(select(TaskModel))).all() == []
+ assert await submit("pbs_live") == "pbs_live"
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("failure_point", ["assignment", "pin"])
async def test_forced_session_assignment_precedes_required_pin(