Skip to content

Automate browser based workflows with AI

[codex] Add workflow runtime limit backend (#6201)

Suchintand654db4b10b8
diff --git a/alembic/versions/2026_05_27_1503-4b254743ea85_add_max_elapsed_time_to_workflows_and_workflow_runs.py b/alembic/versions/2026_05_27_1503-4b254743ea85_add_max_elapsed_time_to_workflows_and_workflow_runs.py
new file mode 100644
index 000000000..0d708628a
--- /dev/null
+++ b/alembic/versions/2026_05_27_1503-4b254743ea85_add_max_elapsed_time_to_workflows_and_workflow_runs.py
@@ -0,0 +1,35 @@
+"""add max elapsed time to workflows and workflow_runs
+
+Revision ID: 4b254743ea85
+Revises: 8a7754a48701
+Create Date: 2026-05-27T15:03:10.396250+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "4b254743ea85"
+down_revision: Union[str, None] = "8a7754a48701"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+_COLUMN_NAME = "max_elapsed_time_minutes"
+_LOCK_TIMEOUT = "5s"
+
+
+def upgrade() -> None:
+ op.execute(f"SET LOCAL lock_timeout = '{_LOCK_TIMEOUT}'")
+ op.add_column("workflows", sa.Column(_COLUMN_NAME, sa.Integer(), nullable=True))
+ # Nullable ADD COLUMN without a default is metadata-only on PG 11+, including on high-traffic workflow_runs.
+ op.add_column("workflow_runs", sa.Column(_COLUMN_NAME, sa.Integer(), nullable=True))
+
+
+def downgrade() -> None:
+ op.execute(f"SET LOCAL lock_timeout = '{_LOCK_TIMEOUT}'")
+ op.drop_column("workflow_runs", _COLUMN_NAME)
+ op.drop_column("workflows", _COLUMN_NAME)
diff --git a/skyvern/client/client.py b/skyvern/client/client.py
index d7411d832..e61082be9 100644
--- a/skyvern/client/client.py
+++ b/skyvern/client/client.py
@@ -329,6 +329,7 @@ class Skyvern:
browser_session_id: typing.Optional[str] = OMIT,
browser_profile_id: typing.Optional[str] = OMIT,
max_screenshot_scrolls: typing.Optional[int] = OMIT,
+ max_elapsed_time_minutes: typing.Optional[int] = OMIT,
extra_http_headers: typing.Optional[typing.Dict[str, typing.Optional[str]]] = OMIT,
browser_address: typing.Optional[str] = OMIT,
ai_fallback: typing.Optional[bool] = OMIT,
@@ -410,6 +411,9 @@ class Skyvern:
max_screenshot_scrolls : typing.Optional[int]
The maximum number of scrolls for the post action screenshot. When it's None or 0, it takes the current viewpoint screenshot.
+ max_elapsed_time_minutes : typing.Optional[int]
+ Timeout this workflow run after the configured elapsed runtime in minutes. Maximum runtime is 4 hours.
+
extra_http_headers : typing.Optional[typing.Dict[str, typing.Optional[str]]]
The extra HTTP headers for the requests in browser.
@@ -461,6 +465,7 @@ class Skyvern:
browser_session_id=browser_session_id,
browser_profile_id=browser_profile_id,
max_screenshot_scrolls=max_screenshot_scrolls,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
extra_http_headers=extra_http_headers,
browser_address=browser_address,
ai_fallback=ai_fallback,
@@ -2887,6 +2892,7 @@ class AsyncSkyvern:
browser_session_id: typing.Optional[str] = OMIT,
browser_profile_id: typing.Optional[str] = OMIT,
max_screenshot_scrolls: typing.Optional[int] = OMIT,
+ max_elapsed_time_minutes: typing.Optional[int] = OMIT,
extra_http_headers: typing.Optional[typing.Dict[str, typing.Optional[str]]] = OMIT,
browser_address: typing.Optional[str] = OMIT,
ai_fallback: typing.Optional[bool] = OMIT,
@@ -2968,6 +2974,9 @@ class AsyncSkyvern:
max_screenshot_scrolls : typing.Optional[int]
The maximum number of scrolls for the post action screenshot. When it's None or 0, it takes the current viewpoint screenshot.
+ max_elapsed_time_minutes : typing.Optional[int]
+ Timeout this workflow run after the configured elapsed runtime in minutes. Maximum runtime is 4 hours.
+
extra_http_headers : typing.Optional[typing.Dict[str, typing.Optional[str]]]
The extra HTTP headers for the requests in browser.
@@ -3027,6 +3036,7 @@ class AsyncSkyvern:
browser_session_id=browser_session_id,
browser_profile_id=browser_profile_id,
max_screenshot_scrolls=max_screenshot_scrolls,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
extra_http_headers=extra_http_headers,
browser_address=browser_address,
ai_fallback=ai_fallback,
diff --git a/skyvern/client/raw_client.py b/skyvern/client/raw_client.py
index ef984144f..4b8668351 100644
--- a/skyvern/client/raw_client.py
+++ b/skyvern/client/raw_client.py
@@ -289,6 +289,7 @@ class RawSkyvern:
browser_session_id: typing.Optional[str] = OMIT,
browser_profile_id: typing.Optional[str] = OMIT,
max_screenshot_scrolls: typing.Optional[int] = OMIT,
+ max_elapsed_time_minutes: typing.Optional[int] = OMIT,
extra_http_headers: typing.Optional[typing.Dict[str, typing.Optional[str]]] = OMIT,
browser_address: typing.Optional[str] = OMIT,
ai_fallback: typing.Optional[bool] = OMIT,
@@ -370,6 +371,9 @@ class RawSkyvern:
max_screenshot_scrolls : typing.Optional[int]
The maximum number of scrolls for the post action screenshot. When it's None or 0, it takes the current viewpoint screenshot.
+ max_elapsed_time_minutes : typing.Optional[int]
+ Timeout this workflow run after the configured elapsed runtime in minutes. Maximum runtime is 4 hours.
+
extra_http_headers : typing.Optional[typing.Dict[str, typing.Optional[str]]]
The extra HTTP headers for the requests in browser.
@@ -412,6 +416,7 @@ class RawSkyvern:
"browser_session_id": browser_session_id,
"browser_profile_id": browser_profile_id,
"max_screenshot_scrolls": max_screenshot_scrolls,
+ "max_elapsed_time_minutes": max_elapsed_time_minutes,
"extra_http_headers": extra_http_headers,
"browser_address": browser_address,
"ai_fallback": ai_fallback,
@@ -3864,6 +3869,7 @@ class AsyncRawSkyvern:
browser_session_id: typing.Optional[str] = OMIT,
browser_profile_id: typing.Optional[str] = OMIT,
max_screenshot_scrolls: typing.Optional[int] = OMIT,
+ max_elapsed_time_minutes: typing.Optional[int] = OMIT,
extra_http_headers: typing.Optional[typing.Dict[str, typing.Optional[str]]] = OMIT,
browser_address: typing.Optional[str] = OMIT,
ai_fallback: typing.Optional[bool] = OMIT,
@@ -3945,6 +3951,9 @@ class AsyncRawSkyvern:
max_screenshot_scrolls : typing.Optional[int]
The maximum number of scrolls for the post action screenshot. When it's None or 0, it takes the current viewpoint screenshot.
+ max_elapsed_time_minutes : typing.Optional[int]
+ Timeout this workflow run after the configured elapsed runtime in minutes. Maximum runtime is 4 hours.
+
extra_http_headers : typing.Optional[typing.Dict[str, typing.Optional[str]]]
The extra HTTP headers for the requests in browser.
@@ -3987,6 +3996,7 @@ class AsyncRawSkyvern:
"browser_session_id": browser_session_id,
"browser_profile_id": browser_profile_id,
"max_screenshot_scrolls": max_screenshot_scrolls,
+ "max_elapsed_time_minutes": max_elapsed_time_minutes,
"extra_http_headers": extra_http_headers,
"browser_address": browser_address,
"ai_fallback": ai_fallback,
diff --git a/skyvern/client/types/workflow_run_request.py b/skyvern/client/types/workflow_run_request.py
index a8fa5d491..57fdb1cd2 100644
--- a/skyvern/client/types/workflow_run_request.py
+++ b/skyvern/client/types/workflow_run_request.py
@@ -91,6 +91,11 @@ class WorkflowRunRequest(UniversalBaseModel):
The maximum number of scrolls for the post action screenshot. When it's None or 0, it takes the current viewpoint screenshot.
"""
+ max_elapsed_time_minutes: typing.Optional[int] = pydantic.Field(default=None)
+ """
+ Timeout this workflow run after the configured elapsed runtime in minutes. Maximum runtime is 4 hours.
+ """
+
extra_http_headers: typing.Optional[typing.Dict[str, typing.Optional[str]]] = pydantic.Field(default=None)
"""
The extra HTTP headers for the requests in browser.
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index c06011acb..262ea63bf 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -438,6 +438,7 @@ class WorkflowModel(SoftDeleteMixin, Base):
proxy_location = Column(String)
webhook_callback_url = Column(String)
max_screenshot_scrolling_times = Column(Integer, nullable=True)
+ max_elapsed_time_minutes = Column(Integer, nullable=True)
extra_http_headers = Column(JSON, nullable=True)
cdp_connect_headers = Column(JSON, nullable=True)
totp_verification_url = Column(String)
@@ -557,6 +558,7 @@ class WorkflowRunModel(Base):
totp_verification_url = Column(String)
totp_identifier = Column(String)
max_screenshot_scrolling_times = Column(Integer, nullable=True)
+ max_elapsed_time_minutes = Column(Integer, nullable=True)
extra_http_headers = Column(JSON, nullable=True)
cdp_connect_headers = Column(JSON, nullable=True)
browser_address = Column(String, nullable=True, index=True)
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index 1f438edfc..e1ecdb45c 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -154,6 +154,7 @@ class WorkflowRunsRepository(BaseRepository):
totp_identifier: str | None = None,
parent_workflow_run_id: str | None = None,
max_screenshot_scrolling_times: int | None = None,
+ max_elapsed_time_minutes: int | None = None,
extra_http_headers: dict[str, str] | None = None,
cdp_connect_headers: dict[str, str] | None = None,
browser_address: str | None = None,
@@ -185,6 +186,7 @@ class WorkflowRunsRepository(BaseRepository):
totp_identifier=totp_identifier,
parent_workflow_run_id=parent_workflow_run_id,
max_screenshot_scrolling_times=max_screenshot_scrolling_times,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
extra_http_headers=extra_http_headers,
cdp_connect_headers=cdp_connect_headers,
browser_address=browser_address,
diff --git a/skyvern/forge/sdk/db/repositories/workflows.py b/skyvern/forge/sdk/db/repositories/workflows.py
index 51791cb7d..03eb235d1 100644
--- a/skyvern/forge/sdk/db/repositories/workflows.py
+++ b/skyvern/forge/sdk/db/repositories/workflows.py
@@ -95,6 +95,7 @@ class WorkflowsRepository(BaseRepository):
proxy_location: ProxyLocationInput = None,
webhook_callback_url: str | None = None,
max_screenshot_scrolling_times: int | None = None,
+ max_elapsed_time_minutes: int | None = None,
extra_http_headers: dict[str, str] | None = None,
cdp_connect_headers: dict[str, str] | None = None,
totp_verification_url: str | None = None,
@@ -129,6 +130,7 @@ class WorkflowsRepository(BaseRepository):
totp_verification_url=totp_verification_url,
totp_identifier=totp_identifier,
max_screenshot_scrolling_times=max_screenshot_scrolling_times,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
extra_http_headers=extra_http_headers,
cdp_connect_headers=cdp_connect_headers,
persist_browser_session=persist_browser_session,
@@ -613,6 +615,7 @@ class WorkflowsRepository(BaseRepository):
browser_profile_id: str | None | object = _UNSET,
model: dict[str, Any] | None | object = _UNSET,
max_screenshot_scrolling_times: int | None | object = _UNSET,
+ max_elapsed_time_minutes: int | None | object = _UNSET,
extra_http_headers: dict[str, str] | None | object = _UNSET,
cdp_connect_headers: dict[str, str] | None | object = _UNSET,
ai_fallback: bool | None = None,
@@ -658,6 +661,8 @@ class WorkflowsRepository(BaseRepository):
workflow.model = model
if max_screenshot_scrolling_times is not _UNSET:
workflow.max_screenshot_scrolling_times = max_screenshot_scrolling_times
+ if max_elapsed_time_minutes is not _UNSET:
+ workflow.max_elapsed_time_minutes = max_elapsed_time_minutes
if extra_http_headers is not _UNSET:
workflow.extra_http_headers = extra_http_headers
if cdp_connect_headers is not _UNSET:
@@ -910,6 +915,7 @@ class WorkflowsRepository(BaseRepository):
browser_profile_id: str | None | object = _UNSET,
model: dict[str, Any] | None | object = _UNSET,
max_screenshot_scrolling_times: int | None | object = _UNSET,
+ max_elapsed_time_minutes: int | None | object = _UNSET,
extra_http_headers: dict[str, str] | None | object = _UNSET,
cdp_connect_headers: dict[str, str] | None | object = _UNSET,
ai_fallback: bool | None = None,
@@ -989,6 +995,8 @@ class WorkflowsRepository(BaseRepository):
workflow.model = model
if max_screenshot_scrolling_times is not _UNSET:
workflow.max_screenshot_scrolling_times = max_screenshot_scrolling_times
+ if max_elapsed_time_minutes is not _UNSET:
+ workflow.max_elapsed_time_minutes = max_elapsed_time_minutes
if extra_http_headers is not _UNSET:
workflow.extra_http_headers = extra_http_headers
if cdp_connect_headers is not _UNSET:
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index d800fc37b..335cac726 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -532,6 +532,7 @@ def convert_to_workflow(
model=workflow_model.model,
proxy_location=deserialize_proxy_location(workflow_model.proxy_location),
max_screenshot_scrolls=workflow_model.max_screenshot_scrolling_times,
+ max_elapsed_time_minutes=workflow_model.max_elapsed_time_minutes,
version=workflow_model.version,
is_saved_task=workflow_model.is_saved_task,
is_template=is_template,
@@ -590,6 +591,7 @@ def convert_to_workflow_run(
modified_at=workflow_run_model.modified_at,
workflow_title=workflow_title,
max_screenshot_scrolls=workflow_run_model.max_screenshot_scrolling_times,
+ max_elapsed_time_minutes=workflow_run_model.max_elapsed_time_minutes,
extra_http_headers=workflow_run_model.extra_http_headers,
cdp_connect_headers=workflow_run_model.cdp_connect_headers,
browser_address=workflow_run_model.browser_address,
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 7ed3fd022..1ccc8f17c 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -398,6 +398,26 @@ async def run_task(
raise HTTPException(status_code=400, detail=f"Invalid agent engine: {run_request.engine}")
+def _workflow_run_request_to_legacy_request(workflow_run_request: WorkflowRunRequest) -> WorkflowRequestBody:
+ return WorkflowRequestBody(
+ data=workflow_run_request.parameters,
+ proxy_location=workflow_run_request.proxy_location,
+ webhook_callback_url=workflow_run_request.webhook_url,
+ totp_identifier=workflow_run_request.totp_identifier,
+ totp_verification_url=workflow_run_request.totp_url,
+ browser_session_id=workflow_run_request.browser_session_id,
+ browser_profile_id=workflow_run_request.browser_profile_id,
+ max_screenshot_scrolls=workflow_run_request.max_screenshot_scrolls,
+ max_elapsed_time_minutes=workflow_run_request.max_elapsed_time_minutes,
+ extra_http_headers=workflow_run_request.extra_http_headers,
+ cdp_connect_headers=workflow_run_request.cdp_connect_headers,
+ browser_address=workflow_run_request.browser_address,
+ run_with=workflow_run_request.run_with,
+ ai_fallback=workflow_run_request.ai_fallback,
+ run_metadata=workflow_run_request.run_metadata,
+ )
+
+
@base_router.post(
"/run/workflows",
tags=["Workflow Runs"],
@@ -438,22 +458,7 @@ async def run_workflow(
workflow_id = workflow_run_request.workflow_id
context = skyvern_context.ensure_context()
request_id = context.request_id
- legacy_workflow_request = WorkflowRequestBody(
- data=workflow_run_request.parameters,
- proxy_location=workflow_run_request.proxy_location,
- webhook_callback_url=workflow_run_request.webhook_url,
- totp_identifier=workflow_run_request.totp_identifier,
- totp_verification_url=workflow_run_request.totp_url,
- browser_session_id=workflow_run_request.browser_session_id,
- browser_profile_id=workflow_run_request.browser_profile_id,
- max_screenshot_scrolls=workflow_run_request.max_screenshot_scrolls,
- extra_http_headers=workflow_run_request.extra_http_headers,
- cdp_connect_headers=workflow_run_request.cdp_connect_headers,
- browser_address=workflow_run_request.browser_address,
- run_with=workflow_run_request.run_with,
- ai_fallback=workflow_run_request.ai_fallback,
- run_metadata=workflow_run_request.run_metadata,
- )
+ legacy_workflow_request = _workflow_run_request_to_legacy_request(workflow_run_request)
trigger_type = workflow_run_trigger_type_from_user_agent(x_user_agent)
try:
diff --git a/skyvern/forge/sdk/workflow/models/run_limits.py b/skyvern/forge/sdk/workflow/models/run_limits.py
new file mode 100644
index 000000000..bf1a7e091
--- /dev/null
+++ b/skyvern/forge/sdk/workflow/models/run_limits.py
@@ -0,0 +1,16 @@
+DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES = 4 * 60
+
+
+def reject_bool_max_elapsed_time_minutes(max_elapsed_time_minutes: object) -> object:
+ if isinstance(max_elapsed_time_minutes, bool):
+ raise ValueError("max_elapsed_time_minutes must be an integer, not a boolean")
+ return max_elapsed_time_minutes
+
+
+def get_effective_workflow_run_max_elapsed_time_minutes(max_elapsed_time_minutes: int | None) -> int:
+ if not isinstance(max_elapsed_time_minutes, int) or isinstance(max_elapsed_time_minutes, bool):
+ return DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES
+ if max_elapsed_time_minutes <= 0:
+ return DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES
+ # Product hard ceiling: explicit workflow/run settings cannot extend past the default 4 hour cap.
+ return min(max_elapsed_time_minutes, DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES)
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index 11f4610a2..32d2f6717 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -15,6 +15,10 @@ from skyvern.forge.sdk.workflow.exceptions import (
)
from skyvern.forge.sdk.workflow.models.block import BlockTypeVar, ForLoopBlock, WhileLoopBlock, get_all_blocks
from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE, OutputParameter
+from skyvern.forge.sdk.workflow.models.run_limits import (
+ DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES,
+ reject_bool_max_elapsed_time_minutes,
+)
from skyvern.forge.sdk.workflow.models.validators import normalize_run_metadata, normalize_run_with
from skyvern.schemas.runs import ProxyLocationInput, ScriptRunResponse
from skyvern.schemas.workflows import BlockType, WorkflowStatus
@@ -32,6 +36,7 @@ class WorkflowRequestBody(BaseModel):
browser_session_id: str | None = None
browser_profile_id: str | None = None
max_screenshot_scrolls: int | None = None
+ max_elapsed_time_minutes: int | None = Field(default=None, ge=1, le=DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES)
extra_http_headers: dict[str, str] | None = None
cdp_connect_headers: dict[str, str] | None = None
browser_address: str | None = None
@@ -39,6 +44,11 @@ class WorkflowRequestBody(BaseModel):
ai_fallback: bool | None = None
run_metadata: dict[str, str] | None = None
+ @field_validator("max_elapsed_time_minutes", mode="before")
+ @classmethod
+ def validate_max_elapsed_time_minutes(cls, value: object) -> object:
+ return reject_bool_max_elapsed_time_minutes(value)
+
@field_validator("webhook_callback_url", "totp_verification_url")
@classmethod
def validate_urls(cls, url: str | None) -> str | None:
@@ -116,6 +126,7 @@ class Workflow(BaseModel):
model: dict[str, Any] | None = None
status: WorkflowStatus = WorkflowStatus.published
max_screenshot_scrolls: int | None = None
+ max_elapsed_time_minutes: int | None = None
extra_http_headers: dict[str, str] | None = None
cdp_connect_headers: dict[str, str] | None = None
run_with: str = "agent"
@@ -213,6 +224,7 @@ class WorkflowRun(BaseModel):
parent_workflow_run_id: str | None = None
workflow_title: str | None = None
max_screenshot_scrolls: int | None = None
+ max_elapsed_time_minutes: int | None = None
browser_address: str | None = None
run_with: str | None = None
script_run: ScriptRunResponse | None = None
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index cb7d06e38..f4c959def 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -99,6 +99,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
WorkflowParameter,
WorkflowParameterType,
)
+from skyvern.forge.sdk.workflow.models.run_limits import get_effective_workflow_run_max_elapsed_time_minutes
from skyvern.forge.sdk.workflow.models.workflow import (
Workflow,
WorkflowDefinition,
@@ -153,12 +154,26 @@ DEFAULT_WORKFLOW_TITLE = "New Workflow"
# Empirical S3 upload SLA; no start buffer (back-to-back leakage is worse than late uploads to the next run).
RECORDING_WINDOW_END_BUFFER = timedelta(minutes=15)
+# Skip post-run work when only a sub-millisecond budget remains; asyncio.timeout would fire on the first await.
+POST_RUN_TIMEOUT_EXHAUSTED_THRESHOLD_SECONDS = 0.001
def _as_utc(dt: datetime) -> datetime:
return dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
+def _get_workflow_run_max_elapsed_timeout_seconds(workflow_run: WorkflowRun) -> float:
+ effective_minutes = get_effective_workflow_run_max_elapsed_time_minutes(workflow_run.max_elapsed_time_minutes)
+ started_at = _as_utc(workflow_run.started_at or workflow_run.created_at)
+ elapsed_seconds = (datetime.now(UTC) - started_at).total_seconds()
+ return max(0.0, effective_minutes * 60 - elapsed_seconds)
+
+
+def _format_workflow_run_elapsed_timeout_failure_reason(effective_minutes: int) -> str:
+ minute_label = "minute" if effective_minutes == 1 else "minutes"
+ return f"Workflow run exceeded max elapsed runtime limit of {effective_minutes} {minute_label}."
+
+
def _select_recording_urls_in_window(
recordings: Sequence[FileInfo],
lower_bound: datetime,
@@ -958,6 +973,11 @@ class WorkflowService:
organization_id=organization.organization_id,
parent_workflow_run_id=parent_workflow_run_id,
sequential_key=workflow.sequential_key,
+ # Run-request nulls intentionally inherit the workflow default; workflow definition updates
+ # use model_fields_set so explicit null can clear the saved workflow-level setting.
+ max_elapsed_time_minutes=workflow_request.max_elapsed_time_minutes
+ if workflow_request.max_elapsed_time_minutes is not None
+ else workflow.max_elapsed_time_minutes,
debug_session_id=debug_session_id,
code_gen=code_gen,
workflow_run_id=workflow_run_id,
@@ -1318,6 +1338,83 @@ class WorkflowService:
prompts.reverse()
return "\n\n".join(prompts)
+ async def _handle_post_run_elapsed_timeout(
+ self,
+ *,
+ workflow_run_id: str,
+ organization_id: str | None,
+ workflow_run: WorkflowRun,
+ pre_finally_status: WorkflowRunStatus | None,
+ pre_finally_failure_reason: str | None,
+ timeout_failure_reason: str,
+ ) -> tuple[WorkflowRun, WorkflowRunStatus | None, str | None]:
+ if pre_finally_status is None:
+ if refreshed_workflow_run := await app.DATABASE.workflow_runs.get_workflow_run(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ ):
+ workflow_run = refreshed_workflow_run
+ if workflow_run.status.is_final():
+ pre_finally_status = workflow_run.status
+ pre_finally_failure_reason = workflow_run.failure_reason
+ if pre_finally_status and pre_finally_status.is_final():
+ LOG.info(
+ "Preserving terminal workflow run status after post-run elapsed timeout",
+ workflow_run_id=workflow_run_id,
+ pre_finally_status=pre_finally_status,
+ )
+ else:
+ workflow_run = await self.mark_workflow_run_as_timed_out(
+ workflow_run_id=workflow_run_id,
+ failure_reason=timeout_failure_reason,
+ )
+ pre_finally_status = WorkflowRunStatus.timed_out
+ pre_finally_failure_reason = timeout_failure_reason
+ return workflow_run, pre_finally_status, pre_finally_failure_reason
+
+ async def _shield_post_run_elapsed_timeout(
+ self,
+ *,
+ workflow_run_id: str,
+ organization_id: str | None,
+ workflow_run: WorkflowRun,
+ pre_finally_status: WorkflowRunStatus | None,
+ pre_finally_failure_reason: str | None,
+ timeout_failure_reason: str,
+ ) -> tuple[WorkflowRun, WorkflowRunStatus | None, str | None]:
+ handle_timeout_task = asyncio.create_task(
+ self._handle_post_run_elapsed_timeout(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ workflow_run=workflow_run,
+ pre_finally_status=pre_finally_status,
+ pre_finally_failure_reason=pre_finally_failure_reason,
+ timeout_failure_reason=timeout_failure_reason,
+ ),
+ name=f"post_run_elapsed_timeout_{workflow_run_id}",
+ )
+ try:
+ return await asyncio.shield(handle_timeout_task)
+ except asyncio.CancelledError:
+ LOG.warning(
+ "Cancellation received while handling post-run elapsed timeout; waiting for status write",
+ workflow_run_id=workflow_run_id,
+ )
+ try:
+ return await handle_timeout_task
+ except Exception:
+ LOG.exception(
+ "Post-run elapsed timeout handler failed after cancellation; falling back to direct timeout write",
+ workflow_run_id=workflow_run_id,
+ )
+ fallback_workflow_run = await asyncio.shield(
+ self.mark_workflow_run_as_timed_out(
+ workflow_run_id=workflow_run_id,
+ failure_reason=timeout_failure_reason,
+ )
+ )
+ return fallback_workflow_run, WorkflowRunStatus.timed_out, timeout_failure_reason
+
@traced(name="skyvern.workflow.execute", role="wrapper")
async def execute_workflow(
self,
@@ -1526,132 +1623,193 @@ class WorkflowService:
pre_finally_failure_reason: str | None = None
try:
- # Check if there's a related workflow script that should be used instead
- workflow_script, _, script_is_pinned = await workflow_script_service.get_workflow_script(
- workflow, workflow_run, block_labels
- )
- current_context = skyvern_context.current()
- if current_context:
- if workflow_script:
- current_context.generate_script = False
- if workflow_run.code_gen:
- current_context.generate_script = True
- workflow_run, blocks_to_update = await self._execute_workflow_blocks(
- workflow=workflow,
- workflow_run=workflow_run,
- organization=organization,
- browser_session_id=browser_session_id,
- browser_profile_id=browser_profile_id,
- block_labels=block_labels,
- block_outputs=block_outputs,
- script=workflow_script,
- script_is_pinned=script_is_pinned,
- )
-
- # Check if there's a finally block configured
- finally_block_label = workflow.workflow_definition.finally_block_label
-
- # Refresh workflow_run from DB to pick up status/failure_reason
- # set by _execute_workflow_blocks.
- if refreshed_workflow_run := await app.DATABASE.workflow_runs.get_workflow_run(
- workflow_run_id=workflow_run_id,
- organization_id=organization_id,
- ):
- workflow_run = refreshed_workflow_run
-
- pre_finally_status = workflow_run.status
- pre_finally_failure_reason = workflow_run.failure_reason
-
- # Statuses that always skip script generation
- skip_statuses = {WorkflowRunStatus.canceled, WorkflowRunStatus.failed, WorkflowRunStatus.timed_out}
- # When generate_script_on_terminal is enabled, allow terminated runs to generate scripts
- if not workflow.generate_script_on_terminal:
- skip_statuses.add(WorkflowRunStatus.terminated)
-
- if pre_finally_status not in skip_statuses:
- await self.generate_script_if_needed(
- workflow=workflow,
- workflow_run=workflow_run,
- block_labels=block_labels,
- blocks_to_update=blocks_to_update,
- finalize=True, # Force regeneration to ensure field mappings have complete action data
- has_conditionals=has_conditionals,
+ effective_max_elapsed_minutes = get_effective_workflow_run_max_elapsed_time_minutes(
+ workflow_run.max_elapsed_time_minutes
+ )
+ timeout_failure_reason = _format_workflow_run_elapsed_timeout_failure_reason(effective_max_elapsed_minutes)
+ max_elapsed_timeout_seconds = _get_workflow_run_max_elapsed_timeout_seconds(workflow_run)
+ blocks_to_update: set[str] = set()
+ if max_elapsed_timeout_seconds <= 0:
+ workflow_run = await self.mark_workflow_run_as_timed_out(
+ workflow_run_id=workflow_run_id,
+ failure_reason=timeout_failure_reason,
)
+ return workflow_run
else:
- LOG.info(
- "Skipping post-run script generation due to run status",
+ timeout_context = asyncio.timeout(max_elapsed_timeout_seconds)
+ try:
+ async with timeout_context:
+ # Check if there's a related workflow script that should be used instead.
+ workflow_script, _, script_is_pinned = await workflow_script_service.get_workflow_script(
+ workflow, workflow_run, block_labels
+ )
+ current_context = skyvern_context.current()
+ if current_context:
+ if workflow_script:
+ current_context.generate_script = False
+ if workflow_run.code_gen:
+ current_context.generate_script = True
+ workflow_run, blocks_to_update = await self._execute_workflow_blocks(
+ workflow=workflow,
+ workflow_run=workflow_run,
+ organization=organization,
+ browser_session_id=browser_session_id,
+ browser_profile_id=browser_profile_id,
+ block_labels=block_labels,
+ block_outputs=block_outputs,
+ script=workflow_script,
+ script_is_pinned=script_is_pinned,
+ )
+ except TimeoutError:
+ if not timeout_context.expired():
+ raise
+ workflow_run = await self.mark_workflow_run_as_timed_out(
+ workflow_run_id=workflow_run_id,
+ failure_reason=timeout_failure_reason,
+ )
+ return workflow_run
+
+ post_run_timeout_seconds = _get_workflow_run_max_elapsed_timeout_seconds(workflow_run)
+ if post_run_timeout_seconds <= POST_RUN_TIMEOUT_EXHAUSTED_THRESHOLD_SECONDS:
+ (
+ workflow_run,
+ pre_finally_status,
+ pre_finally_failure_reason,
+ ) = await self._shield_post_run_elapsed_timeout(
workflow_run_id=workflow_run_id,
- workflow_permanent_id=workflow.workflow_permanent_id,
+ organization_id=organization_id,
+ workflow_run=workflow_run,
pre_finally_status=pre_finally_status,
- blocks_to_update_count=len(blocks_to_update),
+ pre_finally_failure_reason=pre_finally_failure_reason,
+ timeout_failure_reason=timeout_failure_reason,
)
+ return workflow_run
- # Trigger AI Script Reviewer for adaptive caching workflows
- # Include terminated and failed runs — the reviewer filters to only
- # episodes where the AI fallback succeeded (actionable signal).
- # Skip canceled (user stopped) and timed_out (infrastructure issue)
- # Only trigger if the script was actually executed this run — reviewing based on
- # agent-only runs provides no signal about script quality and wastes LLM tokens.
- # Only trigger if this run used the latest script version — stale runs produce
- # episodes that may already be fixed in newer versions, and reviewing them creates
- # redundant/regressive versions.
- is_script_execution = await self.should_run_script(workflow, workflow_run)
- if (
- is_adaptive_caching(workflow, workflow_run)
- and is_script_execution
- and pre_finally_status
- not in (
- WorkflowRunStatus.canceled,
- WorkflowRunStatus.timed_out,
- )
- ):
- should_trigger_reviewer = True
- current_ctx = skyvern_context.current()
- if current_ctx and current_ctx.script_id:
- latest_script = await app.DATABASE.scripts.get_latest_script_version(
- script_id=current_ctx.script_id,
- organization_id=workflow.organization_id,
- )
- if latest_script and latest_script.script_revision_id != current_ctx.script_revision_id:
- should_trigger_reviewer = False
+ post_run_timeout_context = asyncio.timeout(post_run_timeout_seconds)
+ try:
+ async with post_run_timeout_context:
+ # Check if there's a finally block configured
+ finally_block_label = workflow.workflow_definition.finally_block_label
+
+ # Refresh workflow_run from DB to pick up status/failure_reason
+ # set by _execute_workflow_blocks.
+ if refreshed_workflow_run := await app.DATABASE.workflow_runs.get_workflow_run(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ ):
+ workflow_run = refreshed_workflow_run
+
+ pre_finally_status = workflow_run.status
+ pre_finally_failure_reason = workflow_run.failure_reason
+
+ # Statuses that always skip script generation
+ skip_statuses = {WorkflowRunStatus.canceled, WorkflowRunStatus.failed, WorkflowRunStatus.timed_out}
+ # When generate_script_on_terminal is enabled, allow terminated runs to generate scripts
+ if not workflow.generate_script_on_terminal:
+ skip_statuses.add(WorkflowRunStatus.terminated)
+
+ if pre_finally_status not in skip_statuses:
+ await self.generate_script_if_needed(
+ workflow=workflow,
+ workflow_run=workflow_run,
+ block_labels=block_labels,
+ blocks_to_update=blocks_to_update,
+ finalize=True, # Force regeneration to ensure field mappings have complete action data
+ has_conditionals=has_conditionals,
+ )
+ else:
LOG.info(
- "Skipping script reviewer - run used stale script version",
+ "Skipping post-run script generation due to run status",
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ pre_finally_status=pre_finally_status,
+ blocks_to_update_count=len(blocks_to_update),
+ )
+
+ # Trigger AI Script Reviewer for adaptive caching workflows
+ # Include terminated and failed runs — the reviewer filters to only
+ # episodes where the AI fallback succeeded (actionable signal).
+ # Skip canceled (user stopped) and timed_out (infrastructure issue)
+ # Only trigger if the script was actually executed this run — reviewing based on
+ # agent-only runs provides no signal about script quality and wastes LLM tokens.
+ # Only trigger if this run used the latest script version — stale runs produce
+ # episodes that may already be fixed in newer versions, and reviewing them creates
+ # redundant/regressive versions.
+ is_script_execution = await self.should_run_script(workflow, workflow_run)
+ if (
+ is_adaptive_caching(workflow, workflow_run)
+ and is_script_execution
+ and pre_finally_status
+ not in (
+ WorkflowRunStatus.canceled,
+ WorkflowRunStatus.timed_out,
+ )
+ ):
+ should_trigger_reviewer = True
+ current_ctx = skyvern_context.current()
+ if current_ctx and current_ctx.script_id:
+ latest_script = await app.DATABASE.scripts.get_latest_script_version(
+ script_id=current_ctx.script_id,
+ organization_id=workflow.organization_id,
+ )
+ if latest_script and latest_script.script_revision_id != current_ctx.script_revision_id:
+ should_trigger_reviewer = False
+ LOG.info(
+ "Skipping script reviewer - run used stale script version",
+ workflow_run_id=workflow_run.workflow_run_id,
+ used_revision=current_ctx.script_revision_id,
+ latest_revision=latest_script.script_revision_id,
+ latest_version=latest_script.version,
+ )
+ if should_trigger_reviewer:
+ asyncio.create_task(
+ self._trigger_script_reviewer(
+ workflow, workflow_run, pre_finally_status=pre_finally_status
+ ),
+ name=f"script_reviewer_{workflow_run.workflow_run_id}",
+ )
+ elif is_adaptive_caching(workflow, workflow_run):
+ LOG.info(
+ "Skipping script reviewer - script was not executed this run",
workflow_run_id=workflow_run.workflow_run_id,
- used_revision=current_ctx.script_revision_id,
- latest_revision=latest_script.script_revision_id,
- latest_version=latest_script.version,
+ run_with=workflow_run.run_with,
)
- if should_trigger_reviewer:
- asyncio.create_task(
- self._trigger_script_reviewer(workflow, workflow_run, pre_finally_status=pre_finally_status),
- name=f"script_reviewer_{workflow_run.workflow_run_id}",
- )
- elif is_adaptive_caching(workflow, workflow_run):
- LOG.info(
- "Skipping script reviewer - script was not executed this run",
- workflow_run_id=workflow_run.workflow_run_id,
- run_with=workflow_run.run_with,
- )
- # Execute finally block if configured. Skip for: canceled (user explicitly stopped)
- should_run_finally = finally_block_label and pre_finally_status != WorkflowRunStatus.canceled
- if should_run_finally:
- # Temporarily set to running for terminal workflows (for frontend UX)
- if pre_finally_status in (
- WorkflowRunStatus.failed,
- WorkflowRunStatus.terminated,
- WorkflowRunStatus.timed_out,
- ):
- workflow_run = await self._update_workflow_run_status(
- workflow_run_id=workflow_run_id,
- status=WorkflowRunStatus.running,
- failure_reason=None,
- )
- await self._execute_finally_block_if_configured(
- workflow=workflow,
+ # Execute finally block if configured. Skip only for canceled runs; elapsed-time timeouts return
+ # before post-run work, while other timeout statuses can still run cleanup within the remaining cap.
+ should_run_finally = finally_block_label and pre_finally_status != WorkflowRunStatus.canceled
+ if should_run_finally:
+ # Temporarily set to running for terminal workflows (for frontend UX)
+ if pre_finally_status in (
+ WorkflowRunStatus.failed,
+ WorkflowRunStatus.terminated,
+ WorkflowRunStatus.timed_out,
+ ):
+ workflow_run = await self._update_workflow_run_status(
+ workflow_run_id=workflow_run_id,
+ status=WorkflowRunStatus.running,
+ failure_reason=None,
+ )
+ await self._execute_finally_block_if_configured(
+ workflow=workflow,
+ workflow_run=workflow_run,
+ organization=organization,
+ browser_session_id=browser_session_id,
+ )
+ except TimeoutError:
+ if not post_run_timeout_context.expired():
+ raise
+ (
+ workflow_run,
+ pre_finally_status,
+ pre_finally_failure_reason,
+ ) = await self._shield_post_run_elapsed_timeout(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
workflow_run=workflow_run,
- organization=organization,
- browser_session_id=browser_session_id,
+ pre_finally_status=pre_finally_status,
+ pre_finally_failure_reason=pre_finally_failure_reason,
+ timeout_failure_reason=timeout_failure_reason,
)
finally:
# Shielded finalize runs even when the try body was cancelled
@@ -3500,6 +3658,7 @@ class WorkflowService:
description: str | None = None,
proxy_location: ProxyLocationInput = None,
max_screenshot_scrolling_times: int | None = None,
+ max_elapsed_time_minutes: int | None = None,
webhook_callback_url: str | None = None,
totp_verification_url: str | None = None,
totp_identifier: str | None = None,
@@ -3533,6 +3692,7 @@ class WorkflowService:
proxy_location=proxy_location,
webhook_callback_url=webhook_callback_url,
max_screenshot_scrolling_times=max_screenshot_scrolling_times,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
totp_verification_url=totp_verification_url,
totp_identifier=totp_identifier,
persist_browser_session=persist_browser_session,
@@ -3924,6 +4084,7 @@ class WorkflowService:
browser_profile_id: str | None | object = _UNSET,
model: dict[str, Any] | None | object = _UNSET,
max_screenshot_scrolling_times: int | None | object = _UNSET,
+ max_elapsed_time_minutes: int | None | object = _UNSET,
extra_http_headers: dict[str, str] | None | object = _UNSET,
cdp_connect_headers: dict[str, str] | None | object = _UNSET,
run_with: str | None = None,
@@ -3947,6 +4108,7 @@ class WorkflowService:
browser_profile_id=browser_profile_id,
model=model,
max_screenshot_scrolling_times=max_screenshot_scrolling_times,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
extra_http_headers=extra_http_headers,
cdp_connect_headers=cdp_connect_headers,
run_with=run_with,
@@ -3971,6 +4133,7 @@ class WorkflowService:
browser_profile_id=browser_profile_id,
model=model,
max_screenshot_scrolling_times=max_screenshot_scrolling_times,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
extra_http_headers=extra_http_headers,
cdp_connect_headers=cdp_connect_headers,
run_with=run_with,
@@ -4236,6 +4399,7 @@ class WorkflowService:
organization_id: str,
parent_workflow_run_id: str | None = None,
sequential_key: str | None = None,
+ max_elapsed_time_minutes: int | None = None,
debug_session_id: str | None = None,
code_gen: bool | None = None,
workflow_run_id: str | None = None,
@@ -4323,6 +4487,7 @@ class WorkflowService:
totp_identifier=workflow_request.totp_identifier,
parent_workflow_run_id=parent_workflow_run_id,
max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolls,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
extra_http_headers=workflow_request.extra_http_headers,
cdp_connect_headers=workflow_request.cdp_connect_headers,
browser_address=workflow_request.browser_address,
@@ -5815,6 +5980,11 @@ class WorkflowService:
request.cdp_connect_headers,
existing_latest_workflow.cdp_connect_headers,
)
+ effective_max_elapsed_time_minutes = (
+ request.max_elapsed_time_minutes
+ if "max_elapsed_time_minutes" in request.model_fields_set
+ else existing_latest_workflow.max_elapsed_time_minutes
+ )
# NOTE: it's only potential, as it may be immediately deleted!
potential_workflow = await self.create_workflow(
@@ -5830,6 +6000,7 @@ class WorkflowService:
browser_profile_id=request.browser_profile_id,
model=request.model,
max_screenshot_scrolling_times=request.max_screenshot_scrolls,
+ max_elapsed_time_minutes=effective_max_elapsed_time_minutes,
extra_http_headers=request.extra_http_headers,
cdp_connect_headers=effective_cdp_connect_headers,
workflow_permanent_id=existing_latest_workflow.workflow_permanent_id,
@@ -5869,6 +6040,7 @@ class WorkflowService:
browser_profile_id=request.browser_profile_id,
model=request.model,
max_screenshot_scrolling_times=request.max_screenshot_scrolls,
+ max_elapsed_time_minutes=request.max_elapsed_time_minutes,
extra_http_headers=request.extra_http_headers,
cdp_connect_headers=new_cdp_connect_headers,
is_saved_task=request.is_saved_task,
diff --git a/skyvern/library/skyvern.py b/skyvern/library/skyvern.py
index 521670623..7dae02d78 100644
--- a/skyvern/library/skyvern.py
+++ b/skyvern/library/skyvern.py
@@ -360,6 +360,7 @@ class Skyvern(AsyncSkyvern):
user_agent: str | None = None,
browser_profile_id: str | None = None,
max_screenshot_scrolls: int | None = None,
+ max_elapsed_time_minutes: int | None = None,
extra_http_headers: dict[str, str] | None = None,
browser_address: str | None = None,
ai_fallback: bool | None = None,
@@ -383,6 +384,7 @@ class Skyvern(AsyncSkyvern):
user_agent=user_agent,
browser_profile_id=browser_profile_id,
max_screenshot_scrolls=max_screenshot_scrolls,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
extra_http_headers=extra_http_headers,
browser_address=browser_address,
ai_fallback=ai_fallback,
diff --git a/skyvern/library/skyvern_browser_page_agent.py b/skyvern/library/skyvern_browser_page_agent.py
index aad82920d..5fa9ebe37 100644
--- a/skyvern/library/skyvern_browser_page_agent.py
+++ b/skyvern/library/skyvern_browser_page_agent.py
@@ -349,6 +349,7 @@ class SkyvernBrowserPageAgent:
webhook_url: str | None = None,
totp_url: str | None = None,
totp_identifier: str | None = None,
+ max_elapsed_time_minutes: int | None = None,
run_metadata: dict[str, str] | None = None,
timeout: float = DEFAULT_AGENT_TIMEOUT,
) -> WorkflowRunResponse:
@@ -362,6 +363,7 @@ class SkyvernBrowserPageAgent:
webhook_url: URL to receive webhook notifications about workflow progress.
totp_url: URL to fetch TOTP codes from.
totp_identifier: Identifier for TOTP authentication.
+ max_elapsed_time_minutes: Timeout this workflow run after the configured elapsed runtime in minutes.
run_metadata: String key/value metadata to attach to this workflow run.
timeout: Maximum time in seconds to wait for workflow completion.
@@ -379,6 +381,7 @@ class SkyvernBrowserPageAgent:
webhook_url=webhook_url,
totp_url=totp_url,
totp_identifier=totp_identifier,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
run_metadata=run_metadata,
browser_session_id=self._browser.browser_session_id,
browser_address=self._browser.browser_address,
diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.py
index bb2e80d21..f9a380372 100644
--- a/skyvern/schemas/runs.py
+++ b/skyvern/schemas/runs.py
@@ -6,6 +6,10 @@ from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeAlias, Union
from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_validator
from skyvern.forge.sdk.schemas.files import FileInfo
+from skyvern.forge.sdk.workflow.models.run_limits import (
+ DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES,
+ reject_bool_max_elapsed_time_minutes,
+)
from skyvern.forge.sdk.workflow.models.validators import normalize_run_metadata, normalize_run_with
from skyvern.schemas.docs.doc_examples import (
BROWSER_SESSION_ID_EXAMPLES,
@@ -240,6 +244,12 @@ class WorkflowRunRequest(BaseModel):
default=None,
description="The maximum number of scrolls for the post action screenshot. When it's None or 0, it takes the current viewpoint screenshot.",
)
+ max_elapsed_time_minutes: int | None = Field(
+ default=None,
+ ge=1,
+ le=DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES,
+ description="Timeout this workflow run after the configured elapsed runtime in minutes. Maximum runtime is 4 hours.",
+ )
extra_http_headers: dict[str, str] | None = Field(
default=None,
description="The extra HTTP headers for the requests in browser.",
@@ -279,6 +289,11 @@ class WorkflowRunRequest(BaseModel):
return None
return normalize_run_with(v)
+ @field_validator("max_elapsed_time_minutes", mode="before")
+ @classmethod
+ def validate_max_elapsed_time_minutes(cls, value: object) -> object:
+ return reject_bool_max_elapsed_time_minutes(value)
+
@field_validator("run_metadata")
@classmethod
def _validate_run_metadata(cls, v: dict[str, str] | None) -> dict[str, str] | None:
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index 6b7902d66..f11b4b469 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -11,6 +11,10 @@ from skyvern.config import settings
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType, WorkflowParameterType
+from skyvern.forge.sdk.workflow.models.run_limits import (
+ DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES,
+ reject_bool_max_elapsed_time_minutes,
+)
from skyvern.forge.sdk.workflow.models.validators import normalize_run_with
from skyvern.schemas.runs import GeoTarget, ProxyLocation, RunEngine
from skyvern.utils.secret_headers import mask_header_values
@@ -1198,6 +1202,7 @@ class WorkflowCreateYAMLRequest(BaseModel):
workflow_definition: WorkflowDefinitionYAML
is_saved_task: bool = False
max_screenshot_scrolls: int | None = None
+ max_elapsed_time_minutes: int | None = Field(default=None, ge=1, le=DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES)
extra_http_headers: dict[str, str] | None = None
cdp_connect_headers: dict[str, str] | None = None
status: WorkflowStatus = WorkflowStatus.published
@@ -1211,6 +1216,11 @@ class WorkflowCreateYAMLRequest(BaseModel):
sequential_key: str | None = None
folder_id: str | None = None
+ @field_validator("max_elapsed_time_minutes", mode="before")
+ @classmethod
+ def validate_max_elapsed_time_minutes(cls, value: object) -> object:
+ return reject_bool_max_elapsed_time_minutes(value)
+
@field_validator("run_with", mode="before")
@classmethod
def _normalize_run_with(cls, v: str | None) -> str:
diff --git a/tests/unit/test_sdk_run_metadata.py b/tests/unit/test_sdk_run_metadata.py
index ee4ee94a0..dc7c8589c 100644
--- a/tests/unit/test_sdk_run_metadata.py
+++ b/tests/unit/test_sdk_run_metadata.py
@@ -14,6 +14,12 @@ def test_workflow_run_request_accepts_run_metadata() -> None:
assert request.run_metadata == {"customer": "acme"}
+def test_workflow_run_request_accepts_max_elapsed_time() -> None:
+ request = WorkflowRunRequest(workflow_id="wpid_123", max_elapsed_time_minutes=10)
+
+ assert request.max_elapsed_time_minutes == 10
+
+
def test_run_workflow_sends_run_metadata() -> None:
captured_bodies: list[dict[str, object]] = []
@@ -49,6 +55,41 @@ def test_run_workflow_sends_run_metadata() -> None:
]
+def test_run_workflow_sends_max_elapsed_time() -> None:
+ captured_bodies: list[dict[str, object]] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ captured_bodies.append(json.loads(request.content))
+ return httpx.Response(
+ status_code=200,
+ json={
+ "run_id": "wr_123",
+ "status": "queued",
+ "created_at": datetime.now(UTC).isoformat(),
+ "modified_at": datetime.now(UTC).isoformat(),
+ },
+ )
+
+ client = Skyvern(
+ base_url="https://api.example.test",
+ api_key="test-key",
+ httpx_client=httpx.Client(transport=httpx.MockTransport(handler)),
+ )
+
+ response = client.run_workflow(
+ workflow_id="wpid_123",
+ max_elapsed_time_minutes=10,
+ )
+
+ assert response.run_id == "wr_123"
+ assert captured_bodies == [
+ {
+ "workflow_id": "wpid_123",
+ "max_elapsed_time_minutes": 10,
+ }
+ ]
+
+
def _workflow_run_payload() -> dict[str, object]:
now = datetime.now(UTC).isoformat()
return {
diff --git a/tests/unit/test_workflow_browser_profile_id.py b/tests/unit/test_workflow_browser_profile_id.py
index b592f8fb7..40893bd23 100644
--- a/tests/unit/test_workflow_browser_profile_id.py
+++ b/tests/unit/test_workflow_browser_profile_id.py
@@ -1,8 +1,10 @@
from __future__ import annotations
+from collections.abc import Generator
from datetime import datetime, timezone
from types import SimpleNamespace
-from unittest.mock import AsyncMock, patch
+from typing import Any, cast
+from unittest.mock import AsyncMock, Mock, patch
import pytest
@@ -69,6 +71,81 @@ def test_workflow_create_yaml_request_masks_cdp_connect_headers_on_dump() -> Non
}
+async def test_create_workflow_from_request_preserves_existing_max_elapsed_time_when_omitted() -> None:
+ service, updated_workflow = _make_workflow_update_service(existing_max_elapsed_time_minutes=90)
+
+ request = WorkflowCreateYAMLRequest(
+ title="test",
+ workflow_definition=WorkflowDefinitionYAML(parameters=[], blocks=[]),
+ )
+
+ result = await service.create_workflow_from_request(
+ organization=cast(Any, SimpleNamespace(organization_id="org_1")),
+ request=request,
+ workflow_permanent_id="wpid_test",
+ )
+
+ assert result is updated_workflow
+ create_workflow_mock = service.create_workflow
+ assert isinstance(create_workflow_mock, AsyncMock)
+ create_workflow_mock.assert_awaited_once()
+ assert create_workflow_mock.await_args is not None
+ assert create_workflow_mock.await_args.kwargs["max_elapsed_time_minutes"] == 90
+
+
+async def test_create_workflow_from_request_allows_explicit_null_to_clear_existing_max_elapsed_time() -> None:
+ service, updated_workflow = _make_workflow_update_service(existing_max_elapsed_time_minutes=90)
+
+ request = WorkflowCreateYAMLRequest(
+ title="test",
+ workflow_definition=WorkflowDefinitionYAML(parameters=[], blocks=[]),
+ max_elapsed_time_minutes=None,
+ )
+
+ result = await service.create_workflow_from_request(
+ organization=cast(Any, SimpleNamespace(organization_id="org_1")),
+ request=request,
+ workflow_permanent_id="wpid_test",
+ )
+
+ assert result is updated_workflow
+ create_workflow_mock = service.create_workflow
+ assert isinstance(create_workflow_mock, AsyncMock)
+ create_workflow_mock.assert_awaited_once()
+ assert create_workflow_mock.await_args is not None
+ assert create_workflow_mock.await_args.kwargs["max_elapsed_time_minutes"] is None
+
+
+def _make_workflow_update_service(
+ existing_max_elapsed_time_minutes: int | None,
+) -> tuple[WorkflowService, SimpleNamespace]:
+ service = WorkflowService()
+ existing_workflow = SimpleNamespace(
+ version=2,
+ cdp_connect_headers=None,
+ workflow_permanent_id="wpid_test",
+ folder_id=None,
+ code_version=None,
+ max_elapsed_time_minutes=existing_max_elapsed_time_minutes,
+ )
+ potential_workflow = SimpleNamespace(workflow_id="wf_new")
+ updated_workflow = SimpleNamespace(workflow_id="wf_new")
+
+ service.get_workflow_by_permanent_id = AsyncMock(return_value=existing_workflow) # type: ignore[method-assign]
+ service.create_workflow = AsyncMock(return_value=potential_workflow) # type: ignore[method-assign]
+ service.make_workflow_definition = AsyncMock( # type: ignore[method-assign]
+ return_value=WorkflowDefinition(parameters=[], blocks=[])
+ )
+ service.validate_workflow_block_graph = Mock() # type: ignore[method-assign]
+ service._validate_payload_templates = Mock() # type: ignore[method-assign]
+ service.update_workflow_definition = AsyncMock(return_value=updated_workflow) # type: ignore[method-assign]
+ service.maybe_delete_cached_code = AsyncMock() # type: ignore[method-assign]
+
+ return service, updated_workflow
+
+
def _make_setup_service(workflow: SimpleNamespace) -> tuple[WorkflowService, SimpleNamespace, SimpleNamespace]:
service = WorkflowService()
workflow_run = SimpleNamespace(workflow_run_id="wr_test", workflow_permanent_id="wpid_test")
@@ -83,7 +160,10 @@ def _make_setup_service(workflow: SimpleNamespace) -> tuple[WorkflowService, Sim
return service, organization, workflow_run
-def _make_workflow_stub(browser_profile_id: str | None) -> SimpleNamespace:
+def _make_workflow_stub(
+ browser_profile_id: str | None,
+ max_elapsed_time_minutes: int | None = None,
+) -> SimpleNamespace:
return SimpleNamespace(
workflow_id="wf_test",
workflow_permanent_id="wpid_test",
@@ -93,6 +173,7 @@ def _make_workflow_stub(browser_profile_id: str | None) -> SimpleNamespace:
extra_http_headers=None,
cdp_connect_headers=None,
browser_profile_id=browser_profile_id,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
run_with="agent",
code_version=None,
adaptive_caching=False,
@@ -101,7 +182,7 @@ def _make_workflow_stub(browser_profile_id: str | None) -> SimpleNamespace:
@pytest.fixture(autouse=True)
-def reset_context() -> None:
+def reset_context() -> Generator[None]:
skyvern_context.reset()
yield
skyvern_context.reset()
@@ -195,3 +276,29 @@ async def test_setup_workflow_run_session_present_skips_workflow_default() -> No
)
assert request.browser_profile_id is None
+
+
+async def test_setup_workflow_run_prefers_request_max_elapsed_time_over_workflow_default() -> None:
+ """A run-level runtime cap should override the workflow default for that run snapshot."""
+ workflow_stub = _make_workflow_stub(browser_profile_id=None, max_elapsed_time_minutes=120)
+ service, organization, _ = _make_setup_service(workflow_stub)
+
+ request = WorkflowRequestBody(data={}, max_elapsed_time_minutes=10)
+
+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
+ mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
+ mock_app.AGENT_FUNCTION.should_use_flex_llm_routing = AsyncMock(return_value=False)
+
+ await service.setup_workflow_run(
+ request_id="req_test",
+ workflow_request=request,
+ workflow_permanent_id="wpid_test",
+ organization=organization,
+ )
+
+ create_workflow_run_mock = service.create_workflow_run
+ assert isinstance(create_workflow_run_mock, AsyncMock)
+ create_workflow_run_mock.assert_awaited_once()
+ assert create_workflow_run_mock.await_args is not None
+ assert create_workflow_run_mock.await_args.kwargs["max_elapsed_time_minutes"] == 10
diff --git a/tests/unit/test_workflow_elapsed_timeout.py b/tests/unit/test_workflow_elapsed_timeout.py
new file mode 100644
index 000000000..d680c8e82
--- /dev/null
+++ b/tests/unit/test_workflow_elapsed_timeout.py
@@ -0,0 +1,891 @@
+import asyncio
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from typing import cast
+from unittest.mock import AsyncMock
+
+import pytest
+from pydantic import ValidationError
+
+from skyvern.forge.sdk.routes.agent_protocol import _workflow_run_request_to_legacy_request
+from skyvern.forge.sdk.workflow import service as service_module
+from skyvern.forge.sdk.workflow.models.run_limits import (
+ DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES,
+ get_effective_workflow_run_max_elapsed_time_minutes,
+)
+from skyvern.forge.sdk.workflow.models.workflow import (
+ Workflow,
+ WorkflowDefinition,
+ WorkflowRequestBody,
+ WorkflowRun,
+ WorkflowRunStatus,
+)
+from skyvern.forge.sdk.workflow.service import WorkflowService
+from skyvern.schemas.runs import WorkflowRunRequest
+from skyvern.schemas.workflows import WorkflowCreateYAMLRequest, WorkflowDefinitionYAML
+
+
+def _workflow_run(status: WorkflowRunStatus, *, started_at: datetime) -> SimpleNamespace:
+ return SimpleNamespace(
+ workflow_run_id="wr_1",
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ organization_id="org_1",
+ browser_profile_id="bp_1",
+ browser_address=None,
+ status=status,
+ failure_reason=None,
+ ignore_inherited_workflow_system_prompt=False,
+ parent_workflow_run_id=None,
+ proxy_location=None,
+ max_elapsed_time_minutes=1,
+ started_at=started_at,
+ created_at=started_at,
+ code_gen=False,
+ run_with="agent",
+ )
+
+
+def test_workflow_request_rejects_max_elapsed_time_above_cap() -> None:
+ with pytest.warns(DeprecationWarning), pytest.raises(ValidationError):
+ WorkflowRequestBody(max_elapsed_time_minutes=DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES + 1)
+
+ assert (
+ get_effective_workflow_run_max_elapsed_time_minutes(DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES + 1)
+ == DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES
+ )
+
+
+def test_workflow_request_rejects_bool_max_elapsed_time() -> None:
+ with pytest.warns(DeprecationWarning), pytest.raises(ValidationError):
+ WorkflowRequestBody(max_elapsed_time_minutes=True)
+
+
+def test_workflow_run_request_accepts_max_elapsed_time() -> None:
+ request = WorkflowRunRequest(workflow_id="wpid_test", max_elapsed_time_minutes=10)
+
+ assert request.max_elapsed_time_minutes == 10
+
+
+def test_workflow_run_request_rejects_invalid_max_elapsed_time() -> None:
+ with pytest.raises(ValidationError):
+ WorkflowRunRequest(
+ workflow_id="wpid_test",
+ max_elapsed_time_minutes=DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES + 1,
+ )
+
+ with pytest.raises(ValidationError):
+ WorkflowRunRequest(workflow_id="wpid_test", max_elapsed_time_minutes=True)
+
+
+def test_public_workflow_run_request_preserves_max_elapsed_time_for_legacy_runner() -> None:
+ request = WorkflowRunRequest(workflow_id="wpid_test", max_elapsed_time_minutes=10)
+
+ with pytest.warns(DeprecationWarning):
+ legacy_request = _workflow_run_request_to_legacy_request(request)
+
+ assert legacy_request.max_elapsed_time_minutes == 10
+
+
+def test_workflow_create_yaml_request_rejects_bool_max_elapsed_time() -> None:
+ with pytest.raises(ValidationError):
+ WorkflowCreateYAMLRequest(
+ title="test",
+ workflow_definition=WorkflowDefinitionYAML(parameters=[], blocks=[]),
+ max_elapsed_time_minutes=True,
+ )
+
+
+def test_workflow_domain_models_accept_db_values_above_current_elapsed_cap() -> None:
+ now = datetime.now(timezone.utc)
+ max_elapsed_time_minutes = DEFAULT_WORKFLOW_RUN_MAX_ELAPSED_TIME_MINUTES + 1
+
+ workflow = Workflow(
+ workflow_id="wf_1",
+ organization_id="org_1",
+ title="test",
+ workflow_permanent_id="wp_1",
+ version=1,
+ is_saved_task=False,
+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
+ created_at=now,
+ modified_at=now,
+ )
+ workflow_run = WorkflowRun(
+ workflow_run_id="wr_1",
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ organization_id="org_1",
+ status=WorkflowRunStatus.running,
+ max_elapsed_time_minutes=max_elapsed_time_minutes,
+ created_at=now,
+ modified_at=now,
+ )
+
+ assert workflow.max_elapsed_time_minutes == max_elapsed_time_minutes
+ assert workflow_run.max_elapsed_time_minutes == max_elapsed_time_minutes
+
+
+async def test_execute_workflow_returns_after_elapsed_timeout_without_finally(monkeypatch: pytest.MonkeyPatch) -> None:
+ started_at = datetime.now(timezone.utc) - timedelta(minutes=2)
+ created_run = _workflow_run(WorkflowRunStatus.created, started_at=started_at)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+ timed_out_run.failure_reason = "Workflow run exceeded max elapsed runtime limit of 1 minute."
+
+ workflow = SimpleNamespace(
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ title="Timeout workflow",
+ organization_id="org_1",
+ generate_script_on_terminal=False,
+ workflow_definition=SimpleNamespace(
+ parameters=[],
+ finally_block_label="cleanup",
+ blocks=[],
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org_1")
+
+ workflow_context_manager = SimpleNamespace(
+ initialize_workflow_run_context=AsyncMock(),
+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),
+ )
+ monkeypatch.setattr(service_module.app, "WORKFLOW_CONTEXT_MANAGER", workflow_context_manager)
+ monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)
+ monkeypatch.setattr(
+ service_module.workflow_script_service,
+ "get_workflow_script",
+ AsyncMock(return_value=(None, None, False)),
+ )
+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ execute_workflow_blocks = AsyncMock()
+ generate_script_if_needed = AsyncMock()
+ execute_finally_block_if_configured = AsyncMock()
+ clean_up_workflow = AsyncMock()
+
+ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=created_run))
+ monkeypatch.setattr(svc, "get_workflow_by_permanent_id", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+ monkeypatch.setattr(svc, "_execute_workflow_blocks", execute_workflow_blocks)
+ monkeypatch.setattr(svc, "generate_script_if_needed", generate_script_if_needed)
+ monkeypatch.setattr(svc, "_execute_finally_block_if_configured", execute_finally_block_if_configured)
+ monkeypatch.setattr(svc, "clean_up_workflow", clean_up_workflow)
+
+ result = await svc.execute_workflow(
+ workflow_run_id="wr_1",
+ api_key=None,
+ organization=organization,
+ )
+
+ assert result is timed_out_run
+ mark_workflow_run_as_timed_out.assert_awaited_once()
+ assert mark_workflow_run_as_timed_out.await_args is not None
+ assert (
+ mark_workflow_run_as_timed_out.await_args.kwargs["failure_reason"]
+ == "Workflow run exceeded max elapsed runtime limit of 1 minute."
+ )
+ execute_workflow_blocks.assert_not_awaited()
+ generate_script_if_needed.assert_not_awaited()
+ execute_finally_block_if_configured.assert_not_awaited()
+ clean_up_workflow.assert_awaited_once()
+
+
+async def test_execute_workflow_times_out_slow_pre_block_script_lookup(monkeypatch: pytest.MonkeyPatch) -> None:
+ started_at = datetime.now(timezone.utc)
+ created_run = _workflow_run(WorkflowRunStatus.created, started_at=started_at)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+ timed_out_run.failure_reason = "Workflow run exceeded max elapsed runtime limit of 1 minute."
+
+ workflow = SimpleNamespace(
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ title="Timeout workflow",
+ organization_id="org_1",
+ generate_script_on_terminal=False,
+ workflow_definition=SimpleNamespace(
+ parameters=[],
+ finally_block_label="cleanup",
+ blocks=[],
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org_1")
+
+ workflow_context_manager = SimpleNamespace(
+ initialize_workflow_run_context=AsyncMock(),
+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),
+ )
+
+ async def slow_script_lookup(*_args: object, **_kwargs: object) -> tuple[None, None, bool]:
+ await asyncio.sleep(0.05)
+ return None, None, False
+
+ monkeypatch.setattr(service_module.app, "WORKFLOW_CONTEXT_MANAGER", workflow_context_manager)
+ monkeypatch.setattr(
+ service_module.workflow_script_service,
+ "get_workflow_script",
+ AsyncMock(side_effect=slow_script_lookup),
+ )
+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(service_module, "_get_workflow_run_max_elapsed_timeout_seconds", lambda _workflow_run: 0.01)
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ execute_workflow_blocks = AsyncMock()
+ clean_up_workflow = AsyncMock()
+
+ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=created_run))
+ monkeypatch.setattr(svc, "get_workflow_by_permanent_id", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+ monkeypatch.setattr(svc, "_execute_workflow_blocks", execute_workflow_blocks)
+ monkeypatch.setattr(svc, "clean_up_workflow", clean_up_workflow)
+
+ result = await svc.execute_workflow(
+ workflow_run_id="wr_1",
+ api_key=None,
+ organization=organization,
+ )
+
+ assert result is timed_out_run
+ mark_workflow_run_as_timed_out.assert_awaited_once()
+ execute_workflow_blocks.assert_not_awaited()
+ clean_up_workflow.assert_awaited_once()
+
+
+async def test_execute_workflow_preserves_completed_status_after_post_run_timeout(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ started_at = datetime.now(timezone.utc)
+ created_run = _workflow_run(WorkflowRunStatus.created, started_at=started_at)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ completed_run = _workflow_run(WorkflowRunStatus.completed, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+
+ workflow = SimpleNamespace(
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ title="Timeout workflow",
+ organization_id="org_1",
+ generate_script_on_terminal=False,
+ workflow_definition=SimpleNamespace(
+ parameters=[],
+ finally_block_label="cleanup",
+ blocks=[],
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org_1")
+
+ workflow_context_manager = SimpleNamespace(
+ initialize_workflow_run_context=AsyncMock(),
+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),
+ )
+ database = SimpleNamespace(
+ workflow_runs=SimpleNamespace(
+ get_workflow_run=AsyncMock(return_value=completed_run),
+ ),
+ )
+ timeout_seconds = iter([10.0, 0.01])
+
+ monkeypatch.setattr(service_module.app, "WORKFLOW_CONTEXT_MANAGER", workflow_context_manager)
+ monkeypatch.setattr(service_module.app, "DATABASE", database)
+ monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)
+ monkeypatch.setattr(
+ service_module.workflow_script_service,
+ "get_workflow_script",
+ AsyncMock(return_value=(None, None, False)),
+ )
+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(service_module, "is_adaptive_caching", lambda _workflow, _workflow_run: False)
+ monkeypatch.setattr(
+ service_module,
+ "_get_workflow_run_max_elapsed_timeout_seconds",
+ lambda _workflow_run: next(timeout_seconds),
+ )
+
+ async def slow_finally(**_kwargs: object) -> None:
+ await asyncio.sleep(0.05)
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ execute_finally_block_if_configured = AsyncMock(side_effect=slow_finally)
+ finalize_workflow_run_status = AsyncMock(return_value=completed_run)
+ clean_up_workflow = AsyncMock()
+
+ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=created_run))
+ monkeypatch.setattr(svc, "get_workflow_by_permanent_id", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+ monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(return_value=(completed_run, set())))
+ monkeypatch.setattr(svc, "generate_script_if_needed", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "should_run_script", AsyncMock(return_value=False))
+ monkeypatch.setattr(svc, "_execute_finally_block_if_configured", execute_finally_block_if_configured)
+ monkeypatch.setattr(svc, "_finalize_workflow_run_status", finalize_workflow_run_status)
+ monkeypatch.setattr(svc, "clean_up_workflow", clean_up_workflow)
+
+ result = await svc.execute_workflow(
+ workflow_run_id="wr_1",
+ api_key=None,
+ organization=organization,
+ )
+
+ assert result is completed_run
+ execute_finally_block_if_configured.assert_awaited_once()
+ mark_workflow_run_as_timed_out.assert_not_awaited()
+ finalize_workflow_run_status.assert_awaited_once()
+ clean_up_workflow.assert_awaited_once()
+
+
+async def test_execute_workflow_preserves_timed_out_status_after_non_terminal_post_run_timeout(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ started_at = datetime.now(timezone.utc)
+ created_run = _workflow_run(WorkflowRunStatus.created, started_at=started_at)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+ timed_out_run.failure_reason = "Workflow run exceeded max elapsed runtime limit of 1 minute."
+
+ workflow = SimpleNamespace(
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ title="Timeout workflow",
+ organization_id="org_1",
+ generate_script_on_terminal=False,
+ workflow_definition=SimpleNamespace(
+ parameters=[],
+ finally_block_label="cleanup",
+ blocks=[],
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org_1")
+
+ workflow_context_manager = SimpleNamespace(
+ initialize_workflow_run_context=AsyncMock(),
+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),
+ )
+ database = SimpleNamespace(
+ workflow_runs=SimpleNamespace(
+ get_workflow_run=AsyncMock(return_value=running_run),
+ ),
+ )
+ timeout_seconds = iter([10.0, 0.01])
+
+ monkeypatch.setattr(service_module.app, "WORKFLOW_CONTEXT_MANAGER", workflow_context_manager)
+ monkeypatch.setattr(service_module.app, "DATABASE", database)
+ monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)
+ monkeypatch.setattr(
+ service_module.workflow_script_service,
+ "get_workflow_script",
+ AsyncMock(return_value=(None, None, False)),
+ )
+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(service_module, "is_adaptive_caching", lambda _workflow, _workflow_run: False)
+ monkeypatch.setattr(
+ service_module,
+ "_get_workflow_run_max_elapsed_timeout_seconds",
+ lambda _workflow_run: next(timeout_seconds),
+ )
+
+ async def slow_finally(**_kwargs: object) -> None:
+ await asyncio.sleep(0.05)
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ execute_finally_block_if_configured = AsyncMock(side_effect=slow_finally)
+ finalize_workflow_run_status = AsyncMock(return_value=timed_out_run)
+ clean_up_workflow = AsyncMock()
+
+ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=created_run))
+ monkeypatch.setattr(svc, "get_workflow_by_permanent_id", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+ monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(return_value=(running_run, set())))
+ monkeypatch.setattr(svc, "generate_script_if_needed", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "should_run_script", AsyncMock(return_value=False))
+ monkeypatch.setattr(svc, "_execute_finally_block_if_configured", execute_finally_block_if_configured)
+ monkeypatch.setattr(svc, "_finalize_workflow_run_status", finalize_workflow_run_status)
+ monkeypatch.setattr(svc, "clean_up_workflow", clean_up_workflow)
+
+ result = await svc.execute_workflow(
+ workflow_run_id="wr_1",
+ api_key=None,
+ organization=organization,
+ )
+
+ assert result is timed_out_run
+ mark_workflow_run_as_timed_out.assert_awaited_once()
+ execute_finally_block_if_configured.assert_awaited_once()
+ finalize_workflow_run_status.assert_awaited_once()
+ assert finalize_workflow_run_status.await_args is not None
+ assert finalize_workflow_run_status.await_args.kwargs["pre_finally_status"] == WorkflowRunStatus.timed_out
+ assert (
+ finalize_workflow_run_status.await_args.kwargs["pre_finally_failure_reason"]
+ == "Workflow run exceeded max elapsed runtime limit of 1 minute."
+ )
+ clean_up_workflow.assert_awaited_once()
+
+
+async def test_execute_workflow_marks_timed_out_when_post_run_budget_is_exhausted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ started_at = datetime.now(timezone.utc)
+ created_run = _workflow_run(WorkflowRunStatus.created, started_at=started_at)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+ timed_out_run.failure_reason = "Workflow run exceeded max elapsed runtime limit of 1 minute."
+
+ workflow = SimpleNamespace(
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ title="Timeout workflow",
+ organization_id="org_1",
+ generate_script_on_terminal=False,
+ workflow_definition=SimpleNamespace(
+ parameters=[],
+ finally_block_label="cleanup",
+ blocks=[],
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org_1")
+
+ workflow_context_manager = SimpleNamespace(
+ initialize_workflow_run_context=AsyncMock(),
+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),
+ )
+ database = SimpleNamespace(
+ workflow_runs=SimpleNamespace(
+ get_workflow_run=AsyncMock(return_value=running_run),
+ ),
+ )
+ timeout_seconds = iter([10.0, 0.0001])
+
+ monkeypatch.setattr(service_module.app, "WORKFLOW_CONTEXT_MANAGER", workflow_context_manager)
+ monkeypatch.setattr(service_module.app, "DATABASE", database)
+ monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)
+ monkeypatch.setattr(
+ service_module.workflow_script_service,
+ "get_workflow_script",
+ AsyncMock(return_value=(None, None, False)),
+ )
+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(service_module, "is_adaptive_caching", lambda _workflow, _workflow_run: False)
+ monkeypatch.setattr(
+ service_module,
+ "_get_workflow_run_max_elapsed_timeout_seconds",
+ lambda _workflow_run: next(timeout_seconds),
+ )
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ generate_script_if_needed = AsyncMock()
+ execute_finally_block_if_configured = AsyncMock()
+ finalize_workflow_run_status = AsyncMock(return_value=timed_out_run)
+ clean_up_workflow = AsyncMock()
+
+ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=created_run))
+ monkeypatch.setattr(svc, "get_workflow_by_permanent_id", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+ monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(return_value=(running_run, set())))
+ monkeypatch.setattr(svc, "generate_script_if_needed", generate_script_if_needed)
+ monkeypatch.setattr(svc, "should_run_script", AsyncMock(return_value=False))
+ monkeypatch.setattr(svc, "_execute_finally_block_if_configured", execute_finally_block_if_configured)
+ monkeypatch.setattr(svc, "_finalize_workflow_run_status", finalize_workflow_run_status)
+ monkeypatch.setattr(svc, "clean_up_workflow", clean_up_workflow)
+
+ result = await svc.execute_workflow(
+ workflow_run_id="wr_1",
+ api_key=None,
+ organization=organization,
+ )
+
+ assert result is timed_out_run
+ mark_workflow_run_as_timed_out.assert_awaited_once()
+ generate_script_if_needed.assert_not_awaited()
+ execute_finally_block_if_configured.assert_not_awaited()
+ finalize_workflow_run_status.assert_awaited_once()
+ assert finalize_workflow_run_status.await_args is not None
+ assert finalize_workflow_run_status.await_args.kwargs["pre_finally_status"] == WorkflowRunStatus.timed_out
+ clean_up_workflow.assert_awaited_once()
+
+
+async def test_shield_post_run_elapsed_timeout_waits_for_status_write_after_cancellation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ started_at = datetime.now(timezone.utc)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+ outer_task = asyncio.current_task()
+ assert outer_task is not None
+
+ async def refresh_workflow_run(**_kwargs: object) -> SimpleNamespace:
+ outer_task.cancel()
+ await asyncio.sleep(0)
+ return running_run
+
+ database = SimpleNamespace(
+ workflow_runs=SimpleNamespace(
+ get_workflow_run=AsyncMock(side_effect=refresh_workflow_run),
+ ),
+ )
+ monkeypatch.setattr(service_module.app, "DATABASE", database)
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+
+ result_run, pre_finally_status, pre_finally_failure_reason = await svc._shield_post_run_elapsed_timeout(
+ workflow_run_id="wr_1",
+ organization_id="org_1",
+ workflow_run=cast(WorkflowRun, running_run),
+ pre_finally_status=None,
+ pre_finally_failure_reason=None,
+ timeout_failure_reason="timed out",
+ )
+
+ assert result_run is timed_out_run
+ assert pre_finally_status == WorkflowRunStatus.timed_out
+ assert pre_finally_failure_reason == "timed out"
+ mark_workflow_run_as_timed_out.assert_awaited_once_with(
+ workflow_run_id="wr_1",
+ failure_reason="timed out",
+ )
+
+
+async def test_shield_post_run_elapsed_timeout_falls_back_when_handler_fails_after_cancellation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ started_at = datetime.now(timezone.utc)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+ outer_task = asyncio.current_task()
+ assert outer_task is not None
+
+ async def fail_after_cancellation(**_kwargs: object) -> WorkflowRun:
+ outer_task.cancel()
+ await asyncio.sleep(0)
+ raise RuntimeError("status write failed")
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ monkeypatch.setattr(svc, "_handle_post_run_elapsed_timeout", fail_after_cancellation)
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+
+ result_run, pre_finally_status, pre_finally_failure_reason = await svc._shield_post_run_elapsed_timeout(
+ workflow_run_id="wr_1",
+ organization_id="org_1",
+ workflow_run=cast(WorkflowRun, running_run),
+ pre_finally_status=None,
+ pre_finally_failure_reason=None,
+ timeout_failure_reason="timed out",
+ )
+
+ assert result_run is timed_out_run
+ assert pre_finally_status == WorkflowRunStatus.timed_out
+ assert pre_finally_failure_reason == "timed out"
+ mark_workflow_run_as_timed_out.assert_awaited_once_with(
+ workflow_run_id="wr_1",
+ failure_reason="timed out",
+ )
+
+
+async def test_execute_workflow_refreshes_terminal_status_after_immediate_post_run_timeout(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ started_at = datetime.now(timezone.utc)
+ created_run = _workflow_run(WorkflowRunStatus.created, started_at=started_at)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ completed_run = _workflow_run(WorkflowRunStatus.completed, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+
+ workflow = SimpleNamespace(
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ title="Timeout workflow",
+ organization_id="org_1",
+ generate_script_on_terminal=False,
+ workflow_definition=SimpleNamespace(
+ parameters=[],
+ finally_block_label="cleanup",
+ blocks=[],
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org_1")
+
+ workflow_context_manager = SimpleNamespace(
+ initialize_workflow_run_context=AsyncMock(),
+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),
+ )
+
+ async def refresh_workflow_run(**_kwargs: object) -> SimpleNamespace:
+ await asyncio.sleep(0.01)
+ return completed_run
+
+ database = SimpleNamespace(
+ workflow_runs=SimpleNamespace(
+ get_workflow_run=AsyncMock(side_effect=refresh_workflow_run),
+ ),
+ )
+ timeout_seconds = iter([10.0, 0.0])
+
+ monkeypatch.setattr(service_module.app, "WORKFLOW_CONTEXT_MANAGER", workflow_context_manager)
+ monkeypatch.setattr(service_module.app, "DATABASE", database)
+ monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)
+ monkeypatch.setattr(
+ service_module.workflow_script_service,
+ "get_workflow_script",
+ AsyncMock(return_value=(None, None, False)),
+ )
+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(service_module, "is_adaptive_caching", lambda _workflow, _workflow_run: False)
+ monkeypatch.setattr(
+ service_module,
+ "_get_workflow_run_max_elapsed_timeout_seconds",
+ lambda _workflow_run: next(timeout_seconds),
+ )
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ finalize_workflow_run_status = AsyncMock(return_value=completed_run)
+ clean_up_workflow = AsyncMock()
+
+ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=created_run))
+ monkeypatch.setattr(svc, "get_workflow_by_permanent_id", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+ monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(return_value=(completed_run, set())))
+ monkeypatch.setattr(svc, "generate_script_if_needed", AsyncMock())
+ monkeypatch.setattr(svc, "should_run_script", AsyncMock(return_value=False))
+ monkeypatch.setattr(svc, "_execute_finally_block_if_configured", AsyncMock())
+ monkeypatch.setattr(svc, "_finalize_workflow_run_status", finalize_workflow_run_status)
+ monkeypatch.setattr(svc, "clean_up_workflow", clean_up_workflow)
+
+ result = await svc.execute_workflow(
+ workflow_run_id="wr_1",
+ api_key=None,
+ organization=organization,
+ )
+
+ assert result is completed_run
+ mark_workflow_run_as_timed_out.assert_not_awaited()
+ finalize_workflow_run_status.assert_awaited_once()
+ clean_up_workflow.assert_awaited_once()
+
+
+async def test_execute_workflow_returns_finalized_status_after_post_run_timeout(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ started_at = datetime.now(timezone.utc)
+ created_run = _workflow_run(WorkflowRunStatus.created, started_at=started_at)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ failed_run = _workflow_run(WorkflowRunStatus.failed, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+
+ workflow = SimpleNamespace(
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ title="Timeout workflow",
+ organization_id="org_1",
+ generate_script_on_terminal=False,
+ workflow_definition=SimpleNamespace(
+ parameters=[],
+ finally_block_label="cleanup",
+ blocks=[],
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org_1")
+
+ workflow_context_manager = SimpleNamespace(
+ initialize_workflow_run_context=AsyncMock(),
+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),
+ )
+ database = SimpleNamespace(
+ workflow_runs=SimpleNamespace(
+ get_workflow_run=AsyncMock(return_value=failed_run),
+ ),
+ )
+ timeout_seconds = iter([10.0, 0.01])
+
+ monkeypatch.setattr(service_module.app, "WORKFLOW_CONTEXT_MANAGER", workflow_context_manager)
+ monkeypatch.setattr(service_module.app, "DATABASE", database)
+ monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)
+ monkeypatch.setattr(
+ service_module.workflow_script_service,
+ "get_workflow_script",
+ AsyncMock(return_value=(None, None, False)),
+ )
+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(service_module, "is_adaptive_caching", lambda _workflow, _workflow_run: False)
+ monkeypatch.setattr(
+ service_module,
+ "_get_workflow_run_max_elapsed_timeout_seconds",
+ lambda _workflow_run: next(timeout_seconds),
+ )
+
+ async def slow_finally(**_kwargs: object) -> None:
+ await asyncio.sleep(0.05)
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ update_workflow_run_status = AsyncMock(return_value=running_run)
+ execute_finally_block_if_configured = AsyncMock(side_effect=slow_finally)
+ finalize_workflow_run_status = AsyncMock(return_value=failed_run)
+ clean_up_workflow = AsyncMock()
+
+ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=created_run))
+ monkeypatch.setattr(svc, "get_workflow_by_permanent_id", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+ monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(return_value=(failed_run, set())))
+ monkeypatch.setattr(svc, "generate_script_if_needed", AsyncMock())
+ monkeypatch.setattr(svc, "should_run_script", AsyncMock(return_value=False))
+ monkeypatch.setattr(svc, "_update_workflow_run_status", update_workflow_run_status)
+ monkeypatch.setattr(svc, "_execute_finally_block_if_configured", execute_finally_block_if_configured)
+ monkeypatch.setattr(svc, "_finalize_workflow_run_status", finalize_workflow_run_status)
+ monkeypatch.setattr(svc, "clean_up_workflow", clean_up_workflow)
+
+ result = await svc.execute_workflow(
+ workflow_run_id="wr_1",
+ api_key=None,
+ organization=organization,
+ )
+
+ assert result is failed_run
+ mark_workflow_run_as_timed_out.assert_not_awaited()
+ update_workflow_run_status.assert_awaited_once_with(
+ workflow_run_id="wr_1",
+ status=WorkflowRunStatus.running,
+ failure_reason=None,
+ )
+ execute_finally_block_if_configured.assert_awaited_once()
+ finalize_workflow_run_status.assert_awaited_once()
+ clean_up_workflow.assert_awaited_once()
+
+
+async def test_execute_workflow_runs_finally_for_existing_timed_out_status(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ started_at = datetime.now(timezone.utc)
+ created_run = _workflow_run(WorkflowRunStatus.created, started_at=started_at)
+ running_run = _workflow_run(WorkflowRunStatus.running, started_at=started_at)
+ timed_out_run = _workflow_run(WorkflowRunStatus.timed_out, started_at=started_at)
+
+ workflow = SimpleNamespace(
+ workflow_id="wf_1",
+ workflow_permanent_id="wp_1",
+ title="Timeout workflow",
+ organization_id="org_1",
+ generate_script_on_terminal=False,
+ workflow_definition=SimpleNamespace(
+ parameters=[],
+ finally_block_label="cleanup",
+ blocks=[],
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org_1")
+
+ workflow_context_manager = SimpleNamespace(
+ initialize_workflow_run_context=AsyncMock(),
+ get_workflow_run_context=lambda _workflow_run_id: SimpleNamespace(browser_session_id=None),
+ )
+ database = SimpleNamespace(
+ workflow_runs=SimpleNamespace(
+ get_workflow_run=AsyncMock(return_value=timed_out_run),
+ ),
+ )
+
+ monkeypatch.setattr(service_module.app, "WORKFLOW_CONTEXT_MANAGER", workflow_context_manager)
+ monkeypatch.setattr(service_module.app, "DATABASE", database)
+ monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)
+ monkeypatch.setattr(
+ service_module.workflow_script_service,
+ "get_workflow_script",
+ AsyncMock(return_value=(None, None, False)),
+ )
+ monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(service_module, "is_adaptive_caching", lambda _workflow, _workflow_run: False)
+ monkeypatch.setattr(service_module, "_get_workflow_run_max_elapsed_timeout_seconds", lambda _workflow_run: 10.0)
+
+ svc = WorkflowService()
+ mark_workflow_run_as_timed_out = AsyncMock(return_value=timed_out_run)
+ update_workflow_run_status = AsyncMock(return_value=running_run)
+ execute_finally_block_if_configured = AsyncMock()
+ finalize_workflow_run_status = AsyncMock(return_value=timed_out_run)
+ clean_up_workflow = AsyncMock()
+
+ monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=created_run))
+ monkeypatch.setattr(svc, "get_workflow_by_permanent_id", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(return_value=running_run))
+ monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
+ monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
+ monkeypatch.setattr(svc, "mark_workflow_run_as_timed_out", mark_workflow_run_as_timed_out)
+ monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(return_value=(timed_out_run, set())))
+ monkeypatch.setattr(svc, "generate_script_if_needed", AsyncMock())
+ monkeypatch.setattr(svc, "should_run_script", AsyncMock(return_value=False))
+ monkeypatch.setattr(svc, "_update_workflow_run_status", update_workflow_run_status)
+ monkeypatch.setattr(svc, "_execute_finally_block_if_configured", execute_finally_block_if_configured)
+ monkeypatch.setattr(svc, "_finalize_workflow_run_status", finalize_workflow_run_status)
+ monkeypatch.setattr(svc, "clean_up_workflow", clean_up_workflow)
+
+ result = await svc.execute_workflow(
+ workflow_run_id="wr_1",
+ api_key=None,
+ organization=organization,
+ )
+
+ assert result is timed_out_run
+ mark_workflow_run_as_timed_out.assert_not_awaited()
+ update_workflow_run_status.assert_awaited_once_with(
+ workflow_run_id="wr_1",
+ status=WorkflowRunStatus.running,
+ failure_reason=None,
+ )
+ execute_finally_block_if_configured.assert_awaited_once()
+ finalize_workflow_run_status.assert_awaited_once()
+ clean_up_workflow.assert_awaited_once()
diff --git a/tests/unit/test_workflow_run_parameter_batching.py b/tests/unit/test_workflow_run_parameter_batching.py
index 5001c2a6a..b2ca3ca35 100644
--- a/tests/unit/test_workflow_run_parameter_batching.py
+++ b/tests/unit/test_workflow_run_parameter_batching.py
@@ -7,6 +7,7 @@ persists them in a single batch insert, and that validation failures
from __future__ import annotations
+from collections.abc import Generator
from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
@@ -58,6 +59,7 @@ def _make_service_with_mocks(
extra_http_headers=None,
cdp_connect_headers=None,
browser_profile_id=None,
+ max_elapsed_time_minutes=None,
run_with="agent",
code_version=None,
adaptive_caching=False,
@@ -83,7 +85,7 @@ def _make_service_with_mocks(
@pytest.fixture(autouse=True)
-def reset_context() -> None:
+def reset_context() -> Generator[None]:
skyvern_context.reset()
yield
skyvern_context.reset()