Skip to content

Automate browser based workflows with AI

Retry a CodeBlock browser session that times out starting instead of failing the run (#8034)

Aaron Perezd961500f2b7c
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 37dea619c..8d3eee9dc 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -44,6 +44,7 @@ from skyvern.exceptions import (
BrowserProfileNotFound,
BrowserSessionNotFound,
BrowserSessionNotRenewable,
+ BrowserSessionStartupTimeout,
DisabledBlockExecutionError,
InProcessScriptExecutionDenied,
InvalidCredentialId,
@@ -264,6 +265,10 @@ WORKFLOW_RUN_INTERRUPTED_FAILURE_REASON = (
# Short lifespan for auto-provisioned CodeBlock sessions; the renewal loop extends it while the
# run is active, so this only bounds how long a leaked session lingers if cleanup never runs.
CODE_BLOCK_SESSION_TIMEOUT_MINUTES = 20
+# Wall clock, not an attempt count: a waiting run holds a whole worker pod
+# (max_concurrent_activities=1), so what has to stay bounded is the time pinned, whatever the
+# per-attempt startup deadline happens to be.
+CODE_BLOCK_SESSION_STARTUP_BUDGET_SECONDS = 90.0
# Structured warning emitted when a debug-session run's visible PBS profile is
# incompatible with the LoginBlock credential's saved profile. Observability
@@ -3055,12 +3060,11 @@ class WorkflowService:
return None
try:
- browser_session = await app.PERSISTENT_SESSIONS_MANAGER.create_session(
+ browser_session = await self._create_code_block_browser_session(
organization_id=organization_id,
- timeout_minutes=CODE_BLOCK_SESSION_TIMEOUT_MINUTES,
browser_profile_id=browser_profile_id,
proxy_location=proxy_location,
- inherit_profile_proxy=True,
+ workflow_run_id=workflow_run_id,
)
except Exception:
LOG.error(
@@ -3081,6 +3085,59 @@ class WorkflowService:
)
return browser_session
+ async def _create_code_block_browser_session(
+ self,
+ *,
+ organization_id: str,
+ browser_profile_id: str | None,
+ proxy_location: ProxyLocationInput,
+ workflow_run_id: str,
+ ) -> PersistentBrowserSession:
+ """Create the CodeBlock session, retrying only a startup timeout.
+
+ A startup timeout means the session fleet had no free capacity inside the deadline,
+ not that the request was invalid — the fleet scales on demand, so a fresh attempt
+ often lands on pods that came up just after the previous deadline expired. Every
+ other failure is deterministic, so retrying it only delays the run.
+
+ Retrying is bounded by wall clock rather than by attempts because the waiting run
+ occupies a whole worker pod, and it does so during exactly the capacity crunch that
+ causes the timeout — so the cost to bound is time held, not requests made.
+ """
+ deadline = time.monotonic() + CODE_BLOCK_SESSION_STARTUP_BUDGET_SECONDS
+ attempt = 0
+ while True:
+ attempt += 1
+ try:
+ return await app.PERSISTENT_SESSIONS_MANAGER.create_session(
+ organization_id=organization_id,
+ timeout_minutes=CODE_BLOCK_SESSION_TIMEOUT_MINUTES,
+ browser_profile_id=browser_profile_id,
+ proxy_location=proxy_location,
+ inherit_profile_proxy=True,
+ )
+ except BrowserSessionStartupTimeout:
+ if time.monotonic() >= deadline:
+ raise
+ # A cancel can land while we wait. Provisioning another billed session for a
+ # run that already finished helps nobody and deepens the shortage.
+ workflow_run = await self.get_workflow_run(workflow_run_id=workflow_run_id)
+ if workflow_run.status.is_final():
+ LOG.info(
+ "Workflow run reached a final state while its CodeBlock session was starting; not retrying",
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ workflow_status=workflow_run.status,
+ )
+ raise
+ LOG.warning(
+ "Browser session for CodeBlock run timed out starting; retrying",
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ attempt=attempt,
+ budget_seconds=CODE_BLOCK_SESSION_STARTUP_BUDGET_SECONDS,
+ )
+
async def _collect_inherited_workflow_system_prompt(
self,
parent_workflow_run_id: str | None,
@@ -3469,8 +3526,13 @@ class WorkflowService:
"Failed to create the browser session required for secure code execution: "
f"{get_user_facing_exception_message(e)}"
)
- workflow_run = await self.mark_workflow_run_as_failed(
- workflow_run_id=workflow_run_id, failure_reason=failure_reason
+ # Conditional: a cancel can land while the session is still starting, and an
+ # unconditional write would overwrite that terminal status with ``failed``.
+ workflow_run = (
+ await self.mark_workflow_run_as_failed_if_not_final(
+ workflow_run_id=workflow_run_id, failure_reason=failure_reason
+ )
+ or workflow_run
)
await self.clean_up_workflow(
workflow=workflow,
diff --git a/tests/unit/test_workflow_service_managed_browser_profile.py b/tests/unit/test_workflow_service_managed_browser_profile.py
index d3892cfcd..851e97aa7 100644
--- a/tests/unit/test_workflow_service_managed_browser_profile.py
+++ b/tests/unit/test_workflow_service_managed_browser_profile.py
@@ -108,6 +108,9 @@ def _patch_execute_workflow_deps(
workflow_runs=SimpleNamespace(
get_workflow_run=AsyncMock(return_value=refreshed_run),
update_workflow_run=AsyncMock(),
+ # Conditional finalize used by mark_workflow_run_as_failed_if_not_final: returns the
+ # run when this caller won the terminal transition, None when it was already final.
+ update_workflow_run_if_not_final=AsyncMock(return_value=refreshed_run),
),
artifacts=SimpleNamespace(claim_session_download_artifacts_for_run=AsyncMock(return_value=0)),
)