Skip to content

Automate browser based workflows with AI

SKY-18797: Retry event-trigger submissions that never reached Batch (#8999)

Aaron Perez18b4a4db238f
diff --git a/skyvern-frontend/src/util/failureCategoryDisplay.ts b/skyvern-frontend/src/util/failureCategoryDisplay.ts
index 199518036..a4f99f3a3 100644
--- a/skyvern-frontend/src/util/failureCategoryDisplay.ts
+++ b/skyvern-frontend/src/util/failureCategoryDisplay.ts
@@ -22,7 +22,7 @@ const FAILURE_CATEGORY_DISPLAY = {
BROWSER_ERROR: {
label: "Browser error",
description:
- "The browser crashed or the page closed unexpectedly. Retrying usually helps.",
+ "The browser crashed, stopped responding, or closed the page unexpectedly. Retrying usually helps, because a new run gets a fresh browser. If the run uses a browser session or connects to your own browser, start a new session or restart that browser first.",
},
BROWSER_SESSION_EXPIRED: {
label: "Browser session expired",
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 409e9abc9..171107faa 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -233,7 +233,7 @@ from skyvern.forge.validation_evidence_router import (
ValidationRouterResult,
route_validation_evidence,
)
-from skyvern.schemas.runs import CUA_ENGINES, RunEngine
+from skyvern.schemas.runs import CUA_ENGINES, ProxyLocation, RunEngine
from skyvern.schemas.steps import AgentStepOutput
from skyvern.services import run_service, service_utils, uploaded_file_service
from skyvern.services.action_service import get_action_history
@@ -2732,6 +2732,8 @@ class ForgeAgent:
)
if not new_files:
return None
+ if context is not None:
+ context.download_completed_task_ids.add(task.task_id)
download_finalized = True
download_completion_reason = f"download completed ({len(new_files)} new file(s))"
LOG.info(
@@ -4825,24 +4827,37 @@ class ForgeAgent:
return None
async def _enrich_failure_reason_with_download_status(self, task: Task, reason: str) -> str:
- """Append one bounded sentence to a terminal failure reason when the run's latest download-intent
- action received no file and passively observed a server 5xx status.
-
- Folds over every persisted step's output in execution order; only the latest download-intent
- action's own evidence counts. The original reason is preserved verbatim and the sentence is
- appended once (deduplicated). Returns the (possibly unchanged) reason to store; never raises.
- """
+ """Append terminal download evidence without changing the original reason or its classification."""
try:
steps = await app.DATABASE.tasks.get_task_steps(task_id=task.task_id, organization_id=task.organization_id)
status = self._latest_download_failure_status(steps)
- if status is None:
- return reason
- sentence = (
- f"An HTTP request made during the download action returned HTTP {status}, and no file was received."
- )
- if sentence in reason:
+ if status is not None:
+ sentence = (
+ f"An HTTP request made during the download action returned HTTP {status}, and no file was received."
+ )
+ if sentence not in reason:
+ reason = f"{reason} {sentence}"
+ context = skyvern_context.current()
+ if (
+ context is None
+ or context.task_id != task.task_id
+ or context.workflow_run_id != task.workflow_run_id
+ or context.organization_id != task.organization_id
+ or task.task_id not in context.residential_download_interrupted_task_ids
+ or task.task_id in context.download_completed_task_ids
+ or task.proxy_location == ProxyLocation.RESIDENTIAL_ISP
+ or task.status in (TaskStatus.completed, TaskStatus.canceled, TaskStatus.timed_out)
+ or any(
+ result.downloaded_files
+ for step in steps
+ if step.output and step.output.actions_and_results
+ for _action, results in step.output.actions_and_results
+ for result in results
+ )
+ ):
return reason
- return f"{reason} {sentence}"
+ advice = "Try using an ISP proxy."
+ return reason if advice in reason else f"{reason} {advice}"
except Exception:
LOG.warning(
"Failed to enrich failure reason with download status",
@@ -9880,9 +9895,7 @@ class ForgeAgent:
if task_from_db:
task = task_from_db
- # Enrich only the stored reason with a server 5xx observed on an admitted xhr/fetch request during
- # the run's latest download action that received no file. Runs at the shared terminal seam so every
- # failed/terminated path inherits it; the caller's classification of the original reason is untouched.
+ # Enrich only the stored terminal reason; callers classify the original before this seam.
if status in (TaskStatus.failed, TaskStatus.terminated) and failure_reason is not None:
failure_reason = await self._enrich_failure_reason_with_download_status(task, failure_reason)
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 527a1def2..98f30f0c6 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -1864,6 +1864,9 @@ class AgentFunction:
async def setup_browser_context_extensions(self, browser_context: Any, **kwargs: Any) -> None:
"""Attach cloud-only listeners/route handlers to a fresh BrowserContext. OSS no-op."""
+ def is_residential_browser_egress(self, browser_context: BrowserContext) -> bool:
+ return False
+
async def on_browser_context_acquired(
self,
browser_context: Any,
diff --git a/skyvern/forge/sdk/api/aws.py b/skyvern/forge/sdk/api/aws.py
index 0e4367d11..8952f335b 100644
--- a/skyvern/forge/sdk/api/aws.py
+++ b/skyvern/forge/sdk/api/aws.py
@@ -1648,28 +1648,51 @@ class AsyncAWSClient:
container_overrides = container_overrides or {}
depends_on = [{"jobId": job_id} for job_id in depends_on_ids or []]
async with self._batch_client() as client:
- if job_priority is None or share_identifier is None:
- response = await client.submit_job(
- jobName=job_name,
- jobQueue=job_queue,
- jobDefinition=job_definition,
- parameters=params,
- containerOverrides=container_overrides,
- dependsOn=depends_on,
- )
- return response.get("jobId")
- else:
- response = await client.submit_job(
- jobName=job_name,
- jobQueue=job_queue,
- jobDefinition=job_definition,
- parameters=params,
- schedulingPriorityOverride=job_priority,
- shareIdentifier=share_identifier,
- containerOverrides=container_overrides,
- dependsOn=depends_on,
+ attempts: list[tuple[type[BaseException] | None, type[BaseException] | None, int | None]] = []
+
+ def record_attempt(**kwargs: Any) -> None:
+ error = kwargs.get("caught_exception")
+ cause = getattr(error, "kwargs", {}).get("error")
+ response = kwargs.get("response")
+ status_code = getattr(response[0], "status_code", None) if response else None
+ attempts.append(
+ (
+ type(error) if error is not None else None,
+ type(cause) if cause is not None else None,
+ status_code,
+ )
)
+
+ # Keep lightweight per-attempt outcomes; the final exception can hide an earlier ambiguity.
+ event_name = "needs-retry.batch.SubmitJob"
+ client.meta.events.register(event_name, record_attempt)
+ try:
+ if job_priority is None or share_identifier is None:
+ response = await client.submit_job(
+ jobName=job_name,
+ jobQueue=job_queue,
+ jobDefinition=job_definition,
+ parameters=params,
+ containerOverrides=container_overrides,
+ dependsOn=depends_on,
+ )
+ else:
+ response = await client.submit_job(
+ jobName=job_name,
+ jobQueue=job_queue,
+ jobDefinition=job_definition,
+ parameters=params,
+ schedulingPriorityOverride=job_priority,
+ shareIdentifier=share_identifier,
+ containerOverrides=container_overrides,
+ dependsOn=depends_on,
+ )
return response.get("jobId")
+ except Exception as error:
+ vars(error)["_skyvern_submit_job_attempts"] = attempts
+ raise
+ finally:
+ client.meta.events.unregister(event_name, record_attempt)
class S3Uri:
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index 7db24d94f..db1eda026 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -714,6 +714,8 @@ class SkyvernContext:
# Explicit download_timeout of the Task V3 block or task now running; the browser download listener
# reads it when a download starts, so it also covers the initial navigation and a relaunched browser.
download_timeout: float | None = None
+ residential_download_interrupted_task_ids: set[str] = field(default_factory=builtins.set)
+ download_completed_task_ids: set[str] = field(default_factory=builtins.set)
# Keep source names with exact CDP targets; finalization must not infer provenance from affixes.
download_suffix_applied_files: dict[str, tuple[str, str]] = field(default_factory=dict, repr=False)
totp_codes: dict[str, str | None] = field(default_factory=dict)
diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py
index f2810bc9b..e13bc4e1a 100644
--- a/skyvern/webeye/browser_factory.py
+++ b/skyvern/webeye/browser_factory.py
@@ -17,7 +17,7 @@ from contextlib import suppress
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
-from typing import Any, Awaitable, Callable, Literal, Protocol, cast
+from typing import Any, Awaitable, Callable, Coroutine, Literal, Protocol, cast
from urllib.parse import parse_qsl, urlparse
import psutil
@@ -64,7 +64,7 @@ from skyvern.forge.sdk.core.http_request_authorization import (
RunScopedRedirectHopAuthorizer,
deny_unenrolled_redirect_hop,
)
-from skyvern.forge.sdk.core.skyvern_context import current, ensure_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext, current, ensure_context
from skyvern.schemas.runs import ProxyLocation, ProxyLocationInput, get_tzinfo_from_proxy
from skyvern.utils.url_validators import redacted_url_origin, signed_url_ttl_remaining_seconds
from skyvern.webeye.attach_only import forbid
@@ -439,11 +439,12 @@ def _redact_url_query(url: str) -> str:
def set_download_file_listener(
browser_context: BrowserContext, download_timeout: float | None = None, **kwargs: Any
) -> None:
- async def listen_to_download(download: Download) -> None:
+ async def save_download(
+ download: Download, context: SkyvernContext | None, owner_task_id: str | None, residential_egress: bool
+ ) -> None:
download_started_at = time.monotonic()
- context = current()
workflow_run_id = (context.workflow_run_id if context else None) or kwargs.get("workflow_run_id")
- task_id = (context.task_id if context else None) or kwargs.get("task_id")
+ task_id = owner_task_id or (context.task_id if context else None) or kwargs.get("task_id")
# Read when the download starts, not when the context is built: one browser serves many tasks.
timeout_seconds = (
download_timeout or (context.download_timeout if context else None) or BROWSER_DOWNLOAD_TIMEOUT
@@ -463,6 +464,8 @@ def set_download_file_listener(
mark_download_cancelled_by_skyvern(download)
await download.cancel()
return
+ if context and owner_task_id:
+ context.download_completed_task_ids.add(owner_task_id)
file_path = Path(resolved_path)
if not file_path.exists():
# On an adopted persistent session the bytes live on the run connection, not
@@ -526,6 +529,15 @@ def set_download_file_listener(
# An aborted transfer surfaces here as a path() error; reporting it as a rename failure
# sends triage to the wrong subsystem and hides that no file was ever saved.
failure = await read_download_failure(download)
+ if (
+ failure == "canceled"
+ and not was_download_cancelled_by_skyvern(download)
+ and not download.page.is_closed()
+ and context
+ and owner_task_id
+ and residential_egress
+ ):
+ context.residential_download_interrupted_task_ids.add(owner_task_id)
if failure is not None:
LOG.warning(
"Browser aborted the download before any file was saved",
@@ -544,6 +556,25 @@ def set_download_file_listener(
task_id=task_id,
)
+ def listen_to_download(download: Download) -> Coroutine[Any, Any, None]:
+ # Snapshot the task synchronously at event delivery; the context can change before the coroutine runs.
+ # Callback kwargs can belong to an older task or run and cannot establish ownership.
+ context = current()
+ owner_task_id = context.task_id if context else None
+ residential_egress = False
+ try:
+ state = app.BROWSER_MANAGER.get_for_task(owner_task_id) if owner_task_id else None
+ # Workflow blocks register their browser under the run id only. A plain lookup avoids aliasing the task.
+ if state is None and owner_task_id and context and context.workflow_run_id:
+ state = app.BROWSER_MANAGER.get_for_workflow_run(context.workflow_run_id)
+ if state is None or state.browser_context is not browser_context:
+ owner_task_id = None
+ elif owner_task_id:
+ residential_egress = app.AGENT_FUNCTION.is_residential_browser_egress(browser_context)
+ except Exception:
+ owner_task_id = None
+ return save_download(download, context, owner_task_id, residential_egress)
+
def listen_to_new_page(page: Page) -> None:
page.on("download", listen_to_download)