Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-18797: Retry event-trigger submissions that never reached Batch (#8999)
Aaron Perez18b4a4db238f
diff --git a/skyvern-frontend/src/util/failureCategoryDisplay.ts b/skyvern-frontend/src/util/failureCategoryDisplay.tsindex 199518036..a4f99f3a3 100644--- a/skyvern-frontend/src/util/failureCategoryDisplay.ts+++ b/skyvern-frontend/src/util/failureCategoryDisplay.ts@@ -22,7 +22,7 @@ const FAILURE_CATEGORY_DISPLAY = {BROWSER_ERROR: {label: "Browser error",description:- "The browser crashed or the page closed unexpectedly. Retrying usually helps.",+ "The browser crashed, stopped responding, or closed the page unexpectedly. Retrying usually helps, because a new run gets a fresh browser. If the run uses a browser session or connects to your own browser, start a new session or restart that browser first.",},BROWSER_SESSION_EXPIRED: {label: "Browser session expired",diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 409e9abc9..171107faa 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -233,7 +233,7 @@ from skyvern.forge.validation_evidence_router import (ValidationRouterResult,route_validation_evidence,)-from skyvern.schemas.runs import CUA_ENGINES, RunEngine+from skyvern.schemas.runs import CUA_ENGINES, ProxyLocation, RunEnginefrom skyvern.schemas.steps import AgentStepOutputfrom skyvern.services import run_service, service_utils, uploaded_file_servicefrom skyvern.services.action_service import get_action_history@@ -2732,6 +2732,8 @@ class ForgeAgent:)if not new_files:return None+ if context is not None:+ context.download_completed_task_ids.add(task.task_id)download_finalized = Truedownload_completion_reason = f"download completed ({len(new_files)} new file(s))"LOG.info(@@ -4825,24 +4827,37 @@ class ForgeAgent:return Noneasync def _enrich_failure_reason_with_download_status(self, task: Task, reason: str) -> str:- """Append one bounded sentence to a terminal failure reason when the run's latest download-intent- action received no file and passively observed a server 5xx status.-- Folds over every persisted step's output in execution order; only the latest download-intent- action's own evidence counts. The original reason is preserved verbatim and the sentence is- appended once (deduplicated). Returns the (possibly unchanged) reason to store; never raises.- """+ """Append terminal download evidence without changing the original reason or its classification."""try:steps = await app.DATABASE.tasks.get_task_steps(task_id=task.task_id, organization_id=task.organization_id)status = self._latest_download_failure_status(steps)- if status is None:- return reason- sentence = (- f"An HTTP request made during the download action returned HTTP {status}, and no file was received."- )- if sentence in reason:+ if status is not None:+ sentence = (+ f"An HTTP request made during the download action returned HTTP {status}, and no file was received."+ )+ if sentence not in reason:+ reason = f"{reason} {sentence}"+ context = skyvern_context.current()+ if (+ context is None+ or context.task_id != task.task_id+ or context.workflow_run_id != task.workflow_run_id+ or context.organization_id != task.organization_id+ or task.task_id not in context.residential_download_interrupted_task_ids+ or task.task_id in context.download_completed_task_ids+ or task.proxy_location == ProxyLocation.RESIDENTIAL_ISP+ or task.status in (TaskStatus.completed, TaskStatus.canceled, TaskStatus.timed_out)+ or any(+ result.downloaded_files+ for step in steps+ if step.output and step.output.actions_and_results+ for _action, results in step.output.actions_and_results+ for result in results+ )+ ):return reason- return f"{reason} {sentence}"+ advice = "Try using an ISP proxy."+ return reason if advice in reason else f"{reason} {advice}"except Exception:LOG.warning("Failed to enrich failure reason with download status",@@ -9880,9 +9895,7 @@ class ForgeAgent:if task_from_db:task = task_from_db- # Enrich only the stored reason with a server 5xx observed on an admitted xhr/fetch request during- # the run's latest download action that received no file. Runs at the shared terminal seam so every- # failed/terminated path inherits it; the caller's classification of the original reason is untouched.+ # Enrich only the stored terminal reason; callers classify the original before this seam.if status in (TaskStatus.failed, TaskStatus.terminated) and failure_reason is not None:failure_reason = await self._enrich_failure_reason_with_download_status(task, failure_reason)diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 527a1def2..98f30f0c6 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -1864,6 +1864,9 @@ class AgentFunction:async def setup_browser_context_extensions(self, browser_context: Any, **kwargs: Any) -> None:"""Attach cloud-only listeners/route handlers to a fresh BrowserContext. OSS no-op."""+ def is_residential_browser_egress(self, browser_context: BrowserContext) -> bool:+ return False+async def on_browser_context_acquired(self,browser_context: Any,diff --git a/skyvern/forge/sdk/api/aws.py b/skyvern/forge/sdk/api/aws.pyindex 0e4367d11..8952f335b 100644--- a/skyvern/forge/sdk/api/aws.py+++ b/skyvern/forge/sdk/api/aws.py@@ -1648,28 +1648,51 @@ class AsyncAWSClient:container_overrides = container_overrides or {}depends_on = [{"jobId": job_id} for job_id in depends_on_ids or []]async with self._batch_client() as client:- if job_priority is None or share_identifier is None:- response = await client.submit_job(- jobName=job_name,- jobQueue=job_queue,- jobDefinition=job_definition,- parameters=params,- containerOverrides=container_overrides,- dependsOn=depends_on,- )- return response.get("jobId")- else:- response = await client.submit_job(- jobName=job_name,- jobQueue=job_queue,- jobDefinition=job_definition,- parameters=params,- schedulingPriorityOverride=job_priority,- shareIdentifier=share_identifier,- containerOverrides=container_overrides,- dependsOn=depends_on,+ attempts: list[tuple[type[BaseException] | None, type[BaseException] | None, int | None]] = []++ def record_attempt(**kwargs: Any) -> None:+ error = kwargs.get("caught_exception")+ cause = getattr(error, "kwargs", {}).get("error")+ response = kwargs.get("response")+ status_code = getattr(response[0], "status_code", None) if response else None+ attempts.append(+ (+ type(error) if error is not None else None,+ type(cause) if cause is not None else None,+ status_code,+ ))++ # Keep lightweight per-attempt outcomes; the final exception can hide an earlier ambiguity.+ event_name = "needs-retry.batch.SubmitJob"+ client.meta.events.register(event_name, record_attempt)+ try:+ if job_priority is None or share_identifier is None:+ response = await client.submit_job(+ jobName=job_name,+ jobQueue=job_queue,+ jobDefinition=job_definition,+ parameters=params,+ containerOverrides=container_overrides,+ dependsOn=depends_on,+ )+ else:+ response = await client.submit_job(+ jobName=job_name,+ jobQueue=job_queue,+ jobDefinition=job_definition,+ parameters=params,+ schedulingPriorityOverride=job_priority,+ shareIdentifier=share_identifier,+ containerOverrides=container_overrides,+ dependsOn=depends_on,+ )return response.get("jobId")+ except Exception as error:+ vars(error)["_skyvern_submit_job_attempts"] = attempts+ raise+ finally:+ client.meta.events.unregister(event_name, record_attempt)class S3Uri:diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 7db24d94f..db1eda026 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -714,6 +714,8 @@ class SkyvernContext:# Explicit download_timeout of the Task V3 block or task now running; the browser download listener# reads it when a download starts, so it also covers the initial navigation and a relaunched browser.download_timeout: float | None = None+ residential_download_interrupted_task_ids: set[str] = field(default_factory=builtins.set)+ download_completed_task_ids: set[str] = field(default_factory=builtins.set)# Keep source names with exact CDP targets; finalization must not infer provenance from affixes.download_suffix_applied_files: dict[str, tuple[str, str]] = field(default_factory=dict, repr=False)totp_codes: dict[str, str | None] = field(default_factory=dict)diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.pyindex f2810bc9b..e13bc4e1a 100644--- a/skyvern/webeye/browser_factory.py+++ b/skyvern/webeye/browser_factory.py@@ -17,7 +17,7 @@ from contextlib import suppressfrom dataclasses import dataclassfrom datetime import UTC, datetimefrom pathlib import Path-from typing import Any, Awaitable, Callable, Literal, Protocol, cast+from typing import Any, Awaitable, Callable, Coroutine, Literal, Protocol, castfrom urllib.parse import parse_qsl, urlparseimport psutil@@ -64,7 +64,7 @@ from skyvern.forge.sdk.core.http_request_authorization import (RunScopedRedirectHopAuthorizer,deny_unenrolled_redirect_hop,)-from skyvern.forge.sdk.core.skyvern_context import current, ensure_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext, current, ensure_contextfrom skyvern.schemas.runs import ProxyLocation, ProxyLocationInput, get_tzinfo_from_proxyfrom skyvern.utils.url_validators import redacted_url_origin, signed_url_ttl_remaining_secondsfrom skyvern.webeye.attach_only import forbid@@ -439,11 +439,12 @@ def _redact_url_query(url: str) -> str:def set_download_file_listener(browser_context: BrowserContext, download_timeout: float | None = None, **kwargs: Any) -> None:- async def listen_to_download(download: Download) -> None:+ async def save_download(+ download: Download, context: SkyvernContext | None, owner_task_id: str | None, residential_egress: bool+ ) -> None:download_started_at = time.monotonic()- context = current()workflow_run_id = (context.workflow_run_id if context else None) or kwargs.get("workflow_run_id")- task_id = (context.task_id if context else None) or kwargs.get("task_id")+ task_id = owner_task_id or (context.task_id if context else None) or kwargs.get("task_id")# Read when the download starts, not when the context is built: one browser serves many tasks.timeout_seconds = (download_timeout or (context.download_timeout if context else None) or BROWSER_DOWNLOAD_TIMEOUT@@ -463,6 +464,8 @@ def set_download_file_listener(mark_download_cancelled_by_skyvern(download)await download.cancel()return+ if context and owner_task_id:+ context.download_completed_task_ids.add(owner_task_id)file_path = Path(resolved_path)if not file_path.exists():# On an adopted persistent session the bytes live on the run connection, not@@ -526,6 +529,15 @@ def set_download_file_listener(# An aborted transfer surfaces here as a path() error; reporting it as a rename failure# sends triage to the wrong subsystem and hides that no file was ever saved.failure = await read_download_failure(download)+ if (+ failure == "canceled"+ and not was_download_cancelled_by_skyvern(download)+ and not download.page.is_closed()+ and context+ and owner_task_id+ and residential_egress+ ):+ context.residential_download_interrupted_task_ids.add(owner_task_id)if failure is not None:LOG.warning("Browser aborted the download before any file was saved",@@ -544,6 +556,25 @@ def set_download_file_listener(task_id=task_id,)+ def listen_to_download(download: Download) -> Coroutine[Any, Any, None]:+ # Snapshot the task synchronously at event delivery; the context can change before the coroutine runs.+ # Callback kwargs can belong to an older task or run and cannot establish ownership.+ context = current()+ owner_task_id = context.task_id if context else None+ residential_egress = False+ try:+ state = app.BROWSER_MANAGER.get_for_task(owner_task_id) if owner_task_id else None+ # Workflow blocks register their browser under the run id only. A plain lookup avoids aliasing the task.+ if state is None and owner_task_id and context and context.workflow_run_id:+ state = app.BROWSER_MANAGER.get_for_workflow_run(context.workflow_run_id)+ if state is None or state.browser_context is not browser_context:+ owner_task_id = None+ elif owner_task_id:+ residential_egress = app.AGENT_FUNCTION.is_residential_browser_egress(browser_context)+ except Exception:+ owner_task_id = None+ return save_download(download, context, owner_task_id, residential_egress)+def listen_to_new_page(page: Page) -> None:page.on("download", listen_to_download)