Skip to content

Automate browser based workflows with AI

fix(SKY-8924): retry webhook delivery on transient 403/5xx/network errors (#5952)

Aaron Perezee6ca1769ce5
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 129657f96..2518f213d 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -39,6 +39,7 @@ import { useSettingsStore } from "@/store/SettingsStore";
import { wssBaseUrl, newWssBaseUrl, getRuntimeApiKey } from "@/util/env";
import { copyText } from "@/util/copyText";
import { cn } from "@/util/utils";
+import { captureRecordBrowser } from "@/util/recordBrowserTelemetry";
import { RotateThrough } from "./RotateThrough";
import "./browser-stream.css";
@@ -830,7 +831,7 @@ function BrowserStream({
<div className="relative h-full w-full">
<div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2 text-red-500">
<div className="truncate">Browser is recording</div>
- <Tip content="To finish the recording, press stop on the animated recording button in the workflow.">
+ <Tip content="To finish and turn your actions into blocks, click the stop icon.">
<div className="cursor-pointer">
<InfoCircledIcon />
</div>
@@ -850,6 +851,12 @@ function BrowserStream({
recordingStore.compressedChunks.length > 0;
if (!hasEvents) {
e.preventDefault();
+ captureRecordBrowser("record_browser.cancelled", {
+ event_count_at_cancel:
+ recordingStore.getEventCount(),
+ seconds_recording:
+ recordingStore.getSecondsRecording(),
+ });
recordingStore.setIsRecording(false);
recordingStore.reset();
}
@@ -874,6 +881,12 @@ function BrowserStream({
<Button
variant="destructive"
onClick={() => {
+ captureRecordBrowser("record_browser.cancelled", {
+ event_count_at_cancel:
+ recordingStore.getEventCount(),
+ seconds_recording:
+ recordingStore.getSecondsRecording(),
+ });
recordingStore.setIsRecording(false);
recordingStore.reset();
}}
diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
index f7e76eddb..79f392527 100644
--- a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
@@ -1,5 +1,6 @@
import { useParams } from "react-router-dom";
import { useMutation } from "@tanstack/react-query";
+import { useRef } from "react";
import { getClient } from "@/api/AxiosClient";
import { toast } from "@/components/ui/use-toast";
@@ -9,6 +10,10 @@ import {
type WorkflowBlock,
type WorkflowParameter,
} from "@/routes/workflows/types/workflowTypes";
+import {
+ captureRecordBrowser,
+ markRecordBrowserProcessed,
+} from "@/util/recordBrowserTelemetry";
const FAIL_QUIET_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;
@@ -25,6 +30,7 @@ const useProcessRecordingMutation = ({
const credentialGetter = useCredentialGetter();
const recordingStore = useRecordingStore();
const { workflowPermanentId } = useParams();
+ const mutationStartedAtRef = useRef<number | null>(null);
const processRecordingMutation = useMutation({
mutationFn: async () => {
@@ -40,16 +46,24 @@ const useProcessRecordingMutation = ({
);
}
+ mutationStartedAtRef.current = Date.now();
+
const eventCount = recordingStore.getEventCount();
if (eventCount === 0) {
+ captureRecordBrowser("record_browser.empty_blocked", {
+ seconds_recording: recordingStore.getSecondsRecording(),
+ });
throw new Error(FAIL_QUIET_NO_EVENTS);
}
- // (this flushes any pending events)
const compressedChunks = await recordingStore.getCompressedChunks();
- // TODO: Replace this mock with actual API call when endpoint is ready
+ captureRecordBrowser("record_browser.process_attempted", {
+ event_count: recordingStore.getEventCount(),
+ compressed_chunk_count: compressedChunks.length,
+ });
+
const client = await getClient(credentialGetter, "sans-api-v1");
return client
.post<
@@ -70,6 +84,20 @@ const useProcessRecordingMutation = ({
}));
},
onSuccess: ({ blocks, parameters }) => {
+ const latencyMs =
+ mutationStartedAtRef.current !== null
+ ? Date.now() - mutationStartedAtRef.current
+ : 0;
+ mutationStartedAtRef.current = null;
+
+ markRecordBrowserProcessed(blocks?.length ?? 0);
+
+ captureRecordBrowser("record_browser.processed", {
+ block_count: blocks?.length ?? 0,
+ parameter_count: parameters?.length ?? 0,
+ latency_ms: latencyMs,
+ });
+
recordingStore.clear();
if (blocks && blocks.length > 0) {
@@ -91,14 +119,32 @@ const useProcessRecordingMutation = ({
});
},
onError: (error) => {
+ const latencyMs =
+ mutationStartedAtRef.current !== null
+ ? Date.now() - mutationStartedAtRef.current
+ : 0;
+ mutationStartedAtRef.current = null;
+
if (error instanceof Error && error.message === FAIL_QUIET_NO_EVENTS) {
+ recordingStore.reset();
+ toast({
+ variant: "warning",
+ title: "Nothing was recorded",
+ description:
+ "Interact with the live browser (clicks, typing, navigation), then stop recording again to generate blocks.",
+ });
return;
}
+ captureRecordBrowser("record_browser.processing_failed", {
+ error_message: error instanceof Error ? error.message : String(error),
+ latency_ms: latencyMs,
+ });
+
toast({
variant: "destructive",
title: "Error Processing Recording",
- description: error.message,
+ description: error instanceof Error ? error.message : String(error),
});
},
});
diff --git a/skyvern-frontend/src/routes/history/RunHistory.tsx b/skyvern-frontend/src/routes/history/RunHistory.tsx
index 367aa3a48..59e1adf51 100644
--- a/skyvern-frontend/src/routes/history/RunHistory.tsx
+++ b/skyvern-frontend/src/routes/history/RunHistory.tsx
@@ -16,10 +16,12 @@ import {
Status,
TaskRunListItem,
TaskRunType,
+ TriggerType,
WorkflowRunStatusApiResponse,
} from "@/api/types";
import { StatusBadge } from "@/components/StatusBadge";
import { StatusFilterDropdown } from "@/components/StatusFilterDropdown";
+import { TriggerTypeBadge } from "@/components/TriggerTypeBadge";
import {
Pagination,
PaginationContent,
@@ -69,6 +71,17 @@ function isKnownStatus(value: string): value is Status {
return statusValues.has(value);
}
+// Scheduled workflow runs carry a deterministic `wr_sched_<hash>` id prefix.
+function inferTriggerType(run: TaskRunListItem): TriggerType | null {
+ if (
+ run.task_run_type === TaskRunType.WorkflowRun &&
+ run.run_id.startsWith("wr_sched_")
+ ) {
+ return TriggerType.Scheduled;
+ }
+ return null;
+}
+
function getRunNavigationPath(run: TaskRunListItem): string {
switch (run.task_run_type) {
case TaskRunType.WorkflowRun:
@@ -195,10 +208,12 @@ function RunHistory() {
const isWorkflowRun = run.task_run_type === TaskRunType.WorkflowRun;
const isExpanded = isWorkflowRun && expandedRows.has(run.run_id);
const navPath = getRunNavigationPath(run);
+ const triggerType = inferTriggerType(run);
const titleContent =
- run.script_run || run.workflow_deleted ? (
+ triggerType || run.script_run || run.workflow_deleted ? (
<div className="flex items-center gap-2">
+ {triggerType && <TriggerTypeBadge triggerType={triggerType} />}
{run.script_run && (
<Tip content="Ran with code">
<LightningBoltIcon className="text-[gold]" />
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx
index a7451b17d..fb7433b71 100644
--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx
@@ -1,3 +1,4 @@
+import { StopIcon } from "@radix-ui/react-icons";
import { ReactNode, useEffect, useRef, useState } from "react";
import { Button } from "@/components/ui/button";
@@ -127,7 +128,12 @@ function WorkflowAdderBusy({
style={{ backgroundColor: color }}
onClick={handleClick}
/>
- <div className="pointer-events-none flex items-center justify-center">
+ <div
+ className={cn(
+ "pointer-events-none flex items-center justify-center",
+ operation === "recording" && "opacity-0",
+ )}
+ >
{children}
</div>
<div className="pointer-events-none absolute inset-0">
@@ -154,7 +160,17 @@ function WorkflowAdderBusy({
/>
</svg>
</div>
- {isHovered && (
+ {operation === "recording" && (
+ <div className="pointer-events-none absolute inset-0 flex items-center justify-center">
+ <StopIcon
+ className={cn(
+ "text-white drop-shadow-md",
+ size === "small" ? "h-5 w-5" : "h-10 w-10",
+ )}
+ />
+ </div>
+ )}
+ {isHovered && operation !== "recording" && (
<div className="pointer-events-none absolute inset-0">
<svg
className="h-full w-full"
@@ -185,14 +201,14 @@ function WorkflowAdderBusy({
<TooltipContent>
<p>
{operation === "recording"
- ? "Finish Recording"
+ ? "Stop recording & generate blocks"
: operation === "uploading"
? "Converting SOP... (click to cancel)"
: "Processing..."}
</p>
</TooltipContent>
</Tooltip>
- {recordingStore.isRecording && eventCount > 0 && (
+ {operation === "recording" && (
<Tooltip delayDuration={0}>
<TooltipTrigger asChild>
<div
@@ -201,6 +217,7 @@ function WorkflowAdderBusy({
{
"scale-125": shouldBump,
"scale-100": !shouldBump,
+ "opacity-90": eventCount === 0,
},
)}
style={{
@@ -212,7 +229,11 @@ function WorkflowAdderBusy({
</div>
</TooltipTrigger>
<TooltipContent>
- <p>Event Count</p>
+ <p>
+ {eventCount === 0
+ ? "No events yet — click and type in the live browser"
+ : "Events captured in this recording"}
+ </p>
</TooltipContent>
</Tooltip>
)}
diff --git a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
index aff88a3c8..fa11b8853 100644
--- a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
@@ -7,6 +7,7 @@ import {
useNodes,
} from "@xyflow/react";
import { useRef } from "react";
+import { useParams } from "react-router-dom";
import { Button } from "@/components/ui/button";
import {
@@ -39,6 +40,7 @@ function EdgeWithAddButton({
style = {},
markerEnd,
}: EdgeProps) {
+ const { workflowPermanentId } = useParams();
const nodes = useNodes();
const [edgePath, labelX, labelY] = getBezierPath({
sourceX,
@@ -178,7 +180,10 @@ function EdgeWithAddButton({
if (recordingStore.isRecording) {
recordingStore.setIsRecording(false);
} else {
- recordingStore.setIsRecording(true);
+ recordingStore.setIsRecording(true, {
+ workflowPermanentId: workflowPermanentId ?? null,
+ browserSessionId: settingsStore.browserSessionId,
+ });
updateWorkflowPanelState(false);
}
};
diff --git a/skyvern-frontend/src/routes/workflows/editor/hooks/useWorkflowHistory.ts b/skyvern-frontend/src/routes/workflows/editor/hooks/useWorkflowHistory.ts
index 4398d4628..6e569b5d5 100644
--- a/skyvern-frontend/src/routes/workflows/editor/hooks/useWorkflowHistory.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/hooks/useWorkflowHistory.ts
@@ -11,6 +11,7 @@ import type { Edge } from "@xyflow/react";
import { usePostHog } from "posthog-js/react";
import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";
+import { captureRecordBrowserUndoAfterRecordingIfRecent } from "@/util/recordBrowserTelemetry";
import type { AppNode } from "../nodes";
import {
canRedo as historyCanRedo,
@@ -331,11 +332,16 @@ export function useWorkflowHistory({
if (latestNodesRef.current.some((n) => n.dragging)) return;
// Flush first so an edit still pending in the debounce window isn't dropped.
flushPendingCapture();
+ const presentBeforeUndo = historyRef.current.present;
+ const nodesBeforeUndo = presentBeforeUndo?.nodes.length ?? 0;
const result = historyUndo(historyRef.current);
if (result === null) return;
historyRef.current = result.state;
applySnapshot(result.applied.nodes, result.applied.edges);
refreshFlags();
+ captureRecordBrowserUndoAfterRecordingIfRecent(
+ nodesBeforeUndo - result.applied.nodes.length,
+ );
posthog.capture("builder.undo_redo.used", {
action: "undo",
node_count: result.applied.nodes.length,
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
index e12336f1c..662ed3c03 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
@@ -1,6 +1,7 @@
import { PlusIcon } from "@radix-ui/react-icons";
import { Handle, NodeProps, Position, useEdges, useNodes } from "@xyflow/react";
import { useRef } from "react";
+import { useParams } from "react-router-dom";
import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";
import { useSopToBlocksMutation } from "@/routes/workflows/hooks/useSopToBlocksMutation";
@@ -22,6 +23,7 @@ import { WorkflowAddMenu } from "../../WorkflowAddMenu";
import { WorkflowAdderBusy } from "../../WorkflowAdderBusy";
function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
+ const { workflowPermanentId } = useParams();
const edges = useEdges();
const nodes = useNodes();
const debugStore = useDebugStore();
@@ -189,7 +191,10 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
if (recordingStore.isRecording) {
recordingStore.setIsRecording(false);
} else {
- recordingStore.setIsRecording(true);
+ recordingStore.setIsRecording(true, {
+ workflowPermanentId: workflowPermanentId ?? null,
+ browserSessionId: settingsStore.browserSessionId,
+ });
updateWorkflowPanelState(false);
}
};
diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.ts
index 76802d829..2303e4f00 100644
--- a/skyvern-frontend/src/store/useRecordingStore.ts
+++ b/skyvern-frontend/src/store/useRecordingStore.ts
@@ -1,5 +1,9 @@
import { create } from "zustand";
+import { captureRecordBrowser } from "@/util/recordBrowserTelemetry";
+
+const EVENT_CAPTURED_SAMPLE_RATE = 0.1;
+
/**
* example: {
* 'targetInfo': {
@@ -124,6 +128,7 @@ interface RecordingStore {
* Whether the user is currently in browser recording mode.
*/
isRecording: boolean;
+ recordingStartedAtMs: number | null;
/**
* Add a new recorded event. Triggers async compression when buffer is full.
*/
@@ -139,7 +144,13 @@ interface RecordingStore {
/**
* Set whether the user is in browser recording mode.
*/
- setIsRecording: (isRecording: boolean) => void;
+ setIsRecording: (
+ isRecording: boolean,
+ meta?: {
+ workflowPermanentId?: string | null;
+ browserSessionId?: string | null;
+ },
+ ) => void;
/**
* Flush any pending events into a compressed chunk.
* Call this before consuming the data.
@@ -153,6 +164,7 @@ interface RecordingStore {
* Get the total number of events (compressed + pending).
*/
getEventCount: () => number;
+ getSecondsRecording: () => number;
}
/**
@@ -223,6 +235,7 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
pendingEvents: [],
isCompressing: false,
isRecording: false,
+ recordingStartedAtMs: null,
add: (event) => {
const state = get();
@@ -233,6 +246,13 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
}
if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) {
+ if (Math.random() < EVENT_CAPTURED_SAMPLE_RATE) {
+ captureRecordBrowser("record_browser.event_captured", {
+ event_count: state.compressedChunks.length * CHUNK_SIZE + CHUNK_SIZE,
+ source: event.source,
+ });
+ }
+
const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE);
const remainingEvents = newPendingEvents.slice(CHUNK_SIZE);
@@ -264,7 +284,12 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
}
},
- clear: () => set({ compressedChunks: [], pendingEvents: [] }),
+ clear: () =>
+ set({
+ compressedChunks: [],
+ pendingEvents: [],
+ recordingStartedAtMs: null,
+ }),
reset: () =>
set({
@@ -273,15 +298,28 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
pendingEvents: [],
isCompressing: false,
isRecording: false,
+ recordingStartedAtMs: null,
}),
- setIsRecording: (isRecording) => {
+ setIsRecording: (isRecording, meta) => {
const state = get();
- // clear events on rising edge
if (!state.isRecording && isRecording) {
get().clear();
+ set({
+ isRecording: true,
+ recordingStartedAtMs: Date.now(),
+ });
+ captureRecordBrowser("record_browser.started", {
+ workflow_permanent_id: meta?.workflowPermanentId ?? undefined,
+ browser_session_id: meta?.browserSessionId ?? undefined,
+ });
+ return;
}
- set({ isRecording });
+
+ set({
+ isRecording,
+ recordingStartedAtMs: isRecording ? state.recordingStartedAtMs : null,
+ });
},
flush: async () => {
@@ -324,4 +362,12 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
state.compressedChunks.length * CHUNK_SIZE + state.pendingEvents.length
);
},
+
+ getSecondsRecording: () => {
+ const started = get().recordingStartedAtMs;
+ if (started === null) {
+ return 0;
+ }
+ return (Date.now() - started) / 1000;
+ },
}));
diff --git a/skyvern-frontend/src/util/recordBrowserTelemetry.ts b/skyvern-frontend/src/util/recordBrowserTelemetry.ts
new file mode 100644
index 000000000..bfbbf8542
--- /dev/null
+++ b/skyvern-frontend/src/util/recordBrowserTelemetry.ts
@@ -0,0 +1,52 @@
+import posthog from "posthog-js";
+
+let lastProcessedAtMs: number | null = null;
+let lastRecordingGeneratedBlockCount = 0;
+
+export function captureRecordBrowser(
+ event: string,
+ properties?: Record<string, unknown>,
+): void {
+ try {
+ posthog.capture(event, properties);
+ } catch {
+ // PostHog may be unavailable in tests or before init.
+ }
+}
+
+export function markRecordBrowserProcessed(blockCount: number): void {
+ if (blockCount <= 0) {
+ lastProcessedAtMs = null;
+ lastRecordingGeneratedBlockCount = 0;
+ return;
+ }
+ lastProcessedAtMs = Date.now();
+ lastRecordingGeneratedBlockCount = blockCount;
+}
+
+export function captureRecordBrowserUndoAfterRecordingIfRecent(
+ nodesRemovedCount: number,
+): void {
+ if (lastProcessedAtMs === null) {
+ return;
+ }
+
+ if (Date.now() - lastProcessedAtMs > 60_000) {
+ lastProcessedAtMs = null;
+ lastRecordingGeneratedBlockCount = 0;
+ return;
+ }
+
+ if (lastRecordingGeneratedBlockCount === 0) {
+ return;
+ }
+
+ if (nodesRemovedCount <= 0) {
+ return;
+ }
+
+ captureRecordBrowser("record_browser.undo_after_recording", {
+ nodes_removed_count: nodesRemovedCount,
+ recording_generated_block_count: lastRecordingGeneratedBlockCount,
+ });
+}
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index e4287e20c..d6b2f6a0b 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -113,6 +113,7 @@ from skyvern.services.otp_service import (
poll_otp_value,
try_generate_totp_from_credential,
)
+from skyvern.services.webhook_delivery import WEBHOOK_DELIVERY_MAX_ATTEMPTS, deliver_webhook_with_retries
from skyvern.utils.image_resizer import Resolution
from skyvern.utils.prompt_engine import (
PROMPT_HARD_CEILING_TOKENS,
@@ -4013,6 +4014,7 @@ class ForgeAgent:
self,
task: Task,
api_key: str | None,
+ enable_retries: bool = True,
) -> None:
if not api_key:
LOG.warning(
@@ -4060,13 +4062,14 @@ class ForgeAgent:
headers=signed_data.headers,
)
- resp = await app.AGENT_FUNCTION.deliver_webhook(
+ resp = await deliver_webhook_with_retries(
url=task.webhook_callback_url,
payload=signed_data.signed_payload,
headers=signed_data.headers,
timeout_seconds=30.0,
organization_id=task.organization_id,
run_id=task.task_id,
+ max_attempts=WEBHOOK_DELIVERY_MAX_ATTEMPTS if enable_retries else 1,
)
if resp.status_code >= 200 and resp.status_code < 300:
LOG.info(
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 69f2992b3..040730f44 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -1032,7 +1032,7 @@ class TaskRunModel(Base):
"organization_id",
desc("created_at"),
postgresql_using="btree",
- postgresql_where=text("parent_workflow_run_id IS NULL AND debug_session_id IS NULL AND status IS NOT NULL"),
+ postgresql_where=text("parent_workflow_run_id IS NULL AND debug_session_id IS NULL"),
),
Index(
"ix_task_runs_org_status_created",
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index f7fab6d47..6053d6205 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -558,7 +558,8 @@ class WorkflowRunsRepository(BaseRepository):
),
)
.filter(TaskRunModel.organization_id == organization_id)
- .filter(TaskRunModel.status.isnot(None))
+ # Coalesced filter so rows with NULL task_runs.status but a set workflow_runs.status are visible.
+ .filter(effective_status.isnot(None))
.filter(TaskRunModel.parent_workflow_run_id.is_(None))
.filter(TaskRunModel.debug_session_id.is_(None))
.filter(WorkflowRunModel.copilot_session_id.is_(None))
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 15dfd2fc8..c0693513b 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -2235,8 +2235,8 @@ async def retry_webhook(
if not latest_step:
return await app.agent.build_task_response(task=task_obj)
- # retry the webhook
- await app.agent.execute_task_webhook(task=task_obj, api_key=x_api_key)
+ # retry the webhook (single-shot - manual replay is itself the retry)
+ await app.agent.execute_task_webhook(task=task_obj, api_key=x_api_key, enable_retries=False)
return await app.agent.build_task_response(task=task_obj, last_step=latest_step)
diff --git a/skyvern/forge/sdk/routes/streaming/cdp_input.py b/skyvern/forge/sdk/routes/streaming/cdp_input.py
index 46f7f0926..772967d6e 100644
--- a/skyvern/forge/sdk/routes/streaming/cdp_input.py
+++ b/skyvern/forge/sdk/routes/streaming/cdp_input.py
@@ -30,12 +30,13 @@ LOG = structlog.get_logger()
_VALID_MOUSE_TYPES = {"mousePressed", "mouseReleased", "mouseMoved"}
_VALID_MOUSE_BUTTONS = {"left", "middle", "right", "none"}
-_VALID_KEY_TYPES = {"keyDown", "keyUp"}
+_VALID_KEY_TYPES = {"keyDown", "keyUp", "rawKeyDown"}
_MAX_COORD = 10000
_MAX_DELTA = 10000
_MAX_KEY_LEN = 32
_MAX_CODE_LEN = 32
_MODIFIER_MASK = 0xF
+_MAX_VK_CODE = 0xFE
@dataclasses.dataclass
@@ -124,6 +125,12 @@ def _validate_key_event(msg: dict) -> dict | None:
if isinstance(text, str) and len(text) == 1 and text.isprintable() and event_type == "keyDown":
result["text"] = text
+ # Forward `windowsVirtualKeyCode` so CDP can resolve non-printable keys
+ # (Backspace, Enter, Arrow*, etc.) to actual editing actions.
+ vk = msg.get("windowsVirtualKeyCode")
+ if isinstance(vk, int) and 0 <= vk <= _MAX_VK_CODE:
+ result["windowsVirtualKeyCode"] = vk
+
return result
@@ -379,9 +386,6 @@ async def cdp_input_browser_session_stream(
return
cdp_session = await page.context.new_cdp_session(page)
- # stream_ref_inc/dec is intentionally omitted for browser sessions.
- # Browser state lives in PersistentSessionsManager._browser_sessions,
- # not BrowserManager.pages, so there is no entry to protect from eviction.
LOG.info("CDP input channel ready", browser_session_id=browser_session_id, client_id=client_id)
await websocket.send_json({"kind": "ready"})
diff --git a/skyvern/forge/sdk/routes/streaming/channels/cdp.py b/skyvern/forge/sdk/routes/streaming/channels/cdp.py
index 500b99c1b..413c95f19 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/cdp.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/cdp.py
@@ -58,7 +58,7 @@ class CdpChannel:
def identity(self) -> t.Dict[str, t.Any]:
base = self.vnc_channel.identity
- return base | {"url": self.url}
+ return base | {"cdp_url": self.url}
async def connect(self, cdp_url: str | None = None) -> t.Self:
"""
diff --git a/skyvern/forge/sdk/routes/streaming/channels/execution.py b/skyvern/forge/sdk/routes/streaming/channels/execution.py
index 0c44d7e3d..21879cafd 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/execution.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/execution.py
@@ -12,15 +12,21 @@ Channel data:
from __future__ import annotations
+import base64
import typing as t
from contextlib import asynccontextmanager
+from urllib.parse import urlparse
import structlog
from playwright.async_api import Browser, BrowserContext, Page, Playwright
+from skyvern.config import settings
from skyvern.forge.sdk.routes.streaming.channels.cdp import CdpChannel
+from skyvern.forge.sdk.routes.streaming.payload_limits import MAX_SCREENSHOT_BYTES
+from skyvern.forge.sdk.routes.streaming.registries import get_vnc_channel
if t.TYPE_CHECKING:
+ from skyvern.forge.sdk.routes.streaming.channels.message import MessageChannel
from skyvern.forge.sdk.routes.streaming.channels.vnc import VncChannel
LOG = structlog.get_logger()
@@ -62,27 +68,176 @@ class ExecutionChannel(CdpChannel):
raise RuntimeError(f"{self.class_name} selected text is not a string, but a(n) '{type(selected_text)}'")
+ async def get_current_url(self) -> str:
+ LOG.debug(f"{self.class_name} getting current URL", **self.identity)
+
+ if not self.page:
+ raise RuntimeError(f"{self.class_name} get_current_url: not connected to a page.")
+
+ url = self.page.url
+ LOG.debug(f"{self.class_name} got current URL", url=url, **self.identity)
+ return url or ""
+
async def paste_text(self, text: str) -> None:
LOG.info(f"{self.class_name} pasting text", **self.identity)
- js_expression = """
- (text) => {
- const activeElement = document.activeElement;
- if (activeElement && (activeElement.tagName === 'INPUT' || activeElement.tagName === 'TEXTAREA' || activeElement.isContentEditable)) {
- const start = activeElement.selectionStart || 0;
- const end = activeElement.selectionEnd || 0;
- const value = activeElement.value || '';
- activeElement.value = value.slice(0, start) + text + value.slice(end);
- const newCursorPos = start + text.length;
- activeElement.setSelectionRange(newCursorPos, newCursorPos);
- }
- }
- """
+ if not self.page:
+ raise RuntimeError(f"{self.class_name} paste_text: not connected to a page.")
- await self.evaluate_js(js_expression, text)
+ await self.page.keyboard.insert_text(text)
LOG.info(f"{self.class_name} pasted text successfully", **self.identity)
+ async def navigate(self, url: str) -> None:
+ LOG.info(f"{self.class_name} navigating", target_url=url, **self.identity)
+
+ if not self.page:
+ raise RuntimeError(f"{self.class_name} navigate: not connected to a page.")
+
+ normalized = self._normalize_url(url)
+
+ await self.page.goto(
+ normalized,
+ wait_until="domcontentloaded",
+ timeout=settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+
+ LOG.info(f"{self.class_name} navigated", target_url=normalized, **self.identity)
+
+ async def reload(self, hard: bool = False) -> None:
+ LOG.info(f"{self.class_name} reloading", hard=hard, **self.identity)
+
+ if not self.page:
+ raise RuntimeError(f"{self.class_name} reload: not connected to a page.")
+
+ if hard:
+ cdp_session = await self.page.context.new_cdp_session(self.page)
+ try:
+ await cdp_session.send("Network.clearBrowserCache")
+ finally:
+ await cdp_session.detach()
+
+ await self.page.reload(
+ wait_until="domcontentloaded",
+ timeout=settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+
+ async def go_back(self) -> None:
+ LOG.info(f"{self.class_name} going back", **self.identity)
+
+ if not self.page:
+ raise RuntimeError(f"{self.class_name} go_back: not connected to a page.")
+
+ # Playwright returns None when there is no history to go back to; that is fine.
+ await self.page.go_back(
+ wait_until="domcontentloaded",
+ timeout=settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+
+ async def go_forward(self) -> None:
+ LOG.info(f"{self.class_name} going forward", **self.identity)
+
+ if not self.page:
+ raise RuntimeError(f"{self.class_name} go_forward: not connected to a page.")
+
+ await self.page.go_forward(
+ wait_until="domcontentloaded",
+ timeout=settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+
+ async def take_screenshot(self) -> str:
+ LOG.info(f"{self.class_name} taking screenshot", **self.identity)
+
+ if not self.page:
+ raise RuntimeError(f"{self.class_name} take_screenshot: not connected to a page.")
+
+ png_bytes = await self.page.screenshot(type="png", full_page=False)
+
+ if len(png_bytes) > MAX_SCREENSHOT_BYTES:
+ raise RuntimeError(
+ f"{self.class_name} screenshot too large: {len(png_bytes)} bytes (max {MAX_SCREENSHOT_BYTES})",
+ )
+
+ return base64.b64encode(png_bytes).decode("ascii")
+
+ async def clear_cookies(self) -> None:
+ LOG.info(f"{self.class_name} clearing cookies", **self.identity)
+
+ if not self.browser_context:
+ raise RuntimeError(f"{self.class_name} clear_cookies: no browser context.")
+
+ await self.browser_context.clear_cookies()
+
+ async def clear_storage(self) -> None:
+ """Clear storage for the current page's origin: cookies, local/session storage, IndexedDB, caches, service workers."""
+
+ LOG.info(f"{self.class_name} clearing all storage for origin", **self.identity)
+
+ if not self.page or not self.browser_context:
+ raise RuntimeError(f"{self.class_name} clear_storage: not connected.")
+
+ origin = self._origin_of(self.page.url)
+
+ cdp_session = await self.browser_context.new_cdp_session(self.page)
+ try:
+ await cdp_session.send(
+ "Storage.clearDataForOrigin",
+ {
+ "origin": origin,
+ "storageTypes": (
+ "cookies,local_storage,session_storage,indexeddb,websql,"
+ "service_workers,cache_storage,shader_cache,file_systems"
+ ),
+ },
+ )
+ finally:
+ await cdp_session.detach()
+
+ async def clear_history(self) -> None:
+ LOG.info(
+ f"{self.class_name} clearing history (cache, SW/cache storage, navigation stack)",
+ **self.identity,
+ )
+
+ if not self.page or not self.browser_context:
+ raise RuntimeError(f"{self.class_name} clear_history: not connected.")
+
+ origin = self._origin_of(self.page.url)
+
+ cdp_session = await self.browser_context.new_cdp_session(self.page)
+ try:
+ await cdp_session.send("Network.clearBrowserCache")
+ await cdp_session.send(
+ "Storage.clearDataForOrigin",
+ {"origin": origin, "storageTypes": "service_workers,cache_storage"},
+ )
+ await cdp_session.send("Page.resetNavigationHistory")
+ finally:
+ await cdp_session.detach()
+
+ @staticmethod
+ def _normalize_url(url: str) -> str:
+ candidate = url.strip()
+ if not candidate:
+ raise ValueError("URL must not be empty")
+ if candidate.startswith("//"):
+ raise ValueError("URL must include an explicit http(s) scheme or host")
+
+ parsed = urlparse(candidate)
+ if parsed.scheme in ("http", "https"):
+ return candidate
+ if parsed.scheme:
+ raise ValueError(f"refusing to navigate to non-http(s) scheme: {parsed.scheme}")
+ # Bare host or path -- prepend https://.
+ return f"https://{candidate}"
+
+ @staticmethod
+ def _origin_of(url: str) -> str:
+ parsed = urlparse(url)
+ if not parsed.scheme or not parsed.netloc:
+ return ""
+ return f"{parsed.scheme}://{parsed.netloc}"
+
async def close(self) -> None:
LOG.info(f"{self.class_name} closing connection", **self.identity)
@@ -99,6 +254,85 @@ class ExecutionChannel(CdpChannel):
LOG.info(f"{self.class_name} closed", **self.identity)
+class LocalExecutionChannel(ExecutionChannel):
+ def __init__(self, *, page: Page) -> None: # type: ignore[override]
+ # bypass CdpChannel.__init__ which requires a VncChannel
+ self.vnc_channel = None # type: ignore[assignment]
+ self.browser = None
+ self.browser_context = page.context
+ self.page = page
+ self.pw = None
+ self.url = None
+
+ @property
+ def identity(self) -> dict[str, t.Any]:
+ # Use `page_url` (not `url`) to avoid colliding with the inherited
+ # get_current_url log call which already passes `url=...` as a kwarg.
+ return {"local_execution": True, "page_url": self.page.url if self.page else None}
+
+ async def evaluate_js(
+ self,
+ expression: str,
+ arg: str | int | float | bool | list | dict | None = None,
+ ) -> str | int | float | bool | list | dict | None:
+ # Skip the inherited connect() — it dereferences self.vnc_channel,
+ # which LocalExecutionChannel intentionally leaves None.
+ if not self.page:
+ raise RuntimeError(f"{self.class_name} evaluate_js: no page available.")
+ return await self.page.evaluate(expression, arg)
+
+ async def close(self) -> None:
+ # We don't own the page or context; do not close.
+ return None
+
+
+@asynccontextmanager
+async def execution_for_message_channel(
+ message_channel: MessageChannel,
+) -> t.AsyncIterator[ExecutionChannel]:
+ """
+ Resolve an ExecutionChannel for a given message channel.
+
+ Prefers the VNC pipeline (remote CDP via VncChannel.browser_session.browser_address) when a
+ VncChannel is registered for this client. Falls back to the in-process Page held by
+ PERSISTENT_SESSIONS_MANAGER, which is what local dev (and any single-process deployment) uses.
+ """
+
+ vnc_channel = get_vnc_channel(message_channel.client_id)
+
+ if vnc_channel is not None:
+ async with execution_channel(vnc_channel) as execute:
+ yield execute
+ return
+
+ # Imports kept local to avoid a circular import (this module is imported by message.py,
+ # and the persistent sessions manager pulls in app/cloud bootstrapping).
+ from skyvern.forge import app
+ from skyvern.forge.sdk.routes.streaming.screencast import wait_for_browser_state
+
+ if message_channel.browser_session is None:
+ raise RuntimeError("execution_for_message_channel: no browser session on message channel")
+
+ browser_session_id = message_channel.browser_session.persistent_browser_session_id
+
+ session = await app.PERSISTENT_SESSIONS_MANAGER.get_session(
+ session_id=browser_session_id,
+ organization_id=message_channel.organization_id,
+ )
+ if session is None:
+ raise RuntimeError(f"execution_for_message_channel: session {browser_session_id} not found")
+
+ browser_state = await wait_for_browser_state(browser_session_id, "browser_session")
+ if browser_state is None:
+ raise RuntimeError(f"execution_for_message_channel: browser state timeout for {browser_session_id}")
+
+ page = await browser_state.get_working_page()
+ if page is None:
+ raise RuntimeError(f"execution_for_message_channel: no working page for {browser_session_id}")
+
+ yield LocalExecutionChannel(page=page)
+
+
@asynccontextmanager
async def execution_channel(vnc_channel: VncChannel) -> t.AsyncIterator[ExecutionChannel]:
"""
@@ -118,10 +352,6 @@ async def execution_channel(vnc_channel: VncChannel) -> t.AsyncIterator[Executio
try:
await channel.connect()
- # NOTE(jdo:streaming-local-dev)
- # from skyvern.config import settings
- # await channel.connect(settings.BROWSER_REMOTE_DEBUGGING_URL)
-
yield channel
finally:
await channel.close()
diff --git a/skyvern/forge/sdk/routes/streaming/channels/message.py b/skyvern/forge/sdk/routes/streaming/channels/message.py
index 27691e8bf..1769f8761 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/message.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/message.py
@@ -22,8 +22,9 @@ from fastapi import WebSocket, WebSocketDisconnect
from starlette.websockets import WebSocketState
from websockets.exceptions import ConnectionClosedError
-from skyvern.forge.sdk.routes.streaming.channels.execution import execution_channel
+from skyvern.forge.sdk.routes.streaming.channels.execution import execution_for_message_channel
from skyvern.forge.sdk.routes.streaming.channels.exfiltration import ExfiltratedEvent, ExfiltrationChannel
+from skyvern.forge.sdk.routes.streaming.payload_limits import MAX_CLIPBOARD_PASTE_BYTES
from skyvern.forge.sdk.routes.streaming.registries import (
add_message_channel,
del_message_channel,
@@ -48,10 +49,24 @@ class MessageKind(enum.StrEnum):
ASK_FOR_CLIPBOARD_RESPONSE = "ask-for-clipboard-response"
BEGIN_EXFILTRATION = "begin-exfiltration"
BROWSER_TABS = "browser-tabs"
+ BROWSER_URL = "browser-url"
CEDE_CONTROL = "cede-control"
+ CLEAR_ALL_DATA = "clear-all-data"
+ CLEAR_COOKIES = "clear-cookies"
+ CLEAR_HISTORY = "clear-history"
+ CLIPBOARD_COPY = "clipboard-copy"
+ CLIPBOARD_PASTE = "clipboard-paste"
END_EXFILTRATION = "end-exfiltration"
+ ERROR = "error"
EXFILTRATED_EVENT = "exfiltrated-event"
+ GET_BROWSER_URL = "get-browser-url"
+ GO_BACK = "go-back"
+ GO_FORWARD = "go-forward"
+ NAVIGATE = "navigate"
+ RELOAD = "reload"
+ SCREENSHOT = "screenshot"
TAKE_CONTROL = "take-control"
+ TAKE_SCREENSHOT = "take-screenshot"
class ExfiltratedEventSource(enum.StrEnum):
@@ -76,10 +91,24 @@ MessageKinds = t.Literal[
MessageKind.ASK_FOR_CLIPBOARD_RESPONSE,
MessageKind.BEGIN_EXFILTRATION,
MessageKind.BROWSER_TABS,
+ MessageKind.BROWSER_URL,
MessageKind.CEDE_CONTROL,
+ MessageKind.CLEAR_ALL_DATA,
+ MessageKind.CLEAR_COOKIES,
+ MessageKind.CLEAR_HISTORY,
+ MessageKind.CLIPBOARD_COPY,
+ MessageKind.CLIPBOARD_PASTE,
MessageKind.END_EXFILTRATION,
+ MessageKind.ERROR,
MessageKind.EXFILTRATED_EVENT,
+ MessageKind.GET_BROWSER_URL,
+ MessageKind.GO_BACK,
+ MessageKind.GO_FORWARD,
+ MessageKind.NAVIGATE,
+ MessageKind.RELOAD,
+ MessageKind.SCREENSHOT,
MessageKind.TAKE_CONTROL,
+ MessageKind.TAKE_SCREENSHOT,
]
@@ -114,6 +143,86 @@ class MessageInAskForClipboardResponse(Message):
text: str = ""
+class MessageInClipboardCopy(Message):
+ kind: t.Literal[MessageKind.CLIPBOARD_COPY] = MessageKind.CLIPBOARD_COPY
+
+
+class MessageInClipboardPaste(Message):
+ kind: t.Literal[MessageKind.CLIPBOARD_PASTE] = MessageKind.CLIPBOARD_PASTE
+ text: str = ""
+
+
+class MessageInGetBrowserUrl(Message):
+ kind: t.Literal[MessageKind.GET_BROWSER_URL] = MessageKind.GET_BROWSER_URL
+
+
+class MessageInNavigate(Message):
+ kind: t.Literal[MessageKind.NAVIGATE] = MessageKind.NAVIGATE
+ url: str = ""
+
+
+class MessageInReload(Message):
+ kind: t.Literal[MessageKind.RELOAD] = MessageKind.RELOAD
+ hard: bool = False
+
+
+class MessageInGoBack(Message):
+ kind: t.Literal[MessageKind.GO_BACK] = MessageKind.GO_BACK
+
+
+class MessageInGoForward(Message):
+ kind: t.Literal[MessageKind.GO_FORWARD] = MessageKind.GO_FORWARD
+
+
+class MessageInTakeScreenshot(Message):
+ kind: t.Literal[MessageKind.TAKE_SCREENSHOT] = MessageKind.TAKE_SCREENSHOT
+
+
+class MessageInClearCookies(Message):
+ kind: t.Literal[MessageKind.CLEAR_COOKIES] = MessageKind.CLEAR_COOKIES
+
+
+class MessageInClearHistory(Message):
+ kind: t.Literal[MessageKind.CLEAR_HISTORY] = MessageKind.CLEAR_HISTORY
+
+
+class MessageInClearAllData(Message):
+ kind: t.Literal[MessageKind.CLEAR_ALL_DATA] = MessageKind.CLEAR_ALL_DATA
+
+
+class MessageOutBrowserUrl(Message):
+ kind: t.Literal[MessageKind.BROWSER_URL] = MessageKind.BROWSER_URL
+ url: str = ""
+
+
+class MessageOutScreenshot(Message):
+ kind: t.Literal[MessageKind.SCREENSHOT] = MessageKind.SCREENSHOT
+ data: str = "" # base64-encoded PNG payload
+ mime_type: str = "image/png"
+
+
+class MessageOutError(Message):
+ """Surfaces a backend handler failure to the frontend so it can toast."""
+
+ kind: t.Literal[MessageKind.ERROR] = MessageKind.ERROR
+ failed_kind: str = ""
+ message: str = ""
+
+
@dataclasses.dataclass
class MessageOutExfiltratedEvent(Message):
kind: t.Literal[MessageKind.EXFILTRATED_EVENT] = MessageKind.EXFILTRATED_EVENT
@@ -135,12 +244,25 @@ MessageIn = (
MessageInAskForClipboardResponse
| MessageInBeginExfiltration
| MessageInCedeControl
+ | MessageInClearAllData
+ | MessageInClearCookies
+ | MessageInClearHistory
+ | MessageInClipboardCopy
+ | MessageInClipboardPaste
| MessageInEndExfiltration
+ | MessageInGetBrowserUrl
+ | MessageInGoBack
+ | MessageInGoForward
+ | MessageInNavigate
+ | MessageInReload
| MessageInTakeControl
+ | MessageInTakeScreenshot
)
-MessageOut = MessageOutExfiltratedEvent | MessageOutTabInfo
+MessageOut = (
+ MessageOutBrowserUrl | MessageOutError | MessageOutExfiltratedEvent | MessageOutScreenshot | MessageOutTabInfo
+)
ChannelMessage = MessageIn | MessageOut
@@ -157,10 +279,35 @@ def reify_channel_message(data: dict) -> ChannelMessage:
return MessageInBeginExfiltration()
case MessageKind.CEDE_CONTROL:
return MessageInCedeControl()
+ case MessageKind.CLEAR_ALL_DATA:
+ return MessageInClearAllData()
+ case MessageKind.CLEAR_COOKIES:
+ return MessageInClearCookies()
+ case MessageKind.CLEAR_HISTORY:
+ return MessageInClearHistory()
+ case MessageKind.CLIPBOARD_COPY:
+ return MessageInClipboardCopy()
+ case MessageKind.CLIPBOARD_PASTE:
+ text = data.get("text") or ""
+ return MessageInClipboardPaste(text=text)
case MessageKind.END_EXFILTRATION:
return MessageInEndExfiltration()
+ case MessageKind.GET_BROWSER_URL:
+ return MessageInGetBrowserUrl()
+ case MessageKind.GO_BACK:
+ return MessageInGoBack()
+ case MessageKind.GO_FORWARD:
+ return MessageInGoForward()
+ case MessageKind.NAVIGATE:
+ url = data.get("url") or ""
+ return MessageInNavigate(url=url)
+ case MessageKind.RELOAD:
+ hard = bool(data.get("hard") or False)
+ return MessageInReload(hard=hard)
case MessageKind.TAKE_CONTROL:
return MessageInTakeControl()
+ case MessageKind.TAKE_SCREENSHOT:
+ return MessageInTakeScreenshot()
case _:
raise ValueError(f"Unknown message kind: '{kind}'")
@@ -393,22 +540,37 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:
)
return
+ async def send_error(failed_kind: MessageKind, reason: str) -> None:
+ await send(MessageOutError(failed_kind=str(failed_kind), message=reason))
+
+ async def send_current_browser_url(execute: t.Any) -> None:
+ url = await execute.get_current_url()
+ await send(MessageOutBrowserUrl(url=url))
+
match message.kind:
case MessageKind.ASK_FOR_CLIPBOARD_RESPONSE:
- vnc_channel = get_vnc_channel(message_channel.client_id)
+ text = message.text
- if not vnc_channel:
- LOG.error(
- f"{class_name} no vnc channel found for message channel.",
- message=message,
+ paste_byte_len = len(text.encode("utf-8"))
+ if paste_byte_len > MAX_CLIPBOARD_PASTE_BYTES:
+ LOG.warning(
+ f"{class_name} ask-for-clipboard-response paste exceeds size cap; rejecting.",
+ size=paste_byte_len,
+ max_size=MAX_CLIPBOARD_PASTE_BYTES,
**message_channel.identity,
)
+ await send_error(message.kind, "Clipboard payload too large.")
return
- text = message.text
-
- async with execution_channel(vnc_channel) as execute:
- await execute.paste_text(text)
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ await execute.paste_text(text)
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to paste clipboard response into browser.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Failed to paste into browser.")
case MessageKind.BEGIN_EXFILTRATION:
if exfiltration_channel is not None:
@@ -447,6 +609,9 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:
case MessageKind.BROWSER_TABS:
await send(message)
+ case MessageKind.BROWSER_URL:
+ await send(message)
+
case MessageKind.CEDE_CONTROL:
vnc_channel = get_vnc_channel(message_channel.client_id)
@@ -459,6 +624,76 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:
return
vnc_channel.interactor = "agent"
+ case MessageKind.CLEAR_ALL_DATA:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ await execute.clear_storage()
+ await execute.clear_cookies()
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to clear all data.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Failed to clear all browsing data.")
+
+ case MessageKind.CLEAR_COOKIES:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ await execute.clear_cookies()
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to clear cookies.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Failed to clear cookies.")
+
+ case MessageKind.CLEAR_HISTORY:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ await execute.clear_history()
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to clear history.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Failed to clear browsing history.")
+
+ case MessageKind.CLIPBOARD_COPY:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ copied_text = await execute.get_selected_text()
+ await message_channel.send_copied_text(copied_text)
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to copy text from browser.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Failed to copy selected text.")
+
+ case MessageKind.CLIPBOARD_PASTE:
+ text = message.text
+
+ paste_byte_len = len(text.encode("utf-8"))
+ if paste_byte_len > MAX_CLIPBOARD_PASTE_BYTES:
+ LOG.warning(
+ f"{class_name} clipboard-paste payload exceeds size cap; rejecting.",
+ size=paste_byte_len,
+ max_size=MAX_CLIPBOARD_PASTE_BYTES,
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Clipboard payload too large.")
+ return
+
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ await execute.paste_text(text)
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to paste text into browser.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Failed to paste into browser.")
+
case MessageKind.END_EXFILTRATION:
if exfiltration_channel is None:
return
@@ -467,14 +702,89 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:
exfiltration_channel = None
+ case MessageKind.ERROR:
+ await send(message)
+
case MessageKind.EXFILTRATED_EVENT:
await send(message)
- # case MessageKind.GET_TAB_INFO:
- # """
- # TODO(jdo): implement - this is an on-demand request for tab info, which is
- # required when connecting to an existing browser session.
- # """
+ case MessageKind.GET_BROWSER_URL:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ url = await execute.get_current_url()
+ url_message = MessageOutBrowserUrl(url=url)
+ await send(url_message)
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to get browser URL.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Failed to read browser URL.")
+
+ case MessageKind.GO_BACK:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ await execute.go_back()
+ await send_current_browser_url(execute)
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to go back.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Couldn't go back.")
+
+ case MessageKind.GO_FORWARD:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ await execute.go_forward()
+ await send_current_browser_url(execute)
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to go forward.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Couldn't go forward.")
+
+ case MessageKind.NAVIGATE:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ await execute.navigate(message.url)
+ # Push the new URL back so the URL input reflects it immediately,
+ # rather than waiting for the next poll.
+ await send_current_browser_url(execute)
+ except ValueError as exc:
+ LOG.warning(
+ f"{class_name} rejected navigate.",
+ url=message.url,
+ error=str(exc),
+ **message_channel.identity,
+ )
+ await send_error(message.kind, str(exc))
+ except Exception:
+ # The target URL is user-controlled; log it server-side
+ # but don't reflect it back in the toast.
+ LOG.exception(
+ f"{class_name} failed to navigate.",
+ url=message.url,
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Navigation failed.")
+
+ case MessageKind.RELOAD:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ await execute.reload(hard=message.hard)
+ await send_current_browser_url(execute)
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to reload.",
+ hard=message.hard,
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Failed to reload.")
+
+ case MessageKind.SCREENSHOT:
+ await send(message)
case MessageKind.TAKE_CONTROL:
LOG.info(f"{class_name} processing take-control message.", **message_channel.identity)
@@ -489,6 +799,18 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:
return
vnc_channel.interactor = "user"
+ case MessageKind.TAKE_SCREENSHOT:
+ try:
+ async with execution_for_message_channel(message_channel) as execute:
+ screenshot_b64 = await execute.take_screenshot()
+ await send(MessageOutScreenshot(data=screenshot_b64))
+ except Exception:
+ LOG.exception(
+ f"{class_name} failed to take screenshot.",
+ **message_channel.identity,
+ )
+ await send_error(message.kind, "Failed to capture screenshot.")
+
case _:
t.assert_never(message.kind)
diff --git a/skyvern/forge/sdk/routes/streaming/payload_limits.py b/skyvern/forge/sdk/routes/streaming/payload_limits.py
new file mode 100644
index 000000000..1e233ec0a
--- /dev/null
+++ b/skyvern/forge/sdk/routes/streaming/payload_limits.py
@@ -0,0 +1,2 @@
+MAX_CLIPBOARD_PASTE_BYTES = 1 * 1024 * 1024
+MAX_SCREENSHOT_BYTES = 5 * 1024 * 1024
diff --git a/skyvern/forge/sdk/routes/streaming/verify.py b/skyvern/forge/sdk/routes/streaming/verify.py
index 500cfd171..0593b984a 100644
--- a/skyvern/forge/sdk/routes/streaming/verify.py
+++ b/skyvern/forge/sdk/routes/streaming/verify.py
@@ -21,6 +21,7 @@ from datetime import datetime, timedelta
import structlog
+from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.schemas.persistent_browser_sessions import AddressablePersistentBrowserSession, is_final_status
from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
@@ -79,25 +80,28 @@ async def verify_browser_session(
browser_address = browser_session.browser_address
if not browser_address:
- LOG.info(
- "Waiting for browser session address.",
- browser_session_id=browser_session_id,
- organization_id=organization_id,
- )
-
- try:
- browser_address = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_address(
- session_id=browser_session_id,
- organization_id=organization_id,
- )
- except Exception as ex:
+ if settings.BROWSER_STREAMING_MODE == "cdp":
+ browser_address = ""
+ else:
LOG.info(
- "Browser session address not found for browser session.",
+ "Waiting for browser session address.",
browser_session_id=browser_session_id,
organization_id=organization_id,
- ex=ex,
)
- return None
+
+ try:
+ browser_address = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_address(
+ session_id=browser_session_id,
+ organization_id=organization_id,
+ )
+ except Exception as ex:
+ LOG.info(
+ "Browser session address not found for browser session.",
+ browser_session_id=browser_session_id,
+ organization_id=organization_id,
+ ex=ex,
+ )
+ return None
try:
addressable_browser_session = AddressablePersistentBrowserSession(
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index fcb363618..84aca45d2 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -57,6 +57,7 @@ from skyvern.exceptions import (
get_user_facing_exception_message,
)
from skyvern.forge import app
+from skyvern.forge.failure_classifier import classify_from_failure_reason
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api import email
from skyvern.forge.sdk.api.aws import AsyncAWSClient
@@ -777,6 +778,19 @@ class Block(BaseModel, abc.ABC):
pass
+def _should_skip_retry_on_anti_bot_detection(task: Task) -> bool:
+ categories = task.failure_category
+ if categories:
+ return any(c.get("category") == "ANTI_BOT_DETECTION" for c in categories)
+
+ if task.failure_reason:
+ result = classify_from_failure_reason(task.failure_reason)
+ if result and any(c.get("category") == "ANTI_BOT_DETECTION" for c in result):
+ return True
+
+ return False
+
+
class BaseTaskBlock(Block):
task_type: str = TaskType.general
url: str | None = None
@@ -1368,6 +1382,19 @@ class BaseTaskBlock(Block):
else:
current_retry += 1
will_retry = current_retry <= self.max_retries
+ if will_retry and _should_skip_retry_on_anti_bot_detection(updated_task):
+ LOG.warning(
+ "Skipping retry - task failed due to anti-bot detection",
+ task_id=updated_task.task_id,
+ workflow_run_id=workflow_run_id,
+ workflow_id=workflow.workflow_id,
+ organization_id=workflow_run.organization_id,
+ current_retry=current_retry,
+ max_retries=self.max_retries,
+ failure_reason=updated_task.failure_reason,
+ failure_category=updated_task.failure_category,
+ )
+ will_retry = False
retry_message = f", retrying task {current_retry}/{self.max_retries}" if will_retry else ""
downloaded_files = []
try:
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index a527bb8f2..1ed1338f6 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -129,6 +129,7 @@ from skyvern.schemas.workflows import (
WorkflowStatus,
)
from skyvern.services import script_service, workflow_script_service
+from skyvern.services.webhook_delivery import deliver_webhook_with_retries
from skyvern.utils.css_selector import build_action_summaries_with_timing # shared with script_service
from skyvern.utils.url_validators import validate_url as validate_url_with_blocked_host_check
from skyvern.webeye.browser_state import BrowserState
@@ -5281,7 +5282,7 @@ class WorkflowService:
headers=signed_data.headers,
)
try:
- resp = await app.AGENT_FUNCTION.deliver_webhook(
+ resp = await deliver_webhook_with_retries(
url=workflow_run.webhook_callback_url,
payload=signed_data.signed_payload,
headers=signed_data.headers,
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index 1c275da1b..68a2b0e0f 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -65,6 +65,7 @@ from skyvern.schemas.workflows import (
WorkflowDefinitionYAML,
WorkflowStatus,
)
+from skyvern.services.webhook_delivery import deliver_webhook_with_retries
from skyvern.utils.prompt_engine import load_prompt_with_elements
from skyvern.utils.strings import generate_random_string
from skyvern.webeye.browser_state import BrowserState
@@ -2095,7 +2096,7 @@ async def send_task_v2_webhook(task_v2: TaskV2) -> None:
payload_length=len(payload),
header_keys=sorted(headers.keys()),
)
- resp = await app.AGENT_FUNCTION.deliver_webhook(
+ resp = await deliver_webhook_with_retries(
url=task_v2.webhook_callback_url,
payload=payload,
headers=headers,
diff --git a/skyvern/services/webhook_delivery.py b/skyvern/services/webhook_delivery.py
new file mode 100644
index 000000000..aa0a98733
--- /dev/null
+++ b/skyvern/services/webhook_delivery.py
@@ -0,0 +1,125 @@
+"""Bounded-retry wrapper around ``app.AGENT_FUNCTION.deliver_webhook``."""
+
+from __future__ import annotations
+
+import asyncio
+import random
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+
+import httpx
+import structlog
+
+from skyvern.forge import app
+
+LOG = structlog.get_logger()
+
+NON_5XX_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset(
+ {
+ 403, # included for intermittent WAF/proxy denials; permanent auth failures stay observable via status_code in the retry log
+ 408,
+ 425,
+ 429,
+ }
+)
+
+WEBHOOK_DELIVERY_MAX_ATTEMPTS = 3
+WEBHOOK_DELIVERY_RETRY_BASE_DELAY_SECONDS = 1.0
+WEBHOOK_DELIVERY_MAX_RETRY_AFTER_SECONDS = 30.0
+
+
+def is_retryable_status(status_code: int) -> bool:
+ return status_code in NON_5XX_RETRYABLE_STATUS_CODES or 500 <= status_code < 600
+
+
+def _parse_retry_after(value: str | None) -> float | None:
+ if value is None:
+ return None
+ stripped = value.strip()
+ if not stripped:
+ return None
+ try:
+ return max(float(stripped), 0.0)
+ except ValueError:
+ pass
+ try:
+ target = parsedate_to_datetime(stripped)
+ except (TypeError, ValueError):
+ return None
+ if target is None:
+ return None
+ if target.tzinfo is None:
+ target = target.replace(tzinfo=timezone.utc)
+ return max((target - datetime.now(timezone.utc)).total_seconds(), 0.0)
+
+
+def _compute_backoff_delay(attempt: int, base_delay_seconds: float, response: httpx.Response | None) -> float:
+ hinted = _parse_retry_after(response.headers.get("Retry-After")) if response is not None else None
+ if hinted is not None:
+ return min(hinted, WEBHOOK_DELIVERY_MAX_RETRY_AFTER_SECONDS)
+ return base_delay_seconds * (2**attempt) + random.uniform(0, base_delay_seconds)
+
+
+async def deliver_webhook_with_retries(
+ url: str,
+ payload: str,
+ headers: dict[str, str],
+ timeout_seconds: float,
+ organization_id: str | None,
+ run_id: str | None,
+ max_attempts: int = WEBHOOK_DELIVERY_MAX_ATTEMPTS,
+ base_delay_seconds: float = WEBHOOK_DELIVERY_RETRY_BASE_DELAY_SECONDS,
+) -> httpx.Response:
+ last_response: httpx.Response | None = None
+ last_exc: Exception | None = None
+
+ for attempt in range(max_attempts):
+ try:
+ response = await app.AGENT_FUNCTION.deliver_webhook(
+ url=url,
+ payload=payload,
+ headers=headers,
+ timeout_seconds=timeout_seconds,
+ organization_id=organization_id,
+ run_id=run_id,
+ )
+ last_response = response
+ last_exc = None
+ if 200 <= response.status_code < 300:
+ return response
+ if not is_retryable_status(response.status_code):
+ return response
+ except httpx.HTTPStatusError as exc:
+ # NAT egress proxy client calls resp.raise_for_status(), so a proxy-side
+ # 5xx surfaces here rather than as a returned Response.
+ last_response = exc.response
+ last_exc = exc
+ if not is_retryable_status(exc.response.status_code):
+ raise
+ except (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError) as exc:
+ last_response = None
+ last_exc = exc
+
+ if attempt < max_attempts - 1:
+ delay = _compute_backoff_delay(attempt, base_delay_seconds, last_response)
+ status_code = last_response.status_code if last_response is not None else None
+ log_fn = LOG.warning if status_code == 403 else LOG.info
+ log_fn(
+ "Retrying webhook delivery after transient failure",
+ url=url,
+ run_id=run_id,
+ organization_id=organization_id,
+ attempt=attempt + 1,
+ max_attempts=max_attempts,
+ status_code=status_code,
+ error=str(last_exc) if last_exc is not None else None,
+ sleep_seconds=delay,
+ retry_after_present=last_response is not None and "Retry-After" in last_response.headers,
+ )
+ await asyncio.sleep(delay)
+
+ if last_exc is not None:
+ raise last_exc
+ if last_response is None:
+ raise RuntimeError("deliver_webhook_with_retries exited without a response or exception")
+ return last_response
diff --git a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
index facddce66..b6deffb17 100644
--- a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
+++ b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
@@ -228,6 +228,26 @@ async def test_get_all_runs_v2_selects_workflow_deleted_flag() -> None:
assert "coalesce(task_runs.workflow_permanent_id, workflow_runs.workflow_permanent_id)" in rendered
+async def test_get_all_runs_v2_status_filter_uses_coalesced_effective_status() -> None:
+ captured: dict[str, Any] = {}
+
+ async def _execute(query):
+ captured["query"] = query
+ return _EmptyExecuteResult()
+
+ session = MagicMock()
+ session.execute = AsyncMock(side_effect=_execute)
+
+ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)
+
+ await repo.get_all_runs_v2(organization_id="o_test")
+
+ where_clause = _where_clause_sql(captured["query"]).lower()
+ assert "coalesce(workflow_runs.status, task_runs.status) is not null" in where_clause
+ assert "task_runs.status is not null" not in where_clause
+
+
@pytest.mark.asyncio
async def test_get_all_runs_v2_excludes_copilot_session_workflow_runs() -> None:
captured: dict[str, Any] = {}
diff --git a/tests/unit/forge/sdk/routes/__init__.py b/tests/unit/forge/sdk/routes/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/forge/sdk/routes/streaming/__init__.py b/tests/unit/forge/sdk/routes/streaming/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py
new file mode 100644
index 000000000..885c6bfe8
--- /dev/null
+++ b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py
@@ -0,0 +1,280 @@
+"""Tests for `reify_channel_message` and lightweight ExecutionChannel helpers."""
+
+from __future__ import annotations
+
+import typing as t
+from contextlib import asynccontextmanager
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from starlette.websockets import WebSocketState
+
+from skyvern.config import settings
+from skyvern.forge.sdk.routes.streaming.channels import message as message_module
+from skyvern.forge.sdk.routes.streaming.channels.execution import ExecutionChannel, LocalExecutionChannel
+from skyvern.forge.sdk.routes.streaming.channels.message import (
+ MessageChannel,
+ MessageInClearAllData,
+ MessageInClearCookies,
+ MessageInClearHistory,
+ MessageInClipboardPaste,
+ MessageInGoBack,
+ MessageInGoForward,
+ MessageInNavigate,
+ MessageInReload,
+ MessageInTakeScreenshot,
+ MessageKind,
+ loop_stream_messages,
+ reify_channel_message,
+)
+
+
+class TestReifyChannelMessage:
+ def test_navigate_with_url(self) -> None:
+ msg = reify_channel_message({"kind": "navigate", "url": "https://example.com"})
+ assert isinstance(msg, MessageInNavigate)
+ assert msg.url == "https://example.com"
+ assert msg.kind == MessageKind.NAVIGATE
+
+ def test_navigate_missing_url_defaults_to_empty(self) -> None:
+ msg = reify_channel_message({"kind": "navigate"})
+ assert isinstance(msg, MessageInNavigate)
+ assert msg.url == ""
+
+ def test_reload_soft(self) -> None:
+ msg = reify_channel_message({"kind": "reload", "hard": False})
+ assert isinstance(msg, MessageInReload)
+ assert msg.hard is False
+
+ def test_reload_hard(self) -> None:
+ msg = reify_channel_message({"kind": "reload", "hard": True})
+ assert isinstance(msg, MessageInReload)
+ assert msg.hard is True
+
+ def test_reload_default_is_soft(self) -> None:
+ msg = reify_channel_message({"kind": "reload"})
+ assert isinstance(msg, MessageInReload)
+ assert msg.hard is False
+
+ def test_go_back(self) -> None:
+ assert isinstance(reify_channel_message({"kind": "go-back"}), MessageInGoBack)
+
+ def test_go_forward(self) -> None:
+ assert isinstance(reify_channel_message({"kind": "go-forward"}), MessageInGoForward)
+
+ def test_take_screenshot(self) -> None:
+ assert isinstance(
+ reify_channel_message({"kind": "take-screenshot"}),
+ MessageInTakeScreenshot,
+ )
+
+ def test_clear_cookies(self) -> None:
+ assert isinstance(reify_channel_message({"kind": "clear-cookies"}), MessageInClearCookies)
+
+ def test_clear_history(self) -> None:
+ assert isinstance(reify_channel_message({"kind": "clear-history"}), MessageInClearHistory)
+
+ def test_clear_all_data(self) -> None:
+ assert isinstance(reify_channel_message({"kind": "clear-all-data"}), MessageInClearAllData)
+
+ def test_clipboard_paste_carries_text(self) -> None:
+ msg = reify_channel_message({"kind": "clipboard-paste", "text": "hello"})
+ assert isinstance(msg, MessageInClipboardPaste)
+ assert msg.text == "hello"
+
+ def test_unknown_kind_raises(self) -> None:
+ with pytest.raises(ValueError):
+ reify_channel_message({"kind": "totally-not-a-real-kind"})
+
+
+class TestExecutionChannelHelpers:
+ @pytest.mark.asyncio
+ async def test_get_current_url_does_not_collide_with_cdp_url_identity(self) -> None:
+ vnc_channel = MagicMock()
+ vnc_channel.identity = {"client_id": "abc"}
+
+ page = MagicMock()
+ page.url = "https://example.com/current"
+
+ channel = ExecutionChannel(vnc_channel=vnc_channel)
+ channel.url = "ws://cdp.example/devtools/browser/123"
+ channel.page = page
+
+ assert await channel.get_current_url() == "https://example.com/current"
+ assert channel.identity["cdp_url"] == "ws://cdp.example/devtools/browser/123"
+ assert "url" not in channel.identity
+
+ def test_normalize_url_passes_https(self) -> None:
+ assert ExecutionChannel._normalize_url("https://example.com/path") == "https://example.com/path"
+
+ def test_normalize_url_passes_http(self) -> None:
+ assert ExecutionChannel._normalize_url("http://example.com") == "http://example.com"
+
+ def test_normalize_url_prepends_https_for_bare_host(self) -> None:
+ assert ExecutionChannel._normalize_url("example.com") == "https://example.com"
+
+ def test_normalize_url_strips_whitespace(self) -> None:
+ assert ExecutionChannel._normalize_url(" https://example.com ") == "https://example.com"
+
+ def test_normalize_url_rejects_non_http_scheme(self) -> None:
+ with pytest.raises(ValueError):
+ ExecutionChannel._normalize_url("javascript:alert(1)")
+
+ def test_normalize_url_rejects_file_scheme(self) -> None:
+ with pytest.raises(ValueError):
+ ExecutionChannel._normalize_url("file:///etc/passwd")
+
+ @pytest.mark.parametrize("url", ["", " "])
+ def test_normalize_url_rejects_empty_url(self, url: str) -> None:
+ with pytest.raises(ValueError, match="URL must not be empty"):
+ ExecutionChannel._normalize_url(url)
+
+ def test_normalize_url_rejects_protocol_relative_url(self) -> None:
+ with pytest.raises(ValueError, match="explicit http\\(s\\) scheme or host"):
+ ExecutionChannel._normalize_url("//evil.com")
+
+ def test_origin_of_extracts_origin(self) -> None:
+ assert ExecutionChannel._origin_of("https://example.com/foo/bar?q=1") == "https://example.com"
+
+ def test_origin_of_includes_port(self) -> None:
+ assert ExecutionChannel._origin_of("http://localhost:8080/x") == "http://localhost:8080"
+
+ def test_origin_of_returns_empty_for_blank(self) -> None:
+ assert ExecutionChannel._origin_of("") == ""
+
+ def test_origin_of_returns_empty_for_about_blank(self) -> None:
+ # No scheme/netloc on "about:blank" -- treat as empty so CDP doesn't choke.
+ assert ExecutionChannel._origin_of("about:blank") == ""
+
+ @pytest.mark.asyncio
+ async def test_navigation_helpers_use_action_timeout(self) -> None:
+ vnc_channel = MagicMock()
+ vnc_channel.identity = {"client_id": "abc"}
+
+ page = MagicMock()
+ page.url = "https://example.com"
+ page.goto = AsyncMock()
+ page.reload = AsyncMock()
+ page.go_back = AsyncMock()
+ page.go_forward = AsyncMock()
+ page.context.new_cdp_session = AsyncMock()
+
+ channel = ExecutionChannel(vnc_channel=vnc_channel)
+ channel.page = page
+
+ await channel.navigate("example.com")
+ await channel.reload()
+ await channel.go_back()
+ await channel.go_forward()
+
+ timeout_kwargs = {
+ "wait_until": "domcontentloaded",
+ "timeout": settings.BROWSER_ACTION_TIMEOUT_MS,
+ }
+ page.goto.assert_awaited_once_with("https://example.com", **timeout_kwargs)
+ page.reload.assert_awaited_once_with(**timeout_kwargs)
+ page.go_back.assert_awaited_once_with(**timeout_kwargs)
+ page.go_forward.assert_awaited_once_with(**timeout_kwargs)
+
+
+class TestLocalExecutionChannel:
+ @pytest.mark.asyncio
+ async def test_evaluate_js_calls_page_evaluate_directly(self) -> None:
+ page = MagicMock()
+ page.url = "https://example.com"
+ page.context = MagicMock()
+ page.evaluate = AsyncMock(return_value="hello")
+
+ channel = LocalExecutionChannel(page=page)
+ result = await channel.evaluate_js("() => 'hello'")
+
+ assert result == "hello"
+ page.evaluate.assert_awaited_once_with("() => 'hello'", None)
+
+ @pytest.mark.asyncio
+ async def test_evaluate_js_forwards_arg(self) -> None:
+ page = MagicMock()
+ page.url = "https://example.com"
+ page.context = MagicMock()
+ page.evaluate = AsyncMock(return_value=None)
+
+ channel = LocalExecutionChannel(page=page)
+ await channel.evaluate_js("(text) => text", "pasted")
+
+ page.evaluate.assert_awaited_once_with("(text) => text", "pasted")
+
+ @pytest.mark.asyncio
+ async def test_paste_text_uses_keyboard_insert_text(self) -> None:
+ page = MagicMock()
+ page.url = "https://example.com"
+ page.context = MagicMock()
+ page.keyboard.insert_text = AsyncMock()
+ page.evaluate = AsyncMock()
+
+ channel = LocalExecutionChannel(page=page)
+ await channel.paste_text("pasted")
+
+ page.keyboard.insert_text.assert_awaited_once_with("pasted")
+ page.evaluate.assert_not_awaited()
+
+
+class TestMessageLoopUrlUpdates:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("payload", "method_name", "expected_kwargs", "current_url"),
+ [
+ ({"kind": "go-back"}, "go_back", {}, "https://example.com/previous"),
+ ({"kind": "go-forward"}, "go_forward", {}, "https://example.com/next"),
+ ({"kind": "reload", "hard": True}, "reload", {"hard": True}, "https://example.com/redirected"),
+ ],
+ )
+ async def test_browser_history_actions_push_current_url(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ payload: dict[str, object],
+ method_name: str,
+ expected_kwargs: dict[str, object],
+ current_url: str,
+ ) -> None:
+ websocket = MagicMock()
+ websocket.client_state = WebSocketState.CONNECTED
+ websocket.send_json = AsyncMock()
+ websocket.close = AsyncMock()
+
+ message_channel = MessageChannel(
+ client_id="client-1",
+ organization_id="org-1",
+ websocket=websocket,
+ )
+
+ drain_calls = 0
+
+ async def drain_once_then_close() -> list[dict[str, object]]:
+ nonlocal drain_calls
+ drain_calls += 1
+ if drain_calls == 1:
+ return [payload]
+ websocket.client_state = WebSocketState.DISCONNECTED
+ return []
+
+ message_channel.drain = drain_once_then_close # type: ignore[method-assign]
+
+ execute = AsyncMock()
+ execute.get_current_url = AsyncMock(return_value=current_url)
+
+ @asynccontextmanager
+ async def fake_execution_for_message_channel(_: MessageChannel) -> t.AsyncIterator[AsyncMock]:
+ yield execute
+
+ monkeypatch.setattr(
+ message_module,
+ "execution_for_message_channel",
+ fake_execution_for_message_channel,
+ )
+
+ await loop_stream_messages(message_channel)
+
+ method = getattr(execute, method_name)
+ method.assert_awaited_once_with(**expected_kwargs)
+ execute.get_current_url.assert_awaited_once_with()
+ websocket.send_json.assert_any_await({"kind": "browser-url", "url": current_url})
diff --git a/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py b/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py
new file mode 100644
index 000000000..8b0b197a9
--- /dev/null
+++ b/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py
@@ -0,0 +1,99 @@
+"""Tests for verify_browser_session — local-mode short-circuit (SKY-8017)."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.forge.sdk.routes.streaming.verify import verify_browser_session
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
+ PersistentBrowserSession,
+ PersistentBrowserSessionStatus,
+)
+
+
+def _make_session(*, browser_address: str | None, status: PersistentBrowserSessionStatus) -> PersistentBrowserSession:
+ now = datetime.now(timezone.utc)
+ return PersistentBrowserSession(
+ persistent_browser_session_id="bs_test",
+ organization_id="o_test",
+ status=status,
+ browser_address=browser_address,
+ created_at=now,
+ modified_at=now,
+ )
+
+
+class TestVerifyBrowserSessionLocalShortCircuit:
+ @pytest.mark.asyncio
+ async def test_short_circuits_in_cdp_mode_with_no_address(self) -> None:
+ # Local CDP mode: browser runs in-process, browser_address is never
+ # populated. Verify must not block on the address wait.
+ session = _make_session(
+ browser_address=None,
+ status=PersistentBrowserSessionStatus.running,
+ )
+
+ manager = MagicMock()
+ manager.get_session = AsyncMock(return_value=session)
+ manager.get_browser_address = AsyncMock(
+ side_effect=AssertionError("must not wait for address in cdp mode"),
+ )
+
+ with (
+ patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock,
+ patch("skyvern.forge.sdk.routes.streaming.verify.settings") as settings_mock,
+ ):
+ app_mock.PERSISTENT_SESSIONS_MANAGER = manager
+ settings_mock.BROWSER_STREAMING_MODE = "cdp"
+ result = await verify_browser_session("bs_test", "o_test")
+
+ assert result is not None
+ assert result.browser_address == ""
+ assert result.persistent_browser_session_id == "bs_test"
+ manager.get_browser_address.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_waits_for_address_in_non_cdp_mode(self) -> None:
+ session = _make_session(
+ browser_address=None,
+ status=PersistentBrowserSessionStatus.running,
+ )
+
+ manager = MagicMock()
+ manager.get_session = AsyncMock(return_value=session)
+ manager.get_browser_address = AsyncMock(return_value="ws://remote:9222")
+
+ with (
+ patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock,
+ patch("skyvern.forge.sdk.routes.streaming.verify.settings") as settings_mock,
+ ):
+ app_mock.PERSISTENT_SESSIONS_MANAGER = manager
+ settings_mock.BROWSER_STREAMING_MODE = "vnc"
+ result = await verify_browser_session("bs_test", "o_test")
+
+ assert result is not None
+ assert result.browser_address == "ws://remote:9222"
+ manager.get_browser_address.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_uses_existing_address_without_polling(self) -> None:
+ session = _make_session(
+ browser_address="ws://remote:9222",
+ status=PersistentBrowserSessionStatus.running,
+ )
+
+ manager = MagicMock()
+ manager.get_session = AsyncMock(return_value=session)
+ manager.get_browser_address = AsyncMock(
+ side_effect=AssertionError("must not wait when address present"),
+ )
+
+ with patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock:
+ app_mock.PERSISTENT_SESSIONS_MANAGER = manager
+ result = await verify_browser_session("bs_test", "o_test")
+
+ assert result is not None
+ assert result.browser_address == "ws://remote:9222"
diff --git a/tests/unit/test_webhook_delivery_retry.py b/tests/unit/test_webhook_delivery_retry.py
new file mode 100644
index 000000000..182dbc41a
--- /dev/null
+++ b/tests/unit/test_webhook_delivery_retry.py
@@ -0,0 +1,516 @@
+"""Tests for webhook delivery retry behavior."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, patch
+
+import httpx
+import pytest
+
+from skyvern.services.webhook_delivery import (
+ WEBHOOK_DELIVERY_MAX_ATTEMPTS,
+ WEBHOOK_DELIVERY_MAX_RETRY_AFTER_SECONDS,
+ deliver_webhook_with_retries,
+ is_retryable_status,
+)
+
+
+def _response(status_code: int, body: str = "", headers: dict[str, str] | None = None) -> httpx.Response:
+ return httpx.Response(status_code=status_code, content=body.encode("utf-8"), headers=headers or {})
+
+
+def fake_sleep(monkeypatch: pytest.MonkeyPatch) -> list[float]:
+ """Replace asyncio.sleep with a no-op recorder so tests don't actually wait."""
+ recorded: list[float] = []
+
+ async def _sleep(delay: float) -> None:
+ recorded.append(delay)
+
+ monkeypatch.setattr("skyvern.services.webhook_delivery.asyncio.sleep", _sleep)
+ return recorded
+
+
+def no_jitter(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Pin random.uniform to 0 so backoff delays are deterministic."""
+ monkeypatch.setattr("skyvern.services.webhook_delivery.random.uniform", lambda a, b: 0.0)
+
+
+async def test_returns_immediately_on_success(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(return_value=_response(200, "ok"))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 200
+ assert deliver.await_count == 1
+ assert fake_sleep == []
+
+
+async def test_retries_on_403_then_succeeds(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(side_effect=[_response(403, "forbidden"), _response(200, "ok")])
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 200
+ assert deliver.await_count == 2
+ assert len(fake_sleep) == 1
+
+
+async def test_retries_on_5xx_then_succeeds(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(side_effect=[_response(503, "unavailable"), _response(200)])
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 200
+ assert deliver.await_count == 2
+
+
+async def test_retries_on_429_then_succeeds(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(side_effect=[_response(429), _response(200)])
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 200
+ assert deliver.await_count == 2
+
+
+async def test_does_not_retry_on_400_bad_request(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(return_value=_response(400, "bad request"))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 400
+ assert deliver.await_count == 1
+ assert fake_sleep == []
+
+
+async def test_does_not_retry_on_401_unauthorized(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(return_value=_response(401))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 401
+ assert deliver.await_count == 1
+
+
+async def test_does_not_retry_on_404(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(return_value=_response(404))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 404
+ assert deliver.await_count == 1
+
+
+async def test_returns_final_failure_when_all_attempts_fail(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(return_value=_response(503, "still down"))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 503
+ assert deliver.await_count == WEBHOOK_DELIVERY_MAX_ATTEMPTS
+ assert len(fake_sleep) == WEBHOOK_DELIVERY_MAX_ATTEMPTS - 1
+
+
+async def test_retries_on_network_error_then_succeeds(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(side_effect=[httpx.ConnectError("conn refused"), _response(200)])
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 200
+ assert deliver.await_count == 2
+
+
+async def test_retries_on_timeout_then_succeeds(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(side_effect=[httpx.ReadTimeout("slow"), _response(200)])
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 200
+ assert deliver.await_count == 2
+
+
+async def test_retries_on_remote_protocol_error_then_succeeds(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(
+ side_effect=[
+ httpx.RemoteProtocolError("Server disconnected without sending a response"),
+ _response(200),
+ ],
+ )
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 200
+ assert deliver.await_count == 2
+
+
+async def test_raises_when_all_network_attempts_fail(fake_sleep: list[float]) -> None:
+ err = httpx.ConnectError("conn refused")
+ deliver = AsyncMock(side_effect=[err, err, err])
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ with pytest.raises(httpx.ConnectError):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert deliver.await_count == WEBHOOK_DELIVERY_MAX_ATTEMPTS
+
+
+async def test_backoff_is_exponential(fake_sleep: list[float], no_jitter: None) -> None:
+ deliver = AsyncMock(return_value=_response(503))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ base_delay_seconds=1.0,
+ )
+
+ assert fake_sleep == pytest.approx([1.0, 2.0])
+
+
+def test_is_retryable_status_covers_full_5xx_range_and_curated_4xx() -> None:
+ for code in (403, 408, 425, 429):
+ assert is_retryable_status(code)
+ for code in (500, 501, 502, 503, 504, 520, 522, 599):
+ assert is_retryable_status(code)
+ for code in (200, 201, 301, 400, 401, 404, 405, 410, 418, 422):
+ assert not is_retryable_status(code)
+
+
+async def test_retries_on_521_then_succeeds(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(side_effect=[_response(521, "origin down"), _response(200)])
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 200
+ assert deliver.await_count == 2
+
+
+async def test_retries_on_http_status_error_5xx_then_succeeds(fake_sleep: list[float]) -> None:
+ """NAT proxy client raises HTTPStatusError on proxy-side 5xx via raise_for_status()."""
+ request = httpx.Request("POST", "https://proxy.example/proxy/webhook")
+ err = httpx.HTTPStatusError("503", request=request, response=_response(503))
+ deliver = AsyncMock(side_effect=[err, _response(200)])
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert resp.status_code == 200
+ assert deliver.await_count == 2
+
+
+async def test_re_raises_http_status_error_on_non_retryable(fake_sleep: list[float]) -> None:
+ request = httpx.Request("POST", "https://proxy.example/proxy/webhook")
+ err = httpx.HTTPStatusError("400", request=request, response=_response(400))
+ deliver = AsyncMock(side_effect=err)
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ with pytest.raises(httpx.HTTPStatusError):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert deliver.await_count == 1
+ assert fake_sleep == []
+
+
+async def test_raises_last_http_status_error_after_exhaustion(fake_sleep: list[float]) -> None:
+ request = httpx.Request("POST", "https://proxy.example/proxy/webhook")
+ err = httpx.HTTPStatusError("502", request=request, response=_response(502))
+ deliver = AsyncMock(side_effect=[err, err, err])
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ with pytest.raises(httpx.HTTPStatusError):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+
+ assert deliver.await_count == WEBHOOK_DELIVERY_MAX_ATTEMPTS
+
+
+async def test_backoff_adds_jitter_within_base_window(fake_sleep: list[float], monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("skyvern.services.webhook_delivery.random.uniform", lambda a, b: 0.7)
+ deliver = AsyncMock(return_value=_response(503))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ base_delay_seconds=1.0,
+ )
+
+ # attempt 0 -> 1.0 * 2**0 + 0.7 = 1.7; attempt 1 -> 1.0 * 2**1 + 0.7 = 2.7
+ assert fake_sleep == pytest.approx([1.7, 2.7])
+
+
+async def test_backoff_jitter_keeps_delays_non_decreasing(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(return_value=_response(503))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ base_delay_seconds=1.0,
+ )
+
+ for attempt, delay in enumerate(fake_sleep):
+ base = 1.0 * (2**attempt)
+ assert base <= delay <= base + 1.0
+ for prev, curr in zip(fake_sleep, fake_sleep[1:]):
+ assert curr >= prev
+
+
+async def test_honors_numeric_retry_after_header(fake_sleep: list[float], no_jitter: None) -> None:
+ deliver = AsyncMock(
+ side_effect=[_response(429, headers={"Retry-After": "5"}), _response(200)],
+ )
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ base_delay_seconds=1.0,
+ )
+
+ assert resp.status_code == 200
+ assert fake_sleep == [5.0]
+
+
+async def test_caps_retry_after_at_max(fake_sleep: list[float], no_jitter: None) -> None:
+ deliver = AsyncMock(
+ side_effect=[
+ _response(429, headers={"Retry-After": str(int(WEBHOOK_DELIVERY_MAX_RETRY_AFTER_SECONDS * 10))}),
+ _response(200),
+ ],
+ )
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ base_delay_seconds=1.0,
+ )
+
+ assert fake_sleep == [WEBHOOK_DELIVERY_MAX_RETRY_AFTER_SECONDS]
+
+
+async def test_honors_http_date_retry_after(fake_sleep: list[float], no_jitter: None) -> None:
+ from datetime import datetime, timedelta, timezone
+
+ future = datetime.now(timezone.utc) + timedelta(seconds=4)
+ http_date = future.strftime("%a, %d %b %Y %H:%M:%S GMT")
+ deliver = AsyncMock(
+ side_effect=[_response(429, headers={"Retry-After": http_date}), _response(200)],
+ )
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ base_delay_seconds=1.0,
+ )
+
+ assert len(fake_sleep) == 1
+ assert 0.0 <= fake_sleep[0] <= WEBHOOK_DELIVERY_MAX_RETRY_AFTER_SECONDS
+
+
+async def test_ignores_unparseable_retry_after(fake_sleep: list[float], no_jitter: None) -> None:
+ deliver = AsyncMock(
+ side_effect=[_response(429, headers={"Retry-After": "soon"}), _response(200)],
+ )
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ base_delay_seconds=1.0,
+ )
+
+ assert fake_sleep == [1.0]
+
+
+async def test_max_attempts_one_returns_retryable_failure_without_retry(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(return_value=_response(503, "still down"))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ resp = await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ max_attempts=1,
+ )
+
+ assert resp.status_code == 503
+ assert deliver.await_count == 1
+ assert fake_sleep == []
+
+
+async def test_max_attempts_one_raises_network_error_without_retry(fake_sleep: list[float]) -> None:
+ deliver = AsyncMock(side_effect=httpx.ConnectError("down"))
+ with patch("skyvern.services.webhook_delivery.app.AGENT_FUNCTION.deliver_webhook", deliver):
+ with pytest.raises(httpx.ConnectError):
+ await deliver_webhook_with_retries(
+ url="https://example.com/hook",
+ payload="{}",
+ headers={},
+ timeout_seconds=30.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ max_attempts=1,
+ )
+
+ assert deliver.await_count == 1
+ assert fake_sleep == []
diff --git a/tests/unit/workflow/test_block_anti_bot_retry_skip.py b/tests/unit/workflow/test_block_anti_bot_retry_skip.py
new file mode 100644
index 000000000..08d5b5bdf
--- /dev/null
+++ b/tests/unit/workflow/test_block_anti_bot_retry_skip.py
@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
+from skyvern.forge.sdk.workflow.models.block import _should_skip_retry_on_anti_bot_detection
+
+
+def _make_failed_task(
+ *,
+ failure_reason: str | None = None,
+ failure_category: list[dict] | None = None,
+) -> Task:
+ now = datetime.now(timezone.utc)
+ return Task(
+ task_id="tsk_test",
+ organization_id="o_test",
+ url="https://example.com/login",
+ created_at=now,
+ modified_at=now,
+ status=TaskStatus.failed,
+ failure_reason=failure_reason,
+ failure_category=failure_category,
+ )
+
+
+class TestPersistedCategory:
+ def test_primary_anti_bot_category_skips_retry(self) -> None:
+ task = _make_failed_task(
+ failure_category=[
+ {"category": "ANTI_BOT_DETECTION", "confidence_float": 0.9, "reasoning": "captcha"},
+ ],
+ )
+ assert _should_skip_retry_on_anti_bot_detection(task) is True
+
+ def test_anti_bot_secondary_position_still_skips_retry(self) -> None:
+ task = _make_failed_task(
+ failure_category=[
+ {"category": "MAX_STEPS_EXCEEDED", "confidence_float": 0.9, "reasoning": "max"},
+ {"category": "ANTI_BOT_DETECTION", "confidence_float": 0.7, "reasoning": "captcha"},
+ ],
+ )
+ assert _should_skip_retry_on_anti_bot_detection(task) is True
+
+ def test_auth_failure_does_not_skip_retry(self) -> None:
+ task = _make_failed_task(
+ failure_category=[
+ {"category": "AUTH_FAILURE", "confidence_float": 0.9, "reasoning": "bad password"},
+ ],
+ )
+ assert _should_skip_retry_on_anti_bot_detection(task) is False
+
+ def test_persisted_primary_wins_over_reason_keywords(self) -> None:
+ """A persisted non-anti-bot primary must NOT be overridden by reason text.
+
+ Example: persisted AUTH_FAILURE whose reason happens to mention a captcha;
+ retry should still proceed because the persisted classification is the
+ authoritative signal.
+ """
+ task = _make_failed_task(
+ failure_category=[
+ {"category": "AUTH_FAILURE", "confidence_float": 0.9, "reasoning": "bad password"},
+ ],
+ failure_reason="Login failed; the page also showed a captcha challenge after submit",
+ )
+ assert _should_skip_retry_on_anti_bot_detection(task) is False
+
+
+class TestFallbackFromReason:
+ def test_keyword_reason_skips_retry_when_no_category(self) -> None:
+ task = _make_failed_task(failure_reason="Page blocked by captcha challenge")
+ assert _should_skip_retry_on_anti_bot_detection(task) is True
+
+ def test_managed_challenge_service_reason_skips_retry(self) -> None:
+ task = _make_failed_task(failure_reason="Bot detection service blocked access")
+ assert _should_skip_retry_on_anti_bot_detection(task) is True
+
+ def test_max_steps_with_captcha_in_reason_skips_retry(self) -> None:
+ """Exact ticket scenario for tasks without a persisted category list.
+
+ The agent stores max_steps failures as 'Reached the maximum steps (...)'.
+ That phrase ranks higher in the classifier than the captcha keyword,
+ so [0] is MAX_STEPS_EXCEEDED and [1] is ANTI_BOT_DETECTION. The fallback
+ must still recognize the anti-bot signal in any position.
+ """
+ task = _make_failed_task(
+ failure_reason=(
+ "Reached the maximum steps (10). Possible failure reasons: "
+ "the page presented a captcha challenge after the login form submit"
+ ),
+ )
+ assert _should_skip_retry_on_anti_bot_detection(task) is True
+
+ def test_benign_reason_does_not_skip_retry(self) -> None:
+ task = _make_failed_task(failure_reason="Could not find the submit button on the page")
+ assert _should_skip_retry_on_anti_bot_detection(task) is False
+
+ def test_no_reason_and_no_category_does_not_skip_retry(self) -> None:
+ task = _make_failed_task()
+ assert _should_skip_retry_on_anti_bot_detection(task) is False
+
+ def test_access_denied_with_auth_context_does_not_skip(self) -> None:
+ """Auth-context 'access denied' classifies as AUTH_FAILURE, not bot detection."""
+ task = _make_failed_task(
+ failure_reason="Access denied after login - user does not have permission",
+ )
+ assert _should_skip_retry_on_anti_bot_detection(task) is False