Skip to content

Automate browser based workflows with AI

[codex] Add workflow runtime limit frontend (#6248)

Suchintan3897cd8cfd2f
diff --git a/docs/assets/pr/cdp-browser-controls.png b/docs/assets/pr/cdp-browser-controls.png
deleted file mode 100644
index 84458e623..000000000
Binary files a/docs/assets/pr/cdp-browser-controls.png and /dev/null differ
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 37cf6334f..5eb5b0355 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -4,7 +4,7 @@
import _RFB from "@novnc/novnc/lib/rfb.js";
type RFB = _RFB;
const RFB = (_RFB as typeof _RFB & { default?: typeof _RFB }).default ?? _RFB;
-import { InfoCircledIcon } from "@radix-ui/react-icons";
+import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons";
import { useEffect, useState, useRef, useCallback } from "react";
import { useQuery } from "@tanstack/react-query";
@@ -826,40 +826,33 @@ function BrowserStream({
{isReady && isVisible && (
<div className="overlay z-10 flex items-center justify-center overflow-hidden">
{showControlButtons && (
- <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center bg-[radial-gradient(circle_at_center,rgba(15,23,42,0.22)_0%,rgba(15,23,42,0.12)_18%,transparent_42%)]">
+ <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">
<Button
- aria-label="Take control of this browser session"
onClick={() => {
setUserIsControlling(true);
}}
- className={cn(
- "control-button group pointer-events-auto h-10 rounded-full border border-white/15 bg-slate-950/90 px-5 text-sm font-semibold text-white shadow-xl shadow-slate-950/25 ring-1 ring-slate-950/10 backdrop-blur-sm transition duration-200 ease-out hover:-translate-y-0.5 hover:bg-slate-900 hover:shadow-2xl focus-visible:ring-2 focus-visible:ring-sky-400 focus-visible:ring-offset-2 motion-reduce:transition-none motion-reduce:hover:translate-y-0",
- {
- hide: userIsControlling,
- },
- )}
+ className={cn("control-button pointer-events-auto border", {
+ hide: userIsControlling,
+ })}
+ size="sm"
>
- Take the wheel
- <span
- aria-hidden="true"
- className="ml-2 transition-transform duration-200 group-hover:translate-x-0.5 motion-reduce:transition-none"
- >
- →
- </span>
+ <HandIcon className="mr-2 h-4 w-4" />
+ take control
</Button>
<Button
- aria-label="Stop controlling this browser session"
onClick={() => {
setUserIsControlling(false);
}}
className={cn(
- "control-button pointer-events-auto absolute bottom-0 h-9 rounded-full border border-white/15 bg-slate-950/90 px-4 text-sm font-semibold text-white shadow-xl shadow-slate-950/25 ring-1 ring-slate-950/10 backdrop-blur-sm transition duration-200 ease-out hover:-translate-y-0.5 hover:bg-slate-900 focus-visible:ring-2 focus-visible:ring-sky-400 focus-visible:ring-offset-2 motion-reduce:transition-none motion-reduce:hover:translate-y-0",
+ "control-button pointer-events-auto absolute bottom-0 border",
{
hide: !userIsControlling,
},
)}
+ size="sm"
>
- Release control
+ <ExitIcon className="mr-2 h-4 w-4" />
+ stop controlling
</Button>
</div>
)}
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
index b487d8a8d..bfbb84b1c 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
@@ -109,10 +109,8 @@ function BrowserSessionStream({
userIsControlling,
setUserIsControlling,
inputReady,
- browserCommandError,
containerRef,
handlers,
- browserControls,
} = useCdpInput({
inputWsUrl,
interactive,
@@ -238,11 +236,9 @@ function BrowserSessionStream({
userIsControlling={userIsControlling}
setUserIsControlling={setUserIsControlling}
inputReady={inputReady}
- browserCommandError={interactive ? browserCommandError : null}
containerRef={containerRef}
showControlButtons={showControlButtons}
handlers={handlers}
- browserControls={interactive ? browserControls : undefined}
currentUrl={currentUrl}
/>
);
diff --git a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx
index bfc65860a..b99bf31eb 100644
--- a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx
+++ b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx
@@ -1,13 +1,7 @@
-import { useEffect, useState, type FormEvent, type RefObject } from "react";
-import {
- ArrowLeftIcon,
- ArrowRightIcon,
- GlobeIcon,
- ReloadIcon,
-} from "@radix-ui/react-icons";
+import type { RefObject } from "react";
+import { GlobeIcon } from "@radix-ui/react-icons";
import { ZoomableImage } from "@/components/ZoomableImage";
import { Button } from "@/components/ui/button";
-import { Input } from "@/components/ui/input";
import { cn } from "@/util/utils";
interface InteractiveStreamViewProps {
@@ -17,7 +11,6 @@ interface InteractiveStreamViewProps {
userIsControlling: boolean;
setUserIsControlling: (v: boolean) => void;
inputReady: boolean;
- browserCommandError?: string | null;
containerRef: RefObject<HTMLDivElement>;
showControlButtons: boolean;
handlers: {
@@ -27,99 +20,14 @@ interface InteractiveStreamViewProps {
handleKeyDown: (e: React.KeyboardEvent) => void;
handleKeyUp: (e: React.KeyboardEvent) => void;
};
- browserControls?: {
- navigate: (url: string) => void;
- reload: () => void;
- goBack: () => void;
- goForward: () => void;
- };
currentUrl?: string;
}
-function UrlBar({
- url,
- inputReady,
- browserCommandError,
- browserControls,
-}: {
- url: string;
- inputReady: boolean;
- browserCommandError?: string | null;
- browserControls?: InteractiveStreamViewProps["browserControls"];
-}) {
- const [draftUrl, setDraftUrl] = useState(url);
- const controlsEnabled = inputReady && Boolean(browserControls);
-
- useEffect(() => {
- setDraftUrl(url);
- }, [url]);
-
- function handleSubmit(e: FormEvent<HTMLFormElement>) {
- e.preventDefault();
- e.stopPropagation();
- browserControls?.navigate(draftUrl);
- }
-
+function UrlBar({ url }: { url: string }) {
return (
- <div
- className="flex h-8 w-full items-center gap-1 rounded-t-md bg-slate-800 px-2 text-xs text-slate-300"
- onKeyDown={(e) => e.stopPropagation()}
- onKeyUp={(e) => e.stopPropagation()}
- onMouseDown={(e) => e.stopPropagation()}
- onMouseUp={(e) => e.stopPropagation()}
- >
- <Button
- type="button"
- variant="ghost"
- size="icon"
- className="h-6 w-6 flex-shrink-0 text-slate-300 hover:bg-slate-700 hover:text-white"
- disabled={!controlsEnabled}
- aria-label="Go back"
- title="Go back"
- onClick={() => browserControls?.goBack()}
- >
- <ArrowLeftIcon className="h-3.5 w-3.5" />
- </Button>
- <Button
- type="button"
- variant="ghost"
- size="icon"
- className="h-6 w-6 flex-shrink-0 text-slate-300 hover:bg-slate-700 hover:text-white"
- disabled={!controlsEnabled}
- aria-label="Go forward"
- title="Go forward"
- onClick={() => browserControls?.goForward()}
- >
- <ArrowRightIcon className="h-3.5 w-3.5" />
- </Button>
- <Button
- type="button"
- variant="ghost"
- size="icon"
- className="h-6 w-6 flex-shrink-0 text-slate-300 hover:bg-slate-700 hover:text-white"
- disabled={!controlsEnabled}
- aria-label="Reload"
- title="Reload"
- onClick={() => browserControls?.reload()}
- >
- <ReloadIcon className="h-3.5 w-3.5" />
- </Button>
- <form className="min-w-0 flex-1" onSubmit={handleSubmit}>
- <div className="relative">
- <GlobeIcon className="pointer-events-none absolute left-2 top-1/2 h-3 w-3 -translate-y-1/2 text-slate-400" />
- <Input
- value={draftUrl}
- disabled={!controlsEnabled}
- aria-label="Current browser URL"
- title={browserCommandError || draftUrl}
- onChange={(e) => setDraftUrl(e.target.value)}
- className={cn(
- "h-6 rounded border-slate-700 bg-slate-900 py-0 pl-7 pr-2 text-xs text-slate-200 shadow-none focus-visible:ring-slate-500",
- browserCommandError ? "border-red-500 text-red-200" : "",
- )}
- />
- </div>
- </form>
+ <div className="flex h-8 w-full items-center gap-2 rounded-t-md bg-slate-800 px-3 text-xs text-slate-300">
+ <GlobeIcon className="h-3 w-3 flex-shrink-0 text-slate-400" />
+ <span className="truncate">{url}</span>
</div>
);
}
@@ -131,15 +39,12 @@ function InteractiveStreamView({
userIsControlling,
setUserIsControlling,
inputReady,
- browserCommandError,
containerRef,
showControlButtons,
handlers,
- browserControls,
currentUrl,
}: InteractiveStreamViewProps) {
const imgDataUrl = `data:image/${streamFormat};base64,${streamImgSrc}`;
- const showUrlBar = Boolean(currentUrl) || Boolean(browserControls);
if (interactive) {
return (
@@ -150,45 +55,27 @@ function InteractiveStreamView({
onKeyDown={handlers.handleKeyDown}
onKeyUp={handlers.handleKeyUp}
>
- {showUrlBar && (
- <UrlBar
- url={currentUrl ?? ""}
- inputReady={inputReady}
- browserCommandError={browserCommandError}
- browserControls={browserControls}
- />
- )}
+ {currentUrl && <UrlBar url={currentUrl} />}
{showControlButtons && !userIsControlling && inputReady && (
- <div className="pointer-events-none absolute inset-0 z-10 flex items-center justify-center bg-[radial-gradient(circle_at_center,rgba(15,23,42,0.22)_0%,rgba(15,23,42,0.12)_18%,transparent_42%)]">
- <Button
- aria-label="Take control of this browser session"
- className="group pointer-events-auto h-10 rounded-full border border-white/15 bg-slate-950/90 px-5 text-sm font-semibold text-white shadow-xl shadow-slate-950/25 ring-1 ring-slate-950/10 backdrop-blur-sm transition duration-200 ease-out hover:-translate-y-0.5 hover:bg-slate-900 hover:shadow-2xl focus-visible:ring-2 focus-visible:ring-sky-400 focus-visible:ring-offset-2 motion-reduce:transition-none motion-reduce:hover:translate-y-0"
- onClick={() => setUserIsControlling(true)}
- >
- Take the wheel
- <span
- aria-hidden="true"
- className="ml-2 transition-transform duration-200 group-hover:translate-x-0.5 motion-reduce:transition-none"
- >
- →
- </span>
+ <div className="absolute inset-0 z-10 flex items-center justify-center">
+ <Button onClick={() => setUserIsControlling(true)}>
+ take control
</Button>
</div>
)}
{showControlButtons && userIsControlling && (
<Button
- aria-label="Stop controlling this browser session"
- className="absolute bottom-2 left-1/2 z-10 h-9 -translate-x-1/2 rounded-full border border-white/15 bg-slate-950/90 px-4 text-sm font-semibold text-white shadow-xl shadow-slate-950/25 ring-1 ring-slate-950/10 backdrop-blur-sm transition duration-200 ease-out hover:-translate-x-1/2 hover:-translate-y-0.5 hover:bg-slate-900 focus-visible:ring-2 focus-visible:ring-sky-400 focus-visible:ring-offset-2 motion-reduce:transition-none motion-reduce:hover:-translate-x-1/2 motion-reduce:hover:translate-y-0"
+ className="absolute bottom-2 left-1/2 z-10 -translate-x-1/2"
onClick={() => setUserIsControlling(false)}
>
- Release control
+ stop controlling
</Button>
)}
<img
src={imgDataUrl}
className={cn(
"w-full rounded-md object-contain",
- showUrlBar ? "h-[calc(100%-2rem)] rounded-t-none" : "h-full",
+ currentUrl ? "h-[calc(100%-2rem)]" : "h-full",
{ "cursor-default": userIsControlling },
)}
onMouseDown={handlers.handleMouseDown}
@@ -203,18 +90,11 @@ function InteractiveStreamView({
return (
<div className="h-full w-full">
- {showUrlBar && (
- <UrlBar
- url={currentUrl ?? ""}
- inputReady={inputReady}
- browserCommandError={browserCommandError}
- browserControls={browserControls}
- />
- )}
+ {currentUrl && <UrlBar url={currentUrl} />}
<ZoomableImage
src={imgDataUrl}
className={
- showUrlBar ? "h-[calc(100%-2rem)] rounded-b-md" : "rounded-md"
+ currentUrl ? "h-[calc(100%-2rem)] rounded-b-md" : "rounded-md"
}
/>
</div>
diff --git a/skyvern-frontend/src/routes/streaming/cdpInputUtils.test.ts b/skyvern-frontend/src/routes/streaming/cdpInputUtils.test.ts
deleted file mode 100644
index 427109320..000000000
--- a/skyvern-frontend/src/routes/streaming/cdpInputUtils.test.ts
+++ /dev/null
@@ -1,83 +0,0 @@
-import { describe, expect, test } from "vitest";
-import { buildKeyDownPayload, buildKeyUpPayload } from "./cdpInputUtils";
-
-function makeKeyboardEvent(
- key: string,
- code: string,
- modifiers: Partial<KeyboardEvent> = {},
-): KeyboardEvent {
- return new KeyboardEvent("keydown", {
- key,
- code,
- altKey: Boolean(modifiers.altKey),
- ctrlKey: Boolean(modifiers.ctrlKey),
- metaKey: Boolean(modifiers.metaKey),
- shiftKey: Boolean(modifiers.shiftKey),
- });
-}
-
-describe("CDP keyboard payloads", () => {
- test("sends non-printable keys as rawKeyDown with a Windows virtual key code", () => {
- expect(buildKeyDownPayload(makeKeyboardEvent("Enter", "Enter"))).toEqual({
- type: "keyEvent",
- eventType: "rawKeyDown",
- key: "Enter",
- code: "Enter",
- windowsVirtualKeyCode: 13,
- modifiers: 0,
- });
-
- expect(
- buildKeyDownPayload(makeKeyboardEvent("ArrowLeft", "ArrowLeft")),
- ).toMatchObject({
- eventType: "rawKeyDown",
- windowsVirtualKeyCode: 37,
- });
- });
-
- test("includes the Windows virtual key code on keyUp for non-printable keys", () => {
- expect(
- buildKeyUpPayload(makeKeyboardEvent("Backspace", "Backspace")),
- ).toEqual({
- type: "keyEvent",
- eventType: "keyUp",
- key: "Backspace",
- code: "Backspace",
- windowsVirtualKeyCode: 8,
- modifiers: 0,
- });
- });
-
- test("keeps printable characters as keyDown with text", () => {
- expect(buildKeyDownPayload(makeKeyboardEvent("a", "KeyA"))).toEqual({
- type: "keyEvent",
- eventType: "keyDown",
- key: "a",
- code: "KeyA",
- text: "a",
- windowsVirtualKeyCode: 65,
- modifiers: 0,
- });
- });
-
- test("keeps unmapped multi-character keys as keyDown", () => {
- expect(buildKeyDownPayload(makeKeyboardEvent("Dead", "Quote"))).toEqual({
- type: "keyEvent",
- eventType: "keyDown",
- key: "Dead",
- code: "Quote",
- modifiers: 0,
- });
- });
-
- test("preserves modifier bitmask", () => {
- expect(
- buildKeyDownPayload(
- makeKeyboardEvent("Tab", "Tab", { ctrlKey: true, shiftKey: true }),
- ),
- ).toMatchObject({
- modifiers: 10,
- windowsVirtualKeyCode: 9,
- });
- });
-});
diff --git a/skyvern-frontend/src/routes/streaming/cdpInputUtils.ts b/skyvern-frontend/src/routes/streaming/cdpInputUtils.ts
index 8c2552d0a..9937da288 100644
--- a/skyvern-frontend/src/routes/streaming/cdpInputUtils.ts
+++ b/skyvern-frontend/src/routes/streaming/cdpInputUtils.ts
@@ -15,102 +15,6 @@ export function getModifiers(
return m;
}
-const WINDOWS_VIRTUAL_KEY_CODES: Record<string, number> = {
- Backspace: 8,
- Tab: 9,
- Enter: 13,
- Shift: 16,
- Control: 17,
- Alt: 18,
- Pause: 19,
- CapsLock: 20,
- Escape: 27,
- " ": 32,
- PageUp: 33,
- PageDown: 34,
- End: 35,
- Home: 36,
- ArrowLeft: 37,
- ArrowUp: 38,
- ArrowRight: 39,
- ArrowDown: 40,
- Insert: 45,
- Delete: 46,
- Meta: 91,
- ContextMenu: 93,
- F1: 112,
- F2: 113,
- F3: 114,
- F4: 115,
- F5: 116,
- F6: 117,
- F7: 118,
- F8: 119,
- F9: 120,
- F10: 121,
- F11: 122,
- F12: 123,
- NumLock: 144,
- ScrollLock: 145,
-};
-
-export type CdpKeyEventPayload = {
- type: "keyEvent";
- eventType: "keyDown" | "keyUp" | "rawKeyDown";
- key: string;
- code: string;
- modifiers: number;
- text?: string;
- windowsVirtualKeyCode?: number;
-};
-
-type KeyboardEventLike = Pick<
- KeyboardEvent,
- "altKey" | "code" | "ctrlKey" | "key" | "metaKey" | "shiftKey"
->;
-
-function windowsVirtualKeyCodeForKey(key: string): number | undefined {
- if (/^[a-z]$/i.test(key)) {
- return key.toUpperCase().charCodeAt(0);
- }
- if (/^[0-9]$/.test(key)) {
- return key.charCodeAt(0);
- }
- return WINDOWS_VIRTUAL_KEY_CODES[key];
-}
-
-export function buildKeyDownPayload(e: KeyboardEventLike): CdpKeyEventPayload {
- const windowsVirtualKeyCode = windowsVirtualKeyCodeForKey(e.key);
- const isPrintableSingleCharacter = e.key.length === 1;
- const eventType =
- !isPrintableSingleCharacter && windowsVirtualKeyCode !== undefined
- ? "rawKeyDown"
- : "keyDown";
-
- return {
- type: "keyEvent",
- eventType,
- key: e.key,
- code: e.code,
- ...(isPrintableSingleCharacter ? { text: e.key } : {}),
- ...(windowsVirtualKeyCode !== undefined ? { windowsVirtualKeyCode } : {}),
- modifiers: getModifiers(e),
- };
-}
-
-export function buildKeyUpPayload(e: KeyboardEventLike): CdpKeyEventPayload {
- const windowsVirtualKeyCode = windowsVirtualKeyCodeForKey(e.key);
-
- return {
- type: "keyEvent",
- eventType: "keyUp",
- key: e.key,
- code: e.code,
- ...(windowsVirtualKeyCode !== undefined ? { windowsVirtualKeyCode } : {}),
- modifiers: getModifiers(e),
- };
-}
-
/**
* Map pixel coordinates from a rendered image back to viewport coordinates,
* accounting for object-contain letterboxing.
diff --git a/skyvern-frontend/src/routes/streaming/useCdpInput.ts b/skyvern-frontend/src/routes/streaming/useCdpInput.ts
index 3664b23d1..72f7119d7 100644
--- a/skyvern-frontend/src/routes/streaming/useCdpInput.ts
+++ b/skyvern-frontend/src/routes/streaming/useCdpInput.ts
@@ -4,8 +4,6 @@ import { getCredentialParam } from "@/util/env";
import { useClientIdStore } from "@/store/useClientIdStore";
import {
mouseButtonName,
- buildKeyDownPayload,
- buildKeyUpPayload,
getModifiers,
mapCoordinates,
mapMouseCoordinates,
@@ -24,7 +22,6 @@ interface UseCdpInputReturn {
userIsControlling: boolean;
setUserIsControlling: (v: boolean) => void;
inputReady: boolean;
- browserCommandError: string | null;
containerRef: React.RefObject<HTMLDivElement>;
handlers: {
handleMouseDown: (e: React.MouseEvent<HTMLImageElement>) => void;
@@ -33,12 +30,6 @@ interface UseCdpInputReturn {
handleKeyDown: (e: React.KeyboardEvent) => void;
handleKeyUp: (e: React.KeyboardEvent) => void;
};
- browserControls: {
- navigate: (url: string) => void;
- reload: () => void;
- goBack: () => void;
- goForward: () => void;
- };
}
export function useCdpInput({
@@ -49,9 +40,6 @@ export function useCdpInput({
}: UseCdpInputOptions): UseCdpInputReturn {
const [userIsControlling, setUserIsControlling] = useState(false);
const [inputReady, setInputReady] = useState(false);
- const [browserCommandError, setBrowserCommandError] = useState<string | null>(
- null,
- );
const credentialGetter = useCredentialGetter();
const clientId = useClientIdStore((s) => s.clientId);
@@ -105,8 +93,6 @@ export function useCdpInput({
if (userIsControllingRef.current) {
ws.send(JSON.stringify({ kind: "take-control" }));
}
- } else if (msg.kind === "browserCommandResult") {
- setBrowserCommandError(msg.ok ? null : String(msg.error || ""));
}
} catch {
// ignore non-JSON messages
@@ -251,44 +237,6 @@ export function useCdpInput({
ws.send(JSON.stringify(payload));
}, []);
- const sendBrowserCommand = useCallback((payload: Record<string, unknown>) => {
- const ws = inputSocketRef.current;
- if (!ws || ws.readyState !== WebSocket.OPEN) {
- setBrowserCommandError("Browser controls are not connected.");
- return;
- }
- setBrowserCommandError(null);
- ws.send(JSON.stringify({ type: "browserCommand", ...payload }));
- }, []);
-
- const navigate = useCallback(
- (rawUrl: string) => {
- const url = rawUrl.trim();
- if (!url) {
- setBrowserCommandError("URL must not be empty.");
- return;
- }
- if (url.toLowerCase().startsWith("chrome://")) {
- setBrowserCommandError("chrome:// URLs are not allowed.");
- return;
- }
- sendBrowserCommand({ command: "navigate", url });
- },
- [sendBrowserCommand],
- );
-
- const reload = useCallback(() => {
- sendBrowserCommand({ command: "reload" });
- }, [sendBrowserCommand]);
-
- const goBack = useCallback(() => {
- sendBrowserCommand({ command: "goBack" });
- }, [sendBrowserCommand]);
-
- const goForward = useCallback(() => {
- sendBrowserCommand({ command: "goForward" });
- }, [sendBrowserCommand]);
-
const handleMouseDown = useCallback(
(e: React.MouseEvent<HTMLImageElement>) => {
if (!interactive || !userIsControlling) return;
@@ -368,7 +316,14 @@ export function useCdpInput({
(e: React.KeyboardEvent) => {
if (!interactive || !userIsControlling) return;
e.preventDefault();
- sendInputEvent(buildKeyDownPayload(e.nativeEvent));
+ sendInputEvent({
+ type: "keyEvent",
+ eventType: "keyDown",
+ key: e.key,
+ code: e.code,
+ text: e.key.length === 1 ? e.key : "",
+ modifiers: getModifiers(e),
+ });
},
[interactive, userIsControlling, sendInputEvent],
);
@@ -377,7 +332,13 @@ export function useCdpInput({
(e: React.KeyboardEvent) => {
if (!interactive || !userIsControlling) return;
e.preventDefault();
- sendInputEvent(buildKeyUpPayload(e.nativeEvent));
+ sendInputEvent({
+ type: "keyEvent",
+ eventType: "keyUp",
+ key: e.key,
+ code: e.code,
+ modifiers: getModifiers(e),
+ });
},
[interactive, userIsControlling, sendInputEvent],
);
@@ -386,7 +347,6 @@ export function useCdpInput({
userIsControlling,
setUserIsControlling,
inputReady,
- browserCommandError,
containerRef,
handlers: {
handleMouseDown,
@@ -395,11 +355,5 @@ export function useCdpInput({
handleKeyDown,
handleKeyUp,
},
- browserControls: {
- navigate,
- reload,
- goBack,
- goForward,
- },
};
}
diff --git a/skyvern-frontend/src/routes/workflows/components/WorkflowVisualComparisonDrawer.tsx b/skyvern-frontend/src/routes/workflows/components/WorkflowVisualComparisonDrawer.tsx
index c35fe3198..e709034cc 100644
--- a/skyvern-frontend/src/routes/workflows/components/WorkflowVisualComparisonDrawer.tsx
+++ b/skyvern-frontend/src/routes/workflows/components/WorkflowVisualComparisonDrawer.tsx
@@ -106,6 +106,7 @@ function getWorkflowElements(version: WorkflowVersion) {
browserProfileId: version.browser_profile_id ?? null,
model: version.model,
maxScreenshotScrolls: version.max_screenshot_scrolls || 3,
+ maxElapsedTimeMinutes: version.max_elapsed_time_minutes ?? null,
extraHttpHeaders: version.extra_http_headers
? JSON.stringify(version.extra_http_headers)
: null,
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 1306ba3f5..b6f6d1f1a 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -791,6 +791,8 @@ export function WorkflowCopilotChat({
browser_profile_id: saveData.settings.browserProfileId,
model: saveData.settings.model,
max_screenshot_scrolls: saveData.settings.maxScreenshotScrolls,
+ max_elapsed_time_minutes:
+ saveData.settings.maxElapsedTimeMinutes ?? null,
totp_verification_url: saveData.workflow.totp_verification_url,
extra_http_headers: extraHttpHeaders,
run_with: saveData.settings.runWith,
diff --git a/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx b/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx
index 5cc8c0017..8182deb06 100644
--- a/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx
+++ b/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx
@@ -80,6 +80,7 @@ function Debugger() {
webhookCallbackUrl: workflow.webhook_callback_url,
model: workflow.model,
maxScreenshotScrolls: workflow.max_screenshot_scrolls,
+ maxElapsedTimeMinutes: workflow.max_elapsed_time_minutes ?? null,
extraHttpHeaders: workflow.extra_http_headers
? JSON.stringify(workflow.extra_http_headers)
: null,
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
index ba61effbe..2639d73b3 100644
--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
@@ -82,6 +82,7 @@ function WorkflowEditor() {
webhookCallbackUrl: workflow.webhook_callback_url,
model: workflow.model,
maxScreenshotScrolls: workflow.max_screenshot_scrolls,
+ maxElapsedTimeMinutes: workflow.max_elapsed_time_minutes ?? null,
extraHttpHeaders: workflow.extra_http_headers
? JSON.stringify(workflow.extra_http_headers)
: null,
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index d12ed60ce..8b6c19440 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -1240,6 +1240,7 @@ function Workspace({
browserProfileId: workflowData.browser_profile_id ?? null,
model: workflowData.model ?? null,
maxScreenshotScrolls: workflowData.max_screenshot_scrolls || 3,
+ maxElapsedTimeMinutes: workflowData.max_elapsed_time_minutes ?? null,
extraHttpHeaders: workflowData.extra_http_headers
? JSON.stringify(workflowData.extra_http_headers)
: null,
@@ -1308,6 +1309,7 @@ function Workspace({
browserProfileId: selectedVersion.browser_profile_id ?? null,
model: selectedVersion.model,
maxScreenshotScrolls: selectedVersion.max_screenshot_scrolls || 3,
+ maxElapsedTimeMinutes: selectedVersion.max_elapsed_time_minutes ?? null,
extraHttpHeaders: selectedVersion.extra_http_headers
? JSON.stringify(selectedVersion.extra_http_headers)
: null,
@@ -2121,6 +2123,8 @@ function Workspace({
totp_verification_url: saveData.workflow.totp_verification_url,
totp_identifier: null,
max_screenshot_scrolls: saveData.settings.maxScreenshotScrolls,
+ max_elapsed_time_minutes:
+ saveData.settings.maxElapsedTimeMinutes ?? null,
status: saveData.workflow.status,
created_at: new Date().toISOString(),
modified_at: new Date().toISOString(),
diff --git a/skyvern-frontend/src/routes/workflows/editor/nestedConditionalSave.test.ts b/skyvern-frontend/src/routes/workflows/editor/nestedConditionalSave.test.ts
index 7c9446594..5a7b8bf35 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nestedConditionalSave.test.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nestedConditionalSave.test.ts
@@ -23,6 +23,7 @@ const DEFAULT_SETTINGS: WorkflowSettings = {
browserProfileId: null,
model: null,
maxScreenshotScrolls: null,
+ maxElapsedTimeMinutes: null,
extraHttpHeaders: null,
cdpConnectHeaders: null,
runWith: "code",
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/WorkflowSettingsEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/WorkflowSettingsEditor.tsx
index 68621304b..0d0e5119a 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/WorkflowSettingsEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/WorkflowSettingsEditor.tsx
@@ -400,6 +400,7 @@ function WorkflowSettingsEditorBody({
<Input
value={data.maxScreenshotScrolls ?? ""}
placeholder={`Default: ${MAX_SCREENSHOT_SCROLLS_DEFAULT}`}
+ type="number"
onChange={(event) => {
const value =
event.target.value === "" ? null : Number(event.target.value);
@@ -407,6 +408,29 @@ function WorkflowSettingsEditorBody({
}}
/>
</div>
+ <div className="space-y-2">
+ <div className="flex items-center gap-2">
+ <Label>Max Run Time (minutes)</Label>
+ <HelpTooltip content="Times out this workflow after the configured elapsed runtime. Maximum runtime is 4 hours (240 minutes)." />
+ </div>
+ <Input
+ value={data.maxElapsedTimeMinutes ?? ""}
+ placeholder="Default: 4 hours"
+ type="number"
+ min={1}
+ max={240}
+ step={1}
+ onChange={(event) => {
+ const rawValue = event.target.value;
+ const parsedValue = Number(rawValue);
+ const value =
+ rawValue === "" || !Number.isFinite(parsedValue)
+ ? null
+ : Math.min(240, Math.max(1, Math.trunc(parsedValue)));
+ update({ maxElapsedTimeMinutes: value });
+ }}
+ />
+ </div>
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Execute on Any Outcome</Label>
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts
index 8f34aa437..742e2e799 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts
@@ -11,6 +11,7 @@ export type WorkflowStartNodeData = {
browserProfileId: string | null;
model: WorkflowModel | null;
maxScreenshotScrolls: number | null;
+ maxElapsedTimeMinutes: number | null;
extraHttpHeaders: string | Record<string, unknown> | null;
cdpConnectHeaders: string | Record<string, unknown> | null;
editable: boolean;
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/WorkflowSettingsBlockForm.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/WorkflowSettingsBlockForm.tsx
index aad9e3824..15533b71b 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/WorkflowSettingsBlockForm.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/WorkflowSettingsBlockForm.tsx
@@ -40,6 +40,7 @@ function WorkflowSettingsBlockFormBody({
persistBrowserSession: data.persistBrowserSession,
extraHttpHeaders: data.extraHttpHeaders,
maxScreenshotScrolls: data.maxScreenshotScrolls,
+ maxElapsedTimeMinutes: data.maxElapsedTimeMinutes,
finallyBlockLabel: data.finallyBlockLabel,
workflowSystemPrompt: data.workflowSystemPrompt,
};
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowComparisonPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowComparisonPanel.tsx
index ad4429891..20f76541b 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowComparisonPanel.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowComparisonPanel.tsx
@@ -141,6 +141,7 @@ function getWorkflowElements(version: WorkflowVersion) {
browserProfileId: version.browser_profile_id ?? null,
model: version.model,
maxScreenshotScrolls: version.max_screenshot_scrolls || 3,
+ maxElapsedTimeMinutes: version.max_elapsed_time_minutes ?? null,
extraHttpHeaders: version.extra_http_headers
? JSON.stringify(version.extra_http_headers)
: null,
diff --git a/skyvern-frontend/src/routes/workflows/editor/sortable/perf.test.ts b/skyvern-frontend/src/routes/workflows/editor/sortable/perf.test.ts
index 9a5ff3362..eea5cbe66 100644
--- a/skyvern-frontend/src/routes/workflows/editor/sortable/perf.test.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/sortable/perf.test.ts
@@ -52,6 +52,7 @@ const DEFAULT_SETTINGS: WorkflowSettings = {
browserProfileId: null,
model: null,
maxScreenshotScrolls: null,
+ maxElapsedTimeMinutes: null,
extraHttpHeaders: null,
cdpConnectHeaders: null,
runWith: "code",
diff --git a/skyvern-frontend/src/routes/workflows/editor/sortable/roundTrip.test.ts b/skyvern-frontend/src/routes/workflows/editor/sortable/roundTrip.test.ts
index d6eb17539..cb8e028ce 100644
--- a/skyvern-frontend/src/routes/workflows/editor/sortable/roundTrip.test.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/sortable/roundTrip.test.ts
@@ -42,6 +42,7 @@ const DEFAULT_SETTINGS: WorkflowSettings = {
browserProfileId: null,
model: null,
maxScreenshotScrolls: null,
+ maxElapsedTimeMinutes: null,
extraHttpHeaders: null,
cdpConnectHeaders: null,
runWith: "code",
diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
index 3763b041f..f75d41ff1 100644
--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
@@ -1959,6 +1959,7 @@ function getElements(
webhookCallbackUrl: settings.webhookCallbackUrl ?? "",
model: settings.model,
maxScreenshotScrolls: settings.maxScreenshotScrolls,
+ maxElapsedTimeMinutes: settings.maxElapsedTimeMinutes ?? null,
extraHttpHeaders: settings.extraHttpHeaders,
cdpConnectHeaders: settings.cdpConnectHeaders,
editable,
@@ -3335,6 +3336,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {
webhookCallbackUrl: null,
model: null,
maxScreenshotScrolls: null,
+ maxElapsedTimeMinutes: null,
extraHttpHeaders: null,
cdpConnectHeaders: null,
runWith: "code",
@@ -3362,6 +3364,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {
webhookCallbackUrl: data.webhookCallbackUrl,
model: data.model,
maxScreenshotScrolls: data.maxScreenshotScrolls,
+ maxElapsedTimeMinutes: data.maxElapsedTimeMinutes,
extraHttpHeaders:
data.extraHttpHeaders && typeof data.extraHttpHeaders === "object"
? JSON.stringify(data.extraHttpHeaders)
@@ -4457,6 +4460,7 @@ function convert(workflow: WorkflowApiResponse): WorkflowCreateYAMLRequest {
model: workflow.model,
totp_verification_url: workflow.totp_verification_url,
max_screenshot_scrolls: workflow.max_screenshot_scrolls,
+ max_elapsed_time_minutes: workflow.max_elapsed_time_minutes,
extra_http_headers: workflow.extra_http_headers,
workflow_definition: {
version: workflowDefinitionVersion,
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
index 18284d9a3..412d6e867 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
@@ -642,6 +642,7 @@ export type WorkflowApiResponse = {
totp_verification_url: string | null;
totp_identifier: string | null;
max_screenshot_scrolls: number | null;
+ max_elapsed_time_minutes: number | null;
status: string | null;
created_at: string;
modified_at: string;
@@ -664,6 +665,7 @@ export type WorkflowSettings = {
browserProfileId: string | null;
model: WorkflowModel | null;
maxScreenshotScrolls: number | null;
+ maxElapsedTimeMinutes: number | null;
extraHttpHeaders: string | null;
cdpConnectHeaders: string | null;
runWith: string; // 'agent' or 'code'
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
index 44c160775..21edfdef3 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
@@ -13,6 +13,7 @@ export type WorkflowCreateYAMLRequest = {
workflow_definition: WorkflowDefinitionYAML;
is_saved_task?: boolean;
max_screenshot_scrolls?: number | null;
+ max_elapsed_time_minutes?: number | null;
extra_http_headers?: Record<string, string> | null;
cdp_connect_headers?: Record<string, string> | null;
status?: string | null;
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx
index 31ceead26..9284ba720 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx
@@ -15,6 +15,7 @@ vi.mock("./WorkflowRunHumanInteraction", () => ({
}));
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
+import { MemoryRouter } from "react-router-dom";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ActionTypes, Status, type ActionsApiResponse } from "@/api/types";
@@ -137,6 +138,26 @@ describe("WorkflowRunBlockDetail router", () => {
expect(screen.getByText("https://example.test/report")).toBeDefined();
});
+ it("shows a diagnostics link in the inspector for any block with a task id", () => {
+ const block = buildBlock({
+ block_type: "file_download",
+ task_id: "tsk_123",
+ });
+
+ render(
+ <MemoryRouter>
+ <WorkflowRunBlockDetail activeItem={block} timeline={[]} />
+ </MemoryRouter>,
+ );
+
+ const diagnosticsLink = screen.getByRole("link", {
+ name: /diagnostics/i,
+ });
+ expect(diagnosticsLink.getAttribute("href")).toBe(
+ "/tasks/tsk_123/diagnostics",
+ );
+ });
+
it("renders the conditional detail (branch evaluation) for a conditional block", () => {
const block = buildBlock({
block_type: "conditional",
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx
index ded4e53f0..78e6a0ca5 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx
@@ -22,7 +22,6 @@ type StreamMessage = {
format?: string;
viewport_width?: number;
viewport_height?: number;
- url?: string;
};
const STARTING_DIAGNOSTIC: StreamDiagnostic = {
@@ -94,7 +93,6 @@ function WorkflowRunStream({
const [streamFormat, setStreamFormat] = useState<string>("png");
const [viewportWidth, setViewportWidth] = useState(1280);
const [viewportHeight, setViewportHeight] = useState(720);
- const [currentUrl, setCurrentUrl] = useState("");
const [diagnostic, setDiagnostic] =
useState<StreamDiagnostic>(STARTING_DIAGNOSTIC);
const showStream =
@@ -116,10 +114,8 @@ function WorkflowRunStream({
userIsControlling,
setUserIsControlling,
inputReady,
- browserCommandError,
containerRef,
handlers,
- browserControls,
} = useCdpInput({
inputWsUrl,
interactive,
@@ -132,7 +128,6 @@ function WorkflowRunStream({
return;
}
setDiagnostic(STARTING_DIAGNOSTIC);
- setCurrentUrl("");
hasFrameRef.current = false;
let cancelled = false;
@@ -172,9 +167,6 @@ function WorkflowRunStream({
if (message.viewport_height) {
setViewportHeight(message.viewport_height);
}
- if (message.url !== undefined) {
- setCurrentUrl(message.url);
- }
if (!message.screenshot && message.status) {
setDiagnostic(diagnosticForStatus(message.status));
}
@@ -297,12 +289,9 @@ function WorkflowRunStream({
userIsControlling={userIsControlling}
setUserIsControlling={setUserIsControlling}
inputReady={inputReady}
- browserCommandError={interactive ? browserCommandError : null}
containerRef={containerRef}
showControlButtons={showControlButtons}
handlers={handlers}
- browserControls={interactive ? browserControls : undefined}
- currentUrl={currentUrl}
/>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx
index ff66608f7..866ddb8b4 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx
@@ -1,5 +1,10 @@
-import { ChevronDownIcon, ChevronRightIcon } from "@radix-ui/react-icons";
+import {
+ ChevronDownIcon,
+ ChevronRightIcon,
+ ExternalLinkIcon,
+} from "@radix-ui/react-icons";
import { useEffect, useMemo, useState } from "react";
+import { Link } from "react-router-dom";
import { Input } from "@/components/ui/input";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
@@ -410,6 +415,20 @@ function BlockInspector({ block }: { block: WorkflowRunBlock }) {
>
Outputs
</TabsTrigger>
+ {block.task_id && (
+ <Link
+ to={`/tasks/${block.task_id}/diagnostics`}
+ title="Go to diagnostics"
+ onClick={(event) => event.stopPropagation()}
+ className={cn(
+ triggerClassName,
+ "inline-flex items-center gap-1.5",
+ )}
+ >
+ <ExternalLinkIcon className="size-3.5" />
+ <span>Diagnostics</span>
+ </Link>
+ )}
</TabsList>
<TabsContent value="summary" className="m-0">
<FieldList
diff --git a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
index 2916eb385..5890504e2 100644
--- a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
+++ b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
@@ -183,6 +183,8 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {
browser_profile_id: saveData.settings.browserProfileId,
model: saveData.settings.model,
max_screenshot_scrolls: saveData.settings.maxScreenshotScrolls,
+ max_elapsed_time_minutes:
+ saveData.settings.maxElapsedTimeMinutes ?? null,
totp_verification_url: saveData.workflow.totp_verification_url,
extra_http_headers: extraHttpHeaders,
cdp_connect_headers: cdpConnectHeaders,
diff --git a/skyvern/forge/sdk/routes/debug_sessions.py b/skyvern/forge/sdk/routes/debug_sessions.py
index 1cef17bf7..9dba437f0 100644
--- a/skyvern/forge/sdk/routes/debug_sessions.py
+++ b/skyvern/forge/sdk/routes/debug_sessions.py
@@ -16,7 +16,6 @@ from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
from skyvern.forge.sdk.services import org_auth_service
from skyvern.schemas.runs import ProxyLocation
-from skyvern.webeye.vnc_streaming import browser_session_supports_vnc_streaming
LOG = structlog.get_logger()
@@ -246,10 +245,9 @@ async def new_debug_session(
organization_id=current_org.organization_id,
user_id=current_user_id,
workflow_permanent_id=workflow_permanent_id,
- vnc_streaming_supported=browser_session_supports_vnc_streaming(
- browser_address=new_browser_session.browser_address,
- ip_address=new_browser_session.ip_address,
- ),
+ vnc_streaming_supported=True if new_browser_session.ip_address else False,
+ # NOTE(jdo:streaming-local-dev)
+ # vnc_streaming_supported=True,
)
LOG.info(
diff --git a/skyvern/forge/sdk/routes/streaming/cdp_input.py b/skyvern/forge/sdk/routes/streaming/cdp_input.py
index 5c0f7d3d0..772967d6e 100644
--- a/skyvern/forge/sdk/routes/streaming/cdp_input.py
+++ b/skyvern/forge/sdk/routes/streaming/cdp_input.py
@@ -5,17 +5,14 @@ CDP input channel for interactive browser control via Chrome DevTools Protocol.
import asyncio
import dataclasses
import json
-import re
import time
import typing as t
-from urllib.parse import urlparse
import structlog
from fastapi import WebSocket, WebSocketDisconnect
-from playwright.async_api import CDPSession, Page
+from playwright.async_api import CDPSession
from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK
-from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router
from skyvern.forge.sdk.routes.streaming.auth import auth, require_client_id
@@ -40,8 +37,6 @@ _MAX_KEY_LEN = 32
_MAX_CODE_LEN = 32
_MODIFIER_MASK = 0xF
_MAX_VK_CODE = 0xFE
-_MAX_NAVIGATION_URL_LEN = 2048
-_BARE_HOST_PORT_RE = re.compile(r"^[A-Za-z0-9.-]+:\d+(?:[/?#].*)?$")
@dataclasses.dataclass
@@ -201,86 +196,10 @@ async def _dispatch_event(
)
-def _normalize_navigation_url(raw_url: str) -> str:
- candidate = raw_url.strip()
- if not candidate:
- raise ValueError("URL must not be empty")
- if len(candidate) > _MAX_NAVIGATION_URL_LEN:
- raise ValueError("URL is too long")
- if candidate.lower().startswith("chrome://"):
- raise ValueError("chrome:// URLs are not allowed")
- if candidate.startswith("//"):
- raise ValueError("URL must include an explicit http(s) scheme or host")
- if _BARE_HOST_PORT_RE.match(candidate):
- return f"http://{candidate}"
-
- parsed = urlparse(candidate)
- if parsed.scheme in ("http", "https"):
- return candidate
- if parsed.scheme:
- raise ValueError(f"refusing to navigate to non-http(s) scheme: {parsed.scheme}")
- return f"https://{candidate}"
-
-
-async def _dispatch_browser_command(
- websocket: WebSocket,
- page: Page,
- msg: dict,
- log_id_key: str,
- log_id_value: str,
-) -> None:
- command = msg.get("command")
- try:
- if command == "reload":
- await page.reload(wait_until="domcontentloaded", timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
- elif command == "goBack":
- await page.go_back(wait_until="domcontentloaded", timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
- elif command == "goForward":
- await page.go_forward(wait_until="domcontentloaded", timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
- elif command == "navigate":
- raw_url = msg.get("url")
- if not isinstance(raw_url, str):
- raise ValueError("URL must be a string")
- await page.goto(
- _normalize_navigation_url(raw_url),
- wait_until="domcontentloaded",
- timeout=settings.BROWSER_ACTION_TIMEOUT_MS,
- )
- else:
- raise ValueError("unsupported browser command")
- except Exception as exc:
- LOG.warning(
- "CDP input: browser command failed",
- **{log_id_key: log_id_value},
- command=command,
- error=str(exc),
- )
- await websocket.send_json(
- {
- "kind": "browserCommandResult",
- "ok": False,
- "command": command,
- "error": str(exc),
- "url": page.url,
- }
- )
- return
-
- await websocket.send_json(
- {
- "kind": "browserCommandResult",
- "ok": True,
- "command": command,
- "url": page.url,
- }
- )
-
-
async def _run_input_loop(
websocket: WebSocket,
channel: CdpInputChannel,
cdp_session: CDPSession,
- page: Page,
log_id_key: str,
log_id_value: str,
) -> None:
@@ -307,9 +226,6 @@ async def _run_input_loop(
channel.interactor = "agent"
LOG.info("CDP input: cede-control received", **{log_id_key: log_id_value}, client_id=channel.client_id)
continue
- if kind == "browserCommand":
- await _dispatch_browser_command(websocket, page, msg, log_id_key, log_id_value)
- continue
if channel.interactor != "user":
if dropped_log_count < 5:
@@ -398,7 +314,7 @@ async def cdp_input_stream(
LOG.info("CDP input channel ready", workflow_run_id=workflow_run_id, client_id=client_id)
await websocket.send_json({"kind": "ready"})
- await _run_input_loop(websocket, channel, cdp_session, page, "workflow_run_id", workflow_run_id)
+ await _run_input_loop(websocket, channel, cdp_session, "workflow_run_id", workflow_run_id)
except ConnectionClosedOK:
LOG.info("CDP input: WS closed cleanly", workflow_run_id=workflow_run_id)
@@ -474,7 +390,7 @@ async def cdp_input_browser_session_stream(
LOG.info("CDP input channel ready", browser_session_id=browser_session_id, client_id=client_id)
await websocket.send_json({"kind": "ready"})
- await _run_input_loop(websocket, channel, cdp_session, page, "browser_session_id", browser_session_id)
+ await _run_input_loop(websocket, channel, cdp_session, "browser_session_id", browser_session_id)
except ConnectionClosedOK:
LOG.info("CDP input: WS closed cleanly", browser_session_id=browser_session_id)
diff --git a/skyvern/forge/sdk/routes/streaming/channels/vnc.py b/skyvern/forge/sdk/routes/streaming/channels/vnc.py
index 680d27a47..809887bb7 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/vnc.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/vnc.py
@@ -20,6 +20,7 @@ import asyncio
import dataclasses
import typing as t
from enum import IntEnum
+from urllib.parse import urlparse
import structlog
import websockets
@@ -48,7 +49,6 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import AddressablePer
from skyvern.forge.sdk.schemas.tasks import Task
from skyvern.forge.sdk.utils.aio import collect
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun
-from skyvern.webeye.vnc_streaming import build_vnc_streaming_url
LOG = structlog.get_logger()
@@ -281,6 +281,43 @@ async def ask_for_clipboard(vnc_channel: VncChannel) -> None:
LOG.exception(f"{class_name} Failed to ask for clipboard via CDP", **vnc_channel.identity)
+# TODO(benji): I hate this function. It's messy and gross. Once we remove v1,
+# we should clean this up.
+def _build_vnc_url_from_browser_address(browser_address: str) -> str | None:
+ """
+ Build a routed VNC URL from a V2 K8s routed browser_address.
+
+ V2 K8s routed browser_address format:
+ wss://{domain}/{session_id}/{token}/devtools/browser/{browser_id}
+
+ Returns VNC URL in format:
+ wss://{domain}/vnc/{session_id}/{token}
+
+ Returns None if browser_address is not a V2 routed URL.
+ """
+ if not browser_address:
+ return None
+
+ parsed = urlparse(browser_address)
+
+ # Check if this looks like a V2 routed URL (wss:// with token in path)
+ if parsed.scheme not in ("wss", "ws"):
+ return None
+
+ # Parse path: /{session_id}/{token}/devtools/browser/{browser_id}
+ path_parts = parsed.path.strip("/").split("/")
+ if len(path_parts) < 4 or path_parts[2] != "devtools":
+ return None
+
+ session_id = path_parts[0]
+ token = path_parts[1]
+ domain = parsed.netloc
+
+ # Build VNC URL with same domain and token
+ scheme = "wss" if parsed.scheme == "wss" else "ws"
+ return f"{scheme}://{domain}/vnc/{session_id}/{token}"
+
+
async def loop_stream_vnc(vnc_channel: VncChannel) -> None:
"""
Actually stream the VNC data between a frontend and a browser.
@@ -288,19 +325,32 @@ async def loop_stream_vnc(vnc_channel: VncChannel) -> None:
Loops until the task is cleared or the websocket is closed.
"""
+ vnc_url: str = ""
browser_session = vnc_channel.browser_session
class_name = vnc_channel.class_name
if not browser_session:
raise Exception(f"{class_name} No browser session associated with vnc channel.")
- vnc_url = build_vnc_streaming_url(
- browser_address=browser_session.browser_address,
- ip_address=browser_session.ip_address,
- vnc_port=vnc_channel.vnc_port,
- )
- if not vnc_url:
- raise Exception(f"{class_name} Browser session does not expose a VNC endpoint.")
+ # First, check if this is a V2 K8s routed session by examining browser_address
+ # V2 sessions have browser_address like: wss://{domain}/{session_id}/{token}/devtools/...
+ # For these, we need to route VNC through the same nginx proxy
+ routed_vnc_url = _build_vnc_url_from_browser_address(browser_session.browser_address)
+ if routed_vnc_url:
+ vnc_url = routed_vnc_url
+ elif browser_session.ip_address:
+ # V1 ECS sessions: Direct IP connection (ip_address is a public/reachable IP)
+ if ":" in browser_session.ip_address:
+ ip, _ = browser_session.ip_address.split(":")
+ vnc_url = f"ws://{ip}:{vnc_channel.vnc_port}"
+ else:
+ vnc_url = f"ws://{browser_session.ip_address}:{vnc_channel.vnc_port}"
+ else:
+ # Last resort: parse browser_address hostname
+ browser_address = browser_session.browser_address
+ parsed_browser_address = urlparse(browser_address)
+ host = parsed_browser_address.hostname
+ vnc_url = f"ws://{host}:{vnc_channel.vnc_port}"
LOG.debug(
f"{class_name} Connecting to vnc url.",
diff --git a/skyvern/forge/sdk/routes/streaming/verify.py b/skyvern/forge/sdk/routes/streaming/verify.py
index 916a2b55b..0593b984a 100644
--- a/skyvern/forge/sdk/routes/streaming/verify.py
+++ b/skyvern/forge/sdk/routes/streaming/verify.py
@@ -26,7 +26,6 @@ from skyvern.forge import app
from skyvern.forge.sdk.schemas.persistent_browser_sessions import AddressablePersistentBrowserSession, is_final_status
from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
-from skyvern.webeye.vnc_streaming import browser_session_supports_vnc_streaming
if t.TYPE_CHECKING:
from skyvern.forge.sdk.routes.streaming.channels.message import MessageChannel
@@ -81,10 +80,7 @@ async def verify_browser_session(
browser_address = browser_session.browser_address
if not browser_address:
- if settings.BROWSER_STREAMING_MODE == "cdp" or browser_session_supports_vnc_streaming(
- browser_address=browser_session.browser_address,
- ip_address=browser_session.ip_address,
- ):
+ if settings.BROWSER_STREAMING_MODE == "cdp":
browser_address = ""
else:
LOG.info(
diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.py
index 1427869f1..f318d584b 100644
--- a/skyvern/webeye/default_persistent_sessions_manager.py
+++ b/skyvern/webeye/default_persistent_sessions_manager.py
@@ -26,7 +26,6 @@ from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.cdp_ports import _release_cdp_port
from skyvern.webeye.persistent_sessions_manager import PersistentSessionsManager
from skyvern.webeye.real_browser_manager import RealBrowserManager
-from skyvern.webeye.vnc_streaming import is_local_vnc_streaming_enabled
LOG = structlog.get_logger()
@@ -301,11 +300,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
# screencast/CDP input endpoints can connect. Triggered both by the
# in-process CDP streaming mode and by cdp-connect, which forwards to a
# remote browser and still needs a registered BrowserState locally.
- should_launch = (
- settings.BROWSER_STREAMING_MODE == "cdp"
- or is_local_vnc_streaming_enabled()
- or settings.BROWSER_TYPE == "cdp-connect"
- )
+ should_launch = settings.BROWSER_STREAMING_MODE == "cdp" or settings.BROWSER_TYPE == "cdp-connect"
if should_launch and runnable_id is None:
session_id = session.persistent_browser_session_id
task = asyncio.create_task(
@@ -513,7 +508,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
async def cleanup_stale_sessions(self) -> None:
"""Close sessions left active by a previous process."""
- if settings.BROWSER_STREAMING_MODE != "cdp" and not is_local_vnc_streaming_enabled():
+ if settings.BROWSER_STREAMING_MODE != "cdp":
return
stale_sessions = await self.database.browser_sessions.get_uncompleted_persistent_browser_sessions()
for db_session in stale_sessions:
diff --git a/skyvern/webeye/schemas.py b/skyvern/webeye/schemas.py
index 371c777db..9e0db1421 100644
--- a/skyvern/webeye/schemas.py
+++ b/skyvern/webeye/schemas.py
@@ -15,7 +15,6 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
PersistentBrowserSession,
PersistentBrowserType,
)
-from skyvern.webeye.vnc_streaming import browser_session_supports_vnc_streaming
LOG = structlog.get_logger()
@@ -141,10 +140,7 @@ class BrowserSessionResponse(BaseModel):
runnable_id=browser_session.runnable_id,
timeout=browser_session.timeout_minutes,
browser_address=browser_session.browser_address,
- vnc_streaming_supported=browser_session_supports_vnc_streaming(
- browser_address=browser_session.browser_address,
- ip_address=browser_session.ip_address,
- ),
+ vnc_streaming_supported=True if browser_session.ip_address else False,
app_url=app_url,
started_at=browser_session.started_at,
completed_at=browser_session.completed_at,
diff --git a/skyvern/webeye/vnc_streaming.py b/skyvern/webeye/vnc_streaming.py
deleted file mode 100644
index 83d0fdcc1..000000000
--- a/skyvern/webeye/vnc_streaming.py
+++ /dev/null
@@ -1,72 +0,0 @@
-from __future__ import annotations
-
-from urllib.parse import urlparse
-
-from skyvern.config import settings
-
-
-def _browser_streaming_mode() -> str:
- return settings.BROWSER_STREAMING_MODE.strip().lower()
-
-
-def is_local_vnc_streaming_enabled() -> bool:
- return _browser_streaming_mode() == "vnc"
-
-
-def build_routed_vnc_url(browser_address: str | None) -> str | None:
- """
- Build a routed VNC URL from a V2 K8s routed browser address.
-
- V2 browser addresses look like:
- wss://{domain}/{session_id}/{token}/devtools/browser/{browser_id}
- """
- if not browser_address:
- return None
-
- parsed = urlparse(browser_address)
- if parsed.scheme not in ("wss", "ws"):
- return None
-
- path_parts = parsed.path.strip("/").split("/")
- if len(path_parts) < 4 or path_parts[2] != "devtools":
- return None
-
- session_id = path_parts[0]
- token = path_parts[1]
- scheme = "wss" if parsed.scheme == "wss" else "ws"
- return f"{scheme}://{parsed.netloc}/vnc/{session_id}/{token}"
-
-
-def browser_session_supports_vnc_streaming(
- *,
- browser_address: str | None = None,
- ip_address: str | None = None,
-) -> bool:
- return bool(build_routed_vnc_url(browser_address) or ip_address or is_local_vnc_streaming_enabled())
-
-
-def build_vnc_streaming_url(
- *,
- browser_address: str | None = None,
- ip_address: str | None = None,
- vnc_port: int,
-) -> str | None:
- routed_vnc_url = build_routed_vnc_url(browser_address)
- if routed_vnc_url:
- return routed_vnc_url
-
- if ip_address:
- if ":" in ip_address:
- ip, _ = ip_address.split(":", maxsplit=1)
- return f"ws://{ip}:{vnc_port}"
- return f"ws://{ip_address}:{vnc_port}"
-
- if is_local_vnc_streaming_enabled():
- return f"ws://127.0.0.1:{vnc_port}"
-
- if browser_address:
- parsed_browser_address = urlparse(browser_address)
- if parsed_browser_address.hostname:
- return f"ws://{parsed_browser_address.hostname}:{vnc_port}"
-
- return None
diff --git a/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py b/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py
index 773093aa5..8b0b197a9 100644
--- a/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py
+++ b/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py
@@ -69,44 +69,15 @@ class TestVerifyBrowserSessionLocalShortCircuit:
with (
patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock,
patch("skyvern.forge.sdk.routes.streaming.verify.settings") as settings_mock,
- patch("skyvern.webeye.vnc_streaming.settings") as vnc_settings_mock,
):
app_mock.PERSISTENT_SESSIONS_MANAGER = manager
- settings_mock.BROWSER_STREAMING_MODE = "remote"
- vnc_settings_mock.BROWSER_STREAMING_MODE = "remote"
+ settings_mock.BROWSER_STREAMING_MODE = "vnc"
result = await verify_browser_session("bs_test", "o_test")
assert result is not None
assert result.browser_address == "ws://remote:9222"
manager.get_browser_address.assert_awaited_once()
- @pytest.mark.asyncio
- async def test_short_circuits_in_vnc_mode_with_no_address(self) -> None:
- session = _make_session(
- browser_address=None,
- status=PersistentBrowserSessionStatus.running,
- )
-
- manager = MagicMock()
- manager.get_session = AsyncMock(return_value=session)
- manager.get_browser_address = AsyncMock(
- side_effect=AssertionError("must not wait for address in local vnc mode"),
- )
-
- with (
- patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock,
- patch("skyvern.forge.sdk.routes.streaming.verify.settings") as settings_mock,
- patch("skyvern.webeye.vnc_streaming.settings") as vnc_settings_mock,
- ):
- app_mock.PERSISTENT_SESSIONS_MANAGER = manager
- settings_mock.BROWSER_STREAMING_MODE = "vnc"
- vnc_settings_mock.BROWSER_STREAMING_MODE = "vnc"
- result = await verify_browser_session("bs_test", "o_test")
-
- assert result is not None
- assert result.browser_address == ""
- manager.get_browser_address.assert_not_called()
-
@pytest.mark.asyncio
async def test_uses_existing_address_without_polling(self) -> None:
session = _make_session(
diff --git a/tests/unit/test_vnc_streaming.py b/tests/unit/test_vnc_streaming.py
deleted file mode 100644
index 04a4e260d..000000000
--- a/tests/unit/test_vnc_streaming.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from __future__ import annotations
-
-from skyvern.webeye import vnc_streaming
-
-
-def test_builds_routed_vnc_url_from_browser_address() -> None:
- assert (
- vnc_streaming.build_routed_vnc_url(
- "wss://browser.example.com/pbs_123/token_456/devtools/browser/browser_789",
- )
- == "wss://browser.example.com/vnc/pbs_123/token_456"
- )
-
-
-def test_local_vnc_mode_supports_session_without_remote_address(monkeypatch) -> None:
- monkeypatch.setattr(vnc_streaming.settings, "BROWSER_STREAMING_MODE", "vnc")
-
- assert vnc_streaming.browser_session_supports_vnc_streaming(
- browser_address=None,
- ip_address=None,
- )
- assert (
- vnc_streaming.build_vnc_streaming_url(
- browser_address=None,
- ip_address=None,
- vnc_port=6080,
- )
- == "ws://127.0.0.1:6080"
- )
-
-
-def test_cdp_mode_does_not_advertise_vnc_for_local_session(monkeypatch) -> None:
- monkeypatch.setattr(vnc_streaming.settings, "BROWSER_STREAMING_MODE", "cdp")
-
- assert not vnc_streaming.browser_session_supports_vnc_streaming(
- browser_address=None,
- ip_address=None,
- )
- assert (
- vnc_streaming.build_vnc_streaming_url(
- browser_address=None,
- ip_address=None,
- vnc_port=6080,
- )
- is None
- )
-
-
-def test_ip_address_takes_precedence_over_local_mode(monkeypatch) -> None:
- monkeypatch.setattr(vnc_streaming.settings, "BROWSER_STREAMING_MODE", "vnc")
-
- assert (
- vnc_streaming.build_vnc_streaming_url(
- browser_address=None,
- ip_address="10.0.0.5:9222",
- vnc_port=6080,
- )
- == "ws://10.0.0.5:6080"
- )