Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[codex] Add workflow runtime limit frontend (#6248)
Suchintan3897cd8cfd2f
diff --git a/docs/assets/pr/cdp-browser-controls.png b/docs/assets/pr/cdp-browser-controls.pngdeleted file mode 100644index 84458e623..000000000Binary files a/docs/assets/pr/cdp-browser-controls.png and /dev/null differdiff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 37cf6334f..5eb5b0355 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -4,7 +4,7 @@import _RFB from "@novnc/novnc/lib/rfb.js";type RFB = _RFB;const RFB = (_RFB as typeof _RFB & { default?: typeof _RFB }).default ?? _RFB;-import { InfoCircledIcon } from "@radix-ui/react-icons";+import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons";import { useEffect, useState, useRef, useCallback } from "react";import { useQuery } from "@tanstack/react-query";@@ -826,40 +826,33 @@ function BrowserStream({{isReady && isVisible && (<div className="overlay z-10 flex items-center justify-center overflow-hidden">{showControlButtons && (- <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center bg-[radial-gradient(circle_at_center,rgba(15,23,42,0.22)_0%,rgba(15,23,42,0.12)_18%,transparent_42%)]">+ <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center"><Button- aria-label="Take control of this browser session"onClick={() => {setUserIsControlling(true);}}- className={cn(- "control-button group pointer-events-auto h-10 rounded-full border border-white/15 bg-slate-950/90 px-5 text-sm font-semibold text-white shadow-xl shadow-slate-950/25 ring-1 ring-slate-950/10 backdrop-blur-sm transition duration-200 ease-out hover:-translate-y-0.5 hover:bg-slate-900 hover:shadow-2xl focus-visible:ring-2 focus-visible:ring-sky-400 focus-visible:ring-offset-2 motion-reduce:transition-none motion-reduce:hover:translate-y-0",- {- hide: userIsControlling,- },- )}+ className={cn("control-button pointer-events-auto border", {+ hide: userIsControlling,+ })}+ size="sm">- Take the wheel- <span- aria-hidden="true"- className="ml-2 transition-transform duration-200 group-hover:translate-x-0.5 motion-reduce:transition-none"- >- →- </span>+ <HandIcon className="mr-2 h-4 w-4" />+ take control</Button><Button- aria-label="Stop controlling this browser session"onClick={() => {setUserIsControlling(false);}}className={cn(- "control-button pointer-events-auto absolute bottom-0 h-9 rounded-full border border-white/15 bg-slate-950/90 px-4 text-sm font-semibold text-white shadow-xl shadow-slate-950/25 ring-1 ring-slate-950/10 backdrop-blur-sm transition duration-200 ease-out hover:-translate-y-0.5 hover:bg-slate-900 focus-visible:ring-2 focus-visible:ring-sky-400 focus-visible:ring-offset-2 motion-reduce:transition-none motion-reduce:hover:translate-y-0",+ "control-button pointer-events-auto absolute bottom-0 border",{hide: !userIsControlling,},)}+ size="sm">- Release control+ <ExitIcon className="mr-2 h-4 w-4" />+ stop controlling</Button></div>)}diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsxindex b487d8a8d..bfbb84b1c 100644--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx@@ -109,10 +109,8 @@ function BrowserSessionStream({userIsControlling,setUserIsControlling,inputReady,- browserCommandError,containerRef,handlers,- browserControls,} = useCdpInput({inputWsUrl,interactive,@@ -238,11 +236,9 @@ function BrowserSessionStream({userIsControlling={userIsControlling}setUserIsControlling={setUserIsControlling}inputReady={inputReady}- browserCommandError={interactive ? browserCommandError : null}containerRef={containerRef}showControlButtons={showControlButtons}handlers={handlers}- browserControls={interactive ? browserControls : undefined}currentUrl={currentUrl}/>);diff --git a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsxindex bfc65860a..b99bf31eb 100644--- a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx+++ b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx@@ -1,13 +1,7 @@-import { useEffect, useState, type FormEvent, type RefObject } from "react";-import {- ArrowLeftIcon,- ArrowRightIcon,- GlobeIcon,- ReloadIcon,-} from "@radix-ui/react-icons";+import type { RefObject } from "react";+import { GlobeIcon } from "@radix-ui/react-icons";import { ZoomableImage } from "@/components/ZoomableImage";import { Button } from "@/components/ui/button";-import { Input } from "@/components/ui/input";import { cn } from "@/util/utils";interface InteractiveStreamViewProps {@@ -17,7 +11,6 @@ interface InteractiveStreamViewProps {userIsControlling: boolean;setUserIsControlling: (v: boolean) => void;inputReady: boolean;- browserCommandError?: string | null;containerRef: RefObject<HTMLDivElement>;showControlButtons: boolean;handlers: {@@ -27,99 +20,14 @@ interface InteractiveStreamViewProps {handleKeyDown: (e: React.KeyboardEvent) => void;handleKeyUp: (e: React.KeyboardEvent) => void;};- browserControls?: {- navigate: (url: string) => void;- reload: () => void;- goBack: () => void;- goForward: () => void;- };currentUrl?: string;}-function UrlBar({- url,- inputReady,- browserCommandError,- browserControls,-}: {- url: string;- inputReady: boolean;- browserCommandError?: string | null;- browserControls?: InteractiveStreamViewProps["browserControls"];-}) {- const [draftUrl, setDraftUrl] = useState(url);- const controlsEnabled = inputReady && Boolean(browserControls);-- useEffect(() => {- setDraftUrl(url);- }, [url]);-- function handleSubmit(e: FormEvent<HTMLFormElement>) {- e.preventDefault();- e.stopPropagation();- browserControls?.navigate(draftUrl);- }-+function UrlBar({ url }: { url: string }) {return (- <div- className="flex h-8 w-full items-center gap-1 rounded-t-md bg-slate-800 px-2 text-xs text-slate-300"- onKeyDown={(e) => e.stopPropagation()}- onKeyUp={(e) => e.stopPropagation()}- onMouseDown={(e) => e.stopPropagation()}- onMouseUp={(e) => e.stopPropagation()}- >- <Button- type="button"- variant="ghost"- size="icon"- className="h-6 w-6 flex-shrink-0 text-slate-300 hover:bg-slate-700 hover:text-white"- disabled={!controlsEnabled}- aria-label="Go back"- title="Go back"- onClick={() => browserControls?.goBack()}- >- <ArrowLeftIcon className="h-3.5 w-3.5" />- </Button>- <Button- type="button"- variant="ghost"- size="icon"- className="h-6 w-6 flex-shrink-0 text-slate-300 hover:bg-slate-700 hover:text-white"- disabled={!controlsEnabled}- aria-label="Go forward"- title="Go forward"- onClick={() => browserControls?.goForward()}- >- <ArrowRightIcon className="h-3.5 w-3.5" />- </Button>- <Button- type="button"- variant="ghost"- size="icon"- className="h-6 w-6 flex-shrink-0 text-slate-300 hover:bg-slate-700 hover:text-white"- disabled={!controlsEnabled}- aria-label="Reload"- title="Reload"- onClick={() => browserControls?.reload()}- >- <ReloadIcon className="h-3.5 w-3.5" />- </Button>- <form className="min-w-0 flex-1" onSubmit={handleSubmit}>- <div className="relative">- <GlobeIcon className="pointer-events-none absolute left-2 top-1/2 h-3 w-3 -translate-y-1/2 text-slate-400" />- <Input- value={draftUrl}- disabled={!controlsEnabled}- aria-label="Current browser URL"- title={browserCommandError || draftUrl}- onChange={(e) => setDraftUrl(e.target.value)}- className={cn(- "h-6 rounded border-slate-700 bg-slate-900 py-0 pl-7 pr-2 text-xs text-slate-200 shadow-none focus-visible:ring-slate-500",- browserCommandError ? "border-red-500 text-red-200" : "",- )}- />- </div>- </form>+ <div className="flex h-8 w-full items-center gap-2 rounded-t-md bg-slate-800 px-3 text-xs text-slate-300">+ <GlobeIcon className="h-3 w-3 flex-shrink-0 text-slate-400" />+ <span className="truncate">{url}</span></div>);}@@ -131,15 +39,12 @@ function InteractiveStreamView({userIsControlling,setUserIsControlling,inputReady,- browserCommandError,containerRef,showControlButtons,handlers,- browserControls,currentUrl,}: InteractiveStreamViewProps) {const imgDataUrl = `data:image/${streamFormat};base64,${streamImgSrc}`;- const showUrlBar = Boolean(currentUrl) || Boolean(browserControls);if (interactive) {return (@@ -150,45 +55,27 @@ function InteractiveStreamView({onKeyDown={handlers.handleKeyDown}onKeyUp={handlers.handleKeyUp}>- {showUrlBar && (- <UrlBar- url={currentUrl ?? ""}- inputReady={inputReady}- browserCommandError={browserCommandError}- browserControls={browserControls}- />- )}+ {currentUrl && <UrlBar url={currentUrl} />}{showControlButtons && !userIsControlling && inputReady && (- <div className="pointer-events-none absolute inset-0 z-10 flex items-center justify-center bg-[radial-gradient(circle_at_center,rgba(15,23,42,0.22)_0%,rgba(15,23,42,0.12)_18%,transparent_42%)]">- <Button- aria-label="Take control of this browser session"- className="group pointer-events-auto h-10 rounded-full border border-white/15 bg-slate-950/90 px-5 text-sm font-semibold text-white shadow-xl shadow-slate-950/25 ring-1 ring-slate-950/10 backdrop-blur-sm transition duration-200 ease-out hover:-translate-y-0.5 hover:bg-slate-900 hover:shadow-2xl focus-visible:ring-2 focus-visible:ring-sky-400 focus-visible:ring-offset-2 motion-reduce:transition-none motion-reduce:hover:translate-y-0"- onClick={() => setUserIsControlling(true)}- >- Take the wheel- <span- aria-hidden="true"- className="ml-2 transition-transform duration-200 group-hover:translate-x-0.5 motion-reduce:transition-none"- >- →- </span>+ <div className="absolute inset-0 z-10 flex items-center justify-center">+ <Button onClick={() => setUserIsControlling(true)}>+ take control</Button></div>)}{showControlButtons && userIsControlling && (<Button- aria-label="Stop controlling this browser session"- className="absolute bottom-2 left-1/2 z-10 h-9 -translate-x-1/2 rounded-full border border-white/15 bg-slate-950/90 px-4 text-sm font-semibold text-white shadow-xl shadow-slate-950/25 ring-1 ring-slate-950/10 backdrop-blur-sm transition duration-200 ease-out hover:-translate-x-1/2 hover:-translate-y-0.5 hover:bg-slate-900 focus-visible:ring-2 focus-visible:ring-sky-400 focus-visible:ring-offset-2 motion-reduce:transition-none motion-reduce:hover:-translate-x-1/2 motion-reduce:hover:translate-y-0"+ className="absolute bottom-2 left-1/2 z-10 -translate-x-1/2"onClick={() => setUserIsControlling(false)}>- Release control+ stop controlling</Button>)}<imgsrc={imgDataUrl}className={cn("w-full rounded-md object-contain",- showUrlBar ? "h-[calc(100%-2rem)] rounded-t-none" : "h-full",+ currentUrl ? "h-[calc(100%-2rem)]" : "h-full",{ "cursor-default": userIsControlling },)}onMouseDown={handlers.handleMouseDown}@@ -203,18 +90,11 @@ function InteractiveStreamView({return (<div className="h-full w-full">- {showUrlBar && (- <UrlBar- url={currentUrl ?? ""}- inputReady={inputReady}- browserCommandError={browserCommandError}- browserControls={browserControls}- />- )}+ {currentUrl && <UrlBar url={currentUrl} />}<ZoomableImagesrc={imgDataUrl}className={- showUrlBar ? "h-[calc(100%-2rem)] rounded-b-md" : "rounded-md"+ currentUrl ? "h-[calc(100%-2rem)] rounded-b-md" : "rounded-md"}/></div>diff --git a/skyvern-frontend/src/routes/streaming/cdpInputUtils.test.ts b/skyvern-frontend/src/routes/streaming/cdpInputUtils.test.tsdeleted file mode 100644index 427109320..000000000--- a/skyvern-frontend/src/routes/streaming/cdpInputUtils.test.ts+++ /dev/null@@ -1,83 +0,0 @@-import { describe, expect, test } from "vitest";-import { buildKeyDownPayload, buildKeyUpPayload } from "./cdpInputUtils";--function makeKeyboardEvent(- key: string,- code: string,- modifiers: Partial<KeyboardEvent> = {},-): KeyboardEvent {- return new KeyboardEvent("keydown", {- key,- code,- altKey: Boolean(modifiers.altKey),- ctrlKey: Boolean(modifiers.ctrlKey),- metaKey: Boolean(modifiers.metaKey),- shiftKey: Boolean(modifiers.shiftKey),- });-}--describe("CDP keyboard payloads", () => {- test("sends non-printable keys as rawKeyDown with a Windows virtual key code", () => {- expect(buildKeyDownPayload(makeKeyboardEvent("Enter", "Enter"))).toEqual({- type: "keyEvent",- eventType: "rawKeyDown",- key: "Enter",- code: "Enter",- windowsVirtualKeyCode: 13,- modifiers: 0,- });-- expect(- buildKeyDownPayload(makeKeyboardEvent("ArrowLeft", "ArrowLeft")),- ).toMatchObject({- eventType: "rawKeyDown",- windowsVirtualKeyCode: 37,- });- });-- test("includes the Windows virtual key code on keyUp for non-printable keys", () => {- expect(- buildKeyUpPayload(makeKeyboardEvent("Backspace", "Backspace")),- ).toEqual({- type: "keyEvent",- eventType: "keyUp",- key: "Backspace",- code: "Backspace",- windowsVirtualKeyCode: 8,- modifiers: 0,- });- });-- test("keeps printable characters as keyDown with text", () => {- expect(buildKeyDownPayload(makeKeyboardEvent("a", "KeyA"))).toEqual({- type: "keyEvent",- eventType: "keyDown",- key: "a",- code: "KeyA",- text: "a",- windowsVirtualKeyCode: 65,- modifiers: 0,- });- });-- test("keeps unmapped multi-character keys as keyDown", () => {- expect(buildKeyDownPayload(makeKeyboardEvent("Dead", "Quote"))).toEqual({- type: "keyEvent",- eventType: "keyDown",- key: "Dead",- code: "Quote",- modifiers: 0,- });- });-- test("preserves modifier bitmask", () => {- expect(- buildKeyDownPayload(- makeKeyboardEvent("Tab", "Tab", { ctrlKey: true, shiftKey: true }),- ),- ).toMatchObject({- modifiers: 10,- windowsVirtualKeyCode: 9,- });- });-});diff --git a/skyvern-frontend/src/routes/streaming/cdpInputUtils.ts b/skyvern-frontend/src/routes/streaming/cdpInputUtils.tsindex 8c2552d0a..9937da288 100644--- a/skyvern-frontend/src/routes/streaming/cdpInputUtils.ts+++ b/skyvern-frontend/src/routes/streaming/cdpInputUtils.ts@@ -15,102 +15,6 @@ export function getModifiers(return m;}-const WINDOWS_VIRTUAL_KEY_CODES: Record<string, number> = {- Backspace: 8,- Tab: 9,- Enter: 13,- Shift: 16,- Control: 17,- Alt: 18,- Pause: 19,- CapsLock: 20,- Escape: 27,- " ": 32,- PageUp: 33,- PageDown: 34,- End: 35,- Home: 36,- ArrowLeft: 37,- ArrowUp: 38,- ArrowRight: 39,- ArrowDown: 40,- Insert: 45,- Delete: 46,- Meta: 91,- ContextMenu: 93,- F1: 112,- F2: 113,- F3: 114,- F4: 115,- F5: 116,- F6: 117,- F7: 118,- F8: 119,- F9: 120,- F10: 121,- F11: 122,- F12: 123,- NumLock: 144,- ScrollLock: 145,-};--export type CdpKeyEventPayload = {- type: "keyEvent";- eventType: "keyDown" | "keyUp" | "rawKeyDown";- key: string;- code: string;- modifiers: number;- text?: string;- windowsVirtualKeyCode?: number;-};--type KeyboardEventLike = Pick<- KeyboardEvent,- "altKey" | "code" | "ctrlKey" | "key" | "metaKey" | "shiftKey"->;--function windowsVirtualKeyCodeForKey(key: string): number | undefined {- if (/^[a-z]$/i.test(key)) {- return key.toUpperCase().charCodeAt(0);- }- if (/^[0-9]$/.test(key)) {- return key.charCodeAt(0);- }- return WINDOWS_VIRTUAL_KEY_CODES[key];-}--export function buildKeyDownPayload(e: KeyboardEventLike): CdpKeyEventPayload {- const windowsVirtualKeyCode = windowsVirtualKeyCodeForKey(e.key);- const isPrintableSingleCharacter = e.key.length === 1;- const eventType =- !isPrintableSingleCharacter && windowsVirtualKeyCode !== undefined- ? "rawKeyDown"- : "keyDown";-- return {- type: "keyEvent",- eventType,- key: e.key,- code: e.code,- ...(isPrintableSingleCharacter ? { text: e.key } : {}),- ...(windowsVirtualKeyCode !== undefined ? { windowsVirtualKeyCode } : {}),- modifiers: getModifiers(e),- };-}--export function buildKeyUpPayload(e: KeyboardEventLike): CdpKeyEventPayload {- const windowsVirtualKeyCode = windowsVirtualKeyCodeForKey(e.key);-- return {- type: "keyEvent",- eventType: "keyUp",- key: e.key,- code: e.code,- ...(windowsVirtualKeyCode !== undefined ? { windowsVirtualKeyCode } : {}),- modifiers: getModifiers(e),- };-}-/*** Map pixel coordinates from a rendered image back to viewport coordinates,* accounting for object-contain letterboxing.diff --git a/skyvern-frontend/src/routes/streaming/useCdpInput.ts b/skyvern-frontend/src/routes/streaming/useCdpInput.tsindex 3664b23d1..72f7119d7 100644--- a/skyvern-frontend/src/routes/streaming/useCdpInput.ts+++ b/skyvern-frontend/src/routes/streaming/useCdpInput.ts@@ -4,8 +4,6 @@ import { getCredentialParam } from "@/util/env";import { useClientIdStore } from "@/store/useClientIdStore";import {mouseButtonName,- buildKeyDownPayload,- buildKeyUpPayload,getModifiers,mapCoordinates,mapMouseCoordinates,@@ -24,7 +22,6 @@ interface UseCdpInputReturn {userIsControlling: boolean;setUserIsControlling: (v: boolean) => void;inputReady: boolean;- browserCommandError: string | null;containerRef: React.RefObject<HTMLDivElement>;handlers: {handleMouseDown: (e: React.MouseEvent<HTMLImageElement>) => void;@@ -33,12 +30,6 @@ interface UseCdpInputReturn {handleKeyDown: (e: React.KeyboardEvent) => void;handleKeyUp: (e: React.KeyboardEvent) => void;};- browserControls: {- navigate: (url: string) => void;- reload: () => void;- goBack: () => void;- goForward: () => void;- };}export function useCdpInput({@@ -49,9 +40,6 @@ export function useCdpInput({}: UseCdpInputOptions): UseCdpInputReturn {const [userIsControlling, setUserIsControlling] = useState(false);const [inputReady, setInputReady] = useState(false);- const [browserCommandError, setBrowserCommandError] = useState<string | null>(- null,- );const credentialGetter = useCredentialGetter();const clientId = useClientIdStore((s) => s.clientId);@@ -105,8 +93,6 @@ export function useCdpInput({if (userIsControllingRef.current) {ws.send(JSON.stringify({ kind: "take-control" }));}- } else if (msg.kind === "browserCommandResult") {- setBrowserCommandError(msg.ok ? null : String(msg.error || ""));}} catch {// ignore non-JSON messages@@ -251,44 +237,6 @@ export function useCdpInput({ws.send(JSON.stringify(payload));}, []);- const sendBrowserCommand = useCallback((payload: Record<string, unknown>) => {- const ws = inputSocketRef.current;- if (!ws || ws.readyState !== WebSocket.OPEN) {- setBrowserCommandError("Browser controls are not connected.");- return;- }- setBrowserCommandError(null);- ws.send(JSON.stringify({ type: "browserCommand", ...payload }));- }, []);-- const navigate = useCallback(- (rawUrl: string) => {- const url = rawUrl.trim();- if (!url) {- setBrowserCommandError("URL must not be empty.");- return;- }- if (url.toLowerCase().startsWith("chrome://")) {- setBrowserCommandError("chrome:// URLs are not allowed.");- return;- }- sendBrowserCommand({ command: "navigate", url });- },- [sendBrowserCommand],- );-- const reload = useCallback(() => {- sendBrowserCommand({ command: "reload" });- }, [sendBrowserCommand]);-- const goBack = useCallback(() => {- sendBrowserCommand({ command: "goBack" });- }, [sendBrowserCommand]);-- const goForward = useCallback(() => {- sendBrowserCommand({ command: "goForward" });- }, [sendBrowserCommand]);-const handleMouseDown = useCallback((e: React.MouseEvent<HTMLImageElement>) => {if (!interactive || !userIsControlling) return;@@ -368,7 +316,14 @@ export function useCdpInput({(e: React.KeyboardEvent) => {if (!interactive || !userIsControlling) return;e.preventDefault();- sendInputEvent(buildKeyDownPayload(e.nativeEvent));+ sendInputEvent({+ type: "keyEvent",+ eventType: "keyDown",+ key: e.key,+ code: e.code,+ text: e.key.length === 1 ? e.key : "",+ modifiers: getModifiers(e),+ });},[interactive, userIsControlling, sendInputEvent],);@@ -377,7 +332,13 @@ export function useCdpInput({(e: React.KeyboardEvent) => {if (!interactive || !userIsControlling) return;e.preventDefault();- sendInputEvent(buildKeyUpPayload(e.nativeEvent));+ sendInputEvent({+ type: "keyEvent",+ eventType: "keyUp",+ key: e.key,+ code: e.code,+ modifiers: getModifiers(e),+ });},[interactive, userIsControlling, sendInputEvent],);@@ -386,7 +347,6 @@ export function useCdpInput({userIsControlling,setUserIsControlling,inputReady,- browserCommandError,containerRef,handlers: {handleMouseDown,@@ -395,11 +355,5 @@ export function useCdpInput({handleKeyDown,handleKeyUp,},- browserControls: {- navigate,- reload,- goBack,- goForward,- },};}diff --git a/skyvern-frontend/src/routes/workflows/components/WorkflowVisualComparisonDrawer.tsx b/skyvern-frontend/src/routes/workflows/components/WorkflowVisualComparisonDrawer.tsxindex c35fe3198..e709034cc 100644--- a/skyvern-frontend/src/routes/workflows/components/WorkflowVisualComparisonDrawer.tsx+++ b/skyvern-frontend/src/routes/workflows/components/WorkflowVisualComparisonDrawer.tsx@@ -106,6 +106,7 @@ function getWorkflowElements(version: WorkflowVersion) {browserProfileId: version.browser_profile_id ?? null,model: version.model,maxScreenshotScrolls: version.max_screenshot_scrolls || 3,+ maxElapsedTimeMinutes: version.max_elapsed_time_minutes ?? null,extraHttpHeaders: version.extra_http_headers? JSON.stringify(version.extra_http_headers): null,diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 1306ba3f5..b6f6d1f1a 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -791,6 +791,8 @@ export function WorkflowCopilotChat({browser_profile_id: saveData.settings.browserProfileId,model: saveData.settings.model,max_screenshot_scrolls: saveData.settings.maxScreenshotScrolls,+ max_elapsed_time_minutes:+ saveData.settings.maxElapsedTimeMinutes ?? null,totp_verification_url: saveData.workflow.totp_verification_url,extra_http_headers: extraHttpHeaders,run_with: saveData.settings.runWith,diff --git a/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx b/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsxindex 5cc8c0017..8182deb06 100644--- a/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx+++ b/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx@@ -80,6 +80,7 @@ function Debugger() {webhookCallbackUrl: workflow.webhook_callback_url,model: workflow.model,maxScreenshotScrolls: workflow.max_screenshot_scrolls,+ maxElapsedTimeMinutes: workflow.max_elapsed_time_minutes ?? null,extraHttpHeaders: workflow.extra_http_headers? JSON.stringify(workflow.extra_http_headers): null,diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsxindex ba61effbe..2639d73b3 100644--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx@@ -82,6 +82,7 @@ function WorkflowEditor() {webhookCallbackUrl: workflow.webhook_callback_url,model: workflow.model,maxScreenshotScrolls: workflow.max_screenshot_scrolls,+ maxElapsedTimeMinutes: workflow.max_elapsed_time_minutes ?? null,extraHttpHeaders: workflow.extra_http_headers? JSON.stringify(workflow.extra_http_headers): null,diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex d12ed60ce..8b6c19440 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -1240,6 +1240,7 @@ function Workspace({browserProfileId: workflowData.browser_profile_id ?? null,model: workflowData.model ?? null,maxScreenshotScrolls: workflowData.max_screenshot_scrolls || 3,+ maxElapsedTimeMinutes: workflowData.max_elapsed_time_minutes ?? null,extraHttpHeaders: workflowData.extra_http_headers? JSON.stringify(workflowData.extra_http_headers): null,@@ -1308,6 +1309,7 @@ function Workspace({browserProfileId: selectedVersion.browser_profile_id ?? null,model: selectedVersion.model,maxScreenshotScrolls: selectedVersion.max_screenshot_scrolls || 3,+ maxElapsedTimeMinutes: selectedVersion.max_elapsed_time_minutes ?? null,extraHttpHeaders: selectedVersion.extra_http_headers? JSON.stringify(selectedVersion.extra_http_headers): null,@@ -2121,6 +2123,8 @@ function Workspace({totp_verification_url: saveData.workflow.totp_verification_url,totp_identifier: null,max_screenshot_scrolls: saveData.settings.maxScreenshotScrolls,+ max_elapsed_time_minutes:+ saveData.settings.maxElapsedTimeMinutes ?? null,status: saveData.workflow.status,created_at: new Date().toISOString(),modified_at: new Date().toISOString(),diff --git a/skyvern-frontend/src/routes/workflows/editor/nestedConditionalSave.test.ts b/skyvern-frontend/src/routes/workflows/editor/nestedConditionalSave.test.tsindex 7c9446594..5a7b8bf35 100644--- a/skyvern-frontend/src/routes/workflows/editor/nestedConditionalSave.test.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nestedConditionalSave.test.ts@@ -23,6 +23,7 @@ const DEFAULT_SETTINGS: WorkflowSettings = {browserProfileId: null,model: null,maxScreenshotScrolls: null,+ maxElapsedTimeMinutes: null,extraHttpHeaders: null,cdpConnectHeaders: null,runWith: "code",diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/WorkflowSettingsEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/WorkflowSettingsEditor.tsxindex 68621304b..0d0e5119a 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/WorkflowSettingsEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/WorkflowSettingsEditor.tsx@@ -400,6 +400,7 @@ function WorkflowSettingsEditorBody({<Inputvalue={data.maxScreenshotScrolls ?? ""}placeholder={`Default: ${MAX_SCREENSHOT_SCROLLS_DEFAULT}`}+ type="number"onChange={(event) => {const value =event.target.value === "" ? null : Number(event.target.value);@@ -407,6 +408,29 @@ function WorkflowSettingsEditorBody({}}/></div>+ <div className="space-y-2">+ <div className="flex items-center gap-2">+ <Label>Max Run Time (minutes)</Label>+ <HelpTooltip content="Times out this workflow after the configured elapsed runtime. Maximum runtime is 4 hours (240 minutes)." />+ </div>+ <Input+ value={data.maxElapsedTimeMinutes ?? ""}+ placeholder="Default: 4 hours"+ type="number"+ min={1}+ max={240}+ step={1}+ onChange={(event) => {+ const rawValue = event.target.value;+ const parsedValue = Number(rawValue);+ const value =+ rawValue === "" || !Number.isFinite(parsedValue)+ ? null+ : Math.min(240, Math.max(1, Math.trunc(parsedValue)));+ update({ maxElapsedTimeMinutes: value });+ }}+ />+ </div><div className="space-y-2"><div className="flex items-center gap-2"><Label>Execute on Any Outcome</Label>diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.tsindex 8f34aa437..742e2e799 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts@@ -11,6 +11,7 @@ export type WorkflowStartNodeData = {browserProfileId: string | null;model: WorkflowModel | null;maxScreenshotScrolls: number | null;+ maxElapsedTimeMinutes: number | null;extraHttpHeaders: string | Record<string, unknown> | null;cdpConnectHeaders: string | Record<string, unknown> | null;editable: boolean;diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/WorkflowSettingsBlockForm.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/WorkflowSettingsBlockForm.tsxindex aad9e3824..15533b71b 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/WorkflowSettingsBlockForm.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/WorkflowSettingsBlockForm.tsx@@ -40,6 +40,7 @@ function WorkflowSettingsBlockFormBody({persistBrowserSession: data.persistBrowserSession,extraHttpHeaders: data.extraHttpHeaders,maxScreenshotScrolls: data.maxScreenshotScrolls,+ maxElapsedTimeMinutes: data.maxElapsedTimeMinutes,finallyBlockLabel: data.finallyBlockLabel,workflowSystemPrompt: data.workflowSystemPrompt,};diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowComparisonPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowComparisonPanel.tsxindex ad4429891..20f76541b 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowComparisonPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowComparisonPanel.tsx@@ -141,6 +141,7 @@ function getWorkflowElements(version: WorkflowVersion) {browserProfileId: version.browser_profile_id ?? null,model: version.model,maxScreenshotScrolls: version.max_screenshot_scrolls || 3,+ maxElapsedTimeMinutes: version.max_elapsed_time_minutes ?? null,extraHttpHeaders: version.extra_http_headers? JSON.stringify(version.extra_http_headers): null,diff --git a/skyvern-frontend/src/routes/workflows/editor/sortable/perf.test.ts b/skyvern-frontend/src/routes/workflows/editor/sortable/perf.test.tsindex 9a5ff3362..eea5cbe66 100644--- a/skyvern-frontend/src/routes/workflows/editor/sortable/perf.test.ts+++ b/skyvern-frontend/src/routes/workflows/editor/sortable/perf.test.ts@@ -52,6 +52,7 @@ const DEFAULT_SETTINGS: WorkflowSettings = {browserProfileId: null,model: null,maxScreenshotScrolls: null,+ maxElapsedTimeMinutes: null,extraHttpHeaders: null,cdpConnectHeaders: null,runWith: "code",diff --git a/skyvern-frontend/src/routes/workflows/editor/sortable/roundTrip.test.ts b/skyvern-frontend/src/routes/workflows/editor/sortable/roundTrip.test.tsindex d6eb17539..cb8e028ce 100644--- a/skyvern-frontend/src/routes/workflows/editor/sortable/roundTrip.test.ts+++ b/skyvern-frontend/src/routes/workflows/editor/sortable/roundTrip.test.ts@@ -42,6 +42,7 @@ const DEFAULT_SETTINGS: WorkflowSettings = {browserProfileId: null,model: null,maxScreenshotScrolls: null,+ maxElapsedTimeMinutes: null,extraHttpHeaders: null,cdpConnectHeaders: null,runWith: "code",diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex 3763b041f..f75d41ff1 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -1959,6 +1959,7 @@ function getElements(webhookCallbackUrl: settings.webhookCallbackUrl ?? "",model: settings.model,maxScreenshotScrolls: settings.maxScreenshotScrolls,+ maxElapsedTimeMinutes: settings.maxElapsedTimeMinutes ?? null,extraHttpHeaders: settings.extraHttpHeaders,cdpConnectHeaders: settings.cdpConnectHeaders,editable,@@ -3335,6 +3336,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {webhookCallbackUrl: null,model: null,maxScreenshotScrolls: null,+ maxElapsedTimeMinutes: null,extraHttpHeaders: null,cdpConnectHeaders: null,runWith: "code",@@ -3362,6 +3364,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {webhookCallbackUrl: data.webhookCallbackUrl,model: data.model,maxScreenshotScrolls: data.maxScreenshotScrolls,+ maxElapsedTimeMinutes: data.maxElapsedTimeMinutes,extraHttpHeaders:data.extraHttpHeaders && typeof data.extraHttpHeaders === "object"? JSON.stringify(data.extraHttpHeaders)@@ -4457,6 +4460,7 @@ function convert(workflow: WorkflowApiResponse): WorkflowCreateYAMLRequest {model: workflow.model,totp_verification_url: workflow.totp_verification_url,max_screenshot_scrolls: workflow.max_screenshot_scrolls,+ max_elapsed_time_minutes: workflow.max_elapsed_time_minutes,extra_http_headers: workflow.extra_http_headers,workflow_definition: {version: workflowDefinitionVersion,diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.tsindex 18284d9a3..412d6e867 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts@@ -642,6 +642,7 @@ export type WorkflowApiResponse = {totp_verification_url: string | null;totp_identifier: string | null;max_screenshot_scrolls: number | null;+ max_elapsed_time_minutes: number | null;status: string | null;created_at: string;modified_at: string;@@ -664,6 +665,7 @@ export type WorkflowSettings = {browserProfileId: string | null;model: WorkflowModel | null;maxScreenshotScrolls: number | null;+ maxElapsedTimeMinutes: number | null;extraHttpHeaders: string | null;cdpConnectHeaders: string | null;runWith: string; // 'agent' or 'code'diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.tsindex 44c160775..21edfdef3 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts@@ -13,6 +13,7 @@ export type WorkflowCreateYAMLRequest = {workflow_definition: WorkflowDefinitionYAML;is_saved_task?: boolean;max_screenshot_scrolls?: number | null;+ max_elapsed_time_minutes?: number | null;extra_http_headers?: Record<string, string> | null;cdp_connect_headers?: Record<string, string> | null;status?: string | null;diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsxindex 31ceead26..9284ba720 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockDetail.test.tsx@@ -15,6 +15,7 @@ vi.mock("./WorkflowRunHumanInteraction", () => ({}));import { cleanup, fireEvent, render, screen } from "@testing-library/react";+import { MemoryRouter } from "react-router-dom";import { afterEach, describe, expect, it, vi } from "vitest";import { ActionTypes, Status, type ActionsApiResponse } from "@/api/types";@@ -137,6 +138,26 @@ describe("WorkflowRunBlockDetail router", () => {expect(screen.getByText("https://example.test/report")).toBeDefined();});+ it("shows a diagnostics link in the inspector for any block with a task id", () => {+ const block = buildBlock({+ block_type: "file_download",+ task_id: "tsk_123",+ });++ render(+ <MemoryRouter>+ <WorkflowRunBlockDetail activeItem={block} timeline={[]} />+ </MemoryRouter>,+ );++ const diagnosticsLink = screen.getByRole("link", {+ name: /diagnostics/i,+ });+ expect(diagnosticsLink.getAttribute("href")).toBe(+ "/tasks/tsk_123/diagnostics",+ );+ });+it("renders the conditional detail (branch evaluation) for a conditional block", () => {const block = buildBlock({block_type: "conditional",diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsxindex ded4e53f0..78e6a0ca5 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx@@ -22,7 +22,6 @@ type StreamMessage = {format?: string;viewport_width?: number;viewport_height?: number;- url?: string;};const STARTING_DIAGNOSTIC: StreamDiagnostic = {@@ -94,7 +93,6 @@ function WorkflowRunStream({const [streamFormat, setStreamFormat] = useState<string>("png");const [viewportWidth, setViewportWidth] = useState(1280);const [viewportHeight, setViewportHeight] = useState(720);- const [currentUrl, setCurrentUrl] = useState("");const [diagnostic, setDiagnostic] =useState<StreamDiagnostic>(STARTING_DIAGNOSTIC);const showStream =@@ -116,10 +114,8 @@ function WorkflowRunStream({userIsControlling,setUserIsControlling,inputReady,- browserCommandError,containerRef,handlers,- browserControls,} = useCdpInput({inputWsUrl,interactive,@@ -132,7 +128,6 @@ function WorkflowRunStream({return;}setDiagnostic(STARTING_DIAGNOSTIC);- setCurrentUrl("");hasFrameRef.current = false;let cancelled = false;@@ -172,9 +167,6 @@ function WorkflowRunStream({if (message.viewport_height) {setViewportHeight(message.viewport_height);}- if (message.url !== undefined) {- setCurrentUrl(message.url);- }if (!message.screenshot && message.status) {setDiagnostic(diagnosticForStatus(message.status));}@@ -297,12 +289,9 @@ function WorkflowRunStream({userIsControlling={userIsControlling}setUserIsControlling={setUserIsControlling}inputReady={inputReady}- browserCommandError={interactive ? browserCommandError : null}containerRef={containerRef}showControlButtons={showControlButtons}handlers={handlers}- browserControls={interactive ? browserControls : undefined}- currentUrl={currentUrl}/>);}diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsxindex ff66608f7..866ddb8b4 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockDetail/BlockInspector.tsx@@ -1,5 +1,10 @@-import { ChevronDownIcon, ChevronRightIcon } from "@radix-ui/react-icons";+import {+ ChevronDownIcon,+ ChevronRightIcon,+ ExternalLinkIcon,+} from "@radix-ui/react-icons";import { useEffect, useMemo, useState } from "react";+import { Link } from "react-router-dom";import { Input } from "@/components/ui/input";import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";@@ -410,6 +415,20 @@ function BlockInspector({ block }: { block: WorkflowRunBlock }) {>Outputs</TabsTrigger>+ {block.task_id && (+ <Link+ to={`/tasks/${block.task_id}/diagnostics`}+ title="Go to diagnostics"+ onClick={(event) => event.stopPropagation()}+ className={cn(+ triggerClassName,+ "inline-flex items-center gap-1.5",+ )}+ >+ <ExternalLinkIcon className="size-3.5" />+ <span>Diagnostics</span>+ </Link>+ )}</TabsList><TabsContent value="summary" className="m-0"><FieldListdiff --git a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts b/skyvern-frontend/src/store/WorkflowHasChangesStore.tsindex 2916eb385..5890504e2 100644--- a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts+++ b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts@@ -183,6 +183,8 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {browser_profile_id: saveData.settings.browserProfileId,model: saveData.settings.model,max_screenshot_scrolls: saveData.settings.maxScreenshotScrolls,+ max_elapsed_time_minutes:+ saveData.settings.maxElapsedTimeMinutes ?? null,totp_verification_url: saveData.workflow.totp_verification_url,extra_http_headers: extraHttpHeaders,cdp_connect_headers: cdpConnectHeaders,diff --git a/skyvern/forge/sdk/routes/debug_sessions.py b/skyvern/forge/sdk/routes/debug_sessions.pyindex 1cef17bf7..9dba437f0 100644--- a/skyvern/forge/sdk/routes/debug_sessions.py+++ b/skyvern/forge/sdk/routes/debug_sessions.py@@ -16,7 +16,6 @@ from skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSessionfrom skyvern.forge.sdk.services import org_auth_servicefrom skyvern.schemas.runs import ProxyLocation-from skyvern.webeye.vnc_streaming import browser_session_supports_vnc_streamingLOG = structlog.get_logger()@@ -246,10 +245,9 @@ async def new_debug_session(organization_id=current_org.organization_id,user_id=current_user_id,workflow_permanent_id=workflow_permanent_id,- vnc_streaming_supported=browser_session_supports_vnc_streaming(- browser_address=new_browser_session.browser_address,- ip_address=new_browser_session.ip_address,- ),+ vnc_streaming_supported=True if new_browser_session.ip_address else False,+ # NOTE(jdo:streaming-local-dev)+ # vnc_streaming_supported=True,)LOG.info(diff --git a/skyvern/forge/sdk/routes/streaming/cdp_input.py b/skyvern/forge/sdk/routes/streaming/cdp_input.pyindex 5c0f7d3d0..772967d6e 100644--- a/skyvern/forge/sdk/routes/streaming/cdp_input.py+++ b/skyvern/forge/sdk/routes/streaming/cdp_input.py@@ -5,17 +5,14 @@ CDP input channel for interactive browser control via Chrome DevTools Protocol.import asyncioimport dataclassesimport json-import reimport timeimport typing as t-from urllib.parse import urlparseimport structlogfrom fastapi import WebSocket, WebSocketDisconnect-from playwright.async_api import CDPSession, Page+from playwright.async_api import CDPSessionfrom websockets.exceptions import ConnectionClosedError, ConnectionClosedOK-from skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.routes.routers import base_router, legacy_base_routerfrom skyvern.forge.sdk.routes.streaming.auth import auth, require_client_id@@ -40,8 +37,6 @@ _MAX_KEY_LEN = 32_MAX_CODE_LEN = 32_MODIFIER_MASK = 0xF_MAX_VK_CODE = 0xFE-_MAX_NAVIGATION_URL_LEN = 2048-_BARE_HOST_PORT_RE = re.compile(r"^[A-Za-z0-9.-]+:\d+(?:[/?#].*)?$")@dataclasses.dataclass@@ -201,86 +196,10 @@ async def _dispatch_event()-def _normalize_navigation_url(raw_url: str) -> str:- candidate = raw_url.strip()- if not candidate:- raise ValueError("URL must not be empty")- if len(candidate) > _MAX_NAVIGATION_URL_LEN:- raise ValueError("URL is too long")- if candidate.lower().startswith("chrome://"):- raise ValueError("chrome:// URLs are not allowed")- if candidate.startswith("//"):- raise ValueError("URL must include an explicit http(s) scheme or host")- if _BARE_HOST_PORT_RE.match(candidate):- return f"http://{candidate}"-- parsed = urlparse(candidate)- if parsed.scheme in ("http", "https"):- return candidate- if parsed.scheme:- raise ValueError(f"refusing to navigate to non-http(s) scheme: {parsed.scheme}")- return f"https://{candidate}"---async def _dispatch_browser_command(- websocket: WebSocket,- page: Page,- msg: dict,- log_id_key: str,- log_id_value: str,-) -> None:- command = msg.get("command")- try:- if command == "reload":- await page.reload(wait_until="domcontentloaded", timeout=settings.BROWSER_ACTION_TIMEOUT_MS)- elif command == "goBack":- await page.go_back(wait_until="domcontentloaded", timeout=settings.BROWSER_ACTION_TIMEOUT_MS)- elif command == "goForward":- await page.go_forward(wait_until="domcontentloaded", timeout=settings.BROWSER_ACTION_TIMEOUT_MS)- elif command == "navigate":- raw_url = msg.get("url")- if not isinstance(raw_url, str):- raise ValueError("URL must be a string")- await page.goto(- _normalize_navigation_url(raw_url),- wait_until="domcontentloaded",- timeout=settings.BROWSER_ACTION_TIMEOUT_MS,- )- else:- raise ValueError("unsupported browser command")- except Exception as exc:- LOG.warning(- "CDP input: browser command failed",- **{log_id_key: log_id_value},- command=command,- error=str(exc),- )- await websocket.send_json(- {- "kind": "browserCommandResult",- "ok": False,- "command": command,- "error": str(exc),- "url": page.url,- }- )- return-- await websocket.send_json(- {- "kind": "browserCommandResult",- "ok": True,- "command": command,- "url": page.url,- }- )--async def _run_input_loop(websocket: WebSocket,channel: CdpInputChannel,cdp_session: CDPSession,- page: Page,log_id_key: str,log_id_value: str,) -> None:@@ -307,9 +226,6 @@ async def _run_input_loop(channel.interactor = "agent"LOG.info("CDP input: cede-control received", **{log_id_key: log_id_value}, client_id=channel.client_id)continue- if kind == "browserCommand":- await _dispatch_browser_command(websocket, page, msg, log_id_key, log_id_value)- continueif channel.interactor != "user":if dropped_log_count < 5:@@ -398,7 +314,7 @@ async def cdp_input_stream(LOG.info("CDP input channel ready", workflow_run_id=workflow_run_id, client_id=client_id)await websocket.send_json({"kind": "ready"})- await _run_input_loop(websocket, channel, cdp_session, page, "workflow_run_id", workflow_run_id)+ await _run_input_loop(websocket, channel, cdp_session, "workflow_run_id", workflow_run_id)except ConnectionClosedOK:LOG.info("CDP input: WS closed cleanly", workflow_run_id=workflow_run_id)@@ -474,7 +390,7 @@ async def cdp_input_browser_session_stream(LOG.info("CDP input channel ready", browser_session_id=browser_session_id, client_id=client_id)await websocket.send_json({"kind": "ready"})- await _run_input_loop(websocket, channel, cdp_session, page, "browser_session_id", browser_session_id)+ await _run_input_loop(websocket, channel, cdp_session, "browser_session_id", browser_session_id)except ConnectionClosedOK:LOG.info("CDP input: WS closed cleanly", browser_session_id=browser_session_id)diff --git a/skyvern/forge/sdk/routes/streaming/channels/vnc.py b/skyvern/forge/sdk/routes/streaming/channels/vnc.pyindex 680d27a47..809887bb7 100644--- a/skyvern/forge/sdk/routes/streaming/channels/vnc.py+++ b/skyvern/forge/sdk/routes/streaming/channels/vnc.py@@ -20,6 +20,7 @@ import asyncioimport dataclassesimport typing as tfrom enum import IntEnum+from urllib.parse import urlparseimport structlogimport websockets@@ -48,7 +49,6 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import AddressablePerfrom skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.forge.sdk.utils.aio import collectfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRun-from skyvern.webeye.vnc_streaming import build_vnc_streaming_urlLOG = structlog.get_logger()@@ -281,6 +281,43 @@ async def ask_for_clipboard(vnc_channel: VncChannel) -> None:LOG.exception(f"{class_name} Failed to ask for clipboard via CDP", **vnc_channel.identity)+# TODO(benji): I hate this function. It's messy and gross. Once we remove v1,+# we should clean this up.+def _build_vnc_url_from_browser_address(browser_address: str) -> str | None:+ """+ Build a routed VNC URL from a V2 K8s routed browser_address.++ V2 K8s routed browser_address format:+ wss://{domain}/{session_id}/{token}/devtools/browser/{browser_id}++ Returns VNC URL in format:+ wss://{domain}/vnc/{session_id}/{token}++ Returns None if browser_address is not a V2 routed URL.+ """+ if not browser_address:+ return None++ parsed = urlparse(browser_address)++ # Check if this looks like a V2 routed URL (wss:// with token in path)+ if parsed.scheme not in ("wss", "ws"):+ return None++ # Parse path: /{session_id}/{token}/devtools/browser/{browser_id}+ path_parts = parsed.path.strip("/").split("/")+ if len(path_parts) < 4 or path_parts[2] != "devtools":+ return None++ session_id = path_parts[0]+ token = path_parts[1]+ domain = parsed.netloc++ # Build VNC URL with same domain and token+ scheme = "wss" if parsed.scheme == "wss" else "ws"+ return f"{scheme}://{domain}/vnc/{session_id}/{token}"++async def loop_stream_vnc(vnc_channel: VncChannel) -> None:"""Actually stream the VNC data between a frontend and a browser.@@ -288,19 +325,32 @@ async def loop_stream_vnc(vnc_channel: VncChannel) -> None:Loops until the task is cleared or the websocket is closed."""+ vnc_url: str = ""browser_session = vnc_channel.browser_sessionclass_name = vnc_channel.class_nameif not browser_session:raise Exception(f"{class_name} No browser session associated with vnc channel.")- vnc_url = build_vnc_streaming_url(- browser_address=browser_session.browser_address,- ip_address=browser_session.ip_address,- vnc_port=vnc_channel.vnc_port,- )- if not vnc_url:- raise Exception(f"{class_name} Browser session does not expose a VNC endpoint.")+ # First, check if this is a V2 K8s routed session by examining browser_address+ # V2 sessions have browser_address like: wss://{domain}/{session_id}/{token}/devtools/...+ # For these, we need to route VNC through the same nginx proxy+ routed_vnc_url = _build_vnc_url_from_browser_address(browser_session.browser_address)+ if routed_vnc_url:+ vnc_url = routed_vnc_url+ elif browser_session.ip_address:+ # V1 ECS sessions: Direct IP connection (ip_address is a public/reachable IP)+ if ":" in browser_session.ip_address:+ ip, _ = browser_session.ip_address.split(":")+ vnc_url = f"ws://{ip}:{vnc_channel.vnc_port}"+ else:+ vnc_url = f"ws://{browser_session.ip_address}:{vnc_channel.vnc_port}"+ else:+ # Last resort: parse browser_address hostname+ browser_address = browser_session.browser_address+ parsed_browser_address = urlparse(browser_address)+ host = parsed_browser_address.hostname+ vnc_url = f"ws://{host}:{vnc_channel.vnc_port}"LOG.debug(f"{class_name} Connecting to vnc url.",diff --git a/skyvern/forge/sdk/routes/streaming/verify.py b/skyvern/forge/sdk/routes/streaming/verify.pyindex 916a2b55b..0593b984a 100644--- a/skyvern/forge/sdk/routes/streaming/verify.py+++ b/skyvern/forge/sdk/routes/streaming/verify.py@@ -26,7 +26,6 @@ from skyvern.forge import appfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import AddressablePersistentBrowserSession, is_final_statusfrom skyvern.forge.sdk.schemas.tasks import Task, TaskStatusfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus-from skyvern.webeye.vnc_streaming import browser_session_supports_vnc_streamingif t.TYPE_CHECKING:from skyvern.forge.sdk.routes.streaming.channels.message import MessageChannel@@ -81,10 +80,7 @@ async def verify_browser_session(browser_address = browser_session.browser_addressif not browser_address:- if settings.BROWSER_STREAMING_MODE == "cdp" or browser_session_supports_vnc_streaming(- browser_address=browser_session.browser_address,- ip_address=browser_session.ip_address,- ):+ if settings.BROWSER_STREAMING_MODE == "cdp":browser_address = ""else:LOG.info(diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.pyindex 1427869f1..f318d584b 100644--- a/skyvern/webeye/default_persistent_sessions_manager.py+++ b/skyvern/webeye/default_persistent_sessions_manager.py@@ -26,7 +26,6 @@ from skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.cdp_ports import _release_cdp_portfrom skyvern.webeye.persistent_sessions_manager import PersistentSessionsManagerfrom skyvern.webeye.real_browser_manager import RealBrowserManager-from skyvern.webeye.vnc_streaming import is_local_vnc_streaming_enabledLOG = structlog.get_logger()@@ -301,11 +300,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):# screencast/CDP input endpoints can connect. Triggered both by the# in-process CDP streaming mode and by cdp-connect, which forwards to a# remote browser and still needs a registered BrowserState locally.- should_launch = (- settings.BROWSER_STREAMING_MODE == "cdp"- or is_local_vnc_streaming_enabled()- or settings.BROWSER_TYPE == "cdp-connect"- )+ should_launch = settings.BROWSER_STREAMING_MODE == "cdp" or settings.BROWSER_TYPE == "cdp-connect"if should_launch and runnable_id is None:session_id = session.persistent_browser_session_idtask = asyncio.create_task(@@ -513,7 +508,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):async def cleanup_stale_sessions(self) -> None:"""Close sessions left active by a previous process."""- if settings.BROWSER_STREAMING_MODE != "cdp" and not is_local_vnc_streaming_enabled():+ if settings.BROWSER_STREAMING_MODE != "cdp":returnstale_sessions = await self.database.browser_sessions.get_uncompleted_persistent_browser_sessions()for db_session in stale_sessions:diff --git a/skyvern/webeye/schemas.py b/skyvern/webeye/schemas.pyindex 371c777db..9e0db1421 100644--- a/skyvern/webeye/schemas.py+++ b/skyvern/webeye/schemas.py@@ -15,7 +15,6 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import (PersistentBrowserSession,PersistentBrowserType,)-from skyvern.webeye.vnc_streaming import browser_session_supports_vnc_streamingLOG = structlog.get_logger()@@ -141,10 +140,7 @@ class BrowserSessionResponse(BaseModel):runnable_id=browser_session.runnable_id,timeout=browser_session.timeout_minutes,browser_address=browser_session.browser_address,- vnc_streaming_supported=browser_session_supports_vnc_streaming(- browser_address=browser_session.browser_address,- ip_address=browser_session.ip_address,- ),+ vnc_streaming_supported=True if browser_session.ip_address else False,app_url=app_url,started_at=browser_session.started_at,completed_at=browser_session.completed_at,diff --git a/skyvern/webeye/vnc_streaming.py b/skyvern/webeye/vnc_streaming.pydeleted file mode 100644index 83d0fdcc1..000000000--- a/skyvern/webeye/vnc_streaming.py+++ /dev/null@@ -1,72 +0,0 @@-from __future__ import annotations--from urllib.parse import urlparse--from skyvern.config import settings---def _browser_streaming_mode() -> str:- return settings.BROWSER_STREAMING_MODE.strip().lower()---def is_local_vnc_streaming_enabled() -> bool:- return _browser_streaming_mode() == "vnc"---def build_routed_vnc_url(browser_address: str | None) -> str | None:- """- Build a routed VNC URL from a V2 K8s routed browser address.-- V2 browser addresses look like:- wss://{domain}/{session_id}/{token}/devtools/browser/{browser_id}- """- if not browser_address:- return None-- parsed = urlparse(browser_address)- if parsed.scheme not in ("wss", "ws"):- return None-- path_parts = parsed.path.strip("/").split("/")- if len(path_parts) < 4 or path_parts[2] != "devtools":- return None-- session_id = path_parts[0]- token = path_parts[1]- scheme = "wss" if parsed.scheme == "wss" else "ws"- return f"{scheme}://{parsed.netloc}/vnc/{session_id}/{token}"---def browser_session_supports_vnc_streaming(- *,- browser_address: str | None = None,- ip_address: str | None = None,-) -> bool:- return bool(build_routed_vnc_url(browser_address) or ip_address or is_local_vnc_streaming_enabled())---def build_vnc_streaming_url(- *,- browser_address: str | None = None,- ip_address: str | None = None,- vnc_port: int,-) -> str | None:- routed_vnc_url = build_routed_vnc_url(browser_address)- if routed_vnc_url:- return routed_vnc_url-- if ip_address:- if ":" in ip_address:- ip, _ = ip_address.split(":", maxsplit=1)- return f"ws://{ip}:{vnc_port}"- return f"ws://{ip_address}:{vnc_port}"-- if is_local_vnc_streaming_enabled():- return f"ws://127.0.0.1:{vnc_port}"-- if browser_address:- parsed_browser_address = urlparse(browser_address)- if parsed_browser_address.hostname:- return f"ws://{parsed_browser_address.hostname}:{vnc_port}"-- return Nonediff --git a/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py b/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.pyindex 773093aa5..8b0b197a9 100644--- a/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py+++ b/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py@@ -69,44 +69,15 @@ class TestVerifyBrowserSessionLocalShortCircuit:with (patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock,patch("skyvern.forge.sdk.routes.streaming.verify.settings") as settings_mock,- patch("skyvern.webeye.vnc_streaming.settings") as vnc_settings_mock,):app_mock.PERSISTENT_SESSIONS_MANAGER = manager- settings_mock.BROWSER_STREAMING_MODE = "remote"- vnc_settings_mock.BROWSER_STREAMING_MODE = "remote"+ settings_mock.BROWSER_STREAMING_MODE = "vnc"result = await verify_browser_session("bs_test", "o_test")assert result is not Noneassert result.browser_address == "ws://remote:9222"manager.get_browser_address.assert_awaited_once()- @pytest.mark.asyncio- async def test_short_circuits_in_vnc_mode_with_no_address(self) -> None:- session = _make_session(- browser_address=None,- status=PersistentBrowserSessionStatus.running,- )-- manager = MagicMock()- manager.get_session = AsyncMock(return_value=session)- manager.get_browser_address = AsyncMock(- side_effect=AssertionError("must not wait for address in local vnc mode"),- )-- with (- patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock,- patch("skyvern.forge.sdk.routes.streaming.verify.settings") as settings_mock,- patch("skyvern.webeye.vnc_streaming.settings") as vnc_settings_mock,- ):- app_mock.PERSISTENT_SESSIONS_MANAGER = manager- settings_mock.BROWSER_STREAMING_MODE = "vnc"- vnc_settings_mock.BROWSER_STREAMING_MODE = "vnc"- result = await verify_browser_session("bs_test", "o_test")-- assert result is not None- assert result.browser_address == ""- manager.get_browser_address.assert_not_called()-@pytest.mark.asyncioasync def test_uses_existing_address_without_polling(self) -> None:session = _make_session(diff --git a/tests/unit/test_vnc_streaming.py b/tests/unit/test_vnc_streaming.pydeleted file mode 100644index 04a4e260d..000000000--- a/tests/unit/test_vnc_streaming.py+++ /dev/null@@ -1,59 +0,0 @@-from __future__ import annotations--from skyvern.webeye import vnc_streaming---def test_builds_routed_vnc_url_from_browser_address() -> None:- assert (- vnc_streaming.build_routed_vnc_url(- "wss://browser.example.com/pbs_123/token_456/devtools/browser/browser_789",- )- == "wss://browser.example.com/vnc/pbs_123/token_456"- )---def test_local_vnc_mode_supports_session_without_remote_address(monkeypatch) -> None:- monkeypatch.setattr(vnc_streaming.settings, "BROWSER_STREAMING_MODE", "vnc")-- assert vnc_streaming.browser_session_supports_vnc_streaming(- browser_address=None,- ip_address=None,- )- assert (- vnc_streaming.build_vnc_streaming_url(- browser_address=None,- ip_address=None,- vnc_port=6080,- )- == "ws://127.0.0.1:6080"- )---def test_cdp_mode_does_not_advertise_vnc_for_local_session(monkeypatch) -> None:- monkeypatch.setattr(vnc_streaming.settings, "BROWSER_STREAMING_MODE", "cdp")-- assert not vnc_streaming.browser_session_supports_vnc_streaming(- browser_address=None,- ip_address=None,- )- assert (- vnc_streaming.build_vnc_streaming_url(- browser_address=None,- ip_address=None,- vnc_port=6080,- )- is None- )---def test_ip_address_takes_precedence_over_local_mode(monkeypatch) -> None:- monkeypatch.setattr(vnc_streaming.settings, "BROWSER_STREAMING_MODE", "vnc")-- assert (- vnc_streaming.build_vnc_streaming_url(- browser_address=None,- ip_address="10.0.0.5:9222",- vnc_port=6080,- )- == "ws://10.0.0.5:6080"- )