Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Paste your clipboard while in control of a live browser, with one control bar for CDP and VNC (SKY-17125) (#8699)
Shuchang Zheng89cc7f3b4c6e
diff --git a/skyvern-frontend/src/components/BrowserStream.test.tsx b/skyvern-frontend/src/components/BrowserStream.test.tsxindex 915b59b1f..190d52d8a 100644--- a/skyvern-frontend/src/components/BrowserStream.test.tsx+++ b/skyvern-frontend/src/components/BrowserStream.test.tsx@@ -20,6 +20,7 @@ const mocks = vi.hoisted(() => {clipboardPasteFrom: ReturnType<typeof vi.fn>;sendKey: ReturnType<typeof vi.fn>;disconnect: ReturnType<typeof vi.fn>;+ focus: ReturnType<typeof vi.fn>;_framebufferUpdate: () => boolean;}> = [];const autoConnect = { value: true };@@ -40,6 +41,7 @@ const mocks = vi.hoisted(() => {clipboardPasteFrom = vi.fn();sendKey = vi.fn();disconnect = vi.fn();+ focus = vi.fn();_framebufferUpdate = vi.fn(() => true);private listeners: Record<string, RfbListener[]> = {};@@ -192,6 +194,10 @@ vi.mock("@/store/SettingsStore", () => ({useSettingsStore: () => mocks.settingsStore,}));+vi.mock("@/components/RecordingPill", () => ({+ RecordingPill: () => null,+}));+vi.mock("@/components/ui/use-toast", () => ({toast: mocks.toast,}));@@ -603,6 +609,99 @@ describe("BrowserStream", () => {expect(consoleError).toHaveBeenCalledTimes(1);});+ it("pastes the local clipboard over VNC from the control bar and stops control from it", async () => {+ renderBrowserStream();+ fireEvent.click(+ await screen.findByRole(+ "button",+ { name: /take control/i },+ { timeout: 10000 },+ ),+ );++ fireEvent.click(screen.getByRole("button", { name: /^paste$/i }));++ const rfb = mocks.rfbInstances[0]!;+ await waitFor(() =>+ expect(rfb.clipboardPasteFrom).toHaveBeenCalledWith(+ "https://example.test",+ ),+ );+ await waitFor(() =>+ expect(screen.getByRole("status").textContent).toContain(+ "Sent 20 characters",+ ),+ );+ expect(rfb.sendKey.mock.calls).toEqual([+ [0xffe3, "ControlLeft", true],+ [0x0076, "KeyV", true],+ [0x0076, "KeyV", false],+ [0xffe3, "ControlLeft", false],+ ]);+ expect(rfb.focus).toHaveBeenCalled();++ fireEvent.click(screen.getByRole("button", { name: /stop controlling/i }));+ expect(screen.queryByRole("button", { name: /^paste$/i })).toBeNull();+ expect(+ document+ .querySelector(".browser-stream")+ ?.classList.contains("user-is-controlling"),+ ).toBe(false);+ });++ it("pastes an Edit-menu paste over VNC only while in control", async () => {+ const { container } = renderBrowserStream();+ const takeControlButton = await screen.findByRole(+ "button",+ { name: /take control/i },+ { timeout: 10000 },+ );+ const canvas = container.querySelector("canvas")!;+ const menuPaste = () =>+ fireEvent.paste(canvas, {+ clipboardData: { getData: () => "from the menu" },+ });+ const rfb = mocks.rfbInstances[0]!;++ menuPaste();+ expect(rfb.clipboardPasteFrom).not.toHaveBeenCalled();++ fireEvent.click(takeControlButton);+ menuPaste();++ await waitFor(() =>+ expect(rfb.clipboardPasteFrom).toHaveBeenCalledWith("from the menu"),+ );+ await waitFor(() =>+ expect(screen.getByRole("status").textContent).toContain(+ "Sent 13 characters",+ ),+ );+ expect(rfb.sendKey).toHaveBeenCalledWith(0x0076, "KeyV", true);+ });++ it("offers no control bar while recording holds control", async () => {+ mocks.recordingStore.isRecording = true;+ try {+ const { container } = renderBrowserStream();+ await waitFor(+ () =>+ expect(+ container+ .querySelector(".browser-stream")+ ?.classList.contains("user-is-controlling"),+ ).toBe(true),+ { timeout: 10000 },+ );+ await screen.findByTestId("browser-stream-overlay");+ expect(+ screen.queryByRole("button", { name: /stop controlling/i }),+ ).toBeNull();+ } finally {+ mocks.recordingStore.isRecording = false;+ }+ });+it("takes control on a click anywhere on the read-only picture, not just the button", async () => {const { container } = renderBrowserStream();await screen.findByRole(diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 90bf52243..ca585a876 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -4,7 +4,7 @@import _RFB, { type RfbEvent } from "@novnc/novnc/lib/rfb.js";type RFB = _RFB;const RFB = (_RFB as typeof _RFB & { default?: typeof _RFB }).default ?? _RFB;-import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons";+import { InfoCircledIcon } from "@radix-ui/react-icons";import { useEffect, useState, useRef, useCallback } from "react";import { useQuery } from "@tanstack/react-query";@@ -17,7 +17,6 @@ import {} from "@/api/types";import { RecordingPill } from "@/components/RecordingPill";import { Tip } from "@/components/Tip";-import { Button } from "@/components/ui/button";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { statusIsNotFinalized } from "@/routes/tasks/types";@@ -38,8 +37,20 @@ import type {import {VNC_SUPER_L_KEYSYM,handleVncClipboardPasteShortcut,+ pasteTextIntoVnc,type HeldMetaSides,} from "@/components/browserStreamClipboard";+import {+ STREAM_CONTAINER_CLASS,+ toastClipboardReadFailed,+ toastNothingToPaste,+ usePastedNotice,+} from "@/routes/streaming/pasteFeedback";+import {+ PastedNotice,+ StreamControlBar,+ TakeControlButton,+} from "@/routes/streaming/StreamControlBar";import { useRecordingMessageChannel } from "@/routes/streaming/useRecordingMessageChannel";import { useWebSocketParams } from "@/routes/streaming/webSocketParams";@@ -723,6 +734,43 @@ function BrowserStream({const theUserIsControlling =userIsControlling || (interactive && !showControlButtons);+ const [pastedCharacters, showPasted] = usePastedNotice();++ const sendTextToVnc = useCallback(+ async (text: string) => {+ if (!text) {+ toastNothingToPaste();+ return;+ }+ // The clipboard read can wait on a permission prompt; the stream or the+ // user's control may have changed by the time it resolves.+ const rfb = rfbRef.current;+ if (!rfb || !userCanSendVncInputRef.current) {+ return;+ }+ await pasteTextIntoVnc(rfb, text);+ if (rfbRef.current !== rfb) {+ return;+ }+ showPasted(text);+ // Keystrokes after a paste belong to the page, not the button.+ rfb.focus({ preventScroll: true });+ },+ [showPasted],+ );++ const pasteIntoVnc = useCallback(async () => {+ let text: string;+ try {+ text = await navigator.clipboard.readText();+ } catch (err) {+ console.error("Failed to read the clipboard for VNC:", err);+ toastClipboardReadFailed();+ return;+ }+ await sendTextToVnc(text);+ }, [sendTextToVnc]);+useEffect(() => {userCanSendVncInputRef.current = theUserIsControlling;}, [theUserIsControlling]);@@ -765,14 +813,9 @@ function BrowserStream({void handleVncClipboardPasteShortcut(event, rfbRef.current, {getHeldMetaSides: () => heldMetaSidesRef.current,- onPasteError: () => {- toast({- title: "Paste failed",- description:- "Skyvern couldn't read your clipboard. Allow clipboard access for this site and try again.",- variant: "destructive",- });- },+ onPasteError: toastClipboardReadFailed,+ onPasted: showPasted,+ onEmptyClipboard: toastNothingToPaste,});};@@ -803,15 +846,27 @@ function BrowserStream({releaseLeftCmd();};+ // Edit-menu paste arrives only as a paste event; Cmd/Ctrl+V never does,+ // because the shortcut handler prevents its default.+ const handlePaste = (event: ClipboardEvent) => {+ if (!userCanSendVncInputRef.current) {+ return;+ }+ event.preventDefault();+ void sendTextToVnc(event.clipboardData?.getData("text/plain") ?? "");+ };+canvasContainer.addEventListener("keydown", handleKeyDown, true);+ canvasContainer.addEventListener("paste", handlePaste);window.addEventListener("keyup", handleKeyUp, true);window.addEventListener("blur", handleBlur);return () => {canvasContainer.removeEventListener("keydown", handleKeyDown, true);+ canvasContainer.removeEventListener("paste", handlePaste);window.removeEventListener("keyup", handleKeyUp, true);window.removeEventListener("blur", handleBlur);};- }, [canvasContainer]);+ }, [canvasContainer, showPasted, sendTextToVnc]);// Read the flag through a ref so the unmount cleanup stays mount-scoped: a// StrictMode double-mount or transport swap must not cancel a live recording.@@ -889,6 +944,7 @@ function BrowserStream({<divclassName={cn("browser-stream relative flex flex-col items-center justify-center",+ STREAM_CONTAINER_CLASS,{"user-is-controlling": theUserIsControlling,},@@ -911,35 +967,22 @@ function BrowserStream({: undefined}>+ <PastedNotice characters={pastedCharacters} />{showControlButtons && (<div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">- <Button- onClick={() => {- setUserIsControlling(true);- }}- className={cn("control-button pointer-events-auto border", {+ <TakeControlButton+ onClick={() => setUserIsControlling(true)}+ className={cn("control-button pointer-events-auto", {hide: userIsControlling,})}- size="sm"- >- <HandIcon className="mr-2 h-4 w-4" />- take control- </Button>- <Button- onClick={() => {- setUserIsControlling(false);- }}- className={cn(- "control-button pointer-events-auto absolute bottom-0 border",- {- hide: !userIsControlling,- },- )}- size="sm"- >- <ExitIcon className="mr-2 h-4 w-4" />- stop controlling- </Button>+ />+ {/* Recording holds control for capture, so it offers no way to cede it. */}+ {userIsControlling && !isRecording && (+ <StreamControlBar+ onStop={() => setUserIsControlling(false)}+ onPaste={pasteIntoVnc}+ />+ )}</div>)}</div>diff --git a/skyvern-frontend/src/components/browserStreamClipboard.test.ts b/skyvern-frontend/src/components/browserStreamClipboard.test.tsindex 53eb978e9..af7e213df 100644--- a/skyvern-frontend/src/components/browserStreamClipboard.test.ts+++ b/skyvern-frontend/src/components/browserStreamClipboard.test.ts@@ -251,6 +251,25 @@ describe("browserStreamClipboard", () => {expect(rfb.sendKey).not.toHaveBeenCalled();});+ it("reports an empty clipboard instead of pasting nothing", async () => {+ const event = pasteEvent();+ const rfb = rfbMock();+ const onEmptyClipboard = vi.fn();+ const onPasted = vi.fn();++ await handleVncClipboardPasteShortcut(event, rfb, {+ onEmptyClipboard,+ onPasted,+ readClipboardText: async () => "",+ syncDelayMs: 0,+ });++ expect(onEmptyClipboard).toHaveBeenCalledTimes(1);+ expect(onPasted).not.toHaveBeenCalled();+ expect(rfb.clipboardPasteFrom).not.toHaveBeenCalled();+ expect(rfb.sendKey).not.toHaveBeenCalled();+ });+it("does not intercept non-paste keys", async () => {const event = pasteEvent({ key: "x" });const rfb = rfbMock();diff --git a/skyvern-frontend/src/components/browserStreamClipboard.ts b/skyvern-frontend/src/components/browserStreamClipboard.tsindex 7ae4d841b..3e309392d 100644--- a/skyvern-frontend/src/components/browserStreamClipboard.ts+++ b/skyvern-frontend/src/components/browserStreamClipboard.ts@@ -60,6 +60,23 @@ function sendVncPasteShortcut(rfb.sendKey(VNC_CONTROL_LEFT_KEYSYM, "ControlLeft", false);}+async function syncTextToVnc(+ rfb: VncClipboardRfb,+ text: string,+ syncDelayMs = DEFAULT_VNC_CLIPBOARD_SYNC_DELAY_MS,+) {+ rfb.clipboardPasteFrom(text);+ if (syncDelayMs > 0) {+ await sleep(syncDelayMs);+ }+}++// For a paste started without a keystroke (a button), so no modifiers are held.+async function pasteTextIntoVnc(rfb: VncClipboardRfb, text: string) {+ await syncTextToVnc(rfb, text);+ sendVncPasteShortcut(rfb);+}+async function handleVncClipboardPasteShortcut(event: PasteShortcutEvent,rfb: VncClipboardRfb | null,@@ -67,6 +84,8 @@ async function handleVncClipboardPasteShortcut(readClipboardText?: () => Promise<string>;syncDelayMs?: number;onPasteError?: (err: unknown) => void;+ onPasted?: (text: string) => void;+ onEmptyClipboard?: () => void;getHeldMetaSides?: () => HeldMetaSides;},) {@@ -76,8 +95,10 @@ async function handleVncClipboardPasteShortcut(const {readClipboardText = () => navigator.clipboard.readText(),- syncDelayMs = DEFAULT_VNC_CLIPBOARD_SYNC_DELAY_MS,+ syncDelayMs,onPasteError,+ onPasted,+ onEmptyClipboard,getHeldMetaSides,} = options ?? {};@@ -85,17 +106,20 @@ async function handleVncClipboardPasteShortcut(event.stopPropagation();event.stopImmediatePropagation();+ let text: string;try {- const text = await readClipboardText();- rfb.clipboardPasteFrom(text);- if (syncDelayMs > 0) {- await sleep(syncDelayMs);+ text = await readClipboardText();+ if (!text) {+ onEmptyClipboard?.();+ return true;}+ await syncTextToVnc(rfb, text, syncDelayMs);} catch (err) {console.error("Failed to sync clipboard contents to VNC:", err);onPasteError?.(err);return true;}+ onPasted?.(text);if (!event.metaKey) {sendVncPasteShortcut(rfb);@@ -131,5 +155,6 @@ export {VNC_SUPER_L_KEYSYM,handleVncClipboardPasteShortcut,isClipboardPasteShortcut,+ pasteTextIntoVnc,sendVncPasteShortcut,};diff --git a/skyvern-frontend/src/novnc.d.ts b/skyvern-frontend/src/novnc.d.tsindex ebbbd40aa..8132f4fde 100644--- a/skyvern-frontend/src/novnc.d.ts+++ b/skyvern-frontend/src/novnc.d.ts@@ -39,6 +39,7 @@ declare module "@novnc/novnc/lib/rfb.js" {removeEventListener(event: string, listener: (e: RfbEvent) => void): void;clipboardPasteFrom(text: string): void;disconnect(): void;+ focus(options?: FocusOptions): void;sendKey(keysym: number, code: string, down?: boolean): void;viewportChange(): void;}diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.test.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.test.tsxindex f5b235cb8..69bd0f902 100644--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.test.tsx+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.test.tsx@@ -24,6 +24,7 @@ const cdpInputState = vi.hoisted(() => ({viewportHeight: 0,clipboardPasteEnabled: false,clipboardCopyEnabled: false,+ onClipboardPaste: undefined as ((text: string) => void) | undefined,userIsControlling: false,inputReady: false,setUserIsControlling: undefined as ((value: boolean) => void) | undefined,@@ -71,6 +72,7 @@ vi.mock("@/routes/streaming/useCdpInput", async (importOriginal) => {cdpInputState.viewportHeight = viewportHeight;cdpInputState.clipboardPasteEnabled = Boolean(options.onClipboardPaste);cdpInputState.clipboardCopyEnabled = Boolean(options.onClipboardCopy);+ cdpInputState.onClipboardPaste = options.onClipboardPaste;cdpInputState.onInput = options.onInput;const result = actual.useCdpInput(options);cdpInputState.userIsControlling = result.userIsControlling;@@ -149,6 +151,14 @@ class FakeStreamSocket {}}+const messageSockets = (browserSessionId: string) =>+ FakeStreamSocket.instances.filter((socket) =>+ socket.url.includes(`/stream/messages/browser_session/${browserSessionId}`),+ );++const sentMessages = (socket: FakeStreamSocket) =>+ socket.send.mock.calls.map((call) => JSON.parse(String(call[0])));+function stubAnimationFrame(): void {vi.stubGlobal("requestAnimationFrame",@@ -379,39 +389,85 @@ describe("BrowserSessionStream terminal statuses", () => {});});- it("enables recording clipboard interception only while the message socket is connected", async () => {+ it("opens the message channel while the user is in control and pastes over it without recording", async () => {+ vi.useFakeTimers();const view = render(- <BrowserSessionStream browserSessionId="pbs_test" exfiltrate={true} />,+ <BrowserSessionStream browserSessionId="pbs_test" showControlButtons />,);+ const sentKinds = (socket: FakeStreamSocket) =>+ sentMessages(socket).map((message) => message.kind);+ try {+ await act(async () => Promise.resolve());+ expect(messageSockets("pbs_test")).toHaveLength(0);- await waitFor(() => expect(FakeStreamSocket.instances).toHaveLength(2));- const messageSocket = FakeStreamSocket.instances.find((socket) =>- socket.url.includes("/stream/messages/browser_session/pbs_test"),- );- expect(messageSocket).toBeTruthy();- expect(cdpInputState).toMatchObject({- clipboardPasteEnabled: false,- clipboardCopyEnabled: false,- });+ act(() => cdpInputState.setUserIsControlling?.(true));+ await act(async () => Promise.resolve());+ expect(messageSockets("pbs_test")).toHaveLength(1);+ const first = messageSockets("pbs_test")[0]!;+ act(() => first.emit("open", new Event("open")));++ act(() => cdpInputState.onClipboardPaste?.("hello"));+ expect(sentMessages(first)).toContainEqual({+ kind: "clipboard-paste",+ text: "hello",+ });+ expect(sentKinds(first)).not.toContain("begin-exfiltration");- act(() => messageSocket?.emit("open", new Event("open")));- await waitFor(() =>- expect(cdpInputState).toMatchObject({- clipboardPasteEnabled: true,- clipboardCopyEnabled: true,- }),- );+ // A drop while in control reconnects instead of leaving paste dead.+ act(() => first.emit("close", new CloseEvent("close", { code: 1006 })));+ await act(async () => vi.advanceTimersByTimeAsync(1000));+ await act(async () => Promise.resolve());+ expect(messageSockets("pbs_test")).toHaveLength(2);+ const second = messageSockets("pbs_test")[1]!;+ act(() => second.emit("open", new Event("open")));+ expect(sentKinds(second)).not.toContain("begin-exfiltration");- act(() =>- messageSocket?.emit("close", new CloseEvent("close", { code: 1006 })),- );- await waitFor(() =>- expect(cdpInputState).toMatchObject({- clipboardPasteEnabled: false,- clipboardCopyEnabled: false,- }),+ act(() => cdpInputState.setUserIsControlling?.(false));+ await act(async () => Promise.resolve());+ expect(second.close).toHaveBeenCalled();+ await act(async () => vi.advanceTimersByTimeAsync(1000));+ expect(messageSockets("pbs_test")).toHaveLength(2);+ } finally {+ view.unmount();+ vi.useRealTimers();+ }+ });++ it("recycles the message socket when a recording ends while the user keeps control", async () => {+ vi.useFakeTimers();+ const view = render(+ <BrowserSessionStream browserSessionId="pbs_test" showControlButtons />,);- view.unmount();+ try {+ await act(async () => Promise.resolve());+ act(() => cdpInputState.setUserIsControlling?.(true));+ await act(async () => Promise.resolve());+ const controlSocket = messageSockets("pbs_test")[0]!;+ act(() => controlSocket.emit("open", new Event("open")));++ view.rerender(+ <BrowserSessionStream+ browserSessionId="pbs_test"+ showControlButtons+ exfiltrate={true}+ />,+ );+ await act(async () => Promise.resolve());+ expect(messageSockets("pbs_test")).toHaveLength(1);+ view.rerender(+ <BrowserSessionStream browserSessionId="pbs_test" showControlButtons />,+ );+ await act(async () => Promise.resolve());++ // The recording's socket must close (its cleanup finalizes the recording)+ // rather than carry that recording's identity into the next one.+ expect(controlSocket.close).toHaveBeenCalled();+ expect(messageSockets("pbs_test")).toHaveLength(2);+ expect(cdpInputState.userIsControlling).toBe(true);+ } finally {+ view.unmount();+ vi.useRealTimers();+ }});it("takes control for recording and only cedes a recording-owned grab", async () => {diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsxindex 8d008e1d0..56763d48a 100644--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx@@ -2,8 +2,17 @@ import { useCallback, useEffect, useRef, useState } from "react";import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { newWssBaseUrl, getCredentialParam } from "@/util/env";import { useCdpInput } from "@/routes/streaming/useCdpInput";-import { useRecordingMessageChannel } from "@/routes/streaming/useRecordingMessageChannel";+import {+ useRecordingMessageChannel,+ type Command,+} from "@/routes/streaming/useRecordingMessageChannel";+import { toast } from "@/components/ui/use-toast";import { InteractiveStreamView } from "@/routes/streaming/InteractiveStreamView";+import {+ toastClipboardReadFailed,+ toastNothingToPaste,+ usePastedNotice,+} from "@/routes/streaming/pasteFeedback";import {markCommit,markLoad,@@ -48,6 +57,9 @@ type StreamMessage = {url?: string;};+// Mirrors MAX_CLIPBOARD_PASTE_BYTES in skyvern/forge/sdk/routes/streaming/payload_limits.py.+const MAX_CLIPBOARD_PASTE_BYTES = 1024 * 1024;+const STARTING_DIAGNOSTIC: StreamDiagnostic = {title: "Waking up your local browser",detail: "Opening the stream and waiting for the first frame...",@@ -159,6 +171,13 @@ function BrowserSessionStream({// the close handler so a reconnect notice augments that reason instead of// replacing it with a generic "closed with code 1000".const streamEndedDiagnosticRef = useRef<StreamDiagnostic | null>(null);+ const keepMessageChannelAliveRef = useRef(false);+ // Clipboard callbacks go to useCdpInput before the message channel exists (the+ // channel opens on its userIsControlling), so they reach the channel through this.+ const sendMessageCommandRef = useRef<(command: Command) => boolean>(+ () => false,+ );+ const [pastedCharacters, showPasted] = usePastedNotice();exfiltrateRef.current = !!exfiltrate;const scheduleRecordingReconnect = useCallback(() => {@@ -167,7 +186,7 @@ function BrowserSessionStream({}recordingReconnectTimerRef.current = setTimeout(() => {recordingReconnectTimerRef.current = null;- if (exfiltrateRef.current) {+ if (keepMessageChannelAliveRef.current) {setRecordingReconnectTrigger((trigger) => trigger + 1);}}, 1000);@@ -187,7 +206,7 @@ function BrowserSessionStream({return;}recordingChannelDisconnectedRef.current = true;- if (exfiltrateRef.current) {+ if (keepMessageChannelAliveRef.current) {scheduleRecordingReconnect();}},@@ -202,36 +221,50 @@ function BrowserSessionStream({};}, []);- const recordingChannelEnabled = exfiltrate !== undefined;- const { isMessageConnected, sendCommand: sendRecordingCommand } =- useRecordingMessageChannel({- browserSessionId,- enabled: recordingChannelEnabled,- exfiltrate: !!exfiltrate,- workflowPermanentId: workflowPermanentId ?? null,- clipboard: "message",- reconnectTrigger: recordingReconnectTrigger,- onConnectionChange: handleRecordingConnectionChange,- });+ const sendClipboardCommand = useCallback(+ (command: Command, failureTitle: string) => {+ const sent = sendMessageCommandRef.current(command);+ if (!sent) {+ toast({+ variant: "destructive",+ title: failureTitle,+ description:+ "Still connecting to the browser. Try again in a moment.",+ });+ }+ return sent;+ },+ [],+ );- useEffect(() => {- if (- exfiltrate &&- !isMessageConnected &&- recordingChannelDisconnectedRef.current- ) {- scheduleRecordingReconnect();- }- }, [exfiltrate, isMessageConnected, scheduleRecordingReconnect]);const onClipboardPaste = useCallback((text: string) => {- sendRecordingCommand({ kind: "clipboard-paste", text });+ if (!text) {+ toastNothingToPaste();+ return;+ }+ // Checked here too so an oversized paste never flashes a success notice+ // before the backend's rejection arrives.+ if (new TextEncoder().encode(text).length > MAX_CLIPBOARD_PASTE_BYTES) {+ toast({+ variant: "destructive",+ title: "Paste failed",+ description: "Your clipboard text is over 1 MB, too large to paste.",+ });+ return;+ }+ if (+ !sendClipboardCommand({ kind: "clipboard-paste", text }, "Paste failed")+ ) {+ return;+ }+ showPasted(text);},- [sendRecordingCommand],+ [sendClipboardCommand, showPasted],);const onClipboardCopy = useCallback(() => {- sendRecordingCommand({ kind: "clipboard-copy" });- }, [sendRecordingCommand]);+ sendClipboardCommand({ kind: "clipboard-copy" }, "Copy failed");+ }, [sendClipboardCommand]);// The CDP input socket must be wired whenever the stream can be controlled,// whether by default interaction or via the take-control button.@@ -250,16 +283,64 @@ function BrowserSessionStream({navigate,historyNavigate,navigateError,+ pasteClipboard,} = useCdpInput({inputWsUrl,interactive: controllable,viewportWidth,viewportHeight,- onClipboardPaste:- exfiltrate && isMessageConnected ? onClipboardPaste : undefined,- onClipboardCopy:- exfiltrate && isMessageConnected ? onClipboardCopy : undefined,+ onClipboardPaste: controllable ? onClipboardPaste : undefined,+ onClipboardPasteError: toastClipboardReadFailed,+ onClipboardCopy: controllable ? onClipboardCopy : undefined,+ // Recording keeps copy local so the keystroke never lands in the capture.+ forwardCopyShortcut: !exfiltrate,});+ const keepMessageChannelAlive = !!exfiltrate || userIsControlling;+ keepMessageChannelAliveRef.current = keepMessageChannelAlive;++ // Outside a recording the channel is open only while the user is in control, so+ // pasting works without holding a socket per passive viewer. exfiltrate stays+ // false then, so opening it never sends begin-exfiltration.+ const recordingChannelActive = exfiltrate !== undefined;+ const messageChannelEnabled = recordingChannelActive || userIsControlling;+ const previousRecordingChannelActiveRef = useRef(recordingChannelActive);+ useEffect(() => {+ // Closing the socket is what finalizes a recording and clears its retained+ // identity, so a user still in control after one gets a fresh socket.+ if (previousRecordingChannelActiveRef.current && !recordingChannelActive) {+ setRecordingReconnectTrigger((trigger) => trigger + 1);+ }+ previousRecordingChannelActiveRef.current = recordingChannelActive;+ }, [recordingChannelActive]);+ const { isMessageConnected, sendCommand: sendMessageCommand } =+ useRecordingMessageChannel({+ browserSessionId,+ enabled: messageChannelEnabled,+ exfiltrate: !!exfiltrate,+ workflowPermanentId: workflowPermanentId ?? null,+ clipboard: "message",+ reconnectTrigger: recordingReconnectTrigger,+ onConnectionChange: handleRecordingConnectionChange,+ });+ sendMessageCommandRef.current = sendMessageCommand;++ useEffect(() => {+ if (!messageChannelEnabled) {+ // A drop from a closed channel must not trigger a reconnect of the next one.+ recordingChannelDisconnectedRef.current = false;+ } else if (+ keepMessageChannelAlive &&+ !isMessageConnected &&+ recordingChannelDisconnectedRef.current+ ) {+ scheduleRecordingReconnect();+ }+ }, [+ keepMessageChannelAlive,+ isMessageConnected,+ messageChannelEnabled,+ scheduleRecordingReconnect,+ ]);useEffect(() => {const recordingStarted =@@ -671,6 +752,8 @@ function BrowserSessionStream({onFrameWidthChange={onFrameWidthChange}frameToken={streamImgToken}onFrameLoad={markLoad}+ onPasteClipboard={controllable ? pasteClipboard : undefined}+ pastedCharacters={pastedCharacters}/>);}diff --git a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.test.tsx b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.test.tsxindex d1c1f4db5..b89506d39 100644--- a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.test.tsx+++ b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.test.tsx@@ -35,6 +35,7 @@ function baseProps() {handleMouseMove: vi.fn(),handleKeyDown: vi.fn(),handleKeyUp: vi.fn(),+ handlePaste: vi.fn(),},currentUrl: "https://example.com/",};@@ -235,9 +236,28 @@ describe("InteractiveStreamView take-control overlay", () => {expect(screen.queryByTestId("take-control-overlay")).toBeNull();expect(- screen.getByRole("button", { name: /stop controlling/ }),+ screen.getByRole("button", { name: /stop controlling/i }),).toBeTruthy();});++ it("offers Paste in the control bar only when the stream can paste", () => {+ const { rerender } = render(+ <InteractiveStreamView {...baseProps()} userIsControlling />,+ );+ expect(screen.queryByRole("button", { name: /paste/i })).toBeNull();++ const onPasteClipboard = vi.fn();+ rerender(+ <InteractiveStreamView+ {...baseProps()}+ userIsControlling+ onPasteClipboard={onPasteClipboard}+ />,+ );+ fireEvent.click(screen.getByRole("button", { name: /paste/i }));++ expect(onPasteClipboard).toHaveBeenCalledTimes(1);+ });});describe("InteractiveStreamView browser chrome", () => {diff --git a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsxindex 9714402d5..7e227c07e 100644--- a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx+++ b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx@@ -2,13 +2,10 @@ import { useEffect, useRef, useState, type RefObject } from "react";import {ArrowLeftIcon,ArrowRightIcon,- ExitIcon,GlobeIcon,- HandIcon,ReloadIcon,} from "@radix-ui/react-icons";import { ZoomableImage } from "@/components/ZoomableImage";-import { Button } from "@/components/ui/button";import { Input } from "@/components/ui/input";import {Tooltip,@@ -17,6 +14,12 @@ import {TooltipTrigger,} from "@/components/ui/tooltip";import { cn } from "@/util/utils";+import { STREAM_CONTAINER_CLASS } from "./pasteFeedback";+import {+ PastedNotice,+ StreamControlBar,+ TakeControlButton,+} from "./StreamControlBar";import type { HistoryAction } from "./useCdpInput";interface InteractiveStreamViewProps {@@ -34,6 +37,7 @@ interface InteractiveStreamViewProps {handleMouseMove: (e: React.MouseEvent<HTMLImageElement>) => void;handleKeyDown: (e: React.KeyboardEvent) => void;handleKeyUp: (e: React.KeyboardEvent) => void;+ handlePaste: (e: React.ClipboardEvent) => void;};currentUrl?: string;centered?: boolean;@@ -51,6 +55,10 @@ interface InteractiveStreamViewProps {onFrameWidthChange?: (width: number | null) => void;frameToken?: number;onFrameLoad?: (token: number) => void;+ // Omitted when the stream can't paste; the control bar then hides its paste controls.+ onPasteClipboard?: () => void;+ // Set briefly after each paste is sent; null hides the confirmation.+ pastedCharacters?: number | null;}function UrlBar({@@ -202,6 +210,8 @@ function InteractiveStreamView({onFrameWidthChange,frameToken,onFrameLoad,+ onPasteClipboard,+ pastedCharacters,}: InteractiveStreamViewProps) {const imgDataUrl = `data:image/${streamFormat};base64,${streamImgSrc}`;const imgRef = useRef<HTMLImageElement>(null);@@ -343,25 +353,22 @@ function InteractiveStreamView({className="absolute inset-0 z-10 flex cursor-pointer items-center justify-center"onClick={() => setUserIsControlling(true)}>- <Button- size="sm"- className="border"- onClick={() => setUserIsControlling(true)}- >- <HandIcon className="mr-2 h-4 w-4" />- take control- </Button>+ <TakeControlButton onClick={() => setUserIsControlling(true)} /></div>)}+ <PastedNotice characters={pastedCharacters ?? null} />{showControlButtons && userIsControlling && (- <Button- size="sm"- className="absolute bottom-2 left-1/2 z-10 -translate-x-1/2 border"- onClick={() => setUserIsControlling(false)}- >- <ExitIcon className="mr-2 h-4 w-4" />- stop controlling- </Button>+ <StreamControlBar+ onStop={() => setUserIsControlling(false)}+ onPaste={+ onPasteClipboard &&+ (() => {+ onPasteClipboard();+ // Keystrokes after a paste belong to the page, not the button.+ containerRef.current?.focus();+ })+ }+ />)}</>);@@ -369,10 +376,14 @@ function InteractiveStreamView({return (<divref={containerRef}- className="relative h-full w-full outline-none"+ className={cn(+ "relative h-full w-full outline-none",+ STREAM_CONTAINER_CLASS,+ )}tabIndex={0}onKeyDown={handlers.handleKeyDown}onKeyUp={handlers.handleKeyUp}+ onPaste={handlers.handlePaste}>{/* Chrome and viewport share previewWidth so the window sizes to the letterboxedpicture rather than the pane, the way a real browser window frames its page.diff --git a/skyvern-frontend/src/routes/streaming/StreamControlBar.tsx b/skyvern-frontend/src/routes/streaming/StreamControlBar.tsxnew file mode 100644index 000000000..c6c1df1ca--- /dev/null+++ b/skyvern-frontend/src/routes/streaming/StreamControlBar.tsx@@ -0,0 +1,102 @@+import { ClipboardIcon, ExitIcon, HandIcon } from "@radix-ui/react-icons";+import { Button } from "@/components/ui/button";+import { isMacPlatform } from "@/util/platform";+import { cn } from "@/util/utils";++const PASTE_SHORTCUT = isMacPlatform() ? "⌘V" : "Ctrl+V";++export function PastedNotice({ characters }: { characters: number | null }) {+ return (+ // Stays mounted so screen readers announce the text when it changes.+ <div+ role="status"+ className="pointer-events-none absolute bottom-14 left-1/2 z-10 -translate-x-1/2"+ >+ {characters != null && (+ <div className="whitespace-nowrap rounded-md bg-foreground px-3 py-1.5 text-xs text-background shadow-md">+ Sent {characters} {characters === 1 ? "character" : "characters"} to+ the browser+ </div>+ )}+ </div>+ );+}++export function StreamControlBar({+ onStop,+ onPaste,+}: {+ onStop: () => void;+ // Omitted when the stream can't paste; the bar then hides its paste controls.+ onPaste?: () => void;+}) {+ return (+ <div+ className="pointer-events-auto absolute bottom-2 left-1/2 z-10 flex max-w-[calc(100%-1rem)] -translate-x-1/2 items-center gap-2 overflow-hidden whitespace-nowrap rounded-md border bg-background py-1 pl-3 pr-1 text-xs text-foreground shadow-md"+ // The stream container forwards keys to the remote page; Enter/Space here+ // must activate the bar's own buttons instead.+ onKeyDown={(e) => e.stopPropagation()}+ onKeyUp={(e) => e.stopPropagation()}+ >+ <span className="flex min-w-0 items-center gap-1.5 font-medium">+ <span className="h-1.5 w-1.5 shrink-0 rounded-full bg-destructive" />+ <span className="truncate">You're in control</span>+ </span>+ {onPaste && (+ <>+ {/* The bar measures ~511px with the full hint, ~382px with the key alone. */}+ <span className="hidden shrink-0 items-center gap-1.5 text-muted-foreground [@container_stream_(min-width:420px)]:flex">+ <kbd className="rounded border bg-muted px-1.5 font-mono text-[11px] text-foreground">+ {PASTE_SHORTCUT}+ </kbd>+ <span className="hidden [@container_stream_(min-width:540px)]:inline">+ pastes your clipboard+ </span>+ </span>+ <Button+ size="sm"+ variant="outline"+ // Too narrow for both buttons: keep Stop controlling, the only way out.+ className="h-7 shrink-0 px-2.5 text-xs [@container_stream_(max-width:280px)]:hidden"+ onClick={onPaste}+ >+ <ClipboardIcon className="mr-1.5 h-3.5 w-3.5" />+ Paste+ </Button>+ </>+ )}+ <Button+ size="sm"+ className="h-7 shrink-0 px-2.5 text-xs"+ onClick={onStop}+ >+ <ExitIcon className="mr-1.5 h-3.5 w-3.5" />+ Stop controlling+ </Button>+ </div>+ );+}++export function TakeControlButton({+ onClick,+ className,+}: {+ onClick: () => void;+ className?: string;+}) {+ // Same surface as StreamControlBar so the two control states read as one set.+ return (+ <Button+ size="sm"+ variant="outline"+ className={cn(+ "border-border bg-background text-foreground shadow-md",+ className,+ )}+ onClick={onClick}+ >+ <HandIcon className="mr-2 h-4 w-4" />+ Take control+ </Button>+ );+}diff --git a/skyvern-frontend/src/routes/streaming/pasteFeedback.ts b/skyvern-frontend/src/routes/streaming/pasteFeedback.tsnew file mode 100644index 000000000..8d7ed7f8a--- /dev/null+++ b/skyvern-frontend/src/routes/streaming/pasteFeedback.ts@@ -0,0 +1,50 @@+import { useCallback, useEffect, useRef, useState } from "react";+import { toast } from "@/components/ui/use-toast";++const PASTE_NOTICE_MS = 2500;++// The bar's container queries resolve against the nearest element carrying these.+export const STREAM_CONTAINER_CLASS =+ "[container-name:stream] [container-type:inline-size]";++export function toastNothingToPaste() {+ toast({+ title: "Nothing to paste",+ description: "Your clipboard has no text to paste.",+ });+}++export function toastClipboardReadFailed() {+ toast({+ variant: "destructive",+ title: "Paste failed",+ description:+ "Skyvern couldn't read your clipboard. Allow clipboard access for this site and try again.",+ });+}++export function usePastedNotice() {+ const [pastedCharacters, setPastedCharacters] = useState<number | null>(null);+ const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);++ useEffect(() => {+ return () => {+ if (timerRef.current) {+ clearTimeout(timerRef.current);+ }+ };+ }, []);++ const showPasted = useCallback((text: string) => {+ setPastedCharacters(Array.from(text).length);+ if (timerRef.current) {+ clearTimeout(timerRef.current);+ }+ timerRef.current = setTimeout(() => {+ timerRef.current = null;+ setPastedCharacters(null);+ }, PASTE_NOTICE_MS);+ }, []);++ return [pastedCharacters, showPasted] as const;+}diff --git a/skyvern-frontend/src/routes/streaming/useCdpInput.test.ts b/skyvern-frontend/src/routes/streaming/useCdpInput.test.tsindex 493df3307..83319d390 100644--- a/skyvern-frontend/src/routes/streaming/useCdpInput.test.ts+++ b/skyvern-frontend/src/routes/streaming/useCdpInput.test.ts@@ -76,6 +76,7 @@ async function renderControllingInputHook(clipboardCallbacks: {onClipboardPaste?: (text: string) => void;onClipboardCopy?: () => void;+ forwardCopyShortcut?: boolean;} = {},) {const { result } = renderHook(() =>@@ -703,6 +704,31 @@ describe("useCdpInput key handling", () => {expect(onClipboardPaste).not.toHaveBeenCalled();});+ it("syncs Cmd+C and still delivers it to the page only when forwarding is on", async () => {+ for (const forwardCopyShortcut of [true, false]) {+ const onClipboardCopy = vi.fn();+ const result = await renderControllingInputHook({+ onClipboardCopy,+ forwardCopyShortcut,+ });+ const send = latestSocketSend();+ send.mockClear();++ act(() => {+ result.current.handlers.handleKeyDown(+ fakeKeyboardEvent("c", "KeyC", { metaKey: true }),+ );+ result.current.handlers.handleKeyUp(fakeKeyboardEvent("c", "KeyC"));+ });++ expect(onClipboardCopy).toHaveBeenCalledTimes(1);+ const sentCodes = send.mock.calls.map(+ (call) => JSON.parse(String(call[0])).code,+ );+ expect(sentCodes).toEqual(forwardCopyShortcut ? ["KeyC", "KeyC"] : []);+ }+ });+it("forwards an ordinary key pair after intercepting a clipboard chord", async () => {Object.defineProperty(navigator, "clipboard", {configurable: true,diff --git a/skyvern-frontend/src/routes/streaming/useCdpInput.ts b/skyvern-frontend/src/routes/streaming/useCdpInput.tsindex 36e711e98..70bbfde51 100644--- a/skyvern-frontend/src/routes/streaming/useCdpInput.ts+++ b/skyvern-frontend/src/routes/streaming/useCdpInput.ts@@ -21,7 +21,11 @@ interface UseCdpInputOptions {viewportWidth: number;viewportHeight: number;onClipboardPaste?: (text: string) => void;+ onClipboardPasteError?: () => void;onClipboardCopy?: () => void;+ // Also deliver Cmd/Ctrl+C to the remote page, so apps with their own+ // selection model (canvas grids, editors) still copy there.+ forwardCopyShortcut?: boolean;onInput?: () => void;}@@ -36,10 +40,12 @@ interface UseCdpInputReturn {handleMouseMove: (e: React.MouseEvent<HTMLImageElement>) => void;handleKeyDown: (e: React.KeyboardEvent) => void;handleKeyUp: (e: React.KeyboardEvent) => void;+ handlePaste: (e: React.ClipboardEvent) => void;};navigate: (url: string) => void;historyNavigate: (action: HistoryAction) => void;navigateError: string | null;+ pasteClipboard: () => void;}export type HistoryAction = "back" | "forward" | "reload";@@ -55,13 +61,23 @@ const NAVIGATE_ERROR_MESSAGES: Record<string, string> = {invalid_url: "Enter a valid http(s) URL.",};+function isEditableTarget(target: EventTarget | null) {+ return (+ target instanceof HTMLInputElement ||+ target instanceof HTMLTextAreaElement ||+ (target instanceof HTMLElement && target.isContentEditable)+ );+}+export function useCdpInput({inputWsUrl,interactive,viewportWidth,viewportHeight,onClipboardPaste,+ onClipboardPasteError,onClipboardCopy,+ forwardCopyShortcut = false,onInput,}: UseCdpInputOptions): UseCdpInputReturn {const [userIsControlling, setUserIsControlling] = useState(false);@@ -98,12 +114,33 @@ export function useCdpInput({};const interceptedClipboardKeysRef = useRef(new Set<string>());const onClipboardPasteRef = useRef(onClipboardPaste);+ const onClipboardPasteErrorRef = useRef(onClipboardPasteError);const onClipboardCopyRef = useRef(onClipboardCopy);const onInputRef = useRef(onInput);onClipboardPasteRef.current = onClipboardPaste;+ onClipboardPasteErrorRef.current = onClipboardPasteError;onClipboardCopyRef.current = onClipboardCopy;+ const forwardCopyShortcutRef = useRef(forwardCopyShortcut);+ forwardCopyShortcutRef.current = forwardCopyShortcut;onInputRef.current = onInput;+ const pasteClipboard = useCallback(() => {+ if (!interactive || !userIsControlling || !onClipboardPasteRef.current) {+ return;+ }+ if (typeof navigator.clipboard?.readText !== "function") {+ onClipboardPasteErrorRef.current?.();+ return;+ }+ navigator.clipboard+ .readText()+ .then((text) => onClipboardPasteRef.current?.(text))+ .catch((error) => {+ console.error("Failed to read clipboard contents:", error);+ onClipboardPasteErrorRef.current?.();+ });+ }, [interactive, userIsControlling]);+useEffect(() => {if (!interactive || !inputWsUrl) return;@@ -440,22 +477,15 @@ export function useCdpInput({const key = e.key.toLowerCase();if (clipboardModifier && key === "v" && onClipboardPasteRef.current) {interceptedClipboardKeysRef.current.add(e.code);- if (!navigator.clipboard) {- console.warn("Clipboard API not available.");- return;- }- navigator.clipboard- .readText()- .then((text) => onClipboardPasteRef.current?.(text))- .catch((error) => {- console.error("Failed to read clipboard contents:", error);- });+ pasteClipboard();return;}if (clipboardModifier && key === "c" && onClipboardCopyRef.current) {- interceptedClipboardKeysRef.current.add(e.code);onClipboardCopyRef.current();- return;+ if (!forwardCopyShortcutRef.current) {+ interceptedClipboardKeysRef.current.add(e.code);+ return;+ }}const isPrintable = e.key.length === 1;const windowsVirtualKeyCode = virtualKeyCodeFor(e);@@ -472,7 +502,20 @@ export function useCdpInput({}sendInputEvent(payload);},- [interactive, userIsControlling, sendInputEvent],+ [interactive, userIsControlling, sendInputEvent, pasteClipboard],+ );++ // Menu-driven pastes arrive only as a paste event; Cmd/Ctrl+V never does,+ // because handleKeyDown prevents its default.+ const handlePaste = useCallback(+ (e: React.ClipboardEvent) => {+ if (!interactive || !userIsControlling) return;+ // The URL bar input lives inside the container; its own paste must stay local.+ if (!onClipboardPasteRef.current || isEditableTarget(e.target)) return;+ e.preventDefault();+ onClipboardPasteRef.current(e.clipboardData.getData("text/plain"));+ },+ [interactive, userIsControlling],);const handleKeyUp = useCallback(@@ -527,9 +570,11 @@ export function useCdpInput({handleMouseMove,handleKeyDown,handleKeyUp,+ handlePaste,},navigate,historyNavigate,navigateError,+ pasteClipboard,};}diff --git a/skyvern-frontend/src/routes/streaming/useRecordingMessageChannel.ts b/skyvern-frontend/src/routes/streaming/useRecordingMessageChannel.tsindex a6357ad55..9bb8fcb7a 100644--- a/skyvern-frontend/src/routes/streaming/useRecordingMessageChannel.ts+++ b/skyvern-frontend/src/routes/streaming/useRecordingMessageChannel.ts@@ -96,8 +96,8 @@ export type Command =export interface UseRecordingMessageChannelResult {messageSocket: WebSocket | null;isMessageConnected: boolean;- /** send a command over the message socket if open */- sendCommand: (command: Command) => void;+ /** send a command over the message socket if open; false when it was not sent */+ sendCommand: (command: Command) => boolean;}const messageInKinds = [@@ -247,6 +247,11 @@ function getMessage(data: unknown): MessageIn | undefined {}}+const CLIPBOARD_FAILURE_TITLES: Record<string, string> = {+ "clipboard-paste": "Paste failed",+ "clipboard-copy": "Copy failed",+};+function handleMessage(data: unknown,ws: WebSocket | null,@@ -303,10 +308,12 @@ function handleMessage(break;}case "copied-text": {- if (clipboard === "none") {+ const text = message.text;+ // Copying with nothing selected must leave the local clipboard alone, as a+ // native Cmd/Ctrl+C does.+ if (clipboard === "none" || !text) {break;}- const text = message.text;copyText(text).then((success) => {@@ -338,6 +345,12 @@ function handleMessage(case "error": {if (message.failed_kind === "begin-exfiltration") {onBeginExfiltrationError();+ } else if (CLIPBOARD_FAILURE_TITLES[message.failed_kind]) {+ toast({+ variant: "destructive",+ title: CLIPBOARD_FAILURE_TITLES[message.failed_kind],+ description: message.message,+ });} else {console.warn("Message channel command failed:", message);}@@ -618,11 +631,13 @@ export function useRecordingMessageChannel(]);const sendCommand = useCallback((command: Command) => {- if (!messageSocketRef.current) {- return;+ const socket = messageSocketRef.current;+ if (!socket || socket.readyState !== WebSocket.OPEN) {+ return false;}- messageSocketRef.current.send(JSON.stringify(command));+ socket.send(JSON.stringify(command));+ return true;}, []);// effect for exfiltrationdiff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex 4da7a928d..f02097d1b 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -3185,8 +3185,9 @@ function Workspace({// to type into and a session resting on about:blank has// no way out (SKY-13705).enableUrlInput={true}- // undefined keeps the recording message channel closed on- // the non-recording live view; a defined value opens it.+ // A defined value starts recording capture; undefined+ // leaves the message channel open only while the user is+ // in control.exfiltrate={recordingStore.isRecording? !recordingStore.finishRequesteddiff --git a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsxindex df7398029..81dba4b08 100644--- a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx@@ -73,8 +73,8 @@ export function StreamPresenter({interactive={interactive}showControlButtons={showControlButtons}enableUrlInput={enableUrlInput}- // undefined keeps the recording message channel closed on the- // non-recording live view; a defined value opens it.+ // A defined value starts recording capture; undefined leaves the+ // message channel open only while the user is in control.exfiltrate={isRecording ? !finishRequested : undefined}workflowPermanentId={workflowPermanentId}onUrlChange={onUrlChange}