Skip to content

Automate browser based workflows with AI

Paste your clipboard while in control of a live browser, with one control bar for CDP and VNC (SKY-17125) (#8699)

Shuchang Zheng89cc7f3b4c6e
diff --git a/skyvern-frontend/src/components/BrowserStream.test.tsx b/skyvern-frontend/src/components/BrowserStream.test.tsx
index 915b59b1f..190d52d8a 100644
--- a/skyvern-frontend/src/components/BrowserStream.test.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.test.tsx
@@ -20,6 +20,7 @@ const mocks = vi.hoisted(() => {
clipboardPasteFrom: ReturnType<typeof vi.fn>;
sendKey: ReturnType<typeof vi.fn>;
disconnect: ReturnType<typeof vi.fn>;
+ focus: ReturnType<typeof vi.fn>;
_framebufferUpdate: () => boolean;
}> = [];
const autoConnect = { value: true };
@@ -40,6 +41,7 @@ const mocks = vi.hoisted(() => {
clipboardPasteFrom = vi.fn();
sendKey = vi.fn();
disconnect = vi.fn();
+ focus = vi.fn();
_framebufferUpdate = vi.fn(() => true);
private listeners: Record<string, RfbListener[]> = {};
@@ -192,6 +194,10 @@ vi.mock("@/store/SettingsStore", () => ({
useSettingsStore: () => mocks.settingsStore,
}));
+vi.mock("@/components/RecordingPill", () => ({
+ RecordingPill: () => null,
+}));
+
vi.mock("@/components/ui/use-toast", () => ({
toast: mocks.toast,
}));
@@ -603,6 +609,99 @@ describe("BrowserStream", () => {
expect(consoleError).toHaveBeenCalledTimes(1);
});
+ it("pastes the local clipboard over VNC from the control bar and stops control from it", async () => {
+ renderBrowserStream();
+ fireEvent.click(
+ await screen.findByRole(
+ "button",
+ { name: /take control/i },
+ { timeout: 10000 },
+ ),
+ );
+
+ fireEvent.click(screen.getByRole("button", { name: /^paste$/i }));
+
+ const rfb = mocks.rfbInstances[0]!;
+ await waitFor(() =>
+ expect(rfb.clipboardPasteFrom).toHaveBeenCalledWith(
+ "https://example.test",
+ ),
+ );
+ await waitFor(() =>
+ expect(screen.getByRole("status").textContent).toContain(
+ "Sent 20 characters",
+ ),
+ );
+ expect(rfb.sendKey.mock.calls).toEqual([
+ [0xffe3, "ControlLeft", true],
+ [0x0076, "KeyV", true],
+ [0x0076, "KeyV", false],
+ [0xffe3, "ControlLeft", false],
+ ]);
+ expect(rfb.focus).toHaveBeenCalled();
+
+ fireEvent.click(screen.getByRole("button", { name: /stop controlling/i }));
+ expect(screen.queryByRole("button", { name: /^paste$/i })).toBeNull();
+ expect(
+ document
+ .querySelector(".browser-stream")
+ ?.classList.contains("user-is-controlling"),
+ ).toBe(false);
+ });
+
+ it("pastes an Edit-menu paste over VNC only while in control", async () => {
+ const { container } = renderBrowserStream();
+ const takeControlButton = await screen.findByRole(
+ "button",
+ { name: /take control/i },
+ { timeout: 10000 },
+ );
+ const canvas = container.querySelector("canvas")!;
+ const menuPaste = () =>
+ fireEvent.paste(canvas, {
+ clipboardData: { getData: () => "from the menu" },
+ });
+ const rfb = mocks.rfbInstances[0]!;
+
+ menuPaste();
+ expect(rfb.clipboardPasteFrom).not.toHaveBeenCalled();
+
+ fireEvent.click(takeControlButton);
+ menuPaste();
+
+ await waitFor(() =>
+ expect(rfb.clipboardPasteFrom).toHaveBeenCalledWith("from the menu"),
+ );
+ await waitFor(() =>
+ expect(screen.getByRole("status").textContent).toContain(
+ "Sent 13 characters",
+ ),
+ );
+ expect(rfb.sendKey).toHaveBeenCalledWith(0x0076, "KeyV", true);
+ });
+
+ it("offers no control bar while recording holds control", async () => {
+ mocks.recordingStore.isRecording = true;
+ try {
+ const { container } = renderBrowserStream();
+ await waitFor(
+ () =>
+ expect(
+ container
+ .querySelector(".browser-stream")
+ ?.classList.contains("user-is-controlling"),
+ ).toBe(true),
+ { timeout: 10000 },
+ );
+ await screen.findByTestId("browser-stream-overlay");
+ expect(
+ screen.queryByRole("button", { name: /stop controlling/i }),
+ ).toBeNull();
+ } finally {
+ mocks.recordingStore.isRecording = false;
+ }
+ });
+
it("takes control on a click anywhere on the read-only picture, not just the button", async () => {
const { container } = renderBrowserStream();
await screen.findByRole(
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 90bf52243..ca585a876 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -4,7 +4,7 @@
import _RFB, { type RfbEvent } from "@novnc/novnc/lib/rfb.js";
type RFB = _RFB;
const RFB = (_RFB as typeof _RFB & { default?: typeof _RFB }).default ?? _RFB;
-import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons";
+import { InfoCircledIcon } from "@radix-ui/react-icons";
import { useEffect, useState, useRef, useCallback } from "react";
import { useQuery } from "@tanstack/react-query";
@@ -17,7 +17,6 @@ import {
} from "@/api/types";
import { RecordingPill } from "@/components/RecordingPill";
import { Tip } from "@/components/Tip";
-import { Button } from "@/components/ui/button";
import { toast } from "@/components/ui/use-toast";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { statusIsNotFinalized } from "@/routes/tasks/types";
@@ -38,8 +37,20 @@ import type {
import {
VNC_SUPER_L_KEYSYM,
handleVncClipboardPasteShortcut,
+ pasteTextIntoVnc,
type HeldMetaSides,
} from "@/components/browserStreamClipboard";
+import {
+ STREAM_CONTAINER_CLASS,
+ toastClipboardReadFailed,
+ toastNothingToPaste,
+ usePastedNotice,
+} from "@/routes/streaming/pasteFeedback";
+import {
+ PastedNotice,
+ StreamControlBar,
+ TakeControlButton,
+} from "@/routes/streaming/StreamControlBar";
import { useRecordingMessageChannel } from "@/routes/streaming/useRecordingMessageChannel";
import { useWebSocketParams } from "@/routes/streaming/webSocketParams";
@@ -723,6 +734,43 @@ function BrowserStream({
const theUserIsControlling =
userIsControlling || (interactive && !showControlButtons);
+ const [pastedCharacters, showPasted] = usePastedNotice();
+
+ const sendTextToVnc = useCallback(
+ async (text: string) => {
+ if (!text) {
+ toastNothingToPaste();
+ return;
+ }
+ // The clipboard read can wait on a permission prompt; the stream or the
+ // user's control may have changed by the time it resolves.
+ const rfb = rfbRef.current;
+ if (!rfb || !userCanSendVncInputRef.current) {
+ return;
+ }
+ await pasteTextIntoVnc(rfb, text);
+ if (rfbRef.current !== rfb) {
+ return;
+ }
+ showPasted(text);
+ // Keystrokes after a paste belong to the page, not the button.
+ rfb.focus({ preventScroll: true });
+ },
+ [showPasted],
+ );
+
+ const pasteIntoVnc = useCallback(async () => {
+ let text: string;
+ try {
+ text = await navigator.clipboard.readText();
+ } catch (err) {
+ console.error("Failed to read the clipboard for VNC:", err);
+ toastClipboardReadFailed();
+ return;
+ }
+ await sendTextToVnc(text);
+ }, [sendTextToVnc]);
+
useEffect(() => {
userCanSendVncInputRef.current = theUserIsControlling;
}, [theUserIsControlling]);
@@ -765,14 +813,9 @@ function BrowserStream({
void handleVncClipboardPasteShortcut(event, rfbRef.current, {
getHeldMetaSides: () => heldMetaSidesRef.current,
- onPasteError: () => {
- toast({
- title: "Paste failed",
- description:
- "Skyvern couldn't read your clipboard. Allow clipboard access for this site and try again.",
- variant: "destructive",
- });
- },
+ onPasteError: toastClipboardReadFailed,
+ onPasted: showPasted,
+ onEmptyClipboard: toastNothingToPaste,
});
};
@@ -803,15 +846,27 @@ function BrowserStream({
releaseLeftCmd();
};
+ // Edit-menu paste arrives only as a paste event; Cmd/Ctrl+V never does,
+ // because the shortcut handler prevents its default.
+ const handlePaste = (event: ClipboardEvent) => {
+ if (!userCanSendVncInputRef.current) {
+ return;
+ }
+ event.preventDefault();
+ void sendTextToVnc(event.clipboardData?.getData("text/plain") ?? "");
+ };
+
canvasContainer.addEventListener("keydown", handleKeyDown, true);
+ canvasContainer.addEventListener("paste", handlePaste);
window.addEventListener("keyup", handleKeyUp, true);
window.addEventListener("blur", handleBlur);
return () => {
canvasContainer.removeEventListener("keydown", handleKeyDown, true);
+ canvasContainer.removeEventListener("paste", handlePaste);
window.removeEventListener("keyup", handleKeyUp, true);
window.removeEventListener("blur", handleBlur);
};
- }, [canvasContainer]);
+ }, [canvasContainer, showPasted, sendTextToVnc]);
// Read the flag through a ref so the unmount cleanup stays mount-scoped: a
// StrictMode double-mount or transport swap must not cancel a live recording.
@@ -889,6 +944,7 @@ function BrowserStream({
<div
className={cn(
"browser-stream relative flex flex-col items-center justify-center",
+ STREAM_CONTAINER_CLASS,
{
"user-is-controlling": theUserIsControlling,
},
@@ -911,35 +967,22 @@ function BrowserStream({
: undefined
}
>
+ <PastedNotice characters={pastedCharacters} />
{showControlButtons && (
<div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">
- <Button
- onClick={() => {
- setUserIsControlling(true);
- }}
- className={cn("control-button pointer-events-auto border", {
+ <TakeControlButton
+ onClick={() => setUserIsControlling(true)}
+ className={cn("control-button pointer-events-auto", {
hide: userIsControlling,
})}
- size="sm"
- >
- <HandIcon className="mr-2 h-4 w-4" />
- take control
- </Button>
- <Button
- onClick={() => {
- setUserIsControlling(false);
- }}
- className={cn(
- "control-button pointer-events-auto absolute bottom-0 border",
- {
- hide: !userIsControlling,
- },
- )}
- size="sm"
- >
- <ExitIcon className="mr-2 h-4 w-4" />
- stop controlling
- </Button>
+ />
+ {/* Recording holds control for capture, so it offers no way to cede it. */}
+ {userIsControlling && !isRecording && (
+ <StreamControlBar
+ onStop={() => setUserIsControlling(false)}
+ onPaste={pasteIntoVnc}
+ />
+ )}
</div>
)}
</div>
diff --git a/skyvern-frontend/src/components/browserStreamClipboard.test.ts b/skyvern-frontend/src/components/browserStreamClipboard.test.ts
index 53eb978e9..af7e213df 100644
--- a/skyvern-frontend/src/components/browserStreamClipboard.test.ts
+++ b/skyvern-frontend/src/components/browserStreamClipboard.test.ts
@@ -251,6 +251,25 @@ describe("browserStreamClipboard", () => {
expect(rfb.sendKey).not.toHaveBeenCalled();
});
+ it("reports an empty clipboard instead of pasting nothing", async () => {
+ const event = pasteEvent();
+ const rfb = rfbMock();
+ const onEmptyClipboard = vi.fn();
+ const onPasted = vi.fn();
+
+ await handleVncClipboardPasteShortcut(event, rfb, {
+ onEmptyClipboard,
+ onPasted,
+ readClipboardText: async () => "",
+ syncDelayMs: 0,
+ });
+
+ expect(onEmptyClipboard).toHaveBeenCalledTimes(1);
+ expect(onPasted).not.toHaveBeenCalled();
+ expect(rfb.clipboardPasteFrom).not.toHaveBeenCalled();
+ expect(rfb.sendKey).not.toHaveBeenCalled();
+ });
+
it("does not intercept non-paste keys", async () => {
const event = pasteEvent({ key: "x" });
const rfb = rfbMock();
diff --git a/skyvern-frontend/src/components/browserStreamClipboard.ts b/skyvern-frontend/src/components/browserStreamClipboard.ts
index 7ae4d841b..3e309392d 100644
--- a/skyvern-frontend/src/components/browserStreamClipboard.ts
+++ b/skyvern-frontend/src/components/browserStreamClipboard.ts
@@ -60,6 +60,23 @@ function sendVncPasteShortcut(
rfb.sendKey(VNC_CONTROL_LEFT_KEYSYM, "ControlLeft", false);
}
+async function syncTextToVnc(
+ rfb: VncClipboardRfb,
+ text: string,
+ syncDelayMs = DEFAULT_VNC_CLIPBOARD_SYNC_DELAY_MS,
+) {
+ rfb.clipboardPasteFrom(text);
+ if (syncDelayMs > 0) {
+ await sleep(syncDelayMs);
+ }
+}
+
+// For a paste started without a keystroke (a button), so no modifiers are held.
+async function pasteTextIntoVnc(rfb: VncClipboardRfb, text: string) {
+ await syncTextToVnc(rfb, text);
+ sendVncPasteShortcut(rfb);
+}
+
async function handleVncClipboardPasteShortcut(
event: PasteShortcutEvent,
rfb: VncClipboardRfb | null,
@@ -67,6 +84,8 @@ async function handleVncClipboardPasteShortcut(
readClipboardText?: () => Promise<string>;
syncDelayMs?: number;
onPasteError?: (err: unknown) => void;
+ onPasted?: (text: string) => void;
+ onEmptyClipboard?: () => void;
getHeldMetaSides?: () => HeldMetaSides;
},
) {
@@ -76,8 +95,10 @@ async function handleVncClipboardPasteShortcut(
const {
readClipboardText = () => navigator.clipboard.readText(),
- syncDelayMs = DEFAULT_VNC_CLIPBOARD_SYNC_DELAY_MS,
+ syncDelayMs,
onPasteError,
+ onPasted,
+ onEmptyClipboard,
getHeldMetaSides,
} = options ?? {};
@@ -85,17 +106,20 @@ async function handleVncClipboardPasteShortcut(
event.stopPropagation();
event.stopImmediatePropagation();
+ let text: string;
try {
- const text = await readClipboardText();
- rfb.clipboardPasteFrom(text);
- if (syncDelayMs > 0) {
- await sleep(syncDelayMs);
+ text = await readClipboardText();
+ if (!text) {
+ onEmptyClipboard?.();
+ return true;
}
+ await syncTextToVnc(rfb, text, syncDelayMs);
} catch (err) {
console.error("Failed to sync clipboard contents to VNC:", err);
onPasteError?.(err);
return true;
}
+ onPasted?.(text);
if (!event.metaKey) {
sendVncPasteShortcut(rfb);
@@ -131,5 +155,6 @@ export {
VNC_SUPER_L_KEYSYM,
handleVncClipboardPasteShortcut,
isClipboardPasteShortcut,
+ pasteTextIntoVnc,
sendVncPasteShortcut,
};
diff --git a/skyvern-frontend/src/novnc.d.ts b/skyvern-frontend/src/novnc.d.ts
index ebbbd40aa..8132f4fde 100644
--- a/skyvern-frontend/src/novnc.d.ts
+++ b/skyvern-frontend/src/novnc.d.ts
@@ -39,6 +39,7 @@ declare module "@novnc/novnc/lib/rfb.js" {
removeEventListener(event: string, listener: (e: RfbEvent) => void): void;
clipboardPasteFrom(text: string): void;
disconnect(): void;
+ focus(options?: FocusOptions): void;
sendKey(keysym: number, code: string, down?: boolean): void;
viewportChange(): void;
}
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.test.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.test.tsx
index f5b235cb8..69bd0f902 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.test.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.test.tsx
@@ -24,6 +24,7 @@ const cdpInputState = vi.hoisted(() => ({
viewportHeight: 0,
clipboardPasteEnabled: false,
clipboardCopyEnabled: false,
+ onClipboardPaste: undefined as ((text: string) => void) | undefined,
userIsControlling: false,
inputReady: false,
setUserIsControlling: undefined as ((value: boolean) => void) | undefined,
@@ -71,6 +72,7 @@ vi.mock("@/routes/streaming/useCdpInput", async (importOriginal) => {
cdpInputState.viewportHeight = viewportHeight;
cdpInputState.clipboardPasteEnabled = Boolean(options.onClipboardPaste);
cdpInputState.clipboardCopyEnabled = Boolean(options.onClipboardCopy);
+ cdpInputState.onClipboardPaste = options.onClipboardPaste;
cdpInputState.onInput = options.onInput;
const result = actual.useCdpInput(options);
cdpInputState.userIsControlling = result.userIsControlling;
@@ -149,6 +151,14 @@ class FakeStreamSocket {
}
}
+const messageSockets = (browserSessionId: string) =>
+ FakeStreamSocket.instances.filter((socket) =>
+ socket.url.includes(`/stream/messages/browser_session/${browserSessionId}`),
+ );
+
+const sentMessages = (socket: FakeStreamSocket) =>
+ socket.send.mock.calls.map((call) => JSON.parse(String(call[0])));
+
function stubAnimationFrame(): void {
vi.stubGlobal(
"requestAnimationFrame",
@@ -379,39 +389,85 @@ describe("BrowserSessionStream terminal statuses", () => {
});
});
- it("enables recording clipboard interception only while the message socket is connected", async () => {
+ it("opens the message channel while the user is in control and pastes over it without recording", async () => {
+ vi.useFakeTimers();
const view = render(
- <BrowserSessionStream browserSessionId="pbs_test" exfiltrate={true} />,
+ <BrowserSessionStream browserSessionId="pbs_test" showControlButtons />,
);
+ const sentKinds = (socket: FakeStreamSocket) =>
+ sentMessages(socket).map((message) => message.kind);
+ try {
+ await act(async () => Promise.resolve());
+ expect(messageSockets("pbs_test")).toHaveLength(0);
- await waitFor(() => expect(FakeStreamSocket.instances).toHaveLength(2));
- const messageSocket = FakeStreamSocket.instances.find((socket) =>
- socket.url.includes("/stream/messages/browser_session/pbs_test"),
- );
- expect(messageSocket).toBeTruthy();
- expect(cdpInputState).toMatchObject({
- clipboardPasteEnabled: false,
- clipboardCopyEnabled: false,
- });
+ act(() => cdpInputState.setUserIsControlling?.(true));
+ await act(async () => Promise.resolve());
+ expect(messageSockets("pbs_test")).toHaveLength(1);
+ const first = messageSockets("pbs_test")[0]!;
+ act(() => first.emit("open", new Event("open")));
+
+ act(() => cdpInputState.onClipboardPaste?.("hello"));
+ expect(sentMessages(first)).toContainEqual({
+ kind: "clipboard-paste",
+ text: "hello",
+ });
+ expect(sentKinds(first)).not.toContain("begin-exfiltration");
- act(() => messageSocket?.emit("open", new Event("open")));
- await waitFor(() =>
- expect(cdpInputState).toMatchObject({
- clipboardPasteEnabled: true,
- clipboardCopyEnabled: true,
- }),
- );
+ // A drop while in control reconnects instead of leaving paste dead.
+ act(() => first.emit("close", new CloseEvent("close", { code: 1006 })));
+ await act(async () => vi.advanceTimersByTimeAsync(1000));
+ await act(async () => Promise.resolve());
+ expect(messageSockets("pbs_test")).toHaveLength(2);
+ const second = messageSockets("pbs_test")[1]!;
+ act(() => second.emit("open", new Event("open")));
+ expect(sentKinds(second)).not.toContain("begin-exfiltration");
- act(() =>
- messageSocket?.emit("close", new CloseEvent("close", { code: 1006 })),
- );
- await waitFor(() =>
- expect(cdpInputState).toMatchObject({
- clipboardPasteEnabled: false,
- clipboardCopyEnabled: false,
- }),
+ act(() => cdpInputState.setUserIsControlling?.(false));
+ await act(async () => Promise.resolve());
+ expect(second.close).toHaveBeenCalled();
+ await act(async () => vi.advanceTimersByTimeAsync(1000));
+ expect(messageSockets("pbs_test")).toHaveLength(2);
+ } finally {
+ view.unmount();
+ vi.useRealTimers();
+ }
+ });
+
+ it("recycles the message socket when a recording ends while the user keeps control", async () => {
+ vi.useFakeTimers();
+ const view = render(
+ <BrowserSessionStream browserSessionId="pbs_test" showControlButtons />,
);
- view.unmount();
+ try {
+ await act(async () => Promise.resolve());
+ act(() => cdpInputState.setUserIsControlling?.(true));
+ await act(async () => Promise.resolve());
+ const controlSocket = messageSockets("pbs_test")[0]!;
+ act(() => controlSocket.emit("open", new Event("open")));
+
+ view.rerender(
+ <BrowserSessionStream
+ browserSessionId="pbs_test"
+ showControlButtons
+ exfiltrate={true}
+ />,
+ );
+ await act(async () => Promise.resolve());
+ expect(messageSockets("pbs_test")).toHaveLength(1);
+ view.rerender(
+ <BrowserSessionStream browserSessionId="pbs_test" showControlButtons />,
+ );
+ await act(async () => Promise.resolve());
+
+ // The recording's socket must close (its cleanup finalizes the recording)
+ // rather than carry that recording's identity into the next one.
+ expect(controlSocket.close).toHaveBeenCalled();
+ expect(messageSockets("pbs_test")).toHaveLength(2);
+ expect(cdpInputState.userIsControlling).toBe(true);
+ } finally {
+ view.unmount();
+ vi.useRealTimers();
+ }
});
it("takes control for recording and only cedes a recording-owned grab", async () => {
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
index 8d008e1d0..56763d48a 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
@@ -2,8 +2,17 @@ import { useCallback, useEffect, useRef, useState } from "react";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { newWssBaseUrl, getCredentialParam } from "@/util/env";
import { useCdpInput } from "@/routes/streaming/useCdpInput";
-import { useRecordingMessageChannel } from "@/routes/streaming/useRecordingMessageChannel";
+import {
+ useRecordingMessageChannel,
+ type Command,
+} from "@/routes/streaming/useRecordingMessageChannel";
+import { toast } from "@/components/ui/use-toast";
import { InteractiveStreamView } from "@/routes/streaming/InteractiveStreamView";
+import {
+ toastClipboardReadFailed,
+ toastNothingToPaste,
+ usePastedNotice,
+} from "@/routes/streaming/pasteFeedback";
import {
markCommit,
markLoad,
@@ -48,6 +57,9 @@ type StreamMessage = {
url?: string;
};
+// Mirrors MAX_CLIPBOARD_PASTE_BYTES in skyvern/forge/sdk/routes/streaming/payload_limits.py.
+const MAX_CLIPBOARD_PASTE_BYTES = 1024 * 1024;
+
const STARTING_DIAGNOSTIC: StreamDiagnostic = {
title: "Waking up your local browser",
detail: "Opening the stream and waiting for the first frame...",
@@ -159,6 +171,13 @@ function BrowserSessionStream({
// the close handler so a reconnect notice augments that reason instead of
// replacing it with a generic "closed with code 1000".
const streamEndedDiagnosticRef = useRef<StreamDiagnostic | null>(null);
+ const keepMessageChannelAliveRef = useRef(false);
+ // Clipboard callbacks go to useCdpInput before the message channel exists (the
+ // channel opens on its userIsControlling), so they reach the channel through this.
+ const sendMessageCommandRef = useRef<(command: Command) => boolean>(
+ () => false,
+ );
+ const [pastedCharacters, showPasted] = usePastedNotice();
exfiltrateRef.current = !!exfiltrate;
const scheduleRecordingReconnect = useCallback(() => {
@@ -167,7 +186,7 @@ function BrowserSessionStream({
}
recordingReconnectTimerRef.current = setTimeout(() => {
recordingReconnectTimerRef.current = null;
- if (exfiltrateRef.current) {
+ if (keepMessageChannelAliveRef.current) {
setRecordingReconnectTrigger((trigger) => trigger + 1);
}
}, 1000);
@@ -187,7 +206,7 @@ function BrowserSessionStream({
return;
}
recordingChannelDisconnectedRef.current = true;
- if (exfiltrateRef.current) {
+ if (keepMessageChannelAliveRef.current) {
scheduleRecordingReconnect();
}
},
@@ -202,36 +221,50 @@ function BrowserSessionStream({
};
}, []);
- const recordingChannelEnabled = exfiltrate !== undefined;
- const { isMessageConnected, sendCommand: sendRecordingCommand } =
- useRecordingMessageChannel({
- browserSessionId,
- enabled: recordingChannelEnabled,
- exfiltrate: !!exfiltrate,
- workflowPermanentId: workflowPermanentId ?? null,
- clipboard: "message",
- reconnectTrigger: recordingReconnectTrigger,
- onConnectionChange: handleRecordingConnectionChange,
- });
+ const sendClipboardCommand = useCallback(
+ (command: Command, failureTitle: string) => {
+ const sent = sendMessageCommandRef.current(command);
+ if (!sent) {
+ toast({
+ variant: "destructive",
+ title: failureTitle,
+ description:
+ "Still connecting to the browser. Try again in a moment.",
+ });
+ }
+ return sent;
+ },
+ [],
+ );
- useEffect(() => {
- if (
- exfiltrate &&
- !isMessageConnected &&
- recordingChannelDisconnectedRef.current
- ) {
- scheduleRecordingReconnect();
- }
- }, [exfiltrate, isMessageConnected, scheduleRecordingReconnect]);
const onClipboardPaste = useCallback(
(text: string) => {
- sendRecordingCommand({ kind: "clipboard-paste", text });
+ if (!text) {
+ toastNothingToPaste();
+ return;
+ }
+ // Checked here too so an oversized paste never flashes a success notice
+ // before the backend's rejection arrives.
+ if (new TextEncoder().encode(text).length > MAX_CLIPBOARD_PASTE_BYTES) {
+ toast({
+ variant: "destructive",
+ title: "Paste failed",
+ description: "Your clipboard text is over 1 MB, too large to paste.",
+ });
+ return;
+ }
+ if (
+ !sendClipboardCommand({ kind: "clipboard-paste", text }, "Paste failed")
+ ) {
+ return;
+ }
+ showPasted(text);
},
- [sendRecordingCommand],
+ [sendClipboardCommand, showPasted],
);
const onClipboardCopy = useCallback(() => {
- sendRecordingCommand({ kind: "clipboard-copy" });
- }, [sendRecordingCommand]);
+ sendClipboardCommand({ kind: "clipboard-copy" }, "Copy failed");
+ }, [sendClipboardCommand]);
// The CDP input socket must be wired whenever the stream can be controlled,
// whether by default interaction or via the take-control button.
@@ -250,16 +283,64 @@ function BrowserSessionStream({
navigate,
historyNavigate,
navigateError,
+ pasteClipboard,
} = useCdpInput({
inputWsUrl,
interactive: controllable,
viewportWidth,
viewportHeight,
- onClipboardPaste:
- exfiltrate && isMessageConnected ? onClipboardPaste : undefined,
- onClipboardCopy:
- exfiltrate && isMessageConnected ? onClipboardCopy : undefined,
+ onClipboardPaste: controllable ? onClipboardPaste : undefined,
+ onClipboardPasteError: toastClipboardReadFailed,
+ onClipboardCopy: controllable ? onClipboardCopy : undefined,
+ // Recording keeps copy local so the keystroke never lands in the capture.
+ forwardCopyShortcut: !exfiltrate,
});
+ const keepMessageChannelAlive = !!exfiltrate || userIsControlling;
+ keepMessageChannelAliveRef.current = keepMessageChannelAlive;
+
+ // Outside a recording the channel is open only while the user is in control, so
+ // pasting works without holding a socket per passive viewer. exfiltrate stays
+ // false then, so opening it never sends begin-exfiltration.
+ const recordingChannelActive = exfiltrate !== undefined;
+ const messageChannelEnabled = recordingChannelActive || userIsControlling;
+ const previousRecordingChannelActiveRef = useRef(recordingChannelActive);
+ useEffect(() => {
+ // Closing the socket is what finalizes a recording and clears its retained
+ // identity, so a user still in control after one gets a fresh socket.
+ if (previousRecordingChannelActiveRef.current && !recordingChannelActive) {
+ setRecordingReconnectTrigger((trigger) => trigger + 1);
+ }
+ previousRecordingChannelActiveRef.current = recordingChannelActive;
+ }, [recordingChannelActive]);
+ const { isMessageConnected, sendCommand: sendMessageCommand } =
+ useRecordingMessageChannel({
+ browserSessionId,
+ enabled: messageChannelEnabled,
+ exfiltrate: !!exfiltrate,
+ workflowPermanentId: workflowPermanentId ?? null,
+ clipboard: "message",
+ reconnectTrigger: recordingReconnectTrigger,
+ onConnectionChange: handleRecordingConnectionChange,
+ });
+ sendMessageCommandRef.current = sendMessageCommand;
+
+ useEffect(() => {
+ if (!messageChannelEnabled) {
+ // A drop from a closed channel must not trigger a reconnect of the next one.
+ recordingChannelDisconnectedRef.current = false;
+ } else if (
+ keepMessageChannelAlive &&
+ !isMessageConnected &&
+ recordingChannelDisconnectedRef.current
+ ) {
+ scheduleRecordingReconnect();
+ }
+ }, [
+ keepMessageChannelAlive,
+ isMessageConnected,
+ messageChannelEnabled,
+ scheduleRecordingReconnect,
+ ]);
useEffect(() => {
const recordingStarted =
@@ -671,6 +752,8 @@ function BrowserSessionStream({
onFrameWidthChange={onFrameWidthChange}
frameToken={streamImgToken}
onFrameLoad={markLoad}
+ onPasteClipboard={controllable ? pasteClipboard : undefined}
+ pastedCharacters={pastedCharacters}
/>
);
}
diff --git a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.test.tsx b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.test.tsx
index d1c1f4db5..b89506d39 100644
--- a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.test.tsx
+++ b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.test.tsx
@@ -35,6 +35,7 @@ function baseProps() {
handleMouseMove: vi.fn(),
handleKeyDown: vi.fn(),
handleKeyUp: vi.fn(),
+ handlePaste: vi.fn(),
},
currentUrl: "https://example.com/",
};
@@ -235,9 +236,28 @@ describe("InteractiveStreamView take-control overlay", () => {
expect(screen.queryByTestId("take-control-overlay")).toBeNull();
expect(
- screen.getByRole("button", { name: /stop controlling/ }),
+ screen.getByRole("button", { name: /stop controlling/i }),
).toBeTruthy();
});
+
+ it("offers Paste in the control bar only when the stream can paste", () => {
+ const { rerender } = render(
+ <InteractiveStreamView {...baseProps()} userIsControlling />,
+ );
+ expect(screen.queryByRole("button", { name: /paste/i })).toBeNull();
+
+ const onPasteClipboard = vi.fn();
+ rerender(
+ <InteractiveStreamView
+ {...baseProps()}
+ userIsControlling
+ onPasteClipboard={onPasteClipboard}
+ />,
+ );
+ fireEvent.click(screen.getByRole("button", { name: /paste/i }));
+
+ expect(onPasteClipboard).toHaveBeenCalledTimes(1);
+ });
});
describe("InteractiveStreamView browser chrome", () => {
diff --git a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx
index 9714402d5..7e227c07e 100644
--- a/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx
+++ b/skyvern-frontend/src/routes/streaming/InteractiveStreamView.tsx
@@ -2,13 +2,10 @@ import { useEffect, useRef, useState, type RefObject } from "react";
import {
ArrowLeftIcon,
ArrowRightIcon,
- ExitIcon,
GlobeIcon,
- HandIcon,
ReloadIcon,
} from "@radix-ui/react-icons";
import { ZoomableImage } from "@/components/ZoomableImage";
-import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import {
Tooltip,
@@ -17,6 +14,12 @@ import {
TooltipTrigger,
} from "@/components/ui/tooltip";
import { cn } from "@/util/utils";
+import { STREAM_CONTAINER_CLASS } from "./pasteFeedback";
+import {
+ PastedNotice,
+ StreamControlBar,
+ TakeControlButton,
+} from "./StreamControlBar";
import type { HistoryAction } from "./useCdpInput";
interface InteractiveStreamViewProps {
@@ -34,6 +37,7 @@ interface InteractiveStreamViewProps {
handleMouseMove: (e: React.MouseEvent<HTMLImageElement>) => void;
handleKeyDown: (e: React.KeyboardEvent) => void;
handleKeyUp: (e: React.KeyboardEvent) => void;
+ handlePaste: (e: React.ClipboardEvent) => void;
};
currentUrl?: string;
centered?: boolean;
@@ -51,6 +55,10 @@ interface InteractiveStreamViewProps {
onFrameWidthChange?: (width: number | null) => void;
frameToken?: number;
onFrameLoad?: (token: number) => void;
+ // Omitted when the stream can't paste; the control bar then hides its paste controls.
+ onPasteClipboard?: () => void;
+ // Set briefly after each paste is sent; null hides the confirmation.
+ pastedCharacters?: number | null;
}
function UrlBar({
@@ -202,6 +210,8 @@ function InteractiveStreamView({
onFrameWidthChange,
frameToken,
onFrameLoad,
+ onPasteClipboard,
+ pastedCharacters,
}: InteractiveStreamViewProps) {
const imgDataUrl = `data:image/${streamFormat};base64,${streamImgSrc}`;
const imgRef = useRef<HTMLImageElement>(null);
@@ -343,25 +353,22 @@ function InteractiveStreamView({
className="absolute inset-0 z-10 flex cursor-pointer items-center justify-center"
onClick={() => setUserIsControlling(true)}
>
- <Button
- size="sm"
- className="border"
- onClick={() => setUserIsControlling(true)}
- >
- <HandIcon className="mr-2 h-4 w-4" />
- take control
- </Button>
+ <TakeControlButton onClick={() => setUserIsControlling(true)} />
</div>
)}
+ <PastedNotice characters={pastedCharacters ?? null} />
{showControlButtons && userIsControlling && (
- <Button
- size="sm"
- className="absolute bottom-2 left-1/2 z-10 -translate-x-1/2 border"
- onClick={() => setUserIsControlling(false)}
- >
- <ExitIcon className="mr-2 h-4 w-4" />
- stop controlling
- </Button>
+ <StreamControlBar
+ onStop={() => setUserIsControlling(false)}
+ onPaste={
+ onPasteClipboard &&
+ (() => {
+ onPasteClipboard();
+ // Keystrokes after a paste belong to the page, not the button.
+ containerRef.current?.focus();
+ })
+ }
+ />
)}
</>
);
@@ -369,10 +376,14 @@ function InteractiveStreamView({
return (
<div
ref={containerRef}
- className="relative h-full w-full outline-none"
+ className={cn(
+ "relative h-full w-full outline-none",
+ STREAM_CONTAINER_CLASS,
+ )}
tabIndex={0}
onKeyDown={handlers.handleKeyDown}
onKeyUp={handlers.handleKeyUp}
+ onPaste={handlers.handlePaste}
>
{/* Chrome and viewport share previewWidth so the window sizes to the letterboxed
picture rather than the pane, the way a real browser window frames its page.
diff --git a/skyvern-frontend/src/routes/streaming/StreamControlBar.tsx b/skyvern-frontend/src/routes/streaming/StreamControlBar.tsx
new file mode 100644
index 000000000..c6c1df1ca
--- /dev/null
+++ b/skyvern-frontend/src/routes/streaming/StreamControlBar.tsx
@@ -0,0 +1,102 @@
+import { ClipboardIcon, ExitIcon, HandIcon } from "@radix-ui/react-icons";
+import { Button } from "@/components/ui/button";
+import { isMacPlatform } from "@/util/platform";
+import { cn } from "@/util/utils";
+
+const PASTE_SHORTCUT = isMacPlatform() ? "⌘V" : "Ctrl+V";
+
+export function PastedNotice({ characters }: { characters: number | null }) {
+ return (
+ // Stays mounted so screen readers announce the text when it changes.
+ <div
+ role="status"
+ className="pointer-events-none absolute bottom-14 left-1/2 z-10 -translate-x-1/2"
+ >
+ {characters != null && (
+ <div className="whitespace-nowrap rounded-md bg-foreground px-3 py-1.5 text-xs text-background shadow-md">
+ Sent {characters} {characters === 1 ? "character" : "characters"} to
+ the browser
+ </div>
+ )}
+ </div>
+ );
+}
+
+export function StreamControlBar({
+ onStop,
+ onPaste,
+}: {
+ onStop: () => void;
+ // Omitted when the stream can't paste; the bar then hides its paste controls.
+ onPaste?: () => void;
+}) {
+ return (
+ <div
+ className="pointer-events-auto absolute bottom-2 left-1/2 z-10 flex max-w-[calc(100%-1rem)] -translate-x-1/2 items-center gap-2 overflow-hidden whitespace-nowrap rounded-md border bg-background py-1 pl-3 pr-1 text-xs text-foreground shadow-md"
+ // The stream container forwards keys to the remote page; Enter/Space here
+ // must activate the bar's own buttons instead.
+ onKeyDown={(e) => e.stopPropagation()}
+ onKeyUp={(e) => e.stopPropagation()}
+ >
+ <span className="flex min-w-0 items-center gap-1.5 font-medium">
+ <span className="h-1.5 w-1.5 shrink-0 rounded-full bg-destructive" />
+ <span className="truncate">You're in control</span>
+ </span>
+ {onPaste && (
+ <>
+ {/* The bar measures ~511px with the full hint, ~382px with the key alone. */}
+ <span className="hidden shrink-0 items-center gap-1.5 text-muted-foreground [@container_stream_(min-width:420px)]:flex">
+ <kbd className="rounded border bg-muted px-1.5 font-mono text-[11px] text-foreground">
+ {PASTE_SHORTCUT}
+ </kbd>
+ <span className="hidden [@container_stream_(min-width:540px)]:inline">
+ pastes your clipboard
+ </span>
+ </span>
+ <Button
+ size="sm"
+ variant="outline"
+ // Too narrow for both buttons: keep Stop controlling, the only way out.
+ className="h-7 shrink-0 px-2.5 text-xs [@container_stream_(max-width:280px)]:hidden"
+ onClick={onPaste}
+ >
+ <ClipboardIcon className="mr-1.5 h-3.5 w-3.5" />
+ Paste
+ </Button>
+ </>
+ )}
+ <Button
+ size="sm"
+ className="h-7 shrink-0 px-2.5 text-xs"
+ onClick={onStop}
+ >
+ <ExitIcon className="mr-1.5 h-3.5 w-3.5" />
+ Stop controlling
+ </Button>
+ </div>
+ );
+}
+
+export function TakeControlButton({
+ onClick,
+ className,
+}: {
+ onClick: () => void;
+ className?: string;
+}) {
+ // Same surface as StreamControlBar so the two control states read as one set.
+ return (
+ <Button
+ size="sm"
+ variant="outline"
+ className={cn(
+ "border-border bg-background text-foreground shadow-md",
+ className,
+ )}
+ onClick={onClick}
+ >
+ <HandIcon className="mr-2 h-4 w-4" />
+ Take control
+ </Button>
+ );
+}
diff --git a/skyvern-frontend/src/routes/streaming/pasteFeedback.ts b/skyvern-frontend/src/routes/streaming/pasteFeedback.ts
new file mode 100644
index 000000000..8d7ed7f8a
--- /dev/null
+++ b/skyvern-frontend/src/routes/streaming/pasteFeedback.ts
@@ -0,0 +1,50 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+import { toast } from "@/components/ui/use-toast";
+
+const PASTE_NOTICE_MS = 2500;
+
+// The bar's container queries resolve against the nearest element carrying these.
+export const STREAM_CONTAINER_CLASS =
+ "[container-name:stream] [container-type:inline-size]";
+
+export function toastNothingToPaste() {
+ toast({
+ title: "Nothing to paste",
+ description: "Your clipboard has no text to paste.",
+ });
+}
+
+export function toastClipboardReadFailed() {
+ toast({
+ variant: "destructive",
+ title: "Paste failed",
+ description:
+ "Skyvern couldn't read your clipboard. Allow clipboard access for this site and try again.",
+ });
+}
+
+export function usePastedNotice() {
+ const [pastedCharacters, setPastedCharacters] = useState<number | null>(null);
+ const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+ useEffect(() => {
+ return () => {
+ if (timerRef.current) {
+ clearTimeout(timerRef.current);
+ }
+ };
+ }, []);
+
+ const showPasted = useCallback((text: string) => {
+ setPastedCharacters(Array.from(text).length);
+ if (timerRef.current) {
+ clearTimeout(timerRef.current);
+ }
+ timerRef.current = setTimeout(() => {
+ timerRef.current = null;
+ setPastedCharacters(null);
+ }, PASTE_NOTICE_MS);
+ }, []);
+
+ return [pastedCharacters, showPasted] as const;
+}
diff --git a/skyvern-frontend/src/routes/streaming/useCdpInput.test.ts b/skyvern-frontend/src/routes/streaming/useCdpInput.test.ts
index 493df3307..83319d390 100644
--- a/skyvern-frontend/src/routes/streaming/useCdpInput.test.ts
+++ b/skyvern-frontend/src/routes/streaming/useCdpInput.test.ts
@@ -76,6 +76,7 @@ async function renderControllingInputHook(
clipboardCallbacks: {
onClipboardPaste?: (text: string) => void;
onClipboardCopy?: () => void;
+ forwardCopyShortcut?: boolean;
} = {},
) {
const { result } = renderHook(() =>
@@ -703,6 +704,31 @@ describe("useCdpInput key handling", () => {
expect(onClipboardPaste).not.toHaveBeenCalled();
});
+ it("syncs Cmd+C and still delivers it to the page only when forwarding is on", async () => {
+ for (const forwardCopyShortcut of [true, false]) {
+ const onClipboardCopy = vi.fn();
+ const result = await renderControllingInputHook({
+ onClipboardCopy,
+ forwardCopyShortcut,
+ });
+ const send = latestSocketSend();
+ send.mockClear();
+
+ act(() => {
+ result.current.handlers.handleKeyDown(
+ fakeKeyboardEvent("c", "KeyC", { metaKey: true }),
+ );
+ result.current.handlers.handleKeyUp(fakeKeyboardEvent("c", "KeyC"));
+ });
+
+ expect(onClipboardCopy).toHaveBeenCalledTimes(1);
+ const sentCodes = send.mock.calls.map(
+ (call) => JSON.parse(String(call[0])).code,
+ );
+ expect(sentCodes).toEqual(forwardCopyShortcut ? ["KeyC", "KeyC"] : []);
+ }
+ });
+
it("forwards an ordinary key pair after intercepting a clipboard chord", async () => {
Object.defineProperty(navigator, "clipboard", {
configurable: true,
diff --git a/skyvern-frontend/src/routes/streaming/useCdpInput.ts b/skyvern-frontend/src/routes/streaming/useCdpInput.ts
index 36e711e98..70bbfde51 100644
--- a/skyvern-frontend/src/routes/streaming/useCdpInput.ts
+++ b/skyvern-frontend/src/routes/streaming/useCdpInput.ts
@@ -21,7 +21,11 @@ interface UseCdpInputOptions {
viewportWidth: number;
viewportHeight: number;
onClipboardPaste?: (text: string) => void;
+ onClipboardPasteError?: () => void;
onClipboardCopy?: () => void;
+ // Also deliver Cmd/Ctrl+C to the remote page, so apps with their own
+ // selection model (canvas grids, editors) still copy there.
+ forwardCopyShortcut?: boolean;
onInput?: () => void;
}
@@ -36,10 +40,12 @@ interface UseCdpInputReturn {
handleMouseMove: (e: React.MouseEvent<HTMLImageElement>) => void;
handleKeyDown: (e: React.KeyboardEvent) => void;
handleKeyUp: (e: React.KeyboardEvent) => void;
+ handlePaste: (e: React.ClipboardEvent) => void;
};
navigate: (url: string) => void;
historyNavigate: (action: HistoryAction) => void;
navigateError: string | null;
+ pasteClipboard: () => void;
}
export type HistoryAction = "back" | "forward" | "reload";
@@ -55,13 +61,23 @@ const NAVIGATE_ERROR_MESSAGES: Record<string, string> = {
invalid_url: "Enter a valid http(s) URL.",
};
+function isEditableTarget(target: EventTarget | null) {
+ return (
+ target instanceof HTMLInputElement ||
+ target instanceof HTMLTextAreaElement ||
+ (target instanceof HTMLElement && target.isContentEditable)
+ );
+}
+
export function useCdpInput({
inputWsUrl,
interactive,
viewportWidth,
viewportHeight,
onClipboardPaste,
+ onClipboardPasteError,
onClipboardCopy,
+ forwardCopyShortcut = false,
onInput,
}: UseCdpInputOptions): UseCdpInputReturn {
const [userIsControlling, setUserIsControlling] = useState(false);
@@ -98,12 +114,33 @@ export function useCdpInput({
};
const interceptedClipboardKeysRef = useRef(new Set<string>());
const onClipboardPasteRef = useRef(onClipboardPaste);
+ const onClipboardPasteErrorRef = useRef(onClipboardPasteError);
const onClipboardCopyRef = useRef(onClipboardCopy);
const onInputRef = useRef(onInput);
onClipboardPasteRef.current = onClipboardPaste;
+ onClipboardPasteErrorRef.current = onClipboardPasteError;
onClipboardCopyRef.current = onClipboardCopy;
+ const forwardCopyShortcutRef = useRef(forwardCopyShortcut);
+ forwardCopyShortcutRef.current = forwardCopyShortcut;
onInputRef.current = onInput;
+ const pasteClipboard = useCallback(() => {
+ if (!interactive || !userIsControlling || !onClipboardPasteRef.current) {
+ return;
+ }
+ if (typeof navigator.clipboard?.readText !== "function") {
+ onClipboardPasteErrorRef.current?.();
+ return;
+ }
+ navigator.clipboard
+ .readText()
+ .then((text) => onClipboardPasteRef.current?.(text))
+ .catch((error) => {
+ console.error("Failed to read clipboard contents:", error);
+ onClipboardPasteErrorRef.current?.();
+ });
+ }, [interactive, userIsControlling]);
+
useEffect(() => {
if (!interactive || !inputWsUrl) return;
@@ -440,22 +477,15 @@ export function useCdpInput({
const key = e.key.toLowerCase();
if (clipboardModifier && key === "v" && onClipboardPasteRef.current) {
interceptedClipboardKeysRef.current.add(e.code);
- if (!navigator.clipboard) {
- console.warn("Clipboard API not available.");
- return;
- }
- navigator.clipboard
- .readText()
- .then((text) => onClipboardPasteRef.current?.(text))
- .catch((error) => {
- console.error("Failed to read clipboard contents:", error);
- });
+ pasteClipboard();
return;
}
if (clipboardModifier && key === "c" && onClipboardCopyRef.current) {
- interceptedClipboardKeysRef.current.add(e.code);
onClipboardCopyRef.current();
- return;
+ if (!forwardCopyShortcutRef.current) {
+ interceptedClipboardKeysRef.current.add(e.code);
+ return;
+ }
}
const isPrintable = e.key.length === 1;
const windowsVirtualKeyCode = virtualKeyCodeFor(e);
@@ -472,7 +502,20 @@ export function useCdpInput({
}
sendInputEvent(payload);
},
- [interactive, userIsControlling, sendInputEvent],
+ [interactive, userIsControlling, sendInputEvent, pasteClipboard],
+ );
+
+ // Menu-driven pastes arrive only as a paste event; Cmd/Ctrl+V never does,
+ // because handleKeyDown prevents its default.
+ const handlePaste = useCallback(
+ (e: React.ClipboardEvent) => {
+ if (!interactive || !userIsControlling) return;
+ // The URL bar input lives inside the container; its own paste must stay local.
+ if (!onClipboardPasteRef.current || isEditableTarget(e.target)) return;
+ e.preventDefault();
+ onClipboardPasteRef.current(e.clipboardData.getData("text/plain"));
+ },
+ [interactive, userIsControlling],
);
const handleKeyUp = useCallback(
@@ -527,9 +570,11 @@ export function useCdpInput({
handleMouseMove,
handleKeyDown,
handleKeyUp,
+ handlePaste,
},
navigate,
historyNavigate,
navigateError,
+ pasteClipboard,
};
}
diff --git a/skyvern-frontend/src/routes/streaming/useRecordingMessageChannel.ts b/skyvern-frontend/src/routes/streaming/useRecordingMessageChannel.ts
index a6357ad55..9bb8fcb7a 100644
--- a/skyvern-frontend/src/routes/streaming/useRecordingMessageChannel.ts
+++ b/skyvern-frontend/src/routes/streaming/useRecordingMessageChannel.ts
@@ -96,8 +96,8 @@ export type Command =
export interface UseRecordingMessageChannelResult {
messageSocket: WebSocket | null;
isMessageConnected: boolean;
- /** send a command over the message socket if open */
- sendCommand: (command: Command) => void;
+ /** send a command over the message socket if open; false when it was not sent */
+ sendCommand: (command: Command) => boolean;
}
const messageInKinds = [
@@ -247,6 +247,11 @@ function getMessage(data: unknown): MessageIn | undefined {
}
}
+const CLIPBOARD_FAILURE_TITLES: Record<string, string> = {
+ "clipboard-paste": "Paste failed",
+ "clipboard-copy": "Copy failed",
+};
+
function handleMessage(
data: unknown,
ws: WebSocket | null,
@@ -303,10 +308,12 @@ function handleMessage(
break;
}
case "copied-text": {
- if (clipboard === "none") {
+ const text = message.text;
+ // Copying with nothing selected must leave the local clipboard alone, as a
+ // native Cmd/Ctrl+C does.
+ if (clipboard === "none" || !text) {
break;
}
- const text = message.text;
copyText(text)
.then((success) => {
@@ -338,6 +345,12 @@ function handleMessage(
case "error": {
if (message.failed_kind === "begin-exfiltration") {
onBeginExfiltrationError();
+ } else if (CLIPBOARD_FAILURE_TITLES[message.failed_kind]) {
+ toast({
+ variant: "destructive",
+ title: CLIPBOARD_FAILURE_TITLES[message.failed_kind],
+ description: message.message,
+ });
} else {
console.warn("Message channel command failed:", message);
}
@@ -618,11 +631,13 @@ export function useRecordingMessageChannel(
]);
const sendCommand = useCallback((command: Command) => {
- if (!messageSocketRef.current) {
- return;
+ const socket = messageSocketRef.current;
+ if (!socket || socket.readyState !== WebSocket.OPEN) {
+ return false;
}
- messageSocketRef.current.send(JSON.stringify(command));
+ socket.send(JSON.stringify(command));
+ return true;
}, []);
// effect for exfiltration
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index 4da7a928d..f02097d1b 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -3185,8 +3185,9 @@ function Workspace({
// to type into and a session resting on about:blank has
// no way out (SKY-13705).
enableUrlInput={true}
- // undefined keeps the recording message channel closed on
- // the non-recording live view; a defined value opens it.
+ // A defined value starts recording capture; undefined
+ // leaves the message channel open only while the user is
+ // in control.
exfiltrate={
recordingStore.isRecording
? !recordingStore.finishRequested
diff --git a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx
index df7398029..81dba4b08 100644
--- a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx
@@ -73,8 +73,8 @@ export function StreamPresenter({
interactive={interactive}
showControlButtons={showControlButtons}
enableUrlInput={enableUrlInput}
- // undefined keeps the recording message channel closed on the
- // non-recording live view; a defined value opens it.
+ // A defined value starts recording capture; undefined leaves the
+ // message channel open only while the user is in control.
exfiltrate={isRecording ? !finishRequested : undefined}
workflowPermanentId={workflowPermanentId}
onUrlChange={onUrlChange}