Skip to content

Automate browser based workflows with AI

FE portion of seamless clipboard transfer in browser stream (#3799)

Jonathan Dobsonf66f50fc9391
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 34cda4a31..b3a5473f8 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -39,8 +39,31 @@ interface CommandCedeControl {
kind: "cede-control";
}
+// a "Command" is an fire-n-forget out-message - it does not require a response
type Command = CommandTakeControl | CommandCedeControl;
+const messageInKinds = ["ask-for-clipboard", "copied-text"] as const;
+
+type MessageInKind = (typeof messageInKinds)[number];
+
+interface MessageInAskForClipboard {
+ kind: "ask-for-clipboard";
+}
+
+interface MessageInCopiedText {
+ kind: "copied-text";
+ text: string;
+}
+
+type MessageIn = MessageInCopiedText | MessageInAskForClipboard;
+
+interface MessageOutAskForClipboardResponse {
+ kind: "ask-for-clipboard-response";
+ text: string;
+}
+
+type MessageOut = MessageOutAskForClipboardResponse;
+
type Props = {
browserSessionId?: string;
interactive?: boolean;
@@ -102,6 +125,20 @@ function BrowserStream({
if (!browserSession || browserSession.completed_at) {
setHasBrowserSession(false);
return false;
+
+ // NOTE(jdo:streaming-local-dev): remove above and use this instead
+ // if (browserSession && browserSession.completed_at) {
+ // console.warn(
+ // "Completed at:",
+ // browserSession.completed_at,
+ // "continuing anyway!",
+ // );
+ // setHasBrowserSession(true);
+ // return true;
+ // } else {
+ // setHasBrowserSession(false);
+ // return false;
+ // }
}
setHasBrowserSession(true);
@@ -117,16 +154,16 @@ function BrowserStream({
const [hasBrowserSession, setHasBrowserSession] = useState(true); // be optimistic
const [userIsControlling, setUserIsControlling] = useState(false);
- const [commandSocket, setCommandSocket] = useState<WebSocket | null>(null);
+ const [messageSocket, setMessageSocket] = useState<WebSocket | null>(null);
const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
const prevVncConnectedRef = useRef<boolean>(false);
const [isVncConnected, setIsVncConnected] = useState<boolean>(false);
const [isCanvasReady, setIsCanvasReady] = useState<boolean>(false);
const [isReady, setIsReady] = useState(false);
- const [commandDisconnectedTrigger, setCommandDisconnectedTrigger] =
+ const [messagesDisconnectedTrigger, setMessagesDisconnectedTrigger] =
useState(0);
- const prevCommandConnectedRef = useRef<boolean>(false);
- const [isCommandConnected, setIsCommandConnected] = useState<boolean>(false);
+ const prevMessageConnectedRef = useRef<boolean>(false);
+ const [isMessageConnected, setIsMessageConnected] = useState<boolean>(false);
const [canvasContainer, setCanvasContainer] = useState<HTMLDivElement | null>(
null,
);
@@ -168,14 +205,14 @@ function BrowserStream({
prevVncConnectedRef.current = isVncConnected;
}, [isVncConnected, onClose]);
- // effect for command disconnects only
+ // effect for message disconnects only
useEffect(() => {
- if (prevCommandConnectedRef.current && !isCommandConnected) {
- setCommandDisconnectedTrigger((x) => x + 1);
+ if (prevMessageConnectedRef.current && !isMessageConnected) {
+ setMessagesDisconnectedTrigger((x) => x + 1);
onClose?.();
}
- prevCommandConnectedRef.current = isCommandConnected;
- }, [isCommandConnected, onClose]);
+ prevMessageConnectedRef.current = isMessageConnected;
+ }, [isMessageConnected, onClose]);
// vnc socket
useEffect(
@@ -289,7 +326,6 @@ function BrowserStream({
],
);
- // command socket
useEffect(() => {
if (!showStream || !canvasContainer || !runId) {
return;
@@ -300,34 +336,52 @@ function BrowserStream({
const connect = async () => {
const wsParams = await getWebSocketParams();
- const commandUrl =
+ const messageUrl =
entity === "browserSession"
- ? `${newWssBaseUrl}/stream/commands/browser_session/${runId}?${wsParams}`
+ ? `${newWssBaseUrl}/stream/messages/browser_session/${runId}?${wsParams}`
: entity === "task"
- ? `${wssBaseUrl}/stream/commands/task/${runId}?${wsParams}`
+ ? `${wssBaseUrl}/stream/messages/task/${runId}?${wsParams}`
: entity === "workflow"
- ? `${wssBaseUrl}/stream/commands/workflow_run/${runId}?${wsParams}`
+ ? `${wssBaseUrl}/stream/messages/workflow_run/${runId}?${wsParams}`
: null;
- if (!commandUrl) {
- throw new Error("No command url");
+ if (!messageUrl) {
+ throw new Error("No message url");
}
if (!hasBrowserSession) {
- setIsCommandConnected(false);
+ setIsMessageConnected(false);
return;
}
- ws = new WebSocket(commandUrl);
+ ws = new WebSocket(messageUrl);
ws.onopen = () => {
- setIsCommandConnected(true);
- setCommandSocket(ws);
+ setIsMessageConnected(true);
+ setMessageSocket(ws);
+ };
+
+ ws.onmessage = (event) => {
+ const data = event.data;
+
+ try {
+ const message = JSON.parse(data);
+
+ handleMessage(message, ws);
+
+ // handle incoming messages if needed
+ } catch (e) {
+ console.error(
+ "Error parsing message from message channel:",
+ e,
+ event,
+ );
+ }
};
ws.onclose = () => {
- setIsCommandConnected(false);
- setCommandSocket(null);
+ setIsMessageConnected(false);
+ setMessageSocket(null);
};
};
@@ -340,31 +394,33 @@ function BrowserStream({
// pass
}
};
+ // NOTE: adding getWebSocketParams causes constant reconnects of message channel when,
+ // for instance, take-control or cede-control is clicked
+ // eslint-disable-next-line react-hooks/exhaustive-deps
}, [
browserSessionId,
canvasContainer,
- commandDisconnectedTrigger,
+ messagesDisconnectedTrigger,
entity,
- getWebSocketParams,
hasBrowserSession,
runId,
showStream,
]);
- // effect to send a command when the user is controlling, vs not controlling
+ // effect to send a message when the user is controlling, vs not controlling
useEffect(() => {
- if (!isCommandConnected) {
+ if (!isMessageConnected) {
return;
}
const sendCommand = (command: Command) => {
- if (!commandSocket) {
+ if (!messageSocket) {
console.warn("Cannot send command, as command socket is closed.");
console.warn(command);
return;
}
- commandSocket.send(JSON.stringify(command));
+ messageSocket.send(JSON.stringify(command));
};
if (interactive || userIsControlling) {
@@ -373,7 +429,7 @@ function BrowserStream({
sendCommand({ kind: "cede-control" });
}
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [interactive, isCommandConnected, userIsControlling]);
+ }, [interactive, isMessageConnected, userIsControlling]);
// Effect to handle window resize trigger for NoVNC canvas
useEffect(() => {
@@ -420,6 +476,134 @@ function BrowserStream({
}
}, [task, workflow]);
+ /**
+ * TODO(jdo): could use zod or smth similar
+ */
+ const getMessage = (data: unknown): MessageIn | undefined => {
+ if (!data) {
+ return;
+ }
+
+ if (typeof data !== "object") {
+ return;
+ }
+
+ if (!("kind" in data)) {
+ return;
+ }
+
+ const k = data.kind;
+
+ if (typeof k !== "string") {
+ return;
+ }
+
+ if (!messageInKinds.includes(k as MessageInKind)) {
+ return;
+ }
+
+ const kind = k as MessageInKind;
+
+ switch (kind) {
+ case "ask-for-clipboard": {
+ return data as MessageInAskForClipboard;
+ }
+ case "copied-text": {
+ if ("text" in data && typeof data.text === "string") {
+ return {
+ kind: "copied-text",
+ text: data.text,
+ };
+ }
+ break;
+ }
+ default: {
+ const _exhaustive: never = kind;
+ return _exhaustive;
+ }
+ }
+ };
+
+ const handleMessage = (data: unknown, ws: WebSocket | null) => {
+ const message = getMessage(data);
+
+ if (!message) {
+ console.warn("Unknown message received on message channel:", data);
+ return;
+ }
+
+ const kind = message.kind;
+
+ const respond = (message: MessageOut) => {
+ if (!ws) {
+ console.warn("Cannot send message, as message socket is null.");
+ console.warn(message);
+ return;
+ }
+
+ ws.send(JSON.stringify(message));
+ };
+
+ switch (kind) {
+ case "ask-for-clipboard": {
+ if (!navigator.clipboard) {
+ console.warn("Clipboard API not available.");
+ return;
+ }
+
+ navigator.clipboard
+ .readText()
+ .then((text) => {
+ toast({
+ title: "Pasting Into Browser",
+ description:
+ "Pasting your current clipboard text into the web page. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).",
+ });
+
+ const response: MessageOutAskForClipboardResponse = {
+ kind: "ask-for-clipboard-response",
+ text,
+ };
+
+ respond(response);
+ })
+ .catch((err) => {
+ console.error("Failed to read clipboard contents: ", err);
+ });
+
+ break;
+ }
+ case "copied-text": {
+ const text = message.text;
+
+ navigator.clipboard
+ .writeText(text)
+ .then(() => {
+ toast({
+ title: "Copied to Clipboard",
+ description:
+ "The text has been copied to your clipboard. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).",
+ });
+ })
+ .catch((err) => {
+ console.error("Failed to write to clipboard:", err);
+
+ toast({
+ variant: "destructive",
+ title: "Failed to write to Clipboard",
+ description: "The text could not be copied to your clipboard.",
+ });
+ });
+
+ break;
+ }
+ default: {
+ const _exhaustive: never = kind;
+ return _exhaustive;
+ }
+ }
+ };
+
const theUserIsControlling =
userIsControlling || (interactive && !showControlButtons);