Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
FE portion of seamless clipboard transfer in browser stream (#3799)
Jonathan Dobsonf66f50fc9391
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 34cda4a31..b3a5473f8 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -39,8 +39,31 @@ interface CommandCedeControl {kind: "cede-control";}+// a "Command" is an fire-n-forget out-message - it does not require a responsetype Command = CommandTakeControl | CommandCedeControl;+const messageInKinds = ["ask-for-clipboard", "copied-text"] as const;++type MessageInKind = (typeof messageInKinds)[number];++interface MessageInAskForClipboard {+ kind: "ask-for-clipboard";+}++interface MessageInCopiedText {+ kind: "copied-text";+ text: string;+}++type MessageIn = MessageInCopiedText | MessageInAskForClipboard;++interface MessageOutAskForClipboardResponse {+ kind: "ask-for-clipboard-response";+ text: string;+}++type MessageOut = MessageOutAskForClipboardResponse;+type Props = {browserSessionId?: string;interactive?: boolean;@@ -102,6 +125,20 @@ function BrowserStream({if (!browserSession || browserSession.completed_at) {setHasBrowserSession(false);return false;++ // NOTE(jdo:streaming-local-dev): remove above and use this instead+ // if (browserSession && browserSession.completed_at) {+ // console.warn(+ // "Completed at:",+ // browserSession.completed_at,+ // "continuing anyway!",+ // );+ // setHasBrowserSession(true);+ // return true;+ // } else {+ // setHasBrowserSession(false);+ // return false;+ // }}setHasBrowserSession(true);@@ -117,16 +154,16 @@ function BrowserStream({const [hasBrowserSession, setHasBrowserSession] = useState(true); // be optimisticconst [userIsControlling, setUserIsControlling] = useState(false);- const [commandSocket, setCommandSocket] = useState<WebSocket | null>(null);+ const [messageSocket, setMessageSocket] = useState<WebSocket | null>(null);const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);const prevVncConnectedRef = useRef<boolean>(false);const [isVncConnected, setIsVncConnected] = useState<boolean>(false);const [isCanvasReady, setIsCanvasReady] = useState<boolean>(false);const [isReady, setIsReady] = useState(false);- const [commandDisconnectedTrigger, setCommandDisconnectedTrigger] =+ const [messagesDisconnectedTrigger, setMessagesDisconnectedTrigger] =useState(0);- const prevCommandConnectedRef = useRef<boolean>(false);- const [isCommandConnected, setIsCommandConnected] = useState<boolean>(false);+ const prevMessageConnectedRef = useRef<boolean>(false);+ const [isMessageConnected, setIsMessageConnected] = useState<boolean>(false);const [canvasContainer, setCanvasContainer] = useState<HTMLDivElement | null>(null,);@@ -168,14 +205,14 @@ function BrowserStream({prevVncConnectedRef.current = isVncConnected;}, [isVncConnected, onClose]);- // effect for command disconnects only+ // effect for message disconnects onlyuseEffect(() => {- if (prevCommandConnectedRef.current && !isCommandConnected) {- setCommandDisconnectedTrigger((x) => x + 1);+ if (prevMessageConnectedRef.current && !isMessageConnected) {+ setMessagesDisconnectedTrigger((x) => x + 1);onClose?.();}- prevCommandConnectedRef.current = isCommandConnected;- }, [isCommandConnected, onClose]);+ prevMessageConnectedRef.current = isMessageConnected;+ }, [isMessageConnected, onClose]);// vnc socketuseEffect(@@ -289,7 +326,6 @@ function BrowserStream({],);- // command socketuseEffect(() => {if (!showStream || !canvasContainer || !runId) {return;@@ -300,34 +336,52 @@ function BrowserStream({const connect = async () => {const wsParams = await getWebSocketParams();- const commandUrl =+ const messageUrl =entity === "browserSession"- ? `${newWssBaseUrl}/stream/commands/browser_session/${runId}?${wsParams}`+ ? `${newWssBaseUrl}/stream/messages/browser_session/${runId}?${wsParams}`: entity === "task"- ? `${wssBaseUrl}/stream/commands/task/${runId}?${wsParams}`+ ? `${wssBaseUrl}/stream/messages/task/${runId}?${wsParams}`: entity === "workflow"- ? `${wssBaseUrl}/stream/commands/workflow_run/${runId}?${wsParams}`+ ? `${wssBaseUrl}/stream/messages/workflow_run/${runId}?${wsParams}`: null;- if (!commandUrl) {- throw new Error("No command url");+ if (!messageUrl) {+ throw new Error("No message url");}if (!hasBrowserSession) {- setIsCommandConnected(false);+ setIsMessageConnected(false);return;}- ws = new WebSocket(commandUrl);+ ws = new WebSocket(messageUrl);ws.onopen = () => {- setIsCommandConnected(true);- setCommandSocket(ws);+ setIsMessageConnected(true);+ setMessageSocket(ws);+ };++ ws.onmessage = (event) => {+ const data = event.data;++ try {+ const message = JSON.parse(data);++ handleMessage(message, ws);++ // handle incoming messages if needed+ } catch (e) {+ console.error(+ "Error parsing message from message channel:",+ e,+ event,+ );+ }};ws.onclose = () => {- setIsCommandConnected(false);- setCommandSocket(null);+ setIsMessageConnected(false);+ setMessageSocket(null);};};@@ -340,31 +394,33 @@ function BrowserStream({// pass}};+ // NOTE: adding getWebSocketParams causes constant reconnects of message channel when,+ // for instance, take-control or cede-control is clicked+ // eslint-disable-next-line react-hooks/exhaustive-deps}, [browserSessionId,canvasContainer,- commandDisconnectedTrigger,+ messagesDisconnectedTrigger,entity,- getWebSocketParams,hasBrowserSession,runId,showStream,]);- // effect to send a command when the user is controlling, vs not controlling+ // effect to send a message when the user is controlling, vs not controllinguseEffect(() => {- if (!isCommandConnected) {+ if (!isMessageConnected) {return;}const sendCommand = (command: Command) => {- if (!commandSocket) {+ if (!messageSocket) {console.warn("Cannot send command, as command socket is closed.");console.warn(command);return;}- commandSocket.send(JSON.stringify(command));+ messageSocket.send(JSON.stringify(command));};if (interactive || userIsControlling) {@@ -373,7 +429,7 @@ function BrowserStream({sendCommand({ kind: "cede-control" });}// eslint-disable-next-line react-hooks/exhaustive-deps- }, [interactive, isCommandConnected, userIsControlling]);+ }, [interactive, isMessageConnected, userIsControlling]);// Effect to handle window resize trigger for NoVNC canvasuseEffect(() => {@@ -420,6 +476,134 @@ function BrowserStream({}}, [task, workflow]);+ /**+ * TODO(jdo): could use zod or smth similar+ */+ const getMessage = (data: unknown): MessageIn | undefined => {+ if (!data) {+ return;+ }++ if (typeof data !== "object") {+ return;+ }++ if (!("kind" in data)) {+ return;+ }++ const k = data.kind;++ if (typeof k !== "string") {+ return;+ }++ if (!messageInKinds.includes(k as MessageInKind)) {+ return;+ }++ const kind = k as MessageInKind;++ switch (kind) {+ case "ask-for-clipboard": {+ return data as MessageInAskForClipboard;+ }+ case "copied-text": {+ if ("text" in data && typeof data.text === "string") {+ return {+ kind: "copied-text",+ text: data.text,+ };+ }+ break;+ }+ default: {+ const _exhaustive: never = kind;+ return _exhaustive;+ }+ }+ };++ const handleMessage = (data: unknown, ws: WebSocket | null) => {+ const message = getMessage(data);++ if (!message) {+ console.warn("Unknown message received on message channel:", data);+ return;+ }++ const kind = message.kind;++ const respond = (message: MessageOut) => {+ if (!ws) {+ console.warn("Cannot send message, as message socket is null.");+ console.warn(message);+ return;+ }++ ws.send(JSON.stringify(message));+ };++ switch (kind) {+ case "ask-for-clipboard": {+ if (!navigator.clipboard) {+ console.warn("Clipboard API not available.");+ return;+ }++ navigator.clipboard+ .readText()+ .then((text) => {+ toast({+ title: "Pasting Into Browser",+ description:+ "Pasting your current clipboard text into the web page. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).",+ });++ const response: MessageOutAskForClipboardResponse = {+ kind: "ask-for-clipboard-response",+ text,+ };++ respond(response);+ })+ .catch((err) => {+ console.error("Failed to read clipboard contents: ", err);+ });++ break;+ }+ case "copied-text": {+ const text = message.text;++ navigator.clipboard+ .writeText(text)+ .then(() => {+ toast({+ title: "Copied to Clipboard",+ description:+ "The text has been copied to your clipboard. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).",+ });+ })+ .catch((err) => {+ console.error("Failed to write to clipboard:", err);++ toast({+ variant: "destructive",+ title: "Failed to write to Clipboard",+ description: "The text could not be copied to your clipboard.",+ });+ });++ break;+ }+ default: {+ const _exhaustive: never = kind;+ return _exhaustive;+ }+ }+ };+const theUserIsControlling =userIsControlling || (interactive && !showControlButtons);