Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add take control/cede control buttons to browser stream view; improve branding yar (#2989)
Jonathan Dobson7cba56c0e02b
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 3733257db..9456c5813 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -1,18 +1,26 @@-import { Status } from "@/api/types";-import { useEffect, useState, useRef, useCallback } from "react";-import { Skeleton } from "@/components/ui/skeleton";-import { statusIsNotFinalized } from "@/routes/tasks/types";-import { useCredentialGetter } from "@/hooks/useCredentialGetter";-import { envCredential } from "@/util/env";-import { toast } from "@/components/ui/use-toast";import RFB from "@novnc/novnc/lib/rfb.js";-import { environment, wssBaseUrl, newWssBaseUrl } from "@/util/env";-import { cn } from "@/util/utils";-import { useClientIdStore } from "@/store/useClientIdStore";+import { ExitIcon, HandIcon } from "@radix-ui/react-icons";+import { useEffect, useState, useRef, useCallback } from "react";++import { Status } from "@/api/types";import type {TaskApiResponse,WorkflowRunStatusApiResponse,} from "@/api/types";+import { Button } from "@/components/ui/button";+import { Skeleton } from "@/components/ui/skeleton";+import { toast } from "@/components/ui/use-toast";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { statusIsNotFinalized } from "@/routes/tasks/types";+import { useClientIdStore } from "@/store/useClientIdStore";+import {+ envCredential,+ environment,+ wssBaseUrl,+ newWssBaseUrl,+} from "@/util/env";+import { cn } from "@/util/utils";+import "./browser-stream.css";interface CommandTakeControl {@@ -28,6 +36,7 @@ type Command = CommandTakeControl | CommandCedeControl;type Props = {browserSessionId?: string;interactive?: boolean;+ showControlButtons?: boolean;task?: {run: TaskApiResponse;};@@ -41,6 +50,7 @@ type Props = {function BrowserStream({browserSessionId = undefined,interactive = true,+ showControlButtons = undefined,task = undefined,workflow = undefined,// --@@ -65,7 +75,7 @@ function BrowserStream({} else {throw new Error("No browser session, task or workflow provided");}-+ const [userIsControlling, setUserIsControlling] = useState(false);const [commandSocket, setCommandSocket] = useState<WebSocket | null>(null);const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);const prevVncConnectedRef = useRef<boolean>(false);@@ -312,14 +322,51 @@ function BrowserStream({}}, [task, workflow]);+ const theUserIsControlling =+ userIsControlling || (interactive && !showControlButtons);+return (<divclassName={cn("browser-stream", {- "user-is-controlling": interactive,+ "user-is-controlling": theUserIsControlling,})}ref={setCanvasContainerRef}>- {isVncConnected && <div className="overlay" />}+ {isVncConnected && (+ <div className="overlay z-10 flex items-center justify-center">+ {showControlButtons && (+ <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">+ <Button+ onClick={() => {+ setUserIsControlling(true);+ }}+ className={cn("control-button pointer-events-auto border", {+ hide: userIsControlling,+ })}+ size="sm"+ >+ <HandIcon className="mr-2 h-4 w-4" />+ take control+ </Button>+ <Button+ onClick={() => {+ setUserIsControlling(false);+ }}+ className={cn(+ "control-button pointer-events-auto absolute bottom-0 border",+ {+ hide: !userIsControlling,+ },+ )}+ size="sm"+ >+ <ExitIcon className="mr-2 h-4 w-4" />+ cede control+ </Button>+ </div>+ )}+ </div>+ )}{!isVncConnected && (<div className="absolute left-0 top-0 flex h-full w-full items-center justify-center bg-black"><Skeleton className="aspect-[16/9] h-auto max-h-full w-full max-w-full rounded-lg object-cover" />diff --git a/skyvern-frontend/src/components/browser-stream.css b/skyvern-frontend/src/components/browser-stream.cssindex fe3b92371..52f54b280 100644--- a/skyvern-frontend/src/components/browser-stream.css+++ b/skyvern-frontend/src/components/browser-stream.css@@ -73,6 +73,17 @@background: transparent !important;}+.browser-stream .control-button {+ transition: 0.3s all ease-in-out;+ transform: translateY(0px);+}++.browser-stream .control-button.hide {+ opacity: 0;+ pointer-events: none;+ transform: translateY(15px);+}+@keyframes skyvern-anim-fadeIn {from {opacity: 0;diff --git a/skyvern-frontend/src/routes/browserSession/BrowserSession.tsx b/skyvern-frontend/src/routes/browserSession/BrowserSession.tsxindex fded8e38a..e4f10ea19 100644--- a/skyvern-frontend/src/routes/browserSession/BrowserSession.tsx+++ b/skyvern-frontend/src/routes/browserSession/BrowserSession.tsx@@ -1,9 +1,10 @@import { useState } from "react";import { useParams } from "react-router-dom";-import { BrowserStream } from "@/components/BrowserStream";-import { getClient } from "@/api/AxiosClient";-import { useQuery } from "@tanstack/react-query";++import { getClient } from "@/api/AxiosClient";+import { BrowserStream } from "@/components/BrowserStream";+import { LogoMinimized } from "@/components/LogoMinimized";import { useCredentialGetter } from "@/hooks/useCredentialGetter";function BrowserSession() {@@ -52,8 +53,20 @@ function BrowserSession() {return (<div className="h-screen w-full gap-4 p-6">- <div className="flex h-full w-full items-center justify-center">- <BrowserStream browserSessionId={browserSessionId} />+ <div className="flex h-full w-full flex-col items-start justify-start gap-2">+ <div className="flex w-full flex-shrink-0 flex-row items-center justify-between rounded-lg border p-4">+ <div className="flex flex-row items-center justify-start gap-2">+ <LogoMinimized />+ <div className="text-xl">browser session</div>+ </div>+ </div>+ <div className="min-h-0 w-full flex-1 rounded-lg border p-4">+ <BrowserStream+ browserSessionId={browserSessionId}+ interactive={false}+ showControlButtons={true}+ />+ </div></div></div>);