Skip to content

Automate browser based workflows with AI

add take control/cede control buttons to browser stream view; improve branding yar (#2989)

Jonathan Dobson7cba56c0e02b
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 3733257db..9456c5813 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -1,18 +1,26 @@
-import { Status } from "@/api/types";
-import { useEffect, useState, useRef, useCallback } from "react";
-import { Skeleton } from "@/components/ui/skeleton";
-import { statusIsNotFinalized } from "@/routes/tasks/types";
-import { useCredentialGetter } from "@/hooks/useCredentialGetter";
-import { envCredential } from "@/util/env";
-import { toast } from "@/components/ui/use-toast";
import RFB from "@novnc/novnc/lib/rfb.js";
-import { environment, wssBaseUrl, newWssBaseUrl } from "@/util/env";
-import { cn } from "@/util/utils";
-import { useClientIdStore } from "@/store/useClientIdStore";
+import { ExitIcon, HandIcon } from "@radix-ui/react-icons";
+import { useEffect, useState, useRef, useCallback } from "react";
+
+import { Status } from "@/api/types";
import type {
TaskApiResponse,
WorkflowRunStatusApiResponse,
} from "@/api/types";
+import { Button } from "@/components/ui/button";
+import { Skeleton } from "@/components/ui/skeleton";
+import { toast } from "@/components/ui/use-toast";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { statusIsNotFinalized } from "@/routes/tasks/types";
+import { useClientIdStore } from "@/store/useClientIdStore";
+import {
+ envCredential,
+ environment,
+ wssBaseUrl,
+ newWssBaseUrl,
+} from "@/util/env";
+import { cn } from "@/util/utils";
+
import "./browser-stream.css";
interface CommandTakeControl {
@@ -28,6 +36,7 @@ type Command = CommandTakeControl | CommandCedeControl;
type Props = {
browserSessionId?: string;
interactive?: boolean;
+ showControlButtons?: boolean;
task?: {
run: TaskApiResponse;
};
@@ -41,6 +50,7 @@ type Props = {
function BrowserStream({
browserSessionId = undefined,
interactive = true,
+ showControlButtons = undefined,
task = undefined,
workflow = undefined,
// --
@@ -65,7 +75,7 @@ function BrowserStream({
} else {
throw new Error("No browser session, task or workflow provided");
}
-
+ const [userIsControlling, setUserIsControlling] = useState(false);
const [commandSocket, setCommandSocket] = useState<WebSocket | null>(null);
const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
const prevVncConnectedRef = useRef<boolean>(false);
@@ -312,14 +322,51 @@ function BrowserStream({
}
}, [task, workflow]);
+ const theUserIsControlling =
+ userIsControlling || (interactive && !showControlButtons);
+
return (
<div
className={cn("browser-stream", {
- "user-is-controlling": interactive,
+ "user-is-controlling": theUserIsControlling,
})}
ref={setCanvasContainerRef}
>
- {isVncConnected && <div className="overlay" />}
+ {isVncConnected && (
+ <div className="overlay z-10 flex items-center justify-center">
+ {showControlButtons && (
+ <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">
+ <Button
+ onClick={() => {
+ setUserIsControlling(true);
+ }}
+ className={cn("control-button pointer-events-auto border", {
+ hide: userIsControlling,
+ })}
+ size="sm"
+ >
+ <HandIcon className="mr-2 h-4 w-4" />
+ take control
+ </Button>
+ <Button
+ onClick={() => {
+ setUserIsControlling(false);
+ }}
+ className={cn(
+ "control-button pointer-events-auto absolute bottom-0 border",
+ {
+ hide: !userIsControlling,
+ },
+ )}
+ size="sm"
+ >
+ <ExitIcon className="mr-2 h-4 w-4" />
+ cede control
+ </Button>
+ </div>
+ )}
+ </div>
+ )}
{!isVncConnected && (
<div className="absolute left-0 top-0 flex h-full w-full items-center justify-center bg-black">
<Skeleton className="aspect-[16/9] h-auto max-h-full w-full max-w-full rounded-lg object-cover" />
diff --git a/skyvern-frontend/src/components/browser-stream.css b/skyvern-frontend/src/components/browser-stream.css
index fe3b92371..52f54b280 100644
--- a/skyvern-frontend/src/components/browser-stream.css
+++ b/skyvern-frontend/src/components/browser-stream.css
@@ -73,6 +73,17 @@
background: transparent !important;
}
+.browser-stream .control-button {
+ transition: 0.3s all ease-in-out;
+ transform: translateY(0px);
+}
+
+.browser-stream .control-button.hide {
+ opacity: 0;
+ pointer-events: none;
+ transform: translateY(15px);
+}
+
@keyframes skyvern-anim-fadeIn {
from {
opacity: 0;
diff --git a/skyvern-frontend/src/routes/browserSession/BrowserSession.tsx b/skyvern-frontend/src/routes/browserSession/BrowserSession.tsx
index fded8e38a..e4f10ea19 100644
--- a/skyvern-frontend/src/routes/browserSession/BrowserSession.tsx
+++ b/skyvern-frontend/src/routes/browserSession/BrowserSession.tsx
@@ -1,9 +1,10 @@
import { useState } from "react";
import { useParams } from "react-router-dom";
-import { BrowserStream } from "@/components/BrowserStream";
-import { getClient } from "@/api/AxiosClient";
-
import { useQuery } from "@tanstack/react-query";
+
+import { getClient } from "@/api/AxiosClient";
+import { BrowserStream } from "@/components/BrowserStream";
+import { LogoMinimized } from "@/components/LogoMinimized";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
function BrowserSession() {
@@ -52,8 +53,20 @@ function BrowserSession() {
return (
<div className="h-screen w-full gap-4 p-6">
- <div className="flex h-full w-full items-center justify-center">
- <BrowserStream browserSessionId={browserSessionId} />
+ <div className="flex h-full w-full flex-col items-start justify-start gap-2">
+ <div className="flex w-full flex-shrink-0 flex-row items-center justify-between rounded-lg border p-4">
+ <div className="flex flex-row items-center justify-start gap-2">
+ <LogoMinimized />
+ <div className="text-xl">browser session</div>
+ </div>
+ </div>
+ <div className="min-h-0 w-full flex-1 rounded-lg border p-4">
+ <BrowserStream
+ browserSessionId={browserSessionId}
+ interactive={false}
+ showControlButtons={true}
+ />
+ </div>
</div>
</div>
);