Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Jon/browser session view (#2911)
Jonathan Dobsondd0be005b6fa
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex a62dafdd1..fd4945d01 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -19,6 +19,7 @@ import type {import "./browser-stream.css";const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;+const newWssBaseUrl = wssBaseUrl.replace("/api", "");interface CommandTakeControl {kind: "take-control";@@ -31,6 +32,7 @@ interface CommandCedeControl {type Command = CommandTakeControl | CommandCedeControl;type Props = {+ browserSessionId?: string;task?: {run: TaskApiResponse;};@@ -42,6 +44,7 @@ type Props = {};function BrowserStream({+ browserSessionId = undefined,task = undefined,workflow = undefined,// --@@ -49,9 +52,13 @@ function BrowserStream({}: Props) {let showStream: boolean = false;let runId: string;- let entity: "task" | "workflow";+ let entity: "browserSession" | "task" | "workflow";- if (task) {+ if (browserSessionId) {+ runId = browserSessionId;+ entity = "browserSession";+ showStream = true;+ } else if (task) {runId = task.run.task_id;showStream = statusIsNotFinalized(task.run);entity = "task";@@ -60,7 +67,7 @@ function BrowserStream({showStream = statusIsNotFinalized(workflow.run);entity = "workflow";} else {- throw new Error("No task or workflow provided");+ throw new Error("No browser session, task or workflow provided");}const [commandSocket, setCommandSocket] = useState<WebSocket | null>(null);@@ -141,11 +148,13 @@ function BrowserStream({const wsParams = await getWebSocketParams();const vncUrl =- entity === "task"- ? `${wssBaseUrl}/stream/vnc/task/${runId}?${wsParams}`- : entity === "workflow"- ? `${wssBaseUrl}/stream/vnc/workflow_run/${runId}?${wsParams}`- : null;+ entity === "browserSession"+ ? `${newWssBaseUrl}/stream/vnc/browser_session/${runId}?${wsParams}`+ : entity === "task"+ ? `${wssBaseUrl}/stream/vnc/task/${runId}?${wsParams}`+ : entity === "workflow"+ ? `${wssBaseUrl}/stream/vnc/workflow_run/${runId}?${wsParams}`+ : null;if (!vncUrl) {throw new Error("No vnc url");@@ -209,11 +218,13 @@ function BrowserStream({const wsParams = await getWebSocketParams();const commandUrl =- entity === "task"- ? `${wssBaseUrl}/stream/commands/task/${runId}?${wsParams}`- : entity === "workflow"- ? `${wssBaseUrl}/stream/commands/workflow_run/${runId}?${wsParams}`- : null;+ entity === "browserSession"+ ? `${newWssBaseUrl}/stream/commands/browser_session/${runId}?${wsParams}`+ : entity === "task"+ ? `${wssBaseUrl}/stream/commands/task/${runId}?${wsParams}`+ : entity === "workflow"+ ? `${wssBaseUrl}/stream/commands/workflow_run/${runId}?${wsParams}`+ : null;if (!commandUrl) {throw new Error("No command url");diff --git a/skyvern-frontend/src/components/browser-stream.css b/skyvern-frontend/src/components/browser-stream.cssindex f1d47dc63..edefc92d9 100644--- a/skyvern-frontend/src/components/browser-stream.css+++ b/skyvern-frontend/src/components/browser-stream.css@@ -86,6 +86,10 @@animation: skyvern-anim-fadeIn 1s ease-in forwards;}+.browser-stream > div {+ background: transparent !important;+}+@keyframes skyvern-anim-fadeIn {from {opacity: 0;diff --git a/skyvern-frontend/src/router.tsx b/skyvern-frontend/src/router.tsxindex 0b5725dc3..72edd5037 100644--- a/skyvern-frontend/src/router.tsx+++ b/skyvern-frontend/src/router.tsx@@ -1,4 +1,5 @@import { Navigate, Outlet, createBrowserRouter } from "react-router-dom";+import { BrowserSession } from "@/routes/browserSession/BrowserSession";import { PageLayout } from "./components/PageLayout";import { DiscoverPage } from "./routes/discover/DiscoverPage";import { HistoryPage } from "./routes/history/HistoryPage";@@ -25,6 +26,10 @@ import { WorkflowRunRecording } from "./routes/workflows/workflowRun/WorkflowRunimport { DebugStoreProvider } from "@/store/DebugStoreContext";const router = createBrowserRouter([+ {+ path: "browser-session/:browserSessionId",+ element: <BrowserSession />,+ },{path: "/",element: (diff --git a/skyvern-frontend/src/routes/browserSession/BrowserSession.tsx b/skyvern-frontend/src/routes/browserSession/BrowserSession.tsxnew file mode 100644index 000000000..8641ff14a--- /dev/null+++ b/skyvern-frontend/src/routes/browserSession/BrowserSession.tsx@@ -0,0 +1,60 @@+import { useState } from "react";+import { useParams } from "react-router-dom";+import { BrowserStream } from "@/components/BrowserStream";+import { getClient } from "@/api/AxiosClient";++import { useQuery } from "@tanstack/react-query";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";++function BrowserSession() {+ const { browserSessionId } = useParams();+ const [hasBrowserSession, setHasBrowserSession] = useState(false);++ const credentialGetter = useCredentialGetter();++ const query = useQuery({+ queryKey: ["browserSession", browserSessionId],+ queryFn: async () => {+ const client = await getClient(credentialGetter, "sans-api-v1");++ try {+ await client.get(`/browser_sessions/${browserSessionId}`);+ setHasBrowserSession(true);+ } catch (error) {+ setHasBrowserSession(false);+ }+ },+ });++ if (query.isLoading) {+ return (+ <div className="h-screen w-full gap-4 p-6">+ <div className="flex h-full w-full items-center justify-center">+ {/* we need nice artwork here */}+ Loading...+ </div>+ </div>+ );+ }++ if (!hasBrowserSession) {+ return (+ <div className="h-screen w-full gap-4 p-6">+ <div className="flex h-full w-full items-center justify-center">+ {/* we need nice artwork here */}+ No browser session found.+ </div>+ </div>+ );+ }++ return (+ <div className="h-screen w-full gap-4 p-6">+ <div className="flex h-full w-full items-center justify-center">+ <BrowserStream browserSessionId={browserSessionId} />+ </div>+ </div>+ );+}++export { BrowserSession };