Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
multiple UI fixes/updates (#3422)
Jonathan Dobsond82eba77b6ce
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex c69ac7e1b..624923fe8 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -343,9 +343,12 @@ function BrowserStream({return (<div- className={cn("browser-stream flex items-center justify-center", {- "user-is-controlling": theUserIsControlling,- })}+ className={cn(+ "browser-stream relative flex items-center justify-center",+ {+ "user-is-controlling": theUserIsControlling,+ },+ )}ref={setCanvasContainerRef}>{isVncConnected && (@@ -384,7 +387,7 @@ function BrowserStream({</div>)}{!isVncConnected && (- <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">+ <div className="absolute left-0 top-1/2 flex aspect-video w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400"><RotateThrough interval={7 * 1000}><span>Hm, working on the connection...</span><span>Hang tight, we're almost there...</span>diff --git a/skyvern-frontend/src/components/Splitter.tsx b/skyvern-frontend/src/components/Splitter.tsxindex 74fc7f2ee..b0b526707 100644--- a/skyvern-frontend/src/components/Splitter.tsx+++ b/skyvern-frontend/src/components/Splitter.tsx@@ -1,7 +1,7 @@import { useRef, useState, RefObject } from "react";-import { useMountEffect } from "@/hooks/useMountEffect";import { cn } from "@/util/utils";import { useOnChange } from "@/hooks/useOnChange";+import { useMountEffect } from "@/hooks/useMountEffect";function Handle({direction,@@ -341,21 +341,30 @@ function Splitter({const [isDragging, setIsDragging] = useState(false);useMountEffect(() => {- if (containerRef.current) {- const newPosition = normalizeUnitsToPercent(- containerRef,- direction,- firstSizingTarget,- firstSizing,- storageKey,- );+ // small delay here, to allow for arbitrary layout thrashing to settle;+ // otherwise we have to rely on an observer for the container size, and+ // resetting whenever the container resizes it likely incorrect behaviour+ setTimeout(() => {+ if (containerRef.current) {+ const newPosition = normalizeUnitsToPercent(+ containerRef,+ direction,+ firstSizingTarget,+ firstSizing,+ storageKey,+ );- setSplitPosition(newPosition);+ setSplitPosition(newPosition);- if (storageKey) {- setStoredSizing(firstSizingTarget, storageKey, newPosition.toString());+ if (storageKey) {+ setStoredSizing(+ firstSizingTarget,+ storageKey,+ newPosition.toString(),+ );+ }}- }+ }, 100);});useOnChange(isDragging, (newValue, oldValue) => {diff --git a/skyvern-frontend/src/router.tsx b/skyvern-frontend/src/router.tsxindex 72edd5037..9cf9c2a4c 100644--- a/skyvern-frontend/src/router.tsx+++ b/skyvern-frontend/src/router.tsx@@ -13,6 +13,7 @@ import { TaskDetails } from "./routes/tasks/detail/TaskDetails";import { TaskParameters } from "./routes/tasks/detail/TaskParameters";import { TaskRecording } from "./routes/tasks/detail/TaskRecording";import { TasksPage } from "./routes/tasks/list/TasksPage";+import { Debugger } from "@/routes/workflows/debugger/Debugger";import { WorkflowPage } from "./routes/workflows/WorkflowPage";import { WorkflowRun } from "./routes/workflows/WorkflowRun";import { WorkflowRunParameters } from "./routes/workflows/WorkflowRunParameters";@@ -110,11 +111,11 @@ const router = createBrowserRouter([},{path: "debug",- element: <WorkflowEditor />,+ element: <Debugger />,},{path: ":workflowRunId/:blockLabel/debug",- element: <WorkflowEditor />,+ element: <Debugger />,},{path: "edit",diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex 27b7f2ae3..18e1ffd4f 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -1,5 +1,5 @@import { AxiosError } from "axios";-import { useEffect, useRef, useState, useMemo } from "react";+import { useEffect, useRef, useState } from "react";import { nanoid } from "nanoid";import {ChevronRightIcon,@@ -17,6 +17,7 @@ import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { useMountEffect } from "@/hooks/useMountEffect";import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQuery";+import { WorkflowRunStream } from "@/routes/workflows/workflowRun/WorkflowRunStream";import { useCacheKeyValuesQuery } from "../hooks/useCacheKeyValuesQuery";import { useBlockScriptStore } from "@/store/BlockScriptStore";import { useSidebarStore } from "@/store/SidebarStore";@@ -137,12 +138,6 @@ function Workspace({const blockScriptStore = useBlockScriptStore();const cacheKey = workflow?.cache_key ?? "";- const enableDebugBrowser = useMemo(() => {- return (- showBrowser && (activeDebugSession?.vnc_streaming_supported ?? false)- );- }, [showBrowser, activeDebugSession?.vnc_streaming_supported]);-const [cacheKeyValue, setCacheKeyValue] = useState(cacheKey === ""? ""@@ -215,10 +210,10 @@ function Workspace({const hasLoopBlock = nodes.some((node) => node.type === "loop");const hasHttpBlock = nodes.some((node) => node.type === "http_request");const workflowWidth = hasHttpBlock- ? "35.1rem"+ ? "39rem": hasLoopBlock- ? "31.25rem"- : "30rem";+ ? "34.25rem"+ : "34rem";/*** Open a new tab (not window) with the browser session URL.@@ -705,7 +700,7 @@ function Workspace({</div>{/* infinite canvas and sub panels when not in debug mode */}- {!enableDebugBrowser && (+ {!showBrowser && (<div className="relative flex h-full w-full overflow-hidden overflow-x-hidden">{/* infinite canvas */}<FlowRenderer@@ -722,7 +717,7 @@ function Workspace({{/* sub panels */}{workflowPanelState.active && (<div- className="pointer-events-none absolute right-6 top-[5rem] z-30"+ className="absolute right-6 top-[8.5rem] z-30"style={{height:workflowPanelState.content === "nodeLibrary"@@ -750,12 +745,12 @@ function Workspace({/>)}{workflowPanelState.content === "parameters" && (- <div className="pointer-events-auto relative right-0 top-[3.5rem] z-30">+ <div className="z-30"><WorkflowParametersPanel /></div>)}{workflowPanelState.content === "nodeLibrary" && (- <div className="pointer-events-auto relative right-0 top-[3.5rem] z-30 h-full w-[25rem]">+ <div className="z-30 h-full w-[25rem]"><WorkflowNodeLibraryPanelonNodeClick={(props) => {addNode(props);@@ -768,8 +763,51 @@ function Workspace({</div>)}- {/* infinite canvas, sub panels, browser, and timeline when in debug mode */}- {enableDebugBrowser && (+ {/* sub panels when in debug mode */}+ {showBrowser && workflowPanelState.active && (+ <div+ className="absolute right-6 top-[8.5rem] z-30"+ style={{+ height:+ workflowPanelState.content === "nodeLibrary"+ ? "calc(100vh - 14rem)"+ : "unset",+ }}+ >+ {workflowPanelState.content === "cacheKeyValues" && (+ <WorkflowCacheKeyValuesPanel+ cacheKeyValues={cacheKeyValues}+ pending={cacheKeyValuesLoading}+ scriptKey={workflow.cache_key ?? "default"}+ onDelete={(cacheKeyValue) => {+ setToDeleteCacheKeyValue(cacheKeyValue);+ setOpenConfirmCacheKeyValueDeleteDialogue(true);+ }}+ onPaginate={(page) => {+ setPage(page);+ }}+ onSelect={(cacheKeyValue) => {+ setCacheKeyValue(cacheKeyValue);+ setCacheKeyValueFilter("");+ closeWorkflowPanel();+ }}+ />+ )}+ {workflowPanelState.content === "parameters" && (+ <WorkflowParametersPanel />+ )}+ {workflowPanelState.content === "nodeLibrary" && (+ <WorkflowNodeLibraryPanel+ onNodeClick={(props) => {+ addNode(props);+ }}+ />+ )}+ </div>+ )}++ {/* infinite canvas, browser, and timeline when in debug mode */}+ {showBrowser && (<div className="relative flex h-full w-full overflow-hidden overflow-x-hidden"><SplitterclassName="splittah"@@ -795,96 +833,65 @@ function Workspace({/></div>- {/* browser & timeline & sub-panels in debug mode */}+ {/* browser & timeline */}<div className="skyvern-split-right relative flex h-full items-end justify-center bg-[#020617] p-4 pl-6">- {/* sub panels */}- {workflowPanelState.active && (- <div- className={cn("absolute right-6 top-[8.5rem] z-30", {- "left-6": workflowPanelState.content === "nodeLibrary",- })}- style={{- height:- workflowPanelState.content === "nodeLibrary"- ? "calc(100vh - 14rem)"- : "unset",- }}- >- {workflowPanelState.content === "cacheKeyValues" && (- <WorkflowCacheKeyValuesPanel- cacheKeyValues={cacheKeyValues}- pending={cacheKeyValuesLoading}- scriptKey={workflow.cache_key ?? "default"}- onDelete={(cacheKeyValue) => {- setToDeleteCacheKeyValue(cacheKeyValue);- setOpenConfirmCacheKeyValueDeleteDialogue(true);- }}- onPaginate={(page) => {- setPage(page);- }}- onSelect={(cacheKeyValue) => {- setCacheKeyValue(cacheKeyValue);- setCacheKeyValueFilter("");- closeWorkflowPanel();- }}- />- )}- {workflowPanelState.content === "parameters" && (- <WorkflowParametersPanel />- )}- {workflowPanelState.content === "nodeLibrary" && (- <WorkflowNodeLibraryPanel- onNodeClick={(props) => {- addNode(props);- }}- />- )}- </div>- )}-- {/* browser & timeline */}<div className="flex h-[calc(100%_-_8rem)] w-full gap-6">- {/* browser */}- <div className="flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">- <div key={reloadKey} className="w-full flex-1">- {activeDebugSession &&- activeDebugSession.browser_session_id &&- !cycleBrowser.isPending ? (- <BrowserStream- interactive={interactor === "human"}- browserSessionId={activeDebugSession.browser_session_id}- showControlButtons={interactor === "human"}- resizeTrigger={windowResizeTrigger}- />- ) : (- <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">- Connecting to your browser...- <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />+ {/* VNC browser */}+ {!activeDebugSession ||+ (activeDebugSession.vnc_streaming_supported && (+ <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">+ <div key={reloadKey} className="w-full flex-1">+ {activeDebugSession &&+ activeDebugSession.browser_session_id &&+ !cycleBrowser.isPending ? (+ <BrowserStream+ interactive={interactor === "human"}+ browserSessionId={+ activeDebugSession.browser_session_id+ }+ showControlButtons={interactor === "human"}+ resizeTrigger={windowResizeTrigger}+ />+ ) : (+ <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">+ Connecting to your browser...+ <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />+ </div>+ )}</div>- )}- </div>- <footer className="flex h-[2rem] w-full items-center justify-start gap-4">- <div className="flex items-center gap-2">- <GlobeIcon /> Live Browser+ <footer className="flex h-[2rem] w-full items-center justify-start gap-4">+ <div className="flex items-center gap-2">+ <GlobeIcon /> Live Browser+ </div>+ {showBreakoutButton && (+ <BreakoutButton onClick={() => breakout()} />+ )}+ <div+ className={cn("ml-auto flex items-center gap-2", {+ "mr-16": !blockLabel,+ })}+ >+ {showPowerButton && (+ <PowerButton onClick={() => cycle()} />+ )}+ <ReloadButton+ isReloading={isReloading}+ onClick={() => reload()}+ />+ </div>+ </footer></div>- {showBreakoutButton && (- <BreakoutButton onClick={() => breakout()} />- )}- <div- className={cn("ml-auto flex items-center gap-2", {- "mr-16": !blockLabel,- })}- >- {showPowerButton && (- <PowerButton onClick={() => cycle()} />- )}- <ReloadButton- isReloading={isReloading}- onClick={() => reload()}- />+ ))}++ {/* Screenshot browser} */}+ {activeDebugSession &&+ !activeDebugSession.vnc_streaming_supported && (+ <div className="skyvern-screenshot-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">+ <div className="aspect-video w-full">+ <WorkflowRunStream alwaysShowStream={true} />+ </div></div>- </footer>- </div>+ )}{/* timeline */}<divdiff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsxindex 388a8d684..11ac6bc6e 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx@@ -15,14 +15,20 @@ type StreamMessage = {screenshot?: string;};+interface Props {+ alwaysShowStream?: boolean;+}+let socket: WebSocket | null = null;const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;-function WorkflowRunStream() {+function WorkflowRunStream(props?: Props) {+ const alwaysShowStream = props?.alwaysShowStream ?? false;const { data: workflowRun } = useWorkflowRunQuery();const [streamImgSrc, setStreamImgSrc] = useState<string>("");- const showStream = workflowRun && statusIsNotFinalized(workflowRun);+ const showStream =+ alwaysShowStream || (workflowRun && statusIsNotFinalized(workflowRun));const credentialGetter = useCredentialGetter();const { workflowRunId, workflowPermanentId } = useParams();const queryClient = useQueryClient();@@ -149,6 +155,26 @@ function WorkflowRunStream() {</div>);}++ if (alwaysShowStream) {+ if (streamImgSrc?.length > 0) {+ return (+ <div className="h-full w-full">+ <ZoomableImage+ src={`data:image/png;base64,${streamImgSrc}`}+ className="rounded-md"+ />+ </div>+ );+ }++ return (+ <div className="flex h-full w-full items-center justify-center">+ Waiting for stream...+ </div>+ );+ }+return null;}