Skip to content

Automate browser based workflows with AI

multiple UI fixes/updates (#3422)

Jonathan Dobsond82eba77b6ce
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index c69ac7e1b..624923fe8 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -343,9 +343,12 @@ function BrowserStream({
return (
<div
- className={cn("browser-stream flex items-center justify-center", {
- "user-is-controlling": theUserIsControlling,
- })}
+ className={cn(
+ "browser-stream relative flex items-center justify-center",
+ {
+ "user-is-controlling": theUserIsControlling,
+ },
+ )}
ref={setCanvasContainerRef}
>
{isVncConnected && (
@@ -384,7 +387,7 @@ function BrowserStream({
</div>
)}
{!isVncConnected && (
- <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
+ <div className="absolute left-0 top-1/2 flex aspect-video w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
<RotateThrough interval={7 * 1000}>
<span>Hm, working on the connection...</span>
<span>Hang tight, we're almost there...</span>
diff --git a/skyvern-frontend/src/components/Splitter.tsx b/skyvern-frontend/src/components/Splitter.tsx
index 74fc7f2ee..b0b526707 100644
--- a/skyvern-frontend/src/components/Splitter.tsx
+++ b/skyvern-frontend/src/components/Splitter.tsx
@@ -1,7 +1,7 @@
import { useRef, useState, RefObject } from "react";
-import { useMountEffect } from "@/hooks/useMountEffect";
import { cn } from "@/util/utils";
import { useOnChange } from "@/hooks/useOnChange";
+import { useMountEffect } from "@/hooks/useMountEffect";
function Handle({
direction,
@@ -341,21 +341,30 @@ function Splitter({
const [isDragging, setIsDragging] = useState(false);
useMountEffect(() => {
- if (containerRef.current) {
- const newPosition = normalizeUnitsToPercent(
- containerRef,
- direction,
- firstSizingTarget,
- firstSizing,
- storageKey,
- );
+ // small delay here, to allow for arbitrary layout thrashing to settle;
+ // otherwise we have to rely on an observer for the container size, and
+ // resetting whenever the container resizes it likely incorrect behaviour
+ setTimeout(() => {
+ if (containerRef.current) {
+ const newPosition = normalizeUnitsToPercent(
+ containerRef,
+ direction,
+ firstSizingTarget,
+ firstSizing,
+ storageKey,
+ );
- setSplitPosition(newPosition);
+ setSplitPosition(newPosition);
- if (storageKey) {
- setStoredSizing(firstSizingTarget, storageKey, newPosition.toString());
+ if (storageKey) {
+ setStoredSizing(
+ firstSizingTarget,
+ storageKey,
+ newPosition.toString(),
+ );
+ }
}
- }
+ }, 100);
});
useOnChange(isDragging, (newValue, oldValue) => {
diff --git a/skyvern-frontend/src/router.tsx b/skyvern-frontend/src/router.tsx
index 72edd5037..9cf9c2a4c 100644
--- a/skyvern-frontend/src/router.tsx
+++ b/skyvern-frontend/src/router.tsx
@@ -13,6 +13,7 @@ import { TaskDetails } from "./routes/tasks/detail/TaskDetails";
import { TaskParameters } from "./routes/tasks/detail/TaskParameters";
import { TaskRecording } from "./routes/tasks/detail/TaskRecording";
import { TasksPage } from "./routes/tasks/list/TasksPage";
+import { Debugger } from "@/routes/workflows/debugger/Debugger";
import { WorkflowPage } from "./routes/workflows/WorkflowPage";
import { WorkflowRun } from "./routes/workflows/WorkflowRun";
import { WorkflowRunParameters } from "./routes/workflows/WorkflowRunParameters";
@@ -110,11 +111,11 @@ const router = createBrowserRouter([
},
{
path: "debug",
- element: <WorkflowEditor />,
+ element: <Debugger />,
},
{
path: ":workflowRunId/:blockLabel/debug",
- element: <WorkflowEditor />,
+ element: <Debugger />,
},
{
path: "edit",
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index 27b7f2ae3..18e1ffd4f 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -1,5 +1,5 @@
import { AxiosError } from "axios";
-import { useEffect, useRef, useState, useMemo } from "react";
+import { useEffect, useRef, useState } from "react";
import { nanoid } from "nanoid";
import {
ChevronRightIcon,
@@ -17,6 +17,7 @@ import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useMountEffect } from "@/hooks/useMountEffect";
import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";
import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQuery";
+import { WorkflowRunStream } from "@/routes/workflows/workflowRun/WorkflowRunStream";
import { useCacheKeyValuesQuery } from "../hooks/useCacheKeyValuesQuery";
import { useBlockScriptStore } from "@/store/BlockScriptStore";
import { useSidebarStore } from "@/store/SidebarStore";
@@ -137,12 +138,6 @@ function Workspace({
const blockScriptStore = useBlockScriptStore();
const cacheKey = workflow?.cache_key ?? "";
- const enableDebugBrowser = useMemo(() => {
- return (
- showBrowser && (activeDebugSession?.vnc_streaming_supported ?? false)
- );
- }, [showBrowser, activeDebugSession?.vnc_streaming_supported]);
-
const [cacheKeyValue, setCacheKeyValue] = useState(
cacheKey === ""
? ""
@@ -215,10 +210,10 @@ function Workspace({
const hasLoopBlock = nodes.some((node) => node.type === "loop");
const hasHttpBlock = nodes.some((node) => node.type === "http_request");
const workflowWidth = hasHttpBlock
- ? "35.1rem"
+ ? "39rem"
: hasLoopBlock
- ? "31.25rem"
- : "30rem";
+ ? "34.25rem"
+ : "34rem";
/**
* Open a new tab (not window) with the browser session URL.
@@ -705,7 +700,7 @@ function Workspace({
</div>
{/* infinite canvas and sub panels when not in debug mode */}
- {!enableDebugBrowser && (
+ {!showBrowser && (
<div className="relative flex h-full w-full overflow-hidden overflow-x-hidden">
{/* infinite canvas */}
<FlowRenderer
@@ -722,7 +717,7 @@ function Workspace({
{/* sub panels */}
{workflowPanelState.active && (
<div
- className="pointer-events-none absolute right-6 top-[5rem] z-30"
+ className="absolute right-6 top-[8.5rem] z-30"
style={{
height:
workflowPanelState.content === "nodeLibrary"
@@ -750,12 +745,12 @@ function Workspace({
/>
)}
{workflowPanelState.content === "parameters" && (
- <div className="pointer-events-auto relative right-0 top-[3.5rem] z-30">
+ <div className="z-30">
<WorkflowParametersPanel />
</div>
)}
{workflowPanelState.content === "nodeLibrary" && (
- <div className="pointer-events-auto relative right-0 top-[3.5rem] z-30 h-full w-[25rem]">
+ <div className="z-30 h-full w-[25rem]">
<WorkflowNodeLibraryPanel
onNodeClick={(props) => {
addNode(props);
@@ -768,8 +763,51 @@ function Workspace({
</div>
)}
- {/* infinite canvas, sub panels, browser, and timeline when in debug mode */}
- {enableDebugBrowser && (
+ {/* sub panels when in debug mode */}
+ {showBrowser && workflowPanelState.active && (
+ <div
+ className="absolute right-6 top-[8.5rem] z-30"
+ style={{
+ height:
+ workflowPanelState.content === "nodeLibrary"
+ ? "calc(100vh - 14rem)"
+ : "unset",
+ }}
+ >
+ {workflowPanelState.content === "cacheKeyValues" && (
+ <WorkflowCacheKeyValuesPanel
+ cacheKeyValues={cacheKeyValues}
+ pending={cacheKeyValuesLoading}
+ scriptKey={workflow.cache_key ?? "default"}
+ onDelete={(cacheKeyValue) => {
+ setToDeleteCacheKeyValue(cacheKeyValue);
+ setOpenConfirmCacheKeyValueDeleteDialogue(true);
+ }}
+ onPaginate={(page) => {
+ setPage(page);
+ }}
+ onSelect={(cacheKeyValue) => {
+ setCacheKeyValue(cacheKeyValue);
+ setCacheKeyValueFilter("");
+ closeWorkflowPanel();
+ }}
+ />
+ )}
+ {workflowPanelState.content === "parameters" && (
+ <WorkflowParametersPanel />
+ )}
+ {workflowPanelState.content === "nodeLibrary" && (
+ <WorkflowNodeLibraryPanel
+ onNodeClick={(props) => {
+ addNode(props);
+ }}
+ />
+ )}
+ </div>
+ )}
+
+ {/* infinite canvas, browser, and timeline when in debug mode */}
+ {showBrowser && (
<div className="relative flex h-full w-full overflow-hidden overflow-x-hidden">
<Splitter
className="splittah"
@@ -795,96 +833,65 @@ function Workspace({
/>
</div>
- {/* browser & timeline & sub-panels in debug mode */}
+ {/* browser & timeline */}
<div className="skyvern-split-right relative flex h-full items-end justify-center bg-[#020617] p-4 pl-6">
- {/* sub panels */}
- {workflowPanelState.active && (
- <div
- className={cn("absolute right-6 top-[8.5rem] z-30", {
- "left-6": workflowPanelState.content === "nodeLibrary",
- })}
- style={{
- height:
- workflowPanelState.content === "nodeLibrary"
- ? "calc(100vh - 14rem)"
- : "unset",
- }}
- >
- {workflowPanelState.content === "cacheKeyValues" && (
- <WorkflowCacheKeyValuesPanel
- cacheKeyValues={cacheKeyValues}
- pending={cacheKeyValuesLoading}
- scriptKey={workflow.cache_key ?? "default"}
- onDelete={(cacheKeyValue) => {
- setToDeleteCacheKeyValue(cacheKeyValue);
- setOpenConfirmCacheKeyValueDeleteDialogue(true);
- }}
- onPaginate={(page) => {
- setPage(page);
- }}
- onSelect={(cacheKeyValue) => {
- setCacheKeyValue(cacheKeyValue);
- setCacheKeyValueFilter("");
- closeWorkflowPanel();
- }}
- />
- )}
- {workflowPanelState.content === "parameters" && (
- <WorkflowParametersPanel />
- )}
- {workflowPanelState.content === "nodeLibrary" && (
- <WorkflowNodeLibraryPanel
- onNodeClick={(props) => {
- addNode(props);
- }}
- />
- )}
- </div>
- )}
-
- {/* browser & timeline */}
<div className="flex h-[calc(100%_-_8rem)] w-full gap-6">
- {/* browser */}
- <div className="flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
- <div key={reloadKey} className="w-full flex-1">
- {activeDebugSession &&
- activeDebugSession.browser_session_id &&
- !cycleBrowser.isPending ? (
- <BrowserStream
- interactive={interactor === "human"}
- browserSessionId={activeDebugSession.browser_session_id}
- showControlButtons={interactor === "human"}
- resizeTrigger={windowResizeTrigger}
- />
- ) : (
- <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">
- Connecting to your browser...
- <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />
+ {/* VNC browser */}
+ {!activeDebugSession ||
+ (activeDebugSession.vnc_streaming_supported && (
+ <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
+ <div key={reloadKey} className="w-full flex-1">
+ {activeDebugSession &&
+ activeDebugSession.browser_session_id &&
+ !cycleBrowser.isPending ? (
+ <BrowserStream
+ interactive={interactor === "human"}
+ browserSessionId={
+ activeDebugSession.browser_session_id
+ }
+ showControlButtons={interactor === "human"}
+ resizeTrigger={windowResizeTrigger}
+ />
+ ) : (
+ <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">
+ Connecting to your browser...
+ <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />
+ </div>
+ )}
</div>
- )}
- </div>
- <footer className="flex h-[2rem] w-full items-center justify-start gap-4">
- <div className="flex items-center gap-2">
- <GlobeIcon /> Live Browser
+ <footer className="flex h-[2rem] w-full items-center justify-start gap-4">
+ <div className="flex items-center gap-2">
+ <GlobeIcon /> Live Browser
+ </div>
+ {showBreakoutButton && (
+ <BreakoutButton onClick={() => breakout()} />
+ )}
+ <div
+ className={cn("ml-auto flex items-center gap-2", {
+ "mr-16": !blockLabel,
+ })}
+ >
+ {showPowerButton && (
+ <PowerButton onClick={() => cycle()} />
+ )}
+ <ReloadButton
+ isReloading={isReloading}
+ onClick={() => reload()}
+ />
+ </div>
+ </footer>
</div>
- {showBreakoutButton && (
- <BreakoutButton onClick={() => breakout()} />
- )}
- <div
- className={cn("ml-auto flex items-center gap-2", {
- "mr-16": !blockLabel,
- })}
- >
- {showPowerButton && (
- <PowerButton onClick={() => cycle()} />
- )}
- <ReloadButton
- isReloading={isReloading}
- onClick={() => reload()}
- />
+ ))}
+
+ {/* Screenshot browser} */}
+ {activeDebugSession &&
+ !activeDebugSession.vnc_streaming_supported && (
+ <div className="skyvern-screenshot-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
+ <div className="aspect-video w-full">
+ <WorkflowRunStream alwaysShowStream={true} />
+ </div>
</div>
- </footer>
- </div>
+ )}
{/* timeline */}
<div
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx
index 388a8d684..11ac6bc6e 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx
@@ -15,14 +15,20 @@ type StreamMessage = {
screenshot?: string;
};
+interface Props {
+ alwaysShowStream?: boolean;
+}
+
let socket: WebSocket | null = null;
const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;
-function WorkflowRunStream() {
+function WorkflowRunStream(props?: Props) {
+ const alwaysShowStream = props?.alwaysShowStream ?? false;
const { data: workflowRun } = useWorkflowRunQuery();
const [streamImgSrc, setStreamImgSrc] = useState<string>("");
- const showStream = workflowRun && statusIsNotFinalized(workflowRun);
+ const showStream =
+ alwaysShowStream || (workflowRun && statusIsNotFinalized(workflowRun));
const credentialGetter = useCredentialGetter();
const { workflowRunId, workflowPermanentId } = useParams();
const queryClient = useQueryClient();
@@ -149,6 +155,26 @@ function WorkflowRunStream() {
</div>
);
}
+
+ if (alwaysShowStream) {
+ if (streamImgSrc?.length > 0) {
+ return (
+ <div className="h-full w-full">
+ <ZoomableImage
+ src={`data:image/png;base64,${streamImgSrc}`}
+ className="rounded-md"
+ />
+ </div>
+ );
+ }
+
+ return (
+ <div className="flex h-full w-full items-center justify-center">
+ Waiting for stream...
+ </div>
+ );
+ }
+
return null;
}