Skip to content

Automate browser based workflows with AI

add a vnc streaming component for workflows (#2746)

Shuchang Zheng8c02c3c4e038
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index e63ba76d3..ed4a72df3 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -294,6 +294,7 @@ export type WorkflowRunStatusApiResponse = {
total_cost: number | null;
task_v2: TaskV2 | null;
workflow_title: string | null;
+ browser_session_id: string | null;
max_screenshot_scrolling_times: number | null;
};
diff --git a/skyvern-frontend/src/novnc.d.ts b/skyvern-frontend/src/novnc.d.ts
new file mode 100644
index 000000000..68706d6ee
--- /dev/null
+++ b/skyvern-frontend/src/novnc.d.ts
@@ -0,0 +1,39 @@
+declare module "@novnc/novnc/lib/rfb.js" {
+ export interface RfbEvent {
+ detail: {
+ clean: boolean;
+ reason: string;
+ error: {
+ message: string;
+ };
+ message: string;
+ };
+ }
+
+ export interface RfbDisplay {
+ autoscale(): void;
+ _scale: number;
+ }
+
+ export interface RFBOptions {
+ credentials?: { username?: string; password?: string };
+ clipViewport?: boolean;
+ scaleViewport?: boolean;
+ shared?: boolean;
+ resizeSession?: boolean;
+ viewOnly?: boolean;
+ [key: string]: unknown;
+ }
+
+ export default class RFB {
+ _display: RfbDisplay;
+ resizeSession: boolean;
+ scaleViewport: boolean;
+ constructor(target: HTMLElement, url: string, options?: RFBOptions);
+
+ addEventListener(event: string, listener: (e: RfbEvent) => void): void;
+ removeEventListener(event: string, listener: (e: RfbEvent) => void): void;
+ disconnect(): void;
+ viewportChange(): void;
+ }
+}
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
index 9bbba88b0..e0a7ef79b 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
@@ -14,6 +14,7 @@ import {
import { ObserverThoughtScreenshot } from "./ObserverThoughtScreenshot";
import { WorkflowRunBlockScreenshot } from "./WorkflowRunBlockScreenshot";
import { WorkflowRunStream } from "./WorkflowRunStream";
+import { WorkflowRunStreamVnc } from "./WorkflowRunStreamVnc";
import { useSearchParams } from "react-router-dom";
import { findActiveItem } from "./workflowTimelineUtils";
import { Skeleton } from "@/components/ui/skeleton";
@@ -62,9 +63,15 @@ function WorkflowRunOverview() {
workflowRunIsFinalized,
);
+ const streamingComponent = workflowRun.browser_session_id ? (
+ <WorkflowRunStreamVnc />
+ ) : (
+ <WorkflowRunStream />
+ );
+
return (
- <AspectRatio ratio={16 / 9} className="overflow-y-hidden">
- {selection === "stream" && <WorkflowRunStream />}
+ <AspectRatio ratio={16 / 9}>
+ {selection === "stream" && streamingComponent}
{selection !== "stream" && isAction(selection) && (
<ActionScreenshot
index={selection.action_order ?? 0}
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStreamVnc.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStreamVnc.tsx
new file mode 100644
index 000000000..aa25a2440
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStreamVnc.tsx
@@ -0,0 +1,208 @@
+import { Status } from "@/api/types";
+import { useWorkflowRunQuery } from "../hooks/useWorkflowRunQuery";
+import { useEffect, useState, useRef, useCallback } from "react";
+import { HandIcon, PlayIcon } from "@radix-ui/react-icons";
+import { Button } from "@/components/ui/button";
+import { Skeleton } from "@/components/ui/skeleton";
+import { statusIsNotFinalized } from "@/routes/tasks/types";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { useParams } from "react-router-dom";
+import { envCredential } from "@/util/env";
+import { toast } from "@/components/ui/use-toast";
+import { useQueryClient } from "@tanstack/react-query";
+import RFB from "@novnc/novnc/lib/rfb.js";
+import { environment } from "@/util/env";
+import { cn } from "@/util/utils";
+
+import "./workflow-run-stream-vnc.css";
+
+const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;
+
+function WorkflowRunStreamVnc() {
+ const { data: workflowRun } = useWorkflowRunQuery();
+ const { workflowRunId, workflowPermanentId } = useParams<{
+ workflowRunId: string;
+ workflowPermanentId: string;
+ }>();
+
+ const [userIsControlling, setUserIsControlling] = useState<boolean>(false);
+ const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
+ const prevVncConnectedRef = useRef<boolean>(false);
+ const [isVncConnected, setIsVncConnected] = useState<boolean>(false);
+ const showStream = workflowRun && statusIsNotFinalized(workflowRun);
+ const credentialGetter = useCredentialGetter();
+ const queryClient = useQueryClient();
+ const [canvasContainer, setCanvasContainer] = useState<HTMLDivElement | null>(
+ null,
+ );
+ const setCanvasContainerRef = useCallback((node: HTMLDivElement | null) => {
+ setCanvasContainer(node);
+ }, []);
+
+ const rfbRef = useRef<RFB | null>(null);
+
+ // effect for disconnects only
+ useEffect(() => {
+ if (prevVncConnectedRef.current && !isVncConnected) {
+ setVncDisconnectedTrigger((x) => x + 1);
+ }
+ prevVncConnectedRef.current = isVncConnected;
+ }, [isVncConnected]);
+
+ useEffect(
+ () => {
+ if (!showStream || !canvasContainer || !workflowRunId) {
+ if (rfbRef.current) {
+ rfbRef.current.disconnect();
+ rfbRef.current = null;
+ setIsVncConnected(false);
+ }
+ return;
+ }
+
+ async function setupVnc() {
+ let credentialQueryParam = "";
+
+ if (environment === "local") {
+ credentialQueryParam = `?apikey=${envCredential}`;
+ } else {
+ if (credentialGetter) {
+ const token = await credentialGetter();
+ credentialQueryParam = `?token=Bearer ${token}`;
+ } else {
+ credentialQueryParam = `?apikey=${envCredential}`;
+ }
+ }
+
+ if (rfbRef.current && isVncConnected) {
+ return;
+ }
+
+ const vncUrl = `${wssBaseUrl}/stream/vnc/workflow_run/${workflowRunId}${credentialQueryParam}`;
+
+ if (rfbRef.current) {
+ rfbRef.current.disconnect();
+ }
+
+ const canvas = canvasContainer;
+
+ if (!canvas) {
+ throw new Error("Canvas element not found");
+ }
+
+ const rfb = new RFB(canvas, vncUrl);
+
+ rfb.scaleViewport = true;
+
+ rfbRef.current = rfb;
+
+ rfb.addEventListener("connect", () => {
+ setIsVncConnected(true);
+ });
+
+ rfb.addEventListener("disconnect", async (/* e: RfbEvent */) => {
+ setIsVncConnected(false);
+
+ queryClient.invalidateQueries({
+ queryKey: ["workflowRun", workflowPermanentId, workflowRunId],
+ });
+ queryClient.invalidateQueries({ queryKey: ["workflowRuns"] });
+ queryClient.invalidateQueries({
+ queryKey: ["workflowTasks", workflowRunId],
+ });
+ queryClient.invalidateQueries({ queryKey: ["runs"] });
+ });
+ }
+
+ setupVnc();
+
+ return () => {
+ if (rfbRef.current) {
+ rfbRef.current.disconnect();
+ rfbRef.current = null;
+ }
+ setIsVncConnected(false);
+ };
+ },
+ // cannot include isVncConnected in deps as it will cause infinite loop
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ [
+ credentialGetter,
+ workflowRunId,
+ workflowPermanentId,
+ showStream,
+ queryClient,
+ canvasContainer,
+ vncDisconnectedTrigger, // will re-run on disconnects
+ ],
+ );
+
+ // Effect to show toast when workflow reaches a final state based on hook updates
+ useEffect(() => {
+ if (workflowRun) {
+ if (
+ workflowRun.status === Status.Failed ||
+ workflowRun.status === Status.Terminated
+ ) {
+ // Only show toast if VNC is not connected or was never connected,
+ // to avoid double toasting if disconnect handler also triggers similar logic.
+ // However, the disconnect handler now primarily invalidates queries.
+ toast({
+ title: "Run Ended",
+ description: `The workflow run has ${workflowRun.status}.`,
+ variant: "destructive",
+ });
+ } else if (workflowRun.status === Status.Completed) {
+ toast({
+ title: "Run Completed",
+ description: "The workflow run has been completed.",
+ variant: "success",
+ });
+ }
+ }
+ }, [workflowRun, workflowRun?.status]);
+
+ return (
+ <div
+ className={cn("workflow-run-stream-vnc", {
+ "user-is-controlling": userIsControlling,
+ })}
+ ref={setCanvasContainerRef}
+ >
+ {isVncConnected && (
+ <div className="overlay-container">
+ <div className="overlay">
+ <Button
+ // className="take-control"
+ className={cn("take-control", { hide: userIsControlling })}
+ type="button"
+ onClick={() => setUserIsControlling(true)}
+ >
+ <HandIcon className="mr-2 h-4 w-4" />
+ take control
+ </Button>
+ <div className="absolute bottom-[-1rem] right-[1rem]">
+ <Button
+ className={cn("relinquish-control", {
+ hide: !userIsControlling,
+ })}
+ type="button"
+ onClick={() => setUserIsControlling(false)}
+ >
+ <PlayIcon className="mr-2 h-4 w-4" />
+ run agent
+ </Button>
+ </div>
+ </div>
+ </div>
+ )}
+ {!isVncConnected && (
+ <div className="absolute left-0 top-0 flex h-full w-full items-center justify-center bg-black">
+ <Skeleton className="aspect-[16/9] h-auto max-h-full w-full max-w-full rounded-lg object-cover" />
+ </div>
+ )}
+ </div>
+ );
+}
+
+export { WorkflowRunStreamVnc };
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css b/skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css
new file mode 100644
index 000000000..fe0185adb
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css
@@ -0,0 +1,96 @@
+.workflow-run-stream-vnc {
+ position: relative;
+ width: 100%;
+ height: 100%;
+ min-height: 0;
+ padding: 0.5rem;
+ overflow: visible;
+
+ transition: padding 0.2s ease-in-out;
+}
+
+.workflow-run-stream-vnc.user-is-controlling {
+ padding: 0rem;
+}
+
+.workflow-run-stream-vnc .overlay-container {
+ position: absolute;
+ top: 0;
+ left: 0;
+ width: 100%;
+ height: 100%;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+}
+
+.workflow-run-stream-vnc .overlay {
+ position: relative;
+ height: auto;
+ width: 100%;
+ max-height: 100%;
+ aspect-ratio: 16 / 9;
+ object-fit: cover;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+}
+
+.workflow-run-stream-vnc.user-is-controlling .overlay {
+ pointer-events: none;
+}
+
+.workflow-run-stream-vnc.user-is-controlling .overlay-container {
+ pointer-events: none;
+}
+
+.workflow-run-stream-vnc .take-control {
+ transform: translateY(0);
+ transition:
+ transform 0.2s ease-in-out,
+ opacity 0.2s ease-in-out;
+ opacity: 0.3;
+}
+
+.workflow-run-stream-vnc .take-control:not(.hide):hover {
+ opacity: 1;
+}
+
+.workflow-run-stream-vnc .take-control.hide {
+ transform: translateY(100%);
+ opacity: 0;
+ pointer-events: none;
+}
+
+.workflow-run-stream-vnc .relinquish-control {
+ transform: translateY(0);
+ transition:
+ transform 0.2s ease-in-out,
+ opacity 0.2s ease-in-out;
+ opacity: 0.3;
+ pointer-events: all;
+}
+
+.workflow-run-stream-vnc .relinquish-control:not(.hide):hover {
+ opacity: 1;
+}
+
+.workflow-run-stream-vnc .relinquish-control.hide {
+ transform: translateY(100%);
+ opacity: 0;
+ pointer-events: none;
+}
+
+.workflow-run-stream-vnc > div > canvas {
+ opacity: 0;
+ animation: skyvern-anim-fadeIn 1s ease-in forwards;
+}
+
+@keyframes skyvern-anim-fadeIn {
+ from {
+ opacity: 0;
+ }
+ to {
+ opacity: 1;
+ }
+}