Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add a vnc streaming component for workflows (#2746)
Shuchang Zheng8c02c3c4e038
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex e63ba76d3..ed4a72df3 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -294,6 +294,7 @@ export type WorkflowRunStatusApiResponse = {total_cost: number | null;task_v2: TaskV2 | null;workflow_title: string | null;+ browser_session_id: string | null;max_screenshot_scrolling_times: number | null;};diff --git a/skyvern-frontend/src/novnc.d.ts b/skyvern-frontend/src/novnc.d.tsnew file mode 100644index 000000000..68706d6ee--- /dev/null+++ b/skyvern-frontend/src/novnc.d.ts@@ -0,0 +1,39 @@+declare module "@novnc/novnc/lib/rfb.js" {+ export interface RfbEvent {+ detail: {+ clean: boolean;+ reason: string;+ error: {+ message: string;+ };+ message: string;+ };+ }++ export interface RfbDisplay {+ autoscale(): void;+ _scale: number;+ }++ export interface RFBOptions {+ credentials?: { username?: string; password?: string };+ clipViewport?: boolean;+ scaleViewport?: boolean;+ shared?: boolean;+ resizeSession?: boolean;+ viewOnly?: boolean;+ [key: string]: unknown;+ }++ export default class RFB {+ _display: RfbDisplay;+ resizeSession: boolean;+ scaleViewport: boolean;+ constructor(target: HTMLElement, url: string, options?: RFBOptions);++ addEventListener(event: string, listener: (e: RfbEvent) => void): void;+ removeEventListener(event: string, listener: (e: RfbEvent) => void): void;+ disconnect(): void;+ viewportChange(): void;+ }+}diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsxindex 9bbba88b0..e0a7ef79b 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx@@ -14,6 +14,7 @@ import {import { ObserverThoughtScreenshot } from "./ObserverThoughtScreenshot";import { WorkflowRunBlockScreenshot } from "./WorkflowRunBlockScreenshot";import { WorkflowRunStream } from "./WorkflowRunStream";+import { WorkflowRunStreamVnc } from "./WorkflowRunStreamVnc";import { useSearchParams } from "react-router-dom";import { findActiveItem } from "./workflowTimelineUtils";import { Skeleton } from "@/components/ui/skeleton";@@ -62,9 +63,15 @@ function WorkflowRunOverview() {workflowRunIsFinalized,);+ const streamingComponent = workflowRun.browser_session_id ? (+ <WorkflowRunStreamVnc />+ ) : (+ <WorkflowRunStream />+ );+return (- <AspectRatio ratio={16 / 9} className="overflow-y-hidden">- {selection === "stream" && <WorkflowRunStream />}+ <AspectRatio ratio={16 / 9}>+ {selection === "stream" && streamingComponent}{selection !== "stream" && isAction(selection) && (<ActionScreenshotindex={selection.action_order ?? 0}diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStreamVnc.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStreamVnc.tsxnew file mode 100644index 000000000..aa25a2440--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStreamVnc.tsx@@ -0,0 +1,208 @@+import { Status } from "@/api/types";+import { useWorkflowRunQuery } from "../hooks/useWorkflowRunQuery";+import { useEffect, useState, useRef, useCallback } from "react";+import { HandIcon, PlayIcon } from "@radix-ui/react-icons";+import { Button } from "@/components/ui/button";+import { Skeleton } from "@/components/ui/skeleton";+import { statusIsNotFinalized } from "@/routes/tasks/types";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { useParams } from "react-router-dom";+import { envCredential } from "@/util/env";+import { toast } from "@/components/ui/use-toast";+import { useQueryClient } from "@tanstack/react-query";+import RFB from "@novnc/novnc/lib/rfb.js";+import { environment } from "@/util/env";+import { cn } from "@/util/utils";++import "./workflow-run-stream-vnc.css";++const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;++function WorkflowRunStreamVnc() {+ const { data: workflowRun } = useWorkflowRunQuery();+ const { workflowRunId, workflowPermanentId } = useParams<{+ workflowRunId: string;+ workflowPermanentId: string;+ }>();++ const [userIsControlling, setUserIsControlling] = useState<boolean>(false);+ const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);+ const prevVncConnectedRef = useRef<boolean>(false);+ const [isVncConnected, setIsVncConnected] = useState<boolean>(false);+ const showStream = workflowRun && statusIsNotFinalized(workflowRun);+ const credentialGetter = useCredentialGetter();+ const queryClient = useQueryClient();+ const [canvasContainer, setCanvasContainer] = useState<HTMLDivElement | null>(+ null,+ );+ const setCanvasContainerRef = useCallback((node: HTMLDivElement | null) => {+ setCanvasContainer(node);+ }, []);++ const rfbRef = useRef<RFB | null>(null);++ // effect for disconnects only+ useEffect(() => {+ if (prevVncConnectedRef.current && !isVncConnected) {+ setVncDisconnectedTrigger((x) => x + 1);+ }+ prevVncConnectedRef.current = isVncConnected;+ }, [isVncConnected]);++ useEffect(+ () => {+ if (!showStream || !canvasContainer || !workflowRunId) {+ if (rfbRef.current) {+ rfbRef.current.disconnect();+ rfbRef.current = null;+ setIsVncConnected(false);+ }+ return;+ }++ async function setupVnc() {+ let credentialQueryParam = "";++ if (environment === "local") {+ credentialQueryParam = `?apikey=${envCredential}`;+ } else {+ if (credentialGetter) {+ const token = await credentialGetter();+ credentialQueryParam = `?token=Bearer ${token}`;+ } else {+ credentialQueryParam = `?apikey=${envCredential}`;+ }+ }++ if (rfbRef.current && isVncConnected) {+ return;+ }++ const vncUrl = `${wssBaseUrl}/stream/vnc/workflow_run/${workflowRunId}${credentialQueryParam}`;++ if (rfbRef.current) {+ rfbRef.current.disconnect();+ }++ const canvas = canvasContainer;++ if (!canvas) {+ throw new Error("Canvas element not found");+ }++ const rfb = new RFB(canvas, vncUrl);++ rfb.scaleViewport = true;++ rfbRef.current = rfb;++ rfb.addEventListener("connect", () => {+ setIsVncConnected(true);+ });++ rfb.addEventListener("disconnect", async (/* e: RfbEvent */) => {+ setIsVncConnected(false);++ queryClient.invalidateQueries({+ queryKey: ["workflowRun", workflowPermanentId, workflowRunId],+ });+ queryClient.invalidateQueries({ queryKey: ["workflowRuns"] });+ queryClient.invalidateQueries({+ queryKey: ["workflowTasks", workflowRunId],+ });+ queryClient.invalidateQueries({ queryKey: ["runs"] });+ });+ }++ setupVnc();++ return () => {+ if (rfbRef.current) {+ rfbRef.current.disconnect();+ rfbRef.current = null;+ }+ setIsVncConnected(false);+ };+ },+ // cannot include isVncConnected in deps as it will cause infinite loop+ // eslint-disable-next-line react-hooks/exhaustive-deps+ [+ credentialGetter,+ workflowRunId,+ workflowPermanentId,+ showStream,+ queryClient,+ canvasContainer,+ vncDisconnectedTrigger, // will re-run on disconnects+ ],+ );++ // Effect to show toast when workflow reaches a final state based on hook updates+ useEffect(() => {+ if (workflowRun) {+ if (+ workflowRun.status === Status.Failed ||+ workflowRun.status === Status.Terminated+ ) {+ // Only show toast if VNC is not connected or was never connected,+ // to avoid double toasting if disconnect handler also triggers similar logic.+ // However, the disconnect handler now primarily invalidates queries.+ toast({+ title: "Run Ended",+ description: `The workflow run has ${workflowRun.status}.`,+ variant: "destructive",+ });+ } else if (workflowRun.status === Status.Completed) {+ toast({+ title: "Run Completed",+ description: "The workflow run has been completed.",+ variant: "success",+ });+ }+ }+ }, [workflowRun, workflowRun?.status]);++ return (+ <div+ className={cn("workflow-run-stream-vnc", {+ "user-is-controlling": userIsControlling,+ })}+ ref={setCanvasContainerRef}+ >+ {isVncConnected && (+ <div className="overlay-container">+ <div className="overlay">+ <Button+ // className="take-control"+ className={cn("take-control", { hide: userIsControlling })}+ type="button"+ onClick={() => setUserIsControlling(true)}+ >+ <HandIcon className="mr-2 h-4 w-4" />+ take control+ </Button>+ <div className="absolute bottom-[-1rem] right-[1rem]">+ <Button+ className={cn("relinquish-control", {+ hide: !userIsControlling,+ })}+ type="button"+ onClick={() => setUserIsControlling(false)}+ >+ <PlayIcon className="mr-2 h-4 w-4" />+ run agent+ </Button>+ </div>+ </div>+ </div>+ )}+ {!isVncConnected && (+ <div className="absolute left-0 top-0 flex h-full w-full items-center justify-center bg-black">+ <Skeleton className="aspect-[16/9] h-auto max-h-full w-full max-w-full rounded-lg object-cover" />+ </div>+ )}+ </div>+ );+}++export { WorkflowRunStreamVnc };diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css b/skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.cssnew file mode 100644index 000000000..fe0185adb--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css@@ -0,0 +1,96 @@+.workflow-run-stream-vnc {+ position: relative;+ width: 100%;+ height: 100%;+ min-height: 0;+ padding: 0.5rem;+ overflow: visible;++ transition: padding 0.2s ease-in-out;+}++.workflow-run-stream-vnc.user-is-controlling {+ padding: 0rem;+}++.workflow-run-stream-vnc .overlay-container {+ position: absolute;+ top: 0;+ left: 0;+ width: 100%;+ height: 100%;+ display: flex;+ align-items: center;+ justify-content: center;+}++.workflow-run-stream-vnc .overlay {+ position: relative;+ height: auto;+ width: 100%;+ max-height: 100%;+ aspect-ratio: 16 / 9;+ object-fit: cover;+ display: flex;+ align-items: center;+ justify-content: center;+}++.workflow-run-stream-vnc.user-is-controlling .overlay {+ pointer-events: none;+}++.workflow-run-stream-vnc.user-is-controlling .overlay-container {+ pointer-events: none;+}++.workflow-run-stream-vnc .take-control {+ transform: translateY(0);+ transition:+ transform 0.2s ease-in-out,+ opacity 0.2s ease-in-out;+ opacity: 0.3;+}++.workflow-run-stream-vnc .take-control:not(.hide):hover {+ opacity: 1;+}++.workflow-run-stream-vnc .take-control.hide {+ transform: translateY(100%);+ opacity: 0;+ pointer-events: none;+}++.workflow-run-stream-vnc .relinquish-control {+ transform: translateY(0);+ transition:+ transform 0.2s ease-in-out,+ opacity 0.2s ease-in-out;+ opacity: 0.3;+ pointer-events: all;+}++.workflow-run-stream-vnc .relinquish-control:not(.hide):hover {+ opacity: 1;+}++.workflow-run-stream-vnc .relinquish-control.hide {+ transform: translateY(100%);+ opacity: 0;+ pointer-events: none;+}++.workflow-run-stream-vnc > div > canvas {+ opacity: 0;+ animation: skyvern-anim-fadeIn 1s ease-in forwards;+}++@keyframes skyvern-anim-fadeIn {+ from {+ opacity: 0;+ }+ to {+ opacity: 1;+ }+}