Skip to content

Automate browser based workflows with AI

Browser recording action (#4130)

Jonathan Dobsonef3d88c1b9ed
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 3b4d5a2fd..1263bf801 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -1,5 +1,5 @@
import RFB from "@novnc/novnc/lib/rfb.js";
-import { ExitIcon, HandIcon } from "@radix-ui/react-icons";
+import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons";
import { useEffect, useState, useRef, useCallback } from "react";
import { useQuery } from "@tanstack/react-query";
@@ -9,12 +9,28 @@ import type {
TaskApiResponse,
WorkflowRunStatusApiResponse,
} from "@/api/types";
+import { Tip } from "@/components/Tip";
import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogClose,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+ DialogTrigger,
+} from "@/components/ui/dialog";
import { AnimatedWave } from "@/components/AnimatedWave";
import { toast } from "@/components/ui/use-toast";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { statusIsNotFinalized } from "@/routes/tasks/types";
import { useClientIdStore } from "@/store/useClientIdStore";
+import {
+ useRecordingStore,
+ type MessageInExfiltratedEvent,
+} from "@/store/useRecordingStore";
+import { useSettingsStore } from "@/store/SettingsStore";
import {
environment,
wssBaseUrl,
@@ -31,18 +47,34 @@ interface BrowserSession {
completed_at?: string;
}
-interface CommandTakeControl {
- kind: "take-control";
+interface CommandBeginExfiltration {
+ kind: "begin-exfiltration";
}
interface CommandCedeControl {
kind: "cede-control";
}
-// a "Command" is an fire-n-forget out-message - it does not require a response
-type Command = CommandTakeControl | CommandCedeControl;
+interface CommandEndExfiltration {
+ kind: "end-exfiltration";
+}
+
+interface CommandTakeControl {
+ kind: "take-control";
+}
-const messageInKinds = ["ask-for-clipboard", "copied-text"] as const;
+// a "Command" is an fire-n-forget out-message - it does not require a response
+type Command =
+ | CommandBeginExfiltration
+ | CommandCedeControl
+ | CommandEndExfiltration
+ | CommandTakeControl;
+
+const messageInKinds = [
+ "ask-for-clipboard",
+ "copied-text",
+ "exfiltrated-event",
+] as const;
type MessageInKind = (typeof messageInKinds)[number];
@@ -55,7 +87,10 @@ interface MessageInCopiedText {
text: string;
}
-type MessageIn = MessageInCopiedText | MessageInAskForClipboard;
+type MessageIn =
+ | MessageInCopiedText
+ | MessageInAskForClipboard
+ | MessageInExfiltratedEvent;
interface MessageOutAskForClipboardResponse {
kind: "ask-for-clipboard-response";
@@ -66,6 +101,7 @@ type MessageOut = MessageOutAskForClipboardResponse;
type Props = {
browserSessionId?: string;
+ exfiltrate?: boolean;
interactive?: boolean;
showControlButtons?: boolean;
task?: {
@@ -82,6 +118,7 @@ type Props = {
function BrowserStream({
browserSessionId = undefined,
+ exfiltrate = false,
interactive = true,
showControlButtons = undefined,
task = undefined,
@@ -175,6 +212,8 @@ function BrowserStream({
const rfbRef = useRef<RFB | null>(null);
const observerRef = useRef<MutationObserver | null>(null);
const clientId = useClientIdStore((state) => state.clientId);
+ const recordingStore = useRecordingStore();
+ const settingsStore = useSettingsStore();
const credentialGetter = useCredentialGetter();
const getWebSocketParams = useCallback(async () => {
@@ -198,6 +237,15 @@ function BrowserStream({
setIsReady(isVncConnected && isCanvasReady && hasBrowserSession);
}, [hasBrowserSession, isCanvasReady, isVncConnected]);
+ // update global settings store about browser usage
+ useEffect(() => {
+ settingsStore.setIsUsingABrowser(isReady);
+ settingsStore.setBrowserSessionId(
+ isReady ? browserSessionId ?? null : null,
+ );
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [isReady, browserSessionId]);
+
// effect for vnc disconnects only
useEffect(() => {
if (prevVncConnectedRef.current && !isVncConnected) {
@@ -417,8 +465,6 @@ function BrowserStream({
const sendCommand = (command: Command) => {
if (!messageSocket) {
- console.warn("Cannot send command, as command socket is closed.");
- console.warn(command);
return;
}
@@ -478,11 +524,58 @@ function BrowserStream({
}
}, [task, workflow]);
+ // effect for exfiltration
+ useEffect(() => {
+ const sendCommand = (command: Command) => {
+ if (!messageSocket) {
+ return;
+ }
+
+ messageSocket.send(JSON.stringify(command));
+ };
+
+ sendCommand({
+ kind: exfiltrate ? "begin-exfiltration" : "end-exfiltration",
+ });
+ }, [exfiltrate, messageSocket]);
+
useEffect(() => {
if (!interactive) {
setUserIsControlling(false);
}
}, [interactive]);
+
+ // effect to ensure the recordingStore is reset when the component unmounts
+ useEffect(() => {
+ return () => {
+ recordingStore.reset();
+ };
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, []);
+
+ // effect to ensure 'take-control' is sent on the rising edge of
+ // recordingStore.isRecording
+ useEffect(() => {
+ if (!recordingStore.isRecording) {
+ return;
+ }
+
+ if (!isMessageConnected) {
+ return;
+ }
+
+ const sendCommand = (command: Command) => {
+ if (!messageSocket) {
+ return;
+ }
+
+ messageSocket.send(JSON.stringify(command));
+ };
+
+ sendCommand({ kind: "take-control" });
+ setUserIsControlling(true);
+ }, [recordingStore.isRecording, isMessageConnected, messageSocket]);
+
/**
* TODO(jdo): could use zod or smth similar
*/
@@ -524,6 +617,25 @@ function BrowserStream({
}
break;
}
+ case "exfiltrated-event": {
+ if (
+ "event_name" in data &&
+ typeof data.event_name === "string" &&
+ "params" in data &&
+ typeof data.params === "object" &&
+ data.params !== null &&
+ "source" in data &&
+ typeof data.source === "string"
+ ) {
+ return {
+ kind: "exfiltrated-event",
+ event_name: data.event_name,
+ params: data.params,
+ source: data.source,
+ } as MessageInExfiltratedEvent;
+ }
+ break;
+ }
default: {
const _exhaustive: never = kind;
return _exhaustive;
@@ -604,6 +716,10 @@ function BrowserStream({
break;
}
+ case "exfiltrated-event": {
+ recordingStore.add(message);
+ break;
+ }
default: {
const _exhaustive: never = kind;
return _exhaustive;
@@ -615,72 +731,137 @@ function BrowserStream({
userIsControlling || (interactive && !showControlButtons);
return (
- <div
- className={cn(
- "browser-stream relative flex items-center justify-center",
- {
- "user-is-controlling": theUserIsControlling,
- },
- )}
- ref={setCanvasContainerRef}
- >
- {isReady && isVisible && (
- <div className="overlay z-10 flex items-center justify-center overflow-hidden">
- {showControlButtons && (
- <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">
- <Button
- onClick={() => {
- setUserIsControlling(true);
- }}
- className={cn("control-button pointer-events-auto border", {
- hide: userIsControlling,
- })}
- size="sm"
- >
- <HandIcon className="mr-2 h-4 w-4" />
- take control
- </Button>
- <Button
- onClick={() => {
- setUserIsControlling(false);
- }}
- className={cn(
- "control-button pointer-events-auto absolute bottom-0 border",
- {
- hide: !userIsControlling,
- },
- )}
- size="sm"
- >
- <ExitIcon className="mr-2 h-4 w-4" />
- stop controlling
- </Button>
+ <>
+ <div
+ className={cn(
+ "browser-stream relative flex flex-col items-center justify-center",
+ {
+ "user-is-controlling": theUserIsControlling,
+ },
+ )}
+ ref={setCanvasContainerRef}
+ >
+ {isReady && isVisible && (
+ <div className="overlay z-10 flex items-center justify-center overflow-hidden">
+ {showControlButtons && (
+ <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">
+ <Button
+ onClick={() => {
+ setUserIsControlling(true);
+ }}
+ className={cn("control-button pointer-events-auto border", {
+ hide: userIsControlling,
+ })}
+ size="sm"
+ >
+ <HandIcon className="mr-2 h-4 w-4" />
+ take control
+ </Button>
+ <Button
+ onClick={() => {
+ setUserIsControlling(false);
+ }}
+ className={cn(
+ "control-button pointer-events-auto absolute bottom-0 border",
+ {
+ hide: !userIsControlling,
+ },
+ )}
+ size="sm"
+ >
+ <ExitIcon className="mr-2 h-4 w-4" />
+ stop controlling
+ </Button>
+ </div>
+ )}
+ </div>
+ )}
+ {recordingStore.isRecording && (
+ <>
+ <div className="pointer-events-none absolute flex aspect-video w-full items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-red-500 animate-in fade-in">
+ <div className="relative h-full w-full">
+ <div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2 text-red-500">
+ <div className="truncate">Browser is recording</div>
+ <Tip content="To finish the recording, press stop on the animated recording button in the workflow.">
+ <div className="cursor-pointer">
+ <InfoCircledIcon />
+ </div>
+ </Tip>
+ <Dialog>
+ <DialogTrigger asChild>
+ <Button
+ className="ml-auto cursor-pointer"
+ size="sm"
+ variant="destructive"
+ style={{
+ marginTop: "-0.5rem",
+ }}
+ onClick={(e) => {
+ const hasEvents =
+ recordingStore.pendingEvents.length > 0 ||
+ recordingStore.compressedChunks.length > 0;
+ if (!hasEvents) {
+ e.preventDefault();
+ recordingStore.setIsRecording(false);
+ }
+ }}
+ >
+ cancel
+ </Button>
+ </DialogTrigger>
+ <DialogContent>
+ <DialogHeader>
+ <DialogTitle>Cancel recording?</DialogTitle>
+ <DialogDescription>
+ You have recorded events that will be lost if you
+ cancel. Are you sure you want to cancel the recording?
+ </DialogDescription>
+ </DialogHeader>
+ <DialogFooter>
+ <DialogClose asChild>
+ <Button variant="outline">Keep recording</Button>
+ </DialogClose>
+ <DialogClose asChild>
+ <Button
+ variant="destructive"
+ onClick={() => {
+ recordingStore.setIsRecording(false);
+ }}
+ >
+ Cancel recording
+ </Button>
+ </DialogClose>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+ </div>
+ </div>
</div>
- )}
- </div>
- )}
- {!isReady && (
- <div className="absolute left-0 top-1/2 flex aspect-video max-h-full w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
- {browserSessionId && !hasBrowserSession ? (
- <div>This live browser session is no longer streaming.</div>
- ) : (
- <>
- <RotateThrough interval={7 * 1000}>
- <span>Hm, working on the connection...</span>
- <span>Hang tight, we're almost there...</span>
- <span>Just a moment...</span>
- <span>Backpropagating...</span>
- <span>Attention is all I need...</span>
- <span>Consulting the manual...</span>
- <span>Looking for the bat phone...</span>
- <span>Where's Shu?...</span>
- </RotateThrough>
- <AnimatedWave text=".‧₊˚ ⋅ ? ✨ ?★ ‧₊˚ ⋅" />
- </>
- )}
- </div>
- )}
- </div>
+ </>
+ )}
+ {!isReady && (
+ <div className="absolute left-0 top-1/2 flex aspect-video max-h-full w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
+ {browserSessionId && !hasBrowserSession ? (
+ <div>This live browser session is no longer streaming.</div>
+ ) : (
+ <>
+ <RotateThrough interval={7 * 1000}>
+ <span>Hm, working on the connection...</span>
+ <span>Hang tight, we're almost there...</span>
+ <span>Just a moment...</span>
+ <span>Backpropagating...</span>
+ <span>Attention is all I need...</span>
+ <span>Consulting the manual...</span>
+ <span>Looking for the bat phone...</span>
+ <span>Where's Shu?...</span>
+ </RotateThrough>
+ <AnimatedWave text=".‧₊˚ ⋅ ? ✨ ?★ ‧₊˚ ⋅" />
+ </>
+ )}
+ </div>
+ )}
+ </div>
+ </>
);
}
diff --git a/skyvern-frontend/src/components/RadialMenu.tsx b/skyvern-frontend/src/components/RadialMenu.tsx
index 77e8b49b6..fe64fc59a 100644
--- a/skyvern-frontend/src/components/RadialMenu.tsx
+++ b/skyvern-frontend/src/components/RadialMenu.tsx
@@ -1,4 +1,4 @@
-import { useState, useRef, useEffect, ReactNode } from "react";
+import { useState, useRef, useEffect, ReactNode, Fragment } from "react";
export interface RadialMenuItem {
id: string;
@@ -175,9 +175,8 @@ export function RadialMenu({
: "scaleY(1)";
return (
- <>
+ <Fragment key={item.id}>
<button
- key={item.id}
onClick={() => {
item.onClick();
setIsOpen(false);
@@ -225,7 +224,7 @@ export function RadialMenu({
</span>
</div>
)}
- </>
+ </Fragment>
);
})}
</div>
diff --git a/skyvern-frontend/src/components/browser-stream.css b/skyvern-frontend/src/components/browser-stream.css
index 859999829..4c463277c 100644
--- a/skyvern-frontend/src/components/browser-stream.css
+++ b/skyvern-frontend/src/components/browser-stream.css
@@ -70,6 +70,7 @@
.browser-stream > div {
background: transparent !important;
+ aspect-ratio: 16/9 !important;
}
.browser-stream .control-button {
diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
new file mode 100644
index 000000000..79142d5a6
--- /dev/null
+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
@@ -0,0 +1,140 @@
+import { useMutation } from "@tanstack/react-query";
+
+// import { getClient } from "@/api/AxiosClient";
+import { toast } from "@/components/ui/use-toast";
+// import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+// import { type MessageInExfiltratedEvent } from "@/store/useRecordingStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
+import {
+ type ActionBlock,
+ type WorkflowBlock,
+} from "@/routes/workflows/types/workflowTypes";
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const FAIL_QUITE_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;
+
+const useProcessRecordingMutation = ({
+ browserSessionId,
+ onSuccess,
+}: {
+ browserSessionId: string | null;
+ onSuccess?: (workflowBlocks: Array<WorkflowBlock>) => void;
+}) => {
+ // const credentialGetter = useCredentialGetter();
+ const recordingStore = useRecordingStore();
+
+ const processRecordingMutation = useMutation({
+ mutationFn: async () => {
+ if (!browserSessionId) {
+ throw new Error(
+ "Cannot process recording without a valid browser session ID.",
+ );
+ }
+
+ const eventCount = recordingStore.getEventCount();
+
+ if (eventCount === 0) {
+ throw new Error(FAIL_QUITE_NO_EVENTS);
+ }
+
+ // (this flushes any pending events)
+ const compressedChunks = await recordingStore.getCompressedChunks();
+
+ // TODO: Replace this mock with actual API call when endpoint is ready
+ // const client = await getClient(credentialGetter, "sans-api-v1");
+ // return client
+ // .post<
+ // { compressed_chunks: string[] },
+ // { data: Array<WorkflowBlock> }
+ // >(`/browser_sessions/${browserSessionId}/process_recording`, {
+ // compressed_chunks: compressedChunks,
+ // })
+ // .then((response) => response.data);
+
+ // Mock response with 2-second delay
+ console.log(
+ `Processing ${eventCount} events in ${compressedChunks.length} compressed chunks`,
+ );
+ await sleep(2000);
+
+ // Return mock workflow blocks with two ActionBlocks
+ const mockWorkflowBlocks: Array<WorkflowBlock> = [
+ {
+ block_type: "action",
+ label: "action_1",
+ title: "Enter search term",
+ navigation_goal: "Enter 'foo' in the search field",
+ url: null,
+ error_code_mapping: null,
+ parameters: [],
+ engine: null,
+ continue_on_failure: false,
+ output_parameter: {
+ parameter_type: "output",
+ key: "action_1_output",
+ description: null,
+ output_parameter_id: "mock-output-1",
+ workflow_id: browserSessionId || "mock-workflow-id",
+ created_at: new Date().toISOString(),
+ modified_at: new Date().toISOString(),
+ deleted_at: null,
+ },
+ model: null,
+ } satisfies ActionBlock,
+ {
+ block_type: "action",
+ label: "action_2",
+ title: "Click search",
+ navigation_goal: "Click the search button",
+ url: null,
+ error_code_mapping: null,
+ parameters: [],
+ engine: null,
+ continue_on_failure: false,
+ output_parameter: {
+ parameter_type: "output",
+ key: "action_2_output",
+ description: null,
+ output_parameter_id: "mock-output-2",
+ workflow_id: browserSessionId || "mock-workflow-id",
+ created_at: new Date().toISOString(),
+ modified_at: new Date().toISOString(),
+ deleted_at: null,
+ },
+ model: null,
+ } satisfies ActionBlock,
+ ];
+ return mockWorkflowBlocks;
+ },
+ onSuccess: (workflowBlocks) => {
+ // Clear events after successful flush
+ recordingStore.clear();
+
+ toast({
+ variant: "success",
+ title: "Recording Processed",
+ description: "The recording has been successfully processed.",
+ });
+
+ if (workflowBlocks) {
+ onSuccess?.(workflowBlocks);
+ }
+ },
+ onError: (error) => {
+ if (error instanceof Error && error.message === FAIL_QUITE_NO_EVENTS) {
+ return;
+ }
+
+ toast({
+ variant: "destructive",
+ title: "Error Processing Recording",
+ description: error.message,
+ });
+ },
+ });
+
+ return processRecordingMutation;
+};
+
+export { useProcessRecordingMutation };
diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
index 1f172feb1..385ee523b 100644
--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
@@ -11,6 +11,7 @@ import { useOnChange } from "@/hooks/useOnChange";
import { useShouldNotifyWhenClosingTab } from "@/hooks/useShouldNotifyWhenClosingTab";
import { BlockActionContext } from "@/store/BlockActionContext";
import { useDebugStore } from "@/store/useDebugStore";
+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
import {
useWorkflowHasChangesStore,
useWorkflowSave,
@@ -33,6 +34,7 @@ import {
EdgeChange,
} from "@xyflow/react";
import "@xyflow/react/dist/style.css";
+import { nanoid } from "nanoid";
import { useCallback, useEffect, useRef, useState } from "react";
import { useBlocker } from "react-router-dom";
import {
@@ -76,8 +78,10 @@ import {
import "./reactFlowOverrideStyles.css";
import {
convertEchoParameters,
+ convertToNode,
createNode,
descendants,
+ generateNodeLabel,
getAdditionalParametersForEmailBlock,
getOrderedChildrenBlocks,
getOutputParameterKey,
@@ -319,6 +323,13 @@ function FlowRenderer({
const setGetSaveDataRef = useRef(workflowChangesStore.setGetSaveData);
setGetSaveDataRef.current = workflowChangesStore.setGetSaveData;
const saveWorkflow = useWorkflowSave({ status: "published" });
+ const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);
+ const recordedInsertionPoint = useRecordedBlocksStore(
+ (state) => state.insertionPoint,
+ );
+ const clearRecordedBlocks = useRecordedBlocksStore(
+ (state) => state.clearRecordedBlocks,
+ );
useShouldNotifyWhenClosingTab(workflowChangesStore.hasChanges);
const blocker = useBlocker(({ currentLocation, nextLocation }) => {
return (
@@ -572,6 +583,85 @@ function FlowRenderer({
doLayout(nodes, edges);
}
+ // effect to add new blocks that were generated from a browser recording
+ useEffect(() => {
+ if (!recordedBlocks || !recordedInsertionPoint) {
+ return;
+ }
+
+ const { previous, next, parent, connectingEdgeType } =
+ recordedInsertionPoint;
+
+ const newNodes: Array<AppNode> = [];
+ const newEdges: Array<Edge> = [];
+
+ let existingLabels = nodes
+ .filter(isWorkflowBlockNode)
+ .map((node) => node.data.label);
+
+ let prevNodeId = previous;
+
+ // convert each WorkflowBlock to an AppNode
+ recordedBlocks.forEach((block, index) => {
+ const id = nanoid();
+ const label = generateNodeLabel(existingLabels);
+ existingLabels = [...existingLabels, label];
+ const blockWithLabel = { ...block, label: block.label || label };
+
+ const node = convertToNode(
+ { id, parentId: parent },
+ blockWithLabel,
+ true,
+ );
+ newNodes.push(node);
+
+ // create edge from previous node to this one
+ if (prevNodeId) {
+ newEdges.push({
+ id: nanoid(),
+ type: "edgeWithAddButton",
+ source: prevNodeId,
+ target: id,
+ style: { strokeWidth: 2 },
+ });
+ }
+
+ // if this is the last block, connect to next
+ if (index === recordedBlocks.length - 1 && next) {
+ newEdges.push({
+ id: nanoid(),
+ type: connectingEdgeType,
+ source: id,
+ target: next,
+ style: { strokeWidth: 2 },
+ });
+ }
+
+ prevNodeId = id;
+ });
+
+ const editedEdges = previous
+ ? edges.filter((edge) => edge.source !== previous)
+ : edges;
+
+ const previousNode = nodes.find((node) => node.id === previous);
+ const previousNodeIndex = previousNode
+ ? nodes.indexOf(previousNode)
+ : nodes.length - 1;
+
+ const newNodesAfter = [
+ ...nodes.slice(0, previousNodeIndex + 1),
+ ...newNodes,
+ ...nodes.slice(previousNodeIndex + 1),
+ ];
+
+ workflowChangesStore.setHasChanges(true);
+ doLayout(newNodesAfter, [...editedEdges, ...newEdges]);
+
+ clearRecordedBlocks();
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [recordedBlocks, recordedInsertionPoint]);
+
const editorElementRef = useRef<HTMLDivElement>(null);
useAutoPan(editorElementRef, nodes);
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsx
new file mode 100644
index 000000000..60c838dc0
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsx
@@ -0,0 +1,82 @@
+import { SquareIcon, PlusIcon } from "@radix-ui/react-icons";
+import { ReactNode } from "react";
+
+import { RadialMenu } from "@/components/RadialMenu";
+import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser";
+import { useDebugStore } from "@/store/useDebugStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
+import { useSettingsStore } from "@/store/SettingsStore";
+
+type WorkflowAddMenuProps = {
+ buttonSize?: string;
+ children: ReactNode;
+ gap?: number;
+ radius?: string;
+ rotateText?: boolean;
+ startAt?: number;
+ // --
+ onAdd: () => void;
+ onRecord: () => void;
+};
+
+function WorkflowAddMenu({
+ buttonSize,
+ children,
+ gap,
+ radius = "80px",
+ rotateText = true,
+ startAt = 90,
+ // --
+ onAdd,
+ onRecord,
+}: WorkflowAddMenuProps) {
+ const debugStore = useDebugStore();
+ const recordingStore = useRecordingStore();
+ const settingsStore = useSettingsStore();
+ const isSkyvernUser = useIsSkyvernUser();
+
+ if (
+ !isSkyvernUser ||
+ !debugStore.isDebugMode ||
+ !settingsStore.isUsingABrowser
+ ) {
+ return <>{children}</>;
+ }
+
+ return (
+ <RadialMenu
+ items={[
+ {
+ id: "1",
+ icon: <PlusIcon className={buttonSize ? "h-3 w-3" : undefined} />,
+ text: "Add Block",
+ onClick: () => {
+ onAdd();
+ },
+ },
+ {
+ id: "2",
+ icon: <SquareIcon className={buttonSize ? "h-3 w-3" : undefined} />,
+ enabled: !recordingStore.isRecording && settingsStore.isUsingABrowser,
+ text: "Record Browser",
+ onClick: () => {
+ if (!settingsStore.isUsingABrowser) {
+ return;
+ }
+
+ onRecord();
+ },
+ },
+ ]}
+ buttonSize={buttonSize}
+ radius={radius}
+ startAt={startAt}
+ gap={gap}
+ rotateText={rotateText}
+ >
+ {children}
+ </RadialMenu>
+ );
+}
+
+export { WorkflowAddMenu };
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.css b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.css
new file mode 100644
index 000000000..f83cc0259
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.css
@@ -0,0 +1,29 @@
+@keyframes pulse-dash {
+ 0% {
+ stroke-dasharray: 141.4 141.4;
+ stroke-width: 6;
+ }
+ 50% {
+ stroke-dasharray: 10 11.4;
+ stroke-width: 8;
+ }
+ 100% {
+ stroke-dasharray: 141.4 141.4;
+ stroke-width: 6;
+ }
+}
+
+@keyframes pulse-dash-small {
+ 0% {
+ stroke-dasharray: 100.4 100.4;
+ stroke-width: 3;
+ }
+ 50% {
+ stroke-dasharray: 10 11.4;
+ stroke-width: 5;
+ }
+ 100% {
+ stroke-dasharray: 100.4 100.4;
+ stroke-width: 3;
+ }
+}
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx
new file mode 100644
index 000000000..54f341e69
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx
@@ -0,0 +1,196 @@
+import { ReactNode, useEffect, useRef, useState } from "react";
+
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipProvider,
+ TooltipTrigger,
+} from "@/components/ui/tooltip";
+import { useRecordingStore, CHUNK_SIZE } from "@/store/useRecordingStore";
+import { cn } from "@/util/utils";
+
+import "./WorkflowAdderBusy.css";
+
+type Operation = "recording" | "processing";
+
+type Size = "small" | "large";
+
+type Props = {
+ children: ReactNode;
+ /**
+ * The operation being performed (e.g., recording or processing).
+ */
+ operation: Operation;
+ /**
+ * An explicit sizing; otherwise the size will be determined by the child content.
+ */
+ size?: Size;
+ /**
+ * Color for the cover and ellipses. Defaults to "red".
+ */
+ color?: string;
+ // --
+ onComplete: () => void;
+};
+
+function WorkflowAdderBusy({
+ children,
+ operation,
+ size,
+ color = "red",
+ onComplete,
+}: Props) {
+ const recordingStore = useRecordingStore();
+ const [isHovered, setIsHovered] = useState(false);
+ const [shouldBump, setShouldBump] = useState(false);
+ const bumpTimeoutRef = useRef<NodeJS.Timeout | null>(null);
+ const prevCountRef = useRef(0);
+
+ const eventCount =
+ recordingStore.pendingEvents.length +
+ recordingStore.compressedChunks.length * CHUNK_SIZE;
+
+ // effect for bump animation when count changes
+ useEffect(() => {
+ if (eventCount > prevCountRef.current && prevCountRef.current > 0) {
+ if (bumpTimeoutRef.current) {
+ clearTimeout(bumpTimeoutRef.current);
+ }
+
+ setShouldBump(true);
+
+ bumpTimeoutRef.current = setTimeout(() => {
+ setShouldBump(false);
+ }, 300);
+ }
+
+ prevCountRef.current = eventCount;
+
+ return () => {
+ if (bumpTimeoutRef.current) {
+ clearTimeout(bumpTimeoutRef.current);
+ }
+ };
+ }, [eventCount]);
+
+ const handleClick = (e: React.MouseEvent) => {
+ e.stopPropagation();
+ e.preventDefault();
+ onComplete();
+
+ return false;
+ };
+
+ return (
+ <TooltipProvider>
+ <div className="relative inline-block">
+ <Tooltip open={isHovered}>
+ <TooltipTrigger asChild>
+ <div
+ className={cn("relative inline-block", {
+ "flex items-center justify-center": size !== undefined,
+ "min-h-[40px] min-w-[40px]": size === "small",
+ "min-h-[80px] min-w-[80px]": size === "large",
+ })}
+ onMouseEnter={() => setIsHovered(true)}
+ onMouseLeave={() => setIsHovered(false)}
+ >
+ {/* cover */}
+ <div
+ className={cn("absolute inset-0 rounded-full opacity-40", {
+ "opacity-30": isHovered,
+ })}
+ style={{ backgroundColor: color }}
+ onClick={handleClick}
+ />
+ <div className="pointer-events-none flex items-center justify-center">
+ {children}
+ </div>
+ <div className="pointer-events-none absolute inset-0">
+ <svg
+ className="h-full w-full animate-spin"
+ viewBox="0 0 100 100"
+ preserveAspectRatio="none"
+ style={{ transformOrigin: "center" }}
+ >
+ <ellipse
+ cx="50"
+ cy="50"
+ rx="45"
+ ry="45"
+ fill="none"
+ stroke={color}
+ strokeWidth={size === "small" ? "3" : "6"}
+ strokeDasharray="141.4 141.4"
+ strokeLinecap="round"
+ vectorEffect="non-scaling-stroke"
+ style={{
+ animation: `${size === "small" ? "pulse-dash-small" : "pulse-dash"} 10s ease-in-out infinite`,
+ }}
+ />
+ </svg>
+ </div>
+ {isHovered && (
+ <div className="pointer-events-none absolute inset-0">
+ <svg
+ className="h-full w-full"
+ viewBox="0 0 100 100"
+ preserveAspectRatio="none"
+ >
+ <rect
+ x="30"
+ y="30"
+ width="40"
+ height="40"
+ fill={color}
+ vectorEffect="non-scaling-stroke"
+ className="animate-in zoom-in-0"
+ style={{
+ transformOrigin: "center",
+ transformBox: "fill-box",
+ animationDuration: "200ms",
+ animationTimingFunction:
+ "cubic-bezier(0.34, 1.56, 0.64, 1)",
+ }}
+ />
+ </svg>
+ </div>
+ )}
+ </div>
+ </TooltipTrigger>
+ <TooltipContent>
+ <p>
+ {operation === "recording" ? "Finish Recording" : "Processing..."}
+ </p>
+ </TooltipContent>
+ </Tooltip>
+ {recordingStore.isRecording && eventCount > 0 && (
+ <Tooltip delayDuration={0}>
+ <TooltipTrigger asChild>
+ <div
+ className={cn(
+ "absolute -right-2 -top-2 flex h-6 min-w-6 items-center justify-center rounded-full px-1.5 text-xs font-semibold text-white shadow-lg transition-transform",
+ {
+ "scale-125": shouldBump,
+ "scale-100": !shouldBump,
+ },
+ )}
+ style={{
+ backgroundColor: color,
+ transition: "transform 0.6s",
+ }}
+ >
+ {eventCount}
+ </div>
+ </TooltipTrigger>
+ <TooltipContent>
+ <p>Event Count</p>
+ </TooltipContent>
+ </Tooltip>
+ )}
+ </div>
+ </TooltipProvider>
+ );
+}
+
+export { WorkflowAdderBusy };
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index f249c08d0..86d0076d4 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -29,6 +29,7 @@ import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQu
import { WorkflowRunStream } from "@/routes/workflows/workflowRun/WorkflowRunStream";
import { useCacheKeyValuesQuery } from "../hooks/useCacheKeyValuesQuery";
import { useBlockScriptStore } from "@/store/BlockScriptStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
import { useSidebarStore } from "@/store/SidebarStore";
import { AnimatedWave } from "@/components/AnimatedWave";
@@ -238,6 +239,7 @@ function Workspace({
const queryClient = useQueryClient();
const [shouldFetchDebugSession, setShouldFetchDebugSession] = useState(false);
const blockScriptStore = useBlockScriptStore();
+ const recordingStore = useRecordingStore();
const cacheKey = workflow?.cache_key ?? "";
const [cacheKeyValue, setCacheKeyValue] = useState(
@@ -1339,6 +1341,7 @@ function Workspace({
<div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
<div key={reloadKey} className="w-full flex-1">
<BrowserStream
+ exfiltrate={recordingStore.isRecording}
interactive={true}
browserSessionId={
activeDebugSession?.browser_session_id
diff --git a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
index 4dfd57ff1..b9b7bbe71 100644
--- a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
@@ -1,4 +1,4 @@
-import { SquareIcon, PlusIcon } from "@radix-ui/react-icons";
+import { PlusIcon } from "@radix-ui/react-icons";
import {
BaseEdge,
EdgeLabelRenderer,
@@ -8,12 +8,16 @@ import {
} from "@xyflow/react";
import { Button } from "@/components/ui/button";
-import { RadialMenu } from "@/components/RadialMenu";
-import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser";
-import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
+import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";
import { useDebugStore } from "@/store/useDebugStore";
+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
+import { useSettingsStore } from "@/store/SettingsStore";
+import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
import { REACT_FLOW_EDGE_Z_INDEX } from "../constants";
+import { WorkflowAddMenu } from "../WorkflowAddMenu";
+import { WorkflowAdderBusy } from "../WorkflowAdderBusy";
function EdgeWithAddButton({
source,
@@ -27,8 +31,6 @@ function EdgeWithAddButton({
style = {},
markerEnd,
}: EdgeProps) {
- const debugStore = useDebugStore();
- const isSkyvernUser = useIsSkyvernUser();
const nodes = useNodes();
const [edgePath, labelX, labelY] = getBezierPath({
sourceX,
@@ -38,14 +40,35 @@ function EdgeWithAddButton({
targetY,
targetPosition,
});
+ const debugStore = useDebugStore();
+ const recordingStore = useRecordingStore();
+ const settingsStore = useSettingsStore();
+ const workflowStatePanel = useWorkflowPanelStore();
+ const setRecordedBlocks = useRecordedBlocksStore(
+ (state) => state.setRecordedBlocks,
+ );
const setWorkflowPanelState = useWorkflowPanelStore(
(state) => state.setWorkflowPanelState,
);
+ const processRecordingMutation = useProcessRecordingMutation({
+ browserSessionId: settingsStore.browserSessionId,
+ onSuccess: (blocks) => {
+ setRecordedBlocks(blocks, {
+ previous: source,
+ next: target,
+ parent: sourceNode?.parentId,
+ connectingEdgeType: "edgeWithAddButton",
+ });
+ },
+ });
+
+ const isProcessing = processRecordingMutation.isPending;
+
const sourceNode = nodes.find((node) => node.id === source);
- const onAdd = () => {
+ const updateWorkflowPanelState = (active: boolean) => {
setWorkflowPanelState({
- active: true,
+ active,
content: "nodeLibrary",
data: {
previous: source,
@@ -55,6 +78,25 @@ function EdgeWithAddButton({
});
};
+ const onAdd = () => updateWorkflowPanelState(true);
+
+ const onRecord = () => {
+ if (recordingStore.isRecording) {
+ recordingStore.setIsRecording(false);
+ } else {
+ recordingStore.setIsRecording(true);
+ updateWorkflowPanelState(false);
+ }
+ };
+
+ const onEndRecord = () => {
+ if (recordingStore.isRecording) {
+ recordingStore.setIsRecording(false);
+ }
+
+ processRecordingMutation.mutate();
+ };
+
const adder = (
<Button
size="icon"
@@ -65,6 +107,41 @@ function EdgeWithAddButton({
</Button>
);
+ const menu = (
+ <WorkflowAddMenu
+ buttonSize="25px"
+ gap={35}
+ radius="50px"
+ startAt={72.5}
+ onAdd={onAdd}
+ onRecord={onRecord}
+ >
+ {adder}
+ </WorkflowAddMenu>
+ );
+
+ const busy = (
+ <WorkflowAdderBusy
+ color={isProcessing ? "white" : "red"}
+ operation={isProcessing ? "processing" : "recording"}
+ size="small"
+ onComplete={() => {
+ onEndRecord();
+ }}
+ >
+ {adder}
+ </WorkflowAdderBusy>
+ );
+
+ const isBusy =
+ (isProcessing || recordingStore.isRecording) &&
+ debugStore.isDebugMode &&
+ settingsStore.isUsingABrowser &&
+ workflowStatePanel.workflowPanelState.data?.previous === source &&
+ workflowStatePanel.workflowPanelState.data?.next === target &&
+ workflowStatePanel.workflowPanelState.data?.parent ===
+ (sourceNode?.parentId || undefined);
+
return (
<>
<BaseEdge path={edgePath} markerEnd={markerEnd} style={style} />
@@ -81,38 +158,7 @@ function EdgeWithAddButton({
}}
className="nodrag nopan"
>
- {isSkyvernUser && debugStore.isDebugMode ? (
- <RadialMenu
- items={[
- {
- id: "1",
- icon: <PlusIcon className="h-3 w-3" />,
- text: "Add Block",
- onClick: () => {
- onAdd();
- },
- },
- {
- id: "2",
- icon: <SquareIcon className="h-3 w-3" />,
- enabled: false,
- text: "Record Browser",
- onClick: () => {
- console.log("Record");
- },
- },
- ]}
- buttonSize="25px"
- radius="50px"
- startAt={72.5}
- gap={35}
- rotateText={true}
- >
- {adder}
- </RadialMenu>
- ) : (
- adder
- )}
+ {isBusy ? busy : menu}
</div>
</EdgeLabelRenderer>
</>
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
index a2d0e3cb8..c8d46b30a 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
@@ -1,25 +1,51 @@
-import { SquareIcon, PlusIcon } from "@radix-ui/react-icons";
+import { PlusIcon } from "@radix-ui/react-icons";
import { Handle, NodeProps, Position, useEdges } from "@xyflow/react";
-import { RadialMenu } from "@/components/RadialMenu";
-import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser";
+import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";
import { useDebugStore } from "@/store/useDebugStore";
+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
+import { useSettingsStore } from "@/store/SettingsStore";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
import type { NodeAdderNode } from "./types";
+import { WorkflowAddMenu } from "../../WorkflowAddMenu";
+import { WorkflowAdderBusy } from "../../WorkflowAdderBusy";
function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
- const debugStore = useDebugStore();
- const isSkyvernUser = useIsSkyvernUser();
const edges = useEdges();
+ const debugStore = useDebugStore();
+ const recordingStore = useRecordingStore();
+ const settingsStore = useSettingsStore();
const setWorkflowPanelState = useWorkflowPanelStore(
(state) => state.setWorkflowPanelState,
);
+ const workflowStatePanel = useWorkflowPanelStore();
+ const setRecordedBlocks = useRecordedBlocksStore(
+ (state) => state.setRecordedBlocks,
+ );
- const onAdd = () => {
+ const previous = edges.find((edge) => edge.target === id)?.source ?? null;
+
+ const processRecordingMutation = useProcessRecordingMutation({
+ browserSessionId: settingsStore.browserSessionId,
+ onSuccess: (blocks) => {
+ setRecordedBlocks(blocks, {
+ previous,
+ next: id,
+ parent: parentId,
+ connectingEdgeType: "default",
+ });
+ },
+ });
+
+ const isProcessing = processRecordingMutation.isPending;
+
+ const updateWorkflowPanelState = (active: boolean) => {
const previous = edges.find((edge) => edge.target === id)?.source;
+
setWorkflowPanelState({
- active: true,
+ active,
content: "nodeLibrary",
data: {
previous: previous ?? null,
@@ -30,9 +56,30 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
});
};
+ const onAdd = () => {
+ updateWorkflowPanelState(true);
+ };
+
+ const onRecord = () => {
+ if (recordingStore.isRecording) {
+ recordingStore.setIsRecording(false);
+ } else {
+ recordingStore.setIsRecording(true);
+ updateWorkflowPanelState(false);
+ }
+ };
+
+ const onEndRecord = () => {
+ if (recordingStore.isRecording) {
+ recordingStore.setIsRecording(false);
+ }
+
+ processRecordingMutation.mutate();
+ };
+
const adder = (
<div
- className="rounded-full bg-slate-50 p-2"
+ className={"rounded-full bg-slate-50 p-2"}
onClick={() => {
onAdd();
}}
@@ -41,6 +88,33 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
</div>
);
+ const busy = (
+ <WorkflowAdderBusy
+ color={isProcessing ? "gray" : "red"}
+ operation={isProcessing ? "processing" : "recording"}
+ onComplete={() => {
+ onEndRecord();
+ }}
+ >
+ {adder}
+ </WorkflowAdderBusy>
+ );
+
+ const menu = (
+ <WorkflowAddMenu onAdd={onAdd} onRecord={onRecord}>
+ {adder}
+ </WorkflowAddMenu>
+ );
+
+ const isBusy =
+ (isProcessing || recordingStore.isRecording) &&
+ debugStore.isDebugMode &&
+ settingsStore.isUsingABrowser &&
+ workflowStatePanel.workflowPanelState.data?.previous === previous &&
+ workflowStatePanel.workflowPanelState.data?.next === id &&
+ workflowStatePanel.workflowPanelState.data?.parent ===
+ (parentId || undefined);
+
return (
<div>
<Handle
@@ -55,36 +129,7 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
id="b"
className="opacity-0"
/>
- {isSkyvernUser && debugStore.isDebugMode ? (
- <RadialMenu
- items={[
- {
- id: "1",
- icon: <PlusIcon />,
- text: "Add Block",
- onClick: () => {
- onAdd();
- },
- },
- {
- id: "2",
- icon: <SquareIcon />,
- enabled: false,
- text: "Record Browser",
- onClick: () => {
- console.log("Record");
- },
- },
- ]}
- radius="80px"
- startAt={90}
- rotateText={true}
- >
- {adder}
- </RadialMenu>
- ) : (
- adder
- )}
+ {isBusy ? busy : menu}
</div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
index 1bb9912e9..833cbb966 100644
--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
@@ -2458,6 +2458,7 @@ function getLabelForWorkflowParameterType(type: WorkflowParameterValueType) {
export {
convert,
convertEchoParameters,
+ convertToNode,
createNode,
generateNodeData,
generateNodeLabel,
diff --git a/skyvern-frontend/src/store/RecordedBlocksStore.ts b/skyvern-frontend/src/store/RecordedBlocksStore.ts
new file mode 100644
index 000000000..4daa68915
--- /dev/null
+++ b/skyvern-frontend/src/store/RecordedBlocksStore.ts
@@ -0,0 +1,36 @@
+import { create } from "zustand";
+import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes";
+
+type InsertionPoint = {
+ previous: string | null;
+ next: string | null;
+ parent?: string;
+ connectingEdgeType: string;
+};
+
+type RecordedBlocksState = {
+ blocks: Array<WorkflowBlock> | null;
+ insertionPoint: InsertionPoint | null;
+};
+
+type RecordedBlocksStore = RecordedBlocksState & {
+ setRecordedBlocks: (
+ blocks: Array<WorkflowBlock>,
+ insertionPoint: InsertionPoint,
+ ) => void;
+ clearRecordedBlocks: () => void;
+};
+
+const useRecordedBlocksStore = create<RecordedBlocksStore>((set) => ({
+ blocks: null,
+ insertionPoint: null,
+ setRecordedBlocks: (blocks, insertionPoint) => {
+ set({ blocks, insertionPoint });
+ },
+ clearRecordedBlocks: () => {
+ set({ blocks: null, insertionPoint: null });
+ },
+}));
+
+export { useRecordedBlocksStore };
+export type { InsertionPoint };
diff --git a/skyvern-frontend/src/store/SettingsStore.ts b/skyvern-frontend/src/store/SettingsStore.ts
index b8050b59c..57ed62cd0 100644
--- a/skyvern-frontend/src/store/SettingsStore.ts
+++ b/skyvern-frontend/src/store/SettingsStore.ts
@@ -2,16 +2,33 @@ import { create } from "zustand";
type SettingsStore = {
environment: string;
+ /**
+ * The user is currently operating or viewing a live, remote browser. NOTE: if
+ * the browser is still connecting, or otherwise not ready, then this should
+ * be false.
+ */
+ isUsingABrowser: boolean;
+ /**
+ * The current browser session ID when a browser is active.
+ */
+ browserSessionId: string | null;
organization: string;
setEnvironment: (environment: string) => void;
+ setIsUsingABrowser: (isUsing: boolean) => void;
+ setBrowserSessionId: (browserSessionId: string | null) => void;
setOrganization: (organization: string) => void;
};
const useSettingsStore = create<SettingsStore>((set) => {
return {
environment: "local",
+ isUsingABrowser: false,
+ browserSessionId: null,
organization: "skyvern",
setEnvironment: (environment: string) => set({ environment }),
+ setIsUsingABrowser: (isUsing: boolean) => set({ isUsingABrowser: isUsing }),
+ setBrowserSessionId: (browserSessionId: string | null) =>
+ set({ browserSessionId }),
setOrganization: (organization: string) => set({ organization }),
};
});
diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.ts
new file mode 100644
index 000000000..5369d0068
--- /dev/null
+++ b/skyvern-frontend/src/store/useRecordingStore.ts
@@ -0,0 +1,298 @@
+import { create } from "zustand";
+
+/**
+ * example: {
+ * 'targetInfo': {
+ * 'targetId': '8B698E27F1F32372718DA73DCA0C5944',
+ * 'type': 'page',
+ * 'title': 'New Tab',
+ * 'url': 'chrome://newtab/',
+ * 'attached': True,
+ * 'canAccessOpener': False,
+ * 'browserContextId': 'FD13D5C556E681BB49AEED0AB2CA1972',
+ * }
+ */
+export interface ExfiltratedEventCdpParams {
+ targetInfo: {
+ attached?: boolean;
+ browserContextId?: string;
+ canAccessOpener?: boolean;
+ targetId?: string;
+ title?: string;
+ type?: string;
+ url?: string;
+ };
+}
+
+export interface ExfiltratedEventConsoleParams {
+ type: string;
+ url: string;
+ timestamp: number;
+ target: {
+ className?: string;
+ id?: string;
+ innerText?: string;
+ tagName?: string;
+ text: string[];
+ value?: string;
+ };
+ inputValue?: string;
+ mousePosition: {
+ xa: number | null;
+ ya: number | null;
+ xp: number | null;
+ yp: number | null;
+ };
+ key?: string;
+ code?: string;
+ activeElement: {
+ tagName?: string;
+ id?: string;
+ className?: string;
+ boundingRect?: {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+ top: number;
+ right: number;
+ bottom: number;
+ left: number;
+ } | null;
+ scroll?: {
+ scrollTop: number;
+ scrollLeft: number;
+ scrollHeight: number;
+ scrollWidth: number;
+ clientHeight: number;
+ clientWidth: number;
+ } | null;
+ };
+ window: {
+ width: number;
+ height: number;
+ scrollX: number;
+ scrollY: number;
+ };
+}
+
+export interface MessageInExfiltratedCdpEvent {
+ kind: "exfiltrated-event";
+ event_name: string;
+ params: ExfiltratedEventCdpParams;
+ source: "cdp";
+}
+
+export interface MessageInExfiltratedConsoleEvent {
+ kind: "exfiltrated-event";
+ event_name: string;
+ params: ExfiltratedEventConsoleParams;
+ source: "console";
+}
+
+export type MessageInExfiltratedEvent =
+ | MessageInExfiltratedCdpEvent
+ | MessageInExfiltratedConsoleEvent;
+
+/**
+ * Number of events per compressed chunk.
+ */
+export const CHUNK_SIZE = 1000 as const;
+
+interface RecordingStore {
+ /**
+ * Compressed chunks of recorded events (base64 gzip).
+ * Each chunk contains up to CHUNK_SIZE events.
+ */
+ compressedChunks: string[];
+ /**
+ * Buffer of events not yet compressed into a chunk.
+ */
+ pendingEvents: MessageInExfiltratedEvent[];
+ /**
+ * Whether a compression operation is currently in progress.
+ */
+ isCompressing: boolean;
+ /**
+ * Whether the user is currently in browser recording mode.
+ */
+ isRecording: boolean;
+ /**
+ * Add a new recorded event. Triggers async compression when buffer is full.
+ */
+ add: (event: MessageInExfiltratedEvent) => void;
+ /**
+ * Clear all recorded events and compressed chunks.
+ */
+ clear: () => void;
+ /**
+ * Reset the recording store (clear events and set isRecording to false).
+ */
+ reset: () => void;
+ /**
+ * Set whether the user is in browser recording mode.
+ */
+ setIsRecording: (isRecording: boolean) => void;
+ /**
+ * Flush any pending events into a compressed chunk.
+ * Call this before consuming the data.
+ */
+ flush: () => Promise<void>;
+ /**
+ * Get all compressed chunks (after flushing pending events).
+ */
+ getCompressedChunks: () => Promise<string[]>;
+ /**
+ * Get the total number of events (compressed + pending).
+ */
+ getEventCount: () => number;
+}
+
+/**
+ * compresses a JSON string using the Gzip algorithm and returns the result
+ * as a Base64 encoded string
+ */
+async function compressEventsToB64(jsonString: string): Promise<string> {
+ // 1. Convert the string to a Uint8Array (a byte array).
+ const encoder = new TextEncoder();
+ const uint8Array = encoder.encode(jsonString);
+
+ // 2. Create a ReadableStream from the byte array.
+ const readableStream = new ReadableStream({
+ start(controller) {
+ controller.enqueue(uint8Array);
+ controller.close();
+ },
+ });
+
+ // 3. Pipe the data through the Gzip compression stream.
+ const compressedStream = readableStream.pipeThrough(
+ new CompressionStream("gzip"), // Use 'gzip' for standard network transport
+ );
+
+ // 4. Read the entire compressed stream back into a single ArrayBuffer.
+ // The Response object provides an easy way to convert streams into a single buffer.
+ const compressedBuffer = await new Response(compressedStream).arrayBuffer();
+
+ // 5. Convert the ArrayBuffer (binary data) to a Base64 string for transport.
+ // Base64 is used to safely transmit binary data over text-based protocols (like JSON).
+ const bytes = new Uint8Array(compressedBuffer);
+ let binary = "";
+
+ // Convert Uint8Array to a raw binary string (this is needed for btoa)
+ for (let i = 0; i < bytes.length; i++) {
+ const nextByte = bytes[i];
+
+ if (nextByte === undefined) {
+ continue;
+ }
+
+ binary += String.fromCharCode(nextByte);
+ }
+
+ // Convert the raw binary string to Base64
+ return btoa(binary);
+}
+
+export const useRecordingStore = create<RecordingStore>((set, get) => ({
+ compressedChunks: [],
+ pendingEvents: [],
+ isCompressing: false,
+ isRecording: false,
+
+ add: (event) => {
+ const state = get();
+ const newPendingEvents = [...state.pendingEvents, event];
+
+ if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) {
+ const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE);
+ const remainingEvents = newPendingEvents.slice(CHUNK_SIZE);
+
+ set({ pendingEvents: remainingEvents, isCompressing: true });
+
+ // compress asynchronously
+ queueMicrotask(async () => {
+ try {
+ const jsonString = JSON.stringify(eventsToCompress);
+ const compressed = await compressEventsToB64(jsonString);
+ const currentState = get();
+ set({
+ compressedChunks: [...currentState.compressedChunks, compressed],
+ isCompressing: false,
+ });
+ } catch (error) {
+ console.error("Failed to compress events chunk:", error);
+
+ // on error, put events back into pending
+ const currentState = get();
+ set({
+ pendingEvents: [...eventsToCompress, ...currentState.pendingEvents],
+ isCompressing: false,
+ });
+ }
+ });
+ } else {
+ set({ pendingEvents: newPendingEvents });
+ }
+ },
+
+ clear: () => set({ compressedChunks: [], pendingEvents: [] }),
+
+ reset: () =>
+ set({
+ compressedChunks: [],
+ pendingEvents: [],
+ isCompressing: false,
+ isRecording: false,
+ }),
+
+ setIsRecording: (isRecording) => {
+ const state = get();
+ // clear events on rising edge
+ if (!state.isRecording && isRecording) {
+ get().clear();
+ }
+ set({ isRecording });
+ },
+
+ flush: async () => {
+ // Wait for any in-progress compression to complete
+ while (get().isCompressing) {
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+
+ const pending = get().pendingEvents;
+ if (pending.length === 0) {
+ return;
+ }
+
+ set({ isCompressing: true });
+
+ try {
+ const jsonString = JSON.stringify(pending);
+ const compressed = await compressEventsToB64(jsonString);
+ const currentState = get();
+ set({
+ compressedChunks: [...currentState.compressedChunks, compressed],
+ pendingEvents: [],
+ isCompressing: false,
+ });
+ } catch (error) {
+ console.error("Failed to flush pending events:", error);
+ set({ isCompressing: false });
+ throw error;
+ }
+ },
+
+ getCompressedChunks: async () => {
+ await get().flush();
+ return get().compressedChunks;
+ },
+
+ getEventCount: () => {
+ const state = get();
+ return (
+ state.compressedChunks.length * CHUNK_SIZE + state.pendingEvents.length
+ );
+ },
+}));