Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Browser recording action (#4130)
Jonathan Dobsonef3d88c1b9ed
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 3b4d5a2fd..1263bf801 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -1,5 +1,5 @@import RFB from "@novnc/novnc/lib/rfb.js";-import { ExitIcon, HandIcon } from "@radix-ui/react-icons";+import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons";import { useEffect, useState, useRef, useCallback } from "react";import { useQuery } from "@tanstack/react-query";@@ -9,12 +9,28 @@ import type {TaskApiResponse,WorkflowRunStatusApiResponse,} from "@/api/types";+import { Tip } from "@/components/Tip";import { Button } from "@/components/ui/button";+import {+ Dialog,+ DialogClose,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+ DialogTrigger,+} from "@/components/ui/dialog";import { AnimatedWave } from "@/components/AnimatedWave";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { statusIsNotFinalized } from "@/routes/tasks/types";import { useClientIdStore } from "@/store/useClientIdStore";+import {+ useRecordingStore,+ type MessageInExfiltratedEvent,+} from "@/store/useRecordingStore";+import { useSettingsStore } from "@/store/SettingsStore";import {environment,wssBaseUrl,@@ -31,18 +47,34 @@ interface BrowserSession {completed_at?: string;}-interface CommandTakeControl {- kind: "take-control";+interface CommandBeginExfiltration {+ kind: "begin-exfiltration";}interface CommandCedeControl {kind: "cede-control";}-// a "Command" is an fire-n-forget out-message - it does not require a response-type Command = CommandTakeControl | CommandCedeControl;+interface CommandEndExfiltration {+ kind: "end-exfiltration";+}++interface CommandTakeControl {+ kind: "take-control";+}-const messageInKinds = ["ask-for-clipboard", "copied-text"] as const;+// a "Command" is an fire-n-forget out-message - it does not require a response+type Command =+ | CommandBeginExfiltration+ | CommandCedeControl+ | CommandEndExfiltration+ | CommandTakeControl;++const messageInKinds = [+ "ask-for-clipboard",+ "copied-text",+ "exfiltrated-event",+] as const;type MessageInKind = (typeof messageInKinds)[number];@@ -55,7 +87,10 @@ interface MessageInCopiedText {text: string;}-type MessageIn = MessageInCopiedText | MessageInAskForClipboard;+type MessageIn =+ | MessageInCopiedText+ | MessageInAskForClipboard+ | MessageInExfiltratedEvent;interface MessageOutAskForClipboardResponse {kind: "ask-for-clipboard-response";@@ -66,6 +101,7 @@ type MessageOut = MessageOutAskForClipboardResponse;type Props = {browserSessionId?: string;+ exfiltrate?: boolean;interactive?: boolean;showControlButtons?: boolean;task?: {@@ -82,6 +118,7 @@ type Props = {function BrowserStream({browserSessionId = undefined,+ exfiltrate = false,interactive = true,showControlButtons = undefined,task = undefined,@@ -175,6 +212,8 @@ function BrowserStream({const rfbRef = useRef<RFB | null>(null);const observerRef = useRef<MutationObserver | null>(null);const clientId = useClientIdStore((state) => state.clientId);+ const recordingStore = useRecordingStore();+ const settingsStore = useSettingsStore();const credentialGetter = useCredentialGetter();const getWebSocketParams = useCallback(async () => {@@ -198,6 +237,15 @@ function BrowserStream({setIsReady(isVncConnected && isCanvasReady && hasBrowserSession);}, [hasBrowserSession, isCanvasReady, isVncConnected]);+ // update global settings store about browser usage+ useEffect(() => {+ settingsStore.setIsUsingABrowser(isReady);+ settingsStore.setBrowserSessionId(+ isReady ? browserSessionId ?? null : null,+ );+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [isReady, browserSessionId]);+// effect for vnc disconnects onlyuseEffect(() => {if (prevVncConnectedRef.current && !isVncConnected) {@@ -417,8 +465,6 @@ function BrowserStream({const sendCommand = (command: Command) => {if (!messageSocket) {- console.warn("Cannot send command, as command socket is closed.");- console.warn(command);return;}@@ -478,11 +524,58 @@ function BrowserStream({}}, [task, workflow]);+ // effect for exfiltration+ useEffect(() => {+ const sendCommand = (command: Command) => {+ if (!messageSocket) {+ return;+ }++ messageSocket.send(JSON.stringify(command));+ };++ sendCommand({+ kind: exfiltrate ? "begin-exfiltration" : "end-exfiltration",+ });+ }, [exfiltrate, messageSocket]);+useEffect(() => {if (!interactive) {setUserIsControlling(false);}}, [interactive]);++ // effect to ensure the recordingStore is reset when the component unmounts+ useEffect(() => {+ return () => {+ recordingStore.reset();+ };+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, []);++ // effect to ensure 'take-control' is sent on the rising edge of+ // recordingStore.isRecording+ useEffect(() => {+ if (!recordingStore.isRecording) {+ return;+ }++ if (!isMessageConnected) {+ return;+ }++ const sendCommand = (command: Command) => {+ if (!messageSocket) {+ return;+ }++ messageSocket.send(JSON.stringify(command));+ };++ sendCommand({ kind: "take-control" });+ setUserIsControlling(true);+ }, [recordingStore.isRecording, isMessageConnected, messageSocket]);+/*** TODO(jdo): could use zod or smth similar*/@@ -524,6 +617,25 @@ function BrowserStream({}break;}+ case "exfiltrated-event": {+ if (+ "event_name" in data &&+ typeof data.event_name === "string" &&+ "params" in data &&+ typeof data.params === "object" &&+ data.params !== null &&+ "source" in data &&+ typeof data.source === "string"+ ) {+ return {+ kind: "exfiltrated-event",+ event_name: data.event_name,+ params: data.params,+ source: data.source,+ } as MessageInExfiltratedEvent;+ }+ break;+ }default: {const _exhaustive: never = kind;return _exhaustive;@@ -604,6 +716,10 @@ function BrowserStream({break;}+ case "exfiltrated-event": {+ recordingStore.add(message);+ break;+ }default: {const _exhaustive: never = kind;return _exhaustive;@@ -615,72 +731,137 @@ function BrowserStream({userIsControlling || (interactive && !showControlButtons);return (- <div- className={cn(- "browser-stream relative flex items-center justify-center",- {- "user-is-controlling": theUserIsControlling,- },- )}- ref={setCanvasContainerRef}- >- {isReady && isVisible && (- <div className="overlay z-10 flex items-center justify-center overflow-hidden">- {showControlButtons && (- <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">- <Button- onClick={() => {- setUserIsControlling(true);- }}- className={cn("control-button pointer-events-auto border", {- hide: userIsControlling,- })}- size="sm"- >- <HandIcon className="mr-2 h-4 w-4" />- take control- </Button>- <Button- onClick={() => {- setUserIsControlling(false);- }}- className={cn(- "control-button pointer-events-auto absolute bottom-0 border",- {- hide: !userIsControlling,- },- )}- size="sm"- >- <ExitIcon className="mr-2 h-4 w-4" />- stop controlling- </Button>+ <>+ <div+ className={cn(+ "browser-stream relative flex flex-col items-center justify-center",+ {+ "user-is-controlling": theUserIsControlling,+ },+ )}+ ref={setCanvasContainerRef}+ >+ {isReady && isVisible && (+ <div className="overlay z-10 flex items-center justify-center overflow-hidden">+ {showControlButtons && (+ <div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">+ <Button+ onClick={() => {+ setUserIsControlling(true);+ }}+ className={cn("control-button pointer-events-auto border", {+ hide: userIsControlling,+ })}+ size="sm"+ >+ <HandIcon className="mr-2 h-4 w-4" />+ take control+ </Button>+ <Button+ onClick={() => {+ setUserIsControlling(false);+ }}+ className={cn(+ "control-button pointer-events-auto absolute bottom-0 border",+ {+ hide: !userIsControlling,+ },+ )}+ size="sm"+ >+ <ExitIcon className="mr-2 h-4 w-4" />+ stop controlling+ </Button>+ </div>+ )}+ </div>+ )}+ {recordingStore.isRecording && (+ <>+ <div className="pointer-events-none absolute flex aspect-video w-full items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-red-500 animate-in fade-in">+ <div className="relative h-full w-full">+ <div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2 text-red-500">+ <div className="truncate">Browser is recording</div>+ <Tip content="To finish the recording, press stop on the animated recording button in the workflow.">+ <div className="cursor-pointer">+ <InfoCircledIcon />+ </div>+ </Tip>+ <Dialog>+ <DialogTrigger asChild>+ <Button+ className="ml-auto cursor-pointer"+ size="sm"+ variant="destructive"+ style={{+ marginTop: "-0.5rem",+ }}+ onClick={(e) => {+ const hasEvents =+ recordingStore.pendingEvents.length > 0 ||+ recordingStore.compressedChunks.length > 0;+ if (!hasEvents) {+ e.preventDefault();+ recordingStore.setIsRecording(false);+ }+ }}+ >+ cancel+ </Button>+ </DialogTrigger>+ <DialogContent>+ <DialogHeader>+ <DialogTitle>Cancel recording?</DialogTitle>+ <DialogDescription>+ You have recorded events that will be lost if you+ cancel. Are you sure you want to cancel the recording?+ </DialogDescription>+ </DialogHeader>+ <DialogFooter>+ <DialogClose asChild>+ <Button variant="outline">Keep recording</Button>+ </DialogClose>+ <DialogClose asChild>+ <Button+ variant="destructive"+ onClick={() => {+ recordingStore.setIsRecording(false);+ }}+ >+ Cancel recording+ </Button>+ </DialogClose>+ </DialogFooter>+ </DialogContent>+ </Dialog>+ </div>+ </div></div>- )}- </div>- )}- {!isReady && (- <div className="absolute left-0 top-1/2 flex aspect-video max-h-full w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">- {browserSessionId && !hasBrowserSession ? (- <div>This live browser session is no longer streaming.</div>- ) : (- <>- <RotateThrough interval={7 * 1000}>- <span>Hm, working on the connection...</span>- <span>Hang tight, we're almost there...</span>- <span>Just a moment...</span>- <span>Backpropagating...</span>- <span>Attention is all I need...</span>- <span>Consulting the manual...</span>- <span>Looking for the bat phone...</span>- <span>Where's Shu?...</span>- </RotateThrough>- <AnimatedWave text=".‧₊˚ ⋅ ? ✨ ?★ ‧₊˚ ⋅" />- </>- )}- </div>- )}- </div>+ </>+ )}+ {!isReady && (+ <div className="absolute left-0 top-1/2 flex aspect-video max-h-full w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">+ {browserSessionId && !hasBrowserSession ? (+ <div>This live browser session is no longer streaming.</div>+ ) : (+ <>+ <RotateThrough interval={7 * 1000}>+ <span>Hm, working on the connection...</span>+ <span>Hang tight, we're almost there...</span>+ <span>Just a moment...</span>+ <span>Backpropagating...</span>+ <span>Attention is all I need...</span>+ <span>Consulting the manual...</span>+ <span>Looking for the bat phone...</span>+ <span>Where's Shu?...</span>+ </RotateThrough>+ <AnimatedWave text=".‧₊˚ ⋅ ? ✨ ?★ ‧₊˚ ⋅" />+ </>+ )}+ </div>+ )}+ </div>+ </>);}diff --git a/skyvern-frontend/src/components/RadialMenu.tsx b/skyvern-frontend/src/components/RadialMenu.tsxindex 77e8b49b6..fe64fc59a 100644--- a/skyvern-frontend/src/components/RadialMenu.tsx+++ b/skyvern-frontend/src/components/RadialMenu.tsx@@ -1,4 +1,4 @@-import { useState, useRef, useEffect, ReactNode } from "react";+import { useState, useRef, useEffect, ReactNode, Fragment } from "react";export interface RadialMenuItem {id: string;@@ -175,9 +175,8 @@ export function RadialMenu({: "scaleY(1)";return (- <>+ <Fragment key={item.id}><button- key={item.id}onClick={() => {item.onClick();setIsOpen(false);@@ -225,7 +224,7 @@ export function RadialMenu({</span></div>)}- </>+ </Fragment>);})}</div>diff --git a/skyvern-frontend/src/components/browser-stream.css b/skyvern-frontend/src/components/browser-stream.cssindex 859999829..4c463277c 100644--- a/skyvern-frontend/src/components/browser-stream.css+++ b/skyvern-frontend/src/components/browser-stream.css@@ -70,6 +70,7 @@.browser-stream > div {background: transparent !important;+ aspect-ratio: 16/9 !important;}.browser-stream .control-button {diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.tsnew file mode 100644index 000000000..79142d5a6--- /dev/null+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts@@ -0,0 +1,140 @@+import { useMutation } from "@tanstack/react-query";++// import { getClient } from "@/api/AxiosClient";+import { toast } from "@/components/ui/use-toast";+// import { useCredentialGetter } from "@/hooks/useCredentialGetter";+// import { type MessageInExfiltratedEvent } from "@/store/useRecordingStore";+import { useRecordingStore } from "@/store/useRecordingStore";+import {+ type ActionBlock,+ type WorkflowBlock,+} from "@/routes/workflows/types/workflowTypes";++const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++const FAIL_QUITE_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;++const useProcessRecordingMutation = ({+ browserSessionId,+ onSuccess,+}: {+ browserSessionId: string | null;+ onSuccess?: (workflowBlocks: Array<WorkflowBlock>) => void;+}) => {+ // const credentialGetter = useCredentialGetter();+ const recordingStore = useRecordingStore();++ const processRecordingMutation = useMutation({+ mutationFn: async () => {+ if (!browserSessionId) {+ throw new Error(+ "Cannot process recording without a valid browser session ID.",+ );+ }++ const eventCount = recordingStore.getEventCount();++ if (eventCount === 0) {+ throw new Error(FAIL_QUITE_NO_EVENTS);+ }++ // (this flushes any pending events)+ const compressedChunks = await recordingStore.getCompressedChunks();++ // TODO: Replace this mock with actual API call when endpoint is ready+ // const client = await getClient(credentialGetter, "sans-api-v1");+ // return client+ // .post<+ // { compressed_chunks: string[] },+ // { data: Array<WorkflowBlock> }+ // >(`/browser_sessions/${browserSessionId}/process_recording`, {+ // compressed_chunks: compressedChunks,+ // })+ // .then((response) => response.data);++ // Mock response with 2-second delay+ console.log(+ `Processing ${eventCount} events in ${compressedChunks.length} compressed chunks`,+ );+ await sleep(2000);++ // Return mock workflow blocks with two ActionBlocks+ const mockWorkflowBlocks: Array<WorkflowBlock> = [+ {+ block_type: "action",+ label: "action_1",+ title: "Enter search term",+ navigation_goal: "Enter 'foo' in the search field",+ url: null,+ error_code_mapping: null,+ parameters: [],+ engine: null,+ continue_on_failure: false,+ output_parameter: {+ parameter_type: "output",+ key: "action_1_output",+ description: null,+ output_parameter_id: "mock-output-1",+ workflow_id: browserSessionId || "mock-workflow-id",+ created_at: new Date().toISOString(),+ modified_at: new Date().toISOString(),+ deleted_at: null,+ },+ model: null,+ } satisfies ActionBlock,+ {+ block_type: "action",+ label: "action_2",+ title: "Click search",+ navigation_goal: "Click the search button",+ url: null,+ error_code_mapping: null,+ parameters: [],+ engine: null,+ continue_on_failure: false,+ output_parameter: {+ parameter_type: "output",+ key: "action_2_output",+ description: null,+ output_parameter_id: "mock-output-2",+ workflow_id: browserSessionId || "mock-workflow-id",+ created_at: new Date().toISOString(),+ modified_at: new Date().toISOString(),+ deleted_at: null,+ },+ model: null,+ } satisfies ActionBlock,+ ];+ return mockWorkflowBlocks;+ },+ onSuccess: (workflowBlocks) => {+ // Clear events after successful flush+ recordingStore.clear();++ toast({+ variant: "success",+ title: "Recording Processed",+ description: "The recording has been successfully processed.",+ });++ if (workflowBlocks) {+ onSuccess?.(workflowBlocks);+ }+ },+ onError: (error) => {+ if (error instanceof Error && error.message === FAIL_QUITE_NO_EVENTS) {+ return;+ }++ toast({+ variant: "destructive",+ title: "Error Processing Recording",+ description: error.message,+ });+ },+ });++ return processRecordingMutation;+};++export { useProcessRecordingMutation };diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsxindex 1f172feb1..385ee523b 100644--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx@@ -11,6 +11,7 @@ import { useOnChange } from "@/hooks/useOnChange";import { useShouldNotifyWhenClosingTab } from "@/hooks/useShouldNotifyWhenClosingTab";import { BlockActionContext } from "@/store/BlockActionContext";import { useDebugStore } from "@/store/useDebugStore";+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";import {useWorkflowHasChangesStore,useWorkflowSave,@@ -33,6 +34,7 @@ import {EdgeChange,} from "@xyflow/react";import "@xyflow/react/dist/style.css";+import { nanoid } from "nanoid";import { useCallback, useEffect, useRef, useState } from "react";import { useBlocker } from "react-router-dom";import {@@ -76,8 +78,10 @@ import {import "./reactFlowOverrideStyles.css";import {convertEchoParameters,+ convertToNode,createNode,descendants,+ generateNodeLabel,getAdditionalParametersForEmailBlock,getOrderedChildrenBlocks,getOutputParameterKey,@@ -319,6 +323,13 @@ function FlowRenderer({const setGetSaveDataRef = useRef(workflowChangesStore.setGetSaveData);setGetSaveDataRef.current = workflowChangesStore.setGetSaveData;const saveWorkflow = useWorkflowSave({ status: "published" });+ const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);+ const recordedInsertionPoint = useRecordedBlocksStore(+ (state) => state.insertionPoint,+ );+ const clearRecordedBlocks = useRecordedBlocksStore(+ (state) => state.clearRecordedBlocks,+ );useShouldNotifyWhenClosingTab(workflowChangesStore.hasChanges);const blocker = useBlocker(({ currentLocation, nextLocation }) => {return (@@ -572,6 +583,85 @@ function FlowRenderer({doLayout(nodes, edges);}+ // effect to add new blocks that were generated from a browser recording+ useEffect(() => {+ if (!recordedBlocks || !recordedInsertionPoint) {+ return;+ }++ const { previous, next, parent, connectingEdgeType } =+ recordedInsertionPoint;++ const newNodes: Array<AppNode> = [];+ const newEdges: Array<Edge> = [];++ let existingLabels = nodes+ .filter(isWorkflowBlockNode)+ .map((node) => node.data.label);++ let prevNodeId = previous;++ // convert each WorkflowBlock to an AppNode+ recordedBlocks.forEach((block, index) => {+ const id = nanoid();+ const label = generateNodeLabel(existingLabels);+ existingLabels = [...existingLabels, label];+ const blockWithLabel = { ...block, label: block.label || label };++ const node = convertToNode(+ { id, parentId: parent },+ blockWithLabel,+ true,+ );+ newNodes.push(node);++ // create edge from previous node to this one+ if (prevNodeId) {+ newEdges.push({+ id: nanoid(),+ type: "edgeWithAddButton",+ source: prevNodeId,+ target: id,+ style: { strokeWidth: 2 },+ });+ }++ // if this is the last block, connect to next+ if (index === recordedBlocks.length - 1 && next) {+ newEdges.push({+ id: nanoid(),+ type: connectingEdgeType,+ source: id,+ target: next,+ style: { strokeWidth: 2 },+ });+ }++ prevNodeId = id;+ });++ const editedEdges = previous+ ? edges.filter((edge) => edge.source !== previous)+ : edges;++ const previousNode = nodes.find((node) => node.id === previous);+ const previousNodeIndex = previousNode+ ? nodes.indexOf(previousNode)+ : nodes.length - 1;++ const newNodesAfter = [+ ...nodes.slice(0, previousNodeIndex + 1),+ ...newNodes,+ ...nodes.slice(previousNodeIndex + 1),+ ];++ workflowChangesStore.setHasChanges(true);+ doLayout(newNodesAfter, [...editedEdges, ...newEdges]);++ clearRecordedBlocks();+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [recordedBlocks, recordedInsertionPoint]);+const editorElementRef = useRef<HTMLDivElement>(null);useAutoPan(editorElementRef, nodes);diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsxnew file mode 100644index 000000000..60c838dc0--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsx@@ -0,0 +1,82 @@+import { SquareIcon, PlusIcon } from "@radix-ui/react-icons";+import { ReactNode } from "react";++import { RadialMenu } from "@/components/RadialMenu";+import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser";+import { useDebugStore } from "@/store/useDebugStore";+import { useRecordingStore } from "@/store/useRecordingStore";+import { useSettingsStore } from "@/store/SettingsStore";++type WorkflowAddMenuProps = {+ buttonSize?: string;+ children: ReactNode;+ gap?: number;+ radius?: string;+ rotateText?: boolean;+ startAt?: number;+ // --+ onAdd: () => void;+ onRecord: () => void;+};++function WorkflowAddMenu({+ buttonSize,+ children,+ gap,+ radius = "80px",+ rotateText = true,+ startAt = 90,+ // --+ onAdd,+ onRecord,+}: WorkflowAddMenuProps) {+ const debugStore = useDebugStore();+ const recordingStore = useRecordingStore();+ const settingsStore = useSettingsStore();+ const isSkyvernUser = useIsSkyvernUser();++ if (+ !isSkyvernUser ||+ !debugStore.isDebugMode ||+ !settingsStore.isUsingABrowser+ ) {+ return <>{children}</>;+ }++ return (+ <RadialMenu+ items={[+ {+ id: "1",+ icon: <PlusIcon className={buttonSize ? "h-3 w-3" : undefined} />,+ text: "Add Block",+ onClick: () => {+ onAdd();+ },+ },+ {+ id: "2",+ icon: <SquareIcon className={buttonSize ? "h-3 w-3" : undefined} />,+ enabled: !recordingStore.isRecording && settingsStore.isUsingABrowser,+ text: "Record Browser",+ onClick: () => {+ if (!settingsStore.isUsingABrowser) {+ return;+ }++ onRecord();+ },+ },+ ]}+ buttonSize={buttonSize}+ radius={radius}+ startAt={startAt}+ gap={gap}+ rotateText={rotateText}+ >+ {children}+ </RadialMenu>+ );+}++export { WorkflowAddMenu };diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.css b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.cssnew file mode 100644index 000000000..f83cc0259--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.css@@ -0,0 +1,29 @@+@keyframes pulse-dash {+ 0% {+ stroke-dasharray: 141.4 141.4;+ stroke-width: 6;+ }+ 50% {+ stroke-dasharray: 10 11.4;+ stroke-width: 8;+ }+ 100% {+ stroke-dasharray: 141.4 141.4;+ stroke-width: 6;+ }+}++@keyframes pulse-dash-small {+ 0% {+ stroke-dasharray: 100.4 100.4;+ stroke-width: 3;+ }+ 50% {+ stroke-dasharray: 10 11.4;+ stroke-width: 5;+ }+ 100% {+ stroke-dasharray: 100.4 100.4;+ stroke-width: 3;+ }+}diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsxnew file mode 100644index 000000000..54f341e69--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx@@ -0,0 +1,196 @@+import { ReactNode, useEffect, useRef, useState } from "react";++import {+ Tooltip,+ TooltipContent,+ TooltipProvider,+ TooltipTrigger,+} from "@/components/ui/tooltip";+import { useRecordingStore, CHUNK_SIZE } from "@/store/useRecordingStore";+import { cn } from "@/util/utils";++import "./WorkflowAdderBusy.css";++type Operation = "recording" | "processing";++type Size = "small" | "large";++type Props = {+ children: ReactNode;+ /**+ * The operation being performed (e.g., recording or processing).+ */+ operation: Operation;+ /**+ * An explicit sizing; otherwise the size will be determined by the child content.+ */+ size?: Size;+ /**+ * Color for the cover and ellipses. Defaults to "red".+ */+ color?: string;+ // --+ onComplete: () => void;+};++function WorkflowAdderBusy({+ children,+ operation,+ size,+ color = "red",+ onComplete,+}: Props) {+ const recordingStore = useRecordingStore();+ const [isHovered, setIsHovered] = useState(false);+ const [shouldBump, setShouldBump] = useState(false);+ const bumpTimeoutRef = useRef<NodeJS.Timeout | null>(null);+ const prevCountRef = useRef(0);++ const eventCount =+ recordingStore.pendingEvents.length ++ recordingStore.compressedChunks.length * CHUNK_SIZE;++ // effect for bump animation when count changes+ useEffect(() => {+ if (eventCount > prevCountRef.current && prevCountRef.current > 0) {+ if (bumpTimeoutRef.current) {+ clearTimeout(bumpTimeoutRef.current);+ }++ setShouldBump(true);++ bumpTimeoutRef.current = setTimeout(() => {+ setShouldBump(false);+ }, 300);+ }++ prevCountRef.current = eventCount;++ return () => {+ if (bumpTimeoutRef.current) {+ clearTimeout(bumpTimeoutRef.current);+ }+ };+ }, [eventCount]);++ const handleClick = (e: React.MouseEvent) => {+ e.stopPropagation();+ e.preventDefault();+ onComplete();++ return false;+ };++ return (+ <TooltipProvider>+ <div className="relative inline-block">+ <Tooltip open={isHovered}>+ <TooltipTrigger asChild>+ <div+ className={cn("relative inline-block", {+ "flex items-center justify-center": size !== undefined,+ "min-h-[40px] min-w-[40px]": size === "small",+ "min-h-[80px] min-w-[80px]": size === "large",+ })}+ onMouseEnter={() => setIsHovered(true)}+ onMouseLeave={() => setIsHovered(false)}+ >+ {/* cover */}+ <div+ className={cn("absolute inset-0 rounded-full opacity-40", {+ "opacity-30": isHovered,+ })}+ style={{ backgroundColor: color }}+ onClick={handleClick}+ />+ <div className="pointer-events-none flex items-center justify-center">+ {children}+ </div>+ <div className="pointer-events-none absolute inset-0">+ <svg+ className="h-full w-full animate-spin"+ viewBox="0 0 100 100"+ preserveAspectRatio="none"+ style={{ transformOrigin: "center" }}+ >+ <ellipse+ cx="50"+ cy="50"+ rx="45"+ ry="45"+ fill="none"+ stroke={color}+ strokeWidth={size === "small" ? "3" : "6"}+ strokeDasharray="141.4 141.4"+ strokeLinecap="round"+ vectorEffect="non-scaling-stroke"+ style={{+ animation: `${size === "small" ? "pulse-dash-small" : "pulse-dash"} 10s ease-in-out infinite`,+ }}+ />+ </svg>+ </div>+ {isHovered && (+ <div className="pointer-events-none absolute inset-0">+ <svg+ className="h-full w-full"+ viewBox="0 0 100 100"+ preserveAspectRatio="none"+ >+ <rect+ x="30"+ y="30"+ width="40"+ height="40"+ fill={color}+ vectorEffect="non-scaling-stroke"+ className="animate-in zoom-in-0"+ style={{+ transformOrigin: "center",+ transformBox: "fill-box",+ animationDuration: "200ms",+ animationTimingFunction:+ "cubic-bezier(0.34, 1.56, 0.64, 1)",+ }}+ />+ </svg>+ </div>+ )}+ </div>+ </TooltipTrigger>+ <TooltipContent>+ <p>+ {operation === "recording" ? "Finish Recording" : "Processing..."}+ </p>+ </TooltipContent>+ </Tooltip>+ {recordingStore.isRecording && eventCount > 0 && (+ <Tooltip delayDuration={0}>+ <TooltipTrigger asChild>+ <div+ className={cn(+ "absolute -right-2 -top-2 flex h-6 min-w-6 items-center justify-center rounded-full px-1.5 text-xs font-semibold text-white shadow-lg transition-transform",+ {+ "scale-125": shouldBump,+ "scale-100": !shouldBump,+ },+ )}+ style={{+ backgroundColor: color,+ transition: "transform 0.6s",+ }}+ >+ {eventCount}+ </div>+ </TooltipTrigger>+ <TooltipContent>+ <p>Event Count</p>+ </TooltipContent>+ </Tooltip>+ )}+ </div>+ </TooltipProvider>+ );+}++export { WorkflowAdderBusy };diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex f249c08d0..86d0076d4 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -29,6 +29,7 @@ import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQuimport { WorkflowRunStream } from "@/routes/workflows/workflowRun/WorkflowRunStream";import { useCacheKeyValuesQuery } from "../hooks/useCacheKeyValuesQuery";import { useBlockScriptStore } from "@/store/BlockScriptStore";+import { useRecordingStore } from "@/store/useRecordingStore";import { useSidebarStore } from "@/store/SidebarStore";import { AnimatedWave } from "@/components/AnimatedWave";@@ -238,6 +239,7 @@ function Workspace({const queryClient = useQueryClient();const [shouldFetchDebugSession, setShouldFetchDebugSession] = useState(false);const blockScriptStore = useBlockScriptStore();+ const recordingStore = useRecordingStore();const cacheKey = workflow?.cache_key ?? "";const [cacheKeyValue, setCacheKeyValue] = useState(@@ -1339,6 +1341,7 @@ function Workspace({<div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center"><div key={reloadKey} className="w-full flex-1"><BrowserStream+ exfiltrate={recordingStore.isRecording}interactive={true}browserSessionId={activeDebugSession?.browser_session_iddiff --git a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsxindex 4dfd57ff1..b9b7bbe71 100644--- a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx@@ -1,4 +1,4 @@-import { SquareIcon, PlusIcon } from "@radix-ui/react-icons";+import { PlusIcon } from "@radix-ui/react-icons";import {BaseEdge,EdgeLabelRenderer,@@ -8,12 +8,16 @@ import {} from "@xyflow/react";import { Button } from "@/components/ui/button";-import { RadialMenu } from "@/components/RadialMenu";-import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser";-import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";+import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";import { useDebugStore } from "@/store/useDebugStore";+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";+import { useRecordingStore } from "@/store/useRecordingStore";+import { useSettingsStore } from "@/store/SettingsStore";+import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";import { REACT_FLOW_EDGE_Z_INDEX } from "../constants";+import { WorkflowAddMenu } from "../WorkflowAddMenu";+import { WorkflowAdderBusy } from "../WorkflowAdderBusy";function EdgeWithAddButton({source,@@ -27,8 +31,6 @@ function EdgeWithAddButton({style = {},markerEnd,}: EdgeProps) {- const debugStore = useDebugStore();- const isSkyvernUser = useIsSkyvernUser();const nodes = useNodes();const [edgePath, labelX, labelY] = getBezierPath({sourceX,@@ -38,14 +40,35 @@ function EdgeWithAddButton({targetY,targetPosition,});+ const debugStore = useDebugStore();+ const recordingStore = useRecordingStore();+ const settingsStore = useSettingsStore();+ const workflowStatePanel = useWorkflowPanelStore();+ const setRecordedBlocks = useRecordedBlocksStore(+ (state) => state.setRecordedBlocks,+ );const setWorkflowPanelState = useWorkflowPanelStore((state) => state.setWorkflowPanelState,);+ const processRecordingMutation = useProcessRecordingMutation({+ browserSessionId: settingsStore.browserSessionId,+ onSuccess: (blocks) => {+ setRecordedBlocks(blocks, {+ previous: source,+ next: target,+ parent: sourceNode?.parentId,+ connectingEdgeType: "edgeWithAddButton",+ });+ },+ });++ const isProcessing = processRecordingMutation.isPending;+const sourceNode = nodes.find((node) => node.id === source);- const onAdd = () => {+ const updateWorkflowPanelState = (active: boolean) => {setWorkflowPanelState({- active: true,+ active,content: "nodeLibrary",data: {previous: source,@@ -55,6 +78,25 @@ function EdgeWithAddButton({});};+ const onAdd = () => updateWorkflowPanelState(true);++ const onRecord = () => {+ if (recordingStore.isRecording) {+ recordingStore.setIsRecording(false);+ } else {+ recordingStore.setIsRecording(true);+ updateWorkflowPanelState(false);+ }+ };++ const onEndRecord = () => {+ if (recordingStore.isRecording) {+ recordingStore.setIsRecording(false);+ }++ processRecordingMutation.mutate();+ };+const adder = (<Buttonsize="icon"@@ -65,6 +107,41 @@ function EdgeWithAddButton({</Button>);+ const menu = (+ <WorkflowAddMenu+ buttonSize="25px"+ gap={35}+ radius="50px"+ startAt={72.5}+ onAdd={onAdd}+ onRecord={onRecord}+ >+ {adder}+ </WorkflowAddMenu>+ );++ const busy = (+ <WorkflowAdderBusy+ color={isProcessing ? "white" : "red"}+ operation={isProcessing ? "processing" : "recording"}+ size="small"+ onComplete={() => {+ onEndRecord();+ }}+ >+ {adder}+ </WorkflowAdderBusy>+ );++ const isBusy =+ (isProcessing || recordingStore.isRecording) &&+ debugStore.isDebugMode &&+ settingsStore.isUsingABrowser &&+ workflowStatePanel.workflowPanelState.data?.previous === source &&+ workflowStatePanel.workflowPanelState.data?.next === target &&+ workflowStatePanel.workflowPanelState.data?.parent ===+ (sourceNode?.parentId || undefined);+return (<><BaseEdge path={edgePath} markerEnd={markerEnd} style={style} />@@ -81,38 +158,7 @@ function EdgeWithAddButton({}}className="nodrag nopan">- {isSkyvernUser && debugStore.isDebugMode ? (- <RadialMenu- items={[- {- id: "1",- icon: <PlusIcon className="h-3 w-3" />,- text: "Add Block",- onClick: () => {- onAdd();- },- },- {- id: "2",- icon: <SquareIcon className="h-3 w-3" />,- enabled: false,- text: "Record Browser",- onClick: () => {- console.log("Record");- },- },- ]}- buttonSize="25px"- radius="50px"- startAt={72.5}- gap={35}- rotateText={true}- >- {adder}- </RadialMenu>- ) : (- adder- )}+ {isBusy ? busy : menu}</div></EdgeLabelRenderer></>diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsxindex a2d0e3cb8..c8d46b30a 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx@@ -1,25 +1,51 @@-import { SquareIcon, PlusIcon } from "@radix-ui/react-icons";+import { PlusIcon } from "@radix-ui/react-icons";import { Handle, NodeProps, Position, useEdges } from "@xyflow/react";-import { RadialMenu } from "@/components/RadialMenu";-import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser";+import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";import { useDebugStore } from "@/store/useDebugStore";+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";+import { useRecordingStore } from "@/store/useRecordingStore";+import { useSettingsStore } from "@/store/SettingsStore";import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";import type { NodeAdderNode } from "./types";+import { WorkflowAddMenu } from "../../WorkflowAddMenu";+import { WorkflowAdderBusy } from "../../WorkflowAdderBusy";function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {- const debugStore = useDebugStore();- const isSkyvernUser = useIsSkyvernUser();const edges = useEdges();+ const debugStore = useDebugStore();+ const recordingStore = useRecordingStore();+ const settingsStore = useSettingsStore();const setWorkflowPanelState = useWorkflowPanelStore((state) => state.setWorkflowPanelState,);+ const workflowStatePanel = useWorkflowPanelStore();+ const setRecordedBlocks = useRecordedBlocksStore(+ (state) => state.setRecordedBlocks,+ );- const onAdd = () => {+ const previous = edges.find((edge) => edge.target === id)?.source ?? null;++ const processRecordingMutation = useProcessRecordingMutation({+ browserSessionId: settingsStore.browserSessionId,+ onSuccess: (blocks) => {+ setRecordedBlocks(blocks, {+ previous,+ next: id,+ parent: parentId,+ connectingEdgeType: "default",+ });+ },+ });++ const isProcessing = processRecordingMutation.isPending;++ const updateWorkflowPanelState = (active: boolean) => {const previous = edges.find((edge) => edge.target === id)?.source;+setWorkflowPanelState({- active: true,+ active,content: "nodeLibrary",data: {previous: previous ?? null,@@ -30,9 +56,30 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {});};+ const onAdd = () => {+ updateWorkflowPanelState(true);+ };++ const onRecord = () => {+ if (recordingStore.isRecording) {+ recordingStore.setIsRecording(false);+ } else {+ recordingStore.setIsRecording(true);+ updateWorkflowPanelState(false);+ }+ };++ const onEndRecord = () => {+ if (recordingStore.isRecording) {+ recordingStore.setIsRecording(false);+ }++ processRecordingMutation.mutate();+ };+const adder = (<div- className="rounded-full bg-slate-50 p-2"+ className={"rounded-full bg-slate-50 p-2"}onClick={() => {onAdd();}}@@ -41,6 +88,33 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {</div>);+ const busy = (+ <WorkflowAdderBusy+ color={isProcessing ? "gray" : "red"}+ operation={isProcessing ? "processing" : "recording"}+ onComplete={() => {+ onEndRecord();+ }}+ >+ {adder}+ </WorkflowAdderBusy>+ );++ const menu = (+ <WorkflowAddMenu onAdd={onAdd} onRecord={onRecord}>+ {adder}+ </WorkflowAddMenu>+ );++ const isBusy =+ (isProcessing || recordingStore.isRecording) &&+ debugStore.isDebugMode &&+ settingsStore.isUsingABrowser &&+ workflowStatePanel.workflowPanelState.data?.previous === previous &&+ workflowStatePanel.workflowPanelState.data?.next === id &&+ workflowStatePanel.workflowPanelState.data?.parent ===+ (parentId || undefined);+return (<div><Handle@@ -55,36 +129,7 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {id="b"className="opacity-0"/>- {isSkyvernUser && debugStore.isDebugMode ? (- <RadialMenu- items={[- {- id: "1",- icon: <PlusIcon />,- text: "Add Block",- onClick: () => {- onAdd();- },- },- {- id: "2",- icon: <SquareIcon />,- enabled: false,- text: "Record Browser",- onClick: () => {- console.log("Record");- },- },- ]}- radius="80px"- startAt={90}- rotateText={true}- >- {adder}- </RadialMenu>- ) : (- adder- )}+ {isBusy ? busy : menu}</div>);}diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex 1bb9912e9..833cbb966 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -2458,6 +2458,7 @@ function getLabelForWorkflowParameterType(type: WorkflowParameterValueType) {export {convert,convertEchoParameters,+ convertToNode,createNode,generateNodeData,generateNodeLabel,diff --git a/skyvern-frontend/src/store/RecordedBlocksStore.ts b/skyvern-frontend/src/store/RecordedBlocksStore.tsnew file mode 100644index 000000000..4daa68915--- /dev/null+++ b/skyvern-frontend/src/store/RecordedBlocksStore.ts@@ -0,0 +1,36 @@+import { create } from "zustand";+import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes";++type InsertionPoint = {+ previous: string | null;+ next: string | null;+ parent?: string;+ connectingEdgeType: string;+};++type RecordedBlocksState = {+ blocks: Array<WorkflowBlock> | null;+ insertionPoint: InsertionPoint | null;+};++type RecordedBlocksStore = RecordedBlocksState & {+ setRecordedBlocks: (+ blocks: Array<WorkflowBlock>,+ insertionPoint: InsertionPoint,+ ) => void;+ clearRecordedBlocks: () => void;+};++const useRecordedBlocksStore = create<RecordedBlocksStore>((set) => ({+ blocks: null,+ insertionPoint: null,+ setRecordedBlocks: (blocks, insertionPoint) => {+ set({ blocks, insertionPoint });+ },+ clearRecordedBlocks: () => {+ set({ blocks: null, insertionPoint: null });+ },+}));++export { useRecordedBlocksStore };+export type { InsertionPoint };diff --git a/skyvern-frontend/src/store/SettingsStore.ts b/skyvern-frontend/src/store/SettingsStore.tsindex b8050b59c..57ed62cd0 100644--- a/skyvern-frontend/src/store/SettingsStore.ts+++ b/skyvern-frontend/src/store/SettingsStore.ts@@ -2,16 +2,33 @@ import { create } from "zustand";type SettingsStore = {environment: string;+ /**+ * The user is currently operating or viewing a live, remote browser. NOTE: if+ * the browser is still connecting, or otherwise not ready, then this should+ * be false.+ */+ isUsingABrowser: boolean;+ /**+ * The current browser session ID when a browser is active.+ */+ browserSessionId: string | null;organization: string;setEnvironment: (environment: string) => void;+ setIsUsingABrowser: (isUsing: boolean) => void;+ setBrowserSessionId: (browserSessionId: string | null) => void;setOrganization: (organization: string) => void;};const useSettingsStore = create<SettingsStore>((set) => {return {environment: "local",+ isUsingABrowser: false,+ browserSessionId: null,organization: "skyvern",setEnvironment: (environment: string) => set({ environment }),+ setIsUsingABrowser: (isUsing: boolean) => set({ isUsingABrowser: isUsing }),+ setBrowserSessionId: (browserSessionId: string | null) =>+ set({ browserSessionId }),setOrganization: (organization: string) => set({ organization }),};});diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.tsnew file mode 100644index 000000000..5369d0068--- /dev/null+++ b/skyvern-frontend/src/store/useRecordingStore.ts@@ -0,0 +1,298 @@+import { create } from "zustand";++/**+ * example: {+ * 'targetInfo': {+ * 'targetId': '8B698E27F1F32372718DA73DCA0C5944',+ * 'type': 'page',+ * 'title': 'New Tab',+ * 'url': 'chrome://newtab/',+ * 'attached': True,+ * 'canAccessOpener': False,+ * 'browserContextId': 'FD13D5C556E681BB49AEED0AB2CA1972',+ * }+ */+export interface ExfiltratedEventCdpParams {+ targetInfo: {+ attached?: boolean;+ browserContextId?: string;+ canAccessOpener?: boolean;+ targetId?: string;+ title?: string;+ type?: string;+ url?: string;+ };+}++export interface ExfiltratedEventConsoleParams {+ type: string;+ url: string;+ timestamp: number;+ target: {+ className?: string;+ id?: string;+ innerText?: string;+ tagName?: string;+ text: string[];+ value?: string;+ };+ inputValue?: string;+ mousePosition: {+ xa: number | null;+ ya: number | null;+ xp: number | null;+ yp: number | null;+ };+ key?: string;+ code?: string;+ activeElement: {+ tagName?: string;+ id?: string;+ className?: string;+ boundingRect?: {+ x: number;+ y: number;+ width: number;+ height: number;+ top: number;+ right: number;+ bottom: number;+ left: number;+ } | null;+ scroll?: {+ scrollTop: number;+ scrollLeft: number;+ scrollHeight: number;+ scrollWidth: number;+ clientHeight: number;+ clientWidth: number;+ } | null;+ };+ window: {+ width: number;+ height: number;+ scrollX: number;+ scrollY: number;+ };+}++export interface MessageInExfiltratedCdpEvent {+ kind: "exfiltrated-event";+ event_name: string;+ params: ExfiltratedEventCdpParams;+ source: "cdp";+}++export interface MessageInExfiltratedConsoleEvent {+ kind: "exfiltrated-event";+ event_name: string;+ params: ExfiltratedEventConsoleParams;+ source: "console";+}++export type MessageInExfiltratedEvent =+ | MessageInExfiltratedCdpEvent+ | MessageInExfiltratedConsoleEvent;++/**+ * Number of events per compressed chunk.+ */+export const CHUNK_SIZE = 1000 as const;++interface RecordingStore {+ /**+ * Compressed chunks of recorded events (base64 gzip).+ * Each chunk contains up to CHUNK_SIZE events.+ */+ compressedChunks: string[];+ /**+ * Buffer of events not yet compressed into a chunk.+ */+ pendingEvents: MessageInExfiltratedEvent[];+ /**+ * Whether a compression operation is currently in progress.+ */+ isCompressing: boolean;+ /**+ * Whether the user is currently in browser recording mode.+ */+ isRecording: boolean;+ /**+ * Add a new recorded event. Triggers async compression when buffer is full.+ */+ add: (event: MessageInExfiltratedEvent) => void;+ /**+ * Clear all recorded events and compressed chunks.+ */+ clear: () => void;+ /**+ * Reset the recording store (clear events and set isRecording to false).+ */+ reset: () => void;+ /**+ * Set whether the user is in browser recording mode.+ */+ setIsRecording: (isRecording: boolean) => void;+ /**+ * Flush any pending events into a compressed chunk.+ * Call this before consuming the data.+ */+ flush: () => Promise<void>;+ /**+ * Get all compressed chunks (after flushing pending events).+ */+ getCompressedChunks: () => Promise<string[]>;+ /**+ * Get the total number of events (compressed + pending).+ */+ getEventCount: () => number;+}++/**+ * compresses a JSON string using the Gzip algorithm and returns the result+ * as a Base64 encoded string+ */+async function compressEventsToB64(jsonString: string): Promise<string> {+ // 1. Convert the string to a Uint8Array (a byte array).+ const encoder = new TextEncoder();+ const uint8Array = encoder.encode(jsonString);++ // 2. Create a ReadableStream from the byte array.+ const readableStream = new ReadableStream({+ start(controller) {+ controller.enqueue(uint8Array);+ controller.close();+ },+ });++ // 3. Pipe the data through the Gzip compression stream.+ const compressedStream = readableStream.pipeThrough(+ new CompressionStream("gzip"), // Use 'gzip' for standard network transport+ );++ // 4. Read the entire compressed stream back into a single ArrayBuffer.+ // The Response object provides an easy way to convert streams into a single buffer.+ const compressedBuffer = await new Response(compressedStream).arrayBuffer();++ // 5. Convert the ArrayBuffer (binary data) to a Base64 string for transport.+ // Base64 is used to safely transmit binary data over text-based protocols (like JSON).+ const bytes = new Uint8Array(compressedBuffer);+ let binary = "";++ // Convert Uint8Array to a raw binary string (this is needed for btoa)+ for (let i = 0; i < bytes.length; i++) {+ const nextByte = bytes[i];++ if (nextByte === undefined) {+ continue;+ }++ binary += String.fromCharCode(nextByte);+ }++ // Convert the raw binary string to Base64+ return btoa(binary);+}++export const useRecordingStore = create<RecordingStore>((set, get) => ({+ compressedChunks: [],+ pendingEvents: [],+ isCompressing: false,+ isRecording: false,++ add: (event) => {+ const state = get();+ const newPendingEvents = [...state.pendingEvents, event];++ if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) {+ const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE);+ const remainingEvents = newPendingEvents.slice(CHUNK_SIZE);++ set({ pendingEvents: remainingEvents, isCompressing: true });++ // compress asynchronously+ queueMicrotask(async () => {+ try {+ const jsonString = JSON.stringify(eventsToCompress);+ const compressed = await compressEventsToB64(jsonString);+ const currentState = get();+ set({+ compressedChunks: [...currentState.compressedChunks, compressed],+ isCompressing: false,+ });+ } catch (error) {+ console.error("Failed to compress events chunk:", error);++ // on error, put events back into pending+ const currentState = get();+ set({+ pendingEvents: [...eventsToCompress, ...currentState.pendingEvents],+ isCompressing: false,+ });+ }+ });+ } else {+ set({ pendingEvents: newPendingEvents });+ }+ },++ clear: () => set({ compressedChunks: [], pendingEvents: [] }),++ reset: () =>+ set({+ compressedChunks: [],+ pendingEvents: [],+ isCompressing: false,+ isRecording: false,+ }),++ setIsRecording: (isRecording) => {+ const state = get();+ // clear events on rising edge+ if (!state.isRecording && isRecording) {+ get().clear();+ }+ set({ isRecording });+ },++ flush: async () => {+ // Wait for any in-progress compression to complete+ while (get().isCompressing) {+ await new Promise((resolve) => setTimeout(resolve, 10));+ }++ const pending = get().pendingEvents;+ if (pending.length === 0) {+ return;+ }++ set({ isCompressing: true });++ try {+ const jsonString = JSON.stringify(pending);+ const compressed = await compressEventsToB64(jsonString);+ const currentState = get();+ set({+ compressedChunks: [...currentState.compressedChunks, compressed],+ pendingEvents: [],+ isCompressing: false,+ });+ } catch (error) {+ console.error("Failed to flush pending events:", error);+ set({ isCompressing: false });+ throw error;+ }+ },++ getCompressedChunks: async () => {+ await get().flush();+ return get().compressedChunks;+ },++ getEventCount: () => {+ const state = get();+ return (+ state.compressedChunks.length * CHUNK_SIZE + state.pendingEvents.length+ );+ },+}));