Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Enable browser recording (#4182)
Jonathan Dobson26a137418bc2
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 1263bf801..b80ccde9c 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -627,11 +627,14 @@ function BrowserStream({"source" in data &&typeof data.source === "string") {+ const event = data as MessageInExfiltratedEvent;+return {kind: "exfiltrated-event",- event_name: data.event_name,- params: data.params,- source: data.source,+ event_name: event.event_name,+ params: event.params,+ source: event.source,+ timestamp: event.timestamp,} as MessageInExfiltratedEvent;}break;@@ -803,6 +806,7 @@ function BrowserStream({if (!hasEvents) {e.preventDefault();recordingStore.setIsRecording(false);+ recordingStore.reset();}}}>@@ -826,6 +830,7 @@ function BrowserStream({variant="destructive"onClick={() => {recordingStore.setIsRecording(false);+ recordingStore.reset();}}>Cancel recordingdiff --git a/skyvern-frontend/src/components/Flippable.tsx b/skyvern-frontend/src/components/Flippable.tsxindex d6bb2c147..8f43e63d8 100644--- a/skyvern-frontend/src/components/Flippable.tsx+++ b/skyvern-frontend/src/components/Flippable.tsx@@ -1,4 +1,6 @@import { ReactNode, Children, useRef, useEffect } from "react";++import { useRecordingStore } from "@/store/useRecordingStore";import { cn } from "@/util/utils";interface FlippableProps {@@ -22,6 +24,7 @@ export function Flippable({className,preserveFrontsideHeight = false,}: FlippableProps) {+ const recordingStore = useRecordingStore();const childrenArray = Children.toArray(children);const front = childrenArray[0];const back = childrenArray[1];@@ -48,7 +51,12 @@ export function Flippable({}, [facing, preserveFrontsideHeight]);return (- <div className={className} style={{ perspective: "1000px" }}>+ <div+ className={cn(className, {+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ style={{ perspective: "1000px" }}+ ><divclassName={cn("transition-transform duration-700",diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.tsindex 79142d5a6..517fda222 100644--- a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts@@ -1,28 +1,28 @@+import { useParams } from "react-router-dom";import { useMutation } from "@tanstack/react-query";-// import { getClient } from "@/api/AxiosClient";+import { getClient } from "@/api/AxiosClient";import { toast } from "@/components/ui/use-toast";-// import { useCredentialGetter } from "@/hooks/useCredentialGetter";-// import { type MessageInExfiltratedEvent } from "@/store/useRecordingStore";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { useRecordingStore } from "@/store/useRecordingStore";-import {- type ActionBlock,- type WorkflowBlock,-} from "@/routes/workflows/types/workflowTypes";+import { type WorkflowBlock } from "@/routes/workflows/types/workflowTypes";+import { type WorkflowParameter } from "@/routes/workflows/types/workflowTypes";-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));--const FAIL_QUITE_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;+const FAIL_QUIET_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;const useProcessRecordingMutation = ({browserSessionId,onSuccess,}: {browserSessionId: string | null;- onSuccess?: (workflowBlocks: Array<WorkflowBlock>) => void;+ onSuccess?: (args: {+ blocks: Array<WorkflowBlock>;+ parameters: Array<WorkflowParameter>;+ }) => void;}) => {- // const credentialGetter = useCredentialGetter();+ const credentialGetter = useCredentialGetter();const recordingStore = useRecordingStore();+ const { workflowPermanentId } = useParams();const processRecordingMutation = useMutation({mutationFn: async () => {@@ -32,97 +32,64 @@ const useProcessRecordingMutation = ({);}+ if (!workflowPermanentId) {+ throw new Error(+ "Cannot process recording without a valid workflow permanent ID.",+ );+ }+const eventCount = recordingStore.getEventCount();if (eventCount === 0) {- throw new Error(FAIL_QUITE_NO_EVENTS);+ throw new Error(FAIL_QUIET_NO_EVENTS);}// (this flushes any pending events)const compressedChunks = await recordingStore.getCompressedChunks();// TODO: Replace this mock with actual API call when endpoint is ready- // const client = await getClient(credentialGetter, "sans-api-v1");- // return client- // .post<- // { compressed_chunks: string[] },- // { data: Array<WorkflowBlock> }- // >(`/browser_sessions/${browserSessionId}/process_recording`, {- // compressed_chunks: compressedChunks,- // })- // .then((response) => response.data);-- // Mock response with 2-second delay- console.log(- `Processing ${eventCount} events in ${compressedChunks.length} compressed chunks`,- );- await sleep(2000);-- // Return mock workflow blocks with two ActionBlocks- const mockWorkflowBlocks: Array<WorkflowBlock> = [- {- block_type: "action",- label: "action_1",- title: "Enter search term",- navigation_goal: "Enter 'foo' in the search field",- url: null,- error_code_mapping: null,- parameters: [],- engine: null,- continue_on_failure: false,- output_parameter: {- parameter_type: "output",- key: "action_1_output",- description: null,- output_parameter_id: "mock-output-1",- workflow_id: browserSessionId || "mock-workflow-id",- created_at: new Date().toISOString(),- modified_at: new Date().toISOString(),- deleted_at: null,- },- model: null,- } satisfies ActionBlock,- {- block_type: "action",- label: "action_2",- title: "Click search",- navigation_goal: "Click the search button",- url: null,- error_code_mapping: null,- parameters: [],- engine: null,- continue_on_failure: false,- output_parameter: {- parameter_type: "output",- key: "action_2_output",- description: null,- output_parameter_id: "mock-output-2",- workflow_id: browserSessionId || "mock-workflow-id",- created_at: new Date().toISOString(),- modified_at: new Date().toISOString(),- deleted_at: null,- },- model: null,- } satisfies ActionBlock,- ];- return mockWorkflowBlocks;+ const client = await getClient(credentialGetter, "sans-api-v1");+ return client+ .post<+ { compressed_chunks: string[] },+ {+ data: {+ blocks: Array<WorkflowBlock>;+ parameters: Array<WorkflowParameter>;+ };+ }+ >(`/browser_sessions/${browserSessionId}/process_recording`, {+ compressed_chunks: compressedChunks,+ workflow_permanent_id: workflowPermanentId,+ })+ .then((response) => ({+ blocks: response.data.blocks,+ parameters: response.data.parameters,+ }));},- onSuccess: (workflowBlocks) => {- // Clear events after successful flush+ onSuccess: ({ blocks, parameters }) => {recordingStore.clear();- toast({- variant: "success",- title: "Recording Processed",- description: "The recording has been successfully processed.",- });+ if (blocks && blocks.length > 0) {+ toast({+ variant: "success",+ title: "Recording Processed",+ description: "The recording has been successfully processed.",+ });++ onSuccess?.({ blocks, parameters: parameters });- if (workflowBlocks) {- onSuccess?.(workflowBlocks);+ return;}++ toast({+ variant: "warning",+ title: "Recording Processed (No Blocks)",+ description: "No blocks could be created from the recording.",+ });},onError: (error) => {- if (error instanceof Error && error.message === FAIL_QUITE_NO_EVENTS) {+ if (error instanceof Error && error.message === FAIL_QUIET_NO_EVENTS) {return;}diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsxindex 385ee523b..74dd5461e 100644--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx@@ -324,6 +324,9 @@ function FlowRenderer({setGetSaveDataRef.current = workflowChangesStore.setGetSaveData;const saveWorkflow = useWorkflowSave({ status: "published" });const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);+ const recordedParameters = useRecordedBlocksStore(+ (state) => state.parameters,+ );const recordedInsertionPoint = useRecordedBlocksStore((state) => state.insertionPoint,);@@ -583,7 +586,8 @@ function FlowRenderer({doLayout(nodes, edges);}- // effect to add new blocks that were generated from a browser recording+ // effect to add new blocks that were generated from a browser recording,+ // along with any new parametersuseEffect(() => {if (!recordedBlocks || !recordedInsertionPoint) {return;@@ -658,6 +662,30 @@ function FlowRenderer({workflowChangesStore.setHasChanges(true);doLayout(newNodesAfter, [...editedEdges, ...newEdges]);+ const newParameters = Array<ParametersState[number]>();++ for (const newParameter of recordedParameters ?? []) {+ const exists = parameters.some((param) => param.key === newParameter.key);++ if (!exists) {+ newParameters.push({+ key: newParameter.key,+ parameterType: "workflow",+ dataType: newParameter.workflow_parameter_type,+ description: newParameter.description ?? null,+ defaultValue: newParameter.default_value ?? "",+ });+ }+ }++ if (newParameters.length > 0) {+ const workflowParametersStore = useWorkflowParametersStore.getState();+ workflowParametersStore.setParameters([+ ...workflowParametersStore.parameters,+ ...newParameters,+ ]);+ }+clearRecordedBlocks();// eslint-disable-next-line react-hooks/exhaustive-deps}, [recordedBlocks, recordedInsertionPoint]);diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsxindex 54f341e69..f925eca58 100644--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx@@ -6,7 +6,7 @@ import {TooltipProvider,TooltipTrigger,} from "@/components/ui/tooltip";-import { useRecordingStore, CHUNK_SIZE } from "@/store/useRecordingStore";+import { useRecordingStore } from "@/store/useRecordingStore";import { cn } from "@/util/utils";import "./WorkflowAdderBusy.css";@@ -45,10 +45,7 @@ function WorkflowAdderBusy({const [shouldBump, setShouldBump] = useState(false);const bumpTimeoutRef = useRef<NodeJS.Timeout | null>(null);const prevCountRef = useRef(0);-- const eventCount =- recordingStore.pendingEvents.length +- recordingStore.compressedChunks.length * CHUNK_SIZE;+ const eventCount = recordingStore.exposedEventCount;// effect for bump animation when count changesuseEffect(() => {diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsxindex 3c8275970..3b61fff51 100644--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx@@ -26,6 +26,7 @@ import { useCreateWorkflowMutation } from "../hooks/useCreateWorkflowMutation";import { convert } from "./workflowEditorUtils";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useDebugStore } from "@/store/useDebugStore";+import { useRecordingStore } from "@/store/useRecordingStore";import { useWorkflowTitleStore } from "@/store/WorkflowTitleStore";import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";import { cn } from "@/util/utils";@@ -82,6 +83,7 @@ function WorkflowHeader({const createWorkflowMutation = useCreateWorkflowMutation();const { data: workflowRun } = useWorkflowRunQuery();const debugStore = useDebugStore();+ const recordingStore = useRecordingStore();const workflowRunIsRunningOrQueued =workflowRun && statusIsRunningOrQueued(workflowRun);const [chosenCacheKeyValue, setChosenCacheKeyValue] = useState<string | null>(@@ -105,8 +107,10 @@ function WorkflowHeader({// eslint-disable-next-line react-hooks/exhaustive-deps}, [cacheKeyValue]);+ const isRecording = recordingStore.isRecording;+const shouldShowCacheControls =- !isGeneratingCode && (cacheKeyValues?.total_count ?? 0) > 0;+ !isRecording && !isGeneratingCode && (cacheKeyValues?.total_count ?? 0) > 0;if (!globalWorkflows) {return null; // this should be loaded already by some other components@@ -124,7 +128,7 @@ function WorkflowHeader({><div className="flex h-full items-center"><EditableNodeTitle- editable={true}+ editable={!isRecording}onChange={(newTitle) => {setTitle(newTitle);workflowChangesStore.setHasChanges(true);@@ -247,7 +251,7 @@ function WorkflowHeader({size="icon"variant={debugStore.isDebugMode ? "default" : "tertiary"}className="size-10 min-w-[2.5rem]"- disabled={workflowRunIsRunningOrQueued}+ disabled={workflowRunIsRunningOrQueued || isRecording}onClick={() => {if (debugStore.isDebugMode) {navigate(`/workflows/${workflowPermanentId}/edit`);@@ -277,7 +281,7 @@ function WorkflowHeader({size="icon"variant="tertiary"className="size-10 min-w-[2.5rem]"- disabled={isGlobalWorkflow}+ disabled={isGlobalWorkflow || isRecording}onClick={() => {onSave();}}@@ -297,6 +301,7 @@ function WorkflowHeader({<Tooltip><TooltipTrigger asChild><Button+ disabled={isRecording}size="icon"variant="tertiary"className="size-10 min-w-[2.5rem]"@@ -311,7 +316,12 @@ function WorkflowHeader({</Tooltip></TooltipProvider>)}- <Button variant="tertiary" size="lg" onClick={onParametersClick}>+ <Button+ disabled={isRecording}+ variant="tertiary"+ size="lg"+ onClick={onParametersClick}+ ><span className="mr-2">Parameters</span>{parametersPanelOpen ? (<ChevronUpIcon className="h-6 w-6" />@@ -320,6 +330,7 @@ function WorkflowHeader({)}</Button><Button+ disabled={isRecording}size="lg"onClick={() => {onRun?.();diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex 86d0076d4..327e7c46f 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -1362,13 +1362,15 @@ function Workspace({"mr-16": !blockLabel,})}>- {showPowerButton && (+ {!recordingStore.isRecording && showPowerButton && (<PowerButton onClick={() => cycle()} />)}- <ReloadButton- isReloading={isReloading}- onClick={() => reload()}- />+ {!recordingStore.isRecording && (+ <ReloadButton+ isReloading={isReloading}+ onClick={() => reload()}+ />+ )}</div></footer></div>diff --git a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsxindex b9b7bbe71..f54670173 100644--- a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx@@ -14,6 +14,7 @@ import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";import { useRecordingStore } from "@/store/useRecordingStore";import { useSettingsStore } from "@/store/SettingsStore";import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";+import { cn } from "@/util/utils";import { REACT_FLOW_EDGE_Z_INDEX } from "../constants";import { WorkflowAddMenu } from "../WorkflowAddMenu";@@ -52,8 +53,8 @@ function EdgeWithAddButton({);const processRecordingMutation = useProcessRecordingMutation({browserSessionId: settingsStore.browserSessionId,- onSuccess: (blocks) => {- setRecordedBlocks(blocks, {+ onSuccess: (result) => {+ setRecordedBlocks(result, {previous: source,next: target,parent: sourceNode?.parentId,@@ -66,6 +67,17 @@ function EdgeWithAddButton({const sourceNode = nodes.find((node) => node.id === source);+ const isBusy =+ (isProcessing || recordingStore.isRecording) &&+ debugStore.isDebugMode &&+ settingsStore.isUsingABrowser &&+ workflowStatePanel.workflowPanelState.data?.previous === source &&+ workflowStatePanel.workflowPanelState.data?.next === target &&+ workflowStatePanel.workflowPanelState.data?.parent ===+ (sourceNode?.parentId || undefined);++ const isDisabled = !isBusy && recordingStore.isRecording;+const updateWorkflowPanelState = (active: boolean) => {setWorkflowPanelState({active,@@ -78,7 +90,13 @@ function EdgeWithAddButton({});};- const onAdd = () => updateWorkflowPanelState(true);+ const onAdd = () => {+ if (isDisabled) {+ return;+ }++ updateWorkflowPanelState(true);+ };const onRecord = () => {if (recordingStore.isRecording) {@@ -100,7 +118,10 @@ function EdgeWithAddButton({const adder = (<Buttonsize="icon"- className="h-4 w-4 rounded-full transition-all hover:scale-150"+ className={cn("h-4 w-4 rounded-full transition-all hover:scale-150", {+ "cursor-not-allowed bg-[grey] hover:scale-100 hover:bg-[grey] active:bg-[grey]":+ isDisabled,+ })}onClick={() => onAdd()}><PlusIcon />@@ -133,15 +154,6 @@ function EdgeWithAddButton({</WorkflowAdderBusy>);- const isBusy =- (isProcessing || recordingStore.isRecording) &&- debugStore.isDebugMode &&- settingsStore.isUsingABrowser &&- workflowStatePanel.workflowPanelState.data?.previous === source &&- workflowStatePanel.workflowPanelState.data?.next === target &&- workflowStatePanel.workflowPanelState.data?.parent ===- (sourceNode?.parentId || undefined);-return (<><BaseEdge path={edgePath} markerEnd={markerEnd} style={style} />@@ -158,7 +170,7 @@ function EdgeWithAddButton({}}className="nodrag nopan">- {isBusy ? busy : menu}+ {isBusy ? busy : isDisabled ? adder : menu}</div></EdgeLabelRenderer></>diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsxindex ea64be6d4..0fbafd1a0 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx@@ -7,6 +7,7 @@ import { CodeEditor } from "@/routes/workflows/components/CodeEditor";import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";+import { useRecordingStore } from "@/store/useRecordingStore";import { deepEqualStringArrays } from "@/util/equality";import { cn } from "@/util/utils";@@ -17,6 +18,7 @@ function CodeBlockNode({ id, data }: NodeProps<CodeBlockNode>) {const { editable, label } = data;const { blockLabel: urlBlockLabel } = useParams();const { data: workflowRun } = useWorkflowRunQuery();+ const recordingStore = useRecordingStore();const workflowRunIsRunningOrQueued =workflowRun && statusIsRunningOrQueued(workflowRun);const thisBlockIsTargetted =@@ -26,7 +28,11 @@ function CodeBlockNode({ id, data }: NodeProps<CodeBlockNode>) {const update = useUpdate<CodeBlockNode["data"]>({ id, editable });return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/DownloadNode/DownloadNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/DownloadNode/DownloadNode.tsxindex dc37bf4fc..cfa51443f 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/DownloadNode/DownloadNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/DownloadNode/DownloadNode.tsx@@ -9,11 +9,13 @@ import { NodeHeader } from "../components/NodeHeader";import { useParams } from "react-router-dom";import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";+import { useRecordingStore } from "@/store/useRecordingStore";function DownloadNode({ id, data }: NodeProps<DownloadNode>) {const { editable, label } = data;const { blockLabel: urlBlockLabel } = useParams();const { data: workflowRun } = useWorkflowRunQuery();+ const recordingStore = useRecordingStore();const workflowRunIsRunningOrQueued =workflowRun && statusIsRunningOrQueued(workflowRun);const thisBlockIsTargetted =@@ -22,7 +24,11 @@ function DownloadNode({ id, data }: NodeProps<DownloadNode>) {workflowRunIsRunningOrQueued && thisBlockIsTargetted;return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsxindex 5106496a6..cbe80227d 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx@@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";import { ModelSelector } from "@/components/ModelSelector";+import { useRecordingStore } from "@/store/useRecordingStore";function FileParserNode({ id, data }: NodeProps<FileParserNode>) {const { editable, label } = data;@@ -27,9 +28,14 @@ function FileParserNode({ id, data }: NodeProps<FileParserNode>) {workflowRunIsRunningOrQueued && thisBlockIsTargetted;const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });const update = useUpdate<FileParserNode["data"]>({ id, editable });+ const recordingStore = useRecordingStore();return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileUploadNode/FileUploadNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileUploadNode/FileUploadNode.tsxindex 5b02a65e8..eb97c6291 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileUploadNode/FileUploadNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileUploadNode/FileUploadNode.tsx@@ -18,6 +18,7 @@ import {import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";+import { useRecordingStore } from "@/store/useRecordingStore";function FileUploadNode({ id, data }: NodeProps<FileUploadNode>) {const { editable, label } = data;@@ -30,9 +31,14 @@ function FileUploadNode({ id, data }: NodeProps<FileUploadNode>) {const thisBlockIsPlaying =workflowRunIsRunningOrQueued && thisBlockIsTargetted;const update = useUpdate<FileUploadNode["data"]>({ id, editable });+ const recordingStore = useRecordingStore();return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpRequestNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpRequestNode.tsxindex b0c6b8dd2..9fd666407 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpRequestNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpRequestNode.tsx@@ -38,6 +38,8 @@ import { CurlImportDialog } from "./CurlImportDialog";import { QuickHeadersDialog } from "./QuickHeadersDialog";import { MethodBadge, UrlValidator, RequestPreview } from "./HttpUtils";import { useRerender } from "@/hooks/useRerender";+import { useRecordingStore } from "@/store/useRecordingStore";+import { cn } from "@/util/utils";const httpMethods = ["GET",@@ -111,12 +113,17 @@ function HttpRequestNode({ id, data }: NodeProps<HttpRequestNodeType>) {);const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });+ const recordingStore = useRecordingStore();const showBodyEditor =data.method !== "GET" && data.method !== "HEAD" && data.method !== "DELETE";return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsxindex 515133d32..54139c78c 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx@@ -19,6 +19,7 @@ import {AccordionTrigger,} from "@/components/ui/accordion";import { useRerender } from "@/hooks/useRerender";+import { useRecordingStore } from "@/store/useRecordingStore";import { AI_IMPROVE_CONFIGS } from "../../constants";const instructionsTooltip =@@ -38,6 +39,7 @@ function HumanInteractionNode({const { editable, label } = data;const { blockLabel: urlBlockLabel } = useParams();const { data: workflowRun } = useWorkflowRunQuery();+ const recordingStore = useRecordingStore();const workflowRunIsRunningOrQueued =workflowRun && statusIsRunningOrQueued(workflowRun);const thisBlockIsTargetted =@@ -48,7 +50,11 @@ function HumanInteractionNode({const rerender = useRerender({ prefix: "accordian" });return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoopNode/LoopNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoopNode/LoopNode.tsxindex 34f188ee9..5b3fddbcd 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoopNode/LoopNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoopNode/LoopNode.tsx@@ -15,6 +15,7 @@ import { useParams } from "react-router-dom";import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";+import { useRecordingStore } from "@/store/useRecordingStore";function LoopNode({ id, data }: NodeProps<LoopNode>) {const nodes = useNodes<AppNode>();@@ -34,6 +35,7 @@ function LoopNode({ id, data }: NodeProps<LoopNode>) {const update = useUpdate<LoopNode["data"]>({ id, editable });const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });const children = nodes.filter((node) => node.parentId === id);+ const recordingStore = useRecordingStore();const furthestDownChild: Node | null = children.reduce((acc, child) => {@@ -56,7 +58,11 @@ function LoopNode({ id, data }: NodeProps<LoopNode>) {const loopNodeWidth = getLoopNodeWidth(node, nodes);return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeActionMenu.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeActionMenu.tsxindex e022b1322..8186b9860 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeActionMenu.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeActionMenu.tsx@@ -1,3 +1,5 @@+import { DotsHorizontalIcon } from "@radix-ui/react-icons";+import {DropdownMenu,DropdownMenuContent,@@ -6,7 +8,7 @@ import {DropdownMenuSeparator,DropdownMenuTrigger,} from "@/components/ui/dropdown-menu";-import { DotsHorizontalIcon } from "@radix-ui/react-icons";+import { useRecordingStore } from "@/store/useRecordingStore";type Props = {isDeletable?: boolean;@@ -23,6 +25,9 @@ function NodeActionMenu({onDelete,onShowScript,}: Props) {+ const recordingStore = useRecordingStore();+ const isRecording = recordingStore.isRecording;+if (!isDeletable && !isScriptable) {return null;}@@ -37,6 +42,7 @@ function NodeActionMenu({<DropdownMenuSeparator />{isDeletable && (<DropdownMenuItem+ disabled={isRecording}onSelect={() => {onDelete?.();}}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsxindex c8d46b30a..8b76ed83c 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx@@ -7,6 +7,7 @@ import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";import { useRecordingStore } from "@/store/useRecordingStore";import { useSettingsStore } from "@/store/SettingsStore";import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";+import { cn } from "@/util/utils";import type { NodeAdderNode } from "./types";import { WorkflowAddMenu } from "../../WorkflowAddMenu";@@ -29,8 +30,8 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {const processRecordingMutation = useProcessRecordingMutation({browserSessionId: settingsStore.browserSessionId,- onSuccess: (blocks) => {- setRecordedBlocks(blocks, {+ onSuccess: (result) => {+ setRecordedBlocks(result, {previous,next: id,parent: parentId,@@ -41,6 +42,17 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {const isProcessing = processRecordingMutation.isPending;+ const isBusy =+ (isProcessing || recordingStore.isRecording) &&+ debugStore.isDebugMode &&+ settingsStore.isUsingABrowser &&+ workflowStatePanel.workflowPanelState.data?.previous === previous &&+ workflowStatePanel.workflowPanelState.data?.next === id &&+ workflowStatePanel.workflowPanelState.data?.parent ===+ (parentId || undefined);++ const isDisabled = !isBusy && recordingStore.isRecording;+const updateWorkflowPanelState = (active: boolean) => {const previous = edges.find((edge) => edge.target === id)?.source;@@ -57,6 +69,10 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {};const onAdd = () => {+ if (isDisabled) {+ return;+ }+updateWorkflowPanelState(true);};@@ -79,7 +95,9 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {const adder = (<div- className={"rounded-full bg-slate-50 p-2"}+ className={cn("rounded-full bg-slate-50 p-2", {+ "cursor-not-allowed bg-[grey]": isDisabled,+ })}onClick={() => {onAdd();}}@@ -106,15 +124,6 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {</WorkflowAddMenu>);- const isBusy =- (isProcessing || recordingStore.isRecording) &&- debugStore.isDebugMode &&- settingsStore.isUsingABrowser &&- workflowStatePanel.workflowPanelState.data?.previous === previous &&- workflowStatePanel.workflowPanelState.data?.next === id &&- workflowStatePanel.workflowPanelState.data?.parent ===- (parentId || undefined);-return (<div><Handle@@ -129,7 +138,7 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {id="b"className="opacity-0"/>- {isBusy ? busy : menu}+ {isBusy ? busy : isDisabled ? adder : menu}</div>);}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsxindex de171b021..de64993f4 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx@@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";import { ModelSelector } from "@/components/ModelSelector";+import { useRecordingStore } from "@/store/useRecordingStore";function PDFParserNode({ id, data }: NodeProps<PDFParserNode>) {const { editable, label } = data;@@ -27,9 +28,14 @@ function PDFParserNode({ id, data }: NodeProps<PDFParserNode>) {workflowRunIsRunningOrQueued && thisBlockIsTargetted;const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });const update = useUpdate<PDFParserNode["data"]>({ id, editable });+ const recordingStore = useRecordingStore();return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsxindex 30105a998..0b1d52841 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx@@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";import { AI_IMPROVE_CONFIGS } from "../../constants";+import { useRecordingStore } from "@/store/useRecordingStore";function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {const { editable, label } = data;@@ -27,9 +28,14 @@ function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {workflowRunIsRunningOrQueued && thisBlockIsTargetted;const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });const update = useUpdate<SendEmailNode["data"]>({ id, editable });+ const recordingStore = useRecordingStore();return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsxindex 5fa4e3889..e81cdeedb 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx@@ -36,6 +36,7 @@ import {import { Flippable } from "@/components/Flippable";import { useRerender } from "@/hooks/useRerender";import { useBlockScriptStore } from "@/store/BlockScriptStore";+import { useRecordingStore } from "@/store/useRecordingStore";import { BlockCodeEditor } from "@/routes/workflows/components/BlockCodeEditor";import { useUpdate } from "@/routes/workflows/editor/useUpdate";import { cn } from "@/util/utils";@@ -56,9 +57,11 @@ function StartNode({ id, data }: NodeProps<StartNode>) {const reactFlowInstance = useReactFlow();const [facing, setFacing] = useState<"front" | "back">("front");const blockScriptStore = useBlockScriptStore();+ const recordingStore = useRecordingStore();const script = blockScriptStore.scripts.__start_block__;const rerender = useRerender({ prefix: "accordion" });const toggleScriptForNodeCallback = useToggleScriptForNodeCallback();+ const isRecording = recordingStore.isRecording;const makeStartSettings = (data: StartNode["data"]): StartSettings => {return {@@ -396,7 +399,11 @@ function StartNode({ id, data }: NodeProps<StartNode>) {}return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsxindex 545969d37..0d7b97920 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx@@ -16,6 +16,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";import { AI_IMPROVE_CONFIGS } from "../../constants";+import { useRecordingStore } from "@/store/useRecordingStore";function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {const { editable, label } = data;@@ -29,9 +30,14 @@ function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {workflowRunIsRunningOrQueued && thisBlockIsTargetted;const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });const update = useUpdate<TextPromptNode["data"]>({ id, editable });+ const recordingStore = useRecordingStore();return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/WaitNode/WaitNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/WaitNode/WaitNode.tsxindex 285628f56..db239f025 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/WaitNode/WaitNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/WaitNode/WaitNode.tsx@@ -11,6 +11,7 @@ import { useParams } from "react-router-dom";import { statusIsRunningOrQueued } from "@/routes/tasks/types";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";import { useUpdate } from "@/routes/workflows/editor/useUpdate";+import { useRecordingStore } from "@/store/useRecordingStore";function WaitNode({ id, data, type }: NodeProps<WaitNode>) {const { editable, label } = data;@@ -25,9 +26,14 @@ function WaitNode({ id, data, type }: NodeProps<WaitNode>) {const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });const update = useUpdate<WaitNode["data"]>({ id, editable });+ const recordingStore = useRecordingStore();return (- <div>+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ ><Handletype="source"position={Position.Bottom}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsxindex f0f00256a..a3ae7fcde 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx@@ -30,6 +30,7 @@ import {import { getInitialValues } from "@/routes/workflows/utils";import { useBlockOutputStore } from "@/store/BlockOutputStore";import { useDebugStore } from "@/store/useDebugStore";+import { useRecordingStore } from "@/store/useRecordingStore";import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";import { useWorkflowSave } from "@/store/WorkflowHasChangesStore";import {@@ -169,6 +170,7 @@ function NodeHeader({} = useParams();const blockOutputsStore = useBlockOutputStore();const debugStore = useDebugStore();+ const recordingStore = useRecordingStore();const { closeWorkflowPanel } = useWorkflowPanelStore();const workflowSettingsStore = useWorkflowSettingsStore();const [label, setLabel] = useNodeLabelChangeHandler({@@ -204,6 +206,8 @@ function NodeHeader({const thisBlockIsTargetted =urlBlockLabel !== undefined && urlBlockLabel === blockLabel;+ const isRecording = recordingStore.isRecording;+const [workflowRunStatus, setWorkflowRunStatus] = useState(workflowRun?.status,);@@ -590,7 +594,8 @@ function NodeHeader({"pointer-events-none fill-gray-500 text-gray-500":workflowRunIsRunningOrQueued ||!workflowPermanentId ||- debugSession === undefined,+ debugSession === undefined ||+ isRecording,})}onClick={() => {handleOnPlay();diff --git a/skyvern-frontend/src/store/RecordedBlocksStore.ts b/skyvern-frontend/src/store/RecordedBlocksStore.tsindex 4daa68915..336fb667e 100644--- a/skyvern-frontend/src/store/RecordedBlocksStore.ts+++ b/skyvern-frontend/src/store/RecordedBlocksStore.ts@@ -1,5 +1,6 @@import { create } from "zustand";import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes";+import type { WorkflowParameter } from "@/routes/workflows/types/workflowTypes";type InsertionPoint = {previous: string | null;@@ -10,12 +11,16 @@ type InsertionPoint = {type RecordedBlocksState = {blocks: Array<WorkflowBlock> | null;+ parameters: Array<WorkflowParameter> | null;insertionPoint: InsertionPoint | null;};type RecordedBlocksStore = RecordedBlocksState & {setRecordedBlocks: (- blocks: Array<WorkflowBlock>,+ data: {+ blocks: Array<WorkflowBlock>;+ parameters: Array<WorkflowParameter>;+ },insertionPoint: InsertionPoint,) => void;clearRecordedBlocks: () => void;@@ -23,9 +28,10 @@ type RecordedBlocksStore = RecordedBlocksState & {const useRecordedBlocksStore = create<RecordedBlocksStore>((set) => ({blocks: null,+ parameters: null,insertionPoint: null,- setRecordedBlocks: (blocks, insertionPoint) => {- set({ blocks, insertionPoint });+ setRecordedBlocks: ({ blocks, parameters }, insertionPoint) => {+ set({ blocks, parameters, insertionPoint });},clearRecordedBlocks: () => {set({ blocks: null, insertionPoint: null });diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.tsindex 5369d0068..76802d829 100644--- a/skyvern-frontend/src/store/useRecordingStore.ts+++ b/skyvern-frontend/src/store/useRecordingStore.ts@@ -81,6 +81,7 @@ export interface MessageInExfiltratedCdpEvent {event_name: string;params: ExfiltratedEventCdpParams;source: "cdp";+ timestamp: number;}export interface MessageInExfiltratedConsoleEvent {@@ -88,6 +89,7 @@ export interface MessageInExfiltratedConsoleEvent {event_name: string;params: ExfiltratedEventConsoleParams;source: "console";+ timestamp: number;}export type MessageInExfiltratedEvent =@@ -105,6 +107,11 @@ interface RecordingStore {* Each chunk contains up to CHUNK_SIZE events.*/compressedChunks: string[];+ /**+ * The number of events to show the user. This elides noisy events, like+ * `mousemove`.+ */+ exposedEventCount: number;/*** Buffer of events not yet compressed into a chunk.*/@@ -194,8 +201,25 @@ async function compressEventsToB64(jsonString: string): Promise<string> {return btoa(binary);}+const isExposedEvent = (event: MessageInExfiltratedEvent): boolean => {+ const exposedConsoleEventTypes = new Set(["focus", "click", "keypress"]);++ if (event.source === "console") {+ if (exposedConsoleEventTypes.has(event.params.type)) {+ return true;+ }+ }++ if (event.source === "cdp") {+ return true;+ }++ return false;+};+export const useRecordingStore = create<RecordingStore>((set, get) => ({compressedChunks: [],+ exposedEventCount: 0,pendingEvents: [],isCompressing: false,isRecording: false,@@ -204,6 +228,10 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({const state = get();const newPendingEvents = [...state.pendingEvents, event];+ if (isExposedEvent(event)) {+ set({ exposedEventCount: state.exposedEventCount + 1 });+ }+if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) {const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE);const remainingEvents = newPendingEvents.slice(CHUNK_SIZE);@@ -241,6 +269,7 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({reset: () =>set({compressedChunks: [],+ exposedEventCount: 0,pendingEvents: [],isCompressing: false,isRecording: false,