Skip to content

Automate browser based workflows with AI

Enable browser recording (#4182)

Jonathan Dobson26a137418bc2
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 1263bf801..b80ccde9c 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -627,11 +627,14 @@ function BrowserStream({
"source" in data &&
typeof data.source === "string"
) {
+ const event = data as MessageInExfiltratedEvent;
+
return {
kind: "exfiltrated-event",
- event_name: data.event_name,
- params: data.params,
- source: data.source,
+ event_name: event.event_name,
+ params: event.params,
+ source: event.source,
+ timestamp: event.timestamp,
} as MessageInExfiltratedEvent;
}
break;
@@ -803,6 +806,7 @@ function BrowserStream({
if (!hasEvents) {
e.preventDefault();
recordingStore.setIsRecording(false);
+ recordingStore.reset();
}
}}
>
@@ -826,6 +830,7 @@ function BrowserStream({
variant="destructive"
onClick={() => {
recordingStore.setIsRecording(false);
+ recordingStore.reset();
}}
>
Cancel recording
diff --git a/skyvern-frontend/src/components/Flippable.tsx b/skyvern-frontend/src/components/Flippable.tsx
index d6bb2c147..8f43e63d8 100644
--- a/skyvern-frontend/src/components/Flippable.tsx
+++ b/skyvern-frontend/src/components/Flippable.tsx
@@ -1,4 +1,6 @@
import { ReactNode, Children, useRef, useEffect } from "react";
+
+import { useRecordingStore } from "@/store/useRecordingStore";
import { cn } from "@/util/utils";
interface FlippableProps {
@@ -22,6 +24,7 @@ export function Flippable({
className,
preserveFrontsideHeight = false,
}: FlippableProps) {
+ const recordingStore = useRecordingStore();
const childrenArray = Children.toArray(children);
const front = childrenArray[0];
const back = childrenArray[1];
@@ -48,7 +51,12 @@ export function Flippable({
}, [facing, preserveFrontsideHeight]);
return (
- <div className={className} style={{ perspective: "1000px" }}>
+ <div
+ className={cn(className, {
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ style={{ perspective: "1000px" }}
+ >
<div
className={cn(
"transition-transform duration-700",
diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
index 79142d5a6..517fda222 100644
--- a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
@@ -1,28 +1,28 @@
+import { useParams } from "react-router-dom";
import { useMutation } from "@tanstack/react-query";
-// import { getClient } from "@/api/AxiosClient";
+import { getClient } from "@/api/AxiosClient";
import { toast } from "@/components/ui/use-toast";
-// import { useCredentialGetter } from "@/hooks/useCredentialGetter";
-// import { type MessageInExfiltratedEvent } from "@/store/useRecordingStore";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useRecordingStore } from "@/store/useRecordingStore";
-import {
- type ActionBlock,
- type WorkflowBlock,
-} from "@/routes/workflows/types/workflowTypes";
+import { type WorkflowBlock } from "@/routes/workflows/types/workflowTypes";
+import { type WorkflowParameter } from "@/routes/workflows/types/workflowTypes";
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
-const FAIL_QUITE_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;
+const FAIL_QUIET_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;
const useProcessRecordingMutation = ({
browserSessionId,
onSuccess,
}: {
browserSessionId: string | null;
- onSuccess?: (workflowBlocks: Array<WorkflowBlock>) => void;
+ onSuccess?: (args: {
+ blocks: Array<WorkflowBlock>;
+ parameters: Array<WorkflowParameter>;
+ }) => void;
}) => {
- // const credentialGetter = useCredentialGetter();
+ const credentialGetter = useCredentialGetter();
const recordingStore = useRecordingStore();
+ const { workflowPermanentId } = useParams();
const processRecordingMutation = useMutation({
mutationFn: async () => {
@@ -32,97 +32,64 @@ const useProcessRecordingMutation = ({
);
}
+ if (!workflowPermanentId) {
+ throw new Error(
+ "Cannot process recording without a valid workflow permanent ID.",
+ );
+ }
+
const eventCount = recordingStore.getEventCount();
if (eventCount === 0) {
- throw new Error(FAIL_QUITE_NO_EVENTS);
+ throw new Error(FAIL_QUIET_NO_EVENTS);
}
// (this flushes any pending events)
const compressedChunks = await recordingStore.getCompressedChunks();
// TODO: Replace this mock with actual API call when endpoint is ready
- // const client = await getClient(credentialGetter, "sans-api-v1");
- // return client
- // .post<
- // { compressed_chunks: string[] },
- // { data: Array<WorkflowBlock> }
- // >(`/browser_sessions/${browserSessionId}/process_recording`, {
- // compressed_chunks: compressedChunks,
- // })
- // .then((response) => response.data);
-
- // Mock response with 2-second delay
- console.log(
- `Processing ${eventCount} events in ${compressedChunks.length} compressed chunks`,
- );
- await sleep(2000);
-
- // Return mock workflow blocks with two ActionBlocks
- const mockWorkflowBlocks: Array<WorkflowBlock> = [
- {
- block_type: "action",
- label: "action_1",
- title: "Enter search term",
- navigation_goal: "Enter 'foo' in the search field",
- url: null,
- error_code_mapping: null,
- parameters: [],
- engine: null,
- continue_on_failure: false,
- output_parameter: {
- parameter_type: "output",
- key: "action_1_output",
- description: null,
- output_parameter_id: "mock-output-1",
- workflow_id: browserSessionId || "mock-workflow-id",
- created_at: new Date().toISOString(),
- modified_at: new Date().toISOString(),
- deleted_at: null,
- },
- model: null,
- } satisfies ActionBlock,
- {
- block_type: "action",
- label: "action_2",
- title: "Click search",
- navigation_goal: "Click the search button",
- url: null,
- error_code_mapping: null,
- parameters: [],
- engine: null,
- continue_on_failure: false,
- output_parameter: {
- parameter_type: "output",
- key: "action_2_output",
- description: null,
- output_parameter_id: "mock-output-2",
- workflow_id: browserSessionId || "mock-workflow-id",
- created_at: new Date().toISOString(),
- modified_at: new Date().toISOString(),
- deleted_at: null,
- },
- model: null,
- } satisfies ActionBlock,
- ];
- return mockWorkflowBlocks;
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ return client
+ .post<
+ { compressed_chunks: string[] },
+ {
+ data: {
+ blocks: Array<WorkflowBlock>;
+ parameters: Array<WorkflowParameter>;
+ };
+ }
+ >(`/browser_sessions/${browserSessionId}/process_recording`, {
+ compressed_chunks: compressedChunks,
+ workflow_permanent_id: workflowPermanentId,
+ })
+ .then((response) => ({
+ blocks: response.data.blocks,
+ parameters: response.data.parameters,
+ }));
},
- onSuccess: (workflowBlocks) => {
- // Clear events after successful flush
+ onSuccess: ({ blocks, parameters }) => {
recordingStore.clear();
- toast({
- variant: "success",
- title: "Recording Processed",
- description: "The recording has been successfully processed.",
- });
+ if (blocks && blocks.length > 0) {
+ toast({
+ variant: "success",
+ title: "Recording Processed",
+ description: "The recording has been successfully processed.",
+ });
+
+ onSuccess?.({ blocks, parameters: parameters });
- if (workflowBlocks) {
- onSuccess?.(workflowBlocks);
+ return;
}
+
+ toast({
+ variant: "warning",
+ title: "Recording Processed (No Blocks)",
+ description: "No blocks could be created from the recording.",
+ });
},
onError: (error) => {
- if (error instanceof Error && error.message === FAIL_QUITE_NO_EVENTS) {
+ if (error instanceof Error && error.message === FAIL_QUIET_NO_EVENTS) {
return;
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
index 385ee523b..74dd5461e 100644
--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
@@ -324,6 +324,9 @@ function FlowRenderer({
setGetSaveDataRef.current = workflowChangesStore.setGetSaveData;
const saveWorkflow = useWorkflowSave({ status: "published" });
const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);
+ const recordedParameters = useRecordedBlocksStore(
+ (state) => state.parameters,
+ );
const recordedInsertionPoint = useRecordedBlocksStore(
(state) => state.insertionPoint,
);
@@ -583,7 +586,8 @@ function FlowRenderer({
doLayout(nodes, edges);
}
- // effect to add new blocks that were generated from a browser recording
+ // effect to add new blocks that were generated from a browser recording,
+ // along with any new parameters
useEffect(() => {
if (!recordedBlocks || !recordedInsertionPoint) {
return;
@@ -658,6 +662,30 @@ function FlowRenderer({
workflowChangesStore.setHasChanges(true);
doLayout(newNodesAfter, [...editedEdges, ...newEdges]);
+ const newParameters = Array<ParametersState[number]>();
+
+ for (const newParameter of recordedParameters ?? []) {
+ const exists = parameters.some((param) => param.key === newParameter.key);
+
+ if (!exists) {
+ newParameters.push({
+ key: newParameter.key,
+ parameterType: "workflow",
+ dataType: newParameter.workflow_parameter_type,
+ description: newParameter.description ?? null,
+ defaultValue: newParameter.default_value ?? "",
+ });
+ }
+ }
+
+ if (newParameters.length > 0) {
+ const workflowParametersStore = useWorkflowParametersStore.getState();
+ workflowParametersStore.setParameters([
+ ...workflowParametersStore.parameters,
+ ...newParameters,
+ ]);
+ }
+
clearRecordedBlocks();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [recordedBlocks, recordedInsertionPoint]);
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx
index 54f341e69..f925eca58 100644
--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx
@@ -6,7 +6,7 @@ import {
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
-import { useRecordingStore, CHUNK_SIZE } from "@/store/useRecordingStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
import { cn } from "@/util/utils";
import "./WorkflowAdderBusy.css";
@@ -45,10 +45,7 @@ function WorkflowAdderBusy({
const [shouldBump, setShouldBump] = useState(false);
const bumpTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const prevCountRef = useRef(0);
-
- const eventCount =
- recordingStore.pendingEvents.length +
- recordingStore.compressedChunks.length * CHUNK_SIZE;
+ const eventCount = recordingStore.exposedEventCount;
// effect for bump animation when count changes
useEffect(() => {
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx
index 3c8275970..3b61fff51 100644
--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx
@@ -26,6 +26,7 @@ import { useCreateWorkflowMutation } from "../hooks/useCreateWorkflowMutation";
import { convert } from "./workflowEditorUtils";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useDebugStore } from "@/store/useDebugStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
import { useWorkflowTitleStore } from "@/store/WorkflowTitleStore";
import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";
import { cn } from "@/util/utils";
@@ -82,6 +83,7 @@ function WorkflowHeader({
const createWorkflowMutation = useCreateWorkflowMutation();
const { data: workflowRun } = useWorkflowRunQuery();
const debugStore = useDebugStore();
+ const recordingStore = useRecordingStore();
const workflowRunIsRunningOrQueued =
workflowRun && statusIsRunningOrQueued(workflowRun);
const [chosenCacheKeyValue, setChosenCacheKeyValue] = useState<string | null>(
@@ -105,8 +107,10 @@ function WorkflowHeader({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [cacheKeyValue]);
+ const isRecording = recordingStore.isRecording;
+
const shouldShowCacheControls =
- !isGeneratingCode && (cacheKeyValues?.total_count ?? 0) > 0;
+ !isRecording && !isGeneratingCode && (cacheKeyValues?.total_count ?? 0) > 0;
if (!globalWorkflows) {
return null; // this should be loaded already by some other components
@@ -124,7 +128,7 @@ function WorkflowHeader({
>
<div className="flex h-full items-center">
<EditableNodeTitle
- editable={true}
+ editable={!isRecording}
onChange={(newTitle) => {
setTitle(newTitle);
workflowChangesStore.setHasChanges(true);
@@ -247,7 +251,7 @@ function WorkflowHeader({
size="icon"
variant={debugStore.isDebugMode ? "default" : "tertiary"}
className="size-10 min-w-[2.5rem]"
- disabled={workflowRunIsRunningOrQueued}
+ disabled={workflowRunIsRunningOrQueued || isRecording}
onClick={() => {
if (debugStore.isDebugMode) {
navigate(`/workflows/${workflowPermanentId}/edit`);
@@ -277,7 +281,7 @@ function WorkflowHeader({
size="icon"
variant="tertiary"
className="size-10 min-w-[2.5rem]"
- disabled={isGlobalWorkflow}
+ disabled={isGlobalWorkflow || isRecording}
onClick={() => {
onSave();
}}
@@ -297,6 +301,7 @@ function WorkflowHeader({
<Tooltip>
<TooltipTrigger asChild>
<Button
+ disabled={isRecording}
size="icon"
variant="tertiary"
className="size-10 min-w-[2.5rem]"
@@ -311,7 +316,12 @@ function WorkflowHeader({
</Tooltip>
</TooltipProvider>
)}
- <Button variant="tertiary" size="lg" onClick={onParametersClick}>
+ <Button
+ disabled={isRecording}
+ variant="tertiary"
+ size="lg"
+ onClick={onParametersClick}
+ >
<span className="mr-2">Parameters</span>
{parametersPanelOpen ? (
<ChevronUpIcon className="h-6 w-6" />
@@ -320,6 +330,7 @@ function WorkflowHeader({
)}
</Button>
<Button
+ disabled={isRecording}
size="lg"
onClick={() => {
onRun?.();
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index 86d0076d4..327e7c46f 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -1362,13 +1362,15 @@ function Workspace({
"mr-16": !blockLabel,
})}
>
- {showPowerButton && (
+ {!recordingStore.isRecording && showPowerButton && (
<PowerButton onClick={() => cycle()} />
)}
- <ReloadButton
- isReloading={isReloading}
- onClick={() => reload()}
- />
+ {!recordingStore.isRecording && (
+ <ReloadButton
+ isReloading={isReloading}
+ onClick={() => reload()}
+ />
+ )}
</div>
</footer>
</div>
diff --git a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
index b9b7bbe71..f54670173 100644
--- a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
@@ -14,6 +14,7 @@ import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
import { useRecordingStore } from "@/store/useRecordingStore";
import { useSettingsStore } from "@/store/SettingsStore";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
+import { cn } from "@/util/utils";
import { REACT_FLOW_EDGE_Z_INDEX } from "../constants";
import { WorkflowAddMenu } from "../WorkflowAddMenu";
@@ -52,8 +53,8 @@ function EdgeWithAddButton({
);
const processRecordingMutation = useProcessRecordingMutation({
browserSessionId: settingsStore.browserSessionId,
- onSuccess: (blocks) => {
- setRecordedBlocks(blocks, {
+ onSuccess: (result) => {
+ setRecordedBlocks(result, {
previous: source,
next: target,
parent: sourceNode?.parentId,
@@ -66,6 +67,17 @@ function EdgeWithAddButton({
const sourceNode = nodes.find((node) => node.id === source);
+ const isBusy =
+ (isProcessing || recordingStore.isRecording) &&
+ debugStore.isDebugMode &&
+ settingsStore.isUsingABrowser &&
+ workflowStatePanel.workflowPanelState.data?.previous === source &&
+ workflowStatePanel.workflowPanelState.data?.next === target &&
+ workflowStatePanel.workflowPanelState.data?.parent ===
+ (sourceNode?.parentId || undefined);
+
+ const isDisabled = !isBusy && recordingStore.isRecording;
+
const updateWorkflowPanelState = (active: boolean) => {
setWorkflowPanelState({
active,
@@ -78,7 +90,13 @@ function EdgeWithAddButton({
});
};
- const onAdd = () => updateWorkflowPanelState(true);
+ const onAdd = () => {
+ if (isDisabled) {
+ return;
+ }
+
+ updateWorkflowPanelState(true);
+ };
const onRecord = () => {
if (recordingStore.isRecording) {
@@ -100,7 +118,10 @@ function EdgeWithAddButton({
const adder = (
<Button
size="icon"
- className="h-4 w-4 rounded-full transition-all hover:scale-150"
+ className={cn("h-4 w-4 rounded-full transition-all hover:scale-150", {
+ "cursor-not-allowed bg-[grey] hover:scale-100 hover:bg-[grey] active:bg-[grey]":
+ isDisabled,
+ })}
onClick={() => onAdd()}
>
<PlusIcon />
@@ -133,15 +154,6 @@ function EdgeWithAddButton({
</WorkflowAdderBusy>
);
- const isBusy =
- (isProcessing || recordingStore.isRecording) &&
- debugStore.isDebugMode &&
- settingsStore.isUsingABrowser &&
- workflowStatePanel.workflowPanelState.data?.previous === source &&
- workflowStatePanel.workflowPanelState.data?.next === target &&
- workflowStatePanel.workflowPanelState.data?.parent ===
- (sourceNode?.parentId || undefined);
-
return (
<>
<BaseEdge path={edgePath} markerEnd={markerEnd} style={style} />
@@ -158,7 +170,7 @@ function EdgeWithAddButton({
}}
className="nodrag nopan"
>
- {isBusy ? busy : menu}
+ {isBusy ? busy : isDisabled ? adder : menu}
</div>
</EdgeLabelRenderer>
</>
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx
index ea64be6d4..0fbafd1a0 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx
@@ -7,6 +7,7 @@ import { CodeEditor } from "@/routes/workflows/components/CodeEditor";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
+import { useRecordingStore } from "@/store/useRecordingStore";
import { deepEqualStringArrays } from "@/util/equality";
import { cn } from "@/util/utils";
@@ -17,6 +18,7 @@ function CodeBlockNode({ id, data }: NodeProps<CodeBlockNode>) {
const { editable, label } = data;
const { blockLabel: urlBlockLabel } = useParams();
const { data: workflowRun } = useWorkflowRunQuery();
+ const recordingStore = useRecordingStore();
const workflowRunIsRunningOrQueued =
workflowRun && statusIsRunningOrQueued(workflowRun);
const thisBlockIsTargetted =
@@ -26,7 +28,11 @@ function CodeBlockNode({ id, data }: NodeProps<CodeBlockNode>) {
const update = useUpdate<CodeBlockNode["data"]>({ id, editable });
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/DownloadNode/DownloadNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/DownloadNode/DownloadNode.tsx
index dc37bf4fc..cfa51443f 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/DownloadNode/DownloadNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/DownloadNode/DownloadNode.tsx
@@ -9,11 +9,13 @@ import { NodeHeader } from "../components/NodeHeader";
import { useParams } from "react-router-dom";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
+import { useRecordingStore } from "@/store/useRecordingStore";
function DownloadNode({ id, data }: NodeProps<DownloadNode>) {
const { editable, label } = data;
const { blockLabel: urlBlockLabel } = useParams();
const { data: workflowRun } = useWorkflowRunQuery();
+ const recordingStore = useRecordingStore();
const workflowRunIsRunningOrQueued =
workflowRun && statusIsRunningOrQueued(workflowRun);
const thisBlockIsTargetted =
@@ -22,7 +24,11 @@ function DownloadNode({ id, data }: NodeProps<DownloadNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx
index 5106496a6..cbe80227d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx
@@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { ModelSelector } from "@/components/ModelSelector";
+import { useRecordingStore } from "@/store/useRecordingStore";
function FileParserNode({ id, data }: NodeProps<FileParserNode>) {
const { editable, label } = data;
@@ -27,9 +28,14 @@ function FileParserNode({ id, data }: NodeProps<FileParserNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<FileParserNode["data"]>({ id, editable });
+ const recordingStore = useRecordingStore();
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileUploadNode/FileUploadNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileUploadNode/FileUploadNode.tsx
index 5b02a65e8..eb97c6291 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileUploadNode/FileUploadNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileUploadNode/FileUploadNode.tsx
@@ -18,6 +18,7 @@ import {
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
+import { useRecordingStore } from "@/store/useRecordingStore";
function FileUploadNode({ id, data }: NodeProps<FileUploadNode>) {
const { editable, label } = data;
@@ -30,9 +31,14 @@ function FileUploadNode({ id, data }: NodeProps<FileUploadNode>) {
const thisBlockIsPlaying =
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const update = useUpdate<FileUploadNode["data"]>({ id, editable });
+ const recordingStore = useRecordingStore();
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpRequestNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpRequestNode.tsx
index b0c6b8dd2..9fd666407 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpRequestNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpRequestNode.tsx
@@ -38,6 +38,8 @@ import { CurlImportDialog } from "./CurlImportDialog";
import { QuickHeadersDialog } from "./QuickHeadersDialog";
import { MethodBadge, UrlValidator, RequestPreview } from "./HttpUtils";
import { useRerender } from "@/hooks/useRerender";
+import { useRecordingStore } from "@/store/useRecordingStore";
+import { cn } from "@/util/utils";
const httpMethods = [
"GET",
@@ -111,12 +113,17 @@ function HttpRequestNode({ id, data }: NodeProps<HttpRequestNodeType>) {
);
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
+ const recordingStore = useRecordingStore();
const showBodyEditor =
data.method !== "GET" && data.method !== "HEAD" && data.method !== "DELETE";
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
index 515133d32..54139c78c 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
@@ -19,6 +19,7 @@ import {
AccordionTrigger,
} from "@/components/ui/accordion";
import { useRerender } from "@/hooks/useRerender";
+import { useRecordingStore } from "@/store/useRecordingStore";
import { AI_IMPROVE_CONFIGS } from "../../constants";
const instructionsTooltip =
@@ -38,6 +39,7 @@ function HumanInteractionNode({
const { editable, label } = data;
const { blockLabel: urlBlockLabel } = useParams();
const { data: workflowRun } = useWorkflowRunQuery();
+ const recordingStore = useRecordingStore();
const workflowRunIsRunningOrQueued =
workflowRun && statusIsRunningOrQueued(workflowRun);
const thisBlockIsTargetted =
@@ -48,7 +50,11 @@ function HumanInteractionNode({
const rerender = useRerender({ prefix: "accordian" });
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoopNode/LoopNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoopNode/LoopNode.tsx
index 34f188ee9..5b3fddbcd 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoopNode/LoopNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoopNode/LoopNode.tsx
@@ -15,6 +15,7 @@ import { useParams } from "react-router-dom";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
+import { useRecordingStore } from "@/store/useRecordingStore";
function LoopNode({ id, data }: NodeProps<LoopNode>) {
const nodes = useNodes<AppNode>();
@@ -34,6 +35,7 @@ function LoopNode({ id, data }: NodeProps<LoopNode>) {
const update = useUpdate<LoopNode["data"]>({ id, editable });
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const children = nodes.filter((node) => node.parentId === id);
+ const recordingStore = useRecordingStore();
const furthestDownChild: Node | null = children.reduce(
(acc, child) => {
@@ -56,7 +58,11 @@ function LoopNode({ id, data }: NodeProps<LoopNode>) {
const loopNodeWidth = getLoopNodeWidth(node, nodes);
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeActionMenu.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeActionMenu.tsx
index e022b1322..8186b9860 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeActionMenu.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeActionMenu.tsx
@@ -1,3 +1,5 @@
+import { DotsHorizontalIcon } from "@radix-ui/react-icons";
+
import {
DropdownMenu,
DropdownMenuContent,
@@ -6,7 +8,7 @@ import {
DropdownMenuSeparator,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
-import { DotsHorizontalIcon } from "@radix-ui/react-icons";
+import { useRecordingStore } from "@/store/useRecordingStore";
type Props = {
isDeletable?: boolean;
@@ -23,6 +25,9 @@ function NodeActionMenu({
onDelete,
onShowScript,
}: Props) {
+ const recordingStore = useRecordingStore();
+ const isRecording = recordingStore.isRecording;
+
if (!isDeletable && !isScriptable) {
return null;
}
@@ -37,6 +42,7 @@ function NodeActionMenu({
<DropdownMenuSeparator />
{isDeletable && (
<DropdownMenuItem
+ disabled={isRecording}
onSelect={() => {
onDelete?.();
}}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
index c8d46b30a..8b76ed83c 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
@@ -7,6 +7,7 @@ import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
import { useRecordingStore } from "@/store/useRecordingStore";
import { useSettingsStore } from "@/store/SettingsStore";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
+import { cn } from "@/util/utils";
import type { NodeAdderNode } from "./types";
import { WorkflowAddMenu } from "../../WorkflowAddMenu";
@@ -29,8 +30,8 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
const processRecordingMutation = useProcessRecordingMutation({
browserSessionId: settingsStore.browserSessionId,
- onSuccess: (blocks) => {
- setRecordedBlocks(blocks, {
+ onSuccess: (result) => {
+ setRecordedBlocks(result, {
previous,
next: id,
parent: parentId,
@@ -41,6 +42,17 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
const isProcessing = processRecordingMutation.isPending;
+ const isBusy =
+ (isProcessing || recordingStore.isRecording) &&
+ debugStore.isDebugMode &&
+ settingsStore.isUsingABrowser &&
+ workflowStatePanel.workflowPanelState.data?.previous === previous &&
+ workflowStatePanel.workflowPanelState.data?.next === id &&
+ workflowStatePanel.workflowPanelState.data?.parent ===
+ (parentId || undefined);
+
+ const isDisabled = !isBusy && recordingStore.isRecording;
+
const updateWorkflowPanelState = (active: boolean) => {
const previous = edges.find((edge) => edge.target === id)?.source;
@@ -57,6 +69,10 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
};
const onAdd = () => {
+ if (isDisabled) {
+ return;
+ }
+
updateWorkflowPanelState(true);
};
@@ -79,7 +95,9 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
const adder = (
<div
- className={"rounded-full bg-slate-50 p-2"}
+ className={cn("rounded-full bg-slate-50 p-2", {
+ "cursor-not-allowed bg-[grey]": isDisabled,
+ })}
onClick={() => {
onAdd();
}}
@@ -106,15 +124,6 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
</WorkflowAddMenu>
);
- const isBusy =
- (isProcessing || recordingStore.isRecording) &&
- debugStore.isDebugMode &&
- settingsStore.isUsingABrowser &&
- workflowStatePanel.workflowPanelState.data?.previous === previous &&
- workflowStatePanel.workflowPanelState.data?.next === id &&
- workflowStatePanel.workflowPanelState.data?.parent ===
- (parentId || undefined);
-
return (
<div>
<Handle
@@ -129,7 +138,7 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
id="b"
className="opacity-0"
/>
- {isBusy ? busy : menu}
+ {isBusy ? busy : isDisabled ? adder : menu}
</div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx
index de171b021..de64993f4 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx
@@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { ModelSelector } from "@/components/ModelSelector";
+import { useRecordingStore } from "@/store/useRecordingStore";
function PDFParserNode({ id, data }: NodeProps<PDFParserNode>) {
const { editable, label } = data;
@@ -27,9 +28,14 @@ function PDFParserNode({ id, data }: NodeProps<PDFParserNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<PDFParserNode["data"]>({ id, editable });
+ const recordingStore = useRecordingStore();
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx
index 30105a998..0b1d52841 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx
@@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { AI_IMPROVE_CONFIGS } from "../../constants";
+import { useRecordingStore } from "@/store/useRecordingStore";
function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {
const { editable, label } = data;
@@ -27,9 +28,14 @@ function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<SendEmailNode["data"]>({ id, editable });
+ const recordingStore = useRecordingStore();
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx
index 5fa4e3889..e81cdeedb 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx
@@ -36,6 +36,7 @@ import {
import { Flippable } from "@/components/Flippable";
import { useRerender } from "@/hooks/useRerender";
import { useBlockScriptStore } from "@/store/BlockScriptStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
import { BlockCodeEditor } from "@/routes/workflows/components/BlockCodeEditor";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { cn } from "@/util/utils";
@@ -56,9 +57,11 @@ function StartNode({ id, data }: NodeProps<StartNode>) {
const reactFlowInstance = useReactFlow();
const [facing, setFacing] = useState<"front" | "back">("front");
const blockScriptStore = useBlockScriptStore();
+ const recordingStore = useRecordingStore();
const script = blockScriptStore.scripts.__start_block__;
const rerender = useRerender({ prefix: "accordion" });
const toggleScriptForNodeCallback = useToggleScriptForNodeCallback();
+ const isRecording = recordingStore.isRecording;
const makeStartSettings = (data: StartNode["data"]): StartSettings => {
return {
@@ -396,7 +399,11 @@ function StartNode({ id, data }: NodeProps<StartNode>) {
}
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx
index 545969d37..0d7b97920 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx
@@ -16,6 +16,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { AI_IMPROVE_CONFIGS } from "../../constants";
+import { useRecordingStore } from "@/store/useRecordingStore";
function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {
const { editable, label } = data;
@@ -29,9 +30,14 @@ function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<TextPromptNode["data"]>({ id, editable });
+ const recordingStore = useRecordingStore();
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/WaitNode/WaitNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/WaitNode/WaitNode.tsx
index 285628f56..db239f025 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/WaitNode/WaitNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/WaitNode/WaitNode.tsx
@@ -11,6 +11,7 @@ import { useParams } from "react-router-dom";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
+import { useRecordingStore } from "@/store/useRecordingStore";
function WaitNode({ id, data, type }: NodeProps<WaitNode>) {
const { editable, label } = data;
@@ -25,9 +26,14 @@ function WaitNode({ id, data, type }: NodeProps<WaitNode>) {
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<WaitNode["data"]>({ id, editable });
+ const recordingStore = useRecordingStore();
return (
- <div>
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
<Handle
type="source"
position={Position.Bottom}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
index f0f00256a..a3ae7fcde 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
@@ -30,6 +30,7 @@ import {
import { getInitialValues } from "@/routes/workflows/utils";
import { useBlockOutputStore } from "@/store/BlockOutputStore";
import { useDebugStore } from "@/store/useDebugStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
import { useWorkflowSave } from "@/store/WorkflowHasChangesStore";
import {
@@ -169,6 +170,7 @@ function NodeHeader({
} = useParams();
const blockOutputsStore = useBlockOutputStore();
const debugStore = useDebugStore();
+ const recordingStore = useRecordingStore();
const { closeWorkflowPanel } = useWorkflowPanelStore();
const workflowSettingsStore = useWorkflowSettingsStore();
const [label, setLabel] = useNodeLabelChangeHandler({
@@ -204,6 +206,8 @@ function NodeHeader({
const thisBlockIsTargetted =
urlBlockLabel !== undefined && urlBlockLabel === blockLabel;
+ const isRecording = recordingStore.isRecording;
+
const [workflowRunStatus, setWorkflowRunStatus] = useState(
workflowRun?.status,
);
@@ -590,7 +594,8 @@ function NodeHeader({
"pointer-events-none fill-gray-500 text-gray-500":
workflowRunIsRunningOrQueued ||
!workflowPermanentId ||
- debugSession === undefined,
+ debugSession === undefined ||
+ isRecording,
})}
onClick={() => {
handleOnPlay();
diff --git a/skyvern-frontend/src/store/RecordedBlocksStore.ts b/skyvern-frontend/src/store/RecordedBlocksStore.ts
index 4daa68915..336fb667e 100644
--- a/skyvern-frontend/src/store/RecordedBlocksStore.ts
+++ b/skyvern-frontend/src/store/RecordedBlocksStore.ts
@@ -1,5 +1,6 @@
import { create } from "zustand";
import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes";
+import type { WorkflowParameter } from "@/routes/workflows/types/workflowTypes";
type InsertionPoint = {
previous: string | null;
@@ -10,12 +11,16 @@ type InsertionPoint = {
type RecordedBlocksState = {
blocks: Array<WorkflowBlock> | null;
+ parameters: Array<WorkflowParameter> | null;
insertionPoint: InsertionPoint | null;
};
type RecordedBlocksStore = RecordedBlocksState & {
setRecordedBlocks: (
- blocks: Array<WorkflowBlock>,
+ data: {
+ blocks: Array<WorkflowBlock>;
+ parameters: Array<WorkflowParameter>;
+ },
insertionPoint: InsertionPoint,
) => void;
clearRecordedBlocks: () => void;
@@ -23,9 +28,10 @@ type RecordedBlocksStore = RecordedBlocksState & {
const useRecordedBlocksStore = create<RecordedBlocksStore>((set) => ({
blocks: null,
+ parameters: null,
insertionPoint: null,
- setRecordedBlocks: (blocks, insertionPoint) => {
- set({ blocks, insertionPoint });
+ setRecordedBlocks: ({ blocks, parameters }, insertionPoint) => {
+ set({ blocks, parameters, insertionPoint });
},
clearRecordedBlocks: () => {
set({ blocks: null, insertionPoint: null });
diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.ts
index 5369d0068..76802d829 100644
--- a/skyvern-frontend/src/store/useRecordingStore.ts
+++ b/skyvern-frontend/src/store/useRecordingStore.ts
@@ -81,6 +81,7 @@ export interface MessageInExfiltratedCdpEvent {
event_name: string;
params: ExfiltratedEventCdpParams;
source: "cdp";
+ timestamp: number;
}
export interface MessageInExfiltratedConsoleEvent {
@@ -88,6 +89,7 @@ export interface MessageInExfiltratedConsoleEvent {
event_name: string;
params: ExfiltratedEventConsoleParams;
source: "console";
+ timestamp: number;
}
export type MessageInExfiltratedEvent =
@@ -105,6 +107,11 @@ interface RecordingStore {
* Each chunk contains up to CHUNK_SIZE events.
*/
compressedChunks: string[];
+ /**
+ * The number of events to show the user. This elides noisy events, like
+ * `mousemove`.
+ */
+ exposedEventCount: number;
/**
* Buffer of events not yet compressed into a chunk.
*/
@@ -194,8 +201,25 @@ async function compressEventsToB64(jsonString: string): Promise<string> {
return btoa(binary);
}
+const isExposedEvent = (event: MessageInExfiltratedEvent): boolean => {
+ const exposedConsoleEventTypes = new Set(["focus", "click", "keypress"]);
+
+ if (event.source === "console") {
+ if (exposedConsoleEventTypes.has(event.params.type)) {
+ return true;
+ }
+ }
+
+ if (event.source === "cdp") {
+ return true;
+ }
+
+ return false;
+};
+
export const useRecordingStore = create<RecordingStore>((set, get) => ({
compressedChunks: [],
+ exposedEventCount: 0,
pendingEvents: [],
isCompressing: false,
isRecording: false,
@@ -204,6 +228,10 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
const state = get();
const newPendingEvents = [...state.pendingEvents, event];
+ if (isExposedEvent(event)) {
+ set({ exposedEventCount: state.exposedEventCount + 1 });
+ }
+
if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) {
const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE);
const remainingEvents = newPendingEvents.slice(CHUNK_SIZE);
@@ -241,6 +269,7 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
reset: () =>
set({
compressedChunks: [],
+ exposedEventCount: 0,
pendingEvents: [],
isCompressing: false,
isRecording: false,