Skip to content

Automate browser based workflows with AI

Browser recording front-end: live draft panel replaces editor, clean screenshots, new step kinds (#7036)

Cindy Lid10191f6ab5f
diff --git a/skyvern-frontend/src/components/BrowserStream.test.tsx b/skyvern-frontend/src/components/BrowserStream.test.tsx
index abfb0f442..85ed8943e 100644
--- a/skyvern-frontend/src/components/BrowserStream.test.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.test.tsx
@@ -80,10 +80,14 @@ const mocks = vi.hoisted(() => {
const recordingStore = {
add: vi.fn(),
+ addScreenshot: vi.fn(),
+ applyInterpretationUpdate: vi.fn(),
compressedChunks: [],
+ draftEditDepth: 0,
getEventCount: vi.fn(() => 0),
getSecondsRecording: vi.fn(() => 0),
isRecording: false,
+ manualCapturePaused: false,
pendingEvents: [],
reset: vi.fn(),
setIsRecording: vi.fn(),
@@ -120,9 +124,22 @@ vi.mock("@/store/SettingsStore", () => ({
useSettingsStore: () => mocks.settingsStore,
}));
-vi.mock("@/store/useRecordingStore", () => ({
- useRecordingStore: () => mocks.recordingStore,
-}));
+vi.mock("@/store/useRecordingStore", () => {
+ // Honor the selector: BrowserStream reads slices (e.g. state.isRecording) via
+ // useRecordingStore(selector). Ignoring the selector and returning the whole
+ // store makes primitive selectors yield the store object instead of the field
+ // value — truthy where a boolean was expected — spuriously rendering the
+ // recording UI.
+ const useRecordingStore = (
+ selector?: (state: typeof mocks.recordingStore) => unknown,
+ ) => (selector ? selector(mocks.recordingStore) : mocks.recordingStore);
+ // Also read imperatively (reset/addScreenshot/etc.) via getState().
+ useRecordingStore.getState = () => mocks.recordingStore;
+ return {
+ useRecordingStore,
+ countVisibleDraftSteps: (steps: Array<unknown> = []) => steps.length,
+ };
+});
function renderBrowserStream(props: { onActivity?: () => void } = {}) {
const queryClient = new QueryClient({
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 28198468c..4398f0e92 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -5,8 +5,9 @@ import _RFB, { type RfbEvent } from "@novnc/novnc/lib/rfb.js";
type RFB = _RFB;
const RFB = (_RFB as typeof _RFB & { default?: typeof _RFB }).default ?? _RFB;
import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons";
-import { useEffect, useState, useRef, useCallback } from "react";
+import { useEffect, useMemo, useState, useRef, useCallback } from "react";
import { useQuery } from "@tanstack/react-query";
+import { useShallow } from "zustand/react/shallow";
import { getClient } from "@/api/AxiosClient";
import {
@@ -16,29 +17,24 @@ import {
} from "@/api/types";
import { Tip } from "@/components/Tip";
import { Button } from "@/components/ui/button";
-import {
- Dialog,
- DialogClose,
- DialogContent,
- DialogDescription,
- DialogFooter,
- DialogHeader,
- DialogTitle,
- DialogTrigger,
-} from "@/components/ui/dialog";
import { toast } from "@/components/ui/use-toast";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { useRecordingElapsedSeconds } from "@/hooks/useRecordingElapsedSeconds";
import { statusIsNotFinalized } from "@/routes/tasks/types";
+import { buildOptimisticStep } from "@/routes/workflows/editor/recording/optimisticSteps";
import { useClientIdStore } from "@/store/useClientIdStore";
import {
useRecordingStore,
+ countVisibleDraftSteps,
+ type ExfiltratedEventConsoleParams,
type MessageInExfiltratedEvent,
+ type RecordingInterpretationUpdate,
} from "@/store/useRecordingStore";
import { useSettingsStore } from "@/store/SettingsStore";
import { wssBaseUrl, newWssBaseUrl, getCredentialParam } from "@/util/env";
import { copyText } from "@/util/copyText";
+import { formatRecordingClock } from "@/util/recordingClock";
import { cn } from "@/util/utils";
-import { captureRecordBrowser } from "@/util/recordBrowserTelemetry";
import {
StreamStatusPanel,
type StreamDiagnostic,
@@ -66,6 +62,14 @@ interface BrowserSession {
interface CommandBeginExfiltration {
kind: "begin-exfiltration";
+ workflow_permanent_id?: string;
+ live_interpretation_enabled?: boolean;
+ // Declares that this client understands delta interpretation updates, so the
+ // backend may send changed_steps instead of full snapshots.
+ supports_interpretation_deltas?: boolean;
+ // Per-recording id: stable across reconnects, new per recording, so the
+ // backend reuses the session on reconnect but starts fresh on a new recording.
+ recording_attempt_id?: string;
}
interface CommandCedeControl {
@@ -80,17 +84,33 @@ interface CommandTakeControl {
kind: "take-control";
}
+interface CommandRecordingCapturePause {
+ kind: "recording-capture-pause";
+}
+
+interface CommandRecordingCaptureResume {
+ kind: "recording-capture-resume";
+}
+
+interface CommandRecordingRearmCapture {
+ kind: "recording-rearm-capture";
+}
+
// a "Command" is an fire-n-forget out-message - it does not require a response
type Command =
| CommandBeginExfiltration
| CommandCedeControl
| CommandEndExfiltration
+ | CommandRecordingCapturePause
+ | CommandRecordingCaptureResume
+ | CommandRecordingRearmCapture
| CommandTakeControl;
const messageInKinds = [
"ask-for-clipboard",
"copied-text",
"exfiltrated-event",
+ "recording-interpretation-update",
] as const;
type MessageInKind = (typeof messageInKinds)[number];
@@ -104,10 +124,15 @@ interface MessageInCopiedText {
text: string;
}
+interface MessageInRecordingInterpretationUpdate extends RecordingInterpretationUpdate {
+ kind: "recording-interpretation-update";
+}
+
type MessageIn =
| MessageInCopiedText
| MessageInAskForClipboard
- | MessageInExfiltratedEvent;
+ | MessageInExfiltratedEvent
+ | MessageInRecordingInterpretationUpdate;
interface MessageOutAskForClipboardResponse {
kind: "ask-for-clipboard-response";
@@ -130,6 +155,9 @@ type Props = {
resizeTrigger?: number;
isVisible?: boolean;
isExecuting?: boolean;
+ // Hide the REC pill overlay when the recording panel is visible beside the
+ // stream (its header already shows the timer + step count).
+ hideRecordingIndicator?: boolean;
onReadyChange?: (isReady: boolean, browserSessionId: string | null) => void;
onActivity?: () => void;
// --
@@ -140,6 +168,83 @@ type RfbWithFrameUpdates = RFB & {
_framebufferUpdate?: () => boolean;
};
+/** VNC encode settings: favor fast frames when the user is driving the browser. */
+function applyVncStreamProfile(
+ rfb: RFB,
+ profile: "interactive" | "passive",
+): void {
+ if (profile === "interactive") {
+ // Low CPU per frame beats max zlib compression for click/type latency.
+ rfb.compressionLevel = 1;
+ rfb.qualityLevel = 7;
+ return;
+ }
+ rfb.compressionLevel = 2;
+ rfb.qualityLevel = 6;
+}
+
+function RecordingPill() {
+ const {
+ finishRequested,
+ manualCapturePaused,
+ draftSteps,
+ deletedStepIds,
+ exposedEventCount,
+ optimisticStepCount,
+ interpretationEnabled,
+ } = useRecordingStore(
+ useShallow((state) => ({
+ finishRequested: state.finishRequested,
+ manualCapturePaused: state.manualCapturePaused,
+ draftSteps: state.draftSteps,
+ deletedStepIds: state.deletedStepIds,
+ exposedEventCount: state.exposedEventCount,
+ optimisticStepCount: state.optimisticSteps.length,
+ interpretationEnabled: state.workflowPermanentId !== null,
+ })),
+ );
+
+ const interpretedStepCount = useMemo(
+ () => countVisibleDraftSteps(draftSteps, deletedStepIds),
+ [draftSteps, deletedStepIds],
+ );
+ const elapsedSeconds = useRecordingElapsedSeconds();
+ // Show interpreted + optimistic steps whenever interpretation is enabled, not
+ // just after the first snapshot arrives — otherwise the first step waits a
+ // backend round-trip even though the optimistic placeholder is already local.
+ const count = interpretationEnabled
+ ? interpretedStepCount + optimisticStepCount
+ : exposedEventCount;
+
+ const paused = manualCapturePaused && !finishRequested;
+
+ return (
+ <div
+ className={cn(
+ "inline-flex h-6 items-center gap-2 rounded-full border px-3 text-xs font-semibold tabular-nums",
+ paused
+ ? "border-amber-500/50 bg-amber-950 text-amber-200"
+ : "border-red-500/50 bg-red-950 text-red-200",
+ )}
+ >
+ <span
+ className={cn("h-2 w-2 rounded-full", {
+ "bg-amber-500": paused,
+ "bg-red-500": !paused,
+ "animate-pulse": !finishRequested && !paused,
+ "opacity-50": finishRequested,
+ })}
+ />
+ {finishRequested ? "FINISHING" : paused ? "PAUSED" : "REC"}{" "}
+ {formatRecordingClock(elapsedSeconds)}
+ <span className={paused ? "text-amber-400/80" : "text-red-400/80"}>
+ ·
+ </span>
+ {count}
+ </div>
+ );
+}
+
function BrowserStream({
browserSessionId = undefined,
exfiltrate = false,
@@ -150,6 +255,7 @@ function BrowserStream({
resizeTrigger,
isVisible = true,
isExecuting = false,
+ hideRecordingIndicator = false,
onReadyChange,
onActivity,
// --
@@ -240,7 +346,7 @@ function BrowserStream({
null,
);
const clientId = useClientIdStore((state) => state.clientId);
- const recordingStore = useRecordingStore();
+ const isRecording = useRecordingStore((state) => state.isRecording);
const settingsStore = useSettingsStore();
const credentialGetter = useCredentialGetter();
const isBrowserSessionAvailable =
@@ -375,6 +481,11 @@ function BrowserStream({
};
}, []);
+ // The low-latency encode profile is scoped to recording: that's where frame
+ // lag directly delays draft feedback. Other interactive live-browser streams
+ // keep the default profile to avoid a broad bandwidth/CPU bump.
+ const vncInteractive = exfiltrate;
+
// vnc socket
useEffect(
() => {
@@ -442,6 +553,7 @@ function BrowserStream({
const rfb = new RFB(canvas, vncUrl);
rfb.scaleViewport = true;
+ applyVncStreamProfile(rfb, vncInteractive ? "interactive" : "passive");
const frameUpdateRfb = rfb as RfbWithFrameUpdates;
// noVNC does not expose a public framebuffer-update event in 1.5.x.
@@ -527,6 +639,17 @@ function BrowserStream({
],
);
+ // Re-apply encode profile when recording starts without tearing down the socket.
+ useEffect(() => {
+ if (!rfbRef.current) {
+ return;
+ }
+ applyVncStreamProfile(
+ rfbRef.current,
+ vncInteractive ? "interactive" : "passive",
+ );
+ }, [vncInteractive]);
+
useEffect(() => {
if (!showStream || !canvasContainer || !runId) {
return;
@@ -701,6 +824,13 @@ function BrowserStream({
}
}, [task, workflow]);
+ const { workflowPermanentId, recordingAttemptId } = useRecordingStore(
+ useShallow((state) => ({
+ workflowPermanentId: state.workflowPermanentId,
+ recordingAttemptId: state.recordingAttemptId,
+ })),
+ );
+
// effect for exfiltration
useEffect(() => {
const sendCommand = (command: Command) => {
@@ -711,10 +841,56 @@ function BrowserStream({
messageSocket.send(JSON.stringify(command));
};
- sendCommand({
- kind: exfiltrate ? "begin-exfiltration" : "end-exfiltration",
- });
- }, [exfiltrate, messageSocket]);
+ if (exfiltrate) {
+ // Including a workflow id turns on backend live interpretation: the
+ // server streams recording-interpretation-update draft step snapshots
+ // back over this same socket while the user records.
+ sendCommand({
+ kind: "begin-exfiltration",
+ workflow_permanent_id: workflowPermanentId ?? undefined,
+ live_interpretation_enabled: Boolean(workflowPermanentId),
+ // Client capability: the backend only emits per-step delta updates
+ // (changed_steps upserted by step_id) to clients that declare support;
+ // otherwise it falls back to full snapshots.
+ supports_interpretation_deltas: true,
+ // Stable across reconnects of this recording; a new recording mints a
+ // new id so the backend starts a fresh interpretation session.
+ recording_attempt_id: recordingAttemptId ?? undefined,
+ });
+ } else {
+ sendCommand({ kind: "end-exfiltration" });
+ }
+ }, [exfiltrate, messageSocket, workflowPermanentId, recordingAttemptId]);
+
+ const manualCapturePaused = useRecordingStore(
+ (state) => state.manualCapturePaused,
+ );
+ const draftEditDepth = useRecordingStore((state) => state.draftEditDepth);
+ const capturePaused = manualCapturePaused || draftEditDepth > 0;
+ const previousCapturePausedRef = useRef(false);
+
+ // Pause exfiltration + live interpretation while the operator edits drafts
+ // or explicitly pauses capture.
+ useEffect(() => {
+ if (!exfiltrate || !messageSocket) {
+ // Backend pause state is per exfiltration session, so start the next
+ // session's edge detection from "not paused". A recording that ended
+ // while paused would otherwise fire a spurious resume on the next start;
+ // and if capture IS paused on a mid-recording reconnect, this re-sends
+ // the pause to the new socket (idempotent) instead of assuming it.
+ previousCapturePausedRef.current = false;
+ return;
+ }
+
+ const wasPaused = previousCapturePausedRef.current;
+ if (!wasPaused && capturePaused) {
+ messageSocket.send(JSON.stringify({ kind: "recording-capture-pause" }));
+ } else if (wasPaused && !capturePaused) {
+ messageSocket.send(JSON.stringify({ kind: "recording-capture-resume" }));
+ }
+
+ previousCapturePausedRef.current = capturePaused;
+ }, [capturePaused, exfiltrate, messageSocket]);
useEffect(() => {
if (!interactive) {
@@ -726,10 +902,10 @@ function BrowserStream({
// interactive), a prior grab must be released or its input keeps flowing.
// Recording is exempt: it holds take-control for exfiltration.
useEffect(() => {
- if (!interactive && !showControlButtons && !recordingStore.isRecording) {
+ if (!interactive && !showControlButtons && !isRecording) {
setUserIsControlling(false);
}
- }, [interactive, showControlButtons, recordingStore.isRecording]);
+ }, [interactive, showControlButtons, isRecording]);
const theUserIsControlling =
userIsControlling || (interactive && !showControlButtons);
@@ -757,18 +933,21 @@ function BrowserStream({
};
}, [canvasContainer]);
- // effect to ensure the recordingStore is reset when the component unmounts
+ // Reset the recording store when the stream unmounts — including mid-recording.
+ // The stream never unmounts during a live recording flow (the studio keeps it
+ // in a detached host across pane changes; the editor keeps it mounted while
+ // recording), so an unmount means the user abandoned the session (navigated
+ // away / switched workflows) and stale isRecording state would otherwise leak
+ // into the next mounted workflow as a stuck recording panel.
useEffect(() => {
return () => {
- recordingStore.reset();
+ useRecordingStore.getState().reset();
};
- // eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
- // effect to ensure 'take-control' is sent on the rising edge of
- // recordingStore.isRecording
+ // effect to ensure 'take-control' is sent on the rising edge of isRecording
useEffect(() => {
- if (!recordingStore.isRecording) {
+ if (!isRecording) {
return;
}
@@ -786,7 +965,7 @@ function BrowserStream({
sendCommand({ kind: "take-control" });
setUserIsControlling(true);
- }, [recordingStore.isRecording, isMessageConnected, messageSocket]);
+ }, [isRecording, isMessageConnected, messageSocket]);
/**
* TODO(jdo): could use zod or smth similar
@@ -851,6 +1030,38 @@ function BrowserStream({
}
break;
}
+ case "recording-interpretation-update": {
+ // steps is optional: a delta update carries changed_steps instead. Only
+ // session_revision is required to accept the message.
+ if (
+ "session_revision" in data &&
+ typeof data.session_revision === "number"
+ ) {
+ const update = data as MessageInRecordingInterpretationUpdate;
+ return {
+ kind: "recording-interpretation-update",
+ interpretation_session_id:
+ typeof update.interpretation_session_id === "string"
+ ? update.interpretation_session_id
+ : "",
+ session_revision: update.session_revision,
+ steps: Array.isArray(update.steps) ? update.steps : [],
+ changed_steps: Array.isArray(update.changed_steps)
+ ? update.changed_steps
+ : [],
+ // Absent/true => full snapshot (legacy). Only false triggers delta merge.
+ is_snapshot:
+ typeof update.is_snapshot === "boolean"
+ ? update.is_snapshot
+ : true,
+ pending:
+ typeof update.pending === "boolean" ? update.pending : false,
+ finalized:
+ typeof update.finalized === "boolean" ? update.finalized : false,
+ };
+ }
+ break;
+ }
default: {
const _exhaustive: never = kind;
return _exhaustive;
@@ -858,6 +1069,34 @@ function BrowserStream({
}
};
+ // Best-effort frame grab from the VNC canvas at the moment of a recorded
+ // click, so the live feed can show a zoomed shot at the action point.
+ const captureRecordingScreenshot = (
+ params: ExfiltratedEventConsoleParams,
+ ) => {
+ const schedule =
+ typeof requestIdleCallback === "function"
+ ? (fn: () => void) => requestIdleCallback(fn, { timeout: 750 })
+ : (fn: () => void) => window.setTimeout(fn, 0);
+
+ schedule(() => {
+ try {
+ const canvas = canvasContainer?.querySelector("canvas");
+ if (!canvas) {
+ return;
+ }
+ useRecordingStore.getState().addScreenshot({
+ timestampMs: params.timestamp,
+ dataUrl: canvas.toDataURL("image/jpeg", 0.5),
+ xp: params.mousePosition.xp,
+ yp: params.mousePosition.yp,
+ });
+ } catch {
+ // toDataURL can throw on a tainted/headless canvas; shots are optional
+ }
+ });
+ };
+
const handleMessage = (data: unknown, ws: WebSocket | null) => {
const message = getMessage(data);
@@ -938,7 +1177,52 @@ function BrowserStream({
break;
}
case "exfiltrated-event": {
- recordingStore.add(message);
+ // Read store state fresh: this handler is attached once per socket
+ // and would otherwise see a stale isRecording.
+ const store = useRecordingStore.getState();
+ if (!store.isRecording && !store.finishRequested) {
+ break;
+ }
+ if (store.isCapturePaused()) {
+ break;
+ }
+ if (
+ store.isRecording &&
+ message.source === "console" &&
+ message.params.type === "click"
+ ) {
+ captureRecordingScreenshot(message.params);
+ }
+ if (
+ store.isRecording &&
+ !store.finishRequested &&
+ message.source === "cdp" &&
+ (message.event_name === "nav:frame_navigated" ||
+ message.event_name === "nav:navigated_within_document")
+ ) {
+ if (ws) {
+ ws.send(JSON.stringify({ kind: "recording-rearm-capture" }));
+ }
+ }
+ if (store.isRecording && !store.finishRequested) {
+ const optimistic = buildOptimisticStep(message);
+ if (optimistic) {
+ store.addOptimisticStep(optimistic);
+ }
+ }
+ store.add(message);
+ break;
+ }
+ case "recording-interpretation-update": {
+ useRecordingStore.getState().applyInterpretationUpdate({
+ interpretation_session_id: message.interpretation_session_id,
+ session_revision: message.session_revision,
+ steps: message.steps,
+ changed_steps: message.changed_steps,
+ is_snapshot: message.is_snapshot,
+ pending: message.pending,
+ finalized: message.finalized,
+ });
break;
}
default: {
@@ -1034,84 +1318,25 @@ function BrowserStream({
)}
</div>
)}
- {recordingStore.isRecording && (
- <>
- <div className="pointer-events-none absolute flex aspect-video w-full items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-red-500 animate-in fade-in">
+ {isRecording && (
+ <div className="pointer-events-none absolute flex aspect-video w-full items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-red-500 animate-in fade-in">
+ {/* The pill duplicates the recording panel's header (timer + step
+ count), so it's hidden while the panel is visible alongside. */}
+ {!hideRecordingIndicator && (
<div className="relative h-full w-full">
- <div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2 text-red-500">
- <div className="truncate">Browser is recording</div>
- <Tip content="To finish and turn your actions into blocks, click the stop icon.">
- <div className="cursor-pointer">
+ <div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2">
+ <RecordingPill />
+ <Tip content="Your actions appear as blocks in the recording panel. Finish with Done, or use the trash icon to discard.">
+ <div className="cursor-pointer text-red-500">
<InfoCircledIcon />
</div>
</Tip>
- <Dialog>
- <DialogTrigger asChild>
- <Button
- className="ml-auto cursor-pointer"
- size="sm"
- variant="destructive"
- style={{
- marginTop: "-0.5rem",
- }}
- onClick={(e) => {
- const hasEvents =
- recordingStore.pendingEvents.length > 0 ||
- recordingStore.compressedChunks.length > 0;
- if (!hasEvents) {
- e.preventDefault();
- captureRecordBrowser("record_browser.cancelled", {
- event_count_at_cancel:
- recordingStore.getEventCount(),
- seconds_recording:
- recordingStore.getSecondsRecording(),
- });
- recordingStore.setIsRecording(false);
- recordingStore.reset();
- }
- }}
- >
- cancel
- </Button>
- </DialogTrigger>
- <DialogContent>
- <DialogHeader>
- <DialogTitle>Cancel recording?</DialogTitle>
- <DialogDescription>
- You have recorded events that will be lost if you
- cancel. Are you sure you want to cancel the recording?
- </DialogDescription>
- </DialogHeader>
- <DialogFooter>
- <DialogClose asChild>
- <Button variant="outline">Keep recording</Button>
- </DialogClose>
- <DialogClose asChild>
- <Button
- variant="destructive"
- onClick={() => {
- captureRecordBrowser("record_browser.cancelled", {
- event_count_at_cancel:
- recordingStore.getEventCount(),
- seconds_recording:
- recordingStore.getSecondsRecording(),
- });
- recordingStore.setIsRecording(false);
- recordingStore.reset();
- }}
- >
- Cancel recording
- </Button>
- </DialogClose>
- </DialogFooter>
- </DialogContent>
- </Dialog>
</div>
</div>
- </div>
- </>
+ )}
+ </div>
)}
- {isExecuting && !recordingStore.isRecording && (
+ {isExecuting && !isRecording && (
<div className="pointer-events-none absolute flex aspect-video w-full animate-glow items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-yellow-500">
<div className="relative h-full w-full">
<div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2 text-yellow-500">
diff --git a/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.test.ts b/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.test.ts
new file mode 100644
index 000000000..3f70674e7
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.test.ts
@@ -0,0 +1,52 @@
+import { act, renderHook } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { useRecordingStore } from "@/store/useRecordingStore";
+
+import { useRecordingElapsedSeconds } from "./useRecordingElapsedSeconds";
+
+const BASE_MS = 1_700_000_000_000;
+
+describe("useRecordingElapsedSeconds", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.setSystemTime(BASE_MS);
+ useRecordingStore.setState({
+ recordingStartedAtMs: BASE_MS,
+ manualCapturePaused: false,
+ finishRequested: false,
+ });
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("counts up while recording", () => {
+ const { result } = renderHook(() => useRecordingElapsedSeconds());
+ expect(result.current).toBe(0);
+ act(() => vi.advanceTimersByTime(3000));
+ expect(Math.floor(result.current)).toBe(3);
+ });
+
+ it("freezes while paused and resumes without jumping the clock", () => {
+ const { result } = renderHook(() => useRecordingElapsedSeconds());
+ act(() => vi.advanceTimersByTime(3000));
+ expect(Math.floor(result.current)).toBe(3);
+
+ act(() => {
+ useRecordingStore.setState({ manualCapturePaused: true });
+ });
+ act(() => vi.advanceTimersByTime(5000));
+ // Frozen: the 5s spent paused must not count.
+ expect(Math.floor(result.current)).toBe(3);
+
+ act(() => {
+ useRecordingStore.setState({ manualCapturePaused: false });
+ });
+ // Resume must not jump forward by the paused span.
+ expect(Math.floor(result.current)).toBe(3);
+ act(() => vi.advanceTimersByTime(2000));
+ expect(Math.floor(result.current)).toBe(5);
+ });
+});
diff --git a/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.ts b/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.ts
new file mode 100644
index 000000000..d0dcdf234
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.ts
@@ -0,0 +1,55 @@
+import { useEffect, useRef, useState } from "react";
+
+import { useRecordingStore } from "@/store/useRecordingStore";
+
+/**
+ * Elapsed recording time in seconds, excluding any time spent manually paused.
+ * The displayed clock freezes while `manualCapturePaused` is set and resumes
+ * from where it left off — paused spans are folded into an accumulator rather
+ * than counted, so resume never jumps the clock forward.
+ */
+export function useRecordingElapsedSeconds(): number {
+ const startedAtMs = useRecordingStore((state) => state.recordingStartedAtMs);
+ const paused = useRecordingStore((state) => state.manualCapturePaused);
+ const finishRequested = useRecordingStore((state) => state.finishRequested);
+
+ const [nowMs, setNowMs] = useState(() => Date.now());
+ const pausedAccumMsRef = useRef(0);
+ const pauseStartedAtMsRef = useRef<number | null>(null);
+
+ // A new recording (new start timestamp) resets the paused accumulator.
+ useEffect(() => {
+ pausedAccumMsRef.current = 0;
+ pauseStartedAtMsRef.current = null;
+ }, [startedAtMs]);
+
+ useEffect(() => {
+ if (finishRequested) {
+ return;
+ }
+
+ if (paused) {
+ // Freeze: mark when the pause began and stop ticking.
+ if (pauseStartedAtMsRef.current === null) {
+ pauseStartedAtMsRef.current = Date.now();
+ setNowMs(Date.now());
+ }
+ return;
+ }
+
+ // Resuming: fold the just-ended pause into the accumulator so it is not
+ // counted, then keep ticking.
+ if (pauseStartedAtMsRef.current !== null) {
+ pausedAccumMsRef.current += Date.now() - pauseStartedAtMsRef.current;
+ pauseStartedAtMsRef.current = null;
+ }
+ setNowMs(Date.now());
+ const interval = setInterval(() => setNowMs(Date.now()), 1000);
+ return () => clearInterval(interval);
+ }, [paused, finishRequested]);
+
+ if (!startedAtMs) {
+ return 0;
+ }
+ return (nowMs - startedAtMs - pausedAccumMsRef.current) / 1000;
+}
diff --git a/skyvern-frontend/src/novnc.d.ts b/skyvern-frontend/src/novnc.d.ts
index 813f4cc5d..0e4d1ade0 100644
--- a/skyvern-frontend/src/novnc.d.ts
+++ b/skyvern-frontend/src/novnc.d.ts
@@ -29,6 +29,10 @@ declare module "@novnc/novnc/lib/rfb.js" {
_display: RfbDisplay;
resizeSession: boolean;
scaleViewport: boolean;
+ // 0–9; JPEG quality for Tight-encoded regions (default 6).
+ qualityLevel: number;
+ // 0–9; zlib compression for stream data (default 2).
+ compressionLevel: number;
constructor(target: HTMLElement, url: string, options?: RFBOptions);
addEventListener(event: string, listener: (e: RfbEvent) => void): void;
diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
index 995cb4eb8..10a1178f1 100644
--- a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
@@ -5,7 +5,10 @@ import { useRef } from "react";
import { getClient } from "@/api/AxiosClient";
import { toast } from "@/components/ui/use-toast";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
-import { useRecordingStore } from "@/store/useRecordingStore";
+import {
+ useRecordingStore,
+ type RecordingDraftStep,
+} from "@/store/useRecordingStore";
import {
type WorkflowBlock,
type WorkflowParameter,
@@ -33,7 +36,18 @@ const useProcessRecordingMutation = ({
const mutationStartedAtRef = useRef<number | null>(null);
const processRecordingMutation = useMutation({
- mutationFn: async () => {
+ mutationFn: async (
+ variables: {
+ /**
+ * Live-interpreted draft steps (with user edits/deletes applied).
+ * When provided the backend converts them deterministically instead
+ * of re-processing the raw event stream.
+ */
+ draftSteps?: Array<RecordingDraftStep> | null;
+ } | void,
+ ) => {
+ const draftSteps = variables?.draftSteps ?? null;
+
if (!browserSessionId) {
throw new Error(
"Cannot process recording without a valid browser session ID.",
@@ -49,8 +63,9 @@ const useProcessRecordingMutation = ({
mutationStartedAtRef.current = Date.now();
const eventCount = recordingStore.getEventCount();
+ const hasDraftSteps = (draftSteps?.length ?? 0) > 0;
- if (eventCount === 0) {
+ if (eventCount === 0 && !hasDraftSteps) {
captureRecordBrowser("record_browser.empty_blocked", {
seconds_recording: recordingStore.getSecondsRecording(),
});
@@ -62,12 +77,16 @@ const useProcessRecordingMutation = ({
captureRecordBrowser("record_browser.process_attempted", {
event_count: recordingStore.getEventCount(),
compressed_chunk_count: compressedChunks.length,
+ draft_step_count: draftSteps?.length,
});
const client = await getClient(credentialGetter, "sans-api-v1");
return client
.post<
- { compressed_chunks: string[] },
+ {
+ compressed_chunks: string[];
+ draft_steps?: Array<RecordingDraftStep>;
+ },
{
data: {
blocks: Array<WorkflowBlock>;
@@ -77,6 +96,7 @@ const useProcessRecordingMutation = ({
>(`/browser_sessions/${browserSessionId}/process_recording`, {
compressed_chunks: compressedChunks,
workflow_permanent_id: workflowPermanentId,
+ ...(draftSteps !== null ? { draft_steps: draftSteps } : {}),
})
.then((response) => ({
blocks: response.data.blocks,
@@ -112,6 +132,11 @@ const useProcessRecordingMutation = ({
return;
}
+ // A zero-block commit still ends the session: the caller's onSuccess (which
+ // normally exits recording after landing blocks) is skipped, and without
+ // this the user is stranded in the recording panel with a dead Done button.
+ recordingStore.setIsRecording(false);
+
toast({
variant: "warning",
title: "Recording Processed (No Blocks)",
@@ -141,6 +166,8 @@ const useProcessRecordingMutation = ({
latency_ms: latencyMs,
});
+ useRecordingStore.setState({ finishRequested: false });
+
toast({
variant: "destructive",
title: "Error Processing Recording",
diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
index 51bef2b3c..ff14a4782 100644
--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
@@ -14,7 +14,6 @@ import { cn } from "@/util/utils";
import { useShouldNotifyWhenClosingTab } from "@/hooks/useShouldNotifyWhenClosingTab";
import { BlockActionContext } from "@/store/BlockActionContext";
import { useDebugStore } from "@/store/useDebugStore";
-import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
import {
useWorkflowHasChangesStore,
useWorkflowSave,
@@ -123,7 +122,6 @@ import {
import "./reactFlowOverrideStyles.css";
import {
convertEchoParameters,
- convertToNode,
createNode,
descendants,
generateNodeLabel,
@@ -719,16 +717,6 @@ function FlowRenderer({
const setGetSaveDataRef = useRef(workflowChangesStore.setGetSaveData);
setGetSaveDataRef.current = workflowChangesStore.setGetSaveData;
const saveWorkflow = useWorkflowSave({ status: "published" });
- const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);
- const recordedParameters = useRecordedBlocksStore(
- (state) => state.parameters,
- );
- const recordedInsertionPoint = useRecordedBlocksStore(
- (state) => state.insertionPoint,
- );
- const clearRecordedBlocks = useRecordedBlocksStore(
- (state) => state.clearRecordedBlocks,
- );
useShouldNotifyWhenClosingTab(!readOnly && workflowChangesStore.hasChanges);
const blocker = useBlocker(({ currentLocation, nextLocation }) => {
return (
@@ -772,7 +760,6 @@ function FlowRenderer({
useEffect(() => {
if (nodesInitialized && !hasCompletedInitialLoad.current) {
- hasCompletedInitialLoad.current = true;
doLayout(nodes, edges);
// After Dagre computes positions, wait one frame for the DOM to update
// with new positions, then fade in the nodes/edges at their final positions.
@@ -781,6 +768,11 @@ function FlowRenderer({
// After the fade-in animation completes, enable normal transform transitions
fadeTimerRef.current = setTimeout(() => {
setLayoutPhase("ready");
+ // Mark complete only at the terminal phase. Setting it earlier would
+ // strand layoutPhase at "pre-layout" (stuck logo loader) if this async
+ // advance is cancelled by a nodesInitialized re-flip — e.g. the
+ // recording flow toggling editor visibility or injecting fresh nodes.
+ hasCompletedInitialLoad.current = true;
}, 350);
});
return () => {
@@ -1214,118 +1206,6 @@ function FlowRenderer({
doLayout(nodes, edges);
}
- // effect to add new blocks that were generated from a browser recording,
- // along with any new parameters
- useEffect(() => {
- if (readOnly) {
- return;
- }
- if (!recordedBlocks || !recordedInsertionPoint) {
- return;
- }
-
- const { previous, next, parent, connectingEdgeType } =
- recordedInsertionPoint;
-
- const newNodes: Array<AppNode> = [];
- const newEdges: Array<Edge> = [];
-
- let existingLabels = nodes
- .filter(isWorkflowBlockNode)
- .map((node) => node.data.label);
-
- let prevNodeId = previous;
-
- // convert each WorkflowBlock to an AppNode
- recordedBlocks.forEach((block, index) => {
- const id = nanoid();
- const label = generateNodeLabel(existingLabels);
- existingLabels = [...existingLabels, label];
- const blockWithLabel = { ...block, label: block.label || label };
-
- const node = convertToNode(
- { id, parentId: parent },
- blockWithLabel,
- true,
- );
- newNodes.push(node);
-
- // create edge from previous node to this one
- if (prevNodeId) {
- newEdges.push({
- id: nanoid(),
- type: "edgeWithAddButton",
- source: prevNodeId,
- target: id,
- style: { strokeWidth: 2 },
- });
- }
-
- // if this is the last block, connect to next
- if (index === recordedBlocks.length - 1 && next) {
- newEdges.push({
- id: nanoid(),
- type: connectingEdgeType,
- source: id,
- target: next,
- style: { strokeWidth: 2 },
- });
- }
-
- prevNodeId = id;
- });
-
- const editedEdges = previous
- ? edges.filter((edge) => edge.source !== previous)
- : edges;
-
- const previousNode = nodes.find((node) => node.id === previous);
- const previousNodeIndex = previousNode
- ? nodes.indexOf(previousNode)
- : nodes.length - 1;
-
- const newNodesAfter = [
- ...nodes.slice(0, previousNodeIndex + 1),
- ...newNodes,
- ...nodes.slice(previousNodeIndex + 1),
- ];
-
- workflowChangesStore.setHasChanges(true);
- doLayout(newNodesAfter, [...editedEdges, ...newEdges]);
-
- const newParameters = Array<ParametersState[number]>();
-
- for (const newParameter of recordedParameters ?? []) {
- const exists = parameters.some((param) => param.key === newParameter.key);
-
- if (!exists) {
- newParameters.push({
- key: newParameter.key,
- parameterType: "workflow",
- dataType: newParameter.workflow_parameter_type,
- description: newParameter.description ?? null,
- defaultValue: newParameter.default_value ?? "",
- });
- }
- }
-
- if (newParameters.length > 0) {
- const workflowParametersStore = useWorkflowParametersStore.getState();
- workflowParametersStore.setParameters([
- ...workflowParametersStore.parameters,
- ...newParameters,
- ]);
- }
-
- clearRecordedBlocks();
- // Effect runs strictly when the recording store flips a new
- // (blocks, insertionPoint) pair into place. nodes/edges/parameters etc.
- // are read from the latest closure inside the effect body; including
- // them in deps would re-fire on every editor edit and re-apply the
- // recorded blocks.
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [recordedBlocks, recordedInsertionPoint]);
-
const editorElementRef = useRef<HTMLDivElement>(null);
// A collapse/accordion animation settles the node at a new height; the layout
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index 45d87d683..80e60b5b7 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -91,6 +91,8 @@ import {
import { toast } from "@/components/ui/use-toast";
import { DeleteConfirmationDialog } from "@/components/DeleteConfirmationDialog";
import { BrowserStream } from "@/components/BrowserStream";
+import { RecordingPanel } from "@/routes/workflows/editor/recording/RecordingPanel";
+import { useApplyRecordedBlocks } from "@/routes/workflows/editor/recording/useApplyRecordedBlocks";
import { statusIsFinalized } from "@/routes/tasks/types.ts";
import { CodeEditor } from "@/routes/workflows/components/CodeEditor";
import { DebuggerRun } from "@/routes/workflows/debugger/DebuggerRun";
@@ -169,6 +171,7 @@ import { WorkflowCopilotChat } from "../copilot/WorkflowCopilotChat";
import { useStudioRunId } from "../studio/useStudioRunId";
import { copilotRunId } from "./copilotRunId";
import { useStudioShellContext } from "../studio/StudioShellContext";
+import { useRecordingLauncherStore } from "@/store/useRecordingLauncherStore";
import { useStudioPanes } from "../studio/useStudioPanes";
import { WorkflowCopilotButton } from "../copilot/WorkflowCopilotButton";
import { resolveCopilotLiveBrowserReady } from "../copilot/browserReadiness";
@@ -1241,6 +1244,63 @@ function Workspace({
[setNodes, setEdges, blockLabel],
);
+ useApplyRecordedBlocks({
+ // Recording runs in the debugger (build + showBrowser), not only on /edit.
+ enabled: !workflowPanelState.data?.showComparison,
+ nodes,
+ edges,
+ doLayout,
+ });
+
+ // Studio entry point for recording: the Browser-tab Record button lives outside
+ // the canvas, so the canvas-aware Workspace registers a launcher that resolves
+ // the append-at-end insertion point (the trailing top-level NodeAdder, same as
+ // clicking its "+") and starts recording. MVP only appends at the end.
+ const setStartRecordingAtEnd = useRecordingLauncherStore(
+ (s) => s.setStartRecordingAtEnd,
+ );
+ // Stable action ref (vs the whole-store `recordingStore` object, which gets a
+ // new reference on every store write and would re-register the launcher).
+ const setIsRecording = useRecordingStore((s) => s.setIsRecording);
+ const startRecordingAtEnd = useCallback(() => {
+ const currentNodes = getNodes() as Array<AppNode>;
+ const currentEdges = getEdges();
+ const trailingAdder = currentNodes.find(
+ (node) => node.type === "nodeAdder" && !node.parentId,
+ );
+ const incomingEdge = trailingAdder
+ ? currentEdges.find((edge) => edge.target === trailingAdder.id)
+ : undefined;
+ setWorkflowPanelState({
+ active: false,
+ content: "nodeLibrary",
+ data: {
+ previous: incomingEdge?.source ?? null,
+ next: trailingAdder?.id ?? null,
+ parent: undefined,
+ connectingEdgeType: "default",
+ },
+ });
+ setIsRecording(true, {
+ workflowPermanentId: workflowPermanentId ?? null,
+ browserSessionId: liveBrowserSessionId,
+ });
+ }, [
+ getNodes,
+ getEdges,
+ setWorkflowPanelState,
+ setIsRecording,
+ workflowPermanentId,
+ liveBrowserSessionId,
+ ]);
+ useEffect(() => {
+ if (!embedded) {
+ return;
+ }
+ setStartRecordingAtEnd(startRecordingAtEnd);
+ return () => setStartRecordingAtEnd(null);
+ }, [embedded, startRecordingAtEnd, setStartRecordingAtEnd]);
+
// Listen for conditional branch changes to trigger re-layout
useEffect(() => {
const handleBranchChange = () => {
@@ -2189,7 +2249,7 @@ function Workspace({
split={{ left: workflowWidth }}
onResize={() => setContainerResizeTrigger((prev) => prev + 1)}
>
- {/* code, infinite canvas, and block runs */}
+ {/* code + canvas; recording overlays the panel without unmounting FlowRenderer */}
<div className="relative h-full w-full">
<div
className={cn(
@@ -2201,7 +2261,9 @@ function Workspace({
{
"translate-x-0": showAllCode,
},
+ recordingStore.isRecording && "pointer-events-none invisible",
)}
+ aria-hidden={recordingStore.isRecording}
ref={dom.splitLeft}
>
{/* code */}
@@ -2261,6 +2323,14 @@ function Workspace({
)}
</div>
</div>
+ {/* In Studio (embedded) the shell owns the single RecordingPanel
+ in the copilot pane; rendering one here too would mount two
+ panels that each fire their own commit. */}
+ {!embedded && recordingStore.isRecording && (
+ <div className="absolute inset-0 z-20 h-full px-6 pb-4 pt-[8.5rem]">
+ <RecordingPanel browserSessionId={liveBrowserSessionId} />
+ </div>
+ )}
</div>
<div className="skyvern-split-right relative flex h-full items-end justify-center bg-neutral-50 p-4 pl-6 dark:bg-background">
@@ -2327,13 +2397,19 @@ function Workspace({
}}
/>
)
- ) : isFlowCanvasReady ? (
+ ) : isFlowCanvasReady || recordingStore.isRecording ? (
<BrowserStream
key={liveBrowserSessionId}
- exfiltrate={recordingStore.isRecording}
+ exfiltrate={
+ recordingStore.isRecording &&
+ !recordingStore.finishRequested
+ }
interactive={true}
browserSessionId={liveBrowserSessionId}
showControlButtons={true}
+ // The recording panel overlays the canvas whenever a
+ // recording is live here, so the REC pill is redundant.
+ hideRecordingIndicator={true}
resizeTrigger={windowResizeTrigger}
isExecuting={!!workflowRun && !isFinalized}
onReadyChange={handleLiveBrowserReadyChange}
@@ -2416,7 +2492,7 @@ function Workspace({
}}
/>
)
- ) : isFlowCanvasReady ? (
+ ) : isFlowCanvasReady || recordingStore.isRecording ? (
<BrowserSessionStream
browserSessionId={liveBrowserSessionId}
interactive={true}
diff --git a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
index 3561b9bb5..0953a8574 100644
--- a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
@@ -97,7 +97,8 @@ function EdgeWithAddButton({
// Derive upload state directly from mutation to avoid race conditions
const isUploadingSOP = sopToBlocksMutation.isPending;
- const isProcessing = processRecordingMutation.isPending;
+ const isProcessing =
+ processRecordingMutation.isPending || recordingStore.isCommitting;
const isBusy =
(isProcessing || recordingStore.isRecording) &&
@@ -151,6 +152,13 @@ function EdgeWithAddButton({
};
const onEndRecord = () => {
+ // With live interpretation, the recording panel owns the commit so user
+ // edits/deletes to draft steps are honored.
+ if (recordingStore.isRecording && recordingStore.sessionRevision > 0) {
+ recordingStore.requestFinish();
+ return;
+ }
+
if (recordingStore.isRecording) {
recordingStore.setIsRecording(false);
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
index 217a534ff..8b6105cc8 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
@@ -113,7 +113,8 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
// Derive upload state directly from mutation to avoid race conditions
const isUploadingSOP = sopToBlocksMutation.isPending;
- const isProcessing = processRecordingMutation.isPending;
+ const isProcessing =
+ processRecordingMutation.isPending || recordingStore.isCommitting;
const isBusy =
(isProcessing || recordingStore.isRecording) &&
@@ -180,6 +181,13 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
};
const onEndRecord = () => {
+ // With live interpretation, the recording panel owns the commit so user
+ // edits/deletes to draft steps are honored.
+ if (recordingStore.isRecording && recordingStore.sessionRevision > 0) {
+ recordingStore.requestFinish();
+ return;
+ }
+
if (recordingStore.isRecording) {
recordingStore.setIsRecording(false);
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx
new file mode 100644
index 000000000..0bb2a9d12
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx
@@ -0,0 +1,659 @@
+import {
+ CheckIcon,
+ CursorArrowIcon,
+ EnterFullScreenIcon,
+ PauseIcon,
+ Pencil1Icon,
+ PlayIcon,
+ TrashIcon,
+ ZoomInIcon,
+} from "@radix-ui/react-icons";
+import { useEffect, useMemo, useRef, useState } from "react";
+import { useShallow } from "zustand/react/shallow";
+
+import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import { useRecordingElapsedSeconds } from "@/hooks/useRecordingElapsedSeconds";
+import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";
+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
+import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
+import {
+ applyDraftStepOverlays,
+ findScreenshotForStep,
+ useRecordingStore,
+ type RecordingActionKind,
+ type RecordingDraftStep,
+ type RecordingScreenshot,
+} from "@/store/useRecordingStore";
+import { captureRecordBrowser } from "@/util/recordBrowserTelemetry";
+import { formatRecordingClock } from "@/util/recordingClock";
+import { cn } from "@/util/utils";
+import { buildDraftStepTitlePatch } from "./recordingDraftStepEdits";
+
+const KIND_LABELS: Record<RecordingActionKind, string> = {
+ click: "Click",
+ hover: "Hover",
+ input_text: "Input text",
+ url_change: "Navigation",
+ wait: "Wait",
+};
+
+/**
+ * How long Done waits for the backend's finalized interpretation snapshot
+ * (flushed on end-exfiltration) before committing whatever drafts we have.
+ */
+const FINALIZE_TIMEOUT_MS = 5000;
+
+function shortUrl(url: string | null | undefined): string {
+ if (!url) {
+ return "";
+ }
+ try {
+ const parsed = new URL(url);
+ return parsed.host + (parsed.pathname === "/" ? "" : parsed.pathname);
+ } catch {
+ return url;
+ }
+}
+
+function formatDraftStepDisplayTitle(step: RecordingDraftStep): string {
+ if (step.title?.trim()) {
+ return step.title.trim();
+ }
+
+ if (step.action_kind === "url_change" || step.block_type === "goto_url") {
+ const destination = shortUrl(step.url);
+ if (destination) {
+ return `Go to ${destination}`;
+ }
+ }
+
+ if (step.label.startsWith("goto_")) {
+ const slug = step.label.slice("goto_".length).replace(/_/g, ".");
+ return slug ? `Go to ${slug}` : "Go to page";
+ }
+
+ return step.label;
+}
+
+function StepScreenshot({ screenshot }: { screenshot: RecordingScreenshot }) {
+ const [showFullPage, setShowFullPage] = useState(false);
+ const xp = screenshot.xp ?? 0.5;
+ const yp = screenshot.yp ?? 0.5;
+
+ return (
+ <div className="group/shot relative h-36 overflow-hidden rounded-md border bg-black">
+ <div
+ className="absolute inset-0"
+ style={
+ showFullPage
+ ? {
+ backgroundImage: `url(${screenshot.dataUrl})`,
+ backgroundSize: "contain",
+ backgroundRepeat: "no-repeat",
+ backgroundPosition: "center",
+ }
+ : {
+ // background-position p% pins image point p% to container
+ // point p%, keeping the zoomed crop in-bounds with the click
+ // point at the same relative spot as the ring below.
+ backgroundImage: `url(${screenshot.dataUrl})`,
+ backgroundSize: "250% auto",
+ backgroundRepeat: "no-repeat",
+ backgroundPosition: `${xp * 100}% ${yp * 100}%`,
+ }
+ }
+ />
+ <button
+ type="button"
+ className="absolute bottom-2 right-2 inline-flex items-center gap-1 rounded bg-black/70 px-2 py-1 text-[10.5px] text-slate-200 opacity-0 transition-opacity group-hover/shot:opacity-100"
+ onClick={() => setShowFullPage(!showFullPage)}
+ >
+ {showFullPage ? (
+ <>
+ <ZoomInIcon className="h-3 w-3" />
+ Zoom to action
+ </>
+ ) : (
+ <>
+ <EnterFullScreenIcon className="h-3 w-3" />
+ Full page
+ </>
+ )}
+ </button>
+ </div>
+ );
+}
+
+function DraftStepCard({
+ step,
+ index,
+ baselineMs,
+ screenshot,
+ onDelete,
+ onRename,
+}: {
+ step: RecordingDraftStep;
+ index: number;
+ baselineMs: number | null;
+ screenshot: RecordingScreenshot | null;
+ onDelete: () => void;
+ onRename: (value: string) => void;
+}) {
+ const beginDraftEdit = useRecordingStore((state) => state.beginDraftEdit);
+ const endDraftEdit = useRecordingStore((state) => state.endDraftEdit);
+ const [isEditing, setIsEditing] = useState(false);
+ const displayTitle = formatDraftStepDisplayTitle(step);
+ const [draftTitle, setDraftTitle] = useState(displayTitle);
+
+ useEffect(() => {
+ if (!isEditing) {
+ return;
+ }
+ beginDraftEdit();
+ return () => {
+ endDraftEdit();
+ };
+ }, [isEditing, beginDraftEdit, endDraftEdit]);
+
+ useEffect(() => {
+ if (!isEditing) {
+ setDraftTitle(displayTitle);
+ }
+ }, [displayTitle, isEditing]);
+
+ const saveTitle = () => {
+ const trimmed = draftTitle.trim();
+ if (trimmed && trimmed !== displayTitle) {
+ onRename(trimmed);
+ }
+ setIsEditing(false);
+ };
+
+ const relativeSeconds =
+ baselineMs !== null &&
+ step.timestamp_start !== null &&
+ step.timestamp_start !== undefined
+ ? (step.timestamp_start - baselineMs) / 1000
+ : null;
+ const meta = shortUrl(step.url);
+
+ return (
+ <div className="group relative flex flex-col gap-2 rounded-[10px] p-3 transition-colors hover:bg-slate-elevation3">
+ <div className="flex items-start gap-2.5">
+ <span className="mt-px flex h-[22px] w-[22px] flex-none items-center justify-center rounded-full bg-slate-elevation5 text-[11px] font-semibold text-foreground">
+ {index + 1}
+ </span>
+ <div className="min-w-0 flex-1">
+ {isEditing ? (
+ <input
+ autoFocus
+ className="w-full rounded border bg-background px-1.5 py-0.5 text-[13px] font-medium text-foreground outline-none focus:ring-1 focus:ring-ring"
+ value={draftTitle}
+ onChange={(e) => setDraftTitle(e.target.value)}
+ onBlur={saveTitle}
+ onKeyDown={(e) => {
+ if (e.key === "Enter") {
+ saveTitle();
+ }
+ if (e.key === "Escape") {
+ setDraftTitle(displayTitle);
+ setIsEditing(false);
+ }
+ }}
+ />
+ ) : (
+ <div
+ className="cursor-text rounded text-[13px] font-medium leading-snug text-foreground hover:bg-slate-elevation5 hover:shadow-[0_0_0_4px_hsl(var(--slate-elevation-5))]"
+ role="button"
+ tabIndex={0}
+ title="Click to edit"
+ onClick={() => setIsEditing(true)}
+ onKeyDown={(event) => {
+ if (event.key === "Enter" || event.key === " ") {
+ event.preventDefault();
+ setIsEditing(true);
+ }
+ }}
+ >
+ {displayTitle}
+ </div>
+ )}
+ <div className="mt-0.5 truncate text-[11px] text-muted-foreground">
+ {KIND_LABELS[step.action_kind] ?? step.action_kind}
+ {relativeSeconds !== null &&
+ ` · ${formatRecordingClock(relativeSeconds)}`}
+ {meta && (
+ <>
+ {" · "}
+ <span className="font-mono">{meta}</span>
+ </>
+ )}
+ {step.status === "interpreting" && (
+ <span className="animate-pulse text-yellow-500">
+ {" · refining…"}
+ </span>
+ )}
+ </div>
+ </div>
+ <div className="flex flex-none gap-0.5 opacity-0 transition-opacity group-hover:opacity-100">
+ <button
+ type="button"
+ title="Edit title"
+ className="flex h-6 w-6 items-center justify-center rounded text-muted-foreground hover:bg-slate-elevation5 hover:text-foreground"
+ onClick={() => setIsEditing(true)}
+ >
+ <Pencil1Icon className="h-3.5 w-3.5" />
+ </button>
+ <button
+ type="button"
+ title="Delete block"
+ className="flex h-6 w-6 items-center justify-center rounded text-muted-foreground hover:bg-slate-elevation5 hover:text-red-400"
+ onClick={onDelete}
+ >
+ <TrashIcon className="h-3.5 w-3.5" />
+ </button>
+ </div>
+ </div>
+ {screenshot && (
+ <div className="pl-8">
+ <StepScreenshot screenshot={screenshot} />
+ </div>
+ )}
+ </div>
+ );
+}
+
+function InterpretingRow({
+ index,
+ label,
+ title,
+}: {
+ index: number;
+ label: string;
+ title?: string;
+}) {
+ return (
+ <div className="flex items-start gap-2.5 p-3">
+ <span className="mt-px flex h-[22px] w-[22px] flex-none items-center justify-center rounded-full bg-slate-elevation5 text-[11px] font-semibold text-muted-foreground">
+ {index + 1}
+ </span>
+ <div className="min-w-0 flex-1">
+ {title ? (
+ <div className="truncate text-[13px] font-medium leading-snug text-foreground">
+ {title}
+ </div>
+ ) : (
+ <div className="mb-1.5 h-3 w-3/5 animate-pulse rounded bg-slate-elevation5" />
+ )}
+ <div className="text-[11px] text-yellow-500">{label}</div>
+ </div>
+ </div>
+ );
+}
+
+type Props = {
+ browserSessionId: string | null;
+};
+
+function RecordingPanel({ browserSessionId }: Props) {
+ const setRecordedBlocks = useRecordedBlocksStore(
+ (state) => state.setRecordedBlocks,
+ );
+ // Captured once on mount via getState() — deliberately not a subscription:
+ // recording starters stash their insertion point in the workflow panel store
+ // right before recording begins, and later panel-state changes (e.g. the user
+ // browsing the node library mid-recording) must not move the commit target.
+ const [insertionPointState] = useState(() => {
+ const data = useWorkflowPanelStore.getState().workflowPanelState.data;
+ if (!data) {
+ captureRecordBrowser("record_browser.missing_insertion_point");
+ }
+ return {
+ insertionPoint: {
+ previous: data?.previous ?? null,
+ next: data?.next ?? null,
+ parent: data?.parent,
+ connectingEdgeType: data?.connectingEdgeType ?? "default",
+ },
+ isValid: data !== null && data !== undefined,
+ };
+ });
+ const insertionPoint = insertionPointState.insertionPoint;
+ const insertionPointMissing = !insertionPointState.isValid;
+ const [confirmDiscardOpen, setConfirmDiscardOpen] = useState(false);
+ const feedEndRef = useRef<HTMLDivElement | null>(null);
+ const committedRef = useRef(false);
+
+ // Slice the frequently-changing fields with useShallow so live-recording
+ // updates (optimistic appends, exposedEventCount ticks) only re-render when a
+ // field this panel reads actually changes, not on every exfiltrated event.
+ const {
+ draftSteps,
+ deletedStepIds,
+ stepPatches,
+ screenshots,
+ sessionRevision,
+ optimisticSteps: rawOptimisticSteps,
+ workflowPermanentId,
+ interpretationPending,
+ interpretationFinalized,
+ finishRequested,
+ isCommitting,
+ exposedEventCount,
+ manualCapturePaused,
+ } = useRecordingStore(
+ useShallow((state) => ({
+ draftSteps: state.draftSteps,
+ deletedStepIds: state.deletedStepIds,
+ stepPatches: state.stepPatches,
+ screenshots: state.screenshots,
+ sessionRevision: state.sessionRevision,
+ optimisticSteps: state.optimisticSteps,
+ workflowPermanentId: state.workflowPermanentId,
+ interpretationPending: state.interpretationPending,
+ interpretationFinalized: state.interpretationFinalized,
+ finishRequested: state.finishRequested,
+ isCommitting: state.isCommitting,
+ exposedEventCount: state.exposedEventCount,
+ manualCapturePaused: state.manualCapturePaused,
+ })),
+ );
+
+ const interpretationEnabled = workflowPermanentId !== null;
+
+ const visibleSteps = useMemo(
+ () => applyDraftStepOverlays(draftSteps, deletedStepIds, stepPatches),
+ [draftSteps, deletedStepIds, stepPatches],
+ );
+
+ // Surface optimistic placeholders whenever interpretation is enabled (not just
+ // after the first snapshot), so the first steps appear without a round-trip.
+ // When interpretation is disabled they would accumulate unreconciled, so hide.
+ const optimisticSteps = interpretationEnabled ? rawOptimisticSteps : [];
+ const actionCount = visibleSteps.length + optimisticSteps.length;
+
+ // Step times are remote-browser clocks; anchor to the first step instead of
+ // the operator's local clock to dodge skew.
+ const baselineMs = useMemo(() => {
+ for (const step of visibleSteps) {
+ if (step.timestamp_start !== null && step.timestamp_start !== undefined) {
+ return step.timestamp_start;
+ }
+ }
+ return null;
+ }, [visibleSteps]);
+
+ const processRecordingMutation = useProcessRecordingMutation({
+ browserSessionId,
+ onSuccess: (result) => {
+ setRecordedBlocks(result, insertionPoint);
+ useRecordingStore.getState().setIsRecording(false);
+ },
+ });
+
+ const mutationIsPending = processRecordingMutation.isPending;
+ const setIsCommitting = useRecordingStore((state) => state.setIsCommitting);
+ useEffect(() => {
+ setIsCommitting(mutationIsPending);
+ }, [mutationIsPending, setIsCommitting]);
+
+ const mutationIsError = processRecordingMutation.isError;
+ useEffect(() => {
+ if (mutationIsError) {
+ // Allow Done to retry with the drafts we still hold.
+ committedRef.current = false;
+ }
+ }, [mutationIsError]);
+
+ const commit = () => {
+ if (committedRef.current || insertionPointMissing) {
+ return;
+ }
+ committedRef.current = true;
+ processRecordingMutation.mutate({
+ draftSteps: useRecordingStore.getState().getFinalDraftSteps(),
+ });
+ };
+ const commitRef = useRef(commit);
+ commitRef.current = commit;
+
+ // Done flow: requestFinish stops exfiltration; the backend flushes a final
+ // interpretation snapshot, then we commit (or commit anyway on timeout).
+ // commitRef keeps the timeout callback stable; mutationIsError clears
+ // committedRef so the timeout can fire again on retry.
+ useEffect(() => {
+ if (!finishRequested || committedRef.current) {
+ return;
+ }
+ if (interpretationFinalized || sessionRevision === 0) {
+ commitRef.current();
+ return;
+ }
+ const timeout = setTimeout(() => commitRef.current(), FINALIZE_TIMEOUT_MS);
+ return () => clearTimeout(timeout);
+ }, [finishRequested, interpretationFinalized, sessionRevision]);
+
+ // keep the newest block in view
+ useEffect(() => {
+ feedEndRef.current?.scrollIntoView({ behavior: "smooth", block: "end" });
+ }, [actionCount, interpretationPending]);
+
+ const elapsedSeconds = useRecordingElapsedSeconds();
+
+ const discard = () => {
+ const store = useRecordingStore.getState();
+ captureRecordBrowser("record_browser.cancelled", {
+ event_count_at_cancel: store.getEventCount(),
+ seconds_recording: store.getSecondsRecording(),
+ });
+ setConfirmDiscardOpen(false);
+ store.setIsRecording(false);
+ store.reset();
+ };
+
+ const onDiscardClick = () => {
+ if (visibleSteps.length > 0 || exposedEventCount > 0) {
+ setConfirmDiscardOpen(true);
+ } else {
+ discard();
+ }
+ };
+
+ const onDoneClick = () => {
+ if (insertionPointMissing) {
+ return;
+ }
+ if (finishRequested) {
+ // A previous commit attempt failed; retry directly.
+ commit();
+ return;
+ }
+ useRecordingStore.getState().requestFinish();
+ };
+
+ const isFinishing = finishRequested || isCommitting;
+ const showInterpretationFallbackNote =
+ !interpretationEnabled && exposedEventCount > 0;
+ const headerTitle = isFinishing
+ ? "Finishing recording"
+ : manualCapturePaused
+ ? "Recording paused"
+ : "Recording browser";
+
+ return (
+ <div className="flex h-full w-full flex-col overflow-hidden rounded-xl border bg-slate-elevation2">
+ {/* header */}
+ <div className="flex flex-none items-center gap-2.5 border-b px-3.5 py-3">
+ <span
+ className={cn("h-2.5 w-2.5 flex-none rounded-full bg-red-500", {
+ "animate-pulse": !isFinishing && !manualCapturePaused,
+ "opacity-60": isFinishing || manualCapturePaused,
+ })}
+ />
+ <div className="min-w-0 flex-1">
+ <div className="text-[13px] font-semibold text-foreground">
+ {headerTitle}
+ </div>
+ <div className="font-mono text-[11.5px] text-muted-foreground">
+ {formatRecordingClock(elapsedSeconds)} · {actionCount} action
+ {actionCount === 1 ? "" : "s"}
+ </div>
+ </div>
+ </div>
+
+ {/* feed */}
+ <div className="flex min-h-0 flex-1 flex-col gap-1 overflow-y-auto px-2 pb-3 pt-2">
+ {visibleSteps.length === 0 &&
+ optimisticSteps.length === 0 &&
+ !interpretationPending ? (
+ <div className="flex flex-col items-center justify-center gap-2.5 px-5 py-10 text-center text-xs leading-relaxed text-muted-foreground">
+ <CursorArrowIcon className="h-5 w-5" />
+ {manualCapturePaused ? (
+ <>
+ Capture is paused.
+ <br />
+ Resume to continue recording browser actions.
+ </>
+ ) : (
+ <>
+ Interact with the browser.
+ <br />
+ Your clicks, typing and navigation will appear here as blocks.
+ </>
+ )}
+ </div>
+ ) : (
+ visibleSteps.map((step, index) => (
+ <DraftStepCard
+ key={step.step_id}
+ step={step}
+ index={index}
+ baselineMs={baselineMs}
+ screenshot={findScreenshotForStep(step, screenshots)}
+ onDelete={() =>
+ useRecordingStore.getState().deleteDraftStep(step.step_id)
+ }
+ onRename={(value) =>
+ useRecordingStore
+ .getState()
+ .patchDraftStep(
+ step.step_id,
+ buildDraftStepTitlePatch(step, value),
+ )
+ }
+ />
+ ))
+ )}
+ {optimisticSteps.map((step, i) => (
+ <InterpretingRow
+ key={step.local_id}
+ index={visibleSteps.length + i}
+ label={isFinishing ? "Finalizing…" : "Interpreting…"}
+ title={step.title}
+ />
+ ))}
+ {interpretationPending && optimisticSteps.length === 0 && (
+ <InterpretingRow
+ index={visibleSteps.length}
+ label={isFinishing ? "Finalizing…" : "Interpreting…"}
+ />
+ )}
+ {showInterpretationFallbackNote && (
+ <div className="px-3 py-2 text-[11px] leading-relaxed text-muted-foreground">
+ {exposedEventCount} interaction
+ {exposedEventCount === 1 ? "" : "s"} captured — blocks will be
+ generated when you finish.
+ </div>
+ )}
+ <div ref={feedEndRef} />
+ </div>
+
+ {insertionPointMissing && (
+ <div className="flex-none border-t px-3.5 py-2 text-[11px] leading-relaxed text-red-400">
+ Could not determine where to insert recorded blocks. Discard and start
+ recording from the workflow editor again.
+ </div>
+ )}
+
+ {/* controls */}
+ <div className="flex flex-none items-center gap-2 border-t px-3 py-2.5">
+ <Button
+ variant="outline"
+ size="icon"
+ title={manualCapturePaused ? "Resume capture" : "Pause capture"}
+ className="h-8 w-8"
+ disabled={isCommitting || isFinishing}
+ onClick={() =>
+ useRecordingStore
+ .getState()
+ .setManualCapturePaused(!manualCapturePaused)
+ }
+ >
+ {manualCapturePaused ? (
+ <PlayIcon className="h-4 w-4" />
+ ) : (
+ <PauseIcon className="h-4 w-4" />
+ )}
+ </Button>
+ <Button
+ variant="outline"
+ size="icon"
+ title="Discard recording"
+ className="h-8 w-8 hover:border-red-500/40 hover:text-red-400"
+ disabled={isCommitting}
+ onClick={onDiscardClick}
+ >
+ <TrashIcon className="h-4 w-4" />
+ </Button>
+ <Button
+ size="sm"
+ className="ml-auto h-8"
+ disabled={insertionPointMissing || (isFinishing && !mutationIsError)}
+ onClick={onDoneClick}
+ >
+ <CheckIcon className="mr-1.5 h-4 w-4" />
+ {mutationIsError ? "Retry" : isFinishing ? "Processing…" : "Done"}
+ </Button>
+ </div>
+
+ {confirmDiscardOpen && (
+ <Dialog open onOpenChange={setConfirmDiscardOpen}>
+ <DialogContent>
+ <DialogHeader>
+ <DialogTitle>Discard recording?</DialogTitle>
+ <DialogDescription>
+ {visibleSteps.length > 0
+ ? `You have ${visibleSteps.length} recorded block${
+ visibleSteps.length === 1 ? "" : "s"
+ } that will be lost if you discard.`
+ : "Your recorded interactions will be lost if you discard."}{" "}
+ Are you sure you want to discard the recording?
+ </DialogDescription>
+ </DialogHeader>
+ <DialogFooter>
+ <Button
+ variant="outline"
+ onClick={() => setConfirmDiscardOpen(false)}
+ >
+ Keep recording
+ </Button>
+ <Button variant="destructive" onClick={discard}>
+ Discard recording
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+ )}
+ </div>
+ );
+}
+
+export { RecordingPanel };
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/applyRecordedBlocksToGraph.ts b/skyvern-frontend/src/routes/workflows/editor/recording/applyRecordedBlocksToGraph.ts
new file mode 100644
index 000000000..b1ddd8f29
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/applyRecordedBlocksToGraph.ts
@@ -0,0 +1,120 @@
+import { Edge } from "@xyflow/react";
+import { nanoid } from "nanoid";
+
+import type {
+ WorkflowBlock,
+ WorkflowParameter,
+} from "@/routes/workflows/types/workflowTypes";
+import type { InsertionPoint } from "@/store/RecordedBlocksStore";
+import { AppNode, isWorkflowBlockNode } from "../nodes";
+import { ParametersState } from "../types";
+import { convertToNode, generateNodeLabel } from "../workflowEditorUtils";
+
+type ApplyRecordedBlocksArgs = {
+ nodes: Array<AppNode>;
+ edges: Array<Edge>;
+ recordedBlocks: Array<WorkflowBlock>;
+ recordedInsertionPoint: InsertionPoint;
+ recordedParameters: Array<WorkflowParameter> | null;
+ existingParameters: ParametersState;
+};
+
+type ApplyRecordedBlocksResult = {
+ nodes: Array<AppNode>;
+ edges: Array<Edge>;
+ newParameters: ParametersState;
+};
+
+function applyRecordedBlocksToGraph({
+ nodes,
+ edges,
+ recordedBlocks,
+ recordedInsertionPoint,
+ recordedParameters,
+ existingParameters,
+}: ApplyRecordedBlocksArgs): ApplyRecordedBlocksResult {
+ const { previous, next, parent, connectingEdgeType } = recordedInsertionPoint;
+
+ const newNodes: Array<AppNode> = [];
+ const newEdges: Array<Edge> = [];
+
+ let existingLabels = nodes
+ .filter(isWorkflowBlockNode)
+ .map((node) => node.data.label);
+
+ let prevNodeId = previous;
+
+ recordedBlocks.forEach((block, index) => {
+ const id = nanoid();
+ const label = generateNodeLabel(existingLabels);
+ existingLabels = [...existingLabels, label];
+ const blockWithLabel = { ...block, label: block.label || label };
+
+ const node = convertToNode({ id, parentId: parent }, blockWithLabel, true);
+ newNodes.push(node);
+
+ if (prevNodeId) {
+ newEdges.push({
+ id: nanoid(),
+ type: "edgeWithAddButton",
+ source: prevNodeId,
+ target: id,
+ style: { strokeWidth: 2 },
+ });
+ }
+
+ if (index === recordedBlocks.length - 1 && next) {
+ newEdges.push({
+ id: nanoid(),
+ type: connectingEdgeType,
+ source: id,
+ target: next,
+ style: { strokeWidth: 2 },
+ });
+ }
+
+ prevNodeId = id;
+ });
+
+ const editedEdges = previous
+ ? edges.filter((edge) => edge.source !== previous)
+ : edges;
+
+ const previousNode = nodes.find((node) => node.id === previous);
+ const previousNodeIndex = previousNode
+ ? nodes.indexOf(previousNode)
+ : nodes.length - 1;
+
+ const mergedNodes = [
+ ...nodes.slice(0, previousNodeIndex + 1),
+ ...newNodes,
+ ...nodes.slice(previousNodeIndex + 1),
+ ];
+
+ const newParameters: ParametersState = [];
+
+ for (const newParameter of recordedParameters ?? []) {
+ const exists = existingParameters.some(
+ (param) => param.key === newParameter.key,
+ );
+
+ if (!exists) {
+ newParameters.push({
+ key: newParameter.key,
+ parameterType: "workflow",
+ dataType: newParameter.workflow_parameter_type,
+ description: newParameter.description ?? null,
+ defaultValue: newParameter.default_value ?? "",
+ });
+ }
+ }
+
+ return {
+ nodes: mergedNodes,
+ edges: [...editedEdges, ...newEdges],
+ newParameters,
+ };
+}
+
+export { applyRecordedBlocksToGraph };
+export type { ApplyRecordedBlocksArgs, ApplyRecordedBlocksResult };
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.test.ts b/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.test.ts
new file mode 100644
index 000000000..7d689d80b
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.test.ts
@@ -0,0 +1,102 @@
+import { describe, expect, it } from "vitest";
+
+import type { MessageInExfiltratedConsoleEvent } from "@/store/useRecordingStore";
+
+import { buildOptimisticStep } from "./optimisticSteps";
+
+function consoleEvent(
+ type: string,
+ overrides: Partial<MessageInExfiltratedConsoleEvent["params"]> = {},
+): MessageInExfiltratedConsoleEvent {
+ return {
+ kind: "exfiltrated-event",
+ event_name: "user_interaction",
+ source: "console",
+ timestamp: 1.0,
+ params: {
+ type,
+ url: "https://example.com",
+ timestamp: 1000,
+ target: { tagName: "BUTTON", text: ["Submit"] },
+ mousePosition: { xa: null, ya: null, xp: 0.5, yp: 0.5 },
+ activeElement: { tagName: "BUTTON" },
+ window: { width: 1200, height: 800, scrollX: 0, scrollY: 0 },
+ ...overrides,
+ },
+ };
+}
+
+describe("buildOptimisticStep", () => {
+ it("builds a click placeholder from target text", () => {
+ const step = buildOptimisticStep(consoleEvent("click"));
+ expect(step?.action_kind).toBe("click");
+ expect(step?.title).toBe("Click 'Submit'");
+ });
+
+ it("falls back through innerText then tag name for click text", () => {
+ expect(
+ buildOptimisticStep(
+ consoleEvent("click", {
+ target: { tagName: "DIV", text: [], innerText: "Read more" },
+ }),
+ )?.title,
+ ).toBe("Click 'Read more'");
+
+ expect(
+ buildOptimisticStep(
+ consoleEvent("click", { target: { tagName: "DIV", text: [] } }),
+ )?.title,
+ ).toBe("Click 'div'");
+ });
+
+ it("builds an input_text placeholder on change/blur with a value", () => {
+ const changeStep = buildOptimisticStep(
+ consoleEvent("change", {
+ target: { tagName: "INPUT", text: ["Email"], value: "[email protected]" },
+ }),
+ );
+ expect(changeStep?.action_kind).toBe("input_text");
+ expect(changeStep?.title).toBe("Fill 'Email'");
+
+ expect(
+ buildOptimisticStep(
+ consoleEvent("blur", {
+ target: { tagName: "INPUT", text: ["Email"] },
+ inputValue: "typed",
+ }),
+ )?.action_kind,
+ ).toBe("input_text");
+ });
+
+ it("returns null for change/blur without a value", () => {
+ expect(
+ buildOptimisticStep(
+ consoleEvent("blur", { target: { tagName: "INPUT", text: ["Email"] } }),
+ ),
+ ).toBeNull();
+ expect(
+ buildOptimisticStep(
+ consoleEvent("change", {
+ target: { tagName: "INPUT", text: ["Email"], value: " " },
+ }),
+ ),
+ ).toBeNull();
+ });
+
+ it("elides events that do not map to a step", () => {
+ for (const type of ["focus", "input", "keydown", "keypress", "mousemove"]) {
+ expect(buildOptimisticStep(consoleEvent(type))).toBeNull();
+ }
+ });
+
+ it("elides navigation (cdp) events entirely", () => {
+ const nav = {
+ kind: "exfiltrated-event" as const,
+ event_name: "nav:frame_navigated",
+ source: "cdp" as const,
+ timestamp: 2.0,
+ params: { targetInfo: { url: "https://example.com/next" } },
+ };
+ expect(buildOptimisticStep(nav)).toBeNull();
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.ts b/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.ts
new file mode 100644
index 000000000..29e53f2e0
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.ts
@@ -0,0 +1,68 @@
+import {
+ nextOptimisticStepId,
+ type MessageInExfiltratedEvent,
+ type OptimisticStep,
+} from "@/store/useRecordingStore";
+
+/**
+ * Human label for an interaction target, mirroring the backend's
+ * `_action_display_text` (interpretation.py) so optimistic titles match the
+ * interpreted ones they are replaced by.
+ */
+function actionDisplayText(target: {
+ text?: Array<string>;
+ innerText?: string | null;
+ tagName?: string | null;
+}): string {
+ for (const text of target.text ?? []) {
+ const cleaned = text.split(/\s+/).filter(Boolean).join(" ");
+ if (cleaned) {
+ return cleaned.slice(0, 80);
+ }
+ }
+ const inner = target.innerText?.split(/\s+/).filter(Boolean).join(" ");
+ if (inner) {
+ return inner.slice(0, 80);
+ }
+ return (target.tagName ?? "element").toLowerCase();
+}
+
+/**
+ * Build an optimistic placeholder for a single exfiltrated event, or null when
+ * the event does not map to a step. Only clicks and the end of an input
+ * (change/blur with a value) produce placeholders — keystrokes, focus, and
+ * navigations are elided (a link-click navigation is already represented by its
+ * click, and the backend emits no goto for it).
+ */
+export function buildOptimisticStep(
+ message: MessageInExfiltratedEvent,
+): OptimisticStep | null {
+ if (message.source !== "console") {
+ return null;
+ }
+
+ const { params } = message;
+ if (params.type === "click") {
+ return {
+ local_id: nextOptimisticStepId(),
+ action_kind: "click",
+ title: `Click '${actionDisplayText(params.target)}'`,
+ timestamp: params.timestamp,
+ };
+ }
+
+ if (params.type === "change" || params.type === "blur") {
+ const value = params.inputValue ?? params.target.value ?? "";
+ if (value.trim() === "") {
+ return null;
+ }
+ return {
+ local_id: nextOptimisticStepId(),
+ action_kind: "input_text",
+ title: `Fill '${actionDisplayText(params.target)}'`,
+ timestamp: params.timestamp,
+ };
+ }
+
+ return null;
+}
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.test.ts b/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.test.ts
new file mode 100644
index 000000000..b77c25b76
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.test.ts
@@ -0,0 +1,64 @@
+import { describe, expect, it } from "vitest";
+
+import type { RecordingDraftStep } from "@/store/useRecordingStore";
+
+import {
+ buildDraftStepTitlePatch,
+ deriveUrlFromNavigationTitle,
+} from "./recordingDraftStepEdits";
+
+const navigationStep: RecordingDraftStep = {
+ step_id: "bs-recording-step-0",
+ action_kind: "url_change",
+ block_type: "goto_url",
+ label: "goto_wikipedia_com",
+ url: "https://www.wikipedia.com/wiki/Foo",
+ status: "ready",
+ editable_fields: ["label", "url"],
+ parameters: [],
+ parameter_keys: [],
+};
+
+describe("deriveUrlFromNavigationTitle", () => {
+ it("updates hostname while preserving path from the original URL", () => {
+ expect(
+ deriveUrlFromNavigationTitle(navigationStep.url, "Go to wikipedia.org"),
+ ).toBe("https://wikipedia.org/wiki/Foo");
+ });
+
+ it("accepts a full URL in the title", () => {
+ expect(
+ deriveUrlFromNavigationTitle(
+ navigationStep.url,
+ "Go to https://example.org/start",
+ ),
+ ).toBe("https://example.org/start");
+ });
+});
+
+describe("buildDraftStepTitlePatch", () => {
+ it("patches url and label for navigation steps", () => {
+ expect(
+ buildDraftStepTitlePatch(navigationStep, "Go to wikipedia.org"),
+ ).toEqual({
+ title: "Go to wikipedia.org",
+ url: "https://wikipedia.org/wiki/Foo",
+ label: "Go_to_wikipedia_org",
+ });
+ });
+
+ it("patches title only for action steps", () => {
+ const actionStep: RecordingDraftStep = {
+ ...navigationStep,
+ action_kind: "click",
+ block_type: "action",
+ title: "Click 'Search'",
+ };
+
+ expect(
+ buildDraftStepTitlePatch(actionStep, "Click the search button"),
+ ).toEqual({
+ title: "Click the search button",
+ });
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.ts b/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.ts
new file mode 100644
index 000000000..befab5709
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.ts
@@ -0,0 +1,118 @@
+import {
+ type RecordingDraftStep,
+ type RecordingDraftStepPatch,
+} from "@/store/useRecordingStore";
+
+function isNavigationDraftStep(step: RecordingDraftStep): boolean {
+ return step.action_kind === "url_change" || step.block_type === "goto_url";
+}
+
+function normalizeRecordingBlockLabel(label: string, fallback: string): string {
+ let candidate = label.trim().replace(/\W+/g, "_").replace(/_+/g, "_");
+ candidate = candidate.replace(/^_|_$/g, "");
+
+ if (!candidate) {
+ return fallback;
+ }
+
+ if (!/^[A-Za-z_]/.test(candidate)) {
+ candidate = `${fallback}_${candidate}`;
+ }
+
+ return candidate;
+}
+
+/** Mirrors `deterministic_goto_url_label` in browser_recording/service.py. */
+function deriveGotoUrlLabelFromUrl(url: string): string {
+ try {
+ const host = new URL(url).hostname;
+ if (!host) {
+ return "goto_url";
+ }
+ return normalizeRecordingBlockLabel(`goto_${host}`, "goto_url");
+ } catch {
+ return "goto_url";
+ }
+}
+
+/**
+ * When the operator renames "Go to wikipedia.com" → "Go to wikipedia.org", map
+ * that display edit onto the goto_url block's `url` field used at commit time.
+ */
+function deriveUrlFromNavigationTitle(
+ originalUrl: string | null | undefined,
+ title: string,
+): string | null {
+ const trimmed = title.trim();
+ if (!trimmed) {
+ return null;
+ }
+
+ const goToMatch = /^go to\s+(.+)$/i.exec(trimmed);
+ const destination = goToMatch?.[1]?.trim() ?? trimmed;
+
+ try {
+ if (/^https?:\/\//i.test(destination)) {
+ return new URL(destination).toString();
+ }
+
+ const destinationHasPath =
+ destination.includes("/") &&
+ !destination.startsWith("/") &&
+ destination.indexOf("/") < destination.length - 1;
+
+ const destinationUrl = new URL(
+ destination.startsWith("//")
+ ? `https:${destination}`
+ : `https://${destination}`,
+ );
+
+ if (!originalUrl) {
+ return destinationUrl.toString();
+ }
+
+ const original = new URL(originalUrl);
+ original.protocol = destinationUrl.protocol;
+ original.hostname = destinationUrl.hostname;
+ original.port = destinationUrl.port;
+
+ if (destinationHasPath) {
+ original.pathname = destinationUrl.pathname;
+ original.search = destinationUrl.search;
+ if (destinationUrl.hash) {
+ original.hash = destinationUrl.hash;
+ }
+ }
+
+ return original.toString();
+ } catch {
+ return null;
+ }
+}
+
+function buildDraftStepTitlePatch(
+ step: RecordingDraftStep,
+ newTitle: string,
+): RecordingDraftStepPatch {
+ const title = newTitle.trim();
+ const patch: RecordingDraftStepPatch = { title };
+
+ if (isNavigationDraftStep(step)) {
+ const url = deriveUrlFromNavigationTitle(step.url, title);
+ if (url) {
+ patch.url = url;
+ const fallbackLabel = deriveGotoUrlLabelFromUrl(url);
+ patch.label = normalizeRecordingBlockLabel(title, fallbackLabel);
+ }
+ }
+
+ return patch;
+}
+
+export {
+ buildDraftStepTitlePatch,
+ deriveGotoUrlLabelFromUrl,
+ deriveUrlFromNavigationTitle,
+ isNavigationDraftStep,
+ normalizeRecordingBlockLabel,
+};
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.test.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.test.tsx
new file mode 100644
index 000000000..a3815f23e
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.test.tsx
@@ -0,0 +1,134 @@
+// @vitest-environment jsdom
+
+import { Edge } from "@xyflow/react";
+import { renderHook } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
+import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes";
+import { AppNode } from "../nodes";
+import { useApplyRecordedBlocks } from "./useApplyRecordedBlocks";
+
+const initialRecordedBlocksState = useRecordedBlocksStore.getState();
+
+describe("useApplyRecordedBlocks", () => {
+ afterEach(() => {
+ useRecordedBlocksStore.setState(initialRecordedBlocksState, true);
+ });
+
+ it("applies recorded blocks when enabled in debugger/build mode", () => {
+ const doLayout = vi.fn();
+ const nodes = [{ id: "start", data: { label: "start" } }] as Array<AppNode>;
+ const edges = [] as Array<Edge>;
+
+ useRecordedBlocksStore.getState().setRecordedBlocks(
+ {
+ blocks: [
+ {
+ block_type: "action",
+ label: "click_button",
+ title: "Click button",
+ navigation_goal: "Click the button.",
+ url: null,
+ parameters: [],
+ } as unknown as WorkflowBlock,
+ ],
+ parameters: [],
+ },
+ {
+ previous: "start",
+ next: null,
+ connectingEdgeType: "edgeWithAddButton",
+ },
+ );
+
+ renderHook(() =>
+ useApplyRecordedBlocks({
+ enabled: true,
+ nodes,
+ edges,
+ doLayout,
+ }),
+ );
+
+ expect(doLayout).toHaveBeenCalledTimes(1);
+ const layoutArgs = doLayout.mock.calls[0];
+ expect(layoutArgs).toBeDefined();
+ const [mergedNodes, mergedEdges] = layoutArgs!;
+ expect(mergedNodes).toHaveLength(2);
+ expect(mergedEdges.length).toBeGreaterThan(0);
+ });
+
+ it("does not apply recorded blocks when disabled", () => {
+ const doLayout = vi.fn();
+
+ useRecordedBlocksStore.getState().setRecordedBlocks(
+ {
+ blocks: [
+ {
+ block_type: "goto_url",
+ label: "goto_home",
+ url: "https://example.com",
+ } as unknown as WorkflowBlock,
+ ],
+ parameters: [],
+ },
+ {
+ previous: null,
+ next: null,
+ connectingEdgeType: "default",
+ },
+ );
+
+ renderHook(() =>
+ useApplyRecordedBlocks({
+ enabled: false,
+ nodes: [],
+ edges: [],
+ doLayout,
+ }),
+ );
+
+ expect(doLayout).not.toHaveBeenCalled();
+ });
+
+ it("clears a pending payload when the consumer unmounts (interrupted handoff)", () => {
+ const doLayout = vi.fn();
+
+ // enabled: false keeps the payload pending — mirrors a handoff that never
+ // applied before the canvas went away (e.g. navigating right after commit).
+ const { unmount } = renderHook(() =>
+ useApplyRecordedBlocks({
+ enabled: false,
+ nodes: [],
+ edges: [],
+ doLayout,
+ }),
+ );
+
+ useRecordedBlocksStore.getState().setRecordedBlocks(
+ {
+ blocks: [
+ {
+ block_type: "goto_url",
+ label: "goto_home",
+ url: "https://example.com",
+ } as unknown as WorkflowBlock,
+ ],
+ parameters: [],
+ },
+ {
+ previous: null,
+ next: null,
+ connectingEdgeType: "default",
+ },
+ );
+
+ unmount();
+
+ const state = useRecordedBlocksStore.getState();
+ expect(state.blocks).toBeNull();
+ expect(state.parameters).toBeNull();
+ expect(state.insertionPoint).toBeNull();
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.ts b/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.ts
new file mode 100644
index 000000000..77af90faa
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.ts
@@ -0,0 +1,122 @@
+import { Edge } from "@xyflow/react";
+import { useEffect, useRef } from "react";
+
+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
+import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";
+import { useWorkflowParametersStore } from "@/store/WorkflowParametersStore";
+import { AppNode } from "../nodes";
+import { applyRecordedBlocksToGraph } from "./applyRecordedBlocksToGraph";
+
+type UseApplyRecordedBlocksArgs = {
+ enabled: boolean;
+ nodes: Array<AppNode>;
+ edges: Array<Edge>;
+ doLayout: (nodes: Array<AppNode>, edges: Array<Edge>) => void;
+};
+
+function useApplyRecordedBlocks({
+ enabled,
+ nodes,
+ edges,
+ doLayout,
+}: UseApplyRecordedBlocksArgs) {
+ const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);
+ const recordedParameters = useRecordedBlocksStore(
+ (state) => state.parameters,
+ );
+ const recordedInsertionPoint = useRecordedBlocksStore(
+ (state) => state.insertionPoint,
+ );
+ const applicationNonce = useRecordedBlocksStore(
+ (state) => state.applicationNonce,
+ );
+ const clearRecordedBlocks = useRecordedBlocksStore(
+ (state) => state.clearRecordedBlocks,
+ );
+ const parameters = useWorkflowParametersStore((state) => state.parameters);
+
+ const appliedSignatureRef = useRef<string | null>(null);
+ const awaitingClearRef = useRef(false);
+
+ useEffect(() => {
+ if (!enabled) {
+ return;
+ }
+ if (!recordedBlocks?.length || !recordedInsertionPoint) {
+ return;
+ }
+
+ const signature = [
+ String(applicationNonce),
+ recordedInsertionPoint.previous ?? "",
+ recordedInsertionPoint.next ?? "",
+ recordedInsertionPoint.parent ?? "",
+ recordedInsertionPoint.connectingEdgeType,
+ recordedBlocks.length,
+ recordedBlocks
+ .map((block) => `${block.block_type}:${block.label ?? ""}`)
+ .join(","),
+ ].join("|");
+
+ if (appliedSignatureRef.current === signature && awaitingClearRef.current) {
+ return;
+ }
+
+ const {
+ nodes: mergedNodes,
+ edges: mergedEdges,
+ newParameters,
+ } = applyRecordedBlocksToGraph({
+ nodes,
+ edges,
+ recordedBlocks,
+ recordedInsertionPoint,
+ recordedParameters,
+ existingParameters: parameters,
+ });
+
+ useWorkflowHasChangesStore.getState().setHasChanges(true);
+ doLayout(mergedNodes, mergedEdges);
+
+ if (newParameters.length > 0) {
+ const workflowParametersStore = useWorkflowParametersStore.getState();
+ workflowParametersStore.setParameters([
+ ...workflowParametersStore.parameters,
+ ...newParameters,
+ ]);
+ }
+
+ appliedSignatureRef.current = signature;
+ awaitingClearRef.current = true;
+ // nodes/edges/parameters are read from the render that saw the new store
+ // payload; listing them as deps would re-apply on every canvas edit.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [
+ enabled,
+ recordedBlocks,
+ recordedInsertionPoint,
+ applicationNonce,
+ doLayout,
+ ]);
+
+ useEffect(() => {
+ if (!awaitingClearRef.current) {
+ return;
+ }
+ awaitingClearRef.current = false;
+ clearRecordedBlocks();
+ appliedSignatureRef.current = null;
+ }, [nodes, edges, clearRecordedBlocks]);
+
+ // The bridge dies with its consumer: this hook is the only thing that applies
+ // recorded blocks, so if it unmounts while a payload is still pending (e.g.
+ // the user navigates away right after a commit), the blocks must not survive
+ // to be applied to whichever workflow canvas mounts next.
+ useEffect(() => {
+ return () => {
+ useRecordedBlocksStore.getState().clearRecordedBlocks();
+ };
+ }, []);
+}
+
+export { useApplyRecordedBlocks };
diff --git a/skyvern-frontend/src/routes/workflows/studio/BrowserTab.tsx b/skyvern-frontend/src/routes/workflows/studio/BrowserTab.tsx
index de67bb44e..603408d8c 100644
--- a/skyvern-frontend/src/routes/workflows/studio/BrowserTab.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/BrowserTab.tsx
@@ -30,6 +30,8 @@ import {
StreamStatusPanel,
} from "@/routes/streaming/StreamDiagnostics";
import { useStudioBrowserStore } from "@/store/useStudioBrowserStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
+import { useRecordingLauncherStore } from "@/store/useRecordingLauncherStore";
import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";
import { useStudioShellContext } from "./StudioShellContext";
@@ -55,6 +57,11 @@ export function BrowserTab() {
const streamUrl = useStudioBrowserStore((s) => s.streamUrl);
const reload = useStudioBrowserStore((s) => s.reload);
+ const isRecording = useRecordingStore((s) => s.isRecording);
+ const manualCapturePaused = useRecordingStore((s) => s.manualCapturePaused);
+ const startRecordingAtEnd = useRecordingLauncherStore(
+ (s) => s.startRecordingAtEnd,
+ );
const [confirmOff, setConfirmOff] = useState(false);
const cycleBrowser = useMutation({
@@ -106,6 +113,30 @@ export function BrowserTab() {
<span className="min-w-0 flex-1 truncate text-sm font-medium text-foreground">
{streamUrl || "Live browser"}
</span>
+ {isRecording ? (
+ manualCapturePaused ? (
+ <span className="flex shrink-0 items-center gap-1.5 rounded-full bg-amber-500/10 px-2.5 py-1 text-xs font-medium text-amber-500">
+ <span className="h-2 w-2 rounded-full bg-amber-500" />
+ Paused
+ </span>
+ ) : (
+ <span className="flex shrink-0 items-center gap-1.5 rounded-full bg-red-500/10 px-2.5 py-1 text-xs font-medium text-red-500">
+ <span className="h-2 w-2 animate-pulse rounded-full bg-red-500" />
+ Recording
+ </span>
+ )
+ ) : (
+ <Button
+ variant="secondary"
+ size="sm"
+ className="shrink-0 gap-1.5"
+ disabled={!browserSessionId || !startRecordingAtEnd}
+ onClick={() => startRecordingAtEnd?.()}
+ >
+ <span className="h-2 w-2 rounded-full bg-red-500" />
+ Record
+ </Button>
+ )}
<button
type="button"
title="Reconnect"
diff --git a/skyvern-frontend/src/routes/workflows/studio/EditorTab.tsx b/skyvern-frontend/src/routes/workflows/studio/EditorTab.tsx
index 2c5329bde..9147f4541 100644
--- a/skyvern-frontend/src/routes/workflows/studio/EditorTab.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/EditorTab.tsx
@@ -1,6 +1,9 @@
+import { ReloadIcon } from "@radix-ui/react-icons";
import { type ComponentProps } from "react";
import { Workspace } from "@/routes/workflows/editor/Workspace";
+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
// PiP shelved for now — no good placement (copilot left, settings right, Pylon corner).
// import { BrowserPiP } from "./BrowserPiP";
@@ -13,10 +16,30 @@ export type StudioWorkspaceProps = Omit<
// Editor tab: the real Workspace canvas in embedded mode (the shell owns the top
// bar + Copilot spine, so Workspace's floating header is suppressed).
export function EditorTab(props: StudioWorkspaceProps) {
+ // Show the overlay only while a commit is genuinely in flight or its blocks
+ // are landing. Both signals clear on *every* terminal outcome (success, empty,
+ // error), so the overlay can never get stuck — unlike `finishRequested`, which
+ // stays true on a failed commit to allow a retry.
+ const isCommitting = useRecordingStore((s) => s.isCommitting);
+ const recordedBlocksPending = useRecordedBlocksStore(
+ (s) => (s.blocks?.length ?? 0) > 0,
+ );
+ const applyingRecording = isCommitting || recordedBlocksPending;
+
return (
<div className="relative h-full w-full">
<Workspace {...props} showBrowser={false} embedded />
{/* <BrowserPiP /> */}
+ {applyingRecording ? (
+ <div className="absolute inset-0 z-40 flex items-center justify-center bg-background/70 backdrop-blur-sm">
+ <div className="flex items-center gap-3 rounded-lg border bg-slate-elevation2 px-4 py-3 shadow-lg">
+ <ReloadIcon className="h-4 w-4 animate-spin text-muted-foreground" />
+ <span className="text-sm font-medium text-foreground">
+ Adding recorded steps…
+ </span>
+ </div>
+ </div>
+ ) : null}
</div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx
index bd31b6eb2..fe69a9d5f 100644
--- a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx
@@ -7,6 +7,7 @@ type StreamPresenterProps = {
interactive?: boolean;
showControlButtons?: boolean;
isRecording?: boolean;
+ hideRecordingIndicator?: boolean;
// Only the CDP transport carries the page URL; VNC is pixels-only and never
// calls this.
onUrlChange?: (url: string) => void;
@@ -22,6 +23,7 @@ export function StreamPresenter({
interactive = false,
showControlButtons = false,
isRecording = false,
+ hideRecordingIndicator = false,
onUrlChange,
onActivity,
}: StreamPresenterProps) {
@@ -48,6 +50,7 @@ export function StreamPresenter({
interactive={interactive}
showControlButtons={showControlButtons}
exfiltrate={isRecording}
+ hideRecordingIndicator={hideRecordingIndicator}
onActivity={onActivity}
/>
);
diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx
index bc3bd174a..f08222dde 100644
--- a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx
@@ -70,6 +70,10 @@ export function StudioBrowserStream() {
interactive={false}
showControlButtons={browserPaneOpen && !controlLocked}
isRecording={isRecording}
+ // While recording, the Copilot pane hosts the live-drafts panel, whose
+ // header already shows the timer + step count — the REC pill would
+ // duplicate it. Closing that pane brings the pill back as the indicator.
+ hideRecordingIndicator={panes.includes("copilot")}
onUrlChange={handleUrlChange}
onActivity={handleActivity}
/>
diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioShell.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioShell.tsx
index a672df7f5..830b07a53 100644
--- a/skyvern-frontend/src/routes/workflows/studio/StudioShell.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/StudioShell.tsx
@@ -1,10 +1,23 @@
-import { useLayoutEffect, useMemo, useState, type ReactNode } from "react";
+import {
+ useEffect,
+ useLayoutEffect,
+ useMemo,
+ useRef,
+ useState,
+ type ReactNode,
+} from "react";
import { createPortal } from "react-dom";
+import { useParams } from "react-router-dom";
import { Cross2Icon } from "@radix-ui/react-icons";
+import { RecordingPanel } from "@/routes/workflows/editor/recording/RecordingPanel";
+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
+import { useRecordingStore } from "@/store/useRecordingStore";
import { useStudioShellStore } from "@/store/StudioShellStore";
import { cn } from "@/util/utils";
+import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";
+
import { BrowserTab } from "./BrowserTab";
import { EditorTab, type StudioWorkspaceProps } from "./EditorTab";
import { RunTab } from "./RunTab";
@@ -93,8 +106,15 @@ export function StudioShell(props: StudioWorkspaceProps) {
}
function StudioStage(props: StudioWorkspaceProps) {
- const { panes, closePane } = useStudioPanes();
+ const { panes, closePane, openPane } = useStudioPanes();
const { registerStageElement } = useStudioPaneDefaults();
+ const { workflowPermanentId } = useParams();
+ const isRecording = useRecordingStore((s) => s.isRecording);
+ const { data: debugSession } = useDebugSessionQuery({
+ workflowPermanentId,
+ enabled: false,
+ });
+ const browserSessionId = debugSession?.browser_session_id ?? null;
const pipMinimized = useStudioShellStore((s) => s.pipMinimized);
const [copilotPortalEl, setCopilotPortalEl] = useState<HTMLElement | null>(
null,
@@ -166,6 +186,35 @@ function StudioStage(props: StudioWorkspaceProps) {
document.getElementById(studioTabId(id))?.focus();
};
+ // Recording lives in the Browser pane (the live stream is there) with the
+ // live-drafts panel taking over the Copilot pane. Once a commit is in flight
+ // or its blocks are landing, reveal the Editor pane (it shows the loading
+ // overlay). Gated on lifecycle transitions so manual pane changes made
+ // mid-recording are preserved.
+ const isCommitting = useRecordingStore((s) => s.isCommitting);
+ const recordedBlocksPending = useRecordedBlocksStore(
+ (s) => (s.blocks?.length ?? 0) > 0,
+ );
+ const processingRecording = isCommitting || recordedBlocksPending;
+ const prevIsRecordingRef = useRef(false);
+ const prevProcessingRef = useRef(false);
+ useEffect(() => {
+ const wasRecording = prevIsRecordingRef.current;
+ const wasProcessing = prevProcessingRef.current;
+ if (processingRecording && !wasProcessing) {
+ openPane("editor");
+ } else if (isRecording && !wasRecording) {
+ // Recording or finalizing → live browser + drafts.
+ openPane("copilot");
+ openPane("browser");
+ } else if (!isRecording && wasRecording && !processingRecording) {
+ // Recording ended (commit or discard) → back to the canvas.
+ openPane("editor");
+ }
+ prevIsRecordingRef.current = isRecording;
+ prevProcessingRef.current = processingRecording;
+ }, [isRecording, processingRecording, openPane]);
+
const paneProps = (id: StudioPaneId) => {
const index = panes.indexOf(id);
return {
@@ -190,9 +239,23 @@ function StudioStage(props: StudioWorkspaceProps) {
className="relative flex min-h-0 min-w-0 flex-1 gap-3 overflow-hidden p-3"
>
<StudioPane {...paneProps("copilot")}>
- {/* Copilot portal target. Kept mounted while the pane is closed so
- an in-flight Copilot turn isn't torn down. */}
- <div ref={setCopilotPortalEl} className="h-full w-full" />
+ <div className="relative h-full w-full">
+ {/* Copilot portal target. Kept mounted while the pane is closed
+ — or while recording, when the live-drafts panel covers it —
+ so an in-flight Copilot turn isn't torn down. */}
+ <div
+ ref={setCopilotPortalEl}
+ className={cn(
+ "h-full w-full",
+ isRecording && "pointer-events-none",
+ )}
+ />
+ {isRecording && browserSessionId ? (
+ <div className="absolute inset-0 duration-150 animate-in fade-in slide-in-from-left-2">
+ <RecordingPanel browserSessionId={browserSessionId} />
+ </div>
+ ) : null}
+ </div>
</StudioPane>
<StudioPane {...paneProps("editor")}>
<EditorTab {...props} />
diff --git a/skyvern-frontend/src/store/RecordedBlocksStore.ts b/skyvern-frontend/src/store/RecordedBlocksStore.ts
index dd405328e..0f13628ba 100644
--- a/skyvern-frontend/src/store/RecordedBlocksStore.ts
+++ b/skyvern-frontend/src/store/RecordedBlocksStore.ts
@@ -15,6 +15,7 @@ type RecordedBlocksState = {
blocks: Array<WorkflowBlock> | null;
parameters: Array<WorkflowParameter> | null;
insertionPoint: InsertionPoint | null;
+ applicationNonce: number;
};
type RecordedBlocksStore = RecordedBlocksState & {
@@ -32,11 +33,22 @@ const useRecordedBlocksStore = create<RecordedBlocksStore>((set) => ({
blocks: null,
parameters: null,
insertionPoint: null,
+ applicationNonce: 0,
setRecordedBlocks: ({ blocks, parameters }, insertionPoint) => {
- set({ blocks, parameters, insertionPoint });
+ set({
+ blocks,
+ parameters,
+ insertionPoint,
+ applicationNonce: Date.now(),
+ });
},
clearRecordedBlocks: () => {
- set({ blocks: null, insertionPoint: null });
+ set({
+ blocks: null,
+ parameters: null,
+ insertionPoint: null,
+ applicationNonce: 0,
+ });
},
}));
diff --git a/skyvern-frontend/src/store/useRecordingLauncherStore.ts b/skyvern-frontend/src/store/useRecordingLauncherStore.ts
new file mode 100644
index 000000000..ace77fceb
--- /dev/null
+++ b/skyvern-frontend/src/store/useRecordingLauncherStore.ts
@@ -0,0 +1,20 @@
+import { create } from "zustand";
+
+/**
+ * Bridges the studio's Browser-tab Record button to the canvas-aware Workspace.
+ * The Workspace (which has ReactFlow nodes/edges) registers a launcher that
+ * computes the append-at-end insertion point and starts recording; the button,
+ * which lives outside the canvas, just invokes it.
+ */
+type RecordingLauncherState = {
+ startRecordingAtEnd: (() => void) | null;
+ setStartRecordingAtEnd: (launcher: (() => void) | null) => void;
+};
+
+export const useRecordingLauncherStore = create<RecordingLauncherState>(
+ (set) => ({
+ startRecordingAtEnd: null,
+ setStartRecordingAtEnd: (startRecordingAtEnd) =>
+ set({ startRecordingAtEnd }),
+ }),
+);
diff --git a/skyvern-frontend/src/store/useRecordingStore.test.ts b/skyvern-frontend/src/store/useRecordingStore.test.ts
new file mode 100644
index 000000000..777574e1f
--- /dev/null
+++ b/skyvern-frontend/src/store/useRecordingStore.test.ts
@@ -0,0 +1,307 @@
+import { beforeEach, describe, expect, it } from "vitest";
+
+import {
+ upsertDraftSteps,
+ useRecordingStore,
+ type OptimisticActionKind,
+ type OptimisticStep,
+ type RecordingActionKind,
+ type RecordingDraftStep,
+ type RecordingInterpretationUpdate,
+} from "./useRecordingStore";
+
+let seq = 0;
+function opt(kind: OptimisticActionKind = "click"): OptimisticStep {
+ seq += 1;
+ return {
+ local_id: `o-${seq}`,
+ action_kind: kind,
+ title: `${kind} ${seq}`,
+ timestamp: seq,
+ };
+}
+
+function draft(id: string, kind: RecordingActionKind): RecordingDraftStep {
+ return {
+ step_id: id,
+ action_kind: kind,
+ block_type:
+ kind === "url_change" ? "goto_url" : kind === "wait" ? "wait" : "action",
+ label: id,
+ status: "ready",
+ editable_fields: [],
+ parameters: [],
+ parameter_keys: [],
+ };
+}
+
+function update(
+ id: string,
+ revision: number,
+ steps: Array<RecordingDraftStep>,
+ opts: {
+ pending?: boolean;
+ finalized?: boolean;
+ is_snapshot?: boolean;
+ changed_steps?: Array<RecordingDraftStep>;
+ } = {},
+): RecordingInterpretationUpdate {
+ return {
+ interpretation_session_id: id,
+ session_revision: revision,
+ steps,
+ pending: opts.pending ?? false,
+ finalized: opts.finalized ?? false,
+ ...(opts.is_snapshot !== undefined
+ ? { is_snapshot: opts.is_snapshot }
+ : {}),
+ ...(opts.changed_steps !== undefined
+ ? { changed_steps: opts.changed_steps }
+ : {}),
+ };
+}
+
+const store = () => useRecordingStore.getState();
+
+beforeEach(() => {
+ store().reset();
+});
+
+describe("addOptimisticStep", () => {
+ it("appends while recording and unpaused", () => {
+ store().setIsRecording(true);
+ store().addOptimisticStep(opt());
+ expect(store().optimisticSteps).toHaveLength(1);
+ });
+
+ it("no-ops when not recording", () => {
+ store().addOptimisticStep(opt());
+ expect(store().optimisticSteps).toHaveLength(0);
+ });
+
+ it("no-ops when capture is paused", () => {
+ store().setIsRecording(true);
+ store().setManualCapturePaused(true);
+ store().addOptimisticStep(opt());
+ expect(store().optimisticSteps).toHaveLength(0);
+ });
+
+ it("no-ops after finish is requested", () => {
+ store().setIsRecording(true);
+ store().requestFinish();
+ store().addOptimisticStep(opt());
+ expect(store().optimisticSteps).toHaveLength(0);
+ });
+});
+
+describe("applyInterpretationUpdate: transient optimistic steps", () => {
+ beforeEach(() => {
+ store().setIsRecording(true);
+ });
+
+ it("keeps optimistic steps while interpretation is pending", () => {
+ store().addOptimisticStep(opt());
+ store().addOptimisticStep(opt());
+
+ // Immediate pending update (steps unchanged) on the first significant event.
+ store().applyInterpretationUpdate(update("s1", 1, [], { pending: true }));
+ expect(store().optimisticSteps).toHaveLength(2);
+
+ // Another pending bump keeps them.
+ store().applyInterpretationUpdate(update("s1", 2, [], { pending: true }));
+ expect(store().optimisticSteps).toHaveLength(2);
+ });
+
+ it("clears optimistic steps once interpretation settles (pending=false)", () => {
+ store().addOptimisticStep(opt());
+ store().addOptimisticStep(opt());
+
+ store().applyInterpretationUpdate(
+ update("s1", 1, [draft("s1-0", "click")], { pending: false }),
+ );
+ expect(store().optimisticSteps).toHaveLength(0);
+ });
+
+ it("does not strand placeholders for events the backend folds away", () => {
+ // A link click yields a click (+wait) but no goto; the optimistic click must
+ // still clear when interpretation settles rather than linger forever.
+ store().addOptimisticStep(opt("click"));
+ store().applyInterpretationUpdate(
+ update("s1", 1, [draft("s1-0", "click"), draft("s1-1", "wait")], {
+ pending: false,
+ }),
+ );
+ expect(store().optimisticSteps).toHaveLength(0);
+ });
+
+ it("clears optimistic steps on a finalized snapshot", () => {
+ store().addOptimisticStep(opt());
+ store().applyInterpretationUpdate(update("s1", 1, [], { finalized: true }));
+ expect(store().optimisticSteps).toHaveLength(0);
+ });
+
+ it("clears optimistic steps on a genuine session change", () => {
+ store().addOptimisticStep(opt());
+ store().applyInterpretationUpdate(update("s1", 1, [], { pending: true }));
+ expect(store().optimisticSteps).toHaveLength(1);
+
+ store().applyInterpretationUpdate(update("s2", 1, [], { pending: true }));
+ expect(store().optimisticSteps).toHaveLength(0);
+ expect(store().interpretationSessionId).toBe("s2");
+ });
+});
+
+describe("commit safety and resets", () => {
+ it("getFinalDraftSteps never includes optimistic steps", () => {
+ store().setIsRecording(true);
+ store().addOptimisticStep(opt());
+ // pending=true so the optimistic step is still present alongside a draft step
+ store().applyInterpretationUpdate(
+ update("s1", 1, [draft("s1-0", "click")], { pending: true }),
+ );
+
+ const final = store().getFinalDraftSteps();
+ expect(final).toHaveLength(1);
+ expect(final?.every((s) => s.step_id === "s1-0")).toBe(true);
+ expect(store().optimisticSteps).toHaveLength(1);
+ });
+
+ it("reset() and starting a new recording zero optimistic state", () => {
+ store().setIsRecording(true);
+ store().addOptimisticStep(opt());
+ store().reset();
+ expect(store().optimisticSteps).toHaveLength(0);
+
+ store().setIsRecording(true);
+ store().addOptimisticStep(opt());
+ expect(store().optimisticSteps).toHaveLength(1);
+ });
+
+ it("mints a new recordingAttemptId for each new recording", () => {
+ store().setIsRecording(true);
+ const first = store().recordingAttemptId;
+ expect(first).toBeTruthy();
+
+ store().setIsRecording(false);
+ store().setIsRecording(true);
+ const second = store().recordingAttemptId;
+ expect(second).toBeTruthy();
+ expect(second).not.toBe(first);
+ });
+});
+
+describe("upsertDraftSteps", () => {
+ it("returns the same list when there are no changes", () => {
+ const current = [draft("a", "click"), draft("b", "click")];
+ expect(upsertDraftSteps(current, [])).toBe(current);
+ });
+
+ it("replaces a step in place, preserving order", () => {
+ const current = [draft("a", "click"), draft("b", "click")];
+ const updated = { ...draft("a", "click"), label: "updated" };
+ const result = upsertDraftSteps(current, [updated]);
+ expect(result.map((s) => s.step_id)).toEqual(["a", "b"]);
+ expect(result[0]?.label).toBe("updated");
+ expect(result[1]).toBe(current[1]);
+ });
+
+ it("appends genuinely-new steps in arrival order", () => {
+ const current = [draft("a", "click")];
+ const result = upsertDraftSteps(current, [
+ draft("b", "click"),
+ draft("c", "click"),
+ ]);
+ expect(result.map((s) => s.step_id)).toEqual(["a", "b", "c"]);
+ });
+
+ it("handles a mix of replace and append in one call", () => {
+ const current = [draft("a", "click"), draft("b", "click")];
+ const result = upsertDraftSteps(current, [
+ { ...draft("b", "click"), label: "b2" },
+ draft("c", "click"),
+ ]);
+ expect(result.map((s) => s.step_id)).toEqual(["a", "b", "c"]);
+ expect(result[1]?.label).toBe("b2");
+ });
+
+ it("is idempotent for a repeated step id", () => {
+ const current = [draft("a", "click")];
+ const result = upsertDraftSteps(current, [
+ draft("a", "click"),
+ draft("a", "click"),
+ ]);
+ expect(result.map((s) => s.step_id)).toEqual(["a"]);
+ });
+});
+
+describe("applyInterpretationUpdate: snapshot vs delta", () => {
+ beforeEach(() => {
+ store().setIsRecording(true);
+ });
+
+ it("legacy message with no is_snapshot/changed_steps replaces wholesale", () => {
+ store().applyInterpretationUpdate(
+ update("s1", 1, [draft("s1-0", "click"), draft("s1-1", "click")]),
+ );
+ expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-0", "s1-1"]);
+
+ // A later snapshot fully replaces (does not merge).
+ store().applyInterpretationUpdate(
+ update("s1", 2, [draft("s1-2", "click")]),
+ );
+ expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-2"]);
+ });
+
+ it("delta upserts changed steps into the current list", () => {
+ // Seed with a snapshot of two interpreting steps.
+ const a = { ...draft("s1-0", "click"), status: "interpreting" as const };
+ const b = { ...draft("s1-1", "click"), status: "interpreting" as const };
+ store().applyInterpretationUpdate(
+ update("s1", 1, [a, b], { pending: true }),
+ );
+
+ // Delta: s1-0 flips to ready with a real label; s1-1 untouched.
+ const enriched = { ...draft("s1-0", "click"), label: "Click Vancouver" };
+ store().applyInterpretationUpdate(
+ update("s1", 2, [], {
+ pending: true,
+ is_snapshot: false,
+ changed_steps: [enriched],
+ }),
+ );
+
+ const steps = store().draftSteps;
+ expect(steps.map((s) => s.step_id)).toEqual(["s1-0", "s1-1"]);
+ expect(steps[0]?.status).toBe("ready");
+ expect(steps[0]?.label).toBe("Click Vancouver");
+ expect(steps[1]?.status).toBe("interpreting");
+ });
+
+ it("delta appends a brand-new step", () => {
+ store().applyInterpretationUpdate(
+ update("s1", 1, [draft("s1-0", "click")], { pending: true }),
+ );
+ store().applyInterpretationUpdate(
+ update("s1", 2, [], {
+ pending: true,
+ is_snapshot: false,
+ changed_steps: [draft("s1-1", "click")],
+ }),
+ );
+ expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-0", "s1-1"]);
+ });
+
+ it("ignores a stale delta at or below the current revision", () => {
+ store().applyInterpretationUpdate(
+ update("s1", 5, [draft("s1-0", "click")], { pending: true }),
+ );
+ store().applyInterpretationUpdate(
+ update("s1", 5, [], {
+ pending: true,
+ is_snapshot: false,
+ changed_steps: [draft("s1-9", "click")],
+ }),
+ );
+ expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-0"]);
+ });
+});
diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.ts
index 2303e4f00..884a00892 100644
--- a/skyvern-frontend/src/store/useRecordingStore.ts
+++ b/skyvern-frontend/src/store/useRecordingStore.ts
@@ -100,6 +100,120 @@ export type MessageInExfiltratedEvent =
| MessageInExfiltratedCdpEvent
| MessageInExfiltratedConsoleEvent;
+export type RecordingActionKind =
+ | "click"
+ | "hover"
+ | "input_text"
+ | "url_change"
+ | "wait";
+
+/**
+ * Mirrors `RecordingDraftStep` in skyvern/services/browser_recording/types.py.
+ */
+export interface RecordingDraftStep {
+ step_id: string;
+ action_kind: RecordingActionKind;
+ block_type: "action" | "goto_url" | "wait";
+ label: string;
+ title?: string | null;
+ navigation_goal?: string | null;
+ url?: string | null;
+ wait_sec?: number | null;
+ status: "interpreting" | "ready";
+ editable_fields: Array<string>;
+ parameters: Array<Record<string, unknown>>;
+ parameter_keys: Array<string>;
+ timestamp_start?: number | null;
+ timestamp_end?: number | null;
+}
+
+export interface RecordingInterpretationUpdate {
+ interpretation_session_id: string;
+ session_revision: number;
+ // Authoritative full list when is_snapshot is true/absent (legacy shape).
+ steps: Array<RecordingDraftStep>;
+ // Delta upserts (by step_id) when is_snapshot is false. Absent on legacy
+ // full-snapshot messages, so a backend that still sends snapshots is a no-op.
+ changed_steps?: Array<RecordingDraftStep>;
+ is_snapshot?: boolean;
+ pending: boolean;
+ finalized: boolean;
+}
+
+/**
+ * Merge delta step upserts into the current list: replace any step sharing a
+ * step_id in place (preserving order), and append genuinely-new steps in arrival
+ * order. Idempotent — re-applying the same step by id is safe.
+ */
+export function upsertDraftSteps(
+ current: Array<RecordingDraftStep>,
+ changed: Array<RecordingDraftStep>,
+): Array<RecordingDraftStep> {
+ if (changed.length === 0) {
+ return current;
+ }
+ const changedById = new Map(changed.map((step) => [step.step_id, step]));
+ const merged = current.map((step) => changedById.get(step.step_id) ?? step);
+ const existingIds = new Set(current.map((step) => step.step_id));
+ for (const step of changed) {
+ if (!existingIds.has(step.step_id)) {
+ merged.push(step);
+ existingIds.add(step.step_id);
+ }
+ }
+ return merged;
+}
+
+export type RecordingDraftStepPatch = Partial<
+ Pick<
+ RecordingDraftStep,
+ "label" | "title" | "navigation_goal" | "url" | "wait_sec"
+ >
+>;
+
+/**
+ * Action kinds the frontend can optimistically predict from a single exfiltrated
+ * event, before the backend interprets it. Navigations are deliberately excluded:
+ * a link click yields a click action (not a goto) on the backend, so an optimistic
+ * nav would have nothing to resolve into.
+ */
+export type OptimisticActionKind = "click" | "input_text";
+
+/**
+ * A frontend-only placeholder appended the instant a significant interaction is
+ * observed, so the count and feed advance without waiting for the debounced
+ * backend interpretation. Transient: cleared once interpretation settles (the
+ * authoritative steps then represent the interaction). Never sent on commit.
+ */
+export interface OptimisticStep {
+ local_id: string;
+ action_kind: OptimisticActionKind;
+ title: string;
+ timestamp: number;
+}
+
+let optimisticStepSeq = 0;
+export const nextOptimisticStepId = (): string =>
+ `optimistic-${(optimisticStepSeq += 1)}`;
+
+/**
+ * A frame grabbed from the live VNC canvas at the moment of a recorded click,
+ * matched to draft steps by source-event timestamp (ms epoch).
+ */
+export interface RecordingScreenshot {
+ timestampMs: number;
+ dataUrl: string;
+ xp: number | null;
+ yp: number | null;
+}
+
+const MAX_SCREENSHOTS = 40;
+// dataUrl.length tracks base64-encoded JPEG bytes; cap total in-memory footprint.
+const MAX_SCREENSHOT_BYTES = 8 * 1024 * 1024;
+
+/** Tolerance when matching a screenshot to a step's source-event window. */
+const SCREENSHOT_MATCH_TOLERANCE_MS = 750;
+
/**
* Number of events per compressed chunk.
*/
@@ -129,6 +243,85 @@ interface RecordingStore {
*/
isRecording: boolean;
recordingStartedAtMs: number | null;
+ /**
+ * The workflow the recording will be committed to. Required to enable
+ * backend live interpretation.
+ */
+ workflowPermanentId: string | null;
+ /**
+ * Per-recording id sent on begin-exfiltration. Stable across reconnects of
+ * the same recording, regenerated for each new recording, so the backend can
+ * tell a reconnect (reuse session) from a new recording (fresh session).
+ */
+ recordingAttemptId: string | null;
+ /**
+ * Latest live-interpretation snapshot from the backend. Snapshots are
+ * full replacements keyed by monotonically increasing session revision.
+ */
+ draftSteps: Array<RecordingDraftStep>;
+ interpretationSessionId: string | null;
+ sessionRevision: number;
+ /**
+ * Frontend-only optimistic placeholders shown instantly on each significant
+ * interaction, cleared once interpretation settles.
+ */
+ optimisticSteps: Array<OptimisticStep>;
+ /**
+ * True between "significant events arrived" and "interpretation resolved" —
+ * drives the trailing "Interpreting…" placeholder in the feed.
+ */
+ interpretationPending: boolean;
+ /**
+ * True once the backend has flushed the session (after end-exfiltration).
+ */
+ interpretationFinalized: boolean;
+ /**
+ * Local user overlays. Snapshots are authoritative for step content; user
+ * deletes/edits are re-applied on top of every snapshot.
+ */
+ deletedStepIds: Array<string>;
+ stepPatches: Record<string, RecordingDraftStepPatch>;
+ /**
+ * Nested count of in-progress live draft title edits. While > 0, capture is
+ * paused alongside manualCapturePaused.
+ */
+ draftEditDepth: number;
+ /**
+ * Operator toggled pause — stops exfiltration and live interpretation until
+ * resumed.
+ */
+ manualCapturePaused: boolean;
+ screenshots: Array<RecordingScreenshot>;
+ /**
+ * Set when the user hits Done: exfiltration stops, and the commit fires once
+ * the finalized interpretation snapshot arrives (or a timeout elapses).
+ */
+ finishRequested: boolean;
+ /**
+ * True while the process_recording mutation is in flight.
+ */
+ isCommitting: boolean;
+ applyInterpretationUpdate: (update: RecordingInterpretationUpdate) => void;
+ /**
+ * Append an optimistic placeholder. No-op unless actively recording and
+ * capture is not paused.
+ */
+ addOptimisticStep: (step: OptimisticStep) => void;
+ deleteDraftStep: (stepId: string) => void;
+ patchDraftStep: (stepId: string, patch: RecordingDraftStepPatch) => void;
+ beginDraftEdit: () => void;
+ endDraftEdit: () => void;
+ setManualCapturePaused: (paused: boolean) => void;
+ isCapturePaused: () => boolean;
+ addScreenshot: (screenshot: RecordingScreenshot) => void;
+ requestFinish: () => void;
+ setIsCommitting: (isCommitting: boolean) => void;
+ /**
+ * Draft steps to commit: snapshot minus user deletions, with user edits
+ * applied. Null when live interpretation never produced a revision (caller
+ * should fall back to raw event processing).
+ */
+ getFinalDraftSteps: () => Array<RecordingDraftStep> | null;
/**
* Add a new recorded event. Triggers async compression when buffer is full.
*/
@@ -213,22 +406,114 @@ async function compressEventsToB64(jsonString: string): Promise<string> {
return btoa(binary);
}
-const isExposedEvent = (event: MessageInExfiltratedEvent): boolean => {
- const exposedConsoleEventTypes = new Set(["focus", "click", "keypress"]);
+export function applyDraftStepOverlays(
+ steps: Array<RecordingDraftStep>,
+ deletedStepIds: Array<string>,
+ stepPatches: Record<string, RecordingDraftStepPatch>,
+): Array<RecordingDraftStep> {
+ const deleted = new Set(deletedStepIds);
+ return steps
+ .filter((step) => !deleted.has(step.step_id))
+ .map((step) => {
+ const patch = stepPatches[step.step_id];
+ return patch ? { ...step, ...patch } : step;
+ });
+}
+
+export function countVisibleDraftSteps(
+ steps: Array<RecordingDraftStep>,
+ deletedStepIds: Array<string>,
+): number {
+ if (deletedStepIds.length === 0) {
+ return steps.length;
+ }
+
+ const deleted = new Set(deletedStepIds);
+ let count = 0;
+ for (const step of steps) {
+ if (!deleted.has(step.step_id)) {
+ count += 1;
+ }
+ }
+ return count;
+}
+/**
+ * The screenshot taken closest to the step's source-event window, if any.
+ *
+ * Both sides of the comparison are the same clock: the screenshot's
+ * `timestampMs` is the exfiltrated event's `params.timestamp` (remote-browser
+ * `Date.now()`, ms epoch), and the step's `timestamp_start`/`timestamp_end`
+ * are the backend echoing that same source-event timestamp back
+ * (RecordingDraftStep "Source-action event timestamps (ms epoch)").
+ */
+export function findScreenshotForStep(
+ step: RecordingDraftStep,
+ screenshots: Array<RecordingScreenshot>,
+): RecordingScreenshot | null {
+ const start = step.timestamp_start;
+ if (start === null || start === undefined) {
+ return null;
+ }
+ const end = step.timestamp_end ?? start;
+
+ let best: RecordingScreenshot | null = null;
+ let bestDistance = Infinity;
+ for (const screenshot of screenshots) {
+ const t = screenshot.timestampMs;
+ const distance = t < start ? start - t : t > end ? t - end : 0; // 0 inside the window
+ if (distance <= SCREENSHOT_MATCH_TOLERANCE_MS && distance < bestDistance) {
+ best = screenshot;
+ bestDistance = distance;
+ }
+ }
+ return best;
+}
+
+const EXPOSED_CONSOLE_EVENT_TYPES = new Set(["focus", "click", "keypress"]);
+
+const isExposedEvent = (event: MessageInExfiltratedEvent): boolean => {
if (event.source === "console") {
- if (exposedConsoleEventTypes.has(event.params.type)) {
+ if (EXPOSED_CONSOLE_EVENT_TYPES.has(event.params.type)) {
return true;
}
}
if (event.source === "cdp") {
- return true;
+ // net:* events are page-activity telemetry, not user interactions
+ return !event.event_name.startsWith("net:");
}
return false;
};
+/**
+ * Fields cleared by both `clear()` and `reset()`. Returns fresh collections on
+ * each call so no two resets share array/object references. `reset()` layers on
+ * the extra teardown (`isCompressing`, `isRecording`, `workflowPermanentId`).
+ */
+function emptyRecordingState() {
+ return {
+ compressedChunks: [] as string[],
+ exposedEventCount: 0,
+ pendingEvents: [] as MessageInExfiltratedEvent[],
+ recordingStartedAtMs: null as number | null,
+ draftSteps: [] as Array<RecordingDraftStep>,
+ interpretationSessionId: null as string | null,
+ sessionRevision: 0,
+ optimisticSteps: [] as Array<OptimisticStep>,
+ interpretationPending: false,
+ interpretationFinalized: false,
+ deletedStepIds: [] as Array<string>,
+ stepPatches: {} as Record<string, RecordingDraftStepPatch>,
+ draftEditDepth: 0,
+ manualCapturePaused: false,
+ screenshots: [] as Array<RecordingScreenshot>,
+ finishRequested: false,
+ isCommitting: false,
+ };
+}
+
export const useRecordingStore = create<RecordingStore>((set, get) => ({
compressedChunks: [],
exposedEventCount: 0,
@@ -236,6 +521,174 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
isCompressing: false,
isRecording: false,
recordingStartedAtMs: null,
+ workflowPermanentId: null,
+ recordingAttemptId: null,
+ draftSteps: [],
+ interpretationSessionId: null,
+ sessionRevision: 0,
+ optimisticSteps: [],
+ interpretationPending: false,
+ interpretationFinalized: false,
+ deletedStepIds: [],
+ stepPatches: {},
+ draftEditDepth: 0,
+ manualCapturePaused: false,
+ screenshots: [],
+ finishRequested: false,
+ isCommitting: false,
+
+ applyInterpretationUpdate: (update) => {
+ const state = get();
+ if (!state.isRecording && !state.finishRequested) {
+ return;
+ }
+
+ // Drop non-finalized snapshots while capture is paused so in-progress edits
+ // are not overwritten; finalized snapshots still commit after Done.
+ if (
+ state.isCapturePaused() &&
+ !update.finalized &&
+ !state.finishRequested
+ ) {
+ return;
+ }
+
+ // The initial null -> first-id transition is not a session change: it must
+ // not discard optimistic steps captured before the first snapshot.
+ const sessionChanged =
+ state.interpretationSessionId !== null &&
+ update.interpretation_session_id !== state.interpretationSessionId;
+
+ if (!sessionChanged && update.session_revision <= state.sessionRevision) {
+ return;
+ }
+
+ // Optimistic placeholders are transient. Keep them only while interpretation
+ // is in flight (pending); clear them once it settles, on session change, or
+ // when finalized — by then the authoritative steps represent the interaction.
+ // The backend emits an immediate pending=true update (steps unchanged) on each
+ // significant event, so this preserves the placeholders we just added and only
+ // clears them when the debounced interpret pass lands (pending=false). This
+ // avoids permanent residue from events the backend folds away (link-click
+ // navigations, deduped/collapsed actions) or duplicate transport events.
+ const settled = !update.pending;
+ const clearOptimistic = sessionChanged || update.finalized || settled;
+
+ // Delta updates upsert changed steps into the current list; snapshots (the
+ // legacy shape, is_snapshot true/absent) replace it wholesale. This keeps the
+ // reader working unchanged until the backend starts sending deltas.
+ const nextDraftSteps =
+ update.is_snapshot === false
+ ? upsertDraftSteps(state.draftSteps, update.changed_steps ?? [])
+ : update.steps;
+
+ set({
+ draftSteps: nextDraftSteps,
+ interpretationSessionId: update.interpretation_session_id,
+ sessionRevision: update.session_revision,
+ ...(clearOptimistic ? { optimisticSteps: [] } : {}),
+ interpretationPending: update.pending,
+ interpretationFinalized: update.finalized,
+ ...(sessionChanged ? { deletedStepIds: [], stepPatches: {} } : {}),
+ });
+ },
+
+ addOptimisticStep: (step) => {
+ const state = get();
+ if (!state.isRecording || state.finishRequested) {
+ return;
+ }
+ if (state.isCapturePaused()) {
+ return;
+ }
+ set({ optimisticSteps: [...state.optimisticSteps, step] });
+ },
+
+ deleteDraftStep: (stepId) => {
+ const state = get();
+ if (state.deletedStepIds.includes(stepId)) {
+ return;
+ }
+ set({ deletedStepIds: [...state.deletedStepIds, stepId] });
+ captureRecordBrowser("record_browser.draft_step_deleted", {
+ step_id: stepId,
+ });
+ },
+
+ patchDraftStep: (stepId, patch) => {
+ const state = get();
+ set({
+ stepPatches: {
+ ...state.stepPatches,
+ [stepId]: { ...state.stepPatches[stepId], ...patch },
+ },
+ });
+ captureRecordBrowser("record_browser.draft_step_edited", {
+ step_id: stepId,
+ fields: Object.keys(patch),
+ });
+ },
+
+ beginDraftEdit: () => {
+ set({ draftEditDepth: get().draftEditDepth + 1 });
+ },
+
+ endDraftEdit: () => {
+ set({ draftEditDepth: Math.max(0, get().draftEditDepth - 1) });
+ },
+
+ setManualCapturePaused: (paused) => {
+ set({ manualCapturePaused: paused });
+ captureRecordBrowser(
+ paused
+ ? "record_browser.capture_paused"
+ : "record_browser.capture_resumed",
+ );
+ },
+
+ isCapturePaused: () => {
+ const state = get();
+ return state.manualCapturePaused || state.draftEditDepth > 0;
+ },
+
+ addScreenshot: (screenshot) => {
+ const screenshots = [...get().screenshots, screenshot];
+ let totalBytes = screenshots.reduce(
+ (sum, item) => sum + item.dataUrl.length,
+ 0,
+ );
+ while (screenshots.length > 1 && totalBytes > MAX_SCREENSHOT_BYTES) {
+ const removed = screenshots.shift();
+ if (removed) {
+ totalBytes -= removed.dataUrl.length;
+ }
+ }
+ if (screenshots.length > MAX_SCREENSHOTS) {
+ screenshots.splice(0, screenshots.length - MAX_SCREENSHOTS);
+ }
+ set({ screenshots });
+ },
+
+ requestFinish: () => {
+ if (get().finishRequested) {
+ return;
+ }
+ set({ finishRequested: true });
+ },
+
+ setIsCommitting: (isCommitting) => set({ isCommitting }),
+
+ getFinalDraftSteps: () => {
+ const state = get();
+ if (state.sessionRevision === 0) {
+ return null;
+ }
+ return applyDraftStepOverlays(
+ state.draftSteps,
+ state.deletedStepIds,
+ state.stepPatches,
+ );
+ },
add: (event) => {
const state = get();
@@ -284,41 +737,45 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
}
},
- clear: () =>
- set({
- compressedChunks: [],
- pendingEvents: [],
- recordingStartedAtMs: null,
- }),
+ clear: () => {
+ // Reset per-recording so ids restart at 1 each recording instead of drifting
+ // upward across the tab's lifetime.
+ optimisticStepSeq = 0;
+ set(emptyRecordingState());
+ },
reset: () =>
set({
- compressedChunks: [],
- exposedEventCount: 0,
- pendingEvents: [],
+ ...emptyRecordingState(),
isCompressing: false,
isRecording: false,
- recordingStartedAtMs: null,
+ workflowPermanentId: null,
}),
setIsRecording: (isRecording, meta) => {
- const state = get();
- if (!state.isRecording && isRecording) {
- get().clear();
- set({
- isRecording: true,
- recordingStartedAtMs: Date.now(),
- });
- captureRecordBrowser("record_browser.started", {
- workflow_permanent_id: meta?.workflowPermanentId ?? undefined,
- browser_session_id: meta?.browserSessionId ?? undefined,
- });
+ if (isRecording) {
+ const state = get();
+ if (!state.isRecording) {
+ get().clear();
+ set({
+ isRecording: true,
+ recordingStartedAtMs: Date.now(),
+ workflowPermanentId: meta?.workflowPermanentId ?? null,
+ recordingAttemptId: crypto.randomUUID(),
+ });
+ captureRecordBrowser("record_browser.started", {
+ workflow_permanent_id: meta?.workflowPermanentId ?? undefined,
+ browser_session_id: meta?.browserSessionId ?? undefined,
+ });
+ }
return;
}
set({
- isRecording,
- recordingStartedAtMs: isRecording ? state.recordingStartedAtMs : null,
+ isRecording: false,
+ recordingStartedAtMs: null,
+ finishRequested: false,
+ isCommitting: false,
});
},
diff --git a/skyvern-frontend/src/util/recordingClock.ts b/skyvern-frontend/src/util/recordingClock.ts
new file mode 100644
index 000000000..ef8d2d146
--- /dev/null
+++ b/skyvern-frontend/src/util/recordingClock.ts
@@ -0,0 +1,6 @@
+export function formatRecordingClock(totalSeconds: number): string {
+ const seconds = Math.max(0, Math.floor(totalSeconds));
+ const mm = String(Math.floor(seconds / 60)).padStart(2, "0");
+ const ss = String(seconds % 60).padStart(2, "0");
+ return `${mm}:${ss}`;
+}