Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Browser recording front-end: live draft panel replaces editor, clean screenshots, new step kinds (#7036)
Cindy Lid10191f6ab5f
diff --git a/skyvern-frontend/src/components/BrowserStream.test.tsx b/skyvern-frontend/src/components/BrowserStream.test.tsxindex abfb0f442..85ed8943e 100644--- a/skyvern-frontend/src/components/BrowserStream.test.tsx+++ b/skyvern-frontend/src/components/BrowserStream.test.tsx@@ -80,10 +80,14 @@ const mocks = vi.hoisted(() => {const recordingStore = {add: vi.fn(),+ addScreenshot: vi.fn(),+ applyInterpretationUpdate: vi.fn(),compressedChunks: [],+ draftEditDepth: 0,getEventCount: vi.fn(() => 0),getSecondsRecording: vi.fn(() => 0),isRecording: false,+ manualCapturePaused: false,pendingEvents: [],reset: vi.fn(),setIsRecording: vi.fn(),@@ -120,9 +124,22 @@ vi.mock("@/store/SettingsStore", () => ({useSettingsStore: () => mocks.settingsStore,}));-vi.mock("@/store/useRecordingStore", () => ({- useRecordingStore: () => mocks.recordingStore,-}));+vi.mock("@/store/useRecordingStore", () => {+ // Honor the selector: BrowserStream reads slices (e.g. state.isRecording) via+ // useRecordingStore(selector). Ignoring the selector and returning the whole+ // store makes primitive selectors yield the store object instead of the field+ // value — truthy where a boolean was expected — spuriously rendering the+ // recording UI.+ const useRecordingStore = (+ selector?: (state: typeof mocks.recordingStore) => unknown,+ ) => (selector ? selector(mocks.recordingStore) : mocks.recordingStore);+ // Also read imperatively (reset/addScreenshot/etc.) via getState().+ useRecordingStore.getState = () => mocks.recordingStore;+ return {+ useRecordingStore,+ countVisibleDraftSteps: (steps: Array<unknown> = []) => steps.length,+ };+});function renderBrowserStream(props: { onActivity?: () => void } = {}) {const queryClient = new QueryClient({diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsxindex 28198468c..4398f0e92 100644--- a/skyvern-frontend/src/components/BrowserStream.tsx+++ b/skyvern-frontend/src/components/BrowserStream.tsx@@ -5,8 +5,9 @@ import _RFB, { type RfbEvent } from "@novnc/novnc/lib/rfb.js";type RFB = _RFB;const RFB = (_RFB as typeof _RFB & { default?: typeof _RFB }).default ?? _RFB;import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons";-import { useEffect, useState, useRef, useCallback } from "react";+import { useEffect, useMemo, useState, useRef, useCallback } from "react";import { useQuery } from "@tanstack/react-query";+import { useShallow } from "zustand/react/shallow";import { getClient } from "@/api/AxiosClient";import {@@ -16,29 +17,24 @@ import {} from "@/api/types";import { Tip } from "@/components/Tip";import { Button } from "@/components/ui/button";-import {- Dialog,- DialogClose,- DialogContent,- DialogDescription,- DialogFooter,- DialogHeader,- DialogTitle,- DialogTrigger,-} from "@/components/ui/dialog";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { useRecordingElapsedSeconds } from "@/hooks/useRecordingElapsedSeconds";import { statusIsNotFinalized } from "@/routes/tasks/types";+import { buildOptimisticStep } from "@/routes/workflows/editor/recording/optimisticSteps";import { useClientIdStore } from "@/store/useClientIdStore";import {useRecordingStore,+ countVisibleDraftSteps,+ type ExfiltratedEventConsoleParams,type MessageInExfiltratedEvent,+ type RecordingInterpretationUpdate,} from "@/store/useRecordingStore";import { useSettingsStore } from "@/store/SettingsStore";import { wssBaseUrl, newWssBaseUrl, getCredentialParam } from "@/util/env";import { copyText } from "@/util/copyText";+import { formatRecordingClock } from "@/util/recordingClock";import { cn } from "@/util/utils";-import { captureRecordBrowser } from "@/util/recordBrowserTelemetry";import {StreamStatusPanel,type StreamDiagnostic,@@ -66,6 +62,14 @@ interface BrowserSession {interface CommandBeginExfiltration {kind: "begin-exfiltration";+ workflow_permanent_id?: string;+ live_interpretation_enabled?: boolean;+ // Declares that this client understands delta interpretation updates, so the+ // backend may send changed_steps instead of full snapshots.+ supports_interpretation_deltas?: boolean;+ // Per-recording id: stable across reconnects, new per recording, so the+ // backend reuses the session on reconnect but starts fresh on a new recording.+ recording_attempt_id?: string;}interface CommandCedeControl {@@ -80,17 +84,33 @@ interface CommandTakeControl {kind: "take-control";}+interface CommandRecordingCapturePause {+ kind: "recording-capture-pause";+}++interface CommandRecordingCaptureResume {+ kind: "recording-capture-resume";+}++interface CommandRecordingRearmCapture {+ kind: "recording-rearm-capture";+}+// a "Command" is an fire-n-forget out-message - it does not require a responsetype Command =| CommandBeginExfiltration| CommandCedeControl| CommandEndExfiltration+ | CommandRecordingCapturePause+ | CommandRecordingCaptureResume+ | CommandRecordingRearmCapture| CommandTakeControl;const messageInKinds = ["ask-for-clipboard","copied-text","exfiltrated-event",+ "recording-interpretation-update",] as const;type MessageInKind = (typeof messageInKinds)[number];@@ -104,10 +124,15 @@ interface MessageInCopiedText {text: string;}+interface MessageInRecordingInterpretationUpdate extends RecordingInterpretationUpdate {+ kind: "recording-interpretation-update";+}+type MessageIn =| MessageInCopiedText| MessageInAskForClipboard- | MessageInExfiltratedEvent;+ | MessageInExfiltratedEvent+ | MessageInRecordingInterpretationUpdate;interface MessageOutAskForClipboardResponse {kind: "ask-for-clipboard-response";@@ -130,6 +155,9 @@ type Props = {resizeTrigger?: number;isVisible?: boolean;isExecuting?: boolean;+ // Hide the REC pill overlay when the recording panel is visible beside the+ // stream (its header already shows the timer + step count).+ hideRecordingIndicator?: boolean;onReadyChange?: (isReady: boolean, browserSessionId: string | null) => void;onActivity?: () => void;// --@@ -140,6 +168,83 @@ type RfbWithFrameUpdates = RFB & {_framebufferUpdate?: () => boolean;};+/** VNC encode settings: favor fast frames when the user is driving the browser. */+function applyVncStreamProfile(+ rfb: RFB,+ profile: "interactive" | "passive",+): void {+ if (profile === "interactive") {+ // Low CPU per frame beats max zlib compression for click/type latency.+ rfb.compressionLevel = 1;+ rfb.qualityLevel = 7;+ return;+ }+ rfb.compressionLevel = 2;+ rfb.qualityLevel = 6;+}++function RecordingPill() {+ const {+ finishRequested,+ manualCapturePaused,+ draftSteps,+ deletedStepIds,+ exposedEventCount,+ optimisticStepCount,+ interpretationEnabled,+ } = useRecordingStore(+ useShallow((state) => ({+ finishRequested: state.finishRequested,+ manualCapturePaused: state.manualCapturePaused,+ draftSteps: state.draftSteps,+ deletedStepIds: state.deletedStepIds,+ exposedEventCount: state.exposedEventCount,+ optimisticStepCount: state.optimisticSteps.length,+ interpretationEnabled: state.workflowPermanentId !== null,+ })),+ );++ const interpretedStepCount = useMemo(+ () => countVisibleDraftSteps(draftSteps, deletedStepIds),+ [draftSteps, deletedStepIds],+ );+ const elapsedSeconds = useRecordingElapsedSeconds();+ // Show interpreted + optimistic steps whenever interpretation is enabled, not+ // just after the first snapshot arrives — otherwise the first step waits a+ // backend round-trip even though the optimistic placeholder is already local.+ const count = interpretationEnabled+ ? interpretedStepCount + optimisticStepCount+ : exposedEventCount;++ const paused = manualCapturePaused && !finishRequested;++ return (+ <div+ className={cn(+ "inline-flex h-6 items-center gap-2 rounded-full border px-3 text-xs font-semibold tabular-nums",+ paused+ ? "border-amber-500/50 bg-amber-950 text-amber-200"+ : "border-red-500/50 bg-red-950 text-red-200",+ )}+ >+ <span+ className={cn("h-2 w-2 rounded-full", {+ "bg-amber-500": paused,+ "bg-red-500": !paused,+ "animate-pulse": !finishRequested && !paused,+ "opacity-50": finishRequested,+ })}+ />+ {finishRequested ? "FINISHING" : paused ? "PAUSED" : "REC"}{" "}+ {formatRecordingClock(elapsedSeconds)}+ <span className={paused ? "text-amber-400/80" : "text-red-400/80"}>+ ·+ </span>+ {count}+ </div>+ );+}+function BrowserStream({browserSessionId = undefined,exfiltrate = false,@@ -150,6 +255,7 @@ function BrowserStream({resizeTrigger,isVisible = true,isExecuting = false,+ hideRecordingIndicator = false,onReadyChange,onActivity,// --@@ -240,7 +346,7 @@ function BrowserStream({null,);const clientId = useClientIdStore((state) => state.clientId);- const recordingStore = useRecordingStore();+ const isRecording = useRecordingStore((state) => state.isRecording);const settingsStore = useSettingsStore();const credentialGetter = useCredentialGetter();const isBrowserSessionAvailable =@@ -375,6 +481,11 @@ function BrowserStream({};}, []);+ // The low-latency encode profile is scoped to recording: that's where frame+ // lag directly delays draft feedback. Other interactive live-browser streams+ // keep the default profile to avoid a broad bandwidth/CPU bump.+ const vncInteractive = exfiltrate;+// vnc socketuseEffect(() => {@@ -442,6 +553,7 @@ function BrowserStream({const rfb = new RFB(canvas, vncUrl);rfb.scaleViewport = true;+ applyVncStreamProfile(rfb, vncInteractive ? "interactive" : "passive");const frameUpdateRfb = rfb as RfbWithFrameUpdates;// noVNC does not expose a public framebuffer-update event in 1.5.x.@@ -527,6 +639,17 @@ function BrowserStream({],);+ // Re-apply encode profile when recording starts without tearing down the socket.+ useEffect(() => {+ if (!rfbRef.current) {+ return;+ }+ applyVncStreamProfile(+ rfbRef.current,+ vncInteractive ? "interactive" : "passive",+ );+ }, [vncInteractive]);+useEffect(() => {if (!showStream || !canvasContainer || !runId) {return;@@ -701,6 +824,13 @@ function BrowserStream({}}, [task, workflow]);+ const { workflowPermanentId, recordingAttemptId } = useRecordingStore(+ useShallow((state) => ({+ workflowPermanentId: state.workflowPermanentId,+ recordingAttemptId: state.recordingAttemptId,+ })),+ );+// effect for exfiltrationuseEffect(() => {const sendCommand = (command: Command) => {@@ -711,10 +841,56 @@ function BrowserStream({messageSocket.send(JSON.stringify(command));};- sendCommand({- kind: exfiltrate ? "begin-exfiltration" : "end-exfiltration",- });- }, [exfiltrate, messageSocket]);+ if (exfiltrate) {+ // Including a workflow id turns on backend live interpretation: the+ // server streams recording-interpretation-update draft step snapshots+ // back over this same socket while the user records.+ sendCommand({+ kind: "begin-exfiltration",+ workflow_permanent_id: workflowPermanentId ?? undefined,+ live_interpretation_enabled: Boolean(workflowPermanentId),+ // Client capability: the backend only emits per-step delta updates+ // (changed_steps upserted by step_id) to clients that declare support;+ // otherwise it falls back to full snapshots.+ supports_interpretation_deltas: true,+ // Stable across reconnects of this recording; a new recording mints a+ // new id so the backend starts a fresh interpretation session.+ recording_attempt_id: recordingAttemptId ?? undefined,+ });+ } else {+ sendCommand({ kind: "end-exfiltration" });+ }+ }, [exfiltrate, messageSocket, workflowPermanentId, recordingAttemptId]);++ const manualCapturePaused = useRecordingStore(+ (state) => state.manualCapturePaused,+ );+ const draftEditDepth = useRecordingStore((state) => state.draftEditDepth);+ const capturePaused = manualCapturePaused || draftEditDepth > 0;+ const previousCapturePausedRef = useRef(false);++ // Pause exfiltration + live interpretation while the operator edits drafts+ // or explicitly pauses capture.+ useEffect(() => {+ if (!exfiltrate || !messageSocket) {+ // Backend pause state is per exfiltration session, so start the next+ // session's edge detection from "not paused". A recording that ended+ // while paused would otherwise fire a spurious resume on the next start;+ // and if capture IS paused on a mid-recording reconnect, this re-sends+ // the pause to the new socket (idempotent) instead of assuming it.+ previousCapturePausedRef.current = false;+ return;+ }++ const wasPaused = previousCapturePausedRef.current;+ if (!wasPaused && capturePaused) {+ messageSocket.send(JSON.stringify({ kind: "recording-capture-pause" }));+ } else if (wasPaused && !capturePaused) {+ messageSocket.send(JSON.stringify({ kind: "recording-capture-resume" }));+ }++ previousCapturePausedRef.current = capturePaused;+ }, [capturePaused, exfiltrate, messageSocket]);useEffect(() => {if (!interactive) {@@ -726,10 +902,10 @@ function BrowserStream({// interactive), a prior grab must be released or its input keeps flowing.// Recording is exempt: it holds take-control for exfiltration.useEffect(() => {- if (!interactive && !showControlButtons && !recordingStore.isRecording) {+ if (!interactive && !showControlButtons && !isRecording) {setUserIsControlling(false);}- }, [interactive, showControlButtons, recordingStore.isRecording]);+ }, [interactive, showControlButtons, isRecording]);const theUserIsControlling =userIsControlling || (interactive && !showControlButtons);@@ -757,18 +933,21 @@ function BrowserStream({};}, [canvasContainer]);- // effect to ensure the recordingStore is reset when the component unmounts+ // Reset the recording store when the stream unmounts — including mid-recording.+ // The stream never unmounts during a live recording flow (the studio keeps it+ // in a detached host across pane changes; the editor keeps it mounted while+ // recording), so an unmount means the user abandoned the session (navigated+ // away / switched workflows) and stale isRecording state would otherwise leak+ // into the next mounted workflow as a stuck recording panel.useEffect(() => {return () => {- recordingStore.reset();+ useRecordingStore.getState().reset();};- // eslint-disable-next-line react-hooks/exhaustive-deps}, []);- // effect to ensure 'take-control' is sent on the rising edge of- // recordingStore.isRecording+ // effect to ensure 'take-control' is sent on the rising edge of isRecordinguseEffect(() => {- if (!recordingStore.isRecording) {+ if (!isRecording) {return;}@@ -786,7 +965,7 @@ function BrowserStream({sendCommand({ kind: "take-control" });setUserIsControlling(true);- }, [recordingStore.isRecording, isMessageConnected, messageSocket]);+ }, [isRecording, isMessageConnected, messageSocket]);/*** TODO(jdo): could use zod or smth similar@@ -851,6 +1030,38 @@ function BrowserStream({}break;}+ case "recording-interpretation-update": {+ // steps is optional: a delta update carries changed_steps instead. Only+ // session_revision is required to accept the message.+ if (+ "session_revision" in data &&+ typeof data.session_revision === "number"+ ) {+ const update = data as MessageInRecordingInterpretationUpdate;+ return {+ kind: "recording-interpretation-update",+ interpretation_session_id:+ typeof update.interpretation_session_id === "string"+ ? update.interpretation_session_id+ : "",+ session_revision: update.session_revision,+ steps: Array.isArray(update.steps) ? update.steps : [],+ changed_steps: Array.isArray(update.changed_steps)+ ? update.changed_steps+ : [],+ // Absent/true => full snapshot (legacy). Only false triggers delta merge.+ is_snapshot:+ typeof update.is_snapshot === "boolean"+ ? update.is_snapshot+ : true,+ pending:+ typeof update.pending === "boolean" ? update.pending : false,+ finalized:+ typeof update.finalized === "boolean" ? update.finalized : false,+ };+ }+ break;+ }default: {const _exhaustive: never = kind;return _exhaustive;@@ -858,6 +1069,34 @@ function BrowserStream({}};+ // Best-effort frame grab from the VNC canvas at the moment of a recorded+ // click, so the live feed can show a zoomed shot at the action point.+ const captureRecordingScreenshot = (+ params: ExfiltratedEventConsoleParams,+ ) => {+ const schedule =+ typeof requestIdleCallback === "function"+ ? (fn: () => void) => requestIdleCallback(fn, { timeout: 750 })+ : (fn: () => void) => window.setTimeout(fn, 0);++ schedule(() => {+ try {+ const canvas = canvasContainer?.querySelector("canvas");+ if (!canvas) {+ return;+ }+ useRecordingStore.getState().addScreenshot({+ timestampMs: params.timestamp,+ dataUrl: canvas.toDataURL("image/jpeg", 0.5),+ xp: params.mousePosition.xp,+ yp: params.mousePosition.yp,+ });+ } catch {+ // toDataURL can throw on a tainted/headless canvas; shots are optional+ }+ });+ };+const handleMessage = (data: unknown, ws: WebSocket | null) => {const message = getMessage(data);@@ -938,7 +1177,52 @@ function BrowserStream({break;}case "exfiltrated-event": {- recordingStore.add(message);+ // Read store state fresh: this handler is attached once per socket+ // and would otherwise see a stale isRecording.+ const store = useRecordingStore.getState();+ if (!store.isRecording && !store.finishRequested) {+ break;+ }+ if (store.isCapturePaused()) {+ break;+ }+ if (+ store.isRecording &&+ message.source === "console" &&+ message.params.type === "click"+ ) {+ captureRecordingScreenshot(message.params);+ }+ if (+ store.isRecording &&+ !store.finishRequested &&+ message.source === "cdp" &&+ (message.event_name === "nav:frame_navigated" ||+ message.event_name === "nav:navigated_within_document")+ ) {+ if (ws) {+ ws.send(JSON.stringify({ kind: "recording-rearm-capture" }));+ }+ }+ if (store.isRecording && !store.finishRequested) {+ const optimistic = buildOptimisticStep(message);+ if (optimistic) {+ store.addOptimisticStep(optimistic);+ }+ }+ store.add(message);+ break;+ }+ case "recording-interpretation-update": {+ useRecordingStore.getState().applyInterpretationUpdate({+ interpretation_session_id: message.interpretation_session_id,+ session_revision: message.session_revision,+ steps: message.steps,+ changed_steps: message.changed_steps,+ is_snapshot: message.is_snapshot,+ pending: message.pending,+ finalized: message.finalized,+ });break;}default: {@@ -1034,84 +1318,25 @@ function BrowserStream({)}</div>)}- {recordingStore.isRecording && (- <>- <div className="pointer-events-none absolute flex aspect-video w-full items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-red-500 animate-in fade-in">+ {isRecording && (+ <div className="pointer-events-none absolute flex aspect-video w-full items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-red-500 animate-in fade-in">+ {/* The pill duplicates the recording panel's header (timer + step+ count), so it's hidden while the panel is visible alongside. */}+ {!hideRecordingIndicator && (<div className="relative h-full w-full">- <div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2 text-red-500">- <div className="truncate">Browser is recording</div>- <Tip content="To finish and turn your actions into blocks, click the stop icon.">- <div className="cursor-pointer">+ <div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2">+ <RecordingPill />+ <Tip content="Your actions appear as blocks in the recording panel. Finish with Done, or use the trash icon to discard.">+ <div className="cursor-pointer text-red-500"><InfoCircledIcon /></div></Tip>- <Dialog>- <DialogTrigger asChild>- <Button- className="ml-auto cursor-pointer"- size="sm"- variant="destructive"- style={{- marginTop: "-0.5rem",- }}- onClick={(e) => {- const hasEvents =- recordingStore.pendingEvents.length > 0 ||- recordingStore.compressedChunks.length > 0;- if (!hasEvents) {- e.preventDefault();- captureRecordBrowser("record_browser.cancelled", {- event_count_at_cancel:- recordingStore.getEventCount(),- seconds_recording:- recordingStore.getSecondsRecording(),- });- recordingStore.setIsRecording(false);- recordingStore.reset();- }- }}- >- cancel- </Button>- </DialogTrigger>- <DialogContent>- <DialogHeader>- <DialogTitle>Cancel recording?</DialogTitle>- <DialogDescription>- You have recorded events that will be lost if you- cancel. Are you sure you want to cancel the recording?- </DialogDescription>- </DialogHeader>- <DialogFooter>- <DialogClose asChild>- <Button variant="outline">Keep recording</Button>- </DialogClose>- <DialogClose asChild>- <Button- variant="destructive"- onClick={() => {- captureRecordBrowser("record_browser.cancelled", {- event_count_at_cancel:- recordingStore.getEventCount(),- seconds_recording:- recordingStore.getSecondsRecording(),- });- recordingStore.setIsRecording(false);- recordingStore.reset();- }}- >- Cancel recording- </Button>- </DialogClose>- </DialogFooter>- </DialogContent>- </Dialog></div></div>- </div>- </>+ )}+ </div>)}- {isExecuting && !recordingStore.isRecording && (+ {isExecuting && !isRecording && (<div className="pointer-events-none absolute flex aspect-video w-full animate-glow items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-yellow-500"><div className="relative h-full w-full"><div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2 text-yellow-500">diff --git a/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.test.ts b/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.test.tsnew file mode 100644index 000000000..3f70674e7--- /dev/null+++ b/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.test.ts@@ -0,0 +1,52 @@+import { act, renderHook } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import { useRecordingStore } from "@/store/useRecordingStore";++import { useRecordingElapsedSeconds } from "./useRecordingElapsedSeconds";++const BASE_MS = 1_700_000_000_000;++describe("useRecordingElapsedSeconds", () => {+ beforeEach(() => {+ vi.useFakeTimers();+ vi.setSystemTime(BASE_MS);+ useRecordingStore.setState({+ recordingStartedAtMs: BASE_MS,+ manualCapturePaused: false,+ finishRequested: false,+ });+ });++ afterEach(() => {+ vi.useRealTimers();+ });++ it("counts up while recording", () => {+ const { result } = renderHook(() => useRecordingElapsedSeconds());+ expect(result.current).toBe(0);+ act(() => vi.advanceTimersByTime(3000));+ expect(Math.floor(result.current)).toBe(3);+ });++ it("freezes while paused and resumes without jumping the clock", () => {+ const { result } = renderHook(() => useRecordingElapsedSeconds());+ act(() => vi.advanceTimersByTime(3000));+ expect(Math.floor(result.current)).toBe(3);++ act(() => {+ useRecordingStore.setState({ manualCapturePaused: true });+ });+ act(() => vi.advanceTimersByTime(5000));+ // Frozen: the 5s spent paused must not count.+ expect(Math.floor(result.current)).toBe(3);++ act(() => {+ useRecordingStore.setState({ manualCapturePaused: false });+ });+ // Resume must not jump forward by the paused span.+ expect(Math.floor(result.current)).toBe(3);+ act(() => vi.advanceTimersByTime(2000));+ expect(Math.floor(result.current)).toBe(5);+ });+});diff --git a/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.ts b/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.tsnew file mode 100644index 000000000..d0dcdf234--- /dev/null+++ b/skyvern-frontend/src/hooks/useRecordingElapsedSeconds.ts@@ -0,0 +1,55 @@+import { useEffect, useRef, useState } from "react";++import { useRecordingStore } from "@/store/useRecordingStore";++/**+ * Elapsed recording time in seconds, excluding any time spent manually paused.+ * The displayed clock freezes while `manualCapturePaused` is set and resumes+ * from where it left off — paused spans are folded into an accumulator rather+ * than counted, so resume never jumps the clock forward.+ */+export function useRecordingElapsedSeconds(): number {+ const startedAtMs = useRecordingStore((state) => state.recordingStartedAtMs);+ const paused = useRecordingStore((state) => state.manualCapturePaused);+ const finishRequested = useRecordingStore((state) => state.finishRequested);++ const [nowMs, setNowMs] = useState(() => Date.now());+ const pausedAccumMsRef = useRef(0);+ const pauseStartedAtMsRef = useRef<number | null>(null);++ // A new recording (new start timestamp) resets the paused accumulator.+ useEffect(() => {+ pausedAccumMsRef.current = 0;+ pauseStartedAtMsRef.current = null;+ }, [startedAtMs]);++ useEffect(() => {+ if (finishRequested) {+ return;+ }++ if (paused) {+ // Freeze: mark when the pause began and stop ticking.+ if (pauseStartedAtMsRef.current === null) {+ pauseStartedAtMsRef.current = Date.now();+ setNowMs(Date.now());+ }+ return;+ }++ // Resuming: fold the just-ended pause into the accumulator so it is not+ // counted, then keep ticking.+ if (pauseStartedAtMsRef.current !== null) {+ pausedAccumMsRef.current += Date.now() - pauseStartedAtMsRef.current;+ pauseStartedAtMsRef.current = null;+ }+ setNowMs(Date.now());+ const interval = setInterval(() => setNowMs(Date.now()), 1000);+ return () => clearInterval(interval);+ }, [paused, finishRequested]);++ if (!startedAtMs) {+ return 0;+ }+ return (nowMs - startedAtMs - pausedAccumMsRef.current) / 1000;+}diff --git a/skyvern-frontend/src/novnc.d.ts b/skyvern-frontend/src/novnc.d.tsindex 813f4cc5d..0e4d1ade0 100644--- a/skyvern-frontend/src/novnc.d.ts+++ b/skyvern-frontend/src/novnc.d.ts@@ -29,6 +29,10 @@ declare module "@novnc/novnc/lib/rfb.js" {_display: RfbDisplay;resizeSession: boolean;scaleViewport: boolean;+ // 0–9; JPEG quality for Tight-encoded regions (default 6).+ qualityLevel: number;+ // 0–9; zlib compression for stream data (default 2).+ compressionLevel: number;constructor(target: HTMLElement, url: string, options?: RFBOptions);addEventListener(event: string, listener: (e: RfbEvent) => void): void;diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.tsindex 995cb4eb8..10a1178f1 100644--- a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts@@ -5,7 +5,10 @@ import { useRef } from "react";import { getClient } from "@/api/AxiosClient";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";-import { useRecordingStore } from "@/store/useRecordingStore";+import {+ useRecordingStore,+ type RecordingDraftStep,+} from "@/store/useRecordingStore";import {type WorkflowBlock,type WorkflowParameter,@@ -33,7 +36,18 @@ const useProcessRecordingMutation = ({const mutationStartedAtRef = useRef<number | null>(null);const processRecordingMutation = useMutation({- mutationFn: async () => {+ mutationFn: async (+ variables: {+ /**+ * Live-interpreted draft steps (with user edits/deletes applied).+ * When provided the backend converts them deterministically instead+ * of re-processing the raw event stream.+ */+ draftSteps?: Array<RecordingDraftStep> | null;+ } | void,+ ) => {+ const draftSteps = variables?.draftSteps ?? null;+if (!browserSessionId) {throw new Error("Cannot process recording without a valid browser session ID.",@@ -49,8 +63,9 @@ const useProcessRecordingMutation = ({mutationStartedAtRef.current = Date.now();const eventCount = recordingStore.getEventCount();+ const hasDraftSteps = (draftSteps?.length ?? 0) > 0;- if (eventCount === 0) {+ if (eventCount === 0 && !hasDraftSteps) {captureRecordBrowser("record_browser.empty_blocked", {seconds_recording: recordingStore.getSecondsRecording(),});@@ -62,12 +77,16 @@ const useProcessRecordingMutation = ({captureRecordBrowser("record_browser.process_attempted", {event_count: recordingStore.getEventCount(),compressed_chunk_count: compressedChunks.length,+ draft_step_count: draftSteps?.length,});const client = await getClient(credentialGetter, "sans-api-v1");return client.post<- { compressed_chunks: string[] },+ {+ compressed_chunks: string[];+ draft_steps?: Array<RecordingDraftStep>;+ },{data: {blocks: Array<WorkflowBlock>;@@ -77,6 +96,7 @@ const useProcessRecordingMutation = ({>(`/browser_sessions/${browserSessionId}/process_recording`, {compressed_chunks: compressedChunks,workflow_permanent_id: workflowPermanentId,+ ...(draftSteps !== null ? { draft_steps: draftSteps } : {}),}).then((response) => ({blocks: response.data.blocks,@@ -112,6 +132,11 @@ const useProcessRecordingMutation = ({return;}+ // A zero-block commit still ends the session: the caller's onSuccess (which+ // normally exits recording after landing blocks) is skipped, and without+ // this the user is stranded in the recording panel with a dead Done button.+ recordingStore.setIsRecording(false);+toast({variant: "warning",title: "Recording Processed (No Blocks)",@@ -141,6 +166,8 @@ const useProcessRecordingMutation = ({latency_ms: latencyMs,});+ useRecordingStore.setState({ finishRequested: false });+toast({variant: "destructive",title: "Error Processing Recording",diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsxindex 51bef2b3c..ff14a4782 100644--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx@@ -14,7 +14,6 @@ import { cn } from "@/util/utils";import { useShouldNotifyWhenClosingTab } from "@/hooks/useShouldNotifyWhenClosingTab";import { BlockActionContext } from "@/store/BlockActionContext";import { useDebugStore } from "@/store/useDebugStore";-import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";import {useWorkflowHasChangesStore,useWorkflowSave,@@ -123,7 +122,6 @@ import {import "./reactFlowOverrideStyles.css";import {convertEchoParameters,- convertToNode,createNode,descendants,generateNodeLabel,@@ -719,16 +717,6 @@ function FlowRenderer({const setGetSaveDataRef = useRef(workflowChangesStore.setGetSaveData);setGetSaveDataRef.current = workflowChangesStore.setGetSaveData;const saveWorkflow = useWorkflowSave({ status: "published" });- const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);- const recordedParameters = useRecordedBlocksStore(- (state) => state.parameters,- );- const recordedInsertionPoint = useRecordedBlocksStore(- (state) => state.insertionPoint,- );- const clearRecordedBlocks = useRecordedBlocksStore(- (state) => state.clearRecordedBlocks,- );useShouldNotifyWhenClosingTab(!readOnly && workflowChangesStore.hasChanges);const blocker = useBlocker(({ currentLocation, nextLocation }) => {return (@@ -772,7 +760,6 @@ function FlowRenderer({useEffect(() => {if (nodesInitialized && !hasCompletedInitialLoad.current) {- hasCompletedInitialLoad.current = true;doLayout(nodes, edges);// After Dagre computes positions, wait one frame for the DOM to update// with new positions, then fade in the nodes/edges at their final positions.@@ -781,6 +768,11 @@ function FlowRenderer({// After the fade-in animation completes, enable normal transform transitionsfadeTimerRef.current = setTimeout(() => {setLayoutPhase("ready");+ // Mark complete only at the terminal phase. Setting it earlier would+ // strand layoutPhase at "pre-layout" (stuck logo loader) if this async+ // advance is cancelled by a nodesInitialized re-flip — e.g. the+ // recording flow toggling editor visibility or injecting fresh nodes.+ hasCompletedInitialLoad.current = true;}, 350);});return () => {@@ -1214,118 +1206,6 @@ function FlowRenderer({doLayout(nodes, edges);}- // effect to add new blocks that were generated from a browser recording,- // along with any new parameters- useEffect(() => {- if (readOnly) {- return;- }- if (!recordedBlocks || !recordedInsertionPoint) {- return;- }-- const { previous, next, parent, connectingEdgeType } =- recordedInsertionPoint;-- const newNodes: Array<AppNode> = [];- const newEdges: Array<Edge> = [];-- let existingLabels = nodes- .filter(isWorkflowBlockNode)- .map((node) => node.data.label);-- let prevNodeId = previous;-- // convert each WorkflowBlock to an AppNode- recordedBlocks.forEach((block, index) => {- const id = nanoid();- const label = generateNodeLabel(existingLabels);- existingLabels = [...existingLabels, label];- const blockWithLabel = { ...block, label: block.label || label };-- const node = convertToNode(- { id, parentId: parent },- blockWithLabel,- true,- );- newNodes.push(node);-- // create edge from previous node to this one- if (prevNodeId) {- newEdges.push({- id: nanoid(),- type: "edgeWithAddButton",- source: prevNodeId,- target: id,- style: { strokeWidth: 2 },- });- }-- // if this is the last block, connect to next- if (index === recordedBlocks.length - 1 && next) {- newEdges.push({- id: nanoid(),- type: connectingEdgeType,- source: id,- target: next,- style: { strokeWidth: 2 },- });- }-- prevNodeId = id;- });-- const editedEdges = previous- ? edges.filter((edge) => edge.source !== previous)- : edges;-- const previousNode = nodes.find((node) => node.id === previous);- const previousNodeIndex = previousNode- ? nodes.indexOf(previousNode)- : nodes.length - 1;-- const newNodesAfter = [- ...nodes.slice(0, previousNodeIndex + 1),- ...newNodes,- ...nodes.slice(previousNodeIndex + 1),- ];-- workflowChangesStore.setHasChanges(true);- doLayout(newNodesAfter, [...editedEdges, ...newEdges]);-- const newParameters = Array<ParametersState[number]>();-- for (const newParameter of recordedParameters ?? []) {- const exists = parameters.some((param) => param.key === newParameter.key);-- if (!exists) {- newParameters.push({- key: newParameter.key,- parameterType: "workflow",- dataType: newParameter.workflow_parameter_type,- description: newParameter.description ?? null,- defaultValue: newParameter.default_value ?? "",- });- }- }-- if (newParameters.length > 0) {- const workflowParametersStore = useWorkflowParametersStore.getState();- workflowParametersStore.setParameters([- ...workflowParametersStore.parameters,- ...newParameters,- ]);- }-- clearRecordedBlocks();- // Effect runs strictly when the recording store flips a new- // (blocks, insertionPoint) pair into place. nodes/edges/parameters etc.- // are read from the latest closure inside the effect body; including- // them in deps would re-fire on every editor edit and re-apply the- // recorded blocks.- // eslint-disable-next-line react-hooks/exhaustive-deps- }, [recordedBlocks, recordedInsertionPoint]);-const editorElementRef = useRef<HTMLDivElement>(null);// A collapse/accordion animation settles the node at a new height; the layoutdiff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex 45d87d683..80e60b5b7 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -91,6 +91,8 @@ import {import { toast } from "@/components/ui/use-toast";import { DeleteConfirmationDialog } from "@/components/DeleteConfirmationDialog";import { BrowserStream } from "@/components/BrowserStream";+import { RecordingPanel } from "@/routes/workflows/editor/recording/RecordingPanel";+import { useApplyRecordedBlocks } from "@/routes/workflows/editor/recording/useApplyRecordedBlocks";import { statusIsFinalized } from "@/routes/tasks/types.ts";import { CodeEditor } from "@/routes/workflows/components/CodeEditor";import { DebuggerRun } from "@/routes/workflows/debugger/DebuggerRun";@@ -169,6 +171,7 @@ import { WorkflowCopilotChat } from "../copilot/WorkflowCopilotChat";import { useStudioRunId } from "../studio/useStudioRunId";import { copilotRunId } from "./copilotRunId";import { useStudioShellContext } from "../studio/StudioShellContext";+import { useRecordingLauncherStore } from "@/store/useRecordingLauncherStore";import { useStudioPanes } from "../studio/useStudioPanes";import { WorkflowCopilotButton } from "../copilot/WorkflowCopilotButton";import { resolveCopilotLiveBrowserReady } from "../copilot/browserReadiness";@@ -1241,6 +1244,63 @@ function Workspace({[setNodes, setEdges, blockLabel],);+ useApplyRecordedBlocks({+ // Recording runs in the debugger (build + showBrowser), not only on /edit.+ enabled: !workflowPanelState.data?.showComparison,+ nodes,+ edges,+ doLayout,+ });++ // Studio entry point for recording: the Browser-tab Record button lives outside+ // the canvas, so the canvas-aware Workspace registers a launcher that resolves+ // the append-at-end insertion point (the trailing top-level NodeAdder, same as+ // clicking its "+") and starts recording. MVP only appends at the end.+ const setStartRecordingAtEnd = useRecordingLauncherStore(+ (s) => s.setStartRecordingAtEnd,+ );+ // Stable action ref (vs the whole-store `recordingStore` object, which gets a+ // new reference on every store write and would re-register the launcher).+ const setIsRecording = useRecordingStore((s) => s.setIsRecording);+ const startRecordingAtEnd = useCallback(() => {+ const currentNodes = getNodes() as Array<AppNode>;+ const currentEdges = getEdges();+ const trailingAdder = currentNodes.find(+ (node) => node.type === "nodeAdder" && !node.parentId,+ );+ const incomingEdge = trailingAdder+ ? currentEdges.find((edge) => edge.target === trailingAdder.id)+ : undefined;+ setWorkflowPanelState({+ active: false,+ content: "nodeLibrary",+ data: {+ previous: incomingEdge?.source ?? null,+ next: trailingAdder?.id ?? null,+ parent: undefined,+ connectingEdgeType: "default",+ },+ });+ setIsRecording(true, {+ workflowPermanentId: workflowPermanentId ?? null,+ browserSessionId: liveBrowserSessionId,+ });+ }, [+ getNodes,+ getEdges,+ setWorkflowPanelState,+ setIsRecording,+ workflowPermanentId,+ liveBrowserSessionId,+ ]);+ useEffect(() => {+ if (!embedded) {+ return;+ }+ setStartRecordingAtEnd(startRecordingAtEnd);+ return () => setStartRecordingAtEnd(null);+ }, [embedded, startRecordingAtEnd, setStartRecordingAtEnd]);+// Listen for conditional branch changes to trigger re-layoutuseEffect(() => {const handleBranchChange = () => {@@ -2189,7 +2249,7 @@ function Workspace({split={{ left: workflowWidth }}onResize={() => setContainerResizeTrigger((prev) => prev + 1)}>- {/* code, infinite canvas, and block runs */}+ {/* code + canvas; recording overlays the panel without unmounting FlowRenderer */}<div className="relative h-full w-full"><divclassName={cn(@@ -2201,7 +2261,9 @@ function Workspace({{"translate-x-0": showAllCode,},+ recordingStore.isRecording && "pointer-events-none invisible",)}+ aria-hidden={recordingStore.isRecording}ref={dom.splitLeft}>{/* code */}@@ -2261,6 +2323,14 @@ function Workspace({)}</div></div>+ {/* In Studio (embedded) the shell owns the single RecordingPanel+ in the copilot pane; rendering one here too would mount two+ panels that each fire their own commit. */}+ {!embedded && recordingStore.isRecording && (+ <div className="absolute inset-0 z-20 h-full px-6 pb-4 pt-[8.5rem]">+ <RecordingPanel browserSessionId={liveBrowserSessionId} />+ </div>+ )}</div><div className="skyvern-split-right relative flex h-full items-end justify-center bg-neutral-50 p-4 pl-6 dark:bg-background">@@ -2327,13 +2397,19 @@ function Workspace({}}/>)- ) : isFlowCanvasReady ? (+ ) : isFlowCanvasReady || recordingStore.isRecording ? (<BrowserStreamkey={liveBrowserSessionId}- exfiltrate={recordingStore.isRecording}+ exfiltrate={+ recordingStore.isRecording &&+ !recordingStore.finishRequested+ }interactive={true}browserSessionId={liveBrowserSessionId}showControlButtons={true}+ // The recording panel overlays the canvas whenever a+ // recording is live here, so the REC pill is redundant.+ hideRecordingIndicator={true}resizeTrigger={windowResizeTrigger}isExecuting={!!workflowRun && !isFinalized}onReadyChange={handleLiveBrowserReadyChange}@@ -2416,7 +2492,7 @@ function Workspace({}}/>)- ) : isFlowCanvasReady ? (+ ) : isFlowCanvasReady || recordingStore.isRecording ? (<BrowserSessionStreambrowserSessionId={liveBrowserSessionId}interactive={true}diff --git a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsxindex 3561b9bb5..0953a8574 100644--- a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx@@ -97,7 +97,8 @@ function EdgeWithAddButton({// Derive upload state directly from mutation to avoid race conditionsconst isUploadingSOP = sopToBlocksMutation.isPending;- const isProcessing = processRecordingMutation.isPending;+ const isProcessing =+ processRecordingMutation.isPending || recordingStore.isCommitting;const isBusy =(isProcessing || recordingStore.isRecording) &&@@ -151,6 +152,13 @@ function EdgeWithAddButton({};const onEndRecord = () => {+ // With live interpretation, the recording panel owns the commit so user+ // edits/deletes to draft steps are honored.+ if (recordingStore.isRecording && recordingStore.sessionRevision > 0) {+ recordingStore.requestFinish();+ return;+ }+if (recordingStore.isRecording) {recordingStore.setIsRecording(false);}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsxindex 217a534ff..8b6105cc8 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx@@ -113,7 +113,8 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {// Derive upload state directly from mutation to avoid race conditionsconst isUploadingSOP = sopToBlocksMutation.isPending;- const isProcessing = processRecordingMutation.isPending;+ const isProcessing =+ processRecordingMutation.isPending || recordingStore.isCommitting;const isBusy =(isProcessing || recordingStore.isRecording) &&@@ -180,6 +181,13 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {};const onEndRecord = () => {+ // With live interpretation, the recording panel owns the commit so user+ // edits/deletes to draft steps are honored.+ if (recordingStore.isRecording && recordingStore.sessionRevision > 0) {+ recordingStore.requestFinish();+ return;+ }+if (recordingStore.isRecording) {recordingStore.setIsRecording(false);}diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsxnew file mode 100644index 000000000..0bb2a9d12--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx@@ -0,0 +1,659 @@+import {+ CheckIcon,+ CursorArrowIcon,+ EnterFullScreenIcon,+ PauseIcon,+ Pencil1Icon,+ PlayIcon,+ TrashIcon,+ ZoomInIcon,+} from "@radix-ui/react-icons";+import { useEffect, useMemo, useRef, useState } from "react";+import { useShallow } from "zustand/react/shallow";++import { Button } from "@/components/ui/button";+import {+ Dialog,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "@/components/ui/dialog";+import { useRecordingElapsedSeconds } from "@/hooks/useRecordingElapsedSeconds";+import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";+import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";+import {+ applyDraftStepOverlays,+ findScreenshotForStep,+ useRecordingStore,+ type RecordingActionKind,+ type RecordingDraftStep,+ type RecordingScreenshot,+} from "@/store/useRecordingStore";+import { captureRecordBrowser } from "@/util/recordBrowserTelemetry";+import { formatRecordingClock } from "@/util/recordingClock";+import { cn } from "@/util/utils";+import { buildDraftStepTitlePatch } from "./recordingDraftStepEdits";++const KIND_LABELS: Record<RecordingActionKind, string> = {+ click: "Click",+ hover: "Hover",+ input_text: "Input text",+ url_change: "Navigation",+ wait: "Wait",+};++/**+ * How long Done waits for the backend's finalized interpretation snapshot+ * (flushed on end-exfiltration) before committing whatever drafts we have.+ */+const FINALIZE_TIMEOUT_MS = 5000;++function shortUrl(url: string | null | undefined): string {+ if (!url) {+ return "";+ }+ try {+ const parsed = new URL(url);+ return parsed.host + (parsed.pathname === "/" ? "" : parsed.pathname);+ } catch {+ return url;+ }+}++function formatDraftStepDisplayTitle(step: RecordingDraftStep): string {+ if (step.title?.trim()) {+ return step.title.trim();+ }++ if (step.action_kind === "url_change" || step.block_type === "goto_url") {+ const destination = shortUrl(step.url);+ if (destination) {+ return `Go to ${destination}`;+ }+ }++ if (step.label.startsWith("goto_")) {+ const slug = step.label.slice("goto_".length).replace(/_/g, ".");+ return slug ? `Go to ${slug}` : "Go to page";+ }++ return step.label;+}++function StepScreenshot({ screenshot }: { screenshot: RecordingScreenshot }) {+ const [showFullPage, setShowFullPage] = useState(false);+ const xp = screenshot.xp ?? 0.5;+ const yp = screenshot.yp ?? 0.5;++ return (+ <div className="group/shot relative h-36 overflow-hidden rounded-md border bg-black">+ <div+ className="absolute inset-0"+ style={+ showFullPage+ ? {+ backgroundImage: `url(${screenshot.dataUrl})`,+ backgroundSize: "contain",+ backgroundRepeat: "no-repeat",+ backgroundPosition: "center",+ }+ : {+ // background-position p% pins image point p% to container+ // point p%, keeping the zoomed crop in-bounds with the click+ // point at the same relative spot as the ring below.+ backgroundImage: `url(${screenshot.dataUrl})`,+ backgroundSize: "250% auto",+ backgroundRepeat: "no-repeat",+ backgroundPosition: `${xp * 100}% ${yp * 100}%`,+ }+ }+ />+ <button+ type="button"+ className="absolute bottom-2 right-2 inline-flex items-center gap-1 rounded bg-black/70 px-2 py-1 text-[10.5px] text-slate-200 opacity-0 transition-opacity group-hover/shot:opacity-100"+ onClick={() => setShowFullPage(!showFullPage)}+ >+ {showFullPage ? (+ <>+ <ZoomInIcon className="h-3 w-3" />+ Zoom to action+ </>+ ) : (+ <>+ <EnterFullScreenIcon className="h-3 w-3" />+ Full page+ </>+ )}+ </button>+ </div>+ );+}++function DraftStepCard({+ step,+ index,+ baselineMs,+ screenshot,+ onDelete,+ onRename,+}: {+ step: RecordingDraftStep;+ index: number;+ baselineMs: number | null;+ screenshot: RecordingScreenshot | null;+ onDelete: () => void;+ onRename: (value: string) => void;+}) {+ const beginDraftEdit = useRecordingStore((state) => state.beginDraftEdit);+ const endDraftEdit = useRecordingStore((state) => state.endDraftEdit);+ const [isEditing, setIsEditing] = useState(false);+ const displayTitle = formatDraftStepDisplayTitle(step);+ const [draftTitle, setDraftTitle] = useState(displayTitle);++ useEffect(() => {+ if (!isEditing) {+ return;+ }+ beginDraftEdit();+ return () => {+ endDraftEdit();+ };+ }, [isEditing, beginDraftEdit, endDraftEdit]);++ useEffect(() => {+ if (!isEditing) {+ setDraftTitle(displayTitle);+ }+ }, [displayTitle, isEditing]);++ const saveTitle = () => {+ const trimmed = draftTitle.trim();+ if (trimmed && trimmed !== displayTitle) {+ onRename(trimmed);+ }+ setIsEditing(false);+ };++ const relativeSeconds =+ baselineMs !== null &&+ step.timestamp_start !== null &&+ step.timestamp_start !== undefined+ ? (step.timestamp_start - baselineMs) / 1000+ : null;+ const meta = shortUrl(step.url);++ return (+ <div className="group relative flex flex-col gap-2 rounded-[10px] p-3 transition-colors hover:bg-slate-elevation3">+ <div className="flex items-start gap-2.5">+ <span className="mt-px flex h-[22px] w-[22px] flex-none items-center justify-center rounded-full bg-slate-elevation5 text-[11px] font-semibold text-foreground">+ {index + 1}+ </span>+ <div className="min-w-0 flex-1">+ {isEditing ? (+ <input+ autoFocus+ className="w-full rounded border bg-background px-1.5 py-0.5 text-[13px] font-medium text-foreground outline-none focus:ring-1 focus:ring-ring"+ value={draftTitle}+ onChange={(e) => setDraftTitle(e.target.value)}+ onBlur={saveTitle}+ onKeyDown={(e) => {+ if (e.key === "Enter") {+ saveTitle();+ }+ if (e.key === "Escape") {+ setDraftTitle(displayTitle);+ setIsEditing(false);+ }+ }}+ />+ ) : (+ <div+ className="cursor-text rounded text-[13px] font-medium leading-snug text-foreground hover:bg-slate-elevation5 hover:shadow-[0_0_0_4px_hsl(var(--slate-elevation-5))]"+ role="button"+ tabIndex={0}+ title="Click to edit"+ onClick={() => setIsEditing(true)}+ onKeyDown={(event) => {+ if (event.key === "Enter" || event.key === " ") {+ event.preventDefault();+ setIsEditing(true);+ }+ }}+ >+ {displayTitle}+ </div>+ )}+ <div className="mt-0.5 truncate text-[11px] text-muted-foreground">+ {KIND_LABELS[step.action_kind] ?? step.action_kind}+ {relativeSeconds !== null &&+ ` · ${formatRecordingClock(relativeSeconds)}`}+ {meta && (+ <>+ {" · "}+ <span className="font-mono">{meta}</span>+ </>+ )}+ {step.status === "interpreting" && (+ <span className="animate-pulse text-yellow-500">+ {" · refining…"}+ </span>+ )}+ </div>+ </div>+ <div className="flex flex-none gap-0.5 opacity-0 transition-opacity group-hover:opacity-100">+ <button+ type="button"+ title="Edit title"+ className="flex h-6 w-6 items-center justify-center rounded text-muted-foreground hover:bg-slate-elevation5 hover:text-foreground"+ onClick={() => setIsEditing(true)}+ >+ <Pencil1Icon className="h-3.5 w-3.5" />+ </button>+ <button+ type="button"+ title="Delete block"+ className="flex h-6 w-6 items-center justify-center rounded text-muted-foreground hover:bg-slate-elevation5 hover:text-red-400"+ onClick={onDelete}+ >+ <TrashIcon className="h-3.5 w-3.5" />+ </button>+ </div>+ </div>+ {screenshot && (+ <div className="pl-8">+ <StepScreenshot screenshot={screenshot} />+ </div>+ )}+ </div>+ );+}++function InterpretingRow({+ index,+ label,+ title,+}: {+ index: number;+ label: string;+ title?: string;+}) {+ return (+ <div className="flex items-start gap-2.5 p-3">+ <span className="mt-px flex h-[22px] w-[22px] flex-none items-center justify-center rounded-full bg-slate-elevation5 text-[11px] font-semibold text-muted-foreground">+ {index + 1}+ </span>+ <div className="min-w-0 flex-1">+ {title ? (+ <div className="truncate text-[13px] font-medium leading-snug text-foreground">+ {title}+ </div>+ ) : (+ <div className="mb-1.5 h-3 w-3/5 animate-pulse rounded bg-slate-elevation5" />+ )}+ <div className="text-[11px] text-yellow-500">{label}</div>+ </div>+ </div>+ );+}++type Props = {+ browserSessionId: string | null;+};++function RecordingPanel({ browserSessionId }: Props) {+ const setRecordedBlocks = useRecordedBlocksStore(+ (state) => state.setRecordedBlocks,+ );+ // Captured once on mount via getState() — deliberately not a subscription:+ // recording starters stash their insertion point in the workflow panel store+ // right before recording begins, and later panel-state changes (e.g. the user+ // browsing the node library mid-recording) must not move the commit target.+ const [insertionPointState] = useState(() => {+ const data = useWorkflowPanelStore.getState().workflowPanelState.data;+ if (!data) {+ captureRecordBrowser("record_browser.missing_insertion_point");+ }+ return {+ insertionPoint: {+ previous: data?.previous ?? null,+ next: data?.next ?? null,+ parent: data?.parent,+ connectingEdgeType: data?.connectingEdgeType ?? "default",+ },+ isValid: data !== null && data !== undefined,+ };+ });+ const insertionPoint = insertionPointState.insertionPoint;+ const insertionPointMissing = !insertionPointState.isValid;+ const [confirmDiscardOpen, setConfirmDiscardOpen] = useState(false);+ const feedEndRef = useRef<HTMLDivElement | null>(null);+ const committedRef = useRef(false);++ // Slice the frequently-changing fields with useShallow so live-recording+ // updates (optimistic appends, exposedEventCount ticks) only re-render when a+ // field this panel reads actually changes, not on every exfiltrated event.+ const {+ draftSteps,+ deletedStepIds,+ stepPatches,+ screenshots,+ sessionRevision,+ optimisticSteps: rawOptimisticSteps,+ workflowPermanentId,+ interpretationPending,+ interpretationFinalized,+ finishRequested,+ isCommitting,+ exposedEventCount,+ manualCapturePaused,+ } = useRecordingStore(+ useShallow((state) => ({+ draftSteps: state.draftSteps,+ deletedStepIds: state.deletedStepIds,+ stepPatches: state.stepPatches,+ screenshots: state.screenshots,+ sessionRevision: state.sessionRevision,+ optimisticSteps: state.optimisticSteps,+ workflowPermanentId: state.workflowPermanentId,+ interpretationPending: state.interpretationPending,+ interpretationFinalized: state.interpretationFinalized,+ finishRequested: state.finishRequested,+ isCommitting: state.isCommitting,+ exposedEventCount: state.exposedEventCount,+ manualCapturePaused: state.manualCapturePaused,+ })),+ );++ const interpretationEnabled = workflowPermanentId !== null;++ const visibleSteps = useMemo(+ () => applyDraftStepOverlays(draftSteps, deletedStepIds, stepPatches),+ [draftSteps, deletedStepIds, stepPatches],+ );++ // Surface optimistic placeholders whenever interpretation is enabled (not just+ // after the first snapshot), so the first steps appear without a round-trip.+ // When interpretation is disabled they would accumulate unreconciled, so hide.+ const optimisticSteps = interpretationEnabled ? rawOptimisticSteps : [];+ const actionCount = visibleSteps.length + optimisticSteps.length;++ // Step times are remote-browser clocks; anchor to the first step instead of+ // the operator's local clock to dodge skew.+ const baselineMs = useMemo(() => {+ for (const step of visibleSteps) {+ if (step.timestamp_start !== null && step.timestamp_start !== undefined) {+ return step.timestamp_start;+ }+ }+ return null;+ }, [visibleSteps]);++ const processRecordingMutation = useProcessRecordingMutation({+ browserSessionId,+ onSuccess: (result) => {+ setRecordedBlocks(result, insertionPoint);+ useRecordingStore.getState().setIsRecording(false);+ },+ });++ const mutationIsPending = processRecordingMutation.isPending;+ const setIsCommitting = useRecordingStore((state) => state.setIsCommitting);+ useEffect(() => {+ setIsCommitting(mutationIsPending);+ }, [mutationIsPending, setIsCommitting]);++ const mutationIsError = processRecordingMutation.isError;+ useEffect(() => {+ if (mutationIsError) {+ // Allow Done to retry with the drafts we still hold.+ committedRef.current = false;+ }+ }, [mutationIsError]);++ const commit = () => {+ if (committedRef.current || insertionPointMissing) {+ return;+ }+ committedRef.current = true;+ processRecordingMutation.mutate({+ draftSteps: useRecordingStore.getState().getFinalDraftSteps(),+ });+ };+ const commitRef = useRef(commit);+ commitRef.current = commit;++ // Done flow: requestFinish stops exfiltration; the backend flushes a final+ // interpretation snapshot, then we commit (or commit anyway on timeout).+ // commitRef keeps the timeout callback stable; mutationIsError clears+ // committedRef so the timeout can fire again on retry.+ useEffect(() => {+ if (!finishRequested || committedRef.current) {+ return;+ }+ if (interpretationFinalized || sessionRevision === 0) {+ commitRef.current();+ return;+ }+ const timeout = setTimeout(() => commitRef.current(), FINALIZE_TIMEOUT_MS);+ return () => clearTimeout(timeout);+ }, [finishRequested, interpretationFinalized, sessionRevision]);++ // keep the newest block in view+ useEffect(() => {+ feedEndRef.current?.scrollIntoView({ behavior: "smooth", block: "end" });+ }, [actionCount, interpretationPending]);++ const elapsedSeconds = useRecordingElapsedSeconds();++ const discard = () => {+ const store = useRecordingStore.getState();+ captureRecordBrowser("record_browser.cancelled", {+ event_count_at_cancel: store.getEventCount(),+ seconds_recording: store.getSecondsRecording(),+ });+ setConfirmDiscardOpen(false);+ store.setIsRecording(false);+ store.reset();+ };++ const onDiscardClick = () => {+ if (visibleSteps.length > 0 || exposedEventCount > 0) {+ setConfirmDiscardOpen(true);+ } else {+ discard();+ }+ };++ const onDoneClick = () => {+ if (insertionPointMissing) {+ return;+ }+ if (finishRequested) {+ // A previous commit attempt failed; retry directly.+ commit();+ return;+ }+ useRecordingStore.getState().requestFinish();+ };++ const isFinishing = finishRequested || isCommitting;+ const showInterpretationFallbackNote =+ !interpretationEnabled && exposedEventCount > 0;+ const headerTitle = isFinishing+ ? "Finishing recording"+ : manualCapturePaused+ ? "Recording paused"+ : "Recording browser";++ return (+ <div className="flex h-full w-full flex-col overflow-hidden rounded-xl border bg-slate-elevation2">+ {/* header */}+ <div className="flex flex-none items-center gap-2.5 border-b px-3.5 py-3">+ <span+ className={cn("h-2.5 w-2.5 flex-none rounded-full bg-red-500", {+ "animate-pulse": !isFinishing && !manualCapturePaused,+ "opacity-60": isFinishing || manualCapturePaused,+ })}+ />+ <div className="min-w-0 flex-1">+ <div className="text-[13px] font-semibold text-foreground">+ {headerTitle}+ </div>+ <div className="font-mono text-[11.5px] text-muted-foreground">+ {formatRecordingClock(elapsedSeconds)} · {actionCount} action+ {actionCount === 1 ? "" : "s"}+ </div>+ </div>+ </div>++ {/* feed */}+ <div className="flex min-h-0 flex-1 flex-col gap-1 overflow-y-auto px-2 pb-3 pt-2">+ {visibleSteps.length === 0 &&+ optimisticSteps.length === 0 &&+ !interpretationPending ? (+ <div className="flex flex-col items-center justify-center gap-2.5 px-5 py-10 text-center text-xs leading-relaxed text-muted-foreground">+ <CursorArrowIcon className="h-5 w-5" />+ {manualCapturePaused ? (+ <>+ Capture is paused.+ <br />+ Resume to continue recording browser actions.+ </>+ ) : (+ <>+ Interact with the browser.+ <br />+ Your clicks, typing and navigation will appear here as blocks.+ </>+ )}+ </div>+ ) : (+ visibleSteps.map((step, index) => (+ <DraftStepCard+ key={step.step_id}+ step={step}+ index={index}+ baselineMs={baselineMs}+ screenshot={findScreenshotForStep(step, screenshots)}+ onDelete={() =>+ useRecordingStore.getState().deleteDraftStep(step.step_id)+ }+ onRename={(value) =>+ useRecordingStore+ .getState()+ .patchDraftStep(+ step.step_id,+ buildDraftStepTitlePatch(step, value),+ )+ }+ />+ ))+ )}+ {optimisticSteps.map((step, i) => (+ <InterpretingRow+ key={step.local_id}+ index={visibleSteps.length + i}+ label={isFinishing ? "Finalizing…" : "Interpreting…"}+ title={step.title}+ />+ ))}+ {interpretationPending && optimisticSteps.length === 0 && (+ <InterpretingRow+ index={visibleSteps.length}+ label={isFinishing ? "Finalizing…" : "Interpreting…"}+ />+ )}+ {showInterpretationFallbackNote && (+ <div className="px-3 py-2 text-[11px] leading-relaxed text-muted-foreground">+ {exposedEventCount} interaction+ {exposedEventCount === 1 ? "" : "s"} captured — blocks will be+ generated when you finish.+ </div>+ )}+ <div ref={feedEndRef} />+ </div>++ {insertionPointMissing && (+ <div className="flex-none border-t px-3.5 py-2 text-[11px] leading-relaxed text-red-400">+ Could not determine where to insert recorded blocks. Discard and start+ recording from the workflow editor again.+ </div>+ )}++ {/* controls */}+ <div className="flex flex-none items-center gap-2 border-t px-3 py-2.5">+ <Button+ variant="outline"+ size="icon"+ title={manualCapturePaused ? "Resume capture" : "Pause capture"}+ className="h-8 w-8"+ disabled={isCommitting || isFinishing}+ onClick={() =>+ useRecordingStore+ .getState()+ .setManualCapturePaused(!manualCapturePaused)+ }+ >+ {manualCapturePaused ? (+ <PlayIcon className="h-4 w-4" />+ ) : (+ <PauseIcon className="h-4 w-4" />+ )}+ </Button>+ <Button+ variant="outline"+ size="icon"+ title="Discard recording"+ className="h-8 w-8 hover:border-red-500/40 hover:text-red-400"+ disabled={isCommitting}+ onClick={onDiscardClick}+ >+ <TrashIcon className="h-4 w-4" />+ </Button>+ <Button+ size="sm"+ className="ml-auto h-8"+ disabled={insertionPointMissing || (isFinishing && !mutationIsError)}+ onClick={onDoneClick}+ >+ <CheckIcon className="mr-1.5 h-4 w-4" />+ {mutationIsError ? "Retry" : isFinishing ? "Processing…" : "Done"}+ </Button>+ </div>++ {confirmDiscardOpen && (+ <Dialog open onOpenChange={setConfirmDiscardOpen}>+ <DialogContent>+ <DialogHeader>+ <DialogTitle>Discard recording?</DialogTitle>+ <DialogDescription>+ {visibleSteps.length > 0+ ? `You have ${visibleSteps.length} recorded block${+ visibleSteps.length === 1 ? "" : "s"+ } that will be lost if you discard.`+ : "Your recorded interactions will be lost if you discard."}{" "}+ Are you sure you want to discard the recording?+ </DialogDescription>+ </DialogHeader>+ <DialogFooter>+ <Button+ variant="outline"+ onClick={() => setConfirmDiscardOpen(false)}+ >+ Keep recording+ </Button>+ <Button variant="destructive" onClick={discard}>+ Discard recording+ </Button>+ </DialogFooter>+ </DialogContent>+ </Dialog>+ )}+ </div>+ );+}++export { RecordingPanel };diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/applyRecordedBlocksToGraph.ts b/skyvern-frontend/src/routes/workflows/editor/recording/applyRecordedBlocksToGraph.tsnew file mode 100644index 000000000..b1ddd8f29--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/recording/applyRecordedBlocksToGraph.ts@@ -0,0 +1,120 @@+import { Edge } from "@xyflow/react";+import { nanoid } from "nanoid";++import type {+ WorkflowBlock,+ WorkflowParameter,+} from "@/routes/workflows/types/workflowTypes";+import type { InsertionPoint } from "@/store/RecordedBlocksStore";+import { AppNode, isWorkflowBlockNode } from "../nodes";+import { ParametersState } from "../types";+import { convertToNode, generateNodeLabel } from "../workflowEditorUtils";++type ApplyRecordedBlocksArgs = {+ nodes: Array<AppNode>;+ edges: Array<Edge>;+ recordedBlocks: Array<WorkflowBlock>;+ recordedInsertionPoint: InsertionPoint;+ recordedParameters: Array<WorkflowParameter> | null;+ existingParameters: ParametersState;+};++type ApplyRecordedBlocksResult = {+ nodes: Array<AppNode>;+ edges: Array<Edge>;+ newParameters: ParametersState;+};++function applyRecordedBlocksToGraph({+ nodes,+ edges,+ recordedBlocks,+ recordedInsertionPoint,+ recordedParameters,+ existingParameters,+}: ApplyRecordedBlocksArgs): ApplyRecordedBlocksResult {+ const { previous, next, parent, connectingEdgeType } = recordedInsertionPoint;++ const newNodes: Array<AppNode> = [];+ const newEdges: Array<Edge> = [];++ let existingLabels = nodes+ .filter(isWorkflowBlockNode)+ .map((node) => node.data.label);++ let prevNodeId = previous;++ recordedBlocks.forEach((block, index) => {+ const id = nanoid();+ const label = generateNodeLabel(existingLabels);+ existingLabels = [...existingLabels, label];+ const blockWithLabel = { ...block, label: block.label || label };++ const node = convertToNode({ id, parentId: parent }, blockWithLabel, true);+ newNodes.push(node);++ if (prevNodeId) {+ newEdges.push({+ id: nanoid(),+ type: "edgeWithAddButton",+ source: prevNodeId,+ target: id,+ style: { strokeWidth: 2 },+ });+ }++ if (index === recordedBlocks.length - 1 && next) {+ newEdges.push({+ id: nanoid(),+ type: connectingEdgeType,+ source: id,+ target: next,+ style: { strokeWidth: 2 },+ });+ }++ prevNodeId = id;+ });++ const editedEdges = previous+ ? edges.filter((edge) => edge.source !== previous)+ : edges;++ const previousNode = nodes.find((node) => node.id === previous);+ const previousNodeIndex = previousNode+ ? nodes.indexOf(previousNode)+ : nodes.length - 1;++ const mergedNodes = [+ ...nodes.slice(0, previousNodeIndex + 1),+ ...newNodes,+ ...nodes.slice(previousNodeIndex + 1),+ ];++ const newParameters: ParametersState = [];++ for (const newParameter of recordedParameters ?? []) {+ const exists = existingParameters.some(+ (param) => param.key === newParameter.key,+ );++ if (!exists) {+ newParameters.push({+ key: newParameter.key,+ parameterType: "workflow",+ dataType: newParameter.workflow_parameter_type,+ description: newParameter.description ?? null,+ defaultValue: newParameter.default_value ?? "",+ });+ }+ }++ return {+ nodes: mergedNodes,+ edges: [...editedEdges, ...newEdges],+ newParameters,+ };+}++export { applyRecordedBlocksToGraph };+export type { ApplyRecordedBlocksArgs, ApplyRecordedBlocksResult };diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.test.ts b/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.test.tsnew file mode 100644index 000000000..7d689d80b--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.test.ts@@ -0,0 +1,102 @@+import { describe, expect, it } from "vitest";++import type { MessageInExfiltratedConsoleEvent } from "@/store/useRecordingStore";++import { buildOptimisticStep } from "./optimisticSteps";++function consoleEvent(+ type: string,+ overrides: Partial<MessageInExfiltratedConsoleEvent["params"]> = {},+): MessageInExfiltratedConsoleEvent {+ return {+ kind: "exfiltrated-event",+ event_name: "user_interaction",+ source: "console",+ timestamp: 1.0,+ params: {+ type,+ url: "https://example.com",+ timestamp: 1000,+ target: { tagName: "BUTTON", text: ["Submit"] },+ mousePosition: { xa: null, ya: null, xp: 0.5, yp: 0.5 },+ activeElement: { tagName: "BUTTON" },+ window: { width: 1200, height: 800, scrollX: 0, scrollY: 0 },+ ...overrides,+ },+ };+}++describe("buildOptimisticStep", () => {+ it("builds a click placeholder from target text", () => {+ const step = buildOptimisticStep(consoleEvent("click"));+ expect(step?.action_kind).toBe("click");+ expect(step?.title).toBe("Click 'Submit'");+ });++ it("falls back through innerText then tag name for click text", () => {+ expect(+ buildOptimisticStep(+ consoleEvent("click", {+ target: { tagName: "DIV", text: [], innerText: "Read more" },+ }),+ )?.title,+ ).toBe("Click 'Read more'");++ expect(+ buildOptimisticStep(+ consoleEvent("click", { target: { tagName: "DIV", text: [] } }),+ )?.title,+ ).toBe("Click 'div'");+ });++ it("builds an input_text placeholder on change/blur with a value", () => {+ const changeStep = buildOptimisticStep(+ consoleEvent("change", {+ target: { tagName: "INPUT", text: ["Email"], value: "[email protected]" },+ }),+ );+ expect(changeStep?.action_kind).toBe("input_text");+ expect(changeStep?.title).toBe("Fill 'Email'");++ expect(+ buildOptimisticStep(+ consoleEvent("blur", {+ target: { tagName: "INPUT", text: ["Email"] },+ inputValue: "typed",+ }),+ )?.action_kind,+ ).toBe("input_text");+ });++ it("returns null for change/blur without a value", () => {+ expect(+ buildOptimisticStep(+ consoleEvent("blur", { target: { tagName: "INPUT", text: ["Email"] } }),+ ),+ ).toBeNull();+ expect(+ buildOptimisticStep(+ consoleEvent("change", {+ target: { tagName: "INPUT", text: ["Email"], value: " " },+ }),+ ),+ ).toBeNull();+ });++ it("elides events that do not map to a step", () => {+ for (const type of ["focus", "input", "keydown", "keypress", "mousemove"]) {+ expect(buildOptimisticStep(consoleEvent(type))).toBeNull();+ }+ });++ it("elides navigation (cdp) events entirely", () => {+ const nav = {+ kind: "exfiltrated-event" as const,+ event_name: "nav:frame_navigated",+ source: "cdp" as const,+ timestamp: 2.0,+ params: { targetInfo: { url: "https://example.com/next" } },+ };+ expect(buildOptimisticStep(nav)).toBeNull();+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.ts b/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.tsnew file mode 100644index 000000000..29e53f2e0--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/recording/optimisticSteps.ts@@ -0,0 +1,68 @@+import {+ nextOptimisticStepId,+ type MessageInExfiltratedEvent,+ type OptimisticStep,+} from "@/store/useRecordingStore";++/**+ * Human label for an interaction target, mirroring the backend's+ * `_action_display_text` (interpretation.py) so optimistic titles match the+ * interpreted ones they are replaced by.+ */+function actionDisplayText(target: {+ text?: Array<string>;+ innerText?: string | null;+ tagName?: string | null;+}): string {+ for (const text of target.text ?? []) {+ const cleaned = text.split(/\s+/).filter(Boolean).join(" ");+ if (cleaned) {+ return cleaned.slice(0, 80);+ }+ }+ const inner = target.innerText?.split(/\s+/).filter(Boolean).join(" ");+ if (inner) {+ return inner.slice(0, 80);+ }+ return (target.tagName ?? "element").toLowerCase();+}++/**+ * Build an optimistic placeholder for a single exfiltrated event, or null when+ * the event does not map to a step. Only clicks and the end of an input+ * (change/blur with a value) produce placeholders — keystrokes, focus, and+ * navigations are elided (a link-click navigation is already represented by its+ * click, and the backend emits no goto for it).+ */+export function buildOptimisticStep(+ message: MessageInExfiltratedEvent,+): OptimisticStep | null {+ if (message.source !== "console") {+ return null;+ }++ const { params } = message;+ if (params.type === "click") {+ return {+ local_id: nextOptimisticStepId(),+ action_kind: "click",+ title: `Click '${actionDisplayText(params.target)}'`,+ timestamp: params.timestamp,+ };+ }++ if (params.type === "change" || params.type === "blur") {+ const value = params.inputValue ?? params.target.value ?? "";+ if (value.trim() === "") {+ return null;+ }+ return {+ local_id: nextOptimisticStepId(),+ action_kind: "input_text",+ title: `Fill '${actionDisplayText(params.target)}'`,+ timestamp: params.timestamp,+ };+ }++ return null;+}diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.test.ts b/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.test.tsnew file mode 100644index 000000000..b77c25b76--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.test.ts@@ -0,0 +1,64 @@+import { describe, expect, it } from "vitest";++import type { RecordingDraftStep } from "@/store/useRecordingStore";++import {+ buildDraftStepTitlePatch,+ deriveUrlFromNavigationTitle,+} from "./recordingDraftStepEdits";++const navigationStep: RecordingDraftStep = {+ step_id: "bs-recording-step-0",+ action_kind: "url_change",+ block_type: "goto_url",+ label: "goto_wikipedia_com",+ url: "https://www.wikipedia.com/wiki/Foo",+ status: "ready",+ editable_fields: ["label", "url"],+ parameters: [],+ parameter_keys: [],+};++describe("deriveUrlFromNavigationTitle", () => {+ it("updates hostname while preserving path from the original URL", () => {+ expect(+ deriveUrlFromNavigationTitle(navigationStep.url, "Go to wikipedia.org"),+ ).toBe("https://wikipedia.org/wiki/Foo");+ });++ it("accepts a full URL in the title", () => {+ expect(+ deriveUrlFromNavigationTitle(+ navigationStep.url,+ "Go to https://example.org/start",+ ),+ ).toBe("https://example.org/start");+ });+});++describe("buildDraftStepTitlePatch", () => {+ it("patches url and label for navigation steps", () => {+ expect(+ buildDraftStepTitlePatch(navigationStep, "Go to wikipedia.org"),+ ).toEqual({+ title: "Go to wikipedia.org",+ url: "https://wikipedia.org/wiki/Foo",+ label: "Go_to_wikipedia_org",+ });+ });++ it("patches title only for action steps", () => {+ const actionStep: RecordingDraftStep = {+ ...navigationStep,+ action_kind: "click",+ block_type: "action",+ title: "Click 'Search'",+ };++ expect(+ buildDraftStepTitlePatch(actionStep, "Click the search button"),+ ).toEqual({+ title: "Click the search button",+ });+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.ts b/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.tsnew file mode 100644index 000000000..befab5709--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/recording/recordingDraftStepEdits.ts@@ -0,0 +1,118 @@+import {+ type RecordingDraftStep,+ type RecordingDraftStepPatch,+} from "@/store/useRecordingStore";++function isNavigationDraftStep(step: RecordingDraftStep): boolean {+ return step.action_kind === "url_change" || step.block_type === "goto_url";+}++function normalizeRecordingBlockLabel(label: string, fallback: string): string {+ let candidate = label.trim().replace(/\W+/g, "_").replace(/_+/g, "_");+ candidate = candidate.replace(/^_|_$/g, "");++ if (!candidate) {+ return fallback;+ }++ if (!/^[A-Za-z_]/.test(candidate)) {+ candidate = `${fallback}_${candidate}`;+ }++ return candidate;+}++/** Mirrors `deterministic_goto_url_label` in browser_recording/service.py. */+function deriveGotoUrlLabelFromUrl(url: string): string {+ try {+ const host = new URL(url).hostname;+ if (!host) {+ return "goto_url";+ }+ return normalizeRecordingBlockLabel(`goto_${host}`, "goto_url");+ } catch {+ return "goto_url";+ }+}++/**+ * When the operator renames "Go to wikipedia.com" → "Go to wikipedia.org", map+ * that display edit onto the goto_url block's `url` field used at commit time.+ */+function deriveUrlFromNavigationTitle(+ originalUrl: string | null | undefined,+ title: string,+): string | null {+ const trimmed = title.trim();+ if (!trimmed) {+ return null;+ }++ const goToMatch = /^go to\s+(.+)$/i.exec(trimmed);+ const destination = goToMatch?.[1]?.trim() ?? trimmed;++ try {+ if (/^https?:\/\//i.test(destination)) {+ return new URL(destination).toString();+ }++ const destinationHasPath =+ destination.includes("/") &&+ !destination.startsWith("/") &&+ destination.indexOf("/") < destination.length - 1;++ const destinationUrl = new URL(+ destination.startsWith("//")+ ? `https:${destination}`+ : `https://${destination}`,+ );++ if (!originalUrl) {+ return destinationUrl.toString();+ }++ const original = new URL(originalUrl);+ original.protocol = destinationUrl.protocol;+ original.hostname = destinationUrl.hostname;+ original.port = destinationUrl.port;++ if (destinationHasPath) {+ original.pathname = destinationUrl.pathname;+ original.search = destinationUrl.search;+ if (destinationUrl.hash) {+ original.hash = destinationUrl.hash;+ }+ }++ return original.toString();+ } catch {+ return null;+ }+}++function buildDraftStepTitlePatch(+ step: RecordingDraftStep,+ newTitle: string,+): RecordingDraftStepPatch {+ const title = newTitle.trim();+ const patch: RecordingDraftStepPatch = { title };++ if (isNavigationDraftStep(step)) {+ const url = deriveUrlFromNavigationTitle(step.url, title);+ if (url) {+ patch.url = url;+ const fallbackLabel = deriveGotoUrlLabelFromUrl(url);+ patch.label = normalizeRecordingBlockLabel(title, fallbackLabel);+ }+ }++ return patch;+}++export {+ buildDraftStepTitlePatch,+ deriveGotoUrlLabelFromUrl,+ deriveUrlFromNavigationTitle,+ isNavigationDraftStep,+ normalizeRecordingBlockLabel,+};diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.test.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.test.tsxnew file mode 100644index 000000000..a3815f23e--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.test.tsx@@ -0,0 +1,134 @@+// @vitest-environment jsdom++import { Edge } from "@xyflow/react";+import { renderHook } from "@testing-library/react";+import { afterEach, describe, expect, it, vi } from "vitest";++import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";+import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes";+import { AppNode } from "../nodes";+import { useApplyRecordedBlocks } from "./useApplyRecordedBlocks";++const initialRecordedBlocksState = useRecordedBlocksStore.getState();++describe("useApplyRecordedBlocks", () => {+ afterEach(() => {+ useRecordedBlocksStore.setState(initialRecordedBlocksState, true);+ });++ it("applies recorded blocks when enabled in debugger/build mode", () => {+ const doLayout = vi.fn();+ const nodes = [{ id: "start", data: { label: "start" } }] as Array<AppNode>;+ const edges = [] as Array<Edge>;++ useRecordedBlocksStore.getState().setRecordedBlocks(+ {+ blocks: [+ {+ block_type: "action",+ label: "click_button",+ title: "Click button",+ navigation_goal: "Click the button.",+ url: null,+ parameters: [],+ } as unknown as WorkflowBlock,+ ],+ parameters: [],+ },+ {+ previous: "start",+ next: null,+ connectingEdgeType: "edgeWithAddButton",+ },+ );++ renderHook(() =>+ useApplyRecordedBlocks({+ enabled: true,+ nodes,+ edges,+ doLayout,+ }),+ );++ expect(doLayout).toHaveBeenCalledTimes(1);+ const layoutArgs = doLayout.mock.calls[0];+ expect(layoutArgs).toBeDefined();+ const [mergedNodes, mergedEdges] = layoutArgs!;+ expect(mergedNodes).toHaveLength(2);+ expect(mergedEdges.length).toBeGreaterThan(0);+ });++ it("does not apply recorded blocks when disabled", () => {+ const doLayout = vi.fn();++ useRecordedBlocksStore.getState().setRecordedBlocks(+ {+ blocks: [+ {+ block_type: "goto_url",+ label: "goto_home",+ url: "https://example.com",+ } as unknown as WorkflowBlock,+ ],+ parameters: [],+ },+ {+ previous: null,+ next: null,+ connectingEdgeType: "default",+ },+ );++ renderHook(() =>+ useApplyRecordedBlocks({+ enabled: false,+ nodes: [],+ edges: [],+ doLayout,+ }),+ );++ expect(doLayout).not.toHaveBeenCalled();+ });++ it("clears a pending payload when the consumer unmounts (interrupted handoff)", () => {+ const doLayout = vi.fn();++ // enabled: false keeps the payload pending — mirrors a handoff that never+ // applied before the canvas went away (e.g. navigating right after commit).+ const { unmount } = renderHook(() =>+ useApplyRecordedBlocks({+ enabled: false,+ nodes: [],+ edges: [],+ doLayout,+ }),+ );++ useRecordedBlocksStore.getState().setRecordedBlocks(+ {+ blocks: [+ {+ block_type: "goto_url",+ label: "goto_home",+ url: "https://example.com",+ } as unknown as WorkflowBlock,+ ],+ parameters: [],+ },+ {+ previous: null,+ next: null,+ connectingEdgeType: "default",+ },+ );++ unmount();++ const state = useRecordedBlocksStore.getState();+ expect(state.blocks).toBeNull();+ expect(state.parameters).toBeNull();+ expect(state.insertionPoint).toBeNull();+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.ts b/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.tsnew file mode 100644index 000000000..77af90faa--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/recording/useApplyRecordedBlocks.ts@@ -0,0 +1,122 @@+import { Edge } from "@xyflow/react";+import { useEffect, useRef } from "react";++import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";+import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";+import { useWorkflowParametersStore } from "@/store/WorkflowParametersStore";+import { AppNode } from "../nodes";+import { applyRecordedBlocksToGraph } from "./applyRecordedBlocksToGraph";++type UseApplyRecordedBlocksArgs = {+ enabled: boolean;+ nodes: Array<AppNode>;+ edges: Array<Edge>;+ doLayout: (nodes: Array<AppNode>, edges: Array<Edge>) => void;+};++function useApplyRecordedBlocks({+ enabled,+ nodes,+ edges,+ doLayout,+}: UseApplyRecordedBlocksArgs) {+ const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);+ const recordedParameters = useRecordedBlocksStore(+ (state) => state.parameters,+ );+ const recordedInsertionPoint = useRecordedBlocksStore(+ (state) => state.insertionPoint,+ );+ const applicationNonce = useRecordedBlocksStore(+ (state) => state.applicationNonce,+ );+ const clearRecordedBlocks = useRecordedBlocksStore(+ (state) => state.clearRecordedBlocks,+ );+ const parameters = useWorkflowParametersStore((state) => state.parameters);++ const appliedSignatureRef = useRef<string | null>(null);+ const awaitingClearRef = useRef(false);++ useEffect(() => {+ if (!enabled) {+ return;+ }+ if (!recordedBlocks?.length || !recordedInsertionPoint) {+ return;+ }++ const signature = [+ String(applicationNonce),+ recordedInsertionPoint.previous ?? "",+ recordedInsertionPoint.next ?? "",+ recordedInsertionPoint.parent ?? "",+ recordedInsertionPoint.connectingEdgeType,+ recordedBlocks.length,+ recordedBlocks+ .map((block) => `${block.block_type}:${block.label ?? ""}`)+ .join(","),+ ].join("|");++ if (appliedSignatureRef.current === signature && awaitingClearRef.current) {+ return;+ }++ const {+ nodes: mergedNodes,+ edges: mergedEdges,+ newParameters,+ } = applyRecordedBlocksToGraph({+ nodes,+ edges,+ recordedBlocks,+ recordedInsertionPoint,+ recordedParameters,+ existingParameters: parameters,+ });++ useWorkflowHasChangesStore.getState().setHasChanges(true);+ doLayout(mergedNodes, mergedEdges);++ if (newParameters.length > 0) {+ const workflowParametersStore = useWorkflowParametersStore.getState();+ workflowParametersStore.setParameters([+ ...workflowParametersStore.parameters,+ ...newParameters,+ ]);+ }++ appliedSignatureRef.current = signature;+ awaitingClearRef.current = true;+ // nodes/edges/parameters are read from the render that saw the new store+ // payload; listing them as deps would re-apply on every canvas edit.+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [+ enabled,+ recordedBlocks,+ recordedInsertionPoint,+ applicationNonce,+ doLayout,+ ]);++ useEffect(() => {+ if (!awaitingClearRef.current) {+ return;+ }+ awaitingClearRef.current = false;+ clearRecordedBlocks();+ appliedSignatureRef.current = null;+ }, [nodes, edges, clearRecordedBlocks]);++ // The bridge dies with its consumer: this hook is the only thing that applies+ // recorded blocks, so if it unmounts while a payload is still pending (e.g.+ // the user navigates away right after a commit), the blocks must not survive+ // to be applied to whichever workflow canvas mounts next.+ useEffect(() => {+ return () => {+ useRecordedBlocksStore.getState().clearRecordedBlocks();+ };+ }, []);+}++export { useApplyRecordedBlocks };diff --git a/skyvern-frontend/src/routes/workflows/studio/BrowserTab.tsx b/skyvern-frontend/src/routes/workflows/studio/BrowserTab.tsxindex de67bb44e..603408d8c 100644--- a/skyvern-frontend/src/routes/workflows/studio/BrowserTab.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/BrowserTab.tsx@@ -30,6 +30,8 @@ import {StreamStatusPanel,} from "@/routes/streaming/StreamDiagnostics";import { useStudioBrowserStore } from "@/store/useStudioBrowserStore";+import { useRecordingStore } from "@/store/useRecordingStore";+import { useRecordingLauncherStore } from "@/store/useRecordingLauncherStore";import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";import { useStudioShellContext } from "./StudioShellContext";@@ -55,6 +57,11 @@ export function BrowserTab() {const streamUrl = useStudioBrowserStore((s) => s.streamUrl);const reload = useStudioBrowserStore((s) => s.reload);+ const isRecording = useRecordingStore((s) => s.isRecording);+ const manualCapturePaused = useRecordingStore((s) => s.manualCapturePaused);+ const startRecordingAtEnd = useRecordingLauncherStore(+ (s) => s.startRecordingAtEnd,+ );const [confirmOff, setConfirmOff] = useState(false);const cycleBrowser = useMutation({@@ -106,6 +113,30 @@ export function BrowserTab() {<span className="min-w-0 flex-1 truncate text-sm font-medium text-foreground">{streamUrl || "Live browser"}</span>+ {isRecording ? (+ manualCapturePaused ? (+ <span className="flex shrink-0 items-center gap-1.5 rounded-full bg-amber-500/10 px-2.5 py-1 text-xs font-medium text-amber-500">+ <span className="h-2 w-2 rounded-full bg-amber-500" />+ Paused+ </span>+ ) : (+ <span className="flex shrink-0 items-center gap-1.5 rounded-full bg-red-500/10 px-2.5 py-1 text-xs font-medium text-red-500">+ <span className="h-2 w-2 animate-pulse rounded-full bg-red-500" />+ Recording+ </span>+ )+ ) : (+ <Button+ variant="secondary"+ size="sm"+ className="shrink-0 gap-1.5"+ disabled={!browserSessionId || !startRecordingAtEnd}+ onClick={() => startRecordingAtEnd?.()}+ >+ <span className="h-2 w-2 rounded-full bg-red-500" />+ Record+ </Button>+ )}<buttontype="button"title="Reconnect"diff --git a/skyvern-frontend/src/routes/workflows/studio/EditorTab.tsx b/skyvern-frontend/src/routes/workflows/studio/EditorTab.tsxindex 2c5329bde..9147f4541 100644--- a/skyvern-frontend/src/routes/workflows/studio/EditorTab.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/EditorTab.tsx@@ -1,6 +1,9 @@+import { ReloadIcon } from "@radix-ui/react-icons";import { type ComponentProps } from "react";import { Workspace } from "@/routes/workflows/editor/Workspace";+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";+import { useRecordingStore } from "@/store/useRecordingStore";// PiP shelved for now — no good placement (copilot left, settings right, Pylon corner).// import { BrowserPiP } from "./BrowserPiP";@@ -13,10 +16,30 @@ export type StudioWorkspaceProps = Omit<// Editor tab: the real Workspace canvas in embedded mode (the shell owns the top// bar + Copilot spine, so Workspace's floating header is suppressed).export function EditorTab(props: StudioWorkspaceProps) {+ // Show the overlay only while a commit is genuinely in flight or its blocks+ // are landing. Both signals clear on *every* terminal outcome (success, empty,+ // error), so the overlay can never get stuck — unlike `finishRequested`, which+ // stays true on a failed commit to allow a retry.+ const isCommitting = useRecordingStore((s) => s.isCommitting);+ const recordedBlocksPending = useRecordedBlocksStore(+ (s) => (s.blocks?.length ?? 0) > 0,+ );+ const applyingRecording = isCommitting || recordedBlocksPending;+return (<div className="relative h-full w-full"><Workspace {...props} showBrowser={false} embedded />{/* <BrowserPiP /> */}+ {applyingRecording ? (+ <div className="absolute inset-0 z-40 flex items-center justify-center bg-background/70 backdrop-blur-sm">+ <div className="flex items-center gap-3 rounded-lg border bg-slate-elevation2 px-4 py-3 shadow-lg">+ <ReloadIcon className="h-4 w-4 animate-spin text-muted-foreground" />+ <span className="text-sm font-medium text-foreground">+ Adding recorded steps…+ </span>+ </div>+ </div>+ ) : null}</div>);}diff --git a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsxindex bd31b6eb2..fe69a9d5f 100644--- a/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/StreamPresenter.tsx@@ -7,6 +7,7 @@ type StreamPresenterProps = {interactive?: boolean;showControlButtons?: boolean;isRecording?: boolean;+ hideRecordingIndicator?: boolean;// Only the CDP transport carries the page URL; VNC is pixels-only and never// calls this.onUrlChange?: (url: string) => void;@@ -22,6 +23,7 @@ export function StreamPresenter({interactive = false,showControlButtons = false,isRecording = false,+ hideRecordingIndicator = false,onUrlChange,onActivity,}: StreamPresenterProps) {@@ -48,6 +50,7 @@ export function StreamPresenter({interactive={interactive}showControlButtons={showControlButtons}exfiltrate={isRecording}+ hideRecordingIndicator={hideRecordingIndicator}onActivity={onActivity}/>);diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsxindex bc3bd174a..f08222dde 100644--- a/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/StudioBrowserStream.tsx@@ -70,6 +70,10 @@ export function StudioBrowserStream() {interactive={false}showControlButtons={browserPaneOpen && !controlLocked}isRecording={isRecording}+ // While recording, the Copilot pane hosts the live-drafts panel, whose+ // header already shows the timer + step count — the REC pill would+ // duplicate it. Closing that pane brings the pill back as the indicator.+ hideRecordingIndicator={panes.includes("copilot")}onUrlChange={handleUrlChange}onActivity={handleActivity}/>diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioShell.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioShell.tsxindex a672df7f5..830b07a53 100644--- a/skyvern-frontend/src/routes/workflows/studio/StudioShell.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/StudioShell.tsx@@ -1,10 +1,23 @@-import { useLayoutEffect, useMemo, useState, type ReactNode } from "react";+import {+ useEffect,+ useLayoutEffect,+ useMemo,+ useRef,+ useState,+ type ReactNode,+} from "react";import { createPortal } from "react-dom";+import { useParams } from "react-router-dom";import { Cross2Icon } from "@radix-ui/react-icons";+import { RecordingPanel } from "@/routes/workflows/editor/recording/RecordingPanel";+import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";+import { useRecordingStore } from "@/store/useRecordingStore";import { useStudioShellStore } from "@/store/StudioShellStore";import { cn } from "@/util/utils";+import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";+import { BrowserTab } from "./BrowserTab";import { EditorTab, type StudioWorkspaceProps } from "./EditorTab";import { RunTab } from "./RunTab";@@ -93,8 +106,15 @@ export function StudioShell(props: StudioWorkspaceProps) {}function StudioStage(props: StudioWorkspaceProps) {- const { panes, closePane } = useStudioPanes();+ const { panes, closePane, openPane } = useStudioPanes();const { registerStageElement } = useStudioPaneDefaults();+ const { workflowPermanentId } = useParams();+ const isRecording = useRecordingStore((s) => s.isRecording);+ const { data: debugSession } = useDebugSessionQuery({+ workflowPermanentId,+ enabled: false,+ });+ const browserSessionId = debugSession?.browser_session_id ?? null;const pipMinimized = useStudioShellStore((s) => s.pipMinimized);const [copilotPortalEl, setCopilotPortalEl] = useState<HTMLElement | null>(null,@@ -166,6 +186,35 @@ function StudioStage(props: StudioWorkspaceProps) {document.getElementById(studioTabId(id))?.focus();};+ // Recording lives in the Browser pane (the live stream is there) with the+ // live-drafts panel taking over the Copilot pane. Once a commit is in flight+ // or its blocks are landing, reveal the Editor pane (it shows the loading+ // overlay). Gated on lifecycle transitions so manual pane changes made+ // mid-recording are preserved.+ const isCommitting = useRecordingStore((s) => s.isCommitting);+ const recordedBlocksPending = useRecordedBlocksStore(+ (s) => (s.blocks?.length ?? 0) > 0,+ );+ const processingRecording = isCommitting || recordedBlocksPending;+ const prevIsRecordingRef = useRef(false);+ const prevProcessingRef = useRef(false);+ useEffect(() => {+ const wasRecording = prevIsRecordingRef.current;+ const wasProcessing = prevProcessingRef.current;+ if (processingRecording && !wasProcessing) {+ openPane("editor");+ } else if (isRecording && !wasRecording) {+ // Recording or finalizing → live browser + drafts.+ openPane("copilot");+ openPane("browser");+ } else if (!isRecording && wasRecording && !processingRecording) {+ // Recording ended (commit or discard) → back to the canvas.+ openPane("editor");+ }+ prevIsRecordingRef.current = isRecording;+ prevProcessingRef.current = processingRecording;+ }, [isRecording, processingRecording, openPane]);+const paneProps = (id: StudioPaneId) => {const index = panes.indexOf(id);return {@@ -190,9 +239,23 @@ function StudioStage(props: StudioWorkspaceProps) {className="relative flex min-h-0 min-w-0 flex-1 gap-3 overflow-hidden p-3"><StudioPane {...paneProps("copilot")}>- {/* Copilot portal target. Kept mounted while the pane is closed so- an in-flight Copilot turn isn't torn down. */}- <div ref={setCopilotPortalEl} className="h-full w-full" />+ <div className="relative h-full w-full">+ {/* Copilot portal target. Kept mounted while the pane is closed+ — or while recording, when the live-drafts panel covers it —+ so an in-flight Copilot turn isn't torn down. */}+ <div+ ref={setCopilotPortalEl}+ className={cn(+ "h-full w-full",+ isRecording && "pointer-events-none",+ )}+ />+ {isRecording && browserSessionId ? (+ <div className="absolute inset-0 duration-150 animate-in fade-in slide-in-from-left-2">+ <RecordingPanel browserSessionId={browserSessionId} />+ </div>+ ) : null}+ </div></StudioPane><StudioPane {...paneProps("editor")}><EditorTab {...props} />diff --git a/skyvern-frontend/src/store/RecordedBlocksStore.ts b/skyvern-frontend/src/store/RecordedBlocksStore.tsindex dd405328e..0f13628ba 100644--- a/skyvern-frontend/src/store/RecordedBlocksStore.ts+++ b/skyvern-frontend/src/store/RecordedBlocksStore.ts@@ -15,6 +15,7 @@ type RecordedBlocksState = {blocks: Array<WorkflowBlock> | null;parameters: Array<WorkflowParameter> | null;insertionPoint: InsertionPoint | null;+ applicationNonce: number;};type RecordedBlocksStore = RecordedBlocksState & {@@ -32,11 +33,22 @@ const useRecordedBlocksStore = create<RecordedBlocksStore>((set) => ({blocks: null,parameters: null,insertionPoint: null,+ applicationNonce: 0,setRecordedBlocks: ({ blocks, parameters }, insertionPoint) => {- set({ blocks, parameters, insertionPoint });+ set({+ blocks,+ parameters,+ insertionPoint,+ applicationNonce: Date.now(),+ });},clearRecordedBlocks: () => {- set({ blocks: null, insertionPoint: null });+ set({+ blocks: null,+ parameters: null,+ insertionPoint: null,+ applicationNonce: 0,+ });},}));diff --git a/skyvern-frontend/src/store/useRecordingLauncherStore.ts b/skyvern-frontend/src/store/useRecordingLauncherStore.tsnew file mode 100644index 000000000..ace77fceb--- /dev/null+++ b/skyvern-frontend/src/store/useRecordingLauncherStore.ts@@ -0,0 +1,20 @@+import { create } from "zustand";++/**+ * Bridges the studio's Browser-tab Record button to the canvas-aware Workspace.+ * The Workspace (which has ReactFlow nodes/edges) registers a launcher that+ * computes the append-at-end insertion point and starts recording; the button,+ * which lives outside the canvas, just invokes it.+ */+type RecordingLauncherState = {+ startRecordingAtEnd: (() => void) | null;+ setStartRecordingAtEnd: (launcher: (() => void) | null) => void;+};++export const useRecordingLauncherStore = create<RecordingLauncherState>(+ (set) => ({+ startRecordingAtEnd: null,+ setStartRecordingAtEnd: (startRecordingAtEnd) =>+ set({ startRecordingAtEnd }),+ }),+);diff --git a/skyvern-frontend/src/store/useRecordingStore.test.ts b/skyvern-frontend/src/store/useRecordingStore.test.tsnew file mode 100644index 000000000..777574e1f--- /dev/null+++ b/skyvern-frontend/src/store/useRecordingStore.test.ts@@ -0,0 +1,307 @@+import { beforeEach, describe, expect, it } from "vitest";++import {+ upsertDraftSteps,+ useRecordingStore,+ type OptimisticActionKind,+ type OptimisticStep,+ type RecordingActionKind,+ type RecordingDraftStep,+ type RecordingInterpretationUpdate,+} from "./useRecordingStore";++let seq = 0;+function opt(kind: OptimisticActionKind = "click"): OptimisticStep {+ seq += 1;+ return {+ local_id: `o-${seq}`,+ action_kind: kind,+ title: `${kind} ${seq}`,+ timestamp: seq,+ };+}++function draft(id: string, kind: RecordingActionKind): RecordingDraftStep {+ return {+ step_id: id,+ action_kind: kind,+ block_type:+ kind === "url_change" ? "goto_url" : kind === "wait" ? "wait" : "action",+ label: id,+ status: "ready",+ editable_fields: [],+ parameters: [],+ parameter_keys: [],+ };+}++function update(+ id: string,+ revision: number,+ steps: Array<RecordingDraftStep>,+ opts: {+ pending?: boolean;+ finalized?: boolean;+ is_snapshot?: boolean;+ changed_steps?: Array<RecordingDraftStep>;+ } = {},+): RecordingInterpretationUpdate {+ return {+ interpretation_session_id: id,+ session_revision: revision,+ steps,+ pending: opts.pending ?? false,+ finalized: opts.finalized ?? false,+ ...(opts.is_snapshot !== undefined+ ? { is_snapshot: opts.is_snapshot }+ : {}),+ ...(opts.changed_steps !== undefined+ ? { changed_steps: opts.changed_steps }+ : {}),+ };+}++const store = () => useRecordingStore.getState();++beforeEach(() => {+ store().reset();+});++describe("addOptimisticStep", () => {+ it("appends while recording and unpaused", () => {+ store().setIsRecording(true);+ store().addOptimisticStep(opt());+ expect(store().optimisticSteps).toHaveLength(1);+ });++ it("no-ops when not recording", () => {+ store().addOptimisticStep(opt());+ expect(store().optimisticSteps).toHaveLength(0);+ });++ it("no-ops when capture is paused", () => {+ store().setIsRecording(true);+ store().setManualCapturePaused(true);+ store().addOptimisticStep(opt());+ expect(store().optimisticSteps).toHaveLength(0);+ });++ it("no-ops after finish is requested", () => {+ store().setIsRecording(true);+ store().requestFinish();+ store().addOptimisticStep(opt());+ expect(store().optimisticSteps).toHaveLength(0);+ });+});++describe("applyInterpretationUpdate: transient optimistic steps", () => {+ beforeEach(() => {+ store().setIsRecording(true);+ });++ it("keeps optimistic steps while interpretation is pending", () => {+ store().addOptimisticStep(opt());+ store().addOptimisticStep(opt());++ // Immediate pending update (steps unchanged) on the first significant event.+ store().applyInterpretationUpdate(update("s1", 1, [], { pending: true }));+ expect(store().optimisticSteps).toHaveLength(2);++ // Another pending bump keeps them.+ store().applyInterpretationUpdate(update("s1", 2, [], { pending: true }));+ expect(store().optimisticSteps).toHaveLength(2);+ });++ it("clears optimistic steps once interpretation settles (pending=false)", () => {+ store().addOptimisticStep(opt());+ store().addOptimisticStep(opt());++ store().applyInterpretationUpdate(+ update("s1", 1, [draft("s1-0", "click")], { pending: false }),+ );+ expect(store().optimisticSteps).toHaveLength(0);+ });++ it("does not strand placeholders for events the backend folds away", () => {+ // A link click yields a click (+wait) but no goto; the optimistic click must+ // still clear when interpretation settles rather than linger forever.+ store().addOptimisticStep(opt("click"));+ store().applyInterpretationUpdate(+ update("s1", 1, [draft("s1-0", "click"), draft("s1-1", "wait")], {+ pending: false,+ }),+ );+ expect(store().optimisticSteps).toHaveLength(0);+ });++ it("clears optimistic steps on a finalized snapshot", () => {+ store().addOptimisticStep(opt());+ store().applyInterpretationUpdate(update("s1", 1, [], { finalized: true }));+ expect(store().optimisticSteps).toHaveLength(0);+ });++ it("clears optimistic steps on a genuine session change", () => {+ store().addOptimisticStep(opt());+ store().applyInterpretationUpdate(update("s1", 1, [], { pending: true }));+ expect(store().optimisticSteps).toHaveLength(1);++ store().applyInterpretationUpdate(update("s2", 1, [], { pending: true }));+ expect(store().optimisticSteps).toHaveLength(0);+ expect(store().interpretationSessionId).toBe("s2");+ });+});++describe("commit safety and resets", () => {+ it("getFinalDraftSteps never includes optimistic steps", () => {+ store().setIsRecording(true);+ store().addOptimisticStep(opt());+ // pending=true so the optimistic step is still present alongside a draft step+ store().applyInterpretationUpdate(+ update("s1", 1, [draft("s1-0", "click")], { pending: true }),+ );++ const final = store().getFinalDraftSteps();+ expect(final).toHaveLength(1);+ expect(final?.every((s) => s.step_id === "s1-0")).toBe(true);+ expect(store().optimisticSteps).toHaveLength(1);+ });++ it("reset() and starting a new recording zero optimistic state", () => {+ store().setIsRecording(true);+ store().addOptimisticStep(opt());+ store().reset();+ expect(store().optimisticSteps).toHaveLength(0);++ store().setIsRecording(true);+ store().addOptimisticStep(opt());+ expect(store().optimisticSteps).toHaveLength(1);+ });++ it("mints a new recordingAttemptId for each new recording", () => {+ store().setIsRecording(true);+ const first = store().recordingAttemptId;+ expect(first).toBeTruthy();++ store().setIsRecording(false);+ store().setIsRecording(true);+ const second = store().recordingAttemptId;+ expect(second).toBeTruthy();+ expect(second).not.toBe(first);+ });+});++describe("upsertDraftSteps", () => {+ it("returns the same list when there are no changes", () => {+ const current = [draft("a", "click"), draft("b", "click")];+ expect(upsertDraftSteps(current, [])).toBe(current);+ });++ it("replaces a step in place, preserving order", () => {+ const current = [draft("a", "click"), draft("b", "click")];+ const updated = { ...draft("a", "click"), label: "updated" };+ const result = upsertDraftSteps(current, [updated]);+ expect(result.map((s) => s.step_id)).toEqual(["a", "b"]);+ expect(result[0]?.label).toBe("updated");+ expect(result[1]).toBe(current[1]);+ });++ it("appends genuinely-new steps in arrival order", () => {+ const current = [draft("a", "click")];+ const result = upsertDraftSteps(current, [+ draft("b", "click"),+ draft("c", "click"),+ ]);+ expect(result.map((s) => s.step_id)).toEqual(["a", "b", "c"]);+ });++ it("handles a mix of replace and append in one call", () => {+ const current = [draft("a", "click"), draft("b", "click")];+ const result = upsertDraftSteps(current, [+ { ...draft("b", "click"), label: "b2" },+ draft("c", "click"),+ ]);+ expect(result.map((s) => s.step_id)).toEqual(["a", "b", "c"]);+ expect(result[1]?.label).toBe("b2");+ });++ it("is idempotent for a repeated step id", () => {+ const current = [draft("a", "click")];+ const result = upsertDraftSteps(current, [+ draft("a", "click"),+ draft("a", "click"),+ ]);+ expect(result.map((s) => s.step_id)).toEqual(["a"]);+ });+});++describe("applyInterpretationUpdate: snapshot vs delta", () => {+ beforeEach(() => {+ store().setIsRecording(true);+ });++ it("legacy message with no is_snapshot/changed_steps replaces wholesale", () => {+ store().applyInterpretationUpdate(+ update("s1", 1, [draft("s1-0", "click"), draft("s1-1", "click")]),+ );+ expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-0", "s1-1"]);++ // A later snapshot fully replaces (does not merge).+ store().applyInterpretationUpdate(+ update("s1", 2, [draft("s1-2", "click")]),+ );+ expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-2"]);+ });++ it("delta upserts changed steps into the current list", () => {+ // Seed with a snapshot of two interpreting steps.+ const a = { ...draft("s1-0", "click"), status: "interpreting" as const };+ const b = { ...draft("s1-1", "click"), status: "interpreting" as const };+ store().applyInterpretationUpdate(+ update("s1", 1, [a, b], { pending: true }),+ );++ // Delta: s1-0 flips to ready with a real label; s1-1 untouched.+ const enriched = { ...draft("s1-0", "click"), label: "Click Vancouver" };+ store().applyInterpretationUpdate(+ update("s1", 2, [], {+ pending: true,+ is_snapshot: false,+ changed_steps: [enriched],+ }),+ );++ const steps = store().draftSteps;+ expect(steps.map((s) => s.step_id)).toEqual(["s1-0", "s1-1"]);+ expect(steps[0]?.status).toBe("ready");+ expect(steps[0]?.label).toBe("Click Vancouver");+ expect(steps[1]?.status).toBe("interpreting");+ });++ it("delta appends a brand-new step", () => {+ store().applyInterpretationUpdate(+ update("s1", 1, [draft("s1-0", "click")], { pending: true }),+ );+ store().applyInterpretationUpdate(+ update("s1", 2, [], {+ pending: true,+ is_snapshot: false,+ changed_steps: [draft("s1-1", "click")],+ }),+ );+ expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-0", "s1-1"]);+ });++ it("ignores a stale delta at or below the current revision", () => {+ store().applyInterpretationUpdate(+ update("s1", 5, [draft("s1-0", "click")], { pending: true }),+ );+ store().applyInterpretationUpdate(+ update("s1", 5, [], {+ pending: true,+ is_snapshot: false,+ changed_steps: [draft("s1-9", "click")],+ }),+ );+ expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-0"]);+ });+});diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.tsindex 2303e4f00..884a00892 100644--- a/skyvern-frontend/src/store/useRecordingStore.ts+++ b/skyvern-frontend/src/store/useRecordingStore.ts@@ -100,6 +100,120 @@ export type MessageInExfiltratedEvent =| MessageInExfiltratedCdpEvent| MessageInExfiltratedConsoleEvent;+export type RecordingActionKind =+ | "click"+ | "hover"+ | "input_text"+ | "url_change"+ | "wait";++/**+ * Mirrors `RecordingDraftStep` in skyvern/services/browser_recording/types.py.+ */+export interface RecordingDraftStep {+ step_id: string;+ action_kind: RecordingActionKind;+ block_type: "action" | "goto_url" | "wait";+ label: string;+ title?: string | null;+ navigation_goal?: string | null;+ url?: string | null;+ wait_sec?: number | null;+ status: "interpreting" | "ready";+ editable_fields: Array<string>;+ parameters: Array<Record<string, unknown>>;+ parameter_keys: Array<string>;+ timestamp_start?: number | null;+ timestamp_end?: number | null;+}++export interface RecordingInterpretationUpdate {+ interpretation_session_id: string;+ session_revision: number;+ // Authoritative full list when is_snapshot is true/absent (legacy shape).+ steps: Array<RecordingDraftStep>;+ // Delta upserts (by step_id) when is_snapshot is false. Absent on legacy+ // full-snapshot messages, so a backend that still sends snapshots is a no-op.+ changed_steps?: Array<RecordingDraftStep>;+ is_snapshot?: boolean;+ pending: boolean;+ finalized: boolean;+}++/**+ * Merge delta step upserts into the current list: replace any step sharing a+ * step_id in place (preserving order), and append genuinely-new steps in arrival+ * order. Idempotent — re-applying the same step by id is safe.+ */+export function upsertDraftSteps(+ current: Array<RecordingDraftStep>,+ changed: Array<RecordingDraftStep>,+): Array<RecordingDraftStep> {+ if (changed.length === 0) {+ return current;+ }+ const changedById = new Map(changed.map((step) => [step.step_id, step]));+ const merged = current.map((step) => changedById.get(step.step_id) ?? step);+ const existingIds = new Set(current.map((step) => step.step_id));+ for (const step of changed) {+ if (!existingIds.has(step.step_id)) {+ merged.push(step);+ existingIds.add(step.step_id);+ }+ }+ return merged;+}++export type RecordingDraftStepPatch = Partial<+ Pick<+ RecordingDraftStep,+ "label" | "title" | "navigation_goal" | "url" | "wait_sec"+ >+>;++/**+ * Action kinds the frontend can optimistically predict from a single exfiltrated+ * event, before the backend interprets it. Navigations are deliberately excluded:+ * a link click yields a click action (not a goto) on the backend, so an optimistic+ * nav would have nothing to resolve into.+ */+export type OptimisticActionKind = "click" | "input_text";++/**+ * A frontend-only placeholder appended the instant a significant interaction is+ * observed, so the count and feed advance without waiting for the debounced+ * backend interpretation. Transient: cleared once interpretation settles (the+ * authoritative steps then represent the interaction). Never sent on commit.+ */+export interface OptimisticStep {+ local_id: string;+ action_kind: OptimisticActionKind;+ title: string;+ timestamp: number;+}++let optimisticStepSeq = 0;+export const nextOptimisticStepId = (): string =>+ `optimistic-${(optimisticStepSeq += 1)}`;++/**+ * A frame grabbed from the live VNC canvas at the moment of a recorded click,+ * matched to draft steps by source-event timestamp (ms epoch).+ */+export interface RecordingScreenshot {+ timestampMs: number;+ dataUrl: string;+ xp: number | null;+ yp: number | null;+}++const MAX_SCREENSHOTS = 40;+// dataUrl.length tracks base64-encoded JPEG bytes; cap total in-memory footprint.+const MAX_SCREENSHOT_BYTES = 8 * 1024 * 1024;++/** Tolerance when matching a screenshot to a step's source-event window. */+const SCREENSHOT_MATCH_TOLERANCE_MS = 750;+/*** Number of events per compressed chunk.*/@@ -129,6 +243,85 @@ interface RecordingStore {*/isRecording: boolean;recordingStartedAtMs: number | null;+ /**+ * The workflow the recording will be committed to. Required to enable+ * backend live interpretation.+ */+ workflowPermanentId: string | null;+ /**+ * Per-recording id sent on begin-exfiltration. Stable across reconnects of+ * the same recording, regenerated for each new recording, so the backend can+ * tell a reconnect (reuse session) from a new recording (fresh session).+ */+ recordingAttemptId: string | null;+ /**+ * Latest live-interpretation snapshot from the backend. Snapshots are+ * full replacements keyed by monotonically increasing session revision.+ */+ draftSteps: Array<RecordingDraftStep>;+ interpretationSessionId: string | null;+ sessionRevision: number;+ /**+ * Frontend-only optimistic placeholders shown instantly on each significant+ * interaction, cleared once interpretation settles.+ */+ optimisticSteps: Array<OptimisticStep>;+ /**+ * True between "significant events arrived" and "interpretation resolved" —+ * drives the trailing "Interpreting…" placeholder in the feed.+ */+ interpretationPending: boolean;+ /**+ * True once the backend has flushed the session (after end-exfiltration).+ */+ interpretationFinalized: boolean;+ /**+ * Local user overlays. Snapshots are authoritative for step content; user+ * deletes/edits are re-applied on top of every snapshot.+ */+ deletedStepIds: Array<string>;+ stepPatches: Record<string, RecordingDraftStepPatch>;+ /**+ * Nested count of in-progress live draft title edits. While > 0, capture is+ * paused alongside manualCapturePaused.+ */+ draftEditDepth: number;+ /**+ * Operator toggled pause — stops exfiltration and live interpretation until+ * resumed.+ */+ manualCapturePaused: boolean;+ screenshots: Array<RecordingScreenshot>;+ /**+ * Set when the user hits Done: exfiltration stops, and the commit fires once+ * the finalized interpretation snapshot arrives (or a timeout elapses).+ */+ finishRequested: boolean;+ /**+ * True while the process_recording mutation is in flight.+ */+ isCommitting: boolean;+ applyInterpretationUpdate: (update: RecordingInterpretationUpdate) => void;+ /**+ * Append an optimistic placeholder. No-op unless actively recording and+ * capture is not paused.+ */+ addOptimisticStep: (step: OptimisticStep) => void;+ deleteDraftStep: (stepId: string) => void;+ patchDraftStep: (stepId: string, patch: RecordingDraftStepPatch) => void;+ beginDraftEdit: () => void;+ endDraftEdit: () => void;+ setManualCapturePaused: (paused: boolean) => void;+ isCapturePaused: () => boolean;+ addScreenshot: (screenshot: RecordingScreenshot) => void;+ requestFinish: () => void;+ setIsCommitting: (isCommitting: boolean) => void;+ /**+ * Draft steps to commit: snapshot minus user deletions, with user edits+ * applied. Null when live interpretation never produced a revision (caller+ * should fall back to raw event processing).+ */+ getFinalDraftSteps: () => Array<RecordingDraftStep> | null;/*** Add a new recorded event. Triggers async compression when buffer is full.*/@@ -213,22 +406,114 @@ async function compressEventsToB64(jsonString: string): Promise<string> {return btoa(binary);}-const isExposedEvent = (event: MessageInExfiltratedEvent): boolean => {- const exposedConsoleEventTypes = new Set(["focus", "click", "keypress"]);+export function applyDraftStepOverlays(+ steps: Array<RecordingDraftStep>,+ deletedStepIds: Array<string>,+ stepPatches: Record<string, RecordingDraftStepPatch>,+): Array<RecordingDraftStep> {+ const deleted = new Set(deletedStepIds);+ return steps+ .filter((step) => !deleted.has(step.step_id))+ .map((step) => {+ const patch = stepPatches[step.step_id];+ return patch ? { ...step, ...patch } : step;+ });+}++export function countVisibleDraftSteps(+ steps: Array<RecordingDraftStep>,+ deletedStepIds: Array<string>,+): number {+ if (deletedStepIds.length === 0) {+ return steps.length;+ }++ const deleted = new Set(deletedStepIds);+ let count = 0;+ for (const step of steps) {+ if (!deleted.has(step.step_id)) {+ count += 1;+ }+ }+ return count;+}+/**+ * The screenshot taken closest to the step's source-event window, if any.+ *+ * Both sides of the comparison are the same clock: the screenshot's+ * `timestampMs` is the exfiltrated event's `params.timestamp` (remote-browser+ * `Date.now()`, ms epoch), and the step's `timestamp_start`/`timestamp_end`+ * are the backend echoing that same source-event timestamp back+ * (RecordingDraftStep "Source-action event timestamps (ms epoch)").+ */+export function findScreenshotForStep(+ step: RecordingDraftStep,+ screenshots: Array<RecordingScreenshot>,+): RecordingScreenshot | null {+ const start = step.timestamp_start;+ if (start === null || start === undefined) {+ return null;+ }+ const end = step.timestamp_end ?? start;++ let best: RecordingScreenshot | null = null;+ let bestDistance = Infinity;+ for (const screenshot of screenshots) {+ const t = screenshot.timestampMs;+ const distance = t < start ? start - t : t > end ? t - end : 0; // 0 inside the window+ if (distance <= SCREENSHOT_MATCH_TOLERANCE_MS && distance < bestDistance) {+ best = screenshot;+ bestDistance = distance;+ }+ }+ return best;+}++const EXPOSED_CONSOLE_EVENT_TYPES = new Set(["focus", "click", "keypress"]);++const isExposedEvent = (event: MessageInExfiltratedEvent): boolean => {if (event.source === "console") {- if (exposedConsoleEventTypes.has(event.params.type)) {+ if (EXPOSED_CONSOLE_EVENT_TYPES.has(event.params.type)) {return true;}}if (event.source === "cdp") {- return true;+ // net:* events are page-activity telemetry, not user interactions+ return !event.event_name.startsWith("net:");}return false;};+/**+ * Fields cleared by both `clear()` and `reset()`. Returns fresh collections on+ * each call so no two resets share array/object references. `reset()` layers on+ * the extra teardown (`isCompressing`, `isRecording`, `workflowPermanentId`).+ */+function emptyRecordingState() {+ return {+ compressedChunks: [] as string[],+ exposedEventCount: 0,+ pendingEvents: [] as MessageInExfiltratedEvent[],+ recordingStartedAtMs: null as number | null,+ draftSteps: [] as Array<RecordingDraftStep>,+ interpretationSessionId: null as string | null,+ sessionRevision: 0,+ optimisticSteps: [] as Array<OptimisticStep>,+ interpretationPending: false,+ interpretationFinalized: false,+ deletedStepIds: [] as Array<string>,+ stepPatches: {} as Record<string, RecordingDraftStepPatch>,+ draftEditDepth: 0,+ manualCapturePaused: false,+ screenshots: [] as Array<RecordingScreenshot>,+ finishRequested: false,+ isCommitting: false,+ };+}+export const useRecordingStore = create<RecordingStore>((set, get) => ({compressedChunks: [],exposedEventCount: 0,@@ -236,6 +521,174 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({isCompressing: false,isRecording: false,recordingStartedAtMs: null,+ workflowPermanentId: null,+ recordingAttemptId: null,+ draftSteps: [],+ interpretationSessionId: null,+ sessionRevision: 0,+ optimisticSteps: [],+ interpretationPending: false,+ interpretationFinalized: false,+ deletedStepIds: [],+ stepPatches: {},+ draftEditDepth: 0,+ manualCapturePaused: false,+ screenshots: [],+ finishRequested: false,+ isCommitting: false,++ applyInterpretationUpdate: (update) => {+ const state = get();+ if (!state.isRecording && !state.finishRequested) {+ return;+ }++ // Drop non-finalized snapshots while capture is paused so in-progress edits+ // are not overwritten; finalized snapshots still commit after Done.+ if (+ state.isCapturePaused() &&+ !update.finalized &&+ !state.finishRequested+ ) {+ return;+ }++ // The initial null -> first-id transition is not a session change: it must+ // not discard optimistic steps captured before the first snapshot.+ const sessionChanged =+ state.interpretationSessionId !== null &&+ update.interpretation_session_id !== state.interpretationSessionId;++ if (!sessionChanged && update.session_revision <= state.sessionRevision) {+ return;+ }++ // Optimistic placeholders are transient. Keep them only while interpretation+ // is in flight (pending); clear them once it settles, on session change, or+ // when finalized — by then the authoritative steps represent the interaction.+ // The backend emits an immediate pending=true update (steps unchanged) on each+ // significant event, so this preserves the placeholders we just added and only+ // clears them when the debounced interpret pass lands (pending=false). This+ // avoids permanent residue from events the backend folds away (link-click+ // navigations, deduped/collapsed actions) or duplicate transport events.+ const settled = !update.pending;+ const clearOptimistic = sessionChanged || update.finalized || settled;++ // Delta updates upsert changed steps into the current list; snapshots (the+ // legacy shape, is_snapshot true/absent) replace it wholesale. This keeps the+ // reader working unchanged until the backend starts sending deltas.+ const nextDraftSteps =+ update.is_snapshot === false+ ? upsertDraftSteps(state.draftSteps, update.changed_steps ?? [])+ : update.steps;++ set({+ draftSteps: nextDraftSteps,+ interpretationSessionId: update.interpretation_session_id,+ sessionRevision: update.session_revision,+ ...(clearOptimistic ? { optimisticSteps: [] } : {}),+ interpretationPending: update.pending,+ interpretationFinalized: update.finalized,+ ...(sessionChanged ? { deletedStepIds: [], stepPatches: {} } : {}),+ });+ },++ addOptimisticStep: (step) => {+ const state = get();+ if (!state.isRecording || state.finishRequested) {+ return;+ }+ if (state.isCapturePaused()) {+ return;+ }+ set({ optimisticSteps: [...state.optimisticSteps, step] });+ },++ deleteDraftStep: (stepId) => {+ const state = get();+ if (state.deletedStepIds.includes(stepId)) {+ return;+ }+ set({ deletedStepIds: [...state.deletedStepIds, stepId] });+ captureRecordBrowser("record_browser.draft_step_deleted", {+ step_id: stepId,+ });+ },++ patchDraftStep: (stepId, patch) => {+ const state = get();+ set({+ stepPatches: {+ ...state.stepPatches,+ [stepId]: { ...state.stepPatches[stepId], ...patch },+ },+ });+ captureRecordBrowser("record_browser.draft_step_edited", {+ step_id: stepId,+ fields: Object.keys(patch),+ });+ },++ beginDraftEdit: () => {+ set({ draftEditDepth: get().draftEditDepth + 1 });+ },++ endDraftEdit: () => {+ set({ draftEditDepth: Math.max(0, get().draftEditDepth - 1) });+ },++ setManualCapturePaused: (paused) => {+ set({ manualCapturePaused: paused });+ captureRecordBrowser(+ paused+ ? "record_browser.capture_paused"+ : "record_browser.capture_resumed",+ );+ },++ isCapturePaused: () => {+ const state = get();+ return state.manualCapturePaused || state.draftEditDepth > 0;+ },++ addScreenshot: (screenshot) => {+ const screenshots = [...get().screenshots, screenshot];+ let totalBytes = screenshots.reduce(+ (sum, item) => sum + item.dataUrl.length,+ 0,+ );+ while (screenshots.length > 1 && totalBytes > MAX_SCREENSHOT_BYTES) {+ const removed = screenshots.shift();+ if (removed) {+ totalBytes -= removed.dataUrl.length;+ }+ }+ if (screenshots.length > MAX_SCREENSHOTS) {+ screenshots.splice(0, screenshots.length - MAX_SCREENSHOTS);+ }+ set({ screenshots });+ },++ requestFinish: () => {+ if (get().finishRequested) {+ return;+ }+ set({ finishRequested: true });+ },++ setIsCommitting: (isCommitting) => set({ isCommitting }),++ getFinalDraftSteps: () => {+ const state = get();+ if (state.sessionRevision === 0) {+ return null;+ }+ return applyDraftStepOverlays(+ state.draftSteps,+ state.deletedStepIds,+ state.stepPatches,+ );+ },add: (event) => {const state = get();@@ -284,41 +737,45 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({}},- clear: () =>- set({- compressedChunks: [],- pendingEvents: [],- recordingStartedAtMs: null,- }),+ clear: () => {+ // Reset per-recording so ids restart at 1 each recording instead of drifting+ // upward across the tab's lifetime.+ optimisticStepSeq = 0;+ set(emptyRecordingState());+ },reset: () =>set({- compressedChunks: [],- exposedEventCount: 0,- pendingEvents: [],+ ...emptyRecordingState(),isCompressing: false,isRecording: false,- recordingStartedAtMs: null,+ workflowPermanentId: null,}),setIsRecording: (isRecording, meta) => {- const state = get();- if (!state.isRecording && isRecording) {- get().clear();- set({- isRecording: true,- recordingStartedAtMs: Date.now(),- });- captureRecordBrowser("record_browser.started", {- workflow_permanent_id: meta?.workflowPermanentId ?? undefined,- browser_session_id: meta?.browserSessionId ?? undefined,- });+ if (isRecording) {+ const state = get();+ if (!state.isRecording) {+ get().clear();+ set({+ isRecording: true,+ recordingStartedAtMs: Date.now(),+ workflowPermanentId: meta?.workflowPermanentId ?? null,+ recordingAttemptId: crypto.randomUUID(),+ });+ captureRecordBrowser("record_browser.started", {+ workflow_permanent_id: meta?.workflowPermanentId ?? undefined,+ browser_session_id: meta?.browserSessionId ?? undefined,+ });+ }return;}set({- isRecording,- recordingStartedAtMs: isRecording ? state.recordingStartedAtMs : null,+ isRecording: false,+ recordingStartedAtMs: null,+ finishRequested: false,+ isCommitting: false,});},diff --git a/skyvern-frontend/src/util/recordingClock.ts b/skyvern-frontend/src/util/recordingClock.tsnew file mode 100644index 000000000..ef8d2d146--- /dev/null+++ b/skyvern-frontend/src/util/recordingClock.ts@@ -0,0 +1,6 @@+export function formatRecordingClock(totalSeconds: number): string {+ const seconds = Math.max(0, Math.floor(totalSeconds));+ const mm = String(Math.floor(seconds / 60)).padStart(2, "0");+ const ss = String(seconds % 60).padStart(2, "0");+ return `${mm}:${ss}`;+}