Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-12766): render copilot test-run action rows live during runs (#7609)
Celal Zamanoğlu216a63b9da94
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.blockActions.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.blockActions.test.tsxindex d44f58448..8af925044 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.blockActions.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.blockActions.test.tsx@@ -191,6 +191,19 @@ const runOutcomeFrame = (overrides: Partial<Record<string, unknown>> = {}) => ({...overrides,});+const blockProgressFrame = (+ overrides: Partial<Record<string, unknown>> = {},+) => ({+ type: "block_progress",+ workflow_run_block_id: "wrb_1",+ block_label: "block_1",+ block_type: "code",+ status: "running",+ iteration: 0,+ timestamp: "2026-06-10T00:00:00Z",+ ...overrides,+});+beforeEach(() => {HTMLElement.prototype.scrollIntoView = vi.fn();HTMLElement.prototype.scrollTo = vi.fn();@@ -217,13 +230,15 @@ afterEach(() => {cleanup();});-describe("WorkflowCopilotChat — recorded-action fetch wiring", () => {- it("fetches the run timeline exactly once when a run_outcome frame arrives", async () => {+describe("WorkflowCopilotChat — recorded-action live poll wiring", () => {+ it("starts a live poll from the first block_progress that carries a run id", async () => {await renderChat();await submit("build a workflow");fireEvent.change(screen.getByRole("textbox"), { target: { value: "" } });- streamCalls[0]!.onMessage(runOutcomeFrame());+ streamCalls[0]!.onMessage(+ blockProgressFrame({ workflow_run_id: "wr_1", status: "running" }),+ );await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));expect(timelineGet.mock.calls[0]![0]).toBe(@@ -231,22 +246,81 @@ describe("WorkflowCopilotChat — recorded-action fetch wiring", () => {);});- it("does not re-fetch for a second run_outcome frame carrying the same run id", async () => {+ it("polls the timeline repeatedly while the run stays live", async () => {+ // Call through so React/testing-library timers keep working; we only need+ // to capture the registered callback to drive it deterministically.+ const setIntervalSpy = vi.spyOn(window, "setInterval");+ try {+ await renderChat();+ await submit("build a workflow");++ streamCalls[0]!.onMessage(+ blockProgressFrame({ workflow_run_id: "wr_1" }),+ );+ // Immediate fetch on the first sighting.+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));+ expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), 2500);++ // Drive the registered interval callback: each tick re-fetches (the+ // reducer merges by actionId, so repeated fetches never duplicate rows).+ const tick = setIntervalSpy.mock.calls.find((c) => c[1] === 2500)![0] as (+ ...args: unknown[]+ ) => void;+ tick();+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(2));+ tick();+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(3));+ } finally {+ setIntervalSpy.mockRestore();+ }+ });++ it("converges with a final fetch and stops polling on a terminal run_outcome", async () => {+ const clearIntervalSpy = vi.spyOn(window, "clearInterval");+ const setIntervalSpy = vi.spyOn(window, "setInterval");+ try {+ await renderChat();+ await submit("build a workflow");++ streamCalls[0]!.onMessage(+ blockProgressFrame({ workflow_run_id: "wr_1" }),+ );+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));+ const idx = setIntervalSpy.mock.calls.findIndex((c) => c[1] === 2500);+ const intervalId = setIntervalSpy.mock.results[idx]!.value;++ // Terminal verdict: one convergent fetch, then the interval is cleared.+ streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "demonstrated" }));+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(2));+ expect(clearIntervalSpy).toHaveBeenCalledWith(intervalId);+ } finally {+ setIntervalSpy.mockRestore();+ clearIntervalSpy.mockRestore();+ }+ });++ it("falls back to run_outcome(evaluating) when block_progress carries no run id", async () => {await renderChat();await submit("build a workflow");- streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "evaluating" }));- await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));+ // Old backend: block_progress has no run id, so no poll can start yet.+ streamCalls[0]!.onMessage(blockProgressFrame({ status: "running" }));+ streamCalls[0]!.onMessage(blockProgressFrame({ status: "completed" }));+ expect(timelineGet).not.toHaveBeenCalled();- streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "demonstrated" }));- // Give any accidental second fetch a chance to fire before asserting.+ // The run id first arrives on run_outcome — the poll starts there.+ streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "evaluating" }));await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));+ expect(timelineGet.mock.calls[0]![0]).toBe(+ "/workflows/wpid_1/runs/wr_1/timeline",+ );});- it("never fetches when the run_outcome frame carries an empty workflow_run_id", async () => {+ it("never fetches when frames carry an empty workflow_run_id", async () => {await renderChat();await submit("build a workflow");+ streamCalls[0]!.onMessage(blockProgressFrame({ workflow_run_id: "" }));streamCalls[0]!.onMessage(runOutcomeFrame({ workflow_run_id: "" }));expect(timelineGet).not.toHaveBeenCalled();@@ -257,8 +331,16 @@ describe("WorkflowCopilotChat — recorded-action fetch wiring", () => {await renderChat();await submit("build a workflow");- streamCalls[0]!.onMessage(runOutcomeFrame());+ // Cover every entry point: the early poll start, the evaluating fallback,+ // and the terminal convergence fetch must all stay inert flag-off.+ streamCalls[0]!.onMessage(blockProgressFrame({ workflow_run_id: "wr_1" }));+ streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "evaluating" }));+ streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "demonstrated" }));+ // The terminal fetch is async (awaits getClient), so flush pending+ // microtasks before asserting it never fired — a synchronous assert would+ // pass even if the guard were missing.+ await new Promise((resolve) => setTimeout(resolve, 0));expect(timelineGet).not.toHaveBeenCalled();});diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex d8f1a53a5..3e9823f78 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -116,6 +116,11 @@ import { TooltipProvider } from "@/components/ui/tooltip";// handful of turns; this ceiling guards a runaway long-running chat.const MAX_TURN_SNAPSHOTS = 20;+// Cadence for re-fetching a live test run's recorded actions. Mirrors the+// backend block-status poll (5s) closely enough to surface rows soon after+// they land without hammering the timeline endpoint.+const RECORDED_ACTIONS_POLL_INTERVAL_MS = 2500;+function normalizeInline(value: string | null | undefined): string | null {if (!value) return null;const trimmed = value.replace(/\s+/g, " ").trim();@@ -831,10 +836,11 @@ export function WorkflowCopilotChat({// Most recent turn_id observed via turn_start; used by Reject and by// legacy error frames that don't carry a turn_id.const latestTurnId = useRef<string | null>(null);- // One-shot guard for the recorded-actions timeline fetch, keyed by- // workflow_run_id — a run's evaluating and final verdict frames share an- // id, so this stops the same run from being fetched twice.- const fetchedActionRunIds = useRef<Set<string>>(new Set());+ // Active recorded-action timeline polls keyed by workflow_run_id, and the+ // ids whose poll already converged on a terminal verdict. Together they stop+ // a run from being polled twice and from restarting after it finalized.+ const actionPollRef = useRef<Map<string, number>>(new Map());+ const finalizedActionRunIds = useRef<Set<string>>(new Set());// Run ids the copilot claimed via run_outcome — the turn narrates these// itself, so useRunLifecycleAnnouncements suppresses their lifecycle lines by// identity (an unrelated run seen in the same window must still be narrated).@@ -843,8 +849,11 @@ export function WorkflowCopilotChat({workflowCopilotChatIdRef.current = workflowCopilotChatId;}, [workflowCopilotChatId]);useEffect(() => {+ const activePolls = actionPollRef.current;return () => {streamingAbortController.current?.abort();+ activePolls.forEach((timer) => clearInterval(timer));+ activePolls.clear();if (cancelSafetyTimer.current !== null) {clearTimeout(cancelSafetyTimer.current);cancelSafetyTimer.current = null;@@ -902,27 +911,15 @@ export function WorkflowCopilotChat({turnOwnedRunIds,announce: announceRunLifecycle,});- // Recorded actions arrive well after block_progress (they're persisted in- // batch at block end), so fetch them once a run reaches adjudication- // instead of waiting on the narrower block_progress/tool-call cadence.- const maybeFetchRecordedActions = useCallback(- async (payload: WorkflowCopilotRunOutcomeUpdate) => {- if (!copilotUxV1Enabled) return;- const runId = payload.workflow_run_id;- if (- !runId ||- !workflowPermanentId ||- fetchedActionRunIds.current.has(runId)- ) {- return;- }- const seen = fetchedActionRunIds.current;- seen.add(runId);- while (seen.size > MAX_TURN_SNAPSHOTS) {- const oldest = seen.values().next().value;- if (oldest === undefined) break;- seen.delete(oldest);- }+ // Recorded actions stream into the run timeline as the test run executes.+ // Fetch them repeatedly while a run is live (started from the earliest frame+ // that carries the run id: block_progress on a new backend, run_outcome on an+ // old one), so rows resolve during execution instead of only at adjudication.+ // The reducer merges by actionId (idempotent + grow), so re-fetching the same+ // growing set never duplicates rows and the terminal fetch just converges.+ const fetchRecordedActions = useCallback(+ async (runId: string) => {+ if (!copilotUxV1Enabled || !workflowPermanentId) return;try {const client = await getClient(credentialGetter);const response = await client.get<WorkflowRunTimelineItem[]>(@@ -967,6 +964,51 @@ export function WorkflowCopilotChat({workflowPermanentId,],);+ const finalizeRecordedActionsPoll = useCallback((runId: string) => {+ const timer = actionPollRef.current.get(runId);+ if (timer !== undefined) {+ clearInterval(timer);+ actionPollRef.current.delete(runId);+ }+ const done = finalizedActionRunIds.current;+ done.add(runId);+ while (done.size > MAX_TURN_SNAPSHOTS) {+ const oldest = done.values().next().value;+ if (oldest === undefined) break;+ done.delete(oldest);+ }+ }, []);+ const startRecordedActionsPoll = useCallback(+ (runId: string | null | undefined) => {+ if (!copilotUxV1Enabled || !runId || !workflowPermanentId) return;+ if (+ actionPollRef.current.has(runId) ||+ finalizedActionRunIds.current.has(runId)+ ) {+ return;+ }+ const timers = actionPollRef.current;+ while (timers.size >= MAX_TURN_SNAPSHOTS) {+ const oldest = timers.entries().next().value;+ if (oldest === undefined) break;+ clearInterval(oldest[1]);+ timers.delete(oldest[0]);+ }+ void fetchRecordedActions(runId);+ timers.set(+ runId,+ window.setInterval(+ () => void fetchRecordedActions(runId),+ RECORDED_ACTIONS_POLL_INTERVAL_MS,+ ),+ );+ },+ [copilotUxV1Enabled, fetchRecordedActions, workflowPermanentId],+ );+ const stopAllRecordedActionsPolls = useCallback(() => {+ actionPollRef.current.forEach((timer) => clearInterval(timer));+ actionPollRef.current.clear();+ }, []);// Fetch the stored credentials once a pause is live. ponytail: page_size 100,// no pagination — a >100-credential org may miss a match and fall back to the// Connect CTA. Add pagination if that shows up. Setting credentialsList back@@ -2209,8 +2251,13 @@ export function WorkflowCopilotChat({case "tool_call":case "tool_result":case "narration":+ applyStoredNarrativeEvent(payload);+ return false;case "block_progress":applyStoredNarrativeEvent(payload);+ // Earliest frame carrying the run id on a new backend — start+ // the live poll here so rows appear mid-execution.+ startRecordedActionsPoll(payload.workflow_run_id);return false;case "run_outcome":applyStoredNarrativeEvent(payload);@@ -2222,8 +2269,16 @@ export function WorkflowCopilotChat({if (oldest === undefined) break;owned.delete(oldest);}+ if (payload.verdict === "evaluating") {+ // Fallback start against an old backend whose block_progress+ // carried no run id: this is the first sighting.+ startRecordedActionsPoll(payload.workflow_run_id);+ } else {+ // Terminal verdict: one convergent fetch, then stop polling.+ void fetchRecordedActions(payload.workflow_run_id);+ finalizeRecordedActionsPoll(payload.workflow_run_id);+ }}- maybeFetchRecordedActions(payload);return false;case "credential_required":setLivePauseFrame(payload);@@ -2317,6 +2372,9 @@ export function WorkflowCopilotChat({pendingMessageId.current = null;pendingCancelToken.current = null;setIsLoading(false);+ // Backstop: a turn that ends without a terminal run_outcome (thrown+ // stream) would otherwise leave a live poll running past the run.+ stopAllRecordedActionsPolls();}},[@@ -2329,14 +2387,17 @@ export function WorkflowCopilotChat({copilotUxV1Enabled,copilotV2Enabled,credentialGetter,+ fetchRecordedActions,+ finalizeRecordedActionsPoll,getSaveData,inputValue,isSpeechListening,isBuild,isLiveBrowserReady,liveBrowserSessionId,- maybeFetchRecordedActions,pendingProposalTurnId,+ startRecordedActionsPoll,+ stopAllRecordedActionsPolls,requiresLiveBrowser,resyncProposalFromChatRow,stopSpeech,diff --git a/skyvern-frontend/src/routes/workflows/copilot/actionReveal.test.ts b/skyvern-frontend/src/routes/workflows/copilot/actionReveal.test.tsindex a8a324b2f..bdcf5da0b 100644--- a/skyvern-frontend/src/routes/workflows/copilot/actionReveal.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/actionReveal.test.ts@@ -62,4 +62,19 @@ describe("revealedCountAt", () => {it("reveals nothing for negative elapsed", () => {expect(revealedCountAt(offsets, -50)).toBe(0);});++ it("never un-reveals a shown row when the action list grows under live polling", () => {+ // Live polling appends actions to a block whose reveal anchor stays fixed.+ // buildRevealOffsets recomputes from the full (grown) list and may rescale+ // to the 6s cap, shrinking every offset — which can only reveal MORE at a+ // fixed elapsed, never fewer. Guards against a growing set flickering rows+ // back to hidden.+ const base = [900, 900, 900];+ const grown = [900, 900, 900, 900, 900, 900, 900, 900]; // total > 6s cap -> rescaled+ for (let elapsed = 0; elapsed <= 8000; elapsed += 137) {+ const before = revealedCountAt(buildRevealOffsets(base), elapsed);+ const after = revealedCountAt(buildRevealOffsets(grown), elapsed);+ expect(after).toBeGreaterThanOrEqual(before);+ }+ });});diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.blockActions.test.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.blockActions.test.tsindex aa5becfae..be7b5a5b6 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.blockActions.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.blockActions.test.ts@@ -94,7 +94,11 @@ describe("applyNarrativeEvent — client_block_actions", () => {expect(block.recordedActionsAt).toBe(1_000);});- it("is idempotent — a duplicate dispatch for the same block keeps the first application", () => {+ it("grows the action list when a later live-poll fetch returns new actions", () => {+ // First poll sees one action; a later poll returns the growing set. The new+ // action appends (keyed by actionId) and the reveal anchor stays put so+ // already-revealed rows don't restart. Fails on the old freeze-once reducer,+ // which dropped everything after the first application.const first = blockActions({blocks: [{@@ -108,14 +112,43 @@ describe("applyNarrativeEvent — client_block_actions", () => {blocks: [{workflowRunBlockId: "wrb_open_search",- actions: [recordedAction({ actionId: "a2" })],+ actions: [+ recordedAction({ actionId: "a1" }),+ recordedAction({ actionId: "a2" }),+ ],},],});const s = reduce([...oneBlockRunning, first, second]);const block = s.blocks.find((b) => b.label === "open_search")!;+ expect(block.recordedActions!.map((a) => a.actionId)).toEqual(["a1", "a2"]);+ expect(block.recordedActionsAt).toBe(1_000);+ });++ it("is idempotent — a re-fetch returning already-seen actions is a no-op", () => {+ const first = blockActions({+ blocks: [+ {+ workflowRunBlockId: "wrb_open_search",+ actions: [recordedAction({ actionId: "a1" })],+ },+ ],+ });+ const second = blockActions({+ receivedAtMs: 99_999,+ blocks: [+ {+ workflowRunBlockId: "wrb_open_search",+ actions: [recordedAction({ actionId: "a1" })],+ },+ ],+ });+ const withFirst = reduce([...oneBlockRunning, first]);+ const after = applyNarrativeEvent(withFirst, second);+ // No new actionId -> same state reference, so no re-render and no duplicate row.+ expect(after).toBe(withFirst);+ const block = after.blocks.find((b) => b.label === "open_search")!;expect(block.recordedActions).toHaveLength(1);- expect(block.recordedActions![0]!.actionId).toBe("a1");expect(block.recordedActionsAt).toBe(1_000);});diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.tsindex 4a7cd09c3..bb36617d9 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts@@ -20,8 +20,9 @@ import {WorkflowCopilotWorkflowDraftUpdate,} from "./workflowCopilotTypes";-// A code block's recorded actions, fetched once from the run timeline after-// a run_outcome frame arrives (block_progress carries no action detail).+// A block's recorded actions, fetched from the run timeline while the test run+// is live (polled from the first frame that carries the run id) and again at+// adjudication. block_progress carries no action detail, only the run id.export interface RecordedActionSummary {actionId: string;label: string;@@ -104,8 +105,8 @@ export interface BlockState {// ``done · 0:14``-style elapsed pill in the card.startedAt: string | null;endedAt: string | null;- // Recorded actions fetched post-run for progressive reveal. Undefined- // until the one-shot fetch resolves; set at most once (idempotent).+ // Recorded actions for progressive reveal. Undefined until the first fetch+ // resolves; grows by actionId as live polls return more (idempotent merge).recordedActions?: RecordedActionSummary[];// Epoch ms this block's reveal schedule starts counting from — staggered// past preceding blocks' schedules so a multi-block run reveals in order.@@ -716,25 +717,37 @@ export function applyNarrativeEvent(}case "client_block_actions": {- // Idempotent merge: a block's recordedActions is set at most once, so- // a duplicate fetch response (or a re-dispatched event) is a no-op.- // Stagger each matched block's reveal start past the running total of- // its predecessors' own schedules, so a multi-block run replays in- // execution order instead of all at once.+ // Idempotent grow-merge: live polling re-fetches the same run repeatedly+ // and returns a growing action set. First sighting seeds the reveal+ // anchor (recordedActionsAt), staggered past preceding blocks' schedules+ // so a multi-block run replays in execution order. Later fetches append+ // only actions we haven't seen (keyed by actionId) and keep the anchor+ // fixed, so already-revealed rows never restart or duplicate.+ // ponytail: an existing action's fields (duration/summary) are frozen at+ // first sighting; if code-block streaming (null-then-backfill) makes that+ // visibly wrong, re-merge matched actionIds instead of skipping them.let carry = 0;let changed = false;const blocks = prev.blocks.map((b) => {- if (b.recordedActions !== undefined) return b;const match = event.blocks.find((entry) => entry.workflowRunBlockId === b.workflowRunBlockId,);if (!match || match.actions.length === 0) return b;+ const existing = b.recordedActions;+ if (existing === undefined) {+ changed = true;+ const recordedActionsAt = event.receivedAtMs + carry;+ const offsets = buildRevealOffsets(+ match.actions.map((a) => a.durationMs),+ );+ carry += offsets[offsets.length - 1] ?? 0;+ return { ...b, recordedActions: match.actions, recordedActionsAt };+ }+ const known = new Set(existing.map((a) => a.actionId));+ const additions = match.actions.filter((a) => !known.has(a.actionId));+ if (additions.length === 0) return b;changed = true;- const recordedActionsAt = event.receivedAtMs + carry;- const durations = match.actions.map((a) => a.durationMs);- const offsets = buildRevealOffsets(durations);- carry += offsets[offsets.length - 1] ?? 0;- return { ...b, recordedActions: match.actions, recordedActionsAt };+ return { ...b, recordedActions: [...existing, ...additions] };});return changed ? { ...prev, blocks } : prev;}diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex 10aaa797c..afd64b047 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -229,6 +229,10 @@ export interface WorkflowCopilotNarrationUpdate {export interface WorkflowCopilotBlockProgressUpdate {type: "block_progress";workflow_run_block_id: string;+ // Present once the backend stamps the dispatched run id onto progress frames+ // (surfaced mid-execution). Absent against older backends, which only carry+ // the run id on run_outcome; the chat falls back to that.+ workflow_run_id?: string | null;block_label: string;block_type: string;status: string;