Skip to content

Automate browser based workflows with AI

feat(SKY-12766): render copilot test-run action rows live during runs (#7609)

Celal Zamanoğlu216a63b9da94
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.blockActions.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.blockActions.test.tsx
index d44f58448..8af925044 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.blockActions.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.blockActions.test.tsx
@@ -191,6 +191,19 @@ const runOutcomeFrame = (overrides: Partial<Record<string, unknown>> = {}) => ({
...overrides,
});
+const blockProgressFrame = (
+ overrides: Partial<Record<string, unknown>> = {},
+) => ({
+ type: "block_progress",
+ workflow_run_block_id: "wrb_1",
+ block_label: "block_1",
+ block_type: "code",
+ status: "running",
+ iteration: 0,
+ timestamp: "2026-06-10T00:00:00Z",
+ ...overrides,
+});
+
beforeEach(() => {
HTMLElement.prototype.scrollIntoView = vi.fn();
HTMLElement.prototype.scrollTo = vi.fn();
@@ -217,13 +230,15 @@ afterEach(() => {
cleanup();
});
-describe("WorkflowCopilotChat — recorded-action fetch wiring", () => {
- it("fetches the run timeline exactly once when a run_outcome frame arrives", async () => {
+describe("WorkflowCopilotChat — recorded-action live poll wiring", () => {
+ it("starts a live poll from the first block_progress that carries a run id", async () => {
await renderChat();
await submit("build a workflow");
fireEvent.change(screen.getByRole("textbox"), { target: { value: "" } });
- streamCalls[0]!.onMessage(runOutcomeFrame());
+ streamCalls[0]!.onMessage(
+ blockProgressFrame({ workflow_run_id: "wr_1", status: "running" }),
+ );
await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));
expect(timelineGet.mock.calls[0]![0]).toBe(
@@ -231,22 +246,81 @@ describe("WorkflowCopilotChat — recorded-action fetch wiring", () => {
);
});
- it("does not re-fetch for a second run_outcome frame carrying the same run id", async () => {
+ it("polls the timeline repeatedly while the run stays live", async () => {
+ // Call through so React/testing-library timers keep working; we only need
+ // to capture the registered callback to drive it deterministically.
+ const setIntervalSpy = vi.spyOn(window, "setInterval");
+ try {
+ await renderChat();
+ await submit("build a workflow");
+
+ streamCalls[0]!.onMessage(
+ blockProgressFrame({ workflow_run_id: "wr_1" }),
+ );
+ // Immediate fetch on the first sighting.
+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));
+ expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), 2500);
+
+ // Drive the registered interval callback: each tick re-fetches (the
+ // reducer merges by actionId, so repeated fetches never duplicate rows).
+ const tick = setIntervalSpy.mock.calls.find((c) => c[1] === 2500)![0] as (
+ ...args: unknown[]
+ ) => void;
+ tick();
+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(2));
+ tick();
+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(3));
+ } finally {
+ setIntervalSpy.mockRestore();
+ }
+ });
+
+ it("converges with a final fetch and stops polling on a terminal run_outcome", async () => {
+ const clearIntervalSpy = vi.spyOn(window, "clearInterval");
+ const setIntervalSpy = vi.spyOn(window, "setInterval");
+ try {
+ await renderChat();
+ await submit("build a workflow");
+
+ streamCalls[0]!.onMessage(
+ blockProgressFrame({ workflow_run_id: "wr_1" }),
+ );
+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));
+ const idx = setIntervalSpy.mock.calls.findIndex((c) => c[1] === 2500);
+ const intervalId = setIntervalSpy.mock.results[idx]!.value;
+
+ // Terminal verdict: one convergent fetch, then the interval is cleared.
+ streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "demonstrated" }));
+ await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(2));
+ expect(clearIntervalSpy).toHaveBeenCalledWith(intervalId);
+ } finally {
+ setIntervalSpy.mockRestore();
+ clearIntervalSpy.mockRestore();
+ }
+ });
+
+ it("falls back to run_outcome(evaluating) when block_progress carries no run id", async () => {
await renderChat();
await submit("build a workflow");
- streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "evaluating" }));
- await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));
+ // Old backend: block_progress has no run id, so no poll can start yet.
+ streamCalls[0]!.onMessage(blockProgressFrame({ status: "running" }));
+ streamCalls[0]!.onMessage(blockProgressFrame({ status: "completed" }));
+ expect(timelineGet).not.toHaveBeenCalled();
- streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "demonstrated" }));
- // Give any accidental second fetch a chance to fire before asserting.
+ // The run id first arrives on run_outcome — the poll starts there.
+ streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "evaluating" }));
await waitFor(() => expect(timelineGet).toHaveBeenCalledTimes(1));
+ expect(timelineGet.mock.calls[0]![0]).toBe(
+ "/workflows/wpid_1/runs/wr_1/timeline",
+ );
});
- it("never fetches when the run_outcome frame carries an empty workflow_run_id", async () => {
+ it("never fetches when frames carry an empty workflow_run_id", async () => {
await renderChat();
await submit("build a workflow");
+ streamCalls[0]!.onMessage(blockProgressFrame({ workflow_run_id: "" }));
streamCalls[0]!.onMessage(runOutcomeFrame({ workflow_run_id: "" }));
expect(timelineGet).not.toHaveBeenCalled();
@@ -257,8 +331,16 @@ describe("WorkflowCopilotChat — recorded-action fetch wiring", () => {
await renderChat();
await submit("build a workflow");
- streamCalls[0]!.onMessage(runOutcomeFrame());
+ // Cover every entry point: the early poll start, the evaluating fallback,
+ // and the terminal convergence fetch must all stay inert flag-off.
+ streamCalls[0]!.onMessage(blockProgressFrame({ workflow_run_id: "wr_1" }));
+ streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "evaluating" }));
+ streamCalls[0]!.onMessage(runOutcomeFrame({ verdict: "demonstrated" }));
+ // The terminal fetch is async (awaits getClient), so flush pending
+ // microtasks before asserting it never fired — a synchronous assert would
+ // pass even if the guard were missing.
+ await new Promise((resolve) => setTimeout(resolve, 0));
expect(timelineGet).not.toHaveBeenCalled();
});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index d8f1a53a5..3e9823f78 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -116,6 +116,11 @@ import { TooltipProvider } from "@/components/ui/tooltip";
// handful of turns; this ceiling guards a runaway long-running chat.
const MAX_TURN_SNAPSHOTS = 20;
+// Cadence for re-fetching a live test run's recorded actions. Mirrors the
+// backend block-status poll (5s) closely enough to surface rows soon after
+// they land without hammering the timeline endpoint.
+const RECORDED_ACTIONS_POLL_INTERVAL_MS = 2500;
+
function normalizeInline(value: string | null | undefined): string | null {
if (!value) return null;
const trimmed = value.replace(/\s+/g, " ").trim();
@@ -831,10 +836,11 @@ export function WorkflowCopilotChat({
// Most recent turn_id observed via turn_start; used by Reject and by
// legacy error frames that don't carry a turn_id.
const latestTurnId = useRef<string | null>(null);
- // One-shot guard for the recorded-actions timeline fetch, keyed by
- // workflow_run_id — a run's evaluating and final verdict frames share an
- // id, so this stops the same run from being fetched twice.
- const fetchedActionRunIds = useRef<Set<string>>(new Set());
+ // Active recorded-action timeline polls keyed by workflow_run_id, and the
+ // ids whose poll already converged on a terminal verdict. Together they stop
+ // a run from being polled twice and from restarting after it finalized.
+ const actionPollRef = useRef<Map<string, number>>(new Map());
+ const finalizedActionRunIds = useRef<Set<string>>(new Set());
// Run ids the copilot claimed via run_outcome — the turn narrates these
// itself, so useRunLifecycleAnnouncements suppresses their lifecycle lines by
// identity (an unrelated run seen in the same window must still be narrated).
@@ -843,8 +849,11 @@ export function WorkflowCopilotChat({
workflowCopilotChatIdRef.current = workflowCopilotChatId;
}, [workflowCopilotChatId]);
useEffect(() => {
+ const activePolls = actionPollRef.current;
return () => {
streamingAbortController.current?.abort();
+ activePolls.forEach((timer) => clearInterval(timer));
+ activePolls.clear();
if (cancelSafetyTimer.current !== null) {
clearTimeout(cancelSafetyTimer.current);
cancelSafetyTimer.current = null;
@@ -902,27 +911,15 @@ export function WorkflowCopilotChat({
turnOwnedRunIds,
announce: announceRunLifecycle,
});
- // Recorded actions arrive well after block_progress (they're persisted in
- // batch at block end), so fetch them once a run reaches adjudication
- // instead of waiting on the narrower block_progress/tool-call cadence.
- const maybeFetchRecordedActions = useCallback(
- async (payload: WorkflowCopilotRunOutcomeUpdate) => {
- if (!copilotUxV1Enabled) return;
- const runId = payload.workflow_run_id;
- if (
- !runId ||
- !workflowPermanentId ||
- fetchedActionRunIds.current.has(runId)
- ) {
- return;
- }
- const seen = fetchedActionRunIds.current;
- seen.add(runId);
- while (seen.size > MAX_TURN_SNAPSHOTS) {
- const oldest = seen.values().next().value;
- if (oldest === undefined) break;
- seen.delete(oldest);
- }
+ // Recorded actions stream into the run timeline as the test run executes.
+ // Fetch them repeatedly while a run is live (started from the earliest frame
+ // that carries the run id: block_progress on a new backend, run_outcome on an
+ // old one), so rows resolve during execution instead of only at adjudication.
+ // The reducer merges by actionId (idempotent + grow), so re-fetching the same
+ // growing set never duplicates rows and the terminal fetch just converges.
+ const fetchRecordedActions = useCallback(
+ async (runId: string) => {
+ if (!copilotUxV1Enabled || !workflowPermanentId) return;
try {
const client = await getClient(credentialGetter);
const response = await client.get<WorkflowRunTimelineItem[]>(
@@ -967,6 +964,51 @@ export function WorkflowCopilotChat({
workflowPermanentId,
],
);
+ const finalizeRecordedActionsPoll = useCallback((runId: string) => {
+ const timer = actionPollRef.current.get(runId);
+ if (timer !== undefined) {
+ clearInterval(timer);
+ actionPollRef.current.delete(runId);
+ }
+ const done = finalizedActionRunIds.current;
+ done.add(runId);
+ while (done.size > MAX_TURN_SNAPSHOTS) {
+ const oldest = done.values().next().value;
+ if (oldest === undefined) break;
+ done.delete(oldest);
+ }
+ }, []);
+ const startRecordedActionsPoll = useCallback(
+ (runId: string | null | undefined) => {
+ if (!copilotUxV1Enabled || !runId || !workflowPermanentId) return;
+ if (
+ actionPollRef.current.has(runId) ||
+ finalizedActionRunIds.current.has(runId)
+ ) {
+ return;
+ }
+ const timers = actionPollRef.current;
+ while (timers.size >= MAX_TURN_SNAPSHOTS) {
+ const oldest = timers.entries().next().value;
+ if (oldest === undefined) break;
+ clearInterval(oldest[1]);
+ timers.delete(oldest[0]);
+ }
+ void fetchRecordedActions(runId);
+ timers.set(
+ runId,
+ window.setInterval(
+ () => void fetchRecordedActions(runId),
+ RECORDED_ACTIONS_POLL_INTERVAL_MS,
+ ),
+ );
+ },
+ [copilotUxV1Enabled, fetchRecordedActions, workflowPermanentId],
+ );
+ const stopAllRecordedActionsPolls = useCallback(() => {
+ actionPollRef.current.forEach((timer) => clearInterval(timer));
+ actionPollRef.current.clear();
+ }, []);
// Fetch the stored credentials once a pause is live. ponytail: page_size 100,
// no pagination — a >100-credential org may miss a match and fall back to the
// Connect CTA. Add pagination if that shows up. Setting credentialsList back
@@ -2209,8 +2251,13 @@ export function WorkflowCopilotChat({
case "tool_call":
case "tool_result":
case "narration":
+ applyStoredNarrativeEvent(payload);
+ return false;
case "block_progress":
applyStoredNarrativeEvent(payload);
+ // Earliest frame carrying the run id on a new backend — start
+ // the live poll here so rows appear mid-execution.
+ startRecordedActionsPoll(payload.workflow_run_id);
return false;
case "run_outcome":
applyStoredNarrativeEvent(payload);
@@ -2222,8 +2269,16 @@ export function WorkflowCopilotChat({
if (oldest === undefined) break;
owned.delete(oldest);
}
+ if (payload.verdict === "evaluating") {
+ // Fallback start against an old backend whose block_progress
+ // carried no run id: this is the first sighting.
+ startRecordedActionsPoll(payload.workflow_run_id);
+ } else {
+ // Terminal verdict: one convergent fetch, then stop polling.
+ void fetchRecordedActions(payload.workflow_run_id);
+ finalizeRecordedActionsPoll(payload.workflow_run_id);
+ }
}
- maybeFetchRecordedActions(payload);
return false;
case "credential_required":
setLivePauseFrame(payload);
@@ -2317,6 +2372,9 @@ export function WorkflowCopilotChat({
pendingMessageId.current = null;
pendingCancelToken.current = null;
setIsLoading(false);
+ // Backstop: a turn that ends without a terminal run_outcome (thrown
+ // stream) would otherwise leave a live poll running past the run.
+ stopAllRecordedActionsPolls();
}
},
[
@@ -2329,14 +2387,17 @@ export function WorkflowCopilotChat({
copilotUxV1Enabled,
copilotV2Enabled,
credentialGetter,
+ fetchRecordedActions,
+ finalizeRecordedActionsPoll,
getSaveData,
inputValue,
isSpeechListening,
isBuild,
isLiveBrowserReady,
liveBrowserSessionId,
- maybeFetchRecordedActions,
pendingProposalTurnId,
+ startRecordedActionsPoll,
+ stopAllRecordedActionsPolls,
requiresLiveBrowser,
resyncProposalFromChatRow,
stopSpeech,
diff --git a/skyvern-frontend/src/routes/workflows/copilot/actionReveal.test.ts b/skyvern-frontend/src/routes/workflows/copilot/actionReveal.test.ts
index a8a324b2f..bdcf5da0b 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/actionReveal.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/actionReveal.test.ts
@@ -62,4 +62,19 @@ describe("revealedCountAt", () => {
it("reveals nothing for negative elapsed", () => {
expect(revealedCountAt(offsets, -50)).toBe(0);
});
+
+ it("never un-reveals a shown row when the action list grows under live polling", () => {
+ // Live polling appends actions to a block whose reveal anchor stays fixed.
+ // buildRevealOffsets recomputes from the full (grown) list and may rescale
+ // to the 6s cap, shrinking every offset — which can only reveal MORE at a
+ // fixed elapsed, never fewer. Guards against a growing set flickering rows
+ // back to hidden.
+ const base = [900, 900, 900];
+ const grown = [900, 900, 900, 900, 900, 900, 900, 900]; // total > 6s cap -> rescaled
+ for (let elapsed = 0; elapsed <= 8000; elapsed += 137) {
+ const before = revealedCountAt(buildRevealOffsets(base), elapsed);
+ const after = revealedCountAt(buildRevealOffsets(grown), elapsed);
+ expect(after).toBeGreaterThanOrEqual(before);
+ }
+ });
});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.blockActions.test.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.blockActions.test.ts
index aa5becfae..be7b5a5b6 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.blockActions.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.blockActions.test.ts
@@ -94,7 +94,11 @@ describe("applyNarrativeEvent — client_block_actions", () => {
expect(block.recordedActionsAt).toBe(1_000);
});
- it("is idempotent — a duplicate dispatch for the same block keeps the first application", () => {
+ it("grows the action list when a later live-poll fetch returns new actions", () => {
+ // First poll sees one action; a later poll returns the growing set. The new
+ // action appends (keyed by actionId) and the reveal anchor stays put so
+ // already-revealed rows don't restart. Fails on the old freeze-once reducer,
+ // which dropped everything after the first application.
const first = blockActions({
blocks: [
{
@@ -108,14 +112,43 @@ describe("applyNarrativeEvent — client_block_actions", () => {
blocks: [
{
workflowRunBlockId: "wrb_open_search",
- actions: [recordedAction({ actionId: "a2" })],
+ actions: [
+ recordedAction({ actionId: "a1" }),
+ recordedAction({ actionId: "a2" }),
+ ],
},
],
});
const s = reduce([...oneBlockRunning, first, second]);
const block = s.blocks.find((b) => b.label === "open_search")!;
+ expect(block.recordedActions!.map((a) => a.actionId)).toEqual(["a1", "a2"]);
+ expect(block.recordedActionsAt).toBe(1_000);
+ });
+
+ it("is idempotent — a re-fetch returning already-seen actions is a no-op", () => {
+ const first = blockActions({
+ blocks: [
+ {
+ workflowRunBlockId: "wrb_open_search",
+ actions: [recordedAction({ actionId: "a1" })],
+ },
+ ],
+ });
+ const second = blockActions({
+ receivedAtMs: 99_999,
+ blocks: [
+ {
+ workflowRunBlockId: "wrb_open_search",
+ actions: [recordedAction({ actionId: "a1" })],
+ },
+ ],
+ });
+ const withFirst = reduce([...oneBlockRunning, first]);
+ const after = applyNarrativeEvent(withFirst, second);
+ // No new actionId -> same state reference, so no re-render and no duplicate row.
+ expect(after).toBe(withFirst);
+ const block = after.blocks.find((b) => b.label === "open_search")!;
expect(block.recordedActions).toHaveLength(1);
- expect(block.recordedActions![0]!.actionId).toBe("a1");
expect(block.recordedActionsAt).toBe(1_000);
});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
index 4a7cd09c3..bb36617d9 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
@@ -20,8 +20,9 @@ import {
WorkflowCopilotWorkflowDraftUpdate,
} from "./workflowCopilotTypes";
-// A code block's recorded actions, fetched once from the run timeline after
-// a run_outcome frame arrives (block_progress carries no action detail).
+// A block's recorded actions, fetched from the run timeline while the test run
+// is live (polled from the first frame that carries the run id) and again at
+// adjudication. block_progress carries no action detail, only the run id.
export interface RecordedActionSummary {
actionId: string;
label: string;
@@ -104,8 +105,8 @@ export interface BlockState {
// ``done · 0:14``-style elapsed pill in the card.
startedAt: string | null;
endedAt: string | null;
- // Recorded actions fetched post-run for progressive reveal. Undefined
- // until the one-shot fetch resolves; set at most once (idempotent).
+ // Recorded actions for progressive reveal. Undefined until the first fetch
+ // resolves; grows by actionId as live polls return more (idempotent merge).
recordedActions?: RecordedActionSummary[];
// Epoch ms this block's reveal schedule starts counting from — staggered
// past preceding blocks' schedules so a multi-block run reveals in order.
@@ -716,25 +717,37 @@ export function applyNarrativeEvent(
}
case "client_block_actions": {
- // Idempotent merge: a block's recordedActions is set at most once, so
- // a duplicate fetch response (or a re-dispatched event) is a no-op.
- // Stagger each matched block's reveal start past the running total of
- // its predecessors' own schedules, so a multi-block run replays in
- // execution order instead of all at once.
+ // Idempotent grow-merge: live polling re-fetches the same run repeatedly
+ // and returns a growing action set. First sighting seeds the reveal
+ // anchor (recordedActionsAt), staggered past preceding blocks' schedules
+ // so a multi-block run replays in execution order. Later fetches append
+ // only actions we haven't seen (keyed by actionId) and keep the anchor
+ // fixed, so already-revealed rows never restart or duplicate.
+ // ponytail: an existing action's fields (duration/summary) are frozen at
+ // first sighting; if code-block streaming (null-then-backfill) makes that
+ // visibly wrong, re-merge matched actionIds instead of skipping them.
let carry = 0;
let changed = false;
const blocks = prev.blocks.map((b) => {
- if (b.recordedActions !== undefined) return b;
const match = event.blocks.find(
(entry) => entry.workflowRunBlockId === b.workflowRunBlockId,
);
if (!match || match.actions.length === 0) return b;
+ const existing = b.recordedActions;
+ if (existing === undefined) {
+ changed = true;
+ const recordedActionsAt = event.receivedAtMs + carry;
+ const offsets = buildRevealOffsets(
+ match.actions.map((a) => a.durationMs),
+ );
+ carry += offsets[offsets.length - 1] ?? 0;
+ return { ...b, recordedActions: match.actions, recordedActionsAt };
+ }
+ const known = new Set(existing.map((a) => a.actionId));
+ const additions = match.actions.filter((a) => !known.has(a.actionId));
+ if (additions.length === 0) return b;
changed = true;
- const recordedActionsAt = event.receivedAtMs + carry;
- const durations = match.actions.map((a) => a.durationMs);
- const offsets = buildRevealOffsets(durations);
- carry += offsets[offsets.length - 1] ?? 0;
- return { ...b, recordedActions: match.actions, recordedActionsAt };
+ return { ...b, recordedActions: [...existing, ...additions] };
});
return changed ? { ...prev, blocks } : prev;
}
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index 10aaa797c..afd64b047 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -229,6 +229,10 @@ export interface WorkflowCopilotNarrationUpdate {
export interface WorkflowCopilotBlockProgressUpdate {
type: "block_progress";
workflow_run_block_id: string;
+ // Present once the backend stamps the dispatched run id onto progress frames
+ // (surfaced mid-execution). Absent against older backends, which only carry
+ // the run id on run_outcome; the chat falls back to that.
+ workflow_run_id?: string | null;
block_label: string;
block_type: string;
status: string;