Skip to content

Automate browser based workflows with AI

Name the blocks the copilot is drafting while it writes them (SKY-12830) (#8578)

Aaron Perez5f9f7fc16764
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.presentation.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.presentation.test.tsx
index cf7eb0b9c..052dd6f5e 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.presentation.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.presentation.test.tsx
@@ -8,6 +8,8 @@ import {
BlockState,
EMPTY_NARRATIVE,
TurnNarrativeState,
+ applyNarrativeEvent,
+ hydrateNarrativeFromPayload,
} from "./narrativeState";
const completedBlock = (): BlockState => ({
@@ -160,3 +162,156 @@ describe("NarrativeView structured turn presentation", () => {
expect(activityRow.getAttribute("aria-expanded")).toBeNull();
});
});
+
+// codegen_progress frames are live-only and never persisted: the backend
+// streams them while the model writes an authoring tool call's arguments, and
+// the tool_call / workflow_draft frames that follow supersede them.
+const draftingTurn = (frames: string[][]): TurnNarrativeState => {
+ let turn = applyNarrativeEvent(EMPTY_NARRATIVE, {
+ type: "turn_start",
+ turn_id: "turn-1",
+ turn_index: 0,
+ timestamp: "2026-05-30T00:00:00Z",
+ });
+ turn = applyNarrativeEvent(turn, {
+ type: "design_start",
+ timestamp: "2026-05-30T00:00:00Z",
+ });
+ frames.forEach((blocksDrafted, i) => {
+ turn = applyNarrativeEvent(turn, {
+ type: "codegen_progress",
+ tool_name: "update_and_run_blocks",
+ blocks_drafted: blocksDrafted,
+ chars_streamed: (i + 1) * 400,
+ iteration: 1,
+ timestamp: `2026-05-30T00:00:0${i + 1}Z`,
+ });
+ });
+ return turn;
+};
+
+// The log stamps every row with its own id; the drafting row's is stable.
+const draftingRow = (): HTMLElement | null =>
+ document.querySelector('[data-activity-row-id="codegen-progress"]');
+
+const draftingRowText = (): string => {
+ const row = draftingRow();
+ if (row === null) throw new Error("no drafting row rendered");
+ return row.textContent ?? "";
+};
+
+describe("NarrativeView drafting row (codegen_progress)", () => {
+ // The frames carry the raw authored identifiers, so the row is also where
+ // `open_page` would leak into the chat if it were rendered verbatim.
+ it("names the blocks, humanized, in the order the frames drafted them", () => {
+ render(
+ <NarrativeView
+ turn={draftingTurn([
+ [],
+ ["open_page"],
+ ["open_page", "fill_form"],
+ ["open_page", "fill_form", "extract_results"],
+ ])}
+ />,
+ );
+
+ const text = draftingRowText();
+ expect(text).not.toContain("open_page");
+ expect(text).toContain("Open Page");
+ expect(text).toContain("Fill Form");
+ expect(text).toContain("Extract Results");
+ expect(text.indexOf("Open Page")).toBeLessThan(text.indexOf("Fill Form"));
+ expect(text.indexOf("Fill Form")).toBeLessThan(
+ text.indexOf("Extract Results"),
+ );
+ });
+
+ // One generation can carry more than one authoring call. `blocks_drafted` is
+ // cumulative only WITHIN a call — the adapter keys its state by output_index
+ // and opens each new call with an empty frame — so a second call must not
+ // erase what the first one drafted.
+ it("keeps the blocks an earlier authoring call drafted in the same generation", () => {
+ render(
+ <NarrativeView
+ turn={draftingTurn([
+ ["open_page"],
+ ["open_page", "fill_form"],
+ [],
+ ["extract_results"],
+ ])}
+ />,
+ );
+
+ const text = draftingRowText();
+ expect(text).toContain("Open Page");
+ expect(text).toContain("Fill Form");
+ expect(text).toContain("Extract Results");
+ });
+
+ it("keeps the existing placeholder when no frames arrive", () => {
+ render(
+ <NarrativeView
+ turn={applyNarrativeEvent(
+ applyNarrativeEvent(EMPTY_NARRATIVE, {
+ type: "turn_start",
+ turn_id: "turn-1",
+ turn_index: 0,
+ timestamp: "2026-05-30T00:00:00Z",
+ }),
+ { type: "design_start", timestamp: "2026-05-30T00:00:00Z" },
+ )}
+ />,
+ );
+
+ expect(draftingRow()).toBeNull();
+ expect(
+ screen.getByText("Copilot is working on your request…"),
+ ).toBeTruthy();
+ });
+
+ it("does not outlive the live frames", () => {
+ const drafting = draftingTurn([["open_page"], ["open_page", "fill_form"]]);
+ const { rerender } = render(<NarrativeView turn={drafting} />);
+ expect(draftingRowText()).toContain("Fill Form");
+
+ // The authoring call the frames were describing is now executing: its own
+ // row reports the work, so the drafting row must not stay beside it.
+ rerender(
+ <NarrativeView
+ turn={applyNarrativeEvent(drafting, {
+ type: "tool_call",
+ tool_name: "update_and_run_blocks",
+ tool_input: {},
+ iteration: 1,
+ tool_call_id: "call-1",
+ timestamp: "2026-05-30T00:00:05Z",
+ })}
+ />,
+ );
+ expect(draftingRow()).toBeNull();
+
+ // A reload rebuilds the turn from the persisted payload, which carries no
+ // live frames — what the model wrote is reported by the persisted authoring
+ // step, and the drafting row cannot come back stuck and empty beside it.
+ const reloaded = hydrateNarrativeFromPayload({
+ turnId: "turn-1",
+ turnIndex: 0,
+ terminal: "response",
+ draft: { blockCount: 2, blockLabels: ["open_page", "fill_form"] },
+ designActivity: [
+ {
+ kind: "tool_result",
+ id: "tr-call-1",
+ text: "Wrote the workflow code",
+ toolName: "update_and_run_blocks",
+ iteration: 1,
+ success: true,
+ timestamp: "2026-05-30T00:00:09Z",
+ },
+ ],
+ });
+ rerender(<NarrativeView turn={reloaded!} />);
+ expect(draftingRow()).toBeNull();
+ expect(screen.getByText("Wrote the workflow code")).toBeTruthy();
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx
index 883a507cc..ef57bcbb7 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx
@@ -36,6 +36,7 @@ import { useThemeAsDarkOrLight } from "../../../components/useThemeAsDarkOrLight
// Row flashes green/red for 600ms once revealed — must match the tailwind
// copilot-row-flash-* animation duration.
const FLASH_WINDOW_MS = 600;
+const MAX_DRAFTING_LABELS = 6;
const OUTCOME_REASON_PREVIEW_LIMIT = 140;
const OUTCOME_NOT_CONFIRMED_REASON =
"the run finished without showing the goal was met";
@@ -1265,6 +1266,29 @@ function FActivityLogRow({
const lineContent =
soloBlock || last === undefined ? null : entryLine(last, row.label);
+ // These arrive as the raw authored identifiers (`open_page`), so they get the
+ // same humanizing every other surface applies — otherwise a block reads
+ // `open_page` here and "Open Page" in its own card a second later. The
+ // backend caps its own list at 50; a chat pane this narrow reads about three
+ // labels to a line, so capping the head keeps already-read text still while
+ // only the tail counts up.
+ const draftingLine =
+ row.draftingLabels === undefined ? null : (
+ <>
+ <span>Writing the workflow code</span>
+ {row.draftingLabels.length === 0 ? null : (
+ <span className="text-muted-foreground dark:text-slate-500">
+ {` · ${row.draftingLabels
+ .slice(0, MAX_DRAFTING_LABELS)
+ .map(humanizeBlockLabel)
+ .join(", ")}`}
+ {row.draftingLabels.length > MAX_DRAFTING_LABELS
+ ? ` +${row.draftingLabels.length - MAX_DRAFTING_LABELS} more`
+ : ""}
+ </span>
+ )}
+ </>
+ );
// A mark reports an outcome, so only a step that returned can carry one — a
// call still in flight has no outcome yet. Beyond that, a browse or write
// step that worked says so in its own sentence, so only a run's result and
@@ -1372,7 +1396,7 @@ function FActivityLogRow({
onClick={hasDetail ? onToggle : undefined}
expanded={hasDetail ? open : undefined}
>
- {lineContent?.content}
+ {draftingLine ?? lineContent?.content}
{mark === null ? null : (
<span
className={
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.instantAck.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.instantAck.test.tsx
index 846a476c5..dbbbfaf3f 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.instantAck.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.instantAck.test.tsx
@@ -259,6 +259,50 @@ describe("WorkflowCopilotChat — instant acknowledgement", () => {
expect(screen.getAllByRole("status")).toHaveLength(1);
});
+ // Drives the real stream path, not the reducer: an SSE payload whose type has
+ // no case in the chat's switch is silently swallowed by its `default`, so a
+ // test that starts at applyNarrativeEvent cannot tell a wired frame from an
+ // unwired one. This is the only test that fails if the dispatch case goes.
+ it("REGRESSION: names the drafting blocks from codegen_progress frames arriving on the stream", async () => {
+ await renderChat();
+ await submit("build a workflow");
+ expectSomeAckLine();
+
+ await act(async () => {
+ streamCalls[0]!.onMessage({
+ type: "turn_start",
+ turn_id: "turn-1",
+ turn_index: 0,
+ mode: "build",
+ timestamp: "2026-06-10T00:00:00Z",
+ });
+ streamCalls[0]!.onMessage({
+ type: "design_start",
+ timestamp: "2026-06-10T00:00:00Z",
+ });
+ streamCalls[0]!.onMessage({
+ type: "codegen_progress",
+ tool_name: "update_and_run_blocks",
+ blocks_drafted: ["open_page", "fill_form"],
+ chars_streamed: 800,
+ iteration: 1,
+ timestamp: "2026-06-10T00:00:01Z",
+ });
+ });
+
+ const row = await waitFor(() => {
+ const found = document.querySelector(
+ '[data-activity-row-id="codegen-progress"]',
+ );
+ expect(found).not.toBeNull();
+ return found!;
+ });
+ const text = row.textContent ?? "";
+ expect(text).toContain("Writing the workflow code");
+ expect(text).toContain("Open Page");
+ expect(text).toContain("Fill Form");
+ });
+
it("REGRESSION: a narrative-less reply clears the placeholder when the turn completes", async () => {
await renderChat();
await submit("what does this workflow do?");
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 1b9a4b67c..c60f3a424 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -64,6 +64,7 @@ import {
WorkflowCopilotRunOutcomeUpdate,
WorkflowCopilotTurnStartUpdate,
WorkflowCopilotWorkflowDraftUpdate,
+ WorkflowCopilotCodegenProgressUpdate,
WorkflowCopilotCredentialRequiredUpdate,
WorkflowCopilotTitleUpdate,
WorkflowCopilotChatSender,
@@ -698,6 +699,7 @@ type WorkflowCopilotSsePayload =
| WorkflowCopilotDesignStartUpdate
| WorkflowCopilotDesignEndUpdate
| WorkflowCopilotWorkflowDraftUpdate
+ | WorkflowCopilotCodegenProgressUpdate
| WorkflowCopilotTitleUpdate
| WorkflowCopilotCredentialRequiredUpdate
| WorkflowCopilotQuestionRequired
@@ -4799,6 +4801,7 @@ export function WorkflowCopilotChat({
}
case "design_start":
case "design_end":
+ case "codegen_progress":
applyStoredNarrativeEvent(payload);
return false;
case "workflow_draft": {
diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts
index 182c5ce56..60ebd8037 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts
@@ -57,6 +57,9 @@ export interface ActivityRow {
endedAt: string | null;
// Epoch ms the winning narration arrived live; absent on hydrate.
reasonAt?: number;
+ // Block labels the model has written so far in the authoring call it is
+ // still streaming. Present only on the synthetic live drafting row.
+ draftingLabels?: string[];
}
export interface ActivityLog {
@@ -535,6 +538,29 @@ export function deriveActivityLog(turn: TurnNarrativeState): ActivityLog {
if (row.live) liveIndex = i;
});
+ // The model is still writing the authoring call's arguments: no tool call is
+ // in flight and no block is running, so nothing above claims the frontier.
+ // The frames are live-only, which is why a terminated turn never shows this
+ // and a reload — whose hydrated turn carries no progress — cannot strand it.
+ const drafting = turn.codegenProgress;
+ if (!ended && drafting !== null) {
+ rows.push({
+ id: "codegen-progress",
+ kind: "author",
+ entries: [],
+ blocks: [],
+ codeDiffs: [],
+ pending: false,
+ live: true,
+ reason: null,
+ label: null,
+ startedAt: drafting.startedAt,
+ endedAt: null,
+ draftingLabels: drafting.blockLabels,
+ });
+ liveIndex = rows.length - 1;
+ }
+
// While the model is generating, no row has an unmatched call and no block is
// running, so liveIndex is -1 and nothing would be open — the stretch that
// reads as "one collapsed row and nothing going on". Follow the newest row in
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
index d50f85a99..db909e649 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
@@ -11,6 +11,7 @@ import {
ProposalDisposition,
RunOutcomeRole,
WorkflowCopilotBlockProgressUpdate,
+ WorkflowCopilotCodegenProgressUpdate,
WorkflowCopilotDesignEndUpdate,
WorkflowCopilotDesignStartUpdate,
WorkflowCopilotNarrationUpdate,
@@ -111,6 +112,7 @@ export type NarrativeEvent =
| WorkflowCopilotNarrationUpdate
| WorkflowCopilotToolCallUpdate
| WorkflowCopilotToolResultUpdate
+ | WorkflowCopilotCodegenProgressUpdate
| CopilotBlockActionsEvent;
// Block lifecycle states as observed via block_progress. The bubble groups
@@ -399,6 +401,13 @@ export interface TurnNarrativeState {
// Activity events fired BEFORE any block started running this turn
// (design phase + pre-execution tool calls), rendered inside the Design card.
designActivity: ActivityEntry[];
+ // Live-only drafting progress from codegen_progress, never persisted. Holds
+ // only what the row renders: the frames' cumulative character count changes
+ // on every frame and would re-render the chat for nothing.
+ codegenProgress: {
+ blockLabels: string[];
+ startedAt: string | null;
+ } | null;
// Snapshot of the most recent factual run outcome.
lastRunOutcome: {
verdict: BlockOutcome;
@@ -450,6 +459,7 @@ export const EMPTY_NARRATIVE: TurnNarrativeState = Object.freeze({
startedAt: null,
endedAt: null,
designActivity: [],
+ codegenProgress: null,
lastRunOutcome: null,
credentialPrompt: null,
credentialPause: null,
@@ -1131,6 +1141,38 @@ export function applyNarrativeEvent(
},
blocks: withDiffs.blocks,
designActivity: withDiffs.designActivity,
+ codegenProgress: null,
+ };
+ }
+
+ case "codegen_progress": {
+ // `blocks_drafted` is cumulative only WITHIN one authoring call: the
+ // producer keys its state by output_index and opens each call with an
+ // empty frame. One generation can carry several authoring calls, so union
+ // rather than replace — otherwise a second call's opening frame erases
+ // the blocks the first one drafted. The tool_call that ends the
+ // generation is what clears the row, so this cannot accumulate past it.
+ const drafting = prev.codegenProgress;
+ const drafted = drafting?.blockLabels ?? [];
+ const merged = drafted.concat(
+ event.blocks_drafted.filter((label) => !drafted.includes(label)),
+ );
+ // Frames stay throttled but still arrive every couple of seconds naming
+ // nothing new, and their character count — which nothing renders — moves
+ // on every one. Returning prev unchanged is what keeps a fast stream from
+ // re-rendering the chat between labels. The first frame of a generation
+ // still has to land: it is what opens the row, and it carries no labels.
+ if (drafting !== null && merged.length === drafted.length) {
+ return prev;
+ }
+ return {
+ ...prev,
+ codegenProgress: {
+ blockLabels: merged,
+ // First frame of the generation wins, so the row's clock times the
+ // whole draft rather than restarting on each label or each call.
+ startedAt: drafting?.startedAt ?? event.timestamp ?? null,
+ },
};
}
@@ -1254,9 +1296,12 @@ export function applyNarrativeEvent(
case "tool_call": {
const entry = buildActivityFromToolCall(event);
+ // The model has finished streaming arguments: the call the drafting
+ // frames described is executing, and its own row reports it from here.
if (!entry) {
return {
...prev,
+ codegenProgress: null,
};
}
const { blocks, designActivity } = appendActivity(
@@ -1268,6 +1313,7 @@ export function applyNarrativeEvent(
...prev,
blocks,
designActivity,
+ codegenProgress: null,
};
}
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index ca55f37dc..91d9e8530 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -269,6 +269,7 @@ export type WorkflowCopilotStreamMessageType =
| "design_start"
| "design_end"
| "workflow_draft"
+ | "codegen_progress"
| "title_update"
| "credential_required"
| "question_required";
@@ -345,6 +346,19 @@ export interface WorkflowCopilotWorkflowDraftUpdate {
tool_call_id?: string | null;
}
+// Throttled progress while the model streams an authoring tool call's arguments.
+// Live-only and never persisted: a reload has no access to these, and the
+// workflow_draft / tool_call frames that follow supersede them.
+export interface WorkflowCopilotCodegenProgressUpdate {
+ type: "codegen_progress";
+ tool_name: string;
+ // Cumulative and ordered: every frame carries the full list seen so far.
+ blocks_drafted: string[];
+ chars_streamed: number;
+ iteration: number;
+ timestamp: string;
+}
+
// Emitted once the backend has persisted a derived agent name, before any block
// exists. Clients must not treat it as authoritative over a user-chosen title.
export interface WorkflowCopilotTitleUpdate {