Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Name the blocks the copilot is drafting while it writes them (SKY-12830) (#8578)
Aaron Perez5f9f7fc16764
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.presentation.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.presentation.test.tsxindex cf7eb0b9c..052dd6f5e 100644--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.presentation.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.presentation.test.tsx@@ -8,6 +8,8 @@ import {BlockState,EMPTY_NARRATIVE,TurnNarrativeState,+ applyNarrativeEvent,+ hydrateNarrativeFromPayload,} from "./narrativeState";const completedBlock = (): BlockState => ({@@ -160,3 +162,156 @@ describe("NarrativeView structured turn presentation", () => {expect(activityRow.getAttribute("aria-expanded")).toBeNull();});});++// codegen_progress frames are live-only and never persisted: the backend+// streams them while the model writes an authoring tool call's arguments, and+// the tool_call / workflow_draft frames that follow supersede them.+const draftingTurn = (frames: string[][]): TurnNarrativeState => {+ let turn = applyNarrativeEvent(EMPTY_NARRATIVE, {+ type: "turn_start",+ turn_id: "turn-1",+ turn_index: 0,+ timestamp: "2026-05-30T00:00:00Z",+ });+ turn = applyNarrativeEvent(turn, {+ type: "design_start",+ timestamp: "2026-05-30T00:00:00Z",+ });+ frames.forEach((blocksDrafted, i) => {+ turn = applyNarrativeEvent(turn, {+ type: "codegen_progress",+ tool_name: "update_and_run_blocks",+ blocks_drafted: blocksDrafted,+ chars_streamed: (i + 1) * 400,+ iteration: 1,+ timestamp: `2026-05-30T00:00:0${i + 1}Z`,+ });+ });+ return turn;+};++// The log stamps every row with its own id; the drafting row's is stable.+const draftingRow = (): HTMLElement | null =>+ document.querySelector('[data-activity-row-id="codegen-progress"]');++const draftingRowText = (): string => {+ const row = draftingRow();+ if (row === null) throw new Error("no drafting row rendered");+ return row.textContent ?? "";+};++describe("NarrativeView drafting row (codegen_progress)", () => {+ // The frames carry the raw authored identifiers, so the row is also where+ // `open_page` would leak into the chat if it were rendered verbatim.+ it("names the blocks, humanized, in the order the frames drafted them", () => {+ render(+ <NarrativeView+ turn={draftingTurn([+ [],+ ["open_page"],+ ["open_page", "fill_form"],+ ["open_page", "fill_form", "extract_results"],+ ])}+ />,+ );++ const text = draftingRowText();+ expect(text).not.toContain("open_page");+ expect(text).toContain("Open Page");+ expect(text).toContain("Fill Form");+ expect(text).toContain("Extract Results");+ expect(text.indexOf("Open Page")).toBeLessThan(text.indexOf("Fill Form"));+ expect(text.indexOf("Fill Form")).toBeLessThan(+ text.indexOf("Extract Results"),+ );+ });++ // One generation can carry more than one authoring call. `blocks_drafted` is+ // cumulative only WITHIN a call — the adapter keys its state by output_index+ // and opens each new call with an empty frame — so a second call must not+ // erase what the first one drafted.+ it("keeps the blocks an earlier authoring call drafted in the same generation", () => {+ render(+ <NarrativeView+ turn={draftingTurn([+ ["open_page"],+ ["open_page", "fill_form"],+ [],+ ["extract_results"],+ ])}+ />,+ );++ const text = draftingRowText();+ expect(text).toContain("Open Page");+ expect(text).toContain("Fill Form");+ expect(text).toContain("Extract Results");+ });++ it("keeps the existing placeholder when no frames arrive", () => {+ render(+ <NarrativeView+ turn={applyNarrativeEvent(+ applyNarrativeEvent(EMPTY_NARRATIVE, {+ type: "turn_start",+ turn_id: "turn-1",+ turn_index: 0,+ timestamp: "2026-05-30T00:00:00Z",+ }),+ { type: "design_start", timestamp: "2026-05-30T00:00:00Z" },+ )}+ />,+ );++ expect(draftingRow()).toBeNull();+ expect(+ screen.getByText("Copilot is working on your request…"),+ ).toBeTruthy();+ });++ it("does not outlive the live frames", () => {+ const drafting = draftingTurn([["open_page"], ["open_page", "fill_form"]]);+ const { rerender } = render(<NarrativeView turn={drafting} />);+ expect(draftingRowText()).toContain("Fill Form");++ // The authoring call the frames were describing is now executing: its own+ // row reports the work, so the drafting row must not stay beside it.+ rerender(+ <NarrativeView+ turn={applyNarrativeEvent(drafting, {+ type: "tool_call",+ tool_name: "update_and_run_blocks",+ tool_input: {},+ iteration: 1,+ tool_call_id: "call-1",+ timestamp: "2026-05-30T00:00:05Z",+ })}+ />,+ );+ expect(draftingRow()).toBeNull();++ // A reload rebuilds the turn from the persisted payload, which carries no+ // live frames — what the model wrote is reported by the persisted authoring+ // step, and the drafting row cannot come back stuck and empty beside it.+ const reloaded = hydrateNarrativeFromPayload({+ turnId: "turn-1",+ turnIndex: 0,+ terminal: "response",+ draft: { blockCount: 2, blockLabels: ["open_page", "fill_form"] },+ designActivity: [+ {+ kind: "tool_result",+ id: "tr-call-1",+ text: "Wrote the workflow code",+ toolName: "update_and_run_blocks",+ iteration: 1,+ success: true,+ timestamp: "2026-05-30T00:00:09Z",+ },+ ],+ });+ rerender(<NarrativeView turn={reloaded!} />);+ expect(draftingRow()).toBeNull();+ expect(screen.getByText("Wrote the workflow code")).toBeTruthy();+ });+});diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsxindex 883a507cc..ef57bcbb7 100644--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx@@ -36,6 +36,7 @@ import { useThemeAsDarkOrLight } from "../../../components/useThemeAsDarkOrLight// Row flashes green/red for 600ms once revealed — must match the tailwind// copilot-row-flash-* animation duration.const FLASH_WINDOW_MS = 600;+const MAX_DRAFTING_LABELS = 6;const OUTCOME_REASON_PREVIEW_LIMIT = 140;const OUTCOME_NOT_CONFIRMED_REASON ="the run finished without showing the goal was met";@@ -1265,6 +1266,29 @@ function FActivityLogRow({const lineContent =soloBlock || last === undefined ? null : entryLine(last, row.label);+ // These arrive as the raw authored identifiers (`open_page`), so they get the+ // same humanizing every other surface applies — otherwise a block reads+ // `open_page` here and "Open Page" in its own card a second later. The+ // backend caps its own list at 50; a chat pane this narrow reads about three+ // labels to a line, so capping the head keeps already-read text still while+ // only the tail counts up.+ const draftingLine =+ row.draftingLabels === undefined ? null : (+ <>+ <span>Writing the workflow code</span>+ {row.draftingLabels.length === 0 ? null : (+ <span className="text-muted-foreground dark:text-slate-500">+ {` · ${row.draftingLabels+ .slice(0, MAX_DRAFTING_LABELS)+ .map(humanizeBlockLabel)+ .join(", ")}`}+ {row.draftingLabels.length > MAX_DRAFTING_LABELS+ ? ` +${row.draftingLabels.length - MAX_DRAFTING_LABELS} more`+ : ""}+ </span>+ )}+ </>+ );// A mark reports an outcome, so only a step that returned can carry one — a// call still in flight has no outcome yet. Beyond that, a browse or write// step that worked says so in its own sentence, so only a run's result and@@ -1372,7 +1396,7 @@ function FActivityLogRow({onClick={hasDetail ? onToggle : undefined}expanded={hasDetail ? open : undefined}>- {lineContent?.content}+ {draftingLine ?? lineContent?.content}{mark === null ? null : (<spanclassName={diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.instantAck.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.instantAck.test.tsxindex 846a476c5..dbbbfaf3f 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.instantAck.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.instantAck.test.tsx@@ -259,6 +259,50 @@ describe("WorkflowCopilotChat — instant acknowledgement", () => {expect(screen.getAllByRole("status")).toHaveLength(1);});+ // Drives the real stream path, not the reducer: an SSE payload whose type has+ // no case in the chat's switch is silently swallowed by its `default`, so a+ // test that starts at applyNarrativeEvent cannot tell a wired frame from an+ // unwired one. This is the only test that fails if the dispatch case goes.+ it("REGRESSION: names the drafting blocks from codegen_progress frames arriving on the stream", async () => {+ await renderChat();+ await submit("build a workflow");+ expectSomeAckLine();++ await act(async () => {+ streamCalls[0]!.onMessage({+ type: "turn_start",+ turn_id: "turn-1",+ turn_index: 0,+ mode: "build",+ timestamp: "2026-06-10T00:00:00Z",+ });+ streamCalls[0]!.onMessage({+ type: "design_start",+ timestamp: "2026-06-10T00:00:00Z",+ });+ streamCalls[0]!.onMessage({+ type: "codegen_progress",+ tool_name: "update_and_run_blocks",+ blocks_drafted: ["open_page", "fill_form"],+ chars_streamed: 800,+ iteration: 1,+ timestamp: "2026-06-10T00:00:01Z",+ });+ });++ const row = await waitFor(() => {+ const found = document.querySelector(+ '[data-activity-row-id="codegen-progress"]',+ );+ expect(found).not.toBeNull();+ return found!;+ });+ const text = row.textContent ?? "";+ expect(text).toContain("Writing the workflow code");+ expect(text).toContain("Open Page");+ expect(text).toContain("Fill Form");+ });+it("REGRESSION: a narrative-less reply clears the placeholder when the turn completes", async () => {await renderChat();await submit("what does this workflow do?");diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 1b9a4b67c..c60f3a424 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -64,6 +64,7 @@ import {WorkflowCopilotRunOutcomeUpdate,WorkflowCopilotTurnStartUpdate,WorkflowCopilotWorkflowDraftUpdate,+ WorkflowCopilotCodegenProgressUpdate,WorkflowCopilotCredentialRequiredUpdate,WorkflowCopilotTitleUpdate,WorkflowCopilotChatSender,@@ -698,6 +699,7 @@ type WorkflowCopilotSsePayload =| WorkflowCopilotDesignStartUpdate| WorkflowCopilotDesignEndUpdate| WorkflowCopilotWorkflowDraftUpdate+ | WorkflowCopilotCodegenProgressUpdate| WorkflowCopilotTitleUpdate| WorkflowCopilotCredentialRequiredUpdate| WorkflowCopilotQuestionRequired@@ -4799,6 +4801,7 @@ export function WorkflowCopilotChat({}case "design_start":case "design_end":+ case "codegen_progress":applyStoredNarrativeEvent(payload);return false;case "workflow_draft": {diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.tsindex 182c5ce56..60ebd8037 100644--- a/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts@@ -57,6 +57,9 @@ export interface ActivityRow {endedAt: string | null;// Epoch ms the winning narration arrived live; absent on hydrate.reasonAt?: number;+ // Block labels the model has written so far in the authoring call it is+ // still streaming. Present only on the synthetic live drafting row.+ draftingLabels?: string[];}export interface ActivityLog {@@ -535,6 +538,29 @@ export function deriveActivityLog(turn: TurnNarrativeState): ActivityLog {if (row.live) liveIndex = i;});+ // The model is still writing the authoring call's arguments: no tool call is+ // in flight and no block is running, so nothing above claims the frontier.+ // The frames are live-only, which is why a terminated turn never shows this+ // and a reload — whose hydrated turn carries no progress — cannot strand it.+ const drafting = turn.codegenProgress;+ if (!ended && drafting !== null) {+ rows.push({+ id: "codegen-progress",+ kind: "author",+ entries: [],+ blocks: [],+ codeDiffs: [],+ pending: false,+ live: true,+ reason: null,+ label: null,+ startedAt: drafting.startedAt,+ endedAt: null,+ draftingLabels: drafting.blockLabels,+ });+ liveIndex = rows.length - 1;+ }+// While the model is generating, no row has an unmatched call and no block is// running, so liveIndex is -1 and nothing would be open — the stretch that// reads as "one collapsed row and nothing going on". Follow the newest row indiff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.tsindex d50f85a99..db909e649 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts@@ -11,6 +11,7 @@ import {ProposalDisposition,RunOutcomeRole,WorkflowCopilotBlockProgressUpdate,+ WorkflowCopilotCodegenProgressUpdate,WorkflowCopilotDesignEndUpdate,WorkflowCopilotDesignStartUpdate,WorkflowCopilotNarrationUpdate,@@ -111,6 +112,7 @@ export type NarrativeEvent =| WorkflowCopilotNarrationUpdate| WorkflowCopilotToolCallUpdate| WorkflowCopilotToolResultUpdate+ | WorkflowCopilotCodegenProgressUpdate| CopilotBlockActionsEvent;// Block lifecycle states as observed via block_progress. The bubble groups@@ -399,6 +401,13 @@ export interface TurnNarrativeState {// Activity events fired BEFORE any block started running this turn// (design phase + pre-execution tool calls), rendered inside the Design card.designActivity: ActivityEntry[];+ // Live-only drafting progress from codegen_progress, never persisted. Holds+ // only what the row renders: the frames' cumulative character count changes+ // on every frame and would re-render the chat for nothing.+ codegenProgress: {+ blockLabels: string[];+ startedAt: string | null;+ } | null;// Snapshot of the most recent factual run outcome.lastRunOutcome: {verdict: BlockOutcome;@@ -450,6 +459,7 @@ export const EMPTY_NARRATIVE: TurnNarrativeState = Object.freeze({startedAt: null,endedAt: null,designActivity: [],+ codegenProgress: null,lastRunOutcome: null,credentialPrompt: null,credentialPause: null,@@ -1131,6 +1141,38 @@ export function applyNarrativeEvent(},blocks: withDiffs.blocks,designActivity: withDiffs.designActivity,+ codegenProgress: null,+ };+ }++ case "codegen_progress": {+ // `blocks_drafted` is cumulative only WITHIN one authoring call: the+ // producer keys its state by output_index and opens each call with an+ // empty frame. One generation can carry several authoring calls, so union+ // rather than replace — otherwise a second call's opening frame erases+ // the blocks the first one drafted. The tool_call that ends the+ // generation is what clears the row, so this cannot accumulate past it.+ const drafting = prev.codegenProgress;+ const drafted = drafting?.blockLabels ?? [];+ const merged = drafted.concat(+ event.blocks_drafted.filter((label) => !drafted.includes(label)),+ );+ // Frames stay throttled but still arrive every couple of seconds naming+ // nothing new, and their character count — which nothing renders — moves+ // on every one. Returning prev unchanged is what keeps a fast stream from+ // re-rendering the chat between labels. The first frame of a generation+ // still has to land: it is what opens the row, and it carries no labels.+ if (drafting !== null && merged.length === drafted.length) {+ return prev;+ }+ return {+ ...prev,+ codegenProgress: {+ blockLabels: merged,+ // First frame of the generation wins, so the row's clock times the+ // whole draft rather than restarting on each label or each call.+ startedAt: drafting?.startedAt ?? event.timestamp ?? null,+ },};}@@ -1254,9 +1296,12 @@ export function applyNarrativeEvent(case "tool_call": {const entry = buildActivityFromToolCall(event);+ // The model has finished streaming arguments: the call the drafting+ // frames described is executing, and its own row reports it from here.if (!entry) {return {...prev,+ codegenProgress: null,};}const { blocks, designActivity } = appendActivity(@@ -1268,6 +1313,7 @@ export function applyNarrativeEvent(...prev,blocks,designActivity,+ codegenProgress: null,};}diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex ca55f37dc..91d9e8530 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -269,6 +269,7 @@ export type WorkflowCopilotStreamMessageType =| "design_start"| "design_end"| "workflow_draft"+ | "codegen_progress"| "title_update"| "credential_required"| "question_required";@@ -345,6 +346,19 @@ export interface WorkflowCopilotWorkflowDraftUpdate {tool_call_id?: string | null;}+// Throttled progress while the model streams an authoring tool call's arguments.+// Live-only and never persisted: a reload has no access to these, and the+// workflow_draft / tool_call frames that follow supersede them.+export interface WorkflowCopilotCodegenProgressUpdate {+ type: "codegen_progress";+ tool_name: string;+ // Cumulative and ordered: every frame carries the full list seen so far.+ blocks_drafted: string[];+ chars_streamed: number;+ iteration: number;+ timestamp: string;+}+// Emitted once the backend has persisted a derived agent name, before any block// exists. Clients must not treat it as authoritative over a user-chosen title.export interface WorkflowCopilotTitleUpdate {