Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Render the Copilot turn from one fact bundle instead of three verdicts (SKY-14512) (#8316)
Andrew Neilsonc3b6f780408f
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsxindex 98483a294..4206facd9 100644--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx@@ -163,7 +163,7 @@ describe("NarrativeView — activity log", () => {it("moves focus between the terminal rollup and detail disclosures", async () => {render(<NarrativeView turn={repairLoopTurn()} />);const rollup = screen.getByRole("button", {- name: /Built and tested the workflow/,+ name: /Built the workflow/,});expect(rollup.getAttribute("aria-expanded")).toBe("false");expect(rollup.hasAttribute("aria-controls")).toBe(false);@@ -178,7 +178,7 @@ describe("NarrativeView — activity log", () => {fireEvent.click(collapse);const restored = screen.getByRole("button", {- name: /Built and tested the workflow/,+ name: /Built the workflow/,});await waitFor(() => expect(document.activeElement).toBe(restored));});diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsxindex f68f54ff6..e081a110a 100644--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx@@ -60,7 +60,7 @@ const inFlightTurn = (): TurnNarrativeState => ({terminal: null,});-const HEADLINE = "Built and tested the workflow";+const HEADLINE = "Built the workflow";const REVIEW_HEADLINE = "Draft needs review";const REVIEW_TESTED_HEADLINE = "Workflow ready for review";const LONG_OUTCOME_REASON =@@ -124,6 +124,18 @@ describe("NarrativeView collapse default", () => {turn={{...terminalBuildTurn(),proposalDisposition: "review_tested",+ // One block, run clean on current source — what backs a tested review proposal.+ turnFacts: {+ factsAvailable: true,+ authoredBlockCount: 1,+ matchingSourceBlockCount: 1,+ evaluationState: null,+ runId: "wr_1",+ runCompleted: true,+ terminalCause: null,+ blocksRunThisTurn: 1,+ ranCleanOnCurrentSource: true,+ },narrativeSummary: "Workflow ready for review.",}}/>,diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.tsindex 40b06041c..3087b1570 100644--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts@@ -1101,6 +1101,19 @@ const draft3 = {summary: null,};+// What the backend publishes for a draft whose blocks all ran clean on current source.+const testedFacts = {+ factsAvailable: true,+ authoredBlockCount: 3,+ matchingSourceBlockCount: 3,+ evaluationState: null,+ runId: "wr_1",+ runCompleted: true,+ terminalCause: null,+ blocksRunThisTurn: 3,+ ranCleanOnCurrentSource: true,+} as const;+describe("hydrateNarrativeFromPayload — terminal adjudication fields", () => {it("hydrates responseKind without inventing an authoring success stamp", () => {const turn = hydrateNarrativeFromPayload(@@ -1348,8 +1361,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {responseType: "REPLY",});const summary = computeTurnSummary(turn);- expect(summary.headline).toBe("Built and tested the workflow");- expect(summary.accent).toBe("ok");+ expect(summary.headline).toBe("Built the workflow");+ expect(summary.accent).toBe("qa");expect(summary.glyph).toBe("✓");});@@ -1417,7 +1430,7 @@ describe("computeTurnSummary — typed terminal adjudication", () => {},);- it("renders a verdict-authorized tested success green with drafts", () => {+ it("names the build without claiming it was tested when no coverage facts arrive", () => {const summary = computeTurnSummary(buildTurn({draft: draft3,@@ -1425,12 +1438,12 @@ describe("computeTurnSummary — typed terminal adjudication", () => {responseKind: "build",}),);- expect(summary.headline).toBe("Built and tested the workflow");- expect(summary.accent).toBe("ok");+ expect(summary.headline).toBe("Built the workflow");+ expect(summary.accent).toBe("qa");expect(summary.glyph).toBe("✓");});- it("renders a verdict-authorized edit turn green", () => {+ it("names an edit turn without claiming a re-test when no coverage facts arrive", () => {const summary = computeTurnSummary(buildTurn({draft: draft3,@@ -1438,8 +1451,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {responseKind: "build",}),);- expect(summary.headline).toBe("Applied edits and re-tested");- expect(summary.accent).toBe("ok");+ expect(summary.headline).toBe("Applied edits");+ expect(summary.accent).toBe("qa");});it("does not infer a completed run from a build response kind alone", () => {@@ -1471,8 +1484,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {responseKind: "build",}),);- expect(summary.headline).toBe("Built and tested the workflow");- expect(summary.accent).toBe("ok");+ expect(summary.headline).toBe("Built the workflow");+ expect(summary.accent).toBe("qa");expect(summary.glyph).toBe("✓");});@@ -1489,8 +1502,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {responseKind: "build",}),);- expect(summary.headline).toBe("Applied edits and re-tested");- expect(summary.accent).toBe("ok");+ expect(summary.headline).toBe("Applied edits");+ expect(summary.accent).toBe("qa");expect(summary.glyph).toBe("✓");});@@ -1509,7 +1522,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {proposalDisposition: "review_tested",responseKind: "build",}),- "Workflow ready for review",+ // No facts back the tested framing, so the card states what it built and no more.+ "Built the workflow",],])("keeps stopped or review-required states when there is no clean completed run (%#)",@@ -1525,6 +1539,7 @@ describe("computeTurnSummary — typed terminal adjudication", () => {const askTurn = buildTurn({draft: draft3,proposalDisposition: "review_tested",+ turnFacts: testedFacts,responseKind: "clarify",responseType: "ASK_QUESTION",terminalMessage: "Is that output format okay?",@@ -1568,8 +1583,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {it("legacy payload without adjudication renders via the unchanged inference chain", () => {const turn = hydrateNarrativeFromPayload(reproClarifyPayload())!;const summary = computeTurnSummary(turn);- expect(summary.headline).toBe("Built and tested the workflow");- expect(summary.accent).toBe("ok");+ expect(summary.headline).toBe("Built the workflow");+ expect(summary.accent).toBe("qa");expect(summary.stats).toEqual(["15:02", "3 blocks ran", "3 new"]);});});@@ -1593,6 +1608,7 @@ describe("computeTurnSummary — disposition-first reorder", () => {responseKind: "clarify",draft: draft3,proposalDisposition: "review_tested",+ turnFacts: testedFacts,}),);expect(summary.headline).toBe("Workflow ready for review");@@ -1692,8 +1708,8 @@ describe("hydrateHistoryNarrative — persisted turn_outcome graft", () => {})!;expect(turn.responseKind).toBe("build");const summary = computeTurnSummary(turn);- expect(summary.headline).toBe("Built and tested the workflow");- expect(summary.accent).toBe("ok");+ expect(summary.headline).toBe("Built the workflow");+ expect(summary.accent).toBe("qa");});it("tolerates missing or unknown turn_outcome", () => {diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsxindex dfee08197..f55cebd9b 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx@@ -8,8 +8,11 @@ import {} from "@testing-library/react";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import bundles from "./narrativeState.turnFacts.fixture.json";import type { WorkflowCopilotStreamResponseUpdate } from "./workflowCopilotTypes";+const editOneOfTwoBundle = bundles["different-source-edit-one-of-two"];+type StreamBody = {message: string;keep_pending_proposal?: boolean;@@ -466,6 +469,8 @@ describe("WorkflowCopilotChat — g2 review gate", () => {);expect(screen.getByRole("button", { name: "Reject" })).toBeTruthy();expect(screen.getByText("Proposed changes")).toBeTruthy();+ // The gate itself still appears live (SKY-14099); with no turn behind it there are+ // no facts to project, so it carries no verdict pill either way.expect(screen.queryByText("Tested")).toBeNull();});@@ -497,4 +502,30 @@ describe("WorkflowCopilotChat — g2 review gate", () => {).toBeTruthy();expect(screen.queryByRole("button", { name: "Reject" })).toBeNull();});++ it("surfaces per-block coverage tags through the mounted chat, not only in an isolated card", async () => {+ await renderChat();++ await submit("edit one block after it passed.");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));+ await act(async () => {+ streamCalls[0]!.onMessage(+ proposalResponse("Draft needs review.", {+ narrative_payload: {+ ...editOneOfTwoBundle,+ turnId: "turn-1",+ },+ }),+ );+ streamCalls[0]!.resolve();+ });++ await waitFor(() =>+ expect(screen.getByText("Different source")).toBeTruthy(),+ );+ expect(+ screen.getByText(/3 blocks authored · 0\/3 ran on current source/),+ ).toBeTruthy();+ expect(screen.queryByText("Built and tested the workflow")).toBeNull();+ });});diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.test.tsxindex 618dff394..14b95d548 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.test.tsx@@ -52,25 +52,61 @@ const turn = (...overrides,});+const coveredFacts = {+ factsAvailable: true,+ authoredBlockCount: 1,+ matchingSourceBlockCount: 1,+ evaluationState: null,+ runId: "wr_1",+ runCompleted: true,+ terminalCause: null,+ blocksRunThisTurn: 1,+ ranCleanOnCurrentSource: true,+} as const;+describe("getReviewGateVerdict", () => {- it("treats review_tested as tested", () => {+ it("treats review_tested as tested when the turn's coverage facts back it", () => {expect(getReviewGateVerdict(- turn({ proposalDisposition: "review_tested" }),+ turn({ proposalDisposition: "review_tested", turnFacts: coveredFacts }),null,),).toBe("tested");});- it("treats auto_applicable as tested — the backend only assigns this disposition to verified changes", () => {+ it("treats auto_applicable as tested when the turn's coverage facts back it", () => {expect(getReviewGateVerdict(- turn({ proposalDisposition: "auto_applicable" }),+ turn({+ proposalDisposition: "auto_applicable",+ turnFacts: coveredFacts,+ }),null,),).toBe("tested");});+ it.each(["review_tested", "auto_applicable"] as const)(+ "refuses a tested pill for %s when coverage is partial",+ (proposalDisposition) => {+ expect(+ getReviewGateVerdict(+ turn({+ proposalDisposition,+ // Partial coverage is decided by the backend, which publishes both the+ // count and the verdict it implies.+ turnFacts: {+ ...coveredFacts,+ matchingSourceBlockCount: 0,+ ranCleanOnCurrentSource: false,+ },+ }),+ null,+ ),+ ).toBe("untested");+ },+ );+it("treats review_untested as untested", () => {expect(getReviewGateVerdict(@@ -89,11 +125,17 @@ describe("getReviewGateVerdict", () => {).toBe("untested");});- it("has no verdict for a legacy proposal carrying no evidence either way", () => {+ it("refuses a tested pill for a legacy proposal that carries no coverage facts", () => {const legacyProposal = {} as unknown as WorkflowApiResponse;expect(getReviewGateVerdict(turn({ proposalDisposition: null }), legacyProposal),- ).toBe(null);+ ).toBe("untested");+ expect(+ getReviewGateVerdict(+ turn({ proposalDisposition: null, turnFacts: coveredFacts }),+ legacyProposal,+ ),+ ).toBe("tested");});it("returns null with no disposition and no proposal", () => {@@ -397,3 +439,107 @@ describe("ReviewGateCard — recorded review projection", () => {expect(malformed?.review).toBeNull();});});++describe("ReviewGateCard — source coverage", () => {+ const noop = () => {};++ it("separates a block run against different source from one never run", () => {+ render(+ <ReviewGateCard+ turn={turn({+ review: {+ blocks: [+ {+ label: "sign_in",+ blockType: "code",+ change: "changed",+ neverTested: true,+ coverage: "different_source",+ },+ {+ label: "read_metric",+ blockType: "code",+ change: "added",+ neverTested: true,+ coverage: "never_run",+ },+ ],+ duplicateWrites: [],+ },+ })}+ pending+ verdict="untested"+ actionsEnabled+ onAccept={noop}+ onAlwaysAccept={noop}+ onReject={noop}+ onReview={noop}+ />,+ );++ expect(screen.getByText("Different source")).not.toBeNull();+ expect(screen.getByText("Never tested")).not.toBeNull();+ });++ it("says a renamed block is untested under this name, not that it failed", () => {+ render(+ <ReviewGateCard+ turn={turn({+ review: {+ blocks: [+ {+ label: "sign_in_v2",+ blockType: "code",+ change: "added",+ neverTested: true,+ coverage: "unknown",+ },+ {+ label: "read_metric",+ blockType: "code",+ change: "unchanged",+ neverTested: false,+ coverage: "current_source",+ },+ ],+ duplicateWrites: [],+ },+ })}+ pending+ verdict="untested"+ actionsEnabled+ onAccept={noop}+ onAlwaysAccept={noop}+ onReject={noop}+ onReview={noop}+ />,+ );++ expect(screen.getByText("Not tested under this name")).not.toBeNull();+ expect(screen.queryByText("Never tested")).toBeNull();+ expect(screen.queryByText(/failed/i)).toBeNull();+ expect(screen.queryByText(/invalid/i)).toBeNull();+ });++ it("stays non-committal on a proposal whose turn published no facts", () => {+ expect(+ getReviewGateVerdict(+ turn({+ proposalDisposition: "auto_applicable",+ turnFacts: {+ factsAvailable: false,+ authoredBlockCount: null,+ matchingSourceBlockCount: null,+ evaluationState: null,+ runId: null,+ runCompleted: null,+ terminalCause: null,+ blocksRunThisTurn: null,+ ranCleanOnCurrentSource: false,+ },+ }),+ null,+ ),+ ).toBe("untested");+ });+});diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.tsxindex 595d92223..f8ab3ca32 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.tsx@@ -4,7 +4,7 @@ import { WorkflowApiResponse } from "@/routes/workflows/types/workflowTypes";import { humanizeBlockLabel } from "../blockLabel";import { everyTestBlockExecuted, hasFailedTestBlock } from "../copilotPhases";-import { TurnNarrativeState } from "../narrativeState";+import { TurnNarrativeState, ranCleanOnCurrentSource } from "../narrativeState";import { getDiffCardTitle } from "./diffCardTitle";export type ReviewGateVerdict = "tested" | "untested" | null;@@ -25,17 +25,32 @@ export function getReviewGateVerdict(if (turn && hasFailedTestBlock(turn)) {return "untested";}+ // A tested claim needs the turn's own facts behind it, so an absent bundle, partial+ // coverage or a halted turn reads as untested rather than falling through to green.+ const covered = ranCleanOnCurrentSource(turn?.turnFacts ?? null);if (- turn?.proposalDisposition === "review_tested" ||- turn?.proposalDisposition === "auto_applicable"+ covered &&+ (turn?.proposalDisposition === "review_tested" ||+ turn?.proposalDisposition === "auto_applicable")) {return "tested";}if (turn?.proposalDisposition) {return "untested";}- const legacy = proposedWorkflow as LegacyProposedWorkflow | null;- return legacy?._copilot_unvalidated ? "untested" : null;+ if (!proposedWorkflow) {+ return null;+ }+ const legacy = proposedWorkflow as LegacyProposedWorkflow;+ if (legacy._copilot_unvalidated) {+ return "untested";+ }+ // With no turn there are no facts to project from, so a still-pending gate stays+ // silent rather than inventing either verdict.+ if (!turn) {+ return null;+ }+ return covered ? "tested" : "untested";}const VERDICT_PILL_CLASSES: Record<"tested" | "untested", string> = {@@ -169,7 +184,16 @@ export function ReviewGateCard({{section.prefix ? `${section.prefix} ` : ""}{humanizeBlockLabel(block.label)}</span>- {block.neverTested ? (+ {block.coverage === "different_source" ? (+ <span className="rounded-full bg-sky-500/10 px-1.5 py-0.5 text-[10px] font-medium text-sky-700 dark:text-sky-300">+ Different source+ </span>+ ) : block.coverage === "unknown" ? (+ <span className="rounded-full bg-sky-500/10 px-1.5 py-0.5 text-[10px] font-medium text-sky-700 dark:text-sky-300">+ Not tested under this name+ </span>+ ) : block.coverage === "never_run" ||+ (block.coverage === undefined && block.neverTested) ? (<span className="rounded-full bg-sky-500/10 px-1.5 py-0.5 text-[10px] font-medium text-sky-700 dark:text-sky-300">Never tested</span>diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.tsindex e98a488f4..3508cee6b 100644--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts@@ -314,6 +314,83 @@ describe("derivePhases — terminal", () => {expect(phase(rows, "test").stub).toBe("1 block · failed");});+ it("a deadline halt after a clean run stops the rail instead of failing it", () => {+ const t = turn({+ terminal: "error",+ designEnded: true,+ draft: { blockCount: 1, blockLabels: ["block_1"], summary: null },+ blocks: [block()],+ turnFacts: {+ factsAvailable: true,+ authoredBlockCount: 1,+ matchingSourceBlockCount: 1,+ evaluationState: "not_evaluated",+ runId: "wr_1",+ runCompleted: true,+ terminalCause: "deadline_expired",+ blocksRunThisTurn: 1,+ ranCleanOnCurrentSource: false,+ },+ });+ const rows = derivePhases(t);+ expect(phase(rows, "test").status).toBe("stopped");+ expect(phase(rows, "test").stub).toBe("1 block · time limit");+ expect(phase(rows, "done").status).toBe("stopped");+ expect(rows.map((r) => r.status)).not.toContain("fail");+ });++ it("a deadline halt with no block receipts never paints a green Test row", () => {+ const t = turn({+ terminal: "error",+ designEnded: true,+ draft: { blockCount: 1, blockLabels: ["block_1"], summary: null },+ designActivity: [+ entry({+ id: "1",+ kind: "tool_call",+ toolName: "update_and_run_blocks",+ }),+ ],+ turnFacts: {+ factsAvailable: true,+ authoredBlockCount: 1,+ matchingSourceBlockCount: 0,+ evaluationState: null,+ runId: null,+ runCompleted: null,+ terminalCause: "deadline_expired",+ blocksRunThisTurn: 0,+ ranCleanOnCurrentSource: false,+ },+ });+ const rows = derivePhases(t);+ expect(phase(rows, "test").status).toBe("stopped");+ expect(phase(rows, "test").stub).toBe("time limit");+ });++ it("a deadline halt with a failed block keeps the failure row", () => {+ const t = turn({+ terminal: "error",+ designEnded: true,+ draft: { blockCount: 1, blockLabels: ["block_1"], summary: null },+ blocks: [block({ state: "failed" })],+ turnFacts: {+ factsAvailable: true,+ authoredBlockCount: 1,+ matchingSourceBlockCount: 1,+ evaluationState: "not_demonstrated",+ runId: "wr_1",+ runCompleted: false,+ terminalCause: "deadline_expired",+ blocksRunThisTurn: 1,+ ranCleanOnCurrentSource: false,+ },+ });+ const rows = derivePhases(t);+ expect(phase(rows, "test").status).toBe("fail");+ expect(phase(rows, "test").stub).toBe("1 block · failed");+ });+it("draft-only turn (update_workflow, no run): Test renders notrun, Done still succeeds", () => {const t = turn({terminal: "response",diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.tsindex b5f82b258..88ae36fd9 100644--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts@@ -13,6 +13,7 @@ import {condenseActivityEntries,formatElapsed,hasPendingToolCall,+ isDeadlineHalt,isInterimOutcome,latestBlocksByLabel,parseUtcIsoMs,@@ -235,9 +236,10 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {const isTerminal = turn.terminal !== null;const isCancelled = turn.cancelled === true;- // A cancelled turn also lands on terminal "error" — that error is the stop- // itself, so it must not paint the rail red.- const isError = turn.terminal === "error" && !isCancelled;+ const deadlineHalt = isDeadlineHalt(turn);+ // A cancelled turn and a deadline halt both land on terminal "error" — that+ // error is the stop itself, so neither may paint the rail red.+ const isError = turn.terminal === "error" && !isCancelled && !deadlineHalt;const anyFailed = hasFailedTestBlock(turn);const anyStopped = latestBlocks.some((b) => b.state === "stopped");const anyNotDemonstrated = latestBlocks.some(@@ -272,7 +274,7 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {if (!reached(id)) return id === "test" ? "notrun" : "pending";if (chainActive === id) {if (isError) return "fail";- if (isCancelled) return "stopped";+ if (isCancelled || deadlineHalt) return "stopped";}return "done";}@@ -318,9 +320,10 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {if (status === "fail")return `${pluralize(latestBlocks.length, "block")} · failed`;if (status === "stopped") {+ const halt = deadlineHalt ? "time limit" : "stopped";return latestBlocks.length > 0- ? `${pluralize(latestBlocks.length, "block")} · stopped`- : "stopped";+ ? `${pluralize(latestBlocks.length, "block")} · ${halt}`+ : halt;}if (anyNotDemonstrated) {// Surface how many test runs the loop actually made — otherwise adiff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.tsindex 93eda9c7f..509539a0f 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts@@ -1,12 +1,16 @@import { describe, expect, it } from "vitest";+import bundles from "./narrativeState.turnFacts.fixture.json";+import { getReviewGateVerdict } from "./cards/ReviewGateCard";import {EMPTY_NARRATIVE,TurnNarrativeState,applyNarrativeEvent,+ computeTurnSummary,hydrateNarrativeFromPayload,isBlockOk,notConfirmedOutcome,+ ranCleanOnCurrentSource,} from "./narrativeState";import {WorkflowCopilotBlockProgressUpdate,@@ -485,3 +489,351 @@ describe("notConfirmedOutcome — envelope-first", () => {});});});++// A recorded terminal-surface packet: one authored block, one clean run against+// the staged source.+const CLEAN_RECORDED_TURN = "surface-20260820T082523349904";++describe("canonical turn facts — card, pill and prose over one bundle", () => {+ const surfaces = (leg: keyof typeof bundles) => {+ const turn = hydrateNarrativeFromPayload(+ bundles[leg] as unknown as Record<string, unknown>,+ );+ if (!turn) throw new Error(`fixture ${leg} did not hydrate`);+ return {+ turn,+ card: computeTurnSummary(turn),+ pill: getReviewGateVerdict(turn, null),+ prose: turn.terminalMessage ?? "",+ };+ };++ it("states coverage and evaluation as independent facts when one of three blocks ran", () => {+ const { card, pill, prose } = surfaces("partial-coverage");++ expect(card.headline).not.toBe("Built and tested the workflow");+ expect(card.stats).toContain("3 blocks authored");+ expect(card.stats).toContain("1/3 ran on current source");+ expect(card.stats).toContain("outcome not evaluated");+ expect(card.stats.join(" ")).not.toContain("wr_");+ expect(card.accent).not.toBe("ok");+ expect(card.isFail).toBe(false);+ expect(pill).toBe("untested");+ expect(prose).not.toContain("tested draft");+ });++ it("reports a deadline halt without a failure row or a tested pill", () => {+ const { card, pill, prose } = surfaces("deadline-after-one-clean-run");++ expect(card.isFail).toBe(false);+ expect(card.accent).not.toBe("fail");+ expect(card.stats).toContain("time limit reached");+ expect(pill).toBe("untested");+ expect(prose).toContain("1 block ran this turn");+ expect(prose).toContain("outcome was not evaluated");+ expect(prose).toContain("reached its time limit");+ });++ it("marks an edited block as run against different source, never as failed", () => {+ const { turn, card, pill } = surfaces("different-source-edit-one-of-two");++ expect(+ turn.review?.blocks.map((block) => [block.label, block.coverage]),+ ).toEqual([+ ["extract_github_star_count", "different_source"],+ ["append_star_count_to_sheet", "never_run"],+ ["append_star_count_to_sales_marketing", "never_run"],+ ]);+ expect(card.isFail).toBe(false);+ expect(pill).toBe("untested");+ });++ it("excludes a removed block from the authored count", () => {+ const { card } = surfaces("mixed-edit-and-delete");++ expect(card.stats).toContain("2 blocks authored");+ expect(card.stats).toContain("1/2 ran on current source");+ });++ it("names clean lifecycle plus full current-source coverage on the satisfaction path", () => {+ const { card, pill, prose } = surfaces("satisfaction");++ expect(card.stats).toContain("3 blocks authored");+ expect(card.stats).toContain("3/3 ran on current source");+ expect(card.stats).toContain("run completed");+ expect(card.headline).toBe("Every block ran clean on the current draft");+ expect(card.accent).toBe("ok");+ expect(card.isFail).toBe(false);+ expect(pill).toBe("tested");+ for (const claim of [/verified/i, /demonstrated/i, /goal/i, /confirm/i]) {+ expect(card.headline).not.toMatch(claim);+ }+ expect(prose).not.toMatch(/verified|demonstrated/i);+ });++ it("states its counts without a completion claim when no run is anchored to the turn", () => {+ const { turn, card } = surfaces(CLEAN_RECORDED_TURN);++ expect(turn.blocks.map((block) => block.state)).toEqual(["completed"]);+ expect(turn.turnFacts?.runCompleted).toBeNull();+ expect(card.stats).toContain("1 block authored");+ expect(card.stats).toContain("1/1 ran on current source");+ expect(card.stats).not.toContain("run completed");+ expect(card.stats).not.toContain("run did not complete");+ expect(card.isFail).toBe(false);+ });++ it("withholds the satisfaction headline when the recorded run did not complete", () => {+ const recorded = bundles[CLEAN_RECORDED_TURN] as unknown as {+ turnFacts: Record<string, unknown>;+ };+ const turn = hydrateNarrativeFromPayload({+ ...(bundles[CLEAN_RECORDED_TURN] as unknown as Record<string, unknown>),+ turnFacts: {+ ...recorded.turnFacts,+ runCompleted: false,+ ranCleanOnCurrentSource: false,+ },+ });+ if (!turn) throw new Error("fixture did not hydrate");+ const card = computeTurnSummary(turn);++ expect(card.stats).toContain("run did not complete");+ expect(card.headline).not.toBe(+ "Every block ran clean on the current draft",+ );+ expect(getReviewGateVerdict(turn, null)).toBe("untested");+ expect(card.isFail).toBe(false);+ });++ it("withholds success from a fully covered turn whose recorded run failed", () => {+ const recorded = bundles[CLEAN_RECORDED_TURN] as unknown as {+ blocks: Array<Record<string, unknown>>;+ };+ const turn = hydrateNarrativeFromPayload({+ ...(bundles[CLEAN_RECORDED_TURN] as unknown as Record<string, unknown>),+ blocks: recorded.blocks.map((block) => ({ ...block, state: "failed" })),+ });+ if (!turn) throw new Error("fixture did not hydrate");+ const card = computeTurnSummary(turn);++ expect(card.headline).not.toBe(+ "Every block ran clean on the current draft",+ );+ expect(card.isFail).toBe(true);+ expect(card.accent).toBe("fail");+ });++ it("says a renamed block is untested under this name across card and pill", () => {+ const { turn, card, pill } = surfaces("renamed-block");++ expect(+ turn.review?.blocks.map((block) => [block.label, block.coverage]),+ ).toEqual([+ ["extract_github_star_count_v2", "unknown"],+ ["append_star_count_to_sheet", "current_source"],+ ["append_star_count_to_sales_marketing", "current_source"],+ ["extract_github_star_count", undefined],+ ]);+ expect(card.stats).toContain("2/3 ran on current source");+ expect(card.isFail).toBe(false);+ expect(pill).toBe("untested");+ });++ it("settles a block still running at the deadline as stopped, not failed", () => {+ const turn = hydrateNarrativeFromPayload({+ ...(bundles["deadline-after-one-clean-run"] as unknown as Record<+ string,+ unknown+ >),+ blocks: [+ {+ workflowRunBlockId: "wrb_1",+ label: "append_star_count_to_sheet",+ blockType: "code",+ state: "running",+ lastSeenIteration: 0,+ activity: [],+ },+ ],+ });+ if (!turn) throw new Error("fixture did not hydrate");+ const card = computeTurnSummary(turn);++ expect(turn.blocks.map((block) => block.state)).toEqual(["stopped"]);+ expect(card.isFail).toBe(false);+ expect(card.accent).not.toBe("fail");+ });++ it("does not call a deadline halt that delivered nothing a completed run", () => {+ const turn = hydrateNarrativeFromPayload({+ ...(bundles["deadline-after-one-clean-run"] as unknown as Record<+ string,+ unknown+ >),+ proposalDisposition: "no_proposal",+ draft: null,+ terminal: "error",+ blocks: [+ {+ workflowRunBlockId: "wrb_1",+ label: "append_star_count_to_sheet",+ blockType: "code",+ state: "completed",+ lastSeenIteration: 0,+ activity: [],+ },+ ],+ });+ if (!turn) throw new Error("fixture did not hydrate");+ const card = computeTurnSummary(turn);++ // Suppressing the failure treatment for a deadline must not promote the turn+ // into a success: it ran out of time and handed nothing back.+ expect(card.isFail).toBe(false);+ expect(card.accent).not.toBe("ok");+ expect(card.headline).not.toMatch(/completed the run/i);+ });++ it("keeps a failed block a failure even when the deadline expired", () => {+ const turn = hydrateNarrativeFromPayload({+ ...(bundles["deadline-after-one-clean-run"] as unknown as Record<+ string,+ unknown+ >),+ blocks: [+ {+ workflowRunBlockId: "wrb_1",+ label: "append_star_count_to_sheet",+ blockType: "code",+ state: "failed",+ lastSeenIteration: 0,+ activity: [],+ },+ ],+ });+ if (!turn) throw new Error("fixture did not hydrate");+ const card = computeTurnSummary(turn);++ expect(turn.turnFacts?.terminalCause).toBe("deadline_expired");+ expect(card.isFail).toBe(true);+ expect(card.stats).toContain("1 failed");+ });++ it("makes no tested claim for a turn that published no facts or coverage", () => {+ const legacy = bundles[CLEAN_RECORDED_TURN] as unknown as Record<+ string,+ unknown+ >;+ const review = legacy.review as { blocks: Array<Record<string, unknown>> };+ const turn = hydrateNarrativeFromPayload({+ ...legacy,+ turnFacts: undefined,+ review: {+ ...review,+ blocks: review.blocks.map((block) => {+ const withoutCoverage = { ...block };+ delete withoutCoverage.coverage;+ return withoutCoverage;+ }),+ },+ });+ if (!turn) throw new Error("fixture did not hydrate");+ const card = computeTurnSummary(turn);++ expect(turn.turnFacts).toBeNull();+ expect(turn.review?.blocks[0]?.coverage).toBeUndefined();+ expect(card.headline).not.toMatch(/tested/i);+ expect(card.accent).not.toBe("ok");+ expect(card.stats.join(" ")).not.toMatch(/ran on current source|authored/);+ });++ it("keeps the review when the backend sends a coverage value it does not know", () => {+ const known = bundles[CLEAN_RECORDED_TURN] as unknown as Record<+ string,+ unknown+ >;+ const review = known.review as { blocks: Array<Record<string, unknown>> };+ const turn = hydrateNarrativeFromPayload({+ ...known,+ review: {+ ...review,+ blocks: review.blocks.map((block) => ({+ ...block,+ coverage: "some_future_value",+ })),+ },+ });+ if (!turn) throw new Error("fixture did not hydrate");++ expect(turn.review?.blocks).toHaveLength(1);+ expect(turn.review?.blocks[0]?.coverage).toBeUndefined();+ });++ it("refuses a tested pill for a pre-fix payload that carries no fact bundle", () => {+ const turn = hydrateNarrativeFromPayload(+ bundles["facts-absent"] as unknown as Record<string, unknown>,+ );+ if (!turn) throw new Error("fixture did not hydrate");+ const card = computeTurnSummary(turn);++ expect(turn.turnFacts).toBeNull();+ expect(turn.proposalDisposition).toBe("review_tested");+ expect(getReviewGateVerdict(turn, null)).toBe("untested");+ // The tested branch is named "Workflow ready for review", which contains no literal+ // "tested" — assert the branch, not the substring.+ expect(card.headline).not.toBe("Workflow ready for review");+ expect(card.headline).toBe("Built the workflow");+ expect(card.glyph).not.toBe("\u2713");+ expect(card.accent).not.toBe("ok");+ });++ it("keeps card and pill telling one story when the envelope is missing", () => {+ const full = bundles["full-coverage"] as unknown as Record<string, unknown>;+ const facts = full.turnFacts as Record<string, unknown>;+ const turn = hydrateNarrativeFromPayload({+ ...full,+ turnFacts: {+ ...facts,+ evaluationState: null,+ runId: null,+ runCompleted: null,+ terminalCause: null,+ blocksRunThisTurn: null,+ },+ });+ if (!turn) throw new Error("fixture did not hydrate");+ const card = computeTurnSummary(turn);++ expect(turn.turnFacts?.factsAvailable).toBe(true);+ expect(turn.turnFacts?.authoredBlockCount).toBe(facts.authoredBlockCount);+ expect(card.stats).toContain(+ `${facts.matchingSourceBlockCount}/${facts.authoredBlockCount} ran on current source`,+ );+ expect(card.stats).not.toContain("run completed");+ expect(card.isFail).toBe(false);+ expect(ranCleanOnCurrentSource(turn.turnFacts)).toBe(true);+ expect(getReviewGateVerdict(turn, null)).toBe("tested");+ });++ it("stays non-committal when the turn published no facts", () => {+ const turn = hydrateNarrativeFromPayload({+ ...(bundles["full-coverage"] as unknown as Record<string, unknown>),+ review: undefined,+ proposalDisposition: "auto_applicable",+ turnFacts: {+ factsAvailable: false,+ evaluationState: null,+ runId: null,+ terminalCause: null,+ blocksRunThisTurn: null,+ },+ });+ if (!turn) throw new Error("fixture did not hydrate");++ expect(getReviewGateVerdict(turn, null)).toBe("untested");+ expect(computeTurnSummary(turn).headline).not.toBe(+ "Every block ran clean on the current draft",+ );+ });+});diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.tsindex 20c29ba47..231a1ac6f 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts@@ -156,6 +156,37 @@ export interface TerminalEnvelopeFacts {}export type ReviewChange = "added" | "changed" | "unchanged" | "removed";+export type BlockCoverage =+ | "current_source"+ | "different_source"+ | "never_run"+ | "unknown";++const BLOCK_COVERAGE: readonly BlockCoverage[] = [+ "current_source",+ "different_source",+ "never_run",+ "unknown",+];++// The canonical per-turn facts the backend publishes; card, pill and prose each+// render a projection of this bundle rather than deriving their own verdict.+export interface TurnFacts {+ factsAvailable: boolean;+ authoredBlockCount: number | null;+ matchingSourceBlockCount: number | null;+ evaluationState: BlockOutcome | null;+ runId: string | null;+ runCompleted: boolean | null;+ terminalCause: string | null;+ blocksRunThisTurn: number | null;+ ranCleanOnCurrentSource: boolean;+}++// The backend decides the tested claim and publishes it here; no surface recomputes it.+export function ranCleanOnCurrentSource(facts: TurnFacts | null): boolean {+ return facts?.ranCleanOnCurrentSource === true;+}export interface ReviewProjection {blocks: Array<{@@ -163,6 +194,7 @@ export interface ReviewProjection {blockType: string;change: ReviewChange;neverTested?: boolean;+ coverage?: BlockCoverage;}>;duplicateWrites: Array<{blockType: string;@@ -335,6 +367,7 @@ export interface TurnNarrativeState {connectedAccountChoices: ConnectedAccountChoice[];googleConnectionNotices: GoogleConnectionNotice[];review: ReviewProjection | null;+ turnFacts: TurnFacts | null;}export interface GoogleConnectionNotice {@@ -373,6 +406,7 @@ export const EMPTY_NARRATIVE: TurnNarrativeState = Object.freeze({connectedAccountChoices: [],googleConnectionNotices: [],review: null,+ turnFacts: null,}) as TurnNarrativeState;// Caps to keep long-running narrations from unbounded growth (and to keep@@ -1404,6 +1438,34 @@ function normalizeActivityEntries(raw: unknown): ActivityEntry[] {return out;}+function parseTurnFacts(raw: unknown): TurnFacts | null {+ if (!raw || typeof raw !== "object") return null;+ const value = raw as Record<string, unknown>;+ if (typeof value.factsAvailable !== "boolean") return null;+ const count = (key: string): number | null =>+ typeof value[key] === "number" ? (value[key] as number) : null;+ const text = (key: string): string | null =>+ typeof value[key] === "string" ? (value[key] as string) : null;+ const evaluationState = value.evaluationState;+ return {+ factsAvailable: value.factsAvailable,+ authoredBlockCount: count("authoredBlockCount"),+ matchingSourceBlockCount: count("matchingSourceBlockCount"),+ evaluationState:+ evaluationState === "demonstrated" ||+ evaluationState === "not_demonstrated" ||+ evaluationState === "not_evaluated"+ ? evaluationState+ : null,+ runId: text("runId"),+ runCompleted:+ typeof value.runCompleted === "boolean" ? value.runCompleted : null,+ terminalCause: text("terminalCause"),+ blocksRunThisTurn: count("blocksRunThisTurn"),+ ranCleanOnCurrentSource: value.ranCleanOnCurrentSource === true,+ };+}+function parseReviewProjection(raw: unknown): ReviewProjection | null {if (!raw || typeof raw !== "object") return null;const value = raw as Record<string, unknown>;@@ -1426,11 +1488,14 @@ function parseReviewProjection(raw: unknown): ReviewProjection | null {) {return null;}+ // An unrecognised coverage value drops off this block rather than voiding the+ // whole projection, so a newer backend vocabulary cannot blank the review.blocks.push({label: block.label,blockType: block.blockType,change: block.change,neverTested: block.neverTested,+ coverage: BLOCK_COVERAGE.find((known) => known === block.coverage),});}const duplicateWrites: ReviewProjection["duplicateWrites"] = [];@@ -1567,12 +1632,16 @@ export function hydrateNarrativeFromPayload(const endedAtIso =typeof payload.endedAt === "string" ? (payload.endedAt as string) : null;const cancelled = payload.cancelled === true;+ const turnFacts = parseTurnFacts(payload.turnFacts);+ // A deadline cuts a running block off; that is a stop, so it must not be swept+ // to failed and then read back as the turn having failed.+ const halted = cancelled || turnFacts?.terminalCause === "deadline_expired";const sweptBlocks: BlockState[] = terminal? blocks.map((b) =>b.state === "running"? {...b,- state: cancelled+ state: halted? "stopped": terminal === "error"? "failed"@@ -1638,6 +1707,7 @@ export function hydrateNarrativeFromPayload(payload.googleConnectionNotices,),review: parseReviewProjection(payload.review),+ turnFacts,};}@@ -1809,7 +1879,6 @@ function adjudicatedSummaryParts(needsTestedProposalReview: boolean;hasEdited: boolean;hasDrafts: boolean;- hasCleanCompletedBuild: boolean;},): AdjudicatedParts | null {if (turn.responseKind === null) return null;@@ -1856,21 +1925,35 @@ function adjudicatedSummaryParts(return null;}+// A deadline exit stamps terminal="error" for budget reasons; that is a halt,+// not a block failure, so no surface may give it failure's treatment.+export function isDeadlineHalt(turn: TurnNarrativeState): boolean {+ return (+ turn.turnFacts?.terminalCause === "deadline_expired" &&+ !latestBlocksByLabel(turn.blocks).some((b) => b.state === "failed")+ );+}+export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {const rollupBlocks = latestBlocksByLabel(turn.blocks);// A cancelled turn's terminal is "error" purely because the user stopped it,// so that arm must not brand their own stop a failure.+ const anyBlockFailed = rollupBlocks.some((b) => b.state === "failed");+ const facts = turn.turnFacts;+ const deadlineHalt = isDeadlineHalt(turn);const isFail =!turn.cancelled &&- (turn.terminal === "error" ||- rollupBlocks.some((b) => b.state === "failed"));+ !deadlineHalt &&+ (turn.terminal === "error" || anyBlockFailed);// A stop halts the turn without failing it — a user cancel, or a budget halt// that cancels a block mid-run. It suppresses a success verdict exactly like// a failure, but never wears failure's treatment. `turn.cancelled` is load// bearing on its own: a stop during the thinking phase, or on a QA turn,// touches no block at all and would otherwise read as a clean success.const isStopped =- turn.cancelled || rollupBlocks.some((b) => b.state === "stopped");+ turn.cancelled ||+ deadlineHalt ||+ rollupBlocks.some((b) => b.state === "stopped");const mode = effectiveMode(turn);const needsInput = asksUserForInput(turn);const isQA =@@ -1879,15 +1962,26 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {mode === "clarify" ||mode === "refuse";const hasDrafts = (turn.draft?.blockCount ?? 0) > 0;+ const cleanFullCurrentSourceCoverage = ranCleanOnCurrentSource(facts);const needsUntestedProposalReview =hasDrafts && turn.proposalDisposition === "review_untested";+ // The tested framing is a claim like any other, so it reads the published verdict+ // rather than the disposition alone.const needsTestedProposalReview =- hasDrafts && turn.proposalDisposition === "review_tested";- const hasEdited = (turn.priorBlockCount ?? 0) > 0 && hasDrafts;- const hasCleanCompletedBuild =hasDrafts &&- rollupBlocks.length > 0 &&- rollupBlocks.every((block) => isBlockOk(block));+ turn.proposalDisposition === "review_tested" &&+ cleanFullCurrentSourceCoverage;+ const hasEdited = (turn.priorBlockCount ?? 0) > 0 && hasDrafts;+ const authoredBlockCount = facts?.factsAvailable+ ? facts.authoredBlockCount+ : null;+ const matchingSourceBlockCount = facts?.factsAvailable+ ? facts.matchingSourceBlockCount+ : null;+ const coverageKnown =+ authoredBlockCount !== null &&+ authoredBlockCount > 0 &&+ matchingSourceBlockCount !== null;const hasReviewableDraft =hasDrafts &&(turn.proposalDisposition === "review_untested" ||@@ -1904,7 +1998,6 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {needsTestedProposalReview,hasEdited,hasDrafts,- hasCleanCompletedBuild,});const headline = adjudicated@@ -1927,11 +2020,13 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {: mode === "clarify"? "Needs your input": "Answered"- : hasEdited- ? "Applied edits and re-tested"- : hasDrafts- ? "Built and tested the workflow"- : "Completed the run";+ : cleanFullCurrentSourceCoverage+ ? "Every block ran clean on the current draft"+ : hasEdited+ ? "Applied edits"+ : hasDrafts+ ? "Built the workflow"+ : "Completed the run";const stats: string[] = [];const turnElapsed = formatElapsed(turn.startedAt, turn.endedAt);@@ -1941,8 +2036,30 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {const failed = rollupBlocks.filter((b) => b.state === "failed").length;const stopped = rollupBlocks.filter((b) => b.state === "stopped").length;const newBlocks = hasEdited ? 0 : (turn.draft?.blockCount ?? 0);- if (ok) stats.push(`${ok} block${ok === 1 ? "" : "s"} ran`);- if (newBlocks) stats.push(`${newBlocks} new`);+ if (coverageKnown) {+ stats.push(+ `${authoredBlockCount} block${authoredBlockCount === 1 ? "" : "s"} authored`,+ );+ stats.push(+ `${matchingSourceBlockCount}/${authoredBlockCount} ran on current source`,+ );+ if (facts?.runCompleted === true) {+ stats.push("run completed");+ } else if (facts?.runCompleted === false) {+ stats.push("run did not complete");+ }+ if (facts?.evaluationState === "not_evaluated") {+ stats.push("outcome not evaluated");+ } else if (facts?.evaluationState === "not_demonstrated") {+ stats.push("outcome not confirmed");+ }+ if (facts?.terminalCause === "deadline_expired") {+ stats.push("time limit reached");+ }+ } else {+ if (ok) stats.push(`${ok} block${ok === 1 ? "" : "s"} ran`);+ if (newBlocks) stats.push(`${newBlocks} new`);+ }if (failed) stats.push(`${failed} failed`);if (stopped) stats.push(`${stopped} stopped`);}@@ -1956,7 +2073,8 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {: isStopped ||needsUntestedProposalReview ||needsTestedProposalReview ||- isQA+ isQA ||+ (hasDrafts && !cleanFullCurrentSourceCoverage)? "qa": "ok";return {@@ -1967,7 +2085,10 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {? adjudicated.glyph: isStoppedWithDraft ||needsUntestedProposalReview ||- needsTestedProposalReview+ needsTestedProposalReview ||+ // A draft still awaiting review never wears a success tick, whether or not+ // facts back its tested framing.+ (hasDrafts && turn.proposalDisposition === "review_tested")? "!": isFail? "✕"diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.turnFacts.fixture.json b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.turnFacts.fixture.jsonnew file mode 100644index 000000000..de7c18506--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.turnFacts.fixture.json@@ -0,0 +1,807 @@+{+ "deadline-after-one-clean-run": {+ "draft": {+ "blockCount": 3,+ "blockLabels": [+ "extract_github_star_count",+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "summary": null+ },+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard. 1 block ran this turn. The recorded run completed, and its outcome was not evaluated. The turn reached its time limit.",+ "proposalDisposition": "review_untested",+ "review": {+ "blocks": [+ {+ "blockType": "code",+ "change": "changed",+ "coverage": "never_run",+ "label": "extract_github_star_count",+ "neverTested": true+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "append_star_count_to_sheet",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "added",+ "coverage": "never_run",+ "label": "append_star_count_to_sales_marketing",+ "neverTested": true+ }+ ],+ "duplicateWrites": [+ {+ "blockLabels": [+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "blockType": "google_sheets_write"+ }+ ]+ },+ "terminal": "error",+ "terminalEnvelope": {+ "attempted": null,+ "blocker_reason": null,+ "blocks_run_this_turn": 1,+ "envelope_version": 1,+ "halt_kind": null,+ "interruption": null,+ "next_state": "proposal_pending",+ "rendered_from_envelope": false,+ "response_kind": "update",+ "run_completed": true,+ "run_display_reason": null,+ "run_id": "wr_recorded",+ "run_output_report": null,+ "run_verdict": "not_evaluated",+ "terminal_cause": "deadline_expired",+ "user_action_required": false,+ "verified": false,+ "workflow_applied": false+ },+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard. 1 block ran this turn. The recorded run completed, and its outcome was not evaluated. The turn reached its time limit.",+ "turnFacts": {+ "authoredBlockCount": 3,+ "blocksRunThisTurn": 1,+ "evaluationState": "not_evaluated",+ "factsAvailable": true,+ "matchingSourceBlockCount": 1,+ "ranCleanOnCurrentSource": false,+ "runCompleted": true,+ "runId": "wr_recorded",+ "terminalCause": "deadline_expired"+ },+ "turnId": "turn_deadline-after-one-clean-run",+ "turnIndex": 0+ },+ "different-source-edit-one-of-two": {+ "draft": {+ "blockCount": 3,+ "blockLabels": [+ "extract_github_star_count",+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "summary": null+ },+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",+ "proposalDisposition": "review_untested",+ "review": {+ "blocks": [+ {+ "blockType": "code",+ "change": "changed",+ "coverage": "different_source",+ "label": "extract_github_star_count",+ "neverTested": true+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "never_run",+ "label": "append_star_count_to_sheet",+ "neverTested": true+ },+ {+ "blockType": "google_sheets_write",+ "change": "added",+ "coverage": "never_run",+ "label": "append_star_count_to_sales_marketing",+ "neverTested": true+ }+ ],+ "duplicateWrites": [+ {+ "blockLabels": [+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "blockType": "google_sheets_write"+ }+ ]+ },+ "terminal": "response",+ "terminalEnvelope": {+ "attempted": null,+ "blocker_reason": null,+ "blocks_run_this_turn": 1,+ "envelope_version": 1,+ "halt_kind": null,+ "interruption": null,+ "next_state": "proposal_pending",+ "rendered_from_envelope": false,+ "response_kind": "update",+ "run_completed": true,+ "run_display_reason": null,+ "run_id": "wr_recorded",+ "run_output_report": null,+ "run_verdict": "not_evaluated",+ "terminal_cause": null,+ "user_action_required": false,+ "verified": false,+ "workflow_applied": false+ },+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",+ "turnFacts": {+ "authoredBlockCount": 3,+ "blocksRunThisTurn": 1,+ "evaluationState": "not_evaluated",+ "factsAvailable": true,+ "matchingSourceBlockCount": 0,+ "ranCleanOnCurrentSource": false,+ "runCompleted": true,+ "runId": "wr_recorded",+ "terminalCause": null+ },+ "turnId": "turn_different-source-edit-one-of-two",+ "turnIndex": 0+ },+ "facts-absent": {+ "draft": {+ "blockCount": 3,+ "blockLabels": [+ "extract_github_star_count",+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "summary": null+ },+ "narrativeSummary": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",+ "proposalDisposition": "review_tested",+ "review": {+ "blocks": [+ {+ "blockType": "code",+ "change": "unchanged",+ "label": "extract_github_star_count",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "label": "append_star_count_to_sheet",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "label": "append_star_count_to_sales_marketing",+ "neverTested": false+ }+ ],+ "duplicateWrites": [+ {+ "blockLabels": [+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "blockType": "google_sheets_write"+ }+ ]+ },+ "terminal": "response",+ "terminalEnvelope": {+ "attempted": null,+ "blocker_reason": null,+ "blocks_run_this_turn": 3,+ "envelope_version": 1,+ "halt_kind": null,+ "interruption": null,+ "next_state": "proposal_pending",+ "rendered_from_envelope": false,+ "response_kind": "update",+ "run_completed": true,+ "run_display_reason": null,+ "run_id": "wr_recorded",+ "run_output_report": null,+ "run_verdict": "not_evaluated",+ "terminal_cause": null,+ "user_action_required": false,+ "verified": false,+ "workflow_applied": false+ },+ "terminalMessage": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",+ "turnFacts": null,+ "turnId": "turn_facts-absent",+ "turnIndex": 0+ },+ "full-coverage": {+ "draft": {+ "blockCount": 3,+ "blockLabels": [+ "extract_github_star_count",+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "summary": null+ },+ "narrativeSummary": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",+ "proposalDisposition": "review_tested",+ "review": {+ "blocks": [+ {+ "blockType": "code",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "extract_github_star_count",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "append_star_count_to_sheet",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "append_star_count_to_sales_marketing",+ "neverTested": false+ }+ ],+ "duplicateWrites": [+ {+ "blockLabels": [+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "blockType": "google_sheets_write"+ }+ ]+ },+ "terminal": "response",+ "terminalEnvelope": {+ "attempted": null,+ "blocker_reason": null,+ "blocks_run_this_turn": 3,+ "envelope_version": 1,+ "halt_kind": null,+ "interruption": null,+ "next_state": "proposal_pending",+ "rendered_from_envelope": false,+ "response_kind": "update",+ "run_completed": true,+ "run_display_reason": null,+ "run_id": "wr_recorded",+ "run_output_report": null,+ "run_verdict": "not_evaluated",+ "terminal_cause": null,+ "user_action_required": false,+ "verified": false,+ "workflow_applied": false+ },+ "terminalMessage": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",+ "turnFacts": {+ "authoredBlockCount": 3,+ "blocksRunThisTurn": 3,+ "evaluationState": "not_evaluated",+ "factsAvailable": true,+ "matchingSourceBlockCount": 3,+ "ranCleanOnCurrentSource": true,+ "runCompleted": true,+ "runId": "wr_recorded",+ "terminalCause": null+ },+ "turnId": "turn_full-coverage",+ "turnIndex": 0+ },+ "mixed-edit-and-delete": {+ "draft": {+ "blockCount": 2,+ "blockLabels": [+ "extract_github_star_count",+ "append_star_count_to_sales_marketing"+ ],+ "summary": null+ },+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",+ "proposalDisposition": "review_untested",+ "review": {+ "blocks": [+ {+ "blockType": "code",+ "change": "changed",+ "coverage": "different_source",+ "label": "extract_github_star_count",+ "neverTested": true+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "append_star_count_to_sales_marketing",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "removed",+ "label": "append_star_count_to_sheet"+ }+ ],+ "duplicateWrites": []+ },+ "terminal": "response",+ "terminalEnvelope": {+ "attempted": null,+ "blocker_reason": null,+ "blocks_run_this_turn": 3,+ "envelope_version": 1,+ "halt_kind": null,+ "interruption": null,+ "next_state": "proposal_pending",+ "rendered_from_envelope": false,+ "response_kind": "update",+ "run_completed": true,+ "run_display_reason": null,+ "run_id": "wr_recorded",+ "run_output_report": null,+ "run_verdict": "not_evaluated",+ "terminal_cause": null,+ "user_action_required": false,+ "verified": false,+ "workflow_applied": false+ },+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",+ "turnFacts": {+ "authoredBlockCount": 2,+ "blocksRunThisTurn": 3,+ "evaluationState": "not_evaluated",+ "factsAvailable": true,+ "matchingSourceBlockCount": 1,+ "ranCleanOnCurrentSource": false,+ "runCompleted": true,+ "runId": "wr_recorded",+ "terminalCause": null+ },+ "turnId": "turn_mixed-edit-and-delete",+ "turnIndex": 0+ },+ "partial-coverage": {+ "draft": {+ "blockCount": 3,+ "blockLabels": [+ "extract_github_star_count",+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "summary": null+ },+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",+ "proposalDisposition": "review_untested",+ "review": {+ "blocks": [+ {+ "blockType": "code",+ "change": "changed",+ "coverage": "never_run",+ "label": "extract_github_star_count",+ "neverTested": true+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "append_star_count_to_sheet",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "added",+ "coverage": "never_run",+ "label": "append_star_count_to_sales_marketing",+ "neverTested": true+ }+ ],+ "duplicateWrites": [+ {+ "blockLabels": [+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "blockType": "google_sheets_write"+ }+ ]+ },+ "terminal": "response",+ "terminalEnvelope": {+ "attempted": null,+ "blocker_reason": null,+ "blocks_run_this_turn": 1,+ "envelope_version": 1,+ "halt_kind": null,+ "interruption": null,+ "next_state": "proposal_pending",+ "rendered_from_envelope": false,+ "response_kind": "update",+ "run_completed": true,+ "run_display_reason": null,+ "run_id": "wr_recorded",+ "run_output_report": null,+ "run_verdict": "not_evaluated",+ "terminal_cause": null,+ "user_action_required": false,+ "verified": false,+ "workflow_applied": false+ },+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",+ "turnFacts": {+ "authoredBlockCount": 3,+ "blocksRunThisTurn": 1,+ "evaluationState": "not_evaluated",+ "factsAvailable": true,+ "matchingSourceBlockCount": 1,+ "ranCleanOnCurrentSource": false,+ "runCompleted": true,+ "runId": "wr_recorded",+ "terminalCause": null+ },+ "turnId": "turn_partial-coverage",+ "turnIndex": 0+ },+ "renamed-block": {+ "draft": {+ "blockCount": 3,+ "blockLabels": [+ "extract_github_star_count_v2",+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "summary": null+ },+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",+ "proposalDisposition": "review_untested",+ "review": {+ "blocks": [+ {+ "blockType": "code",+ "change": "added",+ "coverage": "unknown",+ "label": "extract_github_star_count_v2",+ "neverTested": true+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "append_star_count_to_sheet",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "append_star_count_to_sales_marketing",+ "neverTested": false+ },+ {+ "blockType": "code",+ "change": "removed",+ "label": "extract_github_star_count"+ }+ ],+ "duplicateWrites": [+ {+ "blockLabels": [+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "blockType": "google_sheets_write"+ }+ ]+ },+ "terminal": "response",+ "terminalEnvelope": {+ "attempted": null,+ "blocker_reason": null,+ "blocks_run_this_turn": 3,+ "envelope_version": 1,+ "halt_kind": null,+ "interruption": null,+ "next_state": "proposal_pending",+ "rendered_from_envelope": false,+ "response_kind": "update",+ "run_completed": true,+ "run_display_reason": null,+ "run_id": "wr_recorded",+ "run_output_report": null,+ "run_verdict": "not_evaluated",+ "terminal_cause": null,+ "user_action_required": false,+ "verified": false,+ "workflow_applied": false+ },+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",+ "turnFacts": {+ "authoredBlockCount": 3,+ "blocksRunThisTurn": 3,+ "evaluationState": "not_evaluated",+ "factsAvailable": true,+ "matchingSourceBlockCount": 2,+ "ranCleanOnCurrentSource": false,+ "runCompleted": true,+ "runId": "wr_recorded",+ "terminalCause": null+ },+ "turnId": "turn_renamed-block",+ "turnIndex": 0+ },+ "satisfaction": {+ "draft": {+ "blockCount": 3,+ "blockLabels": [+ "extract_github_star_count",+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "summary": null+ },+ "narrativeSummary": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",+ "proposalDisposition": "auto_applicable",+ "review": {+ "blocks": [+ {+ "blockType": "code",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "extract_github_star_count",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "append_star_count_to_sheet",+ "neverTested": false+ },+ {+ "blockType": "google_sheets_write",+ "change": "unchanged",+ "coverage": "current_source",+ "label": "append_star_count_to_sales_marketing",+ "neverTested": false+ }+ ],+ "duplicateWrites": [+ {+ "blockLabels": [+ "append_star_count_to_sheet",+ "append_star_count_to_sales_marketing"+ ],+ "blockType": "google_sheets_write"+ }+ ]+ },+ "terminal": "response",+ "terminalEnvelope": {+ "attempted": null,+ "blocker_reason": null,+ "blocks_run_this_turn": 3,+ "envelope_version": 1,+ "halt_kind": null,+ "interruption": null,+ "next_state": "proposal_pending",+ "rendered_from_envelope": false,+ "response_kind": "update",+ "run_completed": true,+ "run_display_reason": null,+ "run_id": "wr_recorded",+ "run_output_report": null,+ "run_verdict": "not_evaluated",+ "terminal_cause": null,+ "user_action_required": false,+ "verified": false,+ "workflow_applied": false+ },+ "terminalMessage": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",+ "turnFacts": {+ "authoredBlockCount": 3,+ "blocksRunThisTurn": 3,+ "evaluationState": "not_evaluated",+ "factsAvailable": true,+ "matchingSourceBlockCount": 3,+ "ranCleanOnCurrentSource": true,+ "runCompleted": true,+ "runId": "wr_recorded",+ "terminalCause": null+ },+ "turnId": "turn_satisfaction",+ "turnIndex": 0+ },+ "surface-20260820T082523349904": {+ "blocks": [+ {+ "activity": [],+ "blockType": "code",+ "endedAt": "2026-08-20T08:25:06.357598+00:00",+ "label": "read_heading",+ "lastSeenIteration": 0,+ "outcome": "not_evaluated",+ "outcomeRole": "recorded",+ "startedAt": null,+ "state": "completed"+ }+ ],+ "designActivity": [+ {+ "displayLabel": "Inspecting the page",+ "id": "tc-call_XNWFbCYGnBdzH5FhZBxmnDjG",+ "iteration": 0,+ "kind": "tool_call",+ "text": "Inspecting the page…",+ "toolName": "inspect_page_for_composition"+ },+ {+ "displayLabel": "Inspecting the page",+ "id": "tr-call_XNWFbCYGnBdzH5FhZBxmnDjG",+ "iteration": 0,+ "kind": "tool_result",+ "success": true,+ "text": "Inspected the page",+ "toolName": "inspect_page_for_composition"+ },+ {+ "displayLabel": "Checking workflow block options",+ "id": "tc-call_rGCSiY2vPKAql7diyBAosK4Y",+ "iteration": 1,+ "kind": "tool_call",+ "text": "Checking workflow block options…",+ "toolName": "get_block_schema"+ },+ {+ "displayLabel": "Checking workflow block options",+ "id": "tr-call_rGCSiY2vPKAql7diyBAosK4Y",+ "iteration": 1,+ "kind": "tool_result",+ "success": true,+ "text": "Schema for code",+ "toolName": "get_block_schema"+ },+ {+ "displayLabel": "Testing workflow",+ "id": "tc-call_RTKazhVldm7xi5e3Z3YRuFXk",+ "iteration": 2,+ "kind": "tool_call",+ "text": "Testing workflow…",+ "toolName": "update_and_run_blocks"+ },+ {+ "displayLabel": "Testing workflow",+ "id": "tr-call_RTKazhVldm7xi5e3Z3YRuFXk",+ "iteration": 2,+ "kind": "tool_result",+ "success": true,+ "text": "Updated the workflow and ran it: completed",+ "toolName": "update_and_run_blocks"+ },+ {+ "displayLabel": "Editing block \"Read Heading\"",+ "id": "tc-call_1HHXFiV5MeiZOEZe4XfOOV3K",+ "iteration": 3,+ "kind": "tool_call",+ "text": "Editing block \"Read Heading\"…",+ "toolName": "edit_block"+ },+ {+ "displayLabel": "Editing block \"Read Heading\"",+ "id": "tr-call_1HHXFiV5MeiZOEZe4XfOOV3K",+ "iteration": 3,+ "kind": "tool_result",+ "success": true,+ "text": "Editing block \"Read Heading\"",+ "toolName": "edit_block"+ },+ {+ "displayLabel": "Testing workflow",+ "id": "tc-call_bQZ2EQkM6gukPffLSwTuVQPk",+ "iteration": 4,+ "kind": "tool_call",+ "text": "Testing workflow…",+ "toolName": "run_blocks_and_collect_debug"+ },+ {+ "displayLabel": "Testing workflow",+ "id": "tr-call_bQZ2EQkM6gukPffLSwTuVQPk",+ "iteration": 4,+ "kind": "tool_result",+ "success": true,+ "text": "Run read_heading: completed",+ "toolName": "run_blocks_and_collect_debug"+ }+ ],+ "designEnded": true,+ "designStarted": true,+ "draft": {+ "blockCount": 1,+ "blockLabels": ["read_heading"],+ "summary": null+ },+ "endedAt": "2026-08-20T08:25:23.337402+00:00",+ "narrativeSummary": "I created and tested the single read_heading block twice. It now returns both the top heading and page title; the latest run completed with non-empty output.",+ "priorBlockCount": 0,+ "proposalDisposition": "auto_applicable",+ "responseKind": "build",+ "responseType": "REPLY",+ "review": {+ "blocks": [+ {+ "blockType": "code",+ "change": "added",+ "coverage": "current_source",+ "label": "read_heading",+ "neverTested": false+ }+ ],+ "duplicateWrites": []+ },+ "startedAt": "2026-08-20T08:23:33.076908+00:00",+ "terminal": "response",+ "terminalEnvelope": {+ "attempted": "I created and tested the single read_heading block twice. It now returns both the top heading and page title; the latest run completed with non-empty output.",+ "blocker_reason": null,+ "blocks_run_this_turn": 1,+ "envelope_version": 1,+ "halt_kind": null,+ "interruption": null,+ "next_state": "stopped",+ "rendered_from_envelope": false,+ "response_kind": "stopped",+ "run_display_reason": null,+ "run_id": "wr_564921777870827706",+ "run_output_report": "Recorded output from the latest completed run: {\"read_heading_output\":{\"evidence_text\":\"Example Domain | Example Domain\",\"heading_text\":\"Example Domain\",\"page_title\":\"Example Domain\"}}",+ "run_verdict": "not_evaluated",+ "terminal_cause": null,+ "user_action_required": false,+ "verified": false,+ "workflow_applied": false+ },+ "terminalMessage": "I created and tested the single read_heading block twice. It now returns both the top heading and page title; the latest run completed with non-empty output.",+ "testedBlockFingerprints": {+ "read_heading": [+ "67244b78a2676852d281c77e0f4ba3310b68d0083180318109a6b1de4e791b82",+ "79e5d6d7ed953e0beb207ebb9033e9d6bbb72d7b68118b1df4133406608e339d"+ ]+ },+ "turnFacts": {+ "authoredBlockCount": 1,+ "blocksRunThisTurn": 1,+ "evaluationState": "not_evaluated",+ "factsAvailable": true,+ "matchingSourceBlockCount": 1,+ "ranCleanOnCurrentSource": true,+ "runCompleted": null,+ "runId": "wr_564921777870827706",+ "terminalCause": null+ },+ "turnId": "466012b32fd94e0ca274b00aa59650db",+ "turnIndex": 0+ }+}diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex c9e42ef12..f28f3e889 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -55,6 +55,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (RecordedBuildTestOutcome,history_has_runtime_block_failure,observed_value_extraction_scaffold_lines,+ unresolved_runtime_block_failure,unresolved_runtime_block_failure_with_disposition,)from skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructions@@ -83,6 +84,7 @@ from skyvern.forge.sdk.copilot.context import (NarrativeActivityEntry,NarrativeBlock,NarrativeDraft,+ NarrativeTurnFacts,ProposalDisposition,ResponseType,StructuredContext,@@ -154,7 +156,12 @@ from skyvern.forge.sdk.copilot.request_policy import (redact_raw_secrets_for_prompt,redact_refused_secret_turns,)-from skyvern.forge.sdk.copilot.review_gate import build_review_projection, serialize_execution_receipts+from skyvern.forge.sdk.copilot.review_gate import (+ NarrativeReviewBlock,+ NarrativeReviewProjection,+ build_review_projection,+ serialize_execution_receipts,+)from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.runtime import (BrowserProbeOutcome,@@ -175,6 +182,7 @@ from skyvern.forge.sdk.copilot.terminal_envelope import (TerminalCause,assemble_terminal_envelope,reason_in_reply_shadow,+ select_run_outcome_anchor,)from skyvern.forge.sdk.copilot.todo_list import todo_list_promptfrom skyvern.forge.sdk.copilot.tools.credentials import _server_verified_google_account_choices@@ -1457,6 +1465,85 @@ def _terminal_cause_for_context(ctx: CopilotContext) -> TerminalCause | None:return None+def _authored_review_blocks(review: NarrativeReviewProjection | None) -> list[NarrativeReviewBlock]:+ if review is None:+ return []+ return [block for block in review["blocks"] if block.get("change") != "removed"]+++def _draft_ran_on_current_source(+ facts: NarrativeTurnFacts, unresolved_failure: UnresolvedRuntimeFailure | None+) -> bool:+ """A tested draft claims full current-source coverage and a lifecycle that contradicts nothing:+ a halted turn, an unfinished or unconfirmed run, or an open block failure each disqualify it.+ ``runCompleted`` is None when no run is anchored to this turn, leaving the source-bound receipts+ as the only claim."""+ authored = facts.get("authoredBlockCount")+ if not facts["factsAvailable"] or not authored or facts.get("matchingSourceBlockCount") != authored:+ return False+ if facts["terminalCause"] is not None or unresolved_failure is not None:+ return False+ return facts["runCompleted"] is not False and facts["evaluationState"] != "not_demonstrated"+++def _review_projection_for(ctx: CopilotContext, proposal_yaml: str | None) -> NarrativeReviewProjection | None:+ if not isinstance(proposal_yaml, str):+ return None+ return build_review_projection(+ ctx.persisted_workflow_yaml or _EMPTY_REVIEW_BASELINE_YAML,+ proposal_yaml,+ ctx.executed_block_fingerprints,+ )+++def _tested_draft_reply(+ ctx: CopilotContext,+ review: NarrativeReviewProjection | None,+ *,+ tested_reply: str,+ unvalidated_reply: str,+) -> str:+ """The reply and the proposal disposition read one predicate, so prose cannot outrun the pill."""+ tested = _turn_facts_for_context(ctx, review)["ranCleanOnCurrentSource"]+ return tested_reply if tested else unvalidated_reply+++def _turn_fact_bundle(+ review: NarrativeReviewProjection | None,+ run_outcome: RecordedRunOutcome | None,+ terminal_cause: TerminalCause | None,+ blocks_run_this_turn: int | None,+ unresolved_failure: UnresolvedRuntimeFailure | None = None,+) -> NarrativeTurnFacts:+ run_id = run_outcome.workflow_run_id if run_outcome is not None else None+ facts: NarrativeTurnFacts = {+ "factsAvailable": review is not None,+ "evaluationState": run_outcome.verdict if run_outcome is not None else None,+ "runId": (run_id.strip() or None) if isinstance(run_id, str) else None,+ "runCompleted": run_outcome.run_completed if run_outcome is not None else None,+ "terminalCause": terminal_cause,+ "blocksRunThisTurn": blocks_run_this_turn,+ # Overwritten below, once the coverage counts it reads are in place.+ "ranCleanOnCurrentSource": False,+ }+ if review is not None:+ authored = _authored_review_blocks(review)+ facts["authoredBlockCount"] = len(authored)+ facts["matchingSourceBlockCount"] = sum(1 for block in authored if block.get("coverage") == "current_source")+ facts["ranCleanOnCurrentSource"] = _draft_ran_on_current_source(facts, unresolved_failure)+ return facts+++def _turn_facts_for_context(ctx: CopilotContext, review: NarrativeReviewProjection | None) -> NarrativeTurnFacts:+ return _turn_fact_bundle(+ review,+ select_run_outcome_anchor(_terminal_envelope_run_outcomes(ctx)),+ _terminal_cause_for_context(ctx),+ len(ctx.executed_block_labels),+ unresolved_runtime_block_failure(ctx),+ )++def _assemble_terminal_envelope_safe(*,response_type: str,@@ -1471,6 +1558,7 @@ def _assemble_terminal_envelope_safe(workflow_attempted: bool,final_message: str,terminal_cause: TerminalCause | None = None,+ blocks_run_this_turn: int | None = None,) -> dict[str, Any] | None:try:envelope = assemble_terminal_envelope(@@ -1485,6 +1573,7 @@ def _assemble_terminal_envelope_safe(workflow_mutated=workflow_mutated,workflow_attempted=workflow_attempted,terminal_cause=terminal_cause,+ blocks_run_this_turn=blocks_run_this_turn,)except Exception:LOG.warning("copilot terminal envelope assembly failed", exc_info=True)@@ -1565,18 +1654,23 @@ def _make_agent_result(if ctx is not None and narrative_payload is None:raise ValueError("_make_agent_result requires narrative_payload when ctx is provided")if ctx is not None and isinstance(narrative_payload, dict) and isinstance(proposal_yaml, str):- review = build_review_projection(- ctx.persisted_workflow_yaml or _EMPTY_REVIEW_BASELINE_YAML,- proposal_yaml,- ctx.executed_block_fingerprints,- )+ review = _review_projection_for(ctx, proposal_yaml)narrative_payload = {key: value for key, value in narrative_payload.items() if key != "review"}if review is not None:narrative_payload["review"] = reviewkwargs["narrative_payload"] = narrative_payload+ review_projection: NarrativeReviewProjection | None = (+ narrative_payload.get("review") if isinstance(narrative_payload, dict) else None+ )+ terminal_cause = _terminal_cause_for_context(ctx) if ctx is not None else None+ turn_facts = _turn_facts_for_context(ctx, review_projection) if ctx is not None else Noneresponse_type = kwargs.get("response_type", "REPLY")response_type_value = response_type if isinstance(response_type, str) else "REPLY"- proposal_disposition = kwargs.get("proposal_disposition")+ raw_disposition = kwargs.get("proposal_disposition")+ proposal_disposition: str | None = raw_disposition if isinstance(raw_disposition, str) else None+ if turn_facts is not None and proposal_disposition == "review_tested" and not turn_facts["ranCleanOnCurrentSource"]:+ proposal_disposition = "review_untested"+ kwargs["proposal_disposition"] = proposal_dispositionresult_carries_workflow = (kwargs.get("updated_workflow") is not Noneor kwargs.get("staged_workflow") is not None@@ -1696,9 +1790,14 @@ def _make_agent_result(workflow_mutated=workflow_mutated,workflow_attempted=ctx.has_genuine_workflow_attempt(),final_message=str(kwargs.get("user_response") or ""),- terminal_cause=_terminal_cause_for_context(ctx),+ terminal_cause=terminal_cause,+ blocks_run_this_turn=len(ctx.executed_block_labels),)kwargs["terminal_envelope"] = terminal_envelope+ if turn_facts is not None:+ payload = kwargs.get("narrative_payload")+ if isinstance(payload, dict):+ kwargs["narrative_payload"] = {**payload, "turnFacts": turn_facts}if ctx is not None and "executed_block_fingerprints" not in kwargs:kwargs["executed_block_fingerprints"] = {label: set(fingerprints) for label, fingerprints in ctx.executed_block_fingerprints.items()@@ -2770,7 +2869,14 @@ def _build_wip_exit_result(verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)if outcome_fully_verified(ctx) and verified_workflow is not None:- final_text, outcome = _guard(tested_reply)+ final_text, outcome = _guard(+ _tested_draft_reply(+ ctx,+ _review_projection_for(ctx, verified_yaml),+ tested_reply=tested_reply,+ unvalidated_reply=unvalidated_reply,+ )+ )proposal_disposition = "auto_applicable"return _finalize_result_with_blocker_override(ctx,@@ -2810,7 +2916,13 @@ def _build_wip_exit_result(and not ctx.last_test_suspicious_success):append_failure_detail = recorded_failure_reply and not deadline_suppresses_failure_detail- reply = _last_good_failure_reply(ctx, tested_reply) if append_failure_detail else tested_reply+ held_reply = _tested_draft_reply(+ ctx,+ _review_projection_for(ctx, ctx.last_good_workflow_yaml),+ tested_reply=tested_reply,+ unvalidated_reply=unvalidated_reply,+ )+ reply = _last_good_failure_reply(ctx, held_reply) if append_failure_detail else held_replyfinal_text, outcome = _guard(reply)return _finalize_result_with_blocker_override(ctx,@@ -2848,7 +2960,16 @@ def _build_wip_exit_result(if unvalidated and recorded_failure_reply:reply = _deadline_owned_or(unvalidated_reply, recorded_failure_reply) or unvalidated_replyelse:- reply = unvalidated_reply if unvalidated else tested_reply+ reply = (+ unvalidated_reply+ if unvalidated+ else _tested_draft_reply(+ ctx,+ _review_projection_for(ctx, ctx.last_workflow_yaml),+ tested_reply=tested_reply,+ unvalidated_reply=unvalidated_reply,+ )+ )final_text, outcome = _guard(reply)proposal_disposition = "review_untested" if unvalidated else "review_tested"if not unvalidated and outcome_fully_verified(ctx):diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex ea5f28f06..c175a358f 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -106,6 +106,19 @@ class NarrativeConnectedAccountChoice(TypedDict):email_address: str | None+class NarrativeTurnFacts(TypedDict):+ factsAvailable: bool+ evaluationState: str | None+ runId: str | None+ runCompleted: bool | None+ terminalCause: str | None+ blocksRunThisTurn: int | None+ authoredBlockCount: NotRequired[int]+ matchingSourceBlockCount: NotRequired[int]+ # The tested claim is decided once, in _turn_fact_bundle, so no surface re-derives it.+ ranCleanOnCurrentSource: bool++# Mirror of the FE TurnNarrativeState; camelCase keys match the wire shape.class TurnNarrativePayload(TypedDict):turnId: str | None@@ -139,6 +152,7 @@ class TurnNarrativePayload(TypedDict):endedAt: str | Nonereview: NotRequired[NarrativeReviewProjection]testedBlockFingerprints: NotRequired[dict[str, list[str]]]+ turnFacts: NotRequired[NarrativeTurnFacts]if TYPE_CHECKING:diff --git a/skyvern/forge/sdk/copilot/review_gate.py b/skyvern/forge/sdk/copilot/review_gate.pyindex b33919a50..9d26c5a4a 100644--- a/skyvern/forge/sdk/copilot/review_gate.py+++ b/skyvern/forge/sdk/copilot/review_gate.py@@ -16,6 +16,7 @@ from skyvern.schemas.google_sheets import extract_a1_sheet_prefix, extract_spreafrom skyvern.schemas.workflows import BlockType, FileStorageTypeReviewChange = Literal["added", "changed", "unchanged", "removed"]+BlockCoverage = Literal["current_source", "different_source", "never_run", "unknown"]ParameterDefault = str | int | float | boolDestinationIdentity = tuple[str, ...]DestinationAdapter = Callable[[dict[str, Any], Mapping[str, ParameterDefault]], DestinationIdentity | None]@@ -31,6 +32,7 @@ class NarrativeReviewBlock(TypedDict):blockType: strchange: ReviewChangeneverTested: NotRequired[bool]+ coverage: NotRequired[BlockCoverage]class NarrativeDuplicateWrite(TypedDict):@@ -353,6 +355,23 @@ def build_review_projection(block["label"]: _block_fingerprint(block, staged_execution_inputs) for block in staged_blocks}+ if isinstance(executed_blocks, Mapping):+ receipts: dict[str, set[str]] = {+ label: {value} if isinstance(value, str) else set(value) for label, value in executed_blocks.items()+ }+ source_bound = True+ else:+ receipts = {label: set() for label in executed_blocks}+ source_bound = False+ # A block's own label is excluded from its fingerprint, so a receipt stranded under a+ # label the staged workflow no longer has still identifies source this turn ran.+ orphan_fingerprints = {+ fingerprint+ for label, recorded in receipts.items()+ if label not in staged_fingerprints+ for fingerprint in recorded+ }+review_blocks: list[NarrativeReviewBlock] = []for block in staged_blocks:label = block["label"]@@ -369,21 +388,20 @@ def build_review_projection(change = "unchanged"else:change = "changed"+ recorded = receipts.get(label)+ if recorded is None:+ coverage: BlockCoverage = "unknown" if staged_fingerprints[label] in orphan_fingerprints else "never_run"+ elif not source_bound or staged_fingerprints[label] in recorded:+ coverage = "current_source"+ else:+ coverage = "different_source"review_blocks.append({"label": label,"blockType": _block_type(block),"change": change,- "neverTested": (- staged_fingerprints[label]- not in (- {executed_blocks[label]}- if isinstance(executed_blocks.get(label), str)- else executed_blocks.get(label, ())- )- if isinstance(executed_blocks, Mapping)- else label not in executed_blocks- ),+ "neverTested": coverage != "current_source",+ "coverage": coverage,})for block in persisted_blocks:diff --git a/skyvern/forge/sdk/copilot/run_outcome.py b/skyvern/forge/sdk/copilot/run_outcome.pyindex 427fe5927..1e1e4bec0 100644--- a/skyvern/forge/sdk/copilot/run_outcome.py+++ b/skyvern/forge/sdk/copilot/run_outcome.py@@ -42,6 +42,9 @@ class RecordedRunOutcome:display_reason: str | None = Noneworkflow_run_id: str | None = Noneoutput_report: str | None = None+ # Recorded lifecycle, kept apart from ``verdict``: reaching a completed status says+ # nothing about whether the outcome was evaluated. ``None`` on frames predating the field.+ run_completed: bool | None = None# ``adjudicated`` remains accepted for persisted legacy frames. New# interactive authoring frames are factual records.role: RunOutcomeRole = "recorded"diff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.pyindex 4fb22ae7b..728158d29 100644--- a/skyvern/forge/sdk/copilot/terminal_envelope.py+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py@@ -16,6 +16,8 @@ _FINAL_RUN_VERDICTS = frozenset({"not_demonstrated", "not_evaluated"})_REVIEW_PROPOSAL_DISPOSITIONS = frozenset({"review_untested", "review_tested"})_SHADOW_REASON_TRAILING_PUNCTUATION = ".,;:!?"+MINIMAL_HONEST_STOP = "I stopped without confirming the goal was met."+INTERRUPTED_TERMINAL_REASON = "interrupted"INTERRUPTED_TERMINAL_HEADLINE = "This turn was interrupted before it could finish."INTERRUPTED_TERMINAL_RETRY = "Send your message again to retry."@@ -42,6 +44,9 @@ class TerminalOutcomeEnvelope(BaseModel):verified: boolworkflow_applied: bool = Falserun_verdict: str | None = None+ run_id: str | None = None+ run_completed: bool | None = None+ blocks_run_this_turn: int | None = Nonerun_display_reason: str | None = Nonerun_output_report: str | None = Noneblocker_reason: str | None = None@@ -68,9 +73,14 @@ def assemble_terminal_envelope(workflow_mutated: bool,workflow_attempted: bool,terminal_cause: TerminalCause | None = None,+ blocks_run_this_turn: int | None = None,) -> TerminalOutcomeEnvelope | None:- run_outcome = _select_run_outcome_anchor(run_outcomes)+ run_outcome = select_run_outcome_anchor(run_outcomes)run_verdict = run_outcome.verdict if run_outcome is not None else None+ run_id = _clean_text(run_outcome.workflow_run_id) if run_outcome is not None else None+ # Lifecycle and identity come from the same archived outcome, which a workflow edit+ # never rewrites, so the two can never name different runs.+ run_completed = run_outcome.run_completed if run_outcome is not None else Nonerun_display_reason = _clean_text(run_outcome.display_reason) if run_outcome is not None else Nonerun_output_report = _safe_output_report(run_outcome.output_report) if run_outcome is not None else Noneuser_action_required = response_type == "ASK_QUESTION"@@ -92,6 +102,9 @@ def assemble_terminal_envelope(verified=verified,workflow_applied=workflow_applied,run_verdict=run_verdict,+ run_id=run_id,+ run_completed=run_completed,+ blocks_run_this_turn=blocks_run_this_turn,run_display_reason=run_display_reason,run_output_report=run_output_report,blocker_reason=_clean_text(blocker_reason),@@ -172,13 +185,20 @@ def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: st# "awaiting_user_input" needs no exclusion: it requires ASK_QUESTION, and a deadline# always exits through _build_wip_exit_result, which only ever builds REPLY results.if envelope.terminal_cause == "deadline_expired" and not cancelled and envelope.next_state != "completed":- if output_report and not _text_contains(agent_message, output_report):- return _append_sentence(agent_message, output_report), True- return agent_message, True+ message = agent_message+ facts = _recorded_run_facts(envelope)+ # The agent's own deadline copy already names time, so the time-limit+ # sentence rides along only with facts that copy does not carry.+ if facts:+ for sentence in [*facts, "The turn reached its time limit."]:+ message = _append_sentence(message, sentence)+ if output_report and not _text_contains(message, output_report):+ return _append_sentence(message, output_report), True+ return message, True# A plain reply with no concrete workflow/run/blocker evidence is an answer- # even though next_state remains "stopped"; only stopped-kind turns get the- # honest-stop replacement.+ # even though next_state remains "stopped"; only stopped-kind turns carry the+ # recorded facts.if cancelled:return agent_message, False@@ -187,12 +207,16 @@ def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: streturn _append_sentence(agent_message, output_report), Truereturn agent_message, False- if envelope.run_verdict == "not_demonstrated":- message = "I ran the workflow, but I could not confirm the goal was met."- elif envelope.run_verdict == "not_evaluated":- message = "I ran the workflow. The latest recorded run completed."- else:- message = "I stopped without confirming the goal was met."+ # Facts are appended to the agent's own text rather than replacing it: a message+ # whose unsupported clause sits beside accurate detail loses that detail if replaced.+ message = agent_message.strip() or MINIMAL_HONEST_STOP+ recorded_facts = _recorded_run_facts(envelope)+ # With no run anchored there is no fact to set beside the agent's text, and silence+ # would let an unsupported success claim stand alone on a turn that stopped.+ if not recorded_facts and message != MINIMAL_HONEST_STOP:+ recorded_facts = [MINIMAL_HONEST_STOP]+ for sentence in recorded_facts:+ message = _append_sentence(message, sentence)if envelope.run_display_reason:message = _append_labeled_sentence(message, label="Reason", text=envelope.run_display_reason)@@ -202,10 +226,25 @@ def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: stblocker_reason = envelope.blocker_reasonif blocker_reason and not _text_contains(message, blocker_reason):message = _append_labeled_sentence(message, label="Evidence", text=blocker_reason)- return message, True+ return message, message != agent_message+++def _recorded_run_facts(envelope: TerminalOutcomeEnvelope) -> list[str]:+ facts: list[str] = []+ ran = envelope.blocks_run_this_turn+ if ran is not None:+ facts.append(f"{ran} block{'' if ran == 1 else 's'} ran this turn.")+ # The completion clause rides on the recorded lifecycle fact, never on the+ # verdict: an unevaluated outcome says nothing about whether the run finished.+ lifecycle = "The recorded run completed, and its" if envelope.run_completed else "The recorded run's"+ if envelope.run_verdict == "not_evaluated":+ facts.append(f"{lifecycle} outcome was not evaluated.")+ elif envelope.run_verdict == "not_demonstrated":+ facts.append(f"{lifecycle} outcome did not confirm the goal was met.")+ return facts-def _select_run_outcome_anchor(run_outcomes: Sequence[RecordedRunOutcome]) -> RecordedRunOutcome | None:+def select_run_outcome_anchor(run_outcomes: Sequence[RecordedRunOutcome]) -> RecordedRunOutcome | None:final_outcomes = [outcome for outcome in run_outcomes if outcome.verdict in _FINAL_RUN_VERDICTS]if not final_outcomes:return Nonediff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex bee6b8657..903c26b0e 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -4052,6 +4052,7 @@ def _record_run_blocks_result(recorded_outcome = _recorded_run_outcome(workflow_run_id=run_id if isinstance(run_id, str) else None,output_report=output_report,+ run_completed=run_ok,)unverified = _unverified_current_workflow_labels(copilot_ctx)composition_unverified = _composition_unverified_current_workflow_labels(copilot_ctx)@@ -4123,6 +4124,7 @@ def _record_run_blocks_result(copilot_ctx.last_test_failure_reason or str(result.get("error") or "The run failed.")),workflow_run_id=run_id if isinstance(run_id, str) else None,+ run_completed=False,)_record_build_test_outcome(copilot_ctx, result, recorded_outcome)return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)@@ -4231,12 +4233,14 @@ def _recorded_run_outcome(*,workflow_run_id: str | None = None,output_report: str | None = None,+ run_completed: bool | None = None,) -> RecordedRunOutcome:"""Record the completed run status without interpreting whether it met the request."""return RecordedRunOutcome(verdict="not_evaluated",workflow_run_id=workflow_run_id,output_report=output_report,+ run_completed=run_completed,)diff --git a/tests/unit/copilot/test_review_gate.py b/tests/unit/copilot/test_review_gate.pyindex 76c7d3f43..9b1eccec5 100644--- a/tests/unit/copilot/test_review_gate.py+++ b/tests/unit/copilot/test_review_gate.py@@ -63,9 +63,22 @@ def test_projection_classifies_all_changes_and_uses_cumulative_execution_evidenc"blockType": "task","change": "unchanged","neverTested": False,+ "coverage": "current_source",+ },+ {+ "label": "changed_step",+ "blockType": "task",+ "change": "changed",+ "neverTested": False,+ "coverage": "current_source",+ },+ {+ "label": "added_step",+ "blockType": "task",+ "change": "added",+ "neverTested": True,+ "coverage": "never_run",},- {"label": "changed_step", "blockType": "task", "change": "changed", "neverTested": False},- {"label": "added_step", "blockType": "task", "change": "added", "neverTested": True},{"label": "removed_step", "blockType": "task", "change": "removed"},]@@ -204,8 +217,20 @@ def test_a_drafted_sheets_append_is_never_tested_until_it_actually_runs() -> Nonassert projection is not Noneassert projection["blocks"] == [- {"label": "count_posts", "blockType": "task", "change": "unchanged", "neverTested": False},- {"label": "append_count", "blockType": "google_sheets_write", "change": "added", "neverTested": True},+ {+ "label": "count_posts",+ "blockType": "task",+ "change": "unchanged",+ "neverTested": False,+ "coverage": "current_source",+ },+ {+ "label": "append_count",+ "blockType": "google_sheets_write",+ "change": "added",+ "neverTested": True,+ "coverage": "never_run",+ },]@@ -227,6 +252,7 @@ def test_parameter_default_change_invalidates_change_and_execution_identity() ->"blockType": "task","change": "changed","neverTested": True,+ "coverage": "different_source",}@@ -278,6 +304,7 @@ def test_nested_block_label_change_invalidates_parent_tested_version() -> None:"blockType": "for_loop","change": "changed","neverTested": True,+ "coverage": "different_source",}@@ -733,3 +760,71 @@ def test_run_scoped_write_outputs_abstain_from_duplicate_claims() -> None:assert projection is not Noneassert projection["duplicateWrites"] == []+++def test_a_comment_only_edit_reports_the_prior_receipt_as_a_different_source() -> None:+ tested = _workflow(""" - block_type: code+ label: sign_in+ code: |+ await page.click("#submit")+""")+ edited = _workflow(""" - block_type: code+ label: sign_in+ code: |+ # retry the submit button+ await page.click("#submit")+""")++ projection = build_review_projection(tested, edited, workflow_block_fingerprints(tested))++ assert projection is not None+ assert projection["blocks"][0]["coverage"] == "different_source"+ assert projection["blocks"][0]["change"] == "changed"+++def test_a_renamed_block_loses_its_receipt_while_siblings_keep_theirs() -> None:+ tested = _workflow(""" - block_type: task+ label: sign_in+ next_block_label: read_metric+ prompt: Sign in+ - block_type: task+ label: read_metric+ prompt: Read it+""")+ renamed = _workflow(""" - block_type: task+ label: sign_in_v2+ next_block_label: read_metric+ prompt: Sign in+ - block_type: task+ label: read_metric+ prompt: Read it+""")++ projection = build_review_projection(tested, renamed, workflow_block_fingerprints(tested))++ assert projection is not None+ assert [(row["label"], row.get("coverage")) for row in projection["blocks"]] == [+ ("sign_in_v2", "unknown"),+ ("read_metric", "current_source"),+ ("sign_in", None),+ ]+++def test_editing_one_of_two_blocks_leaves_the_untouched_block_current() -> None:+ tested = _workflow(""" - block_type: task+ label: sign_in+ next_block_label: read_metric+ prompt: Sign in+ - block_type: task+ label: read_metric+ prompt: Read it+""")+ edited = tested.replace("prompt: Read it", "prompt: Read it twice")++ projection = build_review_projection(tested, edited, workflow_block_fingerprints(tested))++ assert projection is not None+ assert [(row["label"], row["coverage"]) for row in projection["blocks"]] == [+ ("sign_in", "current_source"),+ ("read_metric", "different_source"),+ ]diff --git a/tests/unit/test_copilot_agent_blocker_renderer.py b/tests/unit/test_copilot_agent_blocker_renderer.pyindex f348b7c0f..400bc027e 100644--- a/tests/unit/test_copilot_agent_blocker_renderer.py+++ b/tests/unit/test_copilot_agent_blocker_renderer.py@@ -31,6 +31,7 @@ from skyvern.forge.sdk.copilot.output_policy import (OutputPolicyVerdict,)from skyvern.forge.sdk.copilot.request_policy import LivePageResolutionRecord, RequestPolicy+from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprintsfrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.turn_halt import TurnHalt, TurnHaltKindfrom skyvern.forge.sdk.copilot.turn_origin import TurnOrigin@@ -49,6 +50,16 @@ _ALL_BLOCKER_KINDS: tuple[BlockerKind, ...] = get_args(BlockerKind)_ALL_RECOVERY_HINTS: tuple[RecoveryHint, ...] = get_args(RecoveryHint)+_COVERED_DRAFT_YAML = """title: Draft+workflow_definition:+ parameters: []+ blocks:+ - block_type: task+ label: step+ prompt: Do it+"""++def _signal(*,kind: BlockerKind = "authority_denied",@@ -275,7 +286,10 @@ def test_output_policy_generic_block_uses_only_safety_and_draft_evidence() -> Noctx = _ctx()fake_workflow: Any = object()ctx.last_workflow = fake_workflow- ctx.last_workflow_yaml = "title: Draft\n"+ ctx.last_workflow_yaml = _COVERED_DRAFT_YAML+ ctx.executed_block_fingerprints = {+ label: set(values) for label, values in workflow_block_fingerprints(_COVERED_DRAFT_YAML).items()+ }ctx.workflow_persisted = Truectx.last_test_ok = True_seed_terminal_evidence(ctx)@@ -285,7 +299,7 @@ def test_output_policy_generic_block_uses_only_safety_and_draft_evidence() -> Noassert result.response_type == "ASK_QUESTION"assert result.updated_workflow is fake_workflow- assert result.proposal_disposition == "review_tested"+ assert result.proposal_disposition == "review_untested"assert "latest run recorded workflow output" not in result.user_responseassert "did not demonstrate" not in result.user_responseassert "verification challenge" in result.user_responsediff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 7cec79041..48b3e99d8 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -32,6 +32,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (from skyvern.forge.sdk.copilot.build_test_outcome import (PostRunPagePathFailure,RecordedBuildTestOutcome,+ record_build_test_outcome,)from skyvern.forge.sdk.copilot.completion_criteria_store import (StoredCriteriaSet,@@ -81,6 +82,7 @@ from skyvern.forge.sdk.copilot.request_policy import (redact_raw_secrets_for_prompt,)from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOTS_PROMPT_NAME+from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprintsfrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools import _run_blocks_and_collect_debugfrom skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module@@ -113,8 +115,28 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatSender,)from skyvern.utils.yaml_loader import safe_load_no_dates+from tests.unit.copilot_test_helpers import failed_second_factor_runfrom tests.unit.copilot_test_helpers import make_copilot_ctx as _ctxfrom tests.unit.copilot_test_helpers import make_verified_goal_contract as _verified_goal_contract+from tests.unit.copilot_test_helpers import passing_run, two_page_login_yaml++_COVERED_DRAFT_YAML = """title: Draft+workflow_definition:+ parameters: []+ blocks:+ - block_type: task+ label: open_page+ prompt: Open the page+"""+++def _covered_draft_receipts() -> dict[str, set[str]]:+ return {label: set(values) for label, values in workflow_block_fingerprints(_COVERED_DRAFT_YAML).items()}+++def _clean_recorded_run() -> RecordedRunOutcome:+ return RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_clean", run_completed=True)+_HISTORY_SENTINEL_TS = datetime(2026, 1, 1, tzinfo=timezone.utc)@@ -990,7 +1012,9 @@ class TestVerifiedGoalSatisfiedStop:ctx = _ctx(turn_origin=TurnOrigin.runtime_self_heal,last_workflow=workflow,- last_workflow_yaml="workflow_definition:\n blocks: []\n",+ last_workflow_yaml=_COVERED_DRAFT_YAML,+ executed_block_fingerprints=_covered_draft_receipts(),+ last_run_outcome=_clean_recorded_run(),last_test_ok=True,last_full_workflow_test_ok=True,tool_activity=[{"tool": "update_and_run_blocks", "summary": "OK"}],@@ -999,7 +1023,7 @@ class TestVerifiedGoalSatisfiedStop:result = await _build_goal_satisfied_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is workflow- assert result.workflow_yaml == "workflow_definition:\n blocks: []\n"+ assert result.workflow_yaml == _COVERED_DRAFT_YAMLassert result.proposal_disposition == "review_tested"assert result.user_response == "The unattended recovery check completed."assert result.narrative_payload is not None@@ -2554,7 +2578,9 @@ class TestTranslateToAgentResultGating:ctx = _ctx(block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,last_workflow=workflow,- last_workflow_yaml="title: Structural Draft",+ last_workflow_yaml=_COVERED_DRAFT_YAML,+ executed_block_fingerprints=_covered_draft_receipts(),+ last_run_outcome=_clean_recorded_run(),last_test_ok=True,last_full_workflow_test_ok=True,latest_diagnosis_repair_contract=_unverified_no_repair_contract(),@@ -2592,7 +2618,8 @@ class TestTranslateToAgentResultGating:ctx = _ctx(block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,last_workflow=workflow,- last_workflow_yaml="title: Verified Draft",+ last_workflow_yaml=_COVERED_DRAFT_YAML,+ executed_block_fingerprints=_covered_draft_receipts(),last_test_ok=True,last_full_workflow_test_ok=True,has_staged_proposal=True,@@ -2604,7 +2631,7 @@ class TestTranslateToAgentResultGating:verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],),last_run_blocks_workflow_run_id="wr_1",- last_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1"),+ last_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1", run_completed=True),)agent_result = agent_module._build_wip_exit_result(@@ -2963,7 +2990,10 @@ class TestTranslateToAgentResultGating:)assert agent_result.updated_workflow is replacement- assert agent_result.proposal_disposition == "review_tested"+ # A tested pill now needs receipts bound to the staged source, and this stub proposal+ # carries no blocks to bind, so the full-test flags alone no longer earn one. The+ # satisfaction path is proven against real blocks in TestCanonicalTurnFacts.+ assert agent_result.proposal_disposition == "review_untested"def test_inline_replace_workflow_steers_on_stale_block_metadata(self, monkeypatch) -> None:# A label still describing the prior subject is authoring quality, not disclosure, so the@@ -3540,12 +3570,14 @@ workflow_definition:)return _ctx(last_workflow=verified_wf,- last_workflow_yaml="verified: yaml",+ last_workflow_yaml=_COVERED_DRAFT_YAML,+ executed_block_fingerprints=_covered_draft_receipts(),+ last_run_outcome=_clean_recorded_run(),last_test_ok=True,last_full_workflow_test_ok=True,has_staged_proposal=True,staged_workflow=verified_wf,- staged_workflow_yaml="verified: yaml",+ staged_workflow_yaml=_COVERED_DRAFT_YAML,)ask_ctx = build_ctx()@@ -6119,3 +6151,400 @@ class TestToolFactOwnership:for duplicated in ("- CSS SELECTORS:", "- SCROLL:", "- SELECT_OPTION:", "- PRESS_KEY:", "- CONSOLE_MESSAGES:"):assert duplicated not in promptassert "Use evaluate for reading/inspection" not in prompt+++class TestCanonicalTurnFacts:+ """One fact bundle behind the terminal card, the proposal pill and the terminal prose."""++ @staticmethod+ def _two_block_yaml(second_prompt: str = "Read it") -> str:+ return f"""+title: Sign in and read the metric+workflow_definition:+ parameters: []+ blocks:+ - block_type: task+ label: sign_in+ next_block_label: read_metric+ prompt: Sign in+ - block_type: task+ label: read_metric+ prompt: {second_prompt}+"""++ @staticmethod+ def _render(ctx: Any, staged_yaml: str, **overrides: Any) -> dict[str, Any]:+ kwargs: dict[str, Any] = {+ "user_response": "ok",+ "global_llm_context": None,+ "updated_workflow": None,+ "workflow_yaml": staged_yaml,+ "proposal_disposition": "review_tested",+ "narrative_payload": {+ "turnId": "turn-1",+ "turnIndex": 0,+ "designStarted": True,+ "designEnded": True,+ "draft": None,+ "blocks": [],+ "terminal": "response",+ "terminalMessage": "done",+ "narrativeSummary": "Built it.",+ "priorBlockCount": None,+ "designActivity": [],+ "startedAt": None,+ "endedAt": None,+ },+ }+ kwargs.update(overrides)+ result = agent_module._make_agent_result(ctx, **kwargs)+ assert result.narrative_payload is not None+ return {+ "facts": result.narrative_payload["turnFacts"],+ "review": result.narrative_payload.get("review"),+ "disposition": result.proposal_disposition,+ }++ def test_partial_current_source_coverage_states_counts_and_drops_the_tested_pill(self) -> None:+ staged = self._two_block_yaml()+ ctx = _ctx(persisted_workflow_yaml=staged)+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}+ ctx.executed_block_labels = {"sign_in"}++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["authoredBlockCount"] == 2+ assert rendered["facts"]["matchingSourceBlockCount"] == 1+ assert rendered["facts"]["blocksRunThisTurn"] == 1+ assert rendered["facts"]["factsAvailable"] is True+ assert rendered["disposition"] == "review_untested"++ def test_full_current_source_coverage_keeps_the_tested_pill(self) -> None:+ staged = self._two_block_yaml()+ ctx = _ctx(persisted_workflow_yaml=staged, last_run_outcome=_clean_recorded_run())+ ctx.executed_block_fingerprints = {+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()+ }+ ctx.executed_block_labels = {"sign_in", "read_metric"}++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["authoredBlockCount"] == 2+ assert rendered["facts"]["matchingSourceBlockCount"] == 2+ assert rendered["facts"]["runCompleted"] is True+ assert rendered["facts"]["evaluationState"] == "not_evaluated"+ assert rendered["disposition"] == "review_tested"++ def test_full_coverage_over_an_unfinished_run_drops_the_tested_pill(self) -> None:+ staged = self._two_block_yaml()+ ctx = _ctx(+ persisted_workflow_yaml=staged,+ last_run_outcome=RecordedRunOutcome(+ verdict="not_evaluated", workflow_run_id="wr_partial", run_completed=False+ ),+ )+ ctx.executed_block_fingerprints = {+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()+ }+ ctx.executed_block_labels = {"sign_in", "read_metric"}++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["matchingSourceBlockCount"] == 2+ assert rendered["facts"]["runCompleted"] is False+ assert rendered["disposition"] == "review_untested"++ def test_a_block_failure_no_later_run_re_exercised_drops_the_tested_pill(self) -> None:+ staged = two_page_login_yaml()+ ctx = _ctx(+ persisted_workflow_yaml=staged,+ workflow_yaml=staged,+ last_run_outcome=_clean_recorded_run(),+ )+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))+ ctx.executed_block_fingerprints = {+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()+ }+ ctx.executed_block_labels = {"sign_in_and_read"}++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["matchingSourceBlockCount"] == rendered["facts"]["authoredBlockCount"]+ assert rendered["facts"]["runCompleted"] is True+ assert rendered["disposition"] == "review_untested"++ def test_full_coverage_over_an_unconfirmed_outcome_drops_the_tested_pill(self) -> None:+ staged = self._two_block_yaml()+ ctx = _ctx(+ persisted_workflow_yaml=staged,+ last_run_outcome=RecordedRunOutcome(+ verdict="not_demonstrated", workflow_run_id="wr_blocked", run_completed=True+ ),+ )+ ctx.executed_block_fingerprints = {+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()+ }+ ctx.executed_block_labels = {"sign_in", "read_metric"}++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["matchingSourceBlockCount"] == 2+ assert rendered["facts"]["evaluationState"] == "not_demonstrated"+ assert rendered["disposition"] == "review_untested"++ def test_an_edited_block_is_no_longer_counted_against_its_stale_receipt(self) -> None:+ tested = self._two_block_yaml()+ staged = self._two_block_yaml("Read it twice")+ ctx = _ctx(persisted_workflow_yaml=tested)+ ctx.executed_block_fingerprints = {+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(tested).items()+ }+ ctx.executed_block_labels = {"sign_in", "read_metric"}++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["matchingSourceBlockCount"] == 1+ assert rendered["disposition"] == "review_untested"++ def test_a_deadline_turn_reports_its_cause_and_never_a_tested_draft(self) -> None:+ staged = self._two_block_yaml()+ ctx = _ctx(persisted_workflow_yaml=staged)+ ctx.executed_block_fingerprints = {+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()+ }+ ctx.executed_block_labels = {"sign_in", "read_metric"}+ ctx.copilot_total_timeout_exceeded = True++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["terminalCause"] == "deadline_expired"+ assert rendered["facts"]["blocksRunThisTurn"] == 2+ assert rendered["disposition"] == "review_untested"++ def test_removed_blocks_leave_the_authored_count(self) -> None:+ persisted = self._two_block_yaml()+ staged = """+title: Sign in and read the metric+workflow_definition:+ parameters: []+ blocks:+ - block_type: task+ label: sign_in+ prompt: Sign in+"""+ ctx = _ctx(persisted_workflow_yaml=persisted)+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}+ ctx.executed_block_labels = {"sign_in"}++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["authoredBlockCount"] == 1+ assert rendered["facts"]["matchingSourceBlockCount"] == 1++ def test_unavailable_review_facts_report_no_counts_and_no_tested_pill(self) -> None:+ duplicate_labels = """+title: Sign in twice+workflow_definition:+ parameters: []+ blocks:+ - block_type: task+ label: sign_in+ prompt: Sign in+ - block_type: task+ label: sign_in+ prompt: Sign in again+"""+ ctx = _ctx(persisted_workflow_yaml=duplicate_labels)++ rendered = self._render(ctx, duplicate_labels)++ assert rendered["facts"]["factsAvailable"] is False+ assert "authoredBlockCount" not in rendered["facts"]+ assert "matchingSourceBlockCount" not in rendered["facts"]+ assert rendered["disposition"] == "review_untested"++ def test_a_draft_with_no_authored_blocks_is_not_a_tested_draft(self) -> None:+ empty = "title: Nothing yet\nworkflow_definition:\n parameters: []\n blocks: []\n"+ ctx = _ctx(persisted_workflow_yaml=empty)++ rendered = self._render(ctx, empty)++ assert rendered["facts"]["authoredBlockCount"] == 0+ assert rendered["disposition"] == "review_untested"++ def test_coverage_facts_survive_a_failed_envelope_assembly(self, monkeypatch: pytest.MonkeyPatch) -> None:+ def _explode(**_kwargs: Any) -> None:+ raise RuntimeError("envelope assembly failed")++ monkeypatch.setattr(agent_module, "assemble_terminal_envelope", _explode)+ staged = self._two_block_yaml()+ ctx = _ctx(persisted_workflow_yaml=staged)+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}+ ctx.executed_block_labels = {"sign_in"}++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["factsAvailable"] is True+ assert rendered["facts"]["authoredBlockCount"] == 2+ assert rendered["facts"]["matchingSourceBlockCount"] == 1+ assert rendered["facts"]["runCompleted"] is None+ assert rendered["facts"]["evaluationState"] is None+ assert rendered["disposition"] == "review_untested"++ def test_run_lifecycle_and_evaluation_state_are_separate_facts(self) -> None:+ staged = self._two_block_yaml()+ ctx = _ctx(persisted_workflow_yaml=staged, last_test_ok=None)+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}+ ctx.executed_block_labels = {"sign_in"}+ ctx.last_run_outcome = RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1", run_completed=True)++ rendered = self._render(ctx, staged)++ assert rendered["facts"]["runCompleted"] is True+ assert rendered["facts"]["evaluationState"] == "not_evaluated"+ assert rendered["facts"]["runId"] == "wr_1"+ assert rendered["facts"]["authoredBlockCount"] == 2+ assert rendered["facts"]["matchingSourceBlockCount"] == 1+ assert rendered["disposition"] == "review_untested"++ @pytest.mark.parametrize(+ ("tested_reply", "unvalidated_reply"),+ [+ (agent_module._TIMEOUT_REPLY_TESTED, agent_module._TIMEOUT_REPLY_UNVALIDATED),+ (agent_module._MAX_TURNS_REPLY_TESTED, agent_module._MAX_TURNS_REPLY_UNVALIDATED),+ (agent_module._UNEXPECTED_ERROR_REPLY_TESTED, agent_module._UNEXPECTED_ERROR_REPLY_UNVALIDATED),+ (agent_module._CANCEL_REPLY_TESTED, agent_module._CANCEL_REPLY_UNVALIDATED),+ ],+ )+ def test_a_partially_covered_draft_claims_no_tested_draft_on_any_terminal_cause(+ self, tested_reply: str, unvalidated_reply: str+ ) -> None:+ staged = self._two_block_yaml()+ ctx = _ctx(+ persisted_workflow_yaml=staged,+ last_workflow=SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[])),+ last_workflow_yaml=staged,+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ )+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}+ ctx.executed_block_labels = {"sign_in"}++ result = agent_module._build_wip_exit_result(+ ctx,+ None,+ default_reply="No draft.",+ unvalidated_reply=unvalidated_reply,+ tested_reply=tested_reply,+ terminal_reason="max_turns",+ )++ assert "tested draft" not in result.user_response+ assert result.user_response == unvalidated_reply+ assert result.proposal_disposition == "review_untested"++ def test_a_fully_covered_draft_keeps_its_tested_reply(self) -> None:+ staged = self._two_block_yaml()+ ctx = _ctx(+ persisted_workflow_yaml=staged,+ last_workflow=SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[])),+ last_workflow_yaml=staged,+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ last_run_outcome=_clean_recorded_run(),+ )+ ctx.executed_block_fingerprints = {+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()+ }+ ctx.executed_block_labels = {"sign_in", "read_metric"}++ result = agent_module._build_wip_exit_result(+ ctx,+ None,+ default_reply="No draft.",+ unvalidated_reply=agent_module._TIMEOUT_REPLY_UNVALIDATED,+ tested_reply=agent_module._TIMEOUT_REPLY_TESTED,+ terminal_reason="max_turns",+ )++ assert result.user_response == agent_module._TIMEOUT_REPLY_TESTED+ assert result.proposal_disposition == "review_tested"++ def test_a_renamed_block_reports_unknown_coverage_and_drops_the_tested_claim(self) -> None:+ tested = self._two_block_yaml()+ staged = tested.replace("label: read_metric", "label: read_metric_v2").replace(+ "next_block_label: read_metric", "next_block_label: read_metric_v2"+ )+ ctx = _ctx(persisted_workflow_yaml=tested)+ ctx.executed_block_fingerprints = {+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(tested).items()+ }+ ctx.executed_block_labels = {"sign_in", "read_metric"}++ rendered = self._render(ctx, staged)+ review = rendered["review"]++ assert [(row["label"], row.get("coverage")) for row in review["blocks"]] == [+ ("sign_in", "different_source"),+ ("read_metric_v2", "unknown"),+ ("read_metric", None),+ ]+ assert rendered["disposition"] == "review_untested"++ def test_a_self_heal_verified_exit_claims_no_tested_draft_on_partial_coverage(self) -> None:+ staged = self._two_block_yaml()+ ctx = _ctx(+ turn_origin=TurnOrigin.runtime_self_heal,+ persisted_workflow_yaml=staged,+ last_workflow=SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[])),+ last_workflow_yaml=staged,+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ completion_verification_result=CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],+ ),+ )+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}+ ctx.executed_block_labels = {"sign_in"}++ result = agent_module._build_wip_exit_result(+ ctx,+ None,+ default_reply="No draft.",+ unvalidated_reply=agent_module._TIMEOUT_REPLY_UNVALIDATED,+ tested_reply=agent_module._TIMEOUT_REPLY_TESTED,+ terminal_reason="max_turns",+ )++ assert outcome_fully_verified(ctx) is True+ assert result.proposal_disposition == "auto_applicable"+ assert "tested draft" not in result.user_response++ def test_a_partially_covered_last_good_draft_claims_no_tested_draft(self) -> None:+ staged = self._two_block_yaml()+ good = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))+ ctx = _ctx(+ persisted_workflow_yaml=staged,+ last_workflow=SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[])),+ last_workflow_yaml=self._two_block_yaml("Read it twice"),+ last_good_workflow=good,+ last_good_workflow_yaml=staged,+ )+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}+ ctx.executed_block_labels = {"sign_in"}++ result = agent_module._build_wip_exit_result(+ ctx,+ None,+ default_reply="No draft.",+ unvalidated_reply=agent_module._TIMEOUT_REPLY_UNVALIDATED,+ tested_reply=agent_module._TIMEOUT_REPLY_TESTED,+ terminal_reason="max_turns",+ )++ assert result.updated_workflow is good+ assert "tested draft" not in result.user_response+ assert result.proposal_disposition == "review_untested"diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.pyindex 0ca0eb0a5..6ad619689 100644--- a/tests/unit/test_copilot_frontier.py+++ b/tests/unit/test_copilot_frontier.py@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.output_utils import (summarize_tool_result,)from skyvern.forge.sdk.copilot.request_policy import RequestPolicy+from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprintsfrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.session_factory import copilot_session_input_callbackfrom skyvern.forge.sdk.copilot.tools import (@@ -3735,6 +3736,12 @@ async def test_completed_test_result_handoff_uses_ordinary_acting_agent_surface(ctx.workflow_verification_evidence.full_workflow_verified = Truectx.workflow_verification_evidence.workflow_run_id = "wr_completed_test_result_handoff"ctx.last_executed_block_labels = ["inspect_result"]+ # The real run path records a source-bound receipt per executed block+ # (run_execution._record_executed_block_labels); this stub replaces that path.+ ctx.executed_block_labels.add("inspect_result")+ ctx.executed_block_fingerprints["inspect_result"] = set(+ workflow_block_fingerprints(workflow_yaml)["inspect_result"]+ )ctx.secret_scrub_values.append(fixture["registered_secret_value"])ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(phase="persisted_block_run",diff --git a/tests/unit/test_copilot_narrative_activity.py b/tests/unit/test_copilot_narrative_activity.pyindex 214c1ebf1..cc0b68d70 100644--- a/tests/unit/test_copilot_narrative_activity.py+++ b/tests/unit/test_copilot_narrative_activity.py@@ -303,8 +303,20 @@ workflow_definition:assert payload["review"] == {"blocks": [- {"label": "existing", "blockType": "task", "change": "changed", "neverTested": False},- {"label": "added", "blockType": "task", "change": "added", "neverTested": True},+ {+ "label": "existing",+ "blockType": "task",+ "change": "changed",+ "neverTested": False,+ "coverage": "current_source",+ },+ {+ "label": "added",+ "blockType": "task",+ "change": "added",+ "neverTested": True,+ "coverage": "never_run",+ },],"duplicateWrites": [],}@@ -341,7 +353,13 @@ workflow_definition:assert payload["review"] == {"blocks": [- {"label": "first_draft", "blockType": "task", "change": "added", "neverTested": True},+ {+ "label": "first_draft",+ "blockType": "task",+ "change": "added",+ "neverTested": True,+ "coverage": "never_run",+ },],"duplicateWrites": [],}diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 8a7b90521..bae8591a1 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -39,6 +39,7 @@ from skyvern.forge.sdk.copilot.output_policy import (normalize_response_scaffolding,)from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy+from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprintsfrom skyvern.forge.sdk.copilot.tools import (_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL,NATIVE_TOOLS,@@ -68,6 +69,9 @@ def _policy(**overrides: object) -> RequestPolicy:return RequestPolicy(**defaults)+_COVERED_DRAFT_YAML = """title: Draft\nworkflow_definition:\n parameters: []\n blocks:\n - block_type: task\n label: step\n prompt: Do it\n"""++def _ctx(**overrides: object) -> CopilotContext:defaults = dict(organization_id="org-1",@@ -1643,7 +1647,10 @@ def test_output_policy_credential_block_asks_for_credential_confirmation(def test_output_policy_block_preserves_already_gated_workflow_proposal() -> None:ctx = _ctx()ctx.last_workflow = SimpleNamespace(name="draft")- ctx.last_workflow_yaml = "title: Draft"+ ctx.last_workflow_yaml = _COVERED_DRAFT_YAML+ ctx.executed_block_fingerprints = {+ label: set(values) for label, values in workflow_block_fingerprints(_COVERED_DRAFT_YAML).items()+ }ctx.workflow_persisted = Truectx.last_test_ok = True@@ -1658,7 +1665,7 @@ def test_output_policy_block_preserves_already_gated_workflow_proposal() -> Noneassert result.response_type == "ASK_QUESTION"assert result.updated_workflow is ctx.last_workflow- assert result.workflow_yaml == "title: Draft"+ assert result.workflow_yaml == _COVERED_DRAFT_YAMLassert result.workflow_was_persisted is Trueassert result.clear_proposed_workflow is Falseassert result.proposal_disposition == "review_tested"diff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.pyindex a448f34c9..30e5d59ca 100644--- a/tests/unit/test_copilot_run_outcome_event.py+++ b/tests/unit/test_copilot_run_outcome_event.py@@ -304,6 +304,7 @@ async def test_blocker_run_emits_not_demonstrated() -> None:reason_code=final.reason_code,display_reason=final.display_reason,workflow_run_id="wr_test",+ run_completed=False,)assert ctx.last_run_outcome_block_labels == final.block_labels@@ -319,6 +320,7 @@ def test_challenge_failure_records_observation_without_halting_agent() -> None:reason_code="blocker_reported",display_reason=run_outcome_display_reason("Human verification challenge blocked the search."),workflow_run_id="wr_challenge",+ run_completed=False,)assert ctx.last_run_outcome == outcomeassert ctx.last_test_ok is False@@ -396,6 +398,7 @@ async def test_completion_judge_cannot_overturn_run_output() -> None:assert outcome == RecordedRunOutcome(verdict="not_evaluated",workflow_run_id="wr_test",+ run_completed=True,)assert ctx.completion_verification_result is Noneassert ctx.last_test_suspicious_success is False@@ -457,6 +460,7 @@ async def test_completed_partial_run_does_not_promote_full_workflow() -> None:assert ctx.last_run_outcome == RecordedRunOutcome(verdict="not_evaluated",workflow_run_id="wr_test",+ run_completed=True,)@@ -483,6 +487,7 @@ async def test_failed_run_emits_its_own_outcome() -> None:assert [frame.verdict for frame in frames] == ["not_demonstrated"]assert ctx.last_run_outcome is not Noneassert ctx.last_run_outcome.reason_code == "blocker_reported"+ assert ctx.last_run_outcome.run_completed is False@pytest.mark.asyncio@@ -823,6 +828,7 @@ def test_completed_run_uses_retained_terminal_output_when_parameter_identity_can'Recorded output from the latest completed run: {"retrieve_resale_demand_document":''{"document_name":"Resale Demand Package (Required Statement of Fees - Demand)"}}'),+ run_completed=True,)assert ctx.last_test_ok is Trueassert ctx.last_full_workflow_test_ok is Truediff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.pyindex eabea5bea..d463f6343 100644--- a/tests/unit/test_copilot_terminal_envelope.py+++ b/tests/unit/test_copilot_terminal_envelope.py@@ -14,6 +14,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (from skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.terminal_envelope import (+ MINIMAL_HONEST_STOP,TerminalOutcomeEnvelope,assemble_terminal_envelope,finalize_applied_state,@@ -456,7 +457,7 @@ def test_render_terminal_message_stopped_not_demonstrated_contains_verbatim_reasassert all(phrase not in lowered for phrase in forbidden_phrases)-def test_render_terminal_message_stopped_degraded_envelope_uses_minimal_honest_stop() -> None:+def test_render_terminal_message_stopped_without_recorded_facts_keeps_the_agent_text() -> None:envelope = TerminalOutcomeEnvelope(next_state="stopped",verified=False,@@ -464,11 +465,30 @@ def test_render_terminal_message_stopped_degraded_envelope_uses_minimal_honest_srun_display_reason=None,response_kind="stopped",)+ message = "The portal does not expose invoices; archived statements are emailed instead."- rendered, replaced = render_terminal_message(envelope, "legacy", cancelled=False)+ rendered, replaced = render_terminal_message(envelope, message, cancelled=False)++ # The agent's own explanation survives intact, but a stopped turn with no run behind+ # it still says so -- the renderer cannot tell an honest explanation from a claim.+ assert replaced is True+ assert rendered.startswith(message)+ assert rendered.endswith(MINIMAL_HONEST_STOP)+++def test_render_terminal_message_stopped_falls_back_to_honest_stop_without_agent_text() -> None:+ envelope = TerminalOutcomeEnvelope(+ next_state="stopped",+ verified=False,+ run_verdict=None,+ run_display_reason=None,+ response_kind="stopped",+ )++ rendered, replaced = render_terminal_message(envelope, " ", cancelled=False)assert replaced is True- assert rendered == "I stopped without confirming the goal was met."+ assert rendered == MINIMAL_HONEST_STOPdef test_render_terminal_message_stopped_with_a_recorded_run_reports_that_it_ran() -> None:@@ -476,6 +496,7 @@ def test_render_terminal_message_stopped_with_a_recorded_run_reports_that_it_rannext_state="stopped",verified=False,run_verdict="not_evaluated",+ run_completed=True,run_display_reason=None,response_kind="stopped",)@@ -483,7 +504,14 @@ def test_render_terminal_message_stopped_with_a_recorded_run_reports_that_it_ranrendered, replaced = render_terminal_message(envelope, "legacy", cancelled=False)assert replaced is True- assert rendered == "I ran the workflow. The latest recorded run completed."+ assert rendered == "legacy. The recorded run completed, and its outcome was not evaluated."++ # Lifecycle is read from the recorded run, never inferred from the verdict: without a+ # recorded completion the same verdict may not claim the run finished.+ unknown_lifecycle = envelope.model_copy(update={"run_completed": None})+ rendered_unknown, _ = render_terminal_message(unknown_lifecycle, "legacy", cancelled=False)+ assert rendered_unknown == "legacy. The recorded run's outcome was not evaluated."+ assert "completed" not in rendered_unknowndef test_render_terminal_message_no_run_blocker_stop_keeps_blocker_evidence() -> None:@@ -497,10 +525,12 @@ def test_render_terminal_message_no_run_blocker_stop_keeps_blocker_evidence() ->response_kind="stopped",)- rendered, replaced = render_terminal_message(envelope, "legacy", cancelled=False)+ message = "The portal does not expose invoices; archived statements are emailed instead."++ rendered, replaced = render_terminal_message(envelope, message, cancelled=False)assert replaced is True- assert rendered.startswith("I stopped without confirming the goal was met.")+ assert rendered.startswith(message)assert blocker in rendered@@ -596,18 +626,28 @@ def test_render_terminal_message_keeps_deadline_copy_on_stopped_fallthrough() ->assert envelope.next_state == "stopped"assert envelope.response_kind == "stopped"- assert rendered == message+ assert rendered.startswith(message)assert rendered != "I stopped without confirming the goal was met."assert replaced is True-def test_render_terminal_message_still_generic_without_deadline_cause() -> None:- envelope = _assemble(proposal_disposition="auto_applicable", run_outcomes=[])+def test_render_terminal_message_under_budget_stop_keeps_the_reply_and_names_the_unevaluated_outcome() -> None:+ envelope = _assemble(+ proposal_disposition="auto_applicable",+ run_outcomes=[RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1", run_completed=True)],+ blocks_run_this_turn=1,+ )+ message = "I built and end-to-end tested the workflow, and verified the account and date range match."- rendered, replaced = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_DEFAULT, cancelled=False)+ rendered, replaced = render_terminal_message(envelope, message, cancelled=False)- assert rendered == "I stopped without confirming the goal was met."+ assert envelope.next_state == "stopped"+ assert envelope.response_kind == "stopped"assert replaced is True+ assert rendered.startswith(message)+ assert "1 block ran this turn." in rendered+ assert "The recorded run completed, and its outcome was not evaluated." in rendered+ assert "The latest recorded run completed." not in rendereddef test_render_terminal_message_held_draft_text_survives_with_deadline_cause() -> None:@@ -617,7 +657,7 @@ def test_render_terminal_message_held_draft_text_survives_with_deadline_cause()rendered, replaced = render_terminal_message(envelope, message, cancelled=False)assert envelope.next_state == "proposal_pending"- assert rendered == message+ assert rendered.startswith(message)assert replaced is True@@ -677,7 +717,9 @@ def test_run_without_output_projects_unverified_blocker_never_success() -> None:assert finalized.response_kind == "stopped"assert finalized.run_output_report is Noneassert finalized.blocker_reason == blocker- assert rendered == agent_module._TIMEOUT_REPLY_DEFAULT+ assert rendered.startswith(agent_module._TIMEOUT_REPLY_DEFAULT)+ assert "The recorded run's outcome was not evaluated." in rendered+ assert "The turn reached its time limit." in renderedassert replaced is Trueassert "completed" not in rendered.lower()assert "success" not in rendered.lower()@@ -777,3 +819,98 @@ def test_max_turns_exit_has_no_cause_when_neither_latch_is_set() -> None:assert result.terminal_envelope is not Noneassert result.terminal_envelope["terminal_cause"] is None+++def test_a_deadline_turn_states_blocks_run_evaluation_and_the_time_limit() -> None:+ envelope = _assemble(+ proposal_disposition="review_tested",+ run_outcomes=[RecordedRunOutcome(verdict="not_evaluated", run_completed=True)],+ terminal_cause="deadline_expired",+ blocks_run_this_turn=1,+ )++ rendered, replaced = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_TESTED, cancelled=False)++ assert "1 block ran this turn." in rendered+ assert "The recorded run completed, and its outcome was not evaluated." in rendered+ assert "reached its time limit" in rendered+ assert "failed" not in rendered.lower()+ assert replaced is True+++def test_an_unevaluated_outcome_alone_never_claims_the_run_finished() -> None:+ envelope = _assemble(+ proposal_disposition="review_tested",+ run_outcomes=[_run_outcome("not_evaluated")],+ terminal_cause="deadline_expired",+ blocks_run_this_turn=1,+ )++ rendered, _ = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_TESTED, cancelled=False)++ assert "The recorded run's outcome was not evaluated." in rendered+ assert "run completed" not in rendered++[email protected]("run_outcomes", [[], [_run_outcome("evaluating")]])+def test_a_deadline_turn_without_a_settled_verdict_states_only_what_it_knows(+ run_outcomes: list[RecordedRunOutcome],+) -> None:+ envelope = _assemble(+ proposal_disposition="review_tested",+ run_outcomes=run_outcomes,+ terminal_cause="deadline_expired",+ blocks_run_this_turn=2,+ )++ rendered, _ = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_TESTED, cancelled=False)++ assert "2 blocks ran this turn." in rendered+ assert "reached its time limit" in rendered+ assert "not evaluated" not in rendered+++def test_a_deadline_turn_with_no_run_facts_leaves_the_agent_copy_alone() -> None:+ envelope = _assemble(proposal_disposition="review_tested", terminal_cause="deadline_expired")++ rendered, replaced = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_UNVALIDATED, cancelled=False)++ assert rendered == agent_module._TIMEOUT_REPLY_UNVALIDATED+ assert replaced is True+++def test_a_deadline_turn_omits_the_block_count_it_was_never_given() -> None:+ envelope = _assemble(+ proposal_disposition="review_tested",+ run_outcomes=[_run_outcome("not_evaluated")],+ terminal_cause="deadline_expired",+ )++ rendered, _ = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_TESTED, cancelled=False)++ assert "ran this turn" not in rendered+ assert "reached its time limit" in rendered+++def test_the_envelope_carries_the_recorded_run_id() -> None:+ envelope = _assemble(run_outcomes=[RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1")])++ assert envelope.run_id == "wr_1"+++def test_run_lifecycle_and_run_id_name_the_same_archived_outcome() -> None:+ completed = _assemble(+ run_outcomes=[RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1", run_completed=True)]+ )++ assert completed.run_id == "wr_1"+ assert completed.run_completed is True++ halted = _assemble(+ run_outcomes=[+ RecordedRunOutcome(verdict="not_demonstrated", workflow_run_id="wr_2", run_completed=False),+ ]+ )++ assert halted.run_id == "wr_2"+ assert halted.run_completed is Falsediff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pyindex 9351e1390..0ce67ef86 100644--- a/tests/unit/test_copilot_timeout_wip.py+++ b/tests/unit/test_copilot_timeout_wip.py@@ -37,6 +37,28 @@ from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprintsfrom skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind+def _draft_yaml(marker: str) -> str:+ return (+ f"title: {marker}\nworkflow_definition:\n parameters: []\n"+ f" blocks:\n - block_type: task\n label: step\n prompt: {marker}\n"+ )+++_DRAFT_V1 = _draft_yaml("v1")+_DRAFT_GOOD = _draft_yaml("good")+_DRAFT_BROKEN = _draft_yaml("broken")+_DRAFT_TESTED = _draft_yaml("tested")+_DRAFT_IN_FLIGHT = _draft_yaml("in-flight")+++def _all_draft_receipts() -> dict[str, set[str]]:+ receipts: dict[str, set[str]] = {}+ for draft in (_DRAFT_V1, _DRAFT_GOOD, _DRAFT_BROKEN, _DRAFT_TESTED, _DRAFT_IN_FLIGHT):+ for label, values in workflow_block_fingerprints(draft).items():+ receipts.setdefault(label, set()).update(values)+ return receipts++def _ctx(*,last_workflow: object | None,@@ -62,6 +84,8 @@ def _ctx(ctx.test_after_update_done = last_test_ok is not Nonectx.last_update_block_count = Nonectx.request_policy.selected_connected_account_id = None+ ctx.persisted_workflow_yaml = None+ ctx.executed_block_fingerprints = _all_draft_receipts()return ctx@@ -91,10 +115,10 @@ def _overwrite_ctx(*, last_test_ok: bool | None) -> MagicMock:in_flight = MagicMock(name="wf-in-flight")return _ctx(last_workflow=in_flight,- last_workflow_yaml="version: in-flight",+ last_workflow_yaml=_DRAFT_IN_FLIGHT,last_test_ok=last_test_ok,last_good_workflow=good,- last_good_workflow_yaml="version: good",+ last_good_workflow_yaml=_DRAFT_GOOD,)@@ -112,16 +136,16 @@ def _state_ctx(state_kind: str) -> tuple[MagicMock, object | None]:return _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None), Nonewf = MagicMock(name="wf")if state_kind == "untested":- return _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None), wf+ return _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None), wfif state_kind == "failed_test":- return _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False), wf+ return _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False), wfif state_kind == "passing_test":- return _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=True), wf+ return _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=True), wfif state_kind == "suspicious_success":return (_ctx(last_workflow=wf,- last_workflow_yaml="version: '1.0'",+ last_workflow_yaml=_DRAFT_V1,last_test_ok=None,last_test_suspicious_success=True,),@@ -281,7 +305,7 @@ def test_capacity_exit_state_disposition(assert result.cancelled is expected_cancelledif surfaces:assert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1else:assert result.updated_workflow is Noneassert result.workflow_yaml is None@@ -314,7 +338,7 @@ class TestBuildTimeoutExitResult:wf = MagicMock(name="wf")ctx = _ctx(last_workflow=wf,- last_workflow_yaml="version: '1.0'",+ last_workflow_yaml=_DRAFT_V1,last_test_ok=None,last_test_suspicious_success=True,)@@ -328,7 +352,7 @@ class TestBuildTimeoutExitResult:result = _build_timeout_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1assert result.proposal_disposition == "review_untested"assert result.user_response == _TIMEOUT_REPLY_UNVALIDATEDassert result.clear_proposed_workflow is False@@ -337,7 +361,7 @@ class TestBuildTimeoutExitResult:wf = MagicMock(name="wf")ctx = _ctx(last_workflow=wf,- last_workflow_yaml="version: '1.0'",+ last_workflow_yaml=_DRAFT_V1,last_test_ok=None,last_test_suspicious_success=True,)@@ -348,7 +372,7 @@ class TestBuildTimeoutExitResult:result = _build_timeout_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1assert result.proposal_disposition == "review_untested"assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED@@ -357,11 +381,11 @@ class TestBuildTimeoutExitResult:last_good = MagicMock(name="last_good")ctx = _ctx(last_workflow=wf,- last_workflow_yaml="version: 'broken'",+ last_workflow_yaml=_DRAFT_BROKEN,last_test_ok=None,last_test_suspicious_success=True,last_good_workflow=last_good,- last_good_workflow_yaml="version: 'tested'",+ last_good_workflow_yaml=_DRAFT_TESTED,)result = _build_timeout_exit_result(ctx, global_llm_context=None)@@ -369,7 +393,7 @@ class TestBuildTimeoutExitResult:# The last-good branch stays guarded on a suspicious run -- crediting that# shape as tested would be the false-success this terminal must not make.assert result.updated_workflow is wf- assert result.workflow_yaml == "version: 'broken'"+ assert result.workflow_yaml == _DRAFT_BROKENassert result.proposal_disposition == "review_untested"assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED@@ -377,26 +401,26 @@ class TestBuildTimeoutExitResult:class TestBuildCancelledExitResult:def test_total_timeout_latch_routes_cancel_to_timeout_wip(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)ctx.copilot_total_timeout_exceeded = Trueresult = _build_cancelled_exit_result(ctx, global_llm_context=None)assert result.cancelled is Falseassert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1assert result.proposal_disposition == "review_untested"assert result.user_response == _TIMEOUT_REPLY_UNVALIDATEDdef test_regular_cancel_uses_cancel_wip_path(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)result = _build_cancelled_exit_result(ctx, global_llm_context=None)assert result.cancelled is Trueassert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1assert result.proposal_disposition == "review_untested"assert result.user_response == _CANCEL_REPLY_UNVALIDATED@@ -417,40 +441,40 @@ class TestBuildUnexpectedErrorExitResult:def test_untested_workflow_surfaces_as_unvalidated_wip(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1assert result.proposal_disposition == "review_untested"assert result.user_response == _UNEXPECTED_ERROR_REPLY_UNVALIDATEDdef test_passing_test_surfaces_as_tested_proposal(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=True)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=True)result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1assert result.proposal_disposition == "review_tested"assert result.user_response == _UNEXPECTED_ERROR_REPLY_TESTEDdef test_failed_test_surfaces_proposal_as_unvalidated(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1assert result.proposal_disposition == "review_untested"assert result.user_response == _UNEXPECTED_ERROR_REPLY_UNVALIDATEDdef test_failed_test_uses_recorded_blocker_reply(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)ctx.last_update_block_count = 3ctx.latest_diagnosis_repair_contract = _blocker_contract("Browser session was no longer reachable.",@@ -466,7 +490,7 @@ class TestBuildUnexpectedErrorExitResult:def test_failed_test_scrubs_recorded_internal_tool_instruction(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)ctx.last_update_block_count = 3ctx.latest_diagnosis_repair_contract = _blocker_contract("Less than 90 seconds remain in this Copilot turn after the previous workflow run failed. "@@ -485,7 +509,7 @@ class TestBuildUnexpectedErrorExitResult:def test_aborted_test_surfaces_unvalidated_draft_with_recorded_blocker_reply(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)ctx.test_after_update_done = Truectx.last_update_block_count = 4ctx.latest_diagnosis_repair_contract = _blocker_contract(@@ -505,7 +529,7 @@ class TestBuildUnexpectedErrorExitResult:def test_browser_only_blocker_does_not_claim_tested_and_redacts_internal_details(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)ctx.test_after_update_done = Falsectx.last_update_block_count = 2ctx.latest_diagnosis_repair_contract = _blocker_contract(@@ -530,7 +554,7 @@ class TestBuildUnexpectedErrorExitResult:from skyvern.forge.sdk.copilot.tools.run_execution import _copilot_sandbox_unavailable_resultwf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)ctx.test_after_update_done = Truectx.last_update_block_count = 6ctx.latest_diagnosis_repair_contract = build_diagnosis_repair_contract(@@ -548,7 +572,7 @@ class TestBuildUnexpectedErrorExitResult:wf = MagicMock(name="wf")ctx = _ctx(last_workflow=wf,- last_workflow_yaml="version: '1.0'",+ last_workflow_yaml=_DRAFT_V1,last_test_ok=None,last_test_suspicious_success=True,)@@ -556,7 +580,7 @@ class TestBuildUnexpectedErrorExitResult:result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1assert result.proposal_disposition == "review_untested"assert result.user_response == _UNEXPECTED_ERROR_REPLY_UNVALIDATED@@ -591,7 +615,7 @@ class TestWipExitSurfacesLastGoodWithForceReviewNotUnvalidated:result = builder(ctx, global_llm_context=None)assert result.updated_workflow is ctx.last_good_workflow- assert result.workflow_yaml == "version: good"+ assert result.workflow_yaml == _DRAFT_GOODassert result.proposal_disposition == "review_tested"assert result.user_response == tested_replyassert result.cancelled is expected_cancelled@@ -654,21 +678,21 @@ workflow_definition:)assert "pbs_" not in result.user_response- def test_cancelled_total_timeout_latch_uses_force_review_not_unvalidated(self) -> None:+ def test_cancelled_total_timeout_latch_surfaces_the_draft_without_a_tested_claim(self) -> None:ctx = _overwrite_ctx(last_test_ok=None)ctx.copilot_total_timeout_exceeded = Trueresult = _build_cancelled_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is ctx.last_good_workflow- assert result.proposal_disposition == "review_tested"+ assert result.proposal_disposition == "review_untested"assert result.cancelled is False- assert result.user_response == _TIMEOUT_REPLY_TESTED+ assert result.user_response == _TIMEOUT_REPLY_UNVALIDATEDclass TestTimeoutExitNamesTimeAndDraftState:def test_untested_draft_names_time_and_that_it_is_unverified(self) -> None:- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=None)result = _build_timeout_exit_result(ctx, global_llm_context=None)@@ -677,7 +701,7 @@ class TestTimeoutExitNamesTimeAndDraftState:assert "hasn't been verified end-to-end" in result.user_responsedef test_tested_draft_names_time_and_that_it_is_tested(self) -> None:- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=True)+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=True)result = _build_timeout_exit_result(ctx, global_llm_context=None)@@ -696,7 +720,7 @@ class TestTimeoutExitNamesTimeAndDraftState:def test_untested_draft_with_recorded_failure_still_names_time(self) -> None:# The common shape: the failed test run is what spent the budget, so a# recorded failure is present on almost every real deadline expiry.- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=False)ctx.copilot_total_timeout_exceeded = Truectx.last_update_block_count = 1ctx.latest_diagnosis_repair_contract = _blocker_contract(@@ -723,7 +747,7 @@ class TestTimeoutExitNamesTimeAndDraftState:def test_max_turns_exit_keeps_the_recorded_failure_reply(self) -> None:# The deadline precedence must not leak into sibling capacity exits.- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=False)ctx.last_update_block_count = 1ctx.latest_diagnosis_repair_contract = _blocker_contract("could not convert string to float", run_status="failed"@@ -736,7 +760,7 @@ class TestTimeoutExitNamesTimeAndDraftState:def test_held_draft_shape_names_time_draft_state_and_keeps_evidence(self) -> None:# The shape both live deadline runs produced: review_untested / proposal_pending.- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=None)ctx.copilot_total_timeout_exceeded = Truectx.last_update_block_count = 1ctx.latest_diagnosis_repair_contract = _blocker_contract("Run completed", run_status="completed")@@ -751,7 +775,7 @@ class TestTimeoutExitNamesTimeAndDraftState:def test_interrupted_run_is_not_reported_as_a_failed_verification(self) -> None:# A budget-paced halt was interrupted, not disproven. The deadline copy already says the# work is unverified, so appending "the last test did not verify" would mis-attribute twice.- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=False)ctx.copilot_total_timeout_exceeded = Truectx.last_update_block_count = 3ctx.latest_diagnosis_repair_contract = _blocker_contract(@@ -830,19 +854,19 @@ class TestDeadlineTerminalHandsOverRecordedDraft:def test_failed_test_draft_survives_the_deadline(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)ctx.copilot_total_timeout_exceeded = Truectx.last_failure_category_top = Noneresult = _build_timeout_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is wf- assert result.workflow_yaml == "version: '1.0'"+ assert result.workflow_yaml == _DRAFT_V1assert result.proposal_disposition == "review_untested"def test_failed_test_reply_names_the_failure(self) -> None:wf = MagicMock(name="wf")- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)ctx.copilot_total_timeout_exceeded = Truectx.last_failure_category_top = Nonectx.last_update_block_count = 2diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 314a9c92a..bd166559d 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -138,6 +138,7 @@ def _terminal_payload(workflow_applied: bool,workflow_mutated: bool = True,workflow_attempted: bool = True,+ blocks_run_this_turn: int = 0,) -> dict[str, Any]:envelope = assemble_terminal_envelope(response_type="REPLY",@@ -150,6 +151,7 @@ def _terminal_payload(attempted="Attempted full run.",workflow_mutated=workflow_mutated,workflow_attempted=workflow_attempted,+ blocks_run_this_turn=blocks_run_this_turn,)assert envelope is not Nonereturn envelope.model_dump(mode="json")@@ -662,7 +664,7 @@ async def test_finalise_normal_turn_logs_render_decision(decisions = [entry for entry in logs if entry["event"] == "copilot_terminal_render_decision"]assert len(decisions) == 1assert decisions[0]["flag_enabled"] is flag_enabled- # This fixture's envelope is a stopped/stopped shape, so flag-on replaces.+ # This fixture's envelope is a stopped/stopped shape, so flag-on appends recorded facts.assert decisions[0]["replaced"] is flag_enabledassert decisions[0]["next_state"] == "stopped"assert decisions[0]["response_kind"] == "stopped"@@ -707,7 +709,7 @@ async def test_finalise_normal_turn_flag_on_stopped_envelope_renders_terminal_teagent_result=agent_result,)- expected = "I stopped without confirming the goal was met."+ expected = "done. 0 blocks ran this turn."response_frame = stream.send.await_args.args[0]assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)assert response_frame.message == expected