Skip to content

Automate browser based workflows with AI

Render the Copilot turn from one fact bundle instead of three verdicts (SKY-14512) (#8316)

Andrew Neilsonc3b6f780408f
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx
index 98483a294..4206facd9 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx
@@ -163,7 +163,7 @@ describe("NarrativeView — activity log", () => {
it("moves focus between the terminal rollup and detail disclosures", async () => {
render(<NarrativeView turn={repairLoopTurn()} />);
const rollup = screen.getByRole("button", {
- name: /Built and tested the workflow/,
+ name: /Built the workflow/,
});
expect(rollup.getAttribute("aria-expanded")).toBe("false");
expect(rollup.hasAttribute("aria-controls")).toBe(false);
@@ -178,7 +178,7 @@ describe("NarrativeView — activity log", () => {
fireEvent.click(collapse);
const restored = screen.getByRole("button", {
- name: /Built and tested the workflow/,
+ name: /Built the workflow/,
});
await waitFor(() => expect(document.activeElement).toBe(restored));
});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx
index f68f54ff6..e081a110a 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx
@@ -60,7 +60,7 @@ const inFlightTurn = (): TurnNarrativeState => ({
terminal: null,
});
-const HEADLINE = "Built and tested the workflow";
+const HEADLINE = "Built the workflow";
const REVIEW_HEADLINE = "Draft needs review";
const REVIEW_TESTED_HEADLINE = "Workflow ready for review";
const LONG_OUTCOME_REASON =
@@ -124,6 +124,18 @@ describe("NarrativeView collapse default", () => {
turn={{
...terminalBuildTurn(),
proposalDisposition: "review_tested",
+ // One block, run clean on current source — what backs a tested review proposal.
+ turnFacts: {
+ factsAvailable: true,
+ authoredBlockCount: 1,
+ matchingSourceBlockCount: 1,
+ evaluationState: null,
+ runId: "wr_1",
+ runCompleted: true,
+ terminalCause: null,
+ blocksRunThisTurn: 1,
+ ranCleanOnCurrentSource: true,
+ },
narrativeSummary: "Workflow ready for review.",
}}
/>,
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts
index 40b06041c..3087b1570 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts
@@ -1101,6 +1101,19 @@ const draft3 = {
summary: null,
};
+// What the backend publishes for a draft whose blocks all ran clean on current source.
+const testedFacts = {
+ factsAvailable: true,
+ authoredBlockCount: 3,
+ matchingSourceBlockCount: 3,
+ evaluationState: null,
+ runId: "wr_1",
+ runCompleted: true,
+ terminalCause: null,
+ blocksRunThisTurn: 3,
+ ranCleanOnCurrentSource: true,
+} as const;
+
describe("hydrateNarrativeFromPayload — terminal adjudication fields", () => {
it("hydrates responseKind without inventing an authoring success stamp", () => {
const turn = hydrateNarrativeFromPayload(
@@ -1348,8 +1361,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
responseType: "REPLY",
});
const summary = computeTurnSummary(turn);
- expect(summary.headline).toBe("Built and tested the workflow");
- expect(summary.accent).toBe("ok");
+ expect(summary.headline).toBe("Built the workflow");
+ expect(summary.accent).toBe("qa");
expect(summary.glyph).toBe("✓");
});
@@ -1417,7 +1430,7 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
},
);
- it("renders a verdict-authorized tested success green with drafts", () => {
+ it("names the build without claiming it was tested when no coverage facts arrive", () => {
const summary = computeTurnSummary(
buildTurn({
draft: draft3,
@@ -1425,12 +1438,12 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
responseKind: "build",
}),
);
- expect(summary.headline).toBe("Built and tested the workflow");
- expect(summary.accent).toBe("ok");
+ expect(summary.headline).toBe("Built the workflow");
+ expect(summary.accent).toBe("qa");
expect(summary.glyph).toBe("✓");
});
- it("renders a verdict-authorized edit turn green", () => {
+ it("names an edit turn without claiming a re-test when no coverage facts arrive", () => {
const summary = computeTurnSummary(
buildTurn({
draft: draft3,
@@ -1438,8 +1451,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
responseKind: "build",
}),
);
- expect(summary.headline).toBe("Applied edits and re-tested");
- expect(summary.accent).toBe("ok");
+ expect(summary.headline).toBe("Applied edits");
+ expect(summary.accent).toBe("qa");
});
it("does not infer a completed run from a build response kind alone", () => {
@@ -1471,8 +1484,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
responseKind: "build",
}),
);
- expect(summary.headline).toBe("Built and tested the workflow");
- expect(summary.accent).toBe("ok");
+ expect(summary.headline).toBe("Built the workflow");
+ expect(summary.accent).toBe("qa");
expect(summary.glyph).toBe("✓");
});
@@ -1489,8 +1502,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
responseKind: "build",
}),
);
- expect(summary.headline).toBe("Applied edits and re-tested");
- expect(summary.accent).toBe("ok");
+ expect(summary.headline).toBe("Applied edits");
+ expect(summary.accent).toBe("qa");
expect(summary.glyph).toBe("✓");
});
@@ -1509,7 +1522,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
proposalDisposition: "review_tested",
responseKind: "build",
}),
- "Workflow ready for review",
+ // No facts back the tested framing, so the card states what it built and no more.
+ "Built the workflow",
],
])(
"keeps stopped or review-required states when there is no clean completed run (%#)",
@@ -1525,6 +1539,7 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
const askTurn = buildTurn({
draft: draft3,
proposalDisposition: "review_tested",
+ turnFacts: testedFacts,
responseKind: "clarify",
responseType: "ASK_QUESTION",
terminalMessage: "Is that output format okay?",
@@ -1568,8 +1583,8 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
it("legacy payload without adjudication renders via the unchanged inference chain", () => {
const turn = hydrateNarrativeFromPayload(reproClarifyPayload())!;
const summary = computeTurnSummary(turn);
- expect(summary.headline).toBe("Built and tested the workflow");
- expect(summary.accent).toBe("ok");
+ expect(summary.headline).toBe("Built the workflow");
+ expect(summary.accent).toBe("qa");
expect(summary.stats).toEqual(["15:02", "3 blocks ran", "3 new"]);
});
});
@@ -1593,6 +1608,7 @@ describe("computeTurnSummary — disposition-first reorder", () => {
responseKind: "clarify",
draft: draft3,
proposalDisposition: "review_tested",
+ turnFacts: testedFacts,
}),
);
expect(summary.headline).toBe("Workflow ready for review");
@@ -1692,8 +1708,8 @@ describe("hydrateHistoryNarrative — persisted turn_outcome graft", () => {
})!;
expect(turn.responseKind).toBe("build");
const summary = computeTurnSummary(turn);
- expect(summary.headline).toBe("Built and tested the workflow");
- expect(summary.accent).toBe("ok");
+ expect(summary.headline).toBe("Built the workflow");
+ expect(summary.accent).toBe("qa");
});
it("tolerates missing or unknown turn_outcome", () => {
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx
index dfee08197..f55cebd9b 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx
@@ -8,8 +8,11 @@ import {
} from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import bundles from "./narrativeState.turnFacts.fixture.json";
import type { WorkflowCopilotStreamResponseUpdate } from "./workflowCopilotTypes";
+const editOneOfTwoBundle = bundles["different-source-edit-one-of-two"];
+
type StreamBody = {
message: string;
keep_pending_proposal?: boolean;
@@ -466,6 +469,8 @@ describe("WorkflowCopilotChat — g2 review gate", () => {
);
expect(screen.getByRole("button", { name: "Reject" })).toBeTruthy();
expect(screen.getByText("Proposed changes")).toBeTruthy();
+ // The gate itself still appears live (SKY-14099); with no turn behind it there are
+ // no facts to project, so it carries no verdict pill either way.
expect(screen.queryByText("Tested")).toBeNull();
});
@@ -497,4 +502,30 @@ describe("WorkflowCopilotChat — g2 review gate", () => {
).toBeTruthy();
expect(screen.queryByRole("button", { name: "Reject" })).toBeNull();
});
+
+ it("surfaces per-block coverage tags through the mounted chat, not only in an isolated card", async () => {
+ await renderChat();
+
+ await submit("edit one block after it passed.");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+ await act(async () => {
+ streamCalls[0]!.onMessage(
+ proposalResponse("Draft needs review.", {
+ narrative_payload: {
+ ...editOneOfTwoBundle,
+ turnId: "turn-1",
+ },
+ }),
+ );
+ streamCalls[0]!.resolve();
+ });
+
+ await waitFor(() =>
+ expect(screen.getByText("Different source")).toBeTruthy(),
+ );
+ expect(
+ screen.getByText(/3 blocks authored · 0\/3 ran on current source/),
+ ).toBeTruthy();
+ expect(screen.queryByText("Built and tested the workflow")).toBeNull();
+ });
});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.test.tsx
index 618dff394..14b95d548 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.test.tsx
@@ -52,25 +52,61 @@ const turn = (
...overrides,
});
+const coveredFacts = {
+ factsAvailable: true,
+ authoredBlockCount: 1,
+ matchingSourceBlockCount: 1,
+ evaluationState: null,
+ runId: "wr_1",
+ runCompleted: true,
+ terminalCause: null,
+ blocksRunThisTurn: 1,
+ ranCleanOnCurrentSource: true,
+} as const;
+
describe("getReviewGateVerdict", () => {
- it("treats review_tested as tested", () => {
+ it("treats review_tested as tested when the turn's coverage facts back it", () => {
expect(
getReviewGateVerdict(
- turn({ proposalDisposition: "review_tested" }),
+ turn({ proposalDisposition: "review_tested", turnFacts: coveredFacts }),
null,
),
).toBe("tested");
});
- it("treats auto_applicable as tested — the backend only assigns this disposition to verified changes", () => {
+ it("treats auto_applicable as tested when the turn's coverage facts back it", () => {
expect(
getReviewGateVerdict(
- turn({ proposalDisposition: "auto_applicable" }),
+ turn({
+ proposalDisposition: "auto_applicable",
+ turnFacts: coveredFacts,
+ }),
null,
),
).toBe("tested");
});
+ it.each(["review_tested", "auto_applicable"] as const)(
+ "refuses a tested pill for %s when coverage is partial",
+ (proposalDisposition) => {
+ expect(
+ getReviewGateVerdict(
+ turn({
+ proposalDisposition,
+ // Partial coverage is decided by the backend, which publishes both the
+ // count and the verdict it implies.
+ turnFacts: {
+ ...coveredFacts,
+ matchingSourceBlockCount: 0,
+ ranCleanOnCurrentSource: false,
+ },
+ }),
+ null,
+ ),
+ ).toBe("untested");
+ },
+ );
+
it("treats review_untested as untested", () => {
expect(
getReviewGateVerdict(
@@ -89,11 +125,17 @@ describe("getReviewGateVerdict", () => {
).toBe("untested");
});
- it("has no verdict for a legacy proposal carrying no evidence either way", () => {
+ it("refuses a tested pill for a legacy proposal that carries no coverage facts", () => {
const legacyProposal = {} as unknown as WorkflowApiResponse;
expect(
getReviewGateVerdict(turn({ proposalDisposition: null }), legacyProposal),
- ).toBe(null);
+ ).toBe("untested");
+ expect(
+ getReviewGateVerdict(
+ turn({ proposalDisposition: null, turnFacts: coveredFacts }),
+ legacyProposal,
+ ),
+ ).toBe("tested");
});
it("returns null with no disposition and no proposal", () => {
@@ -397,3 +439,107 @@ describe("ReviewGateCard — recorded review projection", () => {
expect(malformed?.review).toBeNull();
});
});
+
+describe("ReviewGateCard — source coverage", () => {
+ const noop = () => {};
+
+ it("separates a block run against different source from one never run", () => {
+ render(
+ <ReviewGateCard
+ turn={turn({
+ review: {
+ blocks: [
+ {
+ label: "sign_in",
+ blockType: "code",
+ change: "changed",
+ neverTested: true,
+ coverage: "different_source",
+ },
+ {
+ label: "read_metric",
+ blockType: "code",
+ change: "added",
+ neverTested: true,
+ coverage: "never_run",
+ },
+ ],
+ duplicateWrites: [],
+ },
+ })}
+ pending
+ verdict="untested"
+ actionsEnabled
+ onAccept={noop}
+ onAlwaysAccept={noop}
+ onReject={noop}
+ onReview={noop}
+ />,
+ );
+
+ expect(screen.getByText("Different source")).not.toBeNull();
+ expect(screen.getByText("Never tested")).not.toBeNull();
+ });
+
+ it("says a renamed block is untested under this name, not that it failed", () => {
+ render(
+ <ReviewGateCard
+ turn={turn({
+ review: {
+ blocks: [
+ {
+ label: "sign_in_v2",
+ blockType: "code",
+ change: "added",
+ neverTested: true,
+ coverage: "unknown",
+ },
+ {
+ label: "read_metric",
+ blockType: "code",
+ change: "unchanged",
+ neverTested: false,
+ coverage: "current_source",
+ },
+ ],
+ duplicateWrites: [],
+ },
+ })}
+ pending
+ verdict="untested"
+ actionsEnabled
+ onAccept={noop}
+ onAlwaysAccept={noop}
+ onReject={noop}
+ onReview={noop}
+ />,
+ );
+
+ expect(screen.getByText("Not tested under this name")).not.toBeNull();
+ expect(screen.queryByText("Never tested")).toBeNull();
+ expect(screen.queryByText(/failed/i)).toBeNull();
+ expect(screen.queryByText(/invalid/i)).toBeNull();
+ });
+
+ it("stays non-committal on a proposal whose turn published no facts", () => {
+ expect(
+ getReviewGateVerdict(
+ turn({
+ proposalDisposition: "auto_applicable",
+ turnFacts: {
+ factsAvailable: false,
+ authoredBlockCount: null,
+ matchingSourceBlockCount: null,
+ evaluationState: null,
+ runId: null,
+ runCompleted: null,
+ terminalCause: null,
+ blocksRunThisTurn: null,
+ ranCleanOnCurrentSource: false,
+ },
+ }),
+ null,
+ ),
+ ).toBe("untested");
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.tsx
index 595d92223..f8ab3ca32 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/ReviewGateCard.tsx
@@ -4,7 +4,7 @@ import { WorkflowApiResponse } from "@/routes/workflows/types/workflowTypes";
import { humanizeBlockLabel } from "../blockLabel";
import { everyTestBlockExecuted, hasFailedTestBlock } from "../copilotPhases";
-import { TurnNarrativeState } from "../narrativeState";
+import { TurnNarrativeState, ranCleanOnCurrentSource } from "../narrativeState";
import { getDiffCardTitle } from "./diffCardTitle";
export type ReviewGateVerdict = "tested" | "untested" | null;
@@ -25,17 +25,32 @@ export function getReviewGateVerdict(
if (turn && hasFailedTestBlock(turn)) {
return "untested";
}
+ // A tested claim needs the turn's own facts behind it, so an absent bundle, partial
+ // coverage or a halted turn reads as untested rather than falling through to green.
+ const covered = ranCleanOnCurrentSource(turn?.turnFacts ?? null);
if (
- turn?.proposalDisposition === "review_tested" ||
- turn?.proposalDisposition === "auto_applicable"
+ covered &&
+ (turn?.proposalDisposition === "review_tested" ||
+ turn?.proposalDisposition === "auto_applicable")
) {
return "tested";
}
if (turn?.proposalDisposition) {
return "untested";
}
- const legacy = proposedWorkflow as LegacyProposedWorkflow | null;
- return legacy?._copilot_unvalidated ? "untested" : null;
+ if (!proposedWorkflow) {
+ return null;
+ }
+ const legacy = proposedWorkflow as LegacyProposedWorkflow;
+ if (legacy._copilot_unvalidated) {
+ return "untested";
+ }
+ // With no turn there are no facts to project from, so a still-pending gate stays
+ // silent rather than inventing either verdict.
+ if (!turn) {
+ return null;
+ }
+ return covered ? "tested" : "untested";
}
const VERDICT_PILL_CLASSES: Record<"tested" | "untested", string> = {
@@ -169,7 +184,16 @@ export function ReviewGateCard({
{section.prefix ? `${section.prefix} ` : ""}
{humanizeBlockLabel(block.label)}
</span>
- {block.neverTested ? (
+ {block.coverage === "different_source" ? (
+ <span className="rounded-full bg-sky-500/10 px-1.5 py-0.5 text-[10px] font-medium text-sky-700 dark:text-sky-300">
+ Different source
+ </span>
+ ) : block.coverage === "unknown" ? (
+ <span className="rounded-full bg-sky-500/10 px-1.5 py-0.5 text-[10px] font-medium text-sky-700 dark:text-sky-300">
+ Not tested under this name
+ </span>
+ ) : block.coverage === "never_run" ||
+ (block.coverage === undefined && block.neverTested) ? (
<span className="rounded-full bg-sky-500/10 px-1.5 py-0.5 text-[10px] font-medium text-sky-700 dark:text-sky-300">
Never tested
</span>
diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts
index e98a488f4..3508cee6b 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts
@@ -314,6 +314,83 @@ describe("derivePhases — terminal", () => {
expect(phase(rows, "test").stub).toBe("1 block · failed");
});
+ it("a deadline halt after a clean run stops the rail instead of failing it", () => {
+ const t = turn({
+ terminal: "error",
+ designEnded: true,
+ draft: { blockCount: 1, blockLabels: ["block_1"], summary: null },
+ blocks: [block()],
+ turnFacts: {
+ factsAvailable: true,
+ authoredBlockCount: 1,
+ matchingSourceBlockCount: 1,
+ evaluationState: "not_evaluated",
+ runId: "wr_1",
+ runCompleted: true,
+ terminalCause: "deadline_expired",
+ blocksRunThisTurn: 1,
+ ranCleanOnCurrentSource: false,
+ },
+ });
+ const rows = derivePhases(t);
+ expect(phase(rows, "test").status).toBe("stopped");
+ expect(phase(rows, "test").stub).toBe("1 block · time limit");
+ expect(phase(rows, "done").status).toBe("stopped");
+ expect(rows.map((r) => r.status)).not.toContain("fail");
+ });
+
+ it("a deadline halt with no block receipts never paints a green Test row", () => {
+ const t = turn({
+ terminal: "error",
+ designEnded: true,
+ draft: { blockCount: 1, blockLabels: ["block_1"], summary: null },
+ designActivity: [
+ entry({
+ id: "1",
+ kind: "tool_call",
+ toolName: "update_and_run_blocks",
+ }),
+ ],
+ turnFacts: {
+ factsAvailable: true,
+ authoredBlockCount: 1,
+ matchingSourceBlockCount: 0,
+ evaluationState: null,
+ runId: null,
+ runCompleted: null,
+ terminalCause: "deadline_expired",
+ blocksRunThisTurn: 0,
+ ranCleanOnCurrentSource: false,
+ },
+ });
+ const rows = derivePhases(t);
+ expect(phase(rows, "test").status).toBe("stopped");
+ expect(phase(rows, "test").stub).toBe("time limit");
+ });
+
+ it("a deadline halt with a failed block keeps the failure row", () => {
+ const t = turn({
+ terminal: "error",
+ designEnded: true,
+ draft: { blockCount: 1, blockLabels: ["block_1"], summary: null },
+ blocks: [block({ state: "failed" })],
+ turnFacts: {
+ factsAvailable: true,
+ authoredBlockCount: 1,
+ matchingSourceBlockCount: 1,
+ evaluationState: "not_demonstrated",
+ runId: "wr_1",
+ runCompleted: false,
+ terminalCause: "deadline_expired",
+ blocksRunThisTurn: 1,
+ ranCleanOnCurrentSource: false,
+ },
+ });
+ const rows = derivePhases(t);
+ expect(phase(rows, "test").status).toBe("fail");
+ expect(phase(rows, "test").stub).toBe("1 block · failed");
+ });
+
it("draft-only turn (update_workflow, no run): Test renders notrun, Done still succeeds", () => {
const t = turn({
terminal: "response",
diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts
index b5f82b258..88ae36fd9 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts
@@ -13,6 +13,7 @@ import {
condenseActivityEntries,
formatElapsed,
hasPendingToolCall,
+ isDeadlineHalt,
isInterimOutcome,
latestBlocksByLabel,
parseUtcIsoMs,
@@ -235,9 +236,10 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {
const isTerminal = turn.terminal !== null;
const isCancelled = turn.cancelled === true;
- // A cancelled turn also lands on terminal "error" — that error is the stop
- // itself, so it must not paint the rail red.
- const isError = turn.terminal === "error" && !isCancelled;
+ const deadlineHalt = isDeadlineHalt(turn);
+ // A cancelled turn and a deadline halt both land on terminal "error" — that
+ // error is the stop itself, so neither may paint the rail red.
+ const isError = turn.terminal === "error" && !isCancelled && !deadlineHalt;
const anyFailed = hasFailedTestBlock(turn);
const anyStopped = latestBlocks.some((b) => b.state === "stopped");
const anyNotDemonstrated = latestBlocks.some(
@@ -272,7 +274,7 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {
if (!reached(id)) return id === "test" ? "notrun" : "pending";
if (chainActive === id) {
if (isError) return "fail";
- if (isCancelled) return "stopped";
+ if (isCancelled || deadlineHalt) return "stopped";
}
return "done";
}
@@ -318,9 +320,10 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {
if (status === "fail")
return `${pluralize(latestBlocks.length, "block")} · failed`;
if (status === "stopped") {
+ const halt = deadlineHalt ? "time limit" : "stopped";
return latestBlocks.length > 0
- ? `${pluralize(latestBlocks.length, "block")} · stopped`
- : "stopped";
+ ? `${pluralize(latestBlocks.length, "block")} · ${halt}`
+ : halt;
}
if (anyNotDemonstrated) {
// Surface how many test runs the loop actually made — otherwise a
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts
index 93eda9c7f..509539a0f 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts
@@ -1,12 +1,16 @@
import { describe, expect, it } from "vitest";
+import bundles from "./narrativeState.turnFacts.fixture.json";
+import { getReviewGateVerdict } from "./cards/ReviewGateCard";
import {
EMPTY_NARRATIVE,
TurnNarrativeState,
applyNarrativeEvent,
+ computeTurnSummary,
hydrateNarrativeFromPayload,
isBlockOk,
notConfirmedOutcome,
+ ranCleanOnCurrentSource,
} from "./narrativeState";
import {
WorkflowCopilotBlockProgressUpdate,
@@ -485,3 +489,351 @@ describe("notConfirmedOutcome — envelope-first", () => {
});
});
});
+
+// A recorded terminal-surface packet: one authored block, one clean run against
+// the staged source.
+const CLEAN_RECORDED_TURN = "surface-20260820T082523349904";
+
+describe("canonical turn facts — card, pill and prose over one bundle", () => {
+ const surfaces = (leg: keyof typeof bundles) => {
+ const turn = hydrateNarrativeFromPayload(
+ bundles[leg] as unknown as Record<string, unknown>,
+ );
+ if (!turn) throw new Error(`fixture ${leg} did not hydrate`);
+ return {
+ turn,
+ card: computeTurnSummary(turn),
+ pill: getReviewGateVerdict(turn, null),
+ prose: turn.terminalMessage ?? "",
+ };
+ };
+
+ it("states coverage and evaluation as independent facts when one of three blocks ran", () => {
+ const { card, pill, prose } = surfaces("partial-coverage");
+
+ expect(card.headline).not.toBe("Built and tested the workflow");
+ expect(card.stats).toContain("3 blocks authored");
+ expect(card.stats).toContain("1/3 ran on current source");
+ expect(card.stats).toContain("outcome not evaluated");
+ expect(card.stats.join(" ")).not.toContain("wr_");
+ expect(card.accent).not.toBe("ok");
+ expect(card.isFail).toBe(false);
+ expect(pill).toBe("untested");
+ expect(prose).not.toContain("tested draft");
+ });
+
+ it("reports a deadline halt without a failure row or a tested pill", () => {
+ const { card, pill, prose } = surfaces("deadline-after-one-clean-run");
+
+ expect(card.isFail).toBe(false);
+ expect(card.accent).not.toBe("fail");
+ expect(card.stats).toContain("time limit reached");
+ expect(pill).toBe("untested");
+ expect(prose).toContain("1 block ran this turn");
+ expect(prose).toContain("outcome was not evaluated");
+ expect(prose).toContain("reached its time limit");
+ });
+
+ it("marks an edited block as run against different source, never as failed", () => {
+ const { turn, card, pill } = surfaces("different-source-edit-one-of-two");
+
+ expect(
+ turn.review?.blocks.map((block) => [block.label, block.coverage]),
+ ).toEqual([
+ ["extract_github_star_count", "different_source"],
+ ["append_star_count_to_sheet", "never_run"],
+ ["append_star_count_to_sales_marketing", "never_run"],
+ ]);
+ expect(card.isFail).toBe(false);
+ expect(pill).toBe("untested");
+ });
+
+ it("excludes a removed block from the authored count", () => {
+ const { card } = surfaces("mixed-edit-and-delete");
+
+ expect(card.stats).toContain("2 blocks authored");
+ expect(card.stats).toContain("1/2 ran on current source");
+ });
+
+ it("names clean lifecycle plus full current-source coverage on the satisfaction path", () => {
+ const { card, pill, prose } = surfaces("satisfaction");
+
+ expect(card.stats).toContain("3 blocks authored");
+ expect(card.stats).toContain("3/3 ran on current source");
+ expect(card.stats).toContain("run completed");
+ expect(card.headline).toBe("Every block ran clean on the current draft");
+ expect(card.accent).toBe("ok");
+ expect(card.isFail).toBe(false);
+ expect(pill).toBe("tested");
+ for (const claim of [/verified/i, /demonstrated/i, /goal/i, /confirm/i]) {
+ expect(card.headline).not.toMatch(claim);
+ }
+ expect(prose).not.toMatch(/verified|demonstrated/i);
+ });
+
+ it("states its counts without a completion claim when no run is anchored to the turn", () => {
+ const { turn, card } = surfaces(CLEAN_RECORDED_TURN);
+
+ expect(turn.blocks.map((block) => block.state)).toEqual(["completed"]);
+ expect(turn.turnFacts?.runCompleted).toBeNull();
+ expect(card.stats).toContain("1 block authored");
+ expect(card.stats).toContain("1/1 ran on current source");
+ expect(card.stats).not.toContain("run completed");
+ expect(card.stats).not.toContain("run did not complete");
+ expect(card.isFail).toBe(false);
+ });
+
+ it("withholds the satisfaction headline when the recorded run did not complete", () => {
+ const recorded = bundles[CLEAN_RECORDED_TURN] as unknown as {
+ turnFacts: Record<string, unknown>;
+ };
+ const turn = hydrateNarrativeFromPayload({
+ ...(bundles[CLEAN_RECORDED_TURN] as unknown as Record<string, unknown>),
+ turnFacts: {
+ ...recorded.turnFacts,
+ runCompleted: false,
+ ranCleanOnCurrentSource: false,
+ },
+ });
+ if (!turn) throw new Error("fixture did not hydrate");
+ const card = computeTurnSummary(turn);
+
+ expect(card.stats).toContain("run did not complete");
+ expect(card.headline).not.toBe(
+ "Every block ran clean on the current draft",
+ );
+ expect(getReviewGateVerdict(turn, null)).toBe("untested");
+ expect(card.isFail).toBe(false);
+ });
+
+ it("withholds success from a fully covered turn whose recorded run failed", () => {
+ const recorded = bundles[CLEAN_RECORDED_TURN] as unknown as {
+ blocks: Array<Record<string, unknown>>;
+ };
+ const turn = hydrateNarrativeFromPayload({
+ ...(bundles[CLEAN_RECORDED_TURN] as unknown as Record<string, unknown>),
+ blocks: recorded.blocks.map((block) => ({ ...block, state: "failed" })),
+ });
+ if (!turn) throw new Error("fixture did not hydrate");
+ const card = computeTurnSummary(turn);
+
+ expect(card.headline).not.toBe(
+ "Every block ran clean on the current draft",
+ );
+ expect(card.isFail).toBe(true);
+ expect(card.accent).toBe("fail");
+ });
+
+ it("says a renamed block is untested under this name across card and pill", () => {
+ const { turn, card, pill } = surfaces("renamed-block");
+
+ expect(
+ turn.review?.blocks.map((block) => [block.label, block.coverage]),
+ ).toEqual([
+ ["extract_github_star_count_v2", "unknown"],
+ ["append_star_count_to_sheet", "current_source"],
+ ["append_star_count_to_sales_marketing", "current_source"],
+ ["extract_github_star_count", undefined],
+ ]);
+ expect(card.stats).toContain("2/3 ran on current source");
+ expect(card.isFail).toBe(false);
+ expect(pill).toBe("untested");
+ });
+
+ it("settles a block still running at the deadline as stopped, not failed", () => {
+ const turn = hydrateNarrativeFromPayload({
+ ...(bundles["deadline-after-one-clean-run"] as unknown as Record<
+ string,
+ unknown
+ >),
+ blocks: [
+ {
+ workflowRunBlockId: "wrb_1",
+ label: "append_star_count_to_sheet",
+ blockType: "code",
+ state: "running",
+ lastSeenIteration: 0,
+ activity: [],
+ },
+ ],
+ });
+ if (!turn) throw new Error("fixture did not hydrate");
+ const card = computeTurnSummary(turn);
+
+ expect(turn.blocks.map((block) => block.state)).toEqual(["stopped"]);
+ expect(card.isFail).toBe(false);
+ expect(card.accent).not.toBe("fail");
+ });
+
+ it("does not call a deadline halt that delivered nothing a completed run", () => {
+ const turn = hydrateNarrativeFromPayload({
+ ...(bundles["deadline-after-one-clean-run"] as unknown as Record<
+ string,
+ unknown
+ >),
+ proposalDisposition: "no_proposal",
+ draft: null,
+ terminal: "error",
+ blocks: [
+ {
+ workflowRunBlockId: "wrb_1",
+ label: "append_star_count_to_sheet",
+ blockType: "code",
+ state: "completed",
+ lastSeenIteration: 0,
+ activity: [],
+ },
+ ],
+ });
+ if (!turn) throw new Error("fixture did not hydrate");
+ const card = computeTurnSummary(turn);
+
+ // Suppressing the failure treatment for a deadline must not promote the turn
+ // into a success: it ran out of time and handed nothing back.
+ expect(card.isFail).toBe(false);
+ expect(card.accent).not.toBe("ok");
+ expect(card.headline).not.toMatch(/completed the run/i);
+ });
+
+ it("keeps a failed block a failure even when the deadline expired", () => {
+ const turn = hydrateNarrativeFromPayload({
+ ...(bundles["deadline-after-one-clean-run"] as unknown as Record<
+ string,
+ unknown
+ >),
+ blocks: [
+ {
+ workflowRunBlockId: "wrb_1",
+ label: "append_star_count_to_sheet",
+ blockType: "code",
+ state: "failed",
+ lastSeenIteration: 0,
+ activity: [],
+ },
+ ],
+ });
+ if (!turn) throw new Error("fixture did not hydrate");
+ const card = computeTurnSummary(turn);
+
+ expect(turn.turnFacts?.terminalCause).toBe("deadline_expired");
+ expect(card.isFail).toBe(true);
+ expect(card.stats).toContain("1 failed");
+ });
+
+ it("makes no tested claim for a turn that published no facts or coverage", () => {
+ const legacy = bundles[CLEAN_RECORDED_TURN] as unknown as Record<
+ string,
+ unknown
+ >;
+ const review = legacy.review as { blocks: Array<Record<string, unknown>> };
+ const turn = hydrateNarrativeFromPayload({
+ ...legacy,
+ turnFacts: undefined,
+ review: {
+ ...review,
+ blocks: review.blocks.map((block) => {
+ const withoutCoverage = { ...block };
+ delete withoutCoverage.coverage;
+ return withoutCoverage;
+ }),
+ },
+ });
+ if (!turn) throw new Error("fixture did not hydrate");
+ const card = computeTurnSummary(turn);
+
+ expect(turn.turnFacts).toBeNull();
+ expect(turn.review?.blocks[0]?.coverage).toBeUndefined();
+ expect(card.headline).not.toMatch(/tested/i);
+ expect(card.accent).not.toBe("ok");
+ expect(card.stats.join(" ")).not.toMatch(/ran on current source|authored/);
+ });
+
+ it("keeps the review when the backend sends a coverage value it does not know", () => {
+ const known = bundles[CLEAN_RECORDED_TURN] as unknown as Record<
+ string,
+ unknown
+ >;
+ const review = known.review as { blocks: Array<Record<string, unknown>> };
+ const turn = hydrateNarrativeFromPayload({
+ ...known,
+ review: {
+ ...review,
+ blocks: review.blocks.map((block) => ({
+ ...block,
+ coverage: "some_future_value",
+ })),
+ },
+ });
+ if (!turn) throw new Error("fixture did not hydrate");
+
+ expect(turn.review?.blocks).toHaveLength(1);
+ expect(turn.review?.blocks[0]?.coverage).toBeUndefined();
+ });
+
+ it("refuses a tested pill for a pre-fix payload that carries no fact bundle", () => {
+ const turn = hydrateNarrativeFromPayload(
+ bundles["facts-absent"] as unknown as Record<string, unknown>,
+ );
+ if (!turn) throw new Error("fixture did not hydrate");
+ const card = computeTurnSummary(turn);
+
+ expect(turn.turnFacts).toBeNull();
+ expect(turn.proposalDisposition).toBe("review_tested");
+ expect(getReviewGateVerdict(turn, null)).toBe("untested");
+ // The tested branch is named "Workflow ready for review", which contains no literal
+ // "tested" — assert the branch, not the substring.
+ expect(card.headline).not.toBe("Workflow ready for review");
+ expect(card.headline).toBe("Built the workflow");
+ expect(card.glyph).not.toBe("\u2713");
+ expect(card.accent).not.toBe("ok");
+ });
+
+ it("keeps card and pill telling one story when the envelope is missing", () => {
+ const full = bundles["full-coverage"] as unknown as Record<string, unknown>;
+ const facts = full.turnFacts as Record<string, unknown>;
+ const turn = hydrateNarrativeFromPayload({
+ ...full,
+ turnFacts: {
+ ...facts,
+ evaluationState: null,
+ runId: null,
+ runCompleted: null,
+ terminalCause: null,
+ blocksRunThisTurn: null,
+ },
+ });
+ if (!turn) throw new Error("fixture did not hydrate");
+ const card = computeTurnSummary(turn);
+
+ expect(turn.turnFacts?.factsAvailable).toBe(true);
+ expect(turn.turnFacts?.authoredBlockCount).toBe(facts.authoredBlockCount);
+ expect(card.stats).toContain(
+ `${facts.matchingSourceBlockCount}/${facts.authoredBlockCount} ran on current source`,
+ );
+ expect(card.stats).not.toContain("run completed");
+ expect(card.isFail).toBe(false);
+ expect(ranCleanOnCurrentSource(turn.turnFacts)).toBe(true);
+ expect(getReviewGateVerdict(turn, null)).toBe("tested");
+ });
+
+ it("stays non-committal when the turn published no facts", () => {
+ const turn = hydrateNarrativeFromPayload({
+ ...(bundles["full-coverage"] as unknown as Record<string, unknown>),
+ review: undefined,
+ proposalDisposition: "auto_applicable",
+ turnFacts: {
+ factsAvailable: false,
+ evaluationState: null,
+ runId: null,
+ terminalCause: null,
+ blocksRunThisTurn: null,
+ },
+ });
+ if (!turn) throw new Error("fixture did not hydrate");
+
+ expect(getReviewGateVerdict(turn, null)).toBe("untested");
+ expect(computeTurnSummary(turn).headline).not.toBe(
+ "Every block ran clean on the current draft",
+ );
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
index 20c29ba47..231a1ac6f 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
@@ -156,6 +156,37 @@ export interface TerminalEnvelopeFacts {
}
export type ReviewChange = "added" | "changed" | "unchanged" | "removed";
+export type BlockCoverage =
+ | "current_source"
+ | "different_source"
+ | "never_run"
+ | "unknown";
+
+const BLOCK_COVERAGE: readonly BlockCoverage[] = [
+ "current_source",
+ "different_source",
+ "never_run",
+ "unknown",
+];
+
+// The canonical per-turn facts the backend publishes; card, pill and prose each
+// render a projection of this bundle rather than deriving their own verdict.
+export interface TurnFacts {
+ factsAvailable: boolean;
+ authoredBlockCount: number | null;
+ matchingSourceBlockCount: number | null;
+ evaluationState: BlockOutcome | null;
+ runId: string | null;
+ runCompleted: boolean | null;
+ terminalCause: string | null;
+ blocksRunThisTurn: number | null;
+ ranCleanOnCurrentSource: boolean;
+}
+
+// The backend decides the tested claim and publishes it here; no surface recomputes it.
+export function ranCleanOnCurrentSource(facts: TurnFacts | null): boolean {
+ return facts?.ranCleanOnCurrentSource === true;
+}
export interface ReviewProjection {
blocks: Array<{
@@ -163,6 +194,7 @@ export interface ReviewProjection {
blockType: string;
change: ReviewChange;
neverTested?: boolean;
+ coverage?: BlockCoverage;
}>;
duplicateWrites: Array<{
blockType: string;
@@ -335,6 +367,7 @@ export interface TurnNarrativeState {
connectedAccountChoices: ConnectedAccountChoice[];
googleConnectionNotices: GoogleConnectionNotice[];
review: ReviewProjection | null;
+ turnFacts: TurnFacts | null;
}
export interface GoogleConnectionNotice {
@@ -373,6 +406,7 @@ export const EMPTY_NARRATIVE: TurnNarrativeState = Object.freeze({
connectedAccountChoices: [],
googleConnectionNotices: [],
review: null,
+ turnFacts: null,
}) as TurnNarrativeState;
// Caps to keep long-running narrations from unbounded growth (and to keep
@@ -1404,6 +1438,34 @@ function normalizeActivityEntries(raw: unknown): ActivityEntry[] {
return out;
}
+function parseTurnFacts(raw: unknown): TurnFacts | null {
+ if (!raw || typeof raw !== "object") return null;
+ const value = raw as Record<string, unknown>;
+ if (typeof value.factsAvailable !== "boolean") return null;
+ const count = (key: string): number | null =>
+ typeof value[key] === "number" ? (value[key] as number) : null;
+ const text = (key: string): string | null =>
+ typeof value[key] === "string" ? (value[key] as string) : null;
+ const evaluationState = value.evaluationState;
+ return {
+ factsAvailable: value.factsAvailable,
+ authoredBlockCount: count("authoredBlockCount"),
+ matchingSourceBlockCount: count("matchingSourceBlockCount"),
+ evaluationState:
+ evaluationState === "demonstrated" ||
+ evaluationState === "not_demonstrated" ||
+ evaluationState === "not_evaluated"
+ ? evaluationState
+ : null,
+ runId: text("runId"),
+ runCompleted:
+ typeof value.runCompleted === "boolean" ? value.runCompleted : null,
+ terminalCause: text("terminalCause"),
+ blocksRunThisTurn: count("blocksRunThisTurn"),
+ ranCleanOnCurrentSource: value.ranCleanOnCurrentSource === true,
+ };
+}
+
function parseReviewProjection(raw: unknown): ReviewProjection | null {
if (!raw || typeof raw !== "object") return null;
const value = raw as Record<string, unknown>;
@@ -1426,11 +1488,14 @@ function parseReviewProjection(raw: unknown): ReviewProjection | null {
) {
return null;
}
+ // An unrecognised coverage value drops off this block rather than voiding the
+ // whole projection, so a newer backend vocabulary cannot blank the review.
blocks.push({
label: block.label,
blockType: block.blockType,
change: block.change,
neverTested: block.neverTested,
+ coverage: BLOCK_COVERAGE.find((known) => known === block.coverage),
});
}
const duplicateWrites: ReviewProjection["duplicateWrites"] = [];
@@ -1567,12 +1632,16 @@ export function hydrateNarrativeFromPayload(
const endedAtIso =
typeof payload.endedAt === "string" ? (payload.endedAt as string) : null;
const cancelled = payload.cancelled === true;
+ const turnFacts = parseTurnFacts(payload.turnFacts);
+ // A deadline cuts a running block off; that is a stop, so it must not be swept
+ // to failed and then read back as the turn having failed.
+ const halted = cancelled || turnFacts?.terminalCause === "deadline_expired";
const sweptBlocks: BlockState[] = terminal
? blocks.map((b) =>
b.state === "running"
? {
...b,
- state: cancelled
+ state: halted
? "stopped"
: terminal === "error"
? "failed"
@@ -1638,6 +1707,7 @@ export function hydrateNarrativeFromPayload(
payload.googleConnectionNotices,
),
review: parseReviewProjection(payload.review),
+ turnFacts,
};
}
@@ -1809,7 +1879,6 @@ function adjudicatedSummaryParts(
needsTestedProposalReview: boolean;
hasEdited: boolean;
hasDrafts: boolean;
- hasCleanCompletedBuild: boolean;
},
): AdjudicatedParts | null {
if (turn.responseKind === null) return null;
@@ -1856,21 +1925,35 @@ function adjudicatedSummaryParts(
return null;
}
+// A deadline exit stamps terminal="error" for budget reasons; that is a halt,
+// not a block failure, so no surface may give it failure's treatment.
+export function isDeadlineHalt(turn: TurnNarrativeState): boolean {
+ return (
+ turn.turnFacts?.terminalCause === "deadline_expired" &&
+ !latestBlocksByLabel(turn.blocks).some((b) => b.state === "failed")
+ );
+}
+
export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {
const rollupBlocks = latestBlocksByLabel(turn.blocks);
// A cancelled turn's terminal is "error" purely because the user stopped it,
// so that arm must not brand their own stop a failure.
+ const anyBlockFailed = rollupBlocks.some((b) => b.state === "failed");
+ const facts = turn.turnFacts;
+ const deadlineHalt = isDeadlineHalt(turn);
const isFail =
!turn.cancelled &&
- (turn.terminal === "error" ||
- rollupBlocks.some((b) => b.state === "failed"));
+ !deadlineHalt &&
+ (turn.terminal === "error" || anyBlockFailed);
// A stop halts the turn without failing it — a user cancel, or a budget halt
// that cancels a block mid-run. It suppresses a success verdict exactly like
// a failure, but never wears failure's treatment. `turn.cancelled` is load
// bearing on its own: a stop during the thinking phase, or on a QA turn,
// touches no block at all and would otherwise read as a clean success.
const isStopped =
- turn.cancelled || rollupBlocks.some((b) => b.state === "stopped");
+ turn.cancelled ||
+ deadlineHalt ||
+ rollupBlocks.some((b) => b.state === "stopped");
const mode = effectiveMode(turn);
const needsInput = asksUserForInput(turn);
const isQA =
@@ -1879,15 +1962,26 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {
mode === "clarify" ||
mode === "refuse";
const hasDrafts = (turn.draft?.blockCount ?? 0) > 0;
+ const cleanFullCurrentSourceCoverage = ranCleanOnCurrentSource(facts);
const needsUntestedProposalReview =
hasDrafts && turn.proposalDisposition === "review_untested";
+ // The tested framing is a claim like any other, so it reads the published verdict
+ // rather than the disposition alone.
const needsTestedProposalReview =
- hasDrafts && turn.proposalDisposition === "review_tested";
- const hasEdited = (turn.priorBlockCount ?? 0) > 0 && hasDrafts;
- const hasCleanCompletedBuild =
hasDrafts &&
- rollupBlocks.length > 0 &&
- rollupBlocks.every((block) => isBlockOk(block));
+ turn.proposalDisposition === "review_tested" &&
+ cleanFullCurrentSourceCoverage;
+ const hasEdited = (turn.priorBlockCount ?? 0) > 0 && hasDrafts;
+ const authoredBlockCount = facts?.factsAvailable
+ ? facts.authoredBlockCount
+ : null;
+ const matchingSourceBlockCount = facts?.factsAvailable
+ ? facts.matchingSourceBlockCount
+ : null;
+ const coverageKnown =
+ authoredBlockCount !== null &&
+ authoredBlockCount > 0 &&
+ matchingSourceBlockCount !== null;
const hasReviewableDraft =
hasDrafts &&
(turn.proposalDisposition === "review_untested" ||
@@ -1904,7 +1998,6 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {
needsTestedProposalReview,
hasEdited,
hasDrafts,
- hasCleanCompletedBuild,
});
const headline = adjudicated
@@ -1927,11 +2020,13 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {
: mode === "clarify"
? "Needs your input"
: "Answered"
- : hasEdited
- ? "Applied edits and re-tested"
- : hasDrafts
- ? "Built and tested the workflow"
- : "Completed the run";
+ : cleanFullCurrentSourceCoverage
+ ? "Every block ran clean on the current draft"
+ : hasEdited
+ ? "Applied edits"
+ : hasDrafts
+ ? "Built the workflow"
+ : "Completed the run";
const stats: string[] = [];
const turnElapsed = formatElapsed(turn.startedAt, turn.endedAt);
@@ -1941,8 +2036,30 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {
const failed = rollupBlocks.filter((b) => b.state === "failed").length;
const stopped = rollupBlocks.filter((b) => b.state === "stopped").length;
const newBlocks = hasEdited ? 0 : (turn.draft?.blockCount ?? 0);
- if (ok) stats.push(`${ok} block${ok === 1 ? "" : "s"} ran`);
- if (newBlocks) stats.push(`${newBlocks} new`);
+ if (coverageKnown) {
+ stats.push(
+ `${authoredBlockCount} block${authoredBlockCount === 1 ? "" : "s"} authored`,
+ );
+ stats.push(
+ `${matchingSourceBlockCount}/${authoredBlockCount} ran on current source`,
+ );
+ if (facts?.runCompleted === true) {
+ stats.push("run completed");
+ } else if (facts?.runCompleted === false) {
+ stats.push("run did not complete");
+ }
+ if (facts?.evaluationState === "not_evaluated") {
+ stats.push("outcome not evaluated");
+ } else if (facts?.evaluationState === "not_demonstrated") {
+ stats.push("outcome not confirmed");
+ }
+ if (facts?.terminalCause === "deadline_expired") {
+ stats.push("time limit reached");
+ }
+ } else {
+ if (ok) stats.push(`${ok} block${ok === 1 ? "" : "s"} ran`);
+ if (newBlocks) stats.push(`${newBlocks} new`);
+ }
if (failed) stats.push(`${failed} failed`);
if (stopped) stats.push(`${stopped} stopped`);
}
@@ -1956,7 +2073,8 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {
: isStopped ||
needsUntestedProposalReview ||
needsTestedProposalReview ||
- isQA
+ isQA ||
+ (hasDrafts && !cleanFullCurrentSourceCoverage)
? "qa"
: "ok";
return {
@@ -1967,7 +2085,10 @@ export function computeTurnSummary(turn: TurnNarrativeState): TurnSummary {
? adjudicated.glyph
: isStoppedWithDraft ||
needsUntestedProposalReview ||
- needsTestedProposalReview
+ needsTestedProposalReview ||
+ // A draft still awaiting review never wears a success tick, whether or not
+ // facts back its tested framing.
+ (hasDrafts && turn.proposalDisposition === "review_tested")
? "!"
: isFail
? "✕"
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.turnFacts.fixture.json b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.turnFacts.fixture.json
new file mode 100644
index 000000000..de7c18506
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.turnFacts.fixture.json
@@ -0,0 +1,807 @@
+{
+ "deadline-after-one-clean-run": {
+ "draft": {
+ "blockCount": 3,
+ "blockLabels": [
+ "extract_github_star_count",
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "summary": null
+ },
+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard. 1 block ran this turn. The recorded run completed, and its outcome was not evaluated. The turn reached its time limit.",
+ "proposalDisposition": "review_untested",
+ "review": {
+ "blocks": [
+ {
+ "blockType": "code",
+ "change": "changed",
+ "coverage": "never_run",
+ "label": "extract_github_star_count",
+ "neverTested": true
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "append_star_count_to_sheet",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "added",
+ "coverage": "never_run",
+ "label": "append_star_count_to_sales_marketing",
+ "neverTested": true
+ }
+ ],
+ "duplicateWrites": [
+ {
+ "blockLabels": [
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "blockType": "google_sheets_write"
+ }
+ ]
+ },
+ "terminal": "error",
+ "terminalEnvelope": {
+ "attempted": null,
+ "blocker_reason": null,
+ "blocks_run_this_turn": 1,
+ "envelope_version": 1,
+ "halt_kind": null,
+ "interruption": null,
+ "next_state": "proposal_pending",
+ "rendered_from_envelope": false,
+ "response_kind": "update",
+ "run_completed": true,
+ "run_display_reason": null,
+ "run_id": "wr_recorded",
+ "run_output_report": null,
+ "run_verdict": "not_evaluated",
+ "terminal_cause": "deadline_expired",
+ "user_action_required": false,
+ "verified": false,
+ "workflow_applied": false
+ },
+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard. 1 block ran this turn. The recorded run completed, and its outcome was not evaluated. The turn reached its time limit.",
+ "turnFacts": {
+ "authoredBlockCount": 3,
+ "blocksRunThisTurn": 1,
+ "evaluationState": "not_evaluated",
+ "factsAvailable": true,
+ "matchingSourceBlockCount": 1,
+ "ranCleanOnCurrentSource": false,
+ "runCompleted": true,
+ "runId": "wr_recorded",
+ "terminalCause": "deadline_expired"
+ },
+ "turnId": "turn_deadline-after-one-clean-run",
+ "turnIndex": 0
+ },
+ "different-source-edit-one-of-two": {
+ "draft": {
+ "blockCount": 3,
+ "blockLabels": [
+ "extract_github_star_count",
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "summary": null
+ },
+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",
+ "proposalDisposition": "review_untested",
+ "review": {
+ "blocks": [
+ {
+ "blockType": "code",
+ "change": "changed",
+ "coverage": "different_source",
+ "label": "extract_github_star_count",
+ "neverTested": true
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "never_run",
+ "label": "append_star_count_to_sheet",
+ "neverTested": true
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "added",
+ "coverage": "never_run",
+ "label": "append_star_count_to_sales_marketing",
+ "neverTested": true
+ }
+ ],
+ "duplicateWrites": [
+ {
+ "blockLabels": [
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "blockType": "google_sheets_write"
+ }
+ ]
+ },
+ "terminal": "response",
+ "terminalEnvelope": {
+ "attempted": null,
+ "blocker_reason": null,
+ "blocks_run_this_turn": 1,
+ "envelope_version": 1,
+ "halt_kind": null,
+ "interruption": null,
+ "next_state": "proposal_pending",
+ "rendered_from_envelope": false,
+ "response_kind": "update",
+ "run_completed": true,
+ "run_display_reason": null,
+ "run_id": "wr_recorded",
+ "run_output_report": null,
+ "run_verdict": "not_evaluated",
+ "terminal_cause": null,
+ "user_action_required": false,
+ "verified": false,
+ "workflow_applied": false
+ },
+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",
+ "turnFacts": {
+ "authoredBlockCount": 3,
+ "blocksRunThisTurn": 1,
+ "evaluationState": "not_evaluated",
+ "factsAvailable": true,
+ "matchingSourceBlockCount": 0,
+ "ranCleanOnCurrentSource": false,
+ "runCompleted": true,
+ "runId": "wr_recorded",
+ "terminalCause": null
+ },
+ "turnId": "turn_different-source-edit-one-of-two",
+ "turnIndex": 0
+ },
+ "facts-absent": {
+ "draft": {
+ "blockCount": 3,
+ "blockLabels": [
+ "extract_github_star_count",
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "summary": null
+ },
+ "narrativeSummary": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",
+ "proposalDisposition": "review_tested",
+ "review": {
+ "blocks": [
+ {
+ "blockType": "code",
+ "change": "unchanged",
+ "label": "extract_github_star_count",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "label": "append_star_count_to_sheet",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "label": "append_star_count_to_sales_marketing",
+ "neverTested": false
+ }
+ ],
+ "duplicateWrites": [
+ {
+ "blockLabels": [
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "blockType": "google_sheets_write"
+ }
+ ]
+ },
+ "terminal": "response",
+ "terminalEnvelope": {
+ "attempted": null,
+ "blocker_reason": null,
+ "blocks_run_this_turn": 3,
+ "envelope_version": 1,
+ "halt_kind": null,
+ "interruption": null,
+ "next_state": "proposal_pending",
+ "rendered_from_envelope": false,
+ "response_kind": "update",
+ "run_completed": true,
+ "run_display_reason": null,
+ "run_id": "wr_recorded",
+ "run_output_report": null,
+ "run_verdict": "not_evaluated",
+ "terminal_cause": null,
+ "user_action_required": false,
+ "verified": false,
+ "workflow_applied": false
+ },
+ "terminalMessage": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",
+ "turnFacts": null,
+ "turnId": "turn_facts-absent",
+ "turnIndex": 0
+ },
+ "full-coverage": {
+ "draft": {
+ "blockCount": 3,
+ "blockLabels": [
+ "extract_github_star_count",
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "summary": null
+ },
+ "narrativeSummary": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",
+ "proposalDisposition": "review_tested",
+ "review": {
+ "blocks": [
+ {
+ "blockType": "code",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "extract_github_star_count",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "append_star_count_to_sheet",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "append_star_count_to_sales_marketing",
+ "neverTested": false
+ }
+ ],
+ "duplicateWrites": [
+ {
+ "blockLabels": [
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "blockType": "google_sheets_write"
+ }
+ ]
+ },
+ "terminal": "response",
+ "terminalEnvelope": {
+ "attempted": null,
+ "blocker_reason": null,
+ "blocks_run_this_turn": 3,
+ "envelope_version": 1,
+ "halt_kind": null,
+ "interruption": null,
+ "next_state": "proposal_pending",
+ "rendered_from_envelope": false,
+ "response_kind": "update",
+ "run_completed": true,
+ "run_display_reason": null,
+ "run_id": "wr_recorded",
+ "run_output_report": null,
+ "run_verdict": "not_evaluated",
+ "terminal_cause": null,
+ "user_action_required": false,
+ "verified": false,
+ "workflow_applied": false
+ },
+ "terminalMessage": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",
+ "turnFacts": {
+ "authoredBlockCount": 3,
+ "blocksRunThisTurn": 3,
+ "evaluationState": "not_evaluated",
+ "factsAvailable": true,
+ "matchingSourceBlockCount": 3,
+ "ranCleanOnCurrentSource": true,
+ "runCompleted": true,
+ "runId": "wr_recorded",
+ "terminalCause": null
+ },
+ "turnId": "turn_full-coverage",
+ "turnIndex": 0
+ },
+ "mixed-edit-and-delete": {
+ "draft": {
+ "blockCount": 2,
+ "blockLabels": [
+ "extract_github_star_count",
+ "append_star_count_to_sales_marketing"
+ ],
+ "summary": null
+ },
+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",
+ "proposalDisposition": "review_untested",
+ "review": {
+ "blocks": [
+ {
+ "blockType": "code",
+ "change": "changed",
+ "coverage": "different_source",
+ "label": "extract_github_star_count",
+ "neverTested": true
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "append_star_count_to_sales_marketing",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "removed",
+ "label": "append_star_count_to_sheet"
+ }
+ ],
+ "duplicateWrites": []
+ },
+ "terminal": "response",
+ "terminalEnvelope": {
+ "attempted": null,
+ "blocker_reason": null,
+ "blocks_run_this_turn": 3,
+ "envelope_version": 1,
+ "halt_kind": null,
+ "interruption": null,
+ "next_state": "proposal_pending",
+ "rendered_from_envelope": false,
+ "response_kind": "update",
+ "run_completed": true,
+ "run_display_reason": null,
+ "run_id": "wr_recorded",
+ "run_output_report": null,
+ "run_verdict": "not_evaluated",
+ "terminal_cause": null,
+ "user_action_required": false,
+ "verified": false,
+ "workflow_applied": false
+ },
+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",
+ "turnFacts": {
+ "authoredBlockCount": 2,
+ "blocksRunThisTurn": 3,
+ "evaluationState": "not_evaluated",
+ "factsAvailable": true,
+ "matchingSourceBlockCount": 1,
+ "ranCleanOnCurrentSource": false,
+ "runCompleted": true,
+ "runId": "wr_recorded",
+ "terminalCause": null
+ },
+ "turnId": "turn_mixed-edit-and-delete",
+ "turnIndex": 0
+ },
+ "partial-coverage": {
+ "draft": {
+ "blockCount": 3,
+ "blockLabels": [
+ "extract_github_star_count",
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "summary": null
+ },
+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",
+ "proposalDisposition": "review_untested",
+ "review": {
+ "blocks": [
+ {
+ "blockType": "code",
+ "change": "changed",
+ "coverage": "never_run",
+ "label": "extract_github_star_count",
+ "neverTested": true
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "append_star_count_to_sheet",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "added",
+ "coverage": "never_run",
+ "label": "append_star_count_to_sales_marketing",
+ "neverTested": true
+ }
+ ],
+ "duplicateWrites": [
+ {
+ "blockLabels": [
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "blockType": "google_sheets_write"
+ }
+ ]
+ },
+ "terminal": "response",
+ "terminalEnvelope": {
+ "attempted": null,
+ "blocker_reason": null,
+ "blocks_run_this_turn": 1,
+ "envelope_version": 1,
+ "halt_kind": null,
+ "interruption": null,
+ "next_state": "proposal_pending",
+ "rendered_from_envelope": false,
+ "response_kind": "update",
+ "run_completed": true,
+ "run_display_reason": null,
+ "run_id": "wr_recorded",
+ "run_output_report": null,
+ "run_verdict": "not_evaluated",
+ "terminal_cause": null,
+ "user_action_required": false,
+ "verified": false,
+ "workflow_applied": false
+ },
+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",
+ "turnFacts": {
+ "authoredBlockCount": 3,
+ "blocksRunThisTurn": 1,
+ "evaluationState": "not_evaluated",
+ "factsAvailable": true,
+ "matchingSourceBlockCount": 1,
+ "ranCleanOnCurrentSource": false,
+ "runCompleted": true,
+ "runId": "wr_recorded",
+ "terminalCause": null
+ },
+ "turnId": "turn_partial-coverage",
+ "turnIndex": 0
+ },
+ "renamed-block": {
+ "draft": {
+ "blockCount": 3,
+ "blockLabels": [
+ "extract_github_star_count_v2",
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "summary": null
+ },
+ "narrativeSummary": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",
+ "proposalDisposition": "review_untested",
+ "review": {
+ "blocks": [
+ {
+ "blockType": "code",
+ "change": "added",
+ "coverage": "unknown",
+ "label": "extract_github_star_count_v2",
+ "neverTested": true
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "append_star_count_to_sheet",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "append_star_count_to_sales_marketing",
+ "neverTested": false
+ },
+ {
+ "blockType": "code",
+ "change": "removed",
+ "label": "extract_github_star_count"
+ }
+ ],
+ "duplicateWrites": [
+ {
+ "blockLabels": [
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "blockType": "google_sheets_write"
+ }
+ ]
+ },
+ "terminal": "response",
+ "terminalEnvelope": {
+ "attempted": null,
+ "blocker_reason": null,
+ "blocks_run_this_turn": 3,
+ "envelope_version": 1,
+ "halt_kind": null,
+ "interruption": null,
+ "next_state": "proposal_pending",
+ "rendered_from_envelope": false,
+ "response_kind": "update",
+ "run_completed": true,
+ "run_display_reason": null,
+ "run_id": "wr_recorded",
+ "run_output_report": null,
+ "run_verdict": "not_evaluated",
+ "terminal_cause": null,
+ "user_action_required": false,
+ "verified": false,
+ "workflow_applied": false
+ },
+ "terminalMessage": "I ran out of time. I have a draft workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard.",
+ "turnFacts": {
+ "authoredBlockCount": 3,
+ "blocksRunThisTurn": 3,
+ "evaluationState": "not_evaluated",
+ "factsAvailable": true,
+ "matchingSourceBlockCount": 2,
+ "ranCleanOnCurrentSource": false,
+ "runCompleted": true,
+ "runId": "wr_recorded",
+ "terminalCause": null
+ },
+ "turnId": "turn_renamed-block",
+ "turnIndex": 0
+ },
+ "satisfaction": {
+ "draft": {
+ "blockCount": 3,
+ "blockLabels": [
+ "extract_github_star_count",
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "summary": null
+ },
+ "narrativeSummary": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",
+ "proposalDisposition": "auto_applicable",
+ "review": {
+ "blocks": [
+ {
+ "blockType": "code",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "extract_github_star_count",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "append_star_count_to_sheet",
+ "neverTested": false
+ },
+ {
+ "blockType": "google_sheets_write",
+ "change": "unchanged",
+ "coverage": "current_source",
+ "label": "append_star_count_to_sales_marketing",
+ "neverTested": false
+ }
+ ],
+ "duplicateWrites": [
+ {
+ "blockLabels": [
+ "append_star_count_to_sheet",
+ "append_star_count_to_sales_marketing"
+ ],
+ "blockType": "google_sheets_write"
+ }
+ ]
+ },
+ "terminal": "response",
+ "terminalEnvelope": {
+ "attempted": null,
+ "blocker_reason": null,
+ "blocks_run_this_turn": 3,
+ "envelope_version": 1,
+ "halt_kind": null,
+ "interruption": null,
+ "next_state": "proposal_pending",
+ "rendered_from_envelope": false,
+ "response_kind": "update",
+ "run_completed": true,
+ "run_display_reason": null,
+ "run_id": "wr_recorded",
+ "run_output_report": null,
+ "run_verdict": "not_evaluated",
+ "terminal_cause": null,
+ "user_action_required": false,
+ "verified": false,
+ "workflow_applied": false
+ },
+ "terminalMessage": "I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",
+ "turnFacts": {
+ "authoredBlockCount": 3,
+ "blocksRunThisTurn": 3,
+ "evaluationState": "not_evaluated",
+ "factsAvailable": true,
+ "matchingSourceBlockCount": 3,
+ "ranCleanOnCurrentSource": true,
+ "runCompleted": true,
+ "runId": "wr_recorded",
+ "terminalCause": null
+ },
+ "turnId": "turn_satisfaction",
+ "turnIndex": 0
+ },
+ "surface-20260820T082523349904": {
+ "blocks": [
+ {
+ "activity": [],
+ "blockType": "code",
+ "endedAt": "2026-08-20T08:25:06.357598+00:00",
+ "label": "read_heading",
+ "lastSeenIteration": 0,
+ "outcome": "not_evaluated",
+ "outcomeRole": "recorded",
+ "startedAt": null,
+ "state": "completed"
+ }
+ ],
+ "designActivity": [
+ {
+ "displayLabel": "Inspecting the page",
+ "id": "tc-call_XNWFbCYGnBdzH5FhZBxmnDjG",
+ "iteration": 0,
+ "kind": "tool_call",
+ "text": "Inspecting the page…",
+ "toolName": "inspect_page_for_composition"
+ },
+ {
+ "displayLabel": "Inspecting the page",
+ "id": "tr-call_XNWFbCYGnBdzH5FhZBxmnDjG",
+ "iteration": 0,
+ "kind": "tool_result",
+ "success": true,
+ "text": "Inspected the page",
+ "toolName": "inspect_page_for_composition"
+ },
+ {
+ "displayLabel": "Checking workflow block options",
+ "id": "tc-call_rGCSiY2vPKAql7diyBAosK4Y",
+ "iteration": 1,
+ "kind": "tool_call",
+ "text": "Checking workflow block options…",
+ "toolName": "get_block_schema"
+ },
+ {
+ "displayLabel": "Checking workflow block options",
+ "id": "tr-call_rGCSiY2vPKAql7diyBAosK4Y",
+ "iteration": 1,
+ "kind": "tool_result",
+ "success": true,
+ "text": "Schema for code",
+ "toolName": "get_block_schema"
+ },
+ {
+ "displayLabel": "Testing workflow",
+ "id": "tc-call_RTKazhVldm7xi5e3Z3YRuFXk",
+ "iteration": 2,
+ "kind": "tool_call",
+ "text": "Testing workflow…",
+ "toolName": "update_and_run_blocks"
+ },
+ {
+ "displayLabel": "Testing workflow",
+ "id": "tr-call_RTKazhVldm7xi5e3Z3YRuFXk",
+ "iteration": 2,
+ "kind": "tool_result",
+ "success": true,
+ "text": "Updated the workflow and ran it: completed",
+ "toolName": "update_and_run_blocks"
+ },
+ {
+ "displayLabel": "Editing block \"Read Heading\"",
+ "id": "tc-call_1HHXFiV5MeiZOEZe4XfOOV3K",
+ "iteration": 3,
+ "kind": "tool_call",
+ "text": "Editing block \"Read Heading\"…",
+ "toolName": "edit_block"
+ },
+ {
+ "displayLabel": "Editing block \"Read Heading\"",
+ "id": "tr-call_1HHXFiV5MeiZOEZe4XfOOV3K",
+ "iteration": 3,
+ "kind": "tool_result",
+ "success": true,
+ "text": "Editing block \"Read Heading\"",
+ "toolName": "edit_block"
+ },
+ {
+ "displayLabel": "Testing workflow",
+ "id": "tc-call_bQZ2EQkM6gukPffLSwTuVQPk",
+ "iteration": 4,
+ "kind": "tool_call",
+ "text": "Testing workflow…",
+ "toolName": "run_blocks_and_collect_debug"
+ },
+ {
+ "displayLabel": "Testing workflow",
+ "id": "tr-call_bQZ2EQkM6gukPffLSwTuVQPk",
+ "iteration": 4,
+ "kind": "tool_result",
+ "success": true,
+ "text": "Run read_heading: completed",
+ "toolName": "run_blocks_and_collect_debug"
+ }
+ ],
+ "designEnded": true,
+ "designStarted": true,
+ "draft": {
+ "blockCount": 1,
+ "blockLabels": ["read_heading"],
+ "summary": null
+ },
+ "endedAt": "2026-08-20T08:25:23.337402+00:00",
+ "narrativeSummary": "I created and tested the single read_heading block twice. It now returns both the top heading and page title; the latest run completed with non-empty output.",
+ "priorBlockCount": 0,
+ "proposalDisposition": "auto_applicable",
+ "responseKind": "build",
+ "responseType": "REPLY",
+ "review": {
+ "blocks": [
+ {
+ "blockType": "code",
+ "change": "added",
+ "coverage": "current_source",
+ "label": "read_heading",
+ "neverTested": false
+ }
+ ],
+ "duplicateWrites": []
+ },
+ "startedAt": "2026-08-20T08:23:33.076908+00:00",
+ "terminal": "response",
+ "terminalEnvelope": {
+ "attempted": "I created and tested the single read_heading block twice. It now returns both the top heading and page title; the latest run completed with non-empty output.",
+ "blocker_reason": null,
+ "blocks_run_this_turn": 1,
+ "envelope_version": 1,
+ "halt_kind": null,
+ "interruption": null,
+ "next_state": "stopped",
+ "rendered_from_envelope": false,
+ "response_kind": "stopped",
+ "run_display_reason": null,
+ "run_id": "wr_564921777870827706",
+ "run_output_report": "Recorded output from the latest completed run: {\"read_heading_output\":{\"evidence_text\":\"Example Domain | Example Domain\",\"heading_text\":\"Example Domain\",\"page_title\":\"Example Domain\"}}",
+ "run_verdict": "not_evaluated",
+ "terminal_cause": null,
+ "user_action_required": false,
+ "verified": false,
+ "workflow_applied": false
+ },
+ "terminalMessage": "I created and tested the single read_heading block twice. It now returns both the top heading and page title; the latest run completed with non-empty output.",
+ "testedBlockFingerprints": {
+ "read_heading": [
+ "67244b78a2676852d281c77e0f4ba3310b68d0083180318109a6b1de4e791b82",
+ "79e5d6d7ed953e0beb207ebb9033e9d6bbb72d7b68118b1df4133406608e339d"
+ ]
+ },
+ "turnFacts": {
+ "authoredBlockCount": 1,
+ "blocksRunThisTurn": 1,
+ "evaluationState": "not_evaluated",
+ "factsAvailable": true,
+ "matchingSourceBlockCount": 1,
+ "ranCleanOnCurrentSource": true,
+ "runCompleted": null,
+ "runId": "wr_564921777870827706",
+ "terminalCause": null
+ },
+ "turnId": "466012b32fd94e0ca274b00aa59650db",
+ "turnIndex": 0
+ }
+}
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index c9e42ef12..f28f3e889 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -55,6 +55,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
history_has_runtime_block_failure,
observed_value_extraction_scaffold_lines,
+ unresolved_runtime_block_failure,
unresolved_runtime_block_failure_with_disposition,
)
from skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructions
@@ -83,6 +84,7 @@ from skyvern.forge.sdk.copilot.context import (
NarrativeActivityEntry,
NarrativeBlock,
NarrativeDraft,
+ NarrativeTurnFacts,
ProposalDisposition,
ResponseType,
StructuredContext,
@@ -154,7 +156,12 @@ from skyvern.forge.sdk.copilot.request_policy import (
redact_raw_secrets_for_prompt,
redact_refused_secret_turns,
)
-from skyvern.forge.sdk.copilot.review_gate import build_review_projection, serialize_execution_receipts
+from skyvern.forge.sdk.copilot.review_gate import (
+ NarrativeReviewBlock,
+ NarrativeReviewProjection,
+ build_review_projection,
+ serialize_execution_receipts,
+)
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.runtime import (
BrowserProbeOutcome,
@@ -175,6 +182,7 @@ from skyvern.forge.sdk.copilot.terminal_envelope import (
TerminalCause,
assemble_terminal_envelope,
reason_in_reply_shadow,
+ select_run_outcome_anchor,
)
from skyvern.forge.sdk.copilot.todo_list import todo_list_prompt
from skyvern.forge.sdk.copilot.tools.credentials import _server_verified_google_account_choices
@@ -1457,6 +1465,85 @@ def _terminal_cause_for_context(ctx: CopilotContext) -> TerminalCause | None:
return None
+def _authored_review_blocks(review: NarrativeReviewProjection | None) -> list[NarrativeReviewBlock]:
+ if review is None:
+ return []
+ return [block for block in review["blocks"] if block.get("change") != "removed"]
+
+
+def _draft_ran_on_current_source(
+ facts: NarrativeTurnFacts, unresolved_failure: UnresolvedRuntimeFailure | None
+) -> bool:
+ """A tested draft claims full current-source coverage and a lifecycle that contradicts nothing:
+ a halted turn, an unfinished or unconfirmed run, or an open block failure each disqualify it.
+ ``runCompleted`` is None when no run is anchored to this turn, leaving the source-bound receipts
+ as the only claim."""
+ authored = facts.get("authoredBlockCount")
+ if not facts["factsAvailable"] or not authored or facts.get("matchingSourceBlockCount") != authored:
+ return False
+ if facts["terminalCause"] is not None or unresolved_failure is not None:
+ return False
+ return facts["runCompleted"] is not False and facts["evaluationState"] != "not_demonstrated"
+
+
+def _review_projection_for(ctx: CopilotContext, proposal_yaml: str | None) -> NarrativeReviewProjection | None:
+ if not isinstance(proposal_yaml, str):
+ return None
+ return build_review_projection(
+ ctx.persisted_workflow_yaml or _EMPTY_REVIEW_BASELINE_YAML,
+ proposal_yaml,
+ ctx.executed_block_fingerprints,
+ )
+
+
+def _tested_draft_reply(
+ ctx: CopilotContext,
+ review: NarrativeReviewProjection | None,
+ *,
+ tested_reply: str,
+ unvalidated_reply: str,
+) -> str:
+ """The reply and the proposal disposition read one predicate, so prose cannot outrun the pill."""
+ tested = _turn_facts_for_context(ctx, review)["ranCleanOnCurrentSource"]
+ return tested_reply if tested else unvalidated_reply
+
+
+def _turn_fact_bundle(
+ review: NarrativeReviewProjection | None,
+ run_outcome: RecordedRunOutcome | None,
+ terminal_cause: TerminalCause | None,
+ blocks_run_this_turn: int | None,
+ unresolved_failure: UnresolvedRuntimeFailure | None = None,
+) -> NarrativeTurnFacts:
+ run_id = run_outcome.workflow_run_id if run_outcome is not None else None
+ facts: NarrativeTurnFacts = {
+ "factsAvailable": review is not None,
+ "evaluationState": run_outcome.verdict if run_outcome is not None else None,
+ "runId": (run_id.strip() or None) if isinstance(run_id, str) else None,
+ "runCompleted": run_outcome.run_completed if run_outcome is not None else None,
+ "terminalCause": terminal_cause,
+ "blocksRunThisTurn": blocks_run_this_turn,
+ # Overwritten below, once the coverage counts it reads are in place.
+ "ranCleanOnCurrentSource": False,
+ }
+ if review is not None:
+ authored = _authored_review_blocks(review)
+ facts["authoredBlockCount"] = len(authored)
+ facts["matchingSourceBlockCount"] = sum(1 for block in authored if block.get("coverage") == "current_source")
+ facts["ranCleanOnCurrentSource"] = _draft_ran_on_current_source(facts, unresolved_failure)
+ return facts
+
+
+def _turn_facts_for_context(ctx: CopilotContext, review: NarrativeReviewProjection | None) -> NarrativeTurnFacts:
+ return _turn_fact_bundle(
+ review,
+ select_run_outcome_anchor(_terminal_envelope_run_outcomes(ctx)),
+ _terminal_cause_for_context(ctx),
+ len(ctx.executed_block_labels),
+ unresolved_runtime_block_failure(ctx),
+ )
+
+
def _assemble_terminal_envelope_safe(
*,
response_type: str,
@@ -1471,6 +1558,7 @@ def _assemble_terminal_envelope_safe(
workflow_attempted: bool,
final_message: str,
terminal_cause: TerminalCause | None = None,
+ blocks_run_this_turn: int | None = None,
) -> dict[str, Any] | None:
try:
envelope = assemble_terminal_envelope(
@@ -1485,6 +1573,7 @@ def _assemble_terminal_envelope_safe(
workflow_mutated=workflow_mutated,
workflow_attempted=workflow_attempted,
terminal_cause=terminal_cause,
+ blocks_run_this_turn=blocks_run_this_turn,
)
except Exception:
LOG.warning("copilot terminal envelope assembly failed", exc_info=True)
@@ -1565,18 +1654,23 @@ def _make_agent_result(
if ctx is not None and narrative_payload is None:
raise ValueError("_make_agent_result requires narrative_payload when ctx is provided")
if ctx is not None and isinstance(narrative_payload, dict) and isinstance(proposal_yaml, str):
- review = build_review_projection(
- ctx.persisted_workflow_yaml or _EMPTY_REVIEW_BASELINE_YAML,
- proposal_yaml,
- ctx.executed_block_fingerprints,
- )
+ review = _review_projection_for(ctx, proposal_yaml)
narrative_payload = {key: value for key, value in narrative_payload.items() if key != "review"}
if review is not None:
narrative_payload["review"] = review
kwargs["narrative_payload"] = narrative_payload
+ review_projection: NarrativeReviewProjection | None = (
+ narrative_payload.get("review") if isinstance(narrative_payload, dict) else None
+ )
+ terminal_cause = _terminal_cause_for_context(ctx) if ctx is not None else None
+ turn_facts = _turn_facts_for_context(ctx, review_projection) if ctx is not None else None
response_type = kwargs.get("response_type", "REPLY")
response_type_value = response_type if isinstance(response_type, str) else "REPLY"
- proposal_disposition = kwargs.get("proposal_disposition")
+ raw_disposition = kwargs.get("proposal_disposition")
+ proposal_disposition: str | None = raw_disposition if isinstance(raw_disposition, str) else None
+ if turn_facts is not None and proposal_disposition == "review_tested" and not turn_facts["ranCleanOnCurrentSource"]:
+ proposal_disposition = "review_untested"
+ kwargs["proposal_disposition"] = proposal_disposition
result_carries_workflow = (
kwargs.get("updated_workflow") is not None
or kwargs.get("staged_workflow") is not None
@@ -1696,9 +1790,14 @@ def _make_agent_result(
workflow_mutated=workflow_mutated,
workflow_attempted=ctx.has_genuine_workflow_attempt(),
final_message=str(kwargs.get("user_response") or ""),
- terminal_cause=_terminal_cause_for_context(ctx),
+ terminal_cause=terminal_cause,
+ blocks_run_this_turn=len(ctx.executed_block_labels),
)
kwargs["terminal_envelope"] = terminal_envelope
+ if turn_facts is not None:
+ payload = kwargs.get("narrative_payload")
+ if isinstance(payload, dict):
+ kwargs["narrative_payload"] = {**payload, "turnFacts": turn_facts}
if ctx is not None and "executed_block_fingerprints" not in kwargs:
kwargs["executed_block_fingerprints"] = {
label: set(fingerprints) for label, fingerprints in ctx.executed_block_fingerprints.items()
@@ -2770,7 +2869,14 @@ def _build_wip_exit_result(
verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)
if outcome_fully_verified(ctx) and verified_workflow is not None:
- final_text, outcome = _guard(tested_reply)
+ final_text, outcome = _guard(
+ _tested_draft_reply(
+ ctx,
+ _review_projection_for(ctx, verified_yaml),
+ tested_reply=tested_reply,
+ unvalidated_reply=unvalidated_reply,
+ )
+ )
proposal_disposition = "auto_applicable"
return _finalize_result_with_blocker_override(
ctx,
@@ -2810,7 +2916,13 @@ def _build_wip_exit_result(
and not ctx.last_test_suspicious_success
):
append_failure_detail = recorded_failure_reply and not deadline_suppresses_failure_detail
- reply = _last_good_failure_reply(ctx, tested_reply) if append_failure_detail else tested_reply
+ held_reply = _tested_draft_reply(
+ ctx,
+ _review_projection_for(ctx, ctx.last_good_workflow_yaml),
+ tested_reply=tested_reply,
+ unvalidated_reply=unvalidated_reply,
+ )
+ reply = _last_good_failure_reply(ctx, held_reply) if append_failure_detail else held_reply
final_text, outcome = _guard(reply)
return _finalize_result_with_blocker_override(
ctx,
@@ -2848,7 +2960,16 @@ def _build_wip_exit_result(
if unvalidated and recorded_failure_reply:
reply = _deadline_owned_or(unvalidated_reply, recorded_failure_reply) or unvalidated_reply
else:
- reply = unvalidated_reply if unvalidated else tested_reply
+ reply = (
+ unvalidated_reply
+ if unvalidated
+ else _tested_draft_reply(
+ ctx,
+ _review_projection_for(ctx, ctx.last_workflow_yaml),
+ tested_reply=tested_reply,
+ unvalidated_reply=unvalidated_reply,
+ )
+ )
final_text, outcome = _guard(reply)
proposal_disposition = "review_untested" if unvalidated else "review_tested"
if not unvalidated and outcome_fully_verified(ctx):
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index ea5f28f06..c175a358f 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -106,6 +106,19 @@ class NarrativeConnectedAccountChoice(TypedDict):
email_address: str | None
+class NarrativeTurnFacts(TypedDict):
+ factsAvailable: bool
+ evaluationState: str | None
+ runId: str | None
+ runCompleted: bool | None
+ terminalCause: str | None
+ blocksRunThisTurn: int | None
+ authoredBlockCount: NotRequired[int]
+ matchingSourceBlockCount: NotRequired[int]
+ # The tested claim is decided once, in _turn_fact_bundle, so no surface re-derives it.
+ ranCleanOnCurrentSource: bool
+
+
# Mirror of the FE TurnNarrativeState; camelCase keys match the wire shape.
class TurnNarrativePayload(TypedDict):
turnId: str | None
@@ -139,6 +152,7 @@ class TurnNarrativePayload(TypedDict):
endedAt: str | None
review: NotRequired[NarrativeReviewProjection]
testedBlockFingerprints: NotRequired[dict[str, list[str]]]
+ turnFacts: NotRequired[NarrativeTurnFacts]
if TYPE_CHECKING:
diff --git a/skyvern/forge/sdk/copilot/review_gate.py b/skyvern/forge/sdk/copilot/review_gate.py
index b33919a50..9d26c5a4a 100644
--- a/skyvern/forge/sdk/copilot/review_gate.py
+++ b/skyvern/forge/sdk/copilot/review_gate.py
@@ -16,6 +16,7 @@ from skyvern.schemas.google_sheets import extract_a1_sheet_prefix, extract_sprea
from skyvern.schemas.workflows import BlockType, FileStorageType
ReviewChange = Literal["added", "changed", "unchanged", "removed"]
+BlockCoverage = Literal["current_source", "different_source", "never_run", "unknown"]
ParameterDefault = str | int | float | bool
DestinationIdentity = tuple[str, ...]
DestinationAdapter = Callable[[dict[str, Any], Mapping[str, ParameterDefault]], DestinationIdentity | None]
@@ -31,6 +32,7 @@ class NarrativeReviewBlock(TypedDict):
blockType: str
change: ReviewChange
neverTested: NotRequired[bool]
+ coverage: NotRequired[BlockCoverage]
class NarrativeDuplicateWrite(TypedDict):
@@ -353,6 +355,23 @@ def build_review_projection(
block["label"]: _block_fingerprint(block, staged_execution_inputs) for block in staged_blocks
}
+ if isinstance(executed_blocks, Mapping):
+ receipts: dict[str, set[str]] = {
+ label: {value} if isinstance(value, str) else set(value) for label, value in executed_blocks.items()
+ }
+ source_bound = True
+ else:
+ receipts = {label: set() for label in executed_blocks}
+ source_bound = False
+ # A block's own label is excluded from its fingerprint, so a receipt stranded under a
+ # label the staged workflow no longer has still identifies source this turn ran.
+ orphan_fingerprints = {
+ fingerprint
+ for label, recorded in receipts.items()
+ if label not in staged_fingerprints
+ for fingerprint in recorded
+ }
+
review_blocks: list[NarrativeReviewBlock] = []
for block in staged_blocks:
label = block["label"]
@@ -369,21 +388,20 @@ def build_review_projection(
change = "unchanged"
else:
change = "changed"
+ recorded = receipts.get(label)
+ if recorded is None:
+ coverage: BlockCoverage = "unknown" if staged_fingerprints[label] in orphan_fingerprints else "never_run"
+ elif not source_bound or staged_fingerprints[label] in recorded:
+ coverage = "current_source"
+ else:
+ coverage = "different_source"
review_blocks.append(
{
"label": label,
"blockType": _block_type(block),
"change": change,
- "neverTested": (
- staged_fingerprints[label]
- not in (
- {executed_blocks[label]}
- if isinstance(executed_blocks.get(label), str)
- else executed_blocks.get(label, ())
- )
- if isinstance(executed_blocks, Mapping)
- else label not in executed_blocks
- ),
+ "neverTested": coverage != "current_source",
+ "coverage": coverage,
}
)
for block in persisted_blocks:
diff --git a/skyvern/forge/sdk/copilot/run_outcome.py b/skyvern/forge/sdk/copilot/run_outcome.py
index 427fe5927..1e1e4bec0 100644
--- a/skyvern/forge/sdk/copilot/run_outcome.py
+++ b/skyvern/forge/sdk/copilot/run_outcome.py
@@ -42,6 +42,9 @@ class RecordedRunOutcome:
display_reason: str | None = None
workflow_run_id: str | None = None
output_report: str | None = None
+ # Recorded lifecycle, kept apart from ``verdict``: reaching a completed status says
+ # nothing about whether the outcome was evaluated. ``None`` on frames predating the field.
+ run_completed: bool | None = None
# ``adjudicated`` remains accepted for persisted legacy frames. New
# interactive authoring frames are factual records.
role: RunOutcomeRole = "recorded"
diff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.py
index 4fb22ae7b..728158d29 100644
--- a/skyvern/forge/sdk/copilot/terminal_envelope.py
+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py
@@ -16,6 +16,8 @@ _FINAL_RUN_VERDICTS = frozenset({"not_demonstrated", "not_evaluated"})
_REVIEW_PROPOSAL_DISPOSITIONS = frozenset({"review_untested", "review_tested"})
_SHADOW_REASON_TRAILING_PUNCTUATION = ".,;:!?"
+MINIMAL_HONEST_STOP = "I stopped without confirming the goal was met."
+
INTERRUPTED_TERMINAL_REASON = "interrupted"
INTERRUPTED_TERMINAL_HEADLINE = "This turn was interrupted before it could finish."
INTERRUPTED_TERMINAL_RETRY = "Send your message again to retry."
@@ -42,6 +44,9 @@ class TerminalOutcomeEnvelope(BaseModel):
verified: bool
workflow_applied: bool = False
run_verdict: str | None = None
+ run_id: str | None = None
+ run_completed: bool | None = None
+ blocks_run_this_turn: int | None = None
run_display_reason: str | None = None
run_output_report: str | None = None
blocker_reason: str | None = None
@@ -68,9 +73,14 @@ def assemble_terminal_envelope(
workflow_mutated: bool,
workflow_attempted: bool,
terminal_cause: TerminalCause | None = None,
+ blocks_run_this_turn: int | None = None,
) -> TerminalOutcomeEnvelope | None:
- run_outcome = _select_run_outcome_anchor(run_outcomes)
+ run_outcome = select_run_outcome_anchor(run_outcomes)
run_verdict = run_outcome.verdict if run_outcome is not None else None
+ run_id = _clean_text(run_outcome.workflow_run_id) if run_outcome is not None else None
+ # Lifecycle and identity come from the same archived outcome, which a workflow edit
+ # never rewrites, so the two can never name different runs.
+ run_completed = run_outcome.run_completed if run_outcome is not None else None
run_display_reason = _clean_text(run_outcome.display_reason) if run_outcome is not None else None
run_output_report = _safe_output_report(run_outcome.output_report) if run_outcome is not None else None
user_action_required = response_type == "ASK_QUESTION"
@@ -92,6 +102,9 @@ def assemble_terminal_envelope(
verified=verified,
workflow_applied=workflow_applied,
run_verdict=run_verdict,
+ run_id=run_id,
+ run_completed=run_completed,
+ blocks_run_this_turn=blocks_run_this_turn,
run_display_reason=run_display_reason,
run_output_report=run_output_report,
blocker_reason=_clean_text(blocker_reason),
@@ -172,13 +185,20 @@ def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: st
# "awaiting_user_input" needs no exclusion: it requires ASK_QUESTION, and a deadline
# always exits through _build_wip_exit_result, which only ever builds REPLY results.
if envelope.terminal_cause == "deadline_expired" and not cancelled and envelope.next_state != "completed":
- if output_report and not _text_contains(agent_message, output_report):
- return _append_sentence(agent_message, output_report), True
- return agent_message, True
+ message = agent_message
+ facts = _recorded_run_facts(envelope)
+ # The agent's own deadline copy already names time, so the time-limit
+ # sentence rides along only with facts that copy does not carry.
+ if facts:
+ for sentence in [*facts, "The turn reached its time limit."]:
+ message = _append_sentence(message, sentence)
+ if output_report and not _text_contains(message, output_report):
+ return _append_sentence(message, output_report), True
+ return message, True
# A plain reply with no concrete workflow/run/blocker evidence is an answer
- # even though next_state remains "stopped"; only stopped-kind turns get the
- # honest-stop replacement.
+ # even though next_state remains "stopped"; only stopped-kind turns carry the
+ # recorded facts.
if cancelled:
return agent_message, False
@@ -187,12 +207,16 @@ def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: st
return _append_sentence(agent_message, output_report), True
return agent_message, False
- if envelope.run_verdict == "not_demonstrated":
- message = "I ran the workflow, but I could not confirm the goal was met."
- elif envelope.run_verdict == "not_evaluated":
- message = "I ran the workflow. The latest recorded run completed."
- else:
- message = "I stopped without confirming the goal was met."
+ # Facts are appended to the agent's own text rather than replacing it: a message
+ # whose unsupported clause sits beside accurate detail loses that detail if replaced.
+ message = agent_message.strip() or MINIMAL_HONEST_STOP
+ recorded_facts = _recorded_run_facts(envelope)
+ # With no run anchored there is no fact to set beside the agent's text, and silence
+ # would let an unsupported success claim stand alone on a turn that stopped.
+ if not recorded_facts and message != MINIMAL_HONEST_STOP:
+ recorded_facts = [MINIMAL_HONEST_STOP]
+ for sentence in recorded_facts:
+ message = _append_sentence(message, sentence)
if envelope.run_display_reason:
message = _append_labeled_sentence(message, label="Reason", text=envelope.run_display_reason)
@@ -202,10 +226,25 @@ def render_terminal_message(envelope: TerminalOutcomeEnvelope, agent_message: st
blocker_reason = envelope.blocker_reason
if blocker_reason and not _text_contains(message, blocker_reason):
message = _append_labeled_sentence(message, label="Evidence", text=blocker_reason)
- return message, True
+ return message, message != agent_message
+
+
+def _recorded_run_facts(envelope: TerminalOutcomeEnvelope) -> list[str]:
+ facts: list[str] = []
+ ran = envelope.blocks_run_this_turn
+ if ran is not None:
+ facts.append(f"{ran} block{'' if ran == 1 else 's'} ran this turn.")
+ # The completion clause rides on the recorded lifecycle fact, never on the
+ # verdict: an unevaluated outcome says nothing about whether the run finished.
+ lifecycle = "The recorded run completed, and its" if envelope.run_completed else "The recorded run's"
+ if envelope.run_verdict == "not_evaluated":
+ facts.append(f"{lifecycle} outcome was not evaluated.")
+ elif envelope.run_verdict == "not_demonstrated":
+ facts.append(f"{lifecycle} outcome did not confirm the goal was met.")
+ return facts
-def _select_run_outcome_anchor(run_outcomes: Sequence[RecordedRunOutcome]) -> RecordedRunOutcome | None:
+def select_run_outcome_anchor(run_outcomes: Sequence[RecordedRunOutcome]) -> RecordedRunOutcome | None:
final_outcomes = [outcome for outcome in run_outcomes if outcome.verdict in _FINAL_RUN_VERDICTS]
if not final_outcomes:
return None
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index bee6b8657..903c26b0e 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -4052,6 +4052,7 @@ def _record_run_blocks_result(
recorded_outcome = _recorded_run_outcome(
workflow_run_id=run_id if isinstance(run_id, str) else None,
output_report=output_report,
+ run_completed=run_ok,
)
unverified = _unverified_current_workflow_labels(copilot_ctx)
composition_unverified = _composition_unverified_current_workflow_labels(copilot_ctx)
@@ -4123,6 +4124,7 @@ def _record_run_blocks_result(
copilot_ctx.last_test_failure_reason or str(result.get("error") or "The run failed.")
),
workflow_run_id=run_id if isinstance(run_id, str) else None,
+ run_completed=False,
)
_record_build_test_outcome(copilot_ctx, result, recorded_outcome)
return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)
@@ -4231,12 +4233,14 @@ def _recorded_run_outcome(
*,
workflow_run_id: str | None = None,
output_report: str | None = None,
+ run_completed: bool | None = None,
) -> RecordedRunOutcome:
"""Record the completed run status without interpreting whether it met the request."""
return RecordedRunOutcome(
verdict="not_evaluated",
workflow_run_id=workflow_run_id,
output_report=output_report,
+ run_completed=run_completed,
)
diff --git a/tests/unit/copilot/test_review_gate.py b/tests/unit/copilot/test_review_gate.py
index 76c7d3f43..9b1eccec5 100644
--- a/tests/unit/copilot/test_review_gate.py
+++ b/tests/unit/copilot/test_review_gate.py
@@ -63,9 +63,22 @@ def test_projection_classifies_all_changes_and_uses_cumulative_execution_evidenc
"blockType": "task",
"change": "unchanged",
"neverTested": False,
+ "coverage": "current_source",
+ },
+ {
+ "label": "changed_step",
+ "blockType": "task",
+ "change": "changed",
+ "neverTested": False,
+ "coverage": "current_source",
+ },
+ {
+ "label": "added_step",
+ "blockType": "task",
+ "change": "added",
+ "neverTested": True,
+ "coverage": "never_run",
},
- {"label": "changed_step", "blockType": "task", "change": "changed", "neverTested": False},
- {"label": "added_step", "blockType": "task", "change": "added", "neverTested": True},
{"label": "removed_step", "blockType": "task", "change": "removed"},
]
@@ -204,8 +217,20 @@ def test_a_drafted_sheets_append_is_never_tested_until_it_actually_runs() -> Non
assert projection is not None
assert projection["blocks"] == [
- {"label": "count_posts", "blockType": "task", "change": "unchanged", "neverTested": False},
- {"label": "append_count", "blockType": "google_sheets_write", "change": "added", "neverTested": True},
+ {
+ "label": "count_posts",
+ "blockType": "task",
+ "change": "unchanged",
+ "neverTested": False,
+ "coverage": "current_source",
+ },
+ {
+ "label": "append_count",
+ "blockType": "google_sheets_write",
+ "change": "added",
+ "neverTested": True,
+ "coverage": "never_run",
+ },
]
@@ -227,6 +252,7 @@ def test_parameter_default_change_invalidates_change_and_execution_identity() ->
"blockType": "task",
"change": "changed",
"neverTested": True,
+ "coverage": "different_source",
}
@@ -278,6 +304,7 @@ def test_nested_block_label_change_invalidates_parent_tested_version() -> None:
"blockType": "for_loop",
"change": "changed",
"neverTested": True,
+ "coverage": "different_source",
}
@@ -733,3 +760,71 @@ def test_run_scoped_write_outputs_abstain_from_duplicate_claims() -> None:
assert projection is not None
assert projection["duplicateWrites"] == []
+
+
+def test_a_comment_only_edit_reports_the_prior_receipt_as_a_different_source() -> None:
+ tested = _workflow(""" - block_type: code
+ label: sign_in
+ code: |
+ await page.click("#submit")
+""")
+ edited = _workflow(""" - block_type: code
+ label: sign_in
+ code: |
+ # retry the submit button
+ await page.click("#submit")
+""")
+
+ projection = build_review_projection(tested, edited, workflow_block_fingerprints(tested))
+
+ assert projection is not None
+ assert projection["blocks"][0]["coverage"] == "different_source"
+ assert projection["blocks"][0]["change"] == "changed"
+
+
+def test_a_renamed_block_loses_its_receipt_while_siblings_keep_theirs() -> None:
+ tested = _workflow(""" - block_type: task
+ label: sign_in
+ next_block_label: read_metric
+ prompt: Sign in
+ - block_type: task
+ label: read_metric
+ prompt: Read it
+""")
+ renamed = _workflow(""" - block_type: task
+ label: sign_in_v2
+ next_block_label: read_metric
+ prompt: Sign in
+ - block_type: task
+ label: read_metric
+ prompt: Read it
+""")
+
+ projection = build_review_projection(tested, renamed, workflow_block_fingerprints(tested))
+
+ assert projection is not None
+ assert [(row["label"], row.get("coverage")) for row in projection["blocks"]] == [
+ ("sign_in_v2", "unknown"),
+ ("read_metric", "current_source"),
+ ("sign_in", None),
+ ]
+
+
+def test_editing_one_of_two_blocks_leaves_the_untouched_block_current() -> None:
+ tested = _workflow(""" - block_type: task
+ label: sign_in
+ next_block_label: read_metric
+ prompt: Sign in
+ - block_type: task
+ label: read_metric
+ prompt: Read it
+""")
+ edited = tested.replace("prompt: Read it", "prompt: Read it twice")
+
+ projection = build_review_projection(tested, edited, workflow_block_fingerprints(tested))
+
+ assert projection is not None
+ assert [(row["label"], row["coverage"]) for row in projection["blocks"]] == [
+ ("sign_in", "current_source"),
+ ("read_metric", "different_source"),
+ ]
diff --git a/tests/unit/test_copilot_agent_blocker_renderer.py b/tests/unit/test_copilot_agent_blocker_renderer.py
index f348b7c0f..400bc027e 100644
--- a/tests/unit/test_copilot_agent_blocker_renderer.py
+++ b/tests/unit/test_copilot_agent_blocker_renderer.py
@@ -31,6 +31,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
OutputPolicyVerdict,
)
from skyvern.forge.sdk.copilot.request_policy import LivePageResolutionRecord, RequestPolicy
+from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.turn_halt import TurnHalt, TurnHaltKind
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
@@ -49,6 +50,16 @@ _ALL_BLOCKER_KINDS: tuple[BlockerKind, ...] = get_args(BlockerKind)
_ALL_RECOVERY_HINTS: tuple[RecoveryHint, ...] = get_args(RecoveryHint)
+_COVERED_DRAFT_YAML = """title: Draft
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: task
+ label: step
+ prompt: Do it
+"""
+
+
def _signal(
*,
kind: BlockerKind = "authority_denied",
@@ -275,7 +286,10 @@ def test_output_policy_generic_block_uses_only_safety_and_draft_evidence() -> No
ctx = _ctx()
fake_workflow: Any = object()
ctx.last_workflow = fake_workflow
- ctx.last_workflow_yaml = "title: Draft\n"
+ ctx.last_workflow_yaml = _COVERED_DRAFT_YAML
+ ctx.executed_block_fingerprints = {
+ label: set(values) for label, values in workflow_block_fingerprints(_COVERED_DRAFT_YAML).items()
+ }
ctx.workflow_persisted = True
ctx.last_test_ok = True
_seed_terminal_evidence(ctx)
@@ -285,7 +299,7 @@ def test_output_policy_generic_block_uses_only_safety_and_draft_evidence() -> No
assert result.response_type == "ASK_QUESTION"
assert result.updated_workflow is fake_workflow
- assert result.proposal_disposition == "review_tested"
+ assert result.proposal_disposition == "review_untested"
assert "latest run recorded workflow output" not in result.user_response
assert "did not demonstrate" not in result.user_response
assert "verification challenge" in result.user_response
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 7cec79041..48b3e99d8 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -32,6 +32,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.build_test_outcome import (
PostRunPagePathFailure,
RecordedBuildTestOutcome,
+ record_build_test_outcome,
)
from skyvern.forge.sdk.copilot.completion_criteria_store import (
StoredCriteriaSet,
@@ -81,6 +82,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
redact_raw_secrets_for_prompt,
)
from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOTS_PROMPT_NAME
+from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools import _run_blocks_and_collect_debug
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
@@ -113,8 +115,28 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatSender,
)
from skyvern.utils.yaml_loader import safe_load_no_dates
+from tests.unit.copilot_test_helpers import failed_second_factor_run
from tests.unit.copilot_test_helpers import make_copilot_ctx as _ctx
from tests.unit.copilot_test_helpers import make_verified_goal_contract as _verified_goal_contract
+from tests.unit.copilot_test_helpers import passing_run, two_page_login_yaml
+
+_COVERED_DRAFT_YAML = """title: Draft
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: task
+ label: open_page
+ prompt: Open the page
+"""
+
+
+def _covered_draft_receipts() -> dict[str, set[str]]:
+ return {label: set(values) for label, values in workflow_block_fingerprints(_COVERED_DRAFT_YAML).items()}
+
+
+def _clean_recorded_run() -> RecordedRunOutcome:
+ return RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_clean", run_completed=True)
+
_HISTORY_SENTINEL_TS = datetime(2026, 1, 1, tzinfo=timezone.utc)
@@ -990,7 +1012,9 @@ class TestVerifiedGoalSatisfiedStop:
ctx = _ctx(
turn_origin=TurnOrigin.runtime_self_heal,
last_workflow=workflow,
- last_workflow_yaml="workflow_definition:\n blocks: []\n",
+ last_workflow_yaml=_COVERED_DRAFT_YAML,
+ executed_block_fingerprints=_covered_draft_receipts(),
+ last_run_outcome=_clean_recorded_run(),
last_test_ok=True,
last_full_workflow_test_ok=True,
tool_activity=[{"tool": "update_and_run_blocks", "summary": "OK"}],
@@ -999,7 +1023,7 @@ class TestVerifiedGoalSatisfiedStop:
result = await _build_goal_satisfied_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is workflow
- assert result.workflow_yaml == "workflow_definition:\n blocks: []\n"
+ assert result.workflow_yaml == _COVERED_DRAFT_YAML
assert result.proposal_disposition == "review_tested"
assert result.user_response == "The unattended recovery check completed."
assert result.narrative_payload is not None
@@ -2554,7 +2578,9 @@ class TestTranslateToAgentResultGating:
ctx = _ctx(
block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
last_workflow=workflow,
- last_workflow_yaml="title: Structural Draft",
+ last_workflow_yaml=_COVERED_DRAFT_YAML,
+ executed_block_fingerprints=_covered_draft_receipts(),
+ last_run_outcome=_clean_recorded_run(),
last_test_ok=True,
last_full_workflow_test_ok=True,
latest_diagnosis_repair_contract=_unverified_no_repair_contract(),
@@ -2592,7 +2618,8 @@ class TestTranslateToAgentResultGating:
ctx = _ctx(
block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
last_workflow=workflow,
- last_workflow_yaml="title: Verified Draft",
+ last_workflow_yaml=_COVERED_DRAFT_YAML,
+ executed_block_fingerprints=_covered_draft_receipts(),
last_test_ok=True,
last_full_workflow_test_ok=True,
has_staged_proposal=True,
@@ -2604,7 +2631,7 @@ class TestTranslateToAgentResultGating:
verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
),
last_run_blocks_workflow_run_id="wr_1",
- last_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1"),
+ last_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1", run_completed=True),
)
agent_result = agent_module._build_wip_exit_result(
@@ -2963,7 +2990,10 @@ class TestTranslateToAgentResultGating:
)
assert agent_result.updated_workflow is replacement
- assert agent_result.proposal_disposition == "review_tested"
+ # A tested pill now needs receipts bound to the staged source, and this stub proposal
+ # carries no blocks to bind, so the full-test flags alone no longer earn one. The
+ # satisfaction path is proven against real blocks in TestCanonicalTurnFacts.
+ assert agent_result.proposal_disposition == "review_untested"
def test_inline_replace_workflow_steers_on_stale_block_metadata(self, monkeypatch) -> None:
# A label still describing the prior subject is authoring quality, not disclosure, so the
@@ -3540,12 +3570,14 @@ workflow_definition:
)
return _ctx(
last_workflow=verified_wf,
- last_workflow_yaml="verified: yaml",
+ last_workflow_yaml=_COVERED_DRAFT_YAML,
+ executed_block_fingerprints=_covered_draft_receipts(),
+ last_run_outcome=_clean_recorded_run(),
last_test_ok=True,
last_full_workflow_test_ok=True,
has_staged_proposal=True,
staged_workflow=verified_wf,
- staged_workflow_yaml="verified: yaml",
+ staged_workflow_yaml=_COVERED_DRAFT_YAML,
)
ask_ctx = build_ctx()
@@ -6119,3 +6151,400 @@ class TestToolFactOwnership:
for duplicated in ("- CSS SELECTORS:", "- SCROLL:", "- SELECT_OPTION:", "- PRESS_KEY:", "- CONSOLE_MESSAGES:"):
assert duplicated not in prompt
assert "Use evaluate for reading/inspection" not in prompt
+
+
+class TestCanonicalTurnFacts:
+ """One fact bundle behind the terminal card, the proposal pill and the terminal prose."""
+
+ @staticmethod
+ def _two_block_yaml(second_prompt: str = "Read it") -> str:
+ return f"""
+title: Sign in and read the metric
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: task
+ label: sign_in
+ next_block_label: read_metric
+ prompt: Sign in
+ - block_type: task
+ label: read_metric
+ prompt: {second_prompt}
+"""
+
+ @staticmethod
+ def _render(ctx: Any, staged_yaml: str, **overrides: Any) -> dict[str, Any]:
+ kwargs: dict[str, Any] = {
+ "user_response": "ok",
+ "global_llm_context": None,
+ "updated_workflow": None,
+ "workflow_yaml": staged_yaml,
+ "proposal_disposition": "review_tested",
+ "narrative_payload": {
+ "turnId": "turn-1",
+ "turnIndex": 0,
+ "designStarted": True,
+ "designEnded": True,
+ "draft": None,
+ "blocks": [],
+ "terminal": "response",
+ "terminalMessage": "done",
+ "narrativeSummary": "Built it.",
+ "priorBlockCount": None,
+ "designActivity": [],
+ "startedAt": None,
+ "endedAt": None,
+ },
+ }
+ kwargs.update(overrides)
+ result = agent_module._make_agent_result(ctx, **kwargs)
+ assert result.narrative_payload is not None
+ return {
+ "facts": result.narrative_payload["turnFacts"],
+ "review": result.narrative_payload.get("review"),
+ "disposition": result.proposal_disposition,
+ }
+
+ def test_partial_current_source_coverage_states_counts_and_drops_the_tested_pill(self) -> None:
+ staged = self._two_block_yaml()
+ ctx = _ctx(persisted_workflow_yaml=staged)
+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}
+ ctx.executed_block_labels = {"sign_in"}
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["authoredBlockCount"] == 2
+ assert rendered["facts"]["matchingSourceBlockCount"] == 1
+ assert rendered["facts"]["blocksRunThisTurn"] == 1
+ assert rendered["facts"]["factsAvailable"] is True
+ assert rendered["disposition"] == "review_untested"
+
+ def test_full_current_source_coverage_keeps_the_tested_pill(self) -> None:
+ staged = self._two_block_yaml()
+ ctx = _ctx(persisted_workflow_yaml=staged, last_run_outcome=_clean_recorded_run())
+ ctx.executed_block_fingerprints = {
+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()
+ }
+ ctx.executed_block_labels = {"sign_in", "read_metric"}
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["authoredBlockCount"] == 2
+ assert rendered["facts"]["matchingSourceBlockCount"] == 2
+ assert rendered["facts"]["runCompleted"] is True
+ assert rendered["facts"]["evaluationState"] == "not_evaluated"
+ assert rendered["disposition"] == "review_tested"
+
+ def test_full_coverage_over_an_unfinished_run_drops_the_tested_pill(self) -> None:
+ staged = self._two_block_yaml()
+ ctx = _ctx(
+ persisted_workflow_yaml=staged,
+ last_run_outcome=RecordedRunOutcome(
+ verdict="not_evaluated", workflow_run_id="wr_partial", run_completed=False
+ ),
+ )
+ ctx.executed_block_fingerprints = {
+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()
+ }
+ ctx.executed_block_labels = {"sign_in", "read_metric"}
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["matchingSourceBlockCount"] == 2
+ assert rendered["facts"]["runCompleted"] is False
+ assert rendered["disposition"] == "review_untested"
+
+ def test_a_block_failure_no_later_run_re_exercised_drops_the_tested_pill(self) -> None:
+ staged = two_page_login_yaml()
+ ctx = _ctx(
+ persisted_workflow_yaml=staged,
+ workflow_yaml=staged,
+ last_run_outcome=_clean_recorded_run(),
+ )
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_1"))
+ record_build_test_outcome(ctx, passing_run("wr_2", ["sign_in_and_read"]))
+ ctx.executed_block_fingerprints = {
+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()
+ }
+ ctx.executed_block_labels = {"sign_in_and_read"}
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["matchingSourceBlockCount"] == rendered["facts"]["authoredBlockCount"]
+ assert rendered["facts"]["runCompleted"] is True
+ assert rendered["disposition"] == "review_untested"
+
+ def test_full_coverage_over_an_unconfirmed_outcome_drops_the_tested_pill(self) -> None:
+ staged = self._two_block_yaml()
+ ctx = _ctx(
+ persisted_workflow_yaml=staged,
+ last_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated", workflow_run_id="wr_blocked", run_completed=True
+ ),
+ )
+ ctx.executed_block_fingerprints = {
+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()
+ }
+ ctx.executed_block_labels = {"sign_in", "read_metric"}
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["matchingSourceBlockCount"] == 2
+ assert rendered["facts"]["evaluationState"] == "not_demonstrated"
+ assert rendered["disposition"] == "review_untested"
+
+ def test_an_edited_block_is_no_longer_counted_against_its_stale_receipt(self) -> None:
+ tested = self._two_block_yaml()
+ staged = self._two_block_yaml("Read it twice")
+ ctx = _ctx(persisted_workflow_yaml=tested)
+ ctx.executed_block_fingerprints = {
+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(tested).items()
+ }
+ ctx.executed_block_labels = {"sign_in", "read_metric"}
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["matchingSourceBlockCount"] == 1
+ assert rendered["disposition"] == "review_untested"
+
+ def test_a_deadline_turn_reports_its_cause_and_never_a_tested_draft(self) -> None:
+ staged = self._two_block_yaml()
+ ctx = _ctx(persisted_workflow_yaml=staged)
+ ctx.executed_block_fingerprints = {
+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()
+ }
+ ctx.executed_block_labels = {"sign_in", "read_metric"}
+ ctx.copilot_total_timeout_exceeded = True
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["terminalCause"] == "deadline_expired"
+ assert rendered["facts"]["blocksRunThisTurn"] == 2
+ assert rendered["disposition"] == "review_untested"
+
+ def test_removed_blocks_leave_the_authored_count(self) -> None:
+ persisted = self._two_block_yaml()
+ staged = """
+title: Sign in and read the metric
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: task
+ label: sign_in
+ prompt: Sign in
+"""
+ ctx = _ctx(persisted_workflow_yaml=persisted)
+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}
+ ctx.executed_block_labels = {"sign_in"}
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["authoredBlockCount"] == 1
+ assert rendered["facts"]["matchingSourceBlockCount"] == 1
+
+ def test_unavailable_review_facts_report_no_counts_and_no_tested_pill(self) -> None:
+ duplicate_labels = """
+title: Sign in twice
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: task
+ label: sign_in
+ prompt: Sign in
+ - block_type: task
+ label: sign_in
+ prompt: Sign in again
+"""
+ ctx = _ctx(persisted_workflow_yaml=duplicate_labels)
+
+ rendered = self._render(ctx, duplicate_labels)
+
+ assert rendered["facts"]["factsAvailable"] is False
+ assert "authoredBlockCount" not in rendered["facts"]
+ assert "matchingSourceBlockCount" not in rendered["facts"]
+ assert rendered["disposition"] == "review_untested"
+
+ def test_a_draft_with_no_authored_blocks_is_not_a_tested_draft(self) -> None:
+ empty = "title: Nothing yet\nworkflow_definition:\n parameters: []\n blocks: []\n"
+ ctx = _ctx(persisted_workflow_yaml=empty)
+
+ rendered = self._render(ctx, empty)
+
+ assert rendered["facts"]["authoredBlockCount"] == 0
+ assert rendered["disposition"] == "review_untested"
+
+ def test_coverage_facts_survive_a_failed_envelope_assembly(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ def _explode(**_kwargs: Any) -> None:
+ raise RuntimeError("envelope assembly failed")
+
+ monkeypatch.setattr(agent_module, "assemble_terminal_envelope", _explode)
+ staged = self._two_block_yaml()
+ ctx = _ctx(persisted_workflow_yaml=staged)
+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}
+ ctx.executed_block_labels = {"sign_in"}
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["factsAvailable"] is True
+ assert rendered["facts"]["authoredBlockCount"] == 2
+ assert rendered["facts"]["matchingSourceBlockCount"] == 1
+ assert rendered["facts"]["runCompleted"] is None
+ assert rendered["facts"]["evaluationState"] is None
+ assert rendered["disposition"] == "review_untested"
+
+ def test_run_lifecycle_and_evaluation_state_are_separate_facts(self) -> None:
+ staged = self._two_block_yaml()
+ ctx = _ctx(persisted_workflow_yaml=staged, last_test_ok=None)
+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}
+ ctx.executed_block_labels = {"sign_in"}
+ ctx.last_run_outcome = RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1", run_completed=True)
+
+ rendered = self._render(ctx, staged)
+
+ assert rendered["facts"]["runCompleted"] is True
+ assert rendered["facts"]["evaluationState"] == "not_evaluated"
+ assert rendered["facts"]["runId"] == "wr_1"
+ assert rendered["facts"]["authoredBlockCount"] == 2
+ assert rendered["facts"]["matchingSourceBlockCount"] == 1
+ assert rendered["disposition"] == "review_untested"
+
+ @pytest.mark.parametrize(
+ ("tested_reply", "unvalidated_reply"),
+ [
+ (agent_module._TIMEOUT_REPLY_TESTED, agent_module._TIMEOUT_REPLY_UNVALIDATED),
+ (agent_module._MAX_TURNS_REPLY_TESTED, agent_module._MAX_TURNS_REPLY_UNVALIDATED),
+ (agent_module._UNEXPECTED_ERROR_REPLY_TESTED, agent_module._UNEXPECTED_ERROR_REPLY_UNVALIDATED),
+ (agent_module._CANCEL_REPLY_TESTED, agent_module._CANCEL_REPLY_UNVALIDATED),
+ ],
+ )
+ def test_a_partially_covered_draft_claims_no_tested_draft_on_any_terminal_cause(
+ self, tested_reply: str, unvalidated_reply: str
+ ) -> None:
+ staged = self._two_block_yaml()
+ ctx = _ctx(
+ persisted_workflow_yaml=staged,
+ last_workflow=SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[])),
+ last_workflow_yaml=staged,
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ )
+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}
+ ctx.executed_block_labels = {"sign_in"}
+
+ result = agent_module._build_wip_exit_result(
+ ctx,
+ None,
+ default_reply="No draft.",
+ unvalidated_reply=unvalidated_reply,
+ tested_reply=tested_reply,
+ terminal_reason="max_turns",
+ )
+
+ assert "tested draft" not in result.user_response
+ assert result.user_response == unvalidated_reply
+ assert result.proposal_disposition == "review_untested"
+
+ def test_a_fully_covered_draft_keeps_its_tested_reply(self) -> None:
+ staged = self._two_block_yaml()
+ ctx = _ctx(
+ persisted_workflow_yaml=staged,
+ last_workflow=SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[])),
+ last_workflow_yaml=staged,
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ last_run_outcome=_clean_recorded_run(),
+ )
+ ctx.executed_block_fingerprints = {
+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(staged).items()
+ }
+ ctx.executed_block_labels = {"sign_in", "read_metric"}
+
+ result = agent_module._build_wip_exit_result(
+ ctx,
+ None,
+ default_reply="No draft.",
+ unvalidated_reply=agent_module._TIMEOUT_REPLY_UNVALIDATED,
+ tested_reply=agent_module._TIMEOUT_REPLY_TESTED,
+ terminal_reason="max_turns",
+ )
+
+ assert result.user_response == agent_module._TIMEOUT_REPLY_TESTED
+ assert result.proposal_disposition == "review_tested"
+
+ def test_a_renamed_block_reports_unknown_coverage_and_drops_the_tested_claim(self) -> None:
+ tested = self._two_block_yaml()
+ staged = tested.replace("label: read_metric", "label: read_metric_v2").replace(
+ "next_block_label: read_metric", "next_block_label: read_metric_v2"
+ )
+ ctx = _ctx(persisted_workflow_yaml=tested)
+ ctx.executed_block_fingerprints = {
+ label: set(fingerprints) for label, fingerprints in workflow_block_fingerprints(tested).items()
+ }
+ ctx.executed_block_labels = {"sign_in", "read_metric"}
+
+ rendered = self._render(ctx, staged)
+ review = rendered["review"]
+
+ assert [(row["label"], row.get("coverage")) for row in review["blocks"]] == [
+ ("sign_in", "different_source"),
+ ("read_metric_v2", "unknown"),
+ ("read_metric", None),
+ ]
+ assert rendered["disposition"] == "review_untested"
+
+ def test_a_self_heal_verified_exit_claims_no_tested_draft_on_partial_coverage(self) -> None:
+ staged = self._two_block_yaml()
+ ctx = _ctx(
+ turn_origin=TurnOrigin.runtime_self_heal,
+ persisted_workflow_yaml=staged,
+ last_workflow=SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[])),
+ last_workflow_yaml=staged,
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ completion_verification_result=CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
+ ),
+ )
+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}
+ ctx.executed_block_labels = {"sign_in"}
+
+ result = agent_module._build_wip_exit_result(
+ ctx,
+ None,
+ default_reply="No draft.",
+ unvalidated_reply=agent_module._TIMEOUT_REPLY_UNVALIDATED,
+ tested_reply=agent_module._TIMEOUT_REPLY_TESTED,
+ terminal_reason="max_turns",
+ )
+
+ assert outcome_fully_verified(ctx) is True
+ assert result.proposal_disposition == "auto_applicable"
+ assert "tested draft" not in result.user_response
+
+ def test_a_partially_covered_last_good_draft_claims_no_tested_draft(self) -> None:
+ staged = self._two_block_yaml()
+ good = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))
+ ctx = _ctx(
+ persisted_workflow_yaml=staged,
+ last_workflow=SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[])),
+ last_workflow_yaml=self._two_block_yaml("Read it twice"),
+ last_good_workflow=good,
+ last_good_workflow_yaml=staged,
+ )
+ ctx.executed_block_fingerprints = {"sign_in": set(workflow_block_fingerprints(staged)["sign_in"])}
+ ctx.executed_block_labels = {"sign_in"}
+
+ result = agent_module._build_wip_exit_result(
+ ctx,
+ None,
+ default_reply="No draft.",
+ unvalidated_reply=agent_module._TIMEOUT_REPLY_UNVALIDATED,
+ tested_reply=agent_module._TIMEOUT_REPLY_TESTED,
+ terminal_reason="max_turns",
+ )
+
+ assert result.updated_workflow is good
+ assert "tested draft" not in result.user_response
+ assert result.proposal_disposition == "review_untested"
diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.py
index 0ca0eb0a5..6ad619689 100644
--- a/tests/unit/test_copilot_frontier.py
+++ b/tests/unit/test_copilot_frontier.py
@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
summarize_tool_result,
)
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.session_factory import copilot_session_input_callback
from skyvern.forge.sdk.copilot.tools import (
@@ -3735,6 +3736,12 @@ async def test_completed_test_result_handoff_uses_ordinary_acting_agent_surface(
ctx.workflow_verification_evidence.full_workflow_verified = True
ctx.workflow_verification_evidence.workflow_run_id = "wr_completed_test_result_handoff"
ctx.last_executed_block_labels = ["inspect_result"]
+ # The real run path records a source-bound receipt per executed block
+ # (run_execution._record_executed_block_labels); this stub replaces that path.
+ ctx.executed_block_labels.add("inspect_result")
+ ctx.executed_block_fingerprints["inspect_result"] = set(
+ workflow_block_fingerprints(workflow_yaml)["inspect_result"]
+ )
ctx.secret_scrub_values.append(fixture["registered_secret_value"])
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
diff --git a/tests/unit/test_copilot_narrative_activity.py b/tests/unit/test_copilot_narrative_activity.py
index 214c1ebf1..cc0b68d70 100644
--- a/tests/unit/test_copilot_narrative_activity.py
+++ b/tests/unit/test_copilot_narrative_activity.py
@@ -303,8 +303,20 @@ workflow_definition:
assert payload["review"] == {
"blocks": [
- {"label": "existing", "blockType": "task", "change": "changed", "neverTested": False},
- {"label": "added", "blockType": "task", "change": "added", "neverTested": True},
+ {
+ "label": "existing",
+ "blockType": "task",
+ "change": "changed",
+ "neverTested": False,
+ "coverage": "current_source",
+ },
+ {
+ "label": "added",
+ "blockType": "task",
+ "change": "added",
+ "neverTested": True,
+ "coverage": "never_run",
+ },
],
"duplicateWrites": [],
}
@@ -341,7 +353,13 @@ workflow_definition:
assert payload["review"] == {
"blocks": [
- {"label": "first_draft", "blockType": "task", "change": "added", "neverTested": True},
+ {
+ "label": "first_draft",
+ "blockType": "task",
+ "change": "added",
+ "neverTested": True,
+ "coverage": "never_run",
+ },
],
"duplicateWrites": [],
}
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 8a7b90521..bae8591a1 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -39,6 +39,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
normalize_response_scaffolding,
)
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
from skyvern.forge.sdk.copilot.tools import (
_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL,
NATIVE_TOOLS,
@@ -68,6 +69,9 @@ def _policy(**overrides: object) -> RequestPolicy:
return RequestPolicy(**defaults)
+_COVERED_DRAFT_YAML = """title: Draft\nworkflow_definition:\n parameters: []\n blocks:\n - block_type: task\n label: step\n prompt: Do it\n"""
+
+
def _ctx(**overrides: object) -> CopilotContext:
defaults = dict(
organization_id="org-1",
@@ -1643,7 +1647,10 @@ def test_output_policy_credential_block_asks_for_credential_confirmation(
def test_output_policy_block_preserves_already_gated_workflow_proposal() -> None:
ctx = _ctx()
ctx.last_workflow = SimpleNamespace(name="draft")
- ctx.last_workflow_yaml = "title: Draft"
+ ctx.last_workflow_yaml = _COVERED_DRAFT_YAML
+ ctx.executed_block_fingerprints = {
+ label: set(values) for label, values in workflow_block_fingerprints(_COVERED_DRAFT_YAML).items()
+ }
ctx.workflow_persisted = True
ctx.last_test_ok = True
@@ -1658,7 +1665,7 @@ def test_output_policy_block_preserves_already_gated_workflow_proposal() -> None
assert result.response_type == "ASK_QUESTION"
assert result.updated_workflow is ctx.last_workflow
- assert result.workflow_yaml == "title: Draft"
+ assert result.workflow_yaml == _COVERED_DRAFT_YAML
assert result.workflow_was_persisted is True
assert result.clear_proposed_workflow is False
assert result.proposal_disposition == "review_tested"
diff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.py
index a448f34c9..30e5d59ca 100644
--- a/tests/unit/test_copilot_run_outcome_event.py
+++ b/tests/unit/test_copilot_run_outcome_event.py
@@ -304,6 +304,7 @@ async def test_blocker_run_emits_not_demonstrated() -> None:
reason_code=final.reason_code,
display_reason=final.display_reason,
workflow_run_id="wr_test",
+ run_completed=False,
)
assert ctx.last_run_outcome_block_labels == final.block_labels
@@ -319,6 +320,7 @@ def test_challenge_failure_records_observation_without_halting_agent() -> None:
reason_code="blocker_reported",
display_reason=run_outcome_display_reason("Human verification challenge blocked the search."),
workflow_run_id="wr_challenge",
+ run_completed=False,
)
assert ctx.last_run_outcome == outcome
assert ctx.last_test_ok is False
@@ -396,6 +398,7 @@ async def test_completion_judge_cannot_overturn_run_output() -> None:
assert outcome == RecordedRunOutcome(
verdict="not_evaluated",
workflow_run_id="wr_test",
+ run_completed=True,
)
assert ctx.completion_verification_result is None
assert ctx.last_test_suspicious_success is False
@@ -457,6 +460,7 @@ async def test_completed_partial_run_does_not_promote_full_workflow() -> None:
assert ctx.last_run_outcome == RecordedRunOutcome(
verdict="not_evaluated",
workflow_run_id="wr_test",
+ run_completed=True,
)
@@ -483,6 +487,7 @@ async def test_failed_run_emits_its_own_outcome() -> None:
assert [frame.verdict for frame in frames] == ["not_demonstrated"]
assert ctx.last_run_outcome is not None
assert ctx.last_run_outcome.reason_code == "blocker_reported"
+ assert ctx.last_run_outcome.run_completed is False
@pytest.mark.asyncio
@@ -823,6 +828,7 @@ def test_completed_run_uses_retained_terminal_output_when_parameter_identity_can
'Recorded output from the latest completed run: {"retrieve_resale_demand_document":'
'{"document_name":"Resale Demand Package (Required Statement of Fees - Demand)"}}'
),
+ run_completed=True,
)
assert ctx.last_test_ok is True
assert ctx.last_full_workflow_test_ok is True
diff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.py
index eabea5bea..d463f6343 100644
--- a/tests/unit/test_copilot_terminal_envelope.py
+++ b/tests/unit/test_copilot_terminal_envelope.py
@@ -14,6 +14,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.terminal_envelope import (
+ MINIMAL_HONEST_STOP,
TerminalOutcomeEnvelope,
assemble_terminal_envelope,
finalize_applied_state,
@@ -456,7 +457,7 @@ def test_render_terminal_message_stopped_not_demonstrated_contains_verbatim_reas
assert all(phrase not in lowered for phrase in forbidden_phrases)
-def test_render_terminal_message_stopped_degraded_envelope_uses_minimal_honest_stop() -> None:
+def test_render_terminal_message_stopped_without_recorded_facts_keeps_the_agent_text() -> None:
envelope = TerminalOutcomeEnvelope(
next_state="stopped",
verified=False,
@@ -464,11 +465,30 @@ def test_render_terminal_message_stopped_degraded_envelope_uses_minimal_honest_s
run_display_reason=None,
response_kind="stopped",
)
+ message = "The portal does not expose invoices; archived statements are emailed instead."
- rendered, replaced = render_terminal_message(envelope, "legacy", cancelled=False)
+ rendered, replaced = render_terminal_message(envelope, message, cancelled=False)
+
+ # The agent's own explanation survives intact, but a stopped turn with no run behind
+ # it still says so -- the renderer cannot tell an honest explanation from a claim.
+ assert replaced is True
+ assert rendered.startswith(message)
+ assert rendered.endswith(MINIMAL_HONEST_STOP)
+
+
+def test_render_terminal_message_stopped_falls_back_to_honest_stop_without_agent_text() -> None:
+ envelope = TerminalOutcomeEnvelope(
+ next_state="stopped",
+ verified=False,
+ run_verdict=None,
+ run_display_reason=None,
+ response_kind="stopped",
+ )
+
+ rendered, replaced = render_terminal_message(envelope, " ", cancelled=False)
assert replaced is True
- assert rendered == "I stopped without confirming the goal was met."
+ assert rendered == MINIMAL_HONEST_STOP
def test_render_terminal_message_stopped_with_a_recorded_run_reports_that_it_ran() -> None:
@@ -476,6 +496,7 @@ def test_render_terminal_message_stopped_with_a_recorded_run_reports_that_it_ran
next_state="stopped",
verified=False,
run_verdict="not_evaluated",
+ run_completed=True,
run_display_reason=None,
response_kind="stopped",
)
@@ -483,7 +504,14 @@ def test_render_terminal_message_stopped_with_a_recorded_run_reports_that_it_ran
rendered, replaced = render_terminal_message(envelope, "legacy", cancelled=False)
assert replaced is True
- assert rendered == "I ran the workflow. The latest recorded run completed."
+ assert rendered == "legacy. The recorded run completed, and its outcome was not evaluated."
+
+ # Lifecycle is read from the recorded run, never inferred from the verdict: without a
+ # recorded completion the same verdict may not claim the run finished.
+ unknown_lifecycle = envelope.model_copy(update={"run_completed": None})
+ rendered_unknown, _ = render_terminal_message(unknown_lifecycle, "legacy", cancelled=False)
+ assert rendered_unknown == "legacy. The recorded run's outcome was not evaluated."
+ assert "completed" not in rendered_unknown
def test_render_terminal_message_no_run_blocker_stop_keeps_blocker_evidence() -> None:
@@ -497,10 +525,12 @@ def test_render_terminal_message_no_run_blocker_stop_keeps_blocker_evidence() ->
response_kind="stopped",
)
- rendered, replaced = render_terminal_message(envelope, "legacy", cancelled=False)
+ message = "The portal does not expose invoices; archived statements are emailed instead."
+
+ rendered, replaced = render_terminal_message(envelope, message, cancelled=False)
assert replaced is True
- assert rendered.startswith("I stopped without confirming the goal was met.")
+ assert rendered.startswith(message)
assert blocker in rendered
@@ -596,18 +626,28 @@ def test_render_terminal_message_keeps_deadline_copy_on_stopped_fallthrough() ->
assert envelope.next_state == "stopped"
assert envelope.response_kind == "stopped"
- assert rendered == message
+ assert rendered.startswith(message)
assert rendered != "I stopped without confirming the goal was met."
assert replaced is True
-def test_render_terminal_message_still_generic_without_deadline_cause() -> None:
- envelope = _assemble(proposal_disposition="auto_applicable", run_outcomes=[])
+def test_render_terminal_message_under_budget_stop_keeps_the_reply_and_names_the_unevaluated_outcome() -> None:
+ envelope = _assemble(
+ proposal_disposition="auto_applicable",
+ run_outcomes=[RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1", run_completed=True)],
+ blocks_run_this_turn=1,
+ )
+ message = "I built and end-to-end tested the workflow, and verified the account and date range match."
- rendered, replaced = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_DEFAULT, cancelled=False)
+ rendered, replaced = render_terminal_message(envelope, message, cancelled=False)
- assert rendered == "I stopped without confirming the goal was met."
+ assert envelope.next_state == "stopped"
+ assert envelope.response_kind == "stopped"
assert replaced is True
+ assert rendered.startswith(message)
+ assert "1 block ran this turn." in rendered
+ assert "The recorded run completed, and its outcome was not evaluated." in rendered
+ assert "The latest recorded run completed." not in rendered
def test_render_terminal_message_held_draft_text_survives_with_deadline_cause() -> None:
@@ -617,7 +657,7 @@ def test_render_terminal_message_held_draft_text_survives_with_deadline_cause()
rendered, replaced = render_terminal_message(envelope, message, cancelled=False)
assert envelope.next_state == "proposal_pending"
- assert rendered == message
+ assert rendered.startswith(message)
assert replaced is True
@@ -677,7 +717,9 @@ def test_run_without_output_projects_unverified_blocker_never_success() -> None:
assert finalized.response_kind == "stopped"
assert finalized.run_output_report is None
assert finalized.blocker_reason == blocker
- assert rendered == agent_module._TIMEOUT_REPLY_DEFAULT
+ assert rendered.startswith(agent_module._TIMEOUT_REPLY_DEFAULT)
+ assert "The recorded run's outcome was not evaluated." in rendered
+ assert "The turn reached its time limit." in rendered
assert replaced is True
assert "completed" not in rendered.lower()
assert "success" not in rendered.lower()
@@ -777,3 +819,98 @@ def test_max_turns_exit_has_no_cause_when_neither_latch_is_set() -> None:
assert result.terminal_envelope is not None
assert result.terminal_envelope["terminal_cause"] is None
+
+
+def test_a_deadline_turn_states_blocks_run_evaluation_and_the_time_limit() -> None:
+ envelope = _assemble(
+ proposal_disposition="review_tested",
+ run_outcomes=[RecordedRunOutcome(verdict="not_evaluated", run_completed=True)],
+ terminal_cause="deadline_expired",
+ blocks_run_this_turn=1,
+ )
+
+ rendered, replaced = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_TESTED, cancelled=False)
+
+ assert "1 block ran this turn." in rendered
+ assert "The recorded run completed, and its outcome was not evaluated." in rendered
+ assert "reached its time limit" in rendered
+ assert "failed" not in rendered.lower()
+ assert replaced is True
+
+
+def test_an_unevaluated_outcome_alone_never_claims_the_run_finished() -> None:
+ envelope = _assemble(
+ proposal_disposition="review_tested",
+ run_outcomes=[_run_outcome("not_evaluated")],
+ terminal_cause="deadline_expired",
+ blocks_run_this_turn=1,
+ )
+
+ rendered, _ = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_TESTED, cancelled=False)
+
+ assert "The recorded run's outcome was not evaluated." in rendered
+ assert "run completed" not in rendered
+
+
[email protected]("run_outcomes", [[], [_run_outcome("evaluating")]])
+def test_a_deadline_turn_without_a_settled_verdict_states_only_what_it_knows(
+ run_outcomes: list[RecordedRunOutcome],
+) -> None:
+ envelope = _assemble(
+ proposal_disposition="review_tested",
+ run_outcomes=run_outcomes,
+ terminal_cause="deadline_expired",
+ blocks_run_this_turn=2,
+ )
+
+ rendered, _ = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_TESTED, cancelled=False)
+
+ assert "2 blocks ran this turn." in rendered
+ assert "reached its time limit" in rendered
+ assert "not evaluated" not in rendered
+
+
+def test_a_deadline_turn_with_no_run_facts_leaves_the_agent_copy_alone() -> None:
+ envelope = _assemble(proposal_disposition="review_tested", terminal_cause="deadline_expired")
+
+ rendered, replaced = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_UNVALIDATED, cancelled=False)
+
+ assert rendered == agent_module._TIMEOUT_REPLY_UNVALIDATED
+ assert replaced is True
+
+
+def test_a_deadline_turn_omits_the_block_count_it_was_never_given() -> None:
+ envelope = _assemble(
+ proposal_disposition="review_tested",
+ run_outcomes=[_run_outcome("not_evaluated")],
+ terminal_cause="deadline_expired",
+ )
+
+ rendered, _ = render_terminal_message(envelope, agent_module._TIMEOUT_REPLY_TESTED, cancelled=False)
+
+ assert "ran this turn" not in rendered
+ assert "reached its time limit" in rendered
+
+
+def test_the_envelope_carries_the_recorded_run_id() -> None:
+ envelope = _assemble(run_outcomes=[RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1")])
+
+ assert envelope.run_id == "wr_1"
+
+
+def test_run_lifecycle_and_run_id_name_the_same_archived_outcome() -> None:
+ completed = _assemble(
+ run_outcomes=[RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_1", run_completed=True)]
+ )
+
+ assert completed.run_id == "wr_1"
+ assert completed.run_completed is True
+
+ halted = _assemble(
+ run_outcomes=[
+ RecordedRunOutcome(verdict="not_demonstrated", workflow_run_id="wr_2", run_completed=False),
+ ]
+ )
+
+ assert halted.run_id == "wr_2"
+ assert halted.run_completed is False
diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.py
index 9351e1390..0ce67ef86 100644
--- a/tests/unit/test_copilot_timeout_wip.py
+++ b/tests/unit/test_copilot_timeout_wip.py
@@ -37,6 +37,28 @@ from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind
+def _draft_yaml(marker: str) -> str:
+ return (
+ f"title: {marker}\nworkflow_definition:\n parameters: []\n"
+ f" blocks:\n - block_type: task\n label: step\n prompt: {marker}\n"
+ )
+
+
+_DRAFT_V1 = _draft_yaml("v1")
+_DRAFT_GOOD = _draft_yaml("good")
+_DRAFT_BROKEN = _draft_yaml("broken")
+_DRAFT_TESTED = _draft_yaml("tested")
+_DRAFT_IN_FLIGHT = _draft_yaml("in-flight")
+
+
+def _all_draft_receipts() -> dict[str, set[str]]:
+ receipts: dict[str, set[str]] = {}
+ for draft in (_DRAFT_V1, _DRAFT_GOOD, _DRAFT_BROKEN, _DRAFT_TESTED, _DRAFT_IN_FLIGHT):
+ for label, values in workflow_block_fingerprints(draft).items():
+ receipts.setdefault(label, set()).update(values)
+ return receipts
+
+
def _ctx(
*,
last_workflow: object | None,
@@ -62,6 +84,8 @@ def _ctx(
ctx.test_after_update_done = last_test_ok is not None
ctx.last_update_block_count = None
ctx.request_policy.selected_connected_account_id = None
+ ctx.persisted_workflow_yaml = None
+ ctx.executed_block_fingerprints = _all_draft_receipts()
return ctx
@@ -91,10 +115,10 @@ def _overwrite_ctx(*, last_test_ok: bool | None) -> MagicMock:
in_flight = MagicMock(name="wf-in-flight")
return _ctx(
last_workflow=in_flight,
- last_workflow_yaml="version: in-flight",
+ last_workflow_yaml=_DRAFT_IN_FLIGHT,
last_test_ok=last_test_ok,
last_good_workflow=good,
- last_good_workflow_yaml="version: good",
+ last_good_workflow_yaml=_DRAFT_GOOD,
)
@@ -112,16 +136,16 @@ def _state_ctx(state_kind: str) -> tuple[MagicMock, object | None]:
return _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None), None
wf = MagicMock(name="wf")
if state_kind == "untested":
- return _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None), wf
+ return _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None), wf
if state_kind == "failed_test":
- return _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False), wf
+ return _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False), wf
if state_kind == "passing_test":
- return _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=True), wf
+ return _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=True), wf
if state_kind == "suspicious_success":
return (
_ctx(
last_workflow=wf,
- last_workflow_yaml="version: '1.0'",
+ last_workflow_yaml=_DRAFT_V1,
last_test_ok=None,
last_test_suspicious_success=True,
),
@@ -281,7 +305,7 @@ def test_capacity_exit_state_disposition(
assert result.cancelled is expected_cancelled
if surfaces:
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
else:
assert result.updated_workflow is None
assert result.workflow_yaml is None
@@ -314,7 +338,7 @@ class TestBuildTimeoutExitResult:
wf = MagicMock(name="wf")
ctx = _ctx(
last_workflow=wf,
- last_workflow_yaml="version: '1.0'",
+ last_workflow_yaml=_DRAFT_V1,
last_test_ok=None,
last_test_suspicious_success=True,
)
@@ -328,7 +352,7 @@ class TestBuildTimeoutExitResult:
result = _build_timeout_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
assert result.proposal_disposition == "review_untested"
assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED
assert result.clear_proposed_workflow is False
@@ -337,7 +361,7 @@ class TestBuildTimeoutExitResult:
wf = MagicMock(name="wf")
ctx = _ctx(
last_workflow=wf,
- last_workflow_yaml="version: '1.0'",
+ last_workflow_yaml=_DRAFT_V1,
last_test_ok=None,
last_test_suspicious_success=True,
)
@@ -348,7 +372,7 @@ class TestBuildTimeoutExitResult:
result = _build_timeout_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
assert result.proposal_disposition == "review_untested"
assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED
@@ -357,11 +381,11 @@ class TestBuildTimeoutExitResult:
last_good = MagicMock(name="last_good")
ctx = _ctx(
last_workflow=wf,
- last_workflow_yaml="version: 'broken'",
+ last_workflow_yaml=_DRAFT_BROKEN,
last_test_ok=None,
last_test_suspicious_success=True,
last_good_workflow=last_good,
- last_good_workflow_yaml="version: 'tested'",
+ last_good_workflow_yaml=_DRAFT_TESTED,
)
result = _build_timeout_exit_result(ctx, global_llm_context=None)
@@ -369,7 +393,7 @@ class TestBuildTimeoutExitResult:
# The last-good branch stays guarded on a suspicious run -- crediting that
# shape as tested would be the false-success this terminal must not make.
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: 'broken'"
+ assert result.workflow_yaml == _DRAFT_BROKEN
assert result.proposal_disposition == "review_untested"
assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED
@@ -377,26 +401,26 @@ class TestBuildTimeoutExitResult:
class TestBuildCancelledExitResult:
def test_total_timeout_latch_routes_cancel_to_timeout_wip(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)
ctx.copilot_total_timeout_exceeded = True
result = _build_cancelled_exit_result(ctx, global_llm_context=None)
assert result.cancelled is False
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
assert result.proposal_disposition == "review_untested"
assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED
def test_regular_cancel_uses_cancel_wip_path(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)
result = _build_cancelled_exit_result(ctx, global_llm_context=None)
assert result.cancelled is True
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
assert result.proposal_disposition == "review_untested"
assert result.user_response == _CANCEL_REPLY_UNVALIDATED
@@ -417,40 +441,40 @@ class TestBuildUnexpectedErrorExitResult:
def test_untested_workflow_surfaces_as_unvalidated_wip(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)
result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
assert result.proposal_disposition == "review_untested"
assert result.user_response == _UNEXPECTED_ERROR_REPLY_UNVALIDATED
def test_passing_test_surfaces_as_tested_proposal(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=True)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=True)
result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
assert result.proposal_disposition == "review_tested"
assert result.user_response == _UNEXPECTED_ERROR_REPLY_TESTED
def test_failed_test_surfaces_proposal_as_unvalidated(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)
result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
assert result.proposal_disposition == "review_untested"
assert result.user_response == _UNEXPECTED_ERROR_REPLY_UNVALIDATED
def test_failed_test_uses_recorded_blocker_reply(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)
ctx.last_update_block_count = 3
ctx.latest_diagnosis_repair_contract = _blocker_contract(
"Browser session was no longer reachable.",
@@ -466,7 +490,7 @@ class TestBuildUnexpectedErrorExitResult:
def test_failed_test_scrubs_recorded_internal_tool_instruction(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)
ctx.last_update_block_count = 3
ctx.latest_diagnosis_repair_contract = _blocker_contract(
"Less than 90 seconds remain in this Copilot turn after the previous workflow run failed. "
@@ -485,7 +509,7 @@ class TestBuildUnexpectedErrorExitResult:
def test_aborted_test_surfaces_unvalidated_draft_with_recorded_blocker_reply(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)
ctx.test_after_update_done = True
ctx.last_update_block_count = 4
ctx.latest_diagnosis_repair_contract = _blocker_contract(
@@ -505,7 +529,7 @@ class TestBuildUnexpectedErrorExitResult:
def test_browser_only_blocker_does_not_claim_tested_and_redacts_internal_details(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=None)
ctx.test_after_update_done = False
ctx.last_update_block_count = 2
ctx.latest_diagnosis_repair_contract = _blocker_contract(
@@ -530,7 +554,7 @@ class TestBuildUnexpectedErrorExitResult:
from skyvern.forge.sdk.copilot.tools.run_execution import _copilot_sandbox_unavailable_result
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)
ctx.test_after_update_done = True
ctx.last_update_block_count = 6
ctx.latest_diagnosis_repair_contract = build_diagnosis_repair_contract(
@@ -548,7 +572,7 @@ class TestBuildUnexpectedErrorExitResult:
wf = MagicMock(name="wf")
ctx = _ctx(
last_workflow=wf,
- last_workflow_yaml="version: '1.0'",
+ last_workflow_yaml=_DRAFT_V1,
last_test_ok=None,
last_test_suspicious_success=True,
)
@@ -556,7 +580,7 @@ class TestBuildUnexpectedErrorExitResult:
result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
assert result.proposal_disposition == "review_untested"
assert result.user_response == _UNEXPECTED_ERROR_REPLY_UNVALIDATED
@@ -591,7 +615,7 @@ class TestWipExitSurfacesLastGoodWithForceReviewNotUnvalidated:
result = builder(ctx, global_llm_context=None)
assert result.updated_workflow is ctx.last_good_workflow
- assert result.workflow_yaml == "version: good"
+ assert result.workflow_yaml == _DRAFT_GOOD
assert result.proposal_disposition == "review_tested"
assert result.user_response == tested_reply
assert result.cancelled is expected_cancelled
@@ -654,21 +678,21 @@ workflow_definition:
)
assert "pbs_" not in result.user_response
- def test_cancelled_total_timeout_latch_uses_force_review_not_unvalidated(self) -> None:
+ def test_cancelled_total_timeout_latch_surfaces_the_draft_without_a_tested_claim(self) -> None:
ctx = _overwrite_ctx(last_test_ok=None)
ctx.copilot_total_timeout_exceeded = True
result = _build_cancelled_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is ctx.last_good_workflow
- assert result.proposal_disposition == "review_tested"
+ assert result.proposal_disposition == "review_untested"
assert result.cancelled is False
- assert result.user_response == _TIMEOUT_REPLY_TESTED
+ assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED
class TestTimeoutExitNamesTimeAndDraftState:
def test_untested_draft_names_time_and_that_it_is_unverified(self) -> None:
- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=None)
result = _build_timeout_exit_result(ctx, global_llm_context=None)
@@ -677,7 +701,7 @@ class TestTimeoutExitNamesTimeAndDraftState:
assert "hasn't been verified end-to-end" in result.user_response
def test_tested_draft_names_time_and_that_it_is_tested(self) -> None:
- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=True)
+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=True)
result = _build_timeout_exit_result(ctx, global_llm_context=None)
@@ -696,7 +720,7 @@ class TestTimeoutExitNamesTimeAndDraftState:
def test_untested_draft_with_recorded_failure_still_names_time(self) -> None:
# The common shape: the failed test run is what spent the budget, so a
# recorded failure is present on almost every real deadline expiry.
- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=False)
ctx.copilot_total_timeout_exceeded = True
ctx.last_update_block_count = 1
ctx.latest_diagnosis_repair_contract = _blocker_contract(
@@ -723,7 +747,7 @@ class TestTimeoutExitNamesTimeAndDraftState:
def test_max_turns_exit_keeps_the_recorded_failure_reply(self) -> None:
# The deadline precedence must not leak into sibling capacity exits.
- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=False)
ctx.last_update_block_count = 1
ctx.latest_diagnosis_repair_contract = _blocker_contract(
"could not convert string to float", run_status="failed"
@@ -736,7 +760,7 @@ class TestTimeoutExitNamesTimeAndDraftState:
def test_held_draft_shape_names_time_draft_state_and_keeps_evidence(self) -> None:
# The shape both live deadline runs produced: review_untested / proposal_pending.
- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=None)
ctx.copilot_total_timeout_exceeded = True
ctx.last_update_block_count = 1
ctx.latest_diagnosis_repair_contract = _blocker_contract("Run completed", run_status="completed")
@@ -751,7 +775,7 @@ class TestTimeoutExitNamesTimeAndDraftState:
def test_interrupted_run_is_not_reported_as_a_failed_verification(self) -> None:
# A budget-paced halt was interrupted, not disproven. The deadline copy already says the
# work is unverified, so appending "the last test did not verify" would mis-attribute twice.
- ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx = _ctx(last_workflow=MagicMock(name="wf"), last_workflow_yaml=_DRAFT_V1, last_test_ok=False)
ctx.copilot_total_timeout_exceeded = True
ctx.last_update_block_count = 3
ctx.latest_diagnosis_repair_contract = _blocker_contract(
@@ -830,19 +854,19 @@ class TestDeadlineTerminalHandsOverRecordedDraft:
def test_failed_test_draft_survives_the_deadline(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)
ctx.copilot_total_timeout_exceeded = True
ctx.last_failure_category_top = None
result = _build_timeout_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is wf
- assert result.workflow_yaml == "version: '1.0'"
+ assert result.workflow_yaml == _DRAFT_V1
assert result.proposal_disposition == "review_untested"
def test_failed_test_reply_names_the_failure(self) -> None:
wf = MagicMock(name="wf")
- ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=_DRAFT_V1, last_test_ok=False)
ctx.copilot_total_timeout_exceeded = True
ctx.last_failure_category_top = None
ctx.last_update_block_count = 2
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 314a9c92a..bd166559d 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -138,6 +138,7 @@ def _terminal_payload(
workflow_applied: bool,
workflow_mutated: bool = True,
workflow_attempted: bool = True,
+ blocks_run_this_turn: int = 0,
) -> dict[str, Any]:
envelope = assemble_terminal_envelope(
response_type="REPLY",
@@ -150,6 +151,7 @@ def _terminal_payload(
attempted="Attempted full run.",
workflow_mutated=workflow_mutated,
workflow_attempted=workflow_attempted,
+ blocks_run_this_turn=blocks_run_this_turn,
)
assert envelope is not None
return envelope.model_dump(mode="json")
@@ -662,7 +664,7 @@ async def test_finalise_normal_turn_logs_render_decision(
decisions = [entry for entry in logs if entry["event"] == "copilot_terminal_render_decision"]
assert len(decisions) == 1
assert decisions[0]["flag_enabled"] is flag_enabled
- # This fixture's envelope is a stopped/stopped shape, so flag-on replaces.
+ # This fixture's envelope is a stopped/stopped shape, so flag-on appends recorded facts.
assert decisions[0]["replaced"] is flag_enabled
assert decisions[0]["next_state"] == "stopped"
assert decisions[0]["response_kind"] == "stopped"
@@ -707,7 +709,7 @@ async def test_finalise_normal_turn_flag_on_stopped_envelope_renders_terminal_te
agent_result=agent_result,
)
- expected = "I stopped without confirming the goal was met."
+ expected = "done. 0 blocks ran this turn."
response_frame = stream.send.await_args.args[0]
assert isinstance(response_frame, WorkflowCopilotStreamResponseUpdate)
assert response_frame.message == expected