Skip to content

Automate browser based workflows with AI

fix(SKY-13445): render interim copilot build-test runs neutrally, not as an amber outcome warning (#8067)

Celal Zamanoğlud044bec7f656
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx
index be6266111..7bf9f0a7b 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx
@@ -155,6 +155,7 @@ describe("NarrativeView — phase checklist (SKY-11970)", () => {
],
lastRunOutcome: {
verdict: "not_demonstrated",
+ role: "interim_build_test",
displayReason: "outcome not confirmed",
activitySeqAtVerdict: 2,
},
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx
index 93cd1b23a..680710d3d 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx
@@ -63,6 +63,24 @@ const LONG_OUTCOME_REASON =
"The verification challenge kept reappearing after submit, and each retry landed back on the same gate instead of the requested destination page.";
const SHORT_OUTCOME_REASON = "A verification challenge prevented confirmation";
+const interimRunTurn = (): TurnNarrativeState => ({
+ ...inFlightTurn(),
+ blocks: [
+ {
+ ...completedBlock("block_1"),
+ outcome: "not_demonstrated",
+ outcomeRole: "interim_build_test",
+ outcomeReason: SHORT_OUTCOME_REASON,
+ },
+ ],
+ lastRunOutcome: {
+ verdict: "not_demonstrated",
+ role: "interim_build_test",
+ displayReason: SHORT_OUTCOME_REASON,
+ activitySeqAtVerdict: 0,
+ },
+});
+
afterEach(() => {
cleanup();
});
@@ -125,6 +143,64 @@ describe("NarrativeView collapse default", () => {
).toBeTruthy();
});
+ it.each([false, true])(
+ "renders an interim completed row as neutral in uxV1=%s",
+ (uxV1) => {
+ render(<NarrativeView turn={interimRunTurn()} uxV1={uxV1} />);
+
+ const row = screen.getByTitle("Highlight block_1 on canvas");
+ expect(row.textContent).toContain("ran");
+ expect(row.textContent).not.toContain("!");
+ expect(row.textContent).not.toContain("✓");
+ expect(screen.queryByText(/Outcome not confirmed/)).toBeNull();
+ expect(screen.queryByText(SHORT_OUTCOME_REASON)).toBeNull();
+ },
+ );
+
+ it.each([undefined, "adjudicated" as const])(
+ "keeps role=%s not-demonstrated rows amber",
+ (outcomeRole) => {
+ const block: BlockState = {
+ ...completedBlock("block_1"),
+ outcome: "not_demonstrated",
+ outcomeReason: SHORT_OUTCOME_REASON,
+ ...(outcomeRole ? { outcomeRole } : {}),
+ };
+ render(<NarrativeView turn={{ ...inFlightTurn(), blocks: [block] }} />);
+
+ const row = screen.getByTitle("Highlight block_1 on canvas");
+ expect(row.textContent).toContain("!");
+ expect(row.textContent).not.toContain("✓");
+ expect(screen.getByText(/Outcome not confirmed/)).toBeTruthy();
+ expect(screen.getByText(new RegExp(SHORT_OUTCOME_REASON))).toBeTruthy();
+ },
+ );
+
+ it("hydrates an interim outcome into the same neutral row treatment", () => {
+ const hydrated = hydrateNarrativeFromPayload({
+ turnId: "turn-hydrated-interim",
+ turnIndex: 0,
+ mode: "build",
+ blocks: [
+ {
+ ...completedBlock("block_1"),
+ outcome: "not_demonstrated",
+ outcomeRole: "interim_build_test",
+ outcomeReason: SHORT_OUTCOME_REASON,
+ },
+ ],
+ terminal: null,
+ })!;
+ render(<NarrativeView turn={hydrated} />);
+
+ const row = screen.getByTitle("Highlight block_1 on canvas");
+ expect(row.textContent).toContain("ran");
+ expect(row.textContent).not.toContain("!");
+ expect(row.textContent).not.toContain("✓");
+ expect(screen.queryByText(/Outcome not confirmed/)).toBeNull();
+ expect(screen.queryByText(SHORT_OUTCOME_REASON)).toBeNull();
+ });
+
it("expands via the summary card and re-collapses via the labeled control", () => {
render(<NarrativeView turn={terminalBuildTurn()} />);
const head = () =>
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts
index 36a3fd1e6..e8e1cb1f6 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts
@@ -1019,6 +1019,95 @@ describe("computeTurnSummary — typed terminal adjudication", () => {
expect(summary.glyph).toBe("!");
});
+ it("suppresses the live turn warning for an interim last-run outcome", () => {
+ const summary = computeTurnSummary(
+ buildTurn({
+ responseKind: "clarify",
+ responseType: "REPLY",
+ verifiedSuccess: false,
+ lastRunOutcome: {
+ verdict: "not_demonstrated",
+ role: "interim_build_test",
+ displayReason: "The workflow still needs its extraction block.",
+ activitySeqAtVerdict: 3,
+ },
+ }),
+ );
+
+ expect(summary.headline).toBe("Question");
+ expect(summary.accent).toBe("qa");
+ });
+
+ it("suppresses the live turn warning for an interim per-block outcome", () => {
+ const summary = computeTurnSummary(
+ buildTurn({
+ responseKind: "clarify",
+ responseType: "REPLY",
+ verifiedSuccess: false,
+ blocks: [
+ {
+ ...summaryBlock("open_search"),
+ outcome: "not_demonstrated",
+ outcomeRole: "interim_build_test",
+ outcomeReason: "The workflow still needs its extraction block.",
+ },
+ ],
+ }),
+ );
+
+ expect(summary.headline).toBe("Question");
+ expect(summary.accent).toBe("qa");
+ });
+
+ it("keeps an explicitly adjudicated outcome warning", () => {
+ const summary = computeTurnSummary(
+ buildTurn({
+ responseKind: "clarify",
+ responseType: "REPLY",
+ verifiedSuccess: false,
+ lastRunOutcome: {
+ verdict: "not_demonstrated",
+ role: "adjudicated",
+ displayReason: "The goal was not demonstrated.",
+ activitySeqAtVerdict: 3,
+ },
+ }),
+ );
+
+ expect(summary.headline).toBe("Outcome not confirmed");
+ expect(summary.accent).toBe("warn");
+ });
+
+ it("keeps the terminal envelope authoritative over interim live state", () => {
+ const summary = computeTurnSummary(
+ buildTurn({
+ responseKind: "clarify",
+ responseType: "REPLY",
+ verifiedSuccess: false,
+ terminalEnvelope: {
+ runVerdict: "not_demonstrated",
+ runDisplayReason: "The final run did not demonstrate the goal.",
+ },
+ lastRunOutcome: {
+ verdict: "not_demonstrated",
+ role: "interim_build_test",
+ displayReason: "The workflow still needs its extraction block.",
+ activitySeqAtVerdict: 3,
+ },
+ blocks: [
+ {
+ ...summaryBlock("open_search"),
+ outcome: "not_demonstrated",
+ outcomeRole: "interim_build_test",
+ },
+ ],
+ }),
+ );
+
+ expect(summary.headline).toBe("Outcome not confirmed");
+ expect(summary.accent).toBe("warn");
+ });
+
it("uses hydrated not_demonstrated block outcomes when lastRunOutcome is absent", () => {
const turn = hydrateNarrativeFromPayload(
reproClarifyPayload({
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx
index 7050553e6..71b9f5d7a 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx
@@ -19,6 +19,7 @@ import {
effectiveMode,
formatElapsed,
isBlockOk,
+ isInterimOutcome,
latestBlocksByLabel,
notConfirmedOutcome,
parseUtcIsoMs,
@@ -353,11 +354,18 @@ function FBlockRun({
const isRunning = block.state === "running";
const isCompleted = block.state === "completed";
const isEvaluating = isCompleted && block.outcome === "evaluating";
+ const isInterimNotDemonstrated =
+ isCompleted &&
+ block.outcome === "not_demonstrated" &&
+ isInterimOutcome(block.outcomeRole);
// A row stuck in `evaluating` at turn end (dropped stream) renders the
// neutral "ran" treatment — never the live verifying beat, never green.
const isVerifying = isEvaluating && !turnEnded;
- const isRanNeutral = isEvaluating && turnEnded;
- const isOutcomeNotShown = isCompleted && block.outcome === "not_demonstrated";
+ const isRanNeutral = (isEvaluating && turnEnded) || isInterimNotDemonstrated;
+ const isOutcomeNotShown =
+ isCompleted &&
+ block.outcome === "not_demonstrated" &&
+ !isInterimNotDemonstrated;
const isOk = isBlockOk(block);
const isFail = block.state === "failed";
const isDraft = block.state === "drafted";
diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts
index 45cbc9d26..b8a4b4c81 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts
@@ -626,6 +626,7 @@ describe("derivePhases — redraft re-activation (SKY-11970 pin)", () => {
blocks: [block({ state: "completed" })],
lastRunOutcome: {
verdict: "not_demonstrated",
+ role: "interim_build_test",
displayReason: "outcome not confirmed",
activitySeqAtVerdict: 2,
},
@@ -826,12 +827,21 @@ describe("derivePhases — redraft re-activation (SKY-11970 pin)", () => {
});
describe("derivePhases — test-run count on the not-confirmed stub (SKY-11339)", () => {
- const notConfirmedTurn = (runToolNames: string[]): TurnNarrativeState =>
+ const notConfirmedTurn = (
+ runToolNames: string[],
+ outcomeRole?: BlockState["outcomeRole"],
+ ): TurnNarrativeState =>
turn({
terminal: "response",
designEnded: true,
draft: { blockCount: 1, blockLabels: ["block_1"], summary: null },
- blocks: [block({ state: "completed", outcome: "not_demonstrated" })],
+ blocks: [
+ block({
+ state: "completed",
+ outcome: "not_demonstrated",
+ outcomeRole,
+ }),
+ ],
designActivity: runToolNames.map((toolName, i) =>
entry({ id: `r${i}`, kind: "tool_call", toolName }),
),
@@ -852,4 +862,18 @@ describe("derivePhases — test-run count on the not-confirmed stub (SKY-11339)"
const rows = derivePhases(notConfirmedTurn(["update_and_run_blocks"]));
expect(phase(rows, "test").stub).toBe("· not confirmed");
});
+
+ it("keeps an explicitly adjudicated negative outcome in the alarm stub", () => {
+ const rows = derivePhases(
+ notConfirmedTurn(["update_and_run_blocks"], "adjudicated"),
+ );
+ expect(phase(rows, "test").stub).toBe("· not confirmed");
+ });
+
+ it("suppresses the alarm stub for an interim build test", () => {
+ const rows = derivePhases(
+ notConfirmedTurn(["update_and_run_blocks"], "interim_build_test"),
+ );
+ expect(phase(rows, "test").stub).not.toContain("not confirmed");
+ });
});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts
index 67df75772..2037bfbab 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts
@@ -12,6 +12,7 @@ import {
TurnNarrativeState,
condenseActivityEntries,
formatElapsed,
+ isInterimOutcome,
latestBlocksByLabel,
parseUtcIsoMs,
toolCallIdOf,
@@ -252,7 +253,7 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {
const isCancelled = turn.cancelled === true;
const anyFailed = latestBlocks.some((b) => b.state === "failed");
const anyNotDemonstrated = latestBlocks.some(
- (b) => b.outcome === "not_demonstrated",
+ (b) => b.outcome === "not_demonstrated" && !isInterimOutcome(b.outcomeRole),
);
function reached(id: CopilotPhaseId): boolean {
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.phaseHint.test.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.phaseHint.test.ts
index 0823ef36c..bbbb5396b 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.phaseHint.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.phaseHint.test.ts
@@ -196,6 +196,7 @@ describe("applyNarrativeEvent — lastRunOutcome", () => {
);
expect(s.lastRunOutcome).toEqual({
verdict: "not_demonstrated",
+ role: "adjudicated",
displayReason: "outcome not confirmed",
activitySeqAtVerdict: 1,
});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts
index 58020afed..85ac51538 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts
@@ -97,8 +97,30 @@ describe("applyNarrativeEvent — run_outcome", () => {
expect(b.outcomeReason).toBe(
"The search stayed gated by a verification challenge.",
);
+ expect(b.outcomeRole).toBe("adjudicated");
expect(isBlockOk(b)).toBe(false);
}
+ expect(s.lastRunOutcome?.role).toBe("adjudicated");
+ expect(notConfirmedOutcome(s)?.verdict).toBe("not_demonstrated");
+ });
+
+ it("interim verdict stays non-green but suppresses live not-confirmed alarms", () => {
+ const s = reduce([
+ ...bothBlocksRan,
+ runOutcome({
+ verdict: "not_demonstrated",
+ role: "interim_build_test",
+ reason_code: "outcome_not_demonstrated",
+ display_reason: "The workflow still needs its extraction block.",
+ }),
+ ]);
+
+ for (const b of s.blocks) {
+ expect(b.outcomeRole).toBe("interim_build_test");
+ expect(isBlockOk(b)).toBe(false);
+ }
+ expect(s.lastRunOutcome?.role).toBe("interim_build_test");
+ expect(notConfirmedOutcome(s)).toBeNull();
});
it("evaluating hold withholds the success affordance until the final frame", () => {
@@ -255,13 +277,14 @@ describe("hydrateNarrativeFromPayload — outcome", () => {
blocks,
});
- it("round-trips outcome/outcomeReason so reload renders like the live stream", () => {
+ it("round-trips outcome/outcomeReason/outcomeRole so reload renders like the live stream", () => {
const live = reduce([
turnStart(),
blockProgress({ block_label: "search_person", status: "running" }),
blockProgress({ block_label: "search_person", status: "completed" }),
runOutcome({
verdict: "not_demonstrated",
+ role: "interim_build_test",
workflow_run_block_ids: ["wrb_search_person"],
block_labels: ["search_person"],
reason_code: "blocker_reported",
@@ -276,6 +299,7 @@ describe("hydrateNarrativeFromPayload — outcome", () => {
payloadBlock({
outcome: "not_demonstrated",
outcomeReason: "The search stayed gated by a verification challenge.",
+ outcomeRole: "interim_build_test",
}),
]),
)!;
@@ -284,6 +308,7 @@ describe("hydrateNarrativeFromPayload — outcome", () => {
expect(hydratedRow.state).toBe(liveRow.state);
expect(hydratedRow.outcome).toBe(liveRow.outcome);
expect(hydratedRow.outcomeReason).toBe(liveRow.outcomeReason);
+ expect(hydratedRow.outcomeRole).toBe(liveRow.outcomeRole);
expect(isBlockOk(hydratedRow)).toBe(false);
expect(isBlockOk(liveRow)).toBe(false);
});
@@ -293,6 +318,7 @@ describe("hydrateNarrativeFromPayload — outcome", () => {
const row = hydrated.blocks[0]!;
expect(row.outcome).toBeUndefined();
expect(row.outcomeReason).toBeUndefined();
+ expect(row.outcomeRole).toBeUndefined();
expect(isBlockOk(row)).toBe(true);
});
@@ -303,9 +329,29 @@ describe("hydrateNarrativeFromPayload — outcome", () => {
const row = hydrated.blocks[0]!;
expect(row.outcome).toBeUndefined();
expect(row.outcomeReason).toBeUndefined();
+ expect(row.outcomeRole).toBeUndefined();
expect(isBlockOk(row)).toBe(true);
});
+ it("normalizes absent and unknown persisted roles to adjudicated", () => {
+ const hydrated = hydrateNarrativeFromPayload(
+ payload([
+ payloadBlock({ outcome: "not_demonstrated" }),
+ payloadBlock({
+ label: "search_person_2",
+ outcome: "not_demonstrated",
+ outcomeRole: "future_role",
+ }),
+ ]),
+ )!;
+
+ expect(hydrated.blocks.map((block) => block.outcomeRole)).toEqual([
+ "adjudicated",
+ "adjudicated",
+ ]);
+ expect(notConfirmedOutcome(hydrated)?.verdict).toBe("not_demonstrated");
+ });
+
it("hydrate sweep promotes a stuck-running row without inventing a verdict", () => {
const hydrated = hydrateNarrativeFromPayload(
payload([
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
index d21b60851..20e00462d 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
@@ -7,6 +7,7 @@ import { buildRevealOffsets } from "./actionReveal";
import {
CopilotResponseType,
ProposalDisposition,
+ RunOutcomeRole,
WorkflowCopilotBlockProgressUpdate,
WorkflowCopilotDesignEndUpdate,
WorkflowCopilotDesignStartUpdate,
@@ -88,6 +89,10 @@ export type BlockOutcome =
| "not_demonstrated"
| "not_evaluated";
+export function isInterimOutcome(role: RunOutcomeRole | undefined): boolean {
+ return role === "interim_build_test";
+}
+
export interface TerminalEnvelopeFacts {
runVerdict: BlockOutcome | null;
runDisplayReason: string | null;
@@ -122,6 +127,7 @@ export interface BlockState {
// Undefined when no verdict was recorded (older backend or unadjudicated run).
outcome?: BlockOutcome;
outcomeReason?: string;
+ outcomeRole?: RunOutcomeRole;
lastSeenIteration: number;
// Tool calls and results emitted while this block was running are
// appended here so the expanded card shows what the agent did.
@@ -232,6 +238,7 @@ export interface TurnNarrativeState {
// stopped rather than Draft (accepted).
lastRunOutcome: {
verdict: BlockOutcome;
+ role?: RunOutcomeRole;
displayReason: string | null;
activitySeqAtVerdict: number;
} | null;
@@ -797,6 +804,7 @@ export function applyNarrativeEvent(
// either, so always keep the prior values.
outcome: previousBlock?.outcome,
outcomeReason: previousBlock?.outcomeReason,
+ outcomeRole: previousBlock?.outcomeRole,
recordedActions: previousBlock?.recordedActions,
recordedActionsAt: previousBlock?.recordedActionsAt,
lastSeenIteration: event.iteration,
@@ -822,6 +830,7 @@ export function applyNarrativeEvent(
...b,
outcome: event.verdict,
outcomeReason: event.display_reason ?? undefined,
+ outcomeRole: event.role ?? "adjudicated",
}
: b,
);
@@ -832,6 +841,7 @@ export function applyNarrativeEvent(
// verdict from a prior run cycle within the same turn.
lastRunOutcome: {
verdict: event.verdict,
+ role: event.role ?? "adjudicated",
displayReason: event.display_reason ?? null,
activitySeqAtVerdict: prev.activitySeq,
},
@@ -1179,6 +1189,12 @@ export function hydrateNarrativeFromPayload(
return o;
return undefined;
})();
+ const outcomeRole: RunOutcomeRole | undefined =
+ outcome === undefined
+ ? undefined
+ : obj.outcomeRole === "interim_build_test"
+ ? "interim_build_test"
+ : "adjudicated";
return {
workflowRunBlockId:
typeof obj.workflowRunBlockId === "string"
@@ -1187,6 +1203,7 @@ export function hydrateNarrativeFromPayload(
label: typeof obj.label === "string" ? obj.label : "",
blockType: typeof obj.blockType === "string" ? obj.blockType : "task",
outcome,
+ outcomeRole,
outcomeReason:
typeof obj.outcomeReason === "string" ? obj.outcomeReason : undefined,
state: ((): BlockState["state"] => {
@@ -1402,7 +1419,8 @@ export function notConfirmedOutcome(
: null;
}
if (turn.lastRunOutcome !== null) {
- return turn.lastRunOutcome.verdict === "not_demonstrated"
+ return turn.lastRunOutcome.verdict === "not_demonstrated" &&
+ !isInterimOutcome(turn.lastRunOutcome.role)
? {
verdict: "not_demonstrated",
displayReason: turn.lastRunOutcome.displayReason,
@@ -1411,7 +1429,10 @@ export function notConfirmedOutcome(
}
for (let i = turn.blocks.length - 1; i >= 0; i -= 1) {
const block = turn.blocks[i]!;
- if (block.outcome === "not_demonstrated") {
+ if (
+ block.outcome === "not_demonstrated" &&
+ !isInterimOutcome(block.outcomeRole)
+ ) {
return {
verdict: "not_demonstrated",
displayReason: block.outcomeReason ?? null,
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index bba8b174c..b8a3f36d4 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -247,12 +247,15 @@ export type WorkflowCopilotRunOutcomeVerdict =
| "not_demonstrated"
| "not_evaluated";
+export type RunOutcomeRole = "adjudicated" | "interim_build_test";
+
export interface WorkflowCopilotRunOutcomeUpdate {
type: "run_outcome";
workflow_run_id: string;
workflow_run_block_ids: string[];
block_labels: string[];
verdict: WorkflowCopilotRunOutcomeVerdict;
+ role?: RunOutcomeRole;
reason_code?: string | null;
display_reason?: string | null;
iteration: number;
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 9efa23d35..215b59cfb 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -2176,6 +2176,7 @@ def _build_narrative_payload(
}
if recorded_outcome is not None and label in outcome_labels:
block_entry["outcome"] = recorded_outcome.verdict
+ block_entry["outcomeRole"] = recorded_outcome.role
if recorded_outcome.display_reason is not None:
block_entry["outcomeReason"] = recorded_outcome.display_reason
blocks.append(block_entry)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 8727a18b4..f975cbf07 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -20,6 +20,7 @@ from skyvern.forge.sdk.copilot.result_evidence import (
LoadedResultCompositionEvidence,
loaded_result_target_structure_signature,
)
+from skyvern.forge.sdk.copilot.run_outcome import RunOutcomeRole
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from skyvern.forge.sdk.workflow.models.workflow import Workflow
@@ -75,6 +76,7 @@ class NarrativeBlock(TypedDict):
endedAt: str | None
outcome: NotRequired[str]
outcomeReason: NotRequired[str]
+ outcomeRole: NotRequired[RunOutcomeRole]
class NarrativeOutcomeAdjudication(TypedDict):
diff --git a/skyvern/forge/sdk/copilot/run_outcome.py b/skyvern/forge/sdk/copilot/run_outcome.py
index 22bd60db3..c7b3713d5 100644
--- a/skyvern/forge/sdk/copilot/run_outcome.py
+++ b/skyvern/forge/sdk/copilot/run_outcome.py
@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_RE
RunOutcomeVerdict = Literal["evaluating", "demonstrated", "not_demonstrated", "not_evaluated"]
RecordedRunOutcomeVerdict = Literal["demonstrated", "not_demonstrated", "not_evaluated"]
+RunOutcomeRole = Literal["adjudicated", "interim_build_test"]
RunOutcomeReasonCode = Literal[
"blocker_reported",
"terminal_challenge_blocker",
@@ -42,6 +43,7 @@ class RecordedRunOutcome:
reason_code: RunOutcomeReasonCode | None = None
display_reason: str | None = None
workflow_run_id: str | None = None
+ role: RunOutcomeRole = "adjudicated"
def run_outcome_display_reason(text: str | None) -> str | None:
diff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.py
index b1de056cf..16bfeb8c1 100644
--- a/skyvern/forge/sdk/copilot/terminal_envelope.py
+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py
@@ -168,6 +168,18 @@ def _select_run_outcome_anchor(run_outcomes: Sequence[RecordedRunOutcome]) -> Re
final_outcomes = [outcome for outcome in run_outcomes if outcome.verdict in _FINAL_RUN_VERDICTS]
if not final_outcomes:
return None
+ # An interim build-test verdict is a mid-build "keep building" signal, not an
+ # honest turn outcome, so it must not mask a later adjudicated run: prefer the
+ # last not_demonstrated among adjudicated outcomes. Only when the turn produced
+ # no adjudicated outcome at all — e.g. a suspicious-success repair ceiling whose
+ # de-facto-final run is interim-tagged because the loop stopped rather than
+ # continued — do interim verdicts anchor, preserving the honest terminal amber.
+ adjudicated = [outcome for outcome in final_outcomes if outcome.role != "interim_build_test"]
+ if adjudicated:
+ adjudicated_not_demonstrated = [outcome for outcome in adjudicated if outcome.verdict == "not_demonstrated"]
+ if adjudicated_not_demonstrated:
+ return adjudicated_not_demonstrated[-1]
+ return adjudicated[-1]
last_not_demonstrated = [outcome for outcome in final_outcomes if outcome.verdict == "not_demonstrated"]
if last_not_demonstrated:
return last_not_demonstrated[-1]
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 5eb0d543f..ed041ad38 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -102,6 +102,7 @@ from skyvern.forge.sdk.copilot.run_outcome import (
TERMINAL_CHALLENGE_USER_FACING_REASON,
RecordedRunOutcome,
RunOutcomeReasonCode,
+ RunOutcomeRole,
RunOutcomeVerdict,
run_outcome_display_reason,
trusted_terminal_challenge_category_name,
@@ -3345,9 +3346,15 @@ def _adjudicated_run_outcome(
committed = _same_run_committed_demonstrated_outcome(copilot_ctx)
if committed is not None:
return committed
+ # last_test_suspicious_success can also mark a turn's de-facto-final run — a repair
+ # ceiling stops the build loop rather than continuing — so this interim tag neutralizes
+ # only the live row; the honest turn-level amber is still surfaced by the terminal
+ # envelope, which anchors interim verdicts when no adjudicated outcome exists in the
+ # turn (test_run_anchor_keeps_interim_amber_when_no_adjudicated_outcome).
if copilot_ctx.last_test_suspicious_success:
return RecordedRunOutcome(
verdict="not_demonstrated",
+ role="interim_build_test",
reason_code="outcome_not_demonstrated",
display_reason=run_outcome_display_reason(copilot_ctx.last_test_failure_reason),
)
@@ -3381,6 +3388,7 @@ async def _send_run_outcome_update(
verdict: RunOutcomeVerdict,
reason_code: RunOutcomeReasonCode | None,
display_reason: str | None,
+ role: RunOutcomeRole = "adjudicated",
) -> None:
stream = getattr(copilot_ctx, "stream", None)
if stream is None:
@@ -3397,6 +3405,7 @@ async def _send_run_outcome_update(
workflow_run_block_ids=list(getattr(copilot_ctx, "last_run_blocks_block_ids", []) or []),
block_labels=list(getattr(copilot_ctx, "last_run_blocks_block_labels", []) or []),
verdict=verdict,
+ role=role,
reason_code=reason_code,
display_reason=display_reason,
iteration=iteration,
@@ -3449,6 +3458,7 @@ async def _verify_and_record_run_blocks_result(
copilot_ctx,
result,
verdict=final.verdict,
+ role=final.role,
reason_code=final.reason_code,
display_reason=final.display_reason,
)
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 49bbed637..6045184a5 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -6,7 +6,7 @@ from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
from skyvern.forge.sdk.copilot.context import ProposalDisposition, ResponseType, TurnNarrativePayload
-from skyvern.forge.sdk.copilot.run_outcome import RunOutcomeReasonCode, RunOutcomeVerdict
+from skyvern.forge.sdk.copilot.run_outcome import RunOutcomeReasonCode, RunOutcomeRole, RunOutcomeVerdict
from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
@@ -431,6 +431,7 @@ class WorkflowCopilotRunOutcomeUpdate(BaseModel):
default_factory=list, description="Block labels of the adjudicated run; key the persisted narrative payload"
)
verdict: RunOutcomeVerdict = Field(..., description="Recorded outcome verdict for the run")
+ role: RunOutcomeRole = Field("adjudicated", description="Display scope of the recorded run verdict")
reason_code: RunOutcomeReasonCode | None = Field(
None, description="Machine-readable cause for a not_demonstrated verdict"
)
diff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.py
index a7807625b..3b25f66fe 100644
--- a/tests/unit/test_copilot_run_outcome_event.py
+++ b/tests/unit/test_copilot_run_outcome_event.py
@@ -191,6 +191,20 @@ def _run_outcome_frames(stream: _FakeStream) -> list[WorkflowCopilotRunOutcomeUp
return [frame for frame in stream.sent if isinstance(frame, WorkflowCopilotRunOutcomeUpdate)]
+def test_run_outcome_event_role_defaults_to_adjudicated() -> None:
+ frame = WorkflowCopilotRunOutcomeUpdate.model_validate(
+ {
+ "type": "run_outcome",
+ "workflow_run_id": "wr_test",
+ "verdict": "demonstrated",
+ "iteration": 0,
+ "timestamp": "2026-06-10T00:00:00Z",
+ }
+ )
+
+ assert frame.role == "adjudicated"
+
+
@pytest.mark.asyncio
async def test_blocker_run_emits_hold_then_not_demonstrated() -> None:
result = _blocked_run_result()
@@ -206,6 +220,7 @@ async def test_blocker_run_emits_hold_then_not_demonstrated() -> None:
assert final.workflow_run_block_ids == ["wrb_open_registry_search", "wrb_search_registry_person"]
assert final.block_labels == ["open_registry_search", "search_registry_person"]
assert final.display_reason is not None and "human verification challenge" in final.display_reason
+ assert final.role == "adjudicated"
assert ctx.last_test_suspicious_success is False
assert ctx.last_run_outcome == RecordedRunOutcome(
verdict=final.verdict,
@@ -273,7 +288,9 @@ async def test_empty_data_run_emits_no_meaningful_output() -> None:
frames = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]
assert [frame.verdict for frame in frames] == ["evaluating", "not_demonstrated"]
assert frames[-1].reason_code == "no_meaningful_output"
+ assert frames[-1].role == "adjudicated"
assert ctx.last_test_suspicious_success is True
+ assert ctx.last_run_outcome is not None and ctx.last_run_outcome.role == "adjudicated"
def _terminal_metadata_entry(label: str) -> dict[str, Any]:
@@ -312,9 +329,11 @@ async def test_negative_adjudication_emits_outcome_not_demonstrated(monkeypatch:
final = frames[-1]
assert final.reason_code == "outcome_not_demonstrated"
assert final.display_reason is not None and "did not demonstrate the goal outcome" in final.display_reason
+ assert final.role == "interim_build_test"
assert ctx.last_test_suspicious_success is True
assert ctx.last_full_workflow_test_ok is False
assert ctx.last_run_outcome is not None and ctx.last_run_outcome.verdict == "not_demonstrated"
+ assert ctx.last_run_outcome.role == "interim_build_test"
@pytest.mark.asyncio
@@ -348,6 +367,7 @@ async def test_satisfied_adjudication_emits_demonstrated(monkeypatch: pytest.Mon
frames = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]
assert [frame.verdict for frame in frames] == ["evaluating", "demonstrated"]
assert frames[-1].reason_code is None
+ assert frames[-1].role == "adjudicated"
assert ctx.last_full_workflow_test_ok is True
@@ -422,6 +442,7 @@ async def test_evaluating_hold_gets_final_frame_when_recording_raises(monkeypatc
frames = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]
assert [frame.verdict for frame in frames] == ["evaluating", "not_evaluated"]
+ assert frames[-1].role == "adjudicated"
def test_failed_rerun_clears_prior_recorded_outcome() -> None:
@@ -589,6 +610,7 @@ def test_narrative_payload_stamps_outcome_on_adjudicated_labels() -> None:
verdict="not_demonstrated",
reason_code="blocker_reported",
display_reason="The search form is gated by a human verification challenge.",
+ role="interim_build_test",
)
ctx.last_run_outcome_block_labels = ["open_registry_search", "search_registry_person"]
@@ -599,8 +621,10 @@ def test_narrative_payload_stamps_outcome_on_adjudicated_labels() -> None:
assert by_label[label]["state"] == "completed"
assert by_label[label]["outcome"] == "not_demonstrated"
assert by_label[label]["outcomeReason"] == "The search form is gated by a human verification challenge."
+ assert by_label[label]["outcomeRole"] == "interim_build_test"
assert "outcome" not in by_label["untested_block"]
assert "outcomeReason" not in by_label["untested_block"]
+ assert "outcomeRole" not in by_label["untested_block"]
def test_narrative_payload_without_recorded_outcome_has_no_outcome_keys() -> None:
@@ -611,6 +635,7 @@ def test_narrative_payload_without_recorded_outcome_has_no_outcome_keys() -> Non
for block in payload["blocks"]:
assert "outcome" not in block
assert "outcomeReason" not in block
+ assert "outcomeRole" not in block
class TestGenuineAttemptRunStamp:
diff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.py
index 52e306680..d0f1d5745 100644
--- a/tests/unit/test_copilot_terminal_envelope.py
+++ b/tests/unit/test_copilot_terminal_envelope.py
@@ -105,6 +105,53 @@ def test_anchor_supersession_divergence_is_logged() -> None:
assert not any("anchored a not_demonstrated verdict" in log["event"] for log in logs)
+def _interim_outcome(verdict: str, display_reason: str | None = None) -> RecordedRunOutcome:
+ return RecordedRunOutcome(verdict=verdict, display_reason=display_reason, role="interim_build_test")
+
+
+def test_run_anchor_ignores_interim_not_demonstrated_when_adjudicated_run_follows() -> None:
+ # Happy turn: an interim scout test run goes not_demonstrated mid-build, then the
+ # completed workflow demonstrates the goal. The envelope must not resurface the
+ # interim amber over the later adjudicated success.
+ envelope = _assemble(
+ run_outcomes=[
+ _interim_outcome("not_demonstrated", "The scout has not produced the goal yet."),
+ _run_outcome("demonstrated", "The extraction returned the value."),
+ ]
+ )
+
+ assert envelope.run_verdict == "demonstrated"
+ assert envelope.run_display_reason == "The extraction returned the value."
+
+
+def test_run_anchor_keeps_interim_amber_when_no_adjudicated_outcome() -> None:
+ # Repair ceiling: the loop stops after a suspicious-success run without ever
+ # producing an adjudicated outcome, so the interim not_demonstrated is the turn's
+ # honest terminal verdict and must still anchor amber.
+ envelope = _assemble(
+ run_outcomes=[
+ _interim_outcome("not_demonstrated", "The run completed but did not demonstrate the goal."),
+ ]
+ )
+
+ assert envelope.run_verdict == "not_demonstrated"
+ assert envelope.run_display_reason == "The run completed but did not demonstrate the goal."
+
+
+def test_run_anchor_prefers_adjudicated_not_demonstrated_over_earlier_interim() -> None:
+ # A genuine adjudicated failure on the completed workflow anchors amber even when an
+ # earlier interim run also went not_demonstrated.
+ envelope = _assemble(
+ run_outcomes=[
+ _interim_outcome("not_demonstrated", "Interim scout, still building."),
+ _run_outcome("not_demonstrated", "The extraction returned no value."),
+ ]
+ )
+
+ assert envelope.run_verdict == "not_demonstrated"
+ assert envelope.run_display_reason == "The extraction returned no value."
+
+
@pytest.mark.parametrize(
("response_type", "verified", "workflow_applied", "proposal_disposition", "expected_next_state"),
[