Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-13445): render interim copilot build-test runs neutrally, not as an amber outcome warning (#8067)
Celal Zamanoğlud044bec7f656
diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsxindex be6266111..7bf9f0a7b 100644--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.checklist.test.tsx@@ -155,6 +155,7 @@ describe("NarrativeView — phase checklist (SKY-11970)", () => {],lastRunOutcome: {verdict: "not_demonstrated",+ role: "interim_build_test",displayReason: "outcome not confirmed",activitySeqAtVerdict: 2,},diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsxindex 93cd1b23a..680710d3d 100644--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.collapse.test.tsx@@ -63,6 +63,24 @@ const LONG_OUTCOME_REASON ="The verification challenge kept reappearing after submit, and each retry landed back on the same gate instead of the requested destination page.";const SHORT_OUTCOME_REASON = "A verification challenge prevented confirmation";+const interimRunTurn = (): TurnNarrativeState => ({+ ...inFlightTurn(),+ blocks: [+ {+ ...completedBlock("block_1"),+ outcome: "not_demonstrated",+ outcomeRole: "interim_build_test",+ outcomeReason: SHORT_OUTCOME_REASON,+ },+ ],+ lastRunOutcome: {+ verdict: "not_demonstrated",+ role: "interim_build_test",+ displayReason: SHORT_OUTCOME_REASON,+ activitySeqAtVerdict: 0,+ },+});+afterEach(() => {cleanup();});@@ -125,6 +143,64 @@ describe("NarrativeView collapse default", () => {).toBeTruthy();});+ it.each([false, true])(+ "renders an interim completed row as neutral in uxV1=%s",+ (uxV1) => {+ render(<NarrativeView turn={interimRunTurn()} uxV1={uxV1} />);++ const row = screen.getByTitle("Highlight block_1 on canvas");+ expect(row.textContent).toContain("ran");+ expect(row.textContent).not.toContain("!");+ expect(row.textContent).not.toContain("✓");+ expect(screen.queryByText(/Outcome not confirmed/)).toBeNull();+ expect(screen.queryByText(SHORT_OUTCOME_REASON)).toBeNull();+ },+ );++ it.each([undefined, "adjudicated" as const])(+ "keeps role=%s not-demonstrated rows amber",+ (outcomeRole) => {+ const block: BlockState = {+ ...completedBlock("block_1"),+ outcome: "not_demonstrated",+ outcomeReason: SHORT_OUTCOME_REASON,+ ...(outcomeRole ? { outcomeRole } : {}),+ };+ render(<NarrativeView turn={{ ...inFlightTurn(), blocks: [block] }} />);++ const row = screen.getByTitle("Highlight block_1 on canvas");+ expect(row.textContent).toContain("!");+ expect(row.textContent).not.toContain("✓");+ expect(screen.getByText(/Outcome not confirmed/)).toBeTruthy();+ expect(screen.getByText(new RegExp(SHORT_OUTCOME_REASON))).toBeTruthy();+ },+ );++ it("hydrates an interim outcome into the same neutral row treatment", () => {+ const hydrated = hydrateNarrativeFromPayload({+ turnId: "turn-hydrated-interim",+ turnIndex: 0,+ mode: "build",+ blocks: [+ {+ ...completedBlock("block_1"),+ outcome: "not_demonstrated",+ outcomeRole: "interim_build_test",+ outcomeReason: SHORT_OUTCOME_REASON,+ },+ ],+ terminal: null,+ })!;+ render(<NarrativeView turn={hydrated} />);++ const row = screen.getByTitle("Highlight block_1 on canvas");+ expect(row.textContent).toContain("ran");+ expect(row.textContent).not.toContain("!");+ expect(row.textContent).not.toContain("✓");+ expect(screen.queryByText(/Outcome not confirmed/)).toBeNull();+ expect(screen.queryByText(SHORT_OUTCOME_REASON)).toBeNull();+ });+it("expands via the summary card and re-collapses via the labeled control", () => {render(<NarrativeView turn={terminalBuildTurn()} />);const head = () =>diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.tsindex 36a3fd1e6..e8e1cb1f6 100644--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.test.ts@@ -1019,6 +1019,95 @@ describe("computeTurnSummary — typed terminal adjudication", () => {expect(summary.glyph).toBe("!");});+ it("suppresses the live turn warning for an interim last-run outcome", () => {+ const summary = computeTurnSummary(+ buildTurn({+ responseKind: "clarify",+ responseType: "REPLY",+ verifiedSuccess: false,+ lastRunOutcome: {+ verdict: "not_demonstrated",+ role: "interim_build_test",+ displayReason: "The workflow still needs its extraction block.",+ activitySeqAtVerdict: 3,+ },+ }),+ );++ expect(summary.headline).toBe("Question");+ expect(summary.accent).toBe("qa");+ });++ it("suppresses the live turn warning for an interim per-block outcome", () => {+ const summary = computeTurnSummary(+ buildTurn({+ responseKind: "clarify",+ responseType: "REPLY",+ verifiedSuccess: false,+ blocks: [+ {+ ...summaryBlock("open_search"),+ outcome: "not_demonstrated",+ outcomeRole: "interim_build_test",+ outcomeReason: "The workflow still needs its extraction block.",+ },+ ],+ }),+ );++ expect(summary.headline).toBe("Question");+ expect(summary.accent).toBe("qa");+ });++ it("keeps an explicitly adjudicated outcome warning", () => {+ const summary = computeTurnSummary(+ buildTurn({+ responseKind: "clarify",+ responseType: "REPLY",+ verifiedSuccess: false,+ lastRunOutcome: {+ verdict: "not_demonstrated",+ role: "adjudicated",+ displayReason: "The goal was not demonstrated.",+ activitySeqAtVerdict: 3,+ },+ }),+ );++ expect(summary.headline).toBe("Outcome not confirmed");+ expect(summary.accent).toBe("warn");+ });++ it("keeps the terminal envelope authoritative over interim live state", () => {+ const summary = computeTurnSummary(+ buildTurn({+ responseKind: "clarify",+ responseType: "REPLY",+ verifiedSuccess: false,+ terminalEnvelope: {+ runVerdict: "not_demonstrated",+ runDisplayReason: "The final run did not demonstrate the goal.",+ },+ lastRunOutcome: {+ verdict: "not_demonstrated",+ role: "interim_build_test",+ displayReason: "The workflow still needs its extraction block.",+ activitySeqAtVerdict: 3,+ },+ blocks: [+ {+ ...summaryBlock("open_search"),+ outcome: "not_demonstrated",+ outcomeRole: "interim_build_test",+ },+ ],+ }),+ );++ expect(summary.headline).toBe("Outcome not confirmed");+ expect(summary.accent).toBe("warn");+ });+it("uses hydrated not_demonstrated block outcomes when lastRunOutcome is absent", () => {const turn = hydrateNarrativeFromPayload(reproClarifyPayload({diff --git a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsxindex 7050553e6..71b9f5d7a 100644--- a/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/NarrativeView.tsx@@ -19,6 +19,7 @@ import {effectiveMode,formatElapsed,isBlockOk,+ isInterimOutcome,latestBlocksByLabel,notConfirmedOutcome,parseUtcIsoMs,@@ -353,11 +354,18 @@ function FBlockRun({const isRunning = block.state === "running";const isCompleted = block.state === "completed";const isEvaluating = isCompleted && block.outcome === "evaluating";+ const isInterimNotDemonstrated =+ isCompleted &&+ block.outcome === "not_demonstrated" &&+ isInterimOutcome(block.outcomeRole);// A row stuck in `evaluating` at turn end (dropped stream) renders the// neutral "ran" treatment — never the live verifying beat, never green.const isVerifying = isEvaluating && !turnEnded;- const isRanNeutral = isEvaluating && turnEnded;- const isOutcomeNotShown = isCompleted && block.outcome === "not_demonstrated";+ const isRanNeutral = (isEvaluating && turnEnded) || isInterimNotDemonstrated;+ const isOutcomeNotShown =+ isCompleted &&+ block.outcome === "not_demonstrated" &&+ !isInterimNotDemonstrated;const isOk = isBlockOk(block);const isFail = block.state === "failed";const isDraft = block.state === "drafted";diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.tsindex 45cbc9d26..b8a4b4c81 100644--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts@@ -626,6 +626,7 @@ describe("derivePhases — redraft re-activation (SKY-11970 pin)", () => {blocks: [block({ state: "completed" })],lastRunOutcome: {verdict: "not_demonstrated",+ role: "interim_build_test",displayReason: "outcome not confirmed",activitySeqAtVerdict: 2,},@@ -826,12 +827,21 @@ describe("derivePhases — redraft re-activation (SKY-11970 pin)", () => {});describe("derivePhases — test-run count on the not-confirmed stub (SKY-11339)", () => {- const notConfirmedTurn = (runToolNames: string[]): TurnNarrativeState =>+ const notConfirmedTurn = (+ runToolNames: string[],+ outcomeRole?: BlockState["outcomeRole"],+ ): TurnNarrativeState =>turn({terminal: "response",designEnded: true,draft: { blockCount: 1, blockLabels: ["block_1"], summary: null },- blocks: [block({ state: "completed", outcome: "not_demonstrated" })],+ blocks: [+ block({+ state: "completed",+ outcome: "not_demonstrated",+ outcomeRole,+ }),+ ],designActivity: runToolNames.map((toolName, i) =>entry({ id: `r${i}`, kind: "tool_call", toolName }),),@@ -852,4 +862,18 @@ describe("derivePhases — test-run count on the not-confirmed stub (SKY-11339)"const rows = derivePhases(notConfirmedTurn(["update_and_run_blocks"]));expect(phase(rows, "test").stub).toBe("· not confirmed");});++ it("keeps an explicitly adjudicated negative outcome in the alarm stub", () => {+ const rows = derivePhases(+ notConfirmedTurn(["update_and_run_blocks"], "adjudicated"),+ );+ expect(phase(rows, "test").stub).toBe("· not confirmed");+ });++ it("suppresses the alarm stub for an interim build test", () => {+ const rows = derivePhases(+ notConfirmedTurn(["update_and_run_blocks"], "interim_build_test"),+ );+ expect(phase(rows, "test").stub).not.toContain("not confirmed");+ });});diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.tsindex 67df75772..2037bfbab 100644--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.ts@@ -12,6 +12,7 @@ import {TurnNarrativeState,condenseActivityEntries,formatElapsed,+ isInterimOutcome,latestBlocksByLabel,parseUtcIsoMs,toolCallIdOf,@@ -252,7 +253,7 @@ export function derivePhases(turn: TurnNarrativeState): PhaseRowModel[] {const isCancelled = turn.cancelled === true;const anyFailed = latestBlocks.some((b) => b.state === "failed");const anyNotDemonstrated = latestBlocks.some(- (b) => b.outcome === "not_demonstrated",+ (b) => b.outcome === "not_demonstrated" && !isInterimOutcome(b.outcomeRole),);function reached(id: CopilotPhaseId): boolean {diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.phaseHint.test.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.phaseHint.test.tsindex 0823ef36c..bbbb5396b 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.phaseHint.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.phaseHint.test.ts@@ -196,6 +196,7 @@ describe("applyNarrativeEvent — lastRunOutcome", () => {);expect(s.lastRunOutcome).toEqual({verdict: "not_demonstrated",+ role: "adjudicated",displayReason: "outcome not confirmed",activitySeqAtVerdict: 1,});diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.tsindex 58020afed..85ac51538 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.runOutcome.test.ts@@ -97,8 +97,30 @@ describe("applyNarrativeEvent — run_outcome", () => {expect(b.outcomeReason).toBe("The search stayed gated by a verification challenge.",);+ expect(b.outcomeRole).toBe("adjudicated");expect(isBlockOk(b)).toBe(false);}+ expect(s.lastRunOutcome?.role).toBe("adjudicated");+ expect(notConfirmedOutcome(s)?.verdict).toBe("not_demonstrated");+ });++ it("interim verdict stays non-green but suppresses live not-confirmed alarms", () => {+ const s = reduce([+ ...bothBlocksRan,+ runOutcome({+ verdict: "not_demonstrated",+ role: "interim_build_test",+ reason_code: "outcome_not_demonstrated",+ display_reason: "The workflow still needs its extraction block.",+ }),+ ]);++ for (const b of s.blocks) {+ expect(b.outcomeRole).toBe("interim_build_test");+ expect(isBlockOk(b)).toBe(false);+ }+ expect(s.lastRunOutcome?.role).toBe("interim_build_test");+ expect(notConfirmedOutcome(s)).toBeNull();});it("evaluating hold withholds the success affordance until the final frame", () => {@@ -255,13 +277,14 @@ describe("hydrateNarrativeFromPayload — outcome", () => {blocks,});- it("round-trips outcome/outcomeReason so reload renders like the live stream", () => {+ it("round-trips outcome/outcomeReason/outcomeRole so reload renders like the live stream", () => {const live = reduce([turnStart(),blockProgress({ block_label: "search_person", status: "running" }),blockProgress({ block_label: "search_person", status: "completed" }),runOutcome({verdict: "not_demonstrated",+ role: "interim_build_test",workflow_run_block_ids: ["wrb_search_person"],block_labels: ["search_person"],reason_code: "blocker_reported",@@ -276,6 +299,7 @@ describe("hydrateNarrativeFromPayload — outcome", () => {payloadBlock({outcome: "not_demonstrated",outcomeReason: "The search stayed gated by a verification challenge.",+ outcomeRole: "interim_build_test",}),]),)!;@@ -284,6 +308,7 @@ describe("hydrateNarrativeFromPayload — outcome", () => {expect(hydratedRow.state).toBe(liveRow.state);expect(hydratedRow.outcome).toBe(liveRow.outcome);expect(hydratedRow.outcomeReason).toBe(liveRow.outcomeReason);+ expect(hydratedRow.outcomeRole).toBe(liveRow.outcomeRole);expect(isBlockOk(hydratedRow)).toBe(false);expect(isBlockOk(liveRow)).toBe(false);});@@ -293,6 +318,7 @@ describe("hydrateNarrativeFromPayload — outcome", () => {const row = hydrated.blocks[0]!;expect(row.outcome).toBeUndefined();expect(row.outcomeReason).toBeUndefined();+ expect(row.outcomeRole).toBeUndefined();expect(isBlockOk(row)).toBe(true);});@@ -303,9 +329,29 @@ describe("hydrateNarrativeFromPayload — outcome", () => {const row = hydrated.blocks[0]!;expect(row.outcome).toBeUndefined();expect(row.outcomeReason).toBeUndefined();+ expect(row.outcomeRole).toBeUndefined();expect(isBlockOk(row)).toBe(true);});+ it("normalizes absent and unknown persisted roles to adjudicated", () => {+ const hydrated = hydrateNarrativeFromPayload(+ payload([+ payloadBlock({ outcome: "not_demonstrated" }),+ payloadBlock({+ label: "search_person_2",+ outcome: "not_demonstrated",+ outcomeRole: "future_role",+ }),+ ]),+ )!;++ expect(hydrated.blocks.map((block) => block.outcomeRole)).toEqual([+ "adjudicated",+ "adjudicated",+ ]);+ expect(notConfirmedOutcome(hydrated)?.verdict).toBe("not_demonstrated");+ });+it("hydrate sweep promotes a stuck-running row without inventing a verdict", () => {const hydrated = hydrateNarrativeFromPayload(payload([diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.tsindex d21b60851..20e00462d 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts@@ -7,6 +7,7 @@ import { buildRevealOffsets } from "./actionReveal";import {CopilotResponseType,ProposalDisposition,+ RunOutcomeRole,WorkflowCopilotBlockProgressUpdate,WorkflowCopilotDesignEndUpdate,WorkflowCopilotDesignStartUpdate,@@ -88,6 +89,10 @@ export type BlockOutcome =| "not_demonstrated"| "not_evaluated";+export function isInterimOutcome(role: RunOutcomeRole | undefined): boolean {+ return role === "interim_build_test";+}+export interface TerminalEnvelopeFacts {runVerdict: BlockOutcome | null;runDisplayReason: string | null;@@ -122,6 +127,7 @@ export interface BlockState {// Undefined when no verdict was recorded (older backend or unadjudicated run).outcome?: BlockOutcome;outcomeReason?: string;+ outcomeRole?: RunOutcomeRole;lastSeenIteration: number;// Tool calls and results emitted while this block was running are// appended here so the expanded card shows what the agent did.@@ -232,6 +238,7 @@ export interface TurnNarrativeState {// stopped rather than Draft (accepted).lastRunOutcome: {verdict: BlockOutcome;+ role?: RunOutcomeRole;displayReason: string | null;activitySeqAtVerdict: number;} | null;@@ -797,6 +804,7 @@ export function applyNarrativeEvent(// either, so always keep the prior values.outcome: previousBlock?.outcome,outcomeReason: previousBlock?.outcomeReason,+ outcomeRole: previousBlock?.outcomeRole,recordedActions: previousBlock?.recordedActions,recordedActionsAt: previousBlock?.recordedActionsAt,lastSeenIteration: event.iteration,@@ -822,6 +830,7 @@ export function applyNarrativeEvent(...b,outcome: event.verdict,outcomeReason: event.display_reason ?? undefined,+ outcomeRole: event.role ?? "adjudicated",}: b,);@@ -832,6 +841,7 @@ export function applyNarrativeEvent(// verdict from a prior run cycle within the same turn.lastRunOutcome: {verdict: event.verdict,+ role: event.role ?? "adjudicated",displayReason: event.display_reason ?? null,activitySeqAtVerdict: prev.activitySeq,},@@ -1179,6 +1189,12 @@ export function hydrateNarrativeFromPayload(return o;return undefined;})();+ const outcomeRole: RunOutcomeRole | undefined =+ outcome === undefined+ ? undefined+ : obj.outcomeRole === "interim_build_test"+ ? "interim_build_test"+ : "adjudicated";return {workflowRunBlockId:typeof obj.workflowRunBlockId === "string"@@ -1187,6 +1203,7 @@ export function hydrateNarrativeFromPayload(label: typeof obj.label === "string" ? obj.label : "",blockType: typeof obj.blockType === "string" ? obj.blockType : "task",outcome,+ outcomeRole,outcomeReason:typeof obj.outcomeReason === "string" ? obj.outcomeReason : undefined,state: ((): BlockState["state"] => {@@ -1402,7 +1419,8 @@ export function notConfirmedOutcome(: null;}if (turn.lastRunOutcome !== null) {- return turn.lastRunOutcome.verdict === "not_demonstrated"+ return turn.lastRunOutcome.verdict === "not_demonstrated" &&+ !isInterimOutcome(turn.lastRunOutcome.role)? {verdict: "not_demonstrated",displayReason: turn.lastRunOutcome.displayReason,@@ -1411,7 +1429,10 @@ export function notConfirmedOutcome(}for (let i = turn.blocks.length - 1; i >= 0; i -= 1) {const block = turn.blocks[i]!;- if (block.outcome === "not_demonstrated") {+ if (+ block.outcome === "not_demonstrated" &&+ !isInterimOutcome(block.outcomeRole)+ ) {return {verdict: "not_demonstrated",displayReason: block.outcomeReason ?? null,diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex bba8b174c..b8a3f36d4 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -247,12 +247,15 @@ export type WorkflowCopilotRunOutcomeVerdict =| "not_demonstrated"| "not_evaluated";+export type RunOutcomeRole = "adjudicated" | "interim_build_test";+export interface WorkflowCopilotRunOutcomeUpdate {type: "run_outcome";workflow_run_id: string;workflow_run_block_ids: string[];block_labels: string[];verdict: WorkflowCopilotRunOutcomeVerdict;+ role?: RunOutcomeRole;reason_code?: string | null;display_reason?: string | null;iteration: number;diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 9efa23d35..215b59cfb 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -2176,6 +2176,7 @@ def _build_narrative_payload(}if recorded_outcome is not None and label in outcome_labels:block_entry["outcome"] = recorded_outcome.verdict+ block_entry["outcomeRole"] = recorded_outcome.roleif recorded_outcome.display_reason is not None:block_entry["outcomeReason"] = recorded_outcome.display_reasonblocks.append(block_entry)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 8727a18b4..f975cbf07 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -20,6 +20,7 @@ from skyvern.forge.sdk.copilot.result_evidence import (LoadedResultCompositionEvidence,loaded_result_target_structure_signature,)+from skyvern.forge.sdk.copilot.run_outcome import RunOutcomeRolefrom skyvern.forge.sdk.copilot.runtime import AgentContextfrom skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidencefrom skyvern.forge.sdk.workflow.models.workflow import Workflow@@ -75,6 +76,7 @@ class NarrativeBlock(TypedDict):endedAt: str | Noneoutcome: NotRequired[str]outcomeReason: NotRequired[str]+ outcomeRole: NotRequired[RunOutcomeRole]class NarrativeOutcomeAdjudication(TypedDict):diff --git a/skyvern/forge/sdk/copilot/run_outcome.py b/skyvern/forge/sdk/copilot/run_outcome.pyindex 22bd60db3..c7b3713d5 100644--- a/skyvern/forge/sdk/copilot/run_outcome.py+++ b/skyvern/forge/sdk/copilot/run_outcome.py@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_RERunOutcomeVerdict = Literal["evaluating", "demonstrated", "not_demonstrated", "not_evaluated"]RecordedRunOutcomeVerdict = Literal["demonstrated", "not_demonstrated", "not_evaluated"]+RunOutcomeRole = Literal["adjudicated", "interim_build_test"]RunOutcomeReasonCode = Literal["blocker_reported","terminal_challenge_blocker",@@ -42,6 +43,7 @@ class RecordedRunOutcome:reason_code: RunOutcomeReasonCode | None = Nonedisplay_reason: str | None = Noneworkflow_run_id: str | None = None+ role: RunOutcomeRole = "adjudicated"def run_outcome_display_reason(text: str | None) -> str | None:diff --git a/skyvern/forge/sdk/copilot/terminal_envelope.py b/skyvern/forge/sdk/copilot/terminal_envelope.pyindex b1de056cf..16bfeb8c1 100644--- a/skyvern/forge/sdk/copilot/terminal_envelope.py+++ b/skyvern/forge/sdk/copilot/terminal_envelope.py@@ -168,6 +168,18 @@ def _select_run_outcome_anchor(run_outcomes: Sequence[RecordedRunOutcome]) -> Refinal_outcomes = [outcome for outcome in run_outcomes if outcome.verdict in _FINAL_RUN_VERDICTS]if not final_outcomes:return None+ # An interim build-test verdict is a mid-build "keep building" signal, not an+ # honest turn outcome, so it must not mask a later adjudicated run: prefer the+ # last not_demonstrated among adjudicated outcomes. Only when the turn produced+ # no adjudicated outcome at all — e.g. a suspicious-success repair ceiling whose+ # de-facto-final run is interim-tagged because the loop stopped rather than+ # continued — do interim verdicts anchor, preserving the honest terminal amber.+ adjudicated = [outcome for outcome in final_outcomes if outcome.role != "interim_build_test"]+ if adjudicated:+ adjudicated_not_demonstrated = [outcome for outcome in adjudicated if outcome.verdict == "not_demonstrated"]+ if adjudicated_not_demonstrated:+ return adjudicated_not_demonstrated[-1]+ return adjudicated[-1]last_not_demonstrated = [outcome for outcome in final_outcomes if outcome.verdict == "not_demonstrated"]if last_not_demonstrated:return last_not_demonstrated[-1]diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 5eb0d543f..ed041ad38 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -102,6 +102,7 @@ from skyvern.forge.sdk.copilot.run_outcome import (TERMINAL_CHALLENGE_USER_FACING_REASON,RecordedRunOutcome,RunOutcomeReasonCode,+ RunOutcomeRole,RunOutcomeVerdict,run_outcome_display_reason,trusted_terminal_challenge_category_name,@@ -3345,9 +3346,15 @@ def _adjudicated_run_outcome(committed = _same_run_committed_demonstrated_outcome(copilot_ctx)if committed is not None:return committed+ # last_test_suspicious_success can also mark a turn's de-facto-final run — a repair+ # ceiling stops the build loop rather than continuing — so this interim tag neutralizes+ # only the live row; the honest turn-level amber is still surfaced by the terminal+ # envelope, which anchors interim verdicts when no adjudicated outcome exists in the+ # turn (test_run_anchor_keeps_interim_amber_when_no_adjudicated_outcome).if copilot_ctx.last_test_suspicious_success:return RecordedRunOutcome(verdict="not_demonstrated",+ role="interim_build_test",reason_code="outcome_not_demonstrated",display_reason=run_outcome_display_reason(copilot_ctx.last_test_failure_reason),)@@ -3381,6 +3388,7 @@ async def _send_run_outcome_update(verdict: RunOutcomeVerdict,reason_code: RunOutcomeReasonCode | None,display_reason: str | None,+ role: RunOutcomeRole = "adjudicated",) -> None:stream = getattr(copilot_ctx, "stream", None)if stream is None:@@ -3397,6 +3405,7 @@ async def _send_run_outcome_update(workflow_run_block_ids=list(getattr(copilot_ctx, "last_run_blocks_block_ids", []) or []),block_labels=list(getattr(copilot_ctx, "last_run_blocks_block_labels", []) or []),verdict=verdict,+ role=role,reason_code=reason_code,display_reason=display_reason,iteration=iteration,@@ -3449,6 +3458,7 @@ async def _verify_and_record_run_blocks_result(copilot_ctx,result,verdict=final.verdict,+ role=final.role,reason_code=final.reason_code,display_reason=final.display_reason,)diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 49bbed637..6045184a5 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -6,7 +6,7 @@ from typing import Any, Literalfrom pydantic import BaseModel, ConfigDict, Field, field_validatorfrom skyvern.forge.sdk.copilot.context import ProposalDisposition, ResponseType, TurnNarrativePayload-from skyvern.forge.sdk.copilot.run_outcome import RunOutcomeReasonCode, RunOutcomeVerdict+from skyvern.forge.sdk.copilot.run_outcome import RunOutcomeReasonCode, RunOutcomeRole, RunOutcomeVerdictfrom skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome@@ -431,6 +431,7 @@ class WorkflowCopilotRunOutcomeUpdate(BaseModel):default_factory=list, description="Block labels of the adjudicated run; key the persisted narrative payload")verdict: RunOutcomeVerdict = Field(..., description="Recorded outcome verdict for the run")+ role: RunOutcomeRole = Field("adjudicated", description="Display scope of the recorded run verdict")reason_code: RunOutcomeReasonCode | None = Field(None, description="Machine-readable cause for a not_demonstrated verdict")diff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.pyindex a7807625b..3b25f66fe 100644--- a/tests/unit/test_copilot_run_outcome_event.py+++ b/tests/unit/test_copilot_run_outcome_event.py@@ -191,6 +191,20 @@ def _run_outcome_frames(stream: _FakeStream) -> list[WorkflowCopilotRunOutcomeUpreturn [frame for frame in stream.sent if isinstance(frame, WorkflowCopilotRunOutcomeUpdate)]+def test_run_outcome_event_role_defaults_to_adjudicated() -> None:+ frame = WorkflowCopilotRunOutcomeUpdate.model_validate(+ {+ "type": "run_outcome",+ "workflow_run_id": "wr_test",+ "verdict": "demonstrated",+ "iteration": 0,+ "timestamp": "2026-06-10T00:00:00Z",+ }+ )++ assert frame.role == "adjudicated"++@pytest.mark.asyncioasync def test_blocker_run_emits_hold_then_not_demonstrated() -> None:result = _blocked_run_result()@@ -206,6 +220,7 @@ async def test_blocker_run_emits_hold_then_not_demonstrated() -> None:assert final.workflow_run_block_ids == ["wrb_open_registry_search", "wrb_search_registry_person"]assert final.block_labels == ["open_registry_search", "search_registry_person"]assert final.display_reason is not None and "human verification challenge" in final.display_reason+ assert final.role == "adjudicated"assert ctx.last_test_suspicious_success is Falseassert ctx.last_run_outcome == RecordedRunOutcome(verdict=final.verdict,@@ -273,7 +288,9 @@ async def test_empty_data_run_emits_no_meaningful_output() -> None:frames = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]assert [frame.verdict for frame in frames] == ["evaluating", "not_demonstrated"]assert frames[-1].reason_code == "no_meaningful_output"+ assert frames[-1].role == "adjudicated"assert ctx.last_test_suspicious_success is True+ assert ctx.last_run_outcome is not None and ctx.last_run_outcome.role == "adjudicated"def _terminal_metadata_entry(label: str) -> dict[str, Any]:@@ -312,9 +329,11 @@ async def test_negative_adjudication_emits_outcome_not_demonstrated(monkeypatch:final = frames[-1]assert final.reason_code == "outcome_not_demonstrated"assert final.display_reason is not None and "did not demonstrate the goal outcome" in final.display_reason+ assert final.role == "interim_build_test"assert ctx.last_test_suspicious_success is Trueassert ctx.last_full_workflow_test_ok is Falseassert ctx.last_run_outcome is not None and ctx.last_run_outcome.verdict == "not_demonstrated"+ assert ctx.last_run_outcome.role == "interim_build_test"@pytest.mark.asyncio@@ -348,6 +367,7 @@ async def test_satisfied_adjudication_emits_demonstrated(monkeypatch: pytest.Monframes = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]assert [frame.verdict for frame in frames] == ["evaluating", "demonstrated"]assert frames[-1].reason_code is None+ assert frames[-1].role == "adjudicated"assert ctx.last_full_workflow_test_ok is True@@ -422,6 +442,7 @@ async def test_evaluating_hold_gets_final_frame_when_recording_raises(monkeypatcframes = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]assert [frame.verdict for frame in frames] == ["evaluating", "not_evaluated"]+ assert frames[-1].role == "adjudicated"def test_failed_rerun_clears_prior_recorded_outcome() -> None:@@ -589,6 +610,7 @@ def test_narrative_payload_stamps_outcome_on_adjudicated_labels() -> None:verdict="not_demonstrated",reason_code="blocker_reported",display_reason="The search form is gated by a human verification challenge.",+ role="interim_build_test",)ctx.last_run_outcome_block_labels = ["open_registry_search", "search_registry_person"]@@ -599,8 +621,10 @@ def test_narrative_payload_stamps_outcome_on_adjudicated_labels() -> None:assert by_label[label]["state"] == "completed"assert by_label[label]["outcome"] == "not_demonstrated"assert by_label[label]["outcomeReason"] == "The search form is gated by a human verification challenge."+ assert by_label[label]["outcomeRole"] == "interim_build_test"assert "outcome" not in by_label["untested_block"]assert "outcomeReason" not in by_label["untested_block"]+ assert "outcomeRole" not in by_label["untested_block"]def test_narrative_payload_without_recorded_outcome_has_no_outcome_keys() -> None:@@ -611,6 +635,7 @@ def test_narrative_payload_without_recorded_outcome_has_no_outcome_keys() -> Nonfor block in payload["blocks"]:assert "outcome" not in blockassert "outcomeReason" not in block+ assert "outcomeRole" not in blockclass TestGenuineAttemptRunStamp:diff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.pyindex 52e306680..d0f1d5745 100644--- a/tests/unit/test_copilot_terminal_envelope.py+++ b/tests/unit/test_copilot_terminal_envelope.py@@ -105,6 +105,53 @@ def test_anchor_supersession_divergence_is_logged() -> None:assert not any("anchored a not_demonstrated verdict" in log["event"] for log in logs)+def _interim_outcome(verdict: str, display_reason: str | None = None) -> RecordedRunOutcome:+ return RecordedRunOutcome(verdict=verdict, display_reason=display_reason, role="interim_build_test")+++def test_run_anchor_ignores_interim_not_demonstrated_when_adjudicated_run_follows() -> None:+ # Happy turn: an interim scout test run goes not_demonstrated mid-build, then the+ # completed workflow demonstrates the goal. The envelope must not resurface the+ # interim amber over the later adjudicated success.+ envelope = _assemble(+ run_outcomes=[+ _interim_outcome("not_demonstrated", "The scout has not produced the goal yet."),+ _run_outcome("demonstrated", "The extraction returned the value."),+ ]+ )++ assert envelope.run_verdict == "demonstrated"+ assert envelope.run_display_reason == "The extraction returned the value."+++def test_run_anchor_keeps_interim_amber_when_no_adjudicated_outcome() -> None:+ # Repair ceiling: the loop stops after a suspicious-success run without ever+ # producing an adjudicated outcome, so the interim not_demonstrated is the turn's+ # honest terminal verdict and must still anchor amber.+ envelope = _assemble(+ run_outcomes=[+ _interim_outcome("not_demonstrated", "The run completed but did not demonstrate the goal."),+ ]+ )++ assert envelope.run_verdict == "not_demonstrated"+ assert envelope.run_display_reason == "The run completed but did not demonstrate the goal."+++def test_run_anchor_prefers_adjudicated_not_demonstrated_over_earlier_interim() -> None:+ # A genuine adjudicated failure on the completed workflow anchors amber even when an+ # earlier interim run also went not_demonstrated.+ envelope = _assemble(+ run_outcomes=[+ _interim_outcome("not_demonstrated", "Interim scout, still building."),+ _run_outcome("not_demonstrated", "The extraction returned no value."),+ ]+ )++ assert envelope.run_verdict == "not_demonstrated"+ assert envelope.run_display_reason == "The extraction returned no value."++@pytest.mark.parametrize(("response_type", "verified", "workflow_applied", "proposal_disposition", "expected_next_state"),[