Skip to content

Automate browser based workflows with AI

SKY-10576: gate copilot recognition and persistence on runtime outcome evidence, not run status (#6323)

Andrew Neilsonae42d0470ad7
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 5eb5b0355..df22c9280 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -36,7 +36,7 @@ import {
type MessageInExfiltratedEvent,
} from "@/store/useRecordingStore";
import { useSettingsStore } from "@/store/SettingsStore";
-import { wssBaseUrl, newWssBaseUrl, getRuntimeApiKey } from "@/util/env";
+import { wssBaseUrl, newWssBaseUrl, getCredentialParam } from "@/util/env";
import { copyText } from "@/util/copyText";
import { cn } from "@/util/utils";
import { captureRecordBrowser } from "@/util/recordBrowserTelemetry";
@@ -193,7 +193,7 @@ function BrowserStream({
}
},
enabled: entity === "browserSession" && !!browserSessionId,
- refetchInterval: 5000,
+ refetchInterval: (query) => (query.state.data ? 5000 : 1000),
});
const [hasBrowserSession, setHasBrowserSession] = useState(true); // be optimistic
@@ -229,22 +229,22 @@ function BrowserStream({
useEffect(() => {
setIsBrowserSessionStarted(false);
setIsReady(false);
+ setIsVncConnected(false);
+ setIsCanvasReady(false);
+ setIsMessageConnected(false);
+ setHasBrowserSession(true);
+ if (rfbRef.current) {
+ rfbRef.current.disconnect();
+ rfbRef.current = null;
+ }
}, [browserSessionId]);
const getWebSocketParams = useCallback(async () => {
- const clientIdQueryParam = `client_id=${clientId}`;
- const runtimeApiKey = getRuntimeApiKey();
-
- let credentialQueryParam = runtimeApiKey ? `apikey=${runtimeApiKey}` : "";
-
- if (credentialGetter) {
- const token = await credentialGetter();
- credentialQueryParam = token ? `token=Bearer ${token}` : "";
- }
-
- return credentialQueryParam
- ? `${credentialQueryParam}&${clientIdQueryParam}`
- : clientIdQueryParam;
+ const params = new URLSearchParams(
+ await getCredentialParam(credentialGetter),
+ );
+ params.set("client_id", clientId);
+ return params.toString();
}, [clientId, credentialGetter]);
// browser is ready
@@ -784,33 +784,38 @@ function BrowserStream({
const theUserIsControlling =
userIsControlling || (interactive && !showControlButtons);
const streamDiagnostic: StreamDiagnostic =
- entity === "browserSession" && browserSessionId && !hasBrowserSession
+ !showStream || !runId
? {
- title: "Browser session is no longer live",
- detail: "This live browser session is no longer streaming.",
- hint: "Refresh the page or create a new browser session.",
+ title: "Starting browser session",
+ detail: "Waiting for a live browser session to attach.",
}
- : !isBrowserSessionBackendReady
+ : entity === "browserSession" && browserSessionId && !hasBrowserSession
? {
- title: "Waiting for browser session",
- detail:
- "The session exists, but the backend has not marked the browser as ready yet.",
+ title: "Browser session is no longer live",
+ detail: "This live browser session is no longer streaming.",
+ hint: "Refresh the page or create a new browser session.",
}
- : !isVncConnected
+ : !isBrowserSessionBackendReady
? {
- title: "Connecting to VNC stream",
- detail: "Opening the browser stream and message WebSockets.",
- hint: "If this stays here, check VNC support for the session or use local browser streaming.",
+ title: "Waiting for browser session",
+ detail:
+ "The session exists, but the backend has not marked the browser as ready yet.",
}
- : !isCanvasReady
+ : !isVncConnected
? {
- title: "Preparing browser display",
- detail:
- "The VNC connection is open and the UI is waiting for the browser canvas.",
+ title: "Connecting to VNC stream",
+ detail: "Opening the browser stream and message WebSockets.",
+ hint: "If this stays here, check VNC support for the session or use local browser streaming.",
}
- : {
- title: "Connecting to browser stream",
- };
+ : !isCanvasReady
+ ? {
+ title: "Preparing browser display",
+ detail:
+ "The VNC connection is open and the UI is waiting for the browser canvas.",
+ }
+ : {
+ title: "Connecting to browser stream",
+ };
return (
<>
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSession.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSession.tsx
index b35afbd30..93c9c2b86 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSession.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSession.tsx
@@ -73,16 +73,16 @@ function BrowserSession() {
return response.data;
},
refetchInterval: (query) =>
- getBrowserSessionRefetchIntervalMs(query.state.data?.status),
+ getBrowserSessionRefetchIntervalMs(query.state.data),
});
const browserSession = query.data;
- const streamMode: StreamMode = browserSession?.vnc_streaming_supported
- ? vncFailed
- ? "fallback"
- : "vnc"
- : isCdpMode
- ? "cdp"
+ const streamMode: StreamMode = isCdpMode
+ ? "cdp"
+ : browserSession?.vnc_streaming_supported
+ ? vncFailed
+ ? "fallback"
+ : "vnc"
: "unavailable";
const closeBrowserSessionMutation = useCloseBrowserSessionMutation({
@@ -229,23 +229,22 @@ function BrowserSession() {
pointerEvents: activeTab === "stream" ? "auto" : "none",
}}
>
- {/* VNC streaming */}
- {browserSession.vnc_streaming_supported && !vncFailed && (
- <BrowserStream
+ {isCdpMode && browserSessionId && (
+ <BrowserSessionStream
browserSessionId={browserSessionId}
- interactive={false}
+ interactive={true}
showControlButtons={true}
- isVisible={activeTab === "stream"}
- onClose={() => setVncFailed(true)}
/>
)}
- {isCdpMode &&
- browserSessionId &&
- (!browserSession.vnc_streaming_supported || vncFailed) && (
- <BrowserSessionStream
+ {!isCdpMode &&
+ browserSession.vnc_streaming_supported &&
+ !vncFailed && (
+ <BrowserStream
browserSessionId={browserSessionId}
- interactive={true}
+ interactive={false}
showControlButtons={true}
+ isVisible={activeTab === "stream"}
+ onClose={() => setVncFailed(true)}
/>
)}
</div>
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
index 58bb9e5b6..f43e86d6a 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessionStream.tsx
@@ -13,6 +13,7 @@ import {
isTerminalStreamStatus,
shouldReconnectStream,
} from "./BrowserSessionStream.utils";
+import { useSettingsStore } from "@/store/SettingsStore";
type StreamMessage = {
browser_session_id?: string;
@@ -111,6 +112,7 @@ function BrowserSessionStream({
const [diagnostic, setDiagnostic] =
useState<StreamDiagnostic>(STARTING_DIAGNOSTIC);
const credentialGetter = useCredentialGetter();
+ const settingsStore = useSettingsStore();
const socketRef = useRef<WebSocket | null>(null);
const hasFrameRef = useRef(false);
@@ -280,6 +282,7 @@ function BrowserSessionStream({
// browserSessionId intentionally not a dep: re-firing on prop change
// before isReady resets would spuriously report (true, newSessionId).
onReadyChange?.(isReady, isReady ? browserSessionId : null);
+ // Zustand store setters are stable; omit browserSessionId from deps on purpose.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [isReady, onReadyChange]);
@@ -289,6 +292,13 @@ function BrowserSessionStream({
};
}, [onReadyChange]);
+ useEffect(() => {
+ settingsStore.setIsUsingABrowser(isReady);
+ settingsStore.setBrowserSessionId(isReady ? browserSessionId : null);
+ // Zustand store setters are stable; only sync when stream readiness changes.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [isReady, browserSessionId]);
+
if (isReady) {
return (
<InteractiveStreamView
diff --git a/skyvern-frontend/src/routes/browserSessions/browserSessionQueryUtils.test.ts b/skyvern-frontend/src/routes/browserSessions/browserSessionQueryUtils.test.ts
index 8100d77c1..f61e2863d 100644
--- a/skyvern-frontend/src/routes/browserSessions/browserSessionQueryUtils.test.ts
+++ b/skyvern-frontend/src/routes/browserSessions/browserSessionQueryUtils.test.ts
@@ -4,18 +4,37 @@ import { getBrowserSessionRefetchIntervalMs } from "./browserSessionQueryUtils";
describe("getBrowserSessionRefetchIntervalMs", () => {
it("polls while session status is bootstrapping", () => {
- expect(getBrowserSessionRefetchIntervalMs(undefined)).toBe(2000);
- expect(getBrowserSessionRefetchIntervalMs("created")).toBe(2000);
- expect(getBrowserSessionRefetchIntervalMs("retry")).toBe(2000);
+ expect(getBrowserSessionRefetchIntervalMs(undefined)).toBe(1000);
+ expect(getBrowserSessionRefetchIntervalMs({ status: "created" })).toBe(
+ 1000,
+ );
+ expect(getBrowserSessionRefetchIntervalMs({ status: "retry" })).toBe(1000);
});
- it("polls slower while session is running", () => {
- expect(getBrowserSessionRefetchIntervalMs("running")).toBe(5000);
+ it("polls quickly while a running session is waiting for an address", () => {
+ expect(getBrowserSessionRefetchIntervalMs({ status: "running" })).toBe(
+ 1000,
+ );
+ });
+
+ it("polls slower after a running session has an address", () => {
+ expect(
+ getBrowserSessionRefetchIntervalMs({
+ status: "running",
+ browser_address: "ws://127.0.0.1:9222/devtools/browser/test",
+ }),
+ ).toBe(5000);
});
it("stops polling for terminal statuses", () => {
- expect(getBrowserSessionRefetchIntervalMs("completed")).toBe(false);
- expect(getBrowserSessionRefetchIntervalMs("failed")).toBe(false);
- expect(getBrowserSessionRefetchIntervalMs("timeout")).toBe(false);
+ expect(getBrowserSessionRefetchIntervalMs({ status: "completed" })).toBe(
+ false,
+ );
+ expect(getBrowserSessionRefetchIntervalMs({ status: "failed" })).toBe(
+ false,
+ );
+ expect(getBrowserSessionRefetchIntervalMs({ status: "timeout" })).toBe(
+ false,
+ );
});
});
diff --git a/skyvern-frontend/src/routes/browserSessions/browserSessionQueryUtils.ts b/skyvern-frontend/src/routes/browserSessions/browserSessionQueryUtils.ts
index 1e5f8b80d..5a3cc9f2b 100644
--- a/skyvern-frontend/src/routes/browserSessions/browserSessionQueryUtils.ts
+++ b/skyvern-frontend/src/routes/browserSessions/browserSessionQueryUtils.ts
@@ -1,14 +1,26 @@
+import { type BrowserSession } from "@/routes/workflows/types/browserSessionTypes";
+
+type BrowserSessionReadiness = Partial<
+ Pick<BrowserSession, "status" | "browser_address">
+>;
+
function getBrowserSessionRefetchIntervalMs(
- status: string | undefined,
+ browserSession: BrowserSessionReadiness | undefined,
): number | false {
- if (!status) {
- return 2000;
+ if (!browserSession?.status) {
+ return 1000;
}
- if (status === "running") {
+ if (browserSession.status === "running") {
+ if (!browserSession.browser_address) {
+ return 1000;
+ }
return 5000;
}
- if (status === "created" || status === "retry") {
- return 2000;
+ if (
+ browserSession.status === "created" ||
+ browserSession.status === "retry"
+ ) {
+ return 1000;
}
return false;
}
diff --git a/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx b/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx
index 8182deb06..5c977a331 100644
--- a/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx
+++ b/skyvern-frontend/src/routes/workflows/debugger/Debugger.tsx
@@ -105,6 +105,7 @@ function Debugger() {
<div className="relative flex h-screen w-full">
<ReactFlowProvider>
<Workspace
+ key={workflowPermanentId}
initialEdges={elements.edges}
initialNodes={elements.nodes}
initialTitle={workflow.title}
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index 305ff99f7..f5afe69eb 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -43,7 +43,10 @@ import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";
import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQuery";
import { BrowserSessionStream } from "@/routes/browserSessions/BrowserSessionStream";
import { useBrowserStreamingMode } from "@/hooks/useRuntimeConfig";
-import { StreamModeBadge } from "@/routes/streaming/StreamDiagnostics";
+import {
+ StreamModeBadge,
+ StreamStatusPanel,
+} from "@/routes/streaming/StreamDiagnostics";
import { useCacheKeyValuesQuery } from "../hooks/useCacheKeyValuesQuery";
import { useBlockScriptStore } from "@/store/BlockScriptStore";
import { useBlockSidebarWidthStore } from "@/store/BlockSidebarWidthStore";
@@ -143,6 +146,20 @@ import { WorkflowCopilotButton } from "../copilot/WorkflowCopilotButton";
import type { WorkflowYAMLConversionResponse } from "../copilot/workflowCopilotTypes";
import "./workspace-styles.css";
+function getAxiosErrorDetail(error: unknown): string | undefined {
+ if (!(error instanceof AxiosError)) {
+ return undefined;
+ }
+
+ const data = error.response?.data;
+ if (!data || typeof data !== "object" || !("detail" in data)) {
+ return undefined;
+ }
+
+ const detail = data.detail;
+ return typeof detail === "string" ? detail : undefined;
+}
+
const Constants = {
NewBrowserCooldown: 30000,
} as const;
@@ -393,8 +410,6 @@ function Workspace({
);
const [copilotMessageCount, setCopilotMessageCount] = useState(0);
const copilotButtonRef = useRef<HTMLButtonElement>(null);
- const [activeDebugSession, setActiveDebugSession] =
- useState<DebugSessionApiResponse | null>(null);
const [readyBrowserSessionId, setReadyBrowserSessionId] = useState<
string | null
>(null);
@@ -417,6 +432,11 @@ function Workspace({
const [shouldFetchDebugSession, setShouldFetchDebugSession] = useState(false);
const blockScriptStore = useBlockScriptStore();
const recordingStore = useRecordingStore();
+ const isCdpStreamingMode =
+ browserStreamingMode === "cdp" && !recordingStore.isRecording;
+ // Record Browser exfiltration requires VNC even when the org default is CDP streaming.
+ const preferVncStream =
+ browserStreamingMode !== "cdp" || recordingStore.isRecording;
const cacheKey = workflow?.cache_key ?? "";
// Block delete confirmation dialog state
@@ -589,12 +609,19 @@ function Workspace({
const { isRateLimited, recordAttempt, resetOnSuccess } =
useBrowserSessionRateLimit(workflowPermanentId);
- const { data: debugSession } = useDebugSessionQuery({
+ const {
+ data: debugSession,
+ isError: isDebugSessionError,
+ error: debugSessionError,
+ refetch: refetchDebugSession,
+ } = useDebugSessionQuery({
workflowPermanentId,
enabled: shouldFetchDebugSession && !!workflowPermanentId,
isRateLimited,
});
+ const activeDebugSession = debugSession ?? null;
+
const setCollapsed = useSidebarStore((state) => {
return state.setCollapsed;
});
@@ -604,6 +631,13 @@ function Workspace({
const showBreakoutButton =
activeDebugSession && activeDebugSession.browser_session_id;
const liveBrowserSessionId = activeDebugSession?.browser_session_id ?? null;
+ const showVncBrowserPanel =
+ preferVncStream &&
+ shouldFetchDebugSession &&
+ !isRateLimited &&
+ (!activeDebugSession || activeDebugSession.vnc_streaming_supported);
+ const showCdpBrowserPanel =
+ isCdpStreamingMode && shouldFetchDebugSession && !isRateLimited;
const copilotRequiresLiveBrowser =
showBrowser && shouldFetchDebugSession && !isRateLimited;
// readyBrowserSessionId is keyed to the browser session id rather than a
@@ -686,7 +720,16 @@ function Workspace({
useShowAllCodeStore.getState().reset();
useSidebarSaveStateStore.getState().reset();
cacheKeyInitWpidRef.current = null;
- }, [workflowPermanentId]);
+ setReadyBrowserSessionId(null);
+ if (workflowPermanentId) {
+ queryClient.removeQueries({
+ queryKey: ["debugSession", workflowPermanentId],
+ });
+ setShouldFetchDebugSession(true);
+ } else {
+ setShouldFetchDebugSession(false);
+ }
+ }, [workflowPermanentId, queryClient]);
useEffect(() => {
// Gate on workflow payload matching the route wpid: `useWorkflowQuery`
@@ -754,16 +797,10 @@ function Workspace({
setCollapsed(true);
workflowChangesStore.setHasChanges(false);
if (workflowPermanentId) {
- queryClient.removeQueries({
- queryKey: ["debugSession", workflowPermanentId],
- });
- setShouldFetchDebugSession(true);
-
queryClient.invalidateQueries({
queryKey: ["cache-key-values", workflowPermanentId, cacheKey],
});
}
-
closeWorkflowPanel();
});
@@ -883,10 +920,13 @@ function Workspace({
},
onSuccess: (response) => {
const newDebugSession = response.data;
- setActiveDebugSession(newDebugSession);
resetOnSuccess();
- queryClient.invalidateQueries({
+ queryClient.setQueryData(
+ ["debugSession", workflowPermanentId],
+ newDebugSession,
+ );
+ void queryClient.invalidateQueries({
queryKey: ["debugSession", workflowPermanentId],
});
@@ -979,10 +1019,6 @@ function Workspace({
// Reset polling timer so it doesn't carry a stale timestamp into the
// next polling cycle (e.g. after a rate-limit window expires).
pollingStartRef.current = null;
-
- if (debugSession) {
- setActiveDebugSession(debugSession);
- }
}
return () => {
@@ -1813,53 +1849,87 @@ function Workspace({
{/* node library sub panel */}
{/* browser & timeline */}
<div className="flex h-[calc(100%_-_8rem)] w-full gap-6">
- {/* VNC browser */}
- {(!activeDebugSession ||
- activeDebugSession.vnc_streaming_supported) && (
- <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
- {isRateLimited ? (
- <div
- data-testid="browser-rate-limit-message"
- className="flex w-full flex-1 items-center justify-center"
+ {isRateLimited && shouldFetchDebugSession && (
+ <div
+ data-testid="browser-rate-limit-message"
+ className="flex h-full w-[calc(100%_-_6rem)] flex-1 items-center justify-center"
+ >
+ <div className="flex max-w-md flex-col items-center justify-center gap-4 rounded-md border border-neutral-200 bg-white p-8 text-center dark:border-neutral-800 dark:bg-neutral-950">
+ <p className="text-sm text-neutral-600 dark:text-neutral-300">
+ Failed to load a browser. We have a high demand for
+ browsers right now. The browser will become available
+ again automatically in ~30 minutes. If the issue
+ persists, please contact support.
+ </p>
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={() => {
+ resetOnSuccess();
+ queryClient.invalidateQueries({
+ queryKey: ["debugSession", workflowPermanentId],
+ });
+ }}
>
- <div className="flex max-w-md flex-col items-center justify-center gap-4 rounded-md border border-neutral-200 bg-white p-8 text-center dark:border-neutral-800 dark:bg-neutral-950">
- <p className="text-sm text-neutral-600 dark:text-neutral-300">
- Failed to load a browser. We have a high demand for
- browsers right now. The browser will become
- available again automatically in ~30 minutes. If the
- issue persists, please contact support.
- </p>
- <Button
- variant="outline"
- size="sm"
- onClick={() => {
- resetOnSuccess();
- queryClient.invalidateQueries({
- queryKey: ["debugSession", workflowPermanentId],
- });
+ Try again
+ </Button>
+ </div>
+ </div>
+ )}
+
+ {/* Live browser: mode comes from BROWSER_STREAMING_MODE / runtime config */}
+ {showVncBrowserPanel && (
+ <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
+ <div key={reloadKey} className="w-full flex-1">
+ {!liveBrowserSessionId ? (
+ isDebugSessionError ? (
+ <StreamStatusPanel
+ diagnostic={{
+ title: "Could not start browser session",
+ detail:
+ getAxiosErrorDetail(debugSessionError) ??
+ "The backend rejected the browser session request.",
+ hint: "Local dev only supports one browser at a time. Retry after closing other agents.",
}}
>
- Try again
- </Button>
- </div>
- </div>
- ) : (
- <div key={reloadKey} className="w-full flex-1">
- {isFlowCanvasReady ? (
- <BrowserStream
- exfiltrate={recordingStore.isRecording}
- interactive={true}
- browserSessionId={
- activeDebugSession?.browser_session_id
- }
- showControlButtons={true}
- resizeTrigger={windowResizeTrigger}
- isExecuting={!!workflowRun && !isFinalized}
- onReadyChange={handleLiveBrowserReadyChange}
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={() => void refetchDebugSession()}
+ >
+ Retry
+ </Button>
+ </StreamStatusPanel>
+ ) : (
+ <StreamStatusPanel
+ diagnostic={{
+ title: "Starting browser session",
+ detail:
+ "Creating a debug browser session for this agent.",
+ }}
/>
- ) : null}
- </div>
- )}
+ )
+ ) : isFlowCanvasReady ? (
+ <BrowserStream
+ key={liveBrowserSessionId}
+ exfiltrate={recordingStore.isRecording}
+ interactive={true}
+ browserSessionId={liveBrowserSessionId}
+ showControlButtons={true}
+ resizeTrigger={windowResizeTrigger}
+ isExecuting={!!workflowRun && !isFinalized}
+ onReadyChange={handleLiveBrowserReadyChange}
+ />
+ ) : (
+ <StreamStatusPanel
+ diagnostic={{
+ title: "Preparing live browser",
+ detail:
+ "Waiting for the workflow canvas to finish loading.",
+ }}
+ />
+ )}
+ </div>
<footer className="flex h-[2rem] w-full items-center justify-start gap-4 text-muted-foreground">
<WorkflowCopilotButton
ref={copilotButtonRef}
@@ -1894,59 +1964,90 @@ function Workspace({
</div>
)}
- {/* Local browser stream: only in local mode when VNC is not supported */}
- {activeDebugSession &&
- !activeDebugSession.vnc_streaming_supported &&
- browserStreamingMode === "cdp" && (
- <div className="skyvern-screenshot-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
+ {showCdpBrowserPanel && (
+ <div className="skyvern-screenshot-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
+ <div
+ key={reloadKey}
+ className="flex w-full flex-1 items-center justify-center"
+ >
+ {!liveBrowserSessionId ? (
+ isDebugSessionError ? (
+ <StreamStatusPanel
+ diagnostic={{
+ title: "Could not start browser session",
+ detail:
+ getAxiosErrorDetail(debugSessionError) ??
+ "The backend rejected the browser session request.",
+ hint: "Local dev only supports one browser at a time. Retry after closing other agents.",
+ }}
+ >
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={() => void refetchDebugSession()}
+ >
+ Retry
+ </Button>
+ </StreamStatusPanel>
+ ) : (
+ <StreamStatusPanel
+ diagnostic={{
+ title: "Starting browser session",
+ detail:
+ "Creating a debug browser session for this agent.",
+ }}
+ />
+ )
+ ) : isFlowCanvasReady ? (
+ <BrowserSessionStream
+ browserSessionId={liveBrowserSessionId}
+ interactive={true}
+ showControlButtons={true}
+ onReadyChange={handleLiveBrowserReadyChange}
+ />
+ ) : (
+ <StreamStatusPanel
+ diagnostic={{
+ title: "Preparing live browser",
+ detail:
+ "Waiting for the workflow canvas to finish loading.",
+ }}
+ />
+ )}
+ </div>
+ <footer className="flex h-[2rem] w-full items-center justify-start gap-4 text-muted-foreground">
+ <WorkflowCopilotButton
+ ref={copilotButtonRef}
+ messageCount={copilotMessageCount}
+ onClick={() => setIsCopilotOpen((prev) => !prev)}
+ />
+ <div className="flex items-center gap-2 text-muted-foreground">
+ <GlobeIcon /> Live Browser
+ <StreamModeBadge mode="cdp" />
+ </div>
<div
- key={reloadKey}
- className="flex w-full flex-1 items-center justify-center"
+ className={cn("ml-auto flex items-center gap-2", {
+ "mr-16": !blockLabel,
+ })}
>
- {isFlowCanvasReady ? (
- <BrowserSessionStream
- browserSessionId={
- activeDebugSession.browser_session_id
- }
- interactive={true}
- showControlButtons={true}
- onReadyChange={handleLiveBrowserReadyChange}
+ {!recordingStore.isRecording && showPowerButton && (
+ <PowerButton onClick={() => cycle()} />
+ )}
+ {!recordingStore.isRecording && (
+ <ReloadButton
+ isReloading={isReloading}
+ onClick={() => reload()}
/>
- ) : null}
+ )}
</div>
- <footer className="flex h-[2rem] w-full items-center justify-start gap-4 text-muted-foreground">
- <WorkflowCopilotButton
- ref={copilotButtonRef}
- messageCount={copilotMessageCount}
- onClick={() => setIsCopilotOpen((prev) => !prev)}
- />
- <div className="flex items-center gap-2 text-muted-foreground">
- <GlobeIcon /> Live Browser
- <StreamModeBadge mode="cdp" />
- </div>
- <div
- className={cn("ml-auto flex items-center gap-2", {
- "mr-16": !blockLabel,
- })}
- >
- {!recordingStore.isRecording && showPowerButton && (
- <PowerButton onClick={() => cycle()} />
- )}
- {!recordingStore.isRecording && (
- <ReloadButton
- isReloading={isReloading}
- onClick={() => reload()}
- />
- )}
- </div>
- </footer>
- </div>
- )}
+ </footer>
+ </div>
+ )}
- {/* Fallback: non-local without VNC (edge case) */}
{activeDebugSession &&
+ preferVncStream &&
!activeDebugSession.vnc_streaming_supported &&
- browserStreamingMode !== "cdp" && (
+ !showCdpBrowserPanel && (
<div className="flex h-full w-[calc(100%_-_6rem)] flex-1 items-center justify-center text-muted-foreground">
Browser streaming unavailable
</div>
diff --git a/skyvern/config.py b/skyvern/config.py
index be0e42425..116149ea7 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -133,6 +133,9 @@ class Settings(BaseSettings):
LOG_RAW_API_REQUESTS: bool = True
LOG_LEVEL: str = "INFO"
COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS: float = 12.0
+ # Gate copilot verified-success on per-criterion outcome evidence.
+ # Off restores the prior run-status/suspicious-success gate and classifier prompt.
+ COPILOT_OUTCOME_VERIFICATION_ENABLED: bool = True
# Dispatch flag for the workflow copilot v2 (openai-agents-SDK rewrite).
# Off = existing direct-LLM copilot at workflow_copilot_chat_post.
# On = new agent-SDK path under skyvern.forge.sdk.copilot.
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-completion-verification.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-completion-verification.j2
new file mode 100644
index 000000000..b2f9a6500
--- /dev/null
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-completion-verification.j2
@@ -0,0 +1,20 @@
+You verify whether a workflow run achieved the user's outcome goal. You are given a list of completion criteria (the END STATES the user wants to be true) and the EVIDENCE the run actually produced. For each criterion, decide whether the produced evidence demonstrates that outcome.
+
+Return only JSON of the form {"verdicts": [{"criterion_id": "c0", "satisfied": true, "reason_code": "evidence_confirms", "evidence_ref": "confirm_cart"}]}. Emit exactly one verdict object per criterion_id listed below.
+
+Rules:
+- satisfied=true ONLY when the produced evidence directly demonstrates the criterion's end state (an extraction/validation output value, or the observed end-state URL/title). When in doubt, satisfied=false.
+- Judge OUTCOMES, not steps. The fact that a block ran, a label was executed, or a navigation happened is NEVER sufficient on its own — a run can "complete" without achieving the outcome.
+- If the evidence does not contain anything that shows the outcome, set satisfied=false and reason_code=no_evidence. Do not infer the outcome from the presence of a step or from the criterion text itself.
+- If the evidence shows the outcome did NOT happen (e.g. the cart is empty, an error/blocker, a contradicting value), set satisfied=false and reason_code=evidence_contradicts.
+- If you cannot tell from the evidence, set satisfied=false and reason_code=unknown.
+- For an implicit constraint (e.g. "added exactly once"), check the end-state value (e.g. a cart quantity), not whether an add step ran.
+- evidence_ref: a short pointer to the specific evidence you used (a block label or the observed URL), or null.
+
+reason_code is one of: evidence_confirms (satisfied=true), no_evidence, evidence_contradicts, unknown.
+
+Completion criteria:
+```{{ criteria }}```
+
+Evidence the run produced:
+```{{ run_evidence }}```
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index e24db870a..0ff725238 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -1,5 +1,5 @@
Classify the latest Workflow Copilot request. Return only JSON with keys:
-testing_intent=require_test|skip_test|unspecified; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
+testing_intent=require_test|skip_test|unspecified; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; {% if outcome_verification_enabled %}completion_criteria; {% endif %}raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
testing_intent:
- skip_test: the user asks for an untested draft, says they will test it themselves, says draft only, just sketch it, no validation run, or skip testing.
@@ -24,6 +24,15 @@ completion_contract:
- The value must be an exact span copied from the latest user message. Do not paraphrase, summarize, or infer a completion condition.
- Set to null when the user does not provide an explicit completion condition.
+{% if outcome_verification_enabled %}completion_criteria:
+- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, implicit, method_mandated}.
+- outcome: a short statement of an end state the workflow must achieve (an END, not a step). Extract the goal as outcomes the user wants to be true, not the recipe they describe.
+- Capture ALL distinct outcomes the goal requires, including implicit constraints the user did not state as a step (for example "added exactly once / not duplicated", "the item is present in the cart"). Mark such items implicit=true.
+- A prescribed HOW step is NOT a criterion. "search for X", "open the result lightbox", "click add to cart", "go to the cart page" are methods/navigation, not outcomes — do not list them. A different or shorter path that reaches the same end state must still satisfy the goal.
+- method_mandated=true ONLY when the user explicitly makes a specific method the point of the request (for example "verify it works by submitting the real form", "specifically use the search bar"). Otherwise method_mandated=false.
+- Set to [] when the user gives no goal with a checkable outcome (for example a pure draft-only or sketch request).
+{% endif %}
+
raw_secret_evidence:
- When credential_input_kind=raw_secret, set this to the verbatim substring of the LATEST user message that is the pasted secret value (e.g. the password string, JWT, API key, OTP digits, or PEM body). Do not paraphrase. Do not cite a token that appears only in prior turns or in the workflow_yaml — the substring must occur in the latest user message.
- Set to null in every other case, including when prior turns contained pasted credentials but the latest message does not.
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index e5669816e..d1d1a218f 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -51,6 +51,10 @@ from skyvern.forge.sdk.copilot.context import (
TurnNarrativePayload,
finalize_discovery_counter_in_global_llm_context,
)
+from skyvern.forge.sdk.copilot.outcome_verification_trace import (
+ finalize_outcome_verification_trace,
+ record_gate_decision,
+)
from skyvern.forge.sdk.copilot.output_policy import (
UNVALIDATED_DISCLOSURE_PHRASES,
WORKFLOW_PRESENT_SENTINEL,
@@ -754,13 +758,28 @@ def _shape_ask_question_response(user_response: str, ctx: CopilotContext) -> str
return user_response
+def _completion_contract_not_violated(ctx: CopilotContext) -> bool:
+ result = ctx.completion_verification_result
+ if result is None:
+ return True
+ if result.status != "evaluated":
+ # Verification was required for this run but could not produce a verdict
+ # (unavailable): do not surface the workflow as verified on run status alone.
+ return False
+ return result.is_fully_satisfied()
+
+
def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:
- """Surface a proposal only when it passed a test this turn AND yaml is on hand."""
+ """Surface a proposal when it passed a test this turn, or when the outcome judge
+ confirmed the goal from evidence even though the run did not finish cleanly."""
+ from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verified
+
+ run_status_clean = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True
if (
ctx.last_workflow is not None
and ctx.last_workflow_yaml
- and ctx.last_test_ok is True
- and ctx.last_full_workflow_test_ok is True
+ and (run_status_clean or outcome_fully_verified(ctx))
+ and _completion_contract_not_violated(ctx)
):
return ctx.last_workflow, ctx.last_workflow_yaml
return None, None
@@ -2538,6 +2557,7 @@ async def run_copilot_agent(
# is installed; otherwise the very first turn lands the parent span on
# OTel's no-op ProxyTracer when running locally with COPILOT_TRACING_ENABLED.
ensure_tracing_initialized()
+ ctx_sink: list[CopilotContext] = []
with _copilot_turn_span(
chat_request=chat_request,
chat_history=chat_history,
@@ -2560,6 +2580,7 @@ async def run_copilot_agent(
turn_index=normalized_turn_index,
prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
prior_block_count=prior_block_count,
+ ctx_sink=ctx_sink,
)
except Exception as exc:
LOG.error(
@@ -2584,6 +2605,8 @@ async def run_copilot_agent(
turn_index=normalized_turn_index,
)
return _build_unexpected_error_exit_result(ctx, global_llm_context, error=exc, span=turn_span)
+ finally:
+ finalize_outcome_verification_trace(ctx_sink[0] if ctx_sink else None, turn_span)
except Exception as exc:
LOG.error(
"Copilot turn unhandled error",
@@ -2624,6 +2647,7 @@ async def _run_copilot_turn_impl(
turn_index: int,
prior_copilot_workflow_yaml: str | None = None,
prior_block_count: int | None = None,
+ ctx_sink: list[CopilotContext] | None = None,
) -> AgentResult:
copilot_config = config or CopilotConfig(security_rules=security_rules)
chat_history_text = _format_chat_history(chat_history)
@@ -2694,6 +2718,8 @@ async def _run_copilot_turn_impl(
raise RuntimeError(
f"CopilotContext.turn_id ({ctx.turn_id!r}) diverged from route-supplied turn_id ({turn_id!r})"
)
+ if ctx_sink is not None:
+ ctx_sink.append(ctx)
policy_inputs = RequestPolicyGuardrailInputs(
user_message=chat_request.message,
workflow_yaml=safe_workflow_yaml,
@@ -2805,8 +2831,8 @@ async def _run_copilot_turn_impl(
CopilotNonRetriableNavError,
CopilotTotalTimeoutError,
CopilotUnrecoverableToolError,
+ gate_decision_trace_fields,
run_with_enforcement,
- verified_goal_satisfied_context,
)
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer
@@ -3084,7 +3110,11 @@ async def _run_copilot_turn_impl(
)
except Exception as e:
try:
- goal_satisfied = verified_goal_satisfied_context(ctx)
+ # Terminal-path gate-decision record; the per-tool hook records the
+ # in-loop path, and the later write wins on the shared snapshot.
+ gate_fields = gate_decision_trace_fields(ctx)
+ record_gate_decision(ctx, gate_fields)
+ goal_satisfied = gate_fields["gate_satisfied"]
except Exception:
LOG.error("Copilot agent error", error=str(e), exc_info=True)
return _build_unexpected_error_exit_result(ctx, global_llm_context, error=e)
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
new file mode 100644
index 000000000..2eb05121a
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -0,0 +1,186 @@
+"""Outcome-criteria verification judge for the workflow copilot.
+
+For a workflow test run — whether it completed cleanly or was canceled/partial —
+this focused LLM call checks each completion criterion (an end-state outcome the
+user asked for) against the evidence the run actually produced — extraction/
+validation block outputs and the observed end-state URL/title — and returns a typed
+per-criterion verdict. The deterministic gate consumes the typed result; this
+module never decides the gate.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+import structlog
+
+from skyvern.config import settings
+from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.copilot.output_utils import parse_final_response
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, redact_raw_secrets_for_prompt
+from skyvern.utils.strings import escape_code_fences
+
+LOG = structlog.get_logger()
+
+PROMPT_TEMPLATE_NAME = "workflow-copilot-completion-verification"
+_EVIDENCE_VALUE_MAX_CHARS = 2000
+_MAX_BLOCK_OUTPUTS = 20
+_REASON_CODES = frozenset({"evidence_confirms", "no_evidence", "evidence_contradicts", "unknown"})
+
+VerificationStatus = Literal["evaluated", "unavailable"]
+
+
+@dataclass(frozen=True)
+class CriterionVerdict:
+ criterion_id: str
+ satisfied: bool
+ reason_code: str
+ evidence_ref: str | None = None
+
+
+@dataclass(frozen=True)
+class CompletionVerificationResult:
+ status: VerificationStatus
+ criterion_ids: list[str] = field(default_factory=list)
+ verdicts: list[CriterionVerdict] = field(default_factory=list)
+
+ def is_fully_satisfied(self) -> bool:
+ if self.status != "evaluated" or not self.criterion_ids:
+ return False
+ satisfied = {verdict.criterion_id for verdict in self.verdicts if verdict.satisfied}
+ return all(criterion_id in satisfied for criterion_id in self.criterion_ids)
+
+ def to_trace_data(self) -> dict[str, Any]:
+ return {
+ "status": self.status,
+ "criterion_count": len(self.criterion_ids),
+ "satisfied_count": sum(1 for verdict in self.verdicts if verdict.satisfied),
+ "fully_satisfied": self.is_fully_satisfied(),
+ "reason_codes": [verdict.reason_code for verdict in self.verdicts],
+ }
+
+
+@dataclass(frozen=True)
+class RunEvidenceSnapshot:
+ workflow_run_id: str | None = None
+ block_outputs: dict[str, Any] = field(default_factory=dict)
+ current_url: str | None = None
+ page_title: str | None = None
+ executed_block_labels: list[str] = field(default_factory=list)
+
+ def has_evidence(self) -> bool:
+ return bool(self.block_outputs or self.current_url or self.page_title)
+
+ def render_prompt_block(self) -> str:
+ lines: list[str] = []
+ if self.workflow_run_id:
+ lines.append(f"workflow_run_id: {self.workflow_run_id}")
+ if self.current_url:
+ lines.append(f"observed_end_state_url: {self.current_url}")
+ if self.page_title:
+ lines.append(f"observed_end_state_page_title: {self.page_title}")
+ if self.executed_block_labels:
+ lines.append("executed_block_labels: " + ", ".join(self.executed_block_labels[:_MAX_BLOCK_OUTPUTS]))
+ if self.block_outputs:
+ lines.append("produced_block_outputs:")
+ for label, payload in list(self.block_outputs.items())[:_MAX_BLOCK_OUTPUTS]:
+ serialized = payload if isinstance(payload, str) else json.dumps(payload, default=str)
+ serialized = " ".join(serialized.split())[:_EVIDENCE_VALUE_MAX_CHARS]
+ lines.append(f" - {label}: {serialized}")
+ return redact_raw_secrets_for_prompt("\n".join(lines))
+
+
+_UNAVAILABLE = CompletionVerificationResult(status="unavailable")
+
+
+def summarize_unsatisfied_outcomes(result: CompletionVerificationResult, criteria: list[CompletionCriterion]) -> str:
+ outcome_by_id = {criterion.id: criterion.outcome for criterion in criteria}
+ satisfied = {verdict.criterion_id for verdict in result.verdicts if verdict.satisfied}
+ unmet = [outcome_by_id[cid] for cid in result.criterion_ids if cid not in satisfied and cid in outcome_by_id]
+ return "; ".join(unmet)
+
+
+def _render_criteria(criteria: list[CompletionCriterion]) -> str:
+ parts: list[str] = []
+ for criterion in criteria:
+ flags = ""
+ if criterion.implicit:
+ flags += " (implicit)"
+ if criterion.method_mandated:
+ flags += " (method_mandated)"
+ parts.append(f"- {criterion.id}: {criterion.outcome}{flags}")
+ return "\n".join(parts)
+
+
+def _coerce_result(raw: Any, criterion_ids: list[str]) -> CompletionVerificationResult:
+ if isinstance(raw, bytes):
+ raw = raw.decode("utf-8", errors="replace")
+ if isinstance(raw, str):
+ raw = parse_final_response(raw)
+ if not isinstance(raw, dict):
+ return _UNAVAILABLE
+ raw_verdicts = raw.get("verdicts")
+ if not isinstance(raw_verdicts, list):
+ return _UNAVAILABLE
+
+ allowed = set(criterion_ids)
+ by_id: dict[str, CriterionVerdict] = {}
+ for item in raw_verdicts:
+ if not isinstance(item, dict):
+ continue
+ criterion_id = item.get("criterion_id")
+ if criterion_id not in allowed or criterion_id in by_id:
+ continue
+ reason = item.get("reason_code")
+ reason_code = reason if reason in _REASON_CODES else "unknown"
+ # A criterion counts as satisfied only when the judge cites confirming
+ # evidence; an "unknown"/"no_evidence"/"contradicts" verdict never passes.
+ satisfied = bool(item.get("satisfied")) and reason_code == "evidence_confirms"
+ evidence_ref_raw = item.get("evidence_ref")
+ evidence_ref = (
+ evidence_ref_raw.strip() if isinstance(evidence_ref_raw, str) and evidence_ref_raw.strip() else None
+ )
+ by_id[criterion_id] = CriterionVerdict(
+ criterion_id=criterion_id,
+ satisfied=satisfied,
+ reason_code=reason_code,
+ evidence_ref=evidence_ref,
+ )
+
+ verdicts = [
+ by_id.get(criterion_id, CriterionVerdict(criterion_id=criterion_id, satisfied=False, reason_code="no_evidence"))
+ for criterion_id in criterion_ids
+ ]
+ return CompletionVerificationResult(status="evaluated", criterion_ids=list(criterion_ids), verdicts=verdicts)
+
+
+async def evaluate_completion_criteria(
+ criteria: list[CompletionCriterion],
+ snapshot: RunEvidenceSnapshot,
+ handler: Any,
+) -> CompletionVerificationResult:
+ if handler is None or not criteria:
+ return _UNAVAILABLE
+
+ criterion_ids = [criterion.id for criterion in criteria]
+ prompt = prompt_engine.load_prompt(
+ template=PROMPT_TEMPLATE_NAME,
+ criteria=escape_code_fences(_render_criteria(criteria)),
+ run_evidence=escape_code_fences(snapshot.render_prompt_block()),
+ )
+ try:
+ raw = await asyncio.wait_for(
+ handler(prompt=prompt, prompt_name=PROMPT_TEMPLATE_NAME),
+ timeout=settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ LOG.warning("completion-verification judge timed out")
+ return _UNAVAILABLE
+ except Exception as exc:
+ LOG.warning("completion-verification judge failed", error=str(exc))
+ return _UNAVAILABLE
+
+ return _coerce_result(raw, criterion_ids)
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index fdaeda27b..8879010c9 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -10,6 +10,7 @@ from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prom
from skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_RE
if TYPE_CHECKING:
+ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.context import CopilotContext
_TEXT_MAX = 240
@@ -124,7 +125,9 @@ def build_diagnosis_repair_contract(
next_action = _next_action(failure_type, ctx, data)
frontier = _safe_str(data.get("frontier_start_label"))
target_blocks = failed_blocks or ([frontier] if frontier else []) if next_action == RepairNextAction.REPAIR else []
- user_goal_satisfied, completion_contract_satisfied = _verification_satisfaction(run_ok, suspicious, run_status)
+ user_goal_satisfied, completion_contract_satisfied = _verification_satisfaction(
+ run_ok, suspicious, run_status, getattr(ctx, "completion_verification_result", None)
+ )
confidence = (
0.9
if failure_type == DiagnosisFailureType.NO_FAILURE
@@ -357,12 +360,16 @@ def _verification_satisfaction(
run_ok: bool,
suspicious: bool,
run_status: str | None,
+ completion_verification: CompletionVerificationResult | None = None,
) -> tuple[bool | None, bool | None]:
user_goal_satisfied = (not suspicious) if run_ok else None if run_status is None else False
- # Current tool evidence has one completion signal: whether the tested run
- # can be trusted. Keep these fields separate for future richer contracts,
- # but mirror them until an explicit completion-contract signal exists.
- completion_contract_satisfied = user_goal_satisfied
+ # When the verification judge was required (a non-null result), its verdict
+ # is authoritative: an unmet or unavailable verdict cannot claim the outcome.
+ # With no result (no criteria, judge skipped) fall back to run trust.
+ if completion_verification is not None:
+ completion_contract_satisfied: bool | None = completion_verification.is_fully_satisfied()
+ else:
+ completion_contract_satisfied = user_goal_satisfied
return user_goal_satisfied, completion_contract_satisfied
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index cdeb32417..0a8f144c6 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any
import structlog
from agents.run import Runner
+from skyvern.config import settings
from skyvern.forge.sdk.copilot import config as copilot_config_defaults
from skyvern.forge.sdk.copilot.config import (
DEFAULT_ENFORCEMENT_NUDGES,
@@ -226,13 +227,61 @@ def latest_diagnosis_contract_satisfies_goal(ctx: CopilotContext) -> bool:
)
+def _outcome_criteria_evaluated(ctx: CopilotContext) -> bool:
+ if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
+ return False
+ result = ctx.completion_verification_result
+ return result is not None and result.status == "evaluated"
+
+
+def outcome_fully_verified(ctx: CopilotContext) -> bool:
+ """The judge confirmed every outcome criterion from the evidence this run produced.
+
+ This evidence is authoritative over run status: a run that reached the goal is
+ recognized even when it was canceled or only partially completed. Run status must
+ never suppress recognition of an outcome the user can observe was achieved.
+ """
+ if not _outcome_criteria_evaluated(ctx):
+ return False
+ result = ctx.completion_verification_result
+ return result is not None and result.is_fully_satisfied()
+
+
def verified_goal_satisfied_context(ctx: CopilotContext) -> bool:
- return (
+ if outcome_fully_verified(ctx):
+ return True
+ # The judge verdict is authoritative: once it has evaluated, an unconfirmed
+ # criterion means the outcome is unmet regardless of run status. The block-count
+ # heuristic (which counts method verbs in the request) governs only when there
+ # is no evaluated verdict.
+ if _outcome_criteria_evaluated(ctx):
+ return False
+ if not (
ctx.last_test_ok is True
and ctx.last_full_workflow_test_ok is True
and latest_diagnosis_contract_satisfies_goal(ctx)
- and not _verified_goal_likely_needs_more_work(ctx)
- )
+ ):
+ return False
+ return not _verified_goal_likely_needs_more_work(ctx)
+
+
+def gate_decision_trace_fields(ctx: CopilotContext) -> dict[str, bool]:
+ """The terminal-gate decision plus the conjuncts that explain it.
+
+ Captured wherever the gate is evaluated (including when it returns False, the
+ signal that explains why the turn continued) so a single trace shows whether
+ the gate failed on the test, the full-workflow run, the diagnosis contract,
+ the absence of outcome verification, or the block-count heuristic.
+ """
+ return {
+ "gate_satisfied": verified_goal_satisfied_context(ctx),
+ "gate_last_test_ok": ctx.last_test_ok is True,
+ "gate_last_full_workflow_test_ok": ctx.last_full_workflow_test_ok is True,
+ "gate_diagnosis_contract_satisfies_goal": latest_diagnosis_contract_satisfies_goal(ctx),
+ "gate_outcome_criteria_evaluated": _outcome_criteria_evaluated(ctx),
+ "gate_likely_needs_more_work": _verified_goal_likely_needs_more_work(ctx),
+ "gate_evaluated_this_turn": True,
+ }
def _verified_goal_likely_needs_more_work(ctx: CopilotContext) -> bool:
diff --git a/skyvern/forge/sdk/copilot/hooks.py b/skyvern/forge/sdk/copilot/hooks.py
index e08e1d90a..fc01f0f71 100644
--- a/skyvern/forge/sdk/copilot/hooks.py
+++ b/skyvern/forge/sdk/copilot/hooks.py
@@ -12,7 +12,12 @@ from agents.lifecycle import RunHooksBase
from agents.run_context import RunContextWrapper
from agents.tool import Tool
-from skyvern.forge.sdk.copilot.enforcement import CopilotGoalSatisfied, verified_goal_satisfied_context
+from skyvern.forge.sdk.copilot.enforcement import (
+ CopilotGoalSatisfied,
+ gate_decision_trace_fields,
+ outcome_fully_verified,
+)
+from skyvern.forge.sdk.copilot.outcome_verification_trace import record_gate_decision
from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result
from skyvern.forge.sdk.copilot.streaming_adapter import parse_tool_output
@@ -33,6 +38,7 @@ _VERIFIED_GOAL_CONTEXT_ATTRS: frozenset[str] = frozenset(
"last_full_workflow_test_ok",
"last_update_block_count",
"latest_diagnosis_repair_contract",
+ "completion_verification_result",
"user_message",
}
)
@@ -41,11 +47,15 @@ _VERIFIED_GOAL_CONTEXT_ATTRS: frozenset[str] = frozenset(
def _tool_completion_satisfies_turn(ctx: CopilotContext, tool_name: str, parsed: Mapping[str, object]) -> bool:
if tool_name not in {"run_blocks_and_collect_debug", "update_and_run_blocks"}:
return False
- if parsed.get("ok") is not True:
- return False
if not all(hasattr(ctx, attr) for attr in _VERIFIED_GOAL_CONTEXT_ATTRS):
return False
- return verified_goal_satisfied_context(ctx)
+ # An unfinished run (ok != True) can still satisfy the turn when the outcome
+ # judge confirmed the goal from evidence: recognition must not key on run status.
+ if parsed.get("ok") is not True and not outcome_fully_verified(ctx):
+ return False
+ gate_fields = gate_decision_trace_fields(ctx)
+ record_gate_decision(ctx, gate_fields)
+ return gate_fields["gate_satisfied"]
class CopilotRunHooks(RunHooksBase):
diff --git a/skyvern/forge/sdk/copilot/outcome_verification_trace.py b/skyvern/forge/sdk/copilot/outcome_verification_trace.py
new file mode 100644
index 000000000..c8bc4fcfe
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/outcome_verification_trace.py
@@ -0,0 +1,110 @@
+"""Turn-level outcome-verification telemetry for the workflow copilot.
+
+The completion-verification verdict and the terminal-gate decision are produced
+and consumed mid-turn, and ``ctx.completion_verification_result`` is cleared at
+the start of every run-tool call. So the verdict and gate decision are captured
+into a per-turn snapshot at the moment they are produced/consumed, and the
+snapshot is attached to the copilot.turn span once at turn finalization — the
+snapshot survives a later run-tool call that clears the live result field.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import structlog
+from opentelemetry import trace as otel_trace
+
+LOG = structlog.get_logger()
+
+_SNAPSHOT_ATTR = "outcome_verification_trace_snapshot"
+_COMPLETION_PREFIX = "completion_verification_"
+
+
+def _snapshot(ctx: Any) -> dict[str, Any] | None:
+ snapshot = getattr(ctx, _SNAPSHOT_ATTR, None)
+ if isinstance(snapshot, dict):
+ return snapshot
+ snapshot = {}
+ try:
+ setattr(ctx, _SNAPSHOT_ATTR, snapshot)
+ except Exception:
+ return None
+ return snapshot
+
+
+def record_gate_decision(ctx: Any, fields: dict[str, Any]) -> None:
+ try:
+ snapshot = _snapshot(ctx)
+ if snapshot is not None:
+ snapshot.update(fields)
+ except Exception:
+ LOG.warning("failed to record copilot gate decision telemetry", exc_info=True)
+
+
+def record_completion_verification(ctx: Any, result: Any | None) -> None:
+ """Refresh the snapshot's completion-verification block for the latest run.
+
+ Called on every recorded run, evaluated or not, so the snapshot describes the
+ turn's most recent run and never preserves a stale verdict from an earlier one.
+ """
+ try:
+ snapshot = _snapshot(ctx)
+ if snapshot is None:
+ return
+ for key in [key for key in snapshot if key.startswith(_COMPLETION_PREFIX)]:
+ del snapshot[key]
+ if result is not None and getattr(result, "status", None) == "evaluated":
+ snapshot.update({f"{_COMPLETION_PREFIX}{key}": value for key, value in result.to_trace_data().items()})
+ snapshot[f"{_COMPLETION_PREFIX}evaluated_on_final_run"] = True
+ else:
+ snapshot[f"{_COMPLETION_PREFIX}status"] = getattr(result, "status", None) or "not_run"
+ snapshot[f"{_COMPLETION_PREFIX}evaluated_on_final_run"] = False
+ except Exception:
+ LOG.warning("failed to record copilot completion verification telemetry", exc_info=True)
+
+
+def outcome_verification_turn_fields(ctx: Any) -> dict[str, Any]:
+ fields: dict[str, Any] = {}
+ snapshot = getattr(ctx, _SNAPSHOT_ATTR, None)
+ if isinstance(snapshot, dict):
+ fields.update(snapshot)
+ evidence = getattr(ctx, "workflow_verification_evidence", None)
+ if evidence is not None and hasattr(evidence, "to_trace_data"):
+ fields.update({f"verification_evidence_{key}": value for key, value in evidence.to_trace_data().items()})
+ policy = getattr(ctx, "request_policy", None)
+ if policy is not None and hasattr(policy, "to_trace_data"):
+ fields.update({f"request_policy_{key}": value for key, value in policy.to_trace_data().items()})
+ return _otel_safe(fields)
+
+
+def finalize_outcome_verification_trace(ctx: Any, span: Any = None) -> None:
+ """Attach the accumulated snapshot to the copilot.turn span once per turn.
+
+ Strictly best-effort: a telemetry failure here must never convert a valid
+ turn into an unexpected-error result.
+ """
+ try:
+ if ctx is None:
+ return
+ fields = outcome_verification_turn_fields(ctx)
+ if not fields:
+ return
+ target = span if span is not None else otel_trace.get_current_span()
+ target.set_attributes(fields)
+ except Exception:
+ LOG.warning("failed to finalize copilot outcome verification telemetry", exc_info=True)
+
+
+def _otel_safe(fields: dict[str, Any]) -> dict[str, Any]:
+ safe: dict[str, Any] = {}
+ for key, value in fields.items():
+ if isinstance(value, (bool, int, float, str)):
+ safe[key] = value
+ elif isinstance(value, (list, tuple)) and all(isinstance(item, str) for item in value):
+ safe[key] = list(value)
+ elif value is None:
+ continue
+ else:
+ safe[key] = str(value)
+ return safe
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 23d8a056c..86dac24a3 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -118,6 +118,18 @@ _INVALID_CONDITIONAL_CONTAINER_MARKERS = (
)
+_MAX_COMPLETION_CRITERIA = 8
+_COMPLETION_CRITERION_OUTCOME_MAX_CHARS = 200
+
+
+@dataclass(frozen=True)
+class CompletionCriterion:
+ id: str
+ outcome: str
+ implicit: bool = False
+ method_mandated: bool = False
+
+
@dataclass
class RequestPolicy:
testing_intent: str = "unspecified"
@@ -130,6 +142,7 @@ class RequestPolicy:
allow_missing_credentials_in_draft: bool = False
user_response_policy: str = "proceed"
completion_contract: str | None = None
+ completion_criteria: list[CompletionCriterion] = field(default_factory=list)
resolved_credentials: list[Credential] = field(default_factory=list)
invalid_credential_ids: list[str] = field(default_factory=list)
clarification_question: str | None = None
@@ -151,6 +164,13 @@ class RequestPolicy:
"allow_missing_credentials_in_draft": self.allow_missing_credentials_in_draft,
"resolved_credential_count": len(self.resolved_credentials),
"has_completion_contract": bool(self.completion_contract),
+ "completion_criteria_count": len(self.completion_criteria),
+ "completion_criteria_implicit_count": sum(
+ 1 for criterion in self.completion_criteria if criterion.implicit
+ ),
+ "completion_criteria_method_mandated_count": sum(
+ 1 for criterion in self.completion_criteria if criterion.method_mandated
+ ),
"raw_secret_detected": self.raw_secret_detected,
"has_raw_secret_evidence": self.raw_secret_evidence is not None,
"raw_secret_handling": self.raw_secret_handling,
@@ -442,6 +462,7 @@ def _classification_from_raw(raw: Any) -> RequestPolicy:
login_page_urls=_clean_list(raw.get("login_page_urls") or []),
requires_user_clarification=bool(raw.get("requires_user_clarification")),
completion_contract=completion_contract or None,
+ completion_criteria=_parse_completion_criteria(raw.get("completion_criteria")),
raw_secret_evidence=raw_secret_evidence,
raw_secret_handling=_coerce_raw_secret_handling(raw.get("raw_secret_handling")),
clarification_reason=_coerce_clarification_reason(raw.get("clarification_reason")),
@@ -475,6 +496,43 @@ def _ground_completion_contract(user_message: str, value: str | None) -> str | N
return None
+def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
+ """Build outcome criteria from the classifier output.
+
+ IDs are assigned server-side by index after de-duplication; any id the
+ model emits is discarded because the downstream satisfaction check keys on
+ a unique id set.
+ """
+ if not isinstance(raw, list):
+ return []
+ criteria: list[CompletionCriterion] = []
+ seen: set[str] = set()
+ for item in raw:
+ if not isinstance(item, dict):
+ continue
+ outcome_raw = item.get("outcome")
+ if not isinstance(outcome_raw, str):
+ continue
+ outcome = " ".join(outcome_raw.split())[:_COMPLETION_CRITERION_OUTCOME_MAX_CHARS].strip()
+ if not outcome:
+ continue
+ key = outcome.lower()
+ if key in seen:
+ continue
+ seen.add(key)
+ criteria.append(
+ CompletionCriterion(
+ id=f"c{len(criteria)}",
+ outcome=outcome,
+ implicit=bool(item.get("implicit")),
+ method_mandated=bool(item.get("method_mandated")),
+ )
+ )
+ if len(criteria) >= _MAX_COMPLETION_CRITERIA:
+ break
+ return criteria
+
+
def _classifier_fallback_policy(ids: list[str], *, raw_secret_present: bool) -> RequestPolicy:
if raw_secret_present:
return RequestPolicy(
@@ -516,6 +574,7 @@ async def _classify_request(
transcript = build_transcript_context(chat_history, safe_user_message)
prompt = prompt_engine.load_prompt(
template=PROMPT_NAME,
+ outcome_verification_enabled=settings.COPILOT_OUTCOME_VERIFICATION_ENABLED,
user_message=escape_code_fences(safe_user_message),
raw_secret_present=str(raw_secret_present).lower(),
workflow_yaml=escape_code_fences(redact_raw_secrets_for_prompt(workflow_yaml)[:2048]),
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index d36ae65df..f4f3983ac 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -33,6 +33,7 @@ from skyvern.library.skyvern_browser import SkyvernBrowser
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
@@ -170,6 +171,8 @@ class AgentContext:
last_good_workflow: Any | None = None
last_good_workflow_yaml: str | None = None
last_run_blocks_workflow_run_id: str | None = None
+ completion_verification_result: CompletionVerificationResult | None = None
+ outcome_verification_trace_snapshot: dict[str, Any] = field(default_factory=dict)
composition_page_evidence: dict[str, Any] | None = None
post_budget_page_inspection_required: bool = False
post_budget_page_inspection_url: str | None = None
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 62d8f49b8..eb70ec714 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import base64
+import contextlib
import json
import re
import time
@@ -26,6 +27,7 @@ from agents.run_context import RunContextWrapper
from jinja2.sandbox import SandboxedEnvironment
from pydantic import ValidationError
+from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.failure_classifier import classify_from_failure_reason
from skyvern.forge.sdk.artifact.models import ArtifactType
@@ -46,6 +48,13 @@ from skyvern.forge.sdk.copilot.build_phase import (
advance_to_discovering,
advance_to_testing,
)
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ CriterionVerdict,
+ RunEvidenceSnapshot,
+ evaluate_completion_criteria,
+ summarize_unsatisfied_outcomes,
+)
from skyvern.forge.sdk.copilot.composition_evidence import (
composition_page_evidence_error,
merge_visual_composition_evidence,
@@ -80,6 +89,7 @@ from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay
from skyvern.forge.sdk.copilot.narration import NarratorState
from skyvern.forge.sdk.copilot.narration import handler_available as narration_handler_available
from skyvern.forge.sdk.copilot.narration import narrator_poll_tick
+from skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification
from skyvern.forge.sdk.copilot.output_policy import (
evaluate_output_policy,
format_output_policy_tool_error,
@@ -134,6 +144,10 @@ _FAILED_BLOCK_STATUSES: frozenset[str] = frozenset(
}
)
_DATA_PRODUCING_BLOCK_TYPES = frozenset({"EXTRACTION", "TEXT_PROMPT"})
+# Block types whose output can demonstrate an end-state outcome. Until a workflow
+# contains one of these, an unmet outcome criterion means the build is still
+# incomplete (no confirmation step yet), not a completed run that failed the goal.
+_OUTCOME_EVIDENCE_BLOCK_TYPES = frozenset({BlockType.EXTRACTION.value, BlockType.VALIDATION.value})
# Block types whose ``block.output`` is a ``TaskOutput.from_task()`` envelope
# (schemas/tasks.py:TaskOutput) rather than the raw payload. The
@@ -2288,6 +2302,20 @@ def _current_workflow_block_labels(ctx: object) -> list[str]:
return yaml_labels
+def _current_workflow_has_evidence_block(ctx: object) -> bool:
+ workflow = getattr(ctx, "last_workflow", None)
+ blocks = getattr(getattr(workflow, "workflow_definition", None), "blocks", None)
+ if blocks:
+ return any(_block_type_name(block) in _OUTCOME_EVIDENCE_BLOCK_TYPES for block in blocks)
+ workflow_yaml = getattr(ctx, "last_workflow_yaml", None)
+ if not isinstance(workflow_yaml, str):
+ return False
+ return any(
+ isinstance(block, dict) and _enum_or_string_name(block.get("block_type")) in _OUTCOME_EVIDENCE_BLOCK_TYPES
+ for block in (_parse_workflow_blocks(workflow_yaml) or [])
+ )
+
+
def _unverified_current_workflow_labels(ctx: object) -> list[str]:
labels = _current_workflow_block_labels(ctx)
verified = set(getattr(ctx, "verified_prefix_labels", []) or [])
@@ -5129,7 +5157,15 @@ def _update_verification_evidence_from_run_result(copilot_ctx: AgentContext, res
unverified = list(copilot_ctx.last_unverified_block_labels or [])
evidence.unverified_block_labels = list(dict.fromkeys(str(label) for label in unverified if str(label)))
evidence.failed_block_labels = []
- evidence.failure_reason = None
+ # A completed-but-suspicious run (outcome unverified, null data, blocker)
+ # keeps its failure reason so the evidence stays consistent with
+ # test_attempted_but_incomplete instead of reading as a clean success.
+ suspicious_reason = copilot_ctx.last_test_failure_reason if copilot_ctx.last_test_suspicious_success else None
+ evidence.failure_reason = (
+ " ".join(suspicious_reason.split())[:240]
+ if isinstance(suspicious_reason, str) and suspicious_reason.strip()
+ else None
+ )
if evidence.unverified_block_labels:
evidence.verified_from_current_browser_state = True
return
@@ -5143,10 +5179,196 @@ def _update_verification_evidence_from_run_result(copilot_ctx: AgentContext, res
evidence.failure_reason = " ".join(failure_reason.split())[:240]
-def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:
+_COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS = 5.0
+
+
+def _completion_verification_handler() -> Any:
+ with contextlib.suppress(RuntimeError, AttributeError):
+ return app.WORKFLOW_COPILOT_FAST_LLM_API_HANDLER
+ return None
+
+
+def _is_outcome_evidence_candidate(copilot_ctx: Any, result: dict[str, Any]) -> bool:
+ """A clean ok=True run worth judging on its whole-workflow outcome.
+
+ Recognition is governed by the outcome evidence the user can observe, not by
+ whether every block was verified as an end-to-end prefix (SKY-10576). The judge
+ requires positive evidence for every criterion, and ``empty_data_blocks`` rejects
+ a run whose outcome block produced nothing, so an incomplete run never passes --
+ this only lets an already-reached goal be recognized without a redundant
+ full-prefix re-run.
+ """
+ if not bool(result.get("ok", False)):
+ return False
+ if _run_blocks_structured_blocker_message(result):
+ return False
+ _anti_bot, empty_data_blocks, _categories = _analyze_run_blocks(result)
+ return not empty_data_blocks
+
+
+def _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[str, Any]) -> bool:
+ """A canceled/partial run (ok=False) still worth judging because it left runtime
+ evidence behind. The judge confirms a criterion only on positive evidence, so a
+ broken run never spuriously passes; this only lets a reached goal be recognized
+ even though the run did not finish cleanly — recognition must not key on run status.
+ """
+ if bool(result.get("ok", False)):
+ return False
+ if _run_blocks_structured_blocker_message(result):
+ return False
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return False
+ return _valid_runtime_anchor_url(data.get("current_url")) is not None
+
+
+def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:
+ data = result.get("data")
+ data = data if isinstance(data, dict) else {}
+ current_labels = set(_current_workflow_block_labels(copilot_ctx))
+ # Evidence must be what THIS run produced. ``verified_block_outputs`` accumulates
+ # across incremental runs, so sourcing from it would let an output from a prior
+ # run satisfy a criterion the current run never re-produced.
+ blocks = data.get("blocks")
+ block_outputs: dict[str, Any] = {}
+ if isinstance(blocks, list):
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ label = block.get("label")
+ output = block.get("extracted_data")
+ if isinstance(label, str) and label in current_labels and _is_meaningful_extracted_data(output):
+ block_outputs[label] = output
+ executed = data.get("executed_block_labels")
+ page_title = data.get("page_title")
+ run_id = data.get("workflow_run_id")
+ return RunEvidenceSnapshot(
+ workflow_run_id=run_id if isinstance(run_id, str) else None,
+ block_outputs=block_outputs,
+ current_url=_valid_runtime_anchor_url(data.get("current_url")),
+ page_title=page_title if isinstance(page_title, str) and page_title.strip() else None,
+ executed_block_labels=[str(label) for label in executed] if isinstance(executed, list) else [],
+ )
+
+
+async def _maybe_run_completion_verification(
+ copilot_ctx: Any, result: dict[str, Any], handler_start: float
+) -> CompletionVerificationResult | None:
+ if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
+ return None
+ if getattr(copilot_ctx, "copilot_total_timeout_exceeded", False):
+ return None
+ policy = getattr(copilot_ctx, "request_policy", None)
+ # A method-mandated criterion asserts HOW the goal was reached; this judge sees
+ # only end-state evidence and would always return no_evidence, false-blocking a
+ # legitimate success. Verifying method adherence needs an action trace (separate).
+ criteria = [c for c in (policy.completion_criteria if policy is not None else []) if not c.method_mandated]
+ if not criteria:
+ return None
+ if not (
+ _is_outcome_evidence_candidate(copilot_ctx, result)
+ or _is_unfinished_run_verification_candidate(copilot_ctx, result)
+ ):
+ return None
+ # A missing judge handler is an infra/config state, not a transient failure:
+ # fall back to the prior gate rather than fail closed on every run.
+ handler = _completion_verification_handler()
+ if handler is None:
+ return None
+ # Too little budget to verify a candidate run: fail closed (unavailable) rather
+ # than let the run-status proxy claim an unverified outcome as success.
+ remaining = RUN_BLOCKS_SAFETY_CEILING_SECONDS - (time.monotonic() - handler_start)
+ if remaining <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS:
+ return CompletionVerificationResult(status="unavailable")
+ snapshot = _build_run_evidence_snapshot(copilot_ctx, result)
+ if not snapshot.has_evidence():
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[criterion.id for criterion in criteria],
+ verdicts=[
+ CriterionVerdict(criterion_id=criterion.id, satisfied=False, reason_code="no_evidence")
+ for criterion in criteria
+ ],
+ )
+ return await evaluate_completion_criteria(criteria, snapshot, handler)
+
+
+def _outcome_unverified_reason(
+ copilot_ctx: Any, completion_verification: CompletionVerificationResult | None
+) -> str | None:
+ if completion_verification is None:
+ return None
+ if completion_verification.status == "evaluated":
+ if completion_verification.is_fully_satisfied():
+ return None
+ policy = getattr(copilot_ctx, "request_policy", None)
+ criteria = list(policy.completion_criteria) if policy is not None else []
+ unmet = summarize_unsatisfied_outcomes(completion_verification, criteria)
+ detail = f": {unmet}" if unmet else ""
+ return (
+ f"The run completed but did not demonstrate the goal outcome(s){detail}. "
+ "Add an end-state confirmation (an extraction or validation block) that observes the outcome, then re-run."
+ )
+ # An 'unavailable' result reaches here only when verification was required;
+ # fail closed and ask for a re-run rather than claim an unverified success.
+ return (
+ "The run completed but the goal outcome could not be verified (verification was unavailable). "
+ "Re-run to verify the outcome before reporting success."
+ )
+
+
+def _outcome_failure_warrants_repair(
+ copilot_ctx: Any, completion_verification: CompletionVerificationResult | None
+) -> bool:
+ """Whether an unmet outcome should route to suspicious-success/repair rather
+ than continue-building.
+
+ Contradicting evidence is always a real failure. Absence of evidence is a
+ failure only once the workflow has an outcome-producing block; while the agent
+ is still adding blocks toward the goal, an unmet criterion is expected, and the
+ run should keep building. Terminal success stays withheld in both cases via the
+ verification result, so this only governs the repair directive, not the gate.
+ """
+ if completion_verification is None:
+ return False
+ if any(verdict.reason_code == "evidence_contradicts" for verdict in completion_verification.verdicts):
+ return True
+ return _current_workflow_has_evidence_block(copilot_ctx)
+
+
+def _emit_completion_verification_trace(
+ copilot_ctx: Any, completion_verification: CompletionVerificationResult
+) -> None:
+ block_count = getattr(copilot_ctx, "last_update_block_count", None)
+ policy = getattr(copilot_ctx, "request_policy", None)
+ contract = policy.completion_contract if policy is not None else None
+ heuristic_would_block = isinstance(block_count, int) and _goal_likely_needs_more_blocks(
+ getattr(copilot_ctx, "user_message", ""), block_count, contract
+ )
+ trace_data = {
+ **completion_verification.to_trace_data(),
+ "heuristic_would_block": heuristic_would_block,
+ "evidence_block_present": _current_workflow_has_evidence_block(copilot_ctx),
+ "warrants_repair": _outcome_failure_warrants_repair(copilot_ctx, completion_verification),
+ }
+ LOG.info(
+ "copilot completion verification",
+ **{f"completion_verification_{key}": value for key, value in trace_data.items()},
+ )
+ with copilot_span("completion_verification", data=trace_data):
+ pass
+
+
+def _record_run_blocks_result(
+ copilot_ctx: Any, result: dict[str, Any], completion_verification: CompletionVerificationResult | None = None
+) -> None:
run_ok = bool(result.get("ok", False))
data = result.get("data")
run_id = data.get("workflow_run_id") if isinstance(data, dict) else None
+ copilot_ctx.completion_verification_result = completion_verification
+ record_completion_verification(copilot_ctx, completion_verification)
+ if completion_verification is not None and completion_verification.status == "evaluated":
+ _emit_completion_verification_trace(copilot_ctx, completion_verification)
copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else None
copilot_ctx.last_successful_run_blocks_workflow_run_id = run_id if run_ok and isinstance(run_id, str) else None
# Watchdog cancels normally count as ok=False; only a coincident total
@@ -5248,7 +5470,21 @@ def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:
copilot_ctx.non_retriable_nav_error_last_emitted_signature = None
unverified = _unverified_current_workflow_labels(copilot_ctx)
copilot_ctx.last_unverified_block_labels = unverified
- if not unverified:
+ outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
+ if outcome_unverified_reason is not None and _outcome_failure_warrants_repair(
+ copilot_ctx, completion_verification
+ ):
+ # The workflow already has a confirmation block, yet the produced
+ # evidence does not demonstrate the outcome (or contradicts it). Treat
+ # it as a suspicious success so the existing repair/partial machinery
+ # fires. A mid-build run with no confirmation block yet falls through to
+ # keep-building below; terminal success stays withheld either way via
+ # the verification result.
+ copilot_ctx.last_test_suspicious_success = True
+ copilot_ctx.last_test_failure_reason = outcome_unverified_reason
+ if isinstance(data, dict):
+ data.setdefault("failure_reason", outcome_unverified_reason)
+ elif not unverified:
copilot_ctx.last_full_workflow_test_ok = True
copilot_ctx.last_good_workflow = copilot_ctx.last_workflow
copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml
@@ -5484,6 +5720,8 @@ async def run_blocks_tool(
evidence instead of retrying guessed URL params or page structure.
"""
copilot_ctx = ctx.context
+ copilot_ctx.completion_verification_result = None
+ handler_start = time.monotonic()
arguments = {"block_labels": block_labels, "parameters": parameters or {}}
authority_error = _authority_tool_error(copilot_ctx, "run_blocks_and_collect_debug")
if authority_error:
@@ -5525,7 +5763,8 @@ async def run_blocks_tool(
block_outputs_to_seed=block_outputs_to_seed,
frontier_start_label=frontier_start_label,
)
- _record_run_blocks_result(copilot_ctx, result)
+ completion_verification = await _maybe_run_completion_verification(copilot_ctx, result, handler_start)
+ _record_run_blocks_result(copilot_ctx, result, completion_verification=completion_verification)
record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, result)
_record_diagnosis_repair_contract(
copilot_ctx,
@@ -5623,6 +5862,8 @@ async def update_and_run_blocks_tool(
do not compose a click against a control observed as disabled.
"""
copilot_ctx = ctx.context
+ copilot_ctx.completion_verification_result = None
+ handler_start = time.monotonic()
arguments = {"workflow_yaml": workflow_yaml, "block_labels": block_labels, "parameters": parameters or {}}
skip_run_after_update = _request_policy_allows_update_and_skip_run(copilot_ctx, "update_and_run_blocks")
authority_error = _authority_tool_error(
@@ -5764,7 +6005,8 @@ async def update_and_run_blocks_tool(
block_outputs_to_seed=block_outputs_to_seed,
frontier_start_label=frontier_start_label,
)
- _record_run_blocks_result(copilot_ctx, run_result)
+ completion_verification = await _maybe_run_completion_verification(copilot_ctx, run_result, handler_start)
+ _record_run_blocks_result(copilot_ctx, run_result, completion_verification=completion_verification)
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, run_result)
_record_diagnosis_repair_contract(
copilot_ctx,
diff --git a/skyvern/forge/sdk/db/repositories/browser_sessions.py b/skyvern/forge/sdk/db/repositories/browser_sessions.py
index e8d913dba..52e800dff 100644
--- a/skyvern/forge/sdk/db/repositories/browser_sessions.py
+++ b/skyvern/forge/sdk/db/repositories/browser_sessions.py
@@ -6,6 +6,7 @@ from datetime import datetime, timedelta
import structlog
from sqlalchemy import asc, case, or_, select
+from skyvern.config import settings
from skyvern.exceptions import BrowserProfileNotFound
from skyvern.forge.sdk.db._error_handling import db_operation
from skyvern.forge.sdk.db.base_alchemy_db import read_retry
@@ -226,14 +227,16 @@ class BrowserSessionsRepository(BaseRepository):
) -> PersistentBrowserSession | None:
"""Get a specific persistent browser session."""
async with self.Session() as session:
- persistent_browser_session = (
- await session.scalars(
- select(PersistentBrowserSessionModel)
- .filter_by(persistent_browser_session_id=session_id)
- .filter_by(organization_id=organization_id)
- .filter_by(deleted_at=None)
- )
- ).first()
+ query = (
+ select(PersistentBrowserSessionModel)
+ .filter_by(persistent_browser_session_id=session_id)
+ .filter_by(deleted_at=None)
+ )
+ if organization_id is not None or settings.ENV != "local":
+ if organization_id is None:
+ raise ValueError("organization_id is required outside local development")
+ query = query.filter_by(organization_id=organization_id)
+ persistent_browser_session = (await session.scalars(query)).first()
if persistent_browser_session:
return PersistentBrowserSession.model_validate(persistent_browser_session)
return None
diff --git a/skyvern/forge/sdk/routes/debug_sessions.py b/skyvern/forge/sdk/routes/debug_sessions.py
index 9dba437f0..163df100e 100644
--- a/skyvern/forge/sdk/routes/debug_sessions.py
+++ b/skyvern/forge/sdk/routes/debug_sessions.py
@@ -167,7 +167,7 @@ async def new_debug_session(
workflow_permanent_id=workflow_permanent_id,
)
- if completed_debug_sessions:
+ if completed_debug_sessions and settings.ENV != "local":
closeable_browser_sessions: list[PersistentBrowserSession] = []
for debug_session in completed_debug_sessions:
@@ -225,6 +225,14 @@ async def new_debug_session(
current_org.organization_id,
)
)
+ elif completed_debug_sessions:
+ LOG.info(
+ "Local mode: skipping debug-session close tasks before creating replacement browser session",
+ num_completed_debug_sessions=len(completed_debug_sessions),
+ organization_id=current_org.organization_id,
+ user_id=current_user_id,
+ workflow_permanent_id=workflow_permanent_id,
+ )
# Look up the workflow's proxy_location so the debug session browser
# uses the same proxy region the user configured on the workflow.
@@ -238,6 +246,7 @@ async def new_debug_session(
organization_id=current_org.organization_id,
timeout_minutes=settings.DEBUG_SESSION_TIMEOUT_MINUTES,
proxy_location=proxy_location,
+ wait_for_startup=settings.ENV != "local",
)
debug_session = await app.DATABASE.debug.create_debug_session(
@@ -245,9 +254,9 @@ async def new_debug_session(
organization_id=current_org.organization_id,
user_id=current_user_id,
workflow_permanent_id=workflow_permanent_id,
- vnc_streaming_supported=True if new_browser_session.ip_address else False,
- # NOTE(jdo:streaming-local-dev)
- # vnc_streaming_supported=True,
+ vnc_streaming_supported=(
+ settings.ENV == "local" or bool(new_browser_session.ip_address or new_browser_session.browser_address)
+ ),
)
LOG.info(
diff --git a/skyvern/forge/sdk/routes/streaming/cdp_input.py b/skyvern/forge/sdk/routes/streaming/cdp_input.py
index 7a685c764..e4cb21bf2 100644
--- a/skyvern/forge/sdk/routes/streaming/cdp_input.py
+++ b/skyvern/forge/sdk/routes/streaming/cdp_input.py
@@ -455,7 +455,11 @@ async def cdp_input_browser_session_stream(
await websocket.close(code=4404, reason="browser_session_finalized")
return
- browser_state = await wait_for_browser_state(browser_session_id, "browser_session")
+ browser_state = await wait_for_browser_state(
+ browser_session_id,
+ "browser_session",
+ organization_id=organization_id,
+ )
if browser_state is None:
LOG.warning("CDP input: timed out waiting for browser state", browser_session_id=browser_session_id)
await websocket.close(code=4408, reason="browser_state_timeout")
diff --git a/skyvern/forge/sdk/routes/streaming/channels/cdp.py b/skyvern/forge/sdk/routes/streaming/channels/cdp.py
index 413c95f19..605625478 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/cdp.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/cdp.py
@@ -21,7 +21,11 @@ import structlog
from playwright.async_api import Browser, BrowserContext, Page, Playwright, async_playwright
from skyvern.config import settings
-from skyvern.webeye.cdp_connection import connect_over_cdp_with_diagnostics
+from skyvern.webeye.cdp_connection import (
+ connect_over_cdp_with_diagnostics,
+ is_local_pbs_cdp_url,
+ resolve_local_pbs_cdp_url,
+)
if t.TYPE_CHECKING:
from skyvern.forge.sdk.routes.streaming.channels.vnc import VncChannel
@@ -77,6 +81,8 @@ class CdpChannel:
else:
url = settings.BROWSER_REMOTE_DEBUGGING_URL
+ url = resolve_local_pbs_cdp_url(url)
+
self.url = url
LOG.info(f"{self.class_name} connecting to CDP", **self.identity)
@@ -85,13 +91,12 @@ class CdpChannel:
self.pw = pw
- headers = (
- {
- "x-api-key": self.vnc_channel.x_api_key,
- }
- if self.vnc_channel.x_api_key
- else None
- )
+ headers: dict[str, str] | None = None
+ if self.vnc_channel.x_api_key:
+ headers = {"x-api-key": self.vnc_channel.x_api_key}
+ if self.vnc_channel.browser_session and is_local_pbs_cdp_url(url):
+ headers = headers or {}
+ headers["X-Session-Id"] = self.vnc_channel.browser_session.persistent_browser_session_id
def on_close() -> None:
LOG.warning(
@@ -100,7 +105,7 @@ class CdpChannel:
close_task = asyncio.create_task(self.close())
close_task.add_done_callback(lambda _: asyncio.create_task(self.connect())) # TODO: avoid blind reconnect
- self.browser = await connect_over_cdp_with_diagnostics(pw, url, headers=headers)
+ self.browser = await connect_over_cdp_with_diagnostics(pw, url, headers=headers if headers else None)
self.browser.on("disconnected", on_close)
await self.apply_download_behavior(self.browser)
diff --git a/skyvern/forge/sdk/routes/streaming/channels/execution.py b/skyvern/forge/sdk/routes/streaming/channels/execution.py
index 21879cafd..1b59aa060 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/execution.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/execution.py
@@ -322,7 +322,11 @@ async def execution_for_message_channel(
if session is None:
raise RuntimeError(f"execution_for_message_channel: session {browser_session_id} not found")
- browser_state = await wait_for_browser_state(browser_session_id, "browser_session")
+ browser_state = await wait_for_browser_state(
+ browser_session_id,
+ "browser_session",
+ organization_id=message_channel.organization_id,
+ )
if browser_state is None:
raise RuntimeError(f"execution_for_message_channel: browser state timeout for {browser_session_id}")
diff --git a/skyvern/forge/sdk/routes/streaming/screencast.py b/skyvern/forge/sdk/routes/streaming/screencast.py
index 9e0fe5dc1..639f7d3f4 100644
--- a/skyvern/forge/sdk/routes/streaming/screencast.py
+++ b/skyvern/forge/sdk/routes/streaming/screencast.py
@@ -26,12 +26,18 @@ async def wait_for_browser_state(
entity_id: str,
entity_type: str,
workflow_run_id: str | None = None,
+ organization_id: str | None = None,
timeout: float = 120,
- poll_interval: float = 1.0,
+ poll_interval: float = 0.25,
) -> BrowserState | None:
elapsed = 0.0
while elapsed < timeout:
- browser_state = await _resolve_browser_state(entity_id, entity_type, workflow_run_id)
+ browser_state = await _resolve_browser_state(
+ entity_id,
+ entity_type,
+ workflow_run_id,
+ organization_id=organization_id,
+ )
if browser_state is not None:
page = await browser_state.get_working_page()
@@ -48,13 +54,14 @@ async def _resolve_browser_state(
entity_id: str,
entity_type: str,
workflow_run_id: str | None = None,
+ organization_id: str | None = None,
) -> BrowserState | None:
if entity_type == "workflow_run":
return app.BROWSER_MANAGER.get_for_workflow_run(entity_id)
if entity_type == "task":
return app.BROWSER_MANAGER.get_for_task(entity_id, workflow_run_id)
if entity_type == "browser_session":
- return await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(entity_id)
+ return await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(entity_id, organization_id)
return None
diff --git a/skyvern/forge/sdk/routes/streaming/screenshot.py b/skyvern/forge/sdk/routes/streaming/screenshot.py
index 6f03f24c2..b820daf20 100644
--- a/skyvern/forge/sdk/routes/streaming/screenshot.py
+++ b/skyvern/forge/sdk/routes/streaming/screenshot.py
@@ -353,6 +353,7 @@ async def _run_local_screencast(
check_finalized: Callable[[], Awaitable[bool]],
get_current_status: Callable[[], Awaitable[str | None]],
get_workflow_run_id: Callable[[], str | None] | None = None,
+ organization_id: str | None = None,
) -> None:
id_key = f"{entity_type}_id"
try:
@@ -366,6 +367,7 @@ async def _run_local_screencast(
entity_id,
entity_type,
workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
)
if browser_state is None:
LOG.warning("Timed out waiting for browser state", **{id_key: entity_id})
@@ -535,4 +537,5 @@ async def _local_screencast_for_browser_session(
wait_for_running=wait_for_running,
check_finalized=check_finalized,
get_current_status=get_current_status,
+ organization_id=organization_id,
)
diff --git a/skyvern/forge/sdk/routes/streaming/verify.py b/skyvern/forge/sdk/routes/streaming/verify.py
index 0593b984a..c7b63d99d 100644
--- a/skyvern/forge/sdk/routes/streaming/verify.py
+++ b/skyvern/forge/sdk/routes/streaming/verify.py
@@ -84,16 +84,22 @@ async def verify_browser_session(
browser_address = ""
else:
LOG.info(
- "Waiting for browser session address.",
+ "Checking browser session address readiness.",
browser_session_id=browser_session_id,
organization_id=organization_id,
)
try:
- browser_address = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_address(
- session_id=browser_session_id,
- organization_id=organization_id,
- )
+ if settings.ENV == "local":
+ browser_address = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_address_if_ready(
+ session_id=browser_session_id,
+ organization_id=organization_id,
+ )
+ else:
+ browser_address = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_address(
+ session_id=browser_session_id,
+ organization_id=organization_id,
+ )
except Exception as ex:
LOG.info(
"Browser session address not found for browser session.",
@@ -102,6 +108,8 @@ async def verify_browser_session(
ex=ex,
)
return None
+ if not browser_address:
+ return None
try:
addressable_browser_session = AddressablePersistentBrowserSession(
diff --git a/skyvern/webeye/cdp_connection.py b/skyvern/webeye/cdp_connection.py
index 6d1faa404..88f678a30 100644
--- a/skyvern/webeye/cdp_connection.py
+++ b/skyvern/webeye/cdp_connection.py
@@ -1,15 +1,17 @@
from __future__ import annotations
import json
+import os
import re
import socket
from collections.abc import Iterable
from dataclasses import dataclass
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urlunparse
import structlog
from playwright.async_api import Browser, Playwright
+from skyvern.config import settings
from skyvern.exceptions import CdpConnectionConfigurationError
LOG = structlog.get_logger()
@@ -80,6 +82,87 @@ def merge_cdp_connect_headers(
return {**filtered_defaults, **filtered_per_row, **managed_host_header}
+def strip_browser_address_discriminator(url: str) -> str:
+ """Remove local PBS URL fragments used only for browser_address DB uniqueness."""
+ parsed = urlparse(url)
+ if parsed.fragment.startswith("pbs_"):
+ return urlunparse(parsed._replace(fragment=""))
+ return url
+
+
+_LOCAL_CONTAINER_CDP_PORT = 9222
+
+
+def parse_local_cdp_host_port_env() -> int | None:
+ raw = os.environ.get("LOCAL_CDP_HOST_PORT", "").strip()
+ if not raw:
+ return None
+ try:
+ return int(raw)
+ except ValueError:
+ return None
+
+
+def local_pbs_cdp_host_port(url: str) -> int | None:
+ """Host-published CDP port when url targets the local PBS openresty proxy."""
+ if settings.ENV != "local":
+ return None
+
+ parsed = urlparse(url)
+ if parsed.hostname not in ("127.0.0.1", "localhost") or parsed.port is None:
+ return None
+
+ host_port = parse_local_cdp_host_port_env()
+ if host_port is not None:
+ if parsed.port in (_LOCAL_CONTAINER_CDP_PORT, host_port):
+ return host_port
+ return None
+
+ if parsed.port == _LOCAL_CONTAINER_CDP_PORT:
+ return None
+ return parsed.port
+
+
+def resolve_local_pbs_cdp_url(url: str) -> str:
+ """Strip DB-only fragments and rewrite container :9222 to host-published PBS port."""
+ url = strip_browser_address_discriminator(url)
+ host_port = local_pbs_cdp_host_port(url)
+ if host_port is None:
+ return url
+
+ parsed = urlparse(url)
+ if parsed.port == host_port:
+ return url
+ if parsed.port != _LOCAL_CONTAINER_CDP_PORT:
+ return url
+
+ netloc = f"{parsed.hostname}:{host_port}"
+ return urlunparse(parsed._replace(netloc=netloc))
+
+
+def is_local_pbs_cdp_url(url: str) -> bool:
+ return local_pbs_cdp_host_port(url) is not None
+
+
+def prepare_persistent_browser_cdp_connect(
+ browser_address: str,
+ *,
+ browser_session_id: str | None = None,
+ x_api_key: str | None = None,
+ cdp_connect_headers: dict[str, str] | None = None,
+) -> tuple[str, dict[str, str] | None]:
+ """Normalize CDP URL and headers for host API connections to local PBS."""
+ connect_url = resolve_local_pbs_cdp_url(browser_address)
+ headers: dict[str, str] = {}
+ if cdp_connect_headers:
+ headers.update(cdp_connect_headers)
+ if x_api_key:
+ headers["x-api-key"] = x_api_key
+ if browser_session_id and is_local_pbs_cdp_url(connect_url):
+ headers["X-Session-Id"] = browser_session_id
+ return connect_url, headers or None
+
+
def parse_cdp_discovery_error(error: Exception) -> tuple[int, str] | None:
"""Return the HTTP status and discovery URL from a Playwright CDP discovery error."""
match = _CDP_DISCOVERY_ERROR_RE.search(str(error))
@@ -171,6 +254,7 @@ async def connect_over_cdp_with_diagnostics(
headers: dict[str, str] | None = None,
timeout_ms: int = DEFAULT_CDP_CONNECT_TIMEOUT_MS,
) -> Browser:
+ remote_browser_url = strip_browser_address_discriminator(remote_browser_url)
try:
return await playwright.chromium.connect_over_cdp(
remote_browser_url,
diff --git a/skyvern/webeye/cdp_retry.py b/skyvern/webeye/cdp_retry.py
index 5aeb3db01..185509a4f 100644
--- a/skyvern/webeye/cdp_retry.py
+++ b/skyvern/webeye/cdp_retry.py
@@ -8,6 +8,8 @@ from playwright._impl._errors import TargetClosedError as PWTargetClosedError
from playwright._impl._errors import TimeoutError as PWTimeoutError
from playwright.async_api import Browser, Playwright
+from skyvern.webeye.cdp_connection import strip_browser_address_discriminator
+
LOG = structlog.get_logger()
_CDP_CONNECTION_ERROR_SUBSTR_FALLBACK = (
@@ -19,7 +21,7 @@ _CDP_CONNECTION_ERROR_SUBSTR_FALLBACK = (
)
-def _is_cdp_connection_error(exc: Exception) -> bool:
+def is_cdp_connection_error(exc: Exception) -> bool:
if isinstance(
exc, (PWTimeoutError, PWTargetClosedError, ConnectionRefusedError, ConnectionResetError, TimeoutError)
):
@@ -38,6 +40,7 @@ async def connect_over_cdp_with_retry(
browser_address: str,
headers: dict[str, str] | None = None,
) -> Browser:
+ browser_address = strip_browser_address_discriminator(browser_address)
for attempt in range(1, _CDP_RETRY_ATTEMPTS + 1):
try:
browser = await playwright.chromium.connect_over_cdp(browser_address, headers=headers)
@@ -49,7 +52,7 @@ async def connect_over_cdp_with_retry(
)
return browser
except Exception as e:
- if not _is_cdp_connection_error(e) or attempt == _CDP_RETRY_ATTEMPTS:
+ if not is_cdp_connection_error(e) or attempt == _CDP_RETRY_ATTEMPTS:
raise
backoff = (
_CDP_RETRY_BACKOFF_SECONDS[attempt - 1]
diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.py
index 8427741c9..41f60960e 100644
--- a/skyvern/webeye/default_persistent_sessions_manager.py
+++ b/skyvern/webeye/default_persistent_sessions_manager.py
@@ -245,6 +245,31 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
return address
+ async def get_browser_address_if_ready(
+ self,
+ session_id: str,
+ organization_id: str,
+ *,
+ timeout: float = 0.0,
+ poll_interval: float = 0.25,
+ ) -> str | None:
+ browser_session = await self.database.browser_sessions.get_persistent_browser_session(
+ session_id, organization_id
+ )
+ if browser_session is None or is_final_status(browser_session.status):
+ return None
+ if browser_session.browser_address:
+ return browser_session.browser_address
+ if timeout <= 0:
+ return None
+ return await wait_on_persistent_browser_address(
+ self.database,
+ session_id,
+ organization_id,
+ timeout=max(1, int(timeout)),
+ poll_interval=poll_interval,
+ )
+
async def get_session_by_runnable_id(
self, runnable_id: str, organization_id: str
) -> PersistentBrowserSession | None:
@@ -282,6 +307,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
browser_type: PersistentBrowserType | None = None,
is_high_priority: bool = False,
browser_profile_id: str | None = None,
+ wait_for_startup: bool = True,
) -> PersistentBrowserSession:
"""Create a new browser session for an organization and return its ID with the browser state."""
LOG.info(
diff --git a/skyvern/webeye/persistent_sessions_manager.py b/skyvern/webeye/persistent_sessions_manager.py
index cd0b7d4e0..7ebf0113c 100644
--- a/skyvern/webeye/persistent_sessions_manager.py
+++ b/skyvern/webeye/persistent_sessions_manager.py
@@ -37,6 +37,17 @@ class PersistentSessionsManager(Protocol):
"""Get the browser address for a session."""
...
+ async def get_browser_address_if_ready(
+ self,
+ session_id: str,
+ organization_id: str,
+ *,
+ timeout: float = 0.0,
+ poll_interval: float = 0.25,
+ ) -> str | None:
+ """Get the browser address for a session if it is already available."""
+ ...
+
async def get_session_by_runnable_id(
self, runnable_id: str, organization_id: str
) -> PersistentBrowserSession | None:
@@ -71,6 +82,7 @@ class PersistentSessionsManager(Protocol):
browser_type: PersistentBrowserType | None = None,
is_high_priority: bool = False,
browser_profile_id: str | None = None,
+ wait_for_startup: bool = True,
) -> PersistentBrowserSession:
"""Create a new browser session."""
...
diff --git a/skyvern/webeye/schemas.py b/skyvern/webeye/schemas.py
index 9e0db1421..acb69b790 100644
--- a/skyvern/webeye/schemas.py
+++ b/skyvern/webeye/schemas.py
@@ -140,7 +140,7 @@ class BrowserSessionResponse(BaseModel):
runnable_id=browser_session.runnable_id,
timeout=browser_session.timeout_minutes,
browser_address=browser_session.browser_address,
- vnc_streaming_supported=True if browser_session.ip_address else False,
+ vnc_streaming_supported=bool(browser_session.ip_address or browser_session.browser_address),
app_url=app_url,
started_at=browser_session.started_at,
completed_at=browser_session.completed_at,
diff --git a/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py b/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py
index 8b0b197a9..41c41ce73 100644
--- a/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py
+++ b/tests/unit/forge/sdk/routes/streaming/test_verify_browser_session.py
@@ -56,7 +56,35 @@ class TestVerifyBrowserSessionLocalShortCircuit:
manager.get_browser_address.assert_not_called()
@pytest.mark.asyncio
- async def test_waits_for_address_in_non_cdp_mode(self) -> None:
+ async def test_checks_address_readiness_in_non_cdp_mode(self) -> None:
+ session = _make_session(
+ browser_address=None,
+ status=PersistentBrowserSessionStatus.running,
+ )
+
+ manager = MagicMock()
+ manager.get_session = AsyncMock(return_value=session)
+ manager.get_browser_address_if_ready = AsyncMock(return_value="ws://remote:9222")
+ manager.get_browser_address = AsyncMock(
+ side_effect=AssertionError("must not enter blocking address wait"),
+ )
+
+ with (
+ patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock,
+ patch("skyvern.forge.sdk.routes.streaming.verify.settings") as settings_mock,
+ ):
+ app_mock.PERSISTENT_SESSIONS_MANAGER = manager
+ settings_mock.BROWSER_STREAMING_MODE = "vnc"
+ settings_mock.ENV = "local"
+ result = await verify_browser_session("bs_test", "o_test")
+
+ assert result is not None
+ assert result.browser_address == "ws://remote:9222"
+ manager.get_browser_address_if_ready.assert_awaited_once_with(session_id="bs_test", organization_id="o_test")
+ manager.get_browser_address.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_blocks_until_address_ready_in_non_local_non_cdp_mode(self) -> None:
session = _make_session(
browser_address=None,
status=PersistentBrowserSessionStatus.running,
@@ -65,6 +93,9 @@ class TestVerifyBrowserSessionLocalShortCircuit:
manager = MagicMock()
manager.get_session = AsyncMock(return_value=session)
manager.get_browser_address = AsyncMock(return_value="ws://remote:9222")
+ manager.get_browser_address_if_ready = AsyncMock(
+ side_effect=AssertionError("must not use non-blocking address check in production"),
+ )
with (
patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock,
@@ -72,11 +103,40 @@ class TestVerifyBrowserSessionLocalShortCircuit:
):
app_mock.PERSISTENT_SESSIONS_MANAGER = manager
settings_mock.BROWSER_STREAMING_MODE = "vnc"
+ settings_mock.ENV = "production"
result = await verify_browser_session("bs_test", "o_test")
assert result is not None
assert result.browser_address == "ws://remote:9222"
- manager.get_browser_address.assert_awaited_once()
+ manager.get_browser_address.assert_awaited_once_with(session_id="bs_test", organization_id="o_test")
+ manager.get_browser_address_if_ready.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_returns_none_when_address_not_ready_in_non_cdp_mode(self) -> None:
+ session = _make_session(
+ browser_address=None,
+ status=PersistentBrowserSessionStatus.running,
+ )
+
+ manager = MagicMock()
+ manager.get_session = AsyncMock(return_value=session)
+ manager.get_browser_address_if_ready = AsyncMock(return_value=None)
+ manager.get_browser_address = AsyncMock(
+ side_effect=AssertionError("must not enter blocking address wait"),
+ )
+
+ with (
+ patch("skyvern.forge.sdk.routes.streaming.verify.app") as app_mock,
+ patch("skyvern.forge.sdk.routes.streaming.verify.settings") as settings_mock,
+ ):
+ app_mock.PERSISTENT_SESSIONS_MANAGER = manager
+ settings_mock.BROWSER_STREAMING_MODE = "vnc"
+ settings_mock.ENV = "local"
+ result = await verify_browser_session("bs_test", "o_test")
+
+ assert result is None
+ manager.get_browser_address_if_ready.assert_awaited_once_with(session_id="bs_test", organization_id="o_test")
+ manager.get_browser_address.assert_not_called()
@pytest.mark.asyncio
async def test_uses_existing_address_without_polling(self) -> None:
diff --git a/tests/unit/forge/sdk/routes/test_debug_sessions.py b/tests/unit/forge/sdk/routes/test_debug_sessions.py
new file mode 100644
index 000000000..750601db3
--- /dev/null
+++ b/tests/unit/forge/sdk/routes/test_debug_sessions.py
@@ -0,0 +1,56 @@
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.forge.sdk.routes import debug_sessions as debug_sessions_mod
+
+
+async def test_new_debug_session_skips_redundant_local_close_before_create() -> None:
+ created_debug_session = SimpleNamespace(debug_session_id="ds_new")
+ new_browser_session = SimpleNamespace(
+ persistent_browser_session_id="pbs_new",
+ ip_address=None,
+ browser_address=None,
+ )
+
+ app_mock = MagicMock()
+ app_mock.DATABASE.debug.get_debug_session = AsyncMock(return_value=None)
+ app_mock.DATABASE.debug.complete_debug_sessions = AsyncMock(
+ return_value=[SimpleNamespace(browser_session_id="pbs_old")]
+ )
+ app_mock.DATABASE.debug.create_debug_session = AsyncMock(return_value=created_debug_session)
+ app_mock.DATABASE.browser_sessions.get_persistent_browser_session = AsyncMock()
+ app_mock.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(
+ return_value=SimpleNamespace(proxy_location=None)
+ )
+ app_mock.PERSISTENT_SESSIONS_MANAGER.close_session = AsyncMock()
+ app_mock.PERSISTENT_SESSIONS_MANAGER.create_session = AsyncMock(return_value=new_browser_session)
+
+ with (
+ patch.object(debug_sessions_mod, "app", app_mock),
+ patch.object(debug_sessions_mod.settings, "ENV", "local"),
+ ):
+ result = await debug_sessions_mod.new_debug_session(
+ "wpid_test",
+ current_org=SimpleNamespace(organization_id="org_123"),
+ current_user_id="user_123",
+ )
+
+ assert result is created_debug_session
+ app_mock.DATABASE.browser_sessions.get_persistent_browser_session.assert_not_awaited()
+ app_mock.PERSISTENT_SESSIONS_MANAGER.close_session.assert_not_awaited()
+ app_mock.PERSISTENT_SESSIONS_MANAGER.create_session.assert_awaited_once_with(
+ organization_id="org_123",
+ timeout_minutes=debug_sessions_mod.settings.DEBUG_SESSION_TIMEOUT_MINUTES,
+ proxy_location=debug_sessions_mod.ProxyLocation.RESIDENTIAL,
+ wait_for_startup=False,
+ )
+ app_mock.DATABASE.debug.create_debug_session.assert_awaited_once_with(
+ browser_session_id="pbs_new",
+ organization_id="org_123",
+ user_id="user_123",
+ workflow_permanent_id="wpid_test",
+ vnc_streaming_supported=True,
+ )
diff --git a/tests/unit/test_cdp_error_detection.py b/tests/unit/test_cdp_error_detection.py
index 2979f7e81..90f14972e 100644
--- a/tests/unit/test_cdp_error_detection.py
+++ b/tests/unit/test_cdp_error_detection.py
@@ -5,7 +5,7 @@ from playwright._impl._errors import Error as PWError
from playwright._impl._errors import TargetClosedError as PWTargetClosedError
from playwright._impl._errors import TimeoutError as PWTimeoutError
-from skyvern.webeye.cdp_retry import _is_cdp_connection_error
+from skyvern.webeye.cdp_retry import is_cdp_connection_error
class TestConnectionErrorDetection:
@@ -23,7 +23,7 @@ class TestConnectionErrorDetection:
ids=lambda e: type(e).__name__ + ": " + str(e)[:50],
)
def test_connection_errors_detected(self, exc: Exception):
- assert _is_cdp_connection_error(exc), f"Expected connection error: {exc!r}"
+ assert is_cdp_connection_error(exc), f"Expected connection error: {exc!r}"
@pytest.mark.parametrize(
"exc",
@@ -37,4 +37,4 @@ class TestConnectionErrorDetection:
ids=lambda e: type(e).__name__ + ": " + str(e)[:50],
)
def test_app_errors_not_detected(self, exc: Exception):
- assert not _is_cdp_connection_error(exc), f"Expected app error NOT to match: {exc!r}"
+ assert not is_cdp_connection_error(exc), f"Expected app error NOT to match: {exc!r}"
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
new file mode 100644
index 000000000..c119d6034
--- /dev/null
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -0,0 +1,642 @@
+from __future__ import annotations
+
+import time
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.config import settings
+from skyvern.forge.sdk.copilot.agent import _completion_contract_not_violated, _verified_workflow_or_none
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ CriterionVerdict,
+ RunEvidenceSnapshot,
+ _coerce_result,
+ evaluate_completion_criteria,
+ summarize_unsatisfied_outcomes,
+)
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ DiagnosisInput,
+ DiagnosisRepairContract,
+ DiagnosisResult,
+ RepairDecision,
+ RepairNextAction,
+ VerificationResult,
+ _verification_satisfaction,
+)
+from skyvern.forge.sdk.copilot.enforcement import (
+ _outcome_criteria_evaluated,
+ outcome_fully_verified,
+ verified_goal_satisfied_context,
+)
+from skyvern.forge.sdk.copilot.hooks import _tool_completion_satisfies_turn
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy, _parse_completion_criteria
+from skyvern.forge.sdk.copilot.tools import (
+ _build_run_evidence_snapshot,
+ _current_workflow_has_evidence_block,
+ _is_outcome_evidence_candidate,
+ _is_unfinished_run_verification_candidate,
+ _maybe_run_completion_verification,
+ _outcome_failure_warrants_repair,
+ _outcome_unverified_reason,
+ _record_run_blocks_result,
+)
+
+
+def _criterion(cid: str, outcome: str, *, method_mandated: bool = False) -> CompletionCriterion:
+ return CompletionCriterion(id=cid, outcome=outcome, method_mandated=method_mandated)
+
+
+def _evaluated(*satisfied_by_id: tuple[str, bool]) -> CompletionVerificationResult:
+ ids = [cid for cid, _ in satisfied_by_id]
+ verdicts = [
+ CriterionVerdict(criterion_id=cid, satisfied=ok, reason_code="evidence_confirms" if ok else "no_evidence")
+ for cid, ok in satisfied_by_id
+ ]
+ return CompletionVerificationResult(status="evaluated", criterion_ids=ids, verdicts=verdicts)
+
+
+def test_is_fully_satisfied_requires_every_criterion() -> None:
+ assert _evaluated(("c0", True), ("c1", True)).is_fully_satisfied() is True
+ assert _evaluated(("c0", True), ("c1", False)).is_fully_satisfied() is False
+
+
+def test_empty_verdicts_with_criteria_is_not_vacuously_satisfied() -> None:
+ result = CompletionVerificationResult(status="evaluated", criterion_ids=["c0"], verdicts=[])
+ assert result.is_fully_satisfied() is False
+
+
+def test_unavailable_and_empty_criteria_never_satisfied() -> None:
+ assert CompletionVerificationResult(status="unavailable").is_fully_satisfied() is False
+ assert CompletionVerificationResult(status="evaluated", criterion_ids=[]).is_fully_satisfied() is False
+
+
+def test_coerce_requires_evidence_confirms_for_satisfied() -> None:
+ raw = {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "unknown"}]}
+ result = _coerce_result(raw, ["c0"])
+ assert result.status == "evaluated"
+ assert result.verdicts[0].satisfied is False
+
+
+def test_coerce_missing_criterion_defaults_to_no_evidence() -> None:
+ result = _coerce_result({"verdicts": []}, ["c0", "c1"])
+ assert [v.reason_code for v in result.verdicts] == ["no_evidence", "no_evidence"]
+ assert result.is_fully_satisfied() is False
+
+
+def test_coerce_ignores_unknown_ids_and_dedupes_first_wins() -> None:
+ raw = {
+ "verdicts": [
+ {"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"},
+ {"criterion_id": "c0", "satisfied": False, "reason_code": "no_evidence"},
+ {"criterion_id": "ghost", "satisfied": True, "reason_code": "evidence_confirms"},
+ ]
+ }
+ result = _coerce_result(raw, ["c0"])
+ assert len(result.verdicts) == 1
+ assert result.verdicts[0].satisfied is True
+
+
+def test_coerce_accepts_bytes_and_rejects_malformed() -> None:
+ raw_bytes = b'{"verdicts": [{"criterion_id": "c0", "satisfied": true, "reason_code": "evidence_confirms"}]}'
+ assert _coerce_result(raw_bytes, ["c0"]).is_fully_satisfied() is True
+ assert _coerce_result("not json at all", ["c0"]).status == "unavailable"
+ assert _coerce_result({"no_verdicts_key": 1}, ["c0"]).status == "unavailable"
+
+
+async def test_evaluate_no_handler_or_no_criteria_is_unavailable() -> None:
+ snapshot = RunEvidenceSnapshot(current_url="https://example.com")
+ assert (await evaluate_completion_criteria([_criterion("c0", "x")], snapshot, None)).status == "unavailable"
+ assert (await evaluate_completion_criteria([], snapshot, lambda **_: {})).status == "unavailable"
+
+
+async def test_evaluate_handler_exception_is_unavailable() -> None:
+ async def boom(**_: object) -> object:
+ raise RuntimeError("llm down")
+
+ snapshot = RunEvidenceSnapshot(current_url="https://example.com")
+ result = await evaluate_completion_criteria([_criterion("c0", "x")], snapshot, boom)
+ assert result.status == "unavailable"
+
+
+async def test_evaluate_happy_path_returns_evaluated() -> None:
+ async def handler(**_: object) -> dict:
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ snapshot = RunEvidenceSnapshot(block_outputs={"confirm": {"count": 1}})
+ result = await evaluate_completion_criteria([_criterion("c0", "item in cart")], snapshot, handler)
+ assert result.status == "evaluated"
+ assert result.is_fully_satisfied() is True
+
+
+def test_snapshot_has_evidence() -> None:
+ assert RunEvidenceSnapshot().has_evidence() is False
+ assert RunEvidenceSnapshot(current_url="https://example.com").has_evidence() is True
+ assert RunEvidenceSnapshot(block_outputs={"a": 1}).has_evidence() is True
+
+
+def test_summarize_unsatisfied_lists_unmet_outcomes() -> None:
+ criteria = [_criterion("c0", "item in cart"), _criterion("c1", "added exactly once")]
+ result = _evaluated(("c0", True), ("c1", False))
+ assert summarize_unsatisfied_outcomes(result, criteria) == "added exactly once"
+
+
+def test_parse_assigns_deterministic_ids_and_dedupes() -> None:
+ raw = [
+ {"outcome": "Item in cart", "id": "model-supplied-ignored"},
+ {"outcome": "item in cart"},
+ {"outcome": "", "implicit": True},
+ {"outcome": "Added exactly once", "implicit": True},
+ ]
+ criteria = _parse_completion_criteria(raw)
+ assert [c.id for c in criteria] == ["c0", "c1"]
+ assert [c.outcome for c in criteria] == ["Item in cart", "Added exactly once"]
+ assert criteria[1].implicit is True
+
+
+def test_parse_caps_count() -> None:
+ raw = [{"outcome": f"outcome {i}"} for i in range(20)]
+ assert len(_parse_completion_criteria(raw)) == 8
+
+
+def test_verification_satisfaction_no_cvr_uses_prior_proxy() -> None:
+ assert _verification_satisfaction(True, False, "completed", None) == (True, True)
+ assert _verification_satisfaction(True, True, "completed", None) == (False, False)
+ assert _verification_satisfaction(False, False, None, None) == (None, None)
+
+
+def test_verification_satisfaction_evaluated_drives_contract_signal() -> None:
+ assert _verification_satisfaction(True, False, "completed", _evaluated(("c0", True))) == (True, True)
+ _, contract = _verification_satisfaction(True, False, "completed", _evaluated(("c0", False)))
+ assert contract is False
+
+
+def test_verification_satisfaction_unavailable_fails_closed() -> None:
+ _, contract = _verification_satisfaction(True, False, "completed", CompletionVerificationResult("unavailable"))
+ assert contract is False
+
+
+def _satisfied_contract() -> DiagnosisRepairContract:
+ return DiagnosisRepairContract(
+ diagnosis_input=DiagnosisInput(source_tool="run_blocks_and_collect_debug"),
+ diagnosis_result=DiagnosisResult(),
+ repair_decision=RepairDecision(next_action=RepairNextAction.NO_CHANGE),
+ verification_result=VerificationResult(user_goal_satisfied=True, completion_contract_satisfied=True),
+ )
+
+
+def _gate_ctx() -> CopilotContext:
+ ctx = CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ user_message="do A then B",
+ )
+ ctx.last_test_ok = True
+ ctx.last_full_workflow_test_ok = True
+ ctx.latest_diagnosis_repair_contract = _satisfied_contract()
+ ctx.last_update_block_count = 1
+ ctx.request_policy = RequestPolicy(completion_contract="done when B happens")
+ return ctx
+
+
+def test_gate_bypasses_heuristic_only_on_evaluated_verdict() -> None:
+ bypass = _gate_ctx()
+ bypass.completion_verification_result = _evaluated(("c0", True))
+ assert verified_goal_satisfied_context(bypass) is True
+
+ retained = _gate_ctx()
+ retained.completion_verification_result = None
+ assert verified_goal_satisfied_context(retained) is False
+
+
+def test_gate_flag_off_ignores_verdict(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_OUTCOME_VERIFICATION_ENABLED", False)
+ ctx = _gate_ctx()
+ ctx.completion_verification_result = _evaluated(("c0", True))
+ assert _outcome_criteria_evaluated(ctx) is False
+ assert verified_goal_satisfied_context(ctx) is False
+
+
+def test_gate_withholds_on_evaluated_unconfirmed_even_with_clean_run_status() -> None:
+ # The judge verdict is authoritative in both directions: an evaluated-but-
+ # unconfirmed verdict withholds even when run-status latches and the diagnosis
+ # contract would otherwise pass -- recognition must weigh the verdict, not just
+ # whether the judge ran.
+ ctx = _gate_ctx()
+ ctx.completion_verification_result = _evaluated(("c0", True), ("c1", False))
+ assert verified_goal_satisfied_context(ctx) is False
+
+
+def test_completion_contract_not_violated() -> None:
+ ctx = SimpleNamespace(completion_verification_result=None)
+ assert _completion_contract_not_violated(ctx) is True # type: ignore[arg-type]
+ ctx.completion_verification_result = _evaluated(("c0", True))
+ assert _completion_contract_not_violated(ctx) is True # type: ignore[arg-type]
+ ctx.completion_verification_result = _evaluated(("c0", False))
+ assert _completion_contract_not_violated(ctx) is False # type: ignore[arg-type]
+
+
+def test_outcome_unverified_reason_for_unsatisfied_and_unavailable() -> None:
+ policy = RequestPolicy(completion_criteria=[_criterion("c0", "item in cart")])
+ ctx = SimpleNamespace(request_policy=policy)
+ assert _outcome_unverified_reason(ctx, None) is None
+ assert _outcome_unverified_reason(ctx, _evaluated(("c0", True))) is None
+ unsatisfied = _outcome_unverified_reason(ctx, _evaluated(("c0", False)))
+ assert unsatisfied is not None and "item in cart" in unsatisfied
+ unavailable = _outcome_unverified_reason(ctx, CompletionVerificationResult("unavailable"))
+ assert unavailable is not None and "could not be verified" in unavailable
+
+
+def _clean_success_result() -> dict:
+ return {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_x",
+ "overall_status": "completed",
+ "executed_block_labels": ["confirm"],
+ "current_url": "https://example.com/cart",
+ "blocks": [
+ {
+ "label": "confirm",
+ "block_type": "EXTRACTION",
+ "status": "completed",
+ "extracted_data": {"extracted_information": {"items": ["a"]}},
+ }
+ ],
+ },
+ }
+
+
+def _run_ctx() -> CopilotContext:
+ ctx = CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ user_message="add item to cart and confirm",
+ )
+ ctx.request_policy = RequestPolicy(completion_criteria=[_criterion("c0", "item in cart")])
+ return ctx
+
+
+def _ctx_with_blocks(*block_types: str) -> CopilotContext:
+ ctx = _run_ctx()
+ blocks = [SimpleNamespace(block_type=bt, label=f"b{i}") for i, bt in enumerate(block_types)]
+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=blocks))
+ ctx.verified_prefix_labels = [b.label for b in blocks]
+ return ctx
+
+
+def _contradicted(cid: str) -> CompletionVerificationResult:
+ verdict = CriterionVerdict(criterion_id=cid, satisfied=False, reason_code="evidence_contradicts")
+ return CompletionVerificationResult(status="evaluated", criterion_ids=[cid], verdicts=[verdict])
+
+
+def test_record_run_blocks_downgrades_when_confirmation_block_present_but_unmet() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))
+ assert ctx.last_test_suspicious_success is True
+ assert ctx.last_full_workflow_test_ok is False
+ assert ctx.last_good_workflow is None
+ assert ctx.workflow_verification_evidence.full_workflow_verified is False
+ assert "item in cart" in (ctx.last_test_failure_reason or "")
+
+
+def test_record_run_blocks_keeps_building_on_mid_build_no_evidence() -> None:
+ ctx = _ctx_with_blocks("goto_url", "navigation")
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))
+ # A nav-only WIP that has not added a confirmation block yet must keep building,
+ # not enter repair...
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.last_full_workflow_test_ok is True
+ # ...but terminal success stays withheld because the outcome is unverified.
+ assert _completion_contract_not_violated(ctx) is False
+
+
+def test_record_run_blocks_downgrades_on_contradiction_without_confirmation_block() -> None:
+ ctx = _ctx_with_blocks("goto_url", "navigation")
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_contradicted("c0"))
+ assert ctx.last_test_suspicious_success is True
+ assert ctx.last_full_workflow_test_ok is False
+
+
+def test_record_run_blocks_keeps_success_when_outcome_verified() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))
+ assert ctx.last_full_workflow_test_ok is True
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.workflow_verification_evidence.full_workflow_verified is True
+
+
+def test_current_workflow_has_evidence_block() -> None:
+ assert _current_workflow_has_evidence_block(_ctx_with_blocks("extraction")) is True
+ assert _current_workflow_has_evidence_block(_ctx_with_blocks("goto_url", "validation")) is True
+ assert _current_workflow_has_evidence_block(_ctx_with_blocks("goto_url", "navigation")) is False
+ assert _current_workflow_has_evidence_block(_run_ctx()) is False
+
+
+def test_outcome_failure_warrants_repair() -> None:
+ has_block = _ctx_with_blocks("extraction")
+ nav_only = _ctx_with_blocks("goto_url", "navigation")
+ assert _outcome_failure_warrants_repair(nav_only, None) is False
+ # Contradiction is a real failure regardless of which blocks exist.
+ assert _outcome_failure_warrants_repair(nav_only, _contradicted("c0")) is True
+ # Absence of evidence: failure only once a confirmation block exists.
+ assert _outcome_failure_warrants_repair(has_block, _evaluated(("c0", False))) is True
+ assert _outcome_failure_warrants_repair(nav_only, _evaluated(("c0", False))) is False
+
+
+# --- Direction 2: recognition governed by evidence, not run status ---------------
+#
+# A run canceled or only partially completed (ok=False) still produces runtime
+# evidence. When that evidence confirms every outcome criterion, the goal the user
+# can observe was reached, and recognition must not be suppressed by run status.
+
+
+def _canceled_budget_result() -> dict:
+ # The watchdog budget-cancel result shape: ok=False, no "blocks" list (the
+ # result is returned before block harvest), only the reached URL survives.
+ return {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_cancel",
+ "overall_status": "canceled",
+ "current_url": "https://example.com/cart",
+ "failure_reason": "Task wr_cancel was canceled",
+ "failure_categories": [{"category": "PER_TOOL_BUDGET", "confidence_float": 1.0, "reasoning": "budget"}],
+ },
+ }
+
+
+def _canceled_gate_ctx() -> CopilotContext:
+ # A run that did not finish cleanly: every run-status latch is false and the
+ # diagnosis routed to repair, yet the judge confirmed the outcome from evidence.
+ ctx = _gate_ctx()
+ ctx.last_test_ok = False
+ ctx.last_full_workflow_test_ok = False
+ ctx.latest_diagnosis_repair_contract = DiagnosisRepairContract(
+ diagnosis_input=DiagnosisInput(source_tool="run_blocks_and_collect_debug"),
+ diagnosis_result=DiagnosisResult(),
+ repair_decision=RepairDecision(next_action=RepairNextAction.REPAIR),
+ verification_result=VerificationResult(user_goal_satisfied=False, completion_contract_satisfied=True),
+ )
+ return ctx
+
+
+def test_unfinished_run_verification_candidate_admits_canceled_with_evidence() -> None:
+ ctx = _run_ctx()
+ assert _is_unfinished_run_verification_candidate(ctx, _canceled_budget_result()) is True
+ # ok=True belongs to the clean-success candidate path, not this one.
+ assert _is_unfinished_run_verification_candidate(ctx, _clean_success_result()) is False
+ # ok=False with no reached runtime URL leaves nothing to judge.
+ assert _is_unfinished_run_verification_candidate(ctx, {"ok": False, "data": {}}) is False
+
+
+async def test_maybe_run_completion_verification_runs_on_canceled_run(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def handler(**_: object) -> dict:
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._completion_verification_handler", lambda: handler)
+ ctx = _run_ctx()
+ result = await _maybe_run_completion_verification(ctx, _canceled_budget_result(), time.monotonic())
+ assert result is not None
+ assert result.status == "evaluated"
+ assert result.is_fully_satisfied() is True
+
+
+def test_outcome_fully_verified_predicate(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _gate_ctx()
+ ctx.completion_verification_result = _evaluated(("c0", True))
+ assert outcome_fully_verified(ctx) is True
+ ctx.completion_verification_result = _evaluated(("c0", True), ("c1", False))
+ assert outcome_fully_verified(ctx) is False
+ ctx.completion_verification_result = None
+ assert outcome_fully_verified(ctx) is False
+ ctx.completion_verification_result = _evaluated(("c0", True))
+ monkeypatch.setattr(settings, "COPILOT_OUTCOME_VERIFICATION_ENABLED", False)
+ assert outcome_fully_verified(ctx) is False
+
+
+def test_gate_recognizes_canceled_run_on_full_evidence() -> None:
+ ctx = _canceled_gate_ctx()
+ ctx.completion_verification_result = _evaluated(("c0", True))
+ assert verified_goal_satisfied_context(ctx) is True
+
+
+def test_gate_does_not_recognize_partial_canceled_run() -> None:
+ ctx = _canceled_gate_ctx()
+ ctx.completion_verification_result = _evaluated(("c0", True), ("c1", False))
+ assert verified_goal_satisfied_context(ctx) is False
+
+
+def test_tool_completion_recognizes_canceled_run_on_full_evidence() -> None:
+ ctx = _canceled_gate_ctx()
+ parsed = {"ok": False, "data": {"workflow_run_id": "wr_cancel"}}
+ ctx.completion_verification_result = _evaluated(("c0", True))
+ assert _tool_completion_satisfies_turn(ctx, "run_blocks_and_collect_debug", parsed) is True
+ # A canceled run whose outcome is only partially confirmed does not satisfy the turn.
+ ctx.completion_verification_result = _evaluated(("c0", True), ("c1", False))
+ assert _tool_completion_satisfies_turn(ctx, "run_blocks_and_collect_debug", parsed) is False
+
+
+def test_verified_workflow_presented_on_recognized_canceled_run() -> None:
+ ctx = _canceled_gate_ctx()
+ ctx.last_workflow = SimpleNamespace()
+ ctx.last_workflow_yaml = "workflow: {}"
+ ctx.completion_verification_result = _evaluated(("c0", True))
+ assert _verified_workflow_or_none(ctx) == (ctx.last_workflow, "workflow: {}")
+ # Run-status latches false and outcome not fully confirmed: nothing is surfaced.
+ ctx.completion_verification_result = _evaluated(("c0", False))
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+
+# --- SKY-10576: recognition governed by whole-workflow outcome, not per-block prefix ---
+#
+# A clean ok=True run can reach the goal (its outcome block produced data and the
+# browser is on the goal page) while earlier block labels are not in the verified
+# end-to-end prefix. Recognition must come from the outcome judge, not from whether
+# every block was verified as a prefix; otherwise an achieved goal is hedged as an
+# "unverified draft" and the agent overruns (SKY-10576, confirmed in live QA).
+
+
+def _ctx_unverified_prefix() -> CopilotContext:
+ ctx = _run_ctx()
+ blocks = [
+ SimpleNamespace(block_type="navigation", label="b0"),
+ SimpleNamespace(block_type="navigation", label="b1"),
+ SimpleNamespace(block_type="navigation", label="b2"),
+ SimpleNamespace(block_type="extraction", label="b3"),
+ ]
+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=blocks))
+ # Only the suffix is in the verified prefix; the goal was reached on the final
+ # incremental run, but b0/b1 never entered the prefix.
+ ctx.verified_prefix_labels = ["b2", "b3"]
+ ctx.verified_block_outputs = {"b3": {"one_star_review_text": "For the life of me..."}}
+ return ctx
+
+
+def _empty_data_result() -> dict:
+ return {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_empty",
+ "overall_status": "completed",
+ "current_url": "https://example.com/reviews",
+ "blocks": [{"label": "confirm", "block_type": "EXTRACTION", "status": "completed"}],
+ },
+ }
+
+
+def test_outcome_evidence_candidate_admits_clean_run_despite_unverified_prefix() -> None:
+ ctx = _ctx_unverified_prefix()
+ # A clean run is admitted for the judge even though b0/b1 are not in the verified
+ # prefix -- recognition is governed by the outcome judge, not the per-block prefix.
+ assert _is_outcome_evidence_candidate(ctx, _clean_success_result()) is True
+ # It still rejects empty-data runs (no clean outcome to judge) and ok=False runs.
+ assert _is_outcome_evidence_candidate(ctx, _empty_data_result()) is False
+ assert _is_outcome_evidence_candidate(ctx, {"ok": False, "data": {}}) is False
+
+
+async def test_maybe_run_completion_verification_runs_on_unverified_prefix(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def handler(**_: object) -> dict:
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._completion_verification_handler", lambda: handler)
+ ctx = _ctx_unverified_prefix()
+ result = await _maybe_run_completion_verification(ctx, _clean_success_result(), time.monotonic())
+ assert result is not None
+ assert result.status == "evaluated"
+ assert result.is_fully_satisfied() is True
+
+
+def test_gate_recognizes_clean_run_despite_unverified_prefix() -> None:
+ ctx = _ctx_unverified_prefix()
+ # The full-workflow run-status latch is False (incremental run), yet the judge
+ # confirmed the outcome: recognition must fire on the evidence.
+ ctx.last_test_ok = True
+ ctx.last_full_workflow_test_ok = False
+ ctx.completion_verification_result = _evaluated(("c0", True))
+ assert outcome_fully_verified(ctx) is True
+ assert verified_goal_satisfied_context(ctx) is True
+
+
+# --- Review hardening: method-mandated criteria, per-run evidence, fail-closed ---
+
+
+async def test_method_mandated_criteria_excluded_from_verification(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def handler(**_: object) -> dict:
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._completion_verification_handler", lambda: handler)
+ ctx = _run_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion("c0", "item in cart"),
+ _criterion("c1", "use the search bar", method_mandated=True),
+ ]
+ )
+ result = await _maybe_run_completion_verification(ctx, _clean_success_result(), time.monotonic())
+ # The method-mandated criterion is not sent to the end-state judge (it could only
+ # ever return no_evidence), so it cannot false-block a legitimate success.
+ assert result is not None
+ assert result.criterion_ids == ["c0"]
+ assert result.is_fully_satisfied() is True
+
+
+def test_snapshot_uses_current_run_blocks_not_stale_outputs() -> None:
+ ctx = _ctx_unverified_prefix() # verified_block_outputs carries a stale b3 from a prior run
+ stale_run = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_now",
+ "current_url": "https://example.com/x",
+ "executed_block_labels": ["b0"],
+ "blocks": [{"label": "b0", "block_type": "NAVIGATION", "status": "completed"}],
+ },
+ }
+ snap = _build_run_evidence_snapshot(ctx, stale_run)
+ # A prior run's output must not leak in as this run's evidence.
+ assert "b3" not in snap.block_outputs
+ assert snap.block_outputs == {}
+ fresh_run = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_now",
+ "current_url": "https://example.com/x",
+ "executed_block_labels": ["b3"],
+ "blocks": [
+ {
+ "label": "b3",
+ "block_type": "EXTRACTION",
+ "status": "completed",
+ "extracted_data": {"extracted_information": {"price": "9.99"}},
+ }
+ ],
+ },
+ }
+ snap2 = _build_run_evidence_snapshot(ctx, fresh_run)
+ assert snap2.block_outputs.get("b3") == {"extracted_information": {"price": "9.99"}}
+
+
+async def test_maybe_run_completion_verification_unavailable_on_low_budget(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def handler(**_: object) -> dict:
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._completion_verification_handler", lambda: handler)
+ ctx = _run_ctx()
+ starved = time.monotonic() - 100_000 # no budget left to verify this candidate run
+ result = await _maybe_run_completion_verification(ctx, _clean_success_result(), starved)
+ # Fail closed: a candidate run we could not verify must not fall back to the
+ # run-status proxy and claim success.
+ assert result is not None
+ assert result.status == "unavailable"
+ assert result.is_fully_satisfied() is False
+
+ # A missing judge handler stays a soft fallback (None), not a hard block.
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._completion_verification_handler", lambda: None)
+ assert await _maybe_run_completion_verification(ctx, _clean_success_result(), time.monotonic()) is None
+
+
+def test_completion_contract_not_violated_unavailable_blocks_surfacing() -> None:
+ ctx = SimpleNamespace(completion_verification_result=CompletionVerificationResult("unavailable"))
+ # An unavailable verdict means the outcome could not be verified: do not surface
+ # the workflow as verified on run status alone.
+ assert _completion_contract_not_violated(ctx) is False # type: ignore[arg-type]
+
+
+def test_request_policy_prompt_gates_completion_criteria_on_flag() -> None:
+ from skyvern.forge.prompts import prompt_engine
+
+ common = dict(
+ user_message="",
+ raw_secret_present="false",
+ workflow_yaml="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history="",
+ global_llm_context="",
+ )
+ on = prompt_engine.load_prompt(
+ template="workflow-copilot-request-policy", outcome_verification_enabled=True, **common
+ )
+ off = prompt_engine.load_prompt(
+ template="workflow-copilot-request-policy", outcome_verification_enabled=False, **common
+ )
+ assert "completion_criteria" in on
+ # Flag off must not perturb the classifier prompt with the criteria contract.
+ assert "completion_criteria" not in off
diff --git a/tests/unit/test_copilot_outcome_verification_trace.py b/tests/unit/test_copilot_outcome_verification_trace.py
new file mode 100644
index 000000000..cb978e014
--- /dev/null
+++ b/tests/unit/test_copilot_outcome_verification_trace.py
@@ -0,0 +1,173 @@
+"""Unit tests for copilot.turn outcome-verification telemetry."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+from opentelemetry import trace as otel_trace
+
+from skyvern.forge.sdk.copilot import agent as copilot_agent
+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+from skyvern.forge.sdk.copilot.outcome_verification_trace import (
+ finalize_outcome_verification_trace,
+ outcome_verification_turn_fields,
+ record_completion_verification,
+ record_gate_decision,
+)
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest
+
+
+def _evaluated_result() -> CompletionVerificationResult:
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0", "c1"],
+ verdicts=[
+ CriterionVerdict(criterion_id="c0", satisfied=True, reason_code="evidence_confirms", evidence_ref="cart"),
+ CriterionVerdict(criterion_id="c1", satisfied=False, reason_code="no_evidence"),
+ ],
+ )
+
+
+def test_record_completion_verification_populates_evaluated_block() -> None:
+ ctx = SimpleNamespace()
+ record_completion_verification(ctx, _evaluated_result())
+
+ snapshot = ctx.outcome_verification_trace_snapshot
+ assert snapshot["completion_verification_status"] == "evaluated"
+ assert snapshot["completion_verification_criterion_count"] == 2
+ assert snapshot["completion_verification_satisfied_count"] == 1
+ assert snapshot["completion_verification_fully_satisfied"] is False
+ assert snapshot["completion_verification_evaluated_on_final_run"] is True
+
+
+def test_record_completion_verification_replaces_stale_verdict_on_later_unevaluated_run() -> None:
+ ctx = SimpleNamespace()
+ record_completion_verification(ctx, _evaluated_result())
+ # A later recorded run executed but did not evaluate completion criteria.
+ record_completion_verification(ctx, None)
+
+ snapshot = ctx.outcome_verification_trace_snapshot
+ assert snapshot["completion_verification_evaluated_on_final_run"] is False
+ assert snapshot["completion_verification_status"] == "not_run"
+ # The prior run's verdict counts must not linger as if they describe this run.
+ assert "completion_verification_criterion_count" not in snapshot
+ assert "completion_verification_satisfied_count" not in snapshot
+ assert "completion_verification_fully_satisfied" not in snapshot
+
+
+def test_record_gate_decision_merges_into_snapshot() -> None:
+ ctx = SimpleNamespace()
+ record_gate_decision(ctx, {"gate_satisfied": False, "gate_last_full_workflow_test_ok": False})
+
+ snapshot = ctx.outcome_verification_trace_snapshot
+ assert snapshot["gate_satisfied"] is False
+ assert snapshot["gate_last_full_workflow_test_ok"] is False
+
+
+def test_outcome_verification_turn_fields_composes_all_sources() -> None:
+ ctx = SimpleNamespace(
+ workflow_verification_evidence=WorkflowVerificationEvidence(full_workflow_verified=True),
+ request_policy=RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id="c0", outcome="cart shows the item", implicit=True),
+ CompletionCriterion(id="c1", outcome="submitted via the search bar", method_mandated=True),
+ ]
+ ),
+ )
+ record_gate_decision(ctx, {"gate_satisfied": True})
+ record_completion_verification(ctx, _evaluated_result())
+
+ fields = outcome_verification_turn_fields(ctx)
+ assert fields["gate_satisfied"] is True
+ assert fields["completion_verification_status"] == "evaluated"
+ assert fields["verification_evidence_full_workflow_verified"] is True
+ assert fields["request_policy_completion_criteria_count"] == 2
+ assert fields["request_policy_completion_criteria_implicit_count"] == 1
+ assert fields["request_policy_completion_criteria_method_mandated_count"] == 1
+
+
+def test_finalize_is_best_effort_when_source_raises() -> None:
+ class Exploding:
+ def to_trace_data(self) -> dict[str, Any]:
+ raise RuntimeError("boom")
+
+ ctx = SimpleNamespace(workflow_verification_evidence=Exploding())
+ with otel_trace.get_tracer("test.finalize").start_as_current_span("copilot.turn") as span:
+ finalize_outcome_verification_trace(ctx, span) # must not raise
+
+
+def test_finalize_noop_on_none_ctx() -> None:
+ finalize_outcome_verification_trace(None)
+
+
+async def test_finalize_lands_fields_on_finished_turn_span(span_exporter: Any) -> None:
+ ctx = SimpleNamespace(
+ workflow_verification_evidence=WorkflowVerificationEvidence(full_workflow_verified=True),
+ request_policy=RequestPolicy(completion_criteria=[CompletionCriterion(id="c0", outcome="done")]),
+ )
+ record_gate_decision(ctx, {"gate_satisfied": True, "gate_last_full_workflow_test_ok": True})
+ record_completion_verification(ctx, _evaluated_result())
+
+ with otel_trace.get_tracer("test.finalize").start_as_current_span("copilot.turn") as span:
+ finalize_outcome_verification_trace(ctx, span)
+
+ finished = [s for s in span_exporter.get_finished_spans() if s.name == "copilot.turn"]
+ assert len(finished) == 1
+ attrs = dict(finished[0].attributes or {})
+ assert attrs["gate_satisfied"] is True
+ assert attrs["completion_verification_status"] == "evaluated"
+ assert attrs["verification_evidence_full_workflow_verified"] is True
+ assert attrs["request_policy_completion_criteria_count"] == 1
+
+
+async def test_run_copilot_agent_finalizes_false_gate_on_turn_span(
+ span_exporter: Any,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def stub_impl(*, ctx_sink: list[Any] | None = None, **_: Any) -> None:
+ ctx = SimpleNamespace(
+ workflow_verification_evidence=WorkflowVerificationEvidence(test_attempted_but_incomplete=True),
+ request_policy=RequestPolicy(),
+ )
+ record_gate_decision(
+ ctx,
+ {"gate_satisfied": False, "gate_last_full_workflow_test_ok": False, "gate_evaluated_this_turn": True},
+ )
+ record_completion_verification(ctx, None)
+ if ctx_sink is not None:
+ ctx_sink.append(ctx)
+ return None
+
+ monkeypatch.setattr(copilot_agent, "_run_copilot_turn_impl", stub_impl)
+ chat_request = WorkflowCopilotChatRequest(
+ workflow_permanent_id="wpid_xyz",
+ workflow_id="w_001",
+ workflow_copilot_chat_id="chat_abc",
+ message="add the item to my cart",
+ workflow_yaml="",
+ )
+
+ await copilot_agent.run_copilot_agent(
+ stream=object(),
+ organization_id="o_test",
+ chat_request=chat_request,
+ chat_history=[],
+ global_llm_context=None,
+ debug_run_info_text="",
+ llm_api_handler=None,
+ )
+
+ turn_spans = [s for s in span_exporter.get_finished_spans() if s.name == "copilot.turn"]
+ assert len(turn_spans) == 1
+ attrs = dict(turn_spans[0].attributes or {})
+ assert attrs["gate_satisfied"] is False
+ assert attrs["gate_last_full_workflow_test_ok"] is False
+ assert attrs["gate_evaluated_this_turn"] is True
+ assert attrs["completion_verification_evaluated_on_final_run"] is False
+ assert attrs["verification_evidence_test_attempted_but_incomplete"] is True
diff --git a/tests/unit/webeye/test_browser_session_response.py b/tests/unit/webeye/test_browser_session_response.py
new file mode 100644
index 000000000..edd3de90b
--- /dev/null
+++ b/tests/unit/webeye/test_browser_session_response.py
@@ -0,0 +1,24 @@
+from datetime import datetime, timezone
+
+import pytest
+
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
+from skyvern.webeye.schemas import BrowserSessionResponse
+
+
+async def test_browser_session_response_supports_vnc_when_browser_address_is_set() -> None:
+ now = datetime.now(timezone.utc)
+ session = PersistentBrowserSession(
+ persistent_browser_session_id="pbs_123",
+ organization_id="org_123",
+ status="running",
+ browser_address="ws://127.0.0.1:9222/devtools/browser/test",
+ ip_address=None,
+ created_at=now,
+ modified_at=now,
+ )
+
+ response = await BrowserSessionResponse.from_browser_session(session)
+
+ assert response.vnc_streaming_supported is True
diff --git a/tests/unit/webeye/test_local_pbs_cdp_connect.py b/tests/unit/webeye/test_local_pbs_cdp_connect.py
new file mode 100644
index 000000000..00fc13943
--- /dev/null
+++ b/tests/unit/webeye/test_local_pbs_cdp_connect.py
@@ -0,0 +1,32 @@
+import os
+from unittest.mock import patch
+
+from skyvern.webeye.cdp_connection import (
+ prepare_persistent_browser_cdp_connect,
+ resolve_local_pbs_cdp_url,
+ strip_browser_address_discriminator,
+)
+
+
+def test_strip_browser_address_discriminator() -> None:
+ url = "ws://127.0.0.1:9224/devtools/browser/abc#pbs_pbs_123"
+ assert strip_browser_address_discriminator(url) == "ws://127.0.0.1:9224/devtools/browser/abc"
+
+
[email protected](os.environ, {"LOCAL_CDP_HOST_PORT": "9224"}, clear=False)
+@patch("skyvern.webeye.cdp_connection.settings.ENV", "local")
+def test_prepare_local_pbs_cdp_connect_rewrites_port_and_adds_session_header() -> None:
+ browser_address = "ws://127.0.0.1:9222/devtools/browser/abc#pbs_pbs_123"
+ connect_url, headers = prepare_persistent_browser_cdp_connect(
+ browser_address,
+ browser_session_id="pbs_123",
+ cdp_connect_headers={"x-api-key": "test-key"},
+ )
+ assert connect_url == "ws://127.0.0.1:9224/devtools/browser/abc"
+ assert headers == {"x-api-key": "test-key", "X-Session-Id": "pbs_123"}
+
+
+@patch("skyvern.webeye.cdp_connection.settings.ENV", "prod")
+def test_resolve_local_pbs_cdp_url_noop_outside_local() -> None:
+ url = "wss://sessions.example.com/pbs_1/token/devtools/browser/abc"
+ assert resolve_local_pbs_cdp_url(url) == url
diff --git a/tests/unit_tests/test_streaming_screencast.py b/tests/unit_tests/test_streaming_screencast.py
index 3856ee7a4..6f220a2cb 100644
--- a/tests/unit_tests/test_streaming_screencast.py
+++ b/tests/unit_tests/test_streaming_screencast.py
@@ -64,10 +64,10 @@ async def test_resolve_browser_state_for_browser_session(monkeypatch: pytest.Mon
)
monkeypatch.setattr(screencast, "app", fake_app)
- result = await screencast._resolve_browser_state("bs_123", "browser_session")
+ result = await screencast._resolve_browser_state("bs_123", "browser_session", organization_id="o_123")
assert result is expected_state
- fake_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state.assert_awaited_once_with("bs_123")
+ fake_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state.assert_awaited_once_with("bs_123", "o_123")
fake_app.BROWSER_MANAGER.get_for_workflow_run.assert_not_called()
fake_app.BROWSER_MANAGER.get_for_task.assert_not_called()
@@ -101,7 +101,7 @@ async def test_wait_for_browser_state_returns_when_working_page_is_available(
result = await screencast.wait_for_browser_state("wr_123", "workflow_run", timeout=1, poll_interval=0.1)
assert result is browser_state
- resolve_mock.assert_awaited_once_with("wr_123", "workflow_run", None)
+ resolve_mock.assert_awaited_once_with("wr_123", "workflow_run", None, organization_id=None)
browser_state.get_working_page.assert_awaited_once()
sleep_mock.assert_not_awaited()
diff --git a/tests/unit_tests/test_streaming_screenshot_local.py b/tests/unit_tests/test_streaming_screenshot_local.py
index a646fb788..834bcca52 100644
--- a/tests/unit_tests/test_streaming_screenshot_local.py
+++ b/tests/unit_tests/test_streaming_screenshot_local.py
@@ -34,7 +34,9 @@ async def test_run_local_screencast_happy_path(monkeypatch: pytest.MonkeyPatch)
)
wait_for_running.assert_awaited_once()
- wait_for_browser_state_mock.assert_awaited_once_with("task_123", "task", workflow_run_id="wr_123")
+ wait_for_browser_state_mock.assert_awaited_once_with(
+ "task_123", "task", workflow_run_id="wr_123", organization_id=None
+ )
start_screencast_loop_mock.assert_awaited_once_with(
websocket=websocket,
browser_state=browser_state,
@@ -75,6 +77,7 @@ async def test_run_local_screencast_timeout_when_browser_state_not_available(
"bs_123",
"browser_session",
workflow_run_id=None,
+ organization_id=None,
)
start_screencast_loop_mock.assert_not_awaited()
get_current_status.assert_not_awaited()